~launchpad-pqm/launchpad/devel

# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).

"""Mantis ExternalBugTracker utility."""

__metaclass__ = type
__all__ = ['Mantis', 'MantisLoginHandler']

import cgi
import csv
import logging
import urllib
import urllib2
from urlparse import urlunparse

from BeautifulSoup import (
    BeautifulSoup,
    Comment,
    SoupStrainer,
    )

from lp.bugs.externalbugtracker import (
    BugNotFound,
    BugTrackerConnectError,
    BugWatchUpdateError,
    ExternalBugTracker,
    InvalidBugId,
    LookupTree,
    UnknownRemoteStatusError,
    UnparsableBugData,
    )
from lp.bugs.interfaces.bugtask import (
    BugTaskImportance,
    BugTaskStatus,
    )
from lp.bugs.interfaces.externalbugtracker import UNKNOWN_REMOTE_IMPORTANCE
from lp.services.database.isolation import ensure_no_transaction
from lp.services.propertycache import cachedproperty
from lp.services.webapp.url import urlparse


class MantisLoginHandler(urllib2.HTTPRedirectHandler):
    """Handler for urllib2.build_opener to automatically log-in
    to Mantis anonymously if needed.

    The ALSA bug tracker is the only tested Mantis installation that
    actually needs this. For ALSA bugs, the dance is like so:

      1. We request bug 3301 ('jack sensing problem'):
           https://bugtrack.alsa-project.org/alsa-bug/view.php?id=3301

      2. Mantis redirects us to:
           .../alsa-bug/login_page.php?
                 return=%2Falsa-bug%2Fview.php%3Fid%3D3301

      3. We notice this, rewrite the query, and skip to login.php:
           .../alsa-bug/login.php?
                 return=%2Falsa-bug%2Fview.php%3Fid%3D3301&
                 username=guest&password=guest

      4. Mantis accepts our credentials then redirects us to the bug
         view page via a cookie test page (login_cookie_test.php)
    """

    def rewrite_url(self, url):
        scheme, host, path, params, query, fragment = urlparse(url)

        # If we can, skip the login page and submit credentials
        # directly. The query should contain a 'return' parameter
        # which, if our credentials are accepted, means we'll be
        # redirected back from whence we came. In other words, we'll
        # end up back at the bug page we first requested.
        login_page = '/login_page.php'
        if path.endswith(login_page):
            path = path[:-len(login_page)] + '/login.php'
            query = cgi.parse_qs(query, True)
            query['username'] = query['password'] = ['guest']
            if 'return' not in query:
                raise BugTrackerConnectError(
                    url, ("Mantis redirected us to the login page "
                          "but did not set a return path."))

            query = urllib.urlencode(query, True)
            url = urlunparse(
                (scheme, host, path, params, query, fragment))

        # Previous versions of the Mantis external bug tracker fetched
        # login_anon.php in addition to the login.php method above, but none
        # of the Mantis installations tested actually needed this. For
        # example, the ALSA bugtracker actually issues an error "Your account
        # may be disabled" when accessing this page. For now it's better to
        # *not* try this page because we may end up annoying admins with
        # spurious login attempts.

        return url

    def redirect_request(self, request, fp, code, msg, hdrs, new_url):
        return urllib2.HTTPRedirectHandler.redirect_request(
            self, request, fp, code, msg, hdrs, self.rewrite_url(new_url))


class MantisBugBatchParser:
    """A class that parses the batch of bug data.

    Using the CSV reader is pretty much essential since the data that comes
    back can include title text which can in turn contain field separators.
    You don't want to handle the unquoting yourself.
    """

    def __init__(self, csv_data, logger):
        # Clean out stray, unquoted newlines inside csv_data to avoid the CSV
        # module blowing up.  IDEA: perhaps if the size of csv_data is large
        # in the future, this could be moved into a generator.
        csv_data = [s.replace("\r", "") for s in csv_data]
        csv_data = [s.replace("\n", "") for s in csv_data]
        self.reader = csv.reader(csv_data)
        self.logger = logger

    def processCSVBugLine(self, bug_line, headers):
        """Processes a single line of the CSV."""
        bug = {}
        for index, header in enumerate(headers):
            try:
                data = bug_line[index]
            except IndexError:
                self.logger.warning("Line %r incomplete." % bug_line)
                return None
            bug[header] = data
        try:
            bug['id'] = int(bug['id'])
        except ValueError:
            self.logger.warning("Encountered invalid bug ID: %r." % bug['id'])
            return None
        return bug

    def parseHeaderLine(self, reader):
        # The first line of the CSV file is the header. We need to read
        # it because different Mantis instances have different header
        # ordering and even different columns in the export.
        try:
            headers = [h.lower() for h in reader.next()]
        except StopIteration:
            raise UnparsableBugData("Missing header line")
        missing_headers = [
            name for name in ('id', 'status', 'resolution')
            if name not in headers]
        if missing_headers:
            raise UnparsableBugData(
                "CSV header %r missing fields: %r" % (
                    headers, missing_headers))
        return headers

    def getBugs(self):
        headers = self.parseHeaderLine(self.reader)
        bugs = {}
        try:
            for bug_line in self.reader:
                bug = self.processCSVBugLine(bug_line, headers)
                if bug is not None:
                    bugs[bug['id']] = bug
            return bugs
        except csv.Error, error:
            raise UnparsableBugData("Exception parsing CSV file: %s." % error)


class Mantis(ExternalBugTracker):
    """An `ExternalBugTracker` for dealing with Mantis instances.

    For a list of tested Mantis instances and their behaviour when
    exported from, see:

        https://dev.launchpad.net/Bugs/ExternalBugTrackers/Mantis
    """

    def __init__(self, baseurl):
        super(Mantis, self).__init__(baseurl)
        # Custom cookie aware opener that automatically sends anonymous
        # credentials to Mantis if (and only if) needed.
        self._cookie_handler = urllib2.HTTPCookieProcessor()
        self._opener = urllib2.build_opener(
            self._cookie_handler, MantisLoginHandler())
        self._logger = logging.getLogger()

    @ensure_no_transaction
    def urlopen(self, request, data=None):
        # We use urllib2 to make following cookies transparent.
        # This is required for certain bugtrackers that require
        # cookies that actually do anything (as is the case with
        # Mantis). It's basically a drop-in replacement for
        # urllib2.urlopen() that tracks cookies. We also have a
        # customised urllib2 opener to handle transparent
        # authentication.
        return self._opener.open(request, data)

    @cachedproperty
    def csv_data(self):
        """Attempt to retrieve a CSV export from the remote server.

        If the export fails (i.e. the response is 0-length), None will
        be returned.
        """
        return self._csv_data()

    def _csv_data(self):
        """See `csv_data()."""
        # Next step is getting our query filter cookie set up; we need
        # to do this weird submit in order to get the closed bugs
        # included in the results; the default Mantis filter excludes
        # them. It's unlikely that all these parameters are actually
        # necessary, but it's easy to prepare the complete set from a
        # view_all_bugs.php form dump so let's keep it complete.
        data = {
           'type': '1',
           'page_number': '1',
           'view_type': 'simple',
           'reporter_id[]': '0',
           'user_monitor[]': '0',
           'handler_id[]': '0',
           'show_category[]': '0',
           'show_severity[]': '0',
           'show_resolution[]': '0',
           'show_profile[]': '0',
           'show_status[]': '0',
           # Some of the more modern Mantis trackers use
           # a value of 'hide_status[]': '-2' here but it appears that
           # [none] works. Oops, older Mantis uses 'none' here. Gross!
           'hide_status[]': '[none]',
           'show_build[]': '0',
           'show_version[]': '0',
           'fixed_in_version[]': '0',
           'show_priority[]': '0',
           'per_page': '50',
           'view_state': '0',
           'sticky_issues': 'on',
           'highlight_changed': '6',
           'relationship_type': '-1',
           'relationship_bug': '0',
           # Hack around the fact that the sorting parameter has
           # changed over time.
           'sort': 'last_updated',
           'sort_0': 'last_updated',
           'dir': 'DESC',
           'dir_0': 'DESC',
           'search': '',
           'filter': 'Apply Filter',
        }
        try:
            self._postPage("view_all_set.php?f=3", data)
        except BugTrackerConnectError:
            return None

        # Finally grab the full CSV export, which uses the
        # MANTIS_VIEW_ALL_COOKIE set in the previous step to specify
        # what's being viewed.
        try:
            csv_data = self._getPage("csv_export.php")
        except BugTrackerConnectError, value:
            # Some Mantis installations simply return a 500 error
            # when the csv_export.php page is accessed. Since the
            # bug data may be nevertheless available from ordinary
            # web pages, we simply ignore this error.
            if value.error.startswith('HTTP Error 500'):
                return None
            raise

        if not csv_data:
            return None
        else:
            return csv_data

    def canUseCSVExports(self):
        """Return True if a Mantis instance supports CSV exports.

        If the Mantis instance cannot or does not support CSV exports,
        False will be returned.
        """
        return self.csv_data is not None

    def initializeRemoteBugDB(self, bug_ids):
        """See `ExternalBugTracker`.

        This method is overridden so that it can take into account the
        fact that not all Mantis instances support CSV exports. In
        those cases all bugs will be imported individually, regardless
        of how many there are.
        """
        self.bugs = {}

        if (len(bug_ids) > self.batch_query_threshold and
            self.canUseCSVExports()):
            # We only query for batches of bugs if the remote Mantis
            # instance supports CSV exports, otherwise we default to
            # screen-scraping on a per bug basis regardless of how many bugs
            # there are to retrieve.
            self.bugs = self.getRemoteBugBatch(bug_ids)
        else:
            for bug_id in bug_ids:
                bug_id, remote_bug = self.getRemoteBug(bug_id)

                if bug_id is not None:
                    self.bugs[bug_id] = remote_bug

    def getRemoteBug(self, bug_id):
        """See `ExternalBugTracker`."""
        # Only parse tables to save time and memory. If we didn't have
        # to check for application errors in the page (using
        # _checkForApplicationError) then we could be much more
        # specific than this.
        bug_page = BeautifulSoup(
            self._getPage('view.php?id=%s' % bug_id),
            convertEntities=BeautifulSoup.HTML_ENTITIES,
            parseOnlyThese=SoupStrainer('table'))

        app_error = self._checkForApplicationError(bug_page)
        if app_error:
            app_error_code, app_error_message = app_error
            # 1100 is ERROR_BUG_NOT_FOUND in Mantis (see
            # mantisbt/core/constant_inc.php).
            if app_error_code == '1100':
                return None, None
            else:
                raise BugWatchUpdateError(
                    "Mantis APPLICATION ERROR #%s: %s" % (
                    app_error_code, app_error_message))

        bug = {
            'id': bug_id,
            'status': self._findValueRightOfKey(bug_page, 'Status'),
            'resolution': self._findValueRightOfKey(bug_page, 'Resolution')}

        return int(bug_id), bug

    def getRemoteBugBatch(self, bug_ids):
        """See `ExternalBugTracker`."""
        # XXX: Gavin Panella 2007-09-06 bug=137780:
        # You may find this zero in "\r\n0" funny. Well I don't. This is
        # to work around the fact that Mantis' CSV export doesn't cope
        # with the fact that the bug summary can contain embedded "\r\n"
        # characters! I don't see a better way to handle this short of
        # not using the CSV module and forcing all lines to have the
        # same number as fields as the header.
        csv_data = self.csv_data.strip().split("\r\n0")

        if not csv_data:
            raise UnparsableBugData("Empty CSV for %s" % self.baseurl)

        parser = MantisBugBatchParser(csv_data, self._logger)
        return parser.getBugs()

    def _checkForApplicationError(self, page_soup):
        """If Mantis does not find the bug it still returns a 200 OK
        response, so we need to look into the page to figure it out.

        If there is no error, None is returned.

        If there is an error, a 2-tuple of (code, message) is
        returned, both unicode strings.
        """
        app_error = page_soup.find(
            text=lambda node: (node.startswith('APPLICATION ERROR ')
                               and node.parent['class'] == 'form-title'
                               and not isinstance(node, Comment)))
        if app_error:
            app_error_code = ''.join(c for c in app_error if c.isdigit())
            app_error_message = app_error.findNext('p')
            if app_error_message is not None:
                app_error_message = app_error_message.string
            return app_error_code, app_error_message

        return None

    def _findValueRightOfKey(self, page_soup, key):
        """Scrape a value from a Mantis bug view page where the value
        is displayed to the right of the key.

        The Mantis bug view page uses HTML tables for both layout and
        representing tabular data, often within the same table. This
        method assumes that the key and value are on the same row,
        adjacent to one another, with the key preceding the value:

        ...
        <td>Key</td>
        <td>Value</td>
        ...

        This method does not compensate for colspan or rowspan.
        """
        key_node = page_soup.find(
            text=lambda node: (node.strip() == key
                               and not isinstance(node, Comment)))
        if key_node is None:
            raise UnparsableBugData("Key %r not found." % (key,))

        value_cell = key_node.findNext('td')
        if value_cell is None:
            raise UnparsableBugData(
                "Value cell for key %r not found." % (key,))

        value_node = value_cell.string
        if value_node is None:
            raise UnparsableBugData("Value for key %r not found." % (key,))

        return value_node.strip()

    def _findValueBelowKey(self, page_soup, key):
        """Scrape a value from a Mantis bug view page where the value
        is displayed directly below the key.

        The Mantis bug view page uses HTML tables for both layout and
        representing tabular data, often within the same table. This
        method assumes that the key and value are within the same
        column on adjacent rows, with the key preceding the value:

        ...
        <tr>...<td>Key</td>...</tr>
        <tr>...<td>Value</td>...</tr>
        ...

        This method does not compensate for colspan or rowspan.
        """
        key_node = page_soup.find(
            text=lambda node: (node.strip() == key
                               and not isinstance(node, Comment)))
        if key_node is None:
            raise UnparsableBugData("Key %r not found." % (key,))

        key_cell = key_node.parent
        if key_cell is None:
            raise UnparsableBugData("Cell for key %r not found." % (key,))

        key_row = key_cell.parent
        if key_row is None:
            raise UnparsableBugData("Row for key %r not found." % (key,))

        try:
            key_pos = key_row.findAll('td').index(key_cell)
        except ValueError:
            raise UnparsableBugData(
                "Key cell in row for key %r not found." % (key,))

        value_row = key_row.findNextSibling('tr')
        if value_row is None:
            raise UnparsableBugData(
                "Value row for key %r not found." % (key,))

        value_cell = value_row.findAll('td')[key_pos]
        if value_cell is None:
            raise UnparsableBugData(
                "Value cell for key %r not found." % (key,))

        value_node = value_cell.string
        if value_node is None:
            raise UnparsableBugData("Value for key %r not found." % (key,))

        return value_node.strip()

    def getRemoteImportance(self, bug_id):
        """See `ExternalBugTracker`.

        This method is implemented here as a stub to ensure that
        existing functionality is preserved. As a result,
        UNKNOWN_REMOTE_IMPORTANCE will always be returned.
        """
        return UNKNOWN_REMOTE_IMPORTANCE

    def getRemoteStatus(self, bug_id):
        if not bug_id.isdigit():
            raise InvalidBugId(
                "Mantis (%s) bug number not an integer: %s" % (
                    self.baseurl, bug_id))

        try:
            bug = self.bugs[int(bug_id)]
        except KeyError:
            raise BugNotFound(bug_id)

        # Use a colon and a space to join status and resolution because
        # there is a chance that statuses contain spaces, and because
        # it makes display of the data nicer.
        return "%(status)s: %(resolution)s" % bug

    def convertRemoteImportance(self, remote_importance):
        """See `ExternalBugTracker`.

        This method is implemented here as a stub to ensure that
        existing functionality is preserved. As a result,
        BugTaskImportance.UNKNOWN will always be returned.
        """
        return BugTaskImportance.UNKNOWN

    _status_lookup_titles = 'Mantis status', 'Mantis resolution'
    _status_lookup = (
        LookupTree(
            ('assigned', BugTaskStatus.INPROGRESS),
            ('feedback', BugTaskStatus.INCOMPLETE),
            ('new', BugTaskStatus.NEW),
            ('confirmed', 'ackowledged', BugTaskStatus.CONFIRMED),
            ('resolved', 'closed',
                LookupTree(
                    ('reopened', BugTaskStatus.NEW),
                    ('fixed', 'open', 'no change required',
                     BugTaskStatus.FIXRELEASED),
                    ('unable to reproduce', 'not fixable', 'suspended',
                     'duplicate', BugTaskStatus.INVALID),
                    ("won't fix", BugTaskStatus.WONTFIX))),
            )
        )

    def convertRemoteStatus(self, status_and_resolution):
        status, importance = status_and_resolution.split(": ", 1)
        try:
            return self._status_lookup.find(status, importance)
        except KeyError:
            raise UnknownRemoteStatusError(status_and_resolution)