~launchpad-pqm/launchpad/devel

# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).

"""Trac ExternalBugTracker implementation."""

__metaclass__ = type
__all__ = ['Trac', 'TracLPPlugin']

from Cookie import SimpleCookie
from cookielib import CookieJar
import csv
from datetime import datetime
from email.Utils import parseaddr
import time
import urllib2
import xmlrpclib

import pytz
from zope.component import getUtility
from zope.interface import implements

from canonical.config import config
from lp.services.messages.interfaces.message import IMessageSet
from canonical.launchpad.webapp.url import urlappend
from lp.app.validators.email import valid_email
from lp.bugs.externalbugtracker.base import (
    BugNotFound,
    BugTrackerAuthenticationError,
    BugTrackerConnectError,
    ExternalBugTracker,
    InvalidBugId,
    LookupTree,
    UnknownRemoteStatusError,
    UnparsableBugData,
    )
from lp.bugs.externalbugtracker.xmlrpc import UrlLib2Transport
from lp.bugs.interfaces.bugtask import (
    BugTaskImportance,
    BugTaskStatus,
    )
from lp.bugs.interfaces.externalbugtracker import (
    ISupportsBackLinking,
    ISupportsCommentImport,
    ISupportsCommentPushing,
    UNKNOWN_REMOTE_IMPORTANCE,
    )
from lp.services.database.isolation import ensure_no_transaction

# Symbolic constants used for the Trac LP plugin.
LP_PLUGIN_BUG_IDS_ONLY = 0
LP_PLUGIN_METADATA_ONLY = 1
LP_PLUGIN_METADATA_AND_COMMENTS = 2
LP_PLUGIN_FULL = 3

# Fault code constants for the LP Plugin
FAULT_TICKET_NOT_FOUND = 1001


class Trac(ExternalBugTracker):
    """An ExternalBugTracker instance for handling Trac bugtrackers."""

    ticket_url = 'ticket/%i?format=csv'
    batch_url = 'query?%s&order=resolution&format=csv'
    batch_query_threshold = 10

    def getExternalBugTrackerToUse(self):
        """See `IExternalBugTracker`."""
        base_auth_url = urlappend(self.baseurl, 'launchpad-auth')
        # Any token will do.
        auth_url = urlappend(base_auth_url, 'check')
        try:
            response = self.urlopen(auth_url)
        except urllib2.HTTPError, error:
            # If the error is HTTP 401 Unauthorized then we're
            # probably talking to the LP plugin.
            if error.code == 401:
                return TracLPPlugin(self.baseurl)
            else:
                return self
        except urllib2.URLError, error:
            return self
        else:
            # If the response contains a trac_auth cookie then we're
            # talking to the LP plugin. However, it's unlikely that
            # the remote system will authorize the bogus auth token we
            # sent, so this check is really intended to detect broken
            # Trac instances that return HTTP 200 for a missing page.
            for set_cookie in response.headers.getheaders('Set-Cookie'):
                cookie = SimpleCookie(set_cookie)
                if 'trac_auth' in cookie:
                    return TracLPPlugin(self.baseurl)
            else:
                return self

    def supportsSingleExports(self, bug_ids):
        """Return True if the Trac instance provides CSV exports for single
        tickets, False otherwise.

        :bug_ids: A list of bug IDs that we can use for discovery purposes.
        """
        valid_ticket = False
        html_ticket_url = '%s/%s' % (
            self.baseurl, self.ticket_url.replace('?format=csv', ''))

        bug_ids = list(bug_ids)
        while not valid_ticket and len(bug_ids) > 0:
            try:
                # We try to retrive the ticket in HTML form, since that will
                # tell us whether or not it is actually a valid ticket
                ticket_id = int(bug_ids.pop())
                self._fetchPage(html_ticket_url % ticket_id)
            except (ValueError, urllib2.HTTPError):
                # If we get an HTTP error we can consider the ticket to be
                # invalid. If we get a ValueError then the ticket_id couldn't
                # be identified and it's of no use to us anyway.
                pass
            else:
                # If we didn't get an error we can try to get the ticket in
                # CSV form. If this fails then we can consider single ticket
                # exports to be unsupported.
                try:
                    csv_data = self._fetchPage(
                        "%s/%s" % (self.baseurl, self.ticket_url % ticket_id))
                    return csv_data.headers.subtype == 'csv'
                except (urllib2.HTTPError, urllib2.URLError):
                    return False
        else:
            # If we reach this point then we likely haven't had any valid
            # tickets or something else is wrong. Either way, we can only
            # assume that CSV exports of single tickets aren't supported.
            return False

    def _fetchBugData(self, query_url):
        """Retrieve the CSV bug data from a URL and return it.

        :param query_url: The URL from which to retrieve the CSV bug
            data.
        :return: A list of dicts, with each dict representing a single
            row in the CSV data retrieved from `query_url`.
        """
        # We read the remote bugs into a list so that we can check that
        # the data we're getting back from the remote server are valid.
        csv_reader = csv.DictReader(self._fetchPage(query_url))
        remote_bugs = [csv_reader.next()]

        # We consider the data we're getting from the remote server to
        # be valid if there is an ID field and a status field in the CSV
        # header. If the fields don't exist we raise an
        # UnparsableBugData error. If these fields are defined but not
        # filled in for each row, that error will be handled in
        # getRemoteBugStatus() (i.e.  with a BugNotFound or an
        # UnknownRemoteStatusError).
        if ('id' not in csv_reader.fieldnames or
            'status' not in csv_reader.fieldnames):
            raise UnparsableBugData(
                "External bugtracker %s does not define all the necessary "
                "fields for bug status imports (Defined field names: %r)."
                % (self.baseurl, csv_reader.fieldnames))

        remote_bugs = remote_bugs + list(csv_reader)
        return remote_bugs

    def getRemoteBug(self, bug_id):
        """See `ExternalBugTracker`."""
        bug_id = int(bug_id)
        query_url = "%s/%s" % (self.baseurl, self.ticket_url % bug_id)

        bug_data = self._fetchBugData(query_url)
        if len(bug_data) == 1:
            return bug_id, bug_data[0]

        # There should be only one bug returned for a getRemoteBug()
        # call, so if we have more or less than one bug something went
        # wrong.
        raise UnparsableBugData(
            "Remote bugtracker %s returned wrong amount of data for bug "
            "%i (expected 1 bug, got %i bugs)." %
            (self.baseurl, bug_id, len(bug_data)))

    def getRemoteBugBatch(self, bug_ids):
        """See `ExternalBugTracker`."""
        id_string = '&'.join(['id=%s' % id for id in bug_ids])
        query_url = "%s/%s" % (self.baseurl, self.batch_url % id_string)
        remote_bugs = self._fetchBugData(query_url)

        bugs = {}
        for remote_bug in remote_bugs:
            # We're only interested in the bug if it's one of the ones in
            # bug_ids, just in case we get all the tickets in the Trac
            # instance back instead of only the ones we want.
            if remote_bug['id'] not in bug_ids:
                continue

            bugs[int(remote_bug['id'])] = remote_bug

        return bugs

    def initializeRemoteBugDB(self, bug_ids):
        """See `ExternalBugTracker`.

        This method overrides ExternalBugTracker.initializeRemoteBugDB()
        so that the remote Trac instance's support for single ticket
        exports can be taken into account.

        If the URL specified for the bugtracker is not valid a
        BugTrackerConnectError will be raised.
        """
        self.bugs = {}
        # When there are less than batch_query_threshold bugs to update
        # we make one request per bug id to the remote bug tracker,
        # providing it supports CSV exports per-ticket. If the Trac
        # instance doesn't support exports-per-ticket we fail over to
        # using the batch export method for retrieving bug statuses.
        if (len(bug_ids) < self.batch_query_threshold and
            self.supportsSingleExports(bug_ids)):
            for bug_id in bug_ids:
                remote_id, remote_bug = self.getRemoteBug(bug_id)
                self.bugs[remote_id] = remote_bug

        # For large lists of bug ids we retrieve bug statuses as a batch
        # from the remote bug tracker so as to avoid effectively DOSing
        # it.
        else:
            self.bugs = self.getRemoteBugBatch(bug_ids)

    def getRemoteImportance(self, bug_id):
        """See `ExternalBugTracker`.

        This method is implemented here as a stub to ensure that
        existing functionality is preserved. As a result,
        UNKNOWN_REMOTE_IMPORTANCE will always be returned.
        """
        return UNKNOWN_REMOTE_IMPORTANCE

    def getRemoteStatus(self, bug_id):
        """Return the remote status for the given bug id.

        Raise BugNotFound if the bug can't be found.
        Raise InvalidBugId if the bug id has an unexpected format.
        """
        try:
            bug_id = int(bug_id)
        except ValueError:
            raise InvalidBugId(
                "bug_id must be convertable an integer: %s" % str(bug_id))

        try:
            remote_bug = self.bugs[bug_id]
        except KeyError:
            raise BugNotFound(bug_id)

        # If the bug has a valid resolution as well as a status then we return
        # that, since it's more informative than the status field on its own.
        if ('resolution' in remote_bug and
            remote_bug['resolution'] not in ['', '--', None]):
            return remote_bug['resolution']
        else:
            try:
                return remote_bug['status']
            except KeyError:
                # Some Trac instances don't include the bug status in their
                # CSV exports. In those cases we raise a error.
                raise UnknownRemoteStatusError('Status not exported.')

    def convertRemoteImportance(self, remote_importance):
        """See `ExternalBugTracker`.

        This method is implemented here as a stub to ensure that
        existing functionality is preserved. As a result,
        BugTaskImportance.UNKNOWN will always be returned.
        """
        return BugTaskImportance.UNKNOWN

    _status_lookup_titles = 'Trac status',
    _status_lookup = LookupTree(
        ('new', 'open', 'reopened', BugTaskStatus.NEW),
        # XXX: Graham Binns 2007-08-06: We should follow dupes if possible.
        ('accepted', 'assigned', 'duplicate', BugTaskStatus.CONFIRMED),
        # Status fixverified added for bug 667340, for http://trac.yorba.org/,
        # but could be generally useful so adding here.
        ('fixed', 'closed', 'fixverified', BugTaskStatus.FIXRELEASED),
        ('invalid', 'worksforme', BugTaskStatus.INVALID),
        ('wontfix', BugTaskStatus.WONTFIX),
        )

    def convertRemoteStatus(self, remote_status):
        """See `IExternalBugTracker`"""
        try:
            return self._status_lookup.find(remote_status)
        except KeyError:
            raise UnknownRemoteStatusError(remote_status)


def needs_authentication(func):
    """Decorator for automatically authenticating if needed.

    If an `xmlrpclib.ProtocolError` with error code 403 is raised by the
    function, we'll try to authenticate and call the function again.
    """

    def decorator(self, *args, **kwargs):
        try:
            return func(self, *args, **kwargs)
        except xmlrpclib.ProtocolError, error:
            # Catch authentication errors only.
            if error.errcode != 403:
                raise
            self._authenticate()
            return func(self, *args, **kwargs)
    return decorator


class TracLPPlugin(Trac):
    """A Trac instance having the LP plugin installed."""

    implements(
        ISupportsBackLinking, ISupportsCommentImport, ISupportsCommentPushing)

    def __init__(self, baseurl, xmlrpc_transport=None,
                 internal_xmlrpc_transport=None, cookie_jar=None):
        super(TracLPPlugin, self).__init__(baseurl)

        if cookie_jar is None:
            cookie_jar = CookieJar()
        if xmlrpc_transport is None:
            xmlrpc_transport = UrlLib2Transport(baseurl, cookie_jar)

        self._cookie_jar = cookie_jar
        self._xmlrpc_transport = xmlrpc_transport
        self._internal_xmlrpc_transport = internal_xmlrpc_transport

        xmlrpc_endpoint = urlappend(self.baseurl, 'xmlrpc')
        self._server = xmlrpclib.ServerProxy(
            xmlrpc_endpoint, transport=self._xmlrpc_transport)

        self._url_opener = urllib2.build_opener(
            urllib2.HTTPCookieProcessor(cookie_jar))

    @ensure_no_transaction
    @needs_authentication
    def initializeRemoteBugDB(self, bug_ids):
        """See `IExternalBugTracker`."""
        self.bugs = {}

        time_snapshot, remote_bugs = self._server.launchpad.bug_info(
            LP_PLUGIN_METADATA_AND_COMMENTS, dict(bugs=bug_ids))
        for remote_bug in remote_bugs:
            # We only import bugs whose status isn't 'missing', since
            # those bugs don't exist on the remote system.
            if remote_bug['status'] != 'missing':
                self.bugs[int(remote_bug['id'])] = remote_bug

    @ensure_no_transaction
    def urlopen(self, request, data=None):
        """See `ExternalBugTracker`.

        This method is overridden here so that it uses the _url_opener
        attribute in order to maintain the use of Trac authentication
        cookies across requests.
        """
        return self._url_opener.open(request, data)

    @ensure_no_transaction
    def _generateAuthenticationToken(self):
        """Create an authentication token and return it."""
        internal_xmlrpc = xmlrpclib.ServerProxy(
            config.checkwatches.xmlrpc_url,
            transport=self._internal_xmlrpc_transport)
        return internal_xmlrpc.newBugTrackerToken()

    def _authenticate(self):
        """Authenticate with the Trac instance."""
        token_text = self._generateAuthenticationToken()
        base_auth_url = urlappend(self.baseurl, 'launchpad-auth')
        auth_url = urlappend(base_auth_url, token_text)

        try:
            self._fetchPage(auth_url)
        except BugTrackerConnectError, e:
            raise BugTrackerAuthenticationError(self.baseurl, e.error)

    @ensure_no_transaction
    @needs_authentication
    def getCurrentDBTime(self):
        """See `IExternalBugTracker`."""
        time_zone, local_time, utc_time = (
            self._server.launchpad.time_snapshot())

        # Return the UTC time, so we don't have to care about the time
        # zone for now.
        trac_time = datetime.utcfromtimestamp(utc_time)
        return trac_time.replace(tzinfo=pytz.timezone('UTC'))

    @ensure_no_transaction
    @needs_authentication
    def getModifiedRemoteBugs(self, remote_bug_ids, last_checked):
        """See `IExternalBugTracker`."""
        # Convert last_checked into an integer timestamp (which is what
        # the Trac LP plugin expects).
        last_checked_timestamp = int(
            time.mktime(last_checked.timetuple()))

        # We retrieve only the IDs of the modified bugs from the server.
        criteria = {
            'modified_since': last_checked_timestamp,
            'bugs': remote_bug_ids,
            }
        time_snapshot, modified_bugs = self._server.launchpad.bug_info(
            LP_PLUGIN_BUG_IDS_ONLY, criteria)

        return [bug['id'] for bug in modified_bugs]

    def getCommentIds(self, remote_bug_id):
        """See `ISupportsCommentImport`."""
        try:
            bug = self.bugs[int(remote_bug_id)]
        except KeyError:
            raise BugNotFound(remote_bug_id)
        else:
            return [comment_id for comment_id in bug['comments']]

    @ensure_no_transaction
    @needs_authentication
    def fetchComments(self, remote_bug_id, comment_ids):
        """See `ISupportsCommentImport`."""
        bug_comments = {}

        # Use the get_comments() method on the remote server to get the
        # comments specified.
        timestamp, remote_comments = self._server.launchpad.get_comments(
            comment_ids)
        for remote_comment in remote_comments:
            bug_comments[remote_comment['id']] = remote_comment

        # Finally, we overwrite the bug's comments field with the
        # bug_comments dict. The nice upshot of this is that we can
        # still loop over the dict and get IDs back.
        self.bugs[int(remote_bug_id)]['comments'] = bug_comments

    def getPosterForComment(self, remote_bug_id, comment_id):
        """See `ISupportsCommentImport`."""
        bug = self.bugs[int(remote_bug_id)]
        comment = bug['comments'][comment_id]

        display_name, email = parseaddr(comment['user'])

        # If the email isn't valid, return the email address as the
        # display name (a Launchpad Person will be created with this
        # name).
        if not valid_email(email):
            return email, None
        # If the display name is empty, set it to None so that it's
        # useable by IPersonSet.ensurePerson().
        elif display_name == '':
            return None, email
        # Both displayname and email are valid, return both.
        else:
            return display_name, email

    def getMessageForComment(self, remote_bug_id, comment_id, poster):
        """See `ISupportsCommentImport`."""
        bug = self.bugs[int(remote_bug_id)]
        comment = bug['comments'][comment_id]

        comment_datecreated = datetime.fromtimestamp(
            comment['timestamp'], pytz.timezone('UTC'))
        message = getUtility(IMessageSet).fromText(
            subject='', content=comment['comment'],
            datecreated=comment_datecreated, owner=poster)

        return message

    @ensure_no_transaction
    @needs_authentication
    def addRemoteComment(self, remote_bug, comment_body, rfc822msgid):
        """See `ISupportsCommentPushing`."""
        timestamp, comment_id = self._server.launchpad.add_comment(
            remote_bug, comment_body)

        return comment_id

    @ensure_no_transaction
    @needs_authentication
    def getLaunchpadBugId(self, remote_bug):
        """Return the Launchpad bug for a given remote bug.

        :raises BugNotFound: When `remote_bug` doesn't exist.
        """
        try:
            timestamp, lp_bug_id = self._server.launchpad.get_launchpad_bug(
                remote_bug)
        except xmlrpclib.Fault, fault:
            # Deal with "Ticket does not exist" faults. We re-raise
            # anything else, since they're a sign of a bigger problem.
            if fault.faultCode == FAULT_TICKET_NOT_FOUND:
                raise BugNotFound(remote_bug)
            else:
                raise

        # If the returned bug ID is 0, return None, since a 0 means that
        # no LP bug is linked to the remote bug.
        if lp_bug_id == 0:
            return None
        else:
            return lp_bug_id

    @ensure_no_transaction
    @needs_authentication
    def setLaunchpadBugId(self, remote_bug, launchpad_bug_id,
                          launchpad_bug_url):
        """Set the Launchpad bug ID for a given remote bug.

        :raises BugNotFound: When `remote_bug` doesn't exist.
        """
        # If the launchpad_bug_id is None, pass 0 to set_launchpad_bug
        # to delete the bug link, since we can't send None over XML-RPC.
        if launchpad_bug_id == None:
            launchpad_bug_id = 0

        try:
            self._server.launchpad.set_launchpad_bug(
                remote_bug, launchpad_bug_id)
        except xmlrpclib.Fault, fault:
            # Deal with "Ticket does not exist" faults. We re-raise
            # anything else, since they're a sign of a bigger problem.
            if fault.faultCode == FAULT_TICKET_NOT_FOUND:
                raise BugNotFound(remote_bug)
            else:
                raise