~launchpad-pqm/launchpad/devel

# Copyright 2009 Canonical Ltd.  This software is licensed under the

# GNU Affero General Public License version 3 (see the file LICENSE).

"""Bugzilla to Launchpad import logic"""

# Bugzilla schema:

# http://lxr.mozilla.org/mozilla/source/webtools/bugzilla/Bugzilla/DB/Schema.pm

# XXX: jamesh 2005-10-18

# Currently unhandled bug info:

#  * Operating system and platform

#  * version (not really used in Ubuntu bugzilla though)

#  * keywords

#  * private bugs (none of the canonical-only bugs seem sensitive though)

#  * bug dependencies

#  * "bug XYZ" references inside comment text (at the moment we just

#    insert the full URL to the bug afterwards).

#

# Not all of these are necessary though

__metaclass__ = type

from cStringIO import StringIO

import datetime

import logging

import re

import pytz

from storm.store import Store

from zope.component import getUtility

from canonical.launchpad.interfaces.emailaddress import IEmailAddressSet

from canonical.launchpad.interfaces.launchpad import ILaunchpadCelebrities

from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet

from lp.services.messages.interfaces.message import IMessageSet

from canonical.launchpad.webapp import canonical_url

from lp.app.errors import NotFoundError

from lp.bugs.interfaces.bug import (

    CreateBugParams,

    IBugSet,

    )

from lp.bugs.interfaces.bugattachment import (

    BugAttachmentType,

    IBugAttachmentSet,

    )

from lp.bugs.interfaces.bugtask import (

    BugTaskImportance,

    BugTaskStatus,

    IBugTaskSet,

    )

from lp.bugs.interfaces.bugwatch import IBugWatchSet

from lp.bugs.interfaces.cve import ICveSet

from lp.registry.interfaces.person import (

    IPersonSet,

    PersonCreationRationale,

    )

logger = logging.getLogger('lp.bugs.scripts.bugzilla')

def _add_tz(dt):

    """Convert a naiive datetime value to a UTC datetime value."""

    assert dt.tzinfo is None, 'add_tz() only accepts naiive datetime values'

    return datetime.datetime(dt.year, dt.month, dt.day,

                             dt.hour, dt.minute, dt.second,

                             dt.microsecond, tzinfo=pytz.timezone('UTC'))

class BugzillaBackend:

    """A wrapper for all the MySQL database access.

    The main purpose of this is to make it possible to test the rest

    of the import code without access to a MySQL database.

    """

    def __init__(self, conn, charset='UTF-8'):

        self.conn = conn

        self.cursor = conn.cursor()

        self.charset = charset

    def _decode(self, s):

        if s is not None:

            value = s.decode(self.charset, 'replace')

            # Postgres doesn't like values outside of the basic multilingual

            # plane (U+0000 - U+FFFF), so replace them (and surrogates) with

            # U+FFFD (replacement character).

            # Existance of these characters generally indicate an encoding

            # problem in the existing Bugzilla data.

            return re.sub(u'[^\u0000-\ud7ff\ue000-\uffff]', u'\ufffd', value)

        else:

            return None

    def lookupUser(self, user_id):

        """Look up information about a particular Bugzilla user ID"""

        self.cursor.execute('SELECT login_name, realname '

                            '  FROM profiles '

                            '  WHERE userid = %d' % user_id)

        if self.cursor.rowcount != 1:

            raise NotFoundError('could not look up user %d' % user_id)

        (login_name, realname) = self.cursor.fetchone()

        realname = self._decode(realname)

        return (login_name, realname)

    def getBugInfo(self, bug_id):

        """Retrieve information about a bug."""

        self.cursor.execute(

            'SELECT bug_id, assigned_to, bug_file_loc, bug_severity, '

            '    bug_status, creation_ts, short_desc, op_sys, priority, '

            '    products.name, rep_platform, reporter, version, '

            '    components.name, resolution, target_milestone, qa_contact, '

            '    status_whiteboard, keywords, alias '

            '  FROM bugs '

            '    INNER JOIN products ON bugs.product_id = products.id '

            '    INNER JOIN components ON bugs.component_id = components.id '

            '  WHERE bug_id = %d' % bug_id)

        if self.cursor.rowcount != 1:

            raise NotFoundError('could not look up bug %d' % bug_id)

        (bug_id, assigned_to, bug_file_loc, bug_severity, bug_status,

         creation_ts, short_desc, op_sys, priority, product,

         rep_platform, reporter, version, component, resolution,

         target_milestone, qa_contact, status_whiteboard, keywords,

         alias) = self.cursor.fetchone()

        bug_file_loc = self._decode(bug_file_loc)

        creation_ts = _add_tz(creation_ts)

        product = self._decode(product)

        version = self._decode(version)

        component = self._decode(component)

        status_whiteboard = self._decode(status_whiteboard)

        keywords = self._decode(keywords)

        alias = self._decode(alias)

        return (bug_id, assigned_to, bug_file_loc, bug_severity,

                bug_status, creation_ts, short_desc, op_sys, priority,

                product, rep_platform, reporter, version, component,

                resolution, target_milestone, qa_contact,

                status_whiteboard, keywords, alias)

    def getBugCcs(self, bug_id):

        """Get the IDs of the people CC'd to the bug."""

        self.cursor.execute('SELECT who FROM cc WHERE bug_id = %d'

                            % bug_id)

        return [row[0] for row in self.cursor.fetchall()]

    def getBugComments(self, bug_id):

        """Get the comments for the bug."""

        self.cursor.execute('SELECT who, bug_when, thetext '

                            '  FROM longdescs '

                            '  WHERE bug_id = %d '

                            '  ORDER BY bug_when' % bug_id)

        # XXX: jamesh 2005-12-07:

        # Due to a bug in Debzilla, Ubuntu bugzilla bug 248 has > 7800

        # duplicate comments,consisting of someone's signature.

        # For the import, just ignore those comments.

        return [(who, _add_tz(when), self._decode(thetext))

                 for (who, when, thetext) in self.cursor.fetchall()

                 if thetext != '\n--=20\n   Jacobo Tarr=EDo     |     '

                               'http://jacobo.tarrio.org/\n\n\n']

    def getBugAttachments(self, bug_id):

        """Get the attachments for the bug."""

        self.cursor.execute('SELECT attach_id, creation_ts, description, '

                            '    mimetype, ispatch, filename, thedata, '

                            '    submitter_id '

                            '  FROM attachments '

                            '  WHERE bug_id = %d '

                            '  ORDER BY attach_id' % bug_id)

        return [(attach_id, _add_tz(creation_ts),

                 self._decode(description), mimetype,

                 ispatch, self._decode(filename), thedata, submitter_id)

                for (attach_id, creation_ts, description,

                     mimetype, ispatch, filename, thedata,

                     submitter_id) in self.cursor.fetchall()]

    def findBugs(self, product=None, component=None, status=None):

        """Returns the requested bug IDs as a list"""

        if product is None:

            product = []

        if component is None:

            component = []

        if status is None:

            status = []

        joins = []

        conditions = []

        if product:

            joins.append(

                'INNER JOIN products ON bugs.product_id = products.id')

            conditions.append('products.name IN (%s)' %

                ', '.join([self.conn.escape(p) for p in product]))

        if component:

            joins.append(

                'INNER JOIN components ON bugs.component_id = components.id')

            conditions.append('components.name IN (%s)' %

                ', '.join([self.conn.escape(c) for c in component]))

        if status:

            conditions.append('bugs.bug_status IN (%s)' %

                ', '.join([self.conn.escape(s) for s in status]))

        if conditions:

            conditions = 'WHERE %s' % ' AND '.join(conditions)

        else:

            conditions = ''

        self.cursor.execute('SELECT bug_id FROM bugs %s %s ORDER BY bug_id' %

                            (' '.join(joins), conditions))

        return [bug_id for (bug_id,) in self.cursor.fetchall()]

    def getDuplicates(self):

        """Returns a list of (dupe_of, dupe) relations."""

        self.cursor.execute('SELECT dupe_of, dupe FROM duplicates '

                            'ORDER BY dupe, dupe_of')

        return [(dupe_of, dupe) for (dupe_of, dupe) in self.cursor.fetchall()]

class Bug:

    """Representation of a Bugzilla Bug"""

    def __init__(self, backend, bug_id):

        self.backend = backend

        (self.bug_id, self.assigned_to, self.bug_file_loc, self.bug_severity,

         self.bug_status, self.creation_ts, self.short_desc, self.op_sys,

         self.priority, self.product, self.rep_platform, self.reporter,

         self.version, self.component, self.resolution,

         self.target_milestone, self.qa_contact, self.status_whiteboard,

         self.keywords, self.alias) = backend.getBugInfo(bug_id)

        self._ccs = None

        self._comments = None

        self._attachments = None

    @property

    def ccs(self):

        """Return the IDs of people CC'd to this bug"""

        if self._ccs is not None:

            return self._ccs

        self._ccs = self.backend.getBugCcs(self.bug_id)

        return self._ccs

    @property

    def comments(self):

        """Return the comments attached to this bug"""

        if self._comments is not None:

            return self._comments

        self._comments = self.backend.getBugComments(self.bug_id)

        return self._comments

    @property

    def attachments(self):

        """Return the attachments for this bug"""

        if self._attachments is not None:

            return self._attachments

        self._attachments = self.backend.getBugAttachments(self.bug_id)

        return self._attachments

    def mapSeverity(self, bugtask):

        """Set a Launchpad bug task's importance based on this bug's severity.

        """

        bug_importer = getUtility(ILaunchpadCelebrities).bug_importer

        importance_map = {

            'blocker': BugTaskImportance.CRITICAL,

            'critical': BugTaskImportance.CRITICAL,

            'major': BugTaskImportance.HIGH,

            'normal': BugTaskImportance.MEDIUM,

            'minor': BugTaskImportance.LOW,

            'trivial': BugTaskImportance.LOW,

            'enhancement': BugTaskImportance.WISHLIST

            }

        importance = importance_map.get(

            self.bug_severity, BugTaskImportance.UNKNOWN)

        bugtask.transitionToImportance(importance, bug_importer)

    def mapStatus(self, bugtask):

        """Set a Launchpad bug task's status based on this bug's status.

        If the bug is in the RESOLVED, VERIFIED or CLOSED states, the

        bug resolution is also taken into account when mapping the

        status.

        Additional information about the bugzilla status is appended

        to the bug task's status explanation.

        """

        bug_importer = getUtility(ILaunchpadCelebrities).bug_importer

        if self.bug_status == 'ASSIGNED':

            bugtask.transitionToStatus(

                BugTaskStatus.CONFIRMED, bug_importer)

        elif self.bug_status == 'NEEDINFO':

            bugtask.transitionToStatus(

                BugTaskStatus.INCOMPLETE, bug_importer)

        elif self.bug_status == 'PENDINGUPLOAD':

            bugtask.transitionToStatus(

                BugTaskStatus.FIXCOMMITTED, bug_importer)

        elif self.bug_status in ['RESOLVED', 'VERIFIED', 'CLOSED']:

            # depends on the resolution:

            if self.resolution == 'FIXED':

                bugtask.transitionToStatus(

                    BugTaskStatus.FIXRELEASED, bug_importer)

            else:

                bugtask.transitionToStatus(

                    BugTaskStatus.INVALID, bug_importer)

        else:

            bugtask.transitionToStatus(

                BugTaskStatus.NEW, bug_importer)

        # add the status to the notes section, to account for any lost

        # information

        bugzilla_status = 'Bugzilla status=%s' % self.bug_status

        if self.resolution:

            bugzilla_status += ' %s' % self.resolution

        bugzilla_status += ', product=%s' % self.product

        bugzilla_status += ', component=%s' % self.component

        if bugtask.statusexplanation:

            bugtask.statusexplanation = '%s (%s)' % (

                bugtask.statusexplanation, bugzilla_status)

        else:

            bugtask.statusexplanation = bugzilla_status

class Bugzilla:

    """Representation of a bugzilla instance"""

    def __init__(self, conn):

        if conn is not None:

            self.backend = BugzillaBackend(conn)

        else:

            self.backend = None

        self.ubuntu = getUtility(ILaunchpadCelebrities).ubuntu

        self.debian = getUtility(ILaunchpadCelebrities).debian

        self.bugtracker = getUtility(ILaunchpadCelebrities).ubuntu_bugzilla

        self.debbugs = getUtility(ILaunchpadCelebrities).debbugs

        self.bugset = getUtility(IBugSet)

        self.bugtaskset = getUtility(IBugTaskSet)

        self.bugwatchset = getUtility(IBugWatchSet)

        self.cveset = getUtility(ICveSet)

        self.personset = getUtility(IPersonSet)

        self.emailset = getUtility(IEmailAddressSet)

        self.person_mapping = {}

    def person(self, bugzilla_id):

        """Get the Launchpad person corresponding to the given Bugzilla ID"""

        # Bugzilla treats a user ID of 0 as a NULL

        if bugzilla_id == 0:

            return None

        # Try and get the person using a cache of the mapping.  We

        # check to make sure the person still exists and has not been

        # merged.

        person = None

        launchpad_id = self.person_mapping.get(bugzilla_id)

        if launchpad_id is not None:

            person = self.personset.get(launchpad_id)

            if person is not None and person.merged is not None:

                person = None

        # look up the person

        if person is None:

            email, displayname = self.backend.lookupUser(bugzilla_id)

            person = self.personset.ensurePerson(

                email, displayname, PersonCreationRationale.BUGIMPORT,

                comment=('when importing bugs from %s'

                         % self.bugtracker.baseurl))

            # Bugzilla performs similar address checks to Launchpad, so

            # if the Launchpad account has no preferred email, use the

            # Bugzilla one.

            emailaddr = self.emailset.getByEmail(email)

            assert emailaddr is not None

            if person.preferredemail != emailaddr:

                person.validateAndEnsurePreferredEmail(emailaddr)

            self.person_mapping[bugzilla_id] = person.id

        return person

    def _getPackageNames(self, bug):

        """Returns the source and binary package names for the given bug."""

        # we currently only support mapping Ubuntu bugs ...

        if bug.product != 'Ubuntu':

            raise AssertionError('product must be Ubuntu')

        # kernel bugs are currently filed against the "linux"

        # component, which is not a source or binary package.  The

        # following mapping was provided by BenC:

        if bug.component == 'linux':

            cutoffdate = datetime.datetime(2004, 12, 1,

                                           tzinfo=pytz.timezone('UTC'))

            if bug.bug_status == 'NEEDINFO' and bug.creation_ts < cutoffdate:

                pkgname = 'linux-source-2.6.12'

            else:

                pkgname = 'linux-source-2.6.15'

        else:

            pkgname = bug.component.encode('ASCII')

        try:

            srcpkg, binpkg = self.ubuntu.guessPackageNames(pkgname)

        except NotFoundError, e:

            logger.warning('could not find package name for "%s": %s',

                           pkgname, str(e))

            srcpkg = binpkg = None

        return srcpkg, binpkg

    def getLaunchpadBugTarget(self, bug):

        """Returns a dictionary of arguments to createBug() that correspond

        to the given bugzilla bug.

        """

        srcpkg, binpkg = self._getPackageNames(bug)

        return {

            'distribution': self.ubuntu,

            'sourcepackagename': srcpkg,

            }

    def getLaunchpadMilestone(self, bug):

        """Return the Launchpad milestone for a Bugzilla bug.

        If the milestone does not exist, then it is created.

        """

        if bug.product != 'Ubuntu':

            raise AssertionError('product must be Ubuntu')

        # Bugzilla uses a value of "---" to represent "no selected Milestone"

        # Launchpad represents this by setting the milestone column to NULL.

        if bug.target_milestone is None or bug.target_milestone == '---':

            return None

        # generate a Launchpad name from the Milestone name:

        name = re.sub(r'[^a-z0-9\+\.\-]', '-', bug.target_milestone.lower())

        milestone = self.ubuntu.getMilestone(name)

        if milestone is None:

            milestone = self.ubuntu.currentseries.newMilestone(name)

            Store.of(milestone).flush()

        return milestone

    def getLaunchpadUpstreamProduct(self, bug):

        """Find the upstream product for the given Bugzilla bug.

        This function relies on the package -> product linkage having been

        entered in advance.

        """

        srcpkgname, binpkgname = self._getPackageNames(bug)

        # find a product series

        series = None

        for series in self.ubuntu.series:

            srcpkg = series.getSourcePackage(srcpkgname)

            if srcpkg:

                series = srcpkg.productseries

                if series:

                    return series.product

        else:

            logger.warning('could not find upstream product for '

                           'source package "%s"', srcpkgname.name)

            return None

    _bug_re = re.compile('bug\s*#?\s*(?P<id>\d+)', re.IGNORECASE)

    def replaceBugRef(self, match):

        # XXX: jamesh 2005-10-24:

        # this is where bug number rewriting would be plugged in

        bug_id = int(match.group('id'))

        url = '%s/%d' % (canonical_url(self.bugtracker), bug_id)

        return '%s [%s]' % (match.group(0), url)

    def handleBug(self, bug_id):

        """Maybe import a single bug.

        If the bug has already been imported (detected by checking for

        a bug watch), it is skipped.

        """

        logger.info('Handling Bugzilla bug %d', bug_id)

        # is there a bug watch on the bug?

        lp_bug = self.bugset.queryByRemoteBug(self.bugtracker, bug_id)

        # if we already have an associated bug, don't add a new one.

        if lp_bug is not None:

            logger.info('Bugzilla bug %d is already being watched by '

                        'Launchpad bug %d', bug_id, lp_bug.id)

            return lp_bug

        bug = Bug(self.backend, bug_id)

        comments = bug.comments[:]

        # create a message for the initial comment:

        msgset = getUtility(IMessageSet)

        who, when, text = comments.pop(0)

        text = self._bug_re.sub(self.replaceBugRef, text)

        # If a URL is associated with the bug, add it to the description:

        if bug.bug_file_loc:

            text = text + '\n\n' + bug.bug_file_loc

        # the initial comment can't be empty:

        if not text.strip():

            text = '<empty comment>'

        msg = msgset.fromText(bug.short_desc, text, self.person(who), when)

        # create the bug

        target = self.getLaunchpadBugTarget(bug)

        params = CreateBugParams(

            msg=msg, datecreated=bug.creation_ts, title=bug.short_desc,

            owner=self.person(bug.reporter))

        params.setBugTarget(**target)

        lp_bug = self.bugset.createBug(params)

        # add the bug watch:

        lp_bug.addWatch(self.bugtracker, str(bug.bug_id), lp_bug.owner)

        # add remaining comments, and add CVEs found in all text

        lp_bug.findCvesInText(text, lp_bug.owner)

        for (who, when, text) in comments:

            text = self._bug_re.sub(self.replaceBugRef, text)

            msg = msgset.fromText(msg.followup_title, text,

                                  self.person(who), when)

            lp_bug.linkMessage(msg)

        # subscribe QA contact and CC's

        if bug.qa_contact:

            lp_bug.subscribe(

                self.person(bug.qa_contact), self.person(bug.reporter))

        for cc in bug.ccs:

            lp_bug.subscribe(self.person(cc), self.person(bug.reporter))

        # translate bugzilla status and severity to LP equivalents

        task = lp_bug.bugtasks[0]

        task.datecreated = bug.creation_ts

        task.transitionToAssignee(self.person(bug.assigned_to))

        task.statusexplanation = bug.status_whiteboard

        bug.mapSeverity(task)

        bug.mapStatus(task)

        # bugs with an alias of the form "deb1234" have been imported

        # from the Debian bug tracker by the "debzilla" program.  For

        # these bugs, generate a task and watch on the corresponding

        # bugs.debian.org bug.

        if bug.alias:

            if re.match(r'^deb\d+$', bug.alias):

                watch = self.bugwatchset.createBugWatch(

                    lp_bug, lp_bug.owner, self.debbugs, bug.alias[3:])

                debtask = self.bugtaskset.createTask(

                    lp_bug,

                    owner=lp_bug.owner,

                    distribution=self.debian,

                    sourcepackagename=target['sourcepackagename'])

                debtask.datecreated = bug.creation_ts

                debtask.bugwatch = watch

            else:

                # generate a Launchpad name from the alias:

                name = re.sub(r'[^a-z0-9\+\.\-]', '-', bug.alias.lower())

                lp_bug.name = name

        # for UPSTREAM bugs, try to find whether the URL field contains

        # a bug reference.

        if bug.bug_status == 'UPSTREAM':

            # see if the URL field contains a bug tracker reference

            watches = self.bugwatchset.fromText(bug.bug_file_loc,

                                                lp_bug, lp_bug.owner)

            # find the upstream product for this bug

            product = self.getLaunchpadUpstreamProduct(bug)

            # if we created a watch, and there is an upstream product,

            # create a new task and link it to the watch.

            if len(watches) > 0:

                if product:

                    upstreamtask = self.bugtaskset.createTask(

                        lp_bug, product=product, owner=lp_bug.owner)

                    upstreamtask.datecreated = bug.creation_ts

                    upstreamtask.bugwatch = watches[0]

                else:

                    logger.warning('Could not find upstream product to link '

                                   'bug %d to', lp_bug.id)

        # translate milestone linkage

        task.milestone = self.getLaunchpadMilestone(bug)

        # import attachments

        for (attach_id, creation_ts, description, mimetype, ispatch,

             filename, thedata, submitter_id) in bug.attachments:

            # if the filename is missing for some reason, use a generic one.

            if filename is None or filename.strip() == '':

                filename = 'untitled'

            logger.debug('Creating attachment %s for bug %d',

                         filename, bug.bug_id)

            if ispatch:

                attach_type = BugAttachmentType.PATCH

                mimetype = 'text/plain'

            else:

                attach_type = BugAttachmentType.UNSPECIFIED

            # look for a message starting with "Created an attachment (id=NN)"

            for msg in lp_bug.messages:

                if msg.text_contents.startswith(

                        'Created an attachment (id=%d)' % attach_id):

                    break

            else:

                # could not find the add message, so create one:

                msg = msgset.fromText(description,

                                      'Created attachment %s' % filename,

                                      self.person(submitter_id),

                                      creation_ts)

                lp_bug.linkMessage(msg)

            filealias = getUtility(ILibraryFileAliasSet).create(

                name=filename,

                size=len(thedata),

                file=StringIO(thedata),

                contentType=mimetype)

            getUtility(IBugAttachmentSet).create(

                bug=lp_bug, filealias=filealias, attach_type=attach_type,

                title=description, message=msg)

        return lp_bug

    def processDuplicates(self, trans):

        """Mark Launchpad bugs as duplicates based on Bugzilla duplicates.

        Launchpad bug A will be marked as a duplicate of bug B if:

         * bug A watches bugzilla bug A'

         * bug B watches bugzilla bug B'

         * bug A' is a duplicate of bug B'

         * bug A is not currently a duplicate of any other bug.

        """

        logger.info('Processing duplicate bugs')

        bugmap = {}

        def getlpbug(bugid):

            """Get the Launchpad bug corresponding to the given remote ID