1
# Copyright 2009 Canonical Ltd. This software is licensed under the
2
# GNU Affero General Public License version 3 (see the file LICENSE).
4
"""Bugzilla to Launchpad import logic"""
8
# http://lxr.mozilla.org/mozilla/source/webtools/bugzilla/Bugzilla/DB/Schema.pm
10
# XXX: jamesh 2005-10-18
11
# Currently unhandled bug info:
12
# * Operating system and platform
13
# * version (not really used in Ubuntu bugzilla though)
15
# * private bugs (none of the canonical-only bugs seem sensitive though)
17
# * "bug XYZ" references inside comment text (at the moment we just
18
# insert the full URL to the bug afterwards).
20
# Not all of these are necessary though
24
from cStringIO import StringIO
30
from storm.store import Store
31
from zope.component import getUtility
33
from canonical.launchpad.interfaces.emailaddress import IEmailAddressSet
34
from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
35
from canonical.launchpad.webapp import canonical_url
36
from lp.app.errors import NotFoundError
37
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
38
from lp.bugs.interfaces.bug import (
42
from lp.bugs.interfaces.bugattachment import (
46
from lp.bugs.interfaces.bugtask import (
51
from lp.bugs.interfaces.bugwatch import IBugWatchSet
52
from lp.bugs.interfaces.cve import ICveSet
53
from lp.registry.interfaces.person import (
55
PersonCreationRationale,
57
from lp.services.messages.interfaces.message import IMessageSet
60
logger = logging.getLogger('lp.bugs.scripts.bugzilla')
64
"""Convert a naiive datetime value to a UTC datetime value."""
65
assert dt.tzinfo is None, 'add_tz() only accepts naiive datetime values'
66
return datetime.datetime(dt.year, dt.month, dt.day,
67
dt.hour, dt.minute, dt.second,
68
dt.microsecond, tzinfo=pytz.timezone('UTC'))
71
class BugzillaBackend:
72
"""A wrapper for all the MySQL database access.
74
The main purpose of this is to make it possible to test the rest
75
of the import code without access to a MySQL database.
77
def __init__(self, conn, charset='UTF-8'):
79
self.cursor = conn.cursor()
80
self.charset = charset
84
value = s.decode(self.charset, 'replace')
85
# Postgres doesn't like values outside of the basic multilingual
86
# plane (U+0000 - U+FFFF), so replace them (and surrogates) with
87
# U+FFFD (replacement character).
88
# Existance of these characters generally indicate an encoding
89
# problem in the existing Bugzilla data.
90
return re.sub(u'[^\u0000-\ud7ff\ue000-\uffff]', u'\ufffd', value)
94
def lookupUser(self, user_id):
95
"""Look up information about a particular Bugzilla user ID"""
96
self.cursor.execute('SELECT login_name, realname '
98
' WHERE userid = %d' % user_id)
99
if self.cursor.rowcount != 1:
100
raise NotFoundError('could not look up user %d' % user_id)
101
(login_name, realname) = self.cursor.fetchone()
102
realname = self._decode(realname)
103
return (login_name, realname)
105
def getBugInfo(self, bug_id):
106
"""Retrieve information about a bug."""
108
'SELECT bug_id, assigned_to, bug_file_loc, bug_severity, '
109
' bug_status, creation_ts, short_desc, op_sys, priority, '
110
' products.name, rep_platform, reporter, version, '
111
' components.name, resolution, target_milestone, qa_contact, '
112
' status_whiteboard, keywords, alias '
114
' INNER JOIN products ON bugs.product_id = products.id '
115
' INNER JOIN components ON bugs.component_id = components.id '
116
' WHERE bug_id = %d' % bug_id)
117
if self.cursor.rowcount != 1:
118
raise NotFoundError('could not look up bug %d' % bug_id)
119
(bug_id, assigned_to, bug_file_loc, bug_severity, bug_status,
120
creation_ts, short_desc, op_sys, priority, product,
121
rep_platform, reporter, version, component, resolution,
122
target_milestone, qa_contact, status_whiteboard, keywords,
123
alias) = self.cursor.fetchone()
125
bug_file_loc = self._decode(bug_file_loc)
126
creation_ts = _add_tz(creation_ts)
127
product = self._decode(product)
128
version = self._decode(version)
129
component = self._decode(component)
130
status_whiteboard = self._decode(status_whiteboard)
131
keywords = self._decode(keywords)
132
alias = self._decode(alias)
134
return (bug_id, assigned_to, bug_file_loc, bug_severity,
135
bug_status, creation_ts, short_desc, op_sys, priority,
136
product, rep_platform, reporter, version, component,
137
resolution, target_milestone, qa_contact,
138
status_whiteboard, keywords, alias)
140
def getBugCcs(self, bug_id):
141
"""Get the IDs of the people CC'd to the bug."""
142
self.cursor.execute('SELECT who FROM cc WHERE bug_id = %d'
144
return [row[0] for row in self.cursor.fetchall()]
146
def getBugComments(self, bug_id):
147
"""Get the comments for the bug."""
148
self.cursor.execute('SELECT who, bug_when, thetext '
150
' WHERE bug_id = %d '
151
' ORDER BY bug_when' % bug_id)
152
# XXX: jamesh 2005-12-07:
153
# Due to a bug in Debzilla, Ubuntu bugzilla bug 248 has > 7800
154
# duplicate comments,consisting of someone's signature.
155
# For the import, just ignore those comments.
156
return [(who, _add_tz(when), self._decode(thetext))
157
for (who, when, thetext) in self.cursor.fetchall()
158
if thetext != '\n--=20\n Jacobo Tarr=EDo | '
159
'http://jacobo.tarrio.org/\n\n\n']
161
def getBugAttachments(self, bug_id):
162
"""Get the attachments for the bug."""
163
self.cursor.execute('SELECT attach_id, creation_ts, description, '
164
' mimetype, ispatch, filename, thedata, '
167
' WHERE bug_id = %d '
168
' ORDER BY attach_id' % bug_id)
169
return [(attach_id, _add_tz(creation_ts),
170
self._decode(description), mimetype,
171
ispatch, self._decode(filename), thedata, submitter_id)
172
for (attach_id, creation_ts, description,
173
mimetype, ispatch, filename, thedata,
174
submitter_id) in self.cursor.fetchall()]
176
def findBugs(self, product=None, component=None, status=None):
177
"""Returns the requested bug IDs as a list"""
180
if component is None:
188
'INNER JOIN products ON bugs.product_id = products.id')
189
conditions.append('products.name IN (%s)' %
190
', '.join([self.conn.escape(p) for p in product]))
193
'INNER JOIN components ON bugs.component_id = components.id')
194
conditions.append('components.name IN (%s)' %
195
', '.join([self.conn.escape(c) for c in component]))
197
conditions.append('bugs.bug_status IN (%s)' %
198
', '.join([self.conn.escape(s) for s in status]))
200
conditions = 'WHERE %s' % ' AND '.join(conditions)
203
self.cursor.execute('SELECT bug_id FROM bugs %s %s ORDER BY bug_id' %
204
(' '.join(joins), conditions))
205
return [bug_id for (bug_id,) in self.cursor.fetchall()]
207
def getDuplicates(self):
208
"""Returns a list of (dupe_of, dupe) relations."""
209
self.cursor.execute('SELECT dupe_of, dupe FROM duplicates '
210
'ORDER BY dupe, dupe_of')
211
return [(dupe_of, dupe) for (dupe_of, dupe) in self.cursor.fetchall()]
215
"""Representation of a Bugzilla Bug"""
216
def __init__(self, backend, bug_id):
217
self.backend = backend
218
(self.bug_id, self.assigned_to, self.bug_file_loc, self.bug_severity,
219
self.bug_status, self.creation_ts, self.short_desc, self.op_sys,
220
self.priority, self.product, self.rep_platform, self.reporter,
221
self.version, self.component, self.resolution,
222
self.target_milestone, self.qa_contact, self.status_whiteboard,
223
self.keywords, self.alias) = backend.getBugInfo(bug_id)
226
self._comments = None
227
self._attachments = None
231
"""Return the IDs of people CC'd to this bug"""
232
if self._ccs is not None:
234
self._ccs = self.backend.getBugCcs(self.bug_id)
239
"""Return the comments attached to this bug"""
240
if self._comments is not None:
241
return self._comments
242
self._comments = self.backend.getBugComments(self.bug_id)
243
return self._comments
246
def attachments(self):
247
"""Return the attachments for this bug"""
248
if self._attachments is not None:
249
return self._attachments
250
self._attachments = self.backend.getBugAttachments(self.bug_id)
251
return self._attachments
253
def mapSeverity(self, bugtask):
254
"""Set a Launchpad bug task's importance based on this bug's severity.
256
bug_importer = getUtility(ILaunchpadCelebrities).bug_importer
258
'blocker': BugTaskImportance.CRITICAL,
259
'critical': BugTaskImportance.CRITICAL,
260
'major': BugTaskImportance.HIGH,
261
'normal': BugTaskImportance.MEDIUM,
262
'minor': BugTaskImportance.LOW,
263
'trivial': BugTaskImportance.LOW,
264
'enhancement': BugTaskImportance.WISHLIST
266
importance = importance_map.get(
267
self.bug_severity, BugTaskImportance.UNKNOWN)
268
bugtask.transitionToImportance(importance, bug_importer)
270
def mapStatus(self, bugtask):
271
"""Set a Launchpad bug task's status based on this bug's status.
273
If the bug is in the RESOLVED, VERIFIED or CLOSED states, the
274
bug resolution is also taken into account when mapping the
277
Additional information about the bugzilla status is appended
278
to the bug task's status explanation.
280
bug_importer = getUtility(ILaunchpadCelebrities).bug_importer
282
if self.bug_status == 'ASSIGNED':
283
bugtask.transitionToStatus(
284
BugTaskStatus.CONFIRMED, bug_importer)
285
elif self.bug_status == 'NEEDINFO':
286
bugtask.transitionToStatus(
287
BugTaskStatus.INCOMPLETE, bug_importer)
288
elif self.bug_status == 'PENDINGUPLOAD':
289
bugtask.transitionToStatus(
290
BugTaskStatus.FIXCOMMITTED, bug_importer)
291
elif self.bug_status in ['RESOLVED', 'VERIFIED', 'CLOSED']:
292
# depends on the resolution:
293
if self.resolution == 'FIXED':
294
bugtask.transitionToStatus(
295
BugTaskStatus.FIXRELEASED, bug_importer)
297
bugtask.transitionToStatus(
298
BugTaskStatus.INVALID, bug_importer)
300
bugtask.transitionToStatus(
301
BugTaskStatus.NEW, bug_importer)
303
# add the status to the notes section, to account for any lost
305
bugzilla_status = 'Bugzilla status=%s' % self.bug_status
307
bugzilla_status += ' %s' % self.resolution
308
bugzilla_status += ', product=%s' % self.product
309
bugzilla_status += ', component=%s' % self.component
311
if bugtask.statusexplanation:
312
bugtask.statusexplanation = '%s (%s)' % (
313
bugtask.statusexplanation, bugzilla_status)
315
bugtask.statusexplanation = bugzilla_status
319
"""Representation of a bugzilla instance"""
321
def __init__(self, conn):
323
self.backend = BugzillaBackend(conn)
326
self.ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
327
self.debian = getUtility(ILaunchpadCelebrities).debian
328
self.bugtracker = getUtility(ILaunchpadCelebrities).ubuntu_bugzilla
329
self.debbugs = getUtility(ILaunchpadCelebrities).debbugs
330
self.bugset = getUtility(IBugSet)
331
self.bugtaskset = getUtility(IBugTaskSet)
332
self.bugwatchset = getUtility(IBugWatchSet)
333
self.cveset = getUtility(ICveSet)
334
self.personset = getUtility(IPersonSet)
335
self.emailset = getUtility(IEmailAddressSet)
336
self.person_mapping = {}
338
def person(self, bugzilla_id):
339
"""Get the Launchpad person corresponding to the given Bugzilla ID"""
340
# Bugzilla treats a user ID of 0 as a NULL
344
# Try and get the person using a cache of the mapping. We
345
# check to make sure the person still exists and has not been
348
launchpad_id = self.person_mapping.get(bugzilla_id)
349
if launchpad_id is not None:
350
person = self.personset.get(launchpad_id)
351
if person is not None and person.merged is not None:
356
email, displayname = self.backend.lookupUser(bugzilla_id)
358
person = self.personset.ensurePerson(
359
email, displayname, PersonCreationRationale.BUGIMPORT,
360
comment=('when importing bugs from %s'
361
% self.bugtracker.baseurl))
363
# Bugzilla performs similar address checks to Launchpad, so
364
# if the Launchpad account has no preferred email, use the
366
emailaddr = self.emailset.getByEmail(email)
367
assert emailaddr is not None
368
if person.preferredemail != emailaddr:
369
person.validateAndEnsurePreferredEmail(emailaddr)
371
self.person_mapping[bugzilla_id] = person.id
375
def _getPackageName(self, bug):
376
"""Returns the source package name for the given bug."""
377
# we currently only support mapping Ubuntu bugs ...
378
if bug.product != 'Ubuntu':
379
raise AssertionError('product must be Ubuntu')
381
# kernel bugs are currently filed against the "linux"
382
# component, which is not a source or binary package. The
383
# following mapping was provided by BenC:
384
if bug.component == 'linux':
385
cutoffdate = datetime.datetime(2004, 12, 1,
386
tzinfo=pytz.timezone('UTC'))
387
if bug.bug_status == 'NEEDINFO' and bug.creation_ts < cutoffdate:
388
pkgname = 'linux-source-2.6.12'
390
pkgname = 'linux-source-2.6.15'
392
pkgname = bug.component.encode('ASCII')
395
return self.ubuntu.guessPublishedSourcePackageName(pkgname)
396
except NotFoundError, e:
397
logger.warning('could not find package name for "%s": %s',
401
def getLaunchpadBugTarget(self, bug):
402
"""Returns a dictionary of arguments to createBug() that correspond
403
to the given bugzilla bug.
405
srcpkg = self._getPackageName(bug)
407
'distribution': self.ubuntu,
408
'sourcepackagename': srcpkg,
411
def getLaunchpadMilestone(self, bug):
412
"""Return the Launchpad milestone for a Bugzilla bug.
414
If the milestone does not exist, then it is created.
416
if bug.product != 'Ubuntu':
417
raise AssertionError('product must be Ubuntu')
419
# Bugzilla uses a value of "---" to represent "no selected Milestone"
420
# Launchpad represents this by setting the milestone column to NULL.
421
if bug.target_milestone is None or bug.target_milestone == '---':
424
# generate a Launchpad name from the Milestone name:
425
name = re.sub(r'[^a-z0-9\+\.\-]', '-', bug.target_milestone.lower())
427
milestone = self.ubuntu.getMilestone(name)
428
if milestone is None:
429
milestone = self.ubuntu.currentseries.newMilestone(name)
430
Store.of(milestone).flush()
433
def getLaunchpadUpstreamProduct(self, bug):
434
"""Find the upstream product for the given Bugzilla bug.
436
This function relies on the package -> product linkage having been
439
srcpkgname = self._getPackageName(bug)
440
# find a product series
442
for series in self.ubuntu.series:
443
srcpkg = series.getSourcePackage(srcpkgname)
445
series = srcpkg.productseries
447
return series.product
449
logger.warning('could not find upstream product for '
450
'source package "%s"', srcpkgname.name)
453
_bug_re = re.compile('bug\s*#?\s*(?P<id>\d+)', re.IGNORECASE)
455
def replaceBugRef(self, match):
456
# XXX: jamesh 2005-10-24:
457
# this is where bug number rewriting would be plugged in
458
bug_id = int(match.group('id'))
459
url = '%s/%d' % (canonical_url(self.bugtracker), bug_id)
460
return '%s [%s]' % (match.group(0), url)
462
def handleBug(self, bug_id):
463
"""Maybe import a single bug.
465
If the bug has already been imported (detected by checking for
466
a bug watch), it is skipped.
468
logger.info('Handling Bugzilla bug %d', bug_id)
470
# is there a bug watch on the bug?
471
lp_bug = self.bugset.queryByRemoteBug(self.bugtracker, bug_id)
473
# if we already have an associated bug, don't add a new one.
474
if lp_bug is not None:
475
logger.info('Bugzilla bug %d is already being watched by '
476
'Launchpad bug %d', bug_id, lp_bug.id)
479
bug = Bug(self.backend, bug_id)
481
comments = bug.comments[:]
483
# create a message for the initial comment:
484
msgset = getUtility(IMessageSet)
485
who, when, text = comments.pop(0)
486
text = self._bug_re.sub(self.replaceBugRef, text)
487
# If a URL is associated with the bug, add it to the description:
489
text = text + '\n\n' + bug.bug_file_loc
490
# the initial comment can't be empty:
492
text = '<empty comment>'
493
msg = msgset.fromText(bug.short_desc, text, self.person(who), when)
496
target = self.getLaunchpadBugTarget(bug)
497
params = CreateBugParams(
498
msg=msg, datecreated=bug.creation_ts, title=bug.short_desc,
499
owner=self.person(bug.reporter))
500
params.setBugTarget(**target)
501
lp_bug = self.bugset.createBug(params)
504
lp_bug.addWatch(self.bugtracker, str(bug.bug_id), lp_bug.owner)
506
# add remaining comments, and add CVEs found in all text
507
lp_bug.findCvesInText(text, lp_bug.owner)
508
for (who, when, text) in comments:
509
text = self._bug_re.sub(self.replaceBugRef, text)
510
msg = msgset.fromText(msg.followup_title, text,
511
self.person(who), when)
512
lp_bug.linkMessage(msg)
514
# subscribe QA contact and CC's
517
self.person(bug.qa_contact), self.person(bug.reporter))
519
lp_bug.subscribe(self.person(cc), self.person(bug.reporter))
521
# translate bugzilla status and severity to LP equivalents
522
task = lp_bug.bugtasks[0]
523
task.datecreated = bug.creation_ts
524
task.transitionToAssignee(self.person(bug.assigned_to))
525
task.statusexplanation = bug.status_whiteboard
526
bug.mapSeverity(task)
529
# bugs with an alias of the form "deb1234" have been imported
530
# from the Debian bug tracker by the "debzilla" program. For
531
# these bugs, generate a task and watch on the corresponding
532
# bugs.debian.org bug.
534
if re.match(r'^deb\d+$', bug.alias):
535
watch = self.bugwatchset.createBugWatch(
536
lp_bug, lp_bug.owner, self.debbugs, bug.alias[3:])
537
debtask = self.bugtaskset.createTask(
540
distribution=self.debian,
541
sourcepackagename=target['sourcepackagename'])
542
debtask.datecreated = bug.creation_ts
543
debtask.bugwatch = watch
545
# generate a Launchpad name from the alias:
546
name = re.sub(r'[^a-z0-9\+\.\-]', '-', bug.alias.lower())
549
# for UPSTREAM bugs, try to find whether the URL field contains
551
if bug.bug_status == 'UPSTREAM':
552
# see if the URL field contains a bug tracker reference
553
watches = self.bugwatchset.fromText(bug.bug_file_loc,
554
lp_bug, lp_bug.owner)
555
# find the upstream product for this bug
556
product = self.getLaunchpadUpstreamProduct(bug)
558
# if we created a watch, and there is an upstream product,
559
# create a new task and link it to the watch.
562
upstreamtask = self.bugtaskset.createTask(
563
lp_bug, product=product, owner=lp_bug.owner)
564
upstreamtask.datecreated = bug.creation_ts
565
upstreamtask.bugwatch = watches[0]
567
logger.warning('Could not find upstream product to link '
568
'bug %d to', lp_bug.id)
570
# translate milestone linkage
571
task.milestone = self.getLaunchpadMilestone(bug)
574
for (attach_id, creation_ts, description, mimetype, ispatch,
575
filename, thedata, submitter_id) in bug.attachments:
576
# if the filename is missing for some reason, use a generic one.
577
if filename is None or filename.strip() == '':
578
filename = 'untitled'
579
logger.debug('Creating attachment %s for bug %d',
580
filename, bug.bug_id)
582
attach_type = BugAttachmentType.PATCH
583
mimetype = 'text/plain'
585
attach_type = BugAttachmentType.UNSPECIFIED
587
# look for a message starting with "Created an attachment (id=NN)"
588
for msg in lp_bug.messages:
589
if msg.text_contents.startswith(
590
'Created an attachment (id=%d)' % attach_id):
593
# could not find the add message, so create one:
594
msg = msgset.fromText(description,
595
'Created attachment %s' % filename,
596
self.person(submitter_id),
598
lp_bug.linkMessage(msg)
600
filealias = getUtility(ILibraryFileAliasSet).create(
603
file=StringIO(thedata),
604
contentType=mimetype)
606
getUtility(IBugAttachmentSet).create(
607
bug=lp_bug, filealias=filealias, attach_type=attach_type,
608
title=description, message=msg)
612
def processDuplicates(self, trans):
613
"""Mark Launchpad bugs as duplicates based on Bugzilla duplicates.
615
Launchpad bug A will be marked as a duplicate of bug B if:
616
* bug A watches bugzilla bug A'
617
* bug B watches bugzilla bug B'
618
* bug A' is a duplicate of bug B'
619
* bug A is not currently a duplicate of any other bug.
622
logger.info('Processing duplicate bugs')
626
"""Get the Launchpad bug corresponding to the given remote ID
628
This function makes use of a cache dictionary to reduce the
631
lpbugid = bugmap.get(bugid)
632
if lpbugid is not None:
634
lpbug = self.bugset.get(lpbugid)
638
lpbug = self.bugset.queryByRemoteBug(self.bugtracker, bugid)
639
if lpbug is not None:
640
bugmap[bugid] = lpbug.id
645
for (dupe_of, dupe) in self.backend.getDuplicates():
646
# get the Launchpad bugs corresponding to the two Bugzilla bugs:
648
lpdupe_of = getlpbug(dupe_of)
649
lpdupe = getlpbug(dupe)
650
# if both bugs exist in Launchpad, and lpdupe is not already
651
# a duplicate, mark it as a duplicate of lpdupe_of.
652
if (lpdupe_of is not None and lpdupe is not None and
653
lpdupe.duplicateof is None):
654
logger.info('Marking %d as a duplicate of %d',
655
lpdupe.id, lpdupe_of.id)
656
lpdupe.markAsDuplicate(lpdupe_of)
659
def importBugs(self, trans, product=None, component=None, status=None):
660
"""Import Bugzilla bugs matching the given constraints.
662
Each of product, component and status gives a list of
663
products, components or statuses to limit the import to. An
664
empty list matches all products, components or statuses.
668
if component is None:
673
bugs = self.backend.findBugs(product=product,
679
self.handleBug(bug_id)
680
except (SystemExit, KeyboardInterrupt):
683
logger.exception('Could not import Bugzilla bug #%d', bug_id)
added
removed
lib/lp/bugs/scripts/bugzilla.py
