1
# Copyright 2009 Canonical Ltd. This software is licensed under the
2
# GNU Affero General Public License version 3 (see the file LICENSE).
4
"""Bugzilla to Launchpad import logic"""
8
# http://lxr.mozilla.org/mozilla/source/webtools/bugzilla/Bugzilla/DB/Schema.pm
10
# XXX: jamesh 2005-10-18
11
# Currently unhandled bug info:
12
# * Operating system and platform
13
# * version (not really used in Ubuntu bugzilla though)
15
# * private bugs (none of the canonical-only bugs seem sensitive though)
17
# * "bug XYZ" references inside comment text (at the moment we just
18
# insert the full URL to the bug afterwards).
20
# Not all of these are necessary though
24
from cStringIO import StringIO
30
from storm.store import Store
31
from zope.component import getUtility
33
from canonical.launchpad.interfaces.emailaddress import IEmailAddressSet
34
from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
35
from canonical.launchpad.webapp import canonical_url
36
from lp.app.errors import NotFoundError
37
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
38
from lp.bugs.interfaces.bug import (
42
from lp.bugs.interfaces.bugattachment import (
46
from lp.bugs.interfaces.bugtask import (
51
from lp.bugs.interfaces.bugwatch import IBugWatchSet
52
from lp.bugs.interfaces.cve import ICveSet
53
from lp.registry.interfaces.person import (
55
PersonCreationRationale,
57
from lp.services.messages.interfaces.message import IMessageSet
60
logger = logging.getLogger('lp.bugs.scripts.bugzilla')
63
"""Convert a naiive datetime value to a UTC datetime value."""
64
assert dt.tzinfo is None, 'add_tz() only accepts naiive datetime values'
65
return datetime.datetime(dt.year, dt.month, dt.day,
66
dt.hour, dt.minute, dt.second,
67
dt.microsecond, tzinfo=pytz.timezone('UTC'))
69
class BugzillaBackend:
70
"""A wrapper for all the MySQL database access.
72
The main purpose of this is to make it possible to test the rest
73
of the import code without access to a MySQL database.
75
def __init__(self, conn, charset='UTF-8'):
77
self.cursor = conn.cursor()
78
self.charset = charset
82
value = s.decode(self.charset, 'replace')
83
# Postgres doesn't like values outside of the basic multilingual
84
# plane (U+0000 - U+FFFF), so replace them (and surrogates) with
85
# U+FFFD (replacement character).
86
# Existance of these characters generally indicate an encoding
87
# problem in the existing Bugzilla data.
88
return re.sub(u'[^\u0000-\ud7ff\ue000-\uffff]', u'\ufffd', value)
92
def lookupUser(self, user_id):
93
"""Look up information about a particular Bugzilla user ID"""
94
self.cursor.execute('SELECT login_name, realname '
96
' WHERE userid = %d' % user_id)
97
if self.cursor.rowcount != 1:
98
raise NotFoundError('could not look up user %d' % user_id)
99
(login_name, realname) = self.cursor.fetchone()
100
realname = self._decode(realname)
101
return (login_name, realname)
103
def getBugInfo(self, bug_id):
104
"""Retrieve information about a bug."""
106
'SELECT bug_id, assigned_to, bug_file_loc, bug_severity, '
107
' bug_status, creation_ts, short_desc, op_sys, priority, '
108
' products.name, rep_platform, reporter, version, '
109
' components.name, resolution, target_milestone, qa_contact, '
110
' status_whiteboard, keywords, alias '
112
' INNER JOIN products ON bugs.product_id = products.id '
113
' INNER JOIN components ON bugs.component_id = components.id '
114
' WHERE bug_id = %d' % bug_id)
115
if self.cursor.rowcount != 1:
116
raise NotFoundError('could not look up bug %d' % bug_id)
117
(bug_id, assigned_to, bug_file_loc, bug_severity, bug_status,
118
creation_ts, short_desc, op_sys, priority, product,
119
rep_platform, reporter, version, component, resolution,
120
target_milestone, qa_contact, status_whiteboard, keywords,
121
alias) = self.cursor.fetchone()
123
bug_file_loc = self._decode(bug_file_loc)
124
creation_ts = _add_tz(creation_ts)
125
product = self._decode(product)
126
version = self._decode(version)
127
component = self._decode(component)
128
status_whiteboard = self._decode(status_whiteboard)
129
keywords = self._decode(keywords)
130
alias = self._decode(alias)
132
return (bug_id, assigned_to, bug_file_loc, bug_severity,
133
bug_status, creation_ts, short_desc, op_sys, priority,
134
product, rep_platform, reporter, version, component,
135
resolution, target_milestone, qa_contact,
136
status_whiteboard, keywords, alias)
138
def getBugCcs(self, bug_id):
139
"""Get the IDs of the people CC'd to the bug."""
140
self.cursor.execute('SELECT who FROM cc WHERE bug_id = %d'
142
return [row[0] for row in self.cursor.fetchall()]
144
def getBugComments(self, bug_id):
145
"""Get the comments for the bug."""
146
self.cursor.execute('SELECT who, bug_when, thetext '
148
' WHERE bug_id = %d '
149
' ORDER BY bug_when' % bug_id)
150
# XXX: jamesh 2005-12-07:
151
# Due to a bug in Debzilla, Ubuntu bugzilla bug 248 has > 7800
152
# duplicate comments,consisting of someone's signature.
153
# For the import, just ignore those comments.
154
return [(who, _add_tz(when), self._decode(thetext))
155
for (who, when, thetext) in self.cursor.fetchall()
156
if thetext != '\n--=20\n Jacobo Tarr=EDo | '
157
'http://jacobo.tarrio.org/\n\n\n']
159
def getBugAttachments(self, bug_id):
160
"""Get the attachments for the bug."""
161
self.cursor.execute('SELECT attach_id, creation_ts, description, '
162
' mimetype, ispatch, filename, thedata, '
165
' WHERE bug_id = %d '
166
' ORDER BY attach_id' % bug_id)
167
return [(attach_id, _add_tz(creation_ts),
168
self._decode(description), mimetype,
169
ispatch, self._decode(filename), thedata, submitter_id)
170
for (attach_id, creation_ts, description,
171
mimetype, ispatch, filename, thedata,
172
submitter_id) in self.cursor.fetchall()]
174
def findBugs(self, product=None, component=None, status=None):
175
"""Returns the requested bug IDs as a list"""
178
if component is None:
186
'INNER JOIN products ON bugs.product_id = products.id')
187
conditions.append('products.name IN (%s)' %
188
', '.join([self.conn.escape(p) for p in product]))
191
'INNER JOIN components ON bugs.component_id = components.id')
192
conditions.append('components.name IN (%s)' %
193
', '.join([self.conn.escape(c) for c in component]))
195
conditions.append('bugs.bug_status IN (%s)' %
196
', '.join([self.conn.escape(s) for s in status]))
198
conditions = 'WHERE %s' % ' AND '.join(conditions)
201
self.cursor.execute('SELECT bug_id FROM bugs %s %s ORDER BY bug_id' %
202
(' '.join(joins), conditions))
203
return [bug_id for (bug_id,) in self.cursor.fetchall()]
205
def getDuplicates(self):
206
"""Returns a list of (dupe_of, dupe) relations."""
207
self.cursor.execute('SELECT dupe_of, dupe FROM duplicates '
208
'ORDER BY dupe, dupe_of')
209
return [(dupe_of, dupe) for (dupe_of, dupe) in self.cursor.fetchall()]
212
"""Representation of a Bugzilla Bug"""
213
def __init__(self, backend, bug_id):
214
self.backend = backend
215
(self.bug_id, self.assigned_to, self.bug_file_loc, self.bug_severity,
216
self.bug_status, self.creation_ts, self.short_desc, self.op_sys,
217
self.priority, self.product, self.rep_platform, self.reporter,
218
self.version, self.component, self.resolution,
219
self.target_milestone, self.qa_contact, self.status_whiteboard,
220
self.keywords, self.alias) = backend.getBugInfo(bug_id)
223
self._comments = None
224
self._attachments = None
228
"""Return the IDs of people CC'd to this bug"""
229
if self._ccs is not None:
231
self._ccs = self.backend.getBugCcs(self.bug_id)
236
"""Return the comments attached to this bug"""
237
if self._comments is not None:
238
return self._comments
239
self._comments = self.backend.getBugComments(self.bug_id)
240
return self._comments
243
def attachments(self):
244
"""Return the attachments for this bug"""
245
if self._attachments is not None:
246
return self._attachments
247
self._attachments = self.backend.getBugAttachments(self.bug_id)
248
return self._attachments
250
def mapSeverity(self, bugtask):
251
"""Set a Launchpad bug task's importance based on this bug's severity.
253
bug_importer = getUtility(ILaunchpadCelebrities).bug_importer
255
'blocker': BugTaskImportance.CRITICAL,
256
'critical': BugTaskImportance.CRITICAL,
257
'major': BugTaskImportance.HIGH,
258
'normal': BugTaskImportance.MEDIUM,
259
'minor': BugTaskImportance.LOW,
260
'trivial': BugTaskImportance.LOW,
261
'enhancement': BugTaskImportance.WISHLIST
263
importance = importance_map.get(
264
self.bug_severity, BugTaskImportance.UNKNOWN)
265
bugtask.transitionToImportance(importance, bug_importer)
267
def mapStatus(self, bugtask):
268
"""Set a Launchpad bug task's status based on this bug's status.
270
If the bug is in the RESOLVED, VERIFIED or CLOSED states, the
271
bug resolution is also taken into account when mapping the
274
Additional information about the bugzilla status is appended
275
to the bug task's status explanation.
277
bug_importer = getUtility(ILaunchpadCelebrities).bug_importer
279
if self.bug_status == 'ASSIGNED':
280
bugtask.transitionToStatus(
281
BugTaskStatus.CONFIRMED, bug_importer)
282
elif self.bug_status == 'NEEDINFO':
283
bugtask.transitionToStatus(
284
BugTaskStatus.INCOMPLETE, bug_importer)
285
elif self.bug_status == 'PENDINGUPLOAD':
286
bugtask.transitionToStatus(
287
BugTaskStatus.FIXCOMMITTED, bug_importer)
288
elif self.bug_status in ['RESOLVED', 'VERIFIED', 'CLOSED']:
289
# depends on the resolution:
290
if self.resolution == 'FIXED':
291
bugtask.transitionToStatus(
292
BugTaskStatus.FIXRELEASED, bug_importer)
294
bugtask.transitionToStatus(
295
BugTaskStatus.INVALID, bug_importer)
297
bugtask.transitionToStatus(
298
BugTaskStatus.NEW, bug_importer)
300
# add the status to the notes section, to account for any lost
302
bugzilla_status = 'Bugzilla status=%s' % self.bug_status
304
bugzilla_status += ' %s' % self.resolution
305
bugzilla_status += ', product=%s' % self.product
306
bugzilla_status += ', component=%s' % self.component
308
if bugtask.statusexplanation:
309
bugtask.statusexplanation = '%s (%s)' % (
310
bugtask.statusexplanation, bugzilla_status)
312
bugtask.statusexplanation = bugzilla_status
316
"""Representation of a bugzilla instance"""
318
def __init__(self, conn):
320
self.backend = BugzillaBackend(conn)
323
self.ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
324
self.debian = getUtility(ILaunchpadCelebrities).debian
325
self.bugtracker = getUtility(ILaunchpadCelebrities).ubuntu_bugzilla
326
self.debbugs = getUtility(ILaunchpadCelebrities).debbugs
327
self.bugset = getUtility(IBugSet)
328
self.bugtaskset = getUtility(IBugTaskSet)
329
self.bugwatchset = getUtility(IBugWatchSet)
330
self.cveset = getUtility(ICveSet)
331
self.personset = getUtility(IPersonSet)
332
self.emailset = getUtility(IEmailAddressSet)
333
self.person_mapping = {}
335
def person(self, bugzilla_id):
336
"""Get the Launchpad person corresponding to the given Bugzilla ID"""
337
# Bugzilla treats a user ID of 0 as a NULL
341
# Try and get the person using a cache of the mapping. We
342
# check to make sure the person still exists and has not been
345
launchpad_id = self.person_mapping.get(bugzilla_id)
346
if launchpad_id is not None:
347
person = self.personset.get(launchpad_id)
348
if person is not None and person.merged is not None:
353
email, displayname = self.backend.lookupUser(bugzilla_id)
355
person = self.personset.ensurePerson(
356
email, displayname, PersonCreationRationale.BUGIMPORT,
357
comment=('when importing bugs from %s'
358
% self.bugtracker.baseurl))
360
# Bugzilla performs similar address checks to Launchpad, so
361
# if the Launchpad account has no preferred email, use the
363
emailaddr = self.emailset.getByEmail(email)
364
assert emailaddr is not None
365
if person.preferredemail != emailaddr:
366
person.validateAndEnsurePreferredEmail(emailaddr)
368
self.person_mapping[bugzilla_id] = person.id
372
def _getPackageNames(self, bug):
373
"""Returns the source and binary package names for the given bug."""
374
# we currently only support mapping Ubuntu bugs ...
375
if bug.product != 'Ubuntu':
376
raise AssertionError('product must be Ubuntu')
378
# kernel bugs are currently filed against the "linux"
379
# component, which is not a source or binary package. The
380
# following mapping was provided by BenC:
381
if bug.component == 'linux':
382
cutoffdate = datetime.datetime(2004, 12, 1,
383
tzinfo=pytz.timezone('UTC'))
384
if bug.bug_status == 'NEEDINFO' and bug.creation_ts < cutoffdate:
385
pkgname = 'linux-source-2.6.12'
387
pkgname = 'linux-source-2.6.15'
389
pkgname = bug.component.encode('ASCII')
392
srcpkg, binpkg = self.ubuntu.guessPackageNames(pkgname)
393
except NotFoundError, e:
394
logger.warning('could not find package name for "%s": %s',
396
srcpkg = binpkg = None
398
return srcpkg, binpkg
400
def getLaunchpadBugTarget(self, bug):
401
"""Returns a dictionary of arguments to createBug() that correspond
402
to the given bugzilla bug.
404
srcpkg, binpkg = self._getPackageNames(bug)
406
'distribution': self.ubuntu,
407
'sourcepackagename': srcpkg,
410
def getLaunchpadMilestone(self, bug):
411
"""Return the Launchpad milestone for a Bugzilla bug.
413
If the milestone does not exist, then it is created.
415
if bug.product != 'Ubuntu':
416
raise AssertionError('product must be Ubuntu')
418
# Bugzilla uses a value of "---" to represent "no selected Milestone"
419
# Launchpad represents this by setting the milestone column to NULL.
420
if bug.target_milestone is None or bug.target_milestone == '---':
423
# generate a Launchpad name from the Milestone name:
424
name = re.sub(r'[^a-z0-9\+\.\-]', '-', bug.target_milestone.lower())
426
milestone = self.ubuntu.getMilestone(name)
427
if milestone is None:
428
milestone = self.ubuntu.currentseries.newMilestone(name)
429
Store.of(milestone).flush()
432
def getLaunchpadUpstreamProduct(self, bug):
433
"""Find the upstream product for the given Bugzilla bug.
435
This function relies on the package -> product linkage having been
438
srcpkgname, binpkgname = self._getPackageNames(bug)
439
# find a product series
441
for series in self.ubuntu.series:
442
srcpkg = series.getSourcePackage(srcpkgname)
444
series = srcpkg.productseries
446
return series.product
448
logger.warning('could not find upstream product for '
449
'source package "%s"', srcpkgname.name)
452
_bug_re = re.compile('bug\s*#?\s*(?P<id>\d+)', re.IGNORECASE)
453
def replaceBugRef(self, match):
454
# XXX: jamesh 2005-10-24:
455
# this is where bug number rewriting would be plugged in
456
bug_id = int(match.group('id'))
457
url = '%s/%d' % (canonical_url(self.bugtracker), bug_id)
458
return '%s [%s]' % (match.group(0), url)
461
def handleBug(self, bug_id):
462
"""Maybe import a single bug.
464
If the bug has already been imported (detected by checking for
465
a bug watch), it is skipped.
467
logger.info('Handling Bugzilla bug %d', bug_id)
469
# is there a bug watch on the bug?
470
lp_bug = self.bugset.queryByRemoteBug(self.bugtracker, bug_id)
472
# if we already have an associated bug, don't add a new one.
473
if lp_bug is not None:
474
logger.info('Bugzilla bug %d is already being watched by '
475
'Launchpad bug %d', bug_id, lp_bug.id)
478
bug = Bug(self.backend, bug_id)
480
comments = bug.comments[:]
482
# create a message for the initial comment:
483
msgset = getUtility(IMessageSet)
484
who, when, text = comments.pop(0)
485
text = self._bug_re.sub(self.replaceBugRef, text)
486
# If a URL is associated with the bug, add it to the description:
488
text = text + '\n\n' + bug.bug_file_loc
489
# the initial comment can't be empty:
491
text = '<empty comment>'
492
msg = msgset.fromText(bug.short_desc, text, self.person(who), when)
495
target = self.getLaunchpadBugTarget(bug)
496
params = CreateBugParams(
497
msg=msg, datecreated=bug.creation_ts, title=bug.short_desc,
498
owner=self.person(bug.reporter))
499
params.setBugTarget(**target)
500
lp_bug = self.bugset.createBug(params)
503
lp_bug.addWatch(self.bugtracker, str(bug.bug_id), lp_bug.owner)
505
# add remaining comments, and add CVEs found in all text
506
lp_bug.findCvesInText(text, lp_bug.owner)
507
for (who, when, text) in comments:
508
text = self._bug_re.sub(self.replaceBugRef, text)
509
msg = msgset.fromText(msg.followup_title, text,
510
self.person(who), when)
511
lp_bug.linkMessage(msg)
513
# subscribe QA contact and CC's
516
self.person(bug.qa_contact), self.person(bug.reporter))
518
lp_bug.subscribe(self.person(cc), self.person(bug.reporter))
520
# translate bugzilla status and severity to LP equivalents
521
task = lp_bug.bugtasks[0]
522
task.datecreated = bug.creation_ts
523
task.transitionToAssignee(self.person(bug.assigned_to))
524
task.statusexplanation = bug.status_whiteboard
525
bug.mapSeverity(task)
528
# bugs with an alias of the form "deb1234" have been imported
529
# from the Debian bug tracker by the "debzilla" program. For
530
# these bugs, generate a task and watch on the corresponding
531
# bugs.debian.org bug.
533
if re.match(r'^deb\d+$', bug.alias):
534
watch = self.bugwatchset.createBugWatch(
535
lp_bug, lp_bug.owner, self.debbugs, bug.alias[3:])
536
debtask = self.bugtaskset.createTask(
539
distribution=self.debian,
540
sourcepackagename=target['sourcepackagename'])
541
debtask.datecreated = bug.creation_ts
542
debtask.bugwatch = watch
544
# generate a Launchpad name from the alias:
545
name = re.sub(r'[^a-z0-9\+\.\-]', '-', bug.alias.lower())
548
# for UPSTREAM bugs, try to find whether the URL field contains
550
if bug.bug_status == 'UPSTREAM':
551
# see if the URL field contains a bug tracker reference
552
watches = self.bugwatchset.fromText(bug.bug_file_loc,
553
lp_bug, lp_bug.owner)
554
# find the upstream product for this bug
555
product = self.getLaunchpadUpstreamProduct(bug)
557
# if we created a watch, and there is an upstream product,
558
# create a new task and link it to the watch.
561
upstreamtask = self.bugtaskset.createTask(
562
lp_bug, product=product, owner=lp_bug.owner)
563
upstreamtask.datecreated = bug.creation_ts
564
upstreamtask.bugwatch = watches[0]
566
logger.warning('Could not find upstream product to link '
567
'bug %d to', lp_bug.id)
569
# translate milestone linkage
570
task.milestone = self.getLaunchpadMilestone(bug)
573
for (attach_id, creation_ts, description, mimetype, ispatch,
574
filename, thedata, submitter_id) in bug.attachments:
575
# if the filename is missing for some reason, use a generic one.
576
if filename is None or filename.strip() == '':
577
filename = 'untitled'
578
logger.debug('Creating attachment %s for bug %d',
579
filename, bug.bug_id)
581
attach_type = BugAttachmentType.PATCH
582
mimetype = 'text/plain'
584
attach_type = BugAttachmentType.UNSPECIFIED
586
# look for a message starting with "Created an attachment (id=NN)"
587
for msg in lp_bug.messages:
588
if msg.text_contents.startswith(
589
'Created an attachment (id=%d)' % attach_id):
592
# could not find the add message, so create one:
593
msg = msgset.fromText(description,
594
'Created attachment %s' % filename,
595
self.person(submitter_id),
597
lp_bug.linkMessage(msg)
599
filealias = getUtility(ILibraryFileAliasSet).create(
602
file=StringIO(thedata),
603
contentType=mimetype)
605
getUtility(IBugAttachmentSet).create(
606
bug=lp_bug, filealias=filealias, attach_type=attach_type,
607
title=description, message=msg)
611
def processDuplicates(self, trans):
612
"""Mark Launchpad bugs as duplicates based on Bugzilla duplicates.
614
Launchpad bug A will be marked as a duplicate of bug B if:
615
* bug A watches bugzilla bug A'
616
* bug B watches bugzilla bug B'
617
* bug A' is a duplicate of bug B'
618
* bug A is not currently a duplicate of any other bug.
620
logger.info('Processing duplicate bugs')
623
"""Get the Launchpad bug corresponding to the given remote ID
625
This function makes use of a cache dictionary to reduce the
628
lpbugid = bugmap.get(bugid)
629
if lpbugid is not None:
631
lpbug = self.bugset.get(lpbugid)
635
lpbug = self.bugset.queryByRemoteBug(self.bugtracker, bugid)
636
if lpbug is not None:
637
bugmap[bugid] = lpbug.id
642
for (dupe_of, dupe) in self.backend.getDuplicates():
643
# get the Launchpad bugs corresponding to the two Bugzilla bugs:
645
lpdupe_of = getlpbug(dupe_of)
646
lpdupe = getlpbug(dupe)
647
# if both bugs exist in Launchpad, and lpdupe is not already
648
# a duplicate, mark it as a duplicate of lpdupe_of.
649
if (lpdupe_of is not None and lpdupe is not None and
650
lpdupe.duplicateof is None):
651
logger.info('Marking %d as a duplicate of %d',
652
lpdupe.id, lpdupe_of.id)
653
lpdupe.markAsDuplicate(lpdupe_of)
656
def importBugs(self, trans, product=None, component=None, status=None):
657
"""Import Bugzilla bugs matching the given constraints.
659
Each of product, component and status gives a list of
660
products, components or statuses to limit the import to. An
661
empty list matches all products, components or statuses.
665
if component is None:
670
bugs = self.backend.findBugs(product=product,
676
self.handleBug(bug_id)
677
except (SystemExit, KeyboardInterrupt):
680
logger.exception('Could not import Bugzilla bug #%d', bug_id)
added
removed
lib/lp/bugs/scripts/bugzilla.py
