~launchpad-pqm/launchpad/devel

10637.3.1 by Guilherme Salgado
Use the default python version instead of a hard-coded version
1
#!/usr/bin/python -S
8687.15.7 by Karl Fogel
Add the copyright header block to more files.
2
#
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2048 by Canonical.com Patch Queue Manager
debbugssync, hct enabling, and ui fixes. r=jamesh
5
6
# This script runs through the set of Debbugs watches, and tries to
7
# syncronise each of those to the malone bug which is watching it.
8
9641.1.4 by Gary Poster
more updates for _pythonpath. Still need to update all subprocess calls to python to use -S; still need to update z3c.recipe.filetemplate to provide sys.modules from -S run.
9
import _pythonpath
2450 by Canonical.com Patch Queue Manager
[r=jamesh] rework cve structure, and general polish
10
import os
2048 by Canonical.com Patch Queue Manager
debbugssync, hct enabling, and ui fixes. r=jamesh
11
import sys
12
import email
13
import logging
14
15
# zope bits
16
from zope.component import getUtility
17
12929.9.12 by j.c.sackett
Fixed not-updated cronscript.
18
from canonical.database.constants import UTC_NOW
13155.2.9 by Francis J. Lacoste
Fix imports
19
from lp.app.errors import NotFoundError
13130.1.21 by Curtis Hovey
Fixed cronscript imports.
20
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
11882.2.2 by Jonathan Lange
Clear up a heck of a lot of imports from canonical.launchpad.interfaces.
21
from lp.bugs.interfaces.bug import IBugSet
22
from lp.bugs.interfaces.bugtask import (
23
    BugTaskSearchParams,
24
    IBugTaskSet,
25
    )
26
from lp.bugs.interfaces.bugwatch import IBugWatchSet
27
from lp.bugs.interfaces.cve import ICveSet
8523.3.1 by Gavin Panella
Bugs tree reorg after automated migration.
28
from lp.bugs.scripts import debbugs
8356.1.1 by Leonard Richardson
Partial move.
29
from lp.services.scripts.base import (LaunchpadCronScript,
3691.348.19 by kiko
Two more scripts converted.
30
    LaunchpadScriptFailure)
12929.9.12 by j.c.sackett
Fixed not-updated cronscript.
31
from lp.services.messages.interfaces.message import (
11882.2.2 by Jonathan Lange
Clear up a heck of a lot of imports from canonical.launchpad.interfaces.
32
    InvalidEmailMessage,
33
    IMessageSet,
34
    )
12929.9.12 by j.c.sackett
Fixed not-updated cronscript.
35
2048 by Canonical.com Patch Queue Manager
debbugssync, hct enabling, and ui fixes. r=jamesh
36
37
# setup core values and defaults
2450 by Canonical.com Patch Queue Manager
[r=jamesh] rework cve structure, and general polish
38
debbugs_location_default = '/srv/bugs-mirror.debian.org/'
2048 by Canonical.com Patch Queue Manager
debbugssync, hct enabling, and ui fixes. r=jamesh
39
3691.348.19 by kiko
Two more scripts converted.
40
4264.2.1 by James Henstridge
add a LaunchpadCronScript subclass, and make cronscripts/*.py use it
41
class DebWatchUpdater(LaunchpadCronScript):
3691.348.19 by kiko
Two more scripts converted.
42
    loglevel = logging.WARNING
43
44
    def add_my_options(self):
13155.2.7 by Francis J. Lacoste
Lint removal.
45
        self.parser.add_option(
46
            '--max', action='store', type='int', dest='max',
3691.348.19 by kiko
Two more scripts converted.
47
            default=None, help="The maximum number of bugs to synchronise.")
48
        self.parser.add_option('--debbugs', action='store', type='string',
49
            dest='debbugs',
50
            default=debbugs_location_default,
51
            help="The location of your debbugs database.")
52
53
    def main(self):
13155.2.7 by Francis J. Lacoste
Lint removal.
54
        if not os.path.exists(
55
            os.path.join(self.options.debbugs, 'index/index.db')):
3691.348.19 by kiko
Two more scripts converted.
56
            raise LaunchpadScriptFailure('%s is not a debbugs db.'
57
                                         % self.options.debbugs)
58
59
        self.txn.begin()
60
        self.debbugs_db = debbugs.Database(self.options.debbugs)
61
        self.sync()
62
        self.txn.commit()
63
64
        self.logger.info('Done!')
65
66
    def sync(self):
67
        changedcounter = 0
68
69
        self.logger.info('Finding existing debbugs watches...')
70
        debbugs_tracker = getUtility(ILaunchpadCelebrities).debbugs
71
        debwatches = debbugs_tracker.watches
72
73
        previousimportset = set([b.remotebug for b in debwatches])
13155.2.7 by Francis J. Lacoste
Lint removal.
74
        self.logger.info(
75
            '%d debbugs previously imported.' % len(previousimportset))
3691.348.19 by kiko
Two more scripts converted.
76
77
        target_watches = [watch for watch in debwatches if watch.needscheck]
13155.2.7 by Francis J. Lacoste
Lint removal.
78
        self.logger.info(
79
            '%d debbugs watches to syncronise.' % len(target_watches))
3691.348.19 by kiko
Two more scripts converted.
80
81
        self.logger.info('Sorting bug watches...')
82
        target_watches.sort(key=lambda a: a.remotebug)
83
84
        self.logger.info('Syncing bug watches...')
85
        for watch in target_watches:
86
            if self.sync_watch(watch):
87
                changedcounter += 1
88
                self.txn.commit()
89
            if self.options.max:
90
                if changedcounter >= self.options.max:
91
                    self.logger.info('Synchronised %d bugs!' % changedcounter)
92
                    return
93
94
    def sync_watch(self, watch):
95
        # keep track of whether or not something changed
96
        waschanged = False
97
        # find the bug in malone
98
        malone_bug = watch.bug
99
        # find the bug in debbugs
100
        debian_bug = self.debbugs_db[int(watch.remotebug)]
101
        bugset = getUtility(IBugSet)
102
        bugtaskset = getUtility(IBugTaskSet)
103
        bugwatchset = getUtility(IBugWatchSet)
104
        messageset = getUtility(IMessageSet)
105
        debian = getUtility(ILaunchpadCelebrities).debian
106
        debbugs_tracker = getUtility(ILaunchpadCelebrities).debbugs
107
13155.2.7 by Francis J. Lacoste
Lint removal.
108
        # make sure we have tasks for all the debian package linkages, and
109
        # also make sure we have updated their status and severity
110
        # appropriately.
3691.348.19 by kiko
Two more scripts converted.
111
        for packagename in debian_bug.packagelist():
112
            try:
13155.2.1 by Francis J. Lacoste
Made guessPackageNames return only the source package name. Renamed it to guessPublishedSourcePackageName. Convert its tests to unit tests. Get rid of the Binary package hint: comment.
113
                srcpkgname = debian.guessPublishedSourcePackageName(
114
                    packagename)
115
            except NotFoundError:
3691.348.19 by kiko
Two more scripts converted.
116
                self.logger.error(sys.exc_value)
117
                continue
118
            search_params = BugTaskSearchParams(user=None, bug=malone_bug,
119
                sourcepackagename=srcpkgname)
120
            search_params.setDistribution(debian)
121
            bugtasks = bugtaskset.search(search_params)
122
            if len(bugtasks) == 0:
123
                # we need a new task to link the bug to the debian package
13155.2.1 by Francis J. Lacoste
Made guessPackageNames return only the source package name. Renamed it to guessPublishedSourcePackageName. Convert its tests to unit tests. Get rid of the Binary package hint: comment.
124
                self.logger.info('Linking %d and debian %s' % (
125
                    malone_bug.id, srcpkgname.name))
4664.1.1 by Curtis Hovey
Normalized comments for bug 3732.
126
                # XXX: kiko 2007-02-03:
127
                # This code is completely untested and broken.
3691.348.19 by kiko
Two more scripts converted.
128
                bugtask = malone_bug.addTask(
129
                    owner=malone_bug.owner, distribution=debian,
130
                    sourcepackagename=srcpkgname)
131
                bugtask.bugwatch = watch
132
                waschanged = True
133
            else:
134
                assert len(bugtasks) == 1, 'Should only find a single task'
135
                bugtask = bugtasks[0]
136
            status = bugtask.status
13155.2.7 by Francis J. Lacoste
Lint removal.
137
            if status != bugtask.setStatusFromDebbugs(debian_bug.status):
3691.348.19 by kiko
Two more scripts converted.
138
                waschanged = True
139
            severity = bugtask.severity
13155.2.7 by Francis J. Lacoste
Lint removal.
140
            if severity != bugtask.setSeverityFromDebbugs(
141
                debian_bug.severity):
3691.348.19 by kiko
Two more scripts converted.
142
                waschanged = True
143
144
        known_msg_ids = set([msg.rfc822msgid for msg in malone_bug.messages])
145
146
        for raw_msg in debian_bug.comments:
147
148
            # parse it so we can extract the message id easily
149
            message = email.message_from_string(raw_msg)
150
151
            # see if we already have imported a message with this id for this
152
            # bug
153
            message_id = message['message-id']
154
            if message_id in known_msg_ids:
155
                # Skipping msg that is already imported
156
                continue
157
158
            # make sure this message is in the db
159
            msg = None
160
            try:
161
                msg = messageset.fromEmail(raw_msg, parsed_message=message,
162
                    create_missing_persons=True)
163
            except InvalidEmailMessage:
164
                self.logger.error('Invalid email: %s' % sys.exc_value)
165
            if msg is None:
166
                continue
167
13155.2.7 by Francis J. Lacoste
Lint removal.
168
            # Create the link between the bug and this message.
169
            malone_bug.linkMessage(msg)
3691.348.19 by kiko
Two more scripts converted.
170
13155.2.7 by Francis J. Lacoste
Lint removal.
171
            # ok, this is a new message for this bug, so in effect something
172
            # has changed
3691.348.19 by kiko
Two more scripts converted.
173
            waschanged = True
174
175
            # now we need to analyse the message for useful data
176
            watches = bugwatchset.fromMessage(msg, malone_bug)
177
            for watch in watches:
13155.2.7 by Francis J. Lacoste
Lint removal.
178
                self.logger.info(
179
                    'New watch for #%s on %s' % (watch.bug.id, watch.url))
3691.348.19 by kiko
Two more scripts converted.
180
                waschanged = True
181
182
            # and also for CVE ref clues
183
            prior_cves = set(malone_bug.cves)
184
            cveset = getUtility(ICveSet)
185
            cves = cveset.inMessage(msg)
186
            for cve in cves:
187
                malone_bug.linkCVE(cve)
188
                if cve not in prior_cves:
189
                    self.logger.info('CVE-%s (%s) found for Malone #%s' % (
190
                        cve.sequence, cve.status.name, malone_bug.id))
191
192
            # now we know about this message for this bug
193
            known_msg_ids.add(message_id)
194
195
            # and best we commit, so that we can see the email that the
196
            # librarian has created in the db
197
            self.txn.commit()
198
199
        # Mark all merged bugs as duplicates of the lowest-numbered bug
200
        if (len(debian_bug.mergedwith) > 0 and
201
            min(debian_bug.mergedwith) > debian_bug.id):
202
            for merged_id in debian_bug.mergedwith:
13155.2.7 by Francis J. Lacoste
Lint removal.
203
                merged_bug = bugset.queryByRemoteBug(
204
                    debbugs_tracker, merged_id)
3691.348.19 by kiko
Two more scripts converted.
205
                if merged_bug is not None:
206
                    # Bug has been imported already
207
                    if merged_bug.duplicateof == malone_bug:
208
                        # we already know about this
209
                        continue
210
                    elif merged_bug.duplicateof is not None:
211
                        # Interesting, we think it's a dup of something else
13155.2.7 by Francis J. Lacoste
Lint removal.
212
                        self.logger.warning(
213
                            'Debbugs thinks #%d is a dup of #%d' % (
3691.348.19 by kiko
Two more scripts converted.
214
                            merged_bug.id, merged_bug.duplicateof))
215
                        continue
216
                    # Go ahead and merge it
13155.2.7 by Francis J. Lacoste
Lint removal.
217
                    self.logger.info(
218
                        "Malone #%d is a duplicate of Malone #%d" % (
3691.348.19 by kiko
Two more scripts converted.
219
                        merged_bug.id, malone_bug.id))
220
                    merged_bug.duplicateof = malone_bug.id
221
222
                    # the dup status has changed
223
                    waschanged = True
224
225
        # make a note of the remote watch status, if it has changed
13155.2.7 by Francis J. Lacoste
Lint removal.
226
        if watch.remotestatus != debian_bug.status:
3691.348.19 by kiko
Two more scripts converted.
227
            watch.remotestatus = debian_bug.status
228
            waschanged = True
229
230
        # update the watch date details
231
        watch.lastchecked = UTC_NOW
2048 by Canonical.com Patch Queue Manager
debbugssync, hct enabling, and ui fixes. r=jamesh
232
        if waschanged:
3691.348.19 by kiko
Two more scripts converted.
233
            watch.lastchanged = UTC_NOW
234
            self.logger.info('Watch on Malone #%d changed.' % watch.bug.id)
235
        return waschanged
236
2048 by Canonical.com Patch Queue Manager
debbugssync, hct enabling, and ui fixes. r=jamesh
237
238
if __name__ == '__main__':
3691.348.19 by kiko
Two more scripts converted.
239
    script = DebWatchUpdater('launchpad-debbugs-sync')
240
    script.lock_and_run()