~launchpad-pqm/launchpad/devel

12221.1.7 by Jeroen Vermeulen
s/Unparseable/Unparsable/g, plus lint.
1
# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
8687.15.15 by Karl Fogel
Add the copyright header block to files under lib/lp/bugs/.
2
# GNU Affero General Public License version 3 (see the file LICENSE).
5897.3.4 by Graham Binns
Moved trac into its own module.
3
6002.5.18 by Bjorn Tillenius
review fixes.
4
"""Trac ExternalBugTracker implementation."""
5897.3.4 by Graham Binns
Moved trac into its own module.
5
6
__metaclass__ = type
6604.1.2 by Tom Berger
use a new XMLRPC transport which uses urllib2, handles cookies and proxies
7
__all__ = ['Trac', 'TracLPPlugin']
5897.3.4 by Graham Binns
Moved trac into its own module.
8
11403.1.4 by Henning Eggers
Reformatted imports using format-imports script r32.
9
from Cookie import SimpleCookie
10
from cookielib import CookieJar
5897.3.4 by Graham Binns
Moved trac into its own module.
11
import csv
11403.1.4 by Henning Eggers
Reformatted imports using format-imports script r32.
12
from datetime import datetime
13
from email.Utils import parseaddr
6037.3.5 by Graham Binns
Added implementation of getModifiedRemoteBugs().
14
import time
5897.3.4 by Graham Binns
Moved trac into its own module.
15
import urllib2
6037.1.4 by Bjorn Tillenius
add TracLPPlugin, which currently only knows how to get the current time.
16
import xmlrpclib
17
11403.1.4 by Henning Eggers
Reformatted imports using format-imports script r32.
18
import pytz
6037.9.8 by Graham Binns
Added tests and implementation of getMessageForComment().
19
from zope.component import getUtility
6037.9.2 by Graham Binns
Added tests and implementation for getCommentIds().
20
from zope.interface import implements
21
12442.2.2 by j.c.sackett
Moved validators to app, which makes more sense.
22
from lp.app.validators.email import valid_email
8523.3.1 by Gavin Panella
Bugs tree reorg after automated migration.
23
from lp.bugs.externalbugtracker.base import (
11403.1.4 by Henning Eggers
Reformatted imports using format-imports script r32.
24
    BugNotFound,
25
    BugTrackerAuthenticationError,
12558.1.2 by William Grant
Fix Trac to use _fetchPage (which catches URLError and HTTPError) where possible, and catch both where it's not possible. This mostly affects the initial contact with each tracker, getExternalBugTrackerToUse.
26
    BugTrackerConnectError,
11403.1.4 by Henning Eggers
Reformatted imports using format-imports script r32.
27
    ExternalBugTracker,
28
    InvalidBugId,
29
    LookupTree,
30
    UnknownRemoteStatusError,
12221.1.7 by Jeroen Vermeulen
s/Unparseable/Unparsable/g, plus lint.
31
    UnparsableBugData,
6972.6.6 by Graham Binns
Added implementation for ISupportsBackLinking in TracLPPlugin.
32
    )
11403.1.4 by Henning Eggers
Reformatted imports using format-imports script r32.
33
from lp.bugs.externalbugtracker.xmlrpc import UrlLib2Transport
8523.3.1 by Gavin Panella
Bugs tree reorg after automated migration.
34
from lp.bugs.interfaces.bugtask import (
11403.1.4 by Henning Eggers
Reformatted imports using format-imports script r32.
35
    BugTaskImportance,
36
    BugTaskStatus,
6972.6.6 by Graham Binns
Added implementation for ISupportsBackLinking in TracLPPlugin.
37
    )
8523.3.1 by Gavin Panella
Bugs tree reorg after automated migration.
38
from lp.bugs.interfaces.externalbugtracker import (
11403.1.4 by Henning Eggers
Reformatted imports using format-imports script r32.
39
    ISupportsBackLinking,
40
    ISupportsCommentImport,
41
    ISupportsCommentPushing,
42
    UNKNOWN_REMOTE_IMPORTANCE,
6972.6.6 by Graham Binns
Added implementation for ISupportsBackLinking in TracLPPlugin.
43
    )
14612.2.1 by William Grant
format-imports on lib/. So many imports.
44
from lp.services.config import config
12579.1.1 by William Grant
Move lp.bugs.externalbugtracker.isolation to lp.services.database. It's not Bugs-specific.
45
from lp.services.database.isolation import ensure_no_transaction
14550.1.1 by Steve Kowalik
Run format-imports over lib/lp and lib/canonical/launchpad
46
from lp.services.messages.interfaces.message import IMessageSet
14612.2.1 by William Grant
format-imports on lib/. So many imports.
47
from lp.services.webapp.url import urlappend
5897.3.4 by Graham Binns
Moved trac into its own module.
48
6037.9.27 by Graham Binns
Added symbolic constants for trac LP plugin levels.
49
# Symbolic constants used for the Trac LP plugin.
50
LP_PLUGIN_BUG_IDS_ONLY = 0
51
LP_PLUGIN_METADATA_ONLY = 1
52
LP_PLUGIN_METADATA_AND_COMMENTS = 2
53
LP_PLUGIN_FULL = 3
54
6972.6.9 by Graham Binns
Review changes.
55
# Fault code constants for the LP Plugin
56
FAULT_TICKET_NOT_FOUND = 1001
6037.9.27 by Graham Binns
Added symbolic constants for trac LP plugin levels.
57
10136.4.3 by Gavin Panella
Fix lint.
58
5897.3.4 by Graham Binns
Moved trac into its own module.
59
class Trac(ExternalBugTracker):
60
    """An ExternalBugTracker instance for handling Trac bugtrackers."""
61
62
    ticket_url = 'ticket/%i?format=csv'
63
    batch_url = 'query?%s&order=resolution&format=csv'
64
    batch_query_threshold = 10
65
6095.2.1 by Bjorn Tillenius
detect Trac instances having the LP plugin installed.
66
    def getExternalBugTrackerToUse(self):
67
        """See `IExternalBugTracker`."""
68
        base_auth_url = urlappend(self.baseurl, 'launchpad-auth')
69
        # Any token will do.
70
        auth_url = urlappend(base_auth_url, 'check')
71
        try:
10136.4.2 by Gavin Panella
Use the presence of a trac_auth cookie to determine if we're talking to the LP plugin or not.
72
            response = self.urlopen(auth_url)
6095.2.1 by Bjorn Tillenius
detect Trac instances having the LP plugin installed.
73
        except urllib2.HTTPError, error:
10136.4.2 by Gavin Panella
Use the presence of a trac_auth cookie to determine if we're talking to the LP plugin or not.
74
            # If the error is HTTP 401 Unauthorized then we're
75
            # probably talking to the LP plugin.
76
            if error.code == 401:
77
                return TracLPPlugin(self.baseurl)
78
            else:
79
                return self
12558.1.2 by William Grant
Fix Trac to use _fetchPage (which catches URLError and HTTPError) where possible, and catch both where it's not possible. This mostly affects the initial contact with each tracker, getExternalBugTrackerToUse.
80
        except urllib2.URLError, error:
81
            return self
10136.4.2 by Gavin Panella
Use the presence of a trac_auth cookie to determine if we're talking to the LP plugin or not.
82
        else:
83
            # If the response contains a trac_auth cookie then we're
84
            # talking to the LP plugin. However, it's unlikely that
85
            # the remote system will authorize the bogus auth token we
86
            # sent, so this check is really intended to detect broken
87
            # Trac instances that return HTTP 200 for a missing page.
88
            for set_cookie in response.headers.getheaders('Set-Cookie'):
89
                cookie = SimpleCookie(set_cookie)
90
                if 'trac_auth' in cookie:
91
                    return TracLPPlugin(self.baseurl)
92
            else:
93
                return self
6095.2.1 by Bjorn Tillenius
detect Trac instances having the LP plugin installed.
94
5897.3.4 by Graham Binns
Moved trac into its own module.
95
    def supportsSingleExports(self, bug_ids):
96
        """Return True if the Trac instance provides CSV exports for single
97
        tickets, False otherwise.
98
99
        :bug_ids: A list of bug IDs that we can use for discovery purposes.
100
        """
101
        valid_ticket = False
102
        html_ticket_url = '%s/%s' % (
103
            self.baseurl, self.ticket_url.replace('?format=csv', ''))
104
105
        bug_ids = list(bug_ids)
106
        while not valid_ticket and len(bug_ids) > 0:
107
            try:
108
                # We try to retrive the ticket in HTML form, since that will
109
                # tell us whether or not it is actually a valid ticket
110
                ticket_id = int(bug_ids.pop())
12558.1.2 by William Grant
Fix Trac to use _fetchPage (which catches URLError and HTTPError) where possible, and catch both where it's not possible. This mostly affects the initial contact with each tracker, getExternalBugTrackerToUse.
111
                self._fetchPage(html_ticket_url % ticket_id)
5897.3.4 by Graham Binns
Moved trac into its own module.
112
            except (ValueError, urllib2.HTTPError):
113
                # If we get an HTTP error we can consider the ticket to be
114
                # invalid. If we get a ValueError then the ticket_id couldn't
10136.4.3 by Gavin Panella
Fix lint.
115
                # be identified and it's of no use to us anyway.
5897.3.4 by Graham Binns
Moved trac into its own module.
116
                pass
117
            else:
118
                # If we didn't get an error we can try to get the ticket in
119
                # CSV form. If this fails then we can consider single ticket
120
                # exports to be unsupported.
121
                try:
12558.1.2 by William Grant
Fix Trac to use _fetchPage (which catches URLError and HTTPError) where possible, and catch both where it's not possible. This mostly affects the initial contact with each tracker, getExternalBugTrackerToUse.
122
                    csv_data = self._fetchPage(
5897.3.4 by Graham Binns
Moved trac into its own module.
123
                        "%s/%s" % (self.baseurl, self.ticket_url % ticket_id))
124
                    return csv_data.headers.subtype == 'csv'
125
                except (urllib2.HTTPError, urllib2.URLError):
126
                    return False
127
        else:
128
            # If we reach this point then we likely haven't had any valid
129
            # tickets or something else is wrong. Either way, we can only
130
            # assume that CSV exports of single tickets aren't supported.
131
            return False
132
6207.2.3 by Graham Binns
Added _fetchBugData() method. Tests need refactoring to handle this, though.
133
    def _fetchBugData(self, query_url):
134
        """Retrieve the CSV bug data from a URL and return it.
135
136
        :param query_url: The URL from which to retrieve the CSV bug
137
            data.
138
        :return: A list of dicts, with each dict representing a single
139
            row in the CSV data retrieved from `query_url`.
140
        """
6207.2.2 by Graham Binns
Implemented fix for bug 175417. Can be genericised to work for single exports, too.
141
        # We read the remote bugs into a list so that we can check that
142
        # the data we're getting back from the remote server are valid.
143
        csv_reader = csv.DictReader(self._fetchPage(query_url))
6207.2.7 by Graham Binns
Review changes for Barry.
144
        remote_bugs = [csv_reader.next()]
6207.2.2 by Graham Binns
Implemented fix for bug 175417. Can be genericised to work for single exports, too.
145
146
        # We consider the data we're getting from the remote server to
147
        # be valid if there is an ID field and a status field in the CSV
148
        # header. If the fields don't exist we raise an
12221.1.7 by Jeroen Vermeulen
s/Unparseable/Unparsable/g, plus lint.
149
        # UnparsableBugData error. If these fields are defined but not
6207.2.2 by Graham Binns
Implemented fix for bug 175417. Can be genericised to work for single exports, too.
150
        # filled in for each row, that error will be handled in
151
        # getRemoteBugStatus() (i.e.  with a BugNotFound or an
152
        # UnknownRemoteStatusError).
153
        if ('id' not in csv_reader.fieldnames or
154
            'status' not in csv_reader.fieldnames):
12221.1.7 by Jeroen Vermeulen
s/Unparseable/Unparsable/g, plus lint.
155
            raise UnparsableBugData(
6207.2.2 by Graham Binns
Implemented fix for bug 175417. Can be genericised to work for single exports, too.
156
                "External bugtracker %s does not define all the necessary "
157
                "fields for bug status imports (Defined field names: %r)."
158
                % (self.baseurl, csv_reader.fieldnames))
5897.3.4 by Graham Binns
Moved trac into its own module.
159
6207.2.7 by Graham Binns
Review changes for Barry.
160
        remote_bugs = remote_bugs + list(csv_reader)
6207.2.3 by Graham Binns
Added _fetchBugData() method. Tests need refactoring to handle this, though.
161
        return remote_bugs
162
163
    def getRemoteBug(self, bug_id):
164
        """See `ExternalBugTracker`."""
165
        bug_id = int(bug_id)
166
        query_url = "%s/%s" % (self.baseurl, self.ticket_url % bug_id)
167
6207.2.7 by Graham Binns
Review changes for Barry.
168
        bug_data = self._fetchBugData(query_url)
169
        if len(bug_data) == 1:
170
            return bug_id, bug_data[0]
6207.2.3 by Graham Binns
Added _fetchBugData() method. Tests need refactoring to handle this, though.
171
6207.2.7 by Graham Binns
Review changes for Barry.
172
        # There should be only one bug returned for a getRemoteBug()
173
        # call, so if we have more or less than one bug something went
174
        # wrong.
12221.1.7 by Jeroen Vermeulen
s/Unparseable/Unparsable/g, plus lint.
175
        raise UnparsableBugData(
6207.2.7 by Graham Binns
Review changes for Barry.
176
            "Remote bugtracker %s returned wrong amount of data for bug "
177
            "%i (expected 1 bug, got %i bugs)." %
178
            (self.baseurl, bug_id, len(bug_data)))
6207.2.3 by Graham Binns
Added _fetchBugData() method. Tests need refactoring to handle this, though.
179
180
    def getRemoteBugBatch(self, bug_ids):
181
        """See `ExternalBugTracker`."""
182
        id_string = '&'.join(['id=%s' % id for id in bug_ids])
183
        query_url = "%s/%s" % (self.baseurl, self.batch_url % id_string)
184
        remote_bugs = self._fetchBugData(query_url)
185
5897.3.4 by Graham Binns
Moved trac into its own module.
186
        bugs = {}
187
        for remote_bug in remote_bugs:
188
            # We're only interested in the bug if it's one of the ones in
189
            # bug_ids, just in case we get all the tickets in the Trac
190
            # instance back instead of only the ones we want.
191
            if remote_bug['id'] not in bug_ids:
192
                continue
193
194
            bugs[int(remote_bug['id'])] = remote_bug
195
196
        return bugs
197
198
    def initializeRemoteBugDB(self, bug_ids):
199
        """See `ExternalBugTracker`.
200
201
        This method overrides ExternalBugTracker.initializeRemoteBugDB()
202
        so that the remote Trac instance's support for single ticket
203
        exports can be taken into account.
204
205
        If the URL specified for the bugtracker is not valid a
206
        BugTrackerConnectError will be raised.
207
        """
208
        self.bugs = {}
209
        # When there are less than batch_query_threshold bugs to update
210
        # we make one request per bug id to the remote bug tracker,
211
        # providing it supports CSV exports per-ticket. If the Trac
212
        # instance doesn't support exports-per-ticket we fail over to
213
        # using the batch export method for retrieving bug statuses.
214
        if (len(bug_ids) < self.batch_query_threshold and
215
            self.supportsSingleExports(bug_ids)):
216
            for bug_id in bug_ids:
6207.2.7 by Graham Binns
Review changes for Barry.
217
                remote_id, remote_bug = self.getRemoteBug(bug_id)
218
                self.bugs[remote_id] = remote_bug
5897.3.4 by Graham Binns
Moved trac into its own module.
219
220
        # For large lists of bug ids we retrieve bug statuses as a batch
221
        # from the remote bug tracker so as to avoid effectively DOSing
222
        # it.
223
        else:
224
            self.bugs = self.getRemoteBugBatch(bug_ids)
225
226
    def getRemoteImportance(self, bug_id):
227
        """See `ExternalBugTracker`.
228
229
        This method is implemented here as a stub to ensure that
230
        existing functionality is preserved. As a result,
231
        UNKNOWN_REMOTE_IMPORTANCE will always be returned.
232
        """
233
        return UNKNOWN_REMOTE_IMPORTANCE
234
235
    def getRemoteStatus(self, bug_id):
236
        """Return the remote status for the given bug id.
237
238
        Raise BugNotFound if the bug can't be found.
239
        Raise InvalidBugId if the bug id has an unexpected format.
240
        """
241
        try:
242
            bug_id = int(bug_id)
243
        except ValueError:
244
            raise InvalidBugId(
245
                "bug_id must be convertable an integer: %s" % str(bug_id))
246
247
        try:
248
            remote_bug = self.bugs[bug_id]
249
        except KeyError:
250
            raise BugNotFound(bug_id)
251
252
        # If the bug has a valid resolution as well as a status then we return
253
        # that, since it's more informative than the status field on its own.
12221.1.7 by Jeroen Vermeulen
s/Unparseable/Unparsable/g, plus lint.
254
        if ('resolution' in remote_bug and
5897.3.4 by Graham Binns
Moved trac into its own module.
255
            remote_bug['resolution'] not in ['', '--', None]):
256
            return remote_bug['resolution']
257
        else:
258
            try:
259
                return remote_bug['status']
260
            except KeyError:
261
                # Some Trac instances don't include the bug status in their
262
                # CSV exports. In those cases we raise a error.
6253.2.6 by Gavin Panella
Require a message for BugWatchUpdateWarnings. Suggested by bigjools during review.
263
                raise UnknownRemoteStatusError('Status not exported.')
5897.3.4 by Graham Binns
Moved trac into its own module.
264
265
    def convertRemoteImportance(self, remote_importance):
266
        """See `ExternalBugTracker`.
267
268
        This method is implemented here as a stub to ensure that
269
        existing functionality is preserved. As a result,
270
        BugTaskImportance.UNKNOWN will always be returned.
271
        """
272
        return BugTaskImportance.UNKNOWN
273
6326.8.20 by Gavin Panella
Add titles.
274
    _status_lookup_titles = 'Trac status',
6326.8.34 by Gavin Panella
Rename Lookup to LookupTree in the externalbugtracker modules.
275
    _status_lookup = LookupTree(
6326.8.59 by Gavin Panella
A few small post-review changes.
276
        ('new', 'open', 'reopened', BugTaskStatus.NEW),
12225.7.1 by Gavin Panella
Clean up some XXXs.
277
        # XXX: Graham Binns 2007-08-06: We should follow dupes if possible.
6326.8.17 by Gavin Panella
Convert trac.
278
        ('accepted', 'assigned', 'duplicate', BugTaskStatus.CONFIRMED),
11892.2.1 by Gavin Panella
Add new 'fixreleased' status mapping for Trac. Maps to 'Fix Released'.
279
        # Status fixverified added for bug 667340, for http://trac.yorba.org/,
280
        # but could be generally useful so adding here.
281
        ('fixed', 'closed', 'fixverified', BugTaskStatus.FIXRELEASED),
6326.8.17 by Gavin Panella
Convert trac.
282
        ('invalid', 'worksforme', BugTaskStatus.INVALID),
283
        ('wontfix', BugTaskStatus.WONTFIX),
284
        )
285
5897.3.4 by Graham Binns
Moved trac into its own module.
286
    def convertRemoteStatus(self, remote_status):
287
        """See `IExternalBugTracker`"""
288
        try:
6326.8.41 by Gavin Panella
Change __call__ to find in externalbugtracker too.
289
            return self._status_lookup.find(remote_status)
5897.3.4 by Graham Binns
Moved trac into its own module.
290
        except KeyError:
6253.2.1 by Gavin Panella
Put the unknown status in the exception.
291
            raise UnknownRemoteStatusError(remote_status)
5897.3.4 by Graham Binns
Moved trac into its own module.
292
293
6002.5.8 by Bjorn Tillenius
add needs_authentication decorator.
294
def needs_authentication(func):
6002.5.18 by Bjorn Tillenius
review fixes.
295
    """Decorator for automatically authenticating if needed.
6002.5.8 by Bjorn Tillenius
add needs_authentication decorator.
296
297
    If an `xmlrpclib.ProtocolError` with error code 403 is raised by the
298
    function, we'll try to authenticate and call the function again.
299
    """
12221.1.7 by Jeroen Vermeulen
s/Unparseable/Unparsable/g, plus lint.
300
6002.5.8 by Bjorn Tillenius
add needs_authentication decorator.
301
    def decorator(self, *args, **kwargs):
302
        try:
303
            return func(self, *args, **kwargs)
304
        except xmlrpclib.ProtocolError, error:
305
            # Catch authentication errors only.
306
            if error.errcode != 403:
307
                raise
308
            self._authenticate()
309
            return func(self, *args, **kwargs)
310
    return decorator
311
312
6037.5.22 by Graham Binns
Updated tests to include @needs_authentication changes.
313
class TracLPPlugin(Trac):
6037.1.4 by Bjorn Tillenius
add TracLPPlugin, which currently only knows how to get the current time.
314
    """A Trac instance having the LP plugin installed."""
315
6972.6.6 by Graham Binns
Added implementation for ISupportsBackLinking in TracLPPlugin.
316
    implements(
317
        ISupportsBackLinking, ISupportsCommentImport, ISupportsCommentPushing)
6037.9.2 by Graham Binns
Added tests and implementation for getCommentIds().
318
6002.5.10 by Bjorn Tillenius
use XML-RPC to generate the token.
319
    def __init__(self, baseurl, xmlrpc_transport=None,
7130.2.2 by Graham Binns
TracLPPlugin now shares a CookieJar between its url opener (used for authentication) and its XML-RPC transport.
320
                 internal_xmlrpc_transport=None, cookie_jar=None):
6037.1.4 by Bjorn Tillenius
add TracLPPlugin, which currently only knows how to get the current time.
321
        super(TracLPPlugin, self).__init__(baseurl)
6002.5.5 by Bjorn Tillenius
make sure the Cookie header gets set
322
7130.2.2 by Graham Binns
TracLPPlugin now shares a CookieJar between its url opener (used for authentication) and its XML-RPC transport.
323
        if cookie_jar is None:
324
            cookie_jar = CookieJar()
6002.5.5 by Bjorn Tillenius
make sure the Cookie header gets set
325
        if xmlrpc_transport is None:
7130.2.2 by Graham Binns
TracLPPlugin now shares a CookieJar between its url opener (used for authentication) and its XML-RPC transport.
326
            xmlrpc_transport = UrlLib2Transport(baseurl, cookie_jar)
6972.6.1 by Graham Binns
Added _s to the xmlrpc_transport and internal_* attributes of TracLPPlugin. Moved the xmlrpc server proxy that got created in all the methods into an attribute.
327
7130.2.2 by Graham Binns
TracLPPlugin now shares a CookieJar between its url opener (used for authentication) and its XML-RPC transport.
328
        self._cookie_jar = cookie_jar
6972.6.1 by Graham Binns
Added _s to the xmlrpc_transport and internal_* attributes of TracLPPlugin. Moved the xmlrpc server proxy that got created in all the methods into an attribute.
329
        self._xmlrpc_transport = xmlrpc_transport
330
        self._internal_xmlrpc_transport = internal_xmlrpc_transport
331
332
        xmlrpc_endpoint = urlappend(self.baseurl, 'xmlrpc')
333
        self._server = xmlrpclib.ServerProxy(
334
            xmlrpc_endpoint, transport=self._xmlrpc_transport)
6037.1.4 by Bjorn Tillenius
add TracLPPlugin, which currently only knows how to get the current time.
335
7130.2.3 by Graham Binns
Made an unnecessary @cachedproperty into an attribute on TracLPPlugin.
336
        self._url_opener = urllib2.build_opener(
337
            urllib2.HTTPCookieProcessor(cookie_jar))
7130.2.2 by Graham Binns
TracLPPlugin now shares a CookieJar between its url opener (used for authentication) and its XML-RPC transport.
338
10512.4.3 by Gavin Panella
Sprinkle ensure_no_transaction() in good places.
339
    @ensure_no_transaction
6037.5.22 by Graham Binns
Updated tests to include @needs_authentication changes.
340
    @needs_authentication
6037.5.8 by Graham Binns
Added an implementation of TracLPPlugin.initializeRemoteBugDB().
341
    def initializeRemoteBugDB(self, bug_ids):
342
        """See `IExternalBugTracker`."""
343
        self.bugs = {}
344
6972.6.1 by Graham Binns
Added _s to the xmlrpc_transport and internal_* attributes of TracLPPlugin. Moved the xmlrpc server proxy that got created in all the methods into an attribute.
345
        time_snapshot, remote_bugs = self._server.launchpad.bug_info(
6037.9.27 by Graham Binns
Added symbolic constants for trac LP plugin levels.
346
            LP_PLUGIN_METADATA_AND_COMMENTS, dict(bugs=bug_ids))
6037.5.8 by Graham Binns
Added an implementation of TracLPPlugin.initializeRemoteBugDB().
347
        for remote_bug in remote_bugs:
6037.5.18 by Graham Binns
Missing bugs will no longer be imported.
348
            # We only import bugs whose status isn't 'missing', since
349
            # those bugs don't exist on the remote system.
350
            if remote_bug['status'] != 'missing':
351
                self.bugs[int(remote_bug['id'])] = remote_bug
6037.5.8 by Graham Binns
Added an implementation of TracLPPlugin.initializeRemoteBugDB().
352
10512.4.3 by Gavin Panella
Sprinkle ensure_no_transaction() in good places.
353
    @ensure_no_transaction
7130.2.2 by Graham Binns
TracLPPlugin now shares a CookieJar between its url opener (used for authentication) and its XML-RPC transport.
354
    def urlopen(self, request, data=None):
355
        """See `ExternalBugTracker`.
356
7130.2.3 by Graham Binns
Made an unnecessary @cachedproperty into an attribute on TracLPPlugin.
357
        This method is overridden here so that it uses the _url_opener
358
        attribute in order to maintain the use of Trac authentication
7130.2.2 by Graham Binns
TracLPPlugin now shares a CookieJar between its url opener (used for authentication) and its XML-RPC transport.
359
        cookies across requests.
360
        """
7130.2.3 by Graham Binns
Made an unnecessary @cachedproperty into an attribute on TracLPPlugin.
361
        return self._url_opener.open(request, data)
7130.2.2 by Graham Binns
TracLPPlugin now shares a CookieJar between its url opener (used for authentication) and its XML-RPC transport.
362
10512.4.3 by Gavin Panella
Sprinkle ensure_no_transaction() in good places.
363
    @ensure_no_transaction
6002.5.3 by Bjorn Tillenius
make sure a valid authentication token is generated.
364
    def _generateAuthenticationToken(self):
6002.5.18 by Bjorn Tillenius
review fixes.
365
        """Create an authentication token and return it."""
6002.5.10 by Bjorn Tillenius
use XML-RPC to generate the token.
366
        internal_xmlrpc = xmlrpclib.ServerProxy(
6002.5.11 by Bjorn Tillenius
add config option for the XML-RPC URL.
367
            config.checkwatches.xmlrpc_url,
6972.6.1 by Graham Binns
Added _s to the xmlrpc_transport and internal_* attributes of TracLPPlugin. Moved the xmlrpc server proxy that got created in all the methods into an attribute.
368
            transport=self._internal_xmlrpc_transport)
6002.5.10 by Bjorn Tillenius
use XML-RPC to generate the token.
369
        return internal_xmlrpc.newBugTrackerToken()
6002.5.3 by Bjorn Tillenius
make sure a valid authentication token is generated.
370
371
    def _authenticate(self):
372
        """Authenticate with the Trac instance."""
373
        token_text = self._generateAuthenticationToken()
374
        base_auth_url = urlappend(self.baseurl, 'launchpad-auth')
375
        auth_url = urlappend(base_auth_url, token_text)
7130.2.4 by Graham Binns
Added an explanatory comment and removed TracLPPlugin._extractAuthCookie(), which is no longer used.
376
7130.2.6 by Graham Binns
A BugTrackerAuthenticationError will now be raised if we can't auth with the Trac instance.
377
        try:
12558.1.2 by William Grant
Fix Trac to use _fetchPage (which catches URLError and HTTPError) where possible, and catch both where it's not possible. This mostly affects the initial contact with each tracker, getExternalBugTrackerToUse.
378
            self._fetchPage(auth_url)
379
        except BugTrackerConnectError, e:
380
            raise BugTrackerAuthenticationError(self.baseurl, e.error)
6002.5.3 by Bjorn Tillenius
make sure a valid authentication token is generated.
381
10512.4.3 by Gavin Panella
Sprinkle ensure_no_transaction() in good places.
382
    @ensure_no_transaction
6002.5.8 by Bjorn Tillenius
add needs_authentication decorator.
383
    @needs_authentication
6037.1.4 by Bjorn Tillenius
add TracLPPlugin, which currently only knows how to get the current time.
384
    def getCurrentDBTime(self):
385
        """See `IExternalBugTracker`."""
6972.6.8 by Graham Binns
De-linted.
386
        time_zone, local_time, utc_time = (
387
            self._server.launchpad.time_snapshot())
6290.4.10 by Graham Binns
Removed unnecessary XXXs.
388
6037.1.4 by Bjorn Tillenius
add TracLPPlugin, which currently only knows how to get the current time.
389
        # Return the UTC time, so we don't have to care about the time
390
        # zone for now.
6275.1.1 by Bjorn Tillenius
use utcfromtimestamp instead of fromtimestamp.
391
        trac_time = datetime.utcfromtimestamp(utc_time)
6037.1.4 by Bjorn Tillenius
add TracLPPlugin, which currently only knows how to get the current time.
392
        return trac_time.replace(tzinfo=pytz.timezone('UTC'))
6002.5.4 by Bjorn Tillenius
make sure that the auth_cookie gets set on the transport.
393
10512.4.3 by Gavin Panella
Sprinkle ensure_no_transaction() in good places.
394
    @ensure_no_transaction
6037.3.25 by Graham Binns
Added @needs_authentication decorator.
395
    @needs_authentication
6037.3.5 by Graham Binns
Added implementation of getModifiedRemoteBugs().
396
    def getModifiedRemoteBugs(self, remote_bug_ids, last_checked):
397
        """See `IExternalBugTracker`."""
398
        # Convert last_checked into an integer timestamp (which is what
399
        # the Trac LP plugin expects).
400
        last_checked_timestamp = int(
401
            time.mktime(last_checked.timetuple()))
402
403
        # We retrieve only the IDs of the modified bugs from the server.
6037.3.14 by Graham Binns
Review changes.
404
        criteria = {
405
            'modified_since': last_checked_timestamp,
12221.1.7 by Jeroen Vermeulen
s/Unparseable/Unparsable/g, plus lint.
406
            'bugs': remote_bug_ids,
407
            }
6972.6.1 by Graham Binns
Added _s to the xmlrpc_transport and internal_* attributes of TracLPPlugin. Moved the xmlrpc server proxy that got created in all the methods into an attribute.
408
        time_snapshot, modified_bugs = self._server.launchpad.bug_info(
6037.9.27 by Graham Binns
Added symbolic constants for trac LP plugin levels.
409
            LP_PLUGIN_BUG_IDS_ONLY, criteria)
6037.3.5 by Graham Binns
Added implementation of getModifiedRemoteBugs().
410
6037.3.23 by Graham Binns
Removed the logic from getModifiedRemoteBugs() that dropped missing remote bugs.
411
        return [bug['id'] for bug in modified_bugs]
6037.3.5 by Graham Binns
Added implementation of getModifiedRemoteBugs().
412
10122.2.1 by Gavin Panella
Don't pass BugWatch objects (i.e. a db/model object) into code in the externalbugtracker module.
413
    def getCommentIds(self, remote_bug_id):
6037.9.2 by Graham Binns
Added tests and implementation for getCommentIds().
414
        """See `ISupportsCommentImport`."""
415
        try:
10122.2.1 by Gavin Panella
Don't pass BugWatch objects (i.e. a db/model object) into code in the externalbugtracker module.
416
            bug = self.bugs[int(remote_bug_id)]
6037.9.26 by Graham Binns
Review changes for SteveA.
417
        except KeyError:
10122.2.1 by Gavin Panella
Don't pass BugWatch objects (i.e. a db/model object) into code in the externalbugtracker module.
418
            raise BugNotFound(remote_bug_id)
6037.9.26 by Graham Binns
Review changes for SteveA.
419
        else:
6037.9.5 by Graham Binns
getCommentIds() now works correctly.
420
            return [comment_id for comment_id in bug['comments']]
6037.9.2 by Graham Binns
Added tests and implementation for getCommentIds().
421
10512.4.3 by Gavin Panella
Sprinkle ensure_no_transaction() in good places.
422
    @ensure_no_transaction
6037.9.5 by Graham Binns
getCommentIds() now works correctly.
423
    @needs_authentication
10122.2.1 by Gavin Panella
Don't pass BugWatch objects (i.e. a db/model object) into code in the externalbugtracker module.
424
    def fetchComments(self, remote_bug_id, comment_ids):
6037.9.5 by Graham Binns
getCommentIds() now works correctly.
425
        """See `ISupportsCommentImport`."""
6037.9.7 by Graham Binns
Added tests and implementation for getPosterForComment().
426
        bug_comments = {}
427
428
        # Use the get_comments() method on the remote server to get the
429
        # comments specified.
6972.6.1 by Graham Binns
Added _s to the xmlrpc_transport and internal_* attributes of TracLPPlugin. Moved the xmlrpc server proxy that got created in all the methods into an attribute.
430
        timestamp, remote_comments = self._server.launchpad.get_comments(
6037.9.7 by Graham Binns
Added tests and implementation for getPosterForComment().
431
            comment_ids)
432
        for remote_comment in remote_comments:
433
            bug_comments[remote_comment['id']] = remote_comment
434
435
        # Finally, we overwrite the bug's comments field with the
436
        # bug_comments dict. The nice upshot of this is that we can
437
        # still loop over the dict and get IDs back.
10122.2.1 by Gavin Panella
Don't pass BugWatch objects (i.e. a db/model object) into code in the externalbugtracker module.
438
        self.bugs[int(remote_bug_id)]['comments'] = bug_comments
6037.9.7 by Graham Binns
Added tests and implementation for getPosterForComment().
439
10122.2.1 by Gavin Panella
Don't pass BugWatch objects (i.e. a db/model object) into code in the externalbugtracker module.
440
    def getPosterForComment(self, remote_bug_id, comment_id):
6037.9.7 by Graham Binns
Added tests and implementation for getPosterForComment().
441
        """See `ISupportsCommentImport`."""
10122.2.1 by Gavin Panella
Don't pass BugWatch objects (i.e. a db/model object) into code in the externalbugtracker module.
442
        bug = self.bugs[int(remote_bug_id)]
6037.9.8 by Graham Binns
Added tests and implementation of getMessageForComment().
443
        comment = bug['comments'][comment_id]
6037.9.7 by Graham Binns
Added tests and implementation for getPosterForComment().
444
445
        display_name, email = parseaddr(comment['user'])
446
6325.2.7 by Graham Binns
Added some comments.
447
        # If the email isn't valid, return the email address as the
6325.2.23 by Graham Binns
Review changes for barry.
448
        # display name (a Launchpad Person will be created with this
6325.2.7 by Graham Binns
Added some comments.
449
        # name).
6325.2.6 by Graham Binns
Added implementation for Trac.
450
        if not valid_email(email):
451
            return email, None
6325.2.7 by Graham Binns
Added some comments.
452
        # If the display name is empty, set it to None so that it's
453
        # useable by IPersonSet.ensurePerson().
6325.2.6 by Graham Binns
Added implementation for Trac.
454
        elif display_name == '':
455
            return None, email
6325.2.23 by Graham Binns
Review changes for barry.
456
        # Both displayname and email are valid, return both.
6325.2.6 by Graham Binns
Added implementation for Trac.
457
        else:
458
            return display_name, email
6037.9.5 by Graham Binns
getCommentIds() now works correctly.
459
10122.2.1 by Gavin Panella
Don't pass BugWatch objects (i.e. a db/model object) into code in the externalbugtracker module.
460
    def getMessageForComment(self, remote_bug_id, comment_id, poster):
6037.9.8 by Graham Binns
Added tests and implementation of getMessageForComment().
461
        """See `ISupportsCommentImport`."""
10122.2.1 by Gavin Panella
Don't pass BugWatch objects (i.e. a db/model object) into code in the externalbugtracker module.
462
        bug = self.bugs[int(remote_bug_id)]
6037.9.8 by Graham Binns
Added tests and implementation of getMessageForComment().
463
        comment = bug['comments'][comment_id]
464
6037.9.10 by Graham Binns
Added tests and implementation for getMessagesForComment().
465
        comment_datecreated = datetime.fromtimestamp(
6037.9.22 by Graham Binns
Altered Trac LP plugin mockup to always return 'timestamp' for timestamps.
466
            comment['timestamp'], pytz.timezone('UTC'))
6037.9.8 by Graham Binns
Added tests and implementation of getMessageForComment().
467
        message = getUtility(IMessageSet).fromText(
6037.9.10 by Graham Binns
Added tests and implementation for getMessagesForComment().
468
            subject='', content=comment['comment'],
7182.1.1 by Graham Binns
Fixed bug 283275.
469
            datecreated=comment_datecreated, owner=poster)
6037.9.8 by Graham Binns
Added tests and implementation of getMessageForComment().
470
471
        return message
472
10512.4.3 by Gavin Panella
Sprinkle ensure_no_transaction() in good places.
473
    @ensure_no_transaction
6037.10.2 by Graham Binns
Manually recreated the push-lpplugin-comments patch.
474
    @needs_authentication
6253.1.3 by Graham Binns
Updated TracLPPlugin.addRemoteComment() and associated tests.
475
    def addRemoteComment(self, remote_bug, comment_body, rfc822msgid):
6037.10.2 by Graham Binns
Manually recreated the push-lpplugin-comments patch.
476
        """See `ISupportsCommentPushing`."""
6972.6.1 by Graham Binns
Added _s to the xmlrpc_transport and internal_* attributes of TracLPPlugin. Moved the xmlrpc server proxy that got created in all the methods into an attribute.
477
        timestamp, comment_id = self._server.launchpad.add_comment(
6253.1.3 by Graham Binns
Updated TracLPPlugin.addRemoteComment() and associated tests.
478
            remote_bug, comment_body)
6037.10.2 by Graham Binns
Manually recreated the push-lpplugin-comments patch.
479
480
        return comment_id
6972.6.6 by Graham Binns
Added implementation for ISupportsBackLinking in TracLPPlugin.
481
10512.4.3 by Gavin Panella
Sprinkle ensure_no_transaction() in good places.
482
    @ensure_no_transaction
6972.6.6 by Graham Binns
Added implementation for ISupportsBackLinking in TracLPPlugin.
483
    @needs_authentication
484
    def getLaunchpadBugId(self, remote_bug):
485
        """Return the Launchpad bug for a given remote bug.
486
6972.6.9 by Graham Binns
Review changes.
487
        :raises BugNotFound: When `remote_bug` doesn't exist.
6972.6.6 by Graham Binns
Added implementation for ISupportsBackLinking in TracLPPlugin.
488
        """
6972.6.7 by Graham Binns
Updated get* and setLaunchpadBugId() to handle Faults from the remote service correctly.
489
        try:
490
            timestamp, lp_bug_id = self._server.launchpad.get_launchpad_bug(
491
                remote_bug)
492
        except xmlrpclib.Fault, fault:
493
            # Deal with "Ticket does not exist" faults. We re-raise
494
            # anything else, since they're a sign of a bigger problem.
6972.6.9 by Graham Binns
Review changes.
495
            if fault.faultCode == FAULT_TICKET_NOT_FOUND:
6972.6.7 by Graham Binns
Updated get* and setLaunchpadBugId() to handle Faults from the remote service correctly.
496
                raise BugNotFound(remote_bug)
497
            else:
498
                raise
6972.6.6 by Graham Binns
Added implementation for ISupportsBackLinking in TracLPPlugin.
499
500
        # If the returned bug ID is 0, return None, since a 0 means that
501
        # no LP bug is linked to the remote bug.
502
        if lp_bug_id == 0:
503
            return None
504
        else:
505
            return lp_bug_id
506
10512.4.3 by Gavin Panella
Sprinkle ensure_no_transaction() in good places.
507
    @ensure_no_transaction
6972.6.6 by Graham Binns
Added implementation for ISupportsBackLinking in TracLPPlugin.
508
    @needs_authentication
10694.2.24 by Gavin Panella
Change the function signature style as suggested in review.
509
    def setLaunchpadBugId(self, remote_bug, launchpad_bug_id,
510
                          launchpad_bug_url):
6972.6.6 by Graham Binns
Added implementation for ISupportsBackLinking in TracLPPlugin.
511
        """Set the Launchpad bug ID for a given remote bug.
512
6972.6.9 by Graham Binns
Review changes.
513
        :raises BugNotFound: When `remote_bug` doesn't exist.
6972.6.6 by Graham Binns
Added implementation for ISupportsBackLinking in TracLPPlugin.
514
        """
515
        # If the launchpad_bug_id is None, pass 0 to set_launchpad_bug
516
        # to delete the bug link, since we can't send None over XML-RPC.
517
        if launchpad_bug_id == None:
518
            launchpad_bug_id = 0
519
6972.6.7 by Graham Binns
Updated get* and setLaunchpadBugId() to handle Faults from the remote service correctly.
520
        try:
10136.4.3 by Gavin Panella
Fix lint.
521
            self._server.launchpad.set_launchpad_bug(
6972.6.7 by Graham Binns
Updated get* and setLaunchpadBugId() to handle Faults from the remote service correctly.
522
                remote_bug, launchpad_bug_id)
523
        except xmlrpclib.Fault, fault:
524
            # Deal with "Ticket does not exist" faults. We re-raise
525
            # anything else, since they're a sign of a bigger problem.
6972.6.9 by Graham Binns
Review changes.
526
            if fault.faultCode == FAULT_TICKET_NOT_FOUND:
6972.6.7 by Graham Binns
Updated get* and setLaunchpadBugId() to handle Faults from the remote service correctly.
527
                raise BugNotFound(remote_bug)
528
            else:
529
                raise