1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
|
# Copyright 2010-2011 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Job classes related to ApportJobs are in here."""
__metaclass__ = type
__all__ = [
'ApportJob',
'ApportJobDerived',
]
from cStringIO import StringIO
from lazr.delegates import delegates
import simplejson
from sqlobject import SQLObjectNotFound
from storm.expr import And
from storm.locals import (
Int,
Reference,
Unicode,
)
from zope.component import getUtility
from zope.interface import (
classProvides,
implements,
)
from lp.bugs.interfaces.apportjob import (
ApportJobType,
IApportJob,
IApportJobSource,
IProcessApportBlobJob,
IProcessApportBlobJobSource,
)
from lp.bugs.utilities.filebugdataparser import (
FileBugData,
FileBugDataParser,
)
from lp.services.database.enumcol import EnumCol
from lp.services.database.lpstorm import IStore
from lp.services.database.stormbase import StormBase
from lp.services.job.model.job import Job
from lp.services.job.runner import BaseRunnableJob
from lp.services.librarian.interfaces import ILibraryFileAliasSet
from lp.services.temporaryblobstorage.model import TemporaryBlobStorage
from lp.services.webapp.interfaces import (
DEFAULT_FLAVOR,
IStoreSelector,
MAIN_STORE,
)
class ApportJob(StormBase):
"""Base class for jobs related to Apport BLOBs."""
implements(IApportJob)
__storm_table__ = 'ApportJob'
id = Int(primary=True)
job_id = Int(name='job')
job = Reference(job_id, Job.id)
blob_id = Int(name='blob')
blob = Reference(blob_id, TemporaryBlobStorage.id)
job_type = EnumCol(enum=ApportJobType, notNull=True)
_json_data = Unicode('json_data')
# The metadata property because it needs to be modifiable by
# subclasses of ApportJobDerived. However, since ApportJobDerived
# only delegates() to ApportJob we can't simply directly access the
# _json_data property, so we use a getter and setter here instead.
def _set_metadata(self, metadata):
self._json_data = unicode(
simplejson.dumps(metadata, 'utf-8'))
def _get_metadata(self):
return simplejson.loads(self._json_data)
metadata = property(_get_metadata, _set_metadata)
def __init__(self, blob, job_type, metadata):
"""Constructor.
:param blob: The ITemporaryBlobStorage object this job relates to.
:param job_type: The ApportJobType of this job.
:param metadata: The type-specific variables, as a JSON-compatible
dict.
"""
super(ApportJob, self).__init__()
json_data = simplejson.dumps(metadata)
self.job = Job()
self.blob = blob
self.job_type = job_type
# XXX AaronBentley 2009-01-29 bug=322819: This should be a
# bytestring, but the DB representation is unicode.
self._json_data = json_data.decode('utf-8')
@classmethod
def get(cls, key):
"""Return the instance of this class whose key is supplied."""
store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
instance = store.get(cls, key)
if instance is None:
raise SQLObjectNotFound(
'No occurrence of %s has key %s' % (cls.__name__, key))
return instance
class ApportJobDerived(BaseRunnableJob):
"""Intermediate class for deriving from ApportJob."""
delegates(IApportJob)
classProvides(IApportJobSource)
def __init__(self, job):
self.context = job
@classmethod
def create(cls, blob):
"""See `IApportJob`."""
# If there's already a job for the blob, don't create a new one.
job = ApportJob(blob, cls.class_job_type, {})
return cls(job)
@classmethod
def get(cls, job_id):
"""Get a job by id.
:return: the ApportJob with the specified id, as the current
ApportJobDerived subclass.
:raises: SQLObjectNotFound if there is no job with the specified id,
or its job_type does not match the desired subclass.
"""
job = ApportJob.get(job_id)
if job.job_type != cls.class_job_type:
raise SQLObjectNotFound(
'No object found with id %d and type %s' % (job_id,
cls.class_job_type.title))
return cls(job)
@classmethod
def iterReady(cls):
"""Iterate through all ready ApportJobs."""
store = IStore(ApportJob)
jobs = store.find(
ApportJob,
And(ApportJob.job_type == cls.class_job_type,
ApportJob.job == Job.id,
Job.id.is_in(Job.ready_jobs)))
return (cls(job) for job in jobs)
def getOopsVars(self):
"""See `IRunnableJob`."""
vars = BaseRunnableJob.getOopsVars(self)
vars.extend([
('apport_blob_uuid', self.context.blob.uuid),
('apport_blob_librarian_url',
self.context.blob.file_alias.getURL()),
('apport_job_id', self.context.id),
('apport_job_type', self.context.job_type.title),
])
return vars
class ProcessApportBlobJob(ApportJobDerived):
"""A Job to process an Apport BLOB."""
implements(IProcessApportBlobJob)
class_job_type = ApportJobType.PROCESS_BLOB
classProvides(IProcessApportBlobJobSource)
@classmethod
def create(cls, blob):
"""See `IProcessApportBlobJobSource`."""
# If there's already a job for the BLOB, don't create a new one.
# We also include jobs which have been completed when checking
# for exisiting jobs, since a BLOB should only be processed
# once.
store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
job_for_blob = store.find(
ApportJob,
ApportJob.blob == blob,
ApportJob.job_type == cls.class_job_type,
ApportJob.job == Job.id,
).any()
if job_for_blob is not None:
return cls(job_for_blob)
else:
return super(ProcessApportBlobJob, cls).create(blob)
@classmethod
def getByBlobUUID(cls, uuid):
"""See `IApportJobSource`."""
store = IStore(ApportJob)
jobs_for_blob = store.find(
ApportJob,
ApportJob.job == Job.id,
ApportJob.job_type == cls.class_job_type,
ApportJob.blob_id == TemporaryBlobStorage.id,
TemporaryBlobStorage.uuid == uuid)
job_for_blob = jobs_for_blob.one()
if job_for_blob is None:
raise SQLObjectNotFound(
"No ProcessApportBlobJob found for UUID %s" % uuid)
return cls(job_for_blob)
def run(self):
"""See `IRunnableJob`."""
self.blob.file_alias.open()
parser = FileBugDataParser(self.blob.file_alias)
parsed_data = parser.parse()
# We transform the parsed_data object into a dict, because
# that's easier to store in JSON.
parsed_data_dict = parsed_data.asDict()
# If there are attachments, we loop over them and push them to
# the Librarian, since it's easier than trying to serialize file
# data to the ApportJob table.
if len(parsed_data_dict.get('attachments')) > 0:
attachments = parsed_data_dict['attachments']
attachments_to_store = []
for attachment in attachments:
file_content = attachment['content'].read()
file_alias = getUtility(ILibraryFileAliasSet).create(
name=attachment['filename'], size=len(file_content),
file=StringIO(file_content),
contentType=attachment['content_type'])
attachments_to_store.append({
'file_alias_id': file_alias.id,
'description': attachment['description']})
# We cheekily overwrite the 'attachments' value in the
# parsed_data_dict so as to avoid trying to serialize file
# objects to JSON.
parsed_data_dict['attachments'] = attachments_to_store
metadata = self.metadata
metadata.update({'processed_data': parsed_data_dict})
self.metadata = metadata
def getFileBugData(self):
"""Return the parsed data as a FileBugData object."""
processed_data = self.metadata.get('processed_data', None)
if processed_data is not None:
attachment_data = []
for attachment in processed_data.get('attachments', []):
file_alias_id = attachment['file_alias_id']
file_alias = getUtility(ILibraryFileAliasSet)[file_alias_id]
attachment_data.append(
dict(attachment, file_alias=file_alias))
return FileBugData(
initial_summary=processed_data['initial_summary'],
initial_tags=processed_data['initial_tags'],
private=processed_data['private'],
subscribers=processed_data['subscribers'],
extra_description=processed_data['extra_description'],
comments=processed_data['comments'],
hwdb_submission_keys=processed_data['hwdb_submission_keys'],
attachments=attachment_data)
else:
return FileBugData()
|