1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
|
# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
__all__ = [
'PackageDiff',
'PackageDiffSet',
]
import gzip
import itertools
import os
import shutil
import subprocess
import tempfile
from sqlobject import ForeignKey
from storm.expr import Desc
from storm.store import EmptyResultSet
from zope.component import getUtility
from zope.interface import implements
from lp.services.database.bulk import load
from lp.services.database.constants import UTC_NOW
from lp.services.database.datetimecol import UtcDateTimeCol
from lp.services.database.decoratedresultset import DecoratedResultSet
from lp.services.database.enumcol import EnumCol
from lp.services.database.lpstorm import IStore
from lp.services.database.sqlbase import (
SQLBase,
sqlvalues,
)
from lp.services.librarian.interfaces import ILibraryFileAliasSet
from lp.services.librarian.model import (
LibraryFileAlias,
LibraryFileContent,
)
from lp.services.librarian.utils import copy_and_close
from lp.services.webapp.interfaces import (
DEFAULT_FLAVOR,
IStoreSelector,
MAIN_STORE,
)
from lp.soyuz.enums import PackageDiffStatus
from lp.soyuz.interfaces.packagediff import (
IPackageDiff,
IPackageDiffSet,
)
def perform_deb_diff(tmp_dir, out_filename, from_files, to_files):
"""Perform a (deb)diff on two packages.
A debdiff will be invoked on the files associated with the
two packages to be diff'ed. The resulting output will be a tuple
containing the process return code and the STDERR output.
:param tmp_dir: The temporary directory with the package files.
:type tmp_dir: ``str``
:param out_filename: The name of the file that will hold the
resulting debdiff output.
:type tmp_dir: ``str``
:param from_files: A list with the names of the files associated
with the first package.
:type from_files: ``list``
:param to_files: A list with the names of the files associated
with the second package.
:type to_files: ``list``
"""
compressed_bytes = -1
[from_dsc] = [name for name in from_files
if name.lower().endswith('.dsc')]
[to_dsc] = [name for name in to_files
if name.lower().endswith('.dsc')]
args = ['debdiff', from_dsc, to_dsc]
full_path = os.path.join(tmp_dir, out_filename)
out_file = None
try:
out_file = open(full_path, 'w')
process = subprocess.Popen(
args, stdout=out_file, stderr=subprocess.PIPE, cwd=tmp_dir)
stdout, stderr = process.communicate()
finally:
if out_file is not None:
out_file.close()
return process.returncode, stderr
def download_file(destination_path, libraryfile):
"""Download a file from the librarian to the destination path.
:param destination_path: Absolute destination path (where the
file should be downloaded to).
:type destination_path: ``str``
:param libraryfile: The librarian file that is to be downloaded.
:type libraryfile: ``LibraryFileAlias``
"""
libraryfile.open()
destination_file = open(destination_path, 'w')
copy_and_close(libraryfile, destination_file)
class PackageDiff(SQLBase):
"""A Package Diff request."""
implements(IPackageDiff)
_defaultOrder = ['id']
date_requested = UtcDateTimeCol(notNull=False, default=UTC_NOW)
requester = ForeignKey(
dbName='requester', foreignKey='Person', notNull=True)
from_source = ForeignKey(
dbName="from_source", foreignKey='SourcePackageRelease', notNull=True)
to_source = ForeignKey(
dbName="to_source", foreignKey='SourcePackageRelease', notNull=True)
date_fulfilled = UtcDateTimeCol(notNull=False, default=None)
diff_content = ForeignKey(
dbName="diff_content", foreignKey='LibraryFileAlias',
notNull=False, default=None)
status = EnumCol(
dbName='status', notNull=True, schema=PackageDiffStatus,
default=PackageDiffStatus.PENDING)
@property
def title(self):
"""See `IPackageDiff`."""
ancestry_archive = self.from_source.upload_archive
if ancestry_archive == self.to_source.upload_archive:
ancestry_identifier = self.from_source.version
else:
ancestry_identifier = "%s (in %s)" % (
self.from_source.version,
ancestry_archive.distribution.name.capitalize())
return 'diff from %s to %s' % (
ancestry_identifier, self.to_source.version)
@property
def private(self):
"""See `IPackageDiff`."""
return self.to_source.upload_archive.private
def _countDeletedLFAs(self):
"""How many files associated with either source package have been
deleted from the librarian?"""
store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
query = """
SELECT COUNT(lfa.id)
FROM
SourcePackageRelease spr, SourcePackageReleaseFile sprf,
LibraryFileAlias lfa
WHERE
spr.id IN %s
AND sprf.SourcePackageRelease = spr.id
AND sprf.libraryfile = lfa.id
AND lfa.content IS NULL
""" % sqlvalues((self.from_source.id, self.to_source.id))
result = store.execute(query).get_one()
return (0 if result is None else result[0])
def performDiff(self):
"""See `IPackageDiff`.
This involves creating a temporary directory, downloading the files
from both SPRs involved from the librarian, running debdiff, storing
the output in the librarian and updating the PackageDiff record.
"""
# Make sure the files associated with the two source packages are
# still available in the librarian.
if self._countDeletedLFAs() > 0:
self.status = PackageDiffStatus.FAILED
return
# Create the temporary directory where the files will be
# downloaded to and where the debdiff will be performed.
tmp_dir = tempfile.mkdtemp()
try:
directions = ('from', 'to')
# Keep track of the files belonging to the respective packages.
downloaded = dict(zip(directions, ([], [])))
# Please note that packages may have files in common.
files_seen = []
# Make it easy to iterate over packages.
packages = dict(
zip(directions, (self.from_source, self.to_source)))
# Iterate over the packages to be diff'ed.
for direction, package in packages.iteritems():
# Create distinct directory locations for
# 'from' and 'to' files.
absolute_path = os.path.join(tmp_dir, direction)
os.makedirs(absolute_path)
# Download the files associated with each package in
# their corresponding relative location.
for file in package.files:
the_name = file.libraryfile.filename
relative_location = os.path.join(direction, the_name)
downloaded[direction].append(relative_location)
destination_path = os.path.join(absolute_path, the_name)
download_file(destination_path, file.libraryfile)
# All downloads are done. Construct the name of the resulting
# diff file.
result_filename = '%s_%s_%s.diff' % (
self.from_source.sourcepackagename.name,
self.from_source.version,
self.to_source.version)
# Perform the actual diff operation.
return_code, stderr = perform_deb_diff(
tmp_dir, result_filename, downloaded['from'],
downloaded['to'])
# `debdiff` failed, mark the package diff request accordingly
# and return. 0 means no differences, 1 means they differ.
# Note that pre-Karmic debdiff will return 0 even if they differ.
if return_code not in (0, 1):
self.status = PackageDiffStatus.FAILED
return
# Compress the generated diff.
out_file = open(os.path.join(tmp_dir, result_filename))
gzip_result_filename = result_filename + '.gz'
gzip_file_path = os.path.join(tmp_dir, gzip_result_filename)
gzip_file = gzip.GzipFile(gzip_file_path, mode='wb')
copy_and_close(out_file, gzip_file)
# Calculate the compressed size.
gzip_size = os.path.getsize(gzip_file_path)
# Upload the compressed diff to librarian and update
# the package diff request.
gzip_file = open(gzip_file_path)
try:
librarian_set = getUtility(ILibraryFileAliasSet)
self.diff_content = librarian_set.create(
gzip_result_filename, gzip_size, gzip_file,
'application/gzipped-patch', restricted=self.private)
finally:
gzip_file.close()
# Last but not least, mark the diff as COMPLETED.
self.date_fulfilled = UTC_NOW
self.status = PackageDiffStatus.COMPLETED
finally:
shutil.rmtree(tmp_dir)
class PackageDiffSet:
"""This class is to deal with Distribution related stuff"""
implements(IPackageDiffSet)
def __iter__(self):
"""See `IPackageDiffSet`."""
return iter(PackageDiff.select(orderBy=['-id']))
def get(self, diff_id):
"""See `IPackageDiffSet`."""
return PackageDiff.get(diff_id)
def getPendingDiffs(self, limit=None):
store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
result = store.find(
PackageDiff, PackageDiff.status == PackageDiffStatus.PENDING)
result.order_by(PackageDiff.id)
return result.config(limit=limit)
def getDiffsToReleases(self, sprs, preload_for_display=False):
"""See `IPackageDiffSet`."""
from lp.registry.model.distribution import Distribution
from lp.soyuz.model.archive import Archive
from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
if len(sprs) == 0:
return EmptyResultSet()
store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
spr_ids = [spr.id for spr in sprs]
result = store.find(
PackageDiff, PackageDiff.to_sourceID.is_in(spr_ids))
result.order_by(PackageDiff.to_sourceID,
Desc(PackageDiff.date_requested))
def preload_hook(rows):
lfas = load(LibraryFileAlias, (pd.diff_contentID for pd in rows))
lfcs = load(LibraryFileContent, (lfa.contentID for lfa in lfas))
sprs = load(
SourcePackageRelease,
itertools.chain.from_iterable(
(pd.from_sourceID, pd.to_sourceID) for pd in rows))
archives = load(Archive, (spr.upload_archiveID for spr in sprs))
distros = load(Distribution, (a.distributionID for a in archives))
if preload_for_display:
return DecoratedResultSet(result, pre_iter_hook=preload_hook)
else:
return result
def getDiffBetweenReleases(self, from_spr, to_spr):
"""See `IPackageDiffSet`."""
return IStore(PackageDiff).find(
PackageDiff,
from_sourceID=from_spr.id, to_sourceID=to_spr.id).first()
|