1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
|
# Copyright 2009 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Archive Domination class.
We call 'domination' the procedure used to identify and supersede all
old versions for a given publication, source or binary, inside a suite
(distroseries + pocket, for instance, gutsy or gutsy-updates).
It also processes the superseded publications and makes the ones with
unnecessary files 'eligible for removal', which will then be considered
for archive removal. See deathrow.py.
In order to judge if a source is 'eligible for removal' it also checks
if its resulting binaries are not necessary any more in the archive, i.e.,
old binary publications can (and should) hold sources in the archive.
Source version life-cycle example:
* foo_2.1: currently published, source and binary files live in the archive
pool and it is listed in the archive indexes.
* foo_2.0: superseded, it's not listed in archive indexes but one of its
files is used for foo_2.1 (the orig.tar.gz) or foo_2.1 could
not build for one or more architectures that foo_2.0 could;
* foo_1.8: eligible for removal, none of its files are required in the
archive since foo_2.0 was published (new orig.tar.gz) and none
of its binaries are published (foo_2.0 was completely built)
* foo_1.0: removed, it already passed through the quarantine period and its
files got removed from the archive.
Note that:
* PUBLISHED and SUPERSEDED are publishing statuses.
* 'eligible for removal' is a combination of SUPERSEDED or DELETED
publishing status and a defined (non-empty) 'scheduleddeletiondate'.
* 'removed' is a combination of 'eligible for removal' and a defined
(non-empy) 'dateremoved'.
The 'domination' procedure is the 2nd step of the publication pipeline and
it is performed for each suite using:
* judgeAndDominate(distroseries, pocket)
"""
__metaclass__ = type
__all__ = ['Dominator']
from datetime import timedelta
import functools
import operator
import apt_pkg
from storm.expr import And, Count, Select
from canonical.database.constants import UTC_NOW
from canonical.database.sqlbase import (
flush_database_updates,
sqlvalues,
)
from canonical.launchpad.interfaces.lpstorm import IMasterStore
from lp.archivepublisher import ELIGIBLE_DOMINATION_STATES
from lp.registry.model.sourcepackagename import SourcePackageName
from lp.soyuz.enums import (
BinaryPackageFormat,
PackagePublishingStatus,
)
from lp.soyuz.model.binarypackagename import BinaryPackageName
from lp.soyuz.model.binarypackagerelease import BinaryPackageRelease
from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
# Days before a package will be removed from disk.
STAY_OF_EXECUTION = 1
# Ugly, but works
apt_pkg.InitSystem()
def _compare_packages_by_version_and_date(get_release, p1, p2):
"""Compare publications p1 and p2 by their version; using Debian rules.
If the publications are for the same package, compare by datecreated
instead. This lets newer records win.
"""
if get_release(p1).id == get_release(p2).id:
return cmp(p1.datecreated, p2.datecreated)
return apt_pkg.VersionCompare(get_release(p1).version,
get_release(p2).version)
class Dominator:
""" Manage the process of marking packages as superseded.
Packages are marked as superseded when they become obsolete.
"""
def __init__(self, logger, archive):
"""Initialise the dominator.
This process should be run after the publisher has published
new stuff into the distribution but before the publisher
creates the file lists for apt-ftparchive.
"""
self._logger = logger
self.archive = archive
self.debug = self._logger.debug
def _dominatePublications(self, pubs):
"""Perform dominations for the given publications.
:param pubs: A dict mapping names to a list of publications. Every
publication must be PUBLISHED or PENDING, and the first in each
list will be treated as dominant (so should be the latest).
"""
self.debug("Dominating packages...")
for name in pubs.keys():
assert pubs[name], (
"Empty list of publications for %s" % name)
for pubrec in pubs[name][1:]:
pubrec.supersede(pubs[name][0], self)
def _sortPackages(self, pkglist, is_source=True):
# pkglist is a list of packages with the following
# * sourcepackagename or packagename as appropriate
# * version
# * status
# Don't care about any other attributes
outpkgs = {}
self.debug("Sorting packages...")
attr_prefix = 'source' if is_source else 'binary'
get_release = operator.attrgetter(attr_prefix + 'packagerelease')
get_name = operator.attrgetter(attr_prefix + 'packagename')
for inpkg in pkglist:
L = outpkgs.setdefault(
get_name(get_release(inpkg)).name.encode('utf-8'), [])
L.append(inpkg)
for pkgname in outpkgs:
if len(outpkgs[pkgname]) > 1:
outpkgs[pkgname].sort(
functools.partial(
_compare_packages_by_version_and_date, get_release))
outpkgs[pkgname].reverse()
return outpkgs
def _setScheduledDeletionDate(self, pub_record):
"""Set the scheduleddeletiondate on a publishing record.
If the status is DELETED we set the date to UTC_NOW, otherwise
it gets the configured stay of execution period.
"""
if pub_record.status == PackagePublishingStatus.DELETED:
pub_record.scheduleddeletiondate = UTC_NOW
else:
pub_record.scheduleddeletiondate = (
UTC_NOW + timedelta(days=STAY_OF_EXECUTION))
def _judgeSuperseded(self, source_records, binary_records):
"""Determine whether the superseded packages supplied should
be moved to death row or not.
Currently this is done by assuming that any superseded binary
package should be removed. In the future this should attempt
to supersede binaries in build-sized chunks only, bug 55030.
Superseded source packages are considered removable when they
have no binaries in this distroseries which are published or
superseded
When a package is considered for death row it is given a
'scheduled deletion date' of now plus the defined 'stay of execution'
time provided in the configuration parameter.
"""
# Avoid circular imports.
from lp.soyuz.model.publishing import (
BinaryPackagePublishingHistory,
SourcePackagePublishingHistory)
self.debug("Beginning superseded processing...")
# XXX: dsilvers 2005-09-22 bug=55030:
# Need to make binaries go in groups but for now this'll do.
# An example of the concrete problem here is:
# - Upload foo-1.0, which builds foo and foo-common (arch all).
# - Upload foo-1.1, ditto.
# - foo-common-1.1 is built (along with the i386 binary for foo)
# - foo-common-1.0 is superseded
# Foo is now uninstallable on any architectures which don't yet
# have a build of foo-1.1, as the foo-common for foo-1.0 is gone.
# Essentially we ideally don't want to lose superseded binaries
# unless the entire group is ready to be made pending removal.
# In this instance a group is defined as all the binaries from a
# given build. This assumes we've copied the arch_all binaries
# from whichever build provided them into each arch-specific build
# which we publish. If instead we simply publish the arch-all
# binaries from another build then instead we should scan up from
# the binary to its source, and then back from the source to each
# binary published in *this* distroarchseries for that source.
# if the binaries as a group (in that definition) are all superseded
# then we can consider them eligible for removal.
for pub_record in binary_records:
binpkg_release = pub_record.binarypackagerelease
self.debug("%s/%s (%s) has been judged eligible for removal" %
(binpkg_release.binarypackagename.name,
binpkg_release.version,
pub_record.distroarchseries.architecturetag))
self._setScheduledDeletionDate(pub_record)
# XXX cprov 20070820: 'datemadepending' is useless, since it's
# always equals to "scheduleddeletiondate - quarantine".
pub_record.datemadepending = UTC_NOW
for pub_record in source_records:
srcpkg_release = pub_record.sourcepackagerelease
# Attempt to find all binaries of this
# SourcePackageRelease which are/have been in this
# distroseries...
considered_binaries = BinaryPackagePublishingHistory.select("""
binarypackagepublishinghistory.distroarchseries =
distroarchseries.id AND
binarypackagepublishinghistory.scheduleddeletiondate IS NULL AND
binarypackagepublishinghistory.archive = %s AND
binarypackagebuild.source_package_release = %s AND
distroarchseries.distroseries = %s AND
binarypackagepublishinghistory.binarypackagerelease =
binarypackagerelease.id AND
binarypackagerelease.build = binarypackagebuild.id AND
binarypackagepublishinghistory.pocket = %s
""" % sqlvalues(self.archive, srcpkg_release,
pub_record.distroseries, pub_record.pocket),
clauseTables=['DistroArchSeries', 'BinaryPackageRelease',
'BinaryPackageBuild'])
# There is at least one non-removed binary to consider
if considered_binaries.count() > 0:
# However we can still remove *this* record if there's
# at least one other PUBLISHED for the spr. This happens
# when a package is moved between components.
published = SourcePackagePublishingHistory.selectBy(
distroseries=pub_record.distroseries,
pocket=pub_record.pocket,
status=PackagePublishingStatus.PUBLISHED,
archive=self.archive,
sourcepackagereleaseID=srcpkg_release.id)
# Zero PUBLISHED for this spr, so nothing to take over
# for us, so leave it for consideration next time.
if published.count() == 0:
continue
# Okay, so there's no unremoved binaries, let's go for it...
self.debug(
"%s/%s (%s) source has been judged eligible for removal" %
(srcpkg_release.sourcepackagename.name,
srcpkg_release.version, pub_record.id))
self._setScheduledDeletionDate(pub_record)
# XXX cprov 20070820: 'datemadepending' is pointless, since it's
# always equals to "scheduleddeletiondate - quarantine".
pub_record.datemadepending = UTC_NOW
def judgeAndDominate(self, dr, pocket):
"""Perform the domination and superseding calculations
It only works across the distroseries and pocket specified.
"""
# Avoid circular imports.
from lp.soyuz.model.publishing import (
BinaryPackagePublishingHistory,
SourcePackagePublishingHistory)
for distroarchseries in dr.architectures:
self.debug("Performing domination across %s/%s (%s)" % (
dr.name, pocket.title, distroarchseries.architecturetag))
bpph_location_clauses = And(
BinaryPackagePublishingHistory.status ==
PackagePublishingStatus.PUBLISHED,
BinaryPackagePublishingHistory.distroarchseries ==
distroarchseries,
BinaryPackagePublishingHistory.archive == self.archive,
BinaryPackagePublishingHistory.pocket == pocket,
)
candidate_binary_names = Select(
BinaryPackageName.id,
And(
BinaryPackageRelease.binarypackagenameID ==
BinaryPackageName.id,
BinaryPackagePublishingHistory.binarypackagereleaseID ==
BinaryPackageRelease.id,
bpph_location_clauses,
),
group_by=BinaryPackageName.id,
having=Count(BinaryPackagePublishingHistory.id) > 1)
binaries = IMasterStore(BinaryPackagePublishingHistory).find(
BinaryPackagePublishingHistory,
BinaryPackageRelease.id ==
BinaryPackagePublishingHistory.binarypackagereleaseID,
BinaryPackageRelease.binarypackagenameID.is_in(
candidate_binary_names),
BinaryPackageRelease.binpackageformat !=
BinaryPackageFormat.DDEB,
bpph_location_clauses)
self.debug("Dominating binaries...")
self._dominatePublications(self._sortPackages(binaries, False))
self.debug("Performing domination across %s/%s (Source)" %
(dr.name, pocket.title))
spph_location_clauses = And(
SourcePackagePublishingHistory.status ==
PackagePublishingStatus.PUBLISHED,
SourcePackagePublishingHistory.distroseries == dr,
SourcePackagePublishingHistory.archive == self.archive,
SourcePackagePublishingHistory.pocket == pocket,
)
candidate_source_names = Select(
SourcePackageName.id,
And(
SourcePackageRelease.sourcepackagenameID ==
SourcePackageName.id,
SourcePackagePublishingHistory.sourcepackagereleaseID ==
SourcePackageRelease.id,
spph_location_clauses,
),
group_by=SourcePackageName.id,
having=Count(SourcePackagePublishingHistory.id) > 1)
sources = IMasterStore(SourcePackagePublishingHistory).find(
SourcePackagePublishingHistory,
SourcePackageRelease.id ==
SourcePackagePublishingHistory.sourcepackagereleaseID,
SourcePackageRelease.sourcepackagenameID.is_in(
candidate_source_names),
spph_location_clauses)
self.debug("Dominating sources...")
self._dominatePublications(self._sortPackages(sources))
flush_database_updates()
sources = SourcePackagePublishingHistory.select("""
sourcepackagepublishinghistory.distroseries = %s AND
sourcepackagepublishinghistory.archive = %s AND
sourcepackagepublishinghistory.pocket = %s AND
sourcepackagepublishinghistory.status IN %s AND
sourcepackagepublishinghistory.scheduleddeletiondate is NULL
""" % sqlvalues(dr, self.archive, pocket,
ELIGIBLE_DOMINATION_STATES))
binaries = BinaryPackagePublishingHistory.select("""
binarypackagepublishinghistory.distroarchseries =
distroarchseries.id AND
distroarchseries.distroseries = %s AND
binarypackagepublishinghistory.archive = %s AND
binarypackagepublishinghistory.pocket = %s AND
binarypackagepublishinghistory.status IN %s AND
binarypackagepublishinghistory.scheduleddeletiondate is NULL
""" % sqlvalues(dr, self.archive, pocket,
ELIGIBLE_DOMINATION_STATES),
clauseTables=['DistroArchSeries'])
self._judgeSuperseded(sources, binaries)
self.debug("Domination for %s/%s finished" %
(dr.name, pocket.title))
|