72
class ArchiveCruftCheckerError(Exception):
73
"""ArchiveCruftChecker specific exception.
75
Mostly used to describe errors in the initialization of this object.
79
class TagFileNotFound(Exception):
80
"""Raised when an archive tag file could not be found."""
83
class ArchiveCruftChecker:
84
"""Perform overall checks to identify and remove obsolete records.
86
Use initialize() method to validate passed parameters and build the
87
infrastructure variables. It will raise ArchiveCruftCheckerError if
91
# XXX cprov 2006-05-15: the default archive path should come
93
def __init__(self, logger, distribution_name='ubuntu', suite=None,
94
archive_path='/srv/launchpad.net/ubuntu-archive'):
95
"""Store passed arguments.
97
Also Initialize empty variables for storing preliminar results.
99
self.distribution_name = distribution_name
101
self.archive_path = archive_path
103
# initialize a group of variables to store temporary results
104
# available versions of published sources
105
self.source_versions = {}
106
# available binaries produced by published sources
107
self.source_binaries = {}
108
# 'Not Build From Source' binaries
110
# 'All superseded by Any' binaries
112
# published binary package names
114
# Architecture specific binary packages
116
# proposed NBS (before clean up)
117
self.dubious_nbs = {}
120
# definitive NBS organized for clean up
121
self.nbs_to_remove = []
124
def architectures(self):
125
return dict([(a.architecturetag, a)
126
for a in self.distroseries.architectures])
129
def components(self):
130
return dict([(c.name, c) for c in self.distroseries.components])
133
def components_and_di(self):
134
components_and_di = []
135
for component in self.components:
136
components_and_di.append(component)
137
components_and_di.append('%s/debian-installer' % (component))
138
return components_and_di
141
def dist_archive(self):
143
self.archive_path, self.distro.name, 'dists',
144
self.distroseries.name + pocketsuffix[self.pocket])
146
def gunzipTagFileContent(self, filename):
147
"""Gunzip the contents of passed filename.
149
Check filename presence, if not present in the filesystem,
150
raises ArchiveCruftCheckerError. Use an tempfile.mkstemp()
151
to store the uncompressed content. Invoke system available
152
gunzip`, raises ArchiveCruftCheckError if it fails.
154
This method doesn't close the file descriptor used and does not
155
remove the temporary file from the filesystem, those actions
156
are required in the callsite. (apt_pkg.ParseTagFile is lazy)
158
Return a tuple containing:
159
* temp file descriptor
161
* the contents parsed by apt_pkg.ParseTagFile()
163
if not os.path.exists(filename):
164
raise TagFileNotFound("File does not exist: %s" % filename)
166
temp_fd, temp_filename = tempfile.mkstemp()
167
(result, output) = commands.getstatusoutput(
168
"gunzip -c %s > %s" % (filename, temp_filename))
170
raise ArchiveCruftCheckerError(
171
"Gunzip invocation failed!\n%s" % output)
173
temp_file = os.fdopen(temp_fd)
174
# XXX cprov 2006-05-15: maybe we need some sort of data integrity
175
# check at this point, and maybe keep the uncrompressed file
176
# for debug purposes, let's see how it behaves in real conditions.
177
parsed_contents = apt_pkg.ParseTagFile(temp_file)
179
return temp_file, temp_filename, parsed_contents
181
def processSources(self):
182
"""Process archive sources index.
184
Build source_binaries, source_versions and bin_pkgs lists.
186
self.logger.debug("Considering Sources:")
187
for component in self.components:
188
filename = os.path.join(
189
self.dist_archive, "%s/source/Sources.gz" % component)
191
self.logger.debug("Processing %s" % filename)
193
temp_fd, temp_filename, parsed_sources = (
194
self.gunzipTagFileContent(filename))
195
except TagFileNotFound, warning:
196
self.logger.warn(warning)
199
while parsed_sources.Step():
200
source = parsed_sources.Section.Find("Package")
201
source_version = parsed_sources.Section.Find("Version")
202
binaries = parsed_sources.Section.Find("Binary")
204
item.strip() for item in binaries.split(',')]:
205
self.bin_pkgs.setdefault(binary, [])
206
self.bin_pkgs[binary].append(source)
208
self.source_binaries[source] = binaries
209
self.source_versions[source] = source_version
211
# close fd and remove temporary file used to store
212
# uncompressed tag file content from the filesystem.
214
os.unlink(temp_filename)
217
"""Build the group of 'not build from source' binaries"""
218
# Checks based on the Packages files
219
self.logger.debug("Building not build from source list (NBS):")
220
for component in self.components_and_di:
221
for architecture in self.architectures:
222
self.buildArchNBS(component, architecture)
224
def buildArchNBS(self, component, architecture):
225
"""Build NBS per architecture.
227
Store results in self.nbs, also build architecture specific
228
binaries group (stored in self.arch_any)
230
filename = os.path.join(
232
"%s/binary-%s/Packages.gz" % (component, architecture))
234
self.logger.debug("Processing %s" % filename)
236
temp_fd, temp_filename, parsed_packages = (
237
self.gunzipTagFileContent(filename))
238
except TagFileNotFound, warning:
239
self.logger.warn(warning)
243
while parsed_packages.Step():
244
package = parsed_packages.Section.Find('Package')
245
source = parsed_packages.Section.Find('Source', "")
246
version = parsed_packages.Section.Find('Version')
247
architecture = parsed_packages.Section.Find('Architecture')
252
if source.find("(") != -1:
253
m = re_extract_src_version.match(source)
257
if package not in self.bin_pkgs:
258
self.nbs.setdefault(source, {})
259
self.nbs[source].setdefault(package, {})
260
self.nbs[source][package][version] = ""
262
if architecture != "all":
263
self.arch_any.setdefault(package, "0")
264
if apt_pkg.VersionCompare(
265
version, self.arch_any[package]) < 1:
266
self.arch_any[package] = version
268
# close fd and remove temporary file used to store uncompressed
269
# tag file content from the filesystem.
271
os.unlink(temp_filename)
274
"""Build the group of 'all superseded by any' binaries."""
275
self.logger.debug("Building all superseded by any list (ASBA):")
276
for component in self.components_and_di:
277
for architecture in self.architectures:
278
self.buildArchASBA(component, architecture)
280
def buildArchASBA(self, component, architecture):
281
"""Build ASBA per architecture.
283
Store the result in self.asba, require self.arch_any to be built
286
filename = os.path.join(
288
"%s/binary-%s/Packages.gz" % (component, architecture))
291
temp_fd, temp_filename, parsed_packages = (
292
self.gunzipTagFileContent(filename))
293
except TagFileNotFound, warning:
294
self.logger.warn(warning)
298
while parsed_packages.Step():
299
package = parsed_packages.Section.Find('Package')
300
source = parsed_packages.Section.Find('Source', "")
301
version = parsed_packages.Section.Find('Version')
302
architecture = parsed_packages.Section.Find('Architecture')
307
if source.find("(") != -1:
308
m = re_extract_src_version.match(source)
312
if architecture == "all":
313
if (package in self.arch_any and
314
apt_pkg.VersionCompare(
315
version, self.arch_any[package]) > -1):
316
self.asba.setdefault(source, {})
317
self.asba[source].setdefault(package, {})
318
self.asba[source][package].setdefault(version, {})
319
self.asba[source][package][version][architecture] = ""
321
# close fd and remove temporary file used to store uncompressed
322
# tag file content from the filesystem.
324
os.unlink(temp_filename)
326
def addNBS(self, nbs_d, source, version, package):
327
"""Add a new entry in given organized nbs_d list
329
Ensure the package is still published in the suite before add.
331
result = self.distroseries.getBinaryPackagePublishing(name=package)
333
if len(list(result)) == 0:
336
nbs_d.setdefault(source, {})
337
nbs_d[source].setdefault(version, {})
338
nbs_d[source][version][package] = ""
341
""" Distinguish dubious from real NBS.
343
They are 'dubious' if the version numbers match and 'real'
344
if the versions don't match.
345
It stores results in self.dubious_nbs and self.real_nbs.
347
for source in self.nbs.keys():
348
for package in self.nbs[source].keys():
349
versions = self.nbs[source][package].keys()
350
versions.sort(apt_pkg.VersionCompare)
351
latest_version = versions.pop()
353
source_version = self.source_versions.get(source, "0")
355
if apt_pkg.VersionCompare(latest_version,
356
source_version) == 0:
357
self.addNBS(self.dubious_nbs, source, latest_version,
360
self.addNBS(self.real_nbs, source, latest_version,
364
"""Properly display built NBS entries.
366
Also organize the 'real' NBSs for removal in self.nbs_to_remove
369
output = "Not Built from Source\n"
370
output += "---------------------\n\n"
372
nbs_keys = self.real_nbs.keys()
375
for source in nbs_keys:
376
proposed_bin = self.source_binaries.get(
377
source, "(source does not exist)")
378
porposed_version = self.source_versions.get(source, "??")
379
output += (" * %s_%s builds: %s\n"
380
% (source, porposed_version, proposed_bin))
381
output += "\tbut no longer builds:\n"
382
versions = self.real_nbs[source].keys()
383
versions.sort(apt_pkg.VersionCompare)
385
for version in versions:
386
packages = self.real_nbs[source][version].keys()
390
self.nbs_to_remove.append(pkg)
392
output += " o %s: %s\n" % (
393
version, ", ".join(packages))
397
if self.nbs_to_remove:
398
self.logger.info(output)
400
self.logger.debug("No NBS found")
402
def initialize(self):
403
"""Initialize and build required lists of obsolete entries in archive.
405
Check integrity of passed parameters and store organised data.
406
The result list is the self.nbs_to_remove which should contain
407
obsolete packages not currently able to be built from again.
408
Another preliminary lists can be inspected in order to have better
409
idea of what was computed.
410
If anything goes wrong mid-process, it raises ArchiveCruftCheckError,
411
otherwise a list of packages to be removes is printed.
413
if self.distribution_name is None:
414
self.distro = getUtility(ILaunchpadCelebrities).ubuntu
417
self.distro = getUtility(IDistributionSet)[
418
self.distribution_name]
419
except NotFoundError:
420
raise ArchiveCruftCheckerError(
421
"Invalid distribution: '%s'" % self.distribution_name)
424
self.distroseries = self.distro.currentseries
425
self.pocket = PackagePublishingPocket.RELEASE
428
self.distroseries, self.pocket = (
429
self.distro.getDistroSeriesAndPocket(self.suite))
430
except NotFoundError:
431
raise ArchiveCruftCheckerError(
432
"Invalid suite: '%s'" % self.suite)
434
if not os.path.exists(self.dist_archive):
435
raise ArchiveCruftCheckerError(
436
"Invalid archive path: '%s'" % self.dist_archive)
439
self.processSources()
445
def doRemovals(self):
446
"""Perform the removal of the obsolete packages found.
448
It iterates over the previously build list (self.nbs_to_remove)
449
and mark them as 'superseded' in the archive DB model. They will
450
get removed later by the archive sanity check run each cycle
453
for package in self.nbs_to_remove:
455
for distroarchseries in self.distroseries.architectures:
456
binarypackagename = getUtility(IBinaryPackageNameSet)[package]
457
dasbp = distroarchseries.getBinaryPackage(binarypackagename)
458
dasbpr = dasbp.currentrelease
460
bpph = dasbpr.current_publishing_record
462
# We're blindly removing for all arches, if it's not there
463
# for some, that's fine ...
464
except NotFoundError:
467
version = bpph.binarypackagerelease.version
468
self.logger.info("Removed %s_%s from %s/%s ... "
470
self.distroseries.name,
471
distroarchseries.architecturetag))
70
474
class PubBinaryContent:
71
475
"""Binary publication container.
added
removed
lib/lp/soyuz/scripts/ftpmaster.py
