1
# Copyright 2011 Canonical Ltd. This software is licensed under the
2
# GNU Affero General Public License version 3 (see the file LICENSE).
4
"""Master distro publishing script."""
11
from optparse import OptionParser
13
from zope.component import getUtility
15
from canonical.config import config
16
from lp.archivepublisher.config import getPubConfig
17
from lp.registry.interfaces.distribution import IDistributionSet
18
from lp.services.scripts.base import (
20
LaunchpadScriptFailure,
22
from lp.services.utils import file_exists
23
from lp.soyuz.enums import ArchivePurpose
24
from lp.soyuz.scripts import publishdistro
25
from lp.soyuz.scripts.ftpmaster import LpQueryDistro
26
from lp.soyuz.scripts.processaccepted import ProcessAccepted
29
# XXX JeroenVermeulen 2011-03-31 bug=746229: to start publishing debug
30
# archives, get rid of this list.
31
ARCHIVES_TO_PUBLISH = [
32
ArchivePurpose.PRIMARY,
33
ArchivePurpose.PARTNER,
37
def compose_shell_boolean(boolean_value):
38
"""Represent a boolean value as "yes" or "no"."""
43
return boolean_text[boolean_value]
46
def shell_quote(literal):
47
"""Escape `literal` for use in a double-quoted shell string.
49
This is a pretty naive substitution: it doesn't deal well with
50
non-ASCII characters or special characters.
52
# Characters that need backslash-escaping. Do the backslash itself
53
# first; any escapes we introduced *before* the backslash would have
54
# their own backslashes escaped afterwards when we got to the
56
special_characters = '\\"$`\n'
57
for escapee in special_characters:
58
literal = literal.replace(escapee, '\\' + escapee)
59
return '"%s"' % literal
62
def compose_env_string(*env_dicts):
63
"""Turn dict(s) into a series of shell parameter assignments.
65
Values in later dicts override any values with the same respective
66
keys in earlier dicts.
69
for env_dict in env_dicts:
71
return ' '.join(['='.join(pair) for pair in env.iteritems()])
74
def get_backup_dists(archive_config):
75
"""Return the path of an archive's backup dists directory."""
76
return os.path.join(archive_config.archiveroot + "-distscopy", "dists")
79
def get_dists(archive_config):
80
"""Return the path of an archive's dists directory.
82
:param archive_config: Configuration for the archive in question.
84
return archive_config.distsroot
87
def get_working_dists(archive_config):
88
"""Return the working path for an archive's dists directory.
90
In order for publish-distro to operate on an archive, its dists
91
directory must be in the archive root. So we move the backup
92
dists directory to a working location below the archive root just
93
for publish-distro. This method composes the temporary path.
95
return get_dists(archive_config) + ".in-progress"
98
"""Produce env dict for extending $PATH.
100
Adds the Launchpad source tree's cronscripts/publishing to the
103
:return: a dict suitable for passing to compose_env_string.
105
scripts_dir = os.path.join(config.root, "cronscripts", "publishing")
106
return {"PATH": '"$PATH":%s' % shell_quote(scripts_dir)}
110
"""Helper class: receive argument and store it."""
112
def __call__(self, argument):
113
self.argument = argument
116
def find_run_parts_dir(distro, parts):
117
"""Find the requested run-parts directory, if it exists."""
118
run_parts_location = config.archivepublisher.run_parts_location
119
if not run_parts_location:
122
if run_parts_location.startswith("/"):
124
base_dir = run_parts_location
127
base_dir = os.path.join(config.root, run_parts_location)
129
parts_dir = os.path.join(base_dir, distro.name, parts)
130
if file_exists(parts_dir):
136
class PublishFTPMaster(LaunchpadCronScript):
137
"""Publish a distro (update).
139
The publishable files are kept in the filesystem. Most of the work
140
is done in a working "dists" directory, which then replaces the
141
current "dists" in the archive root.
143
For performance reasons, the old "dists" is not discarded. It is
144
kept as the dists-copy version for the next run. Its contents
145
don't matter in detail; an rsync updates it based on the currently
146
published dists directory before we start working with it.
148
At the end of one pass of the script, the "dists" directory in the
149
archive root and its backup copy in the dists-copy root will have
152
However the script normally does 2 passes: once just for security
153
updates, to expedite publication of critical fixes, and once for the
154
whole distribution. At the end of this, the directories will be
155
back in their original places (though with updated contents).
158
def add_my_options(self):
159
"""See `LaunchpadScript`."""
160
self.parser.add_option(
161
'-d', '--distribution', dest='distribution', default=None,
162
help="Distribution to publish.")
163
self.parser.add_option(
164
'-p', '--post-rsync', dest='post_rsync', action='store_true',
166
help="When done, rsync backup dists to speed up the next run.")
167
self.parser.add_option(
168
'-s', '--security-only', dest='security_only',
169
action='store_true', default=False, help="Security upload only.")
171
def processOptions(self):
172
"""Handle command-line options.
174
Sets `self.distribution` to the `Distribution` to publish.
176
if self.options.distribution is None:
177
raise LaunchpadScriptFailure("Specify a distribution.")
179
self.distribution = getUtility(IDistributionSet).getByName(
180
self.options.distribution)
181
if self.distribution is None:
182
raise LaunchpadScriptFailure(
183
"Distribution %s not found." % self.options.distribution)
185
def executeShell(self, command_line, failure=None):
186
"""Run `command_line` through a shell.
188
This won't just load an external program and run it; the command
189
line goes through the full shell treatment including variable
190
substitutions, output redirections, and so on.
192
:param command_line: Shell command.
193
:param failure: Raise `failure` as an exception if the shell
194
command returns a nonzero value. If omitted, nonzero return
197
self.logger.debug("Executing: %s" % command_line)
198
retval = os.system(command_line)
200
self.logger.debug("Command returned %d.", retval)
201
if failure is not None:
202
self.logger.debug("Command failed: %s", failure)
205
def getArchives(self):
206
"""Find archives for `self.distribution` that should be published."""
207
# XXX JeroenVermeulen 2011-03-31 bug=746229: to start publishing
208
# debug archives, change this to return
209
# list(self.distribution.all_distro_archives).
212
for archive in self.distribution.all_distro_archives
213
if archive.purpose in ARCHIVES_TO_PUBLISH]
215
def getConfigs(self):
216
"""Set up configuration objects for archives to be published.
218
The configs dict maps the archive purposes that are relevant for
219
publishing to the respective archives' configurations.
222
(archive.purpose, getPubConfig(archive))
223
for archive in self.archives)
225
def processAccepted(self):
226
"""Run the process-accepted script."""
228
"Processing the accepted queue into the publishing records...")
229
script = ProcessAccepted(test_args=[self.distribution.name])
230
script.txn = self.txn
231
script.logger = self.logger
234
def getDirtySuites(self):
235
"""Return list of suites that have packages pending publication."""
236
self.logger.debug("Querying which suites are pending publication...")
237
query_distro = LpQueryDistro(
238
test_args=['-d', self.distribution.name, "pending_suites"])
239
receiver = StoreArgument()
240
query_distro.runAction(presenter=receiver)
241
return receiver.argument.split()
243
def getDirtySecuritySuites(self):
244
"""List security suites with pending publications."""
245
suites = self.getDirtySuites()
246
return [suite for suite in suites if suite.endswith('-security')]
248
def rsyncBackupDists(self):
249
"""Populate the backup dists with a copy of distsroot.
251
Uses "rsync -aH --delete" so that any obsolete files that may
252
still be in the backup dists are cleaned out (bug 58835).
254
:param archive_purpose: The (purpose of the) archive to copy.
256
for purpose, archive_config in self.configs.iteritems():
257
dists = get_dists(archive_config)
258
backup_dists = get_backup_dists(archive_config)
260
"rsync -aH --delete '%s/' '%s'" % (dists, backup_dists),
261
failure=LaunchpadScriptFailure(
262
"Failed to rsync new dists for %s." % purpose.title))
264
def recoverWorkingDists(self):
265
"""Look for and recover any dists left in transient working state.
267
An archive's dists directory is temporarily moved into the
268
archive root for running publish-distro. If a previous script
269
run died while in this state, restore the directory to its
272
for archive_config in self.configs.itervalues():
273
working_location = get_working_dists(archive_config)
274
if file_exists(working_location):
276
"Recovering working directory %s from failed run.",
278
os.rename(working_location, get_backup_dists(archive_config))
281
"""Create archive roots and such if they did not yet exist."""
282
for archive_purpose, archive_config in self.configs.iteritems():
283
archiveroot = archive_config.archiveroot
284
if not file_exists(archiveroot):
285
self.logger.debug("Creating archive root %s.", archiveroot)
286
os.makedirs(archiveroot)
287
dists = get_dists(archive_config)
288
if not file_exists(dists):
289
self.logger.debug("Creating dists root %s.", dists)
291
distscopy = get_backup_dists(archive_config)
292
if not file_exists(distscopy):
294
"Creating backup dists directory %s", distscopy)
295
os.makedirs(distscopy)
297
def publishDistroArchive(self, archive, security_suites=None):
298
"""Publish the results for an archive.
300
:param archive: Archive to publish.
301
:param security_suites: An optional list of suites to restrict
304
purpose = archive.purpose
305
archive_config = self.configs[purpose]
307
"Publishing the %s %s...", self.distribution.name, purpose.title)
309
# For reasons unknown, publishdistro only seems to work with a
310
# directory that's inside the archive root. So we move it there
312
temporary_dists = get_working_dists(archive_config)
316
'-d', self.distribution.name,
317
'-R', temporary_dists,
320
if archive.purpose == ArchivePurpose.PARTNER:
321
arguments.append('--partner')
323
if security_suites is not None:
324
arguments += sum([['-s', suite] for suite in security_suites], [])
326
parser = OptionParser()
327
publishdistro.add_options(parser)
329
os.rename(get_backup_dists(archive_config), temporary_dists)
331
options, args = parser.parse_args(arguments)
332
publishdistro.run_publisher(
333
options, txn=self.txn, log=self.logger)
335
os.rename(temporary_dists, get_backup_dists(archive_config))
337
self.runPublishDistroParts(archive)
339
def runPublishDistroParts(self, archive):
340
"""Execute the publish-distro hooks."""
341
archive_config = self.configs[archive.purpose]
343
'ARCHIVEROOT': shell_quote(archive_config.archiveroot),
344
'DISTSROOT': shell_quote(get_backup_dists(archive_config)),
346
if archive_config.overrideroot is not None:
347
env["OVERRIDEROOT"] = shell_quote(archive_config.overrideroot)
348
self.runParts('publish-distro.d', env)
350
def installDists(self):
351
"""Put the new dists into place, as near-atomically as possible.
353
For each archive, this switches the dists directory and the
354
backup dists directory around.
356
self.logger.debug("Moving the new dists into place...")
357
for archive_config in self.configs.itervalues():
358
# Use the dists "working location" as a temporary place to
359
# move the current dists out of the way for the switch. If
360
# we die in this state, the next run will know to move the
361
# temporary directory to the backup location.
362
dists = get_dists(archive_config)
363
temp_dists = get_working_dists(archive_config)
364
backup_dists = get_backup_dists(archive_config)
366
os.rename(dists, temp_dists)
367
os.rename(backup_dists, dists)
368
os.rename(temp_dists, backup_dists)
370
def runCommercialCompat(self):
371
"""Generate the -commercial pocket.
373
This is done for backwards compatibility with dapper, edgy, and
374
feisty releases. Failure here is not fatal.
376
# XXX JeroenVermeulen 2011-03-24 bug=741683: Retire
377
# commercial-compat.sh (and this method) as soon as Dapper
379
if self.distribution.name != 'ubuntu':
381
if not config.archivepublisher.run_commercial_compat:
384
env = {"LPCONFIG": shell_quote(config.instance_name)}
386
"env %s commercial-compat.sh"
387
% compose_env_string(env, extend_PATH()))
389
def generateListings(self):
390
"""Create ls-lR.gz listings."""
391
self.logger.debug("Creating ls-lR.gz...")
393
lslr_new = "." + lslr + ".new"
394
for purpose, archive_config in self.configs.iteritems():
395
lslr_file = os.path.join(archive_config.archiveroot, lslr)
396
new_lslr_file = os.path.join(archive_config.archiveroot, lslr_new)
397
if file_exists(new_lslr_file):
398
os.remove(new_lslr_file)
400
"cd -- '%s' ; TZ=UTC ls -lR | gzip -9n >'%s'"
401
% (archive_config.archiveroot, lslr_new),
402
failure=LaunchpadScriptFailure(
403
"Failed to create %s for %s." % (lslr, purpose.title)))
404
os.rename(new_lslr_file, lslr_file)
406
def clearEmptyDirs(self):
407
"""Clear out any redundant empty directories."""
408
for archive_config in self.configs.itervalues():
410
"find '%s' -type d -empty | xargs -r rmdir"
411
% archive_config.archiveroot)
413
def runParts(self, parts, env):
414
"""Execute run-parts.
416
:param parts: The run-parts directory to execute:
417
"publish-distro.d" or "finalize.d".
418
:param env: A dict of environment variables to pass to the
419
scripts in the run-parts directory.
421
parts_dir = find_run_parts_dir(self.distribution, parts)
422
if parts_dir is None:
423
self.logger.debug("Skipping run-parts %s: not configured.", parts)
425
env_string = compose_env_string(env, extend_PATH())
427
"env %s run-parts -- '%s'" % (env_string, parts_dir),
428
failure=LaunchpadScriptFailure(
429
"Failure while executing run-parts %s." % parts_dir))
431
def runFinalizeParts(self, security_only=False):
432
"""Run the finalize.d parts to finalize publication."""
433
archive_roots = shell_quote(' '.join([
434
archive_config.archiveroot
435
for archive_config in self.configs.itervalues()]))
438
'SECURITY_UPLOAD_ONLY': compose_shell_boolean(security_only),
439
'ARCHIVEROOTS': archive_roots,
441
self.runParts('finalize.d', env)
443
def publishSecurityUploads(self):
444
"""Quickly process just the pending security uploads."""
445
self.logger.debug("Expediting security uploads.")
446
security_suites = self.getDirtySecuritySuites()
447
if len(security_suites) == 0:
448
self.logger.debug("Nothing to do for security publisher.")
451
self.publishDistroArchive(
452
self.distribution.main_archive, security_suites=security_suites)
454
def publishAllUploads(self):
455
"""Publish the distro's complete uploads."""
456
self.logger.debug("Full publication. This may take some time.")
457
for archive in self.archives:
458
# This, for the main archive, is where the script spends
460
self.publishDistroArchive(archive)
462
def publish(self, security_only=False):
463
"""Do the main publishing work.
465
:param security_only: If True, limit publication to security
466
updates on the main archive. This is much faster, so it
467
makes sense to do a security-only run before the main
468
event to expedite critical fixes.
472
self.publishSecurityUploads()
474
self.publishAllUploads()
476
# Swizzle the now-updated backup dists and the current dists
480
# If we failed here, there's a chance that we left a
481
# working dists directory in its temporary location. If so,
482
# recover it. The next script run would do that anyway, but
483
# doing it now is easier on admins trying to recover from
485
self.recoverWorkingDists()
489
"""Process options, and set up internal state."""
490
self.processOptions()
491
self.archives = self.getArchives()
492
self.configs = self.getConfigs()
495
"""See `LaunchpadScript`."""
497
self.recoverWorkingDists()
498
self.processAccepted()
501
self.rsyncBackupDists()
502
self.publish(security_only=True)
503
self.runCommercialCompat()
504
self.runFinalizeParts(security_only=True)
506
if not self.options.security_only:
507
self.rsyncBackupDists()
508
self.publish(security_only=False)
509
self.runCommercialCompat()
510
self.generateListings()
511
self.clearEmptyDirs()
512
self.runFinalizeParts(security_only=False)
514
if self.options.post_rsync:
515
# Update the backup dists with the published changes. The
516
# initial rsync on the next run will not need to make any
517
# changes, and so it'll take the next run a little less
518
# time to publish its security updates.
519
self.rsyncBackupDists()
added
removed
lib/lp/archivepublisher/scripts/publish_ftpmaster.py
