1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
|
# Copyright 2011 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Archive Contents files generator."""
__metaclass__ = type
__all__ = [
'GenerateContentsFiles',
]
from optparse import OptionValueError
import os
from zope.component import getUtility
from canonical.config import config
from lp.archivepublisher.config import getPubConfig
from lp.registry.interfaces.distribution import IDistributionSet
from lp.registry.interfaces.pocket import pocketsuffix
from lp.services.command_spawner import (
CommandSpawner,
OutputLineHandler,
ReturnCodeReceiver,
)
from lp.services.scripts.base import (
LaunchpadScript,
LaunchpadScriptFailure,
)
from lp.services.utils import file_exists
from lp.soyuz.scripts.ftpmaster import LpQueryDistro
COMPONENTS = [
'main',
'restricted',
'universe',
'multiverse',
]
def differ_in_content(one_file, other_file):
"""Do the two named files have different contents?"""
one_exists = file_exists(one_file)
other_exists = file_exists(other_file)
if any([one_exists, other_exists]):
return (
one_exists != other_exists or
file(one_file).read() != file(other_file).read())
else:
return False
class StoreArgument:
"""Local helper for receiving `LpQueryDistro` results."""
def __call__(self, argument):
"""Store call argument."""
self.argument = argument
def get_template(template_name):
"""Return path of given template in this script's templates directory."""
return os.path.join(
config.root, "cronscripts", "publishing", "gen-contents",
template_name)
def execute(logger, command, args=None):
"""Execute a shell command.
:param logger: Output from the command will be logged here.
:param command: Command to execute, as a string.
:param args: Optional list of arguments for `command`.
:raises LaunchpadScriptFailure: If the command returns failure.
"""
command_line = [command]
if args is not None:
command_line += args
description = ' '.join(command_line)
logger.debug("Execute: %s", description)
# Some of these commands can take a long time. Use CommandSpawner
# and friends to provide "live" log output. Simpler ways of running
# commands tend to save it all up and then dump it at the end, or
# have trouble logging it as neat lines.
stderr_logger = OutputLineHandler(logger.warn)
stdout_logger = OutputLineHandler(logger.debug)
receiver = ReturnCodeReceiver()
spawner = CommandSpawner()
spawner.start(
command_line, completion_handler=receiver,
stderr_handler=stderr_logger, stdout_handler=stdout_logger)
spawner.complete()
stdout_logger.finalize()
stderr_logger.finalize()
if receiver.returncode != 0:
raise LaunchpadScriptFailure(
"Failure while running command: %s" % description)
def move_file(old_path, new_path):
"""Rename file `old_path` to `new_path`.
Mercilessly delete any file that may already exist at `new_path`.
"""
if file_exists(new_path):
os.remove(new_path)
os.rename(old_path, new_path)
class GenerateContentsFiles(LaunchpadScript):
distribution = None
def add_my_options(self):
"""See `LaunchpadScript`."""
self.parser.add_option(
"-d", "--distribution", dest="distribution", default=None,
help="Distribution to generate Contents files for.")
@property
def name(self):
"""See `LaunchpadScript`."""
# Include distribution name. Clearer to admins, but also
# puts runs for different distributions under separate
# locks so that they can run simultaneously.
return "%s-%s" % (self._name, self.options.distribution)
def processOptions(self):
"""Handle command-line options."""
if self.options.distribution is None:
raise OptionValueError("Specify a distribution.")
self.distribution = getUtility(IDistributionSet).getByName(
self.options.distribution)
if self.distribution is None:
raise OptionValueError(
"Distribution '%s' not found." % self.options.distribution)
def setUpContentArchive(self):
"""Make sure the `content_archive` directories exist."""
self.logger.debug("Ensuring that we have a private tree in place.")
for suffix in ['cache', 'misc']:
dirname = '-'.join([self.distribution.name, suffix])
path = os.path.join(self.content_archive, dirname)
if not file_exists(path):
os.makedirs(path)
def queryDistro(self, request, options=None):
"""Call the query-distro script about `self.distribution`."""
args = ['-d', self.distribution.name]
if options is not None:
args += options
args.append(request)
query_distro = LpQueryDistro(test_args=args)
query_distro.logger = self.logger
query_distro.txn = self.txn
receiver = StoreArgument()
query_distro.runAction(presenter=receiver)
return receiver.argument
def getSuites(self):
"""Query the distribution's suites."""
return self.queryDistro("supported").split()
def getPockets(self):
"""Return suites that are actually supported in this distribution."""
pockets = []
pocket_suffixes = pocketsuffix.values()
for suite in self.getSuites():
for pocket_suffix in pocket_suffixes:
pocket = suite + pocket_suffix
if file_exists(os.path.join(self.config.distsroot, pocket)):
pockets.append(pocket)
return pockets
def getArchs(self):
"""Query architectures supported by the distribution."""
devel = self.queryDistro("development")
return self.queryDistro("archs", options=["-s", devel]).split()
def getDirs(self, archs):
"""Subdirectories needed for each component."""
return ['source', 'debian-installer'] + [
'binary-%s' % arch for arch in archs]
def writeAptContentsConf(self, suites, archs):
"""Write apt-contents.conf file."""
output_dirname = '%s-misc' % self.distribution.name
output_path = os.path.join(
self.content_archive, output_dirname, "apt-contents.conf")
output_file = file(output_path, 'w')
parameters = {
'architectures': ' '.join(archs),
'content_archive': self.content_archive,
'distribution': self.distribution.name,
}
header = get_template('apt_conf_header.template')
output_file.write(file(header).read() % parameters)
dist_template = file(get_template('apt_conf_dist.template')).read()
for suite in suites:
parameters['suite'] = suite
output_file.write(dist_template % parameters)
output_file.close()
def createComponentDirs(self, suites, archs):
"""Create the content archive's tree for all of its components."""
for suite in suites:
for component in COMPONENTS:
for directory in self.getDirs(archs):
path = os.path.join(
self.content_archive, self.distribution.name, 'dists',
suite, component, directory)
if not file_exists(path):
self.logger.debug("Creating %s.", path)
os.makedirs(path)
def copyOverrides(self):
"""Copy overrides into the content archive."""
if file_exists(self.config.overrideroot):
execute(self.logger, "cp", [
"-a",
self.config.overrideroot,
"%s/" % self.content_archive,
])
else:
self.logger.debug("Did not find overrides; not copying.")
def writeContentsTop(self):
"""Write Contents.top file."""
output_filename = os.path.join(
self.content_archive, '%s-misc' % self.distribution.name,
"Contents.top")
parameters = {
'distrotitle': self.distribution.title,
}
output_file = file(output_filename, 'w')
text = file(get_template("Contents.top")).read() % parameters
output_file.write(text)
output_file.close()
def runAptFTPArchive(self):
"""Run apt-ftparchive to produce the Contents files."""
execute(self.logger, "apt-ftparchive", [
"generate",
os.path.join(
self.content_archive, "%s-misc" % self.distribution.name,
"apt-contents.conf"),
])
def generateContentsFiles(self):
"""Generate Contents files."""
self.logger.debug(
"Running apt in private tree to generate new contents.")
self.copyOverrides()
self.writeContentsTop()
self.runAptFTPArchive()
def updateContentsFile(self, suite, arch):
"""Update Contents file, if it has changed."""
contents_dir = os.path.join(
self.content_archive, self.distribution.name, 'dists', suite)
contents_filename = "Contents-%s" % arch
last_contents = os.path.join(contents_dir, ".%s" % contents_filename)
current_contents = os.path.join(contents_dir, contents_filename)
# Avoid rewriting unchanged files; mirrors would have to
# re-fetch them unnecessarily.
if differ_in_content(current_contents, last_contents):
self.logger.debug(
"Installing new Contents file for %s/%s.", suite, arch)
new_contents = os.path.join(
contents_dir, "%s.gz" % contents_filename)
contents_dest = os.path.join(
self.config.distsroot, suite, "%s.gz" % contents_filename)
move_file(current_contents, last_contents)
move_file(new_contents, contents_dest)
os.chmod(contents_dest, 0664)
else:
self.logger.debug(
"Skipping unmodified Contents file for %s/%s.", suite, arch)
def updateContentsFiles(self, suites, archs):
"""Update all Contents files that have changed."""
self.logger.debug("Comparing contents files with public tree.")
for suite in suites:
for arch in archs:
self.updateContentsFile(suite, arch)
def setUp(self):
"""Prepare configuration and filesystem state for the script's work.
This is idempotent: run it as often as you like. (For example,
a test may call `setUp` prior to calling `main` which again
invokes `setUp`).
"""
self.processOptions()
self.config = getPubConfig(self.distribution.main_archive)
self.content_archive = os.path.join(
config.archivepublisher.content_archive_root,
self.distribution.name + "-contents")
self.setUpContentArchive()
def main(self):
"""See `LaunchpadScript`."""
self.setUp()
suites = self.getPockets()
archs = self.getArchs()
self.writeAptContentsConf(suites, archs)
self.createComponentDirs(suites, archs)
self.generateContentsFiles()
self.updateContentsFiles(suites, archs)
|