1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
|
# Copyright 2011 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Generate extra overrides using Germinate."""
__metaclass__ = type
__all__ = [
'GenerateExtraOverrides',
]
from functools import partial
import logging
from optparse import OptionValueError
import os
import re
from germinate.archive import TagFile
from germinate.germinator import Germinator
from germinate.log import GerminateFormatter
from germinate.seeds import (
SeedError,
SeedStructure,
)
from zope.component import getUtility
from lp.archivepublisher.config import getPubConfig
from lp.registry.interfaces.distribution import IDistributionSet
from lp.registry.interfaces.series import SeriesStatus
from lp.services.scripts.base import (
LaunchpadScript,
LaunchpadScriptFailure,
)
from lp.services.utils import file_exists
from lp.services.webapp.dbpolicy import (
DatabaseBlockedPolicy,
SlaveOnlyDatabasePolicy,
)
class AtomicFile:
"""Facilitate atomic writing of files."""
def __init__(self, filename):
self.filename = filename
self.fd = open("%s.new" % self.filename, "w")
def __enter__(self):
return self.fd
def __exit__(self, exc_type, exc_value, exc_tb):
self.fd.close()
if exc_type is None:
os.rename("%s.new" % self.filename, self.filename)
def find_operable_series(distribution):
"""Find all the series we can operate on in this distribution.
We are allowed to modify DEVELOPMENT or FROZEN series, but should leave
series with any other status alone.
"""
return [
series for series in distribution.series
if series.status in (SeriesStatus.DEVELOPMENT, SeriesStatus.FROZEN)]
class GenerateExtraOverrides(LaunchpadScript):
"""Main class for scripts/ftpmaster-tools/generate-task-overrides.py."""
def __init__(self, *args, **kwargs):
super(GenerateExtraOverrides, self).__init__(*args, **kwargs)
self.germinate_logger = None
def add_my_options(self):
"""Add a 'distribution' context option."""
self.parser.add_option(
"-d", "--distribution", dest="distribution",
help="Context distribution name.")
@property
def name(self):
"""See `LaunchpadScript`."""
# Include distribution name. Clearer to admins, but also
# puts runs for different distributions under separate
# locks so that they can run simultaneously.
return "%s-%s" % (self._name, self.options.distribution)
def processOptions(self):
"""Handle command-line options."""
if self.options.distribution is None:
raise OptionValueError("Specify a distribution.")
self.distribution = getUtility(IDistributionSet).getByName(
self.options.distribution)
if self.distribution is None:
raise OptionValueError(
"Distribution '%s' not found." % self.options.distribution)
self.series = find_operable_series(self.distribution)
if not self.series:
raise LaunchpadScriptFailure(
"There is no DEVELOPMENT or FROZEN distroseries for %s." %
self.options.distribution)
def getConfig(self):
"""Set up a configuration object for this archive."""
archive = self.distribution.main_archive
if archive:
return getPubConfig(archive)
else:
raise LaunchpadScriptFailure(
"There is no PRIMARY archive for %s." %
self.options.distribution)
def setUpDirs(self):
"""Create output directories if they did not already exist."""
germinateroot = self.config.germinateroot
if not file_exists(germinateroot):
self.logger.debug("Creating germinate root %s.", germinateroot)
os.makedirs(germinateroot)
miscroot = self.config.miscroot
if not file_exists(miscroot):
self.logger.debug("Creating misc root %s.", miscroot)
os.makedirs(miscroot)
def addLogHandler(self):
"""Send germinate's log output to a separate file."""
if self.germinate_logger is not None:
return
self.germinate_logger = logging.getLogger("germinate")
self.germinate_logger.setLevel(logging.INFO)
log_file = os.path.join(self.config.germinateroot, "germinate.output")
handler = logging.FileHandler(log_file, mode="w")
handler.setFormatter(GerminateFormatter())
self.germinate_logger.addHandler(handler)
self.germinate_logger.propagate = False
def setUp(self):
"""Process options, and set up internal state."""
self.processOptions()
self.config = self.getConfig()
self.setUpDirs()
self.addLogHandler()
def getComponents(self, series):
"""Get the list of components to process for a given distroseries.
Even if DistroSeries.component_names starts including partner,
we don't want it; this applies to the primary archive only.
"""
return [component
for component in series.component_names
if component != "partner"]
def makeSeedStructures(self, series_name, flavours, seed_bases=None):
structures = {}
for flavour in flavours:
try:
structure = SeedStructure(
"%s.%s" % (flavour, series_name), seed_bases=seed_bases)
if len(structure):
structures[flavour] = structure
else:
self.logger.warning(
"Skipping empty seed structure for %s.%s",
flavour, series_name)
except SeedError, e:
self.logger.warning(
"Failed to fetch seeds for %s.%s: %s",
flavour, series_name, e)
return structures
def logGerminateProgress(self, *args):
"""Log a "progress" entry to the germinate log file.
Germinate logs quite a bit of detailed information. To make it
easier to see the structure of its operation, GerminateFormatter
allows tagging some log entries as "progress" entries, which are
printed without a prefix.
"""
self.germinate_logger.info(*args, extra={"progress": True})
def composeOutputPath(self, flavour, series_name, arch, base):
return os.path.join(
self.config.germinateroot,
"%s_%s_%s_%s" % (base, flavour, series_name, arch))
def writeGerminateOutput(self, germinator, structure, flavour,
series_name, arch):
"""Write dependency-expanded output files.
These files are a reduced subset of those written by the germinate
command-line program.
"""
path = partial(self.composeOutputPath, flavour, series_name, arch)
# The structure file makes it possible to figure out how the other
# output files relate to each other.
structure.write(path("structure"))
# "all" and "all.sources" list the full set of binary and source
# packages respectively for a given flavour/suite/architecture
# combination.
germinator.write_all_list(structure, path("all"))
germinator.write_all_source_list(structure, path("all.sources"))
# Write the dependency-expanded output for each seed. Several of
# these are used by archive administration tools, and others are
# useful for debugging, so it's best to just write them all.
for seedname in structure.names:
germinator.write_full_list(structure, path(seedname), seedname)
def parseTaskHeaders(self, seedtext):
"""Parse a seed for Task headers.
seedtext is a file-like object. Return a dictionary of Task headers,
with keys canonicalised to lower-case.
"""
task_headers = {}
task_header_regex = re.compile(
r"task-(.*?):(.*)", flags=re.IGNORECASE)
for line in seedtext:
match = task_header_regex.match(line)
if match is not None:
key, value = match.groups()
task_headers[key.lower()] = value.strip()
return task_headers
def getTaskName(self, task_headers, flavour, seedname, primary_flavour):
"""Work out the name of the Task to be generated from this seed.
If there is a Task-Name header, it wins; otherwise, seeds with a
Task-Per-Derivative header are honoured for all flavours and put in
an appropriate namespace, while other seeds are only honoured for
the first flavour and have archive-global names.
"""
if "name" in task_headers:
return task_headers["name"]
elif "per-derivative" in task_headers:
return "%s-%s" % (flavour, seedname)
elif primary_flavour:
return seedname
else:
return None
def getTaskSeeds(self, task_headers, seedname):
"""Return the list of seeds used to generate a task from this seed.
The list of packages in this task comes from this seed plus any
other seeds listed in a Task-Seeds header.
"""
scan_seeds = set([seedname])
if "seeds" in task_headers:
scan_seeds.update(task_headers["seeds"].split())
return sorted(scan_seeds)
def writeOverrides(self, override_file, germinator, structure, arch,
seedname, key, value):
packages = germinator.get_full(structure, seedname)
for package in sorted(packages):
print >>override_file, "%s/%s %s %s" % (
package, arch, key, value)
def germinateArchFlavour(self, override_file, germinator, series_name,
arch, flavour, structure, primary_flavour):
"""Germinate seeds on a single flavour for a single architecture."""
# Expand dependencies.
germinator.plant_seeds(structure)
germinator.grow(structure)
germinator.add_extras(structure)
self.writeGerminateOutput(germinator, structure, flavour, series_name,
arch)
write_overrides = partial(
self.writeOverrides, override_file, germinator, structure, arch)
# Generate apt-ftparchive "extra overrides" for Task fields.
seednames = [name for name in structure.names if name != "extra"]
for seedname in seednames:
with structure[seedname] as seedtext:
task_headers = self.parseTaskHeaders(seedtext)
if task_headers:
task = self.getTaskName(
task_headers, flavour, seedname, primary_flavour)
if task is not None:
scan_seeds = self.getTaskSeeds(task_headers, seedname)
for scan_seed in scan_seeds:
write_overrides(scan_seed, "Task", task)
# Generate apt-ftparchive "extra overrides" for Build-Essential
# fields.
if "build-essential" in structure.names and primary_flavour:
write_overrides("build-essential", "Build-Essential", "yes")
def germinateArch(self, override_file, series_name, components, arch,
flavours, structures):
"""Germinate seeds on all flavours for a single architecture."""
germinator = Germinator(arch)
# Read archive metadata.
archive = TagFile(
series_name, components, arch,
"file://%s" % self.config.archiveroot, cleanup=True)
germinator.parse_archive(archive)
for flavour in flavours:
self.logger.info(
"Germinating for %s/%s/%s", flavour, series_name, arch)
# Add this to the germinate log as well so that that can be
# debugged more easily. Log a separator line first.
self.logGerminateProgress("")
self.logGerminateProgress(
"Germinating for %s/%s/%s", flavour, series_name, arch)
self.germinateArchFlavour(
override_file, germinator, series_name, arch, flavour,
structures[flavour], flavour == flavours[0])
def generateExtraOverrides(self, series_name, components, architectures,
flavours, seed_bases=None):
structures = self.makeSeedStructures(
series_name, flavours, seed_bases=seed_bases)
if structures:
override_path = os.path.join(
self.config.miscroot,
"more-extra.override.%s.main" % series_name)
with AtomicFile(override_path) as override_file:
for arch in architectures:
self.germinateArch(
override_file, series_name, components, arch,
flavours, structures)
def process(self, seed_bases=None):
"""Do the bulk of the work."""
self.setUp()
for series in self.series:
series_name = series.name
components = self.getComponents(series)
architectures = sorted(
[arch.architecturetag for arch in series.architectures])
# This takes a while. Ensure that we do it without keeping a
# database transaction open.
self.txn.commit()
with DatabaseBlockedPolicy():
self.generateExtraOverrides(
series_name, components, architectures, self.args,
seed_bases=seed_bases)
def main(self):
"""See `LaunchpadScript`."""
# This code has no need to alter the database.
with SlaveOnlyDatabasePolicy():
self.process()
|