1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
|
#!/usr/bin/python
#
# python port of the nice maintainace-check script by Nick Barcet
#
# Taken from:
# https://code.edge.launchpad.net/~mvo/ubuntu-maintenance-check/python-port
# (where it will vanish once taken here)
import logging
from optparse import OptionParser
import os
import sys
import urllib2
import urlparse
import apt
import apt_pkg
# This is fun! We have a bunch of cases for 10.04 LTS
#
# - distro "ubuntu" follows SUPPORT_TIMEFRAME_LTS but only for
# amd64/i386
# - distros "kubuntu", "edubuntu" and "netbook" need to be
# considered *but* only follow SUPPORT_TIMEFRAME
# - anything that is in armel follows SUPPORT_TIMEFRAME
#
# codename of the lts releases
LTS_RELEASES = ["dapper", "hardy", "lucid"]
# architectures that are full supported (including LTS time)
PRIMARY_ARCHES = ["i386", "amd64"]
# architectures we support (but not for LTS time)
SUPPORTED_ARCHES = PRIMARY_ARCHES + ["armel"]
# what defines the seeds is documented in wiki.ubuntu.com/SeedManagement
SERVER_SEEDS = ["supported-server", "server-ship"]
DESKTOP_SEEDS = ["ship", "supported-desktop", "supported-desktop-extra"]
SUPPORTED_SEEDS = ["all"]
# normal support timeframe
# time, seeds, arches
SUPPORT_TIMEFRAME = [
("18m", SUPPORTED_SEEDS),
]
# lts support timeframe
# time, seeds, arches
SUPPORT_TIMEFRAME_LTS = [
("5y", SERVER_SEEDS),
("3y", DESKTOP_SEEDS),
("18m", SUPPORTED_SEEDS),
]
# distro names and if they get LTS support (order is important)
DISTRO_NAMES_AND_LTS_SUPPORT = [
("ubuntu", True),
("kubuntu", True),
("netbook", False),
]
# Names of the distribution releases that are not supported by this
# tool. All later versions are supported.
UNSUPPORTED_DISTRO_RELEASED = [
"dapper",
"edgy",
"feisty",
"gutsy",
"hardy",
"intrepid",
"jaunty",
"karmic",
]
# germinate output base directory
BASE_URL = os.environ.get(
"MAINTENANCE_CHECK_BASE_URL",
"http://people.canonical.com/~ubuntu-archive/germinate-output/")
# hints dir url, hints file is "$distro.hints" by default
# (e.g. lucid.hints)
HINTS_DIR_URL = os.environ.get(
"MAINTENANCE_CHECK_HINTS_DIR_URL",
"http://people.canonical.com/"
"~ubuntu-archive/seeds/platform.%s/SUPPORTED_HINTS")
# we need the archive root to parse the Sources file to support
# by-source hints
ARCHIVE_ROOT = os.environ.get(
"MAINTENANCE_CHECK_ARCHIVE_ROOT", "http://archive.ubuntu.com/ubuntu")
# support timeframe tag used in the Packages file
SUPPORT_TAG = "Supported"
def get_binaries_for_source_pkg(srcname):
""" Return all binary package names for the given source package name.
:param srcname: The source package name.
:return: A list of binary package names.
"""
pkgnames = set()
recs = apt_pkg.SourceRecords()
while recs.lookup(srcname):
for binary in recs.binaries:
pkgnames.add(binary)
return pkgnames
def expand_src_pkgname(pkgname):
""" Expand a package name if it is prefixed with src.
If the package name is prefixed with src it will be expanded
to a list of binary package names. Otherwise the original
package name will be returned.
:param pkgname: The package name (that may include src:prefix).
:return: A list of binary package names (the list may be one element
long).
"""
if not pkgname.startswith("src:"):
return [pkgname]
return get_binaries_for_source_pkg(pkgname.split("src:")[1])
def create_and_update_deb_src_source_list(distroseries):
""" Create sources.list and update cache.
This creates a sources.list file with deb-src entries for a given
distroseries and apt.Cache.update() to make sure the data is up-to-date.
:param distro: The code name of the distribution series (e.g. lucid).
:return: None
:raises: IOError: When cache update fails.
"""
# apt root dir
rootdir = "./aptroot.%s" % distroseries
sources_list_dir = os.path.join(rootdir, "etc", "apt")
if not os.path.exists(sources_list_dir):
os.makedirs(sources_list_dir)
sources_list = open(os.path.join(sources_list_dir, "sources.list"), "w")
for pocket in [
"%s" % distroseries,
"%s-updates" % distroseries,
"%s-security" % distroseries]:
sources_list.write(
"deb-src %s %s main restricted\n" % (
ARCHIVE_ROOT, pocket))
sources_list.write(
"deb %s %s main restricted\n" % (
ARCHIVE_ROOT, pocket))
sources_list.close()
# create required dirs/files for apt.Cache(rootdir) to work on older
# versions of python-apt. once lucid is used it can be removed
for d in ["var/lib/dpkg",
"var/cache/apt/archives/partial",
"var/lib/apt/lists/partial"]:
if not os.path.exists(os.path.join(rootdir, d)):
os.makedirs(os.path.join(rootdir, d))
if not os.path.exists(os.path.join(rootdir, "var/lib/dpkg/status")):
open(os.path.join(rootdir, "var/lib/dpkg/status"), "w")
# open cache with our just prepared rootdir
cache = apt.Cache(rootdir=rootdir)
try:
cache.update()
except SystemError:
logging.exception("cache.update() failed")
def get_structure(distroname, version):
""" Get structure file conent for named distro and distro version.
:param name: Name of the distribution (e.g. kubuntu, ubuntu, xubuntu).
:param version: Code name of the distribution version (e.g. lucid).
:return: List of strings with the structure file content
"""
f = urllib2.urlopen("%s/%s.%s/structure" % (
BASE_URL, distroname, version))
structure = f.readlines()
f.close()
return structure
def expand_seeds(structure, seedname):
"""Expand seed by its dependencies using the strucure file.
:param structure: The content of the STRUCTURE file as string list.
:param seedname: The name of the seed as string that needs to be expanded.
:return: A set() for the seed dependencies (excluding the original
seedname).
"""
seeds = []
for line in structure:
if line.startswith("%s:" % seedname):
seeds += line.split(":")[1].split()
for seed in seeds:
seeds += expand_seeds(structure, seed)
return set(seeds)
def get_packages_for_seeds(name, distro, seeds):
"""
Get packages for the given name (e.g. ubuntu) and distro release
(e.g. lucid) that are in the given list of seeds
returns a set() of package names.
"""
pkgs_in_seeds = {}
for seed in seeds:
pkgs_in_seeds[seed] = set()
seedurl = "%s/%s.%s/%s" % (BASE_URL, name, distro, seed)
logging.debug("looking for '%s'", seedurl)
try:
f = urllib2.urlopen(seedurl)
for line in f:
# Ignore lines that are not package names (headers etc).
if line[0] < 'a' or line[0] > 'z':
continue
# Each line contains these fields:
# (package, source, why, maintainer, size, inst-size)
if options.source_packages:
pkgname = line.split("|")[1]
else:
pkgname = line.split("|")[0]
pkgs_in_seeds[seed].add(pkgname.strip())
f.close()
except Exception as e:
logging.error("seed %s failed (%s)" % (seedurl, e))
return pkgs_in_seeds
def what_seeds(pkgname, seeds):
in_seeds = set()
for s in seeds:
if pkgname in seeds[s]:
in_seeds.add(s)
return in_seeds
def compare_support_level(x, y):
"""
compare two support level strings of the form 18m, 3y etc
:parm x: the first support level
:parm y: the second support level
:return: negative if x < y, zero if x==y, positive if x > y
"""
def support_to_int(support_time):
"""
helper that takes a support time string and converts it to
a integer for cmp()
"""
# allow strings like "5y (kubuntu-common)
x = support_time.split()[0]
if x.endswith("y"):
return 12 * int(x[0:-1])
elif x.endswith("m"):
return int(x[0:-1])
else:
raise ValueError("support time '%s' has to end with y or m" % x)
return cmp(support_to_int(x), support_to_int(y))
def get_packages_support_time(structure, name, pkg_support_time,
support_timeframe_list):
"""
input a structure file and a list of pair<timeframe, seedlist>
return a dict of pkgnames -> support timeframe string
"""
for (timeframe, seedlist) in support_timeframe_list:
expanded = set()
for s in seedlist:
expanded.add(s)
expanded |= expand_seeds(structure, s)
pkgs_in_seeds = get_packages_for_seeds(name, distro, expanded)
for seed in pkgs_in_seeds:
for pkg in pkgs_in_seeds[seed]:
if not pkg in pkg_support_time:
pkg_support_time[pkg] = timeframe
else:
old_timeframe = pkg_support_time[pkg]
if compare_support_level(old_timeframe, timeframe) < 0:
logging.debug("overwriting %s from %s to %s" % (
pkg, old_timeframe, timeframe))
pkg_support_time[pkg] = timeframe
if options.with_seeds:
pkg_support_time[pkg] += " (%s)" % ", ".join(
what_seeds(pkg, pkgs_in_seeds))
return pkg_support_time
if __name__ == "__main__":
parser = OptionParser()
parser.add_option("--with-seeds", "", default=False,
action="store_true",
help="add seed(s) of the package that are responsible "
"for the maintaince time")
parser.add_option("--source-packages", "", default=False,
action="store_true",
help="show as source pkgs")
parser.add_option("--hints-file", "", default=None,
help="use diffenrt use hints file location")
(options, args) = parser.parse_args()
# init
if len(args) > 0:
distro = args[0]
if distro in UNSUPPORTED_DISTRO_RELEASED:
logging.error("only lucid or later is supported")
sys.exit(1)
else:
distro = "lucid"
# make sure our deb-src information is up-to-date
create_and_update_deb_src_source_list(distro)
if options.hints_file:
hints_file = options.hints_file
(schema, netloc, path, query, fragment) = urlparse.urlsplit(
hints_file)
if not schema:
hints_file = "file:%s" % path
else:
hints_file = HINTS_DIR_URL % distro
# go over the distros we need to check
pkg_support_time = {}
for (name, lts_supported) in DISTRO_NAMES_AND_LTS_SUPPORT:
# get basic structure file
try:
structure = get_structure(name, distro)
except urllib2.HTTPError:
logging.error("Can not get structure for '%s'." % name)
continue
# get dicts of pkgname -> support timeframe string
support_timeframe = SUPPORT_TIMEFRAME
if lts_supported and distro in LTS_RELEASES:
support_timeframe = SUPPORT_TIMEFRAME_LTS
else:
support_timeframe = SUPPORT_TIMEFRAME
get_packages_support_time(
structure, name, pkg_support_time, support_timeframe)
# now go over the bits in main that we have not seen (because
# they are not in any seed and got added manually into "main"
for arch in PRIMARY_ARCHES:
rootdir = "./aptroot.%s" % distro
apt_pkg.config.set("APT::Architecture", arch)
cache = apt.Cache(rootdir=rootdir)
try:
cache.update()
except SystemError:
logging.exception("cache.update() failed")
cache.open()
for pkg in cache:
if not pkg.name in pkg_support_time:
pkg_support_time[pkg.name] = support_timeframe[-1][0]
logging.warn(
"add package in main but not in seeds %s with %s" % (
pkg.name, pkg_support_time[pkg.name]))
# now check the hints file that is used to overwrite
# the default seeds
try:
for line in urllib2.urlopen(hints_file):
line = line.strip()
if not line or line.startswith("#"):
continue
try:
(raw_pkgname, support_time) = line.split()
for pkgname in expand_src_pkgname(raw_pkgname):
if support_time == 'unsupported':
try:
del pkg_support_time[pkgname]
sys.stderr.write("hints-file: marking %s "
"unsupported\n" % pkgname)
except KeyError:
pass
else:
if pkg_support_time.get(pkgname) != support_time:
sys.stderr.write(
"hints-file: changing %s from %s to %s\n" % (
pkgname, pkg_support_time.get(pkgname),
support_time))
pkg_support_time[pkgname] = support_time
except:
logging.exception("can not parse line '%s'" % line)
except urllib2.HTTPError, e:
if e.code != 404:
raise
sys.stderr.write("hints-file: %s gave 404 error\n" % hints_file)
# output suitable for the extra-override file
for pkgname in sorted(pkg_support_time.keys()):
# special case, the hints file may contain overrides that
# are arch-specific (like zsh-doc/armel)
if "/" in pkgname:
print "%s %s %s" % (
pkgname, SUPPORT_TAG, pkg_support_time[pkgname])
else:
# go over the supported arches, they are divided in
# first-class (PRIMARY) and second-class with different
# support levels
for arch in SUPPORTED_ARCHES:
# ensure we do not overwrite arch-specific overwrites
pkgname_and_arch = "%s/%s" % (pkgname, arch)
if pkgname_and_arch in pkg_support_time:
break
if arch in PRIMARY_ARCHES:
# arch with full LTS support
print "%s %s %s" % (
pkgname_and_arch, SUPPORT_TAG,
pkg_support_time[pkgname])
else:
# not a LTS supported architecture, gets only regular
# support_timeframe
print "%s %s %s" % (
pkgname_and_arch, SUPPORT_TAG,
SUPPORT_TIMEFRAME[0][0])
|