← Back to branch summary

~launchpad-pqm/launchpad/devel

« back to all changes in this revision

Viewing changes to scripts/ftpmaster-tools/remove-package.py

  • Committer: Steve Kowalik
  • Date: 2011-08-07 04:05:52 UTC
  • mto: This revision was merged to the branch mainline in revision 13626.
  • Revision ID: stevenk@ubuntu.com-20110807040552-mwnxo0flmhvl35e8
Correct the notification based on review comments, and remove request{,ed}
from the function names, switching to create{,d}.

Show diffs side-by-side

added added

removed removed

Lines of Context:
scripts/ftpmaster-tools/remove-package.py
 
1
#!/usr/bin/python -S
 
2
#
 
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
 
4
# GNU Affero General Public License version 3 (see the file LICENSE).
 
5
 
 
6
# General purpose package removal tool for ftpmaster
 
7
 
 
8
################################################################################
 
9
 
 
10
import _pythonpath
 
11
 
 
12
import commands
 
13
import optparse
 
14
import os
 
15
import re
 
16
import sys
 
17
 
 
18
import dak_utils
 
19
 
 
20
import apt_pkg
 
21
 
 
22
from zope.component import getUtility
 
23
 
 
24
from canonical.config import config
 
25
from canonical.database.constants import UTC_NOW
 
26
from canonical.launchpad.scripts import (
 
27
    execute_zcml_for_scripts,
 
28
    logger,
 
29
    logger_options,
 
30
    )
 
31
from canonical.lp import initZopeless
 
32
from lp.registry.interfaces.distribution import IDistributionSet
 
33
from lp.soyuz.enums import PackagePublishingStatus
 
34
from lp.soyuz.model.publishing import (
 
35
    BinaryPackagePublishingHistory,
 
36
    SourcePackagePublishingHistory,
 
37
    )
 
38
 
 
39
from contrib.glock import GlobalLock
 
40
 
 
41
################################################################################
 
42
 
 
43
re_strip_source_version = re.compile (r'\s+.*$')
 
44
re_build_dep_arch = re.compile(r"\[[^]]+\]")
 
45
 
 
46
################################################################################
 
47
 
 
48
Options = None
 
49
Lock = None
 
50
Log = None
 
51
ztm = None
 
52
 
 
53
################################################################################
 
54
 
 
55
def game_over():
 
56
    answer = dak_utils.our_raw_input("Continue (y/N)? ").lower()
 
57
    if answer != "y":
 
58
        print "Aborted."
 
59
        sys.exit(1)
 
60
 
 
61
################################################################################
 
62
 
 
63
# def reverse_depends_check(removals, suites):
 
64
#     print "Checking reverse dependencies..."
 
65
#     components = Cnf.ValueList("Suite::%s::Components" % suites[0])
 
66
#     dep_problem = 0
 
67
#     p2c = {}
 
68
#     for architecture in Cnf.ValueList("Suite::%s::Architectures" % suites[0]):
 
69
#         if architecture in ["source", "all"]:
 
70
#             continue
 
71
#         deps = {}
 
72
#         virtual_packages = {}
 
73
#         for component in components:
 
74
#             filename = "%s/dists/%s/%s/binary-%s/Packages.gz" \
 
75
#                        % (Cnf["Dir::Root"], suites[0], component,
 
76
#                           architecture)
 
77
#             # apt_pkg.ParseTagFile needs a real file handle and can't
 
78
#             # handle a GzipFile instance...
 
79
#             temp_filename = dak_utils.temp_filename()
 
80
#             (result, output) = commands.getstatusoutput("gunzip -c %s > %s" \
 
81
#                                                         % (filename, temp_filename))
 
82
#             if (result != 0):
 
83
#                 dak_utils.fubar("Gunzip invocation failed!\n%s\n" \
 
84
#                                 % (output), result)
 
85
#             packages = open(temp_filename)
 
86
#             Packages = apt_pkg.ParseTagFile(packages)
 
87
#             while Packages.Step():
 
88
#                 package = Packages.Section.Find("Package")
 
89
#                 depends = Packages.Section.Find("Depends")
 
90
#                 if depends:
 
91
#                     deps[package] = depends
 
92
#                 provides = Packages.Section.Find("Provides")
 
93
#                 # Maintain a counter for each virtual package.  If a
 
94
#                 # Provides: exists, set the counter to 0 and count all
 
95
#                 # provides by a package not in the list for removal.
 
96
#                 # If the counter stays 0 at the end, we know that only
 
97
#                 # the to-be-removed packages provided this virtual
 
98
#                 # package.
 
99
#                 if provides:
 
100
#                     for virtual_pkg in provides.split(","):
 
101
#                         virtual_pkg = virtual_pkg.strip()
 
102
#                         if virtual_pkg == package: continue
 
103
#                         if not virtual_packages.has_key(virtual_pkg):
 
104
#                             virtual_packages[virtual_pkg] = 0
 
105
#                         if package not in removals:
 
106
#                             virtual_packages[virtual_pkg] += 1
 
107
#                 p2c[package] = component
 
108
#             packages.close()
 
109
#             os.unlink(temp_filename)
 
110
 
 
111
#         # If a virtual package is only provided by the to-be-removed
 
112
#         # packages, treat the virtual package as to-be-removed too.
 
113
#         for virtual_pkg in virtual_packages.keys():
 
114
#             if virtual_packages[virtual_pkg] == 0:
 
115
#                 removals.append(virtual_pkg)
 
116
 
 
117
#         # Check binary dependencies (Depends)
 
118
#         for package in deps.keys():
 
119
#             if package in removals: continue
 
120
#             parsed_dep = []
 
121
#             try:
 
122
#                 parsed_dep += apt_pkg.ParseDepends(deps[package])
 
123
#             except ValueError, e:
 
124
#                 print "Error for package %s: %s" % (package, e)
 
125
#             for dep in parsed_dep:
 
126
#                 # Check for partial breakage.  If a package has a ORed
 
127
#                 # dependency, there is only a dependency problem if all
 
128
#                 # packages in the ORed depends will be removed.
 
129
#                 unsat = 0
 
130
#                 for dep_package, _, _ in dep:
 
131
#                     if dep_package in removals:
 
132
#                             unsat += 1
 
133
#                 if unsat == len(dep):
 
134
#                     component = p2c[package]
 
135
#                     if component != "main":
 
136
#                         what = "%s/%s" % (package, component)
 
137
#                     else:
 
138
#                         what = "** %s" % (package)
 
139
#                     print "%s has an unsatisfied dependency on %s: %s" \
 
140
#                           % (what, architecture, dak_utils.pp_deps(dep))
 
141
#                     dep_problem = 1
 
142
 
 
143
#     # Check source dependencies (Build-Depends and Build-Depends-Indep)
 
144
#     for component in components:
 
145
#         filename = "%s/dists/%s/%s/source/Sources.gz" \
 
146
#                    % (Cnf["Dir::Root"], suites[0], component)
 
147
#         # apt_pkg.ParseTagFile needs a real file handle and can't
 
148
#         # handle a GzipFile instance...
 
149
#         temp_filename = dak_utils.temp_filename()
 
150
#         result, output = commands.getstatusoutput("gunzip -c %s > %s" \
 
151
#                                                   % (filename, temp_filename))
 
152
#         if result != 0:
 
153
#             sys.stderr.write("Gunzip invocation failed!\n%s\n" \
 
154
#                              % (output))
 
155
#             sys.exit(result)
 
156
#         sources = open(temp_filename)
 
157
#         Sources = apt_pkg.ParseTagFile(sources)
 
158
#         while Sources.Step():
 
159
#             source = Sources.Section.Find("Package")
 
160
#             if source in removals: continue
 
161
#             parsed_dep = []
 
162
#             for build_dep_type in ["Build-Depends", "Build-Depends-Indep"]:
 
163
#                 build_dep = Sources.Section.get(build_dep_type)
 
164
#                 if build_dep:
 
165
#                     # Remove [arch] information since we want to see
 
166
#                     # breakage on all arches
 
167
#                     build_dep = re_build_dep_arch.sub("", build_dep)
 
168
#                     try:
 
169
#                         parsed_dep += apt_pkg.ParseDepends(build_dep)
 
170
#                     except ValueError, e:
 
171
#                         print "Error for source %s: %s" % (source, e)
 
172
#             for dep in parsed_dep:
 
173
#                 unsat = 0
 
174
#                 for dep_package, _, _ in dep:
 
175
#                     if dep_package in removals:
 
176
#                             unsat += 1
 
177
#                 if unsat == len(dep):
 
178
#                     if component != "main":
 
179
#                         source = "%s/%s" % (source, component)
 
180
#                     else:
 
181
#                         source = "** %s" % (source)
 
182
#                     print "%s has an unsatisfied build-dependency: %s" \
 
183
#                           % (source, dak_utils.pp_deps(dep))
 
184
#                     dep_problem = 1
 
185
#         sources.close()
 
186
#         os.unlink(temp_filename)
 
187
 
 
188
#     if dep_problem:
 
189
#         print "Dependency problem found."
 
190
#         if Options.action:
 
191
#             game_over()
 
192
#     else:
 
193
#         print "No dependency problem found."
 
194
#     print
 
195
 
 
196
################################################################################
 
197
 
 
198
def options_init():
 
199
    global Options
 
200
 
 
201
    parser = optparse.OptionParser()
 
202
    logger_options(parser)
 
203
    parser.add_option("-a", "--architecture", dest="architecture",
 
204
                      help="only act on ARCHITECTURE")
 
205
    parser.add_option("-b", "--binary", dest="binaryonly",
 
206
                      default=False, action="store_true",
 
207
                      help="remove binaries only")
 
208
    parser.add_option("-c", "--component", dest="component",
 
209
                      help="only act on COMPONENT")
 
210
    parser.add_option("-d", "--distro", dest="distro",
 
211
                      help="remove from DISTRO")
 
212
    parser.add_option("-m", "--reason", dest="reason",
 
213
                      help="reason for removal")
 
214
    parser.add_option("-n", "--no-action", dest="action",
 
215
                      default=True, action="store_false",
 
216
                      help="don't do anything")
 
217
    parser.add_option("-R", "--rdep-check", dest="rdepcheck",
 
218
                      default=False, action="store_true",
 
219
                      help="check reverse dependencies")
 
220
    parser.add_option("-s", "--suite", dest="suite",
 
221
                      help="only act on SUITE")
 
222
    parser.add_option("-S", "--source-only", dest="sourceonly",
 
223
                      default=False, action="store_true",
 
224
                      help="remove source only")
 
225
 
 
226
    (Options, arguments) = parser.parse_args()
 
227
 
 
228
    # Sanity check options
 
229
    if not arguments:
 
230
        dak_utils.fubar("need at least one package name as an argument.")
 
231
    if Options.architecture and Options.sourceonly:
 
232
        dak_utils.fubar("can't use -a/--architecutre and -S/"
 
233
                        "--source-only options simultaneously.")
 
234
    if Options.binaryonly and Options.sourceonly:
 
235
        dak_utils.fubar("can't use -b/--binary-only and -S/"
 
236
                        "--source-only options simultaneously.")
 
237
 
 
238
    if not Options.reason:
 
239
        Options.reason = ""
 
240
 
 
241
    # XXX malcc 2006-08-03: 'dak rm' used to check here whether or not we're
 
242
    # removing from anything other than << unstable.  This never got ported
 
243
    # to ubuntu anyway, but it might be nice someday.
 
244
 
 
245
    # Additional architecture checks
 
246
    # XXX James Troup 2006-01-30: parse_args.
 
247
    if Options.architecture and 0:
 
248
        dak_utils.warn("'source' in -a/--argument makes no sense and is ignored.")
 
249
 
 
250
    return arguments
 
251
 
 
252
################################################################################
 
253
def init():
 
254
    global Lock, Log, ztm
 
255
 
 
256
    apt_pkg.init()
 
257
 
 
258
    arguments = options_init()
 
259
 
 
260
    Log = logger(Options, "remove-package")
 
261
 
 
262
    Log.debug("Acquiring lock")
 
263
    Lock = GlobalLock('/var/lock/launchpad-remove-package.lock')
 
264
    Lock.acquire(blocking=True)
 
265
 
 
266
    Log.debug("Initializing connection.")
 
267
    execute_zcml_for_scripts()
 
268
    ztm = initZopeless(dbuser=config.archivepublisher.dbuser)
 
269
 
 
270
    if not Options.distro:
 
271
        Options.distro = "ubuntu"
 
272
    Options.distro = getUtility(IDistributionSet)[Options.distro]
 
273
 
 
274
    if not Options.suite:
 
275
        Options.suite = Options.distro.currentseries.name
 
276
 
 
277
    Options.architecture = dak_utils.split_args(Options.architecture)
 
278
    Options.component = dak_utils.split_args(Options.component)
 
279
    Options.suite = dak_utils.split_args(Options.suite)
 
280
 
 
281
    return arguments
 
282
 
 
283
################################################################################
 
284
 
 
285
def summary_to_remove(to_remove):
 
286
    # Generate the summary of what's to be removed
 
287
    d = {}
 
288
    for removal in to_remove:
 
289
        package = removal["package"]
 
290
        version = removal["version"]
 
291
        architecture = removal["architecture"]
 
292
        if not d.has_key(package):
 
293
            d[package] = {}
 
294
        if not d[package].has_key(version):
 
295
            d[package][version] = []
 
296
        if architecture not in d[package][version]:
 
297
            d[package][version].append(architecture)
 
298
 
 
299
    summary = ""
 
300
    removals = d.keys()
 
301
    removals.sort()
 
302
    for package in removals:
 
303
        versions = d[package].keys()
 
304
        versions.sort(apt_pkg.VersionCompare)
 
305
        for version in versions:
 
306
            d[package][version].sort(dak_utils.arch_compare_sw)
 
307
            summary += "%10s | %10s | %s\n" % (package, version,
 
308
                                               ", ".join(d[package][version]))
 
309
 
 
310
    suites_list = dak_utils.join_with_commas_and(Options.suite);
 
311
    print "Will remove the following packages from %s:" % (suites_list)
 
312
    print
 
313
    print summary
 
314
    print
 
315
    print "------------------- Reason -------------------"
 
316
    print Options.reason
 
317
    print "----------------------------------------------"
 
318
    print
 
319
 
 
320
    return summary
 
321
 
 
322
################################################################################
 
323
 
 
324
def what_to_remove(packages):
 
325
    to_remove = []
 
326
 
 
327
    # We have 3 modes of package selection: binary-only, source-only
 
328
    # and source+binary.  The first two are trivial and obvious; the
 
329
    # latter is a nasty mess, but very nice from a UI perspective so
 
330
    # we try to support it.
 
331
 
 
332
    for removal in packages:
 
333
        for suite in Options.suite:
 
334
            distro_series = Options.distro.getSeries(suite)
 
335
 
 
336
            if Options.sourceonly:
 
337
                bpp_list = []
 
338
            else:
 
339
                if Options.binaryonly:
 
340
                    bpp_list = distro_series.getBinaryPackagePublishing(removal)
 
341
                else:
 
342
                    bpp_list = distro_series.getBinaryPackagePublishing(
 
343
                        sourcename=removal)
 
344
 
 
345
            for bpp in bpp_list:
 
346
                package=bpp.binarypackagerelease.binarypackagename.name
 
347
                version=bpp.binarypackagerelease.version
 
348
                architecture=bpp.distroarchseries.architecturetag
 
349
                if (Options.architecture and
 
350
                    architecture not in Options.architecture):
 
351
                    continue
 
352
                if (Options.component and
 
353
                    bpp.component.name not in Options.component):
 
354
                    continue
 
355
                d = dak_utils.Dict(
 
356
                    type="binary", publishing=bpp, package=package,
 
357
                    version=version, architecture=architecture)
 
358
                to_remove.append(d)
 
359
 
 
360
            if not Options.binaryonly:
 
361
                for spp in distro_series.getPublishedSources(removal):
 
362
                    package = spp.sourcepackagerelease.sourcepackagename.name
 
363
                    version = spp.sourcepackagerelease.version
 
364
                    if (Options.component and
 
365
                        spp.component.name not in Options.component):
 
366
                        continue
 
367
                    d = dak_utils.Dict(
 
368
                        type="source",publishing=spp, package=package,
 
369
                        version=version, architecture="source")
 
370
                    to_remove.append(d)
 
371
 
 
372
    return to_remove
 
373
 
 
374
################################################################################
 
375
 
 
376
def do_removal(removal):
 
377
    """Perform published package removal.
 
378
 
 
379
    Mark provided publishing record as SUPERSEDED, such that the Domination
 
380
    procedure will sort out its eventual removal appropriately; obeying the
 
381
    rules for archive consistency.
 
382
    """
 
383
    current = removal["publishing"]
 
384
    if removal["type"] == "binary":
 
385
        real_current = BinaryPackagePublishingHistory.get(current.id)
 
386
    else:
 
387
        real_current = SourcePackagePublishingHistory.get(current.id)
 
388
    real_current.status = PackagePublishingStatus.SUPERSEDED
 
389
    real_current.datesuperseded = UTC_NOW
 
390
 
 
391
################################################################################
 
392
 
 
393
def main ():
 
394
    packages = init()
 
395
 
 
396
    print "Working...",
 
397
    sys.stdout.flush()
 
398
    to_remove = what_to_remove(packages)
 
399
    print "done."
 
400
 
 
401
    if not to_remove:
 
402
        print "Nothing to do."
 
403
        sys.exit(0)
 
404
 
 
405
    # If we don't have a reason; spawn an editor so the user can add one
 
406
    # Write the rejection email out as the <foo>.reason file
 
407
    if not Options.reason and Options.action:
 
408
        temp_filename = dak_utils.temp_filename()
 
409
        editor = os.environ.get("EDITOR","vi")
 
410
        result = os.system("%s %s" % (editor, temp_filename))
 
411
        if result != 0:
 
412
            dak_utils.fubar ("vi invocation failed for `%s'!" % (temp_filename),
 
413
                             result)
 
414
        temp_file = open(temp_filename)
 
415
        for line in temp_file.readlines():
 
416
            Options.reason += line
 
417
        temp_file.close()
 
418
        os.unlink(temp_filename)
 
419
 
 
420
    summary = summary_to_remove(to_remove)
 
421
 
 
422
    if Options.rdepcheck:
 
423
        dak_utils.fubar("Unimplemented, sucks to be you.")
 
424
        #reverse_depends_check(removals, suites)
 
425
 
 
426
    # If -n/--no-action, drop out here
 
427
    if not Options.action:
 
428
        sys.exit(0)
 
429
 
 
430
    print "Going to remove the packages now."
 
431
    game_over()
 
432
 
 
433
    whoami = dak_utils.whoami()
 
434
    date = commands.getoutput('date -R')
 
435
    suites_list = dak_utils.join_with_commas_and(Options.suite);
 
436
 
 
437
    # Log first; if it all falls apart I want a record that we at least tried.
 
438
    # XXX malcc 2006-08-03: de-hardcode me harder
 
439
    logfile = open("/srv/launchpad.net/dak/removals.txt", 'a')
 
440
    logfile.write("==================================="
 
441
                  "======================================\n")
 
442
    logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami))
 
443
    logfile.write("Removed the following packages from %s:\n\n%s"
 
444
                  % (suites_list, summary))
 
445
    logfile.write("\n------------------- Reason -------------------\n%s\n"
 
446
                  % (Options.reason))
 
447
    logfile.write("----------------------------------------------\n")
 
448
    logfile.flush()
 
449
 
 
450
    # Do the actual deletion
 
451
    print "Deleting...",
 
452
    ztm.begin()
 
453
    for removal in to_remove:
 
454
        do_removal(removal)
 
455
    print "done."
 
456
    ztm.commit()
 
457
 
 
458
    logfile.write("==================================="
 
459
                  "======================================\n")
 
460
    logfile.close()
 
461
 
 
462
################################################################################
 
463
 
 
464
if __name__ == '__main__':
 
465
    main()