← Back to branch summary

~launchpad-pqm/launchpad/devel

« back to all changes in this revision

Viewing changes to lib/lp/archivepublisher/domination.py

  • Committer: Julian Edwards
  • Date: 2011-07-28 20:46:18 UTC
  • mfrom: (13553 devel)
  • mto: This revision was merged to the branch mainline in revision 13555.
  • Revision ID: julian.edwards@canonical.com-20110728204618-tivj2wx2oa9s32bx
merge trunk

Show diffs side-by-side

added added

removed removed

Lines of Context:
lib/lp/archivepublisher/domination.py
1
 
# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
 
1
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2
2
# GNU Affero General Public License version 3 (see the file LICENSE).
3
3
 
4
4
"""Archive Domination class.
52
52
 
53
53
__all__ = ['Dominator']
54
54
 
55
 
from collections import defaultdict
56
55
from datetime import timedelta
57
 
from itertools import (
58
 
    ifilter,
59
 
    ifilterfalse,
60
 
    )
61
 
from operator import (
62
 
    attrgetter,
63
 
    itemgetter,
64
 
    )
 
56
import functools
 
57
import operator
65
58
 
66
59
import apt_pkg
67
 
from storm.expr import (
68
 
    And,
69
 
    Count,
70
 
    Desc,
71
 
    Select,
72
 
    )
 
60
from storm.expr import And, Count, Select
73
61
 
74
62
from canonical.database.constants import UTC_NOW
75
63
from canonical.database.sqlbase import (
76
64
    flush_database_updates,
77
65
    sqlvalues,
78
66
    )
79
 
from lp.services.database.lpstorm import IStore
80
 
from lp.services.orderingcheck import OrderingCheck
 
67
from canonical.launchpad.interfaces.lpstorm import IMasterStore
 
68
from lp.archivepublisher import ELIGIBLE_DOMINATION_STATES
81
69
from lp.registry.model.sourcepackagename import SourcePackageName
82
 
from lp.services.database.bulk import load_related
83
 
from lp.services.database.decoratedresultset import DecoratedResultSet
84
70
from lp.soyuz.enums import (
85
71
    BinaryPackageFormat,
86
72
    PackagePublishingStatus,
87
73
    )
88
 
from lp.soyuz.interfaces.publishing import inactive_publishing_status
89
74
from lp.soyuz.model.binarypackagename import BinaryPackageName
90
75
from lp.soyuz.model.binarypackagerelease import BinaryPackageRelease
91
76
from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
92
77
 
 
78
 
93
79
# Days before a package will be removed from disk.
94
80
STAY_OF_EXECUTION = 1
95
81
 
98
84
apt_pkg.InitSystem()
99
85
 
100
86
 
101
 
def join_spph_spn():
102
 
    """Join condition: SourcePackagePublishingHistory/SourcePackageName."""
103
 
    # Avoid circular imports.
104
 
    from lp.soyuz.model.publishing import SourcePackagePublishingHistory
105
 
 
106
 
    SPPH = SourcePackagePublishingHistory
107
 
    SPN = SourcePackageName
108
 
 
109
 
    return SPN.id == SPPH.sourcepackagenameID
110
 
 
111
 
 
112
 
def join_spph_spr():
113
 
    """Join condition: SourcePackageRelease/SourcePackagePublishingHistory.
114
 
    """
115
 
    # Avoid circular imports.
116
 
    from lp.soyuz.model.publishing import SourcePackagePublishingHistory
117
 
 
118
 
    SPPH = SourcePackagePublishingHistory
119
 
    SPR = SourcePackageRelease
120
 
 
121
 
    return SPR.id == SPPH.sourcepackagereleaseID
122
 
 
123
 
 
124
 
class SourcePublicationTraits:
125
 
    """Basic generalized attributes for `SourcePackagePublishingHistory`.
126
 
 
127
 
    Used by `GeneralizedPublication` to hide the differences from
128
 
    `BinaryPackagePublishingHistory`.
129
 
    """
130
 
    release_class = SourcePackageRelease
131
 
    release_reference_name = 'sourcepackagereleaseID'
132
 
 
133
 
    @staticmethod
134
 
    def getPackageName(spph):
135
 
        """Return the name of this publication's source package."""
136
 
        return spph.sourcepackagename.name
137
 
 
138
 
    @staticmethod
139
 
    def getPackageRelease(spph):
140
 
        """Return this publication's `SourcePackageRelease`."""
141
 
        return spph.sourcepackagerelease
142
 
 
143
 
 
144
 
class BinaryPublicationTraits:
145
 
    """Basic generalized attributes for `BinaryPackagePublishingHistory`.
146
 
 
147
 
    Used by `GeneralizedPublication` to hide the differences from
148
 
    `SourcePackagePublishingHistory`.
149
 
    """
150
 
    release_class = BinaryPackageRelease
151
 
    release_reference_name = 'binarypackagereleaseID'
152
 
 
153
 
    @staticmethod
154
 
    def getPackageName(bpph):
155
 
        """Return the name of this publication's binary package."""
156
 
        return bpph.binarypackagename.name
157
 
 
158
 
    @staticmethod
159
 
    def getPackageRelease(bpph):
160
 
        """Return this publication's `BinaryPackageRelease`."""
161
 
        return bpph.binarypackagerelease
162
 
 
163
 
 
164
 
class GeneralizedPublication:
165
 
    """Generalize handling of publication records.
166
 
 
167
 
    This allows us to write code that can be dealing with either
168
 
    `SourcePackagePublishingHistory`s or `BinaryPackagePublishingHistory`s
169
 
    without caring which.  Differences are abstracted away in a traits
170
 
    class.
171
 
    """
172
 
    def __init__(self, is_source=True):
173
 
        self.is_source = is_source
174
 
        if is_source:
175
 
            self.traits = SourcePublicationTraits
176
 
        else:
177
 
            self.traits = BinaryPublicationTraits
178
 
 
179
 
    def getPackageName(self, pub):
180
 
        """Get the package's name."""
181
 
        return self.traits.getPackageName(pub)
182
 
 
183
 
    def getPackageVersion(self, pub):
184
 
        """Obtain the version string for a publication record."""
185
 
        return self.traits.getPackageRelease(pub).version
186
 
 
187
 
    def compare(self, pub1, pub2):
188
 
        """Compare publications by version.
189
 
 
190
 
        If both publications are for the same version, their creation dates
191
 
        break the tie.
192
 
        """
193
 
        version_comparison = apt_pkg.VersionCompare(
194
 
            self.getPackageVersion(pub1), self.getPackageVersion(pub2))
195
 
 
196
 
        if version_comparison == 0:
197
 
            # Use dates as tie breaker.
198
 
            return cmp(pub1.datecreated, pub2.datecreated)
199
 
        else:
200
 
            return version_comparison
201
 
 
202
 
    def sortPublications(self, publications):
203
 
        """Sort publications from most to least current versions."""
204
 
        return sorted(publications, cmp=self.compare, reverse=True)
205
 
 
206
 
 
207
 
def find_live_source_versions(sorted_pubs):
208
 
    """Find versions out of Published publications that should stay live.
209
 
 
210
 
    This particular notion of liveness applies to source domination: the
211
 
    latest version stays live, and that's it.
212
 
 
213
 
    :param sorted_pubs: An iterable of `SourcePackagePublishingHistory`
214
 
        sorted by descending package version.
215
 
    :return: A list of live versions.
216
 
    """
217
 
    # Given the required sort order, the latest version is at the head
218
 
    # of the list.
219
 
    return [sorted_pubs[0].sourcepackagerelease.version]
220
 
 
221
 
 
222
 
def get_binary_versions(binary_publications):
223
 
    """List versions for sequence of `BinaryPackagePublishingHistory`.
224
 
 
225
 
    :param binary_publications: An iterable of
226
 
        `BinaryPackagePublishingHistory`.
227
 
    :return: A list of the publications' respective versions.
228
 
    """
229
 
    return [pub.binarypackagerelease.version for pub in binary_publications]
230
 
 
231
 
 
232
 
def find_live_binary_versions_pass_1(sorted_pubs):
233
 
    """Find versions out of Published `publications` that should stay live.
234
 
 
235
 
    This particular notion of liveness applies to first-pass binary
236
 
    domination: the latest version stays live, and so do publications of
237
 
    binary packages for the "all" architecture.
238
 
 
239
 
    :param sorted_pubs: An iterable of `BinaryPackagePublishingHistory`,
240
 
        sorted by descending package version.
241
 
    :return: A list of live versions.
242
 
    """
243
 
    sorted_pubs = list(sorted_pubs)
244
 
    latest = sorted_pubs.pop(0)
245
 
    return get_binary_versions(
246
 
        [latest] + [
247
 
            pub for pub in sorted_pubs if not pub.architecture_specific])
248
 
 
249
 
 
250
 
class ArchSpecificPublicationsCache:
251
 
    """Cache to track which releases have arch-specific publications.
252
 
 
253
 
    This is used for second-pass binary domination:
254
 
    architecture-independent binary publications cannot be superseded as long
255
 
    as any architecture-dependent binary publications built from the same
256
 
    source package release are still active.  Thus such arch-indep
257
 
    publications are reprieved from domination.
258
 
 
259
 
    This class looks up whether publications for a release need that
260
 
    reprieve.  That only needs to be looked up in the database once per
261
 
    (source package release, archive, distroseries, pocket).  Hence this
262
 
    cache.
263
 
    """
264
 
    def __init__(self):
265
 
        self.cache = {}
266
 
 
267
 
    @staticmethod
268
 
    def getKey(bpph):
269
 
        """Extract just the relevant bits of information from a bpph."""
270
 
        return (
271
 
            bpph.binarypackagerelease.build.source_package_release,
272
 
            bpph.archive,
273
 
            bpph.distroseries,
274
 
            bpph.pocket,
275
 
            )
276
 
 
277
 
    def hasArchSpecificPublications(self, bpph):
278
 
        """Does bpph have active, arch-specific publications?
279
 
 
280
 
        If so, the dominator will want to reprieve `bpph`.
281
 
        """
282
 
        assert not bpph.architecture_specific, (
283
 
            "Wrongly dominating arch-specific binary pub in pass 2.")
284
 
 
285
 
        key = self.getKey(bpph)
286
 
        if key not in self.cache:
287
 
            self.cache[key] = self._lookUp(*key)
288
 
        return self.cache[key]
289
 
 
290
 
    @staticmethod
291
 
    def _lookUp(spr, archive, distroseries, pocket):
292
 
        """Look up an answer in the database."""
293
 
        query = spr.getActiveArchSpecificPublications(
294
 
            archive, distroseries, pocket)
295
 
        return not query.is_empty()
296
 
 
297
 
 
298
 
def find_live_binary_versions_pass_2(sorted_pubs, cache):
299
 
    """Find versions out of Published publications that should stay live.
300
 
 
301
 
    This particular notion of liveness applies to second-pass binary
302
 
    domination: the latest version stays live, and architecture-specific
303
 
    publications stay live (i.e, ones that are not for the "all"
304
 
    architecture).
305
 
 
306
 
    More importantly, any publication for binary packages of the "all"
307
 
    architecture stay live if any of the non-"all" binary packages from
308
 
    the same source package release are still active -- even if they are
309
 
    for other architectures.
310
 
 
311
 
    This is the raison d'etre for the two-pass binary domination algorithm:
312
 
    to let us see which architecture-independent binary publications can be
313
 
    superseded without rendering any architecture-specific binaries from the
314
 
    same source package release uninstallable.
315
 
 
316
 
    (Note that here, "active" includes Published publications but also
317
 
    Pending ones.  This is standard nomenclature in Soyuz.  Some of the
318
 
    domination code confuses matters by using the term "active" to mean only
319
 
    Published publications).
320
 
 
321
 
    :param sorted_pubs: An iterable of `BinaryPackagePublishingHistory`,
322
 
        sorted by descending package version.
323
 
    :param cache: An `ArchSpecificPublicationsCache` to reduce the number of
324
 
        times we need to look up whether an spr/archive/distroseries/pocket
325
 
        has active arch-specific publications.
326
 
    :return: A list of live versions.
327
 
    """
328
 
    # Avoid circular imports
329
 
    from lp.soyuz.model.binarypackagebuild import BinaryPackageBuild
330
 
 
331
 
    sorted_pubs = list(sorted_pubs)
332
 
    latest = sorted_pubs.pop(0)
333
 
    is_arch_specific = attrgetter('architecture_specific')
334
 
    arch_specific_pubs = list(ifilter(is_arch_specific, sorted_pubs))
335
 
    arch_indep_pubs = list(ifilterfalse(is_arch_specific, sorted_pubs))
336
 
 
337
 
    bpbs = load_related(
338
 
        BinaryPackageBuild,
339
 
        [pub.binarypackagerelease for pub in arch_indep_pubs], ['buildID'])
340
 
    load_related(SourcePackageRelease, bpbs, ['source_package_release_id'])
341
 
 
342
 
    reprieved_pubs = [
343
 
        pub
344
 
        for pub in arch_indep_pubs
345
 
            if cache.hasArchSpecificPublications(pub)]
346
 
 
347
 
    return get_binary_versions([latest] + arch_specific_pubs + reprieved_pubs)
348
 
 
349
 
 
350
 
def contains_arch_indep(bpphs):
351
 
    """Are any of the publications among `bpphs` architecture-independent?"""
352
 
    return any(not bpph.architecture_specific for bpph in bpphs)
 
87
def _compare_packages_by_version_and_date(get_release, p1, p2):
 
88
    """Compare publications p1 and p2 by their version; using Debian rules.
 
89
 
 
90
    If the publications are for the same package, compare by datecreated
 
91
    instead. This lets newer records win.
 
92
    """
 
93
    if get_release(p1).id == get_release(p2).id:
 
94
        return cmp(p1.datecreated, p2.datecreated)
 
95
 
 
96
    return apt_pkg.VersionCompare(get_release(p1).version,
 
97
                                  get_release(p2).version)
353
98
 
354
99
 
355
100
class Dominator:
356
 
    """Manage the process of marking packages as superseded.
 
101
    """ Manage the process of marking packages as superseded.
357
102
 
358
103
    Packages are marked as superseded when they become obsolete.
359
104
    """
365
110
        new stuff into the distribution but before the publisher
366
111
        creates the file lists for apt-ftparchive.
367
112
        """
368
 
        self.logger = logger
 
113
        self._logger = logger
369
114
        self.archive = archive
370
 
 
371
 
    def dominatePackage(self, sorted_pubs, live_versions, generalization):
372
 
        """Dominate publications for a single package.
373
 
 
374
 
        The latest publication for any version in `live_versions` stays
375
 
        active.  Any older publications (including older publications for
376
 
        live versions with multiple publications) are marked as superseded by
377
 
        the respective oldest live releases that are newer than the superseded
378
 
        ones.
379
 
 
380
 
        Any versions that are newer than anything in `live_versions` are
381
 
        marked as deleted.  This should not be possible in Soyuz-native
382
 
        archives, but it can happen during archive imports when the
383
 
        previous latest version of a package has disappeared from the Sources
384
 
        list we import.
385
 
 
386
 
        :param sorted_pubs: A list of publications for the same package,
387
 
            in the same archive, series, and pocket, all with status
388
 
            `PackagePublishingStatus.PUBLISHED`.  They must be sorted from
389
 
            most current to least current, as would be the result of
390
 
            `generalization.sortPublications`.
391
 
        :param live_versions: Iterable of versions that are still considered
392
 
            "live" for this package.  For any of these, the latest publication
393
 
            among `publications` will remain Published.  Publications for
394
 
            older releases, as well as older publications of live versions,
395
 
            will be marked as Superseded.  Publications of newer versions than
396
 
            are listed in `live_versions` are marked as Deleted.
397
 
        :param generalization: A `GeneralizedPublication` helper representing
398
 
            the kind of publications these are: source or binary.
399
 
        """
400
 
        live_versions = frozenset(live_versions)
401
 
 
402
 
        self.logger.debug(
403
 
            "Package has %d live publication(s).  Live versions: %s",
404
 
            len(sorted_pubs), live_versions)
405
 
 
406
 
        # Verify that the publications are really sorted properly.
407
 
        check_order = OrderingCheck(cmp=generalization.compare, reverse=True)
408
 
 
409
 
        current_dominant = None
410
 
        dominant_version = None
411
 
 
412
 
        for pub in sorted_pubs:
413
 
            check_order.check(pub)
414
 
 
415
 
            version = generalization.getPackageVersion(pub)
416
 
            # There should never be two published releases with the same
417
 
            # version.  So it doesn't matter whether this comparison is
418
 
            # really a string comparison or a version comparison: if the
419
 
            # versions are equal by either measure, they're from the same
420
 
            # release.
421
 
            if version == dominant_version:
422
 
                # This publication is for a live version, but has been
423
 
                # superseded by a newer publication of the same version.
424
 
                # Supersede it.
425
 
                pub.supersede(current_dominant, logger=self.logger)
426
 
                self.logger.debug2(
427
 
                    "Superseding older publication for version %s.", version)
428
 
            elif version in live_versions:
429
 
                # This publication stays active; if any publications
430
 
                # that follow right after this are to be superseded,
431
 
                # this is the release that they are superseded by.
432
 
                current_dominant = pub
433
 
                dominant_version = version
434
 
                self.logger.debug2("Keeping version %s.", version)
435
 
            elif current_dominant is None:
436
 
                # This publication is no longer live, but there is no
437
 
                # newer version to supersede it either.  Therefore it
438
 
                # must be deleted.
439
 
                pub.requestDeletion(None)
440
 
                self.logger.debug2("Deleting version %s.", version)
441
 
            else:
442
 
                # This publication is superseded.  This is what we're
443
 
                # here to do.
444
 
                pub.supersede(current_dominant, logger=self.logger)
445
 
                self.logger.debug2("Superseding version %s.", version)
446
 
 
447
 
    def _sortPackages(self, publications, generalization):
448
 
        """Partition publications by package name, and sort them.
449
 
 
450
 
        The publications are sorted from most current to least current,
451
 
        as required by `dominatePackage` etc.
452
 
 
453
 
        :param publications: An iterable of `SourcePackagePublishingHistory`
454
 
            or of `BinaryPackagePublishingHistory`.
455
 
        :param generalization: A `GeneralizedPublication` helper representing
456
 
            the kind of publications these are: source or binary.
457
 
        :return: A dict mapping each package name to a sorted list of
458
 
            publications from `publications`.
459
 
        """
460
 
        pubs_by_package = defaultdict(list)
461
 
        for pub in publications:
462
 
            pubs_by_package[generalization.getPackageName(pub)].append(pub)
463
 
 
464
 
        # Sort the publication lists.  This is not an in-place sort, so
465
 
        # it involves altering the dict while we iterate it.  Listify
466
 
        # the keys so that we can be sure that we're not altering the
467
 
        # iteration order while iteration is underway.
468
 
        for package in list(pubs_by_package.keys()):
469
 
            pubs_by_package[package] = generalization.sortPublications(
470
 
                pubs_by_package[package])
471
 
 
472
 
        return pubs_by_package
 
115
        self.debug = self._logger.debug
 
116
 
 
117
    def _dominatePublications(self, pubs):
 
118
        """Perform dominations for the given publications.
 
119
 
 
120
        :param pubs: A dict mapping names to a list of publications. Every
 
121
            publication must be PUBLISHED or PENDING, and the first in each
 
122
            list will be treated as dominant (so should be the latest).
 
123
        """
 
124
        self.debug("Dominating packages...")
 
125
 
 
126
        for name in pubs.keys():
 
127
            assert pubs[name], (
 
128
                "Empty list of publications for %s" % name)
 
129
            for pubrec in pubs[name][1:]:
 
130
                pubrec.supersede(pubs[name][0], self)
 
131
 
 
132
    def _sortPackages(self, pkglist, is_source=True):
 
133
        # pkglist is a list of packages with the following
 
134
        #  * sourcepackagename or packagename as appropriate
 
135
        #  * version
 
136
        #  * status
 
137
        # Don't care about any other attributes
 
138
        outpkgs = {}
 
139
 
 
140
        self.debug("Sorting packages...")
 
141
 
 
142
        attr_prefix = 'source' if is_source else 'binary'
 
143
        get_release = operator.attrgetter(attr_prefix + 'packagerelease')
 
144
        get_name = operator.attrgetter(attr_prefix + 'packagename')
 
145
 
 
146
        for inpkg in pkglist:
 
147
            L = outpkgs.setdefault(
 
148
                get_name(get_release(inpkg)).name.encode('utf-8'), [])
 
149
            L.append(inpkg)
 
150
 
 
151
        for pkgname in outpkgs:
 
152
            if len(outpkgs[pkgname]) > 1:
 
153
                outpkgs[pkgname].sort(
 
154
                    functools.partial(
 
155
                        _compare_packages_by_version_and_date, get_release))
 
156
                outpkgs[pkgname].reverse()
 
157
 
 
158
        return outpkgs
473
159
 
474
160
    def _setScheduledDeletionDate(self, pub_record):
475
161
        """Set the scheduleddeletiondate on a publishing record.
502
188
        # Avoid circular imports.
503
189
        from lp.soyuz.model.publishing import (
504
190
            BinaryPackagePublishingHistory,
505
 
            SourcePackagePublishingHistory,
506
 
            )
507
 
 
508
 
        self.logger.debug("Beginning superseded processing...")
509
 
 
 
191
            SourcePackagePublishingHistory)
 
192
 
 
193
        self.debug("Beginning superseded processing...")
 
194
 
 
195
        # XXX: dsilvers 2005-09-22 bug=55030:
 
196
        # Need to make binaries go in groups but for now this'll do.
 
197
        # An example of the concrete problem here is:
 
198
        # - Upload foo-1.0, which builds foo and foo-common (arch all).
 
199
        # - Upload foo-1.1, ditto.
 
200
        # - foo-common-1.1 is built (along with the i386 binary for foo)
 
201
        # - foo-common-1.0 is superseded
 
202
        # Foo is now uninstallable on any architectures which don't yet
 
203
        # have a build of foo-1.1, as the foo-common for foo-1.0 is gone.
 
204
 
 
205
        # Essentially we ideally don't want to lose superseded binaries
 
206
        # unless the entire group is ready to be made pending removal.
 
207
        # In this instance a group is defined as all the binaries from a
 
208
        # given build. This assumes we've copied the arch_all binaries
 
209
        # from whichever build provided them into each arch-specific build
 
210
        # which we publish. If instead we simply publish the arch-all
 
211
        # binaries from another build then instead we should scan up from
 
212
        # the binary to its source, and then back from the source to each
 
213
        # binary published in *this* distroarchseries for that source.
 
214
        # if the binaries as a group (in that definition) are all superseded
 
215
        # then we can consider them eligible for removal.
510
216
        for pub_record in binary_records:
511
217
            binpkg_release = pub_record.binarypackagerelease
512
 
            self.logger.debug(
513
 
                "%s/%s (%s) has been judged eligible for removal",
514
 
                binpkg_release.binarypackagename.name, binpkg_release.version,
515
 
                pub_record.distroarchseries.architecturetag)
 
218
            self.debug("%s/%s (%s) has been judged eligible for removal" %
 
219
                       (binpkg_release.binarypackagename.name,
 
220
                        binpkg_release.version,
 
221
                        pub_record.distroarchseries.architecturetag))
516
222
            self._setScheduledDeletionDate(pub_record)
517
223
            # XXX cprov 20070820: 'datemadepending' is useless, since it's
518
224
            # always equals to "scheduleddeletiondate - quarantine".
556
262
                    continue
557
263
 
558
264
            # Okay, so there's no unremoved binaries, let's go for it...
559
 
            self.logger.debug(
560
 
                "%s/%s (%s) source has been judged eligible for removal",
561
 
                srcpkg_release.sourcepackagename.name, srcpkg_release.version,
562
 
                pub_record.id)
 
265
            self.debug(
 
266
                "%s/%s (%s) source has been judged eligible for removal" %
 
267
                (srcpkg_release.sourcepackagename.name,
 
268
                 srcpkg_release.version, pub_record.id))
563
269
            self._setScheduledDeletionDate(pub_record)
564
270
            # XXX cprov 20070820: 'datemadepending' is pointless, since it's
565
271
            # always equals to "scheduleddeletiondate - quarantine".
566
272
            pub_record.datemadepending = UTC_NOW
567
273
 
568
 
    def findBinariesForDomination(self, distroarchseries, pocket):
569
 
        """Find binary publications that need dominating.
570
 
 
571
 
        This is only for traditional domination, where the latest published
572
 
        publication is always kept published.  It will ignore publications
573
 
        that have no other publications competing for the same binary package.
574
 
        """
575
 
        # Avoid circular imports.
576
 
        from lp.soyuz.model.publishing import BinaryPackagePublishingHistory
577
 
 
578
 
        BPPH = BinaryPackagePublishingHistory
579
 
        BPR = BinaryPackageRelease
580
 
 
581
 
        bpph_location_clauses = [
582
 
            BPPH.status == PackagePublishingStatus.PUBLISHED,
583
 
            BPPH.distroarchseries == distroarchseries,
584
 
            BPPH.archive == self.archive,
585
 
            BPPH.pocket == pocket,
586
 
            ]
587
 
        candidate_binary_names = Select(
588
 
            BPPH.binarypackagenameID, And(*bpph_location_clauses),
589
 
            group_by=BPPH.binarypackagenameID, having=(Count() > 1))
590
 
        main_clauses = bpph_location_clauses + [
591
 
            BPR.id == BPPH.binarypackagereleaseID,
592
 
            BPR.binarypackagenameID.is_in(candidate_binary_names),
593
 
            BPR.binpackageformat != BinaryPackageFormat.DDEB,
594
 
            ]
595
 
 
596
 
        # We're going to access the BPRs as well.  Since we make the
597
 
        # database look them up anyway, and since there won't be many
598
 
        # duplications among them, load them alongside the publications.
599
 
        # We'll also want their BinaryPackageNames, but adding those to
600
 
        # the join would complicate the query.
601
 
        query = IStore(BPPH).find((BPPH, BPR), *main_clauses)
602
 
        bpphs = list(DecoratedResultSet(query, itemgetter(0)))
603
 
        load_related(BinaryPackageName, bpphs, ['binarypackagenameID'])
604
 
        return bpphs
605
 
 
606
 
    def dominateBinaries(self, distroseries, pocket):
607
 
        """Perform domination on binary package publications.
608
 
 
609
 
        Dominates binaries, restricted to `distroseries`, `pocket`, and
610
 
        `self.archive`.
611
 
        """
612
 
        generalization = GeneralizedPublication(is_source=False)
613
 
 
614
 
        # Domination happens in two passes.  The first tries to
615
 
        # supersede architecture-dependent publications; the second
616
 
        # tries to supersede architecture-independent ones.  An
617
 
        # architecture-independent pub is kept alive as long as any
618
 
        # architecture-dependent pubs from the same source package build
619
 
        # are still live for any architecture, because they may depend
620
 
        # on the architecture-independent package.
621
 
        # Thus we limit the second pass to those packages that have
622
 
        # published, architecture-independent publications; anything
623
 
        # else will have completed domination in the first pass.
624
 
        packages_w_arch_indep = set()
625
 
 
626
 
        for distroarchseries in distroseries.architectures:
627
 
            self.logger.info(
628
 
                "Performing domination across %s/%s (%s)",
629
 
                distroarchseries.distroseries.name, pocket.title,
630
 
                distroarchseries.architecturetag)
631
 
 
632
 
            self.logger.info("Finding binaries...")
633
 
            bins = self.findBinariesForDomination(distroarchseries, pocket)
634
 
            sorted_packages = self._sortPackages(bins, generalization)
635
 
            self.logger.info("Dominating binaries...")
636
 
            for name, pubs in sorted_packages.iteritems():
637
 
                self.logger.debug("Dominating %s" % name)
638
 
                assert len(pubs) > 0, "Dominating zero binaries!"
639
 
                live_versions = find_live_binary_versions_pass_1(pubs)
640
 
                self.dominatePackage(pubs, live_versions, generalization)
641
 
                if contains_arch_indep(pubs):
642
 
                    packages_w_arch_indep.add(name)
643
 
 
644
 
        packages_w_arch_indep = frozenset(packages_w_arch_indep)
645
 
 
646
 
        # The second pass attempts to supersede arch-all publications of
647
 
        # older versions, from source package releases that no longer
648
 
        # have any active arch-specific publications that might depend
649
 
        # on the arch-indep ones.
650
 
        # (In maintaining this code, bear in mind that some or all of a
651
 
        # source package's binary packages may switch between
652
 
        # arch-specific and arch-indep between releases.)
653
 
        reprieve_cache = ArchSpecificPublicationsCache()
654
 
        for distroarchseries in distroseries.architectures:
655
 
            self.logger.info("Finding binaries...(2nd pass)")
656
 
            bins = self.findBinariesForDomination(distroarchseries, pocket)
657
 
            sorted_packages = self._sortPackages(bins, generalization)
658
 
            self.logger.info("Dominating binaries...(2nd pass)")
659
 
            for name in packages_w_arch_indep.intersection(sorted_packages):
660
 
                pubs = sorted_packages[name]
661
 
                self.logger.debug("Dominating %s" % name)
662
 
                assert len(pubs) > 0, "Dominating zero binaries in 2nd pass!"
663
 
                live_versions = find_live_binary_versions_pass_2(
664
 
                    pubs, reprieve_cache)
665
 
                self.dominatePackage(pubs, live_versions, generalization)
666
 
 
667
 
    def _composeActiveSourcePubsCondition(self, distroseries, pocket):
668
 
        """Compose ORM condition for restricting relevant source pubs."""
669
 
        # Avoid circular imports.
670
 
        from lp.soyuz.model.publishing import SourcePackagePublishingHistory
671
 
 
672
 
        SPPH = SourcePackagePublishingHistory
673
 
 
674
 
        return And(
675
 
            SPPH.status == PackagePublishingStatus.PUBLISHED,
676
 
            SPPH.distroseries == distroseries,
677
 
            SPPH.archive == self.archive,
678
 
            SPPH.pocket == pocket,
 
274
    def judgeAndDominate(self, dr, pocket):
 
275
        """Perform the domination and superseding calculations
 
276
 
 
277
        It only works across the distroseries and pocket specified.
 
278
        """
 
279
        # Avoid circular imports.
 
280
        from lp.soyuz.model.publishing import (
 
281
             BinaryPackagePublishingHistory,
 
282
             SourcePackagePublishingHistory)
 
283
 
 
284
        for distroarchseries in dr.architectures:
 
285
            self.debug("Performing domination across %s/%s (%s)" % (
 
286
                dr.name, pocket.title, distroarchseries.architecturetag))
 
287
 
 
288
            bpph_location_clauses = And(
 
289
                BinaryPackagePublishingHistory.status ==
 
290
                    PackagePublishingStatus.PUBLISHED,
 
291
                BinaryPackagePublishingHistory.distroarchseries ==
 
292
                    distroarchseries,
 
293
                BinaryPackagePublishingHistory.archive == self.archive,
 
294
                BinaryPackagePublishingHistory.pocket == pocket,
 
295
                )
 
296
            candidate_binary_names = Select(
 
297
                BinaryPackageName.id,
 
298
                And(
 
299
                    BinaryPackageRelease.binarypackagenameID ==
 
300
                        BinaryPackageName.id,
 
301
                    BinaryPackagePublishingHistory.binarypackagereleaseID ==
 
302
                        BinaryPackageRelease.id,
 
303
                    bpph_location_clauses,
 
304
                ),
 
305
                group_by=BinaryPackageName.id,
 
306
                having=Count(BinaryPackagePublishingHistory.id) > 1)
 
307
            binaries = IMasterStore(BinaryPackagePublishingHistory).find(
 
308
                BinaryPackagePublishingHistory,
 
309
                BinaryPackageRelease.id ==
 
310
                    BinaryPackagePublishingHistory.binarypackagereleaseID,
 
311
                BinaryPackageRelease.binarypackagenameID.is_in(
 
312
                    candidate_binary_names),
 
313
                BinaryPackageRelease.binpackageformat !=
 
314
                    BinaryPackageFormat.DDEB,
 
315
                bpph_location_clauses)
 
316
            self.debug("Dominating binaries...")
 
317
            self._dominatePublications(self._sortPackages(binaries, False))
 
318
 
 
319
        self.debug("Performing domination across %s/%s (Source)" %
 
320
                   (dr.name, pocket.title))
 
321
        spph_location_clauses = And(
 
322
            SourcePackagePublishingHistory.status ==
 
323
                PackagePublishingStatus.PUBLISHED,
 
324
            SourcePackagePublishingHistory.distroseries == dr,
 
325
            SourcePackagePublishingHistory.archive == self.archive,
 
326
            SourcePackagePublishingHistory.pocket == pocket,
679
327
            )
680
 
 
681
 
    def findSourcesForDomination(self, distroseries, pocket):
682
 
        """Find binary publications that need dominating.
683
 
 
684
 
        This is only for traditional domination, where the latest published
685
 
        publication is always kept published.  See `find_live_source_versions`
686
 
        for this logic.
687
 
 
688
 
        To optimize for that logic, `findSourcesForDomination` will ignore
689
 
        publications that have no other publications competing for the same
690
 
        binary package.  There'd be nothing to do for those cases.
691
 
        """
692
 
        # Avoid circular imports.
693
 
        from lp.soyuz.model.publishing import SourcePackagePublishingHistory
694
 
 
695
 
        SPPH = SourcePackagePublishingHistory
696
 
        SPR = SourcePackageRelease
697
 
 
698
 
        spph_location_clauses = self._composeActiveSourcePubsCondition(
699
 
            distroseries, pocket)
700
328
        candidate_source_names = Select(
701
 
            SPPH.sourcepackagenameID,
702
 
            And(join_spph_spr(), spph_location_clauses),
703
 
            group_by=SPPH.sourcepackagenameID,
704
 
            having=(Count() > 1))
705
 
 
706
 
        # We'll also access the SourcePackageReleases associated with
707
 
        # the publications we find.  Since they're in the join anyway,
708
 
        # load them alongside the publications.
709
 
        # Actually we'll also want the SourcePackageNames, but adding
710
 
        # those to the (outer) query would complicate it, and
711
 
        # potentially slow it down.
712
 
        query = IStore(SPPH).find(
713
 
            (SPPH, SPR),
714
 
            join_spph_spr(),
715
 
            SPPH.sourcepackagenameID.is_in(candidate_source_names),
 
329
            SourcePackageName.id,
 
330
            And(
 
331
                SourcePackageRelease.sourcepackagenameID ==
 
332
                    SourcePackageName.id,
 
333
                SourcePackagePublishingHistory.sourcepackagereleaseID ==
 
334
                    SourcePackageRelease.id,
 
335
                spph_location_clauses,
 
336
            ),
 
337
            group_by=SourcePackageName.id,
 
338
            having=Count(SourcePackagePublishingHistory.id) > 1)
 
339
        sources = IMasterStore(SourcePackagePublishingHistory).find(
 
340
            SourcePackagePublishingHistory,
 
341
            SourcePackageRelease.id ==
 
342
                SourcePackagePublishingHistory.sourcepackagereleaseID,
 
343
            SourcePackageRelease.sourcepackagenameID.is_in(
 
344
                candidate_source_names),
716
345
            spph_location_clauses)
717
 
        spphs = DecoratedResultSet(query, itemgetter(0))
718
 
        load_related(SourcePackageName, spphs, ['sourcepackagenameID'])
719
 
        return spphs
720
 
 
721
 
    def dominateSources(self, distroseries, pocket):
722
 
        """Perform domination on source package publications.
723
 
 
724
 
        Dominates sources, restricted to `distroseries`, `pocket`, and
725
 
        `self.archive`.
726
 
        """
727
 
        self.logger.debug(
728
 
            "Performing domination across %s/%s (Source)",
729
 
            distroseries.name, pocket.title)
730
 
 
731
 
        generalization = GeneralizedPublication(is_source=True)
732
 
 
733
 
        self.logger.debug("Finding sources...")
734
 
        sources = self.findSourcesForDomination(distroseries, pocket)
735
 
        sorted_packages = self._sortPackages(sources, generalization)
736
 
 
737
 
        self.logger.debug("Dominating sources...")
738
 
        for name, pubs in sorted_packages.iteritems():
739
 
            self.logger.debug("Dominating %s" % name)
740
 
            assert len(pubs) > 0, "Dominating zero sources!"
741
 
            live_versions = find_live_source_versions(pubs)
742
 
            self.dominatePackage(pubs, live_versions, generalization)
743
 
 
 
346
        self.debug("Dominating sources...")
 
347
        self._dominatePublications(self._sortPackages(sources))
744
348
        flush_database_updates()
745
349
 
746
 
    def findPublishedSourcePackageNames(self, distroseries, pocket):
747
 
        """Find currently published source packages.
748
 
 
749
 
        Returns an iterable of tuples: (name of source package, number of
750
 
        publications in Published state).
751
 
        """
752
 
        # Avoid circular imports.
753
 
        from lp.soyuz.model.publishing import SourcePackagePublishingHistory
754
 
 
755
 
        looking_for = (
756
 
            SourcePackageName.name,
757
 
            Count(SourcePackagePublishingHistory.id),
758
 
            )
759
 
        result = IStore(SourcePackageName).find(
760
 
            looking_for,
761
 
            join_spph_spr(),
762
 
            join_spph_spn(),
763
 
            self._composeActiveSourcePubsCondition(distroseries, pocket))
764
 
        return result.group_by(SourcePackageName.name)
765
 
 
766
 
    def findPublishedSPPHs(self, distroseries, pocket, package_name):
767
 
        """Find currently published source publications for given package."""
768
 
        # Avoid circular imports.
769
 
        from lp.soyuz.model.publishing import SourcePackagePublishingHistory
770
 
 
771
 
        SPPH = SourcePackagePublishingHistory
772
 
        SPR = SourcePackageRelease
773
 
 
774
 
        query = IStore(SourcePackagePublishingHistory).find(
775
 
            SPPH,
776
 
            join_spph_spr(),
777
 
            join_spph_spn(),
778
 
            SourcePackageName.name == package_name,
779
 
            self._composeActiveSourcePubsCondition(distroseries, pocket))
780
 
        # Sort by descending version (SPR.version has type debversion in
781
 
        # the database, so this should be a real proper comparison) so
782
 
        # that _sortPackage will have slightly less work to do later.
783
 
        return query.order_by(Desc(SPR.version), Desc(SPPH.datecreated))
784
 
 
785
 
    def dominateSourceVersions(self, distroseries, pocket, package_name,
786
 
                               live_versions):
787
 
        """Dominate source publications based on a set of "live" versions.
788
 
 
789
 
        Active publications for the "live" versions will remain active.  All
790
 
        other active publications for the same package (and the same archive,
791
 
        distroseries, and pocket) are marked superseded.
792
 
 
793
 
        Unlike traditional domination, this allows multiple versions of a
794
 
        package to stay active in the same distroseries, archive, and pocket.
795
 
 
796
 
        :param distroseries: `DistroSeries` to dominate.
797
 
        :param pocket: `PackagePublishingPocket` to dominate.
798
 
        :param package_name: Source package name, as text.
799
 
        :param live_versions: Iterable of all version strings that are to
800
 
            remain active.
801
 
        """
802
 
        generalization = GeneralizedPublication(is_source=True)
803
 
        pubs = self.findPublishedSPPHs(distroseries, pocket, package_name)
804
 
        pubs = generalization.sortPublications(pubs)
805
 
        self.dominatePackage(pubs, live_versions, generalization)
806
 
 
807
 
    def judge(self, distroseries, pocket):
808
 
        """Judge superseded sources and binaries."""
809
 
        # Avoid circular imports.
810
 
        from lp.soyuz.model.publishing import (
811
 
             BinaryPackagePublishingHistory,
812
 
             SourcePackagePublishingHistory,
813
 
             )
814
 
 
815
350
        sources = SourcePackagePublishingHistory.select("""
816
351
            sourcepackagepublishinghistory.distroseries = %s AND
817
352
            sourcepackagepublishinghistory.archive = %s AND
818
353
            sourcepackagepublishinghistory.pocket = %s AND
819
354
            sourcepackagepublishinghistory.status IN %s AND
820
355
            sourcepackagepublishinghistory.scheduleddeletiondate is NULL
821
 
            """ % sqlvalues(
822
 
                distroseries, self.archive, pocket,
823
 
                inactive_publishing_status))
 
356
            """ % sqlvalues(dr, self.archive, pocket,
 
357
                            ELIGIBLE_DOMINATION_STATES))
824
358
 
825
359
        binaries = BinaryPackagePublishingHistory.select("""
826
360
            binarypackagepublishinghistory.distroarchseries =
830
364
            binarypackagepublishinghistory.pocket = %s AND
831
365
            binarypackagepublishinghistory.status IN %s AND
832
366
            binarypackagepublishinghistory.scheduleddeletiondate is NULL
833
 
            """ % sqlvalues(
834
 
                distroseries, self.archive, pocket,
835
 
                inactive_publishing_status),
 
367
            """ % sqlvalues(dr, self.archive, pocket,
 
368
                            ELIGIBLE_DOMINATION_STATES),
836
369
            clauseTables=['DistroArchSeries'])
837
370
 
838
371
        self._judgeSuperseded(sources, binaries)
839
372
 
840
 
    def judgeAndDominate(self, distroseries, pocket):
841
 
        """Perform the domination and superseding calculations
842
 
 
843
 
        It only works across the distroseries and pocket specified.
844
 
        """
845
 
 
846
 
        self.dominateBinaries(distroseries, pocket)
847
 
        self.dominateSources(distroseries, pocket)
848
 
        self.judge(distroseries, pocket)
849
 
 
850
 
        self.logger.debug(
851
 
            "Domination for %s/%s finished", distroseries.name, pocket.title)
 
373
        self.debug("Domination for %s/%s finished" %
 
374
                   (dr.name, pocket.title))