~launchpad-pqm/launchpad/devel

10637.3.1 by Guilherme Salgado
Use the default python version instead of a hard-coded version
1
#!/usr/bin/python -S
8687.15.7 by Karl Fogel
Add the copyright header block to more files.
2
#
3
# Copyright 2009 Canonical Ltd.  This software is licensed under the
4
# GNU Affero General Public License version 3 (see the file LICENSE).
2705 by Canonical.com Patch Queue Manager
r=spiv, mark's soyuz loving.
5
4935.3.7 by Curtis Hovey
Added bad name suppression to cronscripts.
6
# pylint: disable-msg=C0103,W0403
2705 by Canonical.com Patch Queue Manager
r=spiv, mark's soyuz loving.
7
8
# This script updates the cached source package information in the system.
9
# We use this for fast source package searching (as opposed to joining
10
# through gazillions of publishing tables).
11
12
import _pythonpath
13
14
from zope.component import getUtility
15
11882.2.2 by Jonathan Lange
Clear up a heck of a lot of imports from canonical.launchpad.interfaces.
16
from lp.registry.interfaces.distribution import IDistributionSet
8356.1.1 by Leonard Richardson
Partial move.
17
from lp.services.scripts.base import LaunchpadCronScript
13757.2.7 by William Grant
Update tests.:
18
from lp.soyuz.model.distributionsourcepackagecache import (
19
    DistributionSourcePackageCache)
13757.2.2 by William Grant
DistroSeries.{getBinaryPackageCaches,removeOldCacheItems} are now on DistroSeriesPackageCache.
20
from lp.soyuz.model.distroseriespackagecache import DistroSeriesPackageCache
3691.348.18 by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky...
21
22
4264.2.1 by James Henstridge
add a LaunchpadCronScript subclass, and make cronscripts/*.py use it
23
class PackageCacheUpdater(LaunchpadCronScript):
5781.5.4 by Celso Providelo
Implementing concentrated PPA caches and improving tests.
24
    """Helper class for updating package caches.
25
26
    It iterates over all distributions, distroseries and archives (including
27
    PPAs) updating the package caches to reflect what is currently published
28
    in those locations.
29
    """
30
31
    def updateDistributionPackageCounters(self, distribution):
32
        """Update package counters for a given distribution."""
33
        for distroseries in distribution:
34
            distroseries.updatePackageCount()
35
            self.txn.commit()
36
            for arch in distroseries.architectures:
37
                arch.updatePackageCount()
38
                self.txn.commit()
39
40
    def updateDistributionCache(self, distribution, archive):
41
        """Update package caches for the given location.
42
43
        'archive' can be one of the main archives (PRIMARY, PARTNER or
44
        EMBARGOED) or even a PPA.
45
46
        This method commits the transaction frequently since it deal with
47
        a huge amount of data.
48
49
        PPA archives caches are consolidated in a Archive row to optimize
50
        searches across PPAs.
51
        """
9760.8.1 by Brad Crittenden
Change the non-English 'serieses' to 'series' throughout our codebase.
52
        for distroseries in distribution.series:
5781.5.4 by Celso Providelo
Implementing concentrated PPA caches and improving tests.
53
            self.updateDistroSeriesCache(distroseries, archive)
54
13757.2.7 by William Grant
Update tests.:
55
        DistributionSourcePackageCache.removeOld(
56
            distribution, archive, log=self.logger)
5781.5.4 by Celso Providelo
Implementing concentrated PPA caches and improving tests.
57
13757.2.7 by William Grant
Update tests.:
58
        updates = DistributionSourcePackageCache.updateAll(
59
            distribution, archive=archive, ztm=self.txn, log=self.logger)
5781.5.4 by Celso Providelo
Implementing concentrated PPA caches and improving tests.
60
6096.3.1 by Celso Providelo
Reducing the commit-rate on update-pkgcache by increasing the commit_chunk size in the content classes and returning the number of updates done. It also sanitize the commits done in the script itself.
61
        if updates > 0:
5781.5.4 by Celso Providelo
Implementing concentrated PPA caches and improving tests.
62
            self.txn.commit()
63
64
    def updateDistroSeriesCache(self, distroseries, archive):
65
        """Update package caches for the given location."""
66
        self.logger.info('%s %s %s starting' % (
7976.2.1 by Celso Providelo
Renaming IArchive.title to IArchive.displayname in preparation to have user-defined titles, bug #340457.
67
            distroseries.distribution.name, distroseries.name,
68
            archive.displayname))
5781.5.4 by Celso Providelo
Implementing concentrated PPA caches and improving tests.
69
13757.2.2 by William Grant
DistroSeries.{getBinaryPackageCaches,removeOldCacheItems} are now on DistroSeriesPackageCache.
70
        DistroSeriesPackageCache.removeOld(
71
            distroseries, archive=archive, log=self.logger)
5781.5.4 by Celso Providelo
Implementing concentrated PPA caches and improving tests.
72
13757.2.3 by William Grant
updateCompletePackageCache and updatePackageCache come along for the ride.
73
        updates = DistroSeriesPackageCache.updateAll(
74
            distroseries, archive=archive, ztm=self.txn, log=self.logger)
6096.3.1 by Celso Providelo
Reducing the commit-rate on update-pkgcache by increasing the commit_chunk size in the content classes and returning the number of updates done. It also sanitize the commits done in the script itself.
75
76
        if updates > 0:
77
            self.txn.commit()
3691.348.18 by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky...
78
79
    def main(self):
5781.5.4 by Celso Providelo
Implementing concentrated PPA caches and improving tests.
80
        self.logger.debug('Starting the package cache update')
81
6096.3.1 by Celso Providelo
Reducing the commit-rate on update-pkgcache by increasing the commit_chunk size in the content classes and returning the number of updates done. It also sanitize the commits done in the script itself.
82
        # Do the package counter and cache update for each distribution.
2705 by Canonical.com Patch Queue Manager
r=spiv, mark's soyuz loving.
83
        distroset = getUtility(IDistributionSet)
5781.5.4 by Celso Providelo
Implementing concentrated PPA caches and improving tests.
84
        for distribution in distroset:
85
            self.logger.info(
86
                'Updating %s package counters' % distribution.name)
87
            self.updateDistributionPackageCounters(distribution)
88
89
            self.logger.info(
90
                'Updating %s main archives' % distribution.name)
91
            for archive in distribution.all_distro_archives:
92
                self.updateDistributionCache(distribution, archive)
93
94
            self.logger.info(
95
                'Updating %s PPAs' % distribution.name)
96
            for archive in distribution.getAllPPAs():
97
                self.updateDistributionCache(distribution, archive)
6096.3.1 by Celso Providelo
Reducing the commit-rate on update-pkgcache by increasing the commit_chunk size in the content classes and returning the number of updates done. It also sanitize the commits done in the script itself.
98
                archive.updateArchiveCache()
5781.5.4 by Celso Providelo
Implementing concentrated PPA caches and improving tests.
99
6096.3.1 by Celso Providelo
Reducing the commit-rate on update-pkgcache by increasing the commit_chunk size in the content classes and returning the number of updates done. It also sanitize the commits done in the script itself.
100
            # Commit any remaining update for a distribution.
101
            self.txn.commit()
5781.5.4 by Celso Providelo
Implementing concentrated PPA caches and improving tests.
102
            self.logger.info('%s done' % distribution.name)
103
104
        self.logger.debug('Finished the package cache update')
2705 by Canonical.com Patch Queue Manager
r=spiv, mark's soyuz loving.
105
106
if __name__ == '__main__':
5781.5.4 by Celso Providelo
Implementing concentrated PPA caches and improving tests.
107
    script = PackageCacheUpdater(
10449.1.1 by Julian Edwards
update-pkg-cache gets its own db user
108
        'update-cache', dbuser="update-pkg-cache")
3691.348.18 by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky...
109
    script.lock_and_run()