|
10637.3.1
by Guilherme Salgado
Use the default python version instead of a hard-coded version |
1 |
#!/usr/bin/python -S
|
|
8687.15.7
by Karl Fogel
Add the copyright header block to more files. |
2 |
#
|
3 |
# Copyright 2009 Canonical Ltd. This software is licensed under the
|
|
4 |
# GNU Affero General Public License version 3 (see the file LICENSE).
|
|
|
2705
by Canonical.com Patch Queue Manager
r=spiv, mark's soyuz loving. |
5 |
|
|
4935.3.7
by Curtis Hovey
Added bad name suppression to cronscripts. |
6 |
# pylint: disable-msg=C0103,W0403
|
|
2705
by Canonical.com Patch Queue Manager
r=spiv, mark's soyuz loving. |
7 |
|
8 |
# This script updates the cached source package information in the system.
|
|
9 |
# We use this for fast source package searching (as opposed to joining
|
|
10 |
# through gazillions of publishing tables).
|
|
11 |
||
12 |
import _pythonpath |
|
13 |
||
14 |
from zope.component import getUtility |
|
15 |
||
|
11882.2.2
by Jonathan Lange
Clear up a heck of a lot of imports from canonical.launchpad.interfaces. |
16 |
from lp.registry.interfaces.distribution import IDistributionSet |
|
8356.1.1
by Leonard Richardson
Partial move. |
17 |
from lp.services.scripts.base import LaunchpadCronScript |
|
13757.2.7
by William Grant
Update tests.: |
18 |
from lp.soyuz.model.distributionsourcepackagecache import ( |
|
14612.2.8
by William Grant
cronscripts |
19 |
DistributionSourcePackageCache, |
20 |
)
|
|
|
13757.2.2
by William Grant
DistroSeries.{getBinaryPackageCaches,removeOldCacheItems} are now on DistroSeriesPackageCache. |
21 |
from lp.soyuz.model.distroseriespackagecache import DistroSeriesPackageCache |
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
22 |
|
23 |
||
|
4264.2.1
by James Henstridge
add a LaunchpadCronScript subclass, and make cronscripts/*.py use it |
24 |
class PackageCacheUpdater(LaunchpadCronScript): |
|
5781.5.4
by Celso Providelo
Implementing concentrated PPA caches and improving tests. |
25 |
"""Helper class for updating package caches.
|
26 |
||
27 |
It iterates over all distributions, distroseries and archives (including
|
|
28 |
PPAs) updating the package caches to reflect what is currently published
|
|
29 |
in those locations.
|
|
30 |
"""
|
|
31 |
||
32 |
def updateDistributionPackageCounters(self, distribution): |
|
33 |
"""Update package counters for a given distribution."""
|
|
34 |
for distroseries in distribution: |
|
35 |
distroseries.updatePackageCount() |
|
36 |
self.txn.commit() |
|
37 |
for arch in distroseries.architectures: |
|
38 |
arch.updatePackageCount() |
|
39 |
self.txn.commit() |
|
40 |
||
41 |
def updateDistributionCache(self, distribution, archive): |
|
42 |
"""Update package caches for the given location.
|
|
43 |
||
44 |
'archive' can be one of the main archives (PRIMARY, PARTNER or
|
|
45 |
EMBARGOED) or even a PPA.
|
|
46 |
||
47 |
This method commits the transaction frequently since it deal with
|
|
48 |
a huge amount of data.
|
|
49 |
||
50 |
PPA archives caches are consolidated in a Archive row to optimize
|
|
51 |
searches across PPAs.
|
|
52 |
"""
|
|
|
9760.8.1
by Brad Crittenden
Change the non-English 'serieses' to 'series' throughout our codebase. |
53 |
for distroseries in distribution.series: |
|
5781.5.4
by Celso Providelo
Implementing concentrated PPA caches and improving tests. |
54 |
self.updateDistroSeriesCache(distroseries, archive) |
55 |
||
|
13757.2.7
by William Grant
Update tests.: |
56 |
DistributionSourcePackageCache.removeOld( |
57 |
distribution, archive, log=self.logger) |
|
|
5781.5.4
by Celso Providelo
Implementing concentrated PPA caches and improving tests. |
58 |
|
|
13757.2.7
by William Grant
Update tests.: |
59 |
updates = DistributionSourcePackageCache.updateAll( |
60 |
distribution, archive=archive, ztm=self.txn, log=self.logger) |
|
|
5781.5.4
by Celso Providelo
Implementing concentrated PPA caches and improving tests. |
61 |
|
|
6096.3.1
by Celso Providelo
Reducing the commit-rate on update-pkgcache by increasing the commit_chunk size in the content classes and returning the number of updates done. It also sanitize the commits done in the script itself. |
62 |
if updates > 0: |
|
5781.5.4
by Celso Providelo
Implementing concentrated PPA caches and improving tests. |
63 |
self.txn.commit() |
64 |
||
65 |
def updateDistroSeriesCache(self, distroseries, archive): |
|
66 |
"""Update package caches for the given location."""
|
|
67 |
self.logger.info('%s %s %s starting' % ( |
|
|
7976.2.1
by Celso Providelo
Renaming IArchive.title to IArchive.displayname in preparation to have user-defined titles, bug #340457. |
68 |
distroseries.distribution.name, distroseries.name, |
69 |
archive.displayname)) |
|
|
5781.5.4
by Celso Providelo
Implementing concentrated PPA caches and improving tests. |
70 |
|
|
13757.2.2
by William Grant
DistroSeries.{getBinaryPackageCaches,removeOldCacheItems} are now on DistroSeriesPackageCache. |
71 |
DistroSeriesPackageCache.removeOld( |
72 |
distroseries, archive=archive, log=self.logger) |
|
|
5781.5.4
by Celso Providelo
Implementing concentrated PPA caches and improving tests. |
73 |
|
|
13757.2.3
by William Grant
updateCompletePackageCache and updatePackageCache come along for the ride. |
74 |
updates = DistroSeriesPackageCache.updateAll( |
75 |
distroseries, archive=archive, ztm=self.txn, log=self.logger) |
|
|
6096.3.1
by Celso Providelo
Reducing the commit-rate on update-pkgcache by increasing the commit_chunk size in the content classes and returning the number of updates done. It also sanitize the commits done in the script itself. |
76 |
|
77 |
if updates > 0: |
|
78 |
self.txn.commit() |
|
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
79 |
|
80 |
def main(self): |
|
|
5781.5.4
by Celso Providelo
Implementing concentrated PPA caches and improving tests. |
81 |
self.logger.debug('Starting the package cache update') |
82 |
||
|
6096.3.1
by Celso Providelo
Reducing the commit-rate on update-pkgcache by increasing the commit_chunk size in the content classes and returning the number of updates done. It also sanitize the commits done in the script itself. |
83 |
# Do the package counter and cache update for each distribution.
|
|
2705
by Canonical.com Patch Queue Manager
r=spiv, mark's soyuz loving. |
84 |
distroset = getUtility(IDistributionSet) |
|
5781.5.4
by Celso Providelo
Implementing concentrated PPA caches and improving tests. |
85 |
for distribution in distroset: |
86 |
self.logger.info( |
|
87 |
'Updating %s package counters' % distribution.name) |
|
88 |
self.updateDistributionPackageCounters(distribution) |
|
89 |
||
90 |
self.logger.info( |
|
91 |
'Updating %s main archives' % distribution.name) |
|
92 |
for archive in distribution.all_distro_archives: |
|
93 |
self.updateDistributionCache(distribution, archive) |
|
94 |
||
95 |
self.logger.info( |
|
96 |
'Updating %s PPAs' % distribution.name) |
|
97 |
for archive in distribution.getAllPPAs(): |
|
98 |
self.updateDistributionCache(distribution, archive) |
|
|
6096.3.1
by Celso Providelo
Reducing the commit-rate on update-pkgcache by increasing the commit_chunk size in the content classes and returning the number of updates done. It also sanitize the commits done in the script itself. |
99 |
archive.updateArchiveCache() |
|
5781.5.4
by Celso Providelo
Implementing concentrated PPA caches and improving tests. |
100 |
|
|
6096.3.1
by Celso Providelo
Reducing the commit-rate on update-pkgcache by increasing the commit_chunk size in the content classes and returning the number of updates done. It also sanitize the commits done in the script itself. |
101 |
# Commit any remaining update for a distribution.
|
102 |
self.txn.commit() |
|
|
5781.5.4
by Celso Providelo
Implementing concentrated PPA caches and improving tests. |
103 |
self.logger.info('%s done' % distribution.name) |
104 |
||
105 |
self.logger.debug('Finished the package cache update') |
|
|
2705
by Canonical.com Patch Queue Manager
r=spiv, mark's soyuz loving. |
106 |
|
107 |
if __name__ == '__main__': |
|
|
5781.5.4
by Celso Providelo
Implementing concentrated PPA caches and improving tests. |
108 |
script = PackageCacheUpdater( |
|
10449.1.1
by Julian Edwards
update-pkg-cache gets its own db user |
109 |
'update-cache', dbuser="update-pkg-cache") |
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
110 |
script.lock_and_run() |