|
10637.3.1
by Guilherme Salgado
Use the default python version instead of a hard-coded version |
1 |
#!/usr/bin/python -S
|
|
8687.15.7
by Karl Fogel
Add the copyright header block to more files. |
2 |
#
|
|
14027.3.1
by Jeroen Vermeulen
Fix lots of lint in recently-changed files. |
3 |
# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
|
|
8687.15.7
by Karl Fogel
Add the copyright header block to more files. |
4 |
# GNU Affero General Public License version 3 (see the file LICENSE).
|
5 |
||
|
4935.3.7
by Curtis Hovey
Added bad name suppression to cronscripts. |
6 |
# pylint: disable-msg=C0103,W0403
|
|
2816.1.1
by Stuart Bishop
Stub garbage collector cron script |
7 |
|
8 |
"""Librarian garbage collector.
|
|
9 |
||
10 |
This script is run on the Librarian server to merge duplicate files,
|
|
11 |
remove expired files from the file system and clean up unreachable
|
|
12 |
rows in the database.
|
|
13 |
"""
|
|
14 |
||
15 |
__metaclass__ = type |
|
16 |
||
|
14606.3.5
by William Grant
Reformat |
17 |
import _pythonpath |
18 |
||
|
14027.3.1
by Jeroen Vermeulen
Fix lots of lint in recently-changed files. |
19 |
import logging |
20 |
||
|
14605.1.1
by Curtis Hovey
Moved canonical.config to lp.services. |
21 |
from lp.services.config import config |
|
14560.2.29
by Curtis Hovey
Restored lpstorm module name because it lp engineers know that name. |
22 |
from lp.services.database.lpstorm import IStore |
|
14578.2.1
by William Grant
Move librarian stuff from canonical.launchpad to lp.services.librarian. canonical.librarian remains untouched. |
23 |
from lp.services.librarian.model import LibraryFileAlias |
|
14606.2.5
by William Grant
Move the rest of canonical.librarian to lp.services.librarianserver. |
24 |
from lp.services.librarianserver import librariangc |
|
8356.1.1
by Leonard Richardson
Partial move. |
25 |
from lp.services.scripts.base import LaunchpadCronScript |
|
4264.2.1
by James Henstridge
add a LaunchpadCronScript subclass, and make cronscripts/*.py use it |
26 |
|
27 |
||
28 |
class LibrarianGC(LaunchpadCronScript): |
|
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
29 |
def add_my_options(self): |
30 |
self.parser.add_option( |
|
31 |
'', "--skip-duplicates", action="store_true", default=False, |
|
32 |
dest="skip_duplicates", |
|
33 |
help="Skip duplicate LibraryFileContent merging" |
|
34 |
)
|
|
35 |
self.parser.add_option( |
|
36 |
'', "--skip-aliases", action="store_true", default=False, |
|
37 |
dest="skip_aliases", |
|
38 |
help="Skip unreferenced LibraryFileAlias removal" |
|
39 |
)
|
|
40 |
self.parser.add_option( |
|
41 |
'', "--skip-content", action="store_true", default=False, |
|
42 |
dest="skip_content", |
|
43 |
help="Skip unreferenced LibraryFileContent removal" |
|
44 |
)
|
|
45 |
self.parser.add_option( |
|
46 |
'', "--skip-blobs", action="store_true", default=False, |
|
47 |
dest="skip_blobs", |
|
48 |
help="Skip removing expired TemporaryBlobStorage rows" |
|
49 |
)
|
|
50 |
self.parser.add_option( |
|
51 |
'', "--skip-files", action="store_true", default=False, |
|
52 |
dest="skip_files", |
|
53 |
help="Skip removing files on disk with no database references" |
|
|
7500.2.1
by Stuart Bishop
When all aliases are expired, flag a LibraryFileContent as deleted |
54 |
" or flagged for deletion."
|
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
55 |
)
|
|
9572.1.10
by Stuart Bishop
Expire expired LibraryFileAliases |
56 |
self.parser.add_option( |
57 |
'', "--skip-expiry", action="store_true", default=False, |
|
58 |
dest="skip_expiry", |
|
59 |
help="Skip expiring aliases with an expiry date in the past." |
|
60 |
)
|
|
61 |
||
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
62 |
def main(self): |
63 |
librariangc.log = self.logger |
|
64 |
||
65 |
if self.options.loglevel <= logging.DEBUG: |
|
66 |
librariangc.debug = True |
|
67 |
||
|
13970.9.4
by William Grant
librarian-gc no longer uses ZTM.conn(). |
68 |
# XXX wgrant 2011-09-18 bug=853066: Using Storm's raw connection
|
69 |
# here is wrong. We should either create our own or use
|
|
70 |
# Store.execute or cursor() and the transaction module.
|
|
71 |
conn = IStore(LibraryFileAlias)._connection._raw_connection |
|
|
3691.357.6
by Stuart Bishop
Updates based on review feedback |
72 |
|
73 |
# Refuse to run if we have significant clock skew between the
|
|
74 |
# librarian and the database.
|
|
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
75 |
librariangc.confirm_no_clock_skew(conn) |
|
3691.357.6
by Stuart Bishop
Updates based on review feedback |
76 |
|
|
3322.1.11
by Stuart Bishop
Refactor librarian garbage collection to only use one connection |
77 |
# Note that each of these next steps will issue commit commands
|
78 |
# as appropriate to make this script transaction friendly
|
|
|
9572.1.10
by Stuart Bishop
Expire expired LibraryFileAliases |
79 |
if not self.options.skip_expiry: |
80 |
librariangc.expire_aliases(conn) |
|
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
81 |
if not self.options.skip_content: |
|
14027.3.1
by Jeroen Vermeulen
Fix lots of lint in recently-changed files. |
82 |
# First sweep.
|
83 |
librariangc.delete_unreferenced_content(conn) |
|
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
84 |
if not self.options.skip_blobs: |
85 |
librariangc.delete_expired_blobs(conn) |
|
86 |
if not self.options.skip_duplicates: |
|
87 |
librariangc.merge_duplicates(conn) |
|
88 |
if not self.options.skip_aliases: |
|
89 |
librariangc.delete_unreferenced_aliases(conn) |
|
90 |
if not self.options.skip_content: |
|
|
14027.3.1
by Jeroen Vermeulen
Fix lots of lint in recently-changed files. |
91 |
# Second sweep.
|
92 |
librariangc.delete_unreferenced_content(conn) |
|
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
93 |
if not self.options.skip_files: |
94 |
librariangc.delete_unwanted_files(conn) |
|
|
2816.1.7
by Stuart Bishop
Work in progress |
95 |
|
|
2816.1.1
by Stuart Bishop
Stub garbage collector cron script |
96 |
|
97 |
if __name__ == '__main__': |
|
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
98 |
script = LibrarianGC('librarian-gc', |
|
5855.5.1
by Curtis Hovey
Migrate db configuration to lazr.config. |
99 |
dbuser=config.librarian_gc.dbuser) |
|
14022.3.2
by William Grant
LaunchpadScript no longer uses initZopeless. |
100 |
script.lock_and_run(isolation='autocommit') |