|
10637.3.1
by Guilherme Salgado
Use the default python version instead of a hard-coded version |
1 |
#!/usr/bin/python -S
|
|
8687.15.7
by Karl Fogel
Add the copyright header block to more files. |
2 |
#
|
3 |
# Copyright 2009 Canonical Ltd. This software is licensed under the
|
|
4 |
# GNU Affero General Public License version 3 (see the file LICENSE).
|
|
5 |
||
|
4935.3.7
by Curtis Hovey
Added bad name suppression to cronscripts. |
6 |
# pylint: disable-msg=C0103,W0403
|
|
2816.1.1
by Stuart Bishop
Stub garbage collector cron script |
7 |
|
8 |
"""Librarian garbage collector.
|
|
9 |
||
10 |
This script is run on the Librarian server to merge duplicate files,
|
|
11 |
remove expired files from the file system and clean up unreachable
|
|
12 |
rows in the database.
|
|
13 |
"""
|
|
14 |
||
15 |
__metaclass__ = type |
|
16 |
||
|
2816.1.7
by Stuart Bishop
Work in progress |
17 |
import _pythonpath |
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
18 |
import logging |
19 |
||
|
2816.1.3
by Stuart Bishop
Work in progress |
20 |
from canonical.librarian import librariangc |
|
5842.1.1
by James Henstridge
Rename the transaction isolation level constants to match the psycopg2 |
21 |
from canonical.database.sqlbase import ISOLATION_LEVEL_AUTOCOMMIT |
|
2816.1.7
by Stuart Bishop
Work in progress |
22 |
from canonical.config import config |
|
8356.1.1
by Leonard Richardson
Partial move. |
23 |
from lp.services.scripts.base import LaunchpadCronScript |
|
4264.2.1
by James Henstridge
add a LaunchpadCronScript subclass, and make cronscripts/*.py use it |
24 |
|
25 |
||
26 |
class LibrarianGC(LaunchpadCronScript): |
|
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
27 |
def add_my_options(self): |
28 |
self.parser.add_option( |
|
29 |
'', "--skip-duplicates", action="store_true", default=False, |
|
30 |
dest="skip_duplicates", |
|
31 |
help="Skip duplicate LibraryFileContent merging" |
|
32 |
)
|
|
33 |
self.parser.add_option( |
|
34 |
'', "--skip-aliases", action="store_true", default=False, |
|
35 |
dest="skip_aliases", |
|
36 |
help="Skip unreferenced LibraryFileAlias removal" |
|
37 |
)
|
|
38 |
self.parser.add_option( |
|
39 |
'', "--skip-content", action="store_true", default=False, |
|
40 |
dest="skip_content", |
|
41 |
help="Skip unreferenced LibraryFileContent removal" |
|
42 |
)
|
|
43 |
self.parser.add_option( |
|
44 |
'', "--skip-blobs", action="store_true", default=False, |
|
45 |
dest="skip_blobs", |
|
46 |
help="Skip removing expired TemporaryBlobStorage rows" |
|
47 |
)
|
|
48 |
self.parser.add_option( |
|
49 |
'', "--skip-files", action="store_true", default=False, |
|
50 |
dest="skip_files", |
|
51 |
help="Skip removing files on disk with no database references" |
|
|
7500.2.1
by Stuart Bishop
When all aliases are expired, flag a LibraryFileContent as deleted |
52 |
" or flagged for deletion."
|
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
53 |
)
|
|
9572.1.10
by Stuart Bishop
Expire expired LibraryFileAliases |
54 |
self.parser.add_option( |
55 |
'', "--skip-expiry", action="store_true", default=False, |
|
56 |
dest="skip_expiry", |
|
57 |
help="Skip expiring aliases with an expiry date in the past." |
|
58 |
)
|
|
59 |
||
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
60 |
|
61 |
def main(self): |
|
62 |
librariangc.log = self.logger |
|
63 |
||
64 |
if self.options.loglevel <= logging.DEBUG: |
|
65 |
librariangc.debug = True |
|
66 |
||
67 |
conn = self.txn.conn() |
|
|
3691.357.6
by Stuart Bishop
Updates based on review feedback |
68 |
|
69 |
# Refuse to run if we have significant clock skew between the
|
|
70 |
# librarian and the database.
|
|
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
71 |
librariangc.confirm_no_clock_skew(conn) |
|
3691.357.6
by Stuart Bishop
Updates based on review feedback |
72 |
|
|
3322.1.11
by Stuart Bishop
Refactor librarian garbage collection to only use one connection |
73 |
# Note that each of these next steps will issue commit commands
|
74 |
# as appropriate to make this script transaction friendly
|
|
|
9572.1.10
by Stuart Bishop
Expire expired LibraryFileAliases |
75 |
if not self.options.skip_expiry: |
76 |
librariangc.expire_aliases(conn) |
|
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
77 |
if not self.options.skip_content: |
78 |
librariangc.delete_unreferenced_content(conn) # first sweep |
|
79 |
if not self.options.skip_blobs: |
|
80 |
librariangc.delete_expired_blobs(conn) |
|
81 |
if not self.options.skip_duplicates: |
|
82 |
librariangc.merge_duplicates(conn) |
|
83 |
if not self.options.skip_aliases: |
|
84 |
librariangc.delete_unreferenced_aliases(conn) |
|
85 |
if not self.options.skip_content: |
|
86 |
librariangc.delete_unreferenced_content(conn) # second sweep |
|
87 |
if not self.options.skip_files: |
|
88 |
librariangc.delete_unwanted_files(conn) |
|
|
2816.1.7
by Stuart Bishop
Work in progress |
89 |
|
|
2816.1.1
by Stuart Bishop
Stub garbage collector cron script |
90 |
|
91 |
if __name__ == '__main__': |
|
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
92 |
script = LibrarianGC('librarian-gc', |
|
5855.5.1
by Curtis Hovey
Migrate db configuration to lazr.config. |
93 |
dbuser=config.librarian_gc.dbuser) |
|
5821.2.96
by James Henstridge
Fix librarian GC tests to run under zopeless layer. |
94 |
script.lock_and_run(isolation=ISOLATION_LEVEL_AUTOCOMMIT) |
|
3691.348.18
by kiko
Convert most other scripts; 6 remain, of which one looks like it's going to be a bit tricky... |
95 |