« back to all changes in this revision
Viewing changes to cronscripts/product-release-finder.py
- browse files at revision 3691.105.4
- compare with another revision
- compare with revision 6393.2.1
- download diff
- download tarball
- view history from revision 3691.105.4
- Committer: James Henstridge
- Date: 2006-08-29 10:05:24 UTC
- mto: (3691.86.48 launchpad)
- mto: This revision was merged to the branch mainline in revision 3711.
- Revision ID: james.henstridge@canonical.com-20060829100524-d8ad2e9d66c9f776
update product-release-finder.py to use modularised version of code
- files modified:
- cronscripts/product-release-finder.py
added
removed
cronscripts/product-release-finder.py
1
1
#!/usr/bin/env python
2
# Copyright 2004-2006 Canonical Ltd. All rights reserved.
2
3
"""Upstream Product Release Finder.
3
4
4
5
Scan FTP and HTTP sites specified for each ProductSeries in the database
6
7
"""
7
8
8
9
import _pythonpath
9
import os
10
import mimetypes
11
12
from urllib import urlretrieve
13
from urlparse import urlsplit
14
from optparse import OptionParser
15
16
from zope.component import getUtility
17
18
from canonical.launchpad.scripts.productreleasefinder.hose import Hose
19
from canonical.launchpad.scripts.productreleasefinder.filter import (
20
Cache, FilterPattern)
21
from canonical.launchpad.interfaces import IProductSet, IProductReleaseSet
22
from canonical.librarian.interfaces import IFileUploadClient
23
from canonical.launchpad.validators.version import sane_version
10
import sys
11
import optparse
12
13
from canonical.config import config
24
14
from canonical.lp import initZopeless
25
from canonical.config import config
26
15
from canonical.launchpad.scripts import (execute_zcml_for_scripts,
27
16
logger, logger_options)
28
29
from hct.util import path
30
31
32
def main():
17
from canonical.launchpad.scripts.productreleasefinder.finder import (
18
ProductReleaseFinder)
19
20
21
def main(argv):
33
22
# Parse command-line arguments
34
parser = OptionParser()
23
parser = optparse.OptionParser()
35
24
logger_options(parser)
36
(options, args) = parser.parse_args()
25
(options, args) = parser.parse_args(argv[1:])
37
26
38
global log
39
log = logger(options, "productreleasefinder")
27
execute_zcml_for_scripts()
40
28
ztm = initZopeless(dbuser=config.productreleasefinder.dbuser,
41
29
implicitBegin=False)
42
30
43
cache = Cache(config.productreleasefinder.cache_path, log_parent=log)
44
try:
45
for product_name, filters in get_filters(ztm):
46
hose = Hose(filters, cache)
47
for series_name, url in hose:
48
if series_name is not None:
49
new_release(ztm, product_name, series_name, url)
50
else:
51
log.warning("File in %s found that matched no glob: %s",
52
product_name, url)
53
finally:
54
cache.save()
55
56
def get_filters(ztm):
57
"""Build the list of products and filters.
58
59
Returns a list of (product_name, filters) for each product in the database,
60
where the filter keys are series names.
61
"""
62
todo = []
63
64
ztm.begin()
65
products = getUtility(IProductSet)
66
for product in products:
67
filters = []
68
69
for series in product.serieslist:
70
if series.releasefileglob is None or series.releasefileglob == "":
71
continue
72
else:
73
releasefileglob = series.releasefileglob
74
75
if series.releaseroot is None or series.releaseroot == "":
76
if product.releaseroot is None or product.releaseroot == "":
77
continue
78
else:
79
releaseroot = product.releaseroot
80
else:
81
releaseroot = series.releaseroot
82
83
filters.append(FilterPattern(series.name,
84
releaseroot, releasefileglob))
85
86
if not len(filters):
87
continue
88
89
log.info("%s has %d series with information", product.name,
90
len(filters))
91
92
todo.append((product.name, filters))
93
ztm.abort()
94
95
return todo
96
97
def new_release(ztm, product_name, series_name, url):
98
"""Create a new ProductRelease.
99
100
Downloads the file and creates a new ProductRelease associated with
101
the series.
102
"""
103
filename = urlsplit(url)[2]
104
slash = filename.rfind("/")
105
if slash != -1:
106
filename = filename[slash + 1:]
107
log.debug("Filename portion is %s", filename)
108
109
version = path.split_version(path.name(filename))[1]
110
log.debug("Version is %s", version)
111
if version is None:
112
log.error("Unable to parse version from %s", url)
113
return
114
115
if not sane_version(version):
116
log.error("Version number '%s' for '%s' is not sane", version, url)
117
return
118
119
(mimetype, encoding) = mimetypes.guess_type(url)
120
log.debug("Mime Type is %s", mimetype)
121
if mimetype is None:
122
mimetype = "application/octet-stream"
123
124
log.debug("Downloading %s", url)
125
try:
126
(local, headers) = urlretrieve(url)
127
stat = os.stat(local)
128
except IOError:
129
log.error("Download of %s failed, can't create release")
130
return
131
except OSError:
132
log.error("Unable to stat downloaded file, can't create release")
133
return
134
135
open_file = open(local, "r")
136
os.unlink(local)
137
try:
138
ztm.begin()
139
try:
140
product = getUtility(IProductSet)[product_name]
141
series = product.getSeries(series_name)
142
log.info("Creating new release %s for %s %s",
143
version, product.name, series.name)
144
145
release = getUtility(IProductReleaseSet).\
146
new(version, series, product.owner)
147
log.debug("Created ProductRelease %d", release.id)
148
149
alias_id = getUtility(IFileUploadClient).\
150
addFile(filename, stat.st_size, open_file, mimetype)
151
log.debug("Created LibraryFileAlias %d", alias_id)
152
153
release.addFileAlias(alias_id)
154
ztm.commit()
155
except:
156
ztm.abort()
157
raise
158
finally:
159
open_file.close()
160
31
log = logger(options, "productreleasefinder")
32
33
prf = ProductReleaseFinder(ztm, log)
34
prf.findReleases()
161
35
162
36
if __name__ == "__main__":
163
execute_zcml_for_scripts()
164
165
main()
37
sys.exit(main(sys.argv))
Loggerhead is a web-based interface for Breezy
Version: 2.0.1