← Back to branch summary

~launchpad-pqm/launchpad/devel

« back to all changes in this revision

Viewing changes to scripts/gina.py

  • Committer: Curtis Hovey
  • Date: 2011-05-12 18:25:06 UTC
  • mto: This revision was merged to the branch mainline in revision 13038.
  • Revision ID: curtis.hovey@canonical.com-20110512182506-098n1wovp9m1av59
Renamed licence_reviewed to project_reviewed.

Show diffs side-by-side

added added

removed removed

Lines of Context:
scripts/gina.py
1
1
#!/usr/bin/python -S
2
2
#
3
 
# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
 
3
# Copyright 2009,2010 Canonical Ltd.  This software is licensed under the
4
4
# GNU Affero General Public License version 3 (see the file LICENSE).
5
5
 
6
6
# This module uses relative imports.
19
19
 
20
20
__metaclass__ = type
21
21
 
 
22
import _pythonpath
 
23
 
 
24
# Set to non-zero if you'd like to be warned every so often
 
25
COUNTDOWN = 0
 
26
 
 
27
import os
 
28
import psycopg2
22
29
import sys
23
30
import time
24
31
 
25
 
import _pythonpath
26
 
import psycopg2
27
32
from zope.component import getUtility
28
33
 
 
34
from canonical import lp
29
35
from canonical.config import config
30
36
from canonical.launchpad.scripts import log
 
37
 
31
38
from lp.services.scripts.base import LaunchpadCronScript
32
39
from lp.soyuz.interfaces.component import IComponentSet
33
40
from lp.soyuz.scripts.gina import ExecutionError
34
 
from lp.soyuz.scripts.gina.archive import (
35
 
    ArchiveComponentItems,
36
 
    MangledArchiveError,
37
 
    PackagesMap,
38
 
    )
39
 
from lp.soyuz.scripts.gina.dominate import dominate_imported_source_packages
40
 
from lp.soyuz.scripts.gina.handlers import (
41
 
    DataSetupError,
42
 
    ImporterHandler,
43
 
    MultiplePackageReleaseError,
44
 
    NoSourcePackageError,
45
 
    )
46
 
from lp.soyuz.scripts.gina.packages import (
47
 
    BinaryPackageData,
48
 
    DisplayNameDecodingError,
49
 
    InvalidVersionError,
50
 
    MissingRequiredArguments,
51
 
    PoolFileNotFound,
52
 
    SourcePackageData,
53
 
    )
54
 
 
55
 
# Set to non-zero if you'd like to be warned every so often
56
 
COUNTDOWN = 0
 
41
from lp.soyuz.scripts.gina.katie import Katie
 
42
from lp.soyuz.scripts.gina.archive import (ArchiveComponentItems,
 
43
    PackagesMap, MangledArchiveError)
 
44
from lp.soyuz.scripts.gina.handlers import (ImporterHandler,
 
45
    MultiplePackageReleaseError, NoSourcePackageError, DataSetupError)
 
46
from lp.soyuz.scripts.gina.packages import (SourcePackageData,
 
47
    BinaryPackageData, MissingRequiredArguments, DisplayNameDecodingError,
 
48
    PoolFileNotFound, InvalidVersionError)
 
49
 
 
50
 
 
51
def _get_keyring(keyrings_root):
 
52
    # XXX kiko 2005-10-23: untested
 
53
    keyrings = ""
 
54
    for keyring in os.listdir(keyrings_root):
 
55
        path = os.path.join(keyrings_root, keyring)
 
56
        keyrings += " --keyring=%s" % path
 
57
    if not keyrings:
 
58
        raise AttributeError("Keyrings not found in ./keyrings/")
 
59
    return keyrings
57
60
 
58
61
 
59
62
def run_gina(options, ztm, target_section):
61
64
    from lp.registry.interfaces.pocket import PackagePublishingPocket
62
65
 
63
66
    package_root = target_section.root
 
67
    keyrings_root = target_section.keyrings
64
68
    distro = target_section.distro
 
69
    # XXX kiko 2005-10-23: I honestly think having a separate distroseries
 
70
    # bit silly. Can't we construct this based on `distroseries-pocket`?
65
71
    pocket_distroseries = target_section.pocketrelease
66
72
    distroseries = target_section.distroseries
67
73
    components = [c.strip() for c in target_section.components.split(",")]
71
77
    source_only = target_section.source_only
72
78
    spnames_only = target_section.sourcepackagenames_only
73
79
 
 
80
    dry_run = options.dry_run
 
81
 
 
82
    LPDB = lp.get_dbname()
 
83
    LPDB_HOST = lp.dbhost
 
84
    LPDB_USER = config.gina.dbuser
 
85
    KTDB = target_section.katie_dbname
 
86
 
74
87
    LIBRHOST = config.librarian.upload_host
75
88
    LIBRPORT = config.librarian.upload_port
76
89
 
77
90
    log.info("")
78
 
    log.info("=== Processing %s/%s/%s ===", distro, distroseries, pocket)
79
 
    log.debug("Packages read from: %s", package_root)
80
 
    log.info("Components to import: %s", ", ".join(components))
 
91
    log.info("=== Processing %s/%s/%s ===" % (distro, distroseries, pocket))
 
92
    log.debug("Packages read from: %s" % package_root)
 
93
    log.debug("Keyrings read from: %s" % keyrings_root)
 
94
    log.info("Components to import: %s" % ", ".join(components))
81
95
    if component_override is not None:
82
 
        log.info("Override components to: %s", component_override)
83
 
    log.info("Architectures to import: %s", ", ".join(archs))
84
 
    log.debug("Launchpad database: %s", config.database.rw_main_master)
85
 
    log.info("SourcePackage Only: %s", source_only)
86
 
    log.info("SourcePackageName Only: %s", spnames_only)
87
 
    log.debug("Librarian: %s:%s", LIBRHOST, LIBRPORT)
 
96
        log.info("Override components to: %s" % component_override)
 
97
    log.info("Architectures to import: %s" % ", ".join(archs))
 
98
    log.debug("Launchpad database: %s" % LPDB)
 
99
    log.debug("Launchpad database host: %s" % LPDB_HOST)
 
100
    log.debug("Launchpad database user: %s" % LPDB_USER)
 
101
    log.info("Katie database: %s" % KTDB)
 
102
    log.info("SourcePackage Only: %s" % source_only)
 
103
    log.info("SourcePackageName Only: %s" % spnames_only)
 
104
    log.debug("Librarian: %s:%s" % (LIBRHOST, LIBRPORT))
 
105
    log.info("Dry run: %s" % (dry_run))
88
106
    log.info("")
89
107
 
90
 
    if not hasattr(PackagePublishingPocket, pocket.upper()):
91
 
        log.error("Could not find a pocket schema for %s", pocket)
 
108
    if hasattr(PackagePublishingPocket, pocket.upper()):
 
109
        pocket = getattr(PackagePublishingPocket, pocket.upper())
 
110
    else:
 
111
        log.error("Could not find a pocket schema for %s" % pocket)
92
112
        sys.exit(1)
93
113
 
94
 
    pocket = getattr(PackagePublishingPocket, pocket.upper())
95
 
 
96
114
    if component_override:
97
115
        valid_components = [
98
116
            component.name for component in getUtility(IComponentSet)]
99
117
        if component_override not in valid_components:
100
 
            log.error("Could not find component %s", component_override)
 
118
            log.error("Could not find component %s" % component_override)
101
119
            sys.exit(1)
102
120
 
 
121
    kdb = None
 
122
    keyrings = None
 
123
    if KTDB:
 
124
        kdb = Katie(KTDB, distroseries, dry_run)
 
125
        keyrings = _get_keyring(keyrings_root)
 
126
 
103
127
    try:
104
128
        arch_component_items = ArchiveComponentItems(
105
129
            package_root, pocket_distroseries, components, archs,
106
130
            source_only)
107
131
    except MangledArchiveError:
108
132
        log.exception(
109
 
            "Failed to analyze archive for %s", pocket_distroseries)
 
133
            "Failed to analyze archive for %s" % pocket_distroseries)
110
134
        sys.exit(1)
111
135
 
112
136
    packages_map = PackagesMap(arch_component_items)
113
 
    importer_handler = ImporterHandler(
114
 
        ztm, distro, distroseries, package_root, pocket, component_override)
 
137
    importer_handler = ImporterHandler(ztm, distro, distroseries,
 
138
                                       dry_run, kdb, package_root, keyrings,
 
139
                                       pocket, component_override)
115
140
 
116
 
    import_sourcepackages(packages_map, package_root, importer_handler)
 
141
    import_sourcepackages(packages_map, kdb, package_root, keyrings,
 
142
                          importer_handler)
117
143
    importer_handler.commit()
118
144
 
119
 
    # XXX JeroenVermeulen 2011-09-07 bug=843728: Dominate binaries as well.
120
 
    dominate_imported_source_packages(
121
 
        ztm, log, distro, distroseries, pocket, packages_map)
122
 
    ztm.commit()
123
 
 
124
145
    if source_only:
125
146
        log.info('Source only mode... done')
126
147
        return
129
150
        try:
130
151
            importer_handler.ensure_archinfo(archtag)
131
152
        except DataSetupError:
132
 
            log.exception("Database setup required for run on %s", archtag)
 
153
            log.exception("Database setup required for run on %s" % archtag)
133
154
            sys.exit(1)
134
155
 
135
 
    import_binarypackages(packages_map, package_root, importer_handler)
 
156
    import_binarypackages(packages_map, kdb, package_root, keyrings,
 
157
                          importer_handler)
136
158
    importer_handler.commit()
137
159
 
138
160
 
139
 
def attempt_source_package_import(source, package_root, importer_handler):
140
 
    """Attempt to import a source package, and handle typical errors."""
141
 
    package_name = source.get("Package", "unknown")
142
 
    try:
143
 
        try:
144
 
            do_one_sourcepackage(source, package_root, importer_handler)
145
 
        except psycopg2.Error:
146
 
            log.exception(
147
 
                "Database error: unable to create SourcePackage for %s. "
148
 
                "Retrying once..", package_name)
149
 
            importer_handler.abort()
150
 
            time.sleep(15)
151
 
            do_one_sourcepackage(source, package_root, importer_handler)
152
 
    except (
153
 
        InvalidVersionError, MissingRequiredArguments,
154
 
        DisplayNameDecodingError):
155
 
        log.exception(
156
 
            "Unable to create SourcePackageData for %s", package_name)
157
 
    except (PoolFileNotFound, ExecutionError):
158
 
        # Problems with katie db stuff of opening files
159
 
        log.exception("Error processing package files for %s", package_name)
160
 
    except psycopg2.Error:
161
 
        log.exception(
162
 
            "Database errors made me give up: unable to create "
163
 
            "SourcePackage for %s", package_name)
164
 
        importer_handler.abort()
165
 
    except MultiplePackageReleaseError:
166
 
        log.exception(
167
 
            "Database duplication processing %s", package_name)
168
 
 
169
 
 
170
 
def import_sourcepackages(packages_map, package_root, importer_handler):
 
161
def import_sourcepackages(packages_map, kdb, package_root,
 
162
                          keyrings, importer_handler):
171
163
    # Goes over src_map importing the sourcepackages packages.
172
164
    count = 0
173
165
    npacks = len(packages_map.src_map)
174
 
    log.info('%i Source Packages to be imported', npacks)
 
166
    log.info('%i Source Packages to be imported' % npacks)
175
167
 
176
 
    for package in sorted(packages_map.src_map.iterkeys()):
177
 
        for source in packages_map.src_map[package]:
 
168
    for list_source in sorted(
 
169
        packages_map.src_map.values(), key=lambda x: x[0].get("Package")):
 
170
        for source in list_source:
178
171
            count += 1
179
 
            attempt_source_package_import(
180
 
                source, package_root, importer_handler)
181
 
            if COUNTDOWN and (count % COUNTDOWN == 0):
182
 
                log.warn('%i/%i sourcepackages processed', count, npacks)
183
 
 
184
 
 
185
 
def do_one_sourcepackage(source, package_root, importer_handler):
 
172
            package_name = source.get("Package", "unknown")
 
173
            try:
 
174
                try:
 
175
                    do_one_sourcepackage(
 
176
                        source, kdb, package_root, keyrings, importer_handler)
 
177
                except psycopg2.Error:
 
178
                    log.exception(
 
179
                        "Database error: unable to create SourcePackage "
 
180
                        "for %s. Retrying once.." % package_name)
 
181
                    importer_handler.abort()
 
182
                    time.sleep(15)
 
183
                    do_one_sourcepackage(
 
184
                        source, kdb, package_root, keyrings, importer_handler)
 
185
            except (
 
186
                InvalidVersionError, MissingRequiredArguments,
 
187
                DisplayNameDecodingError):
 
188
                log.exception(
 
189
                    "Unable to create SourcePackageData for %s" %
 
190
                    package_name)
 
191
                continue
 
192
            except (PoolFileNotFound, ExecutionError):
 
193
                # Problems with katie db stuff of opening files
 
194
                log.exception(
 
195
                    "Error processing package files for %s" % package_name)
 
196
                continue
 
197
            except psycopg2.Error:
 
198
                log.exception(
 
199
                    "Database errors made me give up: unable to create "
 
200
                    "SourcePackage for %s" % package_name)
 
201
                importer_handler.abort()
 
202
                continue
 
203
            except MultiplePackageReleaseError:
 
204
                log.exception(
 
205
                    "Database duplication processing %s" % package_name)
 
206
                continue
 
207
 
 
208
            if COUNTDOWN and count % COUNTDOWN == 0:
 
209
                log.warn('%i/%i sourcepackages processed' % (count, npacks))
 
210
 
 
211
 
 
212
def do_one_sourcepackage(source, kdb, package_root, keyrings,
 
213
                         importer_handler):
186
214
    source_data = SourcePackageData(**source)
187
215
    if importer_handler.preimport_sourcecheck(source_data):
188
216
        # Don't bother reading package information if the source package
189
217
        # already exists in the database
190
 
        log.info('%s already exists in the archive', source_data.package)
 
218
        log.info('%s already exists in the archive' % source_data.package)
191
219
        return
192
 
    source_data.process_package(package_root)
193
 
    source_data.ensure_complete()
 
220
    source_data.process_package(kdb, package_root, keyrings)
 
221
    source_data.ensure_complete(kdb)
194
222
    importer_handler.import_sourcepackage(source_data)
195
223
    importer_handler.commit()
196
224
 
197
225
 
198
 
def import_binarypackages(packages_map, package_root, importer_handler):
 
226
def import_binarypackages(packages_map, kdb, package_root, keyrings,
 
227
                          importer_handler):
199
228
    nosource = []
200
229
 
201
230
    # Run over all the architectures we have
202
231
    for archtag in packages_map.bin_map.keys():
203
232
        count = 0
204
233
        npacks = len(packages_map.bin_map[archtag])
205
 
        log.info(
206
 
            '%i Binary Packages to be imported for %s', npacks, archtag)
 
234
        log.info('%i Binary Packages to be imported for %s' %
 
235
                 (npacks, archtag))
207
236
        # Go over binarypackages importing them for this architecture
208
 
        for package_name in sorted(packages_map.bin_map[archtag].iterkeys()):
209
 
            binary = packages_map.bin_map[archtag][package_name]
 
237
        for binary in sorted(packages_map.bin_map[archtag].values(),
 
238
                             key=lambda x: x.get("Package")):
210
239
            count += 1
 
240
            package_name = binary.get("Package", "unknown")
211
241
            try:
212
242
                try:
213
 
                    do_one_binarypackage(
214
 
                        binary, archtag, package_root, importer_handler)
 
243
                    do_one_binarypackage(binary, archtag, kdb, package_root,
 
244
                                         keyrings, importer_handler)
215
245
                except psycopg2.Error:
216
 
                    log.exception(
217
 
                        "Database errors when importing a BinaryPackage "
218
 
                        "for %s. Retrying once..", package_name)
 
246
                    log.exception("Database errors when importing a "
 
247
                                  "BinaryPackage for %s. Retrying once.."
 
248
                                  % package_name)
219
249
                    importer_handler.abort()
220
250
                    time.sleep(15)
221
 
                    do_one_binarypackage(
222
 
                        binary, archtag, package_root, importer_handler)
 
251
                    do_one_binarypackage(binary, archtag, kdb, package_root,
 
252
                                         keyrings, importer_handler)
223
253
            except (InvalidVersionError, MissingRequiredArguments):
224
 
                log.exception(
225
 
                    "Unable to create BinaryPackageData for %s", package_name)
 
254
                log.exception("Unable to create BinaryPackageData for %s" %
 
255
                              package_name)
226
256
                continue
227
257
            except (PoolFileNotFound, ExecutionError):
228
258
                # Problems with katie db stuff of opening files
229
 
                log.exception(
230
 
                    "Error processing package files for %s", package_name)
 
259
                log.exception("Error processing package files for %s" %
 
260
                              package_name)
231
261
                continue
232
262
            except MultiplePackageReleaseError:
233
 
                log.exception(
234
 
                    "Database duplication processing %s", package_name)
 
263
                log.exception("Database duplication processing %s" %
 
264
                              package_name)
235
265
                continue
236
266
            except psycopg2.Error:
237
 
                log.exception(
238
 
                    "Database errors made me give up: unable to create "
239
 
                    "BinaryPackage for %s", package_name)
 
267
                log.exception("Database errors made me give up: unable to "
 
268
                              "create BinaryPackage for %s" % package_name)
240
269
                importer_handler.abort()
241
270
                continue
242
271
            except NoSourcePackageError:
243
 
                log.exception(
244
 
                    "Failed to create Binary Package for %s", package_name)
 
272
                log.exception("Failed to create Binary Package for %s" %
 
273
                              package_name)
245
274
                nosource.append(binary)
246
275
                continue
247
276
 
248
277
            if COUNTDOWN and count % COUNTDOWN == 0:
249
278
                # XXX kiko 2005-10-23: untested
250
 
                log.warn('%i/%i binary packages processed', count, npacks)
 
279
                log.warn('%i/%i binary packages processed' % (count, npacks))
251
280
 
252
281
        if nosource:
253
282
            # XXX kiko 2005-10-23: untested
254
 
            log.warn('%i source packages not found', len(nosource))
 
283
            log.warn('%i source packages not found' % len(nosource))
255
284
            for pkg in nosource:
256
285
                log.warn(pkg)
257
286
 
258
287
 
259
 
def do_one_binarypackage(binary, archtag, package_root, importer_handler):
 
288
def do_one_binarypackage(binary, archtag, kdb, package_root, keyrings,
 
289
                         importer_handler):
260
290
    binary_data = BinaryPackageData(**binary)
261
291
    if importer_handler.preimport_binarycheck(archtag, binary_data):
262
 
        log.info('%s already exists in the archive', binary_data.package)
 
292
        log.info('%s already exists in the archive' % binary_data.package)
263
293
        return
264
 
    binary_data.process_package(package_root)
 
294
    binary_data.process_package(kdb, package_root, keyrings)
265
295
    importer_handler.import_binarypackage(archtag, binary_data)
266
296
    importer_handler.commit()
267
297
 
271
301
    def __init__(self):
272
302
        super(Gina, self).__init__(name='gina', dbuser=config.gina.dbuser)
273
303
 
274
 
    @property
275
 
    def usage(self):
276
 
        return "%s [options] (targets|--all)" % sys.argv[0]
277
 
 
278
304
    def add_my_options(self):
 
305
        self.parser.add_option("-n", "--dry-run", action="store_true",
 
306
            help="Don't commit changes to the database",
 
307
            dest="dry_run", default=False)
279
308
        self.parser.add_option("-a", "--all", action="store_true",
280
309
            help="Run all sections defined in launchpad.conf (in order)",
281
310
            dest="all", default=False)
282
 
        self.parser.add_option("-l", "--list-targets", action="store_true",
283
 
            help="List configured import targets", dest="list_targets",
284
 
            default=False)
285
 
 
286
 
    def getConfiguredTargets(self):
287
 
        """Get the configured import targets.
288
 
 
289
 
        Gina's targets are configured as "[gina_target.*]" sections in the
290
 
        LAZR config.
291
 
        """
292
 
        sections = config.getByCategory('gina_target', [])
293
 
        targets = [
294
 
            target.category_and_section_names[1] for target in sections]
295
 
        if len(targets) == 0:
296
 
            self.logger.warn("No gina_target entries configured.")
297
 
        return targets
298
 
 
299
 
    def listTargets(self, targets):
300
 
        """Print out the given targets list."""
301
 
        for target in targets:
302
 
            self.logger.info("Target: %s", target)
303
 
 
304
 
    def getTargets(self, possible_targets):
305
 
        """Get targets to process."""
 
311
 
 
312
    def main(self):
 
313
        possible_targets = [target.category_and_section_names[1]
 
314
            for target in config.getByCategory('gina_target')]
306
315
        targets = self.args
307
316
        if self.options.all:
308
 
            return list(possible_targets)
 
317
            targets = possible_targets[:]
309
318
        else:
310
319
            if not targets:
311
320
                self.parser.error(
314
323
                if target not in possible_targets:
315
324
                    self.parser.error(
316
325
                        "No Gina target %s in config file" % target)
317
 
            return targets
318
 
 
319
 
    def main(self):
320
 
        possible_targets = self.getConfiguredTargets()
321
 
 
322
 
        if self.options.list_targets:
323
 
            self.listTargets(possible_targets)
324
 
            return
325
 
 
326
 
        for target in self.getTargets(possible_targets):
 
326
 
 
327
        for target in targets:
327
328
            target_section = config['gina_target.%s' % target]
328
329
            run_gina(self.options, self.txn, target_section)
329
330