← Back to branch summary

~launchpad-pqm/launchpad/devel

« back to all changes in this revision

Viewing changes to scripts/gina.py

[r=allenap][bug=850926] Katie/Gina cleanup.

Show diffs side-by-side

added added

removed removed

Lines of Context:
scripts/gina.py
19
19
 
20
20
__metaclass__ = type
21
21
 
22
 
import os
23
22
import sys
24
23
import time
25
24
 
44
43
    MultiplePackageReleaseError,
45
44
    NoSourcePackageError,
46
45
    )
47
 
from lp.soyuz.scripts.gina.katie import Katie
48
46
from lp.soyuz.scripts.gina.packages import (
49
47
    BinaryPackageData,
50
48
    DisplayNameDecodingError,
58
56
COUNTDOWN = 0
59
57
 
60
58
 
61
 
def _get_keyring(keyrings_root):
62
 
    # XXX kiko 2005-10-23: untested
63
 
    keyrings = ""
64
 
    for keyring in os.listdir(keyrings_root):
65
 
        path = os.path.join(keyrings_root, keyring)
66
 
        keyrings += " --keyring=%s" % path
67
 
    if not keyrings:
68
 
        raise AttributeError("Keyrings not found in ./keyrings/")
69
 
    return keyrings
70
 
 
71
 
 
72
59
def run_gina(options, ztm, target_section):
73
60
    # Avoid circular imports.
74
61
    from lp.registry.interfaces.pocket import PackagePublishingPocket
75
62
 
76
63
    package_root = target_section.root
77
 
    keyrings_root = target_section.keyrings
78
64
    distro = target_section.distro
79
 
    # XXX kiko 2005-10-23: I honestly think having a separate distroseries
80
 
    # bit silly. Can't we construct this based on `distroseries-pocket`?
81
65
    pocket_distroseries = target_section.pocketrelease
82
66
    distroseries = target_section.distroseries
83
67
    components = [c.strip() for c in target_section.components.split(",")]
87
71
    source_only = target_section.source_only
88
72
    spnames_only = target_section.sourcepackagenames_only
89
73
 
90
 
    KTDB = target_section.katie_dbname
91
 
 
92
74
    LIBRHOST = config.librarian.upload_host
93
75
    LIBRPORT = config.librarian.upload_port
94
76
 
95
77
    log.info("")
96
78
    log.info("=== Processing %s/%s/%s ===", distro, distroseries, pocket)
97
79
    log.debug("Packages read from: %s", package_root)
98
 
    log.debug("Keyrings read from: %s", keyrings_root)
99
80
    log.info("Components to import: %s", ", ".join(components))
100
81
    if component_override is not None:
101
82
        log.info("Override components to: %s", component_override)
102
83
    log.info("Architectures to import: %s", ", ".join(archs))
103
84
    log.debug("Launchpad database: %s", config.database.rw_main_master)
104
 
    log.info("Katie database: %s", KTDB)
105
85
    log.info("SourcePackage Only: %s", source_only)
106
86
    log.info("SourcePackageName Only: %s", spnames_only)
107
87
    log.debug("Librarian: %s:%s", LIBRHOST, LIBRPORT)
120
100
            log.error("Could not find component %s", component_override)
121
101
            sys.exit(1)
122
102
 
123
 
    kdb = None
124
 
    keyrings = None
125
 
    if KTDB:
126
 
        kdb = Katie(KTDB, distroseries)
127
 
        keyrings = _get_keyring(keyrings_root)
128
 
 
129
103
    try:
130
104
        arch_component_items = ArchiveComponentItems(
131
105
            package_root, pocket_distroseries, components, archs,
137
111
 
138
112
    packages_map = PackagesMap(arch_component_items)
139
113
    importer_handler = ImporterHandler(
140
 
        ztm, distro, distroseries, kdb, package_root, keyrings, pocket,
141
 
        component_override)
 
114
        ztm, distro, distroseries, package_root, pocket, component_override)
142
115
 
143
 
    import_sourcepackages(
144
 
        packages_map, kdb, package_root, keyrings, importer_handler)
 
116
    import_sourcepackages(packages_map, package_root, importer_handler)
145
117
    importer_handler.commit()
146
118
 
147
119
    # XXX JeroenVermeulen 2011-09-07 bug=843728: Dominate binaries as well.
160
132
            log.exception("Database setup required for run on %s", archtag)
161
133
            sys.exit(1)
162
134
 
163
 
    import_binarypackages(
164
 
        packages_map, kdb, package_root, keyrings, importer_handler)
 
135
    import_binarypackages(packages_map, package_root, importer_handler)
165
136
    importer_handler.commit()
166
137
 
167
138
 
168
 
def attempt_source_package_import(source, kdb, package_root, keyrings,
169
 
                                  importer_handler):
 
139
def attempt_source_package_import(source, package_root, importer_handler):
170
140
    """Attempt to import a source package, and handle typical errors."""
171
141
    package_name = source.get("Package", "unknown")
172
142
    try:
173
143
        try:
174
 
            do_one_sourcepackage(
175
 
                source, kdb, package_root, keyrings, importer_handler)
 
144
            do_one_sourcepackage(source, package_root, importer_handler)
176
145
        except psycopg2.Error:
177
146
            log.exception(
178
147
                "Database error: unable to create SourcePackage for %s. "
179
148
                "Retrying once..", package_name)
180
149
            importer_handler.abort()
181
150
            time.sleep(15)
182
 
            do_one_sourcepackage(
183
 
                source, kdb, package_root, keyrings, importer_handler)
 
151
            do_one_sourcepackage(source, package_root, importer_handler)
184
152
    except (
185
153
        InvalidVersionError, MissingRequiredArguments,
186
154
        DisplayNameDecodingError):
199
167
            "Database duplication processing %s", package_name)
200
168
 
201
169
 
202
 
def import_sourcepackages(packages_map, kdb, package_root,
203
 
                          keyrings, importer_handler):
 
170
def import_sourcepackages(packages_map, package_root, importer_handler):
204
171
    # Goes over src_map importing the sourcepackages packages.
205
172
    count = 0
206
173
    npacks = len(packages_map.src_map)
210
177
        for source in packages_map.src_map[package]:
211
178
            count += 1
212
179
            attempt_source_package_import(
213
 
                source, kdb, package_root, keyrings, importer_handler)
 
180
                source, package_root, importer_handler)
214
181
            if COUNTDOWN and (count % COUNTDOWN == 0):
215
182
                log.warn('%i/%i sourcepackages processed', count, npacks)
216
183
 
217
184
 
218
 
def do_one_sourcepackage(source, kdb, package_root, keyrings,
219
 
                         importer_handler):
 
185
def do_one_sourcepackage(source, package_root, importer_handler):
220
186
    source_data = SourcePackageData(**source)
221
187
    if importer_handler.preimport_sourcecheck(source_data):
222
188
        # Don't bother reading package information if the source package
223
189
        # already exists in the database
224
190
        log.info('%s already exists in the archive', source_data.package)
225
191
        return
226
 
    source_data.process_package(kdb, package_root, keyrings)
227
 
    source_data.ensure_complete(kdb)
 
192
    source_data.process_package(package_root)
 
193
    source_data.ensure_complete()
228
194
    importer_handler.import_sourcepackage(source_data)
229
195
    importer_handler.commit()
230
196
 
231
197
 
232
 
def import_binarypackages(packages_map, kdb, package_root, keyrings,
233
 
                          importer_handler):
 
198
def import_binarypackages(packages_map, package_root, importer_handler):
234
199
    nosource = []
235
200
 
236
201
    # Run over all the architectures we have
245
210
            count += 1
246
211
            try:
247
212
                try:
248
 
                    do_one_binarypackage(binary, archtag, kdb, package_root,
249
 
                                         keyrings, importer_handler)
 
213
                    do_one_binarypackage(
 
214
                        binary, archtag, package_root, importer_handler)
250
215
                except psycopg2.Error:
251
216
                    log.exception(
252
217
                        "Database errors when importing a BinaryPackage "
253
218
                        "for %s. Retrying once..", package_name)
254
219
                    importer_handler.abort()
255
220
                    time.sleep(15)
256
 
                    do_one_binarypackage(binary, archtag, kdb, package_root,
257
 
                                         keyrings, importer_handler)
 
221
                    do_one_binarypackage(
 
222
                        binary, archtag, package_root, importer_handler)
258
223
            except (InvalidVersionError, MissingRequiredArguments):
259
224
                log.exception(
260
225
                    "Unable to create BinaryPackageData for %s", package_name)
291
256
                log.warn(pkg)
292
257
 
293
258
 
294
 
def do_one_binarypackage(binary, archtag, kdb, package_root, keyrings,
295
 
                         importer_handler):
 
259
def do_one_binarypackage(binary, archtag, package_root, importer_handler):
296
260
    binary_data = BinaryPackageData(**binary)
297
261
    if importer_handler.preimport_binarycheck(archtag, binary_data):
298
262
        log.info('%s already exists in the archive', binary_data.package)
299
263
        return
300
 
    binary_data.process_package(kdb, package_root, keyrings)
 
264
    binary_data.process_package(package_root)
301
265
    importer_handler.import_binarypackage(archtag, binary_data)
302
266
    importer_handler.commit()
303
267