~launchpad-pqm/launchpad/devel

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
#!/usr/bin/python -S
# pylint: disable-msg=W0403

# Copyright 2010 Canonical Ltd.  This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
#
# This code is based on William Grant's make-ubuntu-sane.py script, but
# reorganized to fit Launchpad coding guidelines, and extended.  The
# code is included under Canonical copyright with his permission
# (2010-02-24).

"""Clean up sample data so it will allow Soyuz to run locally.

DO NOT RUN ON PRODUCTION SYSTEMS.  This script deletes lots of
Ubuntu-related data.

This script creates a user "ppa-user" (email ppa-user@example.com,
password test) who is able to create PPAs.
"""

__metaclass__ = type

import _pythonpath

from optparse import OptionParser
import re
import os
import subprocess
import sys
from textwrap import dedent
import transaction

from zope.component import getUtility
from zope.event import notify
from zope.lifecycleevent import ObjectCreatedEvent
from zope.security.proxy import removeSecurityProxy

from storm.store import Store

from canonical.database.sqlbase import sqlvalues

from canonical.lp import initZopeless

from canonical.launchpad.interfaces.launchpad import (
    ILaunchpadCelebrities)
from canonical.launchpad.scripts import execute_zcml_for_scripts
from canonical.launchpad.scripts.logger import logger, logger_options
from canonical.launchpad.webapp.interfaces import (
    IStoreSelector, MAIN_STORE, MASTER_FLAVOR, SLAVE_FLAVOR)

from lp.registry.interfaces.codeofconduct import ISignedCodeOfConductSet
from lp.registry.interfaces.person import IPersonSet
from lp.registry.interfaces.series import SeriesStatus
from lp.registry.model.codeofconduct import SignedCodeOfConduct
from lp.soyuz.interfaces.component import IComponentSet
from lp.soyuz.interfaces.processor import IProcessorFamilySet
from lp.soyuz.interfaces.section import ISectionSet
from lp.soyuz.interfaces.sourcepackageformat import (
    ISourcePackageFormatSelectionSet, SourcePackageFormat)
from lp.soyuz.model.section import SectionSelection
from lp.soyuz.model.component import ComponentSelection
from lp.testing.factory import LaunchpadObjectFactory


user_name = 'ppa-user'
default_email = '%s@example.com' % user_name


class DoNotRunOnProduction(Exception):
    """Error: do not run this script on production (-like) systems."""


def get_max_id(store, table_name):
    """Find highest assigned id in given table."""
    max_id = store.execute("SELECT max(id) FROM %s" % table_name).get_one()
    if max_id is None:
        return None
    else:
        return max_id[0]


def get_store(flavor=MASTER_FLAVOR):
    """Obtain an ORM store."""
    return getUtility(IStoreSelector).get(MAIN_STORE, flavor)


def check_preconditions(options):
    """Try to ensure that it's safe to run.

    This script must not run on a production server, or anything
    remotely like it.
    """
    store = get_store(SLAVE_FLAVOR)

    # Just a guess, but dev systems aren't likely to have ids this high
    # in this table.  Production data does.
    real_data = (get_max_id(store, "TranslationMessage") >= 1000000)
    if real_data and not options.force:
        raise DoNotRunOnProduction(
            "Refusing to delete Ubuntu data unless you --force me.")

    # For some configs it's just absolutely clear this script shouldn't
    # run.  Don't even accept --force there.
    forbidden_configs = re.compile('(edge|lpnet|production)')
    current_config = os.getenv('LPCONFIG', 'an unknown config')
    if forbidden_configs.match(current_config):
        raise DoNotRunOnProduction(
            "I won't delete Ubuntu data on %s and you can't --force me."
            % current_config)


def parse_args(arguments):
    """Parse command-line arguments.

    :return: (options, args, logger)
    """
    parser = OptionParser(
        description="Set up fresh Ubuntu series and %s identity." % user_name)
    parser.add_option('-f', '--force', action='store_true', dest='force',
        help="DANGEROUS: run even if the database looks production-like.")
    parser.add_option('-e', '--email', action='store', dest='email',
        default=default_email,
        help=(
            "Email address to use for %s.  Should match your GPG key."
            % user_name))

    logger_options(parser)

    options, args = parser.parse_args(arguments)

    return options, args, logger(options)


def get_person_set():
    """Return `IPersonSet` utility."""
    return getUtility(IPersonSet)


def retire_series(distribution):
    """Mark all `DistroSeries` for `distribution` as obsolete."""
    for series in distribution.series:
        series.status = SeriesStatus.OBSOLETE


def retire_active_publishing_histories(histories, requester):
    """Retire all active publishing histories in the given collection."""
    # Avoid circular import.
    from lp.soyuz.interfaces.publishing import active_publishing_status
    for history in histories(status=active_publishing_status):
        history.requestDeletion(
            requester, "Cleaned up because of missing Librarian files.")


def retire_distro_archives(distribution, culprit):
    """Retire all items in `distribution`'s archives."""
    for archive in distribution.all_distro_archives:
        retire_active_publishing_histories(
            archive.getPublishedSources, culprit)
        retire_active_publishing_histories(
            archive.getAllPublishedBinaries, culprit)


def retire_ppas(distribution):
    """Disable all PPAs for `distribution`."""
    for ppa in distribution.getAllPPAs():
        removeSecurityProxy(ppa).publish = False


def set_lucille_config(distribution):
    """Set lucilleconfig on all series of `distribution`."""
    for series in distribution.series:
        removeSecurityProxy(series).lucilleconfig = '''[publishing]
components = main restricted universe multiverse'''


def add_architecture(distroseries, architecture_name):
    """Add a DistroArchSeries for the given architecture to `distroseries`."""
    # Avoid circular import.
    from lp.soyuz.model.distroarchseries import DistroArchSeries

    store = get_store(MASTER_FLAVOR)
    family = getUtility(IProcessorFamilySet).getByName(architecture_name)
    archseries = DistroArchSeries(
        distroseries=distroseries, processorfamily=family,
        owner=distroseries.owner, official=True,
        architecturetag=architecture_name)
    store.add(archseries)


def create_sections(distroseries):
    """Set up some sections for `distroseries`."""
    section_names = (
        'admin', 'cli-mono', 'comm', 'database', 'devel', 'debug', 'doc',
        'editors', 'electronics', 'embedded', 'fonts', 'games', 'gnome',
        'graphics', 'gnu-r', 'gnustep', 'hamradio', 'haskell', 'httpd',
        'interpreters', 'java', 'kde', 'kernel', 'libs', 'libdevel', 'lisp',
        'localization', 'mail', 'math', 'misc', 'net', 'news', 'ocaml',
        'oldlibs', 'otherosfs', 'perl', 'php', 'python', 'ruby', 'science',
        'shells', 'sound', 'tex', 'text', 'utils', 'vcs', 'video', 'web',
        'x11', 'xfce', 'zope')
    store = Store.of(distroseries)
    for section_name in section_names:
        section = getUtility(ISectionSet).ensure(section_name)
        if section not in distroseries.sections:
            store.add(
                SectionSelection(distroseries=distroseries, section=section))


def create_components(distroseries, uploader):
    """Set up some components for `distroseries`."""
    component_names = ('main', 'restricted', 'universe', 'multiverse')
    store = Store.of(distroseries)
    main_archive = distroseries.distribution.main_archive
    for component_name in component_names:
        component = getUtility(IComponentSet).ensure(component_name)
        if component not in distroseries.components:
            store.add(
                ComponentSelection(
                    distroseries=distroseries, component=component))
        main_archive.newComponentUploader(uploader, component)
        main_archive.newQueueAdmin(uploader, component)


def create_series(parent, full_name, version, status):
    """Set up a `DistroSeries`."""
    distribution = parent.distribution
    owner = parent.owner
    name = full_name.split()[0].lower()
    title = "The " + full_name
    displayname = full_name.split()[0]
    new_series = distribution.newSeries(name=name, title=title,
        displayname=displayname, summary='Ubuntu %s is good.' % version,
        description='%s is awesome.' % version, version=version,
        parent_series=parent, owner=owner)
    new_series.status = status
    notify(ObjectCreatedEvent(new_series))

    # This bit copied from scripts/ftpmaster-tools/initialise-from-parent.py.
    assert new_series.architectures.count() == 0, (
        "Cannot copy distroarchseries from parent; this series already has "
        "distroarchseries.")

    store = Store.of(parent)
    store.execute("""
        INSERT INTO DistroArchSeries
          (distroseries, processorfamily, architecturetag, owner, official)
        SELECT %s, processorfamily, architecturetag, %s, official
        FROM DistroArchSeries WHERE distroseries = %s
        """ % sqlvalues(new_series, owner, parent))

    i386 = new_series.getDistroArchSeries('i386')
    i386.supports_virtualized = True
    new_series.nominatedarchindep = i386

    new_series.initialiseFromParent()
    return new_series


def create_sample_series(original_series, log):
    """Set up sample `DistroSeries`.

    :param original_series: The parent for the first new series to be
        created.  The second new series will have the first as a parent,
        and so on.
    """
    series_descriptions = [
        ('Dapper Drake', SeriesStatus.SUPPORTED, '6.06'),
        ('Edgy Eft', SeriesStatus.OBSOLETE, '6.10'),
        ('Feisty Fawn', SeriesStatus.OBSOLETE, '7.04'),
        ('Gutsy Gibbon', SeriesStatus.OBSOLETE, '7.10'),
        ('Hardy Heron', SeriesStatus.SUPPORTED, '8.04'),
        ('Intrepid Ibex', SeriesStatus.SUPPORTED, '8.10'),
        ('Jaunty Jackalope', SeriesStatus.SUPPORTED, '9.04'),
        ('Karmic Koala', SeriesStatus.SUPPORTED, '9.10'),
        ('Lucid Lynx', SeriesStatus.CURRENT, '10.04'),
        ('Maverick Meerkat', SeriesStatus.DEVELOPMENT, '10.10'),
        ]

    parent = original_series
    for full_name, status, version in series_descriptions:
        log.info('Creating %s...' % full_name)
        parent = create_series(parent, full_name, version, status)


def clean_up(distribution, log):
    # First we eliminate all active publishings in the Ubuntu main archives.
    # None of the librarian files exist, so it kills the publisher.

    # Could use IPublishingSet.requestDeletion() on the published sources to
    # get rid of the binaries too, but I don't trust that there aren't
    # published binaries without corresponding sources.

    log.info("Deleting all items in official archives...")
    retire_distro_archives(distribution, get_person_set().getByName('name16'))

    # Disable publishing of all PPAs, as they probably have broken
    # publishings too.
    log.info("Disabling all PPAs...")
    retire_ppas(distribution)

    retire_series(distribution)


def set_source_package_format(distroseries):
    """Register a series' source package format selection."""
    utility = getUtility(ISourcePackageFormatSelectionSet)
    format = SourcePackageFormat.FORMAT_1_0
    if utility.getBySeriesAndFormat(distroseries, format) is None:
        utility.add(distroseries, format)


def populate(distribution, parent_series_name, uploader_name, options, log):
    """Set up sample data on `distribution`."""
    parent_series = distribution.getSeries(parent_series_name)

    # Set up lucilleconfig on all series.  The sample data lacks this.
    log.info("Setting lucilleconfig...")
    set_lucille_config(distribution)

    log.info("Configuring sections...")
    create_sections(parent_series)
    add_architecture(parent_series, 'amd64')

    log.info("Configuring components and permissions...")
    uploader = get_person_set().getByName(uploader_name)
    create_components(parent_series, uploader)

    set_source_package_format(parent_series)

    create_sample_series(parent_series, log)


def sign_code_of_conduct(person, log):
    """Sign Ubuntu Code of Conduct for `person`, if necessary."""
    if person.is_ubuntu_coc_signer:
        # Already signed.
        return

    log.info("Signing Ubuntu code of conduct.")
    signedcocset = getUtility(ISignedCodeOfConductSet)
    person_id = person.id
    if signedcocset.searchByUser(person_id).count() == 0:
        fake_gpg_key = LaunchpadObjectFactory().makeGPGKey(person)
        Store.of(person).add(SignedCodeOfConduct(
            owner=person, signingkey=fake_gpg_key,
            signedcode="Normally a signed CoC would go here.", active=True))


def create_ppa_user(username, options, approver, log):
    """Create new user, with password "test," and sign code of conduct."""
    person = get_person_set().getByName(username)
    if person is None:
        have_email = (options.email != default_email)
        command_line = [
            'utilities/make-lp-user',
            username,
            'ubuntu-team'
            ]
        if have_email:
            command_line += ['--email', options.email]

        pipe = subprocess.Popen(command_line, stderr=subprocess.PIPE)
        stdout, stderr = pipe.communicate()
        if stderr != '':
            print stderr
        if pipe.returncode != 0:
            sys.exit(2)

    transaction.commit()

    person = getUtility(IPersonSet).getByName(username)
    sign_code_of_conduct(person, log)

    return person


def create_ppa(distribution, person, name):
    """Create a PPA for `person`."""
    ppa = LaunchpadObjectFactory().makeArchive(
        distribution=distribution, owner=person, name=name, virtualized=False,
        description="Automatically created test PPA.")

    series_name = distribution.currentseries.name
    ppa.external_dependencies = (
        "deb http://archive.ubuntu.com/ubuntu %s "
        "main restricted universe multiverse\n") % series_name


def main(argv):
    options, args, log = parse_args(argv[1:])

    execute_zcml_for_scripts()
    txn = initZopeless(dbuser='launchpad')

    check_preconditions(options.force)

    ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
    clean_up(ubuntu, log)

    # Use Hoary as the root, as Breezy and Grumpy are broken.
    populate(ubuntu, 'hoary', 'ubuntu-team', options, log)

    admin = get_person_set().getByName('name16')
    person = create_ppa_user(user_name, options, admin, log)

    create_ppa(ubuntu, person, 'test-ppa')

    txn.commit()
    log.info("Done.")

    print dedent("""
        Now start your local Launchpad with "make run_codehosting" and log
        into https://launchpad.dev/ as "%(email)s" with "test" as the
        password.
        Your user name will be %(user_name)s."""
        % {
            'email': options.email,
            'user_name': user_name,
            })


if __name__ == "__main__":
    main(sys.argv)