~launchpad-pqm/launchpad/devel

# Copyright 2009 Canonical Ltd.  This software is licensed under the

# GNU Affero General Public License version 3 (see the file LICENSE).

"""Tools for maintaining the Launchpad source code."""

__metaclass__ = type

__all__ = [

    'interpret_config',

    'parse_config_file',

    'plan_update',

    ]

import errno

import json

import optparse

import os

import shutil

import sys

from bzrlib import ui

from bzrlib.branch import Branch

from bzrlib.errors import (

    BzrError,

    IncompatibleRepositories,

    NotBranchError,

    )

from bzrlib.plugin import load_plugins

from bzrlib.revisionspec import RevisionSpec

from bzrlib.trace import (

    enable_default_logging,

    report_exception,

    )

from bzrlib.upgrade import upgrade

from bzrlib.workingtree import WorkingTree

from devscripts import get_launchpad_root

def parse_config_file(file_handle):

    """Parse the source code config file 'file_handle'.

    :param file_handle: A file-like object containing sourcecode

        configuration.

    :return: A sequence of lines of either '[key, value]' or

        '[key, value, optional]'.

    """

    for line in file_handle:

        if line.startswith('#'):

            continue

        yield line.split()

def interpret_config_entry(entry):

    """Interpret a single parsed line from the config file."""

    branch_name = entry[0]

    components = entry[1].split(';revno=')

    branch_url = components[0]

    if len(components) == 1:

        revision = None

    else:

        assert len(components) == 2, 'Bad branch URL: ' + entry[1]

        revision = components[1] or None

    if len(entry) > 2:

        assert len(entry) == 3 and entry[2].lower() == 'optional', (

            'Bad configuration line: should be space delimited values of '

            'sourcecode directory name, branch URL [, "optional"]\n' +

            ' '.join(entry))

        optional = True

    else:

        optional = False

    return branch_name, branch_url, revision, optional

def load_cache(cache_filename):

    try:

        cache_file = open(cache_filename, 'rb')

    except IOError as e:

        if e.errno == errno.ENOENT:

            return {}

        else:

            raise

    with cache_file:

        return json.load(cache_file)

def interpret_config(config_entries, public_only):

    """Interpret a configuration stream, as parsed by 'parse_config_file'.

    :param configuration: A sequence of parsed configuration entries.

    :param public_only: If true, ignore private/optional branches.

    :return: A dict mapping the names of the sourcecode dependencies to a

        2-tuple of their branches and whether or not they are optional.

    """

    config = {}

    for entry in config_entries:

        branch_name, branch_url, revision, optional = interpret_config_entry(

            entry)

        if not optional or not public_only:

            config[branch_name] = (branch_url, revision, optional)

    return config

def _subset_dict(d, keys):

    """Return a dict that's a subset of 'd', based on the keys in 'keys'."""

    return dict((key, d[key]) for key in keys)

def plan_update(existing_branches, configuration):

    """Plan the update to existing branches based on 'configuration'.

    :param existing_branches: A sequence of branches that already exist.

    :param configuration: A dictionary of sourcecode configuration, such as is

        returned by `interpret_config`.

    :return: (new_branches, update_branches, removed_branches), where

        'new_branches' are the branches in the configuration that don't exist

        yet, 'update_branches' are the branches in the configuration that do

        exist, and 'removed_branches' are the branches that exist locally, but

        not in the configuration. 'new_branches' and 'update_branches' are

        dicts of the same form as 'configuration', 'removed_branches' is a

        set of the same form as 'existing_branches'.

    """

    existing_branches = set(existing_branches)

    config_branches = set(configuration.keys())

    new_branches = config_branches - existing_branches

    removed_branches = existing_branches - config_branches

    update_branches = config_branches.intersection(existing_branches)

    return (

        _subset_dict(configuration, new_branches),

        _subset_dict(configuration, update_branches),

        removed_branches)

def find_branches(directory):

    """List the directory names in 'directory' that are branches."""

    branches = []

    for name in os.listdir(directory):

        if name in ('.', '..'):

            continue

        try:

            Branch.open(os.path.join(directory, name))

            branches.append(name)

        except NotBranchError:

            pass

    return branches

def get_revision_id(revision, from_branch, tip=False):

    """Return revision id for a revision number and a branch.

    If the revision is empty, the revision_id will be None.

    If ``tip`` is True, the revision value will be ignored.

    """

    if not tip and revision:

        spec = RevisionSpec.from_string(revision)

        return spec.as_revision_id(from_branch)

    # else return None

def _format_revision_name(revision, tip=False):

    """Formatting helper to return human-readable identifier for revision.

    If ``tip`` is True, the revision value will be ignored.

    """

    if not tip and revision:

        return 'revision %s' % (revision,)

    else:

        return 'tip'

def get_branches(sourcecode_directory, new_branches,

                 possible_transports=None, tip=False, quiet=False):

    """Get the new branches into sourcecode."""

    for project, (branch_url, revision, optional) in new_branches.iteritems():

        destination = os.path.join(sourcecode_directory, project)

        try:

            remote_branch = Branch.open(

                branch_url, possible_transports=possible_transports)

        except BzrError:

            if optional:

                report_exception(sys.exc_info(), sys.stderr)

                continue

            else:

                raise

        possible_transports.append(

            remote_branch.bzrdir.root_transport)

        if not quiet:

            print 'Getting %s from %s at %s' % (

                    project, branch_url, _format_revision_name(revision, tip))

        # If the 'optional' flag is set, then it's a branch that shares

        # history with Launchpad, so we should share repositories. Otherwise,

        # we should avoid sharing repositories to avoid format

        # incompatibilities.

        force_new_repo = not optional

        revision_id = get_revision_id(revision, remote_branch, tip)

        remote_branch.bzrdir.sprout(

            destination, revision_id=revision_id, create_tree_if_local=True,

            source_branch=remote_branch, force_new_repo=force_new_repo,

            possible_transports=possible_transports)

def find_stale(updated, cache, sourcecode_directory, quiet):

    """Find branches whose revision info doesn't match the cache."""

    new_updated = dict(updated)

    for project, (branch_url, revision, optional) in updated.iteritems():

        cache_revision_info = cache.get(project)

        if cache_revision_info is None:

            continue

        if cache_revision_info[0] != int(revision):

            continue

        destination = os.path.join(sourcecode_directory, project)

        try:

            branch = Branch.open(destination)

        except BzrError:

            continue

        if list(branch.last_revision_info()) != cache_revision_info:

            continue

        if not quiet:

            print '%s is already up to date.' % project

        del new_updated[project]

    return new_updated

def update_cache(cache, cache_filename, changed, sourcecode_directory, quiet):

    """Update the cache with the changed branches."""

    old_cache = dict(cache)

    for project, (branch_url, revision, optional) in changed.iteritems():

        destination = os.path.join(sourcecode_directory, project)

        branch = Branch.open(destination)

        cache[project] = list(branch.last_revision_info())

    if cache == old_cache:

        return

    with open(cache_filename, 'wb') as cache_file:

        json.dump(cache, cache_file, indent=4, sort_keys=True)

    if not quiet:

        print 'Cache updated.  Please commit "%s".' % cache_filename

def update_branches(sourcecode_directory, update_branches,

                    possible_transports=None, tip=False, quiet=False):

    """Update the existing branches in sourcecode."""

    if possible_transports is None:

        possible_transports = []

    # XXX: JonathanLange 2009-11-09: Rather than updating one branch after

    # another, we could instead try to get them in parallel.

    for project, (branch_url, revision, optional) in (

        update_branches.iteritems()):

        # Update project from branch_url.

        destination = os.path.join(sourcecode_directory, project)

        if not quiet:

            print 'Updating %s to %s' % (

                    project, _format_revision_name(revision, tip))

        local_tree = WorkingTree.open(destination)

        try:

            remote_branch = Branch.open(

                branch_url, possible_transports=possible_transports)

        except BzrError:

            if optional:

                report_exception(sys.exc_info(), sys.stderr)

                continue

            else:

                raise

        possible_transports.append(

            remote_branch.bzrdir.root_transport)

        revision_id = get_revision_id(revision, remote_branch, tip)

        try:

            result = local_tree.pull(

                remote_branch, stop_revision=revision_id, overwrite=True,

                possible_transports=possible_transports)

        except IncompatibleRepositories:

            # XXX JRV 20100407: Ideally remote_branch.bzrdir._format

            # should be passed into upgrade() to ensure the format is the same

            # locally and remotely. Unfortunately smart server branches

            # have their _format set to RemoteFormat rather than an actual

            # format instance.

            upgrade(destination)

            # Upgraded, repoen working tree

            local_tree = WorkingTree.open(destination)

            result = local_tree.pull(

                remote_branch, stop_revision=revision_id, overwrite=True,

                possible_transports=possible_transports)

        if result.old_revid == result.new_revid:

            if not quiet:

                print '  (No change)'

        else:

            if result.old_revno < result.new_revno:

                change = 'Updated'

            else:

                change = 'Reverted'

            if not quiet:

                print '  (%s from %s to %s)' % (

                    change, result.old_revno, result.new_revno)

def remove_branches(sourcecode_directory, removed_branches, quiet=False):

    """Remove sourcecode that's no longer there."""

    for project in removed_branches:

        destination = os.path.join(sourcecode_directory, project)

        if not quiet:

            print 'Removing %s' % project

        try:

            shutil.rmtree(destination)

        except OSError:

            os.unlink(destination)

def update_sourcecode(sourcecode_directory, config_filename, cache_filename,

                      public_only, tip, dry_run, quiet=False):

    """Update the sourcecode."""

    config_file = open(config_filename)

    config = interpret_config(parse_config_file(config_file), public_only)

    config_file.close()

    cache = load_cache(cache_filename)

    branches = find_branches(sourcecode_directory)

    new, updated, removed = plan_update(branches, config)

    possible_transports = []

    if dry_run:

        print 'Branches to fetch:', new.keys()

        print 'Branches to update:', updated.keys()

        print 'Branches to remove:', list(removed)

    else:

        get_branches(

            sourcecode_directory, new, possible_transports, tip, quiet)

        updated = find_stale(updated, cache, sourcecode_directory, quiet)

        update_branches(

            sourcecode_directory, updated, possible_transports, tip, quiet)

        changed = dict(updated)

        changed.update(new)

        update_cache(

            cache, cache_filename, changed, sourcecode_directory, quiet)

        remove_branches(sourcecode_directory, removed, quiet)

# XXX: JonathanLange 2009-09-11: By default, the script will operate on the

# current checkout. Most people only have symlinks to sourcecode in their

# checkouts. This is fine for updating, but breaks for removing (you can't

# shutil.rmtree a symlink) and breaks for adding, since it adds the new branch

# to the checkout, rather than to the shared sourcecode area. Ideally, the

# script would see that the sourcecode directory is full of symlinks and then

# follow these symlinks to find the shared source directory. If the symlinks

# differ from each other (because of developers fiddling with things), we can

# take a survey of all of them, and choose the most popular.

def main(args):

    parser = optparse.OptionParser("usage: %prog [options] [root [conffile]]")

    parser.add_option(

        '--public-only', action='store_true',

        help='Only fetch/update the public sourcecode branches.')

    parser.add_option(

        '--tip', action='store_true',

        help='Ignore revision constraints for all branches and pull tip')

    parser.add_option(

        '--dry-run', action='store_true',

        help='Do nothing, but report what would have been done.')

    parser.add_option(

        '--quiet', action='store_true',

        help="Don't print informational messages.")

    options, args = parser.parse_args(args)

    root = get_launchpad_root()

    if len(args) > 1:

        sourcecode_directory = args[1]

    else:

        sourcecode_directory = os.path.join(root, 'sourcecode')

    if len(args) > 2:

        config_filename = args[2]

    else:

        config_filename = os.path.join(root, 'utilities', 'sourcedeps.conf')

    cache_filename = os.path.join(

        root, 'utilities', 'sourcedeps.cache')

    if len(args) > 3:

        parser.error("Too many arguments.")

    if not options.quiet:

        print 'Sourcecode: %s' % (sourcecode_directory,)

        print 'Config: %s' % (config_filename,)

    enable_default_logging()

    # Tell bzr to use the terminal (if any) to show progress bars

    ui.ui_factory = ui.make_ui_for_terminal(

        sys.stdin, sys.stdout, sys.stderr)

    load_plugins()

    update_sourcecode(

        sourcecode_directory, config_filename, cache_filename,

        options.public_only, options.tip, options.dry_run, options.quiet)

    return 0