~launchpad-pqm/launchpad/devel

# Copyright 2009 Canonical Ltd.  This software is licensed under the

# GNU Affero General Public License version 3 (see the file LICENSE).

"""Opening a new DistroSeries for branch based development.

Intended to be run just after a new distro series has been completed, this

script will create an official package branch in the new series for every one

in the old.  The old branch will become stacked on the new, to avoid a using

too much disk space whilst retaining best performance for the new branch.

"""

__metaclass__ = type

__all__ = [

    'DistroBrancher',

    'switch_branches',

    ]

import os

from bzrlib.branch import Branch

from bzrlib.bzrdir import BzrDir

from bzrlib.errors import (

    NotBranchError,

    NotStacked,

    )

from bzrlib.revision import NULL_REVISION

import transaction

from zope.component import getUtility

from canonical.config import config

from lp.services.database.lpstorm import IMasterStore

from lp.code.enums import (

    BranchLifecycleStatus,

    BranchType,

    )

from lp.code.errors import BranchExists

from lp.code.interfaces.branchcollection import IAllBranches

from lp.code.interfaces.branchnamespace import IBranchNamespaceSet

from lp.code.interfaces.seriessourcepackagebranch import (

    IFindOfficialBranchLinks,

    )

from lp.code.model.branchrevision import BranchRevision

from lp.codehosting.vfs import branch_id_to_path

from lp.registry.interfaces.distribution import IDistributionSet

from lp.registry.interfaces.pocket import PackagePublishingPocket

def switch_branches(prefix, scheme, old_db_branch, new_db_branch):

    """Move bzr data from an old to a new branch, leaving old stacked on new.

    This function is intended to be used just after Ubuntu is released to

    create (at the bzr level) a new trunk branch for a source package for the

    next release of the distribution.  We move the bzr data to the location

    for the new branch and replace the trunk branch for the just released

    version with a stacked branch pointing at the new branch.

    The procedure is to complicated to be carried out atomically, so if this

    function is interrupted things may be a little inconsistent (e.g. there

    might be a branch in the old location, but not stacked on the new location

    yet).  There should be no data loss though.

    :param prefix: The non-branch id dependent part of the physical path to

        the branches on disk.

    :param scheme: The branches should be open-able at a URL of the form

        ``scheme + :/// + unique_name``.

    :param old_db_branch: The branch that currently has the trunk bzr data.

    :param old_db_branch: The new trunk branch.  This should not have any

        presence on disk yet.

    """

    # Move .bzr directory from old to new location, crashing through the

    # abstraction we usually hide our branch locations behind.

    old_underlying_path = os.path.join(

        prefix, branch_id_to_path(old_db_branch.id))

    new_underlying_path = os.path.join(

        prefix, branch_id_to_path(new_db_branch.id))

    os.makedirs(new_underlying_path)

    os.rename(

        os.path.join(old_underlying_path, '.bzr'),

        os.path.join(new_underlying_path, '.bzr'))

    # Create branch at old location -- we use the "clone('null:')" trick to

    # preserve the format.  We have to open at the logical, unique_name-based,

    # location so that it works to set the stacked on url to '/' + a

    # unique_name.

    new_location_bzrdir = BzrDir.open(

        scheme + ':///' + new_db_branch.unique_name)

    old_location_bzrdir = new_location_bzrdir.clone(

        scheme + ':///' + old_db_branch.unique_name, revision_id='null:')

    # Set the stacked on url for old location.

    old_location_branch = old_location_bzrdir.open_branch()

    old_location_branch.set_stacked_on_url('/' + new_db_branch.unique_name)

    # Pull from new location to old -- this won't actually transfer any

    # revisions, just update the last revision pointer.

    old_location_branch.pull(new_location_bzrdir.open_branch())

class DistroBrancher:

    """Open a new distroseries for branch based development.

    `makeNewBranches` will create an official package branch in the new series

    for every one in the old.  `checkNewBranches` will check that a previous

    run of this script completed successfully -- this is only likely to be

    really useful if a script run died halfway through or had to be killed.

    """

    def __init__(self, logger, old_distroseries, new_distroseries):

        """Construct a `DistroBrancher`.

        The old and new distroseries must be from the same distribution, but

        not the same distroseries.

        :param logger: A Logger.  Problems will be logged to this object at

            the WARNING level or higher; progress reports will be logged at

            the DEBUG level.

        :param old_distroseries: The distroseries that will be examined to

            find existing source package branches.

        :param new_distroseries: The distroseries that will have new official

            source branches made for it.

        """

        self.logger = logger

        if old_distroseries.distribution != new_distroseries.distribution:

            raise AssertionError(

                "%s and %s are from different distributions!" %

                (old_distroseries, new_distroseries))

        if old_distroseries == new_distroseries:

            raise AssertionError(

                "New and old distributions must be different!")

        self.old_distroseries = old_distroseries

        self.new_distroseries = new_distroseries

    @classmethod

    def fromNames(cls, logger, distribution_name, old_distroseries_name,

                  new_distroseries_name):

        """Make a `DistroBrancher` from the names of a distro and two series.

        """

        distribution = getUtility(IDistributionSet).getByName(

            distribution_name)

        new_distroseries = distribution.getSeries(new_distroseries_name)

        old_distroseries = distribution.getSeries(old_distroseries_name)

        return cls(logger, old_distroseries, new_distroseries)

    def _existingOfficialBranches(self):

        """Return the collection of official branches in the old distroseries.

        """

        branches = getUtility(IAllBranches)

        distroseries_branches = branches.inDistroSeries(self.old_distroseries)

        return distroseries_branches.officialBranches().getBranches(

            eager_load=False)

    def checkConsistentOfficialPackageBranch(self, db_branch):

        """Check that `db_branch` is a consistent official package branch.

        'Consistent official package branch' means:

         * It's a package branch (rather than a personal or junk branch).

         * It's official for its SourcePackage and no other.

        This function simply returns True or False -- any problems will be

        logged to ``self.logger``.

        :param db_branch: The `IBranch` to check.

        :return: ``True`` if the branch is a consistent official package

            branch, ``False`` otherwise.

        """

        if db_branch.product:

            self.logger.warning(

                "Encountered unexpected product branch %r",

                db_branch.unique_name)

            return False

        if not db_branch.distroseries:

            self.logger.warning(

                "Encountered unexpected personal branch %s",

                db_branch.unique_name)

            return False

        find_branch_links = getUtility(IFindOfficialBranchLinks)

        links = list(find_branch_links.findForBranch(db_branch))

        if len(links) == 0:

            self.logger.warning(

                "%s is not an official branch", db_branch.unique_name)

            return False

        elif len(links) > 1:

            series_text = ', '.join([

                link.sourcepackage.path for link in links])

            self.logger.warning(

                "%s is official for multiple series: %s",

                db_branch.unique_name, series_text)

            return False

        elif links[0].sourcepackage != db_branch.sourcepackage:

            self.logger.warning(

                "%s is the official branch for %s but not its "

                "sourcepackage", db_branch.unique_name,

                links[0].sourcepackage.path)

            return False

        return True

    def makeNewBranches(self):

        """Make official branches in the new distroseries."""

        for db_branch in self._existingOfficialBranches():

            self.logger.debug("Processing %s" % db_branch.unique_name)

            try:

                self.makeOneNewBranch(db_branch)

            except BranchExists:

                pass

    def checkNewBranches(self):

        """Check the branches in the new distroseries are present and correct.

        This function checks that every official package branch in the old

        distroseries has a matching branch in the new distroseries and that

        stacking is set up as we expect on disk.

        Every branch will be checked, even if some fail.

        This function simply returns True or False -- any problems will be

        logged to ``self.logger``.

        :return: ``True`` if every branch passes the check, ``False``

            otherwise.

        """

        ok = True

        for db_branch in self._existingOfficialBranches():

            self.logger.debug("Checking %s" % db_branch.unique_name)

            try:

                if not self.checkOneBranch(db_branch):

                    ok = False

            except:

                ok = False

                self.logger.exception(

                    "Unexpected error checking %s!", db_branch)

        return ok

    def checkOneBranch(self, old_db_branch):

        """Check a branch in the old distroseries has been copied to the new.

        This function checks that `old_db_branch` has a matching branch in the

        new distroseries and that stacking is set up as we expect on disk.

        This function simply returns True or False -- any problems will be

        logged to ``self.logger``.

        :param old_db_branch: The branch to check.

        :return: ``True`` if the branch passes the check, ``False`` otherwise.

        """

        ok = self.checkConsistentOfficialPackageBranch(old_db_branch)

        if not ok:

            return ok

        new_sourcepackage = self.new_distroseries.getSourcePackage(

            old_db_branch.sourcepackagename)

        new_db_branch = new_sourcepackage.getBranch(

            PackagePublishingPocket.RELEASE)

        if new_db_branch is None:

            self.logger.warning(

                "No official branch found for %s",

                new_sourcepackage.path)

            return False

        ok = self.checkConsistentOfficialPackageBranch(new_db_branch)

        if not ok:

            return ok

        # the branch in the new distroseries is unstacked

        new_location = 'lp-internal:///' + new_db_branch.unique_name

        try:

            new_bzr_branch = Branch.open(new_location)

        except NotBranchError:

            self.logger.warning(

                "No bzr branch at new location %s", new_location)

            ok = False

        else:

            try:

                new_stacked_on_url = new_bzr_branch.get_stacked_on_url()

                ok = False

                self.logger.warning(

                    "New branch at %s is stacked on %s, should be "

                    "unstacked.", new_location, new_stacked_on_url)

            except NotStacked:

                pass

        # The branch in the old distroseries is stacked on that in the

        # new.

        old_location = 'lp-internal:///' + old_db_branch.unique_name

        try:

            old_bzr_branch = Branch.open(old_location)

        except NotBranchError:

            self.logger.warning(

                "No bzr branch at old location %s", old_location)

            ok = False

        else:

            try:

                old_stacked_on_url = old_bzr_branch.get_stacked_on_url()

                if old_stacked_on_url != '/' + new_db_branch.unique_name:

                    self.logger.warning(

                        "Old branch at %s is stacked on %s, should be "

                        "stacked on %s", old_location, old_stacked_on_url,

                        '/' + new_db_branch.unique_name)

                    ok = False

            except NotStacked:

                self.logger.warning(

                    "Old branch at %s is not stacked, should be stacked "

                    "on %s", old_location,

                    '/' + new_db_branch.unique_name)

                ok = False

            # The branch in the old distroseries has no revisions in its

            # repository.  We open the repository independently of the

            # branch because the branch's repository has had its fallback

            # location activated. Note that this check might fail if new

            # revisions get pushed to the branch in the old distroseries,

            # which shouldn't happen but isn't totally impossible.

            old_repo = BzrDir.open(old_location).open_repository()

            if len(old_repo.all_revision_ids()) > 0:

                self.logger.warning(

                    "Repository at %s has %s revisions.",

                    old_location, len(old_repo.all_revision_ids()))

                ok = False

            # The branch in the old distroseries has at least some

            # history.  (We can't check that the tips are the same because

            # the branch in the new distroseries might have new revisons).

            if old_bzr_branch.last_revision() == 'null:':

                self.logger.warning(

                    "Old branch at %s has null tip revision.",

                    old_location)

                ok = False

        return ok

    def makeOneNewBranch(self, old_db_branch):

        """Copy a branch to the new distroseries.

        This function makes a new database branch for the same source package

        as old_db_branch but in the new distroseries and then uses

        `switch_branches` to move the underlying bzr branch to the new series

        and replace the old branch with a branch stacked on the new series'

        branch.

        :param old_db_branch: The branch to copy into the new distroseries.

        :raises BranchExists: This will be raised if old_db_branch has already

            been copied to the new distroseries (in the database, at least).

        """

        if not self.checkConsistentOfficialPackageBranch(old_db_branch):

            self.logger.warning("Skipping branch")

            return

        new_namespace = getUtility(IBranchNamespaceSet).get(

            person=old_db_branch.owner, product=None,

            distroseries=self.new_distroseries,

            sourcepackagename=old_db_branch.sourcepackagename)

        new_db_branch = new_namespace.createBranch(

            BranchType.HOSTED, self.new_distroseries.name,

            old_db_branch.registrant)

        new_db_branch.sourcepackage.setBranch(

            PackagePublishingPocket.RELEASE, new_db_branch,

            new_db_branch.owner)

        old_db_branch.lifecycle_status = BranchLifecycleStatus.MATURE

        # switch_branches *moves* the data to locations dependent on the

        # new_branch's id, so if the transaction was rolled back we wouldn't

        # know the branch id and thus wouldn't be able to find the branch data

        # again.  So commit before doing that.

        transaction.commit()

        switch_branches(

            config.codehosting.mirrored_branches_root,

            'lp-internal', old_db_branch, new_db_branch)

        # Directly copy the branch revisions from the old branch to the new

        # branch.

        store = IMasterStore(BranchRevision)

        store.execute(

            """

            INSERT INTO BranchRevision (branch, revision, sequence)

            SELECT %s, BranchRevision.revision, BranchRevision.sequence

            FROM BranchRevision

            WHERE branch = %s

            """ % (new_db_branch.id, old_db_branch.id))

        # Update the scanned details first, that way when hooking into

        # branchChanged, it won't try to create a new scan job.

        tip_revision = old_db_branch.getTipRevision()

        new_db_branch.updateScannedDetails(

            tip_revision, old_db_branch.revision_count)

        tip_revision_id = (

            tip_revision.revision_id if tip_revision is not None else

            NULL_REVISION)

        new_db_branch.branchChanged(

            '', tip_revision_id,

            old_db_branch.control_format,

            old_db_branch.branch_format,

            old_db_branch.repository_format)

        old_db_branch.stacked_on = new_db_branch

        transaction.commit()

        return new_db_branch