~launchpad-pqm/launchpad/devel

# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the

# GNU Affero General Public License version 3 (see the file LICENSE).

# pylint: disable-msg=E0611,W0212

__metaclass__ = type

__all__ = [

    'BinaryPackageBuild',

    'BinaryPackageBuildSet',

    ]

import datetime

import logging

import operator

import apt_pkg

from sqlobject import SQLObjectNotFound

from storm.expr import (

    Desc,

    Join,

    LeftJoin,

    SQL,

    )

from storm.locals import (

    Int,

    Reference,

    )

from storm.store import (

    EmptyResultSet,

    Store,

    )

from storm.zope import IResultSet

from zope.component import getUtility

from zope.interface import implements

from lp.app.browser.tales import DurationFormatterAPI

from lp.app.errors import NotFoundError

from lp.app.interfaces.launchpad import ILaunchpadCelebrities

from lp.archivepublisher.utils import get_ppa_reference

from lp.buildmaster.enums import (

    BuildFarmJobType,

    BuildStatus,

    )

from lp.buildmaster.interfaces.packagebuild import IPackageBuildSource

from lp.buildmaster.model.builder import Builder

from lp.buildmaster.model.buildfarmjob import BuildFarmJob

from lp.buildmaster.model.buildqueue import BuildQueue

from lp.buildmaster.model.packagebuild import (

    PackageBuild,

    PackageBuildDerived,

    )

from lp.services.config import config

from lp.services.database.bulk import load_related

from lp.services.database.decoratedresultset import DecoratedResultSet

from lp.services.database.lpstorm import (

    IMasterObject,

    ISlaveStore,

    IStore,

    )

from lp.services.database.sqlbase import (

    quote_like,

    SQLBase,

    sqlvalues,

    )

from lp.services.job.model.job import Job

from lp.services.librarian.browser import ProxiedLibraryFileAlias

from lp.services.librarian.model import (

    LibraryFileAlias,

    LibraryFileContent,

    )

from lp.services.mail.helpers import (

    get_contact_email_addresses,

    get_email_template,

    )

from lp.services.mail.sendmail import (

    format_address,

    simple_sendmail,

    )

from lp.services.webapp import canonical_url

from lp.services.webapp.interfaces import (

    DEFAULT_FLAVOR,

    IStoreSelector,

    MAIN_STORE,

    )

from lp.soyuz.enums import ArchivePurpose

from lp.soyuz.interfaces.binarypackagebuild import (

    BuildSetStatus,

    CannotBeRescored,

    IBinaryPackageBuild,

    IBinaryPackageBuildSet,

    UnparsableDependencies,

    )

from lp.soyuz.interfaces.publishing import active_publishing_status

from lp.soyuz.model.binarypackagename import BinaryPackageName

from lp.soyuz.model.binarypackagerelease import BinaryPackageRelease

from lp.soyuz.model.buildpackagejob import BuildPackageJob

from lp.soyuz.model.files import BinaryPackageFile

from lp.soyuz.model.publishing import SourcePackagePublishingHistory

from lp.soyuz.model.queue import (

    PackageUpload,

    PackageUploadBuild,

    )

class BinaryPackageBuild(PackageBuildDerived, SQLBase):

    implements(IBinaryPackageBuild)

    _table = 'BinaryPackageBuild'

    _defaultOrder = 'id'

    build_farm_job_type = BuildFarmJobType.PACKAGEBUILD

    package_build_id = Int(name='package_build', allow_none=False)

    package_build = Reference(package_build_id, 'PackageBuild.id')

    distro_arch_series_id = Int(name='distro_arch_series', allow_none=False)

    distro_arch_series = Reference(

        distro_arch_series_id, 'DistroArchSeries.id')

    source_package_release_id = Int(

        name='source_package_release', allow_none=False)

    source_package_release = Reference(

        source_package_release_id, 'SourcePackageRelease.id')

    @property

    def buildqueue_record(self):

        """See `IBuild`."""

        store = Store.of(self)

        results = store.find(

            BuildQueue,

            BuildPackageJob.job == BuildQueue.jobID,

            BuildPackageJob.build == self.id)

        return results.one()

    def _getLatestPublication(self):

        store = Store.of(self)

        results = store.find(

            SourcePackagePublishingHistory,

            SourcePackagePublishingHistory.archive == self.archive,

            SourcePackagePublishingHistory.distroseries == self.distro_series,

            SourcePackagePublishingHistory.sourcepackagerelease ==

                self.source_package_release)

        return results.order_by(

            Desc(SourcePackagePublishingHistory.id)).first()

    @property

    def current_component(self):

        """See `IBuild`."""

        latest_publication = self._getLatestPublication()

        # Production has some buggy builds without source publications.

        # They seem to have been created by early versions of gina and

        # the readding of hppa.

        if latest_publication is not None:

            return latest_publication.component

    @property

    def current_source_publication(self):

        """See `IBuild`."""

        latest_publication = self._getLatestPublication()

        if (latest_publication is not None and

            latest_publication.status in active_publishing_status):

            return latest_publication

        return None

    @property

    def upload_changesfile(self):

        """See `IBuild`"""

        package_upload = self.package_upload

        if package_upload is None:

            return None

        return package_upload.changesfile

    @property

    def changesfile_url(self):

        """See `IBinaryPackageBuild`."""

        changesfile = self.upload_changesfile

        if changesfile is None:

            return None

        return ProxiedLibraryFileAlias(changesfile, self).http_url

    @property

    def package_upload(self):

        """See `IBuild`."""

        store = Store.of(self)

        # The join on 'changesfile' is not only used only for

        # pre-fetching the corresponding library file, so callsites

        # don't have to issue an extra query. It is also important

        # for excluding delayed-copies, because they might match

        # the publication context but will not contain as changesfile.

        origin = [

            PackageUploadBuild,

            Join(PackageUpload,

                 PackageUploadBuild.packageuploadID == PackageUpload.id),

            Join(LibraryFileAlias,

                 LibraryFileAlias.id == PackageUpload.changesfileID),

            Join(LibraryFileContent,

                 LibraryFileContent.id == LibraryFileAlias.contentID),

            ]

        results = store.using(*origin).find(

            (PackageUpload, LibraryFileAlias, LibraryFileContent),

            PackageUploadBuild.build == self,

            PackageUpload.archive == self.archive,

            PackageUpload.distroseries == self.distro_series)

        # Return the unique `PackageUpload` record that corresponds to the

        # upload of the result of this `Build`, load the `LibraryFileAlias`

        # and the `LibraryFileContent` in cache because it's most likely

        # they will be needed.

        return DecoratedResultSet(results, operator.itemgetter(0)).one()

    @property

    def distro_series(self):

        """See `IBuild`"""

        return self.distro_arch_series.distroseries

    @property

    def distribution(self):

        """See `IBuild`"""

        return self.distro_series.distribution

    @property

    def is_virtualized(self):

        """See `IBuild`"""

        return self.archive.require_virtualized

    @property

    def title(self):

        """See `IBuild`"""

        return '%s build of %s %s in %s %s %s' % (

            self.distro_arch_series.architecturetag,

            self.source_package_release.name,

            self.source_package_release.version,

            self.distribution.name, self.distro_series.name, self.pocket.name)

    @property

    def was_built(self):

        """See `IBuild`"""

        return self.status not in [BuildStatus.NEEDSBUILD,

                                   BuildStatus.BUILDING,

                                   BuildStatus.UPLOADING,

                                   BuildStatus.SUPERSEDED]

    @property

    def arch_tag(self):

        """See `IBuild`."""

        return self.distro_arch_series.architecturetag

    @property

    def log_url(self):

        """See `IPackageBuild`.

        Overridden here for the case of builds for distro archives,

        currently only supported for binary package builds.

        """

        if self.log is None:

            return None

        return ProxiedLibraryFileAlias(self.log, self).http_url

    @property

    def upload_log_url(self):

        """See `IPackageBuild`.

        Overridden here for the case of builds for distro archives,

        currently only supported for binary package builds.

        """

        if self.upload_log is None:

            return None

        return ProxiedLibraryFileAlias(self.upload_log, self).http_url

    @property

    def distributionsourcepackagerelease(self):

        """See `IBuild`."""

        from lp.soyuz.model.distributionsourcepackagerelease \

             import (

            DistributionSourcePackageRelease)

        return DistributionSourcePackageRelease(

            distribution=self.distribution,

            sourcepackagerelease=self.source_package_release)

    def getBinaryPackageNamesForDisplay(self):

        """See `IBuildView`."""

        store = Store.of(self)

        result = store.find(

            (BinaryPackageRelease, BinaryPackageName),

            BinaryPackageRelease.build == self,

            BinaryPackageRelease.binarypackagename == BinaryPackageName.id,

            BinaryPackageName.id == BinaryPackageRelease.binarypackagenameID)

        return result.order_by(

            [BinaryPackageName.name, BinaryPackageRelease.id])

    def getBinaryFilesForDisplay(self):

        """See `IBuildView`."""

        store = Store.of(self)

        result = store.find(

            (BinaryPackageRelease, BinaryPackageFile, LibraryFileAlias,

             LibraryFileContent),

            BinaryPackageRelease.build == self,

            BinaryPackageRelease.id ==

                BinaryPackageFile.binarypackagereleaseID,

            LibraryFileAlias.id == BinaryPackageFile.libraryfileID,

            LibraryFileContent.id == LibraryFileAlias.contentID)

        return result.order_by(

            [LibraryFileAlias.filename, BinaryPackageRelease.id]).config(

            distinct=True)

    @property

    def binarypackages(self):

        """See `IBuild`."""

        return BinaryPackageRelease.select("""

            BinaryPackageRelease.build = %s AND

            BinaryPackageRelease.binarypackagename = BinaryPackageName.id

            """ % sqlvalues(self),

            clauseTables=["BinaryPackageName"],

            orderBy=["BinaryPackageName.name", "BinaryPackageRelease.id"],

            prejoins=["binarypackagename", "component", "section"])

    @property

    def distroarchseriesbinarypackages(self):

        """See `IBuild`."""

        # Avoid circular import by importing locally.

        from lp.soyuz.model.distroarchseriesbinarypackagerelease import (

            DistroArchSeriesBinaryPackageRelease)

        return [DistroArchSeriesBinaryPackageRelease(

            self.distro_arch_series, bp)

            for bp in self.binarypackages]

    @property

    def can_be_retried(self):

        """See `IBuild`."""

        # First check that the slave scanner would pick up the build record

        # if we reset it.  PPA and Partner builds are always ok.

        if (self.archive.purpose == ArchivePurpose.PRIMARY and

            not self.distro_series.canUploadToPocket(self.pocket)):

            # The slave scanner would not pick this up, so it cannot be

            # re-tried.

            return False

        failed_statuses = [

            BuildStatus.FAILEDTOBUILD,

            BuildStatus.MANUALDEPWAIT,

            BuildStatus.CHROOTWAIT,

            BuildStatus.FAILEDTOUPLOAD,

            ]

        # If the build is currently in any of the failed states,

        # it may be retried.

        return self.status in failed_statuses

    @property

    def can_be_rescored(self):

        """See `IBuild`."""

        return self.status is BuildStatus.NEEDSBUILD

    @property

    def can_be_cancelled(self):

        """See `IBuild`."""

        if not self.buildqueue_record:

            return False

        if self.buildqueue_record.virtualized is False:

            return False

        cancellable_statuses = [

            BuildStatus.BUILDING,

            BuildStatus.NEEDSBUILD,

            ]

        return self.status in cancellable_statuses

    def retry(self):

        """See `IBuild`."""

        assert self.can_be_retried, "Build %s cannot be retried" % self.id

        self.status = BuildStatus.NEEDSBUILD

        self.date_finished = None

        self.date_started = None

        self.builder = None

        self.log = None

        self.upload_log = None

        self.dependencies = None

        self.queueBuild()

    def rescore(self, score):

        """See `IBuild`."""

        if not self.can_be_rescored:

            raise CannotBeRescored("Build cannot be rescored.")

        self.buildqueue_record.manualScore(score)

    @property

    def api_score(self):

        """See `IBinaryPackageBuild`."""

        # Score of the related buildqueue record (if any)

        if self.buildqueue_record is None:

            return None

        else:

            return self.buildqueue_record.lastscore

    def cancel(self):

        """See `IBinaryPackageBuild`."""

        if not self.can_be_cancelled:

            return

        # If the build is currently building we need to tell the

        # buildd-manager to terminate it.

        if self.status == BuildStatus.BUILDING:

            self.status = BuildStatus.CANCELLING

            return

        # Otherwise we can cancel it here.

        self.buildqueue_record.cancel()

    def makeJob(self):

        """See `IBuildFarmJob`."""

        store = Store.of(self)

        job = Job()

        store.add(job)

        specific_job = BuildPackageJob(build=self, job=job)

        store.add(specific_job)

        return specific_job

    def _parseDependencyToken(self, token):

        """Parse the given token.

        Raises AssertionError if the given token couldn't be parsed.

        Return a triple containing the corresponding (name, version,

        relation) for the given dependency token.

        """

        # XXX cprov 2006-02-27: it may not work for and'd and or'd syntax.

        try:

            name, version, relation = token[0]

        except ValueError:

            raise AssertionError(

                "APT is not dealing correctly with a dependency token "

                "'%r' from %s (%s) with the following dependencies: %s\n"

                "It is expected to be a tuple containing only another "

                "tuple with 3 elements  (name, version, relation)."

                % (token, self.title, self.id, self.dependencies))

        # Map relations to the canonical form used in control files.

        if relation == '<':

            relation = '<<'

        elif relation == '>':

            relation = '>>'

        return (name, version, relation)

    def _checkDependencyVersion(self, available, required, relation):

        """Return True if the available version satisfies the context."""

        # This dict maps the package version relationship syntax in lambda

        # functions which returns boolean according the results of

        # apt_pkg.version_compare function (see the order above).

        # For further information about pkg relationship syntax see:

        #

        # http://www.debian.org/doc/debian-policy/ch-relationships.html

        #

        version_relation_map = {

            # any version is acceptable if no relationship is given

            '': lambda x: True,

            # strictly later

            '>>': lambda x: x == 1,

            # later or equal

            '>=': lambda x: x >= 0,

            # strictly equal

            '=': lambda x: x == 0,

            # earlier or equal

            '<=': lambda x: x <= 0,

            # strictly earlier

            '<<': lambda x: x == -1,

            }

        # Use apt_pkg function to compare versions

        # it behaves similar to cmp, i.e. returns negative

        # if first < second, zero if first == second and

        # positive if first > second.

        dep_result = apt_pkg.version_compare(available, required)

        return version_relation_map[relation](dep_result)

    def _isDependencySatisfied(self, token):

        """Check if the given dependency token is satisfied.

        Check if the dependency exists and that its version constraint is

        satisfied.

        """

        name, version, relation = self._parseDependencyToken(token)

        # There may be several published versions in the available

        # archives and pockets. If any one of them satisifies our

        # constraints, the dependency is satisfied.

        dep_candidates = self.archive.findDepCandidates(

            self.distro_arch_series, self.pocket, self.current_component,

            self.source_package_release.sourcepackagename.name, name)

        for dep_candidate in dep_candidates:

            if self._checkDependencyVersion(

                dep_candidate.binarypackagerelease.version, version,

                relation):

                return True

        return False

    def _toAptFormat(self, token):

        """Rebuild dependencies line in apt format."""

        name, version, relation = self._parseDependencyToken(token)

        if relation and version:

            return '%s (%s %s)' % (name, relation, version)

        return '%s' % name

    def updateDependencies(self):

        """See `IBuild`."""

        # apt_pkg requires init_system to get version_compare working

        # properly.

        apt_pkg.init_system()

        # Check package build dependencies using apt_pkg

        try:

            parsed_deps = apt_pkg.parse_depends(self.dependencies)

        except (ValueError, TypeError):

            raise UnparsableDependencies(

                "Build dependencies for %s (%s) could not be parsed: '%s'\n"

                "It indicates that something is wrong in buildd-slaves."

                % (self.title, self.id, self.dependencies))

        remaining_deps = [

            self._toAptFormat(token) for token in parsed_deps

            if not self._isDependencySatisfied(token)]

        # Update dependencies line

        self.dependencies = u", ".join(remaining_deps)

    def __getitem__(self, name):

        return self.getBinaryPackageRelease(name)

    def getBinaryPackageRelease(self, name):

        """See `IBuild`."""

        for binpkg in self.binarypackages:

            if binpkg.name == name:

                return binpkg

        raise NotFoundError('No binary package "%s" in build' % name)

    def createBinaryPackageRelease(

        self, binarypackagename, version, summary, description,

        binpackageformat, component, section, priority, installedsize,

        architecturespecific, shlibdeps=None, depends=None, recommends=None,

        suggests=None, conflicts=None, replaces=None, provides=None,

        pre_depends=None, enhances=None, breaks=None, essential=False,

        debug_package=None, user_defined_fields=None, homepage=None):

        """See IBuild."""

        return BinaryPackageRelease(

            build=self, binarypackagename=binarypackagename, version=version,

            summary=summary, description=description,

            binpackageformat=binpackageformat,

            component=component, section=section, priority=priority,

            shlibdeps=shlibdeps, depends=depends, recommends=recommends,

            suggests=suggests, conflicts=conflicts, replaces=replaces,

            provides=provides, pre_depends=pre_depends, enhances=enhances,

            breaks=breaks, essential=essential, installedsize=installedsize,

            architecturespecific=architecturespecific,

            debug_package=debug_package,

            user_defined_fields=user_defined_fields, homepage=homepage)

    def estimateDuration(self):

        """See `IPackageBuild`."""

        # Always include the primary archive when looking for

        # past build times (just in case that none can be found

        # in a PPA or copy archive).

        archives = [self.archive.id]

        if self.archive.purpose != ArchivePurpose.PRIMARY:

            archives.append(self.distro_arch_series.main_archive.id)

        # Look for all sourcepackagerelease instances that match the name

        # and get the (successfully built) build records for this

        # package.

        completed_builds = BinaryPackageBuild.select("""

            BinaryPackageBuild.source_package_release =

                SourcePackageRelease.id AND

            BinaryPackageBuild.id != %s AND

            BinaryPackageBuild.distro_arch_series = %s AND

            SourcePackageRelease.sourcepackagename = SourcePackageName.id AND

            SourcePackageName.name = %s AND

            BinaryPackageBuild.package_build = PackageBuild.id AND

            PackageBuild.archive IN %s AND

            PackageBuild.build_farm_job = BuildFarmJob.id AND

            BuildFarmJob.date_finished IS NOT NULL AND

            BuildFarmJob.status = %s

            """ % sqlvalues(self, self.distro_arch_series,

                            self.source_package_release.name, archives,

                            BuildStatus.FULLYBUILT),

            orderBy=['-BuildFarmJob.date_finished', '-id'],

            clauseTables=['PackageBuild', 'BuildFarmJob', 'SourcePackageName',

                          'SourcePackageRelease'])

        estimated_duration = None

        if bool(completed_builds):

            # Historic build data exists, use the most recent value -

            # assuming it has valid data.

            most_recent_build = completed_builds[0]

            estimated_duration = most_recent_build.duration

        if estimated_duration is None:

            # Estimate the build duration based on package size if no

            # historic build data exists.

            # Get the package size in KB.

            package_size = self.source_package_release.getPackageSize()

            if package_size > 0:

                # Analysis of previous build data shows that a build rate

                # of 6 KB/second is realistic. Furthermore we have to add

                # another minute for generic build overhead.

                estimate = int(package_size / 6.0 / 60 + 1)

            else:

                # No historic build times and no package size available,

                # assume a build time of 5 minutes.

                estimate = 5

            estimated_duration = datetime.timedelta(minutes=estimate)

        return estimated_duration

    def verifySuccessfulUpload(self):

        return bool(self.binarypackages)

    def notify(self, extra_info=None):

        """See `IPackageBuild`.

        If config.buildmaster.build_notification is disable, simply

        return.

        If config.builddmaster.notify_owner is enabled and SPR.creator

        has preferredemail it will send an email to the creator, Bcc:

        to the config.builddmaster.default_recipient. If one of the

        conditions was not satisfied, no preferredemail found (autosync

        or untouched packages from debian) or config options disabled,

        it will only send email to the specified default recipient.

        This notification will contain useful information about

        the record in question (all states are supported), see

        doc/build-notification.txt for further information.

        """

        if not config.builddmaster.send_build_notification:

            return

        if self.status == BuildStatus.FULLYBUILT:

            return

        recipients = set()

        fromaddress = format_address(

            config.builddmaster.default_sender_name,

            config.builddmaster.default_sender_address)

        extra_headers = {

            'X-Launchpad-Build-State': self.status.name,

            'X-Launchpad-Build-Component': self.current_component.name,

            'X-Launchpad-Build-Arch':

                self.distro_arch_series.architecturetag,

            }

        # XXX cprov 2006-10-27: Temporary extra debug info about the

        # SPR.creator in context, to be used during the service quarantine,

        # notify_owner will be disabled to avoid *spamming* Debian people.

        creator = self.source_package_release.creator

        extra_headers['X-Creator-Recipient'] = ",".join(

            get_contact_email_addresses(creator))

        # Currently there are 7038 SPR published in edgy which the creators

        # have no preferredemail. They are the autosync ones (creator = katie,

        # 3583 packages) and the untouched sources since we have migrated from

        # DAK (the rest). We should not spam Debian maintainers.

        # Please note that both the package creator and the package uploader

        # will be notified of failures if:

        #     * the 'notify_owner' flag is set

        #     * the package build (failure) occurred in the original

        #       archive.

        package_was_not_copied = (

            self.archive == self.source_package_release.upload_archive)

        if package_was_not_copied and config.builddmaster.notify_owner:

            if (self.archive.is_ppa and creator.inTeam(self.archive.owner)

                or

                not self.archive.is_ppa):

                # If this is a PPA, the package creator should only be

                # notified if they are the PPA owner or in the PPA team.