~launchpad-pqm/launchpad/devel

# Copyright 2010 Canonical Ltd.  This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).

"""Tests for BinaryPackageBuildBehavior."""

__metaclass__ = type

import gzip
import os
import shutil
import tempfile

from storm.store import Store
from testtools.deferredruntest import AsynchronousDeferredRunTest
import transaction
from twisted.internet import defer
from zope.component import getUtility
from zope.security.proxy import removeSecurityProxy

from canonical.config import config
from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
from canonical.testing.layers import LaunchpadZopelessLayer
from lp.buildmaster.enums import BuildStatus
from lp.buildmaster.tests.mock_slaves import (
    AbortedSlave,
    AbortingSlave,
    BuildingSlave,
    OkSlave,
    WaitingSlave,
    )
from lp.registry.interfaces.pocket import (
    PackagePublishingPocket,
    pocketsuffix,
    )
from lp.registry.interfaces.series import SeriesStatus
from lp.services.job.interfaces.job import JobStatus
from lp.services.log.logger import BufferLogger
from lp.soyuz.adapters.archivedependencies import (
    get_sources_list_for_building,
    )
from lp.soyuz.enums import ArchivePurpose
from lp.testing import TestCaseWithFactory


class TestBinaryBuildPackageBehavior(TestCaseWithFactory):
    """Tests for the BinaryPackageBuildBehavior.

    In particular, these tests are about how the BinaryPackageBuildBehavior
    interacts with the build slave.  We test this by using a test double that
    implements the same interface as `BuilderSlave` but instead of actually
    making XML-RPC calls, just records any method invocations along with
    interesting parameters.
    """

    layer = LaunchpadZopelessLayer
    run_tests_with = AsynchronousDeferredRunTest

    def setUp(self):
        super(TestBinaryBuildPackageBehavior, self).setUp()
        self.layer.switchDbUser('testadmin')

    def assertExpectedInteraction(self, ignored, call_log, builder, build,
                                  chroot, archive, archive_purpose,
                                  component=None, extra_urls=None,
                                  filemap_names=None):
        expected = self.makeExpectedInteraction(
            builder, build, chroot, archive, archive_purpose, component,
            extra_urls, filemap_names)
        self.assertEqual(call_log, expected)

    def makeExpectedInteraction(self, builder, build, chroot, archive,
                                archive_purpose, component=None,
                                extra_urls=None, filemap_names=None):
        """Build the log of calls that we expect to be made to the slave.

        :param builder: The builder we are using to build the binary package.
        :param build: The build being done on the builder.
        :param chroot: The `LibraryFileAlias` for the chroot in which we are
            building.
        :param archive: The `IArchive` into which we are building.
        :param archive_purpose: The ArchivePurpose we are sending to the
            builder. We specify this separately from the archive because
            sometimes the behavior object has to give a different purpose
            in order to trick the slave into building correctly.
        :return: A list of the calls we expect to be made.
        """
        job = removeSecurityProxy(builder.current_build_behavior).buildfarmjob
        build_id = job.generateSlaveBuildCookie()
        ds_name = build.distro_arch_series.distroseries.name
        suite = ds_name + pocketsuffix[build.pocket]
        archives = get_sources_list_for_building(
            build, build.distro_arch_series,
            build.source_package_release.name)
        arch_indep = build.distro_arch_series.isNominatedArchIndep
        if component is None:
            component = build.current_component.name
        if filemap_names is None:
            filemap_names = []
        if extra_urls is None:
            extra_urls = []

        upload_logs = [
            ('ensurepresent', url, '', '')
            for url in [chroot.http_url] + extra_urls]

        extra_args = {
            'arch_indep': arch_indep,
            'arch_tag': build.distro_arch_series.architecturetag,
            'archive_private': archive.private,
            'archive_purpose': archive_purpose.name,
            'archives': archives,
            'build_debug_symbols': archive.build_debug_symbols,
            'ogrecomponent': component,
            'suite': suite,
            }
        build_log = [
            ('build', build_id, 'binarypackage', chroot.content.sha1,
             filemap_names, extra_args)]
        if builder.virtualized:
            result = [('echo', 'ping')] + upload_logs + build_log
        else:
            result = upload_logs + build_log
        return result

    def startBuild(self, builder, candidate):
        builder = removeSecurityProxy(builder)
        candidate = removeSecurityProxy(candidate)
        return defer.maybeDeferred(
            builder.startBuild, candidate, BufferLogger())

    def test_non_virtual_ppa_dispatch(self):
        # When the BinaryPackageBuildBehavior dispatches PPA builds to
        # non-virtual builders, it stores the chroot on the server and
        # requests a binary package build, lying to say that the archive
        # purpose is "PRIMARY" because this ensures that the package mangling
        # tools will run over the built packages.
        archive = self.factory.makeArchive(virtualized=False)
        slave = OkSlave()
        builder = self.factory.makeBuilder(virtualized=False)
        builder.setSlaveForTesting(slave)
        build = self.factory.makeBinaryPackageBuild(
            builder=builder, archive=archive)
        lf = self.factory.makeLibraryFileAlias()
        transaction.commit()
        build.distro_arch_series.addOrUpdateChroot(lf)
        candidate = build.queueBuild()
        d = self.startBuild(builder, candidate)
        d.addCallback(
            self.assertExpectedInteraction, slave.call_log,
            builder, build, lf, archive, ArchivePurpose.PRIMARY, 'universe')
        return d

    def test_virtual_ppa_dispatch(self):
        # Make sure the builder slave gets reset before a build is
        # dispatched to it.
        archive = self.factory.makeArchive(virtualized=True)
        slave = OkSlave()
        builder = self.factory.makeBuilder(
            virtualized=True, vm_host="foohost")
        builder.setSlaveForTesting(slave)
        build = self.factory.makeBinaryPackageBuild(
            builder=builder, archive=archive)
        lf = self.factory.makeLibraryFileAlias()
        transaction.commit()
        build.distro_arch_series.addOrUpdateChroot(lf)
        candidate = build.queueBuild()
        d = self.startBuild(builder, candidate)
        def check_build(ignored):
            # We expect the first call to the slave to be a resume call,
            # followed by the rest of the usual calls we expect.
            expected_resume_call = slave.call_log.pop(0)
            self.assertEqual('resume', expected_resume_call)
            self.assertExpectedInteraction(
                ignored, slave.call_log,
                builder, build, lf, archive, ArchivePurpose.PPA)
        return d.addCallback(check_build)

    def test_partner_dispatch_no_publishing_history(self):
        archive = self.factory.makeArchive(
            virtualized=False, purpose=ArchivePurpose.PARTNER)
        slave = OkSlave()
        builder = self.factory.makeBuilder(virtualized=False)
        builder.setSlaveForTesting(slave)
        build = self.factory.makeBinaryPackageBuild(
            builder=builder, archive=archive)
        lf = self.factory.makeLibraryFileAlias()
        transaction.commit()
        build.distro_arch_series.addOrUpdateChroot(lf)
        candidate = build.queueBuild()
        d = self.startBuild(builder, candidate)
        d.addCallback(
            self.assertExpectedInteraction, slave.call_log,
            builder, build, lf, archive, ArchivePurpose.PARTNER)
        return d

    def test_dont_dispatch_release_builds(self):
        archive = self.factory.makeArchive(purpose=ArchivePurpose.PRIMARY)
        builder = self.factory.makeBuilder()
        distroseries = self.factory.makeDistroSeries(
            status=SeriesStatus.CURRENT, distribution=archive.distribution)
        distro_arch_series = self.factory.makeDistroArchSeries(
            distroseries=distroseries)
        build = self.factory.makeBinaryPackageBuild(
            builder=builder, archive=archive,
            distroarchseries=distro_arch_series,
            pocket=PackagePublishingPocket.RELEASE)
        lf = self.factory.makeLibraryFileAlias()
        transaction.commit()
        build.distro_arch_series.addOrUpdateChroot(lf)
        candidate = build.queueBuild()
        behavior = candidate.required_build_behavior
        behavior.setBuilder(builder)
        e = self.assertRaises(
            AssertionError, behavior.verifyBuildRequest, BufferLogger())
        expected_message = (
            "%s (%s) can not be built for pocket %s: invalid pocket due "
            "to the series status of %s." % (
                build.title, build.id, build.pocket.name,
                build.distro_series.name))
        self.assertEqual(expected_message, str(e))

    def test_dont_dispatch_security_builds(self):
        archive = self.factory.makeArchive(purpose=ArchivePurpose.PRIMARY)
        builder = self.factory.makeBuilder()
        build = self.factory.makeBinaryPackageBuild(
            builder=builder, archive=archive,
            pocket=PackagePublishingPocket.SECURITY)
        lf = self.factory.makeLibraryFileAlias()
        transaction.commit()
        build.distro_arch_series.addOrUpdateChroot(lf)
        candidate = build.queueBuild()
        behavior = candidate.required_build_behavior
        behavior.setBuilder(builder)
        e = self.assertRaises(
            AssertionError, behavior.verifyBuildRequest, BufferLogger())
        self.assertEqual(
            'Soyuz is not yet capable of building SECURITY uploads.',
            str(e))

    def test_verifyBuildRequest(self):
        # Don't allow a virtual build on a non-virtual builder.
        archive = self.factory.makeArchive(purpose=ArchivePurpose.PPA)
        builder = self.factory.makeBuilder(virtualized=False)
        build = self.factory.makeBinaryPackageBuild(
            builder=builder, archive=archive,
            pocket=PackagePublishingPocket.RELEASE)
        lf = self.factory.makeLibraryFileAlias()
        transaction.commit()
        build.distro_arch_series.addOrUpdateChroot(lf)
        candidate = build.queueBuild()
        behavior = candidate.required_build_behavior
        behavior.setBuilder(builder)
        e = self.assertRaises(
            AssertionError, behavior.verifyBuildRequest, BufferLogger())
        self.assertEqual(
            'Attempt to build virtual item on a non-virtual builder.',
            str(e))


class TestBinaryBuildPackageBehaviorBuildCollection(TestCaseWithFactory):
    """Tests for the BinaryPackageBuildBehavior.

    Using various mock slaves, we check how updateBuild() behaves in
    various scenarios.
    """

    # XXX: These tests replace part of the old buildd-slavescanner.txt
    # It was checking that each call to updateBuild was sending 3 (!)
    # emails but this behaviour is so ill-defined and dependent on the
    # sample data that I've not replicated that here.  We need to
    # examine that behaviour separately somehow, but the old tests gave
    # NO clue as to what, exactly, they were testing.

    layer = LaunchpadZopelessLayer
    run_tests_with = AsynchronousDeferredRunTest

    def _cleanup(self):
        if os.path.exists(config.builddmaster.root):
            shutil.rmtree(config.builddmaster.root)

    def setUp(self):
        super(TestBinaryBuildPackageBehaviorBuildCollection, self).setUp()
        self.layer.switchDbUser('testadmin')

        self.builder = self.factory.makeBuilder()
        self.build = self.factory.makeBinaryPackageBuild(
            builder=self.builder, pocket=PackagePublishingPocket.RELEASE)
        lf = self.factory.makeLibraryFileAlias()
        transaction.commit()
        self.build.distro_arch_series.addOrUpdateChroot(lf)
        self.candidate = self.build.queueBuild()
        self.candidate.markAsBuilding(self.builder)
        self.behavior = self.candidate.required_build_behavior
        self.behavior.setBuilder(self.builder)
        # This is required so that uploaded files from the buildd don't
        # hang around between test runs.
        self.addCleanup(self._cleanup)

    def assertBuildProperties(self, build):
        """Check that a build happened by making sure some of its properties
        are set."""
        self.assertIsNot(None, build.builder)
        self.assertIsNot(None, build.date_finished)
        self.assertIsNot(None, build.duration)
        self.assertIsNot(None, build.log)

    def test_packagefail_collection(self):
        # When a package fails to build, make sure the builder notes are
        # stored and the build status is set as failed.
        waiting_slave = WaitingSlave('BuildStatus.PACKAGEFAIL')
        self.builder.setSlaveForTesting(waiting_slave)

        def got_update(ignored):
            self.assertBuildProperties(self.build)
            self.assertEqual(BuildStatus.FAILEDTOBUILD, self.build.status)

        d = self.builder.updateBuild(self.candidate)
        return d.addCallback(got_update)

    def test_depwait_collection(self):
        # Package build was left in dependency wait.
        DEPENDENCIES = 'baz (>= 1.0.1)'
        waiting_slave = WaitingSlave('BuildStatus.DEPFAIL', DEPENDENCIES)
        self.builder.setSlaveForTesting(waiting_slave)

        def got_update(ignored):
            self.assertBuildProperties(self.build)
            self.assertEqual(BuildStatus.MANUALDEPWAIT, self.build.status)
            self.assertEqual(DEPENDENCIES, self.build.dependencies)

        d = self.builder.updateBuild(self.candidate)
        return d.addCallback(got_update)

    def test_chrootfail_collection(self):
        # There was a chroot problem for this build.
        waiting_slave = WaitingSlave('BuildStatus.CHROOTFAIL')
        self.builder.setSlaveForTesting(waiting_slave)

        def got_update(ignored):
            self.assertBuildProperties(self.build)
            self.assertEqual(BuildStatus.CHROOTWAIT, self.build.status)

        d = self.builder.updateBuild(self.candidate)
        return d.addCallback(got_update)

    def test_builderfail_collection(self):
        # The builder failed after we dispatched the build.
        waiting_slave = WaitingSlave('BuildStatus.BUILDERFAIL')
        self.builder.setSlaveForTesting(waiting_slave)

        def got_update(ignored):
            self.assertEqual(
                "Builder returned BUILDERFAIL when asked for its status",
                self.builder.failnotes)
            self.assertIs(None, self.candidate.builder)
            self.assertEqual(BuildStatus.NEEDSBUILD, self.build.status)
            job = self.candidate.specific_job.job
            self.assertEqual(JobStatus.WAITING, job.status)

        d = self.builder.updateBuild(self.candidate)
        return d.addCallback(got_update)

    def test_building_collection(self):
        # The builder is still building the package.
        self.builder.setSlaveForTesting(BuildingSlave())

        def got_update(ignored):
            # The fake log is returned from the BuildingSlave() mock.
            self.assertEqual("This is a build log", self.candidate.logtail)

        d = self.builder.updateBuild(self.candidate)
        return d.addCallback(got_update)

    def test_aborted_collection(self):
        # The builder aborted the job.
        self.builder.setSlaveForTesting(AbortedSlave())

        def got_update(ignored):
            self.assertEqual(BuildStatus.NEEDSBUILD, self.build.status)

        d = self.builder.updateBuild(self.candidate)
        return d.addCallback(got_update)

    def test_aborting_collection(self):
        # The builder is in the process of aborting.
        self.builder.setSlaveForTesting(AbortingSlave())

        def got_update(ignored):
            self.assertEqual(
                "Waiting for slave process to be terminated",
                self.candidate.logtail)

        d = self.builder.updateBuild(self.candidate)
        return d.addCallback(got_update)

    def test_collection_for_deleted_source(self):
        # If we collected a build for a superseded/deleted source then
        # the build should get marked superseded as the build results
        # get discarded.
        self.builder.setSlaveForTesting(WaitingSlave('BuildStatus.OK'))
        spr = removeSecurityProxy(self.build.source_package_release)
        pub = self.build.current_source_publication
        pub.requestDeletion(spr.creator)

        def got_update(ignored):
            self.assertEqual(
                BuildStatus.SUPERSEDED, self.build.status)

        d = self.builder.updateBuild(self.candidate)
        return d.addCallback(got_update)

    def test_uploading_collection(self):
        # After a successful build, the status should be UPLOADING.
        self.builder.setSlaveForTesting(WaitingSlave('BuildStatus.OK'))

        def got_update(ignored):
            self.assertEqual(self.build.status, BuildStatus.UPLOADING)
            # We do not store any upload log information when the binary
            # upload processing succeeded.
            self.assertIs(None, self.build.upload_log)

        d = self.builder.updateBuild(self.candidate)
        return d.addCallback(got_update)

    def test_givenback_collection(self):
        waiting_slave = WaitingSlave('BuildStatus.GIVENBACK')
        self.builder.setSlaveForTesting(waiting_slave)
        score = self.candidate.lastscore

        def got_update(ignored):
            self.assertIs(None, self.candidate.builder)
            self.assertIs(None, self.candidate.date_started)
            self.assertEqual(score, self.candidate.lastscore)
            self.assertEqual(BuildStatus.NEEDSBUILD, self.build.status)
            job = self.candidate.specific_job.job
            self.assertEqual(JobStatus.WAITING, job.status)

        d = self.builder.updateBuild(self.candidate)
        return d.addCallback(got_update)

    def test_log_file_collection(self):
        self.builder.setSlaveForTesting(WaitingSlave('BuildStatus.OK'))
        self.build.status = BuildStatus.FULLYBUILT
        old_tmps = sorted(os.listdir('/tmp'))

        def got_log(logfile_lfa_id):
            # Grabbing logs should not leave new files in /tmp (bug #172798)
            logfile_lfa = getUtility(ILibraryFileAliasSet)[logfile_lfa_id]
            new_tmps = sorted(os.listdir('/tmp'))
            self.assertEqual(old_tmps, new_tmps)

            # The new librarian file is stored compressed with a .gz
            # extension and text/plain file type for easy viewing in
            # browsers, as it decompresses and displays the file inline.
            self.assertTrue(
                logfile_lfa.filename.endswith('_FULLYBUILT.txt.gz'))
            self.assertEqual('text/plain', logfile_lfa.mimetype)
            self.layer.txn.commit()

            # LibrarianFileAlias does not implement tell() or seek(), which
            # are required by gzip.open(), so we need to read the file out
            # of the librarian first.
            fd, fname = tempfile.mkstemp()
            self.addCleanup(os.remove, fname)
            tmp = os.fdopen(fd, 'wb')
            tmp.write(logfile_lfa.read())
            tmp.close()
            uncompressed_file = gzip.open(fname).read()

            # Now make a temp filename that getFile() can write to.
            fd, tmp_orig_file_name = tempfile.mkstemp()
            self.addCleanup(os.remove, tmp_orig_file_name)

            # Check that the original file from the slave matches the
            # uncompressed file in the librarian.
            def got_orig_log(ignored):
                orig_file_content = open(tmp_orig_file_name).read()
                self.assertEqual(orig_file_content, uncompressed_file)

            d = removeSecurityProxy(self.builder.slave).getFile(
                'buildlog', tmp_orig_file_name)
            return d.addCallback(got_orig_log)

        d = self.build.getLogFromSlave(self.build)
        return d.addCallback(got_log)

    def test_private_build_log_storage(self):
        # Builds in private archives should have their log uploaded to
        # the restricted librarian.
        self.builder.setSlaveForTesting(WaitingSlave('BuildStatus.OK'))

        # Go behind Storm's back since the field validator on
        # Archive.private prevents us from setting it to True with
        # existing published sources.
        Store.of(self.build).execute("""
            UPDATE archive SET private=True,buildd_secret='foo'
            WHERE archive.id = %s""" % self.build.archive.id)
        Store.of(self.build).invalidate()

        def got_update(ignored):
            # Librarian needs a commit.  :(
            self.layer.txn.commit()
            self.assertTrue(self.build.log.restricted)

        d = self.builder.updateBuild(self.candidate)
        return d.addCallback(got_update)