~launchpad-pqm/launchpad/devel

# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the

# GNU Affero General Public License version 3 (see the file LICENSE).

"""Tests for the renovated slave scanner aka BuilddManager."""

from collections import namedtuple

import os

import signal

import time

import xmlrpclib

from lpbuildd.tests import BuilddSlaveTestSetup

from testtools.deferredruntest import (

    assert_fails_with,

    AsynchronousDeferredRunTest,

    )

from twisted.internet import (

    defer,

    reactor,

    task,

    )

from twisted.internet.task import deferLater

from twisted.python.failure import Failure

from zope.component import getUtility

from zope.security.proxy import removeSecurityProxy

from lp.buildmaster.enums import BuildStatus

from lp.buildmaster.interfaces.builder import IBuilderSet

from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet

from lp.buildmaster.manager import (

    assessFailureCounts,

    BuilddManager,

    NewBuildersScanner,

    SlaveScanner,

    )

from lp.buildmaster.model.builder import Builder

from lp.buildmaster.model.packagebuild import PackageBuild

from lp.buildmaster.testing import BuilddManagerTestFixture

from lp.buildmaster.tests.harness import BuilddManagerTestSetup

from lp.buildmaster.tests.mock_slaves import (

    BrokenSlave,

    BuildingSlave,

    make_publisher,

    OkSlave,

    WaitingSlave,

    )

from lp.registry.interfaces.distribution import IDistributionSet

from lp.services.config import config

from lp.services.database.constants import UTC_NOW

from lp.services.log.logger import BufferLogger

from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet

from lp.testing import (

    ANONYMOUS,

    login,

    TestCase,

    TestCaseWithFactory,

    )

from lp.testing.factory import LaunchpadObjectFactory

from lp.testing.fakemethod import FakeMethod

from lp.testing.layers import (

    LaunchpadScriptLayer,

    LaunchpadZopelessLayer,

    ZopelessDatabaseLayer,

    )

from lp.testing.sampledata import (

    BOB_THE_BUILDER_NAME,

    FROG_THE_BUILDER_NAME,

    )

class TestSlaveScannerScan(TestCaseWithFactory):

    """Tests `SlaveScanner.scan` method.

    This method uses the old framework for scanning and dispatching builds.

    """

    layer = LaunchpadZopelessLayer

    run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=20)

    def setUp(self):

        """Set up BuilddSlaveTest.

        Also adjust the sampledata in a way a build can be dispatched to

        'bob' builder.

        """

        super(TestSlaveScannerScan, self).setUp()

        # Creating the required chroots needed for dispatching.

        test_publisher = make_publisher()

        ubuntu = getUtility(IDistributionSet).getByName('ubuntu')

        hoary = ubuntu.getSeries('hoary')

        test_publisher.setUpDefaultDistroSeries(hoary)

        test_publisher.addFakeChroots()

        self.useFixture(BuilddManagerTestFixture())

    def _resetBuilder(self, builder):

        """Reset the given builder and its job."""

        builder.builderok = True

        job = builder.currentjob

        if job is not None:

            job.reset()

    def getFreshBuilder(self, slave=None, name=BOB_THE_BUILDER_NAME,

                        failure_count=0):

        """Return a builder.

        The builder is taken from sample data, but reset to a usable state.

        Be careful: this is not a proper factory method.  Identical calls

        return (and reset) the same builder.  Don't rely on that though;

        maybe someday we'll have a proper factory here.

        """

        if slave is None:

            slave = OkSlave()

        builder = getUtility(IBuilderSet)[name]

        self._resetBuilder(builder)

        builder.setSlaveForTesting(slave)

        builder.failure_count = failure_count

        return builder

    def assertBuildingJob(self, job, builder, logtail=None):

        """Assert the given job is building on the given builder."""

        from lp.services.job.interfaces.job import JobStatus

        if logtail is None:

            logtail = 'Dummy sampledata entry, not processing'

        self.assertTrue(job is not None)

        self.assertEqual(job.builder, builder)

        self.assertTrue(job.date_started is not None)

        self.assertEqual(job.job.status, JobStatus.RUNNING)

        build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(job)

        self.assertEqual(build.status, BuildStatus.BUILDING)

        self.assertEqual(job.logtail, logtail)

    def _getScanner(self, builder_name=None, clock=None):

        """Instantiate a SlaveScanner object.

        Replace its default logging handler by a testing version.

        """

        if builder_name is None:

            builder_name = BOB_THE_BUILDER_NAME

        scanner = SlaveScanner(builder_name, BufferLogger(), clock=clock)

        scanner.logger.name = 'slave-scanner'

        return scanner

    def _checkDispatch(self, slave, builder):

        # SlaveScanner.scan returns a slave when a dispatch was

        # successful.  We also check that the builder has a job on it.

        self.assertTrue(slave is not None, "Expected a slave.")

        self.assertEqual(0, builder.failure_count)

        self.assertTrue(builder.currentjob is not None)

    def testScanDispatchForResetBuilder(self):

        # A job gets dispatched to the sampledata builder after it's reset.

        # Obtain a builder.   Initialize failure count to 1 so that

        # _checkDispatch can make sure that a successful dispatch resets

        # the count to 0.

        with BuilddManagerTestFixture.extraSetUp():

            builder = self.getFreshBuilder(failure_count=1)

        # Run 'scan' and check its result.

        self.layer.switchDbUser(config.builddmaster.dbuser)

        scanner = self._getScanner()

        d = defer.maybeDeferred(scanner.scan)

        d.addCallback(self._checkDispatch, builder)

        return d

    def _checkNoDispatch(self, slave, builder):

        """Assert that no dispatch has occurred.

        'slave' is None, so no interations would be passed

        to the asynchonous dispatcher and the builder remained active

        and IDLE.

        """

        self.assertIs(None, slave, "Unexpected slave.")

        builder = getUtility(IBuilderSet).get(builder.id)

        self.assertTrue(builder.builderok)

        self.assertIs(None, builder.currentjob)

    def testNoDispatchForMissingChroots(self):

        # When a required chroot is not present the `scan` method

        # should not return any `RecordingSlaves` to be processed

        # and the builder used should remain active and IDLE.

        with BuilddManagerTestFixture.extraSetUp():

            builder = self.getFreshBuilder()

            # Remove hoary/i386 chroot.

            login('foo.bar@canonical.com')

            ubuntu = getUtility(IDistributionSet).getByName('ubuntu')

            hoary = ubuntu.getSeries('hoary')

            pocket_chroot = hoary.getDistroArchSeries('i386').getPocketChroot()

            removeSecurityProxy(pocket_chroot).chroot = None

        login(ANONYMOUS)

        # Run 'scan' and check its result.

        self.layer.switchDbUser(config.builddmaster.dbuser)

        scanner = self._getScanner()

        d = defer.maybeDeferred(scanner.singleCycle)

        d.addCallback(self._checkNoDispatch, builder)

        return d

    def _checkJobRescued(self, slave, builder, job):

        """`SlaveScanner.scan` rescued the job.

        Nothing gets dispatched,  the 'broken' builder remained disabled

        and the 'rescued' job is ready to be dispatched.

        """

        self.assertTrue(

            slave is None, "Unexpected slave.")

        builder = getUtility(IBuilderSet).get(builder.id)

        self.assertFalse(builder.builderok)

        job = getUtility(IBuildQueueSet).get(job.id)

        self.assertTrue(job.builder is None)

        self.assertTrue(job.date_started is None)

        build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(job)

        self.assertEqual(build.status, BuildStatus.NEEDSBUILD)

    def testScanRescuesJobFromBrokenBuilder(self):

        # The job assigned to a broken builder is rescued.

        self.useFixture(BuilddSlaveTestSetup())

        # Sampledata builder is enabled and is assigned to an active job.

        builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME]

        self.assertTrue(builder.builderok)

        job = builder.currentjob

        self.assertBuildingJob(job, builder)

        # Disable the sampledata builder

        login('foo.bar@canonical.com')

        with BuilddManagerTestFixture.extraSetUp():

            builder.builderok = False

        login(ANONYMOUS)

        # Run 'scan' and check its result.

        self.layer.switchDbUser(config.builddmaster.dbuser)

        scanner = self._getScanner()

        d = defer.maybeDeferred(scanner.scan)

        d.addCallback(self._checkJobRescued, builder, job)

        return d

    def _checkJobUpdated(self, slave, builder, job):

        """`SlaveScanner.scan` updates legitimate jobs.

        Job is kept assigned to the active builder and its 'logtail' is

        updated.

        """

        self.assertTrue(slave is None, "Unexpected slave.")

        builder = getUtility(IBuilderSet).get(builder.id)

        self.assertTrue(builder.builderok)

        job = getUtility(IBuildQueueSet).get(job.id)

        self.assertBuildingJob(job, builder, logtail='This is a build log')

    def testScanUpdatesBuildingJobs(self):

        # Enable sampledata builder attached to an appropriate testing

        # slave. It will respond as if it was building the sampledata job.

        builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME]

        login('foo.bar@canonical.com')

        with BuilddManagerTestFixture.extraSetUp():

            builder.builderok = True

            builder.setSlaveForTesting(BuildingSlave(build_id='8-1'))

        login(ANONYMOUS)

        job = builder.currentjob

        self.assertBuildingJob(job, builder)

        # Run 'scan' and check its result.

        self.layer.switchDbUser(config.builddmaster.dbuser)

        scanner = self._getScanner()

        d = defer.maybeDeferred(scanner.scan)

        d.addCallback(self._checkJobUpdated, builder, job)

        return d

    def test_scan_with_nothing_to_dispatch(self):

        with BuilddManagerTestFixture.extraSetUp():

            builder = self.factory.makeBuilder()

            builder.setSlaveForTesting(OkSlave())

        scanner = self._getScanner(builder_name=builder.name)

        d = scanner.scan()

        return d.addCallback(self._checkNoDispatch, builder)

    def test_scan_with_manual_builder(self):

        # Reset sampledata builder.

        with BuilddManagerTestFixture.extraSetUp():

            builder = self.getFreshBuilder()

            builder.manual = True

        scanner = self._getScanner()

        d = scanner.scan()

        d.addCallback(self._checkNoDispatch, builder)

        return d

    def test_scan_with_not_ok_builder(self):

        # Reset sampledata builder.

        with BuilddManagerTestFixture.extraSetUp():

            builder = self.getFreshBuilder()

            builder.builderok = False

        scanner = self._getScanner()

        d = scanner.scan()

        # Because the builder is not ok, we can't use _checkNoDispatch.

        d.addCallback(

            lambda ignored: self.assertIs(None, builder.currentjob))

        return d

    def test_scan_of_broken_slave(self):

        with BuilddManagerTestFixture.extraSetUp():

            builder = self.getFreshBuilder(slave=BrokenSlave())

        scanner = self._getScanner(builder_name=builder.name)

        d = scanner.scan()

        return assert_fails_with(d, xmlrpclib.Fault)

    def _assertFailureCounting(self, builder_count, job_count,

                               expected_builder_count, expected_job_count):

        # Avoid circular imports.

        from lp.buildmaster import manager as manager_module

        # If scan() fails with an exception, failure_counts should be

        # incremented.  What we do with the results of the failure

        # counts is tested below separately, this test just makes sure that

        # scan() is setting the counts.

        def failing_scan():

            return defer.fail(Exception("fake exception"))

        with BuilddManagerTestFixture.extraSetUp():

            scanner = self._getScanner()

            scanner.scan = failing_scan

            self.patch(manager_module, 'assessFailureCounts', FakeMethod())

            builder = getUtility(IBuilderSet)[scanner.builder_name]

            builder.failure_count = builder_count

            builder.currentjob.specific_job.build.failure_count = job_count

            # The _scanFailed() calls abort, so make sure our existing failure

            # counts are persisted by exiting the extraSetUp() context (which

            # commits).

        # singleCycle() calls scan() which is our fake one that throws an

        # exception.

        d = scanner.singleCycle()

        # Failure counts should be updated, and the assessment method

        # should have been called.  The actual behaviour is tested below

        # in TestFailureAssessments.

        def got_scan(ignored):

            self.assertEqual(expected_builder_count, builder.failure_count)

            self.assertEqual(

                expected_job_count,

                builder.currentjob.specific_job.build.failure_count)

            self.assertEqual(

                1, manager_module.assessFailureCounts.call_count)

        return d.addCallback(got_scan)

    def test_scan_first_fail(self):

        # The first failure of a job should result in the failure_count

        # on the job and the builder both being incremented.

        self._assertFailureCounting(

            builder_count=0, job_count=0, expected_builder_count=1,

            expected_job_count=1)

    def test_scan_second_builder_fail(self):

        # The first failure of a job should result in the failure_count

        # on the job and the builder both being incremented.

        self._assertFailureCounting(

            builder_count=1, job_count=0, expected_builder_count=2,

            expected_job_count=1)

    def test_scan_second_job_fail(self):

        # The first failure of a job should result in the failure_count

        # on the job and the builder both being incremented.

        self._assertFailureCounting(

            builder_count=0, job_count=1, expected_builder_count=1,

            expected_job_count=2)

    def test_scanFailed_handles_lack_of_a_job_on_the_builder(self):

        def failing_scan():

            return defer.fail(Exception("fake exception"))

        scanner = self._getScanner()

        scanner.scan = failing_scan

        builder = getUtility(IBuilderSet)[scanner.builder_name]

        with BuilddManagerTestFixture.extraSetUp():

            builder.failure_count = Builder.FAILURE_THRESHOLD

            builder.currentjob.reset()

        d = scanner.singleCycle()

        def scan_finished(ignored):

            self.assertFalse(builder.builderok)

        return d.addCallback(scan_finished)

    def test_fail_to_resume_slave_resets_job(self):

        # If an attempt to resume and dispatch a slave fails, it should

        # reset the job via job.reset()

        # Make a slave with a failing resume() method.

        slave = OkSlave()

        slave.resume = lambda: deferLater(

            reactor, 0, defer.fail, Failure(('out', 'err', 1)))

        # Reset sampledata builder.

        builder = removeSecurityProxy(

            getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME])

        with BuilddManagerTestFixture.extraSetUp():

            self._resetBuilder(builder)

            self.assertEqual(0, builder.failure_count)

            builder.setSlaveForTesting(slave)

            builder.vm_host = "fake_vm_host"

        scanner = self._getScanner()

        # Get the next job that will be dispatched.

        job = removeSecurityProxy(builder._findBuildCandidate())

        with BuilddManagerTestFixture.extraSetUp():

            job.virtualized = True

            builder.virtualized = True

        d = scanner.singleCycle()

        def check(ignored):

            # The failure_count will have been incremented on the

            # builder, we can check that to see that a dispatch attempt

            # did indeed occur.

            self.assertEqual(1, builder.failure_count)

            # There should also be no builder set on the job.

            self.assertTrue(job.builder is None)

            build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(job)

            self.assertEqual(build.status, BuildStatus.NEEDSBUILD)

        return d.addCallback(check)

    def test_cancelling_a_build(self):

        # When scanning an in-progress build, if its state is CANCELLING

        # then the build should be stopped and moved to the CANCELLED state.

        # Set up a building slave with a fake resume method so we can see

        # if it got called later.

        slave = BuildingSlave(build_id="8-1")

        call_counter = FakeMethod()

        def fake_resume():

            call_counter()

            return defer.succeed((None, None, 0))

        slave.resume = fake_resume

        # Set the sample data builder building with the slave from above.

        builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME]

        login('foo.bar@canonical.com')

        with BuilddManagerTestFixture.extraSetUp():

            builder.builderok = True

            # For now, we can only cancel virtual builds.

            builder.virtualized = True

            builder.vm_host = "fake_vm_host"

            builder.setSlaveForTesting(slave)

        login(ANONYMOUS)

        buildqueue = builder.currentjob

        self.assertBuildingJob(buildqueue, builder)

        # Now set the build to CANCELLING.

        build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(buildqueue)

        with BuilddManagerTestFixture.extraSetUp():

            build.status = BuildStatus.CANCELLING

        # Run 'scan' and check its results.

        self.layer.switchDbUser(config.builddmaster.dbuser)

        scanner = self._getScanner()

        d = scanner.scan()

        # The build state should be cancelled and we should have also

        # called the resume() method on the slave that resets the virtual

        # machine.

        def check_cancelled(ignore, builder, buildqueue):

            self.assertEqual(1, call_counter.call_count)

            self.assertEqual(BuildStatus.CANCELLED, build.status)

        d.addCallback(check_cancelled, builder, buildqueue)

        return d

    def makeFakeFailure(self):

        """Produce a fake failure for use with SlaveScanner._scanFailed."""

        FakeFailure = namedtuple('FakeFailure', ['getErrorMessage', 'check'])

        return FakeFailure(

            FakeMethod(self.factory.getUniqueString()),

            FakeMethod(True))

    def test_interleaved_success_and_failure_do_not_interfere(self):

        # It's possible for one builder to fail while another continues

        # to function properly.  When that happens, the failed builder

        # may cause database changes to be rolled back.  But that does

        # not affect the functioning builder.

        clock = task.Clock()

        with BuilddManagerTestFixture.extraSetUp():

            broken_builder = self.getFreshBuilder(

                slave=BrokenSlave(), name=BOB_THE_BUILDER_NAME)

            broken_scanner = self._getScanner(

                builder_name=broken_builder.name)

            good_builder = self.getFreshBuilder(

                slave=WaitingSlave(), name=FROG_THE_BUILDER_NAME)

            good_build = self.factory.makeBinaryPackageBuild(

                distroarchseries=self.factory.makeDistroArchSeries())

            # The good build is being handled by the good builder.

            buildqueue = good_build.queueBuild()

            buildqueue.builder = good_builder

            removeSecurityProxy(

                good_build.build_farm_job).date_started = UTC_NOW

        # The good builder requests information from a successful build,

        # and up receiving it, updates the build's metadata.

        # Our dependencies string goes into the build, and its

        # date_finished will be set.

        dependencies = self.factory.getUniqueString()

        PackageBuild.storeBuildInfo(

            good_build, None, {'dependencies': dependencies})

        clock.advance(1)

        # The broken scanner experiences a failure before the good

        # scanner is receiving its data.  This aborts the ongoing

        # transaction.

        # As a somewhat weird example, if the builder changed its own

        # title, that change will be rolled back.

        original_broken_builder_title = broken_builder.title

        broken_builder.title = self.factory.getUniqueString()

        broken_scanner._scanFailed(self.makeFakeFailure())

        # The work done by the good scanner is retained.  The

        # storeBuildInfo code committed it.

        self.assertEqual(dependencies, good_build.dependencies)

        self.assertIsNot(None, good_build.date_finished)

        # The work done by the broken scanner is rolled back.

        self.assertEqual(original_broken_builder_title, broken_builder.title)

class TestCancellationChecking(TestCaseWithFactory):

    """Unit tests for the checkCancellation method."""

    layer = ZopelessDatabaseLayer

    run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=20)

    def setUp(self):

        super(TestCancellationChecking, self).setUp()

        builder_name = BOB_THE_BUILDER_NAME

        self.builder = getUtility(IBuilderSet)[builder_name]

        self.builder.virtualized = True

        self.scanner = SlaveScanner(builder_name, BufferLogger())

        self.scanner.builder = self.builder

        self.scanner.logger.name = 'slave-scanner'

        self.useFixture(BuilddManagerTestFixture())

    def test_ignores_nonvirtual(self):

        # If the builder is nonvirtual make sure we return False.

        with BuilddManagerTestFixture.extraSetUp():

            self.builder.virtualized = False

        d = self.scanner.checkCancellation(self.builder)

        return d.addCallback(self.assertFalse)

    def test_ignores_no_buildqueue(self):

        # If the builder has no buildqueue associated,

        # make sure we return False.

        buildqueue = self.builder.currentjob

        with BuilddManagerTestFixture.extraSetUp():

            buildqueue.reset()

        d = self.scanner.checkCancellation(self.builder)

        return d.addCallback(self.assertFalse)

    def test_ignores_build_not_cancelling(self):

        # If the active build is not in a CANCELLING state, ignore it.

        buildqueue = self.builder.currentjob

        build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(buildqueue)

        with BuilddManagerTestFixture.extraSetUp():

            build.status = BuildStatus.BUILDING

        d = self.scanner.checkCancellation(self.builder)

        return d.addCallback(self.assertFalse)

    def test_cancelling_build_is_cancelled(self):

        # If a build is CANCELLING, make sure True is returned and the

        # slave was resumed.

        call_counter = FakeMethod()

        def fake_resume():

            call_counter()

            return defer.succeed((None, None, 0))

        slave = OkSlave()

        slave.resume = fake_resume

        with BuilddManagerTestFixture.extraSetUp():

            self.builder.vm_host = "fake_vm_host"

            self.builder.setSlaveForTesting(slave)

            buildqueue = self.builder.currentjob

            build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(buildqueue)

            build.status = BuildStatus.CANCELLING

        def check(result):

            self.assertEqual(1, call_counter.call_count)

            self.assertTrue(result)

            self.assertEqual(BuildStatus.CANCELLED, build.status)

        d = self.scanner.checkCancellation(self.builder)

        return d.addCallback(check)

class TestBuilddManager(TestCase):

    layer = LaunchpadZopelessLayer

    def _stub_out_scheduleNextScanCycle(self):

        # stub out the code that adds a callLater, so that later tests

        # don't get surprises.

        self.patch(SlaveScanner, 'startCycle', FakeMethod())

    def test_addScanForBuilders(self):

        # Test that addScanForBuilders generates NewBuildersScanner objects.

        self._stub_out_scheduleNextScanCycle()

        manager = BuilddManager()

        builder_names = set(

            builder.name for builder in getUtility(IBuilderSet))

        scanners = manager.addScanForBuilders(builder_names)

        scanner_names = set(scanner.builder_name for scanner in scanners)

        self.assertEqual(builder_names, scanner_names)

    def test_startService_adds_NewBuildersScanner(self):

        # When startService is called, the manager will start up a

        # NewBuildersScanner object.

        self._stub_out_scheduleNextScanCycle()

        clock = task.Clock()

        manager = BuilddManager(clock=clock)

        # Replace scan() with FakeMethod so we can see if it was called.

        manager.new_builders_scanner.scan = FakeMethod()

        manager.startService()

        advance = NewBuildersScanner.SCAN_INTERVAL + 1

        clock.advance(advance)

        self.assertNotEqual(0, manager.new_builders_scanner.scan.call_count)

class TestFailureAssessments(TestCaseWithFactory):

    layer = ZopelessDatabaseLayer

    def setUp(self):

        TestCaseWithFactory.setUp(self)

        self.builder = self.factory.makeBuilder()

        self.build = self.factory.makeSourcePackageRecipeBuild()

        self.buildqueue = self.build.queueBuild()

        self.buildqueue.markAsBuilding(self.builder)