← Back to branch summary

~launchpad-pqm/launchpad/devel

« back to all changes in this revision

Viewing changes to lib/lp/buildmaster/tests/test_manager.py

[r=allenap, bac, gmb, julian-edwards, wallyworld][bug=905853, 905855,
 906079] In buildmaster,
 always shift into a read-write database transaction access mode before
 updating PackageBuild statuses. Shift into read-write transactions in
 appropriate places in TranslationTemplatesBuildBehavior. Ensure that all
 lp.buildmaster tests to which it is relevant are running with
 BuilddManagerTestFixture.

Show diffs side-by-side

added added

removed removed

Lines of Context:
lib/lp/buildmaster/tests/test_manager.py
1
 
# Copyright 2009-2010 Canonical Ltd.  This software is licensed under the
 
1
# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
2
2
# GNU Affero General Public License version 3 (see the file LICENSE).
3
3
 
4
4
"""Tests for the renovated slave scanner aka BuilddManager."""
5
5
 
 
6
from collections import namedtuple
6
7
import os
7
8
import signal
8
9
import time
13
14
    assert_fails_with,
14
15
    AsynchronousDeferredRunTest,
15
16
    )
16
 
import transaction
17
17
from twisted.internet import (
18
18
    defer,
19
19
    reactor,
34
34
    SlaveScanner,
35
35
    )
36
36
from lp.buildmaster.model.builder import Builder
 
37
from lp.buildmaster.model.packagebuild import PackageBuild
 
38
from lp.buildmaster.testing import BuilddManagerTestFixture
37
39
from lp.buildmaster.tests.harness import BuilddManagerTestSetup
38
40
from lp.buildmaster.tests.mock_slaves import (
39
41
    BrokenSlave,
40
42
    BuildingSlave,
41
43
    make_publisher,
42
44
    OkSlave,
 
45
    WaitingSlave,
43
46
    )
44
47
from lp.registry.interfaces.distribution import IDistributionSet
45
48
from lp.services.config import config
 
49
from lp.services.database.constants import UTC_NOW
46
50
from lp.services.log.logger import BufferLogger
47
51
from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet
48
52
from lp.testing import (
58
62
    LaunchpadZopelessLayer,
59
63
    ZopelessDatabaseLayer,
60
64
    )
61
 
from lp.testing.sampledata import BOB_THE_BUILDER_NAME
62
 
 
63
 
 
64
 
class TestSlaveScannerScan(TestCase):
 
65
from lp.testing.sampledata import (
 
66
    BOB_THE_BUILDER_NAME,
 
67
    FROG_THE_BUILDER_NAME,
 
68
    )
 
69
 
 
70
 
 
71
class TestSlaveScannerScan(TestCaseWithFactory):
65
72
    """Tests `SlaveScanner.scan` method.
66
73
 
67
74
    This method uses the old framework for scanning and dispatching builds.
83
90
        test_publisher.setUpDefaultDistroSeries(hoary)
84
91
        test_publisher.addFakeChroots()
85
92
 
 
93
        self.useFixture(BuilddManagerTestFixture())
 
94
 
86
95
    def _resetBuilder(self, builder):
87
96
        """Reset the given builder and its job."""
88
 
 
89
97
        builder.builderok = True
90
98
        job = builder.currentjob
91
99
        if job is not None:
92
100
            job.reset()
93
101
 
94
 
        transaction.commit()
 
102
    def getFreshBuilder(self, slave=None, name=BOB_THE_BUILDER_NAME,
 
103
                        failure_count=0):
 
104
        """Return a builder.
 
105
 
 
106
        The builder is taken from sample data, but reset to a usable state.
 
107
        Be careful: this is not a proper factory method.  Identical calls
 
108
        return (and reset) the same builder.  Don't rely on that though;
 
109
        maybe someday we'll have a proper factory here.
 
110
        """
 
111
        if slave is None:
 
112
            slave = OkSlave()
 
113
        builder = getUtility(IBuilderSet)[name]
 
114
        self._resetBuilder(builder)
 
115
        builder.setSlaveForTesting(slave)
 
116
        builder.failure_count = failure_count
 
117
        return builder
95
118
 
96
119
    def assertBuildingJob(self, job, builder, logtail=None):
97
120
        """Assert the given job is building on the given builder."""
107
130
        self.assertEqual(build.status, BuildStatus.BUILDING)
108
131
        self.assertEqual(job.logtail, logtail)
109
132
 
110
 
    def _getScanner(self, builder_name=None):
 
133
    def _getScanner(self, builder_name=None, clock=None):
111
134
        """Instantiate a SlaveScanner object.
112
135
 
113
136
        Replace its default logging handler by a testing version.
114
137
        """
115
138
        if builder_name is None:
116
139
            builder_name = BOB_THE_BUILDER_NAME
117
 
        scanner = SlaveScanner(builder_name, BufferLogger())
 
140
        scanner = SlaveScanner(builder_name, BufferLogger(), clock=clock)
118
141
        scanner.logger.name = 'slave-scanner'
119
142
 
120
143
        return scanner
130
153
    def testScanDispatchForResetBuilder(self):
131
154
        # A job gets dispatched to the sampledata builder after it's reset.
132
155
 
133
 
        # Reset sampledata builder.
134
 
        builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME]
135
 
        self._resetBuilder(builder)
136
 
        builder.setSlaveForTesting(OkSlave())
137
 
        # Set this to 1 here so that _checkDispatch can make sure it's
138
 
        # reset to 0 after a successful dispatch.
139
 
        builder.failure_count = 1
 
156
        # Obtain a builder.   Initialize failure count to 1 so that
 
157
        # _checkDispatch can make sure that a successful dispatch resets
 
158
        # the count to 0.
 
159
        with BuilddManagerTestFixture.extraSetUp():
 
160
            builder = self.getFreshBuilder(failure_count=1)
140
161
 
141
162
        # Run 'scan' and check its result.
142
 
        self.layer.txn.commit()
143
163
        self.layer.switchDbUser(config.builddmaster.dbuser)
144
164
        scanner = self._getScanner()
145
165
        d = defer.maybeDeferred(scanner.scan)
153
173
        to the asynchonous dispatcher and the builder remained active
154
174
        and IDLE.
155
175
        """
156
 
        self.assertTrue(slave is None, "Unexpected slave.")
 
176
        self.assertIs(None, slave, "Unexpected slave.")
157
177
 
158
178
        builder = getUtility(IBuilderSet).get(builder.id)
159
179
        self.assertTrue(builder.builderok)
160
 
        self.assertTrue(builder.currentjob is None)
 
180
        self.assertIs(None, builder.currentjob)
161
181
 
162
182
    def testNoDispatchForMissingChroots(self):
163
183
        # When a required chroot is not present the `scan` method
164
184
        # should not return any `RecordingSlaves` to be processed
165
185
        # and the builder used should remain active and IDLE.
166
 
 
167
 
        # Reset sampledata builder.
168
 
        builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME]
169
 
        self._resetBuilder(builder)
170
 
 
171
 
        # Remove hoary/i386 chroot.
172
 
        login('foo.bar@canonical.com')
173
 
        ubuntu = getUtility(IDistributionSet).getByName('ubuntu')
174
 
        hoary = ubuntu.getSeries('hoary')
175
 
        pocket_chroot = hoary.getDistroArchSeries('i386').getPocketChroot()
176
 
        removeSecurityProxy(pocket_chroot).chroot = None
177
 
        transaction.commit()
 
186
        with BuilddManagerTestFixture.extraSetUp():
 
187
            builder = self.getFreshBuilder()
 
188
            # Remove hoary/i386 chroot.
 
189
            login('foo.bar@canonical.com')
 
190
            ubuntu = getUtility(IDistributionSet).getByName('ubuntu')
 
191
            hoary = ubuntu.getSeries('hoary')
 
192
            pocket_chroot = (
 
193
                hoary.getDistroArchSeries('i386').getPocketChroot())
 
194
            removeSecurityProxy(pocket_chroot).chroot = None
 
195
 
178
196
        login(ANONYMOUS)
179
197
 
180
198
        # Run 'scan' and check its result.
214
232
 
215
233
        # Disable the sampledata builder
216
234
        login('foo.bar@canonical.com')
217
 
        builder.builderok = False
218
 
        transaction.commit()
 
235
        with BuilddManagerTestFixture.extraSetUp():
 
236
            builder.builderok = False
219
237
        login(ANONYMOUS)
220
238
 
221
239
        # Run 'scan' and check its result.
245
263
        builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME]
246
264
 
247
265
        login('foo.bar@canonical.com')
248
 
        builder.builderok = True
249
 
        builder.setSlaveForTesting(BuildingSlave(build_id='8-1'))
250
 
        transaction.commit()
 
266
        with BuilddManagerTestFixture.extraSetUp():
 
267
            builder.builderok = True
 
268
            builder.setSlaveForTesting(BuildingSlave(build_id='8-1'))
251
269
        login(ANONYMOUS)
252
270
 
253
271
        job = builder.currentjob
261
279
        return d
262
280
 
263
281
    def test_scan_with_nothing_to_dispatch(self):
264
 
        factory = LaunchpadObjectFactory()
265
 
        builder = factory.makeBuilder()
266
 
        builder.setSlaveForTesting(OkSlave())
 
282
        with BuilddManagerTestFixture.extraSetUp():
 
283
            builder = self.factory.makeBuilder()
 
284
            builder.setSlaveForTesting(OkSlave())
267
285
        scanner = self._getScanner(builder_name=builder.name)
268
286
        d = scanner.scan()
269
287
        return d.addCallback(self._checkNoDispatch, builder)
270
288
 
271
289
    def test_scan_with_manual_builder(self):
272
290
        # Reset sampledata builder.
273
 
        builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME]
274
 
        self._resetBuilder(builder)
275
 
        builder.setSlaveForTesting(OkSlave())
276
 
        builder.manual = True
 
291
        with BuilddManagerTestFixture.extraSetUp():
 
292
            builder = self.getFreshBuilder()
 
293
            builder.manual = True
277
294
        scanner = self._getScanner()
278
295
        d = scanner.scan()
279
296
        d.addCallback(self._checkNoDispatch, builder)
281
298
 
282
299
    def test_scan_with_not_ok_builder(self):
283
300
        # Reset sampledata builder.
284
 
        builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME]
285
 
        self._resetBuilder(builder)
286
 
        builder.setSlaveForTesting(OkSlave())
287
 
        builder.builderok = False
 
301
        with BuilddManagerTestFixture.extraSetUp():
 
302
            builder = self.getFreshBuilder()
 
303
            builder.builderok = False
288
304
        scanner = self._getScanner()
289
305
        d = scanner.scan()
290
306
        # Because the builder is not ok, we can't use _checkNoDispatch.
293
309
        return d
294
310
 
295
311
    def test_scan_of_broken_slave(self):
296
 
        builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME]
297
 
        self._resetBuilder(builder)
298
 
        builder.setSlaveForTesting(BrokenSlave())
299
 
        builder.failure_count = 0
 
312
        with BuilddManagerTestFixture.extraSetUp():
 
313
            builder = self.getFreshBuilder(slave=BrokenSlave())
300
314
        scanner = self._getScanner(builder_name=builder.name)
301
315
        d = scanner.scan()
302
316
        return assert_fails_with(d, xmlrpclib.Fault)
303
317
 
304
318
    def _assertFailureCounting(self, builder_count, job_count,
305
319
                               expected_builder_count, expected_job_count):
 
320
        # Avoid circular imports.
 
321
        from lp.buildmaster import manager as manager_module
 
322
 
306
323
        # If scan() fails with an exception, failure_counts should be
307
324
        # incremented.  What we do with the results of the failure
308
325
        # counts is tested below separately, this test just makes sure that
309
326
        # scan() is setting the counts.
310
327
        def failing_scan():
311
328
            return defer.fail(Exception("fake exception"))
312
 
        scanner = self._getScanner()
313
 
        scanner.scan = failing_scan
314
 
        from lp.buildmaster import manager as manager_module
315
 
        self.patch(manager_module, 'assessFailureCounts', FakeMethod())
316
 
        builder = getUtility(IBuilderSet)[scanner.builder_name]
317
 
 
318
 
        builder.failure_count = builder_count
319
 
        builder.currentjob.specific_job.build.failure_count = job_count
320
 
        # The _scanFailed() calls abort, so make sure our existing
321
 
        # failure counts are persisted.
322
 
        self.layer.txn.commit()
 
329
 
 
330
        with BuilddManagerTestFixture.extraSetUp():
 
331
            scanner = self._getScanner()
 
332
            scanner.scan = failing_scan
 
333
            self.patch(manager_module, 'assessFailureCounts', FakeMethod())
 
334
            builder = getUtility(IBuilderSet)[scanner.builder_name]
 
335
 
 
336
            builder.failure_count = builder_count
 
337
            builder.currentjob.specific_job.build.failure_count = job_count
 
338
            # The _scanFailed() calls abort, so make sure our existing failure
 
339
            # counts are persisted by exiting the extraSetUp() context (which
 
340
            # commits).
323
341
 
324
342
        # singleCycle() calls scan() which is our fake one that throws an
325
343
        # exception.
365
383
        scanner = self._getScanner()
366
384
        scanner.scan = failing_scan
367
385
        builder = getUtility(IBuilderSet)[scanner.builder_name]
368
 
        builder.failure_count = Builder.FAILURE_THRESHOLD
369
 
        builder.currentjob.reset()
370
 
        self.layer.txn.commit()
 
386
        with BuilddManagerTestFixture.extraSetUp():
 
387
            builder.failure_count = Builder.FAILURE_THRESHOLD
 
388
            builder.currentjob.reset()
371
389
 
372
390
        d = scanner.singleCycle()
373
391
 
388
406
        # Reset sampledata builder.
389
407
        builder = removeSecurityProxy(
390
408
            getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME])
391
 
        self._resetBuilder(builder)
392
 
        self.assertEqual(0, builder.failure_count)
393
 
        builder.setSlaveForTesting(slave)
394
 
        builder.vm_host = "fake_vm_host"
 
409
        with BuilddManagerTestFixture.extraSetUp():
 
410
            self._resetBuilder(builder)
 
411
            self.assertEqual(0, builder.failure_count)
 
412
            builder.setSlaveForTesting(slave)
 
413
            builder.vm_host = "fake_vm_host"
395
414
 
396
415
        scanner = self._getScanner()
397
416
 
398
417
        # Get the next job that will be dispatched.
399
418
        job = removeSecurityProxy(builder._findBuildCandidate())
400
 
        job.virtualized = True
401
 
        builder.virtualized = True
 
419
        with BuilddManagerTestFixture.extraSetUp():
 
420
            job.virtualized = True
 
421
            builder.virtualized = True
402
422
        d = scanner.singleCycle()
403
423
 
404
424
        def check(ignored):
430
450
        # Set the sample data builder building with the slave from above.
431
451
        builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME]
432
452
        login('foo.bar@canonical.com')
433
 
        builder.builderok = True
434
 
        # For now, we can only cancel virtual builds.
435
 
        builder.virtualized = True
436
 
        builder.vm_host = "fake_vm_host"
437
 
        builder.setSlaveForTesting(slave)
438
 
        transaction.commit()
 
453
        with BuilddManagerTestFixture.extraSetUp():
 
454
            builder.builderok = True
 
455
            # For now, we can only cancel virtual builds.
 
456
            builder.virtualized = True
 
457
            builder.vm_host = "fake_vm_host"
 
458
            builder.setSlaveForTesting(slave)
439
459
        login(ANONYMOUS)
440
460
        buildqueue = builder.currentjob
441
461
        self.assertBuildingJob(buildqueue, builder)
442
462
 
443
463
        # Now set the build to CANCELLING.
444
464
        build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(buildqueue)
445
 
        build.status = BuildStatus.CANCELLING
 
465
        with BuilddManagerTestFixture.extraSetUp():
 
466
            build.status = BuildStatus.CANCELLING
446
467
 
447
468
        # Run 'scan' and check its results.
448
469
        self.layer.switchDbUser(config.builddmaster.dbuser)
459
480
        d.addCallback(check_cancelled, builder, buildqueue)
460
481
        return d
461
482
 
 
483
    def makeFakeFailure(self):
 
484
        """Produce a fake failure for use with SlaveScanner._scanFailed."""
 
485
        FakeFailure = namedtuple('FakeFailure', ['getErrorMessage', 'check'])
 
486
        return FakeFailure(
 
487
            FakeMethod(self.factory.getUniqueString()),
 
488
            FakeMethod(True))
 
489
 
 
490
    def test_interleaved_success_and_failure_do_not_interfere(self):
 
491
        # It's possible for one builder to fail while another continues
 
492
        # to function properly.  When that happens, the failed builder
 
493
        # may cause database changes to be rolled back.  But that does
 
494
        # not affect the functioning builder.
 
495
        clock = task.Clock()
 
496
 
 
497
        with BuilddManagerTestFixture.extraSetUp():
 
498
            broken_builder = self.getFreshBuilder(
 
499
                slave=BrokenSlave(), name=BOB_THE_BUILDER_NAME)
 
500
            broken_scanner = self._getScanner(
 
501
                builder_name=broken_builder.name)
 
502
            good_builder = self.getFreshBuilder(
 
503
                slave=WaitingSlave(), name=FROG_THE_BUILDER_NAME)
 
504
            good_build = self.factory.makeBinaryPackageBuild(
 
505
                distroarchseries=self.factory.makeDistroArchSeries())
 
506
 
 
507
            # The good build is being handled by the good builder.
 
508
            buildqueue = good_build.queueBuild()
 
509
            buildqueue.builder = good_builder
 
510
 
 
511
            removeSecurityProxy(
 
512
                good_build.build_farm_job).date_started = UTC_NOW
 
513
 
 
514
        # The good builder requests information from a successful build,
 
515
        # and up receiving it, updates the build's metadata.
 
516
        # Our dependencies string goes into the build, and its
 
517
        # date_finished will be set.
 
518
        dependencies = self.factory.getUniqueString()
 
519
        PackageBuild.storeBuildInfo(
 
520
            good_build, None, {'dependencies': dependencies})
 
521
        clock.advance(1)
 
522
 
 
523
        # The broken scanner experiences a failure before the good
 
524
        # scanner is receiving its data.  This aborts the ongoing
 
525
        # transaction.
 
526
        # As a somewhat weird example, if the builder changed its own
 
527
        # title, that change will be rolled back.
 
528
        original_broken_builder_title = broken_builder.title
 
529
        broken_builder.title = self.factory.getUniqueString()
 
530
        broken_scanner._scanFailed(self.makeFakeFailure())
 
531
 
 
532
        # The work done by the good scanner is retained.  The
 
533
        # storeBuildInfo code committed it.
 
534
        self.assertEqual(dependencies, good_build.dependencies)
 
535
        self.assertIsNot(None, good_build.date_finished)
 
536
 
 
537
        # The work done by the broken scanner is rolled back.
 
538
        self.assertEqual(original_broken_builder_title, broken_builder.title)
 
539
 
462
540
 
463
541
class TestCancellationChecking(TestCaseWithFactory):
464
542
    """Unit tests for the checkCancellation method."""
475
553
        self.scanner.builder = self.builder
476
554
        self.scanner.logger.name = 'slave-scanner'
477
555
 
 
556
        self.useFixture(BuilddManagerTestFixture())
 
557
 
478
558
    def test_ignores_nonvirtual(self):
479
559
        # If the builder is nonvirtual make sure we return False.
480
 
        self.builder.virtualized = False
 
560
        with BuilddManagerTestFixture.extraSetUp():
 
561
            self.builder.virtualized = False
481
562
        d = self.scanner.checkCancellation(self.builder)
482
563
        return d.addCallback(self.assertFalse)
483
564
 
485
566
        # If the builder has no buildqueue associated,
486
567
        # make sure we return False.
487
568
        buildqueue = self.builder.currentjob
488
 
        buildqueue.reset()
 
569
        with BuilddManagerTestFixture.extraSetUp():
 
570
            buildqueue.reset()
489
571
        d = self.scanner.checkCancellation(self.builder)
490
572
        return d.addCallback(self.assertFalse)
491
573
 
493
575
        # If the active build is not in a CANCELLING state, ignore it.
494
576
        buildqueue = self.builder.currentjob
495
577
        build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(buildqueue)
496
 
        build.status = BuildStatus.BUILDING
 
578
        with BuilddManagerTestFixture.extraSetUp():
 
579
            build.status = BuildStatus.BUILDING
497
580
        d = self.scanner.checkCancellation(self.builder)
498
581
        return d.addCallback(self.assertFalse)
499
582
 
507
590
            return defer.succeed((None, None, 0))
508
591
        slave = OkSlave()
509
592
        slave.resume = fake_resume
510
 
        self.builder.vm_host = "fake_vm_host"
511
 
        self.builder.setSlaveForTesting(slave)
512
 
        buildqueue = self.builder.currentjob
513
 
        build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(buildqueue)
514
 
        build.status = BuildStatus.CANCELLING
 
593
 
 
594
        with BuilddManagerTestFixture.extraSetUp():
 
595
            self.builder.vm_host = "fake_vm_host"
 
596
            self.builder.setSlaveForTesting(slave)
 
597
            buildqueue = self.builder.currentjob
 
598
            build = getUtility(
 
599
                IBinaryPackageBuildSet).getByQueueEntry(buildqueue)
 
600
            build.status = BuildStatus.CANCELLING
515
601
 
516
602
        def check(result):
517
603
            self.assertEqual(1, call_counter.call_count)