← Back to branch summary

~launchpad-pqm/launchpad/devel

« back to all changes in this revision

Viewing changes to lib/lp/buildmaster/model/buildfarmjobbehavior.py

[r=allenap, bac, gmb, julian-edwards, wallyworld][bug=905853, 905855,
 906079] In buildmaster,
 always shift into a read-write database transaction access mode before
 updating PackageBuild statuses. Shift into read-write transactions in
 appropriate places in TranslationTemplatesBuildBehavior. Ensure that all
 lp.buildmaster tests to which it is relevant are running with
 BuilddManagerTestFixture.

Show diffs side-by-side

added added

removed removed

Lines of Context:
lib/lp/buildmaster/model/buildfarmjobbehavior.py
1
 
# Copyright 2009-2010 Canonical Ltd.  This software is licensed under the
 
1
# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
2
2
# GNU Affero General Public License version 3 (see the file LICENSE).
3
3
 
4
4
# pylint: disable-msg=E0211,E0213
16
16
import socket
17
17
import xmlrpclib
18
18
 
 
19
import transaction
19
20
from twisted.internet import defer
20
21
from zope.component import getUtility
21
22
from zope.interface import implements
30
31
    IBuildFarmJobBehavior,
31
32
    )
32
33
from lp.services import encoding
 
34
from lp.services.database.transaction_policy import DatabaseTransactionPolicy
33
35
from lp.services.job.interfaces.job import JobStatus
34
36
from lp.services.librarian.interfaces.client import ILibrarianClient
35
37
 
69
71
        if slave_build_cookie != expected_cookie:
70
72
            raise CorruptBuildCookie("Invalid slave build cookie.")
71
73
 
 
74
    def _getBuilderStatusHandler(self, status_text, logger):
 
75
        """Look up the handler method for a given builder status.
 
76
 
 
77
        If status is not a known one, logs an error and returns None.
 
78
        """
 
79
        builder_status_handlers = {
 
80
            'BuilderStatus.IDLE': self.updateBuild_IDLE,
 
81
            'BuilderStatus.BUILDING': self.updateBuild_BUILDING,
 
82
            'BuilderStatus.ABORTING': self.updateBuild_ABORTING,
 
83
            'BuilderStatus.ABORTED': self.updateBuild_ABORTED,
 
84
            'BuilderStatus.WAITING': self.updateBuild_WAITING,
 
85
            }
 
86
        handler = builder_status_handlers.get(status_text)
 
87
        if handler is None:
 
88
            logger.critical(
 
89
                "Builder on %s returned unknown status %s; failing it.",
 
90
                self._builder.url, status_text)
 
91
        return handler
 
92
 
72
93
    def updateBuild(self, queueItem):
73
94
        """See `IBuildFarmJobBehavior`."""
74
95
        logger = logging.getLogger('slave-scanner')
76
97
        d = self._builder.slaveStatus()
77
98
 
78
99
        def got_failure(failure):
 
100
            transaction.abort()
79
101
            failure.trap(xmlrpclib.Fault, socket.error)
80
102
            info = failure.value
81
103
            info = ("Could not contact the builder %s, caught a (%s)"
83
105
            raise BuildSlaveFailure(info)
84
106
 
85
107
        def got_status(slave_status):
86
 
            builder_status_handlers = {
87
 
                'BuilderStatus.IDLE': self.updateBuild_IDLE,
88
 
                'BuilderStatus.BUILDING': self.updateBuild_BUILDING,
89
 
                'BuilderStatus.ABORTING': self.updateBuild_ABORTING,
90
 
                'BuilderStatus.ABORTED': self.updateBuild_ABORTED,
91
 
                'BuilderStatus.WAITING': self.updateBuild_WAITING,
92
 
                }
93
 
 
94
108
            builder_status = slave_status['builder_status']
95
 
            if builder_status not in builder_status_handlers:
96
 
                logger.critical(
97
 
                    "Builder on %s returned unknown status %s, failing it"
98
 
                    % (self._builder.url, builder_status))
99
 
                self._builder.failBuilder(
 
109
            status_handler = self._getBuilderStatusHandler(
 
110
                builder_status, logger)
 
111
            if status_handler is None:
 
112
                error = (
100
113
                    "Unknown status code (%s) returned from status() probe."
101
114
                    % builder_status)
102
 
                # XXX: This will leave the build and job in a bad state, but
103
 
                # should never be possible, since our builder statuses are
104
 
                # known.
105
 
                queueItem._builder = None
106
 
                queueItem.setDateStarted(None)
 
115
                transaction.commit()
 
116
                with DatabaseTransactionPolicy(read_only=False):
 
117
                    self._builder.failBuilder(error)
 
118
                    # XXX: This will leave the build and job in a bad
 
119
                    # state, but should never be possible since our
 
120
                    # builder statuses are known.
 
121
                    queueItem._builder = None
 
122
                    queueItem.setDateStarted(None)
 
123
                    transaction.commit()
107
124
                return
108
125
 
109
126
            # Since logtail is a xmlrpclib.Binary container and it is
113
130
            # will simply remove the proxy.
114
131
            logtail = removeSecurityProxy(slave_status.get('logtail'))
115
132
 
116
 
            method = builder_status_handlers[builder_status]
117
133
            return defer.maybeDeferred(
118
 
                method, queueItem, slave_status, logtail, logger)
 
134
                status_handler, queueItem, slave_status, logtail, logger)
119
135
 
120
136
        d.addErrback(got_failure)
121
137
        d.addCallback(got_status)
127
143
        Log this and reset the record.
128
144
        """
129
145
        logger.warn(
130
 
            "Builder %s forgot about buildqueue %d -- resetting buildqueue "
131
 
            "record" % (queueItem.builder.url, queueItem.id))
132
 
        queueItem.reset()
 
146
            "Builder %s forgot about buildqueue %d -- "
 
147
            "resetting buildqueue record.",
 
148
            queueItem.builder.url, queueItem.id)
 
149
        transaction.commit()
 
150
        with DatabaseTransactionPolicy(read_only=False):
 
151
            queueItem.reset()
 
152
            transaction.commit()
133
153
 
134
154
    def updateBuild_BUILDING(self, queueItem, slave_status, logtail, logger):
135
155
        """Build still building, collect the logtail"""
136
 
        if queueItem.job.status != JobStatus.RUNNING:
137
 
            queueItem.job.start()
138
 
        queueItem.logtail = encoding.guess(str(logtail))
 
156
        transaction.commit()
 
157
        with DatabaseTransactionPolicy(read_only=False):
 
158
            if queueItem.job.status != JobStatus.RUNNING:
 
159
                queueItem.job.start()
 
160
            queueItem.logtail = encoding.guess(str(logtail))
 
161
            transaction.commit()
139
162
 
140
163
    def updateBuild_ABORTING(self, queueItem, slave_status, logtail, logger):
141
164
        """Build was ABORTED.
142
165
 
143
166
        Master-side should wait until the slave finish the process correctly.
144
167
        """
145
 
        queueItem.logtail = "Waiting for slave process to be terminated"
 
168
        transaction.commit()
 
169
        with DatabaseTransactionPolicy(read_only=False):
 
170
            queueItem.logtail = "Waiting for slave process to be terminated"
 
171
            transaction.commit()
146
172
 
147
173
    def updateBuild_ABORTED(self, queueItem, slave_status, logtail, logger):
148
174
        """ABORTING process has successfully terminated.
150
176
        Clean the builder for another jobs.
151
177
        """
152
178
        d = queueItem.builder.cleanSlave()
 
179
 
153
180
        def got_cleaned(ignored):
154
 
            queueItem.builder = None
155
 
            if queueItem.job.status != JobStatus.FAILED:
156
 
                queueItem.job.fail()
157
 
            queueItem.specific_job.jobAborted()
 
181
            transaction.commit()
 
182
            with DatabaseTransactionPolicy(read_only=False):
 
183
                queueItem.builder = None
 
184
                if queueItem.job.status != JobStatus.FAILED:
 
185
                    queueItem.job.fail()
 
186
                queueItem.specific_job.jobAborted()
 
187
                transaction.commit()
 
188
 
158
189
        return d.addCallback(got_cleaned)
159
190
 
160
191
    def extractBuildStatus(self, slave_status):