← Back to branch summary

~launchpad-pqm/launchpad/devel

« back to all changes in this revision

Viewing changes to lib/lp/buildmaster/model/buildfarmjobbehavior.py

[r=allenap][bug=905853, 905855,
        906079] Revert r14499 and r14459 because read-only transactions in
        buildmaster are causing production issues.

Show diffs side-by-side

added added

removed removed

Lines of Context:
lib/lp/buildmaster/model/buildfarmjobbehavior.py
1
 
# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
 
1
# Copyright 2009-2010 Canonical Ltd.  This software is licensed under the
2
2
# GNU Affero General Public License version 3 (see the file LICENSE).
3
3
 
4
4
# pylint: disable-msg=E0211,E0213
16
16
import socket
17
17
import xmlrpclib
18
18
 
19
 
import transaction
20
19
from twisted.internet import defer
 
20
 
21
21
from zope.component import getUtility
22
22
from zope.interface import implements
23
23
from zope.security.proxy import removeSecurityProxy
32
32
    IBuildFarmJobBehavior,
33
33
    )
34
34
from lp.services import encoding
35
 
from lp.services.database.transaction_policy import DatabaseTransactionPolicy
36
35
from lp.services.job.interfaces.job import JobStatus
37
36
 
38
37
 
71
70
        if slave_build_cookie != expected_cookie:
72
71
            raise CorruptBuildCookie("Invalid slave build cookie.")
73
72
 
74
 
    def _getBuilderStatusHandler(self, status_text, logger):
75
 
        """Look up the handler method for a given builder status.
76
 
 
77
 
        If status is not a known one, logs an error and returns None.
78
 
        """
79
 
        builder_status_handlers = {
80
 
            'BuilderStatus.IDLE': self.updateBuild_IDLE,
81
 
            'BuilderStatus.BUILDING': self.updateBuild_BUILDING,
82
 
            'BuilderStatus.ABORTING': self.updateBuild_ABORTING,
83
 
            'BuilderStatus.ABORTED': self.updateBuild_ABORTED,
84
 
            'BuilderStatus.WAITING': self.updateBuild_WAITING,
85
 
            }
86
 
        handler = builder_status_handlers.get(status_text)
87
 
        if handler is None:
88
 
            logger.critical(
89
 
                "Builder on %s returned unknown status %s; failing it.",
90
 
                self._builder.url, status_text)
91
 
        return handler
92
 
 
93
73
    def updateBuild(self, queueItem):
94
74
        """See `IBuildFarmJobBehavior`."""
95
75
        logger = logging.getLogger('slave-scanner')
97
77
        d = self._builder.slaveStatus()
98
78
 
99
79
        def got_failure(failure):
100
 
            transaction.abort()
101
80
            failure.trap(xmlrpclib.Fault, socket.error)
102
81
            info = failure.value
103
82
            info = ("Could not contact the builder %s, caught a (%s)"
105
84
            raise BuildSlaveFailure(info)
106
85
 
107
86
        def got_status(slave_status):
 
87
            builder_status_handlers = {
 
88
                'BuilderStatus.IDLE': self.updateBuild_IDLE,
 
89
                'BuilderStatus.BUILDING': self.updateBuild_BUILDING,
 
90
                'BuilderStatus.ABORTING': self.updateBuild_ABORTING,
 
91
                'BuilderStatus.ABORTED': self.updateBuild_ABORTED,
 
92
                'BuilderStatus.WAITING': self.updateBuild_WAITING,
 
93
                }
 
94
 
108
95
            builder_status = slave_status['builder_status']
109
 
            status_handler = self._getBuilderStatusHandler(
110
 
                builder_status, logger)
111
 
            if status_handler is None:
112
 
                error = (
 
96
            if builder_status not in builder_status_handlers:
 
97
                logger.critical(
 
98
                    "Builder on %s returned unknown status %s, failing it"
 
99
                    % (self._builder.url, builder_status))
 
100
                self._builder.failBuilder(
113
101
                    "Unknown status code (%s) returned from status() probe."
114
102
                    % builder_status)
115
 
                transaction.commit()
116
 
                with DatabaseTransactionPolicy(read_only=False):
117
 
                    self._builder.failBuilder(error)
118
 
                    # XXX: This will leave the build and job in a bad
119
 
                    # state, but should never be possible since our
120
 
                    # builder statuses are known.
121
 
                    queueItem._builder = None
122
 
                    queueItem.setDateStarted(None)
123
 
                    transaction.commit()
 
103
                # XXX: This will leave the build and job in a bad state, but
 
104
                # should never be possible, since our builder statuses are
 
105
                # known.
 
106
                queueItem._builder = None
 
107
                queueItem.setDateStarted(None)
124
108
                return
125
109
 
126
110
            # Since logtail is a xmlrpclib.Binary container and it is
130
114
            # will simply remove the proxy.
131
115
            logtail = removeSecurityProxy(slave_status.get('logtail'))
132
116
 
 
117
            method = builder_status_handlers[builder_status]
133
118
            return defer.maybeDeferred(
134
 
                status_handler, queueItem, slave_status, logtail, logger)
 
119
                method, queueItem, slave_status, logtail, logger)
135
120
 
136
121
        d.addErrback(got_failure)
137
122
        d.addCallback(got_status)
143
128
        Log this and reset the record.
144
129
        """
145
130
        logger.warn(
146
 
            "Builder %s forgot about buildqueue %d -- "
147
 
            "resetting buildqueue record.",
148
 
            queueItem.builder.url, queueItem.id)
149
 
        transaction.commit()
150
 
        with DatabaseTransactionPolicy(read_only=False):
151
 
            queueItem.reset()
152
 
            transaction.commit()
 
131
            "Builder %s forgot about buildqueue %d -- resetting buildqueue "
 
132
            "record" % (queueItem.builder.url, queueItem.id))
 
133
        queueItem.reset()
153
134
 
154
135
    def updateBuild_BUILDING(self, queueItem, slave_status, logtail, logger):
155
136
        """Build still building, collect the logtail"""
156
 
        transaction.commit()
157
 
        with DatabaseTransactionPolicy(read_only=False):
158
 
            if queueItem.job.status != JobStatus.RUNNING:
159
 
                queueItem.job.start()
160
 
            queueItem.logtail = encoding.guess(str(logtail))
161
 
            transaction.commit()
 
137
        if queueItem.job.status != JobStatus.RUNNING:
 
138
            queueItem.job.start()
 
139
        queueItem.logtail = encoding.guess(str(logtail))
162
140
 
163
141
    def updateBuild_ABORTING(self, queueItem, slave_status, logtail, logger):
164
142
        """Build was ABORTED.
165
143
 
166
144
        Master-side should wait until the slave finish the process correctly.
167
145
        """
168
 
        transaction.commit()
169
 
        with DatabaseTransactionPolicy(read_only=False):
170
 
            queueItem.logtail = "Waiting for slave process to be terminated"
171
 
            transaction.commit()
 
146
        queueItem.logtail = "Waiting for slave process to be terminated"
172
147
 
173
148
    def updateBuild_ABORTED(self, queueItem, slave_status, logtail, logger):
174
149
        """ABORTING process has successfully terminated.
176
151
        Clean the builder for another jobs.
177
152
        """
178
153
        d = queueItem.builder.cleanSlave()
179
 
 
180
154
        def got_cleaned(ignored):
181
 
            transaction.commit()
182
 
            with DatabaseTransactionPolicy(read_only=False):
183
 
                queueItem.builder = None
184
 
                if queueItem.job.status != JobStatus.FAILED:
185
 
                    queueItem.job.fail()
186
 
                queueItem.specific_job.jobAborted()
187
 
                transaction.commit()
188
 
 
 
155
            queueItem.builder = None
 
156
            if queueItem.job.status != JobStatus.FAILED:
 
157
                queueItem.job.fail()
 
158
            queueItem.specific_job.jobAborted()
189
159
        return d.addCallback(got_cleaned)
190
160
 
191
161
    def extractBuildStatus(self, slave_status):