1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
|
# Copyright 2010 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""An `IBuildFarmJobBehavior` for `TranslationTemplatesBuildJob`.
Dispatches translation template build jobs to build-farm slaves.
"""
__metaclass__ = type
__all__ = [
'TranslationTemplatesBuildBehavior',
]
import os
import tempfile
from twisted.internet import defer
from zope.component import getUtility
from zope.interface import implements
from zope.security.proxy import removeSecurityProxy
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
from lp.buildmaster.interfaces.buildfarmjobbehavior import (
IBuildFarmJobBehavior,
)
from lp.buildmaster.model.buildfarmjobbehavior import BuildFarmJobBehaviorBase
from lp.registry.interfaces.productseries import IProductSeriesSet
from lp.translations.interfaces.translationimportqueue import (
ITranslationImportQueue,
)
from lp.translations.model.approver import TranslationBuildApprover
class TranslationTemplatesBuildBehavior(BuildFarmJobBehaviorBase):
"""Dispatches `TranslationTemplateBuildJob`s to slaves."""
implements(IBuildFarmJobBehavior)
# Identify the type of job to the slave.
build_type = 'translation-templates'
# Filename for the tarball of templates that the slave builds.
templates_tarball_path = 'translation-templates.tar.gz'
def dispatchBuildToSlave(self, build_queue_item, logger):
"""See `IBuildFarmJobBehavior`."""
chroot = self._getChroot()
chroot_sha1 = chroot.content.sha1
d = self._builder.slave.cacheFile(logger, chroot)
def got_cache_file(ignored):
cookie = self.buildfarmjob.generateSlaveBuildCookie()
args = {
'arch_tag': self._getDistroArchSeries().architecturetag,
'branch_url': self.buildfarmjob.branch.composePublicURL(),
}
filemap = {}
return self._builder.slave.build(
cookie, self.build_type, chroot_sha1, filemap, args)
return d.addCallback(got_cache_file)
def _getChroot(self):
return self._getDistroArchSeries().getChroot()
def _getDistroArchSeries(self):
ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
return ubuntu.currentseries.nominatedarchindep
def logStartBuild(self, logger):
"""See `IBuildFarmJobBehavior`."""
logger.info(
"Starting templates build %s for %s." % (
self.buildfarmjob.getName(),
self.buildfarmjob.branch.bzr_identity))
def _readTarball(self, buildqueue, filemap, logger):
"""Read tarball with generated translation templates from slave."""
if filemap is None:
logger.error("Slave returned no filemap.")
return defer.succeed(None)
slave_filename = filemap.get(self.templates_tarball_path)
if slave_filename is None:
logger.error("Did not find templates tarball in slave output.")
return defer.succeed(None)
slave = removeSecurityProxy(buildqueue.builder.slave)
fd, fname = tempfile.mkstemp()
tarball_file = os.fdopen(fd, 'wb')
d = slave.getFile(slave_filename, tarball_file)
# getFile will close the file object.
return d.addCallback(lambda ignored: fname)
def _uploadTarball(self, branch, tarball, logger):
"""Upload tarball to productseries that want it."""
queue = getUtility(ITranslationImportQueue)
productseriesset = getUtility(IProductSeriesSet)
related_series = (
productseriesset.findByTranslationsImportBranch(branch))
for series in related_series:
queue.addOrUpdateEntriesFromTarball(
tarball, False, branch.owner, productseries=series,
approver_factory=TranslationBuildApprover)
def updateSlaveStatus(self, raw_slave_status, status):
"""See `IBuildFarmJobBehavior`."""
if status['builder_status'] == 'BuilderStatus.WAITING':
if len(raw_slave_status) >= 4:
status['filemap'] = raw_slave_status[3]
def updateBuild_WAITING(self, queue_item, slave_status, logtail, logger):
"""Deal with a finished ("WAITING" state, perversely) build job.
Retrieves tarball and logs from the slave, then cleans up the
slave so it's ready for a next job and destroys the queue item.
If this fails for whatever unforeseen reason, a future run will
retry it.
"""
build_status = self.extractBuildStatus(slave_status)
logger.info(
"Templates generation job %s for %s finished with status %s." % (
queue_item.specific_job.getName(),
queue_item.specific_job.branch.bzr_identity,
build_status))
def clean_slave(ignored):
d = queue_item.builder.cleanSlave()
return d.addCallback(lambda ignored: queue_item.destroySelf())
def got_tarball(filename):
# XXX 2010-11-12 bug=674575
# Please make addOrUpdateEntriesFromTarball() take files on
# disk; reading arbitrarily sized files into memory is
# dangerous.
if filename is None:
logger.error("Build produced no tarball.")
return
tarball_file = open(filename)
try:
tarball = tarball_file.read()
if tarball is None:
logger.error("Build produced empty tarball.")
else:
logger.debug("Uploading translation templates tarball.")
self._uploadTarball(
queue_item.specific_job.branch, tarball, logger)
logger.debug("Upload complete.")
finally:
tarball_file.close()
os.remove(filename)
if build_status == 'OK':
logger.debug("Processing successful templates build.")
filemap = slave_status.get('filemap')
d = self._readTarball(queue_item, filemap, logger)
d.addCallback(got_tarball)
d.addCallback(clean_slave)
return d
return clean_slave(None)
|