133
158
def is_private(self):
134
"""See `IBuildBase`"""
159
"""See `IBuildFarmJob`"""
135
160
return self.archive.private
137
162
def getUploadDirLeaf(self, build_cookie, now=None):
138
163
"""See `IPackageBuild`."""
139
return BuildBase.getUploadDirLeaf(build_cookie, now)
165
now = datetime.datetime.now()
166
timestamp = now.strftime("%Y%m%d-%H%M%S")
167
return '%s-%s' % (timestamp, build_cookie)
141
169
def getUploadDir(self, upload_leaf):
142
170
"""See `IPackageBuild`."""
143
return BuildBase.getUploadDir(upload_leaf)
171
return os.path.join(config.builddmaster.root, 'incoming', upload_leaf)
146
174
def getUploaderCommand(package_build, upload_leaf, upload_logfilename):
147
175
"""See `IPackageBuild`."""
148
return BuildBase.getUploaderCommand(
149
package_build, upload_leaf, upload_logfilename)
176
root = os.path.abspath(config.builddmaster.root)
177
uploader_command = list(config.builddmaster.uploader.split())
179
# Add extra arguments for processing a package upload.
181
"--log-file", "%s" % upload_logfilename,
182
"-d", "%s" % package_build.distribution.name,
184
package_build.distro_series.getSuite(package_build.pocket)),
185
"-b", "%s" % package_build.id,
186
"-J", "%s" % upload_leaf,
187
'--context=%s' % package_build.policy_name,
191
uploader_command.extend(extra_args)
192
return uploader_command
152
195
def getLogFromSlave(package_build):
153
196
"""See `IPackageBuild`."""
154
return BuildBase.getLogFromSlave(package_build)
197
builder = package_build.buildqueue_record.builder
198
return builder.transferSlaveFileToLibrarian(
200
package_build.buildqueue_record.getLogFileName(),
201
package_build.is_private)
157
204
def getUploadLogContent(root, leaf):
158
"""See `IPackageBuild`."""
159
return BuildBase.getUploadLogContent(root, leaf)
205
"""Retrieve the upload log contents.
207
:param root: Root directory for the uploads
208
:param leaf: Leaf for this particular upload
209
:return: Contents of log file or message saying no log file was found.
211
# Retrieve log file content.
212
possible_locations = (
213
'failed', 'failed-to-move', 'rejected', 'accepted')
214
for location_dir in possible_locations:
215
log_filepath = os.path.join(root, location_dir, leaf,
217
if os.path.exists(log_filepath):
218
with open(log_filepath, 'r') as uploader_log_file:
219
return uploader_log_file.read()
221
return 'Could not find upload log file'
161
223
def estimateDuration(self):
162
224
"""See `IPackageBuild`."""
163
225
raise NotImplementedError
166
def storeBuildInfo(package_build, librarian, slave_status):
228
def storeBuildInfo(build, librarian, slave_status):
167
229
"""See `IPackageBuild`."""
168
return BuildBase.storeBuildInfo(package_build, librarian,
230
# log, builder and date_finished are read-only, so we must
231
# currently remove the security proxy to set them.
232
naked_build = removeSecurityProxy(build)
233
naked_build.log = build.getLogFromSlave(build)
234
naked_build.builder = build.buildqueue_record.builder
235
# XXX cprov 20060615 bug=120584: Currently buildduration includes
236
# the scanner latency, it should really be asking the slave for
237
# the duration spent building locally.
238
naked_build.date_finished = datetime.datetime.now(pytz.UTC)
239
if slave_status.get('dependencies') is not None:
240
build.dependencies = unicode(slave_status.get('dependencies'))
242
build.dependencies = None
171
244
def verifySuccessfulUpload(self):
172
245
"""See `IPackageBuild`."""
173
246
raise NotImplementedError
248
def createUploadLog(self, content, filename=None):
249
"""Creates a file on the librarian for the upload log.
251
:return: ILibraryFileAlias for the upload log file.
253
# The given content is stored in the librarian, restricted as
254
# necessary according to the targeted archive's privacy. The content
255
# object's 'upload_log' attribute will point to the
256
# `LibrarianFileAlias`.
258
assert self.upload_log is None, (
259
"Upload log information already exists and cannot be overridden.")
262
filename = 'upload_%s_log.txt' % self.id
263
contentType = filenameToContentType(filename)
264
file_size = len(content)
265
file_content = StringIO(content)
266
restricted = self.is_private
268
return getUtility(ILibraryFileAliasSet).create(
269
filename, file_size, file_content, contentType=contentType,
270
restricted=restricted)
175
272
def storeUploadLog(self, content):
176
273
"""See `IPackageBuild`."""
177
274
filename = "upload_%s_log.txt" % self.build_farm_job.id
178
library_file = BuildBase.createUploadLog(
179
self, content, filename=filename)
275
library_file = self.createUploadLog(content, filename=filename)
180
276
self.upload_log = library_file
182
278
def notify(self, extra_info=None):
203
299
def queueBuild(self, suspended=False):
204
300
"""See `IPackageBuild`."""
205
return BuildBase.queueBuild(self, suspended=suspended)
301
specific_job = self.makeJob()
303
# This build queue job is to be created in a suspended state.
305
specific_job.job.suspend()
307
duration_estimate = self.estimateDuration()
308
queue_entry = BuildQueue(
309
estimated_duration=duration_estimate,
310
job_type=self.build_farm_job_type,
311
job=specific_job.job, processor=specific_job.processor,
312
virtualized=specific_job.virtualized)
313
Store.of(self).add(queue_entry)
207
316
def handleStatus(self, status, librarian, slave_status):
208
317
"""See `IPackageBuild`."""
209
return handle_status_for_build(self, status, librarian, slave_status)
318
logger = logging.getLogger(BUILDD_MANAGER_LOG_NAME)
319
method = getattr(self, '_handleStatus_' + status, None)
321
logger.critical("Unknown BuildStatus '%s' for builder '%s'"
322
% (status, self.buildqueue_record.builder.url))
324
method(librarian, slave_status, logger)
211
# The following private handlers currently re-use the BuildBase
212
# implementation until it is no longer in use. If we find in the
213
# future that it would be useful to delegate these also, they can be
214
# added to IBuildFarmJob or IPackageBuild as necessary.
215
326
def _handleStatus_OK(self, librarian, slave_status, logger):
216
return BuildBase._handleStatus_OK(
217
self, librarian, slave_status, logger)
327
"""Handle a package that built successfully.
329
Once built successfully, we pull the files, store them in a
330
directory, store build information and push them through the
333
filemap = slave_status['filemap']
335
logger.info("Processing successful build %s from builder %s" % (
336
self.buildqueue_record.specific_job.build.title,
337
self.buildqueue_record.builder.name))
338
# Explode before collect a binary that is denied in this
339
# distroseries/pocket
340
if not self.archive.allowUpdatesToReleasePocket():
341
assert self.distro_series.canUploadToPocket(self.pocket), (
342
"%s (%s) can not be built for pocket %s: illegal status"
343
% (self.title, self.id, self.pocket.name))
345
# Ensure we have the correct build root as:
346
# <BUILDMASTER_ROOT>/incoming/<UPLOAD_LEAF>/<TARGET_PATH>/[FILES]
347
root = os.path.abspath(config.builddmaster.root)
349
# Create a single directory to store build result files.
350
upload_leaf = self.getUploadDirLeaf(
351
'%s-%s' % (self.id, self.buildqueue_record.id))
352
upload_dir = self.getUploadDir(upload_leaf)
353
logger.debug("Storing build result at '%s'" % upload_dir)
355
# Build the right UPLOAD_PATH so the distribution and archive
356
# can be correctly found during the upload:
357
# <archive_id>/distribution_name
358
# for all destination archive types.
359
archive = self.archive
360
distribution_name = self.distribution.name
361
target_path = '%s/%s' % (archive.id, distribution_name)
362
upload_path = os.path.join(upload_dir, target_path)
363
os.makedirs(upload_path)
365
slave = removeSecurityProxy(self.buildqueue_record.builder.slave)
366
successful_copy_from_slave = True
367
for filename in filemap:
368
logger.info("Grabbing file: %s" % filename)
369
out_file_name = os.path.join(upload_path, filename)
370
# If the evaluated output file name is not within our
371
# upload path, then we don't try to copy this or any
373
if not os.path.realpath(out_file_name).startswith(upload_path):
374
successful_copy_from_slave = False
376
"A slave tried to upload the file '%s' "
377
"for the build %d." % (filename, self.id))
379
out_file = open(out_file_name, "wb")
380
slave_file = slave.getFile(filemap[filename])
381
copy_and_close(slave_file, out_file)
383
# We only attempt the upload if we successfully copied all the
384
# files from the slave.
385
if successful_copy_from_slave:
386
uploader_logfilename = os.path.join(
387
upload_dir, UPLOAD_LOG_FILENAME)
388
uploader_command = self.getUploaderCommand(
389
self, upload_leaf, uploader_logfilename)
390
logger.debug("Saving uploader log at '%s'" % uploader_logfilename)
392
logger.info("Invoking uploader on %s" % root)
393
logger.info("%s" % uploader_command)
395
uploader_process = subprocess.Popen(
396
uploader_command, stdout=subprocess.PIPE,
397
stderr=subprocess.PIPE)
399
# Nothing should be written to the stdout/stderr.
400
upload_stdout, upload_stderr = uploader_process.communicate()
402
# XXX cprov 2007-04-17: we do not check uploader_result_code
403
# anywhere. We need to find out what will be best strategy
404
# when it failed HARD (there is a huge effort in process-upload
405
# to not return error, it only happen when the code is broken).
406
uploader_result_code = uploader_process.returncode
407
logger.info("Uploader returned %d" % uploader_result_code)
409
# Quick and dirty hack to carry on on process-upload failures
410
if os.path.exists(upload_dir):
411
logger.warning("The upload directory did not get moved.")
412
failed_dir = os.path.join(root, "failed-to-move")
413
if not os.path.exists(failed_dir):
415
os.rename(upload_dir, os.path.join(failed_dir, upload_leaf))
417
# The famous 'flush_updates + clear_cache' will make visible
418
# the DB changes done in process-upload, considering that the
419
# transaction was set with ISOLATION_LEVEL_READ_COMMITED
422
cur.execute('SHOW transaction_isolation')
423
isolation_str = cur.fetchone()[0]
424
assert isolation_str == 'read committed', (
425
'BuildMaster/BuilderGroup transaction isolation should be '
426
'ISOLATION_LEVEL_READ_COMMITTED (not "%s")' % isolation_str)
428
original_slave = self.buildqueue_record.builder.slave
430
# XXX Robert Collins, Celso Providelo 2007-05-26 bug=506256:
431
# 'Refreshing' objects procedure is forced on us by using a
432
# different process to do the upload, but as that process runs
433
# in the same unix account, it is simply double handling and we
434
# would be better off to do it within this process.
435
flush_database_updates()
436
clear_current_connection_cache()
438
# XXX cprov 2007-06-15: Re-issuing removeSecurityProxy is forced on
439
# us by sqlobject refreshing the builder object during the
440
# transaction cache clearing. Once we sort the previous problem
441
# this step should probably not be required anymore.
442
self.buildqueue_record.builder.setSlaveForTesting(
443
removeSecurityProxy(original_slave))
445
# Store build information, build record was already updated during
447
self.storeBuildInfo(self, librarian, slave_status)
449
# Retrive the up-to-date build record and perform consistency
450
# checks. The build record should be updated during the binary
451
# upload processing, if it wasn't something is broken and needs
452
# admins attention. Even when we have a FULLYBUILT build record,
453
# if it is not related with at least one binary, there is also
455
# For both situations we will mark the builder as FAILEDTOUPLOAD
456
# and the and update the build details (datebuilt, duration,
457
# buildlog, builder) in LP. A build-failure-notification will be
458
# sent to the lp-build-admin celebrity and to the sourcepackagerelease
459
# uploader about this occurrence. The failure notification will
460
# also contain the information required to manually reprocess the
461
# binary upload when it was the case.
462
if (self.status != BuildStatus.FULLYBUILT or
463
not successful_copy_from_slave or
464
not self.verifySuccessfulUpload()):
465
logger.warning("Build %s upload failed." % self.id)
466
self.status = BuildStatus.FAILEDTOUPLOAD
467
uploader_log_content = self.getUploadLogContent(root,
469
# Store the upload_log_contents in librarian so it can be
470
# accessed by anyone with permission to see the build.
471
self.storeUploadLog(uploader_log_content)
472
# Notify the build failure.
473
self.notify(extra_info=uploader_log_content)
476
"Gathered %s %d completely" % (
477
self.__class__.__name__, self.id))
479
# Release the builder for another job.
480
self.buildqueue_record.builder.cleanSlave()
481
# Remove BuildQueue record.
482
self.buildqueue_record.destroySelf()
219
484
def _handleStatus_PACKAGEFAIL(self, librarian, slave_status, logger):
220
return BuildBase._handleStatus_PACKAGEFAIL(
221
self, librarian, slave_status, logger)
485
"""Handle a package that had failed to build.
487
Build has failed when trying the work with the target package,
488
set the job status as FAILEDTOBUILD, store available info and
489
remove Buildqueue entry.
491
self.status = BuildStatus.FAILEDTOBUILD
492
self.storeBuildInfo(self, librarian, slave_status)
493
self.buildqueue_record.builder.cleanSlave()
495
self.buildqueue_record.destroySelf()
223
497
def _handleStatus_DEPFAIL(self, librarian, slave_status, logger):
224
return BuildBase._handleStatus_DEPFAIL(
225
self, librarian, slave_status, logger)
498
"""Handle a package that had missing dependencies.
500
Build has failed by missing dependencies, set the job status as
501
MANUALDEPWAIT, store available information, remove BuildQueue
502
entry and release builder slave for another job.
504
self.status = BuildStatus.MANUALDEPWAIT
505
self.storeBuildInfo(self, librarian, slave_status)
506
logger.critical("***** %s is MANUALDEPWAIT *****"
507
% self.buildqueue_record.builder.name)
508
self.buildqueue_record.builder.cleanSlave()
509
self.buildqueue_record.destroySelf()
227
511
def _handleStatus_CHROOTFAIL(self, librarian, slave_status, logger):
228
return BuildBase._handleStatus_CHROOTFAIL(
229
self, librarian, slave_status, logger)
512
"""Handle a package that had failed when unpacking the CHROOT.
514
Build has failed when installing the current CHROOT, mark the
515
job as CHROOTFAIL, store available information, remove BuildQueue
516
and release the builder.
518
self.status = BuildStatus.CHROOTWAIT
519
self.storeBuildInfo(self, librarian, slave_status)
520
logger.critical("***** %s is CHROOTWAIT *****" %
521
self.buildqueue_record.builder.name)
522
self.buildqueue_record.builder.cleanSlave()
524
self.buildqueue_record.destroySelf()
231
526
def _handleStatus_BUILDERFAIL(self, librarian, slave_status, logger):
232
return BuildBase._handleStatus_BUILDERFAIL(
233
self, librarian, slave_status, logger)
527
"""Handle builder failures.
529
Build has been failed when trying to build the target package,
530
The environment is working well, so mark the job as NEEDSBUILD again
531
and 'clean' the builder to do another jobs.
533
logger.warning("***** %s has failed *****"
534
% self.buildqueue_record.builder.name)
535
self.buildqueue_record.builder.failBuilder(
536
"Builder returned BUILDERFAIL when asked for its status")
538
self.storeBuildInfo(self, librarian, slave_status)
539
self.buildqueue_record.reset()
235
541
def _handleStatus_GIVENBACK(self, librarian, slave_status, logger):
236
return BuildBase._handleStatus_GIVENBACK(
237
self, librarian, slave_status, logger)
542
"""Handle automatic retry requested by builder.
544
GIVENBACK pseudo-state represents a request for automatic retry
545
later, the build records is delayed by reducing the lastscore to
548
logger.warning("***** %s is GIVENBACK by %s *****"
549
% (self.buildqueue_record.specific_job.build.title,
550
self.buildqueue_record.builder.name))
551
self.storeBuildInfo(self, librarian, slave_status)
552
# XXX cprov 2006-05-30: Currently this information is not
553
# properly presented in the Web UI. We will discuss it in
554
# the next Paris Summit, infinity has some ideas about how
555
# to use this content. For now we just ensure it's stored.
556
self.buildqueue_record.builder.cleanSlave()
557
self.buildqueue_record.reset()
240
560
class PackageBuildSet:
added
removed
lib/lp/buildmaster/model/packagebuild.py
