1
# Copyright 2009, 2010 Canonical Ltd. This software is licensed under the
2
# GNU Affero General Public License version 3 (see the file LICENSE).
4
# Authors: Daniel Silverstone <daniel.silverstone@canonical.com>
5
# and Adam Conrad <adam.conrad@canonical.com>
7
# Buildd Slave implementation
17
from twisted.internet import protocol
18
from twisted.internet import reactor
19
from twisted.internet import process
20
from twisted.web import xmlrpc
22
# cprov 20080611: in python2.4 posixfile.SEEK_END is deprecated and our
23
# importfascist-check denies its import. When we migrate to python2.5,
24
# we can use os.SEEK_END. See bug #239213.
27
devnull = open("/dev/null", "r")
30
def _sanitizeURLs(text_seq):
31
"""A generator that deletes URL passwords from a string sequence.
33
This generator removes user/password data from URLs if embedded
34
in the latter as follows: scheme://user:passwd@netloc/path.
36
:param text_seq: A sequence of strings (that may contain URLs).
37
:return: A (sanitized) line stripped of authentication credentials.
39
# This regular expression will be used to remove authentication
40
# credentials from URLs.
41
password_re = re.compile('://([^:]+:[^@]+@)(\S+)')
44
sanitized_line = password_re.sub(r'://\2', line)
48
# XXX cprov 2005-06-28:
49
# RunCapture can be replaced with a call to
51
# twisted.internet.utils.getProcessOutputAndValue
53
# when we start using Twisted 2.0.
54
class RunCapture(protocol.ProcessProtocol):
55
"""Run a command and capture its output to a slave's log"""
57
def __init__(self, slave, callback):
59
self.notify = callback
62
def outReceived(self, data):
63
"""Pass on stdout data to the log."""
66
def errReceived(self, data):
67
"""Pass on stderr data to the log.
69
With a bit of luck we won't interleave horribly."""
72
def processEnded(self, statusobject):
73
"""This method is called when a child process got terminated.
75
Three actions are required at this point: identify if we are within an
76
"aborting" process, eliminate pending calls to "kill" and invoke the
77
programmed notification callback. We only really care about invoking
78
the notification callback last thing in this method. The order
79
of the rest of the method is not critical.
81
# finishing the ABORTING workflow
82
if self.slave.builderstatus == BuilderStatus.ABORTING:
83
self.slave.builderstatus = BuilderStatus.ABORTED
85
# check if there is a pending request for kill the process,
86
# in afirmative case simply cancel this request since it
88
if self.killCall and self.killCall.active():
89
self.killCall.cancel()
91
# notify the slave, it'll perform the required actions
92
self.notify(statusobject.value.exitCode)
95
class BuildManager(object):
96
"""Build Daemon slave build manager abstract parent"""
98
def __init__(self, slave, buildid):
99
"""Create a BuildManager.
101
:param slave: A `BuildDSlave`.
102
:param buildid: Identifying string for this build.
104
object.__init__(self)
105
self._buildid = buildid
107
self._unpackpath = slave._config.get("allmanagers", "unpackpath")
108
self._cleanpath = slave._config.get("allmanagers", "cleanpath")
109
self._mountpath = slave._config.get("allmanagers", "mountpath")
110
self._umountpath = slave._config.get("allmanagers", "umountpath")
111
self.is_archive_private = False
112
self.home = os.environ['HOME']
114
def runSubProcess(self, command, args):
115
"""Run a sub process capturing the results in the log."""
116
self._subprocess = RunCapture(self._slave, self.iterate)
117
self._slave.log("RUN: %s %r\n" % (command, args))
118
childfds = {0: devnull.fileno(), 1: "r", 2: "r"}
119
reactor.spawnProcess(
120
self._subprocess, command, args, env=os.environ,
121
path=self.home, childFDs=childfds)
124
"""Unpack the build chroot."""
127
["unpack-chroot", self._buildid, self._chroottarfile])
130
"""Remove the build tree etc."""
131
self.runSubProcess(self._cleanpath, ["remove-build", self._buildid])
133
# Sanitize the URLs in the buildlog file if this is a build
134
# in a private archive.
135
if self.is_archive_private:
136
self._slave.sanitizeBuildlog(self._slave.cachePath("buildlog"))
138
def doMounting(self):
139
"""Mount things in the chroot, e.g. proc."""
140
self.runSubProcess( self._mountpath,
141
["mount-chroot", self._buildid])
143
def doUnmounting(self):
144
"""Unmount the chroot."""
145
self.runSubProcess( self._umountpath,
146
["umount-chroot", self._buildid])
148
def initiate(self, files, chroot, extra_args):
149
"""Initiate a build given the input files.
151
Please note: the 'extra_args' dictionary may contain a boolean
152
value keyed under the 'archive_private' string. If that value
153
evaluates to True the build at hand is for a private archive.
155
os.mkdir("%s/build-%s" % (self.home, self._buildid))
157
os.symlink( self._slave.cachePath(files[f]),
158
"%s/build-%s/%s" % (self.home,
160
self._chroottarfile = self._slave.cachePath(chroot)
162
# Check whether this is a build in a private archive and
163
# whether the URLs in the buildlog file should be sanitized
164
# so that they do not contain any embedded authentication
166
if extra_args.get('archive_private'):
167
self.is_archive_private = True
170
"/bin/echo", ["echo", "Forking build subprocess..."])
172
def iterate(self, success):
173
"""Perform an iteration of the slave.
175
The BuildManager tends to work by invoking several
176
subprocesses in order. the iterate method is called by the
177
object created by runSubProcess to gather the results of the
180
raise NotImplementedError("BuildManager should be subclassed to be "
184
"""Abort the build by killing the subprocess."""
185
if not self.alreadyfailed:
186
self.alreadyfailed = True
187
# Either SIGKILL and SIGTERM presents the same behavior,
188
# the process is just killed some time after the signal was sent
189
# 10 s ~ 40 s, and returns None as exit_code, instead of the normal
190
# interger. See further info on DebianBuildermanager.iterate in
192
# XXX cprov 2005-09-02:
193
# we may want to follow the canonical.tachandler kill process style,
194
# which sends SIGTERM to the process wait a given timeout and if was
195
# not killed sends a SIGKILL. IMO it only would be worth if we found
196
# different behaviour than the previous described.
197
self._subprocess.transport.signalProcess('TERM')
198
# alternativelly to simply send SIGTERM, we can pend a request to
199
# send SIGKILL to the process if nothing happened in 10 seconds
200
# see base class process
201
self._subprocess.killCall = reactor.callLater(10, self.kill)
204
"""Send SIGKILL to child process
206
Mask exception generated when the child process has already exited.
209
self._subprocess.transport.signalProcess('KILL')
210
except process.ProcessExitedAlready:
211
self._slave.log("ABORTING: Process Exited Already\n")
214
"""Status values for the builder."""
216
IDLE = "BuilderStatus.IDLE"
217
BUILDING = "BuilderStatus.BUILDING"
218
WAITING = "BuilderStatus.WAITING"
219
ABORTING = "BuilderStatus.ABORTING"
220
ABORTED = "BuilderStatus.ABORTED"
222
UNKNOWNSUM = "BuilderStatus.UNKNOWNSUM"
223
UNKNOWNBUILDER = "BuilderStatus.UNKNOWNBUILDER"
227
"""Status values for builds themselves."""
229
OK = "BuildStatus.OK"
230
DEPFAIL = "BuildStatus.DEPFAIL"
231
GIVENBACK = "BuildStatus.GIVENBACK"
232
PACKAGEFAIL = "BuildStatus.PACKAGEFAIL"
233
CHROOTFAIL = "BuildStatus.CHROOTFAIL"
234
BUILDERFAIL = "BuildStatus.BUILDERFAIL"
237
class BuildDSlave(object):
238
"""Build Daemon slave. Implementation of most needed functions
239
for a Build-Slave device.
242
def __init__(self, config):
243
object.__init__(self)
244
self._config = config
245
self.builderstatus = BuilderStatus.IDLE
246
self._cachepath = self._config.get("slave","filecache")
247
self.buildstatus = BuildStatus.OK
248
self.waitingfiles = {}
249
self.builddependencies = ""
252
if not os.path.isdir(self._cachepath):
253
raise ValueError("FileCache path is not a dir")
256
"""Return the Architecture tag for the slave."""
257
return self._config.get("slave","architecturetag")
259
def cachePath(self, file):
260
"""Return the path in the cache of the file specified."""
261
return os.path.join(self._cachepath, file)
263
def setupAuthHandler(self, url, username, password):
264
"""Set up a BasicAuthHandler to open the url.
266
:param url: The URL that needs authenticating.
267
:param username: The username for authentication.
268
:param password: The password for authentication.
269
:return: The OpenerDirector instance.
271
This helper installs a urllib2.HTTPBasicAuthHandler that will deal
272
with any HTTP basic authentication required when opening the
275
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
276
password_mgr.add_password(None, url, username, password)
277
handler = urllib2.HTTPBasicAuthHandler(password_mgr)
278
opener = urllib2.build_opener(handler)
281
def ensurePresent(self, sha1sum, url=None, username=None, password=None):
282
"""Ensure we have the file with the checksum specified.
284
Optionally you can provide the librarian URL and
285
the build slave will fetch the file if it doesn't have it.
286
Return a tuple containing: (<present>, <info>)
288
extra_info = 'No URL'
291
if not os.path.exists(self.cachePath(sha1sum)):
292
self.log('Fetching %s by url %s' % (sha1sum, url))
294
opener = self.setupAuthHandler(
295
url, username, password).open
297
opener = urllib2.urlopen
300
# Don't change this to URLError without thoroughly
301
# testing for regressions. For now, just suppress
302
# the PyLint warnings.
303
# pylint: disable-msg=W0703
304
except Exception, info:
305
extra_info = 'Error accessing Librarian: %s' % info
308
of = open(self.cachePath(sha1sum), "w")
309
# Upped for great justice to 256k
310
check_sum = hashlib.sha1()
311
for chunk in iter(lambda: f.read(256*1024), ''):
313
check_sum.update(chunk)
316
extra_info = 'Download'
317
if check_sum.hexdigest() != sha1sum:
318
os.remove(self.cachePath(sha1sum))
319
extra_info = "Digests did not match, removing again!"
321
return (os.path.exists(self.cachePath(sha1sum)), extra_info)
323
def storeFile(self, content):
324
"""Take the provided content and store it in the file cache."""
325
sha1sum = hashlib.sha1(content).hexdigest()
326
present, info = self.ensurePresent(sha1sum)
329
f = open(self.cachePath(sha1sum), "w")
334
def addWaitingFile(self, path):
335
"""Add a file to the cache and store its details for reporting."""
336
fn = os.path.basename(path)
339
self.waitingfiles[fn] = self.storeFile(f.read())
343
def fetchFile(self, sha1sum):
344
"""Fetch the file of the given sha1sum."""
345
present, info = self.ensurePresent(sha1sum)
347
raise ValueError("Unknown SHA1sum %s" % sha1sum)
348
f = open(self.cachePath(sha1sum), "r")
354
"""Abort the current build."""
355
# XXX: dsilvers: 2005-01-21: Current abort mechanism doesn't wait
356
# for abort to complete. This is potentially an issue in a heavy
358
if self.builderstatus != BuilderStatus.BUILDING:
359
# XXX: Should raise a known Fault so that the client can make
360
# useful decisions about the error!
361
raise ValueError("Slave is not BUILDING when asked to abort")
363
self.builderstatus = BuilderStatus.ABORTING
366
"""Clean up pending files and reset the internal build state."""
367
if self.builderstatus not in [BuilderStatus.WAITING,
368
BuilderStatus.ABORTED]:
369
raise ValueError('Slave is not WAITING|ABORTED when asked'
371
for f in self.waitingfiles:
372
os.remove(self.cachePath(self.waitingfiles[f]))
373
self.builderstatus = BuilderStatus.IDLE
374
if self._log is not None:
376
os.remove(self.cachePath("buildlog"))
378
self.waitingfiles = {}
379
self.builddependencies = ""
381
self.buildstatus = BuildStatus.OK
384
"""Write the provided data to the log."""
385
if self._log is not None:
386
self._log.write(data)
388
if data.endswith("\n"):
390
print "Build log: " + data
392
def getLogTail(self):
393
"""Return the tail of the log.
395
If the buildlog is not yet opened for writing (self._log is None),
396
return a empty string.
398
It safely tries to open the 'buildlog', if it doesn't exist, due to
399
job cleanup or buildlog sanitization race-conditions, it also returns
402
When the 'buildlog' is present it return up to 2 KiB character of
405
The returned content will be 'sanitized', see `_sanitizeURLs` for
408
if self._log is None:
414
rlog = open(self.cachePath("buildlog"), "r")
418
# We rely on good OS practices that keep the file handler
419
# usable once it's opened. So, if open() is ok, a subsequent
420
# seek/tell/read will be safe.
421
rlog.seek(0, SEEK_END)
425
rlog.seek(-count, SEEK_END)
426
ret = rlog.read(count)
431
if self.manager.is_archive_private:
432
# This is a build in a private archive. We need to scrub
433
# the URLs contained in the buildlog excerpt in order to
434
# avoid leaking passwords.
435
log_lines = ret.splitlines()
437
# Please note: we are throwing away the first line (of the
438
# excerpt to be scrubbed) because it may be cut off thus
439
# thwarting the detection of embedded passwords.
440
clean_content_iter = _sanitizeURLs(log_lines[1:])
441
ret = '\n'.join(clean_content_iter)
445
def startBuild(self, manager):
446
"""Start a build with the provided BuildManager instance."""
447
if self.builderstatus != BuilderStatus.IDLE:
448
raise ValueError("Slave is not IDLE when asked to start building")
449
self.manager = manager
450
self.builderstatus = BuilderStatus.BUILDING
454
"""Empty the log and start again."""
455
if self._log is not None:
457
self._log = open(self.cachePath("buildlog"), "w")
459
def builderFail(self):
460
"""Cease building because the builder has a problem."""
461
if self.builderstatus != BuilderStatus.BUILDING:
462
raise ValueError("Slave is not BUILDING when set to BUILDERFAIL")
463
self.buildstatus = BuildStatus.BUILDERFAIL
465
def chrootFail(self):
466
"""Cease building because the chroot could not be created or contained
467
a set of package control files which couldn't upgrade themselves, or
468
yet a lot of causes that imply the CHROOT is corrupted not the
471
if self.builderstatus != BuilderStatus.BUILDING:
472
raise ValueError("Slave is not BUILDING when set to CHROOTFAIL")
473
self.buildstatus = BuildStatus.CHROOTFAIL
476
"""Cease building because the package failed to build."""
477
if self.builderstatus != BuilderStatus.BUILDING:
478
raise ValueError("Slave is not BUILDING when set to PACKAGEFAIL")
479
self.buildstatus = BuildStatus.PACKAGEFAIL
482
"""Having passed all possible failure states, mark a build as OK."""
483
if self.builderstatus != BuilderStatus.BUILDING:
484
raise ValueError("Slave is not BUILDING when set to OK")
485
self.buildstatus = BuildStatus.OK
487
def depFail(self, dependencies):
488
"""Cease building due to a dependency issue."""
489
if self.builderstatus != BuilderStatus.BUILDING:
490
raise ValueError("Slave is not BUILDING when set to DEPFAIL")
491
self.buildstatus = BuildStatus.DEPFAIL
492
self.builddependencies = dependencies
495
"""Give-back package due to a transient buildd/archive issue."""
496
if self.builderstatus != BuilderStatus.BUILDING:
497
raise ValueError("Slave is not BUILDING when set to GIVENBACK")
498
self.buildstatus = BuildStatus.GIVENBACK
500
def buildComplete(self):
501
"""Mark the build as complete and waiting interaction from the build
504
if self.builderstatus != BuilderStatus.BUILDING:
505
raise ValueError("Slave is not BUILDING when told build is "
507
self.builderstatus = BuilderStatus.WAITING
509
def sanitizeBuildlog(self, log_path):
510
"""Removes passwords from buildlog URLs.
512
Because none of the URLs to be processed are expected to span
513
multiple lines and because build log files are potentially huge
514
they will be processed line by line.
516
:param log_path: The path to the buildlog file that is to be
518
:type log_path: ``str``
520
# First move the buildlog file that is to be sanitized out of
522
unsanitized_path = self.cachePath(
523
os.path.basename(log_path) + '.unsanitized')
524
os.rename(log_path, unsanitized_path)
526
# Open the unsanitized buildlog file for reading.
527
unsanitized_file = open(unsanitized_path)
529
# Open the file that will hold the resulting, sanitized buildlog
530
# content for writing.
531
sanitized_file = None
534
sanitized_file = open(log_path, 'w')
536
# Scrub the buildlog file line by line
537
clean_content_iter = _sanitizeURLs(unsanitized_file)
538
for line in clean_content_iter:
539
sanitized_file.write(line)
541
# We're done with scrubbing, close the file handles.
542
unsanitized_file.close()
543
if sanitized_file is not None:
544
sanitized_file.close()
547
class XMLRPCBuildDSlave(xmlrpc.XMLRPC):
548
"""XMLRPC build daemon slave management interface"""
550
def __init__(self, config):
551
xmlrpc.XMLRPC.__init__(self, allowNone=True)
552
# The V1.0 new-style protocol introduces string-style protocol
553
# versions of the form 'MAJOR.MINOR', the protocol is '1.0' for now
554
# implying the presence of /filecache/ /filecache/buildlog and
555
# the reduced and optimised XMLRPC interface.
556
self.protocolversion = '1.0'
557
self.slave = BuildDSlave(config)
561
def registerBuilder(self, builderclass, buildertag):
562
self._builders[buildertag] = builderclass
564
def xmlrpc_echo(self, *args):
565
"""Echo the argument back."""
568
def xmlrpc_info(self):
569
"""Return the protocol version and the builder methods supported."""
570
return (self.protocolversion, self.slave.getArch(),
571
self._builders.keys())
573
def xmlrpc_status(self):
574
"""Return the status of the build daemon.
576
Depending on the builder status we return differing amounts of
577
data. We do however always return the builder status as the first
580
status = self.slave.builderstatus
581
statusname = status.split('.')[-1]
582
func = getattr(self, "status_" + statusname, None)
584
raise ValueError("Unknown status '%s'" % status)
585
return (status, ) + func()
587
def status_IDLE(self):
588
"""Handler for xmlrpc_status IDLE.
590
Returns a tuple containing a empty string since there's nothing
593
# keep the result code sane
596
def status_BUILDING(self):
597
"""Handler for xmlrpc_status BUILDING.
599
Returns the build id and up to one kilobyte of log tail
601
tail = self.slave.getLogTail()
602
return (self.buildid, xmlrpclib.Binary(tail))
604
def status_WAITING(self):
605
"""Handler for xmlrpc_status WAITING.
607
Returns the build id and the set of files waiting to be returned
608
unless the builder failed in which case we return the buildstatus
609
and the build id but no file set.
611
if self.slave.buildstatus in (BuildStatus.OK, BuildStatus.PACKAGEFAIL,
612
BuildStatus.DEPFAIL):
613
return (self.slave.buildstatus, self.buildid,
614
self.slave.waitingfiles, self.slave.builddependencies)
615
return (self.slave.buildstatus, self.buildid)
617
def status_ABORTED(self):
618
"""Handler for xmlrpc_status ABORTED.
620
The only action the master can take is clean, other than ask status,
621
of course, it returns the build id only.
623
return (self.buildid, )
625
def status_ABORTING(self):
626
"""Handler for xmlrpc_status ABORTING.
628
This state means the builder performing the ABORT command and is
629
not able to do anything else than answer its status, returns the
632
return (self.buildid, )
634
def xmlrpc_ensurepresent(self, sha1sum, url, username, password):
635
"""Attempt to ensure the given file is present."""
636
return self.slave.ensurePresent(sha1sum, url, username, password)
638
def xmlrpc_abort(self):
639
"""Abort the current build."""
641
return BuilderStatus.ABORTING
643
def xmlrpc_clean(self):
644
"""Clean up the waiting files and reset the slave's internal state."""
646
return BuilderStatus.IDLE
648
def xmlrpc_build(self, buildid, builder, chrootsum, filemap, args):
649
"""Check if requested arguments are sane and initiate build procedure
651
return a tuple containing: (<builder_status>, <info>)
654
# check requested builder
655
if not builder in self._builders:
656
extra_info = "%s not in %r" % (builder, self._builders.keys())
657
return (BuilderStatus.UNKNOWNBUILDER, extra_info)
658
# check requested chroot availability
659
chroot_present, info = self.slave.ensurePresent(chrootsum)
660
if not chroot_present:
661
extra_info = """CHROOTSUM -> %s
665
""" % (chrootsum, info)
666
return (BuilderStatus.UNKNOWNSUM, extra_info)
667
# check requested files availability
668
for filesum in filemap.itervalues():
669
file_present, info = self.slave.ensurePresent(filesum)
671
extra_info = """FILESUM -> %s
675
""" % (filesum, info)
676
return (BuilderStatus.UNKNOWNSUM, extra_info)
677
# check buildid sanity
678
if buildid is None or buildid == "" or buildid == 0:
679
raise ValueError(buildid)
681
# builder is available, buildd is non empty,
682
# filelist is consistent, chrootsum is available, let's initiate...
683
self.buildid = buildid
684
self.slave.startBuild(self._builders[builder](self.slave, buildid))
685
self.slave.manager.initiate(filemap, chrootsum, args)
686
return (BuilderStatus.BUILDING, buildid)