10529.1.3
by Jonathan Lange
Rename the PoppyInterface class to Hooks |
1 |
# Copyright 2009-2010 Canonical Ltd. This software is licensed under the
|
8687.15.13
by Karl Fogel
Add the copyright header block to files under lib/lp/archiveuploader/. |
2 |
# GNU Affero General Public License version 3 (see the file LICENSE).
|
1474
by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon |
3 |
|
10529.1.3
by Jonathan Lange
Rename the PoppyInterface class to Hooks |
4 |
__metaclass__ = type |
5 |
||
12392.7.27
by Julian Edwards
make logging work, and fix the sleep in the tests to look for a special log message instead |
6 |
__all__ = [ |
7 |
'Hooks', |
|
8 |
'PoppyInterfaceFailure', |
|
9 |
]
|
|
10 |
||
11 |
||
2865.6.8
by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential |
12 |
import logging |
11403.1.4
by Henning Eggers
Reformatted imports using format-imports script r32. |
13 |
import os |
2865.6.8
by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential |
14 |
import shutil |
4918.2.1
by Celso Providelo
Fix bug #144392 (grant g+w on the poppy toplevel lockfile allowing lp_queue to be blocked on it). |
15 |
import stat |
3023.3.14
by Daniel Silverstone
Make poppy's resulting directory name contain the timestamp |
16 |
import time |
2865.6.8
by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential |
17 |
|
18 |
from contrib.glock import GlobalLock |
|
1474
by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon |
19 |
|
10529.1.2
by Jonathan Lange
Minor cleanups |
20 |
|
1474
by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon |
21 |
class PoppyInterfaceFailure(Exception): |
22 |
pass
|
|
23 |
||
10529.1.2
by Jonathan Lange
Minor cleanups |
24 |
|
10529.1.3
by Jonathan Lange
Rename the PoppyInterface class to Hooks |
25 |
class Hooks: |
1474
by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon |
26 |
|
27 |
clients = {} |
|
12392.7.27
by Julian Edwards
make logging work, and fix the sleep in the tests to look for a special log message instead |
28 |
LOG_MAGIC = "Post-processing finished" |
12392.7.32
by Julian Edwards
remove the global targetcount |
29 |
_targetcount = 0 |
1474
by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon |
30 |
|
3023.2.20
by Celso Providelo
Fix soyuz-upload.txt test and behaviour of right ANY archhintlist builds. |
31 |
def __init__(self, targetpath, logger, allow_user, cmd=None, |
10935.2.1
by Steve Kowalik
* Export FileIsADirectory from codehosting.sftp, so I can use it for poppy-sftp. |
32 |
targetstart=0, perms=None, prefix=''): |
2865.6.8
by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential |
33 |
self.targetpath = targetpath |
10529.1.3
by Jonathan Lange
Rename the PoppyInterface class to Hooks |
34 |
self.logger = logging.getLogger("%s.Hooks" % logger.name) |
2865.6.8
by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential |
35 |
self.cmd = cmd |
3023.2.20
by Celso Providelo
Fix soyuz-upload.txt test and behaviour of right ANY archhintlist builds. |
36 |
self.allow_user = allow_user |
3023.3.37
by Daniel Silverstone
workaround for annoying permissions on poppy directories |
37 |
self.perms = perms |
10935.2.1
by Steve Kowalik
* Export FileIsADirectory from codehosting.sftp, so I can use it for poppy-sftp. |
38 |
self.prefix = prefix |
2865.6.8
by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential |
39 |
|
12392.7.29
by Julian Edwards
ensure the isolation tests pass now, since there were bugs in both the test and the code - targetcount was not unique as Hooks is re-initialised on each shell, so it's now static |
40 |
@property
|
41 |
def targetcount(self): |
|
42 |
"""A guaranteed unique integer for ensuring unique upload dirs."""
|
|
12392.7.32
by Julian Edwards
remove the global targetcount |
43 |
Hooks._targetcount += 1 |
44 |
return Hooks._targetcount |
|
12392.7.29
by Julian Edwards
ensure the isolation tests pass now, since there were bugs in both the test and the code - targetcount was not unique as Hooks is re-initialised on each shell, so it's now static |
45 |
|
1474
by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon |
46 |
def new_client_hook(self, fsroot, host, port): |
47 |
"""Prepare a new client record indexed by fsroot..."""
|
|
48 |
self.clients[fsroot] = { |
|
49 |
"host": host, |
|
50 |
"port": port |
|
51 |
}
|
|
5554.1.1
by Celso Providelo
Fixing share lockfile issue between poppy (lp_upload) and process-upload (lp_queue). Ignoring exceptions raised while trying to grant the write permission to the group. |
52 |
self.logger.debug("Accepting new session in fsroot: %s" % fsroot) |
53 |
self.logger.debug("Session from %s:%s" % (host, port)) |
|
1474
by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon |
54 |
|
55 |
def client_done_hook(self, fsroot, host, port): |
|
56 |
"""A client has completed. If it authenticated then it stands a chance
|
|
57 |
of having uploaded a file to the set. If not; then it is simply an
|
|
58 |
aborted transaction and we remove the fsroot."""
|
|
2865.6.8
by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential |
59 |
|
1474
by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon |
60 |
if fsroot not in self.clients: |
3023.2.3
by Celso Providelo
Applying Review comments take 1 |
61 |
raise PoppyInterfaceFailure("Unable to find fsroot in client set") |
2865.6.8
by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential |
62 |
|
1474
by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon |
63 |
self.logger.debug("Processing session complete in %s" % fsroot) |
2865.6.8
by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential |
64 |
|
65 |
client = self.clients[fsroot] |
|
66 |
if "distro" not in client: |
|
3023.2.3
by Celso Providelo
Applying Review comments take 1 |
67 |
# Login username defines the distribution context of the upload.
|
68 |
# So abort unauthenticated sessions by removing its contents
|
|
1474
by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon |
69 |
shutil.rmtree(fsroot) |
70 |
return
|
|
2865.6.8
by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential |
71 |
|
72 |
# Protect from race condition between creating the directory
|
|
73 |
# and creating the distro file, and also in cases where the
|
|
74 |
# temporary directory and the upload directory are not in the
|
|
75 |
# same filesystem (non-atomic "rename").
|
|
5089.1.1
by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface. |
76 |
lockfile_path = os.path.join(self.targetpath, ".lock") |
77 |
self.lock = GlobalLock(lockfile_path) |
|
5554.1.2
by Celso Providelo
applying review comments, r=salgado. |
78 |
|
6916.1.1
by Curtis Hovey
Fixed comment formats to fidn missing persons, dates, and some bugs. |
79 |
# XXX cprov 20071024 bug=156795: We try to acquire the lock as soon
|
80 |
# as possible after creating the lockfile but are still open to
|
|
81 |
# a race.
|
|
3691.348.5
by kiko
I changed the semantics for acquire() to be non-blocking by default, so make it explicit in these callsites |
82 |
self.lock.acquire(blocking=True) |
5554.1.2
by Celso Providelo
applying review comments, r=salgado. |
83 |
mode = stat.S_IMODE(os.stat(lockfile_path).st_mode) |
2865.6.8
by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential |
84 |
|
6916.1.1
by Curtis Hovey
Fixed comment formats to fidn missing persons, dates, and some bugs. |
85 |
# XXX cprov 20081024 bug=185731: The lockfile permission can only be
|
86 |
# changed by its owner. Since we can't predict which process will
|
|
87 |
# create it in production systems we simply ignore errors when trying
|
|
88 |
# to grant the right permission. At least, one of the process will
|
|
89 |
# be able to do so.
|
|
5554.1.1
by Celso Providelo
Fixing share lockfile issue between poppy (lp_upload) and process-upload (lp_queue). Ignoring exceptions raised while trying to grant the write permission to the group. |
90 |
try: |
91 |
os.chmod(lockfile_path, mode | stat.S_IWGRP) |
|
92 |
except OSError: |
|
93 |
pass
|
|
4918.2.1
by Celso Providelo
Fix bug #144392 (grant g+w on the poppy toplevel lockfile allowing lp_queue to be blocked on it). |
94 |
|
5089.1.1
by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface. |
95 |
try: |
96 |
timestamp = time.strftime("%Y%m%d-%H%M%S") |
|
10935.2.1
by Steve Kowalik
* Export FileIsADirectory from codehosting.sftp, so I can use it for poppy-sftp. |
97 |
path = "upload%s-%s-%06d" % ( |
98 |
self.prefix, timestamp, self.targetcount) |
|
5089.1.1
by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface. |
99 |
target_fsroot = os.path.join(self.targetpath, path) |
100 |
||
101 |
# Create file to store the distro used.
|
|
102 |
self.logger.debug("Upload was targetted at %s" % client["distro"]) |
|
103 |
distro_filename = target_fsroot + ".distro" |
|
104 |
distro_file = open(distro_filename, "w") |
|
105 |
distro_file.write(client["distro"]) |
|
106 |
distro_file.close() |
|
107 |
||
108 |
# Move the session directory to the target directory.
|
|
109 |
if os.path.exists(target_fsroot): |
|
110 |
self.logger.warn("Targeted upload already present: %s" % path) |
|
5089.1.2
by Celso Providelo
applying review comments, r=kiko. |
111 |
self.logger.warn("System clock skewed ?") |
5089.1.1
by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface. |
112 |
else: |
2865.6.8
by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential |
113 |
try: |
114 |
shutil.move(fsroot, target_fsroot) |
|
115 |
except (OSError, IOError): |
|
116 |
if not os.path.exists(target_fsroot): |
|
117 |
raise
|
|
5089.1.1
by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface. |
118 |
|
5089.1.2
by Celso Providelo
applying review comments, r=kiko. |
119 |
# XXX cprov 20071024: We should replace os.system call by os.chmod
|
120 |
# and fix the default permission value accordingly in poppy-upload
|
|
5089.1.1
by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface. |
121 |
if self.perms is not None: |
8305.2.1
by Celso Providelo
Fixing bug #370513 (adjusting the permission of the uploaded files in a way they do not depend on special enviroments setup). |
122 |
os.system("chmod %s -R %s" % (self.perms, target_fsroot)) |
5089.1.1
by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface. |
123 |
|
124 |
# Invoke processing script, if provided.
|
|
125 |
if self.cmd: |
|
126 |
cmd = self.cmd |
|
127 |
cmd = cmd.replace("@fsroot@", target_fsroot) |
|
128 |
cmd = cmd.replace("@distro@", client["distro"]) |
|
129 |
self.logger.debug("Running upload handler: %s" % cmd) |
|
130 |
os.system(cmd) |
|
131 |
finally: |
|
5089.1.2
by Celso Providelo
applying review comments, r=kiko. |
132 |
# We never delete the lockfile, this way the inode will be
|
133 |
# constant while the machine is up. See comment on 'acquire'
|
|
5089.1.1
by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface. |
134 |
self.lock.release(skip_delete=True) |
2865.6.8
by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential |
135 |
|
1474
by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon |
136 |
self.clients.pop(fsroot) |
12392.7.27
by Julian Edwards
make logging work, and fix the sleep in the tests to look for a special log message instead |
137 |
# This is mainly done so that tests know when the
|
138 |
# post-processing hook has finished.
|
|
139 |
self.logger.info(self.LOG_MAGIC) |
|
1474
by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon |
140 |
|
141 |
def auth_verify_hook(self, fsroot, user, password): |
|
142 |
"""Verify that the username matches a distribution we care about.
|
|
143 |
||
144 |
The password is irrelevant to auth, as is the fsroot"""
|
|
145 |
if fsroot not in self.clients: |
|
3023.2.3
by Celso Providelo
Applying Review comments take 1 |
146 |
raise PoppyInterfaceFailure("Unable to find fsroot in client set") |
3023.3.13
by Daniel Silverstone
Make poppy always assume the ubuntu distro |
147 |
|
3023.2.20
by Celso Providelo
Fix soyuz-upload.txt test and behaviour of right ANY archhintlist builds. |
148 |
# local authentication
|
3023.3.31
by Daniel Silverstone
Allow any user to connect to poppy |
149 |
self.clients[fsroot]["distro"] = self.allow_user |
3023.3.32
by Daniel Silverstone
return True needed in poppyinterface |
150 |
return True |
3023.2.20
by Celso Providelo
Fix soyuz-upload.txt test and behaviour of right ANY archhintlist builds. |
151 |
|
5554.1.1
by Celso Providelo
Fixing share lockfile issue between poppy (lp_upload) and process-upload (lp_queue). Ignoring exceptions raised while trying to grant the write permission to the group. |
152 |
# When we get on with the poppy path stuff, the below may be useful
|
153 |
# and is thus left in rather than being removed.
|
|
4785.3.7
by Jeroen Vermeulen
Removed whitespace at ends of lines |
154 |
|
3023.3.13
by Daniel Silverstone
Make poppy always assume the ubuntu distro |
155 |
#try:
|
156 |
# d = Distribution.byName(user)
|
|
157 |
# if d:
|
|
158 |
# self.logger.debug("Accepting login for %s" % user)
|
|
159 |
# self.clients[fsroot]["distro"] = user
|
|
160 |
# return True
|
|
161 |
#except object, e:
|
|
162 |
# print e
|
|
163 |
#return False
|
|
12392.7.32
by Julian Edwards
remove the global targetcount |
164 |