~launchpad-pqm/launchpad/devel

10529.1.3 by Jonathan Lange
Rename the PoppyInterface class to Hooks
1
# Copyright 2009-2010 Canonical Ltd.  This software is licensed under the
8687.15.13 by Karl Fogel
Add the copyright header block to files under lib/lp/archiveuploader/.
2
# GNU Affero General Public License version 3 (see the file LICENSE).
1474 by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon
3
10529.1.3 by Jonathan Lange
Rename the PoppyInterface class to Hooks
4
__metaclass__ = type
5
12392.7.27 by Julian Edwards
make logging work, and fix the sleep in the tests to look for a special log message instead
6
__all__ = [
7
    'Hooks',
8
    'PoppyInterfaceFailure',
9
    ]
10
11
2865.6.8 by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential
12
import logging
11403.1.4 by Henning Eggers
Reformatted imports using format-imports script r32.
13
import os
2865.6.8 by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential
14
import shutil
4918.2.1 by Celso Providelo
Fix bug #144392 (grant g+w on the poppy toplevel lockfile allowing lp_queue to be blocked on it).
15
import stat
3023.3.14 by Daniel Silverstone
Make poppy's resulting directory name contain the timestamp
16
import time
2865.6.8 by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential
17
18
from contrib.glock import GlobalLock
1474 by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon
19
10529.1.2 by Jonathan Lange
Minor cleanups
20
1474 by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon
21
class PoppyInterfaceFailure(Exception):
22
    pass
23
10529.1.2 by Jonathan Lange
Minor cleanups
24
10529.1.3 by Jonathan Lange
Rename the PoppyInterface class to Hooks
25
class Hooks:
1474 by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon
26
27
    clients = {}
12392.7.27 by Julian Edwards
make logging work, and fix the sleep in the tests to look for a special log message instead
28
    LOG_MAGIC = "Post-processing finished"
12392.7.32 by Julian Edwards
remove the global targetcount
29
    _targetcount = 0
1474 by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon
30
3023.2.20 by Celso Providelo
Fix soyuz-upload.txt test and behaviour of right ANY archhintlist builds.
31
    def __init__(self, targetpath, logger, allow_user, cmd=None,
10935.2.1 by Steve Kowalik
* Export FileIsADirectory from codehosting.sftp, so I can use it for poppy-sftp.
32
                 targetstart=0, perms=None, prefix=''):
2865.6.8 by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential
33
        self.targetpath = targetpath
10529.1.3 by Jonathan Lange
Rename the PoppyInterface class to Hooks
34
        self.logger = logging.getLogger("%s.Hooks" % logger.name)
2865.6.8 by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential
35
        self.cmd = cmd
3023.2.20 by Celso Providelo
Fix soyuz-upload.txt test and behaviour of right ANY archhintlist builds.
36
        self.allow_user = allow_user
3023.3.37 by Daniel Silverstone
workaround for annoying permissions on poppy directories
37
        self.perms = perms
10935.2.1 by Steve Kowalik
* Export FileIsADirectory from codehosting.sftp, so I can use it for poppy-sftp.
38
        self.prefix = prefix
2865.6.8 by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential
39
12392.7.29 by Julian Edwards
ensure the isolation tests pass now, since there were bugs in both the test and the code - targetcount was not unique as Hooks is re-initialised on each shell, so it's now static
40
    @property
41
    def targetcount(self):
42
        """A guaranteed unique integer for ensuring unique upload dirs."""
12392.7.32 by Julian Edwards
remove the global targetcount
43
        Hooks._targetcount += 1
44
        return Hooks._targetcount
12392.7.29 by Julian Edwards
ensure the isolation tests pass now, since there were bugs in both the test and the code - targetcount was not unique as Hooks is re-initialised on each shell, so it's now static
45
1474 by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon
46
    def new_client_hook(self, fsroot, host, port):
47
        """Prepare a new client record indexed by fsroot..."""
48
        self.clients[fsroot] = {
49
            "host": host,
50
            "port": port
51
            }
5554.1.1 by Celso Providelo
Fixing share lockfile issue between poppy (lp_upload) and process-upload (lp_queue). Ignoring exceptions raised while trying to grant the write permission to the group.
52
        self.logger.debug("Accepting new session in fsroot: %s" % fsroot)
53
        self.logger.debug("Session from %s:%s" % (host, port))
1474 by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon
54
55
    def client_done_hook(self, fsroot, host, port):
56
        """A client has completed. If it authenticated then it stands a chance
57
        of having uploaded a file to the set. If not; then it is simply an
58
        aborted transaction and we remove the fsroot."""
2865.6.8 by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential
59
1474 by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon
60
        if fsroot not in self.clients:
3023.2.3 by Celso Providelo
Applying Review comments take 1
61
            raise PoppyInterfaceFailure("Unable to find fsroot in client set")
2865.6.8 by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential
62
1474 by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon
63
        self.logger.debug("Processing session complete in %s" % fsroot)
2865.6.8 by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential
64
65
        client = self.clients[fsroot]
66
        if "distro" not in client:
3023.2.3 by Celso Providelo
Applying Review comments take 1
67
            # Login username defines the distribution context of the upload.
68
            # So abort unauthenticated sessions by removing its contents
1474 by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon
69
            shutil.rmtree(fsroot)
70
            return
2865.6.8 by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential
71
72
        # Protect from race condition between creating the directory
73
        # and creating the distro file, and also in cases where the
74
        # temporary directory and the upload directory are not in the
75
        # same filesystem (non-atomic "rename").
5089.1.1 by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface.
76
        lockfile_path = os.path.join(self.targetpath, ".lock")
77
        self.lock = GlobalLock(lockfile_path)
5554.1.2 by Celso Providelo
applying review comments, r=salgado.
78
6916.1.1 by Curtis Hovey
Fixed comment formats to fidn missing persons, dates, and some bugs.
79
        # XXX cprov 20071024 bug=156795: We try to acquire the lock as soon
80
        # as possible after creating the lockfile but are still open to
81
        # a race.
3691.348.5 by kiko
I changed the semantics for acquire() to be non-blocking by default, so make it explicit in these callsites
82
        self.lock.acquire(blocking=True)
5554.1.2 by Celso Providelo
applying review comments, r=salgado.
83
        mode = stat.S_IMODE(os.stat(lockfile_path).st_mode)
2865.6.8 by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential
84
6916.1.1 by Curtis Hovey
Fixed comment formats to fidn missing persons, dates, and some bugs.
85
        # XXX cprov 20081024 bug=185731: The lockfile permission can only be
86
        # changed by its owner. Since we can't predict which process will
87
        # create it in production systems we simply ignore errors when trying
88
        # to grant the right permission. At least, one of the process will
89
        # be able to do so.
5554.1.1 by Celso Providelo
Fixing share lockfile issue between poppy (lp_upload) and process-upload (lp_queue). Ignoring exceptions raised while trying to grant the write permission to the group.
90
        try:
91
            os.chmod(lockfile_path, mode | stat.S_IWGRP)
92
        except OSError:
93
            pass
4918.2.1 by Celso Providelo
Fix bug #144392 (grant g+w on the poppy toplevel lockfile allowing lp_queue to be blocked on it).
94
5089.1.1 by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface.
95
        try:
96
            timestamp = time.strftime("%Y%m%d-%H%M%S")
10935.2.1 by Steve Kowalik
* Export FileIsADirectory from codehosting.sftp, so I can use it for poppy-sftp.
97
            path = "upload%s-%s-%06d" % (
98
                self.prefix, timestamp, self.targetcount)
5089.1.1 by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface.
99
            target_fsroot = os.path.join(self.targetpath, path)
100
101
            # Create file to store the distro used.
102
            self.logger.debug("Upload was targetted at %s" % client["distro"])
103
            distro_filename = target_fsroot + ".distro"
104
            distro_file = open(distro_filename, "w")
105
            distro_file.write(client["distro"])
106
            distro_file.close()
107
108
            # Move the session directory to the target directory.
109
            if os.path.exists(target_fsroot):
110
                self.logger.warn("Targeted upload already present: %s" % path)
5089.1.2 by Celso Providelo
applying review comments, r=kiko.
111
                self.logger.warn("System clock skewed ?")
5089.1.1 by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface.
112
            else:
2865.6.8 by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential
113
                try:
114
                    shutil.move(fsroot, target_fsroot)
115
                except (OSError, IOError):
116
                    if not os.path.exists(target_fsroot):
117
                        raise
5089.1.1 by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface.
118
5089.1.2 by Celso Providelo
applying review comments, r=kiko.
119
            # XXX cprov 20071024: We should replace os.system call by os.chmod
120
            # and fix the default permission value accordingly in poppy-upload
5089.1.1 by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface.
121
            if self.perms is not None:
8305.2.1 by Celso Providelo
Fixing bug #370513 (adjusting the permission of the uploaded files in a way they do not depend on special enviroments setup).
122
                os.system("chmod %s -R %s" % (self.perms, target_fsroot))
5089.1.1 by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface.
123
124
            # Invoke processing script, if provided.
125
            if self.cmd:
126
                cmd = self.cmd
127
                cmd = cmd.replace("@fsroot@", target_fsroot)
128
                cmd = cmd.replace("@distro@", client["distro"])
129
                self.logger.debug("Running upload handler: %s" % cmd)
130
                os.system(cmd)
131
        finally:
5089.1.2 by Celso Providelo
applying review comments, r=kiko.
132
            # We never delete the lockfile, this way the inode will be
133
            # constant while the machine is up. See comment on 'acquire'
5089.1.1 by Celso Providelo
Fixing misuse of GlobalLock in poppyinterface.
134
            self.lock.release(skip_delete=True)
2865.6.8 by Gustavo Niemeyer
The Soyuz upload mechanism was changed to avoid the potential
135
1474 by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon
136
        self.clients.pop(fsroot)
12392.7.27 by Julian Edwards
make logging work, and fix the sleep in the tests to look for a special log message instead
137
        # This is mainly done so that tests know when the
138
        # post-processing hook has finished.
139
        self.logger.info(self.LOG_MAGIC)
1474 by Canonical.com Patch Queue Manager
GPGV movement, initial poppy-upload and clean up ftpserver signon
140
141
    def auth_verify_hook(self, fsroot, user, password):
142
        """Verify that the username matches a distribution we care about.
143
144
        The password is irrelevant to auth, as is the fsroot"""
145
        if fsroot not in self.clients:
3023.2.3 by Celso Providelo
Applying Review comments take 1
146
            raise PoppyInterfaceFailure("Unable to find fsroot in client set")
3023.3.13 by Daniel Silverstone
Make poppy always assume the ubuntu distro
147
3023.2.20 by Celso Providelo
Fix soyuz-upload.txt test and behaviour of right ANY archhintlist builds.
148
        # local authentication
3023.3.31 by Daniel Silverstone
Allow any user to connect to poppy
149
        self.clients[fsroot]["distro"] = self.allow_user
3023.3.32 by Daniel Silverstone
return True needed in poppyinterface
150
        return True
3023.2.20 by Celso Providelo
Fix soyuz-upload.txt test and behaviour of right ANY archhintlist builds.
151
5554.1.1 by Celso Providelo
Fixing share lockfile issue between poppy (lp_upload) and process-upload (lp_queue). Ignoring exceptions raised while trying to grant the write permission to the group.
152
        # When we get on with the poppy path stuff, the below may be useful
153
        # and is thus left in rather than being removed.
4785.3.7 by Jeroen Vermeulen
Removed whitespace at ends of lines
154
3023.3.13 by Daniel Silverstone
Make poppy always assume the ubuntu distro
155
        #try:
156
        #    d = Distribution.byName(user)
157
        #    if d:
158
        #        self.logger.debug("Accepting login for %s" % user)
159
        #        self.clients[fsroot]["distro"] = user
160
        #        return True
161
        #except object, e:
162
        #    print e
163
        #return False
12392.7.32 by Julian Edwards
remove the global targetcount
164