← Back to branch summary

~launchpad-pqm/launchpad/devel

« back to all changes in this revision

Viewing changes to lib/lp/archivepublisher/generate_contents_files.py

Fairly direct bash-do-python conversion, minus hard-coded distro and paths.

Show diffs side-by-side

added added

removed removed

Lines of Context:
lib/lp/archivepublisher/generate_contents_files.py
 
1
# Copyright 2011 Canonical Ltd.  This software is licensed under the
 
2
# GNU Affero General Public License version 3 (see the file LICENSE).
 
3
 
 
4
"""Archive Contents files generator."""
 
5
 
 
6
__metaclass__ = type
 
7
__all__ = [
 
8
    'GenerateContentsFiles',
 
9
    ]
 
10
 
 
11
from optparse import OptionValueError
 
12
import os
 
13
 
 
14
from canonical.config import config
 
15
from canonical.launchpad.ftests.script import run_command
 
16
from lp.archivepublisher.config import getPubConfig
 
17
from lp.services.scripts.base import (
 
18
    LaunchpadScript,
 
19
    LaunchpadScriptFailure,
 
20
    )
 
21
from lp.services.utils import file_exists
 
22
from lp.soyuz.scripts.ftpmaster import LpQueryDistro
 
23
 
 
24
 
 
25
COMPONENTS = [
 
26
    'main',
 
27
    'restricted',
 
28
    'universe',
 
29
    'multiverse',
 
30
    ]
 
31
 
 
32
 
 
33
def differ_in_content(one_file, other_file):
 
34
    """Do the two named files have different contents?"""
 
35
    return file(one_file).read() != file(other_file).read()
 
36
 
 
37
 
 
38
class StoreArgument:
 
39
    """Local helper for receiving `LpQueryDistro` results."""
 
40
 
 
41
    def __call__(self, argument):
 
42
        """Store call argument."""
 
43
        self.argument = argument
 
44
 
 
45
 
 
46
def get_template(template_name):
 
47
    """Return path of given template in this script's templates directory."""
 
48
    return os.path.join(
 
49
        config.root, "cronscripts", "publishing", "gen-contents",
 
50
        template_name)
 
51
 
 
52
 
 
53
def execute(logger, command_line):
 
54
    description = ' '.join(command_line)
 
55
    logger.debug("Execute: %s", description)
 
56
    retval, stdout, stderr = run_command(command_line)
 
57
    logger.debug(stdout)
 
58
    logger.warn(stderr)
 
59
    if retval != 0:
 
60
        raise LaunchpadScriptFailure(
 
61
            "Failure while running command: %s" % description)
 
62
 
 
63
 
 
64
class GenerateContentsFiles(LaunchpadScript):
 
65
 
 
66
    def add_my_options(self):
 
67
        self.parser.add_option(
 
68
            "-d", "--distribution", dest="distribution", default=None,
 
69
            help="Distribution to generate Contents files for.")
 
70
 
 
71
    def setUpPrivateTree(self):
 
72
        self.logger.debug("Ensuring that we have a private tree in place.")
 
73
        for suffix in ['cache', 'misc']:
 
74
            dirname = '%s-%s' % (self.distribution.name, suffix)
 
75
            os.makedirs(os.path.join(self.content_archive, dirname))
 
76
 
 
77
    def queryDistro(self, request, options=None):
 
78
        """Call the query-distro script about `self.distribution`."""
 
79
        args = ['-d', self.distribution.name]
 
80
        if options is not None:
 
81
            args += options
 
82
        args += request
 
83
        query_distro = LpQueryDistro(test_args=args)
 
84
        receiver = StoreArgument()
 
85
        query_distro.runAction(presenter=receiver)
 
86
        return receiver.argument
 
87
 
 
88
    def getPocketSuffixes(self):
 
89
        return self.queryDistro("pocket_suffixes").split()
 
90
 
 
91
    def getSuites(self):
 
92
        return self.queryDistro("supported").split()
 
93
 
 
94
    def getArchs(self):
 
95
        devel = self.queryDistro("development")
 
96
        return self.queryDistro("archs", options=["-s", devel])
 
97
 
 
98
    def getDirs(self, archs):
 
99
        return ['source', 'debian-installer'] + [
 
100
            'binary-%s' % arch.name for arch in archs]
 
101
 
 
102
    def writeAptConf(self, suites, archs):
 
103
        output_dirname = '%s-misc' % self.distribution.name
 
104
        output_file = file(os.path.join(
 
105
            self.content_archive, output_dirname, "apt-contents.conf"))
 
106
 
 
107
        parameters = {
 
108
            'architectures': archs,
 
109
            'content_archive': self.content_archive,
 
110
            'distribution': self.distribution.name,
 
111
        }
 
112
 
 
113
        header = get_template('apt_conf_header.template')
 
114
        output_file.write(file(header).read() % parameters)
 
115
 
 
116
        dist = get_template('apt_conf_dist.template')
 
117
        dist_template = dist.read()
 
118
 
 
119
        for suite in suites:
 
120
            parameters['suite'] = suite
 
121
            output_file.write(dist_template % parameters)
 
122
 
 
123
        output_file.close()
 
124
 
 
125
    def createComponentDirs(self, suites, archs):
 
126
        for suite in suites:
 
127
            for component in COMPONENTS:
 
128
                for directory in self.getDirs(archs):
 
129
                    path = os.path.join(
 
130
                        self.content_archive, self.distribution.name, 'dists',
 
131
                        suite, component, directory)
 
132
                    if not file_exists(path):
 
133
                        self.logger.debug("Creating %s", path)
 
134
                        os.makedirs(path)
 
135
 
 
136
    def writeContentsTop(self):
 
137
        output_filename = os.path.join(
 
138
            self.content_archive, '%s-misc' % self.distribution.name,
 
139
            "Contents.top")
 
140
        parameters = {
 
141
            'distrotitle': self.distribution.title,
 
142
        }
 
143
        output_file = file(output_filename, 'w')
 
144
        output_file.write(
 
145
            file(get_template('Contents.top')).read() % parameters)
 
146
        output_file.close()
 
147
 
 
148
    def generateContentsFiles(self):
 
149
        self.logger.debug(
 
150
            "Running apt in private tree to generate new contents.")
 
151
        execute(self.logger, [
 
152
            "cp",
 
153
            "-a",
 
154
            self.config.overrideroot,
 
155
            "%s/" % self.content_archive,
 
156
            ])
 
157
        self.writeContentsTop()
 
158
        execute(self.logger, [
 
159
            "apt-ftparchive",
 
160
            "generate",
 
161
            "%s/%s-misc/apt-contents.conf" % (
 
162
                self.content_archive, self.distribution.name),
 
163
            ])
 
164
 
 
165
    def installContentsFile(self, suite, arch):
 
166
        contents_dir = os.path.join(
 
167
            self.content_archive, self.distribution.name, 'dists', suite)
 
168
        contents_filename = "Contents-%s" % arch
 
169
        last_contents = os.path.join(contents_dir, ".%s" % contents_filename)
 
170
        current_contents = os.path.join(contents_dir, contents_filename)
 
171
 
 
172
        # Avoid rewriting unchanged files; mirrors would have to
 
173
        # re-fetch them unnecessarily.
 
174
        if differ_in_content(current_contents, last_contents):
 
175
            self.logger.debug(
 
176
                "Installing new Contents file for %s/%s", suite, arch)
 
177
            if file_exists(last_contents):
 
178
                os.remove(last_contents)
 
179
            os.rename(current_contents, last_contents)
 
180
 
 
181
            new_contents = os.path.join(
 
182
                contents_dir, "%s.gz" % contents_filename)
 
183
            contents_dest = os.path.join(
 
184
                self.config.distsroot, suite, "%s.gz" % contents_filename)
 
185
            if file_exists(contents_dest):
 
186
                os.remove(contents_dest)
 
187
            os.rename(new_contents, contents_dest)
 
188
            os.chmod(contents_dest, 0664)
 
189
        else:
 
190
            self.logger.debug(
 
191
                "Skipping unmodified Contents file for %s/%s", suite, arch)
 
192
 
 
193
    def compareAndUpdate(self, suites, archs):
 
194
        self.logger.debug("Comparing contents files with public tree.")
 
195
        for suite in suites:
 
196
            for arch in archs:
 
197
                self.installContentsFile(suite, arch)
 
198
 
 
199
    def main(self):
 
200
        if self.distribution is None:
 
201
            raise OptionValueError("Specify a distribution using -d.")
 
202
 
 
203
        self.config = getPubConfig(self.distribution.main_archive)
 
204
        self.content_archive = os.path.join(
 
205
            config.archivepublisher.content_archive_root,
 
206
            self.distribution.name + "-contents")
 
207
 
 
208
        self.setUpPrivateTree()
 
209
        suites = self.getSuites()
 
210
        archs = self.getArchs()
 
211
        self.writeAptConf(suites, archs)
 
212
        self.createComponentDirs(suites, archs)
 
213
        self.generateContentsFiles()
 
214
        self.compareAndUpdate(suites, archs)