← Back to branch summary

~launchpad-pqm/launchpad/devel

« back to all changes in this revision

Viewing changes to lib/lp/soyuz/model/distroseriesdifferencejob.py

  • Committer: Curtis Hovey
  • Date: 2011-05-27 21:53:34 UTC
  • mto: This revision was merged to the branch mainline in revision 13136.
  • Revision ID: curtis.hovey@canonical.com-20110527215334-jqlkmt52nnl4bpeh
Moved launchpad.event into registry interfaces.

Show diffs side-by-side

added added

removed removed

Lines of Context:
lib/lp/soyuz/model/distroseriesdifferencejob.py
8
8
    'DistroSeriesDifferenceJob',
9
9
    ]
10
10
 
11
 
from storm.expr import And
12
11
from zope.component import getUtility
13
12
from zope.interface import (
14
13
    classProvides,
15
14
    implements,
16
15
    )
17
16
 
18
 
from canonical.database.sqlbase import sqlvalues
19
 
from canonical.launchpad.interfaces.lpstorm import (
20
 
    IMasterStore,
21
 
    IStore,
22
 
    )
 
17
from canonical.launchpad.interfaces.lpstorm import IMasterStore
23
18
from lp.registry.interfaces.distroseriesdifference import (
24
19
    IDistroSeriesDifferenceSource,
25
20
    )
26
 
from lp.registry.interfaces.distroseriesparent import IDistroSeriesParentSet
27
21
from lp.registry.interfaces.pocket import PackagePublishingPocket
28
 
from lp.registry.model.distroseries import DistroSeries
29
22
from lp.registry.model.distroseriesdifference import DistroSeriesDifference
30
23
from lp.registry.model.sourcepackagename import SourcePackageName
31
24
from lp.services.features import getFeatureFlag
36
29
    IDistroSeriesDifferenceJobSource,
37
30
    )
38
31
from lp.soyuz.interfaces.packageset import IPackagesetSet
39
 
from lp.soyuz.interfaces.publishing import active_publishing_status
40
32
from lp.soyuz.model.distributionjob import (
41
33
    DistributionJob,
42
34
    DistributionJobDerived,
43
35
    )
44
 
from lp.soyuz.model.publishing import SourcePackagePublishingHistory
45
 
from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
46
36
 
47
37
 
48
38
FEATURE_FLAG_ENABLE_MODULE = u"soyuz.derived_series_jobs.enabled"
49
39
 
50
40
 
51
 
def make_metadata(sourcepackagename, parent_series):
 
41
def make_metadata(sourcepackagename):
52
42
    """Return JSON metadata for a job on `sourcepackagename`."""
53
 
    return {
54
 
        'sourcepackagename': sourcepackagename.id,
55
 
        'parent_series': parent_series.id,
56
 
    }
57
 
 
58
 
 
59
 
def create_job(derived_series, sourcepackagename, parent_series):
 
43
    return {'sourcepackagename': sourcepackagename.id}
 
44
 
 
45
 
 
46
def create_job(distroseries, sourcepackagename):
60
47
    """Create a `DistroSeriesDifferenceJob` for a given source package.
61
48
 
62
 
    :param derived_series: A `DistroSeries` that is assumed to be derived
 
49
    :param distroseries: A `DistroSeries` that is assumed to be derived
63
50
        from another one.
64
51
    :param sourcepackagename: The `SourcePackageName` whose publication
65
52
        history has changed.
66
 
    :param parent_series: A `DistroSeries` that is a parent of
67
 
        `derived_series`.  The difference is between the versions of
68
 
        `sourcepackagename` in `parent_series` and `derived_series`.
69
53
    """
70
54
    job = DistributionJob(
71
 
        distribution=derived_series.distribution, distroseries=derived_series,
 
55
        distribution=distroseries.distribution, distroseries=distroseries,
72
56
        job_type=DistributionJobType.DISTROSERIESDIFFERENCE,
73
 
        metadata=make_metadata(sourcepackagename, parent_series))
 
57
        metadata=make_metadata(sourcepackagename))
74
58
    IMasterStore(DistributionJob).add(job)
75
59
    return DistroSeriesDifferenceJob(job)
76
60
 
77
61
 
78
 
def create_multiple_jobs(derived_series, parent_series):
79
 
    """Create a `DistroSeriesDifferenceJob` for all the source packages in
80
 
    archive.
81
 
 
82
 
    :param derived_series: A `DistroSeries` that is assumed to be derived
83
 
        from another one.
84
 
    :param parent_series: A `DistroSeries` that is a parent of
85
 
        `derived_series`.
86
 
    """
87
 
    store = IStore(SourcePackageRelease)
88
 
    source_package_releases = store.find(
89
 
        SourcePackageRelease,
90
 
        And(
91
 
            SourcePackagePublishingHistory.sourcepackagerelease ==
92
 
                SourcePackageRelease.id,
93
 
            SourcePackagePublishingHistory.distroseries == derived_series.id,
94
 
            SourcePackagePublishingHistory.status.is_in(
95
 
                active_publishing_status)))
96
 
    nb_jobs = source_package_releases.count()
97
 
    sourcepackagenames = source_package_releases.values(
98
 
        SourcePackageRelease.sourcepackagenameID)
99
 
    job_ids = Job.createMultiple(store, nb_jobs)
100
 
 
101
 
    def composeJobInsertionTuple(derived_series, parent_series,
102
 
                                 sourcepackagename, job_id):
103
 
        data = (
104
 
            derived_series.distribution.id, derived_series.id,
105
 
            DistributionJobType.DISTROSERIESDIFFERENCE, job_id,
106
 
            DistributionJob.serializeMetadata(
107
 
                {'sourcepackagename': sourcepackagename,
108
 
                 'parent_series': parent_series.id}))
109
 
        format_string = "(%s)" % ", ".join(["%s"] * len(data))
110
 
        return format_string % sqlvalues(*data)
111
 
 
112
 
    job_contents = [
113
 
        composeJobInsertionTuple(
114
 
            derived_series, parent_series, sourcepackagename, job_id)
115
 
        for job_id, sourcepackagename in
116
 
            zip(job_ids, sourcepackagenames)]
117
 
 
118
 
    store = IStore(DistributionJob)
119
 
    result = store.execute("""
120
 
        INSERT INTO DistributionJob (
121
 
            distribution, distroseries, job_type, job, json_data)
122
 
        VALUES %s
123
 
        RETURNING id
124
 
        """ % ", ".join(job_contents))
125
 
    return [job_id for job_id, in result]
126
 
 
127
 
 
128
 
def find_waiting_jobs(derived_series, sourcepackagename, parent_series):
 
62
def find_waiting_jobs(distroseries, sourcepackagename):
129
63
    """Look for pending `DistroSeriesDifference` jobs on a package."""
130
64
    # Look for identical pending jobs.  This compares directly on
131
65
    # the metadata string.  It's fragile, but this is only an
132
66
    # optimization.  It's not actually disastrous to create
133
67
    # redundant jobs occasionally.
134
68
    json_metadata = DistributionJob.serializeMetadata(
135
 
        make_metadata(sourcepackagename, parent_series))
 
69
        make_metadata(sourcepackagename))
136
70
 
137
71
    # Use master store because we don't like outdated information
138
72
    # here.
139
73
    store = IMasterStore(DistributionJob)
140
74
 
141
 
    candidates = store.find(
 
75
    return store.find(
142
76
        DistributionJob,
143
77
        DistributionJob.job_type ==
144
78
            DistributionJobType.DISTROSERIESDIFFERENCE,
145
 
        DistributionJob.distroseries == derived_series,
 
79
        DistributionJob.distroseries == distroseries,
146
80
        DistributionJob._json_data == json_metadata,
147
81
        DistributionJob.job_id.is_in(Job.ready_jobs))
148
82
 
149
 
    return [
150
 
        job
151
 
        for job in candidates
152
 
            if job.metadata["parent_series"] == parent_series.id]
153
 
 
154
 
 
155
 
def may_require_job(derived_series, sourcepackagename, parent_series):
 
83
 
 
84
def may_require_job(distroseries, sourcepackagename):
156
85
    """Might publishing this package require a new job?
157
86
 
158
87
    Use this to determine whether to create a new
161
90
    runner some unnecessary work, but we don't expect a bit of
162
91
    unnecessary work to be a big problem.
163
92
    """
164
 
    if derived_series is None:
165
 
        return False
166
 
    dsp = getUtility(IDistroSeriesParentSet).getByDerivedSeries(
167
 
        derived_series)
168
 
    if dsp.count() == 0:
169
 
        return False
170
 
    for parent in dsp:
171
 
        if parent.parent_series.distribution == derived_series.distribution:
172
 
            # Differences within a distribution are not tracked.
173
 
            return False
174
 
    existing_jobs = find_waiting_jobs(
175
 
        derived_series, sourcepackagename, parent_series)
176
 
    return len(existing_jobs) == 0
 
93
    if distroseries is None:
 
94
        return False
 
95
    previous_series = distroseries.previous_series
 
96
    if previous_series is None:
 
97
        return False
 
98
    if previous_series.distribution == distroseries.distribution:
 
99
        # Differences within a distribution are not tracked.
 
100
        return False
 
101
    return find_waiting_jobs(distroseries, sourcepackagename).is_empty()
177
102
 
178
103
 
179
104
def has_package(distroseries, sourcepackagename):
191
116
    class_job_type = DistributionJobType.DISTROSERIESDIFFERENCE
192
117
 
193
118
    @classmethod
194
 
    def createForPackagePublication(cls, derived_series, sourcepackagename,
 
119
    def createForPackagePublication(cls, distroseries, sourcepackagename,
195
120
                                    pocket):
196
121
        """See `IDistroSeriesDifferenceJobSource`."""
197
122
        if not getFeatureFlag(FEATURE_FLAG_ENABLE_MODULE):
205
130
            PackagePublishingPocket.PROPOSED):
206
131
            return
207
132
        jobs = []
208
 
        parent_series = derived_series.getParentSeries()
209
 
        # Create jobs for DSDs between the derived_series and its
210
 
        # parents.
211
 
        for parent in parent_series:
212
 
            if may_require_job(
213
 
                derived_series, sourcepackagename, parent):
214
 
                jobs.append(create_job(
215
 
                    derived_series, sourcepackagename, parent))
216
 
        # Create jobs for DSDs between the derived_series and its
217
 
        # children.
218
 
        for child in derived_series.getDerivedSeries():
219
 
            if may_require_job(
220
 
                child, sourcepackagename, derived_series):
221
 
                jobs.append(create_job(
222
 
                    child, sourcepackagename, derived_series))
 
133
        children = list(distroseries.getDerivedSeries())
 
134
        for relative in children + [distroseries]:
 
135
            if may_require_job(relative, sourcepackagename):
 
136
                jobs.append(create_job(relative, sourcepackagename))
223
137
        return jobs
224
138
 
225
 
    @classmethod
226
 
    def massCreateForSeries(cls, derived_series):
227
 
        """See `IDistroSeriesDifferenceJobSource`."""
228
 
        if not getFeatureFlag(FEATURE_FLAG_ENABLE_MODULE):
229
 
            return
230
 
        for parent_series in derived_series.getParentSeries():
231
 
            create_multiple_jobs(derived_series, parent_series)
232
 
 
233
 
    @classmethod
234
 
    def getPendingJobsForDifferences(cls, derived_series,
235
 
                                     distroseriesdifferences):
236
 
        """See `IDistroSeriesDifferenceJobSource`."""
237
 
        jobs = IStore(DistributionJob).find(
238
 
            DistributionJob,
239
 
            DistributionJob.job_type == cls.class_job_type,
240
 
            Job.id == DistributionJob.job_id,
241
 
            Job._status.is_in(Job.PENDING_STATUSES),
242
 
            DistributionJob.distroseries == derived_series)
243
 
 
244
 
        parent_series_ids = set(
245
 
            dsd.parent_series.id for dsd in distroseriesdifferences)
246
 
        keyed_dsds = dict(
247
 
            (dsd.source_package_name.id, dsd)
248
 
            for dsd in distroseriesdifferences)
249
 
        jobs_by_dsd = {}
250
 
        for job in jobs:
251
 
            if job.metadata["parent_series"] not in parent_series_ids:
252
 
                continue
253
 
            dsd = keyed_dsds.get(job.metadata["sourcepackagename"])
254
 
            if dsd is not None:
255
 
                jobs_by_dsd.setdefault(dsd, []).append(cls(job))
256
 
        return jobs_by_dsd
257
 
 
258
139
    @property
259
140
    def sourcepackagename(self):
260
141
        return SourcePackageName.get(self.metadata['sourcepackagename'])
261
142
 
262
 
    @property
263
 
    def derived_series(self):
264
 
        return self.distroseries
265
 
 
266
 
    @property
267
 
    def parent_series(self):
268
 
        parent_id = self.metadata['parent_series']
269
 
        return IStore(DistroSeries).get(DistroSeries, parent_id)
270
 
 
271
143
    def passesPackagesetFilter(self):
272
144
        """Is this package of interest as far as packagesets are concerned?
273
145
 
275
147
        missing in the derived series are only of interest if they are
276
148
        in a packageset that the derived series also has.
277
149
        """
278
 
        derived_series = self.derived_series
279
 
        parent_series = self.parent_series
280
 
 
281
 
        sourcepackagename = self.sourcepackagename
282
 
        if has_package(derived_series, sourcepackagename):
 
150
        derived_series = self.distroseries
 
151
        previous_series = derived_series.previous_series
 
152
        if has_package(derived_series, self.sourcepackagename):
283
153
            return True
284
 
        if not has_package(parent_series, sourcepackagename):
 
154
        if not has_package(previous_series, self.sourcepackagename):
285
155
            return True
286
156
        packagesetset = getUtility(IPackagesetSet)
287
 
        if packagesetset.getBySeries(parent_series).is_empty():
 
157
        if packagesetset.getBySeries(previous_series).is_empty():
288
158
            # Parent series does not have packagesets, as would be the
289
159
            # case for e.g. Debian.  In that case, don't filter.
290
160
            return True
291
161
        parent_sets = packagesetset.setsIncludingSource(
292
 
            sourcepackagename, distroseries=parent_series)
 
162
            self.sourcepackagename, distroseries=previous_series)
293
163
        for parent_set in parent_sets:
294
164
            for related_set in parent_set.relatedSets():
295
165
                if related_set.distroseries == derived_series:
296
166
                    return True
297
167
        return False
298
168
 
299
 
    def getMatchingDSD(self):
300
 
        """Find an existing `DistroSeriesDifference` for this difference."""
301
 
        spn_id = self.metadata["sourcepackagename"]
302
 
        parent_id = self.metadata["parent_series"]
303
 
        store = IMasterStore(DistroSeriesDifference)
304
 
        search = store.find(
305
 
            DistroSeriesDifference,
306
 
            DistroSeriesDifference.derived_series == self.derived_series,
307
 
            DistroSeriesDifference.parent_series_id == parent_id,
308
 
            DistroSeriesDifference.source_package_name_id == spn_id)
309
 
        return search.one()
310
 
 
311
169
    def run(self):
312
170
        """See `IRunnableJob`."""
313
171
        if not self.passesPackagesetFilter():
314
172
            return
315
173
 
316
 
        ds_diff = self.getMatchingDSD()
 
174
        store = IMasterStore(DistroSeriesDifference)
 
175
        ds_diff = store.find(
 
176
            DistroSeriesDifference,
 
177
            DistroSeriesDifference.derived_series == self.distroseries,
 
178
            DistroSeriesDifference.source_package_name ==
 
179
            self.sourcepackagename).one()
317
180
        if ds_diff is None:
318
181
            ds_diff = getUtility(IDistroSeriesDifferenceSource).new(
319
 
                self.distroseries, self.sourcepackagename, self.parent_series)
 
182
                self.distroseries, self.sourcepackagename)
320
183
        else:
321
184
            ds_diff.update()