← Back to branch summary

~launchpad-pqm/launchpad/devel

« back to all changes in this revision

Viewing changes to lib/lp/soyuz/model/archive.py

  • Committer: Launchpad Patch Queue Manager
  • Date: 2011-06-25 08:55:37 UTC
  • mfrom: (13287.1.8 bug-800652)
  • Revision ID: launchpad@pqm.canonical.com-20110625085537-moikyoo2pe98zs7r
[r=jcsackett, julian-edwards][bug=800634,
        800652] Enable and display overrides on sync package uploads.

Show diffs side-by-side

added added

removed removed

Lines of Context:
lib/lp/soyuz/model/archive.py
43
43
    implements,
44
44
    )
45
45
 
 
46
from canonical.config import config
 
47
from canonical.database.constants import UTC_NOW
 
48
from canonical.database.datetimecol import UtcDateTimeCol
 
49
from canonical.database.enumcol import EnumCol
 
50
from canonical.database.sqlbase import (
 
51
    cursor,
 
52
    quote,
 
53
    quote_like,
 
54
    SQLBase,
 
55
    sqlvalues,
 
56
    )
 
57
from canonical.launchpad.components.decoratedresultset import (
 
58
    DecoratedResultSet,
 
59
    )
 
60
from canonical.launchpad.components.tokens import (
 
61
    create_unique_token_for_table,
 
62
    create_token,
 
63
    )
 
64
from canonical.launchpad.database.librarian import (
 
65
    LibraryFileAlias,
 
66
    LibraryFileContent,
 
67
    )
 
68
from canonical.launchpad.interfaces.lpstorm import (
 
69
    ISlaveStore,
 
70
    IStore,
 
71
    )
 
72
from canonical.launchpad.webapp.authorization import check_permission
 
73
from canonical.launchpad.webapp.interfaces import (
 
74
    DEFAULT_FLAVOR,
 
75
    IStoreSelector,
 
76
    MAIN_STORE,
 
77
    )
 
78
from canonical.launchpad.webapp.url import urlappend
46
79
from lp.app.errors import NotFoundError
47
80
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
48
81
from lp.app.validators.name import valid_name
69
102
from lp.registry.interfaces.sourcepackagename import ISourcePackageNameSet
70
103
from lp.registry.model.sourcepackagename import SourcePackageName
71
104
from lp.registry.model.teammembership import TeamParticipation
72
 
from lp.services.config import config
73
 
from lp.services.database.bulk import load_related
74
 
from lp.services.database.constants import UTC_NOW
75
 
from lp.services.database.datetimecol import UtcDateTimeCol
76
 
from lp.services.database.decoratedresultset import DecoratedResultSet
77
 
from lp.services.database.enumcol import EnumCol
78
 
from lp.services.database.lpstorm import (
79
 
    ISlaveStore,
80
 
    IStore,
81
 
    )
82
 
from lp.services.database.sqlbase import (
83
 
    cursor,
84
 
    quote,
85
 
    quote_like,
86
 
    SQLBase,
87
 
    sqlvalues,
88
 
    )
89
 
from lp.services.features import getFeatureFlag
90
105
from lp.services.job.interfaces.job import JobStatus
91
 
from lp.services.librarian.model import (
92
 
    LibraryFileAlias,
93
 
    LibraryFileContent,
94
 
    )
95
106
from lp.services.propertycache import (
96
107
    cachedproperty,
97
108
    get_property_cache,
98
109
    )
99
 
from lp.services.tokens import (
100
 
    create_token,
101
 
    create_unique_token_for_table,
102
 
    )
103
 
from lp.services.webapp.authorization import check_permission
104
 
from lp.services.webapp.interfaces import (
105
 
    DEFAULT_FLAVOR,
106
 
    IStoreSelector,
107
 
    MAIN_STORE,
108
 
    )
109
 
from lp.services.webapp.url import urlappend
110
110
from lp.soyuz.adapters.archivedependencies import expand_dependencies
111
111
from lp.soyuz.adapters.packagelocation import PackageLocation
112
112
from lp.soyuz.enums import (
115
115
    ArchivePurpose,
116
116
    ArchiveStatus,
117
117
    ArchiveSubscriberStatus,
118
 
    PackageCopyPolicy,
119
118
    PackagePublishingStatus,
120
119
    PackageUploadStatus,
121
120
    )
131
130
    CannotUploadToPPA,
132
131
    ComponentNotFound,
133
132
    default_name_by_purpose,
134
 
    ForbiddenByFeatureFlag,
135
133
    FULL_COMPONENT_SUPPORT,
136
134
    IArchive,
137
135
    IArchiveSet,
148
146
    NoSuchPPA,
149
147
    NoTokensForTeams,
150
148
    PocketNotFound,
 
149
    VersionRequiresName,
151
150
    validate_external_dependencies,
152
 
    VersionRequiresName,
153
151
    )
154
152
from lp.soyuz.interfaces.archivearch import IArchiveArchSet
155
153
from lp.soyuz.interfaces.archiveauthtoken import IArchiveAuthTokenSet
167
165
    IComponent,
168
166
    IComponentSet,
169
167
    )
170
 
from lp.soyuz.interfaces.packagecopyjob import IPlainPackageCopyJobSource
171
168
from lp.soyuz.interfaces.packagecopyrequest import IPackageCopyRequestSet
172
169
from lp.soyuz.interfaces.processor import IProcessorFamilySet
173
170
from lp.soyuz.interfaces.publishing import (
184
181
    BinaryPackageReleaseDownloadCount,
185
182
    )
186
183
from lp.soyuz.model.component import Component
 
184
from lp.soyuz.model.distributionsourcepackagecache import (
 
185
    DistributionSourcePackageCache,
 
186
    )
 
187
from lp.soyuz.model.distroseriespackagecache import DistroSeriesPackageCache
187
188
from lp.soyuz.model.files import (
188
189
    BinaryPackageFile,
189
190
    SourcePackageReleaseFile,
199
200
    )
200
201
from lp.soyuz.model.section import Section
201
202
from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
202
 
from lp.soyuz.scripts.packagecopier import check_copy_permissions
203
203
 
204
204
 
205
205
def storm_validate_external_dependencies(archive, attr, value):
493
493
    def getPublishedSources(self, name=None, version=None, status=None,
494
494
                            distroseries=None, pocket=None,
495
495
                            exact_match=False, created_since_date=None,
496
 
                            eager_load=False, component_name=None):
 
496
                            eager_load=False):
497
497
        """See `IArchive`."""
498
498
        # clauses contains literal sql expressions for things that don't work
499
499
        # easily in storm : this method was migrated from sqlobject but some
533
533
        else:
534
534
            orderBy.insert(1, Desc(SourcePackageRelease.version))
535
535
 
536
 
        if component_name is not None:
537
 
            storm_clauses.extend(
538
 
                [SourcePackagePublishingHistory.componentID == Component.id,
539
 
                 Component.name == component_name,
540
 
                 ])
541
 
 
542
536
        if status is not None:
543
537
            try:
544
538
                status = tuple(status)
554
548
                    distroseries.id)
555
549
 
556
550
        if pocket is not None:
557
 
            try:
558
 
                pockets = tuple(pocket)
559
 
            except TypeError:
560
 
                pockets = (pocket,)
561
551
            storm_clauses.append(
562
 
                "SourcePackagePublishingHistory.pocket IN %s " %
563
 
                   sqlvalues(pockets))
 
552
                SourcePackagePublishingHistory.pocket == pocket)
564
553
 
565
554
        if created_since_date is not None:
566
555
            clauses.append(
712
701
 
713
702
    def _getBinaryPublishingBaseClauses(
714
703
        self, name=None, version=None, status=None, distroarchseries=None,
715
 
        pocket=None, exact_match=False, created_since_date=None,
716
 
        ordered=True):
 
704
        pocket=None, exact_match=False):
717
705
        """Base clauses and clauseTables for binary publishing queries.
718
706
 
719
707
        Returns a list of 'clauses' (to be joined in the callsite) and
727
715
                BinaryPackageName.id
728
716
        """ % sqlvalues(self)]
729
717
        clauseTables = ['BinaryPackageRelease', 'BinaryPackageName']
730
 
        if ordered:
731
 
            orderBy = ['BinaryPackageName.name',
732
 
                       '-BinaryPackagePublishingHistory.id']
733
 
        else:
734
 
            # Strictly speaking, this is ordering, but it's an indexed
735
 
            # ordering so it will be quick.  It's needed so that we can
736
 
            # batch results on the webservice.
737
 
            orderBy = ['-BinaryPackagePublishingHistory.id']
 
718
        orderBy = ['BinaryPackageName.name',
 
719
                   '-BinaryPackagePublishingHistory.id']
738
720
 
739
721
        if name is not None:
740
722
            if exact_match:
755
737
            clauses.append("""
756
738
                BinaryPackageRelease.version = %s
757
739
            """ % sqlvalues(version))
758
 
        elif ordered:
 
740
        else:
759
741
            order_const = "BinaryPackageRelease.version"
760
742
            desc_version_order = SQLConstant(order_const + " DESC")
761
743
            orderBy.insert(1, desc_version_order)
786
768
                BinaryPackagePublishingHistory.pocket = %s
787
769
            """ % sqlvalues(pocket))
788
770
 
789
 
        if created_since_date is not None:
790
 
            clauses.append(
791
 
                "BinaryPackagePublishingHistory.datecreated >= %s"
792
 
                % sqlvalues(created_since_date))
793
 
 
794
771
        return clauses, clauseTables, orderBy
795
772
 
796
773
    def getAllPublishedBinaries(self, name=None, version=None, status=None,
797
774
                                distroarchseries=None, pocket=None,
798
 
                                exact_match=False, created_since_date=None,
799
 
                                ordered=True):
 
775
                                exact_match=False):
800
776
        """See `IArchive`."""
801
777
        clauses, clauseTables, orderBy = self._getBinaryPublishingBaseClauses(
802
778
            name=name, version=version, status=status, pocket=pocket,
803
 
            distroarchseries=distroarchseries, exact_match=exact_match,
804
 
            created_since_date=created_since_date, ordered=ordered)
 
779
            distroarchseries=distroarchseries, exact_match=exact_match)
805
780
 
806
781
        all_binaries = BinaryPackagePublishingHistory.select(
807
782
            ' AND '.join(clauses), clauseTables=clauseTables,
811
786
 
812
787
    def getPublishedOnDiskBinaries(self, name=None, version=None, status=None,
813
788
                                   distroarchseries=None, pocket=None,
814
 
                                   exact_match=False,
815
 
                                   created_since_date=None):
 
789
                                   exact_match=False):
816
790
        """See `IArchive`."""
817
791
        clauses, clauseTables, orderBy = self._getBinaryPublishingBaseClauses(
818
792
            name=name, version=version, status=status, pocket=pocket,
819
 
            distroarchseries=distroarchseries, exact_match=exact_match,
820
 
            created_since_date=created_since_date)
 
793
            distroarchseries=distroarchseries, exact_match=exact_match)
821
794
 
822
795
        clauses.append("""
823
796
            BinaryPackagePublishingHistory.distroarchseries =
926
899
 
927
900
    def updateArchiveCache(self):
928
901
        """See `IArchive`."""
929
 
        from lp.soyuz.model.distributionsourcepackagecache import (
930
 
            DistributionSourcePackageCache,
931
 
            )
932
 
        from lp.soyuz.model.distroseriespackagecache import (
933
 
            DistroSeriesPackageCache)
934
902
        # Compiled regexp to remove puntication.
935
903
        clean_text = re.compile('(,|;|:|\.|\?|!)')
936
904
 
1539
1507
            sources, to_pocket, to_series, include_binaries,
1540
1508
            person=person)
1541
1509
 
1542
 
    def _validateAndFindSource(self, from_archive, source_name, version):
 
1510
    def syncSource(self, source_name, version, from_archive, to_pocket,
 
1511
                   to_series=None, include_binaries=False, person=None):
 
1512
        """See `IArchive`."""
1543
1513
        # Check to see if the source package exists, and raise a useful error
1544
1514
        # if it doesn't.
1545
1515
        getUtility(ISourcePackageNameSet)[source_name]
1546
1516
        # Find and validate the source package version required.
1547
1517
        source = from_archive.getPublishedSources(
1548
1518
            name=source_name, version=version, exact_match=True).first()
1549
 
        return source
1550
 
 
1551
 
    def syncSource(self, source_name, version, from_archive, to_pocket,
1552
 
                   to_series=None, include_binaries=False, person=None):
1553
 
        """See `IArchive`."""
1554
 
        source = self._validateAndFindSource(
1555
 
            from_archive, source_name, version)
1556
1519
 
1557
1520
        self._copySources(
1558
1521
            [source], to_pocket, to_series, include_binaries,
1559
1522
            person=person)
1560
1523
 
1561
 
    def _checkCopyPackageFeatureFlags(self):
1562
 
        """Prevent copyPackage(s) if these conditions are not met."""
1563
 
        if not getFeatureFlag(u"soyuz.copypackage.enabled"):
1564
 
            raise ForbiddenByFeatureFlag
1565
 
        if (self.is_ppa and
1566
 
            not getFeatureFlag(u"soyuz.copypackageppa.enabled")):
1567
 
            # We have no way of giving feedback about failed jobs yet,
1568
 
            # so this is disabled for now.
1569
 
            raise ForbiddenByFeatureFlag(
1570
 
                "Not enabled for copying to PPAs yet.")
1571
 
 
1572
 
    def copyPackage(self, source_name, version, from_archive, to_pocket,
1573
 
                    person, to_series=None, include_binaries=False,
1574
 
                    sponsored=None):
1575
 
        """See `IArchive`."""
1576
 
        self._checkCopyPackageFeatureFlags()
1577
 
 
1578
 
        # Asynchronously copy a package using the job system.
1579
 
        pocket = self._text_to_pocket(to_pocket)
1580
 
        series = self._text_to_series(to_series)
1581
 
        # Upload permission checks, this will raise CannotCopy as
1582
 
        # necessary.
1583
 
        sourcepackagename = getUtility(ISourcePackageNameSet)[source_name]
1584
 
        check_copy_permissions(
1585
 
            person, self, series, pocket, [sourcepackagename])
1586
 
 
1587
 
        self._validateAndFindSource(from_archive, source_name, version)
1588
 
        job_source = getUtility(IPlainPackageCopyJobSource)
1589
 
        job_source.create(
1590
 
            package_name=source_name, source_archive=from_archive,
1591
 
            target_archive=self, target_distroseries=series,
1592
 
            target_pocket=pocket,
1593
 
            package_version=version, include_binaries=include_binaries,
1594
 
            copy_policy=PackageCopyPolicy.INSECURE, requester=person,
1595
 
            sponsored=sponsored)
1596
 
 
1597
 
    def copyPackages(self, source_names, from_archive, to_pocket,
1598
 
                     person, to_series=None, include_binaries=None,
1599
 
                     sponsored=None):
1600
 
        """See `IArchive`."""
1601
 
        self._checkCopyPackageFeatureFlags()
1602
 
 
1603
 
        sources = self._collectLatestPublishedSources(
1604
 
            from_archive, source_names)
1605
 
        if not sources:
1606
 
            raise CannotCopy(
1607
 
                "None of the supplied package names are published")
1608
 
 
1609
 
        # Bulk-load the sourcepackagereleases so that the list
1610
 
        # comprehension doesn't generate additional queries. The
1611
 
        # sourcepackagenames themselves will already have been loaded when
1612
 
        # generating the list of source publications in "sources".
1613
 
        load_related(
1614
 
            SourcePackageRelease, sources, ["sourcepackagereleaseID"])
1615
 
        sourcepackagenames = [source.sourcepackagerelease.sourcepackagename
1616
 
                              for source in sources]
1617
 
 
1618
 
        # Now do a mass check of permissions.
1619
 
        pocket = self._text_to_pocket(to_pocket)
1620
 
        series = self._text_to_series(to_series)
1621
 
        check_copy_permissions(
1622
 
            person, self, series, pocket, sourcepackagenames)
1623
 
 
1624
 
        # If we get this far then we can create the PackageCopyJob.
1625
 
        copy_tasks = []
1626
 
        for source in sources:
1627
 
            task = (
1628
 
                source.sourcepackagerelease.sourcepackagename.name,
1629
 
                source.sourcepackagerelease.version,
1630
 
                from_archive,
1631
 
                self,
1632
 
                PackagePublishingPocket.RELEASE
1633
 
                )
1634
 
            copy_tasks.append(task)
1635
 
 
1636
 
        job_source = getUtility(IPlainPackageCopyJobSource)
1637
 
        job_source.createMultiple(
1638
 
            series, copy_tasks, person,
1639
 
            copy_policy=PackageCopyPolicy.MASS_SYNC,
1640
 
            include_binaries=include_binaries, sponsored=sponsored)
1641
 
 
1642
1524
    def _collectLatestPublishedSources(self, from_archive, source_names):
1643
1525
        """Private helper to collect the latest published sources for an
1644
1526
        archive.
1646
1528
        :raises NoSuchSourcePackageName: If any of the source_names do not
1647
1529
            exist.
1648
1530
        """
1649
 
        # XXX bigjools bug=810421
1650
 
        # This code is inefficient.  It should try to bulk load all the
1651
 
        # sourcepackagenames and publications instead of iterating.
1652
1531
        sources = []
1653
1532
        for name in source_names:
1654
1533
            # Check to see if the source package exists. This will raise
1659
1538
            # publication.
1660
1539
            published_sources = from_archive.getPublishedSources(
1661
1540
                name=name, exact_match=True,
1662
 
                status=(PackagePublishingStatus.PUBLISHED,
1663
 
                        PackagePublishingStatus.PENDING))
 
1541
                status=PackagePublishingStatus.PUBLISHED)
1664
1542
            first_source = published_sources.first()
1665
1543
            if first_source is not None:
1666
1544
                sources.append(first_source)
1667
1545
        return sources
1668
1546
 
1669
 
    def _text_to_series(self, to_series):
1670
 
        if to_series is not None:
1671
 
            result = getUtility(IDistroSeriesSet).queryByName(
1672
 
                self.distribution, to_series)
1673
 
            if result is None:
1674
 
                raise NoSuchDistroSeries(to_series)
1675
 
            series = result
1676
 
        else:
1677
 
            series = None
1678
 
 
1679
 
        return series
1680
 
 
1681
 
    def _text_to_pocket(self, to_pocket):
1682
 
        # Convert the to_pocket string to its enum.
1683
 
        try:
1684
 
            pocket = PackagePublishingPocket.items[to_pocket.upper()]
1685
 
        except KeyError:
1686
 
            raise PocketNotFound(to_pocket.upper())
1687
 
 
1688
 
        return pocket
1689
 
 
1690
1547
    def _copySources(self, sources, to_pocket, to_series=None,
1691
1548
                     include_binaries=False, person=None):
1692
1549
        """Private helper function to copy sources to this archive.
1696
1553
        """
1697
1554
        # Circular imports.
1698
1555
        from lp.soyuz.scripts.packagecopier import do_copy
 
1556
        # Convert the to_pocket string to its enum.
 
1557
        try:
 
1558
            pocket = PackagePublishingPocket.items[to_pocket.upper()]
 
1559
        except KeyError:
 
1560
            raise PocketNotFound(to_pocket.upper())
1699
1561
 
1700
 
        pocket = self._text_to_pocket(to_pocket)
1701
1562
        # Fail immediately if the destination pocket is not Release and
1702
1563
        # this archive is a PPA.
1703
1564
        if self.is_ppa and pocket != PackagePublishingPocket.RELEASE:
1705
1566
                "Destination pocket must be 'release' for a PPA.")
1706
1567
 
1707
1568
        # Now convert the to_series string to a real distroseries.
1708
 
        series = self._text_to_series(to_series)
 
1569
        if to_series is not None:
 
1570
            result = getUtility(IDistroSeriesSet).queryByName(
 
1571
                self.distribution, to_series)
 
1572
            if result is None:
 
1573
                raise NoSuchDistroSeries(to_series)
 
1574
            series = result
 
1575
        else:
 
1576
            series = None
1709
1577
 
1710
1578
        # Perform the copy, may raise CannotCopy. Don't do any further
1711
1579
        # permission checking: this method is protected by
1904
1772
            "This archive is already deleted.")
1905
1773
 
1906
1774
        # Set all the publications to DELETED.
1907
 
        sources = self.getPublishedSources(status=active_publishing_status)
 
1775
        statuses = (
 
1776
            PackagePublishingStatus.PENDING,
 
1777
            PackagePublishingStatus.PUBLISHED)
 
1778
        sources = list(self.getPublishedSources(status=statuses))
1908
1779
        getUtility(IPublishingSet).requestDeletion(
1909
1780
            sources, removed_by=deleted_by,
1910
1781
            removal_comment="Removed when deleting archive")
1933
1804
        """Retrieve the restricted architecture families this archive can
1934
1805
        build on."""
1935
1806
        # Main archives are always allowed to build on restricted
1936
 
        # architectures if require_virtualized is False.
1937
 
        if self.is_main and not self.require_virtualized:
 
1807
        # architectures.
 
1808
        if self.is_main:
1938
1809
            return getUtility(IProcessorFamilySet).getRestricted()
1939
1810
        archive_arch_set = getUtility(IArchiveArchSet)
1940
1811
        restricted_families = archive_arch_set.getRestrictedFamilies(self)
1944
1815
    def _setEnabledRestrictedFamilies(self, value):
1945
1816
        """Set the restricted architecture families this archive can
1946
1817
        build on."""
1947
 
        # Main archives are not allowed to build on restricted
1948
 
        # architectures unless they are set to build on virtualized
1949
 
        # builders.
1950
 
        if (self.is_main and not self.require_virtualized):
 
1818
        # Main archives are always allowed to build on restricted
 
1819
        # architectures.
 
1820
        if self.is_main:
1951
1821
            proc_family_set = getUtility(IProcessorFamilySet)
1952
1822
            if set(value) != set(proc_family_set.getRestricted()):
1953
 
                raise CannotRestrictArchitectures(
1954
 
                    "Main archives can not be restricted to certain "
1955
 
                    "architectures unless they are set to build on "
1956
 
                    "virtualized builders")
 
1823
                raise CannotRestrictArchitectures("Main archives can not "
 
1824
                        "be restricted to certain architectures")
1957
1825
        archive_arch_set = getUtility(IArchiveArchSet)
1958
1826
        restricted_families = archive_arch_set.getRestrictedFamilies(self)
1959
1827
        for (family, archive_arch) in restricted_families:
1972
1840
        self.enabled_restricted_families = restricted
1973
1841
 
1974
1842
    @classmethod
1975
 
    def validatePPA(self, person, proposed_name, private=False):
 
1843
    def validatePPA(self, person, proposed_name):
1976
1844
        ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
1977
 
        if private:
1978
 
            # NOTE: This duplicates the policy in lp/soyuz/configure.zcml
1979
 
            # which says that one needs 'launchpad.Commercial' permission to
1980
 
            # set 'private', and the logic in `AdminByCommercialTeamOrAdmins`
1981
 
            # which determines who is granted launchpad.Commercial
1982
 
            # permissions.
1983
 
            commercial = getUtility(ILaunchpadCelebrities).commercial_admin
1984
 
            admin = getUtility(ILaunchpadCelebrities).admin
1985
 
            if not person.inTeam(commercial) and not person.inTeam(admin):
1986
 
                return '%s is not allowed to make private PPAs' % person.name
1987
 
        if person.is_team and (
 
1845
        if person.isTeam() and (
1988
1846
            person.subscriptionpolicy in OPEN_TEAM_POLICY):
1989
1847
            return "Open teams cannot have PPAs."
1990
1848
        if proposed_name is not None and proposed_name == ubuntu.name:
1998
1856
            return None
1999
1857
        else:
2000
1858
            text = "You already have a PPA named '%s'." % proposed_name
2001
 
            if person.is_team:
 
1859
            if person.isTeam():
2002
1860
                text = "%s already has a PPA named '%s'." % (
2003
1861
                    person.displayname, proposed_name)
2004
1862
            return text
2107
1965
 
2108
1966
    def new(self, purpose, owner, name=None, displayname=None,
2109
1967
            distribution=None, description=None, enabled=True,
2110
 
            require_virtualized=True, private=False):
 
1968
            require_virtualized=True):
2111
1969
        """See `IArchiveSet`."""
2112
1970
        if distribution is None:
2113
1971
            distribution = getUtility(ILaunchpadCelebrities).ubuntu
2181
2039
            new_archive.buildd_secret = create_unique_token_for_table(
2182
2040
                20, Archive.buildd_secret)
2183
2041
            new_archive.private = True
2184
 
        else:
2185
 
            new_archive.private = private
2186
2042
 
2187
2043
        return new_archive
2188
2044