~launchpad-pqm/launchpad/devel

Upload processing queue with translations
=========================================

This test covers the use case when a package includes translations and is
uploaded into the system.

    >>> from lp.registry.model.gpgkey import GPGKey
    >>> from lp.soyuz.model.component import Component
    >>> from lp.soyuz.model.processor import ProcessorFamily
    >>> from lp.soyuz.model.section import Section
    >>> from lp.soyuz.model.publishing import (
    ...     SourcePackagePublishingHistory)
    >>> from lp.registry.interfaces.distribution import IDistributionSet
    >>> from lp.registry.interfaces.distroseries import (
    ...     IDistroSeries,
    ...     IDistroSeriesSet,
    ...     )
    >>> from lp.registry.interfaces.pocket import PackagePublishingPocket
    >>> from lp.soyuz.interfaces.component import IComponentSet
    >>> from lp.soyuz.enums import (
    ...     PackagePublishingStatus)
    >>> from lp.registry.interfaces.sourcepackagename import (
    ...     ISourcePackageNameSet)

    >>> from lp.archiveuploader.nascentupload import NascentUpload
    >>> from lp.archiveuploader.tests import datadir, getPolicy

    >>> from lp.testing.gpgkeys import import_public_test_keys
    >>> import_public_test_keys()

    >>> from canonical.database.constants import UTC_NOW
    >>> from lp.registry.interfaces.sourcepackage import SourcePackageUrgency

# Login as an admin.
    >>> login('foo.bar@canonical.com')

    # We need to setup our test environment and create the needed objects.
    >>> distro_series_set = getUtility(IDistroSeriesSet)
    >>> ubuntu = getUtility(IDistributionSet)['ubuntu']
    >>> hoary = distro_series_set.queryByName(ubuntu, 'hoary')

# Create the Dapper distro series.
    >>> dapper = ubuntu.newSeries(
    ...     'dapper', 'Dapper', 'Dapper',
    ...     'Dapper', 'Dapper', '06.04', hoary, hoary.owner)

# And an AMD 64 arch series.
    >>> dapper_amd64 = dapper.newArch('amd64', ProcessorFamily.get(3), True,
    ...     dapper.owner)

Only uploads to the RELEASE, UPDATES, SECURITY and PROPOSED pockets are
considered for import. An upload to the BACKPORT pocket won't appear in the
queue:

# We are going to import the pmount build into RELEASE pocket.
    >>> pmount_sourcepackagename = getUtility(ISourcePackageNameSet)['pmount']
    >>> source_package_release = dapper.createUploadedSourcePackageRelease(
    ...     pmount_sourcepackagename, "0.9.7-2ubuntu2", dapper.owner,
    ...     None, None, 'i386', Component.get(1), dapper.owner,
    ...     SourcePackageUrgency.LOW, None, None, None, GPGKey.get(1),
    ...     Section.get(1), '', '', '', '', dapper.main_archive,
    ...     'copyright ?!', '', '')

    >>> publishing_history = SourcePackagePublishingHistory(
    ...     distroseries=dapper.id,
    ...     sourcepackagerelease=source_package_release.id,
    ...     sourcepackagename=source_package_release.sourcepackagename,
    ...     component=source_package_release.component.id,
    ...     section=source_package_release.section.id,
    ...     status=PackagePublishingStatus.PUBLISHED,
    ...     datecreated=UTC_NOW,
    ...     pocket=PackagePublishingPocket.RELEASE,
    ...     archive=dapper.main_archive)

# Do the upload into the system.

    >>> build = source_package_release.createBuild(
    ...      dapper_amd64, PackagePublishingPocket.RELEASE,
    ...      dapper.main_archive)

    >>> buildd_policy = getPolicy(
    ...     name='buildd', distro='ubuntu', distroseries='dapper')

    >>> from lp.services.log.logger import FakeLogger
    >>> pmount_upload = NascentUpload.from_changesfile_path(
    ...     datadir('pmount_0.9.7-2ubuntu2_amd64.changes'),
    ...     buildd_policy, FakeLogger())
    DEBUG pmount_0.9.7-2ubuntu2_amd64.changes can be unsigned.

    >>> pmount_upload.process(build=build)
    DEBUG Beginning processing.
    DEBUG Verifying the changes file.
    DEBUG Verifying files in upload.
    DEBUG Verifying binary pmount_0.9.7-2ubuntu2_amd64.deb
    DEBUG Verifying timestamps in pmount_0.9.7-2ubuntu2_amd64.deb
    DEBUG Finding and applying overrides.
    DEBUG Checking for pmount/0.9.7-2ubuntu2/amd64 binary ancestry
    DEBUG pmount: (binary) NEW
    DEBUG Finished checking upload.

 # It was not rejected.
    >>> pmount_upload.is_rejected
    False

At this point, no translations uploads have been registered for this
package.

    >>> from lp.registry.model.sourcepackage import SourcePackage
    >>> dapper_pmount = SourcePackage(pmount_sourcepackagename, dapper)
    >>> print len(dapper_pmount.getLatestTranslationsUploads())
    0

    >>> success = pmount_upload.do_accept(build=build)
    DEBUG Creating queue entry
    ...

    # And all things worked.
    >>> success
    True

# Ensure 'deb' is NEW and 'translation' is recognized, i.e., ACCEPTED
# XXX julian 2007-05-27 Commented out for now because getNotificationSummary
# no longer exists and this content is impossible to check at the moment
# since no email is generated because the recipients are not LP Persons.
# (So why is it being checked in the first place?)
#>>> print pmount_upload.getNotificationSummary()
#NEW: pmount_0.9.7-2ubuntu2_amd64.deb
#OK: pmount_0.9.7-2ubuntu2_amd64_translations.tar.gz

The upload now shows up as the latest translations upload for the
package.

    >>> latest_translations_uploads = list(
    ...     dapper_pmount.getLatestTranslationsUploads())
    >>> print len(latest_translations_uploads)
    1

We'll get back to that uploaded file later.

    >>> latest_translations_upload = latest_translations_uploads[0]

# Check the import queue content, it should be empty.
    >>> from lp.translations.interfaces.translationimportqueue import (
    ...     ITranslationImportQueue)
    >>> translation_import_queue = getUtility(ITranslationImportQueue)
    >>> translation_import_queue.getAllEntries(target=ubuntu).count()
    0

# We need to commit the transaction to be able to use the librarian files.
    >>> import transaction
    >>> transaction.commit()

An upload to the RELEASE pocket will add items to the import queue:

    >>> from lp.soyuz.enums import PackageUploadStatus
    >>> queue_item = dapper.getPackageUploads(
    ...     status=PackageUploadStatus.NEW)[0]
    >>> queue_item.customfiles[0].publish()

As we can see from the translation import queue content.

    >>> for entry in translation_import_queue.getAllEntries(target=ubuntu):
    ...     print '%s/%s by %s: %s' % (
    ...         entry.distroseries.name, entry.sourcepackagename.name,
    ...         entry.importer.name, entry.path)
    dapper/pmount by ubuntu-team: po/es_ES.po
    dapper/pmount by ubuntu-team: po/ca.po
    dapper/pmount by ubuntu-team: po/de.po
    dapper/pmount by ubuntu-team: po/cs.po
    dapper/pmount by ubuntu-team: po/es.po
    dapper/pmount by ubuntu-team: po/fr.po
    dapper/pmount by ubuntu-team: po/hr.po
    dapper/pmount by ubuntu-team: po/nb.po
    dapper/pmount by ubuntu-team: po/pmount.pot
    dapper/pmount by ubuntu-team: po/it_IT.po

# Abort the transaction so we can check the same upload in a different
# pocket.
    >>> transaction.abort()

# The import queue content should be empty now that the transaction is
# reverted.
    >>> translation_import_queue.getAllEntries(target=ubuntu).count()
    0

An upload to the BACKPORTS pocket will not add items to the import queue:

    >>> ubuntu = getUtility(IDistributionSet)['ubuntu']
    >>> dapper = distro_series_set.queryByName(ubuntu, 'dapper')
    >>> queue_item = dapper.getPackageUploads(PackageUploadStatus.NEW)[0]
    >>> queue_item.pocket = PackagePublishingPocket.BACKPORTS
    >>> queue_item.customfiles[0].publish()

# And this time, we see that there are no entries imported in the queue.
    >>> translation_import_queue.getAllEntries(target=ubuntu).count()
    0

# Let's abort the transaction so we can check the same upload in a different
# pocket.
    >>> transaction.abort()

But an upload to the UPDATE pocket will add items to the import queue:

    >>> ubuntu = getUtility(IDistributionSet)['ubuntu']
    >>> dapper = distro_series_set.queryByName(ubuntu, 'dapper')
    >>> queue_item = dapper.getPackageUploads(PackageUploadStatus.NEW)[0]
    >>> queue_item.pocket = PackagePublishingPocket.UPDATES
    >>> queue_item.customfiles[0].publish()

As we can see from the translation import queue content.

    >>> for entry in translation_import_queue.getAllEntries(target=ubuntu):
    ...     print '%s/%s by %s: %s' % (
    ...         entry.distroseries.name, entry.sourcepackagename.name,
    ...         entry.importer.name, entry.path)
    dapper/pmount by ubuntu-team: po/es_ES.po
    dapper/pmount by ubuntu-team: po/ca.po
    dapper/pmount by ubuntu-team: po/de.po
    dapper/pmount by ubuntu-team: po/cs.po
    dapper/pmount by ubuntu-team: po/es.po
    dapper/pmount by ubuntu-team: po/fr.po
    dapper/pmount by ubuntu-team: po/hr.po
    dapper/pmount by ubuntu-team: po/nb.po
    dapper/pmount by ubuntu-team: po/pmount.pot
    dapper/pmount by ubuntu-team: po/it_IT.po

# Let's abort the transaction so we can check the same upload in a different
# pocket.
    >>> transaction.abort()

Uploads to restricted component are accepted too.

    >>> ubuntu = getUtility(IDistributionSet)['ubuntu']
    >>> dapper = distro_series_set.queryByName(ubuntu, 'dapper')
    >>> restricted_component = getUtility(IComponentSet)['restricted']
    >>> queue_item = dapper.getPackageUploads(PackageUploadStatus.NEW)[0]

# Change the component where this package was attached.
    >>> queue_item.builds[0].build.source_package_release.override(
    ...     component=restricted_component)
    >>> queue_item.customfiles[0].publish()

As we can see from the translation import queue content.

    >>> for entry in translation_import_queue.getAllEntries(target=ubuntu):
    ...     print '%s/%s by %s: %s' % (
    ...         entry.distroseries.name, entry.sourcepackagename.name,
    ...         entry.importer.name, entry.path)
    dapper/pmount by ubuntu-team: po/es_ES.po
    dapper/pmount by ubuntu-team: po/ca.po
    dapper/pmount by ubuntu-team: po/de.po
    dapper/pmount by ubuntu-team: po/cs.po
    dapper/pmount by ubuntu-team: po/es.po
    dapper/pmount by ubuntu-team: po/fr.po
    dapper/pmount by ubuntu-team: po/hr.po
    dapper/pmount by ubuntu-team: po/nb.po
    dapper/pmount by ubuntu-team: po/pmount.pot
    dapper/pmount by ubuntu-team: po/it_IT.po

# Let's abort the transaction so we can check the same upload in a different
# component.
    >>> transaction.abort()

But the ones into universe are not accepted.

    >>> ubuntu = getUtility(IDistributionSet)['ubuntu']
    >>> dapper = distro_series_set.queryByName(ubuntu, 'dapper')
    >>> universe_component = getUtility(IComponentSet)['universe']
    >>> queue_item = dapper.getPackageUploads(PackageUploadStatus.NEW)[0]

# Change the component where this package was attached.
    >>> queue_item.builds[0].build.source_package_release.override(
    ...     component=universe_component)
    >>> queue_item.customfiles[0].publish()

This time, we don't get any entry in the import queue.

    >>> translation_import_queue.getAllEntries(target=ubuntu).count()
    0

# Let's abort the transaction so we can check the same upload in a different
# component.
    >>> transaction.abort()


Translations from PPA build
---------------------------

For now we simply ignore translations for archives other than the
Distribution archives (i.e. PPAs).

    >>> from lp.registry.interfaces.person import IPersonSet
    >>> from lp.soyuz.enums import ArchivePurpose
    >>> from lp.soyuz.interfaces.archive import IArchiveSet

    >>> foobar_archive = getUtility(IArchiveSet).new(
    ...     purpose=ArchivePurpose.PPA,
    ...     owner=getUtility(IPersonSet).getByName('name16'))

    >>> dapper = getUtility(IDistributionSet)['ubuntu']['dapper']
    >>> queue_item = dapper.getPackageUploads(PackageUploadStatus.NEW)[0]
    >>> queue_item.archive = foobar_archive

    >>> queue_item.customfiles[0].publish(FakeLogger())
    DEBUG Publishing custom pmount,
    pmount_0.9.7-2ubuntu2_amd64_translations.tar.gz to ubuntu/dapper
    DEBUG Skipping translations since its purpose is not in
    MAIN_ARCHIVE_PURPOSES.

# And this time, we see that there are no entries imported in the queue.
    >>> translation_import_queue.getAllEntries(target=ubuntu).count()
    0
    >>> transaction.abort()


Translations from a rebuild
---------------------------

Translations coming from rebuilt packages are also ignored.

    >>> from lp.registry.interfaces.person import IPersonSet
    >>> from lp.soyuz.interfaces.archive import ArchivePurpose, IArchiveSet

    >>> foobar_archive = getUtility(IArchiveSet).new(
    ...     purpose=ArchivePurpose.COPY,
    ...     owner=getUtility(IPersonSet).getByName('name16'),
    ...     name='rebuilds')

    >>> dapper = getUtility(IDistributionSet)['ubuntu']['dapper']
    >>> queue_item = dapper.getPackageUploads(PackageUploadStatus.NEW)[0]
    >>> queue_item.archive = foobar_archive

    >>> queue_item.customfiles[0].publish(FakeLogger())
    DEBUG Publishing custom pmount,
    pmount_0.9.7-2ubuntu2_amd64_translations.tar.gz to ubuntu/dapper
    DEBUG Skipping translations since its purpose is not in
    MAIN_ARCHIVE_PURPOSES.

# And this time, we see that there are no entries imported in the queue.
    >>> translation_import_queue.getAllEntries(target=ubuntu).count()
    0


Translations importer: publishRosettaTranslations
-------------------------------------------------

We create mock objects for SourcePackageRelease, PackageUpload and
PackageUploadCustom: these will emulate everything we need to document
different interpretations of "importer" in attachTranslationFiles.

    >>> from zope.interface import implements
    >>> from lp.app.interfaces.launchpad import ILaunchpadCelebrities
    >>> from lp.soyuz.model.queue import PackageUploadCustom
    >>> from lp.soyuz.interfaces.archive import (
    ...     IArchive, ArchivePurpose)
    >>> from lp.soyuz.interfaces.queue import (
    ...     IPackageUpload, IPackageUploadCustom)
    >>> from lp.registry.interfaces.person import IPerson
    >>> from lp.soyuz.enums import PackageUploadCustomFormat
    >>> from lp.soyuz.interfaces.component import IComponentSet
    >>> from lp.soyuz.interfaces.sourcepackagerelease import (
    ...     ISourcePackageRelease)
    >>> from lp.registry.interfaces.pocket import PackagePublishingPocket

    >>> class MockArchive:
    ...       implements(IArchive)
    ...       def __init__(self, purpose):
    ...           self.purpose = purpose

    >>> class MockDistroSeries:
    ...     implements(IDistroSeries)
    ...     def __init__(self, version):
    ...         self.version = version

    >>> class MockSourcePackageRelease:
    ...       implements(ISourcePackageRelease)
    ...       def __init__(self, component, creator, upload_distroseries):
    ...           self.component = getUtility(IComponentSet)[component]
    ...           self.upload_distroseries = upload_distroseries
    ...           self.creator = creator
    ...           self.packageupload = 1
    ...
    ...       def attachTranslationFiles(self, file, imported, importer):
    ...           if (importer is not None and
    ...               not IPerson.providedBy(importer)):
    ...               print "`importer` not a person!"
    ...           print "Imported by: %s" % (
    ...               getattr(importer, "name", "None"))

    >>> class MockPackageUpload:
    ...     implements(IPackageUpload)
    ...     def __init__(self, pocket, auto_sync, sourcepackagerelease,
    ...                  archive):
    ...         self.id = 1
    ...         self.pocket = pocket
    ...         self.auto_sync = auto_sync
    ...         self.sourcepackagerelease = sourcepackagerelease
    ...         self.archive = archive
    ...
    ...     def isAutoSyncUpload(self, changed_by_email=None):
    ...         return self.auto_sync

    >>> class MockPackageUploadCustom(PackageUploadCustom):
    ...     implements(IPackageUploadCustom)
    ...     packageupload = None
    ...
    ...     def __init__(self):
    ...         self.customformat = (
    ...             PackageUploadCustomFormat.ROSETTA_TRANSLATIONS)

For translations from auto-synced packages we consider the importer to be
'katie' (archive@ubuntu.com).

    >>> katie = getUtility(ILaunchpadCelebrities).katie
    >>> release_pocket = PackagePublishingPocket.RELEASE
    >>> archive = MockArchive(ArchivePurpose.PRIMARY)

    >>> distro_series = MockDistroSeries(u'9.04')
    >>> katie_sourcepackagerelease = MockSourcePackageRelease(
    ...     'main', katie, distro_series)
    >>> sync_package_upload = MockPackageUpload(
    ...     release_pocket, True, katie_sourcepackagerelease, archive)
    >>> sync_package_upload.isAutoSyncUpload()
    True
    >>> translations_upload = MockPackageUploadCustom()
    >>> translations_upload.packageupload = sync_package_upload
    >>> translations_upload.publishRosettaTranslations()
    Imported by: katie

Non-auto-sync uploads by 'katie' still indicate 'katie' as the uploader.

    >>> non_sync_package_upload = MockPackageUpload(
    ...     release_pocket, False, katie_sourcepackagerelease, archive)
    >>> non_sync_package_upload.isAutoSyncUpload()
    False
    >>> translations_upload.packageupload = non_sync_package_upload
    >>> translations_upload.publishRosettaTranslations()
    Imported by: katie

Uploads by anyone else are treated as if importer is the packager.

    >>> person_set = getUtility(IPersonSet)
    >>> carlos = person_set.getByName('carlos')
    >>> carlos_sourcepackagerelease = MockSourcePackageRelease(
    ...     'main', carlos, distro_series)
    >>> carlos_package_upload = MockPackageUpload(
    ...     release_pocket, False, carlos_sourcepackagerelease, archive)
    >>> carlos_package_upload.isAutoSyncUpload()
    False
    >>> translations_upload.packageupload = carlos_package_upload
    >>> translations_upload.publishRosettaTranslations()
    Imported by: carlos

Uploads for distroseries before Oneiric or later may not be targeted
to any component but 'main' and 'restricted'.  The upload attempt is ignored.

    >>> katie_sourcepackagerelease = MockSourcePackageRelease(
    ...     'universe', katie, distro_series)
    >>> sync_package_upload = MockPackageUpload(
    ...     release_pocket, True, katie_sourcepackagerelease, archive)
    >>> translations_upload = MockPackageUploadCustom()
    >>> translations_upload.packageupload = sync_package_upload
    >>> translations_upload.publishRosettaTranslations()

For Oneiric the import succeeds for 'universe'.

    >>> distro_series = MockDistroSeries(u'11.10')
    >>> katie_sourcepackagerelease = MockSourcePackageRelease(
    ...     'universe', katie, distro_series)
    >>> sync_package_upload = MockPackageUpload(
    ...     release_pocket, True, katie_sourcepackagerelease, archive)
    >>> translations_upload = MockPackageUploadCustom()
    >>> translations_upload.packageupload = sync_package_upload
    >>> translations_upload.publishRosettaTranslations()
    Imported by: katie

And for the 12.04 release the import succeeds for 'universe'.

    >>> distro_series = MockDistroSeries(u'12.04')
    >>> katie_sourcepackagerelease = MockSourcePackageRelease(
    ...     'universe', katie, distro_series)
    >>> sync_package_upload = MockPackageUpload(
    ...     release_pocket, True, katie_sourcepackagerelease, archive)
    >>> translations_upload = MockPackageUploadCustom()
    >>> translations_upload.packageupload = sync_package_upload
    >>> translations_upload.publishRosettaTranslations()
    Imported by: katie



Translations tarball
~~~~~~~~~~~~~~~~~~~~

The LibraryFileAlias returned by getLatestTranslationsUploads on the
source package points to a tarball with translations files for the
package.

    >>> import tarfile
    >>> from StringIO import StringIO
    >>> tarball = StringIO(latest_translations_upload.read())
    >>> archive = tarfile.open('', 'r|gz', tarball)
    >>> translation_files = sorted([
    ...     entry.name for entry in archive.getmembers()
    ...     if entry.name.endswith('.po') or entry.name.endswith('.pot')
    ...     ])
    >>> for filename in translation_files:
    ...     print filename
    ./source/po/ca.po
    ./source/po/cs.po
    ./source/po/de.po
    ...
    ./source/po/pmount.pot