← Back to branch summary

~launchpad-pqm/launchpad/devel

« back to all changes in this revision

Viewing changes to lib/lp/archiveuploader/nascentuploadfile.py

  • Committer: Raphael Badin
  • Date: 2011-08-12 16:29:57 UTC
  • mto: This revision was merged to the branch mainline in revision 13714.
  • Revision ID: raphael.badin@canonical.com-20110812162957-vymptkg7pksk5r1b
Fix doc.

Show diffs side-by-side

added added

removed removed

Lines of Context:
lib/lp/archiveuploader/nascentuploadfile.py
1
 
# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
 
1
# Copyright 2009 Canonical Ltd.  This software is licensed under the
2
2
# GNU Affero General Public License version 3 (see the file LICENSE).
3
3
 
4
4
"""Specific models for uploaded files"""
19
19
    'splitComponentAndSection',
20
20
    ]
21
21
 
 
22
import apt_inst
 
23
import apt_pkg
 
24
from debian.deb822 import Deb822Dict
22
25
import hashlib
23
26
import os
24
27
import subprocess
25
28
import sys
26
29
import time
27
30
 
28
 
import apt_inst
29
 
import apt_pkg
30
 
from debian.deb822 import Deb822Dict
31
31
from zope.component import getUtility
32
32
 
 
33
from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
 
34
from canonical.librarian.utils import filechunks
33
35
from lp.app.errors import NotFoundError
34
36
from lp.archiveuploader.utils import (
35
37
    determine_source_file_type,
45
47
    )
46
48
from lp.buildmaster.enums import BuildStatus
47
49
from lp.services.encoding import guess as guess_encoding
48
 
from lp.services.librarian.interfaces import ILibraryFileAliasSet
49
 
from lp.services.librarian.utils import filechunks
50
50
from lp.soyuz.enums import (
51
51
    BinaryPackageFormat,
52
52
    PackagePublishingPriority,
58
58
from lp.soyuz.model.files import SourceFileMixin
59
59
 
60
60
 
61
 
apt_pkg.init_system()
 
61
apt_pkg.InitSystem()
62
62
 
63
63
 
64
64
class UploadError(Exception):
86
86
        self.future_files = {}
87
87
        self.ancient_files = {}
88
88
 
89
 
    def callback(self, member, data):
90
 
        """Callback designed to cope with apt_inst.TarFile.go.
 
89
    def callback(self, kind, name, link, mode, uid, gid, size, mtime,
 
90
                 major, minor):
 
91
        """Callback designed to cope with apt_inst.debExtract.
91
92
 
92
93
        It check and store timestamp details of the extracted DEB.
93
94
        """
94
 
        self.check_cutoff(member.name, member.mtime)
 
95
        self.check_cutoff(name, mtime)
95
96
 
96
97
    def check_cutoff(self, name, mtime):
97
98
        """Check the timestamp details of the supplied file.
303
304
 
304
305
        They need to satisfy at least the NEW queue constraints that includes
305
306
        SourcePackageRelease creation, so component and section need to exist.
306
 
        Even if they might be overridden in the future.
 
307
        Even if they might be overriden in the future.
307
308
        """
308
309
        NascentUploadFile.__init__(
309
310
            self, filepath, digest, size, component_and_section,
528
529
                yield error
529
530
 
530
531
    def extractAndParseControl(self):
531
 
        """Extract and parse control information."""
 
532
        """Extract and parse tcontrol information."""
 
533
        deb_file = open(self.filepath, "r")
532
534
        try:
533
 
            deb_file = apt_inst.DebFile(self.filepath)
534
 
            control_file = deb_file.control.extractdata("control")
535
 
            control_lines = apt_pkg.TagSection(control_file)
 
535
            control_file = apt_inst.debExtractControl(deb_file)
 
536
            control_lines = apt_pkg.ParseSection(control_file)
536
537
        except (SystemExit, KeyboardInterrupt):
537
538
            raise
538
539
        except:
 
540
            deb_file.close()
539
541
            yield UploadError(
540
 
                "%s: extracting control file raised %s, giving up."
 
542
                "%s: debExtractControl() raised %s, giving up."
541
543
                 % (self.filename, sys.exc_type))
542
544
            return
543
545
 
544
546
        for mandatory_field in self.mandatory_fields:
545
 
            if control_lines.find(mandatory_field) is None:
 
547
            if control_lines.Find(mandatory_field) is None:
546
548
                yield UploadError(
547
549
                    "%s: control file lacks mandatory field %r"
548
550
                     % (self.filename, mandatory_field))
549
551
        control = {}
550
552
        for key in control_lines.keys():
551
 
            control[key] = control_lines.find(key)
 
553
            control[key] = control_lines.Find(key)
552
554
        self.parseControl(control)
553
555
 
554
556
    def parseControl(self, control):
704
706
            yield UploadError(
705
707
                "%s: second chunk is %s, expected control.tar.gz." % (
706
708
                self.filename, control_tar))
707
 
        if data_tar not in ("data.tar.gz", "data.tar.bz2", "data.tar.lzma",
708
 
                            "data.tar.xz"):
 
709
        if data_tar not in ("data.tar.gz", "data.tar.bz2", "data.tar.lzma"):
709
710
            yield UploadError(
710
711
                "%s: third chunk is %s, expected data.tar.gz, "
711
 
                "data.tar.bz2, data.tar.lzma or data.tar.xz." %
712
 
                (self.filename, data_tar))
713
 
 
714
 
        # xz-compressed debs must pre-depend on dpkg >= 1.15.6~.
715
 
        XZ_REQUIRED_DPKG_VER = '1.15.6~'
716
 
        if data_tar == "data.tar.xz":
717
 
            parsed_deps = []
718
 
            try:
719
 
                parsed_deps = apt_pkg.parse_depends(
720
 
                    self.control['Pre-Depends'])
721
 
            except (ValueError, TypeError):
722
 
                yield UploadError(
723
 
                    "Can't parse Pre-Depends in the control file.")
724
 
                return
725
 
            except KeyError:
726
 
                # Go past the for loop and yield the error below.
727
 
                pass
728
 
 
729
 
            for token in parsed_deps:
730
 
                try:
731
 
                    name, version, relation = token[0]
732
 
                except ValueError:
733
 
                    yield("APT error processing token '%r' from Pre-Depends.")
734
 
                    return
735
 
 
736
 
                if name == 'dpkg':
737
 
                    # VersionCompare returns values similar to cmp;
738
 
                    # negative if first < second, zero if first ==
739
 
                    # second and positive if first > second.
740
 
                    if apt_pkg.version_compare(
741
 
                        version, XZ_REQUIRED_DPKG_VER) >= 0:
742
 
                        # Pre-Depends dpkg is fine.
743
 
                        return
744
 
                    else:
745
 
                        yield UploadError(
746
 
                            "Pre-Depends dpkg version should be >= %s "
747
 
                            "when using xz compression." %
748
 
                            XZ_REQUIRED_DPKG_VER)
749
 
                        return
750
 
 
751
 
            yield UploadError(
752
 
                "Require Pre-Depends: dpkg (>= %s) when using xz "
753
 
                "compression." % XZ_REQUIRED_DPKG_VER)
 
712
                "data.tar.bz2 or data.tar.lzma." % (self.filename, data_tar))
754
713
 
755
714
    def verifyDebTimestamp(self):
756
715
        """Check specific DEB format timestamp checks."""
763
722
        tar_checker = TarFileDateChecker(future_cutoff, past_cutoff)
764
723
        tar_checker.reset()
765
724
        try:
766
 
            deb_file = apt_inst.DebFile(self.filepath)
767
 
        except SystemError, error:
768
 
            # We get an error from the constructor if the .deb does not
769
 
            # contain all the expected top-level members (debian-binary,
770
 
            # control.tar.gz, and data.tar.*).
771
 
            yield UploadError(error)
772
 
        try:
773
 
            deb_file.control.go(tar_checker.callback)
774
 
            deb_file.data.go(tar_checker.callback)
775
 
            future_files = tar_checker.future_files.keys()
776
 
            if future_files:
777
 
                first_file = future_files[0]
778
 
                timestamp = time.ctime(tar_checker.future_files[first_file])
779
 
                yield UploadError(
780
 
                    "%s: has %s file(s) with a time stamp too "
781
 
                    "far into the future (e.g. %s [%s])."
782
 
                     % (self.filename, len(future_files), first_file,
783
 
                        timestamp))
784
 
 
785
 
            ancient_files = tar_checker.ancient_files.keys()
786
 
            if ancient_files:
787
 
                first_file = ancient_files[0]
788
 
                timestamp = time.ctime(tar_checker.ancient_files[first_file])
789
 
                yield UploadError(
790
 
                    "%s: has %s file(s) with a time stamp too "
791
 
                    "far in the past (e.g. %s [%s])."
792
 
                     % (self.filename, len(ancient_files), first_file,
793
 
                        timestamp))
 
725
            deb_file = open(self.filepath, "rb")
 
726
            apt_inst.debExtract(deb_file, tar_checker.callback,
 
727
                                "control.tar.gz")
 
728
            # Only one of these files is present in the archive, so loop
 
729
            # until we find one of them, otherwise fail.
 
730
            data_files = ("data.tar.gz", "data.tar.bz2", "data.tar.lzma")
 
731
            for file in data_files:
 
732
                deb_file.seek(0)
 
733
                try:
 
734
                    apt_inst.debExtract(deb_file, tar_checker.callback, file)
 
735
                except SystemError:
 
736
                    continue
 
737
                else:
 
738
                    deb_file.close()
 
739
 
 
740
                    future_files = tar_checker.future_files.keys()
 
741
                    if future_files:
 
742
                        first_file = future_files[0]
 
743
                        timestamp = time.ctime(
 
744
                            tar_checker.future_files[first_file])
 
745
                        yield UploadError(
 
746
                            "%s: has %s file(s) with a time stamp too "
 
747
                            "far into the future (e.g. %s [%s])."
 
748
                             % (self.filename, len(future_files), first_file,
 
749
                                timestamp))
 
750
 
 
751
                    ancient_files = tar_checker.ancient_files.keys()
 
752
                    if ancient_files:
 
753
                        first_file = ancient_files[0]
 
754
                        timestamp = time.ctime(
 
755
                            tar_checker.ancient_files[first_file])
 
756
                        yield UploadError(
 
757
                            "%s: has %s file(s) with a time stamp too "
 
758
                            "far in the past (e.g. %s [%s])."
 
759
                             % (self.filename, len(ancient_files), first_file,
 
760
                                timestamp))
 
761
                    return
 
762
 
 
763
            deb_file.close()
 
764
            yield UploadError(
 
765
                "Could not find data tarball in %s" % self.filename)
 
766
 
794
767
        except (SystemExit, KeyboardInterrupt):
795
768
            raise
796
769
        except Exception, error:
802
775
            yield UploadError("%s: deb contents timestamp check failed: %s"
803
776
                 % (self.filename, error))
804
777
 
 
778
 
805
779
    #
806
780
    #   Database relationship methods
807
781
    #
831
805
                "Unable to find source package %s/%s in %s" % (
832
806
                self.source_name, self.source_version, distroseries.name))
833
807
 
 
808
 
834
809
    def verifySourcePackageRelease(self, sourcepackagerelease):
835
810
        """Check if the given ISourcePackageRelease matches the context."""
836
811
        assert 'source' in self.changes.architectures, (