~launchpad-pqm/launchpad/devel

#!/usr/bin/python
#
# python port of the nice maintainace-check script by  Nick Barcet
#

import logging
from optparse import OptionParser
import os
import sys
import urllib2
import urlparse

import apt
import apt_pkg


class UbuntuMaintenance(object):
    """ Represents the support timeframe for a regular ubuntu release """

    # architectures that are full supported (including LTS time)
    PRIMARY_ARCHES = [
        "i386",
        "amd64",
        ]

    # architectures we support (but not for LTS time)
    SUPPORTED_ARCHES = PRIMARY_ARCHES + ["armel"]

    # what defines the seeds is documented in wiki.ubuntu.com/SeedManagement
    SERVER_SEEDS = [
        "server-ship",
        "supported-server",
        ]
    DESKTOP_SEEDS = [
        "ship",
        "supported-desktop",
        "supported-desktop-extra",
        ]
    SUPPORTED_SEEDS = ["all"]

    # normal support timeframe
    # time, seeds
    SUPPORT_TIMEFRAME = [
        ("18m", SUPPORTED_SEEDS),
        ]

    # distro names that we check the seeds for
    DISTRO_NAMES = [
        "ubuntu",
        ]


# This is fun! We have a bunch of cases for 10.04 LTS
#
#  - distro "ubuntu" follows SUPPORT_TIMEFRAME_LTS but only for
#    amd64/i386
#  - distros "kubuntu", "edubuntu" and "netbook" need to be
#    considered *but* only follow SUPPORT_TIMEFRAME
#  - anything that is in armel follows SUPPORT_TIMEFRAME
#
class LucidUbuntuMaintenance(UbuntuMaintenance):
    """ Represents the support timeframe for a 10.04 (lucid) LTS release,
        the exact rules differ from LTS release to LTS release
    """

    # lts support timeframe, order is important, least supported must be last
    # time, seeds
    SUPPORT_TIMEFRAME = [
        ("5y",  UbuntuMaintenance.SERVER_SEEDS),
        ("3y",  UbuntuMaintenance.DESKTOP_SEEDS),
        ("18m", UbuntuMaintenance.SUPPORTED_SEEDS),
        ]

    # on a LTS this is significant, it defines what names get LTS support
    DISTRO_NAMES = [
        "ubuntu",
        "kubuntu",
        ]


class PreciseUbuntuMaintenance(UbuntuMaintenance):
    """ The support timeframe for the 12.04 (precise) LTS release.
        This changes the timeframe for desktop packages from 3y to 5y
    """

    # lts support timeframe, order is important, least supported must be last
    # time, seeds
    SUPPORT_TIMEFRAME = [
        ("5y", UbuntuMaintenance.SERVER_SEEDS),
        ("5y", UbuntuMaintenance.DESKTOP_SEEDS),
        ("18m", UbuntuMaintenance.SUPPORTED_SEEDS),
        ]

    # on a LTS this is significant, it defines what names get LTS support
    DISTRO_NAMES = [
        "ubuntu",
        ]


# Names of the distribution releases that are not supported by this
# tool. All later versions are supported.
UNSUPPORTED_DISTRO_RELEASED = [
    "dapper",
    "edgy",
    "feisty",
    "gutsy",
    "hardy",
    "intrepid",
    "jaunty",
    "karmic",
    ]


# germinate output base directory
BASE_URL = os.environ.get(
    "MAINTENANCE_CHECK_BASE_URL",
    "http://people.canonical.com/~ubuntu-archive/germinate-output/")

# hints dir url, hints file is "$distro.hints" by default
# (e.g. lucid.hints)
HINTS_DIR_URL = os.environ.get(
    "MAINTENANCE_CHECK_HINTS_DIR_URL",
    "http://people.canonical.com/"
        "~ubuntu-archive/seeds/platform.%s/SUPPORTED_HINTS")

# we need the archive root to parse the Sources file to support
# by-source hints
ARCHIVE_ROOT = os.environ.get(
    "MAINTENANCE_CHECK_ARCHIVE_ROOT", "http://archive.ubuntu.com/ubuntu")

# support timeframe tag used in the Packages file
SUPPORT_TAG = "Supported"


def get_binaries_for_source_pkg(srcname):
    """ Return all binary package names for the given source package name.

    :param srcname: The source package name.
    :return: A list of binary package names.
    """
    pkgnames = set()
    recs = apt_pkg.SourceRecords()
    while recs.lookup(srcname):
        for binary in recs.binaries:
            pkgnames.add(binary)
    return pkgnames


def expand_src_pkgname(pkgname):
    """ Expand a package name if it is prefixed with src.

    If the package name is prefixed with src it will be expanded
    to a list of binary package names. Otherwise the original
    package name will be returned.

    :param pkgname: The package name (that may include src:prefix).
    :return: A list of binary package names (the list may be one element
             long).
    """
    if not pkgname.startswith("src:"):
        return [pkgname]
    return get_binaries_for_source_pkg(pkgname.split("src:")[1])


def create_and_update_deb_src_source_list(distroseries):
    """ Create sources.list and update cache.

    This creates a sources.list file with deb-src entries for a given
    distroseries and apt.Cache.update() to make sure the data is up-to-date.
    :param distro: The code name of the distribution series (e.g. lucid).
    :return: None
    :raises: IOError: When cache update fails.
    """
    # apt root dir
    rootdir = "./aptroot.%s" % distroseries
    sources_list_dir = os.path.join(rootdir, "etc", "apt")
    if not os.path.exists(sources_list_dir):
        os.makedirs(sources_list_dir)
    sources_list = open(os.path.join(sources_list_dir, "sources.list"), "w")
    for pocket in [
        "%s" % distroseries,
        "%s-updates" % distroseries,
        "%s-security" % distroseries]:
        sources_list.write(
            "deb-src %s %s main restricted\n" % (
                ARCHIVE_ROOT, pocket))
        sources_list.write(
            "deb %s %s main restricted\n" % (
                ARCHIVE_ROOT, pocket))
    sources_list.close()
    # create required dirs/files for apt.Cache(rootdir) to work on older
    # versions of python-apt. once lucid is used it can be removed
    for d in ["var/lib/dpkg",
              "var/cache/apt/archives/partial",
              "var/lib/apt/lists/partial"]:
        if not os.path.exists(os.path.join(rootdir, d)):
            os.makedirs(os.path.join(rootdir, d))
    if not os.path.exists(os.path.join(rootdir, "var/lib/dpkg/status")):
        open(os.path.join(rootdir, "var/lib/dpkg/status"), "w")
    # open cache with our just prepared rootdir
    cache = apt.Cache(rootdir=rootdir)
    try:
        cache.update()
    except SystemError:
        logging.exception("cache.update() failed")


def get_structure(distroname, version):
    """ Get structure file conent for named distro and distro version.

    :param name: Name of the distribution (e.g. kubuntu, ubuntu, xubuntu).
    :param version: Code name of the distribution version (e.g. lucid).
    :return: List of strings with the structure file content
    """
    f = urllib2.urlopen("%s/%s.%s/structure" % (
            BASE_URL, distroname, version))
    structure = f.readlines()
    f.close()
    return structure


def expand_seeds(structure, seedname):
    """Expand seed by its dependencies using the strucure file.

    :param structure: The content of the STRUCTURE file as string list.
    :param seedname: The name of the seed as string that needs to be expanded.
    :return: A set() for the seed dependencies (excluding the original
        seedname).
    """
    seeds = []
    for line in structure:
        if line.startswith("%s:" % seedname):
            seeds += line.split(":")[1].split()
            for seed in seeds:
                seeds += expand_seeds(structure, seed)
    return set(seeds)


def get_packages_for_seeds(name, distro, seeds):
    """
    Get packages for the given name (e.g. ubuntu) and distro release
    (e.g. lucid) that are in the given list of seeds
    returns a set() of package names.
    """
    pkgs_in_seeds = {}
    for seed in seeds:
        pkgs_in_seeds[seed] = set()
        seedurl = "%s/%s.%s/%s" % (BASE_URL, name, distro, seed)
        logging.debug("looking for '%s'", seedurl)
        try:
            f = urllib2.urlopen(seedurl)
            for line in f:
                # Ignore lines that are not package names (headers etc).
                if line[0] < 'a' or line[0] > 'z':
                    continue
                # Each line contains these fields:
                # (package, source, why, maintainer, size, inst-size)
                if options.source_packages:
                    pkgname = line.split("|")[1]
                else:
                    pkgname = line.split("|")[0]
                pkgs_in_seeds[seed].add(pkgname.strip())
            f.close()
        except Exception as e:
            logging.error("seed %s failed (%s)" % (seedurl, e))
    return pkgs_in_seeds


def what_seeds(pkgname, seeds):
    in_seeds = set()
    for s in seeds:
        if pkgname in seeds[s]:
            in_seeds.add(s)
    return in_seeds


def compare_support_level(x, y):
    """
    compare two support level strings of the form 18m, 3y etc
    :parm x: the first support level
    :parm y: the second support level
    :return: negative if x < y, zero if x==y, positive if x > y
    """

    def support_to_int(support_time):
        """
        helper that takes a support time string and converts it to
        a integer for cmp()
        """
        # allow strings like "5y (kubuntu-common)
        x = support_time.split()[0]
        if x.endswith("y"):
            return 12 * int(x[0:-1])
        elif x.endswith("m"):
            return int(x[0:-1])
        else:
            raise ValueError("support time '%s' has to end with y or m" % x)
    return cmp(support_to_int(x), support_to_int(y))


def get_packages_support_time(structure, name, pkg_support_time,
                              support_timeframe_list):
    """
    input a structure file and a list of pair<timeframe, seedlist>
    return a dict of pkgnames -> support timeframe string
    """
    for (timeframe, seedlist) in support_timeframe_list:
        expanded = set()
        for s in seedlist:
            expanded.add(s)
            expanded |= expand_seeds(structure, s)
        pkgs_in_seeds = get_packages_for_seeds(name, distro, expanded)
        for seed in pkgs_in_seeds:
            for pkg in pkgs_in_seeds[seed]:
                if not pkg in pkg_support_time:
                    pkg_support_time[pkg] = timeframe
                else:
                    old_timeframe = pkg_support_time[pkg]
                    if compare_support_level(old_timeframe, timeframe) < 0:
                        logging.debug("overwriting %s from %s to %s" % (
                                pkg, old_timeframe, timeframe))
                        pkg_support_time[pkg] = timeframe
                if options.with_seeds:
                    pkg_support_time[pkg] += " (%s)" % ", ".join(
                        what_seeds(pkg, pkgs_in_seeds))

    return pkg_support_time


if __name__ == "__main__":
    parser = OptionParser()
    parser.add_option("--with-seeds", "", default=False,
                      action="store_true",
                      help="add seed(s) of the package that are responsible "
                           "for the maintaince time")
    parser.add_option("--source-packages", "", default=False,
                      action="store_true",
                      help="show as source pkgs")
    parser.add_option("--hints-file", "", default=None,
                      help="use diffenrt use hints file location")
    (options, args) = parser.parse_args()

    # init
    if len(args) > 0:
        distro = args[0]
        if distro in UNSUPPORTED_DISTRO_RELEASED:
            logging.error("only lucid or later is supported")
            sys.exit(1)
    else:
        distro = "lucid"

    # maintenance class to use
    klass = globals().get("%sUbuntuMaintenance" % distro.capitalize())
    if klass is None:
        klass = UbuntuMaintenance
    ubuntu_maintenance = klass()

    # make sure our deb-src information is up-to-date
    create_and_update_deb_src_source_list(distro)

    if options.hints_file:
        hints_file = options.hints_file
        (schema, netloc, path, query, fragment) = urlparse.urlsplit(
            hints_file)
        if not schema:
            hints_file = "file:%s" % path
    else:
        hints_file = HINTS_DIR_URL % distro

    # go over the distros we need to check
    pkg_support_time = {}
    for name in ubuntu_maintenance.DISTRO_NAMES:

        # get basic structure file
        try:
            structure = get_structure(name, distro)
        except urllib2.HTTPError:
            logging.error("Can not get structure for '%s'." % name)
            continue

        # get dicts of pkgname -> support timeframe string
        support_timeframe = ubuntu_maintenance.SUPPORT_TIMEFRAME
        get_packages_support_time(
            structure, name, pkg_support_time, support_timeframe)

    # now go over the bits in main that we have not seen (because
    # they are not in any seed and got added manually into "main"
    for arch in ubuntu_maintenance.PRIMARY_ARCHES:
        rootdir = "./aptroot.%s" % distro
        apt_pkg.config.set("APT::Architecture", arch)
        cache = apt.Cache(rootdir=rootdir)
        try:
            cache.update()
        except SystemError:
            logging.exception("cache.update() failed")
        cache.open()
        for pkg in cache:
            # ignore multiarch package names
            if ":" in pkg.name:
                continue
            if not pkg.name in pkg_support_time:
                pkg_support_time[pkg.name] = support_timeframe[-1][0]
                logging.warn(
                    "add package in main but not in seeds %s with %s" % (
                        pkg.name, pkg_support_time[pkg.name]))

    # now check the hints file that is used to overwrite
    # the default seeds
    try:
        for line in urllib2.urlopen(hints_file):
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            try:
                (raw_pkgname, support_time) = line.split()
                for pkgname in expand_src_pkgname(raw_pkgname):
                    if support_time == 'unsupported':
                        try:
                            del pkg_support_time[pkgname]
                            sys.stderr.write("hints-file: marking %s "
                                             "unsupported\n" % pkgname)
                        except KeyError:
                            pass
                    else:
                        if pkg_support_time.get(pkgname) != support_time:
                            sys.stderr.write(
                                "hints-file: changing %s from %s to %s\n" % (
                                    pkgname, pkg_support_time.get(pkgname),
                                    support_time))
                            pkg_support_time[pkgname] = support_time
            except:
                logging.exception("can not parse line '%s'" % line)
    except urllib2.HTTPError, e:
        if e.code != 404:
            raise
        sys.stderr.write("hints-file: %s gave 404 error\n" % hints_file)

    # output suitable for the extra-override file
    for pkgname in sorted(pkg_support_time.keys()):
        # special case, the hints file may contain overrides that
        # are arch-specific (like zsh-doc/armel)
        if "/" in pkgname:
            print "%s %s %s" % (
                pkgname, SUPPORT_TAG, pkg_support_time[pkgname])
        else:
            # go over the supported arches, they are divided in
            # first-class (PRIMARY) and second-class with different
            # support levels
            for arch in ubuntu_maintenance.SUPPORTED_ARCHES:
                # ensure we do not overwrite arch-specific overwrites
                pkgname_and_arch = "%s/%s" % (pkgname, arch)
                if pkgname_and_arch in pkg_support_time:
                    break
                if arch in ubuntu_maintenance.PRIMARY_ARCHES:
                    # arch with full LTS support
                    print "%s %s %s" % (
                        pkgname_and_arch, SUPPORT_TAG,
                        pkg_support_time[pkgname])
                else:
                    # not a LTS supported architecture, gets only regular
                    # support_timeframe
                    print "%s %s %s" % (
                        pkgname_and_arch, SUPPORT_TAG,
                        ubuntu_maintenance.SUPPORT_TIMEFRAME[-1][0])