~launchpad-pqm/launchpad/devel

#!/bin/sh
#
# Copyright 2009 Canonical Ltd.  This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).

# LPCONFIG will come from the environment so this script can run unaltered
# on dogfood.
if [ -z $LPCONFIG ]; then
    echo LPCONFIG must be set to run this script.
    exit 1
fi
SECURITY_PUBLISHER="no"
if [ "$1" = "security" ]; then
    # We are running a security publisher run, which skips some steps.
    SECURITY_PUBLISHER="yes"
fi

set -x
set -e
set -u

# Launchpad cron.daily (currently just for Ubuntu).

# Informational -- this *MUST* match the database.
ARCHIVEROOT=/srv/launchpad.net/ubuntu-archive/ubuntu
DISTSROOT=$ARCHIVEROOT/dists
OVERRIDEROOT=$ARCHIVEROOT/../ubuntu-overrides
CACHEROOT=$ARCHIVEROOT/../ubuntu-cache
DISTSCOPYROOT=$ARCHIVEROOT/../ubuntu-distscopy
INDICES=$ARCHIVEROOT/indices
PRODUCTION_CONFIG=ftpmaster

if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
    ARCHIVEROOT_PARTNER=/srv/launchpad.net/ubuntu-archive/ubuntu-partner
    GNUPGHOME=/srv/launchpad.net/ubuntu-archive/gnupg-home
else
    # GNUPGHOME does not need to be set, keys can come from ~/.gnupg.
    ARCHIVEROOT_PARTNER=/srv/launchpad.net/ppa/ubuntu-partner
fi
DISTSROOT_PARTNER=$ARCHIVEROOT_PARTNER/dists
DISTSCOPYROOT_PARTNER=$ARCHIVEROOT_PARTNER/../ubuntu-partner-distscopy

# Configuration options.
LAUNCHPADROOT=/srv/launchpad.net/codelines/current
LOCKFILE=/srv/launchpad.net/ubuntu-archive/cron.daily.lock
DISTRONAME=ubuntu
TOUCHLIST=$ARCHIVEROOT/project/trace/$(hostname --fqdn)
DSYNCLIST=$CACHEROOT/dsync.list
MD5LIST=$INDICES/md5sums.gz

# Manipulate the environment.
export GNUPGHOME
PATH=$PATH:$LAUNCHPADROOT/scripts:$LAUNCHPADROOT/cronscripts:$LAUNCHPADROOT/cronscripts/publishing:$LAUNCHPADROOT/scripts/ftpmaster-tools

# Claim the lock.
if ! lockfile -r1 $LOCKFILE; then
  echo "Could not claim lock file."
  exit 1
fi

DONEPUB=no

cleanup () {
  echo "Cleaning up lockfile."
  rm -f $LOCKFILE
  echo "Moving dists backup to safe keeping for next time."
  if [ "x$DONEPUB" = "xyes" ]; then
    if [ -d ${DISTSROOT}.old ]; then
      mv ${DISTSROOT}.old ${DISTSCOPYROOT}/dists
    fi
    if [ -d ${DISTSROOT_PARTNER}.old ]; then
      mv ${DISTSROOT_PARTNER}.old ${DISTSCOPYROOT_PARTNER}/dists
    fi
  else
    if [ -d ${DISTSROOT}.new ]; then
      mv ${DISTSROOT}.new ${DISTSCOPYROOT}/dists
    fi
    if [ -d ${DISTSROOT_PARTNER}.new ]; then
      mv ${DISTSROOT_PARTNER}.new ${DISTSCOPYROOT_PARTNER}/dists
    fi
  fi
  if [ "$SECURITY_PUBLISHER" = "yes" ]; then
    exec $0
  fi
}

trap cleanup EXIT

# Lock claimed.

# Process the accepted queue into the publishing records.
process-accepted.py -v -v -v $DISTRONAME

# If doing a security run, find out which suites are pending publication.
SUITEOPTS=""
if [ "$SECURITY_PUBLISHER" = "yes" ]; then
    SUITES=$(lp-query-distro.py pending_suites)
    SECURITY_SUITES="no"
    if [ -n "$SUITES" ]; then
        for SUITE in $SUITES; do
            case $SUITE in
                *-security)
                    SECURITY_SUITES="yes"
                    SUITEOPTS="$SUITEOPTS -s $SUITE"
                    ;;
                *)
                    ;;
            esac
        done
    fi
    if [ "$SECURITY_SUITES" != "yes" ]; then
        echo "Nothing to do for security publisher; exiting."
        exit 0
    fi
fi

# Copy the dists tree ready for publishing into so that we don't get
# an inconsistent dists tree at any point during the publishing cycle.
# Because this would cause buildds to explode.
# This is now done through maintaining a persistent backup copy of the
# dists folder, which we move into place and bring up-to-date with rsync.
# This should achieve the same as copying, only faster.

# Create backup dists folder, if this is the first time.
mkdir -p ${DISTSCOPYROOT}/dists
mkdir -p ${DISTSCOPYROOT_PARTNER}/dists

# Move the backup dists folder into place.
mv ${DISTSCOPYROOT}/dists ${ARCHIVEROOT}/dists.new
mv ${DISTSCOPYROOT_PARTNER}/dists ${ARCHIVEROOT_PARTNER}/dists.new

# Bring it up-to-date efficiently with rsync. --delete is required to
# ensure we don't ressurect things previously deleted, bug 58835.
rsync -aH --delete ${DISTSROOT}/ ${ARCHIVEROOT}/dists.new
rsync -aH --delete ${DISTSROOT_PARTNER}/ ${ARCHIVEROOT_PARTNER}/dists.new

# Publish the results for all archives (except PPA).
# The -R only affects the primary and the partner archive.
publish-distro.py -v -v --partner -d $DISTRONAME -R ${DISTSROOT_PARTNER}.new
publish-distro.py -v -v -d $DISTRONAME $SUITEOPTS -R ${DISTSROOT}.new

set +x
# Find all the Release files for which the Release.GPG is missing/too-old
# We use -maxdepth 2 to only sign Release files for distroreleases,
# not distroarchreleases/distrosourcereleases.
# Also we sign the dist-upgrader tarballs because they're handy too.
for CANDIDATE in $(find ${DISTSROOT}.new -maxdepth 2 -name Release) \
       $(find ${DISTSROOT}.new/*/*/dist-upgrader* -name "*.tar.gz"); do
  #  [ Release.gpg missing   ] or [ Release is newer than Release.gpg ]
  if [ ! -f $CANDIDATE.gpg ] || [ $CANDIDATE -nt $CANDIDATE.gpg ]; then
    echo "(re-)signing $CANDIDATE"
    gpg --yes --detach-sign --armor -o $CANDIDATE.gpg --sign $CANDIDATE
  else
    echo "Not re-signing $CANDIDATE"
  fi
done
SIGNLIST_PARTNER=$(find ${DISTSROOT_PARTNER}.new -maxdepth 2 -name Release)
for CANDIDATE in $SIGNLIST_PARTNER; do
  #  [ Release.gpg missing   ] or [ Release is newer than Release.gpg ].
  if [ ! -f $CANDIDATE.gpg ] || [ $CANDIDATE -nt $CANDIDATE.gpg ]; then
    echo "(re-)signing $CANDIDATE"
    gpg --yes --detach-sign --armor -o $CANDIDATE.gpg --sign $CANDIDATE
  else
    echo "Not re-signing $CANDIDATE"
  fi
done

# The Packages and Sources files are very large and would cripple our
# mirrors, so we remove them now that the uncompressed MD5SUMS are in the
# Release files.
echo Removing uncompressed Packages and Sources files
find ${DISTSROOT}.new \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;
find ${DISTSROOT_PARTNER} \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;

# Copy in the indices.
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
    echo Copying the indices into place.
    rm -f $INDICES/override.*
    cp $OVERRIDEROOT/override.* $INDICES
fi

# As close to atomically as possible, put the new dists into place for the
# primary and partner archives.
mv $DISTSROOT ${DISTSROOT}.old
mv ${DISTSROOT}.new $DISTSROOT
mv $DISTSROOT_PARTNER ${DISTSROOT_PARTNER}.old
mv ${DISTSROOT_PARTNER}.new $DISTSROOT_PARTNER
DONEPUB=yes
mv ${DISTSROOT}.old ${DISTSCOPYROOT}/dists
mv ${DISTSROOT_PARTNER}.old ${DISTSCOPYROOT_PARTNER}/dists

# Generate the -commercial pocket for backwards compatibility with
# dapper, edgy and feisty releases.  Don't fail the whole script if it
# fails.
commercial-compat.sh || true

# Touch everything you asked us to do.
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
    for FILE in $TOUCHLIST; do
      touch "$FILE"
    done
fi

# Skip long-running processes when doing a quick security run.
if [ "$SECURITY_PUBLISHER" != "yes" ]; then

    # Make the lslr because we all love those.
    LSLR=ls-lR.gz
    ( cd $ARCHIVEROOT ; \
      rm -f .$LSLR.new ; \
      TZ=UTC ls -lR  | gzip -9n > .$LSLR.new ; \
      mv -f .$LSLR.new $LSLR )
    ( cd $ARCHIVEROOT_PARTNER ; \
      rm -f .$LSLR.new ; \
      TZ=UTC ls -lR  | gzip -9n > .$LSLR.new ; \
      mv -f .$LSLR.new $LSLR )

    # Run dsync over primary archive only.
    ( cd $ARCHIVEROOT ; \
      dsync-flist -q generate $DSYNCLIST -e 'Packages*' -e 'Sources*' -e 'Release*' --md5 ; \
      (dsync-flist -q md5sums $DSYNCLIST; find dists '(' -name 'Packages*' -o -name 'Sources*' -o -name 'Release*' ')' -print | xargs -r md5sum) | gzip -9n > ${MD5LIST} ; \
      dsync-flist -q link-dups $DSYNCLIST || true )

    # Clear out empty and thus redundant dirs.
    find $ARCHIVEROOT -type d -empty | xargs -r rmdir
    find $ARCHIVEROOT_PARTNER -type d -empty | xargs -r rmdir

    # Invoke cron.germinate to fill ubuntu tasks and ignore failures,
    # the mirrors should be always triggered.
    cron.germinate || echo "cron.germinate failed with exit code $?" >&2

# End of block skipped by security publishing.
fi

# Trigger master mirrors.
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
    ssh archvsync@syowa
    ssh archvsync@frei
    ssh archvsync@rockhopper
fi