~launchpad-pqm/launchpad/devel

#!/bin/sh
#
# Copyright 2009 Canonical Ltd.  This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).

# LPCONFIG will come from the environment so this script can run unaltered
# on dogfood.
if [ -z $LPCONFIG ]; then
    echo LPCONFIG must be set to run this script.
    exit 1
fi
SECURITY_PUBLISHER="no"
if [ "$1" = "security" ]; then
    # We are running a security publisher run, which skips some steps.
    SECURITY_PUBLISHER="yes"
fi

set -x
set -e
set -u

# Launchpad cron.daily (currently just for Ubuntu).

# Informational -- this *MUST* match the database.
ARCHIVEROOT=/srv/launchpad.net/ubuntu-archive/ubuntu
DISTSROOT=$ARCHIVEROOT/dists
OVERRIDEROOT=$ARCHIVEROOT/../ubuntu-overrides
CACHEROOT=$ARCHIVEROOT/../ubuntu-cache
DISTSCOPYROOT=$ARCHIVEROOT/../ubuntu-distscopy
INDICES=$ARCHIVEROOT/indices
PRODUCTION_CONFIG=ftpmaster-publish

if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
    ARCHIVEROOT_PARTNER=/srv/launchpad.net/ubuntu-archive/ubuntu-partner
    GNUPGHOME=/srv/launchpad.net/ubuntu-archive/gnupg-home
else
    # GNUPGHOME does not need to be set, keys can come from ~/.gnupg.
    ARCHIVEROOT_PARTNER=/srv/launchpad.net/ppa/ubuntu-partner
fi
DISTSROOT_PARTNER=$ARCHIVEROOT_PARTNER/dists
DISTSCOPYROOT_PARTNER=$ARCHIVEROOT_PARTNER/../ubuntu-partner-distscopy

# Configuration options.
LAUNCHPADROOT=/srv/launchpad.net/codelines/current
LOCKFILE=/srv/launchpad.net/ubuntu-archive/cron.daily.lock
DISTRONAME=ubuntu
TRACEFILE=$ARCHIVEROOT/project/trace/$(hostname --fqdn)
DSYNCLIST=$CACHEROOT/dsync.list
MD5LIST=$INDICES/md5sums.gz

# Mirrors to push at the end of the publishing run.
MASTERMIRRORS="syowa frei wahoo scandium"

# Manipulate the environment.
export GNUPGHOME
PATH=$PATH:$LAUNCHPADROOT/scripts:$LAUNCHPADROOT/cronscripts:$LAUNCHPADROOT/cronscripts/publishing:$LAUNCHPADROOT/scripts/ftpmaster-tools

# Claim the lock.
if ! lockfile -r1 $LOCKFILE; then
  echo "Could not claim lock file."
  exit 1
fi

echo "$(date -R): Initiating archive publishing operations..."

DONEPUB=no

cleanup () {
  echo "$(date -R): Cleaning up lockfile."
  rm -f $LOCKFILE
  echo "$(date -R): Moving dists backup to safe keeping for next time."
  if [ "x$DONEPUB" = "xyes" ]; then
    if [ -d ${DISTSROOT}.old ]; then
      mv ${DISTSROOT}.old ${DISTSCOPYROOT}/dists
    fi
    if [ -d ${DISTSROOT_PARTNER}.old ]; then
      mv ${DISTSROOT_PARTNER}.old ${DISTSCOPYROOT_PARTNER}/dists
    fi
  else
    if [ -d ${DISTSROOT}.new ]; then
      mv ${DISTSROOT}.new ${DISTSCOPYROOT}/dists
    fi
    if [ -d ${DISTSROOT_PARTNER}.new ]; then
      mv ${DISTSROOT_PARTNER}.new ${DISTSCOPYROOT_PARTNER}/dists
    fi
  fi
  if [ "$SECURITY_PUBLISHER" = "yes" ]; then
    exec $0
  fi
}

trap cleanup EXIT

# Lock claimed.

# Process the accepted queue into the publishing records.
echo "$(date -R): Processing the accepted queue into the publishing records..."
process-accepted.py -v -v -v $DISTRONAME

# If doing a security run, find out which suites are pending publication.
SUITEOPTS=""
if [ "$SECURITY_PUBLISHER" = "yes" ]; then
    echo "$(date -R): Querying which suites are pending publication..."
    SUITES=$(lp-query-distro.py pending_suites)
    SECURITY_SUITES="no"
    if [ -n "$SUITES" ]; then
        for SUITE in $SUITES; do
            case $SUITE in
                *-security)
                    SECURITY_SUITES="yes"
                    SUITEOPTS="$SUITEOPTS -s $SUITE"
                    ;;
                *)
                    ;;
            esac
        done
    fi
    if [ "$SECURITY_SUITES" != "yes" ]; then
        echo "$(date -R): Nothing to do for security publisher; exiting."
        exit 0
    fi
fi

# Copy the dists tree ready for publishing into so that we don't get
# an inconsistent dists tree at any point during the publishing cycle.
# Because this would cause buildds to explode.
# This is now done through maintaining a persistent backup copy of the
# dists folder, which we move into place and bring up-to-date with rsync.
# This should achieve the same as copying, only faster.

# Create backup dists folder, if this is the first time.
echo "$(date -R): Creating backup dists directories..."
mkdir -p ${DISTSCOPYROOT}/dists
mkdir -p ${DISTSCOPYROOT_PARTNER}/dists

# Move the backup dists folder into place.
echo "$(date -R): Moving backup dists into place..."
mv ${DISTSCOPYROOT}/dists ${ARCHIVEROOT}/dists.new
mv ${DISTSCOPYROOT_PARTNER}/dists ${ARCHIVEROOT_PARTNER}/dists.new

# Bring it up-to-date efficiently with rsync. --delete is required to
# ensure we don't ressurect things previously deleted, bug 58835.
echo "$(date -R): Updating dists directories..."
rsync -aH --delete ${DISTSROOT}/ ${ARCHIVEROOT}/dists.new
rsync -aH --delete ${DISTSROOT_PARTNER}/ ${ARCHIVEROOT_PARTNER}/dists.new

# Publish the results for all archives (except PPA).
# The -R only affects the primary and the partner archive.
echo "$(date -R): Publishing the $DISTRONAME partner archive..."
publish-distro.py -v -v --partner -d $DISTRONAME -R ${DISTSROOT_PARTNER}.new

echo "$(date -R): Publishing the $DISTRONAME archive..."
publish-distro.py -v -v -d $DISTRONAME $SUITEOPTS -R ${DISTSROOT}.new

set +x
# Find all the Release files for which the Release.GPG is missing/too-old
# We use -maxdepth 2 to only sign Release files for distroreleases,
# not distroarchreleases/distrosourcereleases.
# Also we sign the dist-upgrader tarballs because they're handy too.
for CANDIDATE in $(find ${DISTSROOT}.new -maxdepth 2 -name Release) \
       $(find ${DISTSROOT}.new/*/*/dist-upgrader* -name "*.tar.gz"); do
  #  [ Release.gpg missing   ] or [ Release is newer than Release.gpg ]
  if [ ! -f $CANDIDATE.gpg ] || [ $CANDIDATE -nt $CANDIDATE.gpg ]; then
    echo "$(date -R): (re-)signing $CANDIDATE"
    gpg --yes --detach-sign --armor -o $CANDIDATE.gpg --sign $CANDIDATE
  else
    echo "$(date -R): Not re-signing $CANDIDATE"
  fi
done
SIGNLIST_PARTNER=$(find ${DISTSROOT_PARTNER}.new -maxdepth 2 -name Release)
for CANDIDATE in $SIGNLIST_PARTNER; do
  #  [ Release.gpg missing   ] or [ Release is newer than Release.gpg ].
  if [ ! -f $CANDIDATE.gpg ] || [ $CANDIDATE -nt $CANDIDATE.gpg ]; then
    echo "$(date -R): (re-)signing $CANDIDATE"
    gpg --yes --detach-sign --armor -o $CANDIDATE.gpg --sign $CANDIDATE
  else
    echo "$(date -R): Not re-signing $CANDIDATE"
  fi
done

# The Packages and Sources files are very large and would cripple our
# mirrors, so we remove them now that the uncompressed MD5SUMS are in the
# Release files.
echo "$(date -R): Removing uncompressed Packages and Sources files"
find ${DISTSROOT}.new \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;
find ${DISTSROOT_PARTNER} \( -name "Packages" -o -name "Sources" \) -exec rm "{}" \;

# Copy in the indices.
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
    echo "$(date -R): Copying the indices into place."
    rm -f $INDICES/override.*
    cp $OVERRIDEROOT/override.* $INDICES
fi

# As close to atomically as possible, put the new dists into place for the
# primary and partner archives.
echo "$(date -R): Placing the new dists into place..."
mv $DISTSROOT ${DISTSROOT}.old
mv ${DISTSROOT}.new $DISTSROOT
mv $DISTSROOT_PARTNER ${DISTSROOT_PARTNER}.old
mv ${DISTSROOT_PARTNER}.new $DISTSROOT_PARTNER
DONEPUB=yes
mv ${DISTSROOT}.old ${DISTSCOPYROOT}/dists
mv ${DISTSROOT_PARTNER}.old ${DISTSCOPYROOT_PARTNER}/dists

# Generate the -commercial pocket for backwards compatibility with
# dapper, edgy and feisty releases.  Don't fail the whole script if it
# fails.
echo "$(date -R): Generating -commerical pocket..."
commercial-compat.sh || true

# Timestamp our trace file to track when the last archive publisher run took
# place.
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
    echo "$(date -R): Timestamping trace file..."
    date -u > "$TRACEFILE"
fi

# Skip long-running processes when doing a quick security run.
if [ "$SECURITY_PUBLISHER" != "yes" ]; then

    # Make the lslr because we all love those.
    echo "$(date -R): Creating ls-lR.gz..."
    LSLR=ls-lR.gz
    ( cd $ARCHIVEROOT ; \
      rm -f .$LSLR.new ; \
      TZ=UTC ls -lR  | gzip -9n > .$LSLR.new ; \
      mv -f .$LSLR.new $LSLR )
    ( cd $ARCHIVEROOT_PARTNER ; \
      rm -f .$LSLR.new ; \
      TZ=UTC ls -lR  | gzip -9n > .$LSLR.new ; \
      mv -f .$LSLR.new $LSLR )

    # Run dsync over primary archive only.
    echo "$(date -R): Running dsync over primary archive..."
    ( cd $ARCHIVEROOT ; \
      dsync-flist -q generate $DSYNCLIST -e 'Packages*' -e 'Sources*' -e 'Release*' --md5 ; \
      (dsync-flist -q md5sums $DSYNCLIST; find dists '(' -name 'Packages*' -o -name 'Sources*' -o -name 'Release*' ')' -print | xargs -r md5sum) | gzip -9n > ${MD5LIST} ; \
      dsync-flist -q link-dups $DSYNCLIST || true )

    # Clear out empty and thus redundant dirs.
    echo "$(date -R): Clearing out empty directories..."
    find $ARCHIVEROOT -type d -empty | xargs -r rmdir
    find $ARCHIVEROOT_PARTNER -type d -empty | xargs -r rmdir

    # Invoke cron.germinate to fill ubuntu tasks and ignore failures,
    # the mirrors should be always triggered.
    echo "$(date -R): Running cron.germinate..."
    cron.germinate || echo "$(date -R): cron.germinate failed with exit code $?"

# End of block skipped by security publishing.
fi

# Trigger master mirrors.
if [ "$LPCONFIG" = "$PRODUCTION_CONFIG" ]; then
    echo "$(date -R): Triggering master mirrors..."

    for HOST in $MASTERMIRRORS; do
        echo "$(date -R): Triggering $HOST:"
        ssh archvsync@$HOST
    done

    echo "$(date -R): Master mirror triggers completed."
fi

echo "$(date -R): Archive publishing operations completed."