Mirroring: Provide rsync-based mirror hook script

diff --git a/Makefile b/Makefile
index 0d358c952057e3c1d4200592e9131aecb96113ae..a450e46d5ea9b116dcf1d7ac1a04a38d5c2ec278 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -37,7 +37,7 @@ PERLMODULES=Debian/Dgit.pm
 
 INFRA_PROGRAMS=dgit-repos-server dgit-ssh-dispatch \
        dgit-repos-policy-debian dgit-repos-admin-debian \
 
 INFRA_PROGRAMS=dgit-repos-server dgit-ssh-dispatch \
        dgit-repos-policy-debian dgit-repos-admin-debian \

-       dgit-repos-policy-trusting
+       dgit-repos-policy-trusting dgit-mirror-rsync

 INFRA_EXAMPLES=get-dm-txt ssh-wrap drs-cron-wrap get-suites
 INFRA_PERLMODULES=Debian/Dgit/Policy/Debian.pm
 
 INFRA_EXAMPLES=get-dm-txt ssh-wrap drs-cron-wrap get-suites
 INFRA_PERLMODULES=Debian/Dgit/Policy/Debian.pm
 

diff --git a/infra/dgit-mirror-rsync b/infra/dgit-mirror-rsync
new file mode 100755 (executable)
index 0000000..e73d9a5
--- /dev/null
+++ b/infra/dgit-mirror-rsync
@@ -0,0 +1,158 @@
+#!/bin/bash
+#
+# Mirror script for use as a dgit-repos-server mirror hook
+#
+# In addition to updated-hook (invoked by dgit-repos-server),
+# this script also supports the following ACTIONs:
+#   MIRROR-HOOK-SCRIPT ... setup [...]            create queue dir etc.
+#   MIRROR-HOOK-SCRIPT ... backlog [...]          do all packages which need it
+#   MIRROR-HOOK-SCRIPT ... all [...]              do all packages
+#   MIRROR-HOOK-SCRIPT ... mirror PACKAGE [...]   do just that, longer timeout
+#
+# DISTRO-DIR must contain a file `mirror-settings' which is a bash
+# script fragment assigning the following variables:
+#   remoterepos                for rsync, in form user@host:/dir
+# and optionally
+#   hooktimeout                default 30 [sec]
+#   rsynctimeout       default 900 [sec]
+#   rsyncssh           default 'ssh -o batchmode=yes'
+#   rsync              array, default (rsync -rltH --safe-links --delete)
+#   repos              default DISTRO-DIR/repos
+# (optional settings are all set before mirror-settings is included,
+# so you can modify them with += or some such)
+
+set -e
+set -o pipefail
+shopt -s nullglob
+
+case "$DGIT_DRS_DEBUG" in
+''|0!1)                ;;
+*)             set -x  ;;
+esac
+
+self=$0
+distrodir=$1;  shift
+action=$1;     shift
+package=$1
+
+repos=$distrodir/repos
+
+rsync=(rsync -rltH --safe-links --delete)
+hooktimeout=30
+rsynctimeout=900
+rsyncssh='ssh -o batchmode=yes'
+
+. $distrodir/mirror-settings
+
+# contents of $queue
+# $queue/$package.n    - mirror needed
+# $queue/$package.a    - being attempted, or attempt failed
+# $queue/$package.lock - lock (with-lock-ex)
+# $queue/$package.err  - stderr from failed (or current) run
+# $queue/$package.log  - stderr from last successful run
+
+cd $repos
+queue=_mirror-queue
+
+fail () {
+       echo >&2 "dgit-mirror-rsync: $*"; exit 127
+}
+
+case "$remoterepos" in
+*:/*|/*)       ;;
+'')            fail "remoterepos config not set" ;;
+*)             fail "remoterepos config does not match *:/* or /*" ;;
+esac
+
+actually () {
+       "${rsync[@]}" \
+               --timeout=$rsynctimeout                         \
+               -e "$rsyncssh"                                  \
+               "$repos/$package.git"/.                         \
+               "$remoterepos/$package.git"
+}
+
+reinvoke () {
+       newaction="$1"; shift
+
+       exec                                                    \
+       "$@"                                                    \
+       "$0"    "$distrodir" "reinvoke$newaction" "$package"
+}
+
+check-package-mirrorable () {
+       local repo=$repos/$package.git
+       local mode=$(stat -c%a "$repo")
+       case $mode in
+       *5)     return  0       ;;
+       *0)     return  1       ;;
+       *)      echo >&2 "unexpected mode $mode for $repo"; return 1    ;;
+       esac
+}
+
+lock-and-process () {
+       check-package-mirrorable || return 0
+       reinvoke -locked with-lock-ex -w "$queue/$package.lock"
+}
+
+attempt () {
+       exec >"$queue/$package.err" 2>&1
+       if actually; then
+               rm "$queue/$package.a"
+               mv -f "$queue/$package.err" "$queue/$package.log"
+               rm "$queue/$package.lock"
+       else
+               cat >&2 "$queue/$package.err"
+               exit 127
+       fi
+}
+
+lock-and-process-baseof-f () {
+       package=${f##*/}
+       package=${package%.*}
+       lock-and-process
+}
+
+case "$action" in
+
+updated-hook)
+       check-package-mirrorable || exit 0
+       touch "$queue/$package.n"
+       reinvoke -timed timeout --foreground $hooktimeout
+       ;;
+
+reinvoke-timed)
+       (lock-and-process) >/dev/null 2>&1
+       ;;
+
+mirror)
+       lock-and-process
+       ;;
+
+reinvoke-locked)
+       touch "$queue/$package.a"
+       rm -f "$queue/$package.n"
+       attempt
+       ;;
+
+backlog)
+       for f in $queue/*.[na]; do
+               lock-and-process-baseof-f
+       done
+       ;;
+
+all)
+       for f in [a-z0-9]*.git; do
+               lock-and-process-baseof-f
+       done
+       ;;
+
+setup)
+       test -d "$queue" || mkdir "$queue"
+       ;;
+
+*)
+       fail "bad action $action"
+       ;;
+
+esac