From: Ian Jackson <ijackson@chiark.greenend.org.uk>
Date: Fri, 10 Jul 2015 17:58:28 +0000 (+0100)
Subject: Mirroring: Provide rsync-based mirror hook script
X-Git-Tag: debian/1.0~42
X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=dgit.git;a=commitdiff_plain;h=7de3df349f56e05723b3eeb7972aaea4d9fc1000

Mirroring: Provide rsync-based mirror hook script
---

diff --git a/Makefile b/Makefile
index 0d358c95..a450e46d 100644
--- a/Makefile
+++ b/Makefile
@@ -37,7 +37,7 @@ PERLMODULES=Debian/Dgit.pm
 
 INFRA_PROGRAMS=dgit-repos-server dgit-ssh-dispatch \
 	dgit-repos-policy-debian dgit-repos-admin-debian \
-	dgit-repos-policy-trusting
+	dgit-repos-policy-trusting dgit-mirror-rsync
 INFRA_EXAMPLES=get-dm-txt ssh-wrap drs-cron-wrap get-suites
 INFRA_PERLMODULES=Debian/Dgit/Policy/Debian.pm
 
diff --git a/infra/dgit-mirror-rsync b/infra/dgit-mirror-rsync
new file mode 100755
index 00000000..e73d9a5b
--- /dev/null
+++ b/infra/dgit-mirror-rsync
@@ -0,0 +1,158 @@
+#!/bin/bash
+#
+# Mirror script for use as a dgit-repos-server mirror hook
+#
+# In addition to updated-hook (invoked by dgit-repos-server),
+# this script also supports the following ACTIONs:
+#   MIRROR-HOOK-SCRIPT ... setup [...]            create queue dir etc.
+#   MIRROR-HOOK-SCRIPT ... backlog [...]          do all packages which need it
+#   MIRROR-HOOK-SCRIPT ... all [...]              do all packages
+#   MIRROR-HOOK-SCRIPT ... mirror PACKAGE [...]   do just that, longer timeout
+#
+# DISTRO-DIR must contain a file `mirror-settings' which is a bash
+# script fragment assigning the following variables:
+#   remoterepos		for rsync, in form user@host:/dir
+# and optionally
+#   hooktimeout		default 30 [sec]
+#   rsynctimeout	default 900 [sec]
+#   rsyncssh		default 'ssh -o batchmode=yes'
+#   rsync		array, default (rsync -rltH --safe-links --delete)
+#   repos		default DISTRO-DIR/repos
+# (optional settings are all set before mirror-settings is included,
+# so you can modify them with += or some such)
+
+set -e
+set -o pipefail
+shopt -s nullglob
+
+case "$DGIT_DRS_DEBUG" in
+''|0!1)		;;
+*)		set -x	;;
+esac
+
+self=$0
+distrodir=$1;	shift
+action=$1;	shift
+package=$1
+
+repos=$distrodir/repos
+
+rsync=(rsync -rltH --safe-links --delete)
+hooktimeout=30
+rsynctimeout=900
+rsyncssh='ssh -o batchmode=yes'
+
+. $distrodir/mirror-settings
+
+# contents of $queue
+# $queue/$package.n	- mirror needed
+# $queue/$package.a	- being attempted, or attempt failed
+# $queue/$package.lock	- lock (with-lock-ex)
+# $queue/$package.err	- stderr from failed (or current) run
+# $queue/$package.log	- stderr from last successful run
+
+cd $repos
+queue=_mirror-queue
+
+fail () {
+	echo >&2 "dgit-mirror-rsync: $*"; exit 127
+}
+
+case "$remoterepos" in
+*:/*|/*)	;;
+'')		fail "remoterepos config not set" ;;
+*)		fail "remoterepos config does not match *:/* or /*" ;;
+esac
+
+actually () {
+	"${rsync[@]}" \
+		--timeout=$rsynctimeout				\
+		-e "$rsyncssh"					\
+		"$repos/$package.git"/.				\
+		"$remoterepos/$package.git"
+}
+
+reinvoke () {
+	newaction="$1"; shift
+
+	exec							\
+	"$@"							\
+	"$0"	"$distrodir" "reinvoke$newaction" "$package"
+}
+
+check-package-mirrorable () {
+	local repo=$repos/$package.git
+	local mode=$(stat -c%a "$repo")
+	case $mode in
+	*5)	return	0	;;
+	*0)	return	1	;;
+	*)	echo >&2 "unexpected mode $mode for $repo"; return 1	;;
+	esac
+}
+
+lock-and-process () {
+	check-package-mirrorable || return 0
+	reinvoke -locked with-lock-ex -w "$queue/$package.lock"
+}
+
+attempt () {
+	exec >"$queue/$package.err" 2>&1
+	if actually; then
+		rm "$queue/$package.a"
+		mv -f "$queue/$package.err" "$queue/$package.log"
+		rm "$queue/$package.lock"
+	else
+		cat >&2 "$queue/$package.err"
+		exit 127
+	fi
+}
+
+lock-and-process-baseof-f () {
+	package=${f##*/}
+	package=${package%.*}
+	lock-and-process
+}
+
+case "$action" in
+
+updated-hook)
+	check-package-mirrorable || exit 0
+	touch "$queue/$package.n"
+	reinvoke -timed timeout --foreground $hooktimeout
+	;;
+
+reinvoke-timed)
+	(lock-and-process) >/dev/null 2>&1
+	;;
+
+mirror)
+	lock-and-process
+	;;
+
+reinvoke-locked)
+	touch "$queue/$package.a"
+	rm -f "$queue/$package.n"
+	attempt
+	;;
+
+backlog)
+	for f in $queue/*.[na]; do
+		lock-and-process-baseof-f
+	done
+	;;
+
+all)
+	for f in [a-z0-9]*.git; do
+		lock-and-process-baseof-f
+	done
+	;;
+
+setup)
+	test -d "$queue" || mkdir "$queue"
+	;;
+
+*)
+	fail "bad action $action"
+	;;
+
+esac