chiark / gitweb /
changelog: start 9.13
[dgit.git] / infra / dgit-mirror-rsync
1 #!/bin/bash
2 #
3 # Mirror script for use as a dgit-repos-server mirror hook
4 #
5 # In addition to updated-hook (invoked by dgit-repos-server),
6 # this script also supports the following ACTIONs:
7 #   MIRROR-HOOK-SCRIPT ... setup [...]            create queue dir etc.
8 #   MIRROR-HOOK-SCRIPT ... backlog [...]          do all packages which need it
9 #   MIRROR-HOOK-SCRIPT ... all [...]              do all packages
10 #   MIRROR-HOOK-SCRIPT ... mirror PACKAGE [...]   do just that, longer timeout
11 #
12 # DISTRO-DIR must contain a file `mirror-settings' which is a bash
13 # script fragment assigning the following variables:
14 #   remoterepos         for rsync, in form user@host:/dir
15 # and optionally
16 #   hooktimeout         default 30 [sec]
17 #   rsynctimeout        default 900 [sec]
18 #   rsyncssh            default 'ssh -o batchmode=yes'
19 #   rsync               array, default (rsync -rltH --safe-links --delete)
20 #   repos               default DISTRO-DIR/repos
21 # (optional settings are all set before mirror-settings is included,
22 # so you can modify them with += or some such)
23
24 set -e
25 set -o pipefail
26 shopt -s nullglob
27
28 case "$DGIT_DRS_DEBUG" in
29 ''|0|1)         ;;
30 *)              set -x  ;;
31 esac
32
33 fail () {
34         echo >&2 "dgit-mirror-rsync: $*"; exit 127
35 }
36
37 if [ $# -lt 2 ]; then fail "too few arguments"; fi
38
39 self=$0
40
41 case "$self" in
42 /*)                             ;;
43 */*)    self="$PWD/$self"       ;;
44 *)                              ;;
45 esac
46
47 distrodir=$1;   shift
48 action=$1;      shift
49 package=$1
50
51 repos=$distrodir/repos
52
53 rsync=(rsync -rltH --safe-links --delete)
54 hooktimeout=30
55 rsynctimeout=900
56 rsyncssh='ssh -o batchmode=yes'
57 mirror_gc_cmd='git gc --auto'
58
59 . $distrodir/mirror-settings
60
61 # contents of $queue
62 # $queue/$package.n     - mirror needed
63 # $queue/$package.a     - being attempted, or attempt failed
64 # $queue/$package.lock  - lock (with-lock-ex)
65 # $queue/$package.err   - stderr from failed (or current) run
66 # $queue/$package.log   - stderr from last successful run
67
68 cd $repos
69 queue=_mirror-queue
70
71 case "$remoterepos" in
72 *:/*|/*)        ;;
73 '')             fail "remoterepos config not set" ;;
74 *)              fail "remoterepos config does not match *:/* or /*" ;;
75 esac
76
77 actually () {
78         if [ "x$mirror_gc_cmd" != x ]; then
79                 (
80                         cd "$repos/$package.git"
81                         $mirror_gc_cmd
82                 )
83         fi
84         "${rsync[@]}" \
85                 --timeout=$rsynctimeout                         \
86                 -e "$rsyncssh"                                  \
87                 "$repos/$package.git"/.                         \
88                 "$remoterepos/$package.git"
89 }
90
91 reinvoke () {
92         newaction="$1"; shift
93
94         exec                                                    \
95         "$@"                                                    \
96         "$self" "$distrodir" "reinvoke$newaction" "$package"
97 }
98
99 check-package-mirrorable () {
100         local repo=$repos/$package.git
101         local mode; mode=$(stat -c%a "$repo")
102         case $mode in
103         *5)     return  0       ;;
104         *0)     return  1       ;;
105         *)      echo >&2 "unexpected mode $mode for $repo"; return 1    ;;
106         esac
107 }
108
109 lock-and-process () {
110         check-package-mirrorable || return 0
111         reinvoke -locked with-lock-ex -w "$queue/$package.lock"
112 }
113
114 attempt () {
115         exec 3>&2 >"$queue/$package.err" 2>&1
116         if actually; then
117                 rm -f "$queue/$package.a"
118                 exec 2>&3 2>&1
119                 mv -f "$queue/$package.err" "$queue/$package.log"
120                 if ! [ -s "$queue/$package.log" ]; then
121                         rm "$queue/$package.log"
122                 fi
123                 rm "$queue/$package.lock"
124         else
125                 cat >&3 "$queue/$package.err"
126                 exit 127
127         fi
128 }
129
130 lock-and-process-baseof-f () {
131         package=${f##*/}
132         package=${package%.*}
133         lock-and-process
134 }
135
136 case "$action" in
137
138 updated-hook)
139         check-package-mirrorable || exit 0
140         touch "$queue/$package.n"
141         reinvoke -timed timeout --foreground $hooktimeout
142         ;;
143
144 reinvoke-timed)
145         (lock-and-process) >/dev/null 2>&1
146         ;;
147
148 mirror)
149         lock-and-process
150         ;;
151
152 reinvoke-locked)
153         touch "$queue/$package.a"
154         rm -f "$queue/$package.n"
155         attempt
156         ;;
157
158 backlog)
159         for f in $queue/*.[na]; do
160                 (lock-and-process-baseof-f ||:)
161         done
162         ;;
163
164 all)
165         for f in [a-z0-9]*.git; do
166                 (lock-and-process-baseof-f)
167         done
168         ;;
169
170 setup)
171         test -d "$queue" || mkdir "$queue"
172         ;;
173
174 *)
175         fail "bad action $action"
176         ;;
177
178 esac