infra/dgit-mirror-rsync

   1 #!/bin/bash
   2 #
   3 # Mirror script for use as a dgit-repos-server mirror hook
   4 #
   5 # In addition to updated-hook (invoked by dgit-repos-server),
   6 # this script also supports the following ACTIONs:
   7 #   MIRROR-HOOK-SCRIPT ... setup [...]            create queue dir etc.
   8 #   MIRROR-HOOK-SCRIPT ... backlog [...]          do all packages which need it
   9 #   MIRROR-HOOK-SCRIPT ... all [...]              do all packages
  10 #   MIRROR-HOOK-SCRIPT ... mirror PACKAGE [...]   do just that, longer timeout
  11 #
  12 # DISTRO-DIR must contain a file `mirror-settings' which is a bash
  13 # script fragment assigning the following variables:
  14 #   remoterepos         for rsync, in form user@host:/dir
  15 # and optionally
  16 #   hooktimeout         default 30 [sec]
  17 #   rsynctimeout        default 900 [sec]
  18 #   rsyncssh            default 'ssh -o batchmode=yes'
  19 #   rsync               array, default (rsync -rltH --safe-links --delete)
  20 #   repos               default DISTRO-DIR/repos
  21 # (optional settings are all set before mirror-settings is included,
  22 # so you can modify them with += or some such)
  23
  24 set -e
  25 set -o pipefail
  26 shopt -s nullglob
  27
  28 case "$DGIT_DRS_DEBUG" in
  29 ''|0|1)         ;;
  30 *)              set -x  ;;
  31 esac
  32
  33 fail () {
  34         echo >&2 "dgit-mirror-rsync: $*"; exit 127
  35 }
  36
  37 if [ $# -lt 2 ]; then fail "too few arguments"; fi
  38
  39 self=$0
  40
  41 case "$self" in
  42 /*)                             ;;
  43 */*)    self="$PWD/$self"       ;;
  44 *)                              ;;
  45 esac
  46
  47 distrodir=$1;   shift
  48 action=$1;      shift
  49 package=$1
  50
  51 repos=$distrodir/repos
  52
  53 rsync=(rsync -rltH --safe-links --delete)
  54 hooktimeout=30
  55 rsynctimeout=900
  56 rsyncssh='ssh -o batchmode=yes'
  57 mirror_gc_cmd='git gc --auto'
  58
  59 . $distrodir/mirror-settings
  60
  61 # contents of $queue
  62 # $queue/$package.n     - mirror needed
  63 # $queue/$package.a     - being attempted, or attempt failed
  64 # $queue/$package.lock  - lock (with-lock-ex)
  65 # $queue/$package.err   - stderr from failed (or current) run
  66 # $queue/$package.log   - stderr from last successful run
  67
  68 cd $repos
  69 queue=_mirror-queue
  70
  71 case "$remoterepos" in
  72 *:/*|/*)        ;;
  73 '')             fail "remoterepos config not set" ;;
  74 *)              fail "remoterepos config does not match *:/* or /*" ;;
  75 esac
  76
  77 actually () {
  78         if [ "x$mirror_gc_cmd" != x ]; then
  79                 (
  80                         cd "$repos/$package.git"
  81                         $mirror_gc_cmd
  82                 )
  83         fi
  84         "${rsync[@]}" \
  85                 --timeout=$rsynctimeout                         \
  86                 -e "$rsyncssh"                                  \
  87                 "$repos/$package.git"/.                         \
  88                 "$remoterepos/$package.git"
  89 }
  90
  91 reinvoke () {
  92         newaction="$1"; shift
  93
  94         exec                                                    \
  95         "$@"                                                    \
  96         "$self" "$distrodir" "reinvoke$newaction" "$package"
  97 }
  98
  99 check-package-mirrorable () {
 100         local repo=$repos/$package.git
 101         local mode; mode=$(stat -c%a "$repo")
 102         case $mode in
 103         *5)     return  0       ;;
 104         *0)     return  1       ;;
 105         *)      echo >&2 "unexpected mode $mode for $repo"; return 1    ;;
 106         esac
 107 }
 108
 109 lock-and-process () {
 110         check-package-mirrorable || return 0
 111         reinvoke -locked with-lock-ex -w "$queue/$package.lock"
 112 }
 113
 114 attempt () {
 115         exec 3>&2 >"$queue/$package.err" 2>&1
 116         if actually; then
 117                 rm -f "$queue/$package.a"
 118                 exec 2>&3 2>&1
 119                 mv -f "$queue/$package.err" "$queue/$package.log"
 120                 if ! [ -s "$queue/$package.log" ]; then
 121                         rm "$queue/$package.log"
 122                 fi
 123                 rm "$queue/$package.lock"
 124         else
 125                 cat >&3 "$queue/$package.err"
 126                 exit 127
 127         fi
 128 }
 129
 130 lock-and-process-baseof-f () {
 131         package=${f##*/}
 132         package=${package%.*}
 133         lock-and-process
 134 }
 135
 136 case "$action" in
 137
 138 updated-hook)
 139         check-package-mirrorable || exit 0
 140         touch "$queue/$package.n"
 141         reinvoke -timed timeout --foreground $hooktimeout
 142         ;;
 143
 144 reinvoke-timed)
 145         (lock-and-process) >/dev/null 2>&1
 146         ;;
 147
 148 mirror)
 149         lock-and-process
 150         ;;
 151
 152 reinvoke-locked)
 153         touch "$queue/$package.a"
 154         rm -f "$queue/$package.n"
 155         attempt
 156         ;;
 157
 158 backlog)
 159         for f in $queue/*.[na]; do
 160                 (lock-and-process-baseof-f ||:)
 161         done
 162         ;;
 163
 164 all)
 165         for f in [a-z0-9]*.git; do
 166                 (lock-and-process-baseof-f)
 167         done
 168         ;;
 169
 170 setup)
 171         test -d "$queue" || mkdir "$queue"
 172         ;;
 173
 174 *)
 175         fail "bad action $action"
 176         ;;
 177
 178 esac