rsync-backup.in, rsync-backup.8: Retry backups which fail fshash check.

I'm getting annoyed with vampire:root failing in the morning and then
succeeding after a manual retry.  So get the script to retry automatically.

diff --git a/rsync-backup.8 b/rsync-backup.8
index a265352ced5ebf5c1f3c36ea754784a0ced56d9b..b36153a2def9f3a4f29da86b6aa82caf3f09c56f 100644 (file)
--- a/rsync-backup.8
+++ b/rsync-backup.8
@@ -194,11 +194,28 @@ Expiry considers each existing dump against the policy lines in order:
 the last applicable line determines the dump's fate \(en so you should
 probably write the lines in decreasing order of duration.
 .TP
 the last applicable line determines the dump's fate \(en so you should
 probably write the lines in decreasing order of duration.
 .TP

+.BI "retry " count
+The
+.B live
+snapshot type (see below) doesn't prevent a filesystem from being
+modified while it's being backed up.  If this happens, the
+.B fshash
+pass will detect the difference and fail.  If the filesystem in question
+is relatively quiescent, then maybe retrying the backup will result in a
+successful consistent copy.  Following this command, a backup which
+results in an
+.B fshash
+mismatch will be retried up to
+.I count
+times before being declared a failure.
+.TP

 .BI "snap " type " " \fR[\fIargs\fR...]
 Use the snapshot
 .I type
 for subsequent backups.  Some snapshot types require additional
 .BI "snap " type " " \fR[\fIargs\fR...]
 Use the snapshot
 .I type
 for subsequent backups.  Some snapshot types require additional

-arguments, which may be supplied here.
+arguments, which may be supplied here.  This command clears the
+.B retry
+counter.

 .SS Configuration variables
 The following shell variables may be overridden by the configuration
 file.
 .SS Configuration variables
 The following shell variables may be overridden by the configuration
 file.

diff --git a/rsync-backup.in b/rsync-backup.in
index 705696d2b8cb4f73461c9f8aa288177d48c84d8c..ade72316736378a6b1551c0ed7286b91aada6ede 100644 (file)
--- a/rsync-backup.in
+++ b/rsync-backup.in
@@ -521,62 +521,84 @@ do_backup () {
   ## Back up FS on the current host.
 
   set -e
   ## Back up FS on the current host.
 
   set -e

+  attempt=0

 
   ## Report the start of this attempt.
   log "START BACKUP of $host:$fs"
 
 
   ## Report the start of this attempt.
   log "START BACKUP of $host:$fs"
 

-  ## Create and mount the remote snapshot.
-  snapmnt=$(snap_$snap $snapargs $fs $fsarg) || return $?
-  $verbose "   create snapshot"
+  ## Maybe we need to retry the backup.
+  while :; do

 
 

-  ## Build the list of hardlink sources.
-  linkdests=""
-  for i in $host $like; do
-    d=$STOREDIR/$i/$fs/last/
-    if [ -d $d ]; then linkdests="$linkdests --link-dest=$d"; fi
-  done
+    ## Create and mount the remote snapshot.
+    snapmnt=$(snap_$snap $snapargs $fs $fsarg) || return $?
+    $verbose " create snapshot"

 
 

-  ## Copy files from the remote snapshot.
-  mkdir -p new/
-  $verbose -n "        running rsync..."
-  set +e
-  run "RSYNC of $host:$fs (snapshot on $snapmnt)" do_rsync \
-    $linkdests \
-    $rsyncargs \
-    $snapmnt/ new/
-  rc_rsync=$?
-  set -e
-  $verbose " done"
+    ## Build the list of hardlink sources.
+    linkdests=""
+    for i in $host $like; do
+      d=$STOREDIR/$i/$fs/last/
+      if [ -d $d ]; then linkdests="$linkdests --link-dest=$d"; fi
+    done

 
 

-  ## Collect a map of the snapshot for verification purposes.
-  set +e
-  $verbose -n "        remote fshash..."
-  run "@$host: fshash $fs" remote_fshash
-  rc_fshash=$?
-  set -e
-  $verbose " done"
+    ## Copy files from the remote snapshot.
+    mkdir -p new/
+    $verbose -n "      running rsync..."
+    set +e
+    run "RSYNC of $host:$fs (snapshot on $snapmnt)" do_rsync \
+      $linkdests \
+      $rsyncargs \
+      $snapmnt/ new/
+    rc_rsync=$?
+    set -e
+    $verbose " done"

 
 

-  ## Remove the snapshot.
-  unsnap_$snap $snapargs $fs $fsarg
-  $verbose "   remove snapshot"
+    ## Collect a map of the snapshot for verification purposes.
+    set +e
+    $verbose -n "      remote fshash..."
+    run "@$host: fshash $fs" remote_fshash
+    rc_fshash=$?
+    set -e
+    $verbose " done"

 
 

-  ## If we failed to copy, then give up.
-  case $rc_rsync:$rc_fshash in
-    0:0) ;;
-    0:*) return $rc_fshash ;;
-    *) return $rc_rsync ;;
-  esac
+    ## Remove the snapshot.
+    unsnap_$snap $snapargs $fs $fsarg
+    $verbose " remove snapshot"

 
 

-  ## Get a matching map of the files received.
-  mkdir -m750 -p $STOREDIR/tmp
-  localmap=$STOREDIR/tmp/fshash.$host.$fs.$date
-  $verbose -n "        local fshash..."
-  run "local fshash $host:$fs" local_fshash || return $?
-  $verbose " done"
+    ## If we failed to copy, then give up.
+    case $rc_rsync:$rc_fshash in
+      0:0) ;;
+      0:*) return $rc_fshash ;;
+      *) return $rc_rsync ;;
+    esac
+
+    ## Get a matching map of the files received.
+    mkdir -m750 -p $STOREDIR/tmp
+    localmap=$STOREDIR/tmp/fshash.$host.$fs.$date
+    $verbose -n "      local fshash..."
+    run "local fshash $host:$fs" local_fshash || return $?
+    $verbose " done"
+
+    ## Compare the two maps.
+    set +e
+    run "compare fshash maps for $host:$fs" diff -u new.fshash $localmap
+    rc_diff=$?
+    set -e
+    case $rc_diff in
+      0)
+       break
+       ;;
+      1)
+       if [ $attempt -ge $retry ]; then return $rc; fi
+       $verbose "      fshash mismatch; retrying"
+       attempt=$(( $attempt + 1 ))
+       ;;
+      *)
+       return $rc_diff
+       ;;
+    esac
+  done

 
 

-  ## Compare the two maps.
-  run "compare fshash maps for $host:$fs" \
-    diff -u new.fshash $localmap || return $?
+  ## Glorious success.

   rm -f $localmap
   $verbose "   fshash match"
 
   rm -f $localmap
   $verbose "   fshash match"
 


@@ -687,9 +709,10 @@ backup () {
 ### Configuration functions.
 
 host () { host=$1; like=; $verbose "host $host"; }
 ### Configuration functions.
 
 host () { host=$1; like=; $verbose "host $host"; }

-snaptype () { snap=$1; shift; snapargs="$*"; }
+snaptype () { snap=$1; shift; snapargs="$*"; retry=0; }

 rsyncargs () { rsyncargs="$*"; }
 like () { like="$*"; }
 rsyncargs () { rsyncargs="$*"; }
 like () { like="$*"; }

+retry () { retry="$*"; }

 
 retain () {
   expire_policy="${expire_policy+$expire_policy
 
 retain () {
   expire_policy="${expire_policy+$expire_policy