3 ### Manage the backup archive structure
5 ### (c) 2011 Mark Wooding
8 ###----- Licensing notice ---------------------------------------------------
10 ### This file is part of the distorted.org.uk backup suite.
12 ### distorted-backup is free software; you can redistribute it and/or modify
13 ### it under the terms of the GNU General Public License as published by
14 ### the Free Software Foundation; either version 2 of the License, or
15 ### (at your option) any later version.
17 ### distorted-backup is distributed in the hope that it will be useful,
18 ### but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ### GNU General Public License for more details.
22 ### You should have received a copy of the GNU General Public License along
23 ### with distorted-backup; if not, write to the Free Software Foundation,
24 ### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
28 ## Configuration and testing.
29 : ${BKP=/mnt/bkp} ${META=/mnt/bkpmeta}
32 case $(id -u) in 0) ;; *) exec userv root bkpadmin "$@" ;; esac
34 ###--------------------------------------------------------------------------
41 ## Print a complaint to standard error.
47 ## Print a complaint and exit.
56 ## Add a cleanup command CMD to the list.
62 trap 'rc=$?; for c in $cleanups; do $c; done; exit $rc' \
66 cleanups=${cleanups+$cleanups }$cmd
69 rmtmp () { case ${tmpdir+t} in t) rm -rf "$tmpdir" ;; esac }
72 ## Make a temporary directory and output its name.
80 r=$(openssl rand -base64 12)
81 tmpdir=${TMPDIR-/tmp}/$quis.$$.$r
82 if mkdir -m700 "$tmpdir" >/dev/null 2>&1; then break; fi
83 case $i in ???) die "failed to create temporary directory" ;; esac
91 ###--------------------------------------------------------------------------
94 case "${USERV_USER+t}" in t) uservp=t ;; *) uservp=nil ;; esac
96 USAGE="COMMAND [ARGUMENT ...]"
101 name=$1; shift; args=$*
102 ## Define a command unconditionally.
109 ## Define a command for privileged users only.
111 case $uservp in nil) _defcmd "$@" ;; esac
115 ## Define a command usable via userv.
121 ## Write a usage message for the current command.
123 echo "usage: $quis${cmdname:+ $cmdname}${cmdargs:+ $cmdargs}"
127 ## Fail with a usage error.
135 ## Try to loop up the command CMD.
137 while read cmdname cmdargs; do
138 case $cmdname in "$cmd") return ;; esac
142 die "unknown command \`$cmd'"
147 case $# in 0) ;; *) usage_err ;; esac
150 $quis, version $version
156 while read cmd args; do
157 echo " $cmd${args:+ $args}"
163 ###--------------------------------------------------------------------------
164 ### Utility functions.
168 ## Sign the named FILE, producing a signature FILE.sig.
170 seccure-sign -F$KEYS/priv/backup-auth -cp256 -s"$file.sig" <"$file"
174 ## Check that a host is defined.
177 t) ;; *) die "no host defined (use \`-H')" ;;
182 thing=$1 good=$2 what=$3 string=$4
183 ## Check that STRING is a valid THING -- i.e., it only consists of GOOD
188 die "bad $thing \`$string' given for $what"
195 ## Check that STRING is at least plausibly numeric.
197 checkthing number "0-9" "$what" "$string"
202 ## Check that STRING is a plausible pathname.
205 .* | */.* | *[!-a-zA-Z0-9.,_#!%^+=@/:]*)
206 die "bad pathname \`$string' given for $what"
213 ## Check that THING doesn't need shell quoting, and doesn't interfere with
214 ## other common delimiter characters. (Colons aren't allowed because they
215 ## mess up /etc/passwd; slashes aren't allowed because they're directory
216 ## separators. Leading dots aren't allowed either. Hashes seem OK.)
218 checkthing word "-a-zA-Z0-9.,_#!%^+=@" "$what" "$string"
222 dir=$1 owner=$2 mode=$3
223 ## Make a directory and set permissions on it.
230 ###--------------------------------------------------------------------------
231 ### Volume and volume group maintenance.
234 ## Output the tag of the mounted backup volume group.
238 /dev/mapper/cbkp-*) echo "${dev#*-}"; return ;;
239 *) die "failed to parse tag from device name \`$dev'" ;;
244 ## Guess and print the tag of the available backup volume group. If there
245 ## is not exactly one volume group available, print an error and fail.
247 LVM_SUPPRESS_FD_WARNINGS=t vgs @backup --noheadings -o name,attr | {
249 while read name attr; do
250 case "$name" in bkp-*) ;; *) continue ;; esac
251 case "$attr" in ??x*) continue ;; esac
252 match="$match${match:+ }${name#bkp-}"
255 x) die "no backup volume groups available" ;;
256 x*\ *) die "multiple backup volume groups available: $match" ;;
264 ## Output a device name for the filesystem mounted on DIR.
266 dev=$(mountpoint -d "$dir")
267 devname=$(udevadm info --query=name --path="/dev/block/$dev")
270 devname=mapper/$(dmsetup info -c --noheadings -oname "/dev/$devname")
278 ## Mount the metadata volume of the backup volume group named TAG.
280 if ! mountpoint -q $META; then
281 mount "/dev/bkp-$tag/meta" $META
286 ## Decrypt and output the key for the encrypted volume. This assumes that
287 ## the metadata volume is already mounted on /mnt/bkpmeta.
289 seccure-decrypt -q -m128 -cp256 -F$KEYS/priv/backup-disk <$META/cur/blob
294 ## Decrypt but don't mount the encrypted volume of the backup volume group
298 if [ ! -b "/dev/mapper/cbkp-$tag" ]; then
299 cryptkey | cryptsetup luksOpen --key-file=- \
300 "/dev/bkp-$tag/crypt" "cbkp-$tag"
306 ## Mount the encrypted subvolume of the backup volume group named TAG. The
307 ## metadata volume will be mounted if necessary.
310 if ! mountpoint -q $BKP; then
311 mount "/dev/mapper/cbkp-$tag" $BKP
316 ## Unmounts a backup volume group: both the encrypted and metadata volumes
319 if mountpoint -q $BKP; then
320 tag=$(currenttag) cryptclosep=t
324 for i in bkp bkpmeta; do
325 if mountpoint -q /mnt/$i; then umount /mnt/$i; fi
329 if [ -b "/dev/mapper/cbkp-$tag" ]; then
330 cryptsetup luksClose "cbkp-$tag"
335 defcmd initvol TAG DEVICE
337 case $# in 2) ;; *) usage_err ;; esac
340 vgcreate --addtag @backup "bkp-$tag" "$dev"
342 lvcreate -L4M -nmeta "bkp-$tag"
343 mkfs -text2 -Lmeta "/dev/bkp-$tag/meta"
346 mkdir -m755 $META/new
347 dd if=/dev/random bs=1 count=512 |
348 seccure-encrypt -m128 $(cat $KEYS/pub/backup-disk.pub) >$META/new/blob
349 mv $META/new $META/cur
351 lvcreate -l100%FREE -ncrypt "bkp-$tag"
352 cryptkey | cryptsetup luksFormat \
353 --cipher=twofish-xts-benbi:sha256 --hash=sha256 \
354 "/dev/bkp-$tag/crypt" -
356 mkfs -text2 -Lbackup -i1048576 "/dev/mapper/cbkp-$tag"
360 defucmd mount "[TAG]"
363 0) tag=$(guesstag) check=nil ;;
368 if mountpoint -q $BKP; then
370 case "$check,$curtag" in "t,$tag") ;; t*) exit 1 ;; esac
378 case $# in 0) ;; *) usage_err ;; esac
380 for i in bkp bkpmeta; do
381 if mountpoint -q /mnt/$i; then mntp=t; fi
384 nil) die "backup volume not mounted" ;;
389 ###--------------------------------------------------------------------------
390 ### Archive maintenance.
394 ## Check a directory which has `hashes' and `hashes.sig' files.
396 if ! seccure-verify -q -i"$dir/hashes" -- \
397 $(cat "$KEYS/$key") $(cat "$dir/hashes.sig")
399 die "failed to verify signature for \`$dir'"
403 sha256sum --quiet -c hashes
406 find . -type f -print | sed 's:^\./::' | sort >"$tmpdir/present"
409 sed 's/^[a-f0-9]*[* ] //' hashes
410 } | sort >"$tmpdir/checked"
412 diff -u checked present
416 dir=$1 owner=$2 fmode=$3 dmode=$4
417 ## Fix the directory tree DIR so that everything is owned by OWNER (a
418 ## USER:GROUP pair) and has modes FMODE for files and DMODE for
421 ## Change all of the ownerships. This will prevent anyone else from
422 ## changing the permissions on the files. This assumes that chown(1) is
423 ## secure in recursive mode; I've checked that GNU chown seems correct.
424 chown -R $owner "$dir"
426 ## Paranoia: check that we correctly changed all of the files.
427 u=${owner%:*} g=${owner#*:}
428 (cd "$dir"; find . ! \( -user $u -group $g \) -ls) |
430 moan "failed to fix permssions on \`$dir'"
431 { echo $line; cat; } | sed 's/^/ /'
435 ## Now get to work on the file and directory permissions.
436 find "$dir" -type d -print0 | xargs -0r chmod $dmode
437 find "$dir" ! -type d -print0 | xargs -0r chmod $fmode
442 ## Commit an `prepare' directory DIR, moving its `incoming' files to
443 ## TARGET. This will choose the correct name for the directory, but
444 ## assumes that it's already correctly laid out. We assume that the
445 ## permissions on this directory are safe (e.g., they've already been fixed
446 ## using `fixperms'). On successful exit, DIR won't exist any more. The
447 ## shell variable `label' is set to the resulting archive name.
449 ## If there's no `incoming' directory, then there's nothing to do. Just
450 ## zap the directory and move on.
451 if [ ! -d "$dir/incoming" ]; then
456 ## Find the datestamp and level numbers to use for this directory. These
457 ## are created before the `incoming' directory, so they ought to exist.
458 read level date time tz <"$dir/meta"
460 ## Find a suitable sequence number for the target. This is rather ugly;
465 for i in "$target"/"$date#$seq".*; do
466 if [ -e "$i" ]; then anyp=t; break; fi
468 case $anyp in nil) break ;; esac
472 ## Move the directory.
473 label="$date#$seq.$level"
474 mv "$dir/incoming" "$target/$label"
477 ## Update the catalogue. Replace an existing dump at the same level.
478 ## Assume that dates are monotonically increasing: add the new entry at the
481 while read lab l d t; do
482 if [ $l -ne $level ]; then echo $label $l $d $t; fi
483 done <"$target"/CATALOGUE
484 echo $level $date $time $tz
485 } >"$target"/CATALOGUE.new
486 mv "$target"/CATALOGUE.new "$target"/CATALOGUE
491 case $# in 0) ;; *) usage_err ;; esac
493 ## Make a `new' directory and start recording our files.
499 ## Copy the blob from the existing metadata.
503 ## Archive the key recovery information.
505 tar cfz $META/new/keys.tgz pub/ recov/
508 ## Copy user and group information.
510 for i in passwd group; do
511 grep -E '^(root|backup|bkp-[[:alnum:]]+):' /etc/$i >$i
515 ## Build the hashes file, and sign it.
521 ## Replace the old metadata.
530 case $# in 0) ;; *) usage_err ;; esac
532 checkdir pub/backup-auth.pub $META/cur
536 ## Report the current date, as ISO8601. Allow an override.
538 case "${forceday+t}" in t) echo "$forceday" ;; *) date +%Y-%m-%d ;; esac
541 defucmd prep ASSET LEVEL \[DATE TIME TZ]
544 2) set -- "$@" $(today) $(date +%H:%M:%S) $(date +%z) ;;
548 asset=$1 level=$2 date=$3 time=$4 tz=$5
550 checkword asset "$asset"
551 checknum level "$level"
552 checkthing date -0-9 date "$date"
553 checkthing time :0-9 time "$time"
554 checkthing timezone -+0-9 tz "$tz"
556 ## Make the host and asset directories if necessary.
558 for i in $host $asset; do
559 if [ ! -d $i ]; then domkdir $i root:root 755; fi
562 if [ ! -d failed ]; then domkdir failed root:root 755; fi
563 for i in . failed; do
564 if [ ! -f $i/CATALOGUE ]; then
566 chown root:root $i/CATALOGUE
567 chmod 644 $i/CATALOGUE
571 ## If an existing dump is in progress then archive it as a failure.
572 if [ -d prepare ]; then
573 if [ -d prepare/incoming ]; then
574 fixperms prepare/incoming root:root 640 755
576 commitdir prepare failed/
579 ## Make a new preparation directory.
580 domkdir prepare root:bkp-$host 755
581 echo $level $date $time $tz >prepare/meta
582 domkdir prepare/incoming bkp-$host:bkp-$host 2775
584 ## Print the directory name.
585 echo $BKP/$host/$asset/prepare/incoming
590 case $# in 1) ;; *) usage_err ;; esac
593 checkword asset "$asset"
595 ## Check that there's something to abort.
597 if [ ! -d $host/$asset/prepare ]; then
598 die "no dump in progress for $host/$asset"
601 ## Just throw it away.
602 rm -rf $host/$asset/prepare
607 case $# in 1) ;; *) usage_err ;; esac
610 checkword asset "$asset"
612 ## Check that there's something to fail.
614 if [ ! -d $host/$asset/prepare ]; then
615 die "no dump in progress for $host/$asset"
618 ## Archive the failure. This shouldn't be used to determine dump levels or
619 ## we'll have gaps when things get sorted out.
621 if [ -d prepare/incoming ]; then
622 fixperms prepare/incoming root:root 640 755
624 commitdir prepare failed/
629 ## Convert an ISO8601 DATE to a Julian Day Number.
631 ## Extract the components of the date and trim leading zeros (which will
632 ## cause things to be interpreted as octal and fail).
633 year=${date%%-*} rest=${date#*-}; month=${rest%%-*} day=${rest#*-}
634 year=${year#0} month=${month#0} day=${day#0}
636 ## The actual calculation: convert a (proleptic) Gregorian calendar date
637 ## into a Julian day number. This is taken from Wikipedia's page
638 ## http://en.wikipedia.org/wiki/Julian_day#Calculation but the commentary
639 ## is mine. The epoch is 4713BC-01-01 (proleptic) Julian, or 4714BC-11-24
640 ## proleptic Gregorian.
642 ## If the MONTH is January or February then set a = 1, otherwise set a = 0.
643 a=$(( (14 - $month)/12 ))
645 ## Compute a year offset relative to 4799BC-03-01. This puts the leap day
646 ## as the very last day in a year, which is very convenient. The offset
647 ## here is sufficient to make all y values positive (within the range of
648 ## the JDN calendar), and is a multiple of 400, which is the Gregorian
650 y=$(( $year + 4800 - $a ))
652 ## Compute the offset month number in that year. These months count from
654 m=$(( $month + 12*$a - 3 ))
656 ## Now for the main event. The (153 m + 2)/5 term is a surprising but
657 ## correct trick for obtaining the number of days in the first m months of
658 ## the (shifted) year). The magic offset 32045 is what you get when you
659 ## plug the proper JDN epoch (year = -4713, month = 11, day = 24) into the
661 jdn=$(( $day + (153*$m + 2)/5 + 365*$y + $y/4 - $y/100 + $y/400 - 32045 ))
667 fulldate=$1 lastdate=$2
668 ## Return the dump level, given that the most recent full dump occurred on
669 ## FULLDATE and the most revent dump of any kind occurred on LASTDATE.
671 ## Actually, we're much more interested in the day difference between these
673 fulljdn=$(julian $fulldate)
674 lastjdn=$(julian $lastdate)
675 now=$(today); nowjdn=$(julian $now)
676 lastday=$(( $lastjdn - $fulljdn ))
677 today=$(( $nowjdn - $fulljdn ))
679 ## If the difference is greater than 512 then we know we should do a full
680 ## dump. (This provides an upper bound for the search below. It should
681 ## never happen in practice, of course.)
682 if [ $(( $today - $lastday )) -ge 512 ]; then echo 0; return; fi
684 ## Now we work out the correct dump level. This will assume that the
685 ## previous dump had a sensible level. If dumps are omitted, then we will
686 ## choose a lower (more comprehensive) dump level than the schedule calls
687 ## for; such an overestimation will mean that we will probably end up
688 ## dumping too much again. This is the right error to make.
690 ## We use a Towers of Hanoi schedule. If we're doing dumps every day, then
691 ## on day n since the last full dump, we work out the dump level as
692 ## follows: write n = 2^s t where t is odd (i.e., s is the number of
693 ## trailing zero bits in the binary representation of n); then the dump
694 ## level on day n is 9 - s. This is enough for 512 days without a full
695 ## dump, and it fails gracefully anyway.
697 ## Now we have to deal with the problem of skipping dumps. Suppose the
698 ## last dump was on day m = 2^u v, and it's now day n = 2^s t. We ought to
699 ## take the lowest dump level of any intervening day, i.e., the dump level
700 ## is 9 - a for the largest a such that there exists b with m < l = 2^a b
701 ## <= n. We claim that such an l is unique. Suppose, to the contrary,
702 ## that m < 2^a b < 2^a b' <= n, with both b and b' odd. Then m < 2^{a+1}
703 ## (b + 1)/2 <= n, contradicting maximality of a.
705 ## How does this help? Observe that n = 2^s t = 2^a b + o, for some o <
706 ## 2^a: if o >= 2^a then 2^a (b + 1) <= n contradicting uniqueness of l.
707 ## Similarly, m = 2^u v = 2^a b - r, for some r <= 2^a (otherwise m <
708 ## 2^a (b - 1), again contradicting uniqueness). Therefore, m and n are
709 ## identical from bit a + 1 onwards, and differ at bit a. In other words,
710 ## a is the position of the most significant set bit in m XOR n.
711 diff=$(( lastday ^ today ))
713 ## We know that the bit position must be less than 16.
715 while [ $diff -gt 1 ]; do
716 xx=$(( $diff >> $t ))
717 if [ $xx -gt 0 ]; then
718 diff=$xx n=$(( $n + $t ))
728 case $# in 1) ;; *) usage_err ;; esac
731 checkword asset "$asset"
733 ## Set the correct directory. If it doesn't exist then we obviously need a
736 full="0 1970-01-01 00:00:00 +0000"
737 if [ ! -d $host/$asset ]; then echo $full; return; fi
740 ## We need the time of the most recent dump of any kind, and the most
741 ## recent level-zero dump.
742 fulldate=none lastdate=none
743 while read label level date time tz; do
744 if [ $level -eq 0 ]; then fulldate=$date; fi
747 case $fulldate in none) echo $full; return ;; esac
748 level=$(dumplevel $fulldate $lastdate)
750 ## Determine the time of the most recent dump of the same or more inclusive
753 while read lab l d t; do
754 if [ $l -le $level ]; then date=$d time=$t; fi
756 echo $level $date $time $tz
759 defucmd hash ASSET FILE HASH
761 case $# in 3) ;; *) usage_err ;; esac
762 asset=$1 file=$2 hash=$3
763 checkword asset "$asset"
764 checkpath file "$file"
765 checkword hash "$hash"
767 cd $BKP/$host/$asset/prepare
769 if [ -f hashes ]; then
771 case "$f" in "$file") die "file \`$file' already hashed" ;; esac
775 echo "$hash $file" >>hashes.new
781 case $# in 1) ;; *) usage_err ;; esac
784 checkword asset "$asset"
786 cd $BKP/$host/$asset/prepare
787 fixperms incoming root:bkp-$host 640 755
790 if [ -f hashes ]; then
791 while read hash name; do
792 if [ ! -f "incoming/$name" ]; then
793 die "precomputed hash for nonexistent or non-file \`$name'"
795 findargs="$findargs ! -path incoming/$name"
797 cp hashes hashes.calc
800 find incoming -type f $findargs -print0 | \
801 xargs -0r sha256sum | \
802 sed 's: incoming/: :' \
804 sort -k2 hashes.calc >incoming/hashes
806 chmod 640 incoming/hashes incoming/hashes.sig
807 chown root:bkp-$host incoming/hashes incoming/hashes.sig
814 defucmd check ASSET LABEL
816 case $# in 2) ;; *) usage_err ;; esac
819 checkword asset "$asset"
820 checkword label "$label"
822 checkdir pub/backup-auth.pub $BKP/$host/$asset/$label
825 defucmd catalogue ASSET
827 case $# in 1) ;; *) usage_err ;; esac
830 checkword asset "$asset"
832 cat $BKP/$host/$asset/CATALOGUE
835 defucmd outdated ASSET
837 case $# in 1) ;; *) usage_err ;; esac
840 checkword asset "$asset"
843 for i in [0-9]*#*.*; do
844 if [ -d "$i" ]; then echo "$i"; fi
849 date=${tag%%#*} level=${tag##*.}
850 if [ $level -le $best ]
858 ###--------------------------------------------------------------------------
861 defcmd test CMD '[ARGS ...]'
862 cmd_test () { "$@"; }
866 host=${USERV_USER#bkp-}
875 while getopts "$opts" opt; do
879 D) forceday=$OPTARG ;;
883 shift $(( $OPTIND - 1 ))
885 case $# in 0) usage_err ;; esac
886 lookupcmd "$1"; shift
889 ###----- That's all, folks --------------------------------------------------