3 ### Manage the backup archive structure
5 ### (c) 2011 Mark Wooding
8 ###----- Licensing notice ---------------------------------------------------
10 ### This program is free software; you can redistribute it and/or modify
11 ### it under the terms of the GNU General Public License as published by
12 ### the Free Software Foundation; either version 2 of the License, or
13 ### (at your option) any later version.
15 ### This program is distributed in the hope that it will be useful,
16 ### but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ### GNU General Public License for more details.
20 ### You should have received a copy of the GNU General Public License
21 ### along with this program; if not, write to the Free Software Foundation,
22 ### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
26 ## Configuration and testing.
27 : ${BKP=/mnt/bkp} ${META=/mnt/bkpmeta}
30 case $(id -u) in 0) ;; *) exec userv root bkpadmin "$@" ;; esac
32 ###--------------------------------------------------------------------------
39 ## Print a complaint to standard error.
45 ## Print a complaint and exit.
54 ## Add a cleanup command CMD to the list.
60 trap 'rc=$?; for c in $cleanups; do $c; done; exit $rc' \
64 cleanups=${cleanups+$cleanups }$cmd
67 rmtmp () { case ${tmpdir+t} in t) rm -rf "$tmpdir" ;; esac }
70 ## Make a temporary directory and output its name.
78 r=$(openssl rand -base64 12)
79 tmpdir=${TMPDIR-/tmp}/$quis.$$.$r
80 if mkdir -m700 "$tmpdir" >/dev/null 2>&1; then break; fi
81 case $i in ???) die "failed to create temporary directory" ;; esac
89 ###--------------------------------------------------------------------------
92 case "${USERV_USER+t}" in t) uservp=t ;; *) uservp=nil ;; esac
94 USAGE="COMMAND [ARGUMENT ...]"
99 name=$1; shift; args=$*
100 ## Define a command unconditionally.
107 ## Define a command for privileged users only.
109 case $uservp in nil) _defcmd "$@" ;; esac
113 ## Define a command usable via userv.
119 ## Write a usage message for the current command.
121 echo "usage: $quis${cmdname:+ $cmdname}${cmdargs:+ $cmdargs}"
125 ## Fail with a usage error.
133 ## Try to loop up the command CMD.
135 while read cmdname cmdargs; do
136 case $cmdname in "$cmd") return ;; esac
140 die "unknown command \`$cmd'"
145 case $# in 0) ;; *) usage_err ;; esac
148 $quis, version $version
154 while read cmd args; do
155 echo " $cmd${args:+ $args}"
161 ###--------------------------------------------------------------------------
162 ### Utility functions.
166 ## Sign the named FILE, producing a signature FILE.sig.
168 seccure-sign -F$KEYS/priv/backup-auth -cp256 -s"$file.sig" <"$file"
172 ## Check that a host is defined.
175 t) ;; *) die "no host defined (use \`-H')" ;;
180 thing=$1 good=$2 what=$3 string=$4
181 ## Check that STRING is a valid THING -- i.e., it only consists of GOOD
186 die "bad $thing \`$string' given for $what"
193 ## Check that STRING is at least plausibly numeric.
195 checkthing number "0-9" "$what" "$string"
200 ## Check that STRING is a plausible pathname.
203 .* | */.* | *[!-a-zA-Z0-9.,_#!%^+=@/:]*)
204 die "bad pathname \`$string' given for $what"
211 ## Check that THING doesn't need shell quoting, and doesn't interfere with
212 ## other common delimiter characters. (Colons aren't allowed because they
213 ## mess up /etc/passwd; slashes aren't allowed because they're directory
214 ## separators. Leading dots aren't allowed either. Hashes seem OK.)
216 checkthing word "-a-zA-Z0-9.,_#!%^+=@" "$what" "$string"
220 dir=$1 owner=$2 mode=$3
221 ## Make a directory and set permissions on it.
228 ###--------------------------------------------------------------------------
229 ### Volume and volume group maintenance.
232 ## Output the tag of the mounted backup volume group.
236 /dev/mapper/cbkp-*) echo "${dev#*-}"; return ;;
237 *) die "failed to parse tag from device name \`$dev'" ;;
242 ## Guess and print the tag of the available backup volume group. If there
243 ## is not exactly one volume group available, print an error and fail.
245 LVM_SUPPRESS_FD_WARNINGS=t vgs @backup --noheadings -o name,attr | {
247 while read name attr; do
248 case "$name" in bkp-*) ;; *) continue ;; esac
249 case "$attr" in ??x*) continue ;; esac
250 match="$match${match:+ }${name#bkp-}"
253 x) die "no backup volume groups available" ;;
254 x*\ *) die "multiple backup volume groups available: $match" ;;
262 ## Output a device name for the filesystem mounted on DIR.
264 dev=$(mountpoint -d "$dir")
265 devname=$(udevadm info --query=name --path="/dev/block/$dev")
268 devname=mapper/$(dmsetup info -c --noheadings -oname "/dev/$devname")
276 ## Mount the metadata volume of the backup volume group named TAG.
278 if ! mountpoint -q $META; then
279 mount "/dev/bkp-$tag/meta" $META
284 ## Decrypt and output the key for the encrypted volume. This assumes that
285 ## the metadata volume is already mounted on /mnt/bkpmeta.
287 seccure-decrypt -q -m128 -cp256 -F$KEYS/priv/backup-disk <$META/cur/blob
292 ## Decrypt but don't mount the encrypted volume of the backup volume group
296 if [ ! -b "/dev/mapper/cbkp-$tag" ]; then
297 cryptkey | cryptsetup luksOpen --key-file=- \
298 "/dev/bkp-$tag/crypt" "cbkp-$tag"
304 ## Mount the encrypted subvolume of the backup volume group named TAG. The
305 ## metadata volume will be mounted if necessary.
308 if ! mountpoint -q $BKP; then
309 mount "/dev/mapper/cbkp-$tag" $BKP
314 ## Unmounts a backup volume group: both the encrypted and metadata volumes
317 if mountpoint -q $BKP; then
318 tag=$(currenttag) cryptclosep=t
322 for i in bkp bkpmeta; do
323 if mountpoint -q /mnt/$i; then umount /mnt/$i; fi
327 if [ -b "/dev/mapper/cbkp-$tag" ]; then
328 cryptsetup luksClose "cbkp-$tag"
333 defcmd initvol TAG DEVICE
335 case $# in 2) ;; *) usage_err ;; esac
338 vgcreate --addtag @backup "bkp-$tag" "$dev"
340 lvcreate -L4M -nmeta "bkp-$tag"
341 mkfs -text2 -Lmeta "/dev/bkp-$tag/meta"
344 mkdir -m755 $META/new
345 dd if=/dev/random bs=1 count=512 |
346 seccure-encrypt -m128 $(cat $KEYS/pub/backup-disk.pub) >$META/new/blob
347 mv $META/new $META/cur
349 lvcreate -l100%FREE -ncrypt "bkp-$tag"
350 cryptkey | cryptsetup luksFormat \
351 --cipher=twofish-xts-benbi:sha256 --hash=sha256 \
352 "/dev/bkp-$tag/crypt" -
354 mkfs -text2 -Lbackup -i1048576 "/dev/mapper/cbkp-$tag"
358 defucmd mount "[TAG]"
361 0) tag=$(guesstag) check=nil ;;
366 if mountpoint -q $BKP; then
368 case "$check,$curtag" in "t,$tag") ;; t*) exit 1 ;; esac
376 case $# in 0) ;; *) usage_err ;; esac
378 for i in bkp bkpmeta; do
379 if mountpoint -q /mnt/$i; then mntp=t; fi
382 nil) die "backup volume not mounted" ;;
387 ###--------------------------------------------------------------------------
388 ### Archive maintenance.
392 ## Check a directory which has `hashes' and `hashes.sig' files.
394 if ! seccure-verify -q -i"$dir/hashes" -- \
395 $(cat "$KEYS/$key") $(cat "$dir/hashes.sig")
397 die "failed to verify signature for \`$dir'"
401 sha256sum --quiet -c hashes
404 find . -type f -print | sed 's:^\./::' | sort >"$tmpdir/present"
407 sed 's/^[a-f0-9]*[* ] //' hashes
408 } | sort >"$tmpdir/checked"
410 diff -u checked present
414 dir=$1 owner=$2 fmode=$3 dmode=$4
415 ## Fix the directory tree DIR so that everything is owned by OWNER (a
416 ## USER:GROUP pair) and has modes FMODE for files and DMODE for
419 ## Change all of the ownerships. This will prevent anyone else from
420 ## changing the permissions on the files. This assumes that chown(1) is
421 ## secure in recursive mode; I've checked that GNU chown seems correct.
422 chown -R $owner "$dir"
424 ## Paranoia: check that we correctly changed all of the files.
425 u=${owner%:*} g=${owner#*:}
426 (cd "$dir"; find . ! \( -user $u -group $g \) -ls) |
428 moan "failed to fix permssions on \`$dir'"
429 { echo $line; cat; } | sed 's/^/ /'
433 ## Now get to work on the file and directory permissions.
434 find "$dir" -type d -print0 | xargs -0r chmod $dmode
435 find "$dir" ! -type d -print0 | xargs -0r chmod $fmode
440 ## Commit an `prepare' directory DIR, moving its `incoming' files to
441 ## TARGET. This will choose the correct name for the directory, but
442 ## assumes that it's already correctly laid out. We assume that the
443 ## permissions on this directory are safe (e.g., they've already been fixed
444 ## using `fixperms'). On successful exit, DIR won't exist any more. The
445 ## shell variable `label' is set to the resulting archive name.
447 ## If there's no `incoming' directory, then there's nothing to do. Just
448 ## zap the directory and move on.
449 if [ ! -d "$dir/incoming" ]; then
454 ## Find the datestamp and level numbers to use for this directory. These
455 ## are created before the `incoming' directory, so they ought to exist.
456 read level date time tz <"$dir/meta"
458 ## Find a suitable sequence number for the target. This is rather ugly;
463 for i in "$target"/"$date#$seq".*; do
464 if [ -e "$i" ]; then anyp=t; break; fi
466 case $anyp in nil) break ;; esac
470 ## Move the directory.
471 label="$date#$seq.$level"
472 mv "$dir/incoming" "$target/$label"
475 ## Update the catalogue. Replace an existing dump at the same level.
476 ## Assume that dates are monotonically increasing: add the new entry at the
479 while read lab l d t; do
480 if [ $l -ne $level ]; then echo $label $l $d $t; fi
481 done <"$target"/CATALOGUE
482 echo $level $date $time $tz
483 } >"$target"/CATALOGUE.new
484 mv "$target"/CATALOGUE.new "$target"/CATALOGUE
489 case $# in 0) ;; *) usage_err ;; esac
491 ## Make a `new' directory and start recording our files.
497 ## Copy the blob from the existing metadata.
501 ## Archive the key recovery information.
503 tar cfz $META/new/keys.tgz pub/ recov/
506 ## Copy user and group information.
508 for i in passwd group; do
509 grep -E '^(root|backup|bkp-[[:alnum:]]+):' /etc/$i >$i
513 ## Build the hashes file, and sign it.
519 ## Replace the old metadata.
528 case $# in 0) ;; *) usage_err ;; esac
530 checkdir pub/backup-auth.pub $META/cur
534 ## Report the current date, as ISO8601. Allow an override.
536 case "${forceday+t}" in t) echo "$forceday" ;; *) date +%Y-%m-%d ;; esac
539 defucmd prep ASSET LEVEL \[DATE TIME TZ]
542 2) set -- "$@" $(today) $(date +%H:%M:%S) $(date +%z) ;;
546 asset=$1 level=$2 date=$3 time=$4 tz=$5
548 checkword asset "$asset"
549 checknum level "$level"
550 checkthing date -0-9 date "$date"
551 checkthing time :0-9 time "$time"
552 checkthing timezone -+0-9 tz "$tz"
554 ## Make the host and asset directories if necessary.
556 for i in $host $asset; do
557 if [ ! -d $i ]; then domkdir $i root:root 755; fi
560 if [ ! -d failed ]; then domkdir failed root:root 755; fi
561 for i in . failed; do
562 if [ ! -f $i/CATALOGUE ]; then
564 chown root:root $i/CATALOGUE
565 chmod 644 $i/CATALOGUE
569 ## If an existing dump is in progress then archive it as a failure.
570 if [ -d prepare ]; then
571 if [ -d prepare/incoming ]; then
572 fixperms prepare/incoming root:root 640 755
574 commitdir prepare failed/
577 ## Make a new preparation directory.
578 domkdir prepare root:bkp-$host 755
579 echo $level $date $time $tz >prepare/meta
580 domkdir prepare/incoming bkp-$host:bkp-$host 2775
582 ## Print the directory name.
583 echo $BKP/$host/$asset/prepare/incoming
588 case $# in 1) ;; *) usage_err ;; esac
591 checkword asset "$asset"
593 ## Check that there's something to abort.
595 if [ ! -d $host/$asset/prepare ]; then
596 die "no dump in progress for $host/$asset"
599 ## Just throw it away.
600 rm -rf $host/$asset/prepare
605 case $# in 1) ;; *) usage_err ;; esac
608 checkword asset "$asset"
610 ## Check that there's something to fail.
612 if [ ! -d $host/$asset/prepare ]; then
613 die "no dump in progress for $host/$asset"
616 ## Archive the failure. This shouldn't be used to determine dump levels or
617 ## we'll have gaps when things get sorted out.
619 if [ -d prepare/incoming ]; then
620 fixperms prepare/incoming root:root 640 755
622 commitdir prepare failed/
627 ## Convert an ISO8601 DATE to a Julian Day Number.
629 ## Extract the components of the date and trim leading zeros (which will
630 ## cause things to be interpreted as octal and fail).
631 year=${date%%-*} rest=${date#*-}; month=${rest%%-*} day=${rest#*-}
632 year=${year#0} month=${month#0} day=${day#0}
634 ## The actual calculation: convert a (proleptic) Gregorian calendar date
635 ## into a Julian day number. This is taken from Wikipedia's page
636 ## http://en.wikipedia.org/wiki/Julian_day#Calculation but the commentary
637 ## is mine. The epoch is 4713BC-01-01 (proleptic) Julian, or 4714BC-11-24
638 ## proleptic Gregorian.
640 ## If the MONTH is January or February then set a = 1, otherwise set a = 0.
641 a=$(( (14 - $month)/12 ))
643 ## Compute a year offset relative to 4799BC-03-01. This puts the leap day
644 ## as the very last day in a year, which is very convenient. The offset
645 ## here is sufficient to make all y values positive (within the range of
646 ## the JDN calendar), and is a multiple of 400, which is the Gregorian
648 y=$(( $year + 4800 - $a ))
650 ## Compute the offset month number in that year. These months count from
652 m=$(( $month + 12*$a - 3 ))
654 ## Now for the main event. The (153 m + 2)/5 term is a surprising but
655 ## correct trick for obtaining the number of days in the first m months of
656 ## the (shifted) year). The magic offset 32045 is what you get when you
657 ## plug the proper JDN epoch (year = -4713, month = 11, day = 24) into the
659 jdn=$(( $day + (153*$m + 2)/5 + 365*$y + $y/4 - $y/100 + $y/400 - 32045 ))
665 fulldate=$1 lastdate=$2
666 ## Return the dump level, given that the most recent full dump occurred on
667 ## FULLDATE and the most revent dump of any kind occurred on LASTDATE.
669 ## Actually, we're much more interested in the day difference between these
671 fulljdn=$(julian $fulldate)
672 lastjdn=$(julian $lastdate)
673 now=$(today); nowjdn=$(julian $now)
674 lastday=$(( $lastjdn - $fulljdn ))
675 today=$(( $nowjdn - $fulljdn ))
677 ## If the difference is greater than 512 then we know we should do a full
678 ## dump. (This provides an upper bound for the search below. It should
679 ## never happen in practice, of course.)
680 if [ $(( $today - $lastday )) -ge 512 ]; then echo 0; return; fi
682 ## Now we work out the correct dump level. This will assume that the
683 ## previous dump had a sensible level. If dumps are omitted, then we will
684 ## choose a lower (more comprehensive) dump level than the schedule calls
685 ## for; such an overestimation will mean that we will probably end up
686 ## dumping too much again. This is the right error to make.
688 ## We use a Towers of Hanoi schedule. If we're doing dumps every day, then
689 ## on day n since the last full dump, we work out the dump level as
690 ## follows: write n = 2^s t where t is odd (i.e., s is the number of
691 ## trailing zero bits in the binary representation of n); then the dump
692 ## level on day n is 9 - s. This is enough for 512 days without a full
693 ## dump, and it fails gracefully anyway.
695 ## Now we have to deal with the problem of skipping dumps. Suppose the
696 ## last dump was on day m = 2^u v, and it's now day n = 2^s t. We ought to
697 ## take the lowest dump level of any intervening day, i.e., the dump level
698 ## is 9 - a for the largest a such that there exists b with m < l = 2^a b
699 ## <= n. We claim that such an l is unique. Suppose, to the contrary,
700 ## that m < 2^a b < 2^a b' <= n, with both b and b' odd. Then m < 2^{a+1}
701 ## (b + 1)/2 <= n, contradicting maximality of a.
703 ## How does this help? Observe that n = 2^s t = 2^a b + o, for some o <
704 ## 2^a: if o >= 2^a then 2^a (b + 1) <= n contradicting uniqueness of l.
705 ## Similarly, m = 2^u v = 2^a b - r, for some r <= 2^a (otherwise m <
706 ## 2^a (b - 1), again contradicting uniqueness). Therefore, m and n are
707 ## identical from bit a + 1 onwards, and differ at bit a. In other words,
708 ## a is the position of the most significant set bit in m XOR n.
709 diff=$(( lastday ^ today ))
711 ## We know that the bit position must be less than 16.
713 while [ $diff -gt 1 ]; do
714 xx=$(( $diff >> $t ))
715 if [ $xx -gt 0 ]; then
716 diff=$xx n=$(( $n + $t ))
726 case $# in 1) ;; *) usage_err ;; esac
729 checkword asset "$asset"
731 ## Set the correct directory. If it doesn't exist then we obviously need a
734 full="0 1970-01-01 00:00:00 +0000"
735 if [ ! -d $host/$asset ]; then echo $full; return; fi
738 ## We need the time of the most recent dump of any kind, and the most
739 ## recent level-zero dump.
740 fulldate=none lastdate=none
741 while read label level date time tz; do
742 if [ $level -eq 0 ]; then fulldate=$date; fi
745 case $fulldate in none) echo $full; return ;; esac
746 level=$(dumplevel $fulldate $lastdate)
748 ## Determine the time of the most recent dump of the same or more inclusive
751 while read lab l d t; do
752 if [ $l -le $level ]; then date=$d time=$t; fi
754 echo $level $date $time $tz
757 defucmd hash ASSET FILE HASH
759 case $# in 3) ;; *) usage_err ;; esac
760 asset=$1 file=$2 hash=$3
761 checkword asset "$asset"
762 checkpath file "$file"
763 checkword hash "$hash"
765 cd $BKP/$host/$asset/prepare
767 if [ -f hashes ]; then
769 case "$f" in "$file") die "file \`$file' already hashed" ;; esac
773 echo "$hash $file" >>hashes.new
779 case $# in 1) ;; *) usage_err ;; esac
782 checkword asset "$asset"
784 cd $BKP/$host/$asset/prepare
785 fixperms incoming root:bkp-$host 640 755
788 if [ -f hashes ]; then
789 while read hash name; do
790 if [ ! -f "incoming/$name" ]; then
791 die "precomputed hash for nonexistent or non-file \`$name'"
793 findargs="$findargs ! -path incoming/$name"
795 cp hashes hashes.calc
798 find incoming -type f $findargs -print0 | \
799 xargs -0r sha256sum | \
800 sed 's: incoming/: :' \
802 sort -k2 hashes.calc >incoming/hashes
804 chmod 640 incoming/hashes incoming/hashes.sig
805 chown root:bkp-$host incoming/hashes incoming/hashes.sig
812 defucmd check ASSET LABEL
814 case $# in 2) ;; *) usage_err ;; esac
817 checkword asset "$asset"
818 checkword label "$label"
820 checkdir pub/backup-auth.pub $BKP/$host/$asset/$label
823 defucmd catalogue ASSET
825 case $# in 1) ;; *) usage_err ;; esac
828 checkword asset "$asset"
830 cat $BKP/$host/$asset/CATALOGUE
833 defucmd outdated ASSET
835 case $# in 1) ;; *) usage_err ;; esac
838 checkword asset "$asset"
841 for i in [0-9]*#*.*; do
842 if [ -d "$i" ]; then echo "$i"; fi
847 date=${tag%%#*} level=${tag##*.}
848 if [ $level -le $best ]
856 ###--------------------------------------------------------------------------
859 defcmd test CMD '[ARGS ...]'
860 cmd_test () { "$@"; }
864 host=${USERV_USER#bkp-}
873 while getopts "$opts" opt; do
877 D) forceday=$OPTARG ;;
881 shift $(( $OPTIND - 1 ))
883 case $# in 0) usage_err ;; esac
884 lookupcmd "$1"; shift
887 ###----- That's all, folks --------------------------------------------------