#! /bin/sh ### ### Manage the backup archive structure ### ### (c) 2011 Mark Wooding ### ###----- Licensing notice --------------------------------------------------- ### ### This file is part of the distorted.org.uk backup suite. ### ### distorted-backup is free software; you can redistribute it and/or modify ### it under the terms of the GNU General Public License as published by ### the Free Software Foundation; either version 2 of the License, or ### (at your option) any later version. ### ### distorted-backup is distributed in the hope that it will be useful, ### but WITHOUT ANY WARRANTY; without even the implied warranty of ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ### GNU General Public License for more details. ### ### You should have received a copy of the GNU General Public License along ### with distorted-backup; if not, write to the Free Software Foundation, ### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. set -e ## Configuration and testing. : ${BKP=/mnt/bkp} ${META=/mnt/bkpmeta} : ${KEYS=/etc/keys} case $(id -u) in 0) ;; *) exec userv root bkpadmin "$@" ;; esac ###-------------------------------------------------------------------------- ### Common utilities. quis=${0##*/} version="@VERSION@" moan () { ## Print a complaint to standard error. echo >&2 "$quis: $*" } die () { ## Print a complaint and exit. moan "$*" exit 1 } cleanups="" addcleanup () { cmd=$1 ## Add a cleanup command CMD to the list. case "$cleanups" in ?*) ;; *) trap 'rc=$?; for c in $cleanups; do $c; done; exit $rc' \ EXIT INT TERM ;; esac cleanups=${cleanups+$cleanups }$cmd } rmtmp () { case ${tmpdir+t} in t) rm -rf "$tmpdir" ;; esac } addcleanup rmtmp mktmp () { ## Make a temporary directory and output its name. case "${tmpdir+t}" in t) ;; *) i=0 while :; do r=$(openssl rand -base64 12) tmpdir=${TMPDIR-/tmp}/$quis.$$.$r if mkdir -m700 "$tmpdir" >/dev/null 2>&1; then break; fi case $i in ???) die "failed to create temporary directory" ;; esac i=$(( $i + 1 )) done ;; esac echo "$tmpdir" } ###-------------------------------------------------------------------------- ### Command dispatch. case "${USERV_USER+t}" in t) uservp=t ;; *) uservp=nil ;; esac USAGE="COMMAND [ARGUMENT ...]" cmdname="" cmdargs=$USAGE cmds="" _defcmd () { name=$1; shift; args=$* ## Define a command unconditionally. cmds="${cmds:+$cmds }$name $args" } defcmd () { ## Define a command for privileged users only. case $uservp in nil) _defcmd "$@" ;; esac } defucmd () { ## Define a command usable via userv. _defcmd "$@" } usage () { ## Write a usage message for the current command. echo "usage: $quis${cmdname:+ $cmdname}${cmdargs:+ $cmdargs}" } usage_err () { ## Fail with a usage error. usage >&2 exit 1 } lookupcmd () { cmd=$1 ## Try to loop up the command CMD. while read cmdname cmdargs; do case $cmdname in "$cmd") return ;; esac done <$META/new/blob mv $META/new $META/cur lvcreate -l100%FREE -ncrypt "bkp-$tag" cryptkey | cryptsetup luksFormat \ --cipher=twofish-xts-benbi:sha256 --hash=sha256 \ "/dev/bkp-$tag/crypt" - decrypt "$tag" mkfs -text2 -Lbackup -i1048576 "/dev/mapper/cbkp-$tag" mntcrypt "$tag" } defucmd mount "[TAG]" cmd_mount () { case $# in 0) tag=$(guesstag) check=nil ;; 1) tag=$1 check=t ;; *) usage_err ;; esac if mountpoint -q $BKP; then curtag=$(currenttag) case "$check,$curtag" in "t,$tag") ;; t*) exit 1 ;; esac else mntcrypt "$tag" fi } defcmd umount cmd_umount () { case $# in 0) ;; *) usage_err ;; esac mntp=nil for i in bkp bkpmeta; do if mountpoint -q /mnt/$i; then mntp=t; fi done case $mntp in nil) die "backup volume not mounted" ;; esac umnt } ###-------------------------------------------------------------------------- ### Archive maintenance. checkdir () { key=$1 dir=$2 ## Check a directory which has `hashes' and `hashes.sig' files. if ! seccure-verify -q -i"$dir/hashes" -- \ $(cat "$KEYS/$key") $(cat "$dir/hashes.sig") then die "failed to verify signature for \`$dir'" fi cd "$dir" sha256sum --quiet -c hashes tmpdir=$(mktmp) find . -type f -print | sed 's:^\./::' | sort >"$tmpdir/present" { echo hashes echo hashes.sig sed 's/^[a-f0-9]*[* ] //' hashes } | sort >"$tmpdir/checked" cd "$tmpdir" diff -u checked present } fixperms () { dir=$1 owner=$2 fmode=$3 dmode=$4 ## Fix the directory tree DIR so that everything is owned by OWNER (a ## USER:GROUP pair) and has modes FMODE for files and DMODE for ## directories. ## Change all of the ownerships. This will prevent anyone else from ## changing the permissions on the files. This assumes that chown(1) is ## secure in recursive mode; I've checked that GNU chown seems correct. chown -R $owner "$dir" ## Paranoia: check that we correctly changed all of the files. u=${owner%:*} g=${owner#*:} (cd "$dir"; find . ! \( -user $u -group $g \) -ls) | if read line; then moan "failed to fix permssions on \`$dir'" { echo $line; cat; } | sed 's/^/ /' exit 1 fi ## Now get to work on the file and directory permissions. find "$dir" -type d -print0 | xargs -0r chmod $dmode find "$dir" ! -type d -print0 | xargs -0r chmod $fmode } commitdir () { dir=$1 target=$2 ## Commit an `prepare' directory DIR, moving its `incoming' files to ## TARGET. This will choose the correct name for the directory, but ## assumes that it's already correctly laid out. We assume that the ## permissions on this directory are safe (e.g., they've already been fixed ## using `fixperms'). On successful exit, DIR won't exist any more. The ## shell variable `label' is set to the resulting archive name. ## If there's no `incoming' directory, then there's nothing to do. Just ## zap the directory and move on. if [ ! -d "$dir/incoming" ]; then rm -rf "$dir" return fi ## Find the datestamp and level numbers to use for this directory. These ## are created before the `incoming' directory, so they ought to exist. read level date time tz <"$dir/meta" ## Find a suitable sequence number for the target. This is rather ugly; ## sorry. seq=1 while :; do anyp=nil for i in "$target"/"$date#$seq".*; do if [ -e "$i" ]; then anyp=t; break; fi done case $anyp in nil) break ;; esac seq=$(( $seq + 1 )) done ## Move the directory. label="$date#$seq.$level" mv "$dir/incoming" "$target/$label" rm -rf "$dir" ## Update the catalogue. Replace an existing dump at the same level. ## Assume that dates are monotonically increasing: add the new entry at the ## end. { found=nil while read lab l d t; do if [ $l -ne $level ]; then echo $label $l $d $t; fi done <"$target"/CATALOGUE echo $level $date $time $tz } >"$target"/CATALOGUE.new mv "$target"/CATALOGUE.new "$target"/CATALOGUE } defcmd initmeta cmd_initmeta () { case $# in 0) ;; *) usage_err ;; esac ## Make a `new' directory and start recording our files. cd $META rm -rf new mkdir -m755 new f="" ## Copy the blob from the existing metadata. cp cur/blob new/ f="$f blob" ## Archive the key recovery information. cd $KEYS tar cfz $META/new/keys.tgz pub/ recov/ f="$f keys.tgz" ## Copy user and group information. cd $META/new for i in passwd group; do grep -E '^(root|backup|bkp-[[:alnum:]]+):' /etc/$i >$i done f="$f passwd group" ## Build the hashes file, and sign it. chown root:root $f chmod 644 $f sha256sum $f >hashes sign hashes ## Replace the old metadata. cd $META mv cur old mv new cur rm -rf old } defcmd chkmeta cmd_chkmeta () { case $# in 0) ;; *) usage_err ;; esac checkdir pub/backup-auth.pub $META/cur } today () { ## Report the current date, as ISO8601. Allow an override. case "${forceday+t}" in t) echo "$forceday" ;; *) date +%Y-%m-%d ;; esac } defucmd prep ASSET LEVEL \[DATE TIME TZ] cmd_prep () { case $# in 2) set -- "$@" $(today) $(date +%H:%M:%S) $(date +%z) ;; 5) ;; *) usage_err ;; esac asset=$1 level=$2 date=$3 time=$4 tz=$5 checkhost checkword asset "$asset" checknum level "$level" checkthing date -0-9 date "$date" checkthing time :0-9 time "$time" checkthing timezone -+0-9 tz "$tz" ## Make the host and asset directories if necessary. cd $BKP for i in $host $asset; do if [ ! -d $i ]; then domkdir $i root:root 755; fi cd $i done if [ ! -d failed ]; then domkdir failed root:root 755; fi for i in . failed; do if [ ! -f $i/CATALOGUE ]; then touch $i/CATALOGUE chown root:root $i/CATALOGUE chmod 644 $i/CATALOGUE fi done ## If an existing dump is in progress then archive it as a failure. if [ -d prepare ]; then if [ -d prepare/incoming ]; then fixperms prepare/incoming root:root 640 755 fi commitdir prepare failed/ fi ## Make a new preparation directory. domkdir prepare root:bkp-$host 755 echo $level $date $time $tz >prepare/meta domkdir prepare/incoming bkp-$host:bkp-$host 2775 ## Print the directory name. echo $BKP/$host/$asset/prepare/incoming } defucmd abort ASSET cmd_abort () { case $# in 1) ;; *) usage_err ;; esac asset=$1 checkhost checkword asset "$asset" ## Check that there's something to abort. cd $BKP if [ ! -d $host/$asset/prepare ]; then die "no dump in progress for $host/$asset" fi ## Just throw it away. rm -rf $host/$asset/prepare } defucmd fail ASSET cmd_fail () { case $# in 1) ;; *) usage_err ;; esac asset=$1 checkhost checkword asset "$asset" ## Check that there's something to fail. cd $BKP if [ ! -d $host/$asset/prepare ]; then die "no dump in progress for $host/$asset" fi ## Archive the failure. This shouldn't be used to determine dump levels or ## we'll have gaps when things get sorted out. cd $host/$asset if [ -d prepare/incoming ]; then fixperms prepare/incoming root:root 640 755 fi commitdir prepare failed/ } julian () { date=$1 ## Convert an ISO8601 DATE to a Julian Day Number. ## Extract the components of the date and trim leading zeros (which will ## cause things to be interpreted as octal and fail). year=${date%%-*} rest=${date#*-}; month=${rest%%-*} day=${rest#*-} year=${year#0} month=${month#0} day=${day#0} ## The actual calculation: convert a (proleptic) Gregorian calendar date ## into a Julian day number. This is taken from Wikipedia's page ## http://en.wikipedia.org/wiki/Julian_day#Calculation but the commentary ## is mine. The epoch is 4713BC-01-01 (proleptic) Julian, or 4714BC-11-24 ## proleptic Gregorian. ## If the MONTH is January or February then set a = 1, otherwise set a = 0. a=$(( (14 - $month)/12 )) ## Compute a year offset relative to 4799BC-03-01. This puts the leap day ## as the very last day in a year, which is very convenient. The offset ## here is sufficient to make all y values positive (within the range of ## the JDN calendar), and is a multiple of 400, which is the Gregorian ## cycle length. y=$(( $year + 4800 - $a )) ## Compute the offset month number in that year. These months count from ## zero, not one. m=$(( $month + 12*$a - 3 )) ## Now for the main event. The (153 m + 2)/5 term is a surprising but ## correct trick for obtaining the number of days in the first m months of ## the (shifted) year). The magic offset 32045 is what you get when you ## plug the proper JDN epoch (year = -4713, month = 11, day = 24) into the ## above machinery. jdn=$(( $day + (153*$m + 2)/5 + 365*$y + $y/4 - $y/100 + $y/400 - 32045 )) echo $jdn } dumplevel () { fulldate=$1 lastdate=$2 ## Return the dump level, given that the most recent full dump occurred on ## FULLDATE and the most revent dump of any kind occurred on LASTDATE. ## Actually, we're much more interested in the day difference between these ## two times. fulljdn=$(julian $fulldate) lastjdn=$(julian $lastdate) now=$(today); nowjdn=$(julian $now) lastday=$(( $lastjdn - $fulljdn )) today=$(( $nowjdn - $fulljdn )) ## If the difference is greater than 512 then we know we should do a full ## dump. (This provides an upper bound for the search below. It should ## never happen in practice, of course.) if [ $(( $today - $lastday )) -ge 512 ]; then echo 0; return; fi ## Now we work out the correct dump level. This will assume that the ## previous dump had a sensible level. If dumps are omitted, then we will ## choose a lower (more comprehensive) dump level than the schedule calls ## for; such an overestimation will mean that we will probably end up ## dumping too much again. This is the right error to make. ## ## We use a Towers of Hanoi schedule. If we're doing dumps every day, then ## on day n since the last full dump, we work out the dump level as ## follows: write n = 2^s t where t is odd (i.e., s is the number of ## trailing zero bits in the binary representation of n); then the dump ## level on day n is 9 - s. This is enough for 512 days without a full ## dump, and it fails gracefully anyway. ## ## Now we have to deal with the problem of skipping dumps. Suppose the ## last dump was on day m = 2^u v, and it's now day n = 2^s t. We ought to ## take the lowest dump level of any intervening day, i.e., the dump level ## is 9 - a for the largest a such that there exists b with m < l = 2^a b ## <= n. We claim that such an l is unique. Suppose, to the contrary, ## that m < 2^a b < 2^a b' <= n, with both b and b' odd. Then m < 2^{a+1} ## (b + 1)/2 <= n, contradicting maximality of a. ## ## How does this help? Observe that n = 2^s t = 2^a b + o, for some o < ## 2^a: if o >= 2^a then 2^a (b + 1) <= n contradicting uniqueness of l. ## Similarly, m = 2^u v = 2^a b - r, for some r <= 2^a (otherwise m < ## 2^a (b - 1), again contradicting uniqueness). Therefore, m and n are ## identical from bit a + 1 onwards, and differ at bit a. In other words, ## a is the position of the most significant set bit in m XOR n. diff=$(( lastday ^ today )) ## We know that the bit position must be less than 16. t=16 n=0 while [ $diff -gt 1 ]; do xx=$(( $diff >> $t )) if [ $xx -gt 0 ]; then diff=$xx n=$(( $n + $t )) fi t=$(( $t >> 1 )) done echo $(( 9 - $n )) } defucmd level ASSET cmd_level () { case $# in 1) ;; *) usage_err ;; esac asset=$1 checkhost checkword asset "$asset" ## Set the correct directory. If it doesn't exist then we obviously need a ## level-0 dump. cd $BKP full="0 1970-01-01 00:00:00 +0000" if [ ! -d $host/$asset ]; then echo $full; return; fi cd $host/$asset ## We need the time of the most recent dump of any kind, and the most ## recent level-zero dump. fulldate=none lastdate=none while read label level date time tz; do if [ $level -eq 0 ]; then fulldate=$date; fi lastdate=$date done >hashes.new mv hashes.new hashes } defucmd commit ASSET cmd_commit () { case $# in 1) ;; *) usage_err ;; esac asset=$1 checkhost checkword asset "$asset" cd $BKP/$host/$asset/prepare fixperms incoming root:bkp-$host 640 755 findargs="" if [ -f hashes ]; then while read hash name; do if [ ! -f "incoming/$name" ]; then die "precomputed hash for nonexistent or non-file \`$name'" fi findargs="$findargs ! -path incoming/$name" done >hashes.calc sort -k2 hashes.calc >incoming/hashes sign incoming/hashes chmod 640 incoming/hashes incoming/hashes.sig chown root:bkp-$host incoming/hashes incoming/hashes.sig cd .. commitdir prepare . echo "$label" } defucmd check ASSET LABEL cmd_check () { case $# in 2) ;; *) usage_err ;; esac asset=$1 label=$2 checkhost checkword asset "$asset" checkword label "$label" checkdir pub/backup-auth.pub $BKP/$host/$asset/$label } defucmd catalogue ASSET cmd_catalogue () { case $# in 1) ;; *) usage_err ;; esac asset=$1 checkhost checkword asset "$asset" cat $BKP/$host/$asset/CATALOGUE } defucmd outdated ASSET cmd_outdated () { case $# in 1) ;; *) usage_err ;; esac asset=$1 checkhost checkword asset "$asset" cd $BKP/$host/$asset for i in [0-9]*#*.*; do if [ -d "$i" ]; then echo "$i"; fi done | sort -rn | { best=10 while read tag; do date=${tag%%#*} level=${tag##*.} if [ $level -le $best ] then best=$level else echo "$tag" fi done } } ###-------------------------------------------------------------------------- ### Main program. defcmd test CMD '[ARGS ...]' cmd_test () { "$@"; } case $uservp in t) host=${USERV_USER#bkp-} opts="h" ;; nil) unset host opts="hH:D:" ;; esac while getopts "$opts" opt; do case "$opt" in h) cmd_help; exit ;; H) host=$OPTARG ;; D) forceday=$OPTARG ;; *) usage_err ;; esac done shift $(( $OPTIND - 1 )) case $# in 0) usage_err ;; esac lookupcmd "$1"; shift cmd_$cmdname "$@" ###----- That's all, folks --------------------------------------------------