This is still a little sketchy. Some pieces are lacking documentation.
But the basic code is currently in production use, so it can't be that
bad.
--- /dev/null
+COPYING
+Makefile.in
+aclocal.m4
+autom4te.cache
+config
+configure
--- /dev/null
+config/auto-version
+config/confsubst
+COPYING
--- /dev/null
+;;; -*-emacs-lisp-*-
+
+(setq skel-alist
+ (append
+ '((author . "Mark Wooding")
+ (program . "rsync-backup")
+ (full-title . "the `rsync-backup' program"))
+ skel-alist))
--- /dev/null
+### -*-makefile-*-
+###
+### Build script for rsync-backup
+###
+### (c) 2012 Mark Wooding
+###
+
+###----- Licensing notice ---------------------------------------------------
+###
+### This file is part of the `rsync-backup' program.
+###
+### rsync-backup is free software; you can redistribute it and/or modify
+### it under the terms of the GNU General Public License as published by
+### the Free Software Foundation; either version 2 of the License, or
+### (at your option) any later version.
+###
+### rsync-backup is distributed in the hope that it will be useful,
+### but WITHOUT ANY WARRANTY; without even the implied warranty of
+### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+### GNU General Public License for more details.
+###
+### You should have received a copy of the GNU General Public License
+### along with rsync-backup; if not, write to the Free Software Foundation,
+### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+bin_SCRIPTS =
+sbin_SCRIPTS =
+sbin_PROGRAMS =
+dist_noinst_SCRIPTS =
+dist_man_MANS =
+
+EXTRA_DIST =
+CLEANFILES =
+DISTCLEANFILES =
+
+AM_CFLAGS = $(mLib_CFLAGS)
+
+###--------------------------------------------------------------------------
+### Substitution of configuration data.
+
+confsubst = $(top_srcdir)/config/confsubst
+EXTRA_DIST += config/confsubst
+
+SUBSTVARS = \
+ PACKAGE="$(PACKAGE)" VERSION="$(VERSION)" \
+ BASH="$(BASH)" PYTHON="$(PYTHON)" \
+ sysconfdir="$(sysconfdir)" \
+ mntbkpdir="$(mntbkpdir)" \
+ fshashdir="$(fshashdir)" \
+ logdir="$(logdir)"
+
+V_SUBST = $(V_SUBST_$V)
+V_SUBST_= $(V_SUBST_$(AM_DEFAULT_VERBOSITY))
+V_SUBST_0 = @printf " SUBST %s\n" $@;
+
+SUBST = $(V_SUBST)$(confsubst)
+
+###--------------------------------------------------------------------------
+### Programs and scripts.
+
+sbin_PROGRAMS += rfreezefs
+dist_man_MANS += rfreezefs.8
+rfreezefs_SOURCES = rfreezefs.c
+rfreezefs_LDADD = $(mLib_LIBS)
+
+sbin_SCRIPTS += rsync-backup
+CLEANFILES += rsync-backup
+EXTRA_DIST += rsync-backup.in
+rsync-backup: rsync-backup.in Makefile
+ $(SUBST) >rsync-backup.new \
+ $(srcdir)/rsync-backup.in $(SUBSTVARS) && \
+ chmod +x rsync-backup.new && \
+ mv rsync-backup.new rsync-backup
+
+bin_SCRIPTS += fshash
+CLEANFILES += fshash
+EXTRA_DIST += fshash.in
+fshash: fshash.in Makefile
+ $(SUBST) >fshash.new \
+ $(srcdir)/fshash.in $(SUBSTVARS) && \
+ chmod +x fshash.new && \
+ mv fshash.new fshash
+
+###--------------------------------------------------------------------------
+### Contributed scripts.
+
+## Some simple scripts showing how one might create and manage backup
+## volumes. Assume the distorted.org.uk key management machinery.
+dist_noinst_SCRIPTS += create-backup-volume
+dist_noinst_SCRIPTS += mount-backup-volume
+dist_noinst_SCRIPTS += umount-backup-volume
+
+###--------------------------------------------------------------------------
+### Release machinery.
+
+EXTRA_DIST += config/auto-version
+
+dist-hook:
+ echo $(VERSION) >$(distdir)/RELEASE
+
+###----- That's all, folks --------------------------------------------------
--- /dev/null
+dnl -*-autoconf-*-
+dnl
+dnl Build-time configuration for rsync-backup
+dnl
+dnl (c) 2012 Mark Wooding
+dnl
+
+dnl----- Licensing notice ---------------------------------------------------
+dnl
+dnl This file is part of the `rsync-backup' program.
+dnl
+dnl rsync-backup is free software; you can redistribute it and/or modify
+dnl it under the terms of the GNU General Public License as published by
+dnl the Free Software Foundation; either version 2 of the License, or
+dnl (at your option) any later version.
+dnl
+dnl rsync-backup is distributed in the hope that it will be useful,
+dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
+dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+dnl GNU General Public License for more details.
+dnl
+dnl You should have received a copy of the GNU General Public License
+dnl along with rsync-backup; if not, write to the Free Software Foundation,
+dnl Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+dnl--------------------------------------------------------------------------
+dnl Initialization.
+
+mdw_AUTO_VERSION
+AC_INIT([rsync-backup], AUTO_VERSION, [mdw@distorted.org.uk])
+AC_CONFIG_SRCDIR([rsync-backup.in])
+AC_CONFIG_AUX_DIR([config])
+AM_INIT_AUTOMAKE([foreign])
+mdw_SILENT_RULES
+
+dnl--------------------------------------------------------------------------
+dnl Paths.
+
+AC_ARG_WITH([mntbkpdir],
+ AS_HELP_STRING([--with-mntbkpdir], [location of the backup mount tree]),
+ [mntbkpdir=$withval],
+ [mntbkpdir=/mnt/bkp])
+AC_SUBST([mntbkpdir])
+
+AC_ARG_WITH([fshashdir],
+ AS_HELP_STRING([--with-fshashdir], [location for fshash cache files]),
+ [fshashdir=$withval],
+ [fshashdir=$localstatedir/cache/fshash])
+AC_SUBST([fshashdir])
+
+AC_ARG_WITH([logdir],
+ AS_HELP_STRING([--with-logdir], [location for logfiles]),
+ [logdir=$withval],
+ [logdir=$localstatedir/log/bkp])
+AC_SUBST([logdir])
+
+dnl--------------------------------------------------------------------------
+dnl Programming environments.
+
+dnl C compilers and libraries.
+AC_PROG_CC
+AX_CFLAGS_WARN_ALL
+PKG_CHECK_MODULES([mLib], [mLib >= 2.1.0])
+
+dnl Bourne-Again Shell.
+AC_PATH_PROG([BASH], [bash])
+
+dnl Python.
+AC_PATH_PROG([PYTHON], [python])
+AX_PROG_PYTHON_VERSION([2.5],,
+ [AC_MSG_ERROR([Failed to find suitable Python.])])
+
+dnl--------------------------------------------------------------------------
+dnl Output.
+
+AC_CONFIG_FILES([Makefile])
+AC_OUTPUT
+
+dnl----- That's all, folks --------------------------------------------------
--- /dev/null
+#! /bin/sh -ex
+
+: ${vgtag=@backup} ${vgprefix=vg-backup-}
+: ${mntbkpdir=/mnt/bkp}
+: ${STOREDIR=$mntbkpdir/store} ${METADIR=$mntbkpdir/meta}
+: ${RANDOM=/dev/random}
+
+case $# in
+ 2) tag=$1 pv=$2 ;;
+ *) echo >&2 "usage: $0 TAG PV" ;;
+esac
+vg=$vgprefix$tag
+
+vgcreate --addtag $vgtag $vg $pv
+
+lvcreate -L64M -nmeta $vg
+mkfs -text3 -Lmeta /dev/$vg/meta
+mount /dev/$vg/meta $METADIR
+
+echo $tag >$METADIR/volume
+dd if=$RANDOM bs=1 count=512 |
+ cryptop encrypt backup >$METADIR/crypt.blob
+
+lvcreate -l100%FREE -ncrypt $vg
+cryptop decrypt backup <$METADIR/crypt.blob |
+ cryptsetup luksFormat \
+ --cipher=twofish-xts-benbi:sha256 \
+ --hash=sha256 --key-size=256 \
+ /dev/$vg/crypt -
+
+cryptop decrypt backup <$METADIR/crypt.blob |
+ cryptsetup luksOpen --key-file=- /dev/$vg/crypt cbackup
+
+mkfs -text3 -Lbackup /dev/mapper/cbackup
+
+mount /dev/mapper/cbackup $STOREDIR
+touch $STOREDIR/.rsync-backup-store
--- /dev/null
+#! @PYTHON@
+###
+### Efficiently construct canonical digests of filesystems
+###
+### (c) 2012 Mark Wooding
+###
+
+###----- Licensing notice ---------------------------------------------------
+###
+### This file is part of the `rsync-backup' program.
+###
+### rsync-backup is free software; you can redistribute it and/or modify
+### it under the terms of the GNU General Public License as published by
+### the Free Software Foundation; either version 2 of the License, or
+### (at your option) any later version.
+###
+### rsync-backup is distributed in the hope that it will be useful,
+### but WITHOUT ANY WARRANTY; without even the implied warranty of
+### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+### GNU General Public License for more details.
+###
+### You should have received a copy of the GNU General Public License
+### along with rsync-backup; if not, write to the Free Software Foundation,
+### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+from sys import argv, exit, stdin, stdout, stderr
+import os as OS
+import re as RX
+import time as T
+import stat as ST
+import optparse as OP
+import hashlib as H
+import sqlite3 as DB
+import zlib as Z
+
+PACKAGE = '@PACKAGE@'
+VERSION = '@VERSION@'
+
+###--------------------------------------------------------------------------
+### Utilities.
+
+QUIS = OS.path.basename(argv[0])
+
+def moan(msg):
+ stderr.write('%s: %s\n' % (QUIS, msg))
+
+def die(msg, rc = 1):
+ moan(msg)
+ exit(rc)
+
+SYSERR = 0
+def syserr(msg):
+ global SYSERR
+ moan(msg)
+ SYSERR += 1
+
+###--------------------------------------------------------------------------
+### File system enumeration.
+
+class FileInfo (object):
+ def __init__(me, file, st = None):
+ me.name = file
+ if st:
+ me.st = st
+ me.err = None
+ else:
+ try:
+ me.st = OS.lstat(file)
+ me.err = None
+ except OSError, err:
+ me.st = None
+ me.err = err
+
+def enum_walk(file, func):
+
+ def dirents(name):
+ try:
+ return OS.listdir(name)
+ except OSError, err:
+ syserr("failed to read directory `%s': %s" % (name, err.strerror))
+ return []
+
+ def dir(ee, dev):
+ ff = []
+ dd = []
+ for e in ee:
+ fi = FileInfo(e)
+ if fi.st and fi.st.st_dev != dev: pass
+ if fi.st and ST.S_ISDIR(fi.st.st_mode): dd.append(fi)
+ else: ff.append(fi)
+ ff.sort(key = lambda fi: fi.name)
+ dd.sort(key = lambda fi: fi.name + '/')
+ for f in ff:
+ func(f)
+ for d in dd:
+ if d.st.st_dev == dev:
+ func(d)
+ dir([OS.path.join(d.name, e) for e in dirents(d.name)], dev)
+
+ if file.endswith('/'):
+ OS.chdir(file)
+ fi = FileInfo('.')
+ func(fi)
+ dir(dirents('.'), fi.st.st_dev)
+ else:
+ fi = FileInfo(file)
+ func(fi)
+ if fi.st and ST.S_ISDIR(fi.st.st_mode):
+ dir([OS.path.join(fi.name, e) for e in dirents(fi.name)],
+ fi.st.st_dev)
+
+def enum_find0(f, func):
+ tail = ""
+ while True:
+ buf = f.read(8192)
+ last = len(buf) == 0
+ names = (tail + buf).split('\0')
+ tail = names.pop()
+ for n in names:
+ func(FileInfo(n))
+ if last:
+ break
+ if len(tail):
+ moan("ignored trailing junk after last filename")
+
+RX_RSYNCESC = RX.compile(r'\\ \# ([0-7]{3})', RX.VERBOSE)
+def enum_rsync(f, func):
+
+ ## The format is a little fiddly. Each line consists of PERMS SIZE DATE
+ ## TIME NAME, separated by runs of whitespace, but the NAME starts exactly
+ ## one space character after the TIME and may begin with a space.
+ ## Sequences of the form `\#OOO' where OOO are three octal digits, stand
+ ## for a byte with that value. Newlines and backslashes which would be
+ ## ambiguous are converted into this form; all other characters are
+ ## literal.
+ ##
+ ## We ignore the stat information and retrieve it ourselves, because it's
+ ## incomplete. Hopefully the dcache is still warm.
+
+ for line in f:
+ if line.endswith('\n'): line = line[:-1]
+
+ ## Extract the escaped name.
+ ff = line.split(None, 3)
+ if len(ff) != 4:
+ syserr("ignoring invalid line from rsync: `%s'" % line)
+ continue
+ tail = ff[3]
+ try:
+ spc = tail.index(' ')
+ except ValueError:
+ syserr("ignoring invalid line from rsync: `%s'" % line)
+ continue
+ name = tail[spc + 1:]
+
+ ## Now translate escape sequences.
+ name = RX_RSYNCESC.sub(lambda m: chr(int(m.group(1), 8)), name)
+
+ ## Call the client.
+ try:
+ fi = FileInfo(name)
+ except OSError, err:
+ syserr("failed to stat `%s': %s" % (name, err.strerror))
+ continue
+ func(fi)
+
+###--------------------------------------------------------------------------
+### The hash cache.
+
+class HashCache (object):
+
+ VERSION = 0
+ BUFSZ = 128*1024
+
+ INIT = [
+ """CREATE TABLE meta (
+ version INTEGER NOT NULL,
+ hash TEXT NOT NULL
+ );""",
+ """CREATE TABLE hash (
+ ino INTEGER PRIMARY KEY,
+ mtime INTEGER NOT NULL,
+ ctime INTEGER NOT NULL,
+ size INTEGER NOT NULL,
+ hash TEXT NOT NULL,
+ seen BOOLEAN NOT NULL DEFAULT TRUE
+ );""",
+ """PRAGMA journal_mode = WAL;"""
+ ]
+
+ def __init__(me, file, hash = None):
+
+ if file is None:
+
+ ## We're going this alone, with no cache.
+ db = None
+ if hash is None:
+ die("no hash specified and no database cache to read from")
+ else:
+
+ ## Connect to the database.
+ db = DB.connect(file)
+ db.text_factory = str
+
+ ## See whether we can understand the cache database.
+ c = db.cursor()
+ v = h = None
+ try:
+ c.execute('SELECT version, hash FROM meta')
+ v, h = c.fetchone()
+ if c.fetchone() is not None:
+ die("cache database corrupt: meta table has mutliple rows")
+ except (DB.Error, TypeError):
+ pass
+
+ ## If that didn't work, we'd better clear the thing and start again.
+ ## But only if we know how to initialize it.
+ if v != me.VERSION:
+
+ ## Explain the situation.
+ moan("cache version %s not understood" % v)
+ if hash is None:
+ if h is None:
+ die("can't initialize cache: no hash function set")
+ else:
+ hash = h
+ try:
+ H.new(hash)
+ except Exception:
+ die("unknown hash function `%s'" % hash)
+
+ ## Drop old things.
+ c.execute('SELECT type, name FROM sqlite_master')
+ for type, name in c.fetchall():
+ c.execute('DROP %s IF EXISTS %s' % (type, name))
+
+ ## Now we're ready to go.
+ for stmt in me.INIT:
+ c.execute(stmt)
+ c.execute('INSERT INTO meta VALUES (?, ?)', [me.VERSION, hash])
+ db.commit()
+
+ ## Check the hash function if necessary.
+ if hash is None:
+ hash = h
+ elif h is not None and h != hash:
+ die("hash mismatch: cache uses %s but %s requested" % (h, hash))
+
+ ## All done.
+ me.hash = hash
+ me._db = db
+ me._pend = 0
+
+ def hashfile(me, fi):
+
+ ## If this isn't a proper file then don't try to hash it.
+ if fi.err or not ST.S_ISREG(fi.st.st_mode):
+ return None
+
+ ## See whether there's a valid entry in the cache.
+ if me._db:
+ c = me._db.cursor()
+ c.execute(
+ 'SELECT mtime, size, hash, seen FROM hash WHERE ino = ?;',
+ [fi.st.st_ino])
+ r = c.fetchone()
+ if r is not None:
+ mt, sz, h, s = r
+ if mt == fi.st.st_mtime and \
+ sz == fi.st.st_size:
+ if not s:
+ c.execute('UPDATE hash SET seen = 1 WHERE ino = ?',
+ [fi.st.st_ino])
+ me._update()
+ return h
+
+ ## Hash the file. Beware raciness: update the file information from the
+ ## open descriptor, but set the size from what we actually read.
+ h = H.new(me.hash)
+ try:
+ with open(fi.name, 'rb') as f:
+ sz = 0
+ while True:
+ buf = f.read(me.BUFSZ)
+ if len(buf) == 0:
+ break
+ sz += len(buf)
+ h.update(buf)
+ fi.st = OS.fstat(f.fileno())
+ ##fi.st.st_size = sz
+ hash = h.digest()
+ except (OSError, IOError), err:
+ fi.st = None
+ fi.err = err
+ return None
+ hash = hash.encode('hex')
+
+ ## Insert a record into the database.
+ if me._db:
+ c.execute("""
+ INSERT OR REPLACE INTO hash
+ (ino, mtime, ctime, size, hash, seen)
+ VALUES
+ (?, ?, ?, ?, ?, 1);
+ """, [fi.st.st_ino,
+ fi.st.st_mtime,
+ fi.st.st_ctime,
+ fi.st.st_size,
+ hash])
+ me._update()
+
+ ## Done.
+ return hash
+
+ def _update(me):
+ me._pend += 1
+ if me._pend >= 1024:
+ me.flush()
+
+ def flush(me):
+ if me._db:
+ me._db.commit()
+ me._pend = 0
+
+ def need_db(me):
+ if not me._db:
+ die("no cache database")
+
+ def reset(me):
+ me.need_db()
+ c = me._db.cursor()
+ c.execute('UPDATE hash SET seen = 0 WHERE seen')
+ me.flush()
+
+ def prune(me):
+ me.need_db()
+ c = me._db.cursor()
+ c.execute('DELETE FROM hash WHERE NOT seen')
+ me.flush()
+
+###--------------------------------------------------------------------------
+### Printing output.
+
+class GenericFormatter (object):
+ def __init__(me, fi):
+ me.fi = fi
+ def _fmt_time(me, t):
+ tm = T.gmtime(t)
+ return T.strftime('%Y-%m-%dT%H:%M:%SZ', tm)
+ def _enc_name(me, n):
+ return n.encode('string_escape')
+ def name(me):
+ return me._enc_name(me.fi.name)
+ def info(me):
+ return me.TYPE
+ def mode(me):
+ return '%06o' % me.fi.st.st_mode
+ def size(me):
+ return me.fi.st.st_size
+ def mtime(me):
+ return me._fmt_time(me.fi.st.st_mtime)
+ def owner(me):
+ return '%5d:%d' % (me.fi.st.st_uid, me.fi.st.st_gid)
+
+class ErrorFormatter (GenericFormatter):
+ def info(me):
+ return 'E%d %s' % (me.fi.err.errno, me.fi.err.strerror)
+ def error(me): return 'error'
+ mode = size = mtime = owner = error
+
+class SocketFormatter (GenericFormatter):
+ TYPE = 'socket'
+class PipeFormatter (GenericFormatter):
+ TYPE = 'fifo'
+
+class LinkFormatter (GenericFormatter):
+ TYPE = 'symbolic-link'
+ def name(me):
+ n = GenericFormatter.name(me)
+ try:
+ d = OS.readlink(me.fi.name)
+ return '%s -> %s' % (n, me._enc_name(d))
+ except OSError, err:
+ return '%s -> <E%d %s>' % (n, err.errno, err.strerror)
+
+class DirectoryFormatter (GenericFormatter):
+ TYPE = 'directory'
+ def name(me): return GenericFormatter.name(me) + '/'
+ def size(me): return 'dir'
+
+class DeviceFormatter (GenericFormatter):
+ def info(me):
+ return '%s %d:%d' % (me.TYPE,
+ OS.major(me.fi.st.st_rdev),
+ OS.minor(me.fi.st.st_rdev))
+class BlockDeviceFormatter (DeviceFormatter):
+ TYPE = 'block-device'
+class CharDeviceFormatter (DeviceFormatter):
+ TYPE = 'character-device'
+
+class FileFormatter (GenericFormatter):
+ TYPE = 'regular-file'
+
+class Reporter (object):
+
+ TYMAP = {
+ ST.S_IFSOCK: SocketFormatter,
+ ST.S_IFDIR: DirectoryFormatter,
+ ST.S_IFLNK: LinkFormatter,
+ ST.S_IFREG: FileFormatter,
+ ST.S_IFBLK: BlockDeviceFormatter,
+ ST.S_IFCHR: CharDeviceFormatter,
+ ST.S_IFIFO: PipeFormatter,
+ }
+
+ def __init__(me, db):
+ me._inomap = {}
+ me._vinomap = {}
+ me._db = db
+ me._hsz = int(H.new(db.hash).digest_size)
+
+ def file(me, fi):
+ h = me._db.hashfile(fi)
+ if fi.err:
+ fmt = ErrorFormatter(fi)
+ vino = 'error'
+ else:
+ fmt = me.TYMAP[ST.S_IFMT(fi.st.st_mode)](fi)
+ inoidx = fi.st.st_dev, fi.st.st_ino
+ try:
+ vino = me._inomap[inoidx]
+ except KeyError:
+ suffix = ''
+ seq = 0
+ while True:
+ vino = '%08x' % (Z.crc32(fi.name + suffix) & 0xffffffff)
+ if vino not in me._vinomap: break
+ suffix = '\0%d' % seq
+ seq += 1
+ me._inomap[inoidx] = vino
+ if h: info = h
+ else: info = '[%-*s]' % (2*me._hsz - 2, fmt.info())
+ print '%s %8s %6s %-12s %-20s %20s %s' % (
+ info, vino, fmt.mode(), fmt.owner(),
+ fmt.mtime(), fmt.size(), fmt.name())
+
+###--------------------------------------------------------------------------
+### Main program.
+
+FMTMAP = {
+ 'rsync': lambda f: enum_rsync(stdin, f),
+ 'find0': lambda f: enum_find0(stdin, f)
+}
+op = OP.OptionParser(
+ usage = '%prog [-a] [-c CACHE] [-f FORMAT] [-H HASH] [FILE ...]',
+ version = '%%prog, version %s' % VERSION,
+ description = '''\
+Print a digest of a filesystem (or a collection of specified files) to
+standard output. The idea is that the digest should be mostly /complete/
+(i.e., any `interesting\' change to the filesystem results in a different
+digest) and /canonical/ (i.e., identical filesystem contents result in
+identical output).
+''')
+
+for short, long, props in [
+ ('-a', '--all', { 'action': 'store_true', 'dest': 'all',
+ 'help': 'clear cache of all files not seen' }),
+ ('-c', '--cache', { 'dest': 'cache', 'metavar': 'FILE',
+ 'help': 'use FILE as a cache for file hashes' }),
+ ('-f', '--files', { 'dest': 'files', 'metavar': 'FORMAT',
+ 'type': 'choice', 'choices': FMTMAP.keys(),
+ 'help': 'read files to report in the given FORMAT' }),
+ ('-H', '--hash', { 'dest': 'hash', 'metavar': 'HASH',
+ ##'type': 'choice', 'choices': H.algorithms,
+ 'help': 'use HASH as the hash function' })]:
+ op.add_option(short, long, **props)
+opts, args = op.parse_args(argv)
+
+if not opts.files and len(args) <= 1:
+ die("no filename sources: nothing to do")
+db = HashCache(opts.cache, opts.hash)
+if opts.all:
+ db.reset()
+rep = Reporter(db)
+if opts.files:
+ FMTMAP[opts.files](rep.file)
+for dir in args[1:]:
+ enum_walk(dir, rep.file)
+if opts.all:
+ db.prune()
+db.flush()
+
+###----- That's all, folks --------------------------------------------------
--- /dev/null
+#! /bin/sh -ex
+
+: ${vgtag=@backup} ${vgprefix=vg-backup-}
+: ${mntbkpdir=/mnt/bkp}
+: ${STOREDIR=$mntbkpdir/store} ${METADIR=$mntbkpdir/meta}
+
+vgs=$(vgs --noheadings -oname)
+found=nil
+for vg in $vgs; do
+ case "$found,$vg" in
+ nil,$vgprefix*)
+ found=t
+ tag=${vg#$vgprefix}
+ ;;
+ t,$vgprefix*)
+ echo >&2 "$0: multiple backup volumes attached"
+ exit 1
+ ;;
+ esac
+done
+case $found in
+ nil) echo >&2 "$0: no backup volumes attached"; exit 1 ;;
+esac
+vg=$vgprefix$tag
+
+vgchange -ay $vg
+mount /dev/$vg/meta $METADIR
+
+cryptop decrypt backup <$METADIR/crypt.blob |
+ cryptsetup luksOpen --key-file=- /dev/$vg/crypt cbackup
+mount /dev/mapper/cbackup $STOREDIR
--- /dev/null
+.TH rfreezefs 8 "October 2011" "rsync-backup"
+.SH NAME
+rfreezefs \- freeze a filesystem safely
+.SH SYNOPSIS
+.B rfreezefs
+.RB [ \-n ]
+.RB [ \-a
+.IR address ]
+.RB [ \-p
+.IR loport [\fB\- hiport ]]
+.I filesystem
+\&...
+.SH DESCRIPTION
+The
+.B rfreezefs
+program freezes one or more mounted filesystems for a period of time,
+and then thaws them. For more detail on what this means, why you'd want
+to, and how you might go about using
+.B rfreezefs
+to do it, see below.
+.PP
+The following command-line options are recognized.
+.TP
+.B "\-h, \-\-help"
+Writes a help message to standard output, and exits with status 0.
+.TP
+.B "\-v, \-\-version"
+Writes the version number to standard output, and exits with status 0.
+.TP
+.B "\-u, \-\-usage"
+Writes a command-line usage synopsis to standard output, and exits with
+status 0.
+.TP
+.BI "\-a, \-\-address=" address
+Listen only for incoming connections to the given
+.IR address .
+The default is to listen for connections to any local address.
+.TP
+.B "\-n, \-\-not-really"
+Don't actually freeze or thaw any filesystems; instead, write messages
+to standard error explaining what would be done.
+.TP
+.BI "\-p, \-\-port-range=" loport\fR[ \- hiport \fR]]
+Listen for incoming connections on a port between
+.I loport
+and
+.IR hiport .
+If
+.I hiport
+is omitted, listen for connections only on
+.IR loport .
+The default is to allow the kernel a free choice of local port number.
+.PP
+The
+.I filesystem
+arguments name the filesystems to be frozen. There must be at least one
+such argument. It's conventional to name the filesystem mount points,
+though actually any file or directory in the filesystem will do. The
+files are opened read-only.
+.PP
+The
+.B rfreezefs
+program starts, parses its command line, opens the named files, and
+creates a listening TCP socket according to the command-line options.
+It then prints a sequence of lines to standard output, which may have
+one of the following forms.
+.TP
+.BI "PORT " port
+Announces the TCP
+.I port
+number on which that
+.B rfreezefs
+is listening for incoming connections.
+.TP
+.BI "TOKEN " label " " token
+Declares a `token': a randomly chosen string which is to be used in the
+network connection. The token's value is
+.IR token :
+token values are a sequence of non-whitespace printable ASCII
+characters, but their precise structure is not specified. The token
+value will have the meaning given by the
+.IR label ,
+which is one of the token labels described below.
+.TP
+.B READY
+Marks the end of the lines and announces that
+.B rfreezefs
+is ready to accept connections.
+.PP
+These lines may be sent in any order, except that
+.B READY
+is always last. There may be many
+.B TOKEN
+lines.
+.PP
+Network communications use a simple plain-text line-oriented protocol.
+Each line consists of a token, optionally followed by a carriage return
+(code 13), followed by a linefeed (code 10). No other whitespace is
+permitted. The tokens allowed are precisely those announced in the
+.B TOKEN
+lines written to
+.BR rfreezefs 's
+standard output. Furthermore, only certain tokens are valid at
+particular points in the protocol. For reference, the token labels, and
+the meanings of the corresponding tokens, are as follows.
+.TP
+.B FREEZE
+Sent by a client to freeze the filesystems. This must be the first
+token transmitted by the client. On receipt,
+.B rfreezefs
+will close its listening socket and any other client connections. It
+will then freeze the filesystems.
+.TP
+.B FROZEN
+Sent by
+.B rfreezefs
+to indicate successful freezing of the filesystem.
+.TP
+.B KEEPALIVE
+Sent periodically by the client to prevent filesystems being thawed due
+to a timeout. No explicit acknowledgement is sent.
+.TP
+.B THAW
+Sent by the client to request thawing of the filesystems.
+.TP
+.B THAWED
+Sent by
+.B rfreezefs to indicate successful thawing of the filesystems in response to
+.BR THAW .
+.PP
+The high-level structure of the protocol is then as follows: the client
+sends
+.BR FREEZE ;
+the server freezes and responds with
+.BR FROZEN ;
+the client optionally sends
+.B KEEPALIVE
+at intervals; the client finally sends
+.BR THAW ;
+and the server responds with
+.B THAWED
+and drops the connection.
+.PP
+If sufficient time passes without
+.B rfreezefs
+receiving either
+.B THAW
+or
+.B KEEPALIVE
+tokens, or an invalid token is received, or it receives one of a number
+of signals, currently
+.BR SIGINT ,
+.BR SIGQUIT ,
+.BR SIGTERM ,
+.BR SIGHUP ,
+.BR SIGALRM ,
+.BR SIGILL ,
+.BR SIGSEGV ,
+.BR SIGBUS ,
+.BR SIGFPE ,
+or
+.BR SIGABRT ,
+.B rfreezefs
+will thaw the filesystems and report a failure.
+.PP
+Diagnostics are reported to standard error. Exit statuses have specific
+meanings:
+.TP
+.B 0
+Successful completion. Filesystems were frozen and thawed as required.
+.TP
+.B 1
+Problem with command-line arguments. No filesystems were frozen.
+.TP
+.B 2
+Environmental problem, typically a system call failure: e.g., a file
+failed to open, or there was a problem with the network communications.
+Either no filesystems were frozen, or all filesystems were successfully
+thawed again.
+.TP
+.B 3
+Timeout or invalid data. Either no connections containing the cookie
+were made in time, or no data was received for a long enough period
+after the filesystems were frozen, or an invalid token was received. In
+the first case, no filesystems were frozen; in the other two cases, the
+filesystems were successfully thawed.
+.TP
+.B 4
+Crash. The
+.B rfreezefs
+program received a fatal signal after it had started to freeze
+filesystems. Under these circumstances, it thaws the filesystems,
+removes the signal handler, and sends itself the signal again, but if
+that doesn't work then
+.B rfreezefs
+exits with this status code. All frozen filesystems were successfully
+thawed again.
+.TP
+.B 112
+Failure during filesystem thaw (mnemonic: European emergency number).
+Some filesystems
+.I failed
+to thaw, and are still frozen. You might have some joy with
+.BR SysRq-j ,
+though in the author's experience that doesn't work and you'll probably
+have to reboot. At least your filesystems are consistent...
+.SS Background
+When frozen, a filesystem's backing block device is put in a consistent
+state (as if unmounted), and write operations to it are delayed until
+the filesystem is thawed again. In the meantime, it's possible to take
+a consistent snapshot of the block device. When a filesystem is
+directly mounted on an LVM logical volume, the kernel detects this
+situation and automatically freezes the filesystem while the snapshot is
+being prepared. If the logical volume and filesystem are on separate
+hosts, though, the filesystem must be frozen manually, which is why
+.B rfreezefs
+is useful.
+.PP
+The idea is to run
+.B rfreezefs
+using
+.BR ssh (1)
+or
+.BR userv (1),
+or some other means of acquiring the necessary privilege level. You
+read the port number and tokens, connect to the socket, and send the
+.B FREEZE
+token followed by a newline. You now wait to receive the
+.B FROZEN
+token from
+.BR rfreezefs .
+Once you have received this, the filesystems are frozen: you can safely
+take snapshots. If this will take an extended amount of time, you
+should send
+.B KEEPALIVE
+tokens to the connection at intervals in order to prevent
+.B rfreezefs
+from timing out and thawing the filesystems (but see the
+.B "Security notes"
+below). When your snapshot is prepared, sent the
+.B THAW
+token, and wait for the
+.B THAWED
+token in response. If this is received, the snapshot was completed
+successfully and the filesystems are properly thawed again. If you
+don't receive the
+.B THAWED
+token then something bad might have happened (e.g., the filesystem might
+have been prematurely thawed) and the snapshot is suspect. If the exit
+status is 112 then at least one filesystem is still frozen and some
+emergency action is needed. If you can't retrieve the exit status then
+it's possible that your transport is blocked for trying to write to the
+frozen filesystem (this especially likely if
+.B /
+or
+.B /var
+is frozen) and you should react as if the status was 112.
+.SS Security notes
+The
+.B rfreezefs
+program uses randomly chosen tokens to form a simple code which is
+revealed to the caller. It is assumed that this information is kept
+secret from adversaries, e.g., by ensuring that it is only transmitted
+over local pipes (as used by
+.BR userv (1))
+and/or secure network transports such as SSH (see
+.BR ssh (1)).
+The author believes that the worst possible outcome is that the host
+wedges up because an important filesystem is frozen, and
+.B rfreezefs
+therefore strives to prevent that from happening. In particular,
+cryptographic transport implementations such as SSH may attempt to log
+messages to frozen filesystems or otherwise wedge themselves:
+.B rfreezefs
+deliberately uses only kernel-implemented transports for its
+communication needs once the filesystems are frozen.
+.PP
+Most of the tokens are used at most once in the protocol. In
+particular, the
+.B FROZEN
+token can't be sent by an adversary in advance of the filesystem being
+frozen, since (under the assumption that the tokens are kept secret) it
+only revealed in the clear after a successful freeze. Similarly, the
+.B THAWED
+token is only transmitted if the filesystems are thawed as a result of a
+.B THAW
+request (rather than a dropped connection, timeout, or some other
+problem). If the client only sends the
+.B THAW
+request once its snapshot is complete, then a
+.B THAWED
+response indicates that the filesystems remained frozen until the
+snapshot was indeed completed and therefore the snapshot is consistent.
+.PP
+The exception is the
+.B KEEPALIVE
+token, which may be sent repeatedly. After it is first revealed, an
+adversary can hijack the connection and replay the
+.B KEEPALIVE
+token to keep the filesystems frozen indefinitely. You can recover from
+this by severing the connection somehow, or by sending
+.B rfreezefs
+a signal. It is therefore recommended that
+.B KEEPALIVE
+tokens not be sent unless necessary. The timeout is currently set to
+60s, which ought to be adequate for most snapshot mechanisms.
+.SH BUGS
+There ought to be a better one-time-token protocol for keepalives. I
+want to keep cryptography out of this program, though.
+.SH SEE ALSO
+.BR fsfreeze (8),
+.BR random (4),
+.BR lvm (8),
+.BR ssh (1),
+.BR userv (1).
+.SH AUTHOR
+Mark Wooding, <mdw@distorted.org.uk>
--- /dev/null
+/* -*-c-*-
+ *
+ * Freeze a file system under remote control
+ *
+ * (c) 2012 Mark Wooding
+ */
+
+/*----- Licensing notice --------------------------------------------------*
+ *
+ * This file is part of the `rsync-backup' program.
+ *
+ * rsync-backup is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * rsync-backup is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with rsync-backup; if not, write to the Free Software Foundation,
+ * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+/*----- Header files ------------------------------------------------------*/
+
+#include <assert.h>
+#include <errno.h>
+#include <limits.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <time.h>
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/select.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+
+#include <linux/fs.h>
+
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <netdb.h>
+
+#include <mLib/alloc.h>
+#include <mLib/dstr.h>
+#include <mLib/base64.h>
+#include <mLib/fdflags.h>
+#include <mLib/mdwopt.h>
+#include <mLib/quis.h>
+#include <mLib/report.h>
+#include <mLib/sub.h>
+#include <mLib/tv.h>
+
+/*----- Magic constants ---------------------------------------------------*/
+
+#define COOKIESZ 16 /* Size of authentication cookie */
+#define TO_CONNECT 30 /* Timeout for incoming connection */
+#define TO_KEEPALIVE 60 /* Timeout between keepalives */
+
+/*----- Utility functions -------------------------------------------------*/
+
+static int getuint(const char *p, const char *q)
+{
+ unsigned long i;
+ int e = errno;
+ char *qq;
+
+ if (!q) q = p + strlen(p);
+ errno = 0;
+ i = strtoul(p, &qq, 0);
+ if (errno || qq < q || i > INT_MAX)
+ die(1, "invalid integer `%s'", p);
+ errno = e;
+ return ((int)i);
+}
+
+#ifdef DEBUG
+# define D(x) x
+#else
+# define D(x)
+#endif
+
+/*----- Token management --------------------------------------------------*/
+
+struct token {
+ const char *label;
+ char tok[(COOKIESZ + 2)*4/3 + 1];
+};
+
+#define TOKENS(_) \
+ _(FREEZE) \
+ _(FROZEN) \
+ _(KEEPALIVE) \
+ _(THAW) \
+ _(THAWED)
+
+enum {
+#define ENUM(tok) T_##tok,
+ TOKENS(ENUM)
+#undef ENUM
+ T_LIMIT
+};
+
+enum {
+#define MASK(tok) TF_##tok = 1u << T_##tok,
+ TOKENS(MASK)
+#undef ENUM
+ TF_ALL = (1u << T_LIMIT) - 1u
+};
+
+static struct token toktab[] = {
+#define INIT(tok) { #tok },
+ TOKENS(INIT)
+#undef INIT
+ { 0 }
+};
+
+static void inittoks(void)
+{
+ static struct token *t, *tt;
+ unsigned char buf[COOKIESZ];
+ int fd;
+ ssize_t n;
+ base64_ctx bc;
+ dstr d = DSTR_INIT;
+
+ if ((fd = open("/dev/urandom", O_RDONLY)) < 0)
+ die(2, "open (urandom): %s", strerror(errno));
+
+ for (t = toktab; t->label; t++) {
+ again:
+ n = read(fd, buf, COOKIESZ);
+ if (n < 0) die(2, "read (urandom): %s", strerror(errno));
+ else if (n < COOKIESZ) die(2, "read (urandom): short read");
+ base64_init(&bc);
+ base64_encode(&bc, buf, COOKIESZ, &d);
+ base64_encode(&bc, 0, 0, &d);
+ dstr_putz(&d);
+
+ for (tt = toktab; tt < t; tt++) {
+ if (strcmp(d.buf, tt->tok) == 0)
+ goto again;
+ }
+
+ assert(d.len < sizeof(t->tok));
+ memcpy(t->tok, d.buf, d.len + 1);
+ dstr_reset(&d);
+ }
+}
+
+struct tokmatch {
+ unsigned tf; /* Possible token matches */
+ size_t o; /* Offset into token string */
+ unsigned f; /* Flags */
+#define TMF_CR 1u /* Seen trailing carriage-return */
+};
+
+static void tokmatch_init(struct tokmatch *tm)
+ { tm->tf = TF_ALL; tm->o = 0; tm->f = 0; }
+
+static int tokmatch_update(struct tokmatch *tm, int ch)
+{
+ const struct token *t;
+ unsigned tf;
+
+ switch (ch) {
+ case '\n':
+ for (t = toktab, tf = 1; t->label; t++, tf <<= 1) {
+ if ((tm->tf & tf) && !t->tok[tm->o])
+ return (tf);
+ }
+ return (-1);
+ case '\r':
+ for (t = toktab, tf = 1; t->label; t++, tf <<= 1) {
+ if ((tm->tf & tf) && !t->tok[tm->o] && !(tm->f & TMF_CR))
+ tm->f |= TMF_CR;
+ else
+ tm->tf &= ~tf;
+ }
+ break;
+ default:
+ for (t = toktab, tf = 1; t->label; t++, tf <<= 1) {
+ if ((tm->tf & tf) && ch != t->tok[tm->o])
+ tm->tf &= ~tf;
+ }
+ tm->o++;
+ break;
+ }
+ return (0);
+}
+
+static int writetok(unsigned i, int fd)
+{
+ static const char nl = '\n';
+ const struct token *t = &toktab[i];
+ size_t n = strlen(t->tok);
+
+ errno = EIO;
+ if (write(fd, t->tok, n) < n ||
+ write(fd, &nl, 1) < 1)
+ return (-1);
+ return (0);
+}
+
+/*----- Data structures ---------------------------------------------------*/
+
+struct client {
+ struct client *next; /* Links in the client chain */
+ int fd; /* File descriptor for socket */
+ struct tokmatch tm; /* Token matching context */
+};
+
+/*----- Static variables --------------------------------------------------*/
+
+static int *fs; /* File descriptors for targets */
+static char **fsname; /* File system names */
+static size_t nfs; /* Number of descriptors */
+
+/*----- Cleanup -----------------------------------------------------------*/
+
+#define EOM ((char *)0)
+static void emerg(const char *msg,...)
+{
+ va_list ap;
+
+#define MSG(m) \
+ do { const char *m_ = m; if (write(2, m_, strlen(m_))); } while (0)
+
+ va_start(ap, msg);
+ MSG(QUIS); MSG(": ");
+ do {
+ MSG(msg);
+ msg = va_arg(ap, const char *);
+ } while (msg != EOM);
+ MSG("\n");
+
+#undef MSG
+}
+
+static void partial_cleanup(size_t n)
+{
+ int i;
+ int bad = 0;
+
+ for (i = 0; i < nfs; i++) {
+ if (fs[i] == -1)
+ emerg("not really thawing ", fsname[i], EOM);
+ else if (fs[i] != -2) {
+ if (ioctl(fs[i], FITHAW, 0)) {
+ emerg("VERY BAD! failed to thaw ",
+ fsname[i], ": ", strerror(errno), EOM);
+ bad = 1;
+ }
+ close(fs[i]);
+ }
+ fs[i] = -2;
+ }
+ if (bad) _exit(112);
+}
+
+static void cleanup(void) { partial_cleanup(nfs); }
+
+static int sigcatch[] = {
+ SIGINT, SIGQUIT, SIGTERM, SIGHUP, SIGALRM,
+ SIGILL, SIGSEGV, SIGBUS, SIGFPE, SIGABRT
+};
+
+static void sigmumble(int sig)
+{
+ sigset_t ss;
+
+ cleanup();
+ emerg(strsignal(sig), 0);
+
+ signal(sig, SIG_DFL);
+ sigemptyset(&ss); sigaddset(&ss, sig);
+ sigprocmask(SIG_UNBLOCK, &ss, 0);
+ raise(sig);
+ _exit(4);
+}
+
+/*----- Help functions ----------------------------------------------------*/
+
+static void version(FILE *fp)
+ { pquis(fp, "$, " PACKAGE " version " VERSION "\n"); }
+static void usage(FILE *fp)
+ { pquis(fp, "Usage: $ [-n] [-a ADDR] [-p LOPORT[-HIPORT]] FILSYS ...\n"); }
+
+static void help(FILE *fp)
+{
+ version(fp); putc('\n', fp);
+ usage(fp);
+ fputs("\n\
+Freezes a filesystem temporarily, with some measure of safety.\n\
+\n\
+The program listens for connections on a TCP port, and prints a line\n\
+\n\
+ PORT COOKIE\n\
+\n\
+to standard output. You must connect to this PORT and send the COOKIE\n\
+followed by a newline within a short period of time. The filesystems\n\
+will then be frozen, and `OK' written to the connection. In order to\n\
+keep the file system frozen, you must keep the connection open, and\n\
+feed data into it. If the connection closes, or no data is received\n\
+within a set period of time, or the program receives one of a variety\n\
+of signals or otherwise becomes unhappy, the filesystems are thawed again.\n\
+\n\
+Options:\n\
+\n\
+-h, --help Print this help text.\n\
+-v, --version Print the program version number.\n\
+-u, --usage Print a short usage message.\n\
+\n\
+-a, --address=ADDR Listen only on ADDR.\n\
+-n, --not-really Don't really freeze or thaw filesystems.\n\
+-p, --port-range=LO[-HI] Select a port number between LO and HI.\n\
+ If HI is omitted, choose only LO.\n\
+", fp);
+}
+
+/*----- Main program ------------------------------------------------------*/
+
+int main(int argc, char *argv[])
+{
+ char buf[256];
+ int loport = -1, hiport = -1;
+ int sk, fd, maxfd;
+ struct sockaddr_in sin;
+ socklen_t sasz;
+ struct hostent *h;
+ const char *p, *q;
+ struct timeval now, when, delta;
+ struct client *clients = 0, *c, **cc;
+ const struct token *t;
+ struct tokmatch tm;
+ fd_set fdin;
+ int i;
+ ssize_t n;
+ unsigned f = 0;
+#define f_bogus 0x01u
+#define f_notreally 0x02u
+
+ ego(argv[0]);
+ sub_init();
+
+ /* --- Partially initialize the socket address --- */
+
+ sin.sin_family = AF_INET;
+ sin.sin_addr.s_addr = INADDR_ANY;
+ sin.sin_port = 0;
+
+ /* --- Parse the command line --- */
+
+ for (;;) {
+ static struct option opts[] = {
+ { "help", 0, 0, 'h' },
+ { "version", 0, 0, 'v' },
+ { "usage", 0, 0, 'u' },
+ { "address", OPTF_ARGREQ, 0, 'a' },
+ { "not-really", 0, 0, 'n' },
+ { "port-range", OPTF_ARGREQ, 0, 'p' },
+ { 0, 0, 0, 0 }
+ };
+
+ if ((i = mdwopt(argc, argv, "hvua:np:", opts, 0, 0, 0)) < 0) break;
+ switch (i) {
+ case 'h': help(stdout); exit(0);
+ case 'v': version(stdout); exit(0);
+ case 'u': usage(stdout); exit(0);
+ case 'a':
+ if ((h = gethostbyname(optarg)) == 0) {
+ die(1, "failed to resolve address `%s': %s",
+ optarg, hstrerror(h_errno));
+ }
+ if (h->h_addrtype != AF_INET)
+ die(1, "unexpected address type resolving `%s'", optarg);
+ assert(h->h_length == sizeof(sin.sin_addr));
+ memcpy(&sin.sin_addr, h->h_addr, sizeof(sin.sin_addr));
+ break;
+ case 'n': f |= f_notreally; break;
+ case 'p':
+ if ((p = strchr(optarg, '-')) == 0)
+ loport = hiport = getuint(optarg, 0);
+ else {
+ loport = getuint(optarg, p);
+ hiport = getuint(p + 1, 0);
+ }
+ break;
+ default: f |= f_bogus; break;
+ }
+ }
+ if (f & f_bogus) { usage(stderr); exit(1); }
+ if (optind >= argc) { usage(stderr); exit(1); }
+
+ /* --- Open the file systems --- */
+
+ nfs = argc - optind;
+ fsname = &argv[optind];
+ fs = xmalloc(nfs*sizeof(*fs));
+ for (i = 0; i < nfs; i++) {
+ if ((fs[i] = open(fsname[i], O_RDONLY)) < 0)
+ die(2, "open (%s): %s", fsname[i], strerror(errno));
+ }
+
+ if (f & f_notreally) {
+ for (i = 0; i < nfs; i++) {
+ close(fs[i]);
+ fs[i] = -1;
+ }
+ }
+
+ /* --- Generate random tokens --- */
+
+ inittoks();
+
+ /* --- Create the listening socket --- */
+
+ if ((sk = socket(PF_INET, SOCK_STREAM, 0)) < 0)
+ die(2, "socket: %s", strerror(errno));
+ i = 1;
+ if (setsockopt(sk, SOL_SOCKET, SO_REUSEADDR, &i, sizeof(i)))
+ die(2, "setsockopt (reuseaddr): %s", strerror(errno));
+ if (fdflags(sk, O_NONBLOCK, O_NONBLOCK, FD_CLOEXEC, FD_CLOEXEC))
+ die(2, "fdflags: %s", strerror(errno));
+ if (loport < 0 || loport == hiport) {
+ if (loport >= 0) sin.sin_port = htons(loport);
+ if (bind(sk, (struct sockaddr *)&sin, sizeof(sin)))
+ die(2, "bind: %s", strerror(errno));
+ } else if (hiport != loport) {
+ for (i = loport; i <= hiport; i++) {
+ sin.sin_port = htons(i);
+ if (bind(sk, (struct sockaddr *)&sin, sizeof(sin)) >= 0) break;
+ else if (errno != EADDRINUSE)
+ die(2, "bind: %s", strerror(errno));
+ }
+ if (i > hiport) die(2, "bind: all ports in use");
+ }
+ if (listen(sk, 5)) die(2, "listen: %s", strerror(errno));
+
+ /* --- Tell the caller how to connect to us, and start the timer --- */
+
+ sasz = sizeof(sin);
+ if (getsockname(sk, (struct sockaddr *)&sin, &sasz))
+ die(2, "getsockname (listen): %s", strerror(errno));
+ printf("PORT %d\n", ntohs(sin.sin_port));
+ for (t = toktab; t->label; t++)
+ printf("TOKEN %s %s\n", t->label, t->tok);
+ printf("READY\n");
+ if (fflush(stdout) || ferror(stdout))
+ die(2, "write (stdout, rubric): %s", strerror(errno));
+ gettimeofday(&now, 0); TV_ADDL(&when, &now, TO_CONNECT, 0);
+
+ /* --- Collect incoming connections, and check for the cookie --- *
+ *
+ * This is the tricky part.
+ */
+
+ for (;;) {
+ FD_ZERO(&fdin);
+ FD_SET(sk, &fdin);
+ maxfd = sk;
+ for (c = clients; c; c = c->next) {
+ FD_SET(c->fd, &fdin);
+ if (c->fd > maxfd) maxfd = c->fd;
+ }
+ TV_SUB(&delta, &when, &now);
+ if (select(maxfd + 1, &fdin, 0, 0, &delta) < 0)
+ die(2, "select (accept): %s", strerror(errno));
+ gettimeofday(&now, 0);
+
+ if (TV_CMP(&now, >=, &when)) die(3, "timeout (accept)");
+
+ if (FD_ISSET(sk, &fdin)) {
+ sasz = sizeof(sin);
+ fd = accept(sk, (struct sockaddr *)&sin, &sasz);
+ if (fd >= 0) {
+ if (fdflags(fd, O_NONBLOCK, O_NONBLOCK, FD_CLOEXEC, FD_CLOEXEC) < 0)
+ die(2, "fdflags: %s", strerror(errno));
+ c = CREATE(struct client);
+ c->next = clients; c->fd = fd; tokmatch_init(&c->tm);
+ clients = c;
+ }
+#ifdef DEBUG
+ else if (errno != EAGAIN)
+ moan("accept: %s", strerror(errno));
+#endif
+ }
+
+ for (cc = &clients; *cc;) {
+ c = *cc;
+ if (!FD_ISSET(c->fd, &fdin)) goto next_client;
+ n = read(c->fd, buf, sizeof(buf));
+ if (!n) goto disconn;
+ else if (n < 0) {
+ if (errno == EAGAIN) goto next_client;
+ D( moan("read (client; auth): %s", strerror(errno)); )
+ goto disconn;
+ } else {
+ for (p = buf, q = p + n; p < q; p++) {
+ switch (tokmatch_update(&c->tm, *p)) {
+ case 0: break;
+ case TF_FREEZE: goto connected;
+ default:
+ D( moan("bad token from client"); )
+ goto disconn;
+ }
+ }
+ }
+
+ next_client:
+ cc = &c->next;
+ continue;
+
+ disconn:
+ close(c->fd);
+ *cc = c->next;
+ DESTROY(c);
+ continue;
+ }
+ }
+
+connected:
+ close(sk); sk = c->fd;
+ while (clients) {
+ if (clients->fd != sk) close(clients->fd);
+ c = clients->next;
+ DESTROY(clients);
+ clients = c;
+ }
+
+ /* --- Establish signal handlers --- *
+ *
+ * Hopefully this will prevent bad things happening if we have an accident.
+ */
+
+ for (i = 0; i < sizeof(sigcatch)/sizeof(sigcatch[0]); i++) {
+ if (signal(sigcatch[i], sigmumble) == SIG_ERR)
+ die(2, "signal (%d): %s", i, strerror(errno));
+ }
+ atexit(cleanup);
+
+ /* --- Prevent the OOM killer from clobbering us --- */
+
+ if ((fd = open("/proc/self/oom_adj", O_WRONLY)) < 0 ||
+ write(fd, "-17\n", 4) < 4 ||
+ close(fd))
+ die(2, "set oom_adj: %s", strerror(errno));
+
+ /* --- Actually freeze the filesystem --- */
+
+ for (i = 0; i < nfs; i++) {
+ if (fs[i] == -1)
+ moan("not really freezing %s", fsname[i]);
+ else {
+ if (ioctl(fs[i], FIFREEZE, 0) < 0) {
+ partial_cleanup(i);
+ die(2, "ioctl (freeze %s): %s", fsname[i], strerror(errno));
+ }
+ }
+ }
+ if (writetok(T_FROZEN, sk)) {
+ cleanup();
+ die(2, "write (frozen): %s", strerror(errno));
+ }
+
+ /* --- Now wait for the other end to detach --- */
+
+ tokmatch_init(&tm);
+ TV_ADDL(&when, &now, TO_KEEPALIVE, 0);
+ for (p++; p < q; p++) {
+ switch (tokmatch_update(&tm, *p)) {
+ case 0: break;
+ case TF_KEEPALIVE: tokmatch_init(&tm); break;
+ case TF_THAW: goto done;
+ default: cleanup(); die(3, "unknown token (keepalive)");
+ }
+ }
+ for (;;) {
+ FD_ZERO(&fdin);
+ FD_SET(sk, &fdin);
+ TV_SUB(&delta, &when, &now);
+ if (select(sk + 1, &fdin, 0, 0, &delta) < 0) {
+ cleanup();
+ die(2, "select (keepalive): %s", strerror(errno));
+ }
+
+ gettimeofday(&now, 0);
+ if (TV_CMP(&now, >, &when)) {
+ cleanup(); die(3, "timeout (keepalive)");
+ }
+ if (FD_ISSET(sk, &fdin)) {
+ n = read(sk, buf, sizeof(buf));
+ if (!n) { cleanup(); die(3, "end-of-file (keepalive)"); }
+ else if (n < 0) {
+ if (errno == EAGAIN) ;
+ else {
+ cleanup();
+ die(2, "read (client, keepalive): %s", strerror(errno));
+ }
+ } else {
+ for (p = buf, q = p + n; p < q; p++) {
+ switch (tokmatch_update(&tm, *p)) {
+ case 0: break;
+ case TF_KEEPALIVE:
+ TV_ADDL(&when, &now, TO_KEEPALIVE, 0);
+ tokmatch_init(&tm);
+ break;
+ case TF_THAW:
+ goto done;
+ default:
+ cleanup();
+ die(3, "unknown token (keepalive)");
+ }
+ }
+ }
+ }
+ }
+
+done:
+ cleanup();
+ if (writetok(T_THAWED, sk))
+ die(2, "write (thaw): %s", strerror(errno));
+ close(sk);
+ return (0);
+}
+
+/*----- That's all, folks -------------------------------------------------*/
--- /dev/null
+.TH rsync-backup 8 "7 October 2012" rsync-backup
+.SH SYNOPSIS
+.B rsync-backup
+.RB [ \-v ]
+.RB [ \-c
+.IR config-file ]
+.SH DESCRIPTION
+The
+.B rsync-backup
+script is a backup program of the currently popular
+.RB ` rsync (1)
+.BR \-\-link-dest '
+variety. It uses
+.BR rsync 's
+ability to create hardlinks from (apparently) similar existing local
+trees to make incremental dumps efficient, even from remote sources.
+Restoring files is easy because the backups created are just directories
+full of files, exactly as they were on the source \(en and this is
+verified using the
+.BR fshash (1)
+program.
+.PP
+The script does more than just running
+.BR rsync .
+It is also responsible for creating and removing snapshots of volumes to
+be backed up, and expiring old dumps according to a user-specified
+retention policy.
+.SS Installation
+The idea is that the
+.B rsync-backup
+script should be installed and run on a central backup server with local
+access to the backup volumes.
+.PP
+The script should be run with full (root) privileges, so that it can
+correctly record file ownership information. The server should also be
+able to connect via
+.BR ssh (1)
+to the client machines, and run processes there as root. (This is not a
+security disaster. Remember that the backup server is, in the end,
+responsible for the integrity of the backup data. A dishonest backup
+server can easily compromise a client which is being restored from
+corrupt backup data.)
+.PP
+The
+
+
+.SS Configuration commands
+The configuration file is simply a Bash shell fragment: configuration
+commands are shell functions.
+.TP
+.BI "backup " "fs\fR[:\fIfsarg\fR] ..."
+Back up the named filesystems. The corresponding
+.IR fsarg s
+may be required by the snapshot type.
+.TP
+.BI "host " host
+Future
+.B backup
+commands will back up filesystems on the named
+.IR host .
+This clears the
+.B like
+list.
+.TP
+.BI "like " "host\fR ..."
+Declare that subsequent filesystems are `similar' to like-named
+filesystems on the named
+.IR host s,
+and that
+.B rsync
+should use those trees as potential sources of hardlinkable files. Be
+careful when using this option without
+.BR rsync 's
+.B \-\-checksum
+option: an erroneous hardlink will cause the backup to fail. (The
+backup won't be left silently incorrect.)
+.TP
+.BI "retain " frequency " " duration
+Define part a backup retention policy: backup trees of the
+.I frequency
+should be kept for the
+.IR duration .
+The
+.I frequency
+can be
+.BR daily ,
+.BR weekly ,
+.BR monthly ,
+or
+.B annually
+(or
+.BR yearly ,
+which means the same); the
+.I duration
+may be any of
+.BR week ,
+.BR month ,
+.BR year ,
+or
+.BR forever .
+Expiry considers each existing dump against the policy lines in order:
+the last applicable line determines the dump's fate \(en so you should
+probably write the lines in decreasing order of duration.
+.TP
+.BI "snap " type " " \fR[\fIargs\fR...]
+Use the snapshot
+.I type
+for subsequent backups. Some snapshot types require additional
+arguments, which may be supplied here.
+.SS Configuration variables
+The following shell variables may be overridden by the configuration
+file.
+.TP
+.B MAXLOG
+The number of log files to be kept for each filesystem. Old logfiles
+are deleted to keep the total number below this bound. The default
+value is 14.
+.TP
+.B RSYNCOPTS
+Command-line options to pass to
+.BR rsync (1)
+in addition to the basic set:
+.B \-\-archive \-\-hard-links \-\-numeric-ids \-\-del \-\-sparse
+.B \-\-compress \-\-one-file-system \-\-partial
+.B \-\-filter="dir-merge .rsync-backup"
+The default is
+.BR \-\-verbose .
+.TP
+.B SNAPDIR
+LVM (and
+.BR rfreezefs )
+snapshots are mounted on subdirectories below the
+.B SNAPDIR
+.IR "on backup clients" .
+The default is
+.IB mntbkpdir /snap
+where
+.I mntbkpdir
+is the backup mount directory configured at build time.
+.TP
+.B SNAPSIZE
+The volume size option to pass to
+.BR lvcreate (8)
+when creating a snapshot. The default is
+.B \-l10%ORIGIN
+which seems to work fairly well.
+.TP
+.B STOREDIR
+Where the actual backup trees should be stored. See the section on
+.B Archive structure
+below.
+The default is
+.IB mntbkpdir /store
+where
+.I mntbkpdir
+is the backup mount directory configured at build time.
+.TP
+.B HASH
+The hash function to use for verifying archive integrity. This is
+passed to the
+.B \-H
+option of
+.BR fshash ,
+so it must name one of the hash functions supported by your Python's
+.B hashlib
+module. The default is
+.BR sha256 .
+.SS Hook functions
+The configuration file may define shell functions to perform custom
+actions at various points in the backup process.
+.TP
+.BI "backup_precommit_hook " host " " fs " " date
+Called after a backup has been verified complete and about to be
+committed. The backup tree is in
+.B new
+in the current directory, and the
+.B fshash
+manifest is in
+.BR new.fshash .
+A typical action would be to create a digital signature on the
+manifest.
+.TP
+.BI "backup_commit_hook " host " " fs " " date
+Called during the commit procedure. The backup tree and manifest have
+been renamed into their proper places. Typically one would use this
+hook to rename files created by the
+.B backup_precommit_hook
+function.
+.TP
+.BR "whine " [ \-n ] " " \fItext\fR...
+Called to report `interesting' events when the
+.B \-v
+option is in force. The default action is to echo the
+.I text
+to (what was initially) standard output, followed by a newline unless
+.B \-n
+is given.
+.SS Snapshot types
+The following snapshot types are available.
+.TP
+.B live
+A trivial snapshot type: attempts to back up a live filesystem. How
+well this works depends on how active the filesystem is. If files
+change while the dump is in progress then the
+.B fshash
+verification will likely fail. Backups using this snapshot type must
+specify the filesystem mount point as the
+.IR fsarg .
+.TP
+.B ro
+A slightly less trivial snapshot type: make the filesystem read-only
+while the dump is in progress. Backups using this snapshot type must
+specify the filesystem mount point as the
+.IR fsarg .
+.TP
+.BI "lvm " vg
+Create snapshots using LVM. The snapshot argument is interpreted as the
+relevant volume group. The filesystem name is interpreted as the origin
+volume name; the snapshot will be called
+.IB fs .bkp
+and mounted on
+.IB SNAPDIR / fs \fR;
+space will be allocated to it according to the
+.I SNAPSIZE
+variable.
+.TP
+.BI "rfreezefs " client " " vg
+This gets complicated. Suppose that a server has an LVM volume group,
+and exports (somehow) a logical volume to a client. Examples are a host
+providing a virtual disk to a guest, or a server providing
+network-attached storage to a client. The server can create a snapshot
+of the volume using LVM, but must synchronize with the client to ensure
+that the filesystem image captured in the snapshot is clean. The
+.BR rfreezefs (8)
+program should be installed on the client to perform this rather
+delicate synchronization. Declare the server using the
+.B host
+command as usual; pass the client's name as the
+.I client
+and the
+server's volume group name as the
+.I vg
+snapshot arguments. Finally, backups using this snapshot type must
+specify the filesystem mount point (or, actually, any file in the
+filesystem) on the client, as the
+.IR fsarg .
+.PP
+Additional snapshot types can be defined in the configuration file. A
+snapshot type requires two shell functions.
+.TP
+.BI snap_ type " " snapargs " " fs " " fsarg
+Create the snapshot, and write the mountpoint (on the client host) to
+standard output, in a form suitable as an argument to
+.BR rsync .
+.TP
+.BI unsnap_ type " " snapargs " " fs " " fsarg
+Remove the snapshot.
+.PP
+There are a number of utility functions which can be used by snapshot
+type handlers: please see the script for details. Please send the
+author interesting snapshot handlers for inclusion in the main
+distribution.
+.SS Archive structure
--- /dev/null
+#! @BASH@
+###
+### Backup script
+###
+### (c) 2012 Mark Wooding
+###
+
+###----- Licensing notice ---------------------------------------------------
+###
+### This file is part of the `rsync-backup' program.
+###
+### rsync-backup is free software; you can redistribute it and/or modify
+### it under the terms of the GNU General Public License as published by
+### the Free Software Foundation; either version 2 of the License, or
+### (at your option) any later version.
+###
+### rsync-backup is distributed in the hope that it will be useful,
+### but WITHOUT ANY WARRANTY; without even the implied warranty of
+### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+### GNU General Public License for more details.
+###
+### You should have received a copy of the GNU General Public License
+### along with rsync-backup; if not, write to the Free Software Foundation,
+### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+set -e
+
+thishost=$(hostname -s)
+quis=${0##*/}
+
+VERSION=@VERSION@
+mntbkpdir=@mntbkpdir@
+logdir=@logdir@
+fshashdir=@fshashdir@
+conf=@sysconfdir@/rsync-backup.conf
+
+verbose=:
+
+###--------------------------------------------------------------------------
+### Utility functions.
+
+RSYNCOPTS="--verbose"
+
+do_rsync () {
+ ## Run rsync(1) in an appropriate manner. Configuration should ovrride
+ ## this or set $RSYNCOPTS if it wants to do something weirder. Arguments
+ ## to this function are passed on to rsync.
+
+ rsync \
+ --archive --hard-links --numeric-ids --del \
+ --sparse --compress \
+ --one-file-system \
+ --partial \
+ $RSYNCOPTS \
+ --filter="dir-merge .rsync-backup" \
+ "$@"
+}
+
+log () {
+ now=$(date +"%Y-%m-%d %H:%M:%S %z")
+ echo >&9 "$now $*"
+}
+
+run () {
+ tag=$1 cmd=$2; shift 2
+ ## Run CMD, logging its output in a pleasing manner.
+
+ log "BEGIN $tag"
+ rc=$(
+ { { { ( set +e
+ "$cmd" "$@" 3>&- 4>&- 5>&- 9>&-
+ echo $? >&5; ) |
+ while IFS= read line; do echo "| $line"; done >&4; } 2>&1 |
+ while IFS= read line; do echo "* $line"; done >&4; } 4>&1 |
+ cat >&9; } 5>&1 </dev/null
+ )
+ case $rc in
+ 0) log "END $tag" ;;
+ *) log "FAIL $tag (rc = $rc)" ;;
+ esac
+ return $rc
+}
+
+localp () {
+ h=$1
+ ## Answer whether H is a local host.
+
+ case $h in
+ "$thishost") return 0 ;;
+ *) return 1 ;;
+ esac
+}
+
+hostrun () {
+ tag=$1 cmd=$2
+ ## Run CMD on the current host. If the host seems local then run the
+ ## command through a local shell; otherwise run it through ssh(1). Either
+ ## way it will be processed by a shell.
+
+ if localp $host; then run "@$host: $tag" sh -c "$cmd"
+ else run "@$host: $tag" ssh $host "$cmd"
+ fi
+}
+
+_hostrun () {
+ h=$1 cmd=$2
+ ## Like hostrun, but without the complicated logging, but targetted at a
+ ## specific host.
+
+ if localp $h; then sh -c "$cmd"
+ else ssh $h "$cmd"
+ fi
+}
+
+hostpath () {
+ path=$1
+ ## Output (to stdout) either PATH or HOST:PATH, choosing the former if the
+ ## current host is local.
+
+ if localp $host; then echo $path
+ else echo $host:$path
+ fi
+}
+
+###--------------------------------------------------------------------------
+### Snapshot handling.
+
+## Snapshot protocol. Each snapshot type has a pair of functions snap_TYPE
+## and unsnap_TYPE. Each is given the current snapshot arguments and the
+## filesystem name to back up. The snap_TYPE function should create and
+## mount the snapshot and output an rsync(1) path to where the filesystem can
+## be copied; the unsnap_TYPE function should unmount and tear down the
+## snapshot.
+
+## Fake snapshot by not doing anything. Use only if you have no choice.
+snap_live () { hostpath "$2"; }
+unsnap_live () { :; }
+
+## Fake snapshot by remounting a live filesystem read-only. Useful if the
+## underlying storage isn't in LVM.
+
+snap_ro () {
+ fs=$1 mnt=$2
+
+ ## Place a marker in the filesystem so we know why it was made readonly.
+ ## (Also this serves to ensure that the filesystem was writable before.)
+ hostrun "snap-ro $mnt" "
+ echo rsync-backup >$mnt/.lock
+ mount -oremount,ro $mnt" || return $?
+
+ ## Done.
+ hostpath $mnt
+}
+
+unsnap_ro () {
+ fs=$1 mnt=$2
+
+ ## Check that the filesystem still has our lock marker.
+ hostrun "unsnap-ro $mnt" "
+ case \$(cat $mnt/.lock) in
+ rsync-backup) ;;
+ *) echo unlocked by someone else; exit 31 ;;
+ esac
+ mount -oremount,rw $mnt
+ rm $mnt/.lock" || return $?
+}
+
+## Snapshot using LVM.
+
+SNAPSIZE="-l10%ORIGIN"
+SNAPDIR=@mntbkpdir@/snap
+
+snap_lvm () {
+ vg=$1 lv=$2
+
+ ## Make the snapshot.
+ hostrun "snap-lvm $vg/$lv" "
+ lvcreate --snapshot -n$lv.bkp $SNAPSIZE $vg/$lv
+ mkdir -p $SNAPDIR/$lv
+ mount -oro /dev/$vg/$lv.bkp $SNAPDIR/$lv" || return $?
+
+ ## Done.
+ hostpath $SNAPDIR/$lv
+}
+
+unsnap_lvm () {
+ vg=$1 lv=$2
+
+ ## Remove the snapshot. Sometimes LVM doesn't notice that the snapshot is
+ ## no longer in open immdiately, so try several times.
+ hostrun "unsnap-lvm $vg/$lv" "
+ umount $SNAPDIR/$lv
+ rc=1
+ for i in 1 2 3 4; do
+ if lvremove -f $vg/$lv.bkp; then rc=0; break; fi
+ sleep 2
+ done
+ exit $rc" || return $?
+}
+
+## Complicated snapshot using LVM, where the volume group and filesystem are
+## owned by different machines, so they need to be synchronized during the
+## snapshot.
+
+do_rfreezefs () {
+ lvhost=$1 vg=$2 lv=$3 fshost=$4 fsdir=$5
+
+ ## Engage in the rfreezefs protocol with the filesystem host. This
+ ## involves some hairy plumbing. We want to get exit statuses out of both
+ ## halves.
+ set +e
+ ssh $fshost rfreezefs $fsdir | {
+ set -e
+
+ ## Read the codebook from the remote end.
+ ready=nil
+ while read line; do
+ set -- $line
+ case "$1" in
+ PORT) port=$2 ;;
+ TOKEN) eval tok_$2=$3 ;;
+ READY) ready=t; break ;;
+ *)
+ echo >&2 "$quis: unexpected keyword $1 (rfreezefs to $rhost)"
+ exit 1
+ ;;
+ esac
+ done
+ case $ready in
+ nil)
+ echo >&2 "$quis: unexpected eof (rfreezefs to $rhost)"
+ exit 1
+ ;;
+ esac
+
+ ## Connect to the filesystem host's TCP port and get it to freeze its
+ ## filesystem.
+ exec 3<>/dev/tcp/$fshost/$port
+ echo $tok_FREEZE >&3
+ read tok <&3
+ case $tok in
+ "$tok_FROZEN") ;;
+ *)
+ echo >&2 "$quis: unexpected token $tok (rfreezefs $fsdir on $fshost)"
+ exit 1
+ ;;
+ esac
+
+ ## Get the volume host to create the snapshot.
+ set +e
+ _hostrun >&2 3>&- $lvhost \
+ "lvcreate --snapshot -n$lv.bkp $SNAPSIZE $vg/$lv"
+ snaprc=$?
+ set -e
+
+ ## The filesystem can thaw now.
+ echo $tok_THAW >&3
+ read tok <&3
+ case $tok in
+ "$tok_THAWED") ;;
+ *)
+ _hostrun >&2 3>&- $lvhost "lvremove -f $vg/$lv.bkp" || :
+ echo >&2 "$quis: unexpected token $tok (rfreezefs $fsdir on $fshost)"
+ exit 1
+ ;;
+ esac
+
+ ## Done.
+ exit $snaprc
+ }
+
+ ## Sift through the wreckage to find out what happened.
+ rc_rfreezefs=${PIPESTATUS[0]} rc_snapshot=${PIPESTATUS[1]}
+ set -e
+ case $rc_rfreezefs:$rc_snapshot in
+ 0:0)
+ ;;
+ 112:*)
+ echo >&2 "$quis: EMERGENCY failed to thaw $fsdir on $fshost!"
+ exit 112
+ ;;
+ *)
+ echo >&2 "$quis: failed to snapshot $vg/$lv ($fsdir on $fshost)"
+ exit 1
+ ;;
+ esac
+
+ ## Mount the snapshot on the volume host.
+ _hostrun >&2 $lvhost "
+ mkdir -p $SNAPDIR/$lv
+ mount -oro /dev/$vg/$lv.bkp $SNAPDIR/$lv"
+}
+
+snap_rfreezefs () {
+ rhost=$1 vg=$2 lv=$3 rfs=$4
+
+ set -e
+ run "snap-rfreezefs $host:$vg/$lv $rhost:$rfs" \
+ do_rfreezefs $host $vg $lv $rhost $rfs || return $?
+ hostpath $SNAPDIR/$lv
+}
+
+unsnap_rfreezefs () {
+
+ ## Unshapping is the same as for plain LVM.
+ rhost=$1 vg=$2 lv=$3 rfs=$4
+ unsnap_lvm $vg $lv
+}
+
+###--------------------------------------------------------------------------
+### Expiry computations.
+
+parsedate () {
+ date=$1
+ ## Parse an ISO8601 DATE, and set YEAR, MONTH, DAY appropriately (and
+ ## without leading zeros).
+
+ ## Extract the components of the date and trim leading zeros (which will
+ ## cause things to be interpreted as octal and fail).
+ year=${date%%-*} rest=${date#*-}; month=${rest%%-*} day=${rest#*-}
+ year=${year#0} month=${month#0} day=${day#0}
+}
+
+julian () {
+ date=$1
+ ## Convert an ISO8601 DATE to a Julian Day Number.
+
+ parsedate $date
+
+ ## The actual calculation: convert a (proleptic) Gregorian calendar date
+ ## into a Julian day number. This is taken from Wikipedia's page
+ ## http://en.wikipedia.org/wiki/Julian_day#Calculation but the commentary
+ ## is mine. The epoch is 4713BC-01-01 (proleptic) Julian, or 4714BC-11-24
+ ## proleptic Gregorian.
+
+ ## If the MONTH is January or February then set a = 1, otherwise set a = 0.
+ a=$(( (14 - $month)/12 ))
+
+ ## Compute a year offset relative to 4799BC-03-01. This puts the leap day
+ ## as the very last day in a year, which is very convenient. The offset
+ ## here is sufficient to make all y values positive (within the range of
+ ## the JDN calendar), and is a multiple of 400, which is the Gregorian
+ ## cycle length.
+ y=$(( $year + 4800 - $a ))
+
+ ## Compute the offset month number in that year. These months count from
+ ## zero, not one.
+ m=$(( $month + 12*$a - 3 ))
+
+ ## Now for the main event. The (153 m + 2)/5 term is a surprising but
+ ## correct trick for obtaining the number of days in the first m months of
+ ## the (shifted) year). The magic offset 32045 is what you get when you
+ ## plug the proper JDN epoch (year = -4713, month = 11, day = 24) into the
+ ## above machinery.
+ jdn=$(( $day + (153*$m + 2)/5 + 365*$y + $y/4 - $y/100 + $y/400 - 32045 ))
+
+ echo $jdn
+}
+
+expire () {
+ ## Read dates on stdin; write to stdout `EXPIRE date' for dates which
+ ## should be expired and `RETAIN date' for dates which should be retained.
+
+ ## Get the current date and convert it into useful forms.
+ now=$(date +%Y-%m-%d)
+ parsedate $now
+ now_jdn=$(julian $now) now_year=$year now_month=$month now_day=$day
+ kept=:
+
+ ## Work through each date in the input.
+ while read date; do
+ keep=nil
+
+ ## Convert the date into a useful form.
+ jdn=$(julian $date)
+ parsedate $date
+
+ ## Work through the policy list.
+ if [ $jdn -le $now_jdn ]; then
+ while read ival age; do
+
+ ## Decide whether the policy entry applies to this date.
+ apply=nil
+ case $age in
+ forever)
+ apply=t
+ ;;
+ year)
+ if [ $year -eq $now_year ] ||
+ ([ $year -eq $(( $now_year - 1 )) ] &&
+ [ $month -ge $now_month ])
+ then apply=t; fi
+ ;;
+ month)
+ if ([ $month -eq $now_month ] && [ $year -eq $now_year ]) ||
+ ((([ $month -eq $(( $now_month - 1 )) ] &&
+ [ $year -eq $now_year ]) ||
+ ([ $month -eq 12 ] && [ $now_month -eq 1 ] &&
+ [ $year -eq $(( $now_year - 1 )) ])) &&
+ [ $day -ge $now_day ])
+ then apply=t; fi
+ ;;
+ week)
+ if [ $jdn -ge $(( $now_jdn - 7 )) ]; then apply=t; fi
+ ;;
+ *)
+ echo >&2 "$quis: unknown age symbol \`$age'"
+ exit 1
+ ;;
+ esac
+ case $apply in nil) continue ;; esac
+
+ ## Find the interval marker for this date.
+ case $ival in
+ daily)
+ marker=$date
+ ;;
+ weekly)
+ ydn=$(julian $year-01-01)
+ wk=$(( ($jdn - $ydn)/7 + 1 ))
+ marker=$year-w$wk
+ ;;
+ monthly)
+ marker=$year-$month
+ ;;
+ annually | yearly)
+ marker=$year
+ ;;
+ *)
+ echo >&2 "$quis: unknown interval symbol \`$ival'"
+ exit 1
+ ;;
+ esac
+
+ ## See if we've alredy retained something in this interval.
+ case $kept in
+ *:"$marker":*) ;;
+ *) keep=t kept=$kept$marker: ;;
+ esac
+
+ done <<EOF
+$expire_policy
+EOF
+ fi
+
+ case $keep in
+ t) echo RETAIN $date ;;
+ *) echo EXPIRE $date ;;
+ esac
+
+ done
+}
+
+###--------------------------------------------------------------------------
+### Actually taking backups of filesystems.
+
+STOREDIR=@mntbkpdir@/store
+MAXLOG=14
+HASH=sha256
+
+bkprc=0
+
+remote_fshash () {
+ _hostrun $host "
+ umask 077
+ mkdir -p $fshashdir
+ cd ${snapmnt#*:}
+ echo \"*** $host $fs $date\"; echo
+ rsync -rx --filter='dir-merge .rsync-backup' ./ |
+ fshash -c$fshashdir/$fs.bkp -a -H$HASH -frsync
+ " >new.fshash
+}
+
+local_fshash () {
+ { echo "*** $host $fs $date"; echo
+ fshash -c$STOREDIR/fshash.cache -H$HASH new/
+ } >$localmap
+}
+
+expire_backups () {
+ { seen=:
+ for i in *-*-*; do
+ i=${i%%.*}
+ case $i in *[!-0-9]*) continue ;; esac
+ case $seen in *:"$i":*) continue ;; esac
+ seen=$seen$i:
+ echo $i
+ done; } |
+ expire |
+ while read op date; do
+ case $op in
+ RETAIN)
+ echo "keep $date"
+ ;;
+ EXPIRE)
+ echo "delete $date"
+ $verbose -n " expire $date..."
+ rm -rf $date $date.*
+ $verbose " done"
+ ;;
+ esac
+ done
+}
+
+backup_precommit_hook () {
+ host=$1 fs=$2 date=$3
+ ## Override this hook in the configuration file for special effects.
+
+ :
+}
+
+backup_commit_hook () {
+ host=$1 fs=$2 date=$3
+ ## Override this hook in the configuration file for special effects.
+
+ :
+}
+
+do_backup () {
+ date=$1 fs=$2 fsarg=$3
+ ## Back up FS on the current host.
+
+ set -e
+
+ ## Report the start of this attempt.
+ log "START BACKUP of $host:$fs"
+
+ ## Create and mount the remote snapshot.
+ snapmnt=$(snap_$snap $snapargs $fs $fsarg) || return $?
+ $verbose " create snapshot"
+
+ ## Build the list of hardlink sources.
+ linkdests=""
+ for i in $host $like; do
+ d=$STOREDIR/$i/$fs/last/
+ if [ -d $d ]; then linkdests="$linkdests --link-dest=$d"; fi
+ done
+
+ ## Copy files from the remote snapshot.
+ mkdir -p new/
+ $verbose -n " running rsync..."
+ set +e
+ run "RSYNC of $host:$fs (snapshot on $snapmnt)" do_rsync \
+ $linkdests \
+ $rsyncargs \
+ $snapmnt/ new/
+ rc_rsync=$?
+ set -e
+ $verbose " done"
+
+ ## Collect a map of the snapshot for verification purposes.
+ set +e
+ $verbose -n " remote fshash..."
+ run "@$host: fshash $fs" remote_fshash
+ rc_fshash=$?
+ set -e
+ $verbose " done"
+
+ ## Remove the snapshot.
+ unsnap_$snap $snapargs $fs $fsarg
+ $verbose " remove snapshot"
+
+ ## If we failed to copy, then give up.
+ case $rc_rsync:$rc_fshash in
+ 0:0) ;;
+ 0:*) return $rc_fshash ;;
+ *) return $rc_rsync ;;
+ esac
+
+ ## Get a matching map of the files received.
+ mkdir -m750 -p $STOREDIR/tmp
+ localmap=$STOREDIR/tmp/fshash.$host.$fs.$date
+ $verbose -n " local fshash..."
+ run "local fshash $host:$fs" local_fshash || return $?
+ $verbose " done"
+
+ ## Compare the two maps.
+ run "compare fshash maps for $host:$fs" \
+ diff -u new.fshash $localmap || return $?
+ rm -f $localmap
+ $verbose " fshash match"
+
+ ## Commit this backup.
+ backup_precommit_hook $host $fs $date
+ mv new $date
+ mv new.fshash $date.fshash
+ backup_commit_hook $host $fs $date
+ mkdir hack
+ ln -s $date hack/last
+ mv hack/last .
+ rmdir hack
+ $verbose " commit"
+
+ ## Expire old backups.
+ case "${expire_policy+t}" in
+ t) run "expiry for $host:$fs" expire_backups ;;
+ esac
+
+ ## Report success.
+ log "SUCCESSFUL BACKUP of $host:$fs"
+}
+
+backup () {
+ ## backup FS[:ARG] ...
+ ##
+ ## Back up the filesystems on the currently selected host using the
+ ## currently selected snapshot type.
+
+ for fs in "$@"; do
+
+ ## Parse the argument.
+ case $fs in
+ *:*) fsarg=${fs#*:} fs=${fs%%:*} ;;
+ *) fsarg="" ;;
+ esac
+ $verbose " filesystem $fs"
+
+ ## Move to the store directory and set up somewhere to put this backup.
+ cd $STOREDIR
+ if [ ! -d $host ]; then
+ mkdir -m755 $host
+ chown root:root $host
+ fi
+ if [ ! -d $host/$fs ]; then
+ mkdir -m750 $host/$fs
+ chown root:backup $host/$fs
+ fi
+ cd $host/$fs
+
+ ## Find out if we've already copied this filesystem today.
+ date=$(date +%Y-%m-%d)
+ if [ -d $date ]; then
+ $verbose " already dumped"
+ continue
+ fi
+
+ ## Find a name for the log file. In unusual circumstances, we may have
+ ## deleted old logs from today, so just checking for an unused sequence
+ ## number is insufficient. Instead, check all of the logfiles for today,
+ ## and use a sequence number that's larger than any of them.
+ seq=1
+ for i in "$logdir/$host/$fs.$date#"*; do
+ tail=${i##*#}
+ case "$tail" in [!1-9]* | *[!0-9]*) continue ;; esac
+ if [ -f "$i" -a $tail -ge $seq ]; then seq=$(( tail + 1 )); fi
+ done
+ log="$logdir/$host/$fs.$date#$seq"
+
+ ## Do the backup of this filesystem.
+ mkdir -p $logdir/$host
+ if ! do_backup $date $fs $fsarg 9>$log 1>&9; then
+ echo >&2
+ echo >&2 "$quis: backup of $host:$fs FAILED!"
+ bkprc=1
+ fi
+
+ ## Count up the logfiles.
+ nlog=0
+ for i in "$logdir/$host/$fs".*; do
+ if [ ! -f "$i" ]; then continue; fi
+ nlog=$(( nlog + 1 ))
+ done
+
+ ## If there are too many, go through and delete some early ones.
+ if [ $nlog -gt $MAXLOG ]; then
+ n=$(( nlog - MAXLOG ))
+ for i in "$logdir/$host/$fs".*; do
+ if [ ! -f "$i" ]; then continue; fi
+ rm -f "$i"
+ n=$(( n - 1 ))
+ if [ $n -eq 0 ]; then break; fi
+ done
+ fi
+ done
+}
+
+###--------------------------------------------------------------------------
+### Configuration functions.
+
+host () { host=$1; like=; $verbose "host $host"; }
+snaptype () { snap=$1; shift; snapargs="$*"; }
+rsyncargs () { rsyncargs="$*"; }
+like () { like="$*"; }
+
+retain () {
+ expire_policy="${expire_policy+$expire_policy
+}$*"
+}
+
+###--------------------------------------------------------------------------
+### Read the configuration and we're done.
+
+usage () {
+ echo "usage: $quis [-v] [-c CONF]"
+}
+
+version () {
+ echo "$quis version $VERSION"
+}
+
+config () {
+ echo
+ cat <<EOF
+conf = $conf
+mntbkpdir = $mntbkpdir
+fshashdir = $fshashdir
+logdir = $logdir
+EOF
+}
+
+whine () { echo >&8 "$@"; }
+
+while getopts "hVvc:" opt; do
+ case "$opt" in
+ h) usage; exit 0 ;;
+ V) version; config; exit 0 ;;
+ v) verbose=whine ;;
+ c) conf=$OPTARG ;;
+ *) exit 1 ;;
+ esac
+done
+shift $((OPTIND - 1))
+case $# in 0) ;; *) usage >&2; exit 1 ;; esac
+exec 8>&1
+
+. "$conf"
+
+###----- That's all, folks --------------------------------------------------
+
+exit $bkprc
--- /dev/null
+#! /bin/sh -x
+
+: ${vgtag=@backup} ${vgprefix=vg-backup-}
+: ${mntbkpdir=/mnt/bkp}
+: ${STOREDIR=$mntbkpdir/store} ${METADIR=$mntbkpdir/meta}
+
+umount $STOREDIR
+cryptsetup luksClose cbackup
+umount $METADIR
+vgchange -an @backup
~mdw / rsync-backup / commitdiff