From f6b4ffdc7265b79b945e2350efbd9d2a94df4450 Mon Sep 17 00:00:00 2001 Message-Id: From: Mark Wooding Date: Sun, 7 Oct 2012 14:26:23 +0100 Subject: [PATCH] Initial commit. Organization: Straylight/Edgeware From: Mark Wooding This is still a little sketchy. Some pieces are lacking documentation. But the basic code is currently in production use, so it can't be that bad. --- .gitignore | 6 + .links | 3 + .skelrc | 8 + Makefile.am | 101 ++++++ configure.ac | 79 +++++ create-backup-volume | 37 +++ fshash.in | 493 +++++++++++++++++++++++++++++ mount-backup-volume | 31 ++ rfreezefs.8 | 317 +++++++++++++++++++ rfreezefs.c | 636 +++++++++++++++++++++++++++++++++++++ rsync-backup.8 | 263 ++++++++++++++++ rsync-backup.in | 730 +++++++++++++++++++++++++++++++++++++++++++ umount-backup-volume | 10 + 13 files changed, 2714 insertions(+) create mode 100644 .gitignore create mode 100644 .links create mode 100644 .skelrc create mode 100644 Makefile.am create mode 100644 configure.ac create mode 100755 create-backup-volume create mode 100644 fshash.in create mode 100755 mount-backup-volume create mode 100644 rfreezefs.8 create mode 100644 rfreezefs.c create mode 100644 rsync-backup.8 create mode 100644 rsync-backup.in create mode 100755 umount-backup-volume diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ab9943e --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +COPYING +Makefile.in +aclocal.m4 +autom4te.cache +config +configure diff --git a/.links b/.links new file mode 100644 index 0000000..f1dbbb1 --- /dev/null +++ b/.links @@ -0,0 +1,3 @@ +config/auto-version +config/confsubst +COPYING diff --git a/.skelrc b/.skelrc new file mode 100644 index 0000000..e39c06e --- /dev/null +++ b/.skelrc @@ -0,0 +1,8 @@ +;;; -*-emacs-lisp-*- + +(setq skel-alist + (append + '((author . "Mark Wooding") + (program . "rsync-backup") + (full-title . "the `rsync-backup' program")) + skel-alist)) diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..e2fabd9 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,101 @@ +### -*-makefile-*- +### +### Build script for rsync-backup +### +### (c) 2012 Mark Wooding +### + +###----- Licensing notice --------------------------------------------------- +### +### This file is part of the `rsync-backup' program. +### +### rsync-backup is free software; you can redistribute it and/or modify +### it under the terms of the GNU General Public License as published by +### the Free Software Foundation; either version 2 of the License, or +### (at your option) any later version. +### +### rsync-backup is distributed in the hope that it will be useful, +### but WITHOUT ANY WARRANTY; without even the implied warranty of +### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +### GNU General Public License for more details. +### +### You should have received a copy of the GNU General Public License +### along with rsync-backup; if not, write to the Free Software Foundation, +### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +bin_SCRIPTS = +sbin_SCRIPTS = +sbin_PROGRAMS = +dist_noinst_SCRIPTS = +dist_man_MANS = + +EXTRA_DIST = +CLEANFILES = +DISTCLEANFILES = + +AM_CFLAGS = $(mLib_CFLAGS) + +###-------------------------------------------------------------------------- +### Substitution of configuration data. + +confsubst = $(top_srcdir)/config/confsubst +EXTRA_DIST += config/confsubst + +SUBSTVARS = \ + PACKAGE="$(PACKAGE)" VERSION="$(VERSION)" \ + BASH="$(BASH)" PYTHON="$(PYTHON)" \ + sysconfdir="$(sysconfdir)" \ + mntbkpdir="$(mntbkpdir)" \ + fshashdir="$(fshashdir)" \ + logdir="$(logdir)" + +V_SUBST = $(V_SUBST_$V) +V_SUBST_= $(V_SUBST_$(AM_DEFAULT_VERBOSITY)) +V_SUBST_0 = @printf " SUBST %s\n" $@; + +SUBST = $(V_SUBST)$(confsubst) + +###-------------------------------------------------------------------------- +### Programs and scripts. + +sbin_PROGRAMS += rfreezefs +dist_man_MANS += rfreezefs.8 +rfreezefs_SOURCES = rfreezefs.c +rfreezefs_LDADD = $(mLib_LIBS) + +sbin_SCRIPTS += rsync-backup +CLEANFILES += rsync-backup +EXTRA_DIST += rsync-backup.in +rsync-backup: rsync-backup.in Makefile + $(SUBST) >rsync-backup.new \ + $(srcdir)/rsync-backup.in $(SUBSTVARS) && \ + chmod +x rsync-backup.new && \ + mv rsync-backup.new rsync-backup + +bin_SCRIPTS += fshash +CLEANFILES += fshash +EXTRA_DIST += fshash.in +fshash: fshash.in Makefile + $(SUBST) >fshash.new \ + $(srcdir)/fshash.in $(SUBSTVARS) && \ + chmod +x fshash.new && \ + mv fshash.new fshash + +###-------------------------------------------------------------------------- +### Contributed scripts. + +## Some simple scripts showing how one might create and manage backup +## volumes. Assume the distorted.org.uk key management machinery. +dist_noinst_SCRIPTS += create-backup-volume +dist_noinst_SCRIPTS += mount-backup-volume +dist_noinst_SCRIPTS += umount-backup-volume + +###-------------------------------------------------------------------------- +### Release machinery. + +EXTRA_DIST += config/auto-version + +dist-hook: + echo $(VERSION) >$(distdir)/RELEASE + +###----- That's all, folks -------------------------------------------------- diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..e03e400 --- /dev/null +++ b/configure.ac @@ -0,0 +1,79 @@ +dnl -*-autoconf-*- +dnl +dnl Build-time configuration for rsync-backup +dnl +dnl (c) 2012 Mark Wooding +dnl + +dnl----- Licensing notice --------------------------------------------------- +dnl +dnl This file is part of the `rsync-backup' program. +dnl +dnl rsync-backup is free software; you can redistribute it and/or modify +dnl it under the terms of the GNU General Public License as published by +dnl the Free Software Foundation; either version 2 of the License, or +dnl (at your option) any later version. +dnl +dnl rsync-backup is distributed in the hope that it will be useful, +dnl but WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +dnl GNU General Public License for more details. +dnl +dnl You should have received a copy of the GNU General Public License +dnl along with rsync-backup; if not, write to the Free Software Foundation, +dnl Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +dnl-------------------------------------------------------------------------- +dnl Initialization. + +mdw_AUTO_VERSION +AC_INIT([rsync-backup], AUTO_VERSION, [mdw@distorted.org.uk]) +AC_CONFIG_SRCDIR([rsync-backup.in]) +AC_CONFIG_AUX_DIR([config]) +AM_INIT_AUTOMAKE([foreign]) +mdw_SILENT_RULES + +dnl-------------------------------------------------------------------------- +dnl Paths. + +AC_ARG_WITH([mntbkpdir], + AS_HELP_STRING([--with-mntbkpdir], [location of the backup mount tree]), + [mntbkpdir=$withval], + [mntbkpdir=/mnt/bkp]) +AC_SUBST([mntbkpdir]) + +AC_ARG_WITH([fshashdir], + AS_HELP_STRING([--with-fshashdir], [location for fshash cache files]), + [fshashdir=$withval], + [fshashdir=$localstatedir/cache/fshash]) +AC_SUBST([fshashdir]) + +AC_ARG_WITH([logdir], + AS_HELP_STRING([--with-logdir], [location for logfiles]), + [logdir=$withval], + [logdir=$localstatedir/log/bkp]) +AC_SUBST([logdir]) + +dnl-------------------------------------------------------------------------- +dnl Programming environments. + +dnl C compilers and libraries. +AC_PROG_CC +AX_CFLAGS_WARN_ALL +PKG_CHECK_MODULES([mLib], [mLib >= 2.1.0]) + +dnl Bourne-Again Shell. +AC_PATH_PROG([BASH], [bash]) + +dnl Python. +AC_PATH_PROG([PYTHON], [python]) +AX_PROG_PYTHON_VERSION([2.5],, + [AC_MSG_ERROR([Failed to find suitable Python.])]) + +dnl-------------------------------------------------------------------------- +dnl Output. + +AC_CONFIG_FILES([Makefile]) +AC_OUTPUT + +dnl----- That's all, folks -------------------------------------------------- diff --git a/create-backup-volume b/create-backup-volume new file mode 100755 index 0000000..2839fd1 --- /dev/null +++ b/create-backup-volume @@ -0,0 +1,37 @@ +#! /bin/sh -ex + +: ${vgtag=@backup} ${vgprefix=vg-backup-} +: ${mntbkpdir=/mnt/bkp} +: ${STOREDIR=$mntbkpdir/store} ${METADIR=$mntbkpdir/meta} +: ${RANDOM=/dev/random} + +case $# in + 2) tag=$1 pv=$2 ;; + *) echo >&2 "usage: $0 TAG PV" ;; +esac +vg=$vgprefix$tag + +vgcreate --addtag $vgtag $vg $pv + +lvcreate -L64M -nmeta $vg +mkfs -text3 -Lmeta /dev/$vg/meta +mount /dev/$vg/meta $METADIR + +echo $tag >$METADIR/volume +dd if=$RANDOM bs=1 count=512 | + cryptop encrypt backup >$METADIR/crypt.blob + +lvcreate -l100%FREE -ncrypt $vg +cryptop decrypt backup <$METADIR/crypt.blob | + cryptsetup luksFormat \ + --cipher=twofish-xts-benbi:sha256 \ + --hash=sha256 --key-size=256 \ + /dev/$vg/crypt - + +cryptop decrypt backup <$METADIR/crypt.blob | + cryptsetup luksOpen --key-file=- /dev/$vg/crypt cbackup + +mkfs -text3 -Lbackup /dev/mapper/cbackup + +mount /dev/mapper/cbackup $STOREDIR +touch $STOREDIR/.rsync-backup-store diff --git a/fshash.in b/fshash.in new file mode 100644 index 0000000..1dbd8be --- /dev/null +++ b/fshash.in @@ -0,0 +1,493 @@ +#! @PYTHON@ +### +### Efficiently construct canonical digests of filesystems +### +### (c) 2012 Mark Wooding +### + +###----- Licensing notice --------------------------------------------------- +### +### This file is part of the `rsync-backup' program. +### +### rsync-backup is free software; you can redistribute it and/or modify +### it under the terms of the GNU General Public License as published by +### the Free Software Foundation; either version 2 of the License, or +### (at your option) any later version. +### +### rsync-backup is distributed in the hope that it will be useful, +### but WITHOUT ANY WARRANTY; without even the implied warranty of +### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +### GNU General Public License for more details. +### +### You should have received a copy of the GNU General Public License +### along with rsync-backup; if not, write to the Free Software Foundation, +### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +from sys import argv, exit, stdin, stdout, stderr +import os as OS +import re as RX +import time as T +import stat as ST +import optparse as OP +import hashlib as H +import sqlite3 as DB +import zlib as Z + +PACKAGE = '@PACKAGE@' +VERSION = '@VERSION@' + +###-------------------------------------------------------------------------- +### Utilities. + +QUIS = OS.path.basename(argv[0]) + +def moan(msg): + stderr.write('%s: %s\n' % (QUIS, msg)) + +def die(msg, rc = 1): + moan(msg) + exit(rc) + +SYSERR = 0 +def syserr(msg): + global SYSERR + moan(msg) + SYSERR += 1 + +###-------------------------------------------------------------------------- +### File system enumeration. + +class FileInfo (object): + def __init__(me, file, st = None): + me.name = file + if st: + me.st = st + me.err = None + else: + try: + me.st = OS.lstat(file) + me.err = None + except OSError, err: + me.st = None + me.err = err + +def enum_walk(file, func): + + def dirents(name): + try: + return OS.listdir(name) + except OSError, err: + syserr("failed to read directory `%s': %s" % (name, err.strerror)) + return [] + + def dir(ee, dev): + ff = [] + dd = [] + for e in ee: + fi = FileInfo(e) + if fi.st and fi.st.st_dev != dev: pass + if fi.st and ST.S_ISDIR(fi.st.st_mode): dd.append(fi) + else: ff.append(fi) + ff.sort(key = lambda fi: fi.name) + dd.sort(key = lambda fi: fi.name + '/') + for f in ff: + func(f) + for d in dd: + if d.st.st_dev == dev: + func(d) + dir([OS.path.join(d.name, e) for e in dirents(d.name)], dev) + + if file.endswith('/'): + OS.chdir(file) + fi = FileInfo('.') + func(fi) + dir(dirents('.'), fi.st.st_dev) + else: + fi = FileInfo(file) + func(fi) + if fi.st and ST.S_ISDIR(fi.st.st_mode): + dir([OS.path.join(fi.name, e) for e in dirents(fi.name)], + fi.st.st_dev) + +def enum_find0(f, func): + tail = "" + while True: + buf = f.read(8192) + last = len(buf) == 0 + names = (tail + buf).split('\0') + tail = names.pop() + for n in names: + func(FileInfo(n)) + if last: + break + if len(tail): + moan("ignored trailing junk after last filename") + +RX_RSYNCESC = RX.compile(r'\\ \# ([0-7]{3})', RX.VERBOSE) +def enum_rsync(f, func): + + ## The format is a little fiddly. Each line consists of PERMS SIZE DATE + ## TIME NAME, separated by runs of whitespace, but the NAME starts exactly + ## one space character after the TIME and may begin with a space. + ## Sequences of the form `\#OOO' where OOO are three octal digits, stand + ## for a byte with that value. Newlines and backslashes which would be + ## ambiguous are converted into this form; all other characters are + ## literal. + ## + ## We ignore the stat information and retrieve it ourselves, because it's + ## incomplete. Hopefully the dcache is still warm. + + for line in f: + if line.endswith('\n'): line = line[:-1] + + ## Extract the escaped name. + ff = line.split(None, 3) + if len(ff) != 4: + syserr("ignoring invalid line from rsync: `%s'" % line) + continue + tail = ff[3] + try: + spc = tail.index(' ') + except ValueError: + syserr("ignoring invalid line from rsync: `%s'" % line) + continue + name = tail[spc + 1:] + + ## Now translate escape sequences. + name = RX_RSYNCESC.sub(lambda m: chr(int(m.group(1), 8)), name) + + ## Call the client. + try: + fi = FileInfo(name) + except OSError, err: + syserr("failed to stat `%s': %s" % (name, err.strerror)) + continue + func(fi) + +###-------------------------------------------------------------------------- +### The hash cache. + +class HashCache (object): + + VERSION = 0 + BUFSZ = 128*1024 + + INIT = [ + """CREATE TABLE meta ( + version INTEGER NOT NULL, + hash TEXT NOT NULL + );""", + """CREATE TABLE hash ( + ino INTEGER PRIMARY KEY, + mtime INTEGER NOT NULL, + ctime INTEGER NOT NULL, + size INTEGER NOT NULL, + hash TEXT NOT NULL, + seen BOOLEAN NOT NULL DEFAULT TRUE + );""", + """PRAGMA journal_mode = WAL;""" + ] + + def __init__(me, file, hash = None): + + if file is None: + + ## We're going this alone, with no cache. + db = None + if hash is None: + die("no hash specified and no database cache to read from") + else: + + ## Connect to the database. + db = DB.connect(file) + db.text_factory = str + + ## See whether we can understand the cache database. + c = db.cursor() + v = h = None + try: + c.execute('SELECT version, hash FROM meta') + v, h = c.fetchone() + if c.fetchone() is not None: + die("cache database corrupt: meta table has mutliple rows") + except (DB.Error, TypeError): + pass + + ## If that didn't work, we'd better clear the thing and start again. + ## But only if we know how to initialize it. + if v != me.VERSION: + + ## Explain the situation. + moan("cache version %s not understood" % v) + if hash is None: + if h is None: + die("can't initialize cache: no hash function set") + else: + hash = h + try: + H.new(hash) + except Exception: + die("unknown hash function `%s'" % hash) + + ## Drop old things. + c.execute('SELECT type, name FROM sqlite_master') + for type, name in c.fetchall(): + c.execute('DROP %s IF EXISTS %s' % (type, name)) + + ## Now we're ready to go. + for stmt in me.INIT: + c.execute(stmt) + c.execute('INSERT INTO meta VALUES (?, ?)', [me.VERSION, hash]) + db.commit() + + ## Check the hash function if necessary. + if hash is None: + hash = h + elif h is not None and h != hash: + die("hash mismatch: cache uses %s but %s requested" % (h, hash)) + + ## All done. + me.hash = hash + me._db = db + me._pend = 0 + + def hashfile(me, fi): + + ## If this isn't a proper file then don't try to hash it. + if fi.err or not ST.S_ISREG(fi.st.st_mode): + return None + + ## See whether there's a valid entry in the cache. + if me._db: + c = me._db.cursor() + c.execute( + 'SELECT mtime, size, hash, seen FROM hash WHERE ino = ?;', + [fi.st.st_ino]) + r = c.fetchone() + if r is not None: + mt, sz, h, s = r + if mt == fi.st.st_mtime and \ + sz == fi.st.st_size: + if not s: + c.execute('UPDATE hash SET seen = 1 WHERE ino = ?', + [fi.st.st_ino]) + me._update() + return h + + ## Hash the file. Beware raciness: update the file information from the + ## open descriptor, but set the size from what we actually read. + h = H.new(me.hash) + try: + with open(fi.name, 'rb') as f: + sz = 0 + while True: + buf = f.read(me.BUFSZ) + if len(buf) == 0: + break + sz += len(buf) + h.update(buf) + fi.st = OS.fstat(f.fileno()) + ##fi.st.st_size = sz + hash = h.digest() + except (OSError, IOError), err: + fi.st = None + fi.err = err + return None + hash = hash.encode('hex') + + ## Insert a record into the database. + if me._db: + c.execute(""" + INSERT OR REPLACE INTO hash + (ino, mtime, ctime, size, hash, seen) + VALUES + (?, ?, ?, ?, ?, 1); + """, [fi.st.st_ino, + fi.st.st_mtime, + fi.st.st_ctime, + fi.st.st_size, + hash]) + me._update() + + ## Done. + return hash + + def _update(me): + me._pend += 1 + if me._pend >= 1024: + me.flush() + + def flush(me): + if me._db: + me._db.commit() + me._pend = 0 + + def need_db(me): + if not me._db: + die("no cache database") + + def reset(me): + me.need_db() + c = me._db.cursor() + c.execute('UPDATE hash SET seen = 0 WHERE seen') + me.flush() + + def prune(me): + me.need_db() + c = me._db.cursor() + c.execute('DELETE FROM hash WHERE NOT seen') + me.flush() + +###-------------------------------------------------------------------------- +### Printing output. + +class GenericFormatter (object): + def __init__(me, fi): + me.fi = fi + def _fmt_time(me, t): + tm = T.gmtime(t) + return T.strftime('%Y-%m-%dT%H:%M:%SZ', tm) + def _enc_name(me, n): + return n.encode('string_escape') + def name(me): + return me._enc_name(me.fi.name) + def info(me): + return me.TYPE + def mode(me): + return '%06o' % me.fi.st.st_mode + def size(me): + return me.fi.st.st_size + def mtime(me): + return me._fmt_time(me.fi.st.st_mtime) + def owner(me): + return '%5d:%d' % (me.fi.st.st_uid, me.fi.st.st_gid) + +class ErrorFormatter (GenericFormatter): + def info(me): + return 'E%d %s' % (me.fi.err.errno, me.fi.err.strerror) + def error(me): return 'error' + mode = size = mtime = owner = error + +class SocketFormatter (GenericFormatter): + TYPE = 'socket' +class PipeFormatter (GenericFormatter): + TYPE = 'fifo' + +class LinkFormatter (GenericFormatter): + TYPE = 'symbolic-link' + def name(me): + n = GenericFormatter.name(me) + try: + d = OS.readlink(me.fi.name) + return '%s -> %s' % (n, me._enc_name(d)) + except OSError, err: + return '%s -> ' % (n, err.errno, err.strerror) + +class DirectoryFormatter (GenericFormatter): + TYPE = 'directory' + def name(me): return GenericFormatter.name(me) + '/' + def size(me): return 'dir' + +class DeviceFormatter (GenericFormatter): + def info(me): + return '%s %d:%d' % (me.TYPE, + OS.major(me.fi.st.st_rdev), + OS.minor(me.fi.st.st_rdev)) +class BlockDeviceFormatter (DeviceFormatter): + TYPE = 'block-device' +class CharDeviceFormatter (DeviceFormatter): + TYPE = 'character-device' + +class FileFormatter (GenericFormatter): + TYPE = 'regular-file' + +class Reporter (object): + + TYMAP = { + ST.S_IFSOCK: SocketFormatter, + ST.S_IFDIR: DirectoryFormatter, + ST.S_IFLNK: LinkFormatter, + ST.S_IFREG: FileFormatter, + ST.S_IFBLK: BlockDeviceFormatter, + ST.S_IFCHR: CharDeviceFormatter, + ST.S_IFIFO: PipeFormatter, + } + + def __init__(me, db): + me._inomap = {} + me._vinomap = {} + me._db = db + me._hsz = int(H.new(db.hash).digest_size) + + def file(me, fi): + h = me._db.hashfile(fi) + if fi.err: + fmt = ErrorFormatter(fi) + vino = 'error' + else: + fmt = me.TYMAP[ST.S_IFMT(fi.st.st_mode)](fi) + inoidx = fi.st.st_dev, fi.st.st_ino + try: + vino = me._inomap[inoidx] + except KeyError: + suffix = '' + seq = 0 + while True: + vino = '%08x' % (Z.crc32(fi.name + suffix) & 0xffffffff) + if vino not in me._vinomap: break + suffix = '\0%d' % seq + seq += 1 + me._inomap[inoidx] = vino + if h: info = h + else: info = '[%-*s]' % (2*me._hsz - 2, fmt.info()) + print '%s %8s %6s %-12s %-20s %20s %s' % ( + info, vino, fmt.mode(), fmt.owner(), + fmt.mtime(), fmt.size(), fmt.name()) + +###-------------------------------------------------------------------------- +### Main program. + +FMTMAP = { + 'rsync': lambda f: enum_rsync(stdin, f), + 'find0': lambda f: enum_find0(stdin, f) +} +op = OP.OptionParser( + usage = '%prog [-a] [-c CACHE] [-f FORMAT] [-H HASH] [FILE ...]', + version = '%%prog, version %s' % VERSION, + description = '''\ +Print a digest of a filesystem (or a collection of specified files) to +standard output. The idea is that the digest should be mostly /complete/ +(i.e., any `interesting\' change to the filesystem results in a different +digest) and /canonical/ (i.e., identical filesystem contents result in +identical output). +''') + +for short, long, props in [ + ('-a', '--all', { 'action': 'store_true', 'dest': 'all', + 'help': 'clear cache of all files not seen' }), + ('-c', '--cache', { 'dest': 'cache', 'metavar': 'FILE', + 'help': 'use FILE as a cache for file hashes' }), + ('-f', '--files', { 'dest': 'files', 'metavar': 'FORMAT', + 'type': 'choice', 'choices': FMTMAP.keys(), + 'help': 'read files to report in the given FORMAT' }), + ('-H', '--hash', { 'dest': 'hash', 'metavar': 'HASH', + ##'type': 'choice', 'choices': H.algorithms, + 'help': 'use HASH as the hash function' })]: + op.add_option(short, long, **props) +opts, args = op.parse_args(argv) + +if not opts.files and len(args) <= 1: + die("no filename sources: nothing to do") +db = HashCache(opts.cache, opts.hash) +if opts.all: + db.reset() +rep = Reporter(db) +if opts.files: + FMTMAP[opts.files](rep.file) +for dir in args[1:]: + enum_walk(dir, rep.file) +if opts.all: + db.prune() +db.flush() + +###----- That's all, folks -------------------------------------------------- diff --git a/mount-backup-volume b/mount-backup-volume new file mode 100755 index 0000000..6bb93cb --- /dev/null +++ b/mount-backup-volume @@ -0,0 +1,31 @@ +#! /bin/sh -ex + +: ${vgtag=@backup} ${vgprefix=vg-backup-} +: ${mntbkpdir=/mnt/bkp} +: ${STOREDIR=$mntbkpdir/store} ${METADIR=$mntbkpdir/meta} + +vgs=$(vgs --noheadings -oname) +found=nil +for vg in $vgs; do + case "$found,$vg" in + nil,$vgprefix*) + found=t + tag=${vg#$vgprefix} + ;; + t,$vgprefix*) + echo >&2 "$0: multiple backup volumes attached" + exit 1 + ;; + esac +done +case $found in + nil) echo >&2 "$0: no backup volumes attached"; exit 1 ;; +esac +vg=$vgprefix$tag + +vgchange -ay $vg +mount /dev/$vg/meta $METADIR + +cryptop decrypt backup <$METADIR/crypt.blob | + cryptsetup luksOpen --key-file=- /dev/$vg/crypt cbackup +mount /dev/mapper/cbackup $STOREDIR diff --git a/rfreezefs.8 b/rfreezefs.8 new file mode 100644 index 0000000..9e807f2 --- /dev/null +++ b/rfreezefs.8 @@ -0,0 +1,317 @@ +.TH rfreezefs 8 "October 2011" "rsync-backup" +.SH NAME +rfreezefs \- freeze a filesystem safely +.SH SYNOPSIS +.B rfreezefs +.RB [ \-n ] +.RB [ \-a +.IR address ] +.RB [ \-p +.IR loport [\fB\- hiport ]] +.I filesystem +\&... +.SH DESCRIPTION +The +.B rfreezefs +program freezes one or more mounted filesystems for a period of time, +and then thaws them. For more detail on what this means, why you'd want +to, and how you might go about using +.B rfreezefs +to do it, see below. +.PP +The following command-line options are recognized. +.TP +.B "\-h, \-\-help" +Writes a help message to standard output, and exits with status 0. +.TP +.B "\-v, \-\-version" +Writes the version number to standard output, and exits with status 0. +.TP +.B "\-u, \-\-usage" +Writes a command-line usage synopsis to standard output, and exits with +status 0. +.TP +.BI "\-a, \-\-address=" address +Listen only for incoming connections to the given +.IR address . +The default is to listen for connections to any local address. +.TP +.B "\-n, \-\-not-really" +Don't actually freeze or thaw any filesystems; instead, write messages +to standard error explaining what would be done. +.TP +.BI "\-p, \-\-port-range=" loport\fR[ \- hiport \fR]] +Listen for incoming connections on a port between +.I loport +and +.IR hiport . +If +.I hiport +is omitted, listen for connections only on +.IR loport . +The default is to allow the kernel a free choice of local port number. +.PP +The +.I filesystem +arguments name the filesystems to be frozen. There must be at least one +such argument. It's conventional to name the filesystem mount points, +though actually any file or directory in the filesystem will do. The +files are opened read-only. +.PP +The +.B rfreezefs +program starts, parses its command line, opens the named files, and +creates a listening TCP socket according to the command-line options. +It then prints a sequence of lines to standard output, which may have +one of the following forms. +.TP +.BI "PORT " port +Announces the TCP +.I port +number on which that +.B rfreezefs +is listening for incoming connections. +.TP +.BI "TOKEN " label " " token +Declares a `token': a randomly chosen string which is to be used in the +network connection. The token's value is +.IR token : +token values are a sequence of non-whitespace printable ASCII +characters, but their precise structure is not specified. The token +value will have the meaning given by the +.IR label , +which is one of the token labels described below. +.TP +.B READY +Marks the end of the lines and announces that +.B rfreezefs +is ready to accept connections. +.PP +These lines may be sent in any order, except that +.B READY +is always last. There may be many +.B TOKEN +lines. +.PP +Network communications use a simple plain-text line-oriented protocol. +Each line consists of a token, optionally followed by a carriage return +(code 13), followed by a linefeed (code 10). No other whitespace is +permitted. The tokens allowed are precisely those announced in the +.B TOKEN +lines written to +.BR rfreezefs 's +standard output. Furthermore, only certain tokens are valid at +particular points in the protocol. For reference, the token labels, and +the meanings of the corresponding tokens, are as follows. +.TP +.B FREEZE +Sent by a client to freeze the filesystems. This must be the first +token transmitted by the client. On receipt, +.B rfreezefs +will close its listening socket and any other client connections. It +will then freeze the filesystems. +.TP +.B FROZEN +Sent by +.B rfreezefs +to indicate successful freezing of the filesystem. +.TP +.B KEEPALIVE +Sent periodically by the client to prevent filesystems being thawed due +to a timeout. No explicit acknowledgement is sent. +.TP +.B THAW +Sent by the client to request thawing of the filesystems. +.TP +.B THAWED +Sent by +.B rfreezefs to indicate successful thawing of the filesystems in response to +.BR THAW . +.PP +The high-level structure of the protocol is then as follows: the client +sends +.BR FREEZE ; +the server freezes and responds with +.BR FROZEN ; +the client optionally sends +.B KEEPALIVE +at intervals; the client finally sends +.BR THAW ; +and the server responds with +.B THAWED +and drops the connection. +.PP +If sufficient time passes without +.B rfreezefs +receiving either +.B THAW +or +.B KEEPALIVE +tokens, or an invalid token is received, or it receives one of a number +of signals, currently +.BR SIGINT , +.BR SIGQUIT , +.BR SIGTERM , +.BR SIGHUP , +.BR SIGALRM , +.BR SIGILL , +.BR SIGSEGV , +.BR SIGBUS , +.BR SIGFPE , +or +.BR SIGABRT , +.B rfreezefs +will thaw the filesystems and report a failure. +.PP +Diagnostics are reported to standard error. Exit statuses have specific +meanings: +.TP +.B 0 +Successful completion. Filesystems were frozen and thawed as required. +.TP +.B 1 +Problem with command-line arguments. No filesystems were frozen. +.TP +.B 2 +Environmental problem, typically a system call failure: e.g., a file +failed to open, or there was a problem with the network communications. +Either no filesystems were frozen, or all filesystems were successfully +thawed again. +.TP +.B 3 +Timeout or invalid data. Either no connections containing the cookie +were made in time, or no data was received for a long enough period +after the filesystems were frozen, or an invalid token was received. In +the first case, no filesystems were frozen; in the other two cases, the +filesystems were successfully thawed. +.TP +.B 4 +Crash. The +.B rfreezefs +program received a fatal signal after it had started to freeze +filesystems. Under these circumstances, it thaws the filesystems, +removes the signal handler, and sends itself the signal again, but if +that doesn't work then +.B rfreezefs +exits with this status code. All frozen filesystems were successfully +thawed again. +.TP +.B 112 +Failure during filesystem thaw (mnemonic: European emergency number). +Some filesystems +.I failed +to thaw, and are still frozen. You might have some joy with +.BR SysRq-j , +though in the author's experience that doesn't work and you'll probably +have to reboot. At least your filesystems are consistent... +.SS Background +When frozen, a filesystem's backing block device is put in a consistent +state (as if unmounted), and write operations to it are delayed until +the filesystem is thawed again. In the meantime, it's possible to take +a consistent snapshot of the block device. When a filesystem is +directly mounted on an LVM logical volume, the kernel detects this +situation and automatically freezes the filesystem while the snapshot is +being prepared. If the logical volume and filesystem are on separate +hosts, though, the filesystem must be frozen manually, which is why +.B rfreezefs +is useful. +.PP +The idea is to run +.B rfreezefs +using +.BR ssh (1) +or +.BR userv (1), +or some other means of acquiring the necessary privilege level. You +read the port number and tokens, connect to the socket, and send the +.B FREEZE +token followed by a newline. You now wait to receive the +.B FROZEN +token from +.BR rfreezefs . +Once you have received this, the filesystems are frozen: you can safely +take snapshots. If this will take an extended amount of time, you +should send +.B KEEPALIVE +tokens to the connection at intervals in order to prevent +.B rfreezefs +from timing out and thawing the filesystems (but see the +.B "Security notes" +below). When your snapshot is prepared, sent the +.B THAW +token, and wait for the +.B THAWED +token in response. If this is received, the snapshot was completed +successfully and the filesystems are properly thawed again. If you +don't receive the +.B THAWED +token then something bad might have happened (e.g., the filesystem might +have been prematurely thawed) and the snapshot is suspect. If the exit +status is 112 then at least one filesystem is still frozen and some +emergency action is needed. If you can't retrieve the exit status then +it's possible that your transport is blocked for trying to write to the +frozen filesystem (this especially likely if +.B / +or +.B /var +is frozen) and you should react as if the status was 112. +.SS Security notes +The +.B rfreezefs +program uses randomly chosen tokens to form a simple code which is +revealed to the caller. It is assumed that this information is kept +secret from adversaries, e.g., by ensuring that it is only transmitted +over local pipes (as used by +.BR userv (1)) +and/or secure network transports such as SSH (see +.BR ssh (1)). +The author believes that the worst possible outcome is that the host +wedges up because an important filesystem is frozen, and +.B rfreezefs +therefore strives to prevent that from happening. In particular, +cryptographic transport implementations such as SSH may attempt to log +messages to frozen filesystems or otherwise wedge themselves: +.B rfreezefs +deliberately uses only kernel-implemented transports for its +communication needs once the filesystems are frozen. +.PP +Most of the tokens are used at most once in the protocol. In +particular, the +.B FROZEN +token can't be sent by an adversary in advance of the filesystem being +frozen, since (under the assumption that the tokens are kept secret) it +only revealed in the clear after a successful freeze. Similarly, the +.B THAWED +token is only transmitted if the filesystems are thawed as a result of a +.B THAW +request (rather than a dropped connection, timeout, or some other +problem). If the client only sends the +.B THAW +request once its snapshot is complete, then a +.B THAWED +response indicates that the filesystems remained frozen until the +snapshot was indeed completed and therefore the snapshot is consistent. +.PP +The exception is the +.B KEEPALIVE +token, which may be sent repeatedly. After it is first revealed, an +adversary can hijack the connection and replay the +.B KEEPALIVE +token to keep the filesystems frozen indefinitely. You can recover from +this by severing the connection somehow, or by sending +.B rfreezefs +a signal. It is therefore recommended that +.B KEEPALIVE +tokens not be sent unless necessary. The timeout is currently set to +60s, which ought to be adequate for most snapshot mechanisms. +.SH BUGS +There ought to be a better one-time-token protocol for keepalives. I +want to keep cryptography out of this program, though. +.SH SEE ALSO +.BR fsfreeze (8), +.BR random (4), +.BR lvm (8), +.BR ssh (1), +.BR userv (1). +.SH AUTHOR +Mark Wooding, diff --git a/rfreezefs.c b/rfreezefs.c new file mode 100644 index 0000000..c390c36 --- /dev/null +++ b/rfreezefs.c @@ -0,0 +1,636 @@ +/* -*-c-*- + * + * Freeze a file system under remote control + * + * (c) 2012 Mark Wooding + */ + +/*----- Licensing notice --------------------------------------------------* + * + * This file is part of the `rsync-backup' program. + * + * rsync-backup is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * rsync-backup is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with rsync-backup; if not, write to the Free Software Foundation, + * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +/*----- Header files ------------------------------------------------------*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/*----- Magic constants ---------------------------------------------------*/ + +#define COOKIESZ 16 /* Size of authentication cookie */ +#define TO_CONNECT 30 /* Timeout for incoming connection */ +#define TO_KEEPALIVE 60 /* Timeout between keepalives */ + +/*----- Utility functions -------------------------------------------------*/ + +static int getuint(const char *p, const char *q) +{ + unsigned long i; + int e = errno; + char *qq; + + if (!q) q = p + strlen(p); + errno = 0; + i = strtoul(p, &qq, 0); + if (errno || qq < q || i > INT_MAX) + die(1, "invalid integer `%s'", p); + errno = e; + return ((int)i); +} + +#ifdef DEBUG +# define D(x) x +#else +# define D(x) +#endif + +/*----- Token management --------------------------------------------------*/ + +struct token { + const char *label; + char tok[(COOKIESZ + 2)*4/3 + 1]; +}; + +#define TOKENS(_) \ + _(FREEZE) \ + _(FROZEN) \ + _(KEEPALIVE) \ + _(THAW) \ + _(THAWED) + +enum { +#define ENUM(tok) T_##tok, + TOKENS(ENUM) +#undef ENUM + T_LIMIT +}; + +enum { +#define MASK(tok) TF_##tok = 1u << T_##tok, + TOKENS(MASK) +#undef ENUM + TF_ALL = (1u << T_LIMIT) - 1u +}; + +static struct token toktab[] = { +#define INIT(tok) { #tok }, + TOKENS(INIT) +#undef INIT + { 0 } +}; + +static void inittoks(void) +{ + static struct token *t, *tt; + unsigned char buf[COOKIESZ]; + int fd; + ssize_t n; + base64_ctx bc; + dstr d = DSTR_INIT; + + if ((fd = open("/dev/urandom", O_RDONLY)) < 0) + die(2, "open (urandom): %s", strerror(errno)); + + for (t = toktab; t->label; t++) { + again: + n = read(fd, buf, COOKIESZ); + if (n < 0) die(2, "read (urandom): %s", strerror(errno)); + else if (n < COOKIESZ) die(2, "read (urandom): short read"); + base64_init(&bc); + base64_encode(&bc, buf, COOKIESZ, &d); + base64_encode(&bc, 0, 0, &d); + dstr_putz(&d); + + for (tt = toktab; tt < t; tt++) { + if (strcmp(d.buf, tt->tok) == 0) + goto again; + } + + assert(d.len < sizeof(t->tok)); + memcpy(t->tok, d.buf, d.len + 1); + dstr_reset(&d); + } +} + +struct tokmatch { + unsigned tf; /* Possible token matches */ + size_t o; /* Offset into token string */ + unsigned f; /* Flags */ +#define TMF_CR 1u /* Seen trailing carriage-return */ +}; + +static void tokmatch_init(struct tokmatch *tm) + { tm->tf = TF_ALL; tm->o = 0; tm->f = 0; } + +static int tokmatch_update(struct tokmatch *tm, int ch) +{ + const struct token *t; + unsigned tf; + + switch (ch) { + case '\n': + for (t = toktab, tf = 1; t->label; t++, tf <<= 1) { + if ((tm->tf & tf) && !t->tok[tm->o]) + return (tf); + } + return (-1); + case '\r': + for (t = toktab, tf = 1; t->label; t++, tf <<= 1) { + if ((tm->tf & tf) && !t->tok[tm->o] && !(tm->f & TMF_CR)) + tm->f |= TMF_CR; + else + tm->tf &= ~tf; + } + break; + default: + for (t = toktab, tf = 1; t->label; t++, tf <<= 1) { + if ((tm->tf & tf) && ch != t->tok[tm->o]) + tm->tf &= ~tf; + } + tm->o++; + break; + } + return (0); +} + +static int writetok(unsigned i, int fd) +{ + static const char nl = '\n'; + const struct token *t = &toktab[i]; + size_t n = strlen(t->tok); + + errno = EIO; + if (write(fd, t->tok, n) < n || + write(fd, &nl, 1) < 1) + return (-1); + return (0); +} + +/*----- Data structures ---------------------------------------------------*/ + +struct client { + struct client *next; /* Links in the client chain */ + int fd; /* File descriptor for socket */ + struct tokmatch tm; /* Token matching context */ +}; + +/*----- Static variables --------------------------------------------------*/ + +static int *fs; /* File descriptors for targets */ +static char **fsname; /* File system names */ +static size_t nfs; /* Number of descriptors */ + +/*----- Cleanup -----------------------------------------------------------*/ + +#define EOM ((char *)0) +static void emerg(const char *msg,...) +{ + va_list ap; + +#define MSG(m) \ + do { const char *m_ = m; if (write(2, m_, strlen(m_))); } while (0) + + va_start(ap, msg); + MSG(QUIS); MSG(": "); + do { + MSG(msg); + msg = va_arg(ap, const char *); + } while (msg != EOM); + MSG("\n"); + +#undef MSG +} + +static void partial_cleanup(size_t n) +{ + int i; + int bad = 0; + + for (i = 0; i < nfs; i++) { + if (fs[i] == -1) + emerg("not really thawing ", fsname[i], EOM); + else if (fs[i] != -2) { + if (ioctl(fs[i], FITHAW, 0)) { + emerg("VERY BAD! failed to thaw ", + fsname[i], ": ", strerror(errno), EOM); + bad = 1; + } + close(fs[i]); + } + fs[i] = -2; + } + if (bad) _exit(112); +} + +static void cleanup(void) { partial_cleanup(nfs); } + +static int sigcatch[] = { + SIGINT, SIGQUIT, SIGTERM, SIGHUP, SIGALRM, + SIGILL, SIGSEGV, SIGBUS, SIGFPE, SIGABRT +}; + +static void sigmumble(int sig) +{ + sigset_t ss; + + cleanup(); + emerg(strsignal(sig), 0); + + signal(sig, SIG_DFL); + sigemptyset(&ss); sigaddset(&ss, sig); + sigprocmask(SIG_UNBLOCK, &ss, 0); + raise(sig); + _exit(4); +} + +/*----- Help functions ----------------------------------------------------*/ + +static void version(FILE *fp) + { pquis(fp, "$, " PACKAGE " version " VERSION "\n"); } +static void usage(FILE *fp) + { pquis(fp, "Usage: $ [-n] [-a ADDR] [-p LOPORT[-HIPORT]] FILSYS ...\n"); } + +static void help(FILE *fp) +{ + version(fp); putc('\n', fp); + usage(fp); + fputs("\n\ +Freezes a filesystem temporarily, with some measure of safety.\n\ +\n\ +The program listens for connections on a TCP port, and prints a line\n\ +\n\ + PORT COOKIE\n\ +\n\ +to standard output. You must connect to this PORT and send the COOKIE\n\ +followed by a newline within a short period of time. The filesystems\n\ +will then be frozen, and `OK' written to the connection. In order to\n\ +keep the file system frozen, you must keep the connection open, and\n\ +feed data into it. If the connection closes, or no data is received\n\ +within a set period of time, or the program receives one of a variety\n\ +of signals or otherwise becomes unhappy, the filesystems are thawed again.\n\ +\n\ +Options:\n\ +\n\ +-h, --help Print this help text.\n\ +-v, --version Print the program version number.\n\ +-u, --usage Print a short usage message.\n\ +\n\ +-a, --address=ADDR Listen only on ADDR.\n\ +-n, --not-really Don't really freeze or thaw filesystems.\n\ +-p, --port-range=LO[-HI] Select a port number between LO and HI.\n\ + If HI is omitted, choose only LO.\n\ +", fp); +} + +/*----- Main program ------------------------------------------------------*/ + +int main(int argc, char *argv[]) +{ + char buf[256]; + int loport = -1, hiport = -1; + int sk, fd, maxfd; + struct sockaddr_in sin; + socklen_t sasz; + struct hostent *h; + const char *p, *q; + struct timeval now, when, delta; + struct client *clients = 0, *c, **cc; + const struct token *t; + struct tokmatch tm; + fd_set fdin; + int i; + ssize_t n; + unsigned f = 0; +#define f_bogus 0x01u +#define f_notreally 0x02u + + ego(argv[0]); + sub_init(); + + /* --- Partially initialize the socket address --- */ + + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = INADDR_ANY; + sin.sin_port = 0; + + /* --- Parse the command line --- */ + + for (;;) { + static struct option opts[] = { + { "help", 0, 0, 'h' }, + { "version", 0, 0, 'v' }, + { "usage", 0, 0, 'u' }, + { "address", OPTF_ARGREQ, 0, 'a' }, + { "not-really", 0, 0, 'n' }, + { "port-range", OPTF_ARGREQ, 0, 'p' }, + { 0, 0, 0, 0 } + }; + + if ((i = mdwopt(argc, argv, "hvua:np:", opts, 0, 0, 0)) < 0) break; + switch (i) { + case 'h': help(stdout); exit(0); + case 'v': version(stdout); exit(0); + case 'u': usage(stdout); exit(0); + case 'a': + if ((h = gethostbyname(optarg)) == 0) { + die(1, "failed to resolve address `%s': %s", + optarg, hstrerror(h_errno)); + } + if (h->h_addrtype != AF_INET) + die(1, "unexpected address type resolving `%s'", optarg); + assert(h->h_length == sizeof(sin.sin_addr)); + memcpy(&sin.sin_addr, h->h_addr, sizeof(sin.sin_addr)); + break; + case 'n': f |= f_notreally; break; + case 'p': + if ((p = strchr(optarg, '-')) == 0) + loport = hiport = getuint(optarg, 0); + else { + loport = getuint(optarg, p); + hiport = getuint(p + 1, 0); + } + break; + default: f |= f_bogus; break; + } + } + if (f & f_bogus) { usage(stderr); exit(1); } + if (optind >= argc) { usage(stderr); exit(1); } + + /* --- Open the file systems --- */ + + nfs = argc - optind; + fsname = &argv[optind]; + fs = xmalloc(nfs*sizeof(*fs)); + for (i = 0; i < nfs; i++) { + if ((fs[i] = open(fsname[i], O_RDONLY)) < 0) + die(2, "open (%s): %s", fsname[i], strerror(errno)); + } + + if (f & f_notreally) { + for (i = 0; i < nfs; i++) { + close(fs[i]); + fs[i] = -1; + } + } + + /* --- Generate random tokens --- */ + + inittoks(); + + /* --- Create the listening socket --- */ + + if ((sk = socket(PF_INET, SOCK_STREAM, 0)) < 0) + die(2, "socket: %s", strerror(errno)); + i = 1; + if (setsockopt(sk, SOL_SOCKET, SO_REUSEADDR, &i, sizeof(i))) + die(2, "setsockopt (reuseaddr): %s", strerror(errno)); + if (fdflags(sk, O_NONBLOCK, O_NONBLOCK, FD_CLOEXEC, FD_CLOEXEC)) + die(2, "fdflags: %s", strerror(errno)); + if (loport < 0 || loport == hiport) { + if (loport >= 0) sin.sin_port = htons(loport); + if (bind(sk, (struct sockaddr *)&sin, sizeof(sin))) + die(2, "bind: %s", strerror(errno)); + } else if (hiport != loport) { + for (i = loport; i <= hiport; i++) { + sin.sin_port = htons(i); + if (bind(sk, (struct sockaddr *)&sin, sizeof(sin)) >= 0) break; + else if (errno != EADDRINUSE) + die(2, "bind: %s", strerror(errno)); + } + if (i > hiport) die(2, "bind: all ports in use"); + } + if (listen(sk, 5)) die(2, "listen: %s", strerror(errno)); + + /* --- Tell the caller how to connect to us, and start the timer --- */ + + sasz = sizeof(sin); + if (getsockname(sk, (struct sockaddr *)&sin, &sasz)) + die(2, "getsockname (listen): %s", strerror(errno)); + printf("PORT %d\n", ntohs(sin.sin_port)); + for (t = toktab; t->label; t++) + printf("TOKEN %s %s\n", t->label, t->tok); + printf("READY\n"); + if (fflush(stdout) || ferror(stdout)) + die(2, "write (stdout, rubric): %s", strerror(errno)); + gettimeofday(&now, 0); TV_ADDL(&when, &now, TO_CONNECT, 0); + + /* --- Collect incoming connections, and check for the cookie --- * + * + * This is the tricky part. + */ + + for (;;) { + FD_ZERO(&fdin); + FD_SET(sk, &fdin); + maxfd = sk; + for (c = clients; c; c = c->next) { + FD_SET(c->fd, &fdin); + if (c->fd > maxfd) maxfd = c->fd; + } + TV_SUB(&delta, &when, &now); + if (select(maxfd + 1, &fdin, 0, 0, &delta) < 0) + die(2, "select (accept): %s", strerror(errno)); + gettimeofday(&now, 0); + + if (TV_CMP(&now, >=, &when)) die(3, "timeout (accept)"); + + if (FD_ISSET(sk, &fdin)) { + sasz = sizeof(sin); + fd = accept(sk, (struct sockaddr *)&sin, &sasz); + if (fd >= 0) { + if (fdflags(fd, O_NONBLOCK, O_NONBLOCK, FD_CLOEXEC, FD_CLOEXEC) < 0) + die(2, "fdflags: %s", strerror(errno)); + c = CREATE(struct client); + c->next = clients; c->fd = fd; tokmatch_init(&c->tm); + clients = c; + } +#ifdef DEBUG + else if (errno != EAGAIN) + moan("accept: %s", strerror(errno)); +#endif + } + + for (cc = &clients; *cc;) { + c = *cc; + if (!FD_ISSET(c->fd, &fdin)) goto next_client; + n = read(c->fd, buf, sizeof(buf)); + if (!n) goto disconn; + else if (n < 0) { + if (errno == EAGAIN) goto next_client; + D( moan("read (client; auth): %s", strerror(errno)); ) + goto disconn; + } else { + for (p = buf, q = p + n; p < q; p++) { + switch (tokmatch_update(&c->tm, *p)) { + case 0: break; + case TF_FREEZE: goto connected; + default: + D( moan("bad token from client"); ) + goto disconn; + } + } + } + + next_client: + cc = &c->next; + continue; + + disconn: + close(c->fd); + *cc = c->next; + DESTROY(c); + continue; + } + } + +connected: + close(sk); sk = c->fd; + while (clients) { + if (clients->fd != sk) close(clients->fd); + c = clients->next; + DESTROY(clients); + clients = c; + } + + /* --- Establish signal handlers --- * + * + * Hopefully this will prevent bad things happening if we have an accident. + */ + + for (i = 0; i < sizeof(sigcatch)/sizeof(sigcatch[0]); i++) { + if (signal(sigcatch[i], sigmumble) == SIG_ERR) + die(2, "signal (%d): %s", i, strerror(errno)); + } + atexit(cleanup); + + /* --- Prevent the OOM killer from clobbering us --- */ + + if ((fd = open("/proc/self/oom_adj", O_WRONLY)) < 0 || + write(fd, "-17\n", 4) < 4 || + close(fd)) + die(2, "set oom_adj: %s", strerror(errno)); + + /* --- Actually freeze the filesystem --- */ + + for (i = 0; i < nfs; i++) { + if (fs[i] == -1) + moan("not really freezing %s", fsname[i]); + else { + if (ioctl(fs[i], FIFREEZE, 0) < 0) { + partial_cleanup(i); + die(2, "ioctl (freeze %s): %s", fsname[i], strerror(errno)); + } + } + } + if (writetok(T_FROZEN, sk)) { + cleanup(); + die(2, "write (frozen): %s", strerror(errno)); + } + + /* --- Now wait for the other end to detach --- */ + + tokmatch_init(&tm); + TV_ADDL(&when, &now, TO_KEEPALIVE, 0); + for (p++; p < q; p++) { + switch (tokmatch_update(&tm, *p)) { + case 0: break; + case TF_KEEPALIVE: tokmatch_init(&tm); break; + case TF_THAW: goto done; + default: cleanup(); die(3, "unknown token (keepalive)"); + } + } + for (;;) { + FD_ZERO(&fdin); + FD_SET(sk, &fdin); + TV_SUB(&delta, &when, &now); + if (select(sk + 1, &fdin, 0, 0, &delta) < 0) { + cleanup(); + die(2, "select (keepalive): %s", strerror(errno)); + } + + gettimeofday(&now, 0); + if (TV_CMP(&now, >, &when)) { + cleanup(); die(3, "timeout (keepalive)"); + } + if (FD_ISSET(sk, &fdin)) { + n = read(sk, buf, sizeof(buf)); + if (!n) { cleanup(); die(3, "end-of-file (keepalive)"); } + else if (n < 0) { + if (errno == EAGAIN) ; + else { + cleanup(); + die(2, "read (client, keepalive): %s", strerror(errno)); + } + } else { + for (p = buf, q = p + n; p < q; p++) { + switch (tokmatch_update(&tm, *p)) { + case 0: break; + case TF_KEEPALIVE: + TV_ADDL(&when, &now, TO_KEEPALIVE, 0); + tokmatch_init(&tm); + break; + case TF_THAW: + goto done; + default: + cleanup(); + die(3, "unknown token (keepalive)"); + } + } + } + } + } + +done: + cleanup(); + if (writetok(T_THAWED, sk)) + die(2, "write (thaw): %s", strerror(errno)); + close(sk); + return (0); +} + +/*----- That's all, folks -------------------------------------------------*/ diff --git a/rsync-backup.8 b/rsync-backup.8 new file mode 100644 index 0000000..3d2cffc --- /dev/null +++ b/rsync-backup.8 @@ -0,0 +1,263 @@ +.TH rsync-backup 8 "7 October 2012" rsync-backup +.SH SYNOPSIS +.B rsync-backup +.RB [ \-v ] +.RB [ \-c +.IR config-file ] +.SH DESCRIPTION +The +.B rsync-backup +script is a backup program of the currently popular +.RB ` rsync (1) +.BR \-\-link-dest ' +variety. It uses +.BR rsync 's +ability to create hardlinks from (apparently) similar existing local +trees to make incremental dumps efficient, even from remote sources. +Restoring files is easy because the backups created are just directories +full of files, exactly as they were on the source \(en and this is +verified using the +.BR fshash (1) +program. +.PP +The script does more than just running +.BR rsync . +It is also responsible for creating and removing snapshots of volumes to +be backed up, and expiring old dumps according to a user-specified +retention policy. +.SS Installation +The idea is that the +.B rsync-backup +script should be installed and run on a central backup server with local +access to the backup volumes. +.PP +The script should be run with full (root) privileges, so that it can +correctly record file ownership information. The server should also be +able to connect via +.BR ssh (1) +to the client machines, and run processes there as root. (This is not a +security disaster. Remember that the backup server is, in the end, +responsible for the integrity of the backup data. A dishonest backup +server can easily compromise a client which is being restored from +corrupt backup data.) +.PP +The + + +.SS Configuration commands +The configuration file is simply a Bash shell fragment: configuration +commands are shell functions. +.TP +.BI "backup " "fs\fR[:\fIfsarg\fR] ..." +Back up the named filesystems. The corresponding +.IR fsarg s +may be required by the snapshot type. +.TP +.BI "host " host +Future +.B backup +commands will back up filesystems on the named +.IR host . +This clears the +.B like +list. +.TP +.BI "like " "host\fR ..." +Declare that subsequent filesystems are `similar' to like-named +filesystems on the named +.IR host s, +and that +.B rsync +should use those trees as potential sources of hardlinkable files. Be +careful when using this option without +.BR rsync 's +.B \-\-checksum +option: an erroneous hardlink will cause the backup to fail. (The +backup won't be left silently incorrect.) +.TP +.BI "retain " frequency " " duration +Define part a backup retention policy: backup trees of the +.I frequency +should be kept for the +.IR duration . +The +.I frequency +can be +.BR daily , +.BR weekly , +.BR monthly , +or +.B annually +(or +.BR yearly , +which means the same); the +.I duration +may be any of +.BR week , +.BR month , +.BR year , +or +.BR forever . +Expiry considers each existing dump against the policy lines in order: +the last applicable line determines the dump's fate \(en so you should +probably write the lines in decreasing order of duration. +.TP +.BI "snap " type " " \fR[\fIargs\fR...] +Use the snapshot +.I type +for subsequent backups. Some snapshot types require additional +arguments, which may be supplied here. +.SS Configuration variables +The following shell variables may be overridden by the configuration +file. +.TP +.B MAXLOG +The number of log files to be kept for each filesystem. Old logfiles +are deleted to keep the total number below this bound. The default +value is 14. +.TP +.B RSYNCOPTS +Command-line options to pass to +.BR rsync (1) +in addition to the basic set: +.B \-\-archive \-\-hard-links \-\-numeric-ids \-\-del \-\-sparse +.B \-\-compress \-\-one-file-system \-\-partial +.B \-\-filter="dir-merge .rsync-backup" +The default is +.BR \-\-verbose . +.TP +.B SNAPDIR +LVM (and +.BR rfreezefs ) +snapshots are mounted on subdirectories below the +.B SNAPDIR +.IR "on backup clients" . +The default is +.IB mntbkpdir /snap +where +.I mntbkpdir +is the backup mount directory configured at build time. +.TP +.B SNAPSIZE +The volume size option to pass to +.BR lvcreate (8) +when creating a snapshot. The default is +.B \-l10%ORIGIN +which seems to work fairly well. +.TP +.B STOREDIR +Where the actual backup trees should be stored. See the section on +.B Archive structure +below. +The default is +.IB mntbkpdir /store +where +.I mntbkpdir +is the backup mount directory configured at build time. +.TP +.B HASH +The hash function to use for verifying archive integrity. This is +passed to the +.B \-H +option of +.BR fshash , +so it must name one of the hash functions supported by your Python's +.B hashlib +module. The default is +.BR sha256 . +.SS Hook functions +The configuration file may define shell functions to perform custom +actions at various points in the backup process. +.TP +.BI "backup_precommit_hook " host " " fs " " date +Called after a backup has been verified complete and about to be +committed. The backup tree is in +.B new +in the current directory, and the +.B fshash +manifest is in +.BR new.fshash . +A typical action would be to create a digital signature on the +manifest. +.TP +.BI "backup_commit_hook " host " " fs " " date +Called during the commit procedure. The backup tree and manifest have +been renamed into their proper places. Typically one would use this +hook to rename files created by the +.B backup_precommit_hook +function. +.TP +.BR "whine " [ \-n ] " " \fItext\fR... +Called to report `interesting' events when the +.B \-v +option is in force. The default action is to echo the +.I text +to (what was initially) standard output, followed by a newline unless +.B \-n +is given. +.SS Snapshot types +The following snapshot types are available. +.TP +.B live +A trivial snapshot type: attempts to back up a live filesystem. How +well this works depends on how active the filesystem is. If files +change while the dump is in progress then the +.B fshash +verification will likely fail. Backups using this snapshot type must +specify the filesystem mount point as the +.IR fsarg . +.TP +.B ro +A slightly less trivial snapshot type: make the filesystem read-only +while the dump is in progress. Backups using this snapshot type must +specify the filesystem mount point as the +.IR fsarg . +.TP +.BI "lvm " vg +Create snapshots using LVM. The snapshot argument is interpreted as the +relevant volume group. The filesystem name is interpreted as the origin +volume name; the snapshot will be called +.IB fs .bkp +and mounted on +.IB SNAPDIR / fs \fR; +space will be allocated to it according to the +.I SNAPSIZE +variable. +.TP +.BI "rfreezefs " client " " vg +This gets complicated. Suppose that a server has an LVM volume group, +and exports (somehow) a logical volume to a client. Examples are a host +providing a virtual disk to a guest, or a server providing +network-attached storage to a client. The server can create a snapshot +of the volume using LVM, but must synchronize with the client to ensure +that the filesystem image captured in the snapshot is clean. The +.BR rfreezefs (8) +program should be installed on the client to perform this rather +delicate synchronization. Declare the server using the +.B host +command as usual; pass the client's name as the +.I client +and the +server's volume group name as the +.I vg +snapshot arguments. Finally, backups using this snapshot type must +specify the filesystem mount point (or, actually, any file in the +filesystem) on the client, as the +.IR fsarg . +.PP +Additional snapshot types can be defined in the configuration file. A +snapshot type requires two shell functions. +.TP +.BI snap_ type " " snapargs " " fs " " fsarg +Create the snapshot, and write the mountpoint (on the client host) to +standard output, in a form suitable as an argument to +.BR rsync . +.TP +.BI unsnap_ type " " snapargs " " fs " " fsarg +Remove the snapshot. +.PP +There are a number of utility functions which can be used by snapshot +type handlers: please see the script for details. Please send the +author interesting snapshot handlers for inclusion in the main +distribution. +.SS Archive structure diff --git a/rsync-backup.in b/rsync-backup.in new file mode 100644 index 0000000..5c9a54c --- /dev/null +++ b/rsync-backup.in @@ -0,0 +1,730 @@ +#! @BASH@ +### +### Backup script +### +### (c) 2012 Mark Wooding +### + +###----- Licensing notice --------------------------------------------------- +### +### This file is part of the `rsync-backup' program. +### +### rsync-backup is free software; you can redistribute it and/or modify +### it under the terms of the GNU General Public License as published by +### the Free Software Foundation; either version 2 of the License, or +### (at your option) any later version. +### +### rsync-backup is distributed in the hope that it will be useful, +### but WITHOUT ANY WARRANTY; without even the implied warranty of +### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +### GNU General Public License for more details. +### +### You should have received a copy of the GNU General Public License +### along with rsync-backup; if not, write to the Free Software Foundation, +### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +set -e + +thishost=$(hostname -s) +quis=${0##*/} + +VERSION=@VERSION@ +mntbkpdir=@mntbkpdir@ +logdir=@logdir@ +fshashdir=@fshashdir@ +conf=@sysconfdir@/rsync-backup.conf + +verbose=: + +###-------------------------------------------------------------------------- +### Utility functions. + +RSYNCOPTS="--verbose" + +do_rsync () { + ## Run rsync(1) in an appropriate manner. Configuration should ovrride + ## this or set $RSYNCOPTS if it wants to do something weirder. Arguments + ## to this function are passed on to rsync. + + rsync \ + --archive --hard-links --numeric-ids --del \ + --sparse --compress \ + --one-file-system \ + --partial \ + $RSYNCOPTS \ + --filter="dir-merge .rsync-backup" \ + "$@" +} + +log () { + now=$(date +"%Y-%m-%d %H:%M:%S %z") + echo >&9 "$now $*" +} + +run () { + tag=$1 cmd=$2; shift 2 + ## Run CMD, logging its output in a pleasing manner. + + log "BEGIN $tag" + rc=$( + { { { ( set +e + "$cmd" "$@" 3>&- 4>&- 5>&- 9>&- + echo $? >&5; ) | + while IFS= read line; do echo "| $line"; done >&4; } 2>&1 | + while IFS= read line; do echo "* $line"; done >&4; } 4>&1 | + cat >&9; } 5>&1 $mnt/.lock + mount -oremount,ro $mnt" || return $? + + ## Done. + hostpath $mnt +} + +unsnap_ro () { + fs=$1 mnt=$2 + + ## Check that the filesystem still has our lock marker. + hostrun "unsnap-ro $mnt" " + case \$(cat $mnt/.lock) in + rsync-backup) ;; + *) echo unlocked by someone else; exit 31 ;; + esac + mount -oremount,rw $mnt + rm $mnt/.lock" || return $? +} + +## Snapshot using LVM. + +SNAPSIZE="-l10%ORIGIN" +SNAPDIR=@mntbkpdir@/snap + +snap_lvm () { + vg=$1 lv=$2 + + ## Make the snapshot. + hostrun "snap-lvm $vg/$lv" " + lvcreate --snapshot -n$lv.bkp $SNAPSIZE $vg/$lv + mkdir -p $SNAPDIR/$lv + mount -oro /dev/$vg/$lv.bkp $SNAPDIR/$lv" || return $? + + ## Done. + hostpath $SNAPDIR/$lv +} + +unsnap_lvm () { + vg=$1 lv=$2 + + ## Remove the snapshot. Sometimes LVM doesn't notice that the snapshot is + ## no longer in open immdiately, so try several times. + hostrun "unsnap-lvm $vg/$lv" " + umount $SNAPDIR/$lv + rc=1 + for i in 1 2 3 4; do + if lvremove -f $vg/$lv.bkp; then rc=0; break; fi + sleep 2 + done + exit $rc" || return $? +} + +## Complicated snapshot using LVM, where the volume group and filesystem are +## owned by different machines, so they need to be synchronized during the +## snapshot. + +do_rfreezefs () { + lvhost=$1 vg=$2 lv=$3 fshost=$4 fsdir=$5 + + ## Engage in the rfreezefs protocol with the filesystem host. This + ## involves some hairy plumbing. We want to get exit statuses out of both + ## halves. + set +e + ssh $fshost rfreezefs $fsdir | { + set -e + + ## Read the codebook from the remote end. + ready=nil + while read line; do + set -- $line + case "$1" in + PORT) port=$2 ;; + TOKEN) eval tok_$2=$3 ;; + READY) ready=t; break ;; + *) + echo >&2 "$quis: unexpected keyword $1 (rfreezefs to $rhost)" + exit 1 + ;; + esac + done + case $ready in + nil) + echo >&2 "$quis: unexpected eof (rfreezefs to $rhost)" + exit 1 + ;; + esac + + ## Connect to the filesystem host's TCP port and get it to freeze its + ## filesystem. + exec 3<>/dev/tcp/$fshost/$port + echo $tok_FREEZE >&3 + read tok <&3 + case $tok in + "$tok_FROZEN") ;; + *) + echo >&2 "$quis: unexpected token $tok (rfreezefs $fsdir on $fshost)" + exit 1 + ;; + esac + + ## Get the volume host to create the snapshot. + set +e + _hostrun >&2 3>&- $lvhost \ + "lvcreate --snapshot -n$lv.bkp $SNAPSIZE $vg/$lv" + snaprc=$? + set -e + + ## The filesystem can thaw now. + echo $tok_THAW >&3 + read tok <&3 + case $tok in + "$tok_THAWED") ;; + *) + _hostrun >&2 3>&- $lvhost "lvremove -f $vg/$lv.bkp" || : + echo >&2 "$quis: unexpected token $tok (rfreezefs $fsdir on $fshost)" + exit 1 + ;; + esac + + ## Done. + exit $snaprc + } + + ## Sift through the wreckage to find out what happened. + rc_rfreezefs=${PIPESTATUS[0]} rc_snapshot=${PIPESTATUS[1]} + set -e + case $rc_rfreezefs:$rc_snapshot in + 0:0) + ;; + 112:*) + echo >&2 "$quis: EMERGENCY failed to thaw $fsdir on $fshost!" + exit 112 + ;; + *) + echo >&2 "$quis: failed to snapshot $vg/$lv ($fsdir on $fshost)" + exit 1 + ;; + esac + + ## Mount the snapshot on the volume host. + _hostrun >&2 $lvhost " + mkdir -p $SNAPDIR/$lv + mount -oro /dev/$vg/$lv.bkp $SNAPDIR/$lv" +} + +snap_rfreezefs () { + rhost=$1 vg=$2 lv=$3 rfs=$4 + + set -e + run "snap-rfreezefs $host:$vg/$lv $rhost:$rfs" \ + do_rfreezefs $host $vg $lv $rhost $rfs || return $? + hostpath $SNAPDIR/$lv +} + +unsnap_rfreezefs () { + + ## Unshapping is the same as for plain LVM. + rhost=$1 vg=$2 lv=$3 rfs=$4 + unsnap_lvm $vg $lv +} + +###-------------------------------------------------------------------------- +### Expiry computations. + +parsedate () { + date=$1 + ## Parse an ISO8601 DATE, and set YEAR, MONTH, DAY appropriately (and + ## without leading zeros). + + ## Extract the components of the date and trim leading zeros (which will + ## cause things to be interpreted as octal and fail). + year=${date%%-*} rest=${date#*-}; month=${rest%%-*} day=${rest#*-} + year=${year#0} month=${month#0} day=${day#0} +} + +julian () { + date=$1 + ## Convert an ISO8601 DATE to a Julian Day Number. + + parsedate $date + + ## The actual calculation: convert a (proleptic) Gregorian calendar date + ## into a Julian day number. This is taken from Wikipedia's page + ## http://en.wikipedia.org/wiki/Julian_day#Calculation but the commentary + ## is mine. The epoch is 4713BC-01-01 (proleptic) Julian, or 4714BC-11-24 + ## proleptic Gregorian. + + ## If the MONTH is January or February then set a = 1, otherwise set a = 0. + a=$(( (14 - $month)/12 )) + + ## Compute a year offset relative to 4799BC-03-01. This puts the leap day + ## as the very last day in a year, which is very convenient. The offset + ## here is sufficient to make all y values positive (within the range of + ## the JDN calendar), and is a multiple of 400, which is the Gregorian + ## cycle length. + y=$(( $year + 4800 - $a )) + + ## Compute the offset month number in that year. These months count from + ## zero, not one. + m=$(( $month + 12*$a - 3 )) + + ## Now for the main event. The (153 m + 2)/5 term is a surprising but + ## correct trick for obtaining the number of days in the first m months of + ## the (shifted) year). The magic offset 32045 is what you get when you + ## plug the proper JDN epoch (year = -4713, month = 11, day = 24) into the + ## above machinery. + jdn=$(( $day + (153*$m + 2)/5 + 365*$y + $y/4 - $y/100 + $y/400 - 32045 )) + + echo $jdn +} + +expire () { + ## Read dates on stdin; write to stdout `EXPIRE date' for dates which + ## should be expired and `RETAIN date' for dates which should be retained. + + ## Get the current date and convert it into useful forms. + now=$(date +%Y-%m-%d) + parsedate $now + now_jdn=$(julian $now) now_year=$year now_month=$month now_day=$day + kept=: + + ## Work through each date in the input. + while read date; do + keep=nil + + ## Convert the date into a useful form. + jdn=$(julian $date) + parsedate $date + + ## Work through the policy list. + if [ $jdn -le $now_jdn ]; then + while read ival age; do + + ## Decide whether the policy entry applies to this date. + apply=nil + case $age in + forever) + apply=t + ;; + year) + if [ $year -eq $now_year ] || + ([ $year -eq $(( $now_year - 1 )) ] && + [ $month -ge $now_month ]) + then apply=t; fi + ;; + month) + if ([ $month -eq $now_month ] && [ $year -eq $now_year ]) || + ((([ $month -eq $(( $now_month - 1 )) ] && + [ $year -eq $now_year ]) || + ([ $month -eq 12 ] && [ $now_month -eq 1 ] && + [ $year -eq $(( $now_year - 1 )) ])) && + [ $day -ge $now_day ]) + then apply=t; fi + ;; + week) + if [ $jdn -ge $(( $now_jdn - 7 )) ]; then apply=t; fi + ;; + *) + echo >&2 "$quis: unknown age symbol \`$age'" + exit 1 + ;; + esac + case $apply in nil) continue ;; esac + + ## Find the interval marker for this date. + case $ival in + daily) + marker=$date + ;; + weekly) + ydn=$(julian $year-01-01) + wk=$(( ($jdn - $ydn)/7 + 1 )) + marker=$year-w$wk + ;; + monthly) + marker=$year-$month + ;; + annually | yearly) + marker=$year + ;; + *) + echo >&2 "$quis: unknown interval symbol \`$ival'" + exit 1 + ;; + esac + + ## See if we've alredy retained something in this interval. + case $kept in + *:"$marker":*) ;; + *) keep=t kept=$kept$marker: ;; + esac + + done <new.fshash +} + +local_fshash () { + { echo "*** $host $fs $date"; echo + fshash -c$STOREDIR/fshash.cache -H$HASH new/ + } >$localmap +} + +expire_backups () { + { seen=: + for i in *-*-*; do + i=${i%%.*} + case $i in *[!-0-9]*) continue ;; esac + case $seen in *:"$i":*) continue ;; esac + seen=$seen$i: + echo $i + done; } | + expire | + while read op date; do + case $op in + RETAIN) + echo "keep $date" + ;; + EXPIRE) + echo "delete $date" + $verbose -n " expire $date..." + rm -rf $date $date.* + $verbose " done" + ;; + esac + done +} + +backup_precommit_hook () { + host=$1 fs=$2 date=$3 + ## Override this hook in the configuration file for special effects. + + : +} + +backup_commit_hook () { + host=$1 fs=$2 date=$3 + ## Override this hook in the configuration file for special effects. + + : +} + +do_backup () { + date=$1 fs=$2 fsarg=$3 + ## Back up FS on the current host. + + set -e + + ## Report the start of this attempt. + log "START BACKUP of $host:$fs" + + ## Create and mount the remote snapshot. + snapmnt=$(snap_$snap $snapargs $fs $fsarg) || return $? + $verbose " create snapshot" + + ## Build the list of hardlink sources. + linkdests="" + for i in $host $like; do + d=$STOREDIR/$i/$fs/last/ + if [ -d $d ]; then linkdests="$linkdests --link-dest=$d"; fi + done + + ## Copy files from the remote snapshot. + mkdir -p new/ + $verbose -n " running rsync..." + set +e + run "RSYNC of $host:$fs (snapshot on $snapmnt)" do_rsync \ + $linkdests \ + $rsyncargs \ + $snapmnt/ new/ + rc_rsync=$? + set -e + $verbose " done" + + ## Collect a map of the snapshot for verification purposes. + set +e + $verbose -n " remote fshash..." + run "@$host: fshash $fs" remote_fshash + rc_fshash=$? + set -e + $verbose " done" + + ## Remove the snapshot. + unsnap_$snap $snapargs $fs $fsarg + $verbose " remove snapshot" + + ## If we failed to copy, then give up. + case $rc_rsync:$rc_fshash in + 0:0) ;; + 0:*) return $rc_fshash ;; + *) return $rc_rsync ;; + esac + + ## Get a matching map of the files received. + mkdir -m750 -p $STOREDIR/tmp + localmap=$STOREDIR/tmp/fshash.$host.$fs.$date + $verbose -n " local fshash..." + run "local fshash $host:$fs" local_fshash || return $? + $verbose " done" + + ## Compare the two maps. + run "compare fshash maps for $host:$fs" \ + diff -u new.fshash $localmap || return $? + rm -f $localmap + $verbose " fshash match" + + ## Commit this backup. + backup_precommit_hook $host $fs $date + mv new $date + mv new.fshash $date.fshash + backup_commit_hook $host $fs $date + mkdir hack + ln -s $date hack/last + mv hack/last . + rmdir hack + $verbose " commit" + + ## Expire old backups. + case "${expire_policy+t}" in + t) run "expiry for $host:$fs" expire_backups ;; + esac + + ## Report success. + log "SUCCESSFUL BACKUP of $host:$fs" +} + +backup () { + ## backup FS[:ARG] ... + ## + ## Back up the filesystems on the currently selected host using the + ## currently selected snapshot type. + + for fs in "$@"; do + + ## Parse the argument. + case $fs in + *:*) fsarg=${fs#*:} fs=${fs%%:*} ;; + *) fsarg="" ;; + esac + $verbose " filesystem $fs" + + ## Move to the store directory and set up somewhere to put this backup. + cd $STOREDIR + if [ ! -d $host ]; then + mkdir -m755 $host + chown root:root $host + fi + if [ ! -d $host/$fs ]; then + mkdir -m750 $host/$fs + chown root:backup $host/$fs + fi + cd $host/$fs + + ## Find out if we've already copied this filesystem today. + date=$(date +%Y-%m-%d) + if [ -d $date ]; then + $verbose " already dumped" + continue + fi + + ## Find a name for the log file. In unusual circumstances, we may have + ## deleted old logs from today, so just checking for an unused sequence + ## number is insufficient. Instead, check all of the logfiles for today, + ## and use a sequence number that's larger than any of them. + seq=1 + for i in "$logdir/$host/$fs.$date#"*; do + tail=${i##*#} + case "$tail" in [!1-9]* | *[!0-9]*) continue ;; esac + if [ -f "$i" -a $tail -ge $seq ]; then seq=$(( tail + 1 )); fi + done + log="$logdir/$host/$fs.$date#$seq" + + ## Do the backup of this filesystem. + mkdir -p $logdir/$host + if ! do_backup $date $fs $fsarg 9>$log 1>&9; then + echo >&2 + echo >&2 "$quis: backup of $host:$fs FAILED!" + bkprc=1 + fi + + ## Count up the logfiles. + nlog=0 + for i in "$logdir/$host/$fs".*; do + if [ ! -f "$i" ]; then continue; fi + nlog=$(( nlog + 1 )) + done + + ## If there are too many, go through and delete some early ones. + if [ $nlog -gt $MAXLOG ]; then + n=$(( nlog - MAXLOG )) + for i in "$logdir/$host/$fs".*; do + if [ ! -f "$i" ]; then continue; fi + rm -f "$i" + n=$(( n - 1 )) + if [ $n -eq 0 ]; then break; fi + done + fi + done +} + +###-------------------------------------------------------------------------- +### Configuration functions. + +host () { host=$1; like=; $verbose "host $host"; } +snaptype () { snap=$1; shift; snapargs="$*"; } +rsyncargs () { rsyncargs="$*"; } +like () { like="$*"; } + +retain () { + expire_policy="${expire_policy+$expire_policy +}$*" +} + +###-------------------------------------------------------------------------- +### Read the configuration and we're done. + +usage () { + echo "usage: $quis [-v] [-c CONF]" +} + +version () { + echo "$quis version $VERSION" +} + +config () { + echo + cat <&8 "$@"; } + +while getopts "hVvc:" opt; do + case "$opt" in + h) usage; exit 0 ;; + V) version; config; exit 0 ;; + v) verbose=whine ;; + c) conf=$OPTARG ;; + *) exit 1 ;; + esac +done +shift $((OPTIND - 1)) +case $# in 0) ;; *) usage >&2; exit 1 ;; esac +exec 8>&1 + +. "$conf" + +###----- That's all, folks -------------------------------------------------- + +exit $bkprc diff --git a/umount-backup-volume b/umount-backup-volume new file mode 100755 index 0000000..ce7a2fb --- /dev/null +++ b/umount-backup-volume @@ -0,0 +1,10 @@ +#! /bin/sh -x + +: ${vgtag=@backup} ${vgprefix=vg-backup-} +: ${mntbkpdir=/mnt/bkp} +: ${STOREDIR=$mntbkpdir/store} ${METADIR=$mntbkpdir/meta} + +umount $STOREDIR +cryptsetup luksClose cbackup +umount $METADIR +vgchange -an @backup -- [mdw]