From: ianmdlvl Date: Sun, 7 Oct 2001 17:17:05 +0000 (+0000) Subject: From Peter Maydell (/u2/pmaydell/iwjbackup/) as per in davenant:~ian/mail/INBOX on Sun, 03 Jun 2001 18:45:47 +0100 --- diff --git a/backup/Ucam.comp.unix b/backup/Ucam.comp.unix new file mode 100644 index 0000000..74bddbd --- /dev/null +++ b/backup/Ucam.comp.unix @@ -0,0 +1,251 @@ +From ijackson@chiark.greenend.org.uk Wed Aug 12 19:36:07 BST 1998 +Article: 742 of ucam.comp.unix +Path: ewrotcd!not-for-mail +From: ijackson@chiark.greenend.org.uk (Ian Jackson) +Newsgroups: ucam.comp.unix,comp.sys.sun.admin +Subject: Buffering programs (was Re: Speeding up network dumps) +Date: 21 Jul 1998 12:23:23 +0100 (BST) +Organization: Linux Unlimited +Lines: 233 +Message-ID: +References: <6oiapl$6k5$1@pegasus.csx.cam.ac.uk> <6ol7kf$cln$2@pegasus.csx.cam.ac.uk> +NNTP-Posting-Host: chiark.greenend.org.uk +X-Server-Date: 21 Jul 1998 11:23:25 GMT +Originator: ijackson@news.chiark.greenend.org.uk ([127.0.0.1]) +Xref: news.chiark.greenend.org.uk ucam.comp.unix:742 comp.sys.sun.admin:14526 + +I said I'd post here the buffering programs that I use, so here they +are. + +They're perfectly functional and (I believe) correct, but they do lack +sophistication. The size of the buffer is a #define, and the +diagnostics lack verbosity. I haven't compiled them on anything but +Linux/i386 libc5, but they ought to work on other reasonable Unices. + +writebuffer is intended for writing to tapes; when its buffer is empty +it will write no more data until it gets 3/4 of its buffer filled, or +EOF. + +readbuffer is intended for reading from tapes; when its buffer becomes +full it stops reading until it's down to 1/4. + +Remember that using shell pipes loses the exit status of all but the +last command, and so can cause failures to go unnoticed. + + +/* + * writebuffer.c + * + * Copyright (C) 1997,1998 Ian Jackson + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this file; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#include +#include +#include +#include +#include +#include + +#define BUFFER 16*1024*1024 +#define WAITFILL ((BUFFER*3)/4) + +static inline int min(int a, int b) { return a<=b ? a : b; } + +static void nonblock(int fd) { + int r; + r= fcntl(fd,F_GETFL,0); if (r == -1) { perror("fcntl getfl"); exit(1); } + r |= O_NDELAY; + if (fcntl(fd,F_SETFL,r) == -1) { perror("fcntl setfl"); exit(1); } +} + +int main(void) { + static unsigned char buf[BUFFER]; + + unsigned char *wp, *rp; + int used,r,writing,seeneof; + fd_set readfds; + fd_set writefds; + + used=0; wp=rp=buf; writing=0; seeneof=0; + nonblock(0); nonblock(1); + while (!seeneof || used) { + FD_ZERO(&readfds); if (!seeneof && used+1 WAITFILL) writing=1; + } + if (FD_ISSET(1,&writefds) && used) { + r= write(1,wp,min(used,buf+BUFFER-wp)); + if (r<=0) { + if (!(errno == EAGAIN || errno == EINTR)) { perror("write"); exit(1); } +/*fprintf(stderr,"\t write transient error\n");*/ + } else { +/*fprintf(stderr,"\t wrote %d\n",r);*/ + used-= r; + wp+= r; + if (wp == buf+BUFFER) wp=buf; + } + } + } + exit(0); +} + + +/* + * readbuffer.c + * + * Copyright (C) 1997,1998 Ian Jackson + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this file; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#define BUFFER 16*1024*1024 +#define WAITEMPTY ((BUFFER*1)/4) + +static inline int min(int a, int b) { return a<=b ? a : b; } + +static void nonblock(int fd) { + int r; + r= fcntl(fd,F_GETFL,0); if (r == -1) { perror("fcntl getfl"); exit(1); } + r |= O_NDELAY; + if (fcntl(fd,F_SETFL,r) == -1) { perror("fcntl setfl"); exit(1); } +} + +int main(void) { + static unsigned char buf[BUFFER]; + + unsigned char *wp, *rp; + int used,r,reading,seeneof; + fd_set readfds; + fd_set writefds; + + used=0; wp=rp=buf; reading=1; seeneof=0; + nonblock(0); nonblock(1); + while (!seeneof || used) { + FD_ZERO(&readfds); + if (reading) { + if (used +These opinions are my own. http://www.chiark.greenend.org.uk/~ijackson/ +PGP2 public key id 0x23f5addb, fingerprint 5906F687 BD03ACAD 0D8E602E FCF37657 + + diff --git a/backup/backuplib.pl b/backup/backuplib.pl index d412c94..70d28b3 100644 --- a/backup/backuplib.pl +++ b/backup/backuplib.pl @@ -1,9 +1,14 @@ # +# Assorted useful functions used by the backup scripts. + sub printdate () { print scalar(localtime),"\n"; } +# Set status info -- we write the current status to a file +# so if we hang or crash the last thing written to the file +# will tell us where we were when things went pear-shaped. sub setstatus ($) { open S, ">this-status.new" or die $!; print S $_[0],"\n" or die $!; @@ -11,6 +16,8 @@ sub setstatus ($) { rename "this-status.new","this-status" or die $!; } +# startprocess, endprocesses, killprocesses are +# used to implement the funky pipeline stuff. sub startprocess ($$$) { my ($i,$o,$c) = @_; print LOG " $c\n" or die $!; @@ -42,28 +49,41 @@ sub killprocesses { undef %processes; } +# Read a fsys.foo filesystem group definition file. +# Syntax is: empty lines and those beginning with '#' are ignored. +# Trailing whitespace is ignored. Lines of the form 'prefix foo bar' +# are handled specially, as arex lines 'exclude regexp'; otherwise +# we just shove the line into @fsys and let parsefsys deal with it. sub readfsys ($) { my ($fsnm) = @_; open F, "$etc/fsys.$fsnm" or die "Filesystems $fsnm unknown ($!).\n"; for (;;) { - $_= or die; chomp; s/\s*$//; + $_= or die "unexpected EOF in $etc/fsys.$fsnm\n"; chomp; s/\s*$//; last if m/^end$/; next unless m/\S/; next if m/^\#/; if (m/^prefix\s+(\w+)\s+(\S.*\S)$/) { $prefix{$1}= $2; next; - } elsif (m/^prefix\-df\s+(\w+)\s+(\S.*\S)$/) { - $prefixdf{$1}= $2; - next; } + if (m/^excludedir\s+(\S.*\S)$/) { + push @excldir,$1; + next; + } + if (m/^exclude\s+(\S.*\S)$/) { + push @excl,$1; + next; + } push @fsys,$_; } close F or die $!; } +# Parse a line from a filesystem definition file. We expect the line +# to be in $tf. sub parsefsys () { if ($tf =~ m,^(/\S*)\s+(\w+)$,) { + # Line of form '/file/system dumptype' $atf= $1; $tm= $2; $prefix= ''; @@ -71,6 +91,8 @@ sub parsefsys () { -d _ or die "not a dir: $atf"; $rstr= ''; } elsif ($tf =~ m,^(/\S*)\s+(\w+)\s+(\w+)$,) { + # Line of form '/file/system dumptype prefix' + # (used for remote backups, I think) $atf= $1; $tm= $2; $prefix= $3; diff --git a/backup/bringup b/backup/bringup index 6bcd5d5..ffca871 100755 --- a/backup/bringup +++ b/backup/bringup @@ -1,5 +1,8 @@ #!/bin/sh +# Very simple: extract the default runlevel from /etc/inittab +# and change to it with telinit. runlevel=`sed -ne '/^id:/ s/.*:\([0-9]\):.*/\1/p' /etc/inittab` telinit $runlevel -chvt 11 +# This switches to virtual console 11, but I don't think I want that -- PMM +#chvt 11 diff --git a/backup/checkallused b/backup/checkallused index fd613f3..f50abb7 100755 --- a/backup/checkallused +++ b/backup/checkallused @@ -1,5 +1,9 @@ #!/usr/bin/perl +# Read all the configuration files and check that all filesystems +# are either backed up in both full and incremental dumps or +# listed as exceptions. + BEGIN { $etc= '/etc/backup'; require "$etc/settings.pl"; @@ -38,8 +42,6 @@ for $fsg (sort keys %fsgdone) { for $tf (@fsys) { parsefsys(); $pstr= $prefix ne '' ? "$prefix:$atf" : $atf; - &e("dumped twice ($backed{$pstr}, $fsg): $pstr") - if defined $backed{$pstr}; $backed{$pstr}= $fsg; print " $pstr"; } @@ -59,36 +61,28 @@ print "\n"; for $pfx ('', sort keys %prefix) { $rstr= length($pfx) ? $prefix{$pfx}.' ' : ''; - $dfstr= exists($prefixdf{$pfx}) ? $prefixdf{$pfx} : - 'df --no-sync -xiso9660 -xnfs -xproc'; - $cmd= "$rstr $dfstr"; - open X, "$cmd |" or die $!; - $_= ; m/^Filesystem/ or die "$cmd => $_ ?"; + open X, $rstr." df --no-sync -xnfs |" or die $!; + $_= ; m/^Filesystem/ or die "$_ ?"; $ppstr= length($pfx) ? $pfx : ''; $pstr= length($pfx) ? "$pfx:" : ''; print "mount points: $ppstr:"; while () { chomp; - next if m,^procfs\s,; m,^/dev/(\S+)\s.*\s(/\S*)\s*$, or die "$_ ?"; - ($dev,$mp) = ($1,$2); - $mounted{"$pstr$mp"}="$pstr$dev"; print " $1-$2"; - if (defined($backto= $backed{"$pstr$mp"})) { - if (m,^/dev/\S+\s+\d+\s+(\d+)\s,) { - $usedkb{$backto} += $1; - } else { - $usedkb{$backto} += 0; - $unkkb{$backto} .= " + $pstr$mp"; - } - } + $mounted{"$pstr$2"}="$pstr$1"; print " $1-$2"; } print "\n"; - $!=0; close(X); $? and die "$cmd $? $!"; } +$!=0; close(X); $? and die "$? $!"; -foreach $fsg (keys %usedkb) { - print "filesystem group $fsg: $usedkb{$fsg} 1K-blocks$unkkb{$fsg}\n"; -} +# We check that all mounted filesystems are dumped and all +# filesystems to be dumped are mounted. The expected-diffs +# config file allows us to make exceptions. +# eg: +# #expect disk2 to be mounted but not dumped +# !/disk2 +# # CD may or may not be mounted but should not be dumped in either case +# ?/cdrom open Z,"$etc/expected-diffs" or die $!; for (;;) { @@ -96,7 +90,12 @@ for (;;) { last if m/^end$/; next unless m/^\S/; next if m/^\#/; - if (s/^\!//) { + if (s/^\?//) { + print "non-permanent filesystem expected not to be dumped: $_\n"; + if (defined($mounted{$_})) { + delete $mounted{$_}; + } + } elsif (s/^\!//) { &e("expected not to be dumped, but not a mount point: $_") unless defined($mounted{$_}); print "filesystem expected not to be dumped: $_\n"; diff --git a/backup/driver b/backup/driver index 394b3fb..a9bc37e 100755 --- a/backup/driver +++ b/backup/driver @@ -1,4 +1,5 @@ #!/bin/sh +# This is the driver script that actually runs backups. cd /var/local/backup PATH=/usr/local/lib/backup:$PATH export PATH @@ -11,19 +12,30 @@ fi rm -f this-status p p2 echo 'FAILED to start dump script' >this-status +# Here we go : run 'full', which (name notwithstanding) handles +# both full and incremental backups, according to the ID of the +# tape in the drive. full 2>&1 | tee this-log status=`cat this-status 2>/dev/null` -cat <); $tapeid =~ m/[^0-9a-zA-Z]/ and die "Bad TAPEID ($&).\n"; $tapeid =~ m/[0-9a-zA-Z]/ or die "Empty TAPEID.\n"; @@ -34,6 +43,7 @@ close T; setstatus "FAILED at tape identity check"; +# We don't let the user overwrite the tape used for the last backup. if (open L, "last-tape") { chomp($lasttape= ); close L; @@ -43,13 +53,13 @@ if (open L, "last-tape") { die "Tape $tapeid same as last time.\n" if $tapeid eq $lasttape; -if (defined($tapedesc= readlink "$etc/tape.$tapeid")) { - $tapedesc =~ s/^.*\.//; - $tapedesc .= "($tapeid)"; -} else { - $tapedesc = $tapeid; -} - +# Parse the appropriate tape.nn file. +# Format is: empty lines and lines starting '#' are ignored. Trailing +# whitespace is ignored. File must end with 'end' on a line by itself. +# Either there should be a line 'incremental' to indicate that this is +# a tape for incremental backups, or a pair of lines 'filesystems fsg' +# and 'next tapeid', indicating that this tape is part of a full +# backup, containing the filesystem group fsg. undef $fsys; open D, "$etc/tape.$tapeid" or die "Unknown tape $tapeid ($!).\n"; for (;;) { @@ -69,23 +79,27 @@ for (;;) { } close D or die $!; +# Incremental backups are handled by increm, not us. if ($incremental) { die "incremental tape $tapeid has next or filesystems\n" if defined($next) || defined($fsys); print STDERR "Incremental tape $tapeid.\n\n"; setstatus "FAILED during incremental startup"; - exec "increm",$tapeid,$tapedesc; + exec "increm $tapeid"; die $!; } +# Read the filesystem group definition (file fsys.nnn) readfsys("$fsys"); openlog(); -$doing= "dump of $fsys to tape $tapedesc in drive $tape"; +$doing= "dump of $fsys to tape $tapeid in drive $tape"; print LOG "$doing:\n" or die $!; +setstatus "FAILED writing tape ID"; +# First write the tape ID to this tape. open T, ">TAPEID" or die $!; -print T "$tapeid\n$tapedesc\n" or die $!; +print T "$tapeid\n" or die $!; close T or die $!; system "tar -b$blocksize -vvcf TAPEID.tar TAPEID"; $? and die $?; @@ -105,6 +119,17 @@ sub closepipes () { close(DUMPOW); close(TEEOW); close(BUFOW); close(FINDOW); } +# work out a find option string that will exclude the required files +# Note that dump pays no attention to exclude options. +$exclopt = ''; +foreach $exc (@excldir) { + $exclopt .= "-regex $exc -prune -o "; +} +foreach $exc (@excl) { + $exclopt .= "-regex $exc -o "; +} + +# For each filesystem to be put on this tape: for $tf (@fsys) { printdate(); pipe(FINDOR,FINDOW) or die $!; @@ -112,14 +137,27 @@ for $tf (@fsys) { pipe(TEEOR,TEEOW) or die $!; pipe(BUFOR,BUFOW) or die $!; parsefsys(); + + # We can back up via dump or cpio if ($tm eq 'dump') { $dumpcmd= "dump 0bfu $softblocksizekb - $atf"; $dumpin= '&FINDOW',$rstr."find $atf -xdev -noleaf -print0"; $dumpcmd= "cpio -Hustar -o0C$softblocksizebytes"; $dumpin= '<&FINDOR'; + } elsif ($tm eq 'zafio') { + # compress-each-file-then-archive using afio + startprocess '&FINDOW',$rstr."find $atf -xdev -noleaf $exclopt -print"; + # don't use verbose flag as this generates 2MB report emails :-> + $dumpcmd = "afio -b $softblocksizebytes -Zo -"; + $dumpin = '<&FINDOR'; } + # This is a funky way of doing a pipeline which pays attention + # to the exit status of all the commands in the pipeline. + # It is roughly equivalent to: + # md5sum

>this-md5sums + # dump <$dumpin | tee p | writebuffer | dd >/dev/null startprocess '>this-md5sums','md5sum'; startprocess $dumpin,'>&DUMPOW',$rstr.$dumpcmd; startprocess '<&DUMPOR','>&TEEOW','tee p'; @@ -130,17 +168,21 @@ for $tf (@fsys) { endprocesses(); } +# The backup should now be complete; verify it + setstatus "FAILED during check"; +# Rewind the tape and skip the TAPEID record system "mt -f $tape rewind"; $? and die $?; system "mt -f $ntape fsf 1"; $? and die $?; +# Check the md5sums match for each filesystem on the tape open S,"this-md5sums" or die $!; for $tf (@fsys) { printdate(); chomp($orgsum= ); $orgsum =~ m/^[0-9a-fA-F]{32}$/i or die "orgsum \`$orgsum' ?"; - chomp($csum= `readbuffer <$ntape | md5sum`); + chomp($csum= `dd if=$ntape ibs=$blocksizebytes | readbuffer | md5sum`); $orgsum eq $csum or die "MISMATCH $tf $csum $orgsum\n"; print "checksum ok $csum\t$tf\n" or die $!; print LOG "checksum ok $csum\t$tf\n" or die $!; @@ -150,15 +192,21 @@ system "mt -f $tape rewind"; $? and die $?; setstatus "FAILED during cleanup"; +# Write to some status files to indicate what the backup system +# ought to do when next invoked. +# reset incremental backup count to 1. open IAN,">increm-advance.new" or die $!; print IAN "1\n" or die $!; close IAN or die $!; +# Next full backup is whatever the next link in the tape description +# file says it ought to be. open TN,">next-full.new" or die $!; print TN "$next\n" or die $!; close TN or die $!; unlink 'last-tape','next-full'; +# We are the last tape to have been backed up rename 'TAPEID','last-tape' or die $!; rename 'this-md5sums',"md5sums.$fsys" or die $!; rename 'log',"log.$fsys" or die $!; @@ -167,5 +215,5 @@ rename 'increm-advance.new',"increm-advance" or die $!; print "$doing completed.\nNext dump tape is $next.\n" or die $!; -setstatus "Successful: $tapedesc $fsys, next $next"; +setstatus "Successful ($tapeid $fsys, next $next)"; exit 0; diff --git a/backup/increm b/backup/increm index 1d2aaa7..3120430 100755 --- a/backup/increm +++ b/backup/increm @@ -1,5 +1,10 @@ #!/usr/bin/perl +# Do an incremental backup. We are invoked by full if the tape +# description file says that it is for an incremental backup. +# We expect a single commandline argument which is the ID string +# of the tape to use. + BEGIN { $etc= '/etc/backup'; require "$etc/settings.pl"; @@ -8,30 +13,40 @@ BEGIN { $|=1; -@ARGV==2 or die; -($tapeid,$tapedesc)= @ARGV; +@ARGV==1 or die; +$tapeid= $ARGV[0]; -print "Running incremental onto $tapedesc ...\n" or die $!; +print "Running incremental (tape $tapeid) ...\n" or die $!; +# Just check that we weren't passed a bogus ID (the tape description +# file for incrementals is just 'incremental\nend\n') open T,"$etc/tape.$tapeid" or die "Tape $tapeid not found: $!\n"; close T; +# This is only used for the 'next FULL backup is X' message at the end. open NF,"next-full" or die $!; chomp($next= ); close NF or die $!; setstatus "FAILED during incremental"; +# Read the number of the incremental involved +# (ie, (how many files are already on the tape) - 1) open A,"increm-advance" or die $!; chomp($advance= ); close A or die $!; +# better be a decimal number $advance =~ m/^\d+$/ or die "$advance ?"; +# Rewind the tape and if we are the first incremental on the tape then +# write the TAPEID record, otherwise skip forward to the correct point. +# (full will already have checked that this is the right tape before +# it invoked us, so no need to read the existing TAPEID record first.) system "mt -f $ntape rewind"; $? and die $?; if ($advance == 1) { open TI,">TAPEID" or die $!; - print TI "$tapeid\n$tapedesc\n" or die $!; + print TI "$tapeid" or die $!; close TI or die $!; system "tar -b$blocksize -vvcf TAPEID.tar TAPEID"; $? and die $?; @@ -40,6 +55,7 @@ if ($advance == 1) { system "mt -f $ntape fsf $advance"; $? and die $?; } +# Get a list of all filesystems readfsys('all'); openlog(); @@ -51,6 +67,7 @@ sub closepipes () { setstatus "PROBLEMS during incremental dump"; for $tf (@fsys) { + pipe(DUMPOR,DUMPOW) or die $!; pipe(BUFOR,BUFOW) or die $!; parsefsys(); @@ -59,15 +76,21 @@ for $tf (@fsys) { print LOG "Not dumping $atf ($prefix) - not \`dump'.\n" or die $!; next; } + # Same trick as full uses to do a pipeline whilst keeping track + # of all exit statuses: + # dump /dev/null startprocess '&DUMPOW',$rstr."dump 1bfu $softblocksizekb - $atf"; startprocess '<&DUMPOR','>&BUFOW','writebuffer'; startprocess '<&BUFOR','>/dev/null' ,"dd ibs=$softblocksizebytes obs=$blocksizebytes of=$ntape"; closepipes(); endprocesses(); + # advance is a file counter, so it needs to be updated for each + # dump we do to tape. $advance++; } +# Rewind the tape, and increment the counter of incremental backups. system "mt -f $tape rewind"; $? and die $?; open IAN,">increm-advance.new" or die $!; print IAN "$advance\n" or die $!; @@ -77,5 +100,5 @@ rename 'increm-advance.new','increm-advance' or die $!; print LOG "Next FULL dump tape is $next\n" or die $!; print "Next FULL dump tape is $next\n" or die $!; -setstatus "INCREMENTAL successful: $tapedesc, next full is $next"; +setstatus "INCREMENTAL successful (next full is $next)"; exit 0; diff --git a/backup/increm-advance b/backup/increm-advance index 4099407..d00491f 100644 --- a/backup/increm-advance +++ b/backup/increm-advance @@ -1 +1 @@ -23 +1 diff --git a/backup/iwjbackup.txt b/backup/iwjbackup.txt new file mode 100644 index 0000000..9a2775e --- /dev/null +++ b/backup/iwjbackup.txt @@ -0,0 +1,141 @@ +This is a quick summary of IWJ's backup scripts and my config files: +it's a bit patchy and might have the odd ommission. The canonical +source is the sources, as always :-> + +The three tarfiles in this directory should go in the following +places (the paths can probably be configured/hacked, but this is +where they are on my system and on chiark): + +etc.tgz : /etc/backup/ +lib.tgz : /usr/local/lib/backup/ +var.tgz : /var/local/backup/ + +NOTE: these versions are not those used on chiark; they +are somewhat modified by me (couple of extra features and +lots of comments -- all errors in those are mine.) + +NB: you'll probably need to delete some of the files from +var.tgz (in fact, I think you probably don't want any of +them except maybe last-tape (which you want to change anyway). +You'll also need to recompile readbuffer and writebuffer unless +you're using SPARClinux :-> + +Contents of /etc/backup/: +warnings.* : files defining how many warnings you get as the +system is brought down to do backups. The defaults are fine. +settings.pl : generic config file: in particular, the name of +the tape device is set here. +tape.* : each tape you're going to use in the backup cycle +has a name and a config file. Here the tapes are named 'a'-'e', +although longer names should be OK. You need at least two +tapes defined as the system won't write a backup on the same +tape it wrote the last one to. + +Syntax of the tape.* files: +filesystems X +next N +end + +where N is the next tape in the sequence (which should +be circular; I have a->b->c->d->e->a...) and X is a +filesystem-name (list of filesystems might work?). + +Each defined filesystem has a name and a config file +fsys.. These files define what is backed up and how. +The filesystem 'all' must exist; it's used for incremental +backups (and it must exist even if you don't do incrementals). +I don't have any other filesystems defined as everything fits +on one tape. + +Syntax of these files: +Empty lines and lines starting '#' are comments and ignored. +Lines starting 'excludedir' given regexps of things to exclude +(temp dirs, Netscape's cache, etc). +Lines starting 'prefix' give a command prefix necessary to +run things on a remote machine: +prefix +Other lines should be of the form + +for local backups, or + +for remote backups. +The file must end with the word 'end' on a line of its own. + +Valid values for are 'cpio' (uses cpio to produce +tar-format backups), 'dump' (uses dump to dump entire filesystems; + should be a mount-point for this), and [if you +use my version of the scripts] 'zafio' (uses afio to compress +each file as it is backed up). Only 'dump' type backups permit +incremental backups. + +Finally, expected-diffs is a config file to indicate which +filesystems should *not* be backed up. The scripts do a config +check which involves checking that: + * all filesystems to be backed up are present + * all filesystems that are present are backed up +expected-diffs allows you to make exceptions to this; backing +up your CDROM drive is a bit pointless, frex. +The format here is: + + +where is ?, ! or nothing, and + is : for a remote fs or + for a local one +(examples: "mnementh:/cdrom", "/cdrom"). +If is nothing, the scripts will complain if the fs +is mounted. If it is !, they will complain if it is not mounted. +If ? they won't complain either way (useful for devices that are +not always mounted, like /cdrom). '?' is an enhancement only +present in my version of the scripts. + +Useful scripts: (all in /usr/local/lib/backup) +checkallused: this only does a check of the configuration files. +It should give a cryptic summary of the configuration and print +'configuration ok'. If not, fix your config files :-> + +loaded: this tells the scripts that a currently unlabelled tape +should be treated as tape X: eg: +loaded b +will cause it to treat it as tape 'b'. [NB: this won't override +the TAPEID label written on the tape; it's just for use with +previously unused tapes.] + +driver : this is the script to actually run to do a backup. +If run from the command line, give it the argument 'test' +[otherwise it will attempt to run bringup to change runlevel, +on the assumption that it was run from inittab (see below)]. +You'll need to edit this script to send the status report email +to somewhere right for your system. + +takedown : Run this if you want to run a reduced level of system +services during backups. Usage: +takedown +where can be 'now', 'soon' or nothing depending on number +of warning messages desired. [configured via warnings.* files.] + +To use this you'll need to configure init: + * set up runlevel 5 to provide the level of services you want + (by tweaking the symlinks in /etc/rc5.d or equivalent) + * Add the following to /etc/inittab (tweak paths and VC number + if desired): + +# Runlevel 5 is set up to run a reduced level of services during +# backups. (currently this means: no squid, no webserver, no newsserver) +# We also run the backup script automatically on entering runlevel 5: +# (I/O goes to VC 7, which is also the X server, but never mind -- we +# very seldom run X anyway :->) +dm:5:once:/usr/local/lib/backup/driver /dev/tty7 2>&1 + + * takedown can be run from the command line or via cron. + +whatsthis: a simple script I hacked together to display the +TAPEID of the current tape and optionally list its contents. +Usage: +whatsthis [--list [n]] + +WARNING: it's currently hardwired to assume 'cpio' type backups +when listing; it could be trivially hardwired to assume 'zafio' +or with slightly more effort it could be done properly :-> + +That's all I can think of for now -- comments and +questions to Peter Maydell diff --git a/backup/last-tape b/backup/last-tape index 8c1384d..6178079 100644 --- a/backup/last-tape +++ b/backup/last-tape @@ -1 +1 @@ -v2 +b diff --git a/backup/lib.tgz b/backup/lib.tgz new file mode 100644 index 0000000..208bae8 Binary files /dev/null and b/backup/lib.tgz differ diff --git a/backup/next-full b/backup/next-full index 5abb5bb..f2ad6c7 100644 --- a/backup/next-full +++ b/backup/next-full @@ -1 +1 @@ -s0 +c diff --git a/backup/readbuffer b/backup/readbuffer index c0078a0..b5ab670 100755 Binary files a/backup/readbuffer and b/backup/readbuffer differ diff --git a/backup/readbuffer.c b/backup/readbuffer.c index 4cd31af..2513977 100644 --- a/backup/readbuffer.c +++ b/backup/readbuffer.c @@ -1,4 +1,23 @@ -/**/ +/* + * readbuffer.c + * + * Copyright (C) 1997,1998 Ian Jackson + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this file; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ #include #include @@ -7,9 +26,9 @@ #include #include #include -#include -#define BUFFER 16*1024*1024 +/* was 16MB -- PMM */ +#define BUFFER 6*1024*1024 #define WAITEMPTY ((BUFFER*1)/4) static inline int min(int a, int b) { return a<=b ? a : b; } @@ -21,7 +40,7 @@ static void nonblock(int fd) { if (fcntl(fd,F_SETFL,r) == -1) { perror("fcntl setfl"); exit(1); } } -int main(int argc, const char *const *argv) { +int main(void) { static unsigned char buf[BUFFER]; unsigned char *wp, *rp; @@ -31,11 +50,6 @@ int main(int argc, const char *const *argv) { used=0; wp=rp=buf; reading=1; seeneof=0; nonblock(0); nonblock(1); - if (argv[1] && !strcmp(argv[1],"--mlock")) { - if (mlock(buf,sizeof(buf))) { perror("mlock"); exit(1); } - argv++; argc--; - } - if (argv[1]) { fputs("usage: readbuffer [--mlock]\n",stderr); exit(1); } while (!seeneof || used) { FD_ZERO(&readfds); if (reading) { diff --git a/backup/settings.pl b/backup/settings.pl index 5e9c5d5..4cef23f 100644 --- a/backup/settings.pl +++ b/backup/settings.pl @@ -1,11 +1,17 @@ # chdir '/var/local/backup' or die $!; push(@INC,'/usr/local/lib/backup'); +# huh? AFAICT this adds two entries to the PATH and +# promptly removes them again??? -- PMM $ENV{'PATH'}= '/usr/local/lib/backup:/usr/local/bin:'.$ENV{'PATH'}; $ENV{'PATH'} =~ s,^/usr/local/lib/backup:/usr/local/bin:,,; -$blocksize= 1; +# This sets both blocksizes to 10K. Note that both must be the +# same if using the zftape floppy tape driver, since that requires +# blocks to be the right size, but dd with the obs=10k option +# doesn't pad the final block to the blocksize... +$blocksize= 20; # was 1 -- PMM $blocksizebytes= 512*$blocksize; -$softblocksizekb= 1; +$softblocksizekb= 10; # was 1 -- PMM $softblocksizebytes= 1024*$softblocksizekb; $tapename= 'st0'; $tape= "/dev/$tapename"; diff --git a/backup/takedown b/backup/takedown index 158347b..e41c783 100755 --- a/backup/takedown +++ b/backup/takedown @@ -1,6 +1,18 @@ #!/bin/sh # # Take the system down for backups and then bring it back up. +# Expects a single (possibly empty) argument X which is used to select +# a file /etc/backup/warnings.X. This file will contain lines like: +# T 300 "in 10 minutes" +# T 240 "in 5 minutes" +# T 45 "in 1 minute" +# T 15 "in 15 seconds" +# configuring the frequency of warning messages. If you call the +# files 'warnings.soon', 'warnings.now' and 'warnings.' then +# you can invoke this as: +# takedown lots of warnings +# takedown soon not so many warnings +# takedown now no warning at all set -e cd /etc/backup @@ -30,4 +42,7 @@ END ) & sleep 1 +# We assume that runlevel 5 is set up suitably for doing backups +# (ie non-essential services turned off in an effort to get the +# tape to stream.) telinit 5 diff --git a/backup/tape.b b/backup/tape.b new file mode 100644 index 0000000..ee89381 --- /dev/null +++ b/backup/tape.b @@ -0,0 +1,3 @@ +filesystems all +next c +end diff --git a/backup/tape.c b/backup/tape.c new file mode 100644 index 0000000..0502c06 --- /dev/null +++ b/backup/tape.c @@ -0,0 +1,3 @@ +filesystems all +next d +end diff --git a/backup/tape.d b/backup/tape.d new file mode 100644 index 0000000..2c27037 --- /dev/null +++ b/backup/tape.d @@ -0,0 +1,3 @@ +filesystems all +next e +end diff --git a/backup/tape.e b/backup/tape.e new file mode 100644 index 0000000..09fcda5 --- /dev/null +++ b/backup/tape.e @@ -0,0 +1,3 @@ +filesystems all +next a +end diff --git a/backup/var.tgz b/backup/var.tgz new file mode 100644 index 0000000..a170943 Binary files /dev/null and b/backup/var.tgz differ diff --git a/backup/whatsthis b/backup/whatsthis new file mode 100755 index 0000000..00d5dff --- /dev/null +++ b/backup/whatsthis @@ -0,0 +1,122 @@ +#!/usr/bin/perl + +# whatsthis : just read an ID off the tape and display it to the user. +# Peter Maydell +# First rough hack; mostly just code nabbed from full. +# --list assumes the dump type was 'zafio', which is a bit bogus. + +# whatsthis : no args => just print tapeid +# whatsthis --list [n] : print tapeid then list archive n (if n omitted, +# 0 is assumed.) Note that archives are numbered from zero! + +sub rewind(); +sub stopandsay(@); + +$etc='/etc/backup'; +require "$etc/settings.pl"; +require 'backuplib.pl'; + +$| = 1; + +# This isn't intended to be run automatically, so don't bother +# with setting status. + +# If we are run with the argument --list then list the backup to +# stdout. Otherwise just print the tape ID. +$listing = 0; # default : don't list +$listing = 1 if ($ARGV[0] eq '--list'); +$listarchive = 0; +$listarchive = $ARGV[1] if defined $ARGV[1]; + +print "Trying to read tape ID from currently inserted tape...\n"; + +unlink 'TAPEID'; +system "mt -f $ntape setblk $blocksizebytes"; $? and die $?; +system "dd if=$ntape bs=${blocksize}b count=10 | tar -b$blocksize -vvxf - TAPEID"; +$? and stopandsay "Failed to read TAPEID.\n"; + +if (!open(T, "TAPEID")) +{ + stopandsay "Tape has no ID label.\n"; +} + +# OK, there's a TAPEID file, read the ID and check for sanity. +chomp($tapeid= ); +if ($tapeid =~ m/[^0-9a-zA-Z]/) +{ + stopandsay "Tape has a bad (non-alphanumeric) TAPEID ($&).\n"; +} +elsif (! $tapeid =~ m/[0-9a-zA-Z]/) +{ + stopandsay "Empty TAPEID.\n"; +} + +print "TAPEID is $tapeid.\n"; +close T; + +# If we aren't listing the tape contents, we can just rewind the tape +# and exit. +if (!$listing) +{ + rewind(); + exit; +} + +# List the contents of archive $listarchive on the tape. +# We are already at the right place for the first archive +# (after the TAPEID). +# For any other archive, we skip forwards to the start of that archive. +if ($listarchive) +{ + system "mt -f $ntape fsf $listarchive"; + $? and stopandsay "Couldn't skip forward to archive $listarchive."; +} + +# Use file to figure out what the archive type is +# This doesn't seem to work too well, so I'm disabling it -- PMM +#$ftype = `dd if=$ntape ibs=$blocksizebytes | file -`; +#$? and stopandsay "couldn't determine file type: $?"; +$ftype = 'POSIX tar'; + +# What we want to do here is roughly: +# dd if=$ntape ibs=$blocksizebytes | readbuffer | afio ??? +# +# where the afio options are such as to list an archive created with +# afio -b $softblocksizebytes -Zvo + +if ($ftype =~ /POSIX tar/) { + # POSIX tar archive; we read it with cpio + $reader = "cpio -it -C$softblocksizebytes -Hustar"; +} elsif ($ftype =~ /ASCII cpio/) { + $reader = "afio -b $softblocksizebytes -Zt -"; +} elsif ($ftype =~ /dump file/) { + stopandsay "sorry: can't list dump files yet"; +} else { + stopandsay "listing failed: unknown archive type"; +} + +# Now back up so we can read the file again properly +#system "mt -f $ntape bsf 1"; $? and stopandsay "couldn't backspace tape: $?"; + +system "dd if=$ntape ibs=$blocksizebytes | /usr/local/lib/backup/readbuffer | $reader"; +$? and stopandsay "listing failed: $?"; + +# All's well, stop here. +print "Listing complete.\n"; +rewind(); +exit; + + +# Rewind the tape. +sub rewind () +{ + system "mt -f $tape rewind"; $? and die $?; +} + +# Print the given message, rewind the tape and exit failure. +sub stopandsay(@) +{ + print @_; + rewind(); + exit(1); +} diff --git a/backup/writebuffer b/backup/writebuffer index 533cdb4..67db196 100755 Binary files a/backup/writebuffer and b/backup/writebuffer differ diff --git a/backup/writebuffer.c b/backup/writebuffer.c index c4fa9e7..259f6bb 100644 --- a/backup/writebuffer.c +++ b/backup/writebuffer.c @@ -1,4 +1,23 @@ -/**/ +/* + * writebuffer.c + * + * Copyright (C) 1997,1998 Ian Jackson + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this file; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ #include #include @@ -7,7 +26,8 @@ #include #include -#define BUFFER 16*1024*1024 +/* was 16MB -- PMM */ +#define BUFFER 6*1024*1024 #define WAITFILL ((BUFFER*3)/4) static inline int min(int a, int b) { return a<=b ? a : b; }