From: ianmdlvl Date: Sun, 7 Oct 2001 16:12:21 +0000 (+0000) Subject: Found on davenant in /usr/local/lib (in use on anarres). X-Git-Tag: branchpoint-2001-10-07-from-pmaydell~4 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?p=chiark-utils.git;a=commitdiff_plain;h=3f7f507605d77ece782e847a5a1606197268f220;ds=sidebyside Found on davenant in /usr/local/lib (in use on anarres). --- diff --git a/backup/Makefile b/backup/Makefile new file mode 100644 index 0000000..1315c88 --- /dev/null +++ b/backup/Makefile @@ -0,0 +1,2 @@ +CC=gcc +CFLAGS=-Wall -Wwrite-strings -Wmissing-prototypes -Wstrict-prototypes -Wpointer-arith -O2 -g diff --git a/backup/backuplib.pl b/backup/backuplib.pl new file mode 100644 index 0000000..d412c94 --- /dev/null +++ b/backup/backuplib.pl @@ -0,0 +1,92 @@ +# + +sub printdate () { + print scalar(localtime),"\n"; +} + +sub setstatus ($) { + open S, ">this-status.new" or die $!; + print S $_[0],"\n" or die $!; + close S or die $!; + rename "this-status.new","this-status" or die $!; +} + +sub startprocess ($$$) { + my ($i,$o,$c) = @_; + print LOG " $c\n" or die $!; + print " $c\n" or die $!; + defined($p= fork) or die $!; + if ($p) { $processes{$p}= $c; return; } + open STDIN,"$i" or die "$c stdin $i: $!"; + open STDOUT,"$o" or die "$c stdout $o: $!"; + &closepipes; + exec $c; die "$c: $!"; +} + +sub endprocesses () { + while (keys %processes) { + $p= waitpid(-1,0) or die "wait: $!"; + if (!exists $processes{$p}) { warn "unknown pid exited: $p, code $?\n"; next; } + $c= $processes{$p}; + delete $processes{$p}; + $? && die "error: command gave code $?: $c\n"; + } + print LOG " ok\n" or die $!; + print " ok\n" or die $!; +} + +sub killprocesses { + for $p (keys %processes) { + kill 15,$p or warn "kill process $p: $!"; + } + undef %processes; +} + +sub readfsys ($) { + my ($fsnm) = @_; + open F, "$etc/fsys.$fsnm" or die "Filesystems $fsnm unknown ($!).\n"; + for (;;) { + $_= or die; chomp; s/\s*$//; + last if m/^end$/; + next unless m/\S/; + next if m/^\#/; + if (m/^prefix\s+(\w+)\s+(\S.*\S)$/) { + $prefix{$1}= $2; + next; + } elsif (m/^prefix\-df\s+(\w+)\s+(\S.*\S)$/) { + $prefixdf{$1}= $2; + next; + } + push @fsys,$_; + } + close F or die $!; +} + +sub parsefsys () { + if ($tf =~ m,^(/\S*)\s+(\w+)$,) { + $atf= $1; + $tm= $2; + $prefix= ''; + stat $atf or die "stat $atf: $!"; + -d _ or die "not a dir: $atf"; + $rstr= ''; + } elsif ($tf =~ m,^(/\S*)\s+(\w+)\s+(\w+)$,) { + $atf= $1; + $tm= $2; + $prefix= $3; + defined($prefix{$prefix}) or die "prefix $prefix in $tf ?\n"; + $rstr= $prefix{$prefix}.' '; + } +} + +sub openlog () { + unlink 'log'; + $u= umask(007); + open LOG, ">log" or die $!; + umask $u; + select(LOG); $|=1; select(STDOUT); +} + +$SIG{'__DIE__'}= 'killprocesses'; + +1; diff --git a/backup/bringup b/backup/bringup new file mode 100755 index 0000000..6bcd5d5 --- /dev/null +++ b/backup/bringup @@ -0,0 +1,5 @@ +#!/bin/sh + +runlevel=`sed -ne '/^id:/ s/.*:\([0-9]\):.*/\1/p' /etc/inittab` +telinit $runlevel +chvt 11 diff --git a/backup/checkallused b/backup/checkallused new file mode 100755 index 0000000..fd613f3 --- /dev/null +++ b/backup/checkallused @@ -0,0 +1,121 @@ +#!/usr/bin/perl + +BEGIN { + $etc= '/etc/backup'; + require "$etc/settings.pl"; + require 'backuplib.pl'; +} + +$|=1; + +open X,'last-tape' or die $!; +chomp($tape= ); +close X or die $!; + +while (!defined $tapedone{$tape}) { + open X,"$etc/tape.$tape" or die "$tape $!"; + $fsg=''; $next=''; + for (;;) { + $_= or die $1; chomp; s/\s*$//; + last if m/^end$/; + next unless m/\S/; + next if m/^\#/; + if (m/^filesystems (\w+)$/) { $fsg= $1; } + elsif (m/^next (\w+)$/) { $next=$1; } + else { die "$tape $_ ?"; } + } + length $fsg or die "$tape $!"; + length $next or die "$tape $!"; + push @{$fsgdone{$fsg}}, $tape; + $tapedone{$tape}=1; + $tape= $next; +} + +for $fsg (sort keys %fsgdone) { + print "filesystem group $fsg: ".join(' ',@{$fsgdone{$fsg}}).":\n "; + @fsys= (); + readfsys($fsg); + for $tf (@fsys) { + parsefsys(); + $pstr= $prefix ne '' ? "$prefix:$atf" : $atf; + &e("dumped twice ($backed{$pstr}, $fsg): $pstr") + if defined $backed{$pstr}; + $backed{$pstr}= $fsg; + print " $pstr"; + } + print "\n"; +} + +print "incremental group:\n "; +@fsys= (); +readfsys('all'); +for $tf (@fsys) { + parsefsys(); + $pstr= $prefix ne '' ? "$prefix:$atf" : $atf; + $incrd{$pstr}= $fsg; + print " $pstr"; +} +print "\n"; + +for $pfx ('', sort keys %prefix) { + $rstr= length($pfx) ? $prefix{$pfx}.' ' : ''; + $dfstr= exists($prefixdf{$pfx}) ? $prefixdf{$pfx} : + 'df --no-sync -xiso9660 -xnfs -xproc'; + $cmd= "$rstr $dfstr"; + open X, "$cmd |" or die $!; + $_= ; m/^Filesystem/ or die "$cmd => $_ ?"; + $ppstr= length($pfx) ? $pfx : ''; + $pstr= length($pfx) ? "$pfx:" : ''; + print "mount points: $ppstr:"; + while () { + chomp; + next if m,^procfs\s,; + m,^/dev/(\S+)\s.*\s(/\S*)\s*$, or die "$_ ?"; + ($dev,$mp) = ($1,$2); + $mounted{"$pstr$mp"}="$pstr$dev"; print " $1-$2"; + if (defined($backto= $backed{"$pstr$mp"})) { + if (m,^/dev/\S+\s+\d+\s+(\d+)\s,) { + $usedkb{$backto} += $1; + } else { + $usedkb{$backto} += 0; + $unkkb{$backto} .= " + $pstr$mp"; + } + } + } + print "\n"; + $!=0; close(X); $? and die "$cmd $? $!"; +} + +foreach $fsg (keys %usedkb) { + print "filesystem group $fsg: $usedkb{$fsg} 1K-blocks$unkkb{$fsg}\n"; +} + +open Z,"$etc/expected-diffs" or die $!; +for (;;) { + $_= or die; chomp; s/\s*$//; + last if m/^end$/; + next unless m/^\S/; + next if m/^\#/; + if (s/^\!//) { + &e("expected not to be dumped, but not a mount point: $_") + unless defined($mounted{$_}); + print "filesystem expected not to be dumped: $_\n"; + delete $mounted{$_}; + } else { + &e("non-filesystem expected to be dumped is mounted: $_ on $mounted{$_}") + if defined($mounted{$_}); + $mounted{$_}= 'expected-diffs'; + print "non-filesystem expected to be dumped: $_\n"; + } +} + +for $fs (sort keys %backed) { length($mounted{$fs}) || &e("dumped ($backed{$fs}), not a mount point: $fs"); } +for $fs (sort keys %incrd) { length($mounted{$fs}) || &e("increm'd ($incrd{$fs}), not a mount point: $fs"); } +for $fs (sort keys %mounted) { length($backed{$fs}) || &e("mount point ($mounted{$fs}), not dumped: $fs"); } +for $fs (sort keys %mounted) { length($incrd{$fs}) || &e("mount point ($mounted{$fs}), not increm'd: $fs"); } + +$emsg.= "configuration ok\n" unless $e; +print STDERR $emsg; +exit($e); + +sub e { $emsg.="** @_\n"; $e=1; } diff --git a/backup/driver b/backup/driver new file mode 100755 index 0000000..5060e2f --- /dev/null +++ b/backup/driver @@ -0,0 +1,29 @@ +#!/bin/sh + +cd /var/local/backup +PATH=/usr/local/lib/backup:$PATH export PATH + +if [ "x$1" != test ]; then + stty sane + stty -isig +fi + +rm -f this-status p p2 +echo 'FAILED to start dump script' >this-status + +full 2>&1 | tee this-log + +status=`cat this-status 2>/dev/null` + +cat <); +$tapeid =~ m/[^0-9a-zA-Z]/ and die "Bad TAPEID ($&).\n"; +$tapeid =~ m/[0-9a-zA-Z]/ or die "Empty TAPEID.\n"; +close T; + +setstatus "FAILED at tape identity check"; + +if (open L, "last-tape") { + chomp($lasttape= ); + close L; +} else { + undef $lasttape; +} + +die "Tape $tapeid same as last time.\n" if $tapeid eq $lasttape; + +undef $fsys; +open D, "$etc/tape.$tapeid" or die "Unknown tape $tapeid ($!).\n"; +for (;;) { + $_= or die; chomp; s/\s+$//; + last if m/^end$/; + next unless m/\S/; + next if m/^\#/; + if (m/^filesystems (\w+)$/) { + $fsys= $1; + } elsif (m/^next (\w+)$/) { + $next= $1; + } elsif (m/^incremental$/) { + $incremental= 1; + } else { + die "unknown entry in tape $tapeid at line $.: $_\n"; + } +} +close D or die $!; + +if ($incremental) { + die "incremental tape $tapeid has next or filesystems\n" + if defined($next) || defined($fsys); + print STDERR "Incremental tape $tapeid.\n\n"; + setstatus "FAILED during incremental startup"; + exec "increm $tapeid"; + die $!; +} + +readfsys("$fsys"); +openlog(); + +$doing= "dump of $fsys to tape $tapeid in drive $tape"; +print LOG "$doing:\n" or die $!; + +system "mt -f $tape rewind"; $? and die $?; +system "mt -f $tape retension"; $? and die $?; + +open T, ">TAPEID" or die $!; +print T "$tapeid\n" or die $!; +close T or die $!; + +system "tar -b$blocksize -vvcf TAPEID.tar TAPEID"; $? and die $?; +system "dd if=TAPEID.tar of=$ntape bs=${blocksize}b count=10"; $? and die $?; + +unlink 'this-md5sums'; + +print "Doing $doing ...\n" or die $!; + +unlink 'p'; +system 'mknod p p'; $? and die $?; + +setstatus "FAILED during dump"; + +sub closepipes () { + close(DUMPOR); close(TEEOR); close(BUFOR); close(FINDOR); + close(DUMPOW); close(TEEOW); close(BUFOW); close(FINDOW); +} + +for $tf (@fsys) { + printdate(); + pipe(FINDOR,FINDOW) or die $!; + pipe(DUMPOR,DUMPOW) or die $!; + pipe(TEEOR,TEEOW) or die $!; + pipe(BUFOR,BUFOW) or die $!; + parsefsys(); + if ($tm eq 'dump') { + $dumpcmd= "dump 0bfu $softblocksizekb - $atf"; + $dumpin= '&FINDOW',$rstr."find $atf -xdev -noleaf -print0"; + $dumpcmd= "cpio -Hustar -o0C$softblocksizebytes"; + $dumpin= '<&FINDOR'; + } + startprocess '>this-md5sums','md5sum'; + startprocess $dumpin,'>&DUMPOW',$rstr.$dumpcmd; + startprocess '<&DUMPOR','>&TEEOW','tee p'; + startprocess '<&TEEOR','>&BUFOW','writebuffer'; + startprocess '<&BUFOR','>/dev/null' + ,"dd ibs=$softblocksizebytes obs=$blocksizebytes of=$ntape"; + closepipes(); + endprocesses(); +} + +setstatus "FAILED during check"; + +system "mt -f $tape rewind"; $? and die $?; +system "mt -f $ntape fsf 1"; $? and die $?; + +open S,"this-md5sums" or die $!; +for $tf (@fsys) { + printdate(); + chomp($orgsum= ); + $orgsum =~ m/^[0-9a-fA-F]{32}$/i or die "orgsum \`$orgsum' ?"; + chomp($csum= `readbuffer <$ntape | md5sum`); + $orgsum eq $csum or die "MISMATCH $tf $csum $orgsum\n"; + print "checksum ok $csum\t$tf\n" or die $!; + print LOG "checksum ok $csum\t$tf\n" or die $!; +} +printdate(); +system "mt -f $tape rewind"; $? and die $?; + +setstatus "FAILED during cleanup"; + +open IAN,">increm-advance.new" or die $!; +print IAN "1\n" or die $!; +close IAN or die $!; + +open TN,">next-full.new" or die $!; +print TN "$next\n" or die $!; +close TN or die $!; + +unlink 'last-tape','next-full'; +rename 'TAPEID','last-tape' or die $!; +rename 'this-md5sums',"md5sums.$fsys" or die $!; +rename 'log',"log.$fsys" or die $!; +rename 'next-full.new',"next-full" or die $!; +rename 'increm-advance.new',"increm-advance" or die $!; + +print "$doing completed.\nNext dump tape is $next.\n" or die $!; + +setstatus "Successful ($tapeid $fsys, next $next)"; +exit 0; diff --git a/backup/increm b/backup/increm new file mode 100755 index 0000000..e4ebbf7 --- /dev/null +++ b/backup/increm @@ -0,0 +1,84 @@ +#!/usr/bin/perl + +BEGIN { + $etc= '/etc/backup'; + require "$etc/settings.pl"; + require 'backuplib.pl'; +} + +$|=1; + +@ARGV==1 or die; +$tapeid= $ARGV[0]; + +print "Running incremental (tape $tapeid) ...\n" or die $!; + +open T,"$etc/tape.$tapeid" or die "Tape $tapeid not found: $!\n"; +close T; + +open NF,"next-full" or die $!; +chomp($next= ); +close NF or die $!; + +setstatus "FAILED during incremental"; + +open A,"increm-advance" or die $!; +chomp($advance= ); +close A or die $!; + +$advance =~ m/^\d+$/ or die "$advance ?"; + +system "mt -f $tape rewind"; $? and die $?; +system "mt -f $tape retension"; $? and die $?; + +if ($advance == 1) { + open TI,">TAPEID" or die $!; + print TI "$tapeid" or die $!; + close TI or die $!; + + system "tar -b$blocksize -vvcf TAPEID.tar TAPEID"; $? and die $?; + system "dd if=TAPEID.tar of=$ntape bs=${blocksize}b count=10"; $? and die $?; +} else { + system "mt -f $ntape fsf $advance"; $? and die $?; +} + +readfsys('all'); +openlog(); + +sub closepipes () { + close(DUMPOR); close(BUFOR); + close(DUMPOW); close(BUFOW); +} + +setstatus "PROBLEMS during incremental dump"; + +for $tf (@fsys) { + pipe(DUMPOR,DUMPOW) or die $!; + pipe(BUFOR,BUFOW) or die $!; + parsefsys(); + if ($tm ne 'dump') { + print "Not dumping $atf ($prefix) - not \`dump'.\n" or die $!; + print LOG "Not dumping $atf ($prefix) - not \`dump'.\n" or die $!; + next; + } + startprocess '&DUMPOW',$rstr."dump 1bfu $softblocksizekb - $atf"; + startprocess '<&DUMPOR','>&BUFOW','writebuffer'; + startprocess '<&BUFOR','>/dev/null' + ,"dd ibs=$softblocksizebytes obs=$blocksizebytes of=$ntape"; + closepipes(); + endprocesses(); + $advance++; +} + +system "mt -f $tape rewind"; $? and die $?; + +open IAN,">increm-advance.new" or die $!; +print IAN "$advance\n" or die $!; +close IAN or die $!; +rename 'increm-advance.new','increm-advance' or die $!; + +print LOG "Next FULL dump tape is $next\n" or die $!; +print "Next FULL dump tape is $next\n" or die $!; + +setstatus "INCREMENTAL successful (next full is $next)"; +exit 0; diff --git a/backup/junk b/backup/junk new file mode 100644 index 0000000..92769e4 --- /dev/null +++ b/backup/junk @@ -0,0 +1,34 @@ + + +timestamp () { + echo "Checking timestamp of last full dump of $1;" + if ! test lfull.$which -ot lfull.$1 + then + which=$1 + fi +} + +ok=1 + +for dsk in `cat allincs` +do + which='.' + . inc.$dsk + if test -f "lfull.$which" + then + echo "Using timestamp of last full dump of $which." + else + echo >&2 "No full dump of $which done yet." + exit 1 + fi + + since="`cat lfull.$which`" + doing="incremental of $filesys to $device + files changed since dump of $which at `cat lfull.$which`" + + rm -f log + (umask 007; >log) + + echo Doing "$doing" ... + + diff --git a/backup/loaded b/backup/loaded new file mode 100755 index 0000000..7a8335f --- /dev/null +++ b/backup/loaded @@ -0,0 +1,15 @@ +#!/bin/sh +# +# State that we have loaded a tape + +set -e +cd /var/local/backup + +if test -f "/etc/backup/tape.$1" +then + echo "$1" >really-TAPEID + echo "Will assume tape is $1 unless I discover otherwise." +else + echo "Will only use tape if it has a TAPEID." + rm -f really-TAPEID +fi diff --git a/backup/old-increm b/backup/old-increm new file mode 100755 index 0000000..b036b89 --- /dev/null +++ b/backup/old-increm @@ -0,0 +1,58 @@ +#!/bin/sh + +set -e +false +dir=/var/local/backup +device=/dev/sda4 +blocksize=2048 +filesystems=all + +cd $dir +PATH=/usr/local/lib/backup:$PATH export PATH + +echo "FAILED during startup of incremental" >this-status + +timestamp='.' +for tsfs in `cat all-fsys-n` +do + echo "Checking timestamp of last full dump of $tsfs;" + if ! test "lfull.$timestamp" -ot "lfull.$tsfs" + then + timestamp="$tsfs" + fi +done + +if test -f "lfull.$timestamp" +then + echo "Using timestamp of last full dump of $timestamp." +else + echo >&2 "No full dump of $timestamp done yet." + exit 1 +fi + +rm -f this-fulldump +since="`cat lfull.$timestamp`" +doing="incremental since dump of $timestamp at `cat lfull.$timestamp` to $device" + +>increm-log +chmod 660 increm-log + +echo Doing "$doing" ... +echo "$doing": >>increm-log +cat fsys.all +echo "tar -lvvc \"-N$since\" -T$dir/fsys.all -f - \\" +echo " 2>>increm-log | dd of=$device bs=${blocksize}b" +echo "FAILED during incremental" >this-status +cd / +tar -lvvc "-N$since" -T$dir/fsys.all -f - 2>>$dir/increm-log \ + | dd of=$device bs=${blocksize}b +cd $dir + +echo "FAILED during cleanup" >this-status + +test -f this-fulldump && mv this-fulldump "lfull.$filesystems" +echo "$doing" completed. + +echo "Next dump tape to use would be `cat next-full`". + +echo "INCREMENTAL successful" >this-status diff --git a/backup/readbuffer b/backup/readbuffer new file mode 100755 index 0000000..7b86726 Binary files /dev/null and b/backup/readbuffer differ diff --git a/backup/readbuffer.c b/backup/readbuffer.c new file mode 100644 index 0000000..4cd31af --- /dev/null +++ b/backup/readbuffer.c @@ -0,0 +1,94 @@ +/**/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define BUFFER 16*1024*1024 +#define WAITEMPTY ((BUFFER*1)/4) + +static inline int min(int a, int b) { return a<=b ? a : b; } + +static void nonblock(int fd) { + int r; + r= fcntl(fd,F_GETFL,0); if (r == -1) { perror("fcntl getfl"); exit(1); } + r |= O_NDELAY; + if (fcntl(fd,F_SETFL,r) == -1) { perror("fcntl setfl"); exit(1); } +} + +int main(int argc, const char *const *argv) { + static unsigned char buf[BUFFER]; + + unsigned char *wp, *rp; + int used,r,reading,seeneof; + fd_set readfds; + fd_set writefds; + + used=0; wp=rp=buf; reading=1; seeneof=0; + nonblock(0); nonblock(1); + if (argv[1] && !strcmp(argv[1],"--mlock")) { + if (mlock(buf,sizeof(buf))) { perror("mlock"); exit(1); } + argv++; argc--; + } + if (argv[1]) { fputs("usage: readbuffer [--mlock]\n",stderr); exit(1); } + while (!seeneof || used) { + FD_ZERO(&readfds); + if (reading) { + if (used +#include +#include +#include +#include +#include + +#define BUFFER 16*1024*1024 +#define WAITFILL ((BUFFER*3)/4) + +static inline int min(int a, int b) { return a<=b ? a : b; } + +static void nonblock(int fd) { + int r; + r= fcntl(fd,F_GETFL,0); if (r == -1) { perror("fcntl getfl"); exit(1); } + r |= O_NDELAY; + if (fcntl(fd,F_SETFL,r) == -1) { perror("fcntl setfl"); exit(1); } +} + +int main(void) { + static unsigned char buf[BUFFER]; + + unsigned char *wp, *rp; + int used,r,writing,seeneof; + fd_set readfds; + fd_set writefds; + + used=0; wp=rp=buf; writing=0; seeneof=0; + nonblock(0); nonblock(1); + while (!seeneof || used) { + FD_ZERO(&readfds); if (!seeneof && used+1 WAITFILL) writing=1; + } + if (FD_ISSET(1,&writefds) && used) { + r= write(1,wp,min(used,buf+BUFFER-wp)); + if (r<=0) { + if (!(errno == EAGAIN || errno == EINTR)) { perror("write"); exit(1); } +/*fprintf(stderr,"\t write transient error\n");*/ + } else { +/*fprintf(stderr,"\t wrote %d\n",r);*/ + used-= r; + wp+= r; + if (wp == buf+BUFFER) wp=buf; + } + } + } + exit(0); +}