--- /dev/null
+From ijackson@chiark.greenend.org.uk Wed Aug 12 19:36:07 BST 1998
+Article: 742 of ucam.comp.unix
+Path: ewrotcd!not-for-mail
+From: ijackson@chiark.greenend.org.uk (Ian Jackson)
+Newsgroups: ucam.comp.unix,comp.sys.sun.admin
+Subject: Buffering programs (was Re: Speeding up network dumps)
+Date: 21 Jul 1998 12:23:23 +0100 (BST)
+Organization: Linux Unlimited
+Lines: 233
+Message-ID: <xWb*k6rBn@news.chiark.greenend.org.uk>
+References: <6oiapl$6k5$1@pegasus.csx.cam.ac.uk> <6ol7kf$cln$2@pegasus.csx.cam.ac.uk>
+NNTP-Posting-Host: chiark.greenend.org.uk
+X-Server-Date: 21 Jul 1998 11:23:25 GMT
+Originator: ijackson@news.chiark.greenend.org.uk ([127.0.0.1])
+Xref: news.chiark.greenend.org.uk ucam.comp.unix:742 comp.sys.sun.admin:14526
+
+I said I'd post here the buffering programs that I use, so here they
+are.
+
+They're perfectly functional and (I believe) correct, but they do lack
+sophistication. The size of the buffer is a #define, and the
+diagnostics lack verbosity. I haven't compiled them on anything but
+Linux/i386 libc5, but they ought to work on other reasonable Unices.
+
+writebuffer is intended for writing to tapes; when its buffer is empty
+it will write no more data until it gets 3/4 of its buffer filled, or
+EOF.
+
+readbuffer is intended for reading from tapes; when its buffer becomes
+full it stops reading until it's down to 1/4.
+
+Remember that using shell pipes loses the exit status of all but the
+last command, and so can cause failures to go unnoticed.
+
+
+/*
+ * writebuffer.c
+ *
+ * Copyright (C) 1997,1998 Ian Jackson <ian@chiark.greenend.org.uk>
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2,
+ * or (at your option) any later version.
+ *
+ * This is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this file; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+
+#include <sys/time.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <errno.h>
+#include <unistd.h>
+
+#define BUFFER 16*1024*1024
+#define WAITFILL ((BUFFER*3)/4)
+
+static inline int min(int a, int b) { return a<=b ? a : b; }
+
+static void nonblock(int fd) {
+ int r;
+ r= fcntl(fd,F_GETFL,0); if (r == -1) { perror("fcntl getfl"); exit(1); }
+ r |= O_NDELAY;
+ if (fcntl(fd,F_SETFL,r) == -1) { perror("fcntl setfl"); exit(1); }
+}
+
+int main(void) {
+ static unsigned char buf[BUFFER];
+
+ unsigned char *wp, *rp;
+ int used,r,writing,seeneof;
+ fd_set readfds;
+ fd_set writefds;
+
+ used=0; wp=rp=buf; writing=0; seeneof=0;
+ nonblock(0); nonblock(1);
+ while (!seeneof || used) {
+ FD_ZERO(&readfds); if (!seeneof && used+1<BUFFER) FD_SET(0,&readfds);
+ FD_ZERO(&writefds); if (writing) FD_SET(1,&writefds);
+/*fprintf(stderr,"used %6d writing %d seeneof %d wp %8lx rp %8lx read %d write %d\n",
+ used,writing,seeneof,(unsigned long)(wp-buf),(unsigned long)(rp-buf),
+ FD_ISSET(0,&readfds),FD_ISSET(1,&writefds));*/
+ r= select(2,&readfds,&writefds,0,0);
+/*fprintf(stderr,"\t readable %d writeable %d\n",
+ FD_ISSET(0,&readfds),FD_ISSET(1,&writefds));*/
+ if (r == -1) {
+ if (errno == EINTR) continue;
+ perror("select"); exit(1);
+ }
+ if (FD_ISSET(1,&writefds) && !FD_ISSET(0,&readfds) && !used) {
+ writing= 0;
+ FD_CLR(1,&writefds);
+/*fprintf(stderr,"\t buffers empty - stopping\n");*/
+ }
+ if (FD_ISSET(0,&readfds)) {
+ r= read(0,rp,min(BUFFER-1-used,buf+BUFFER-rp));
+ if (!r) {
+/*fprintf(stderr,"\t eof detected\n");*/
+ seeneof=1; writing=1;
+ } else if (r<0) {
+ if (!(errno == EAGAIN || errno == EINTR)) { perror("read"); exit(1); }
+fprintf(stderr,"\t read transient error\n");
+ } else {
+/*fprintf(stderr,"\t read %d\n",r);*/
+ used+= r;
+ rp+= r;
+ if (rp == buf+BUFFER) rp=buf;
+/*fprintf(stderr,"\t starting writes\n");*/
+ }
+ if (used > WAITFILL) writing=1;
+ }
+ if (FD_ISSET(1,&writefds) && used) {
+ r= write(1,wp,min(used,buf+BUFFER-wp));
+ if (r<=0) {
+ if (!(errno == EAGAIN || errno == EINTR)) { perror("write"); exit(1); }
+/*fprintf(stderr,"\t write transient error\n");*/
+ } else {
+/*fprintf(stderr,"\t wrote %d\n",r);*/
+ used-= r;
+ wp+= r;
+ if (wp == buf+BUFFER) wp=buf;
+ }
+ }
+ }
+ exit(0);
+}
+
+
+/*
+ * readbuffer.c
+ *
+ * Copyright (C) 1997,1998 Ian Jackson <ian@chiark.greenend.org.uk>
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2,
+ * or (at your option) any later version.
+ *
+ * This is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this file; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+
+#include <sys/time.h>
+#include <sys/types.h>
+#include <assert.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <errno.h>
+#include <unistd.h>
+
+#define BUFFER 16*1024*1024
+#define WAITEMPTY ((BUFFER*1)/4)
+
+static inline int min(int a, int b) { return a<=b ? a : b; }
+
+static void nonblock(int fd) {
+ int r;
+ r= fcntl(fd,F_GETFL,0); if (r == -1) { perror("fcntl getfl"); exit(1); }
+ r |= O_NDELAY;
+ if (fcntl(fd,F_SETFL,r) == -1) { perror("fcntl setfl"); exit(1); }
+}
+
+int main(void) {
+ static unsigned char buf[BUFFER];
+
+ unsigned char *wp, *rp;
+ int used,r,reading,seeneof;
+ fd_set readfds;
+ fd_set writefds;
+
+ used=0; wp=rp=buf; reading=1; seeneof=0;
+ nonblock(0); nonblock(1);
+ while (!seeneof || used) {
+ FD_ZERO(&readfds);
+ if (reading) {
+ if (used<BUFFER-1) {
+ FD_SET(0,&readfds);
+ } else {
+/*fprintf(stderr,"\t buffers full - stopping\n");*/
+ reading=0;
+ }
+ }
+ FD_ZERO(&writefds); if (used) FD_SET(1,&writefds);
+/*fprintf(stderr,"used %6d reading %d seeneof %d wp %8lx rp %8lx read %d write %d\n",
+ used,reading,seeneof,(unsigned long)(wp-buf),(unsigned long)(rp-buf),
+ FD_ISSET(0,&readfds),FD_ISSET(1,&writefds));*/
+ r= select(2,&readfds,&writefds,0,0);
+/*fprintf(stderr,"\t readable %d writeable %d\n",
+ FD_ISSET(0,&readfds),FD_ISSET(1,&writefds));*/
+ if (r == -1) {
+ if (errno == EINTR) continue;
+ perror("select"); exit(1);
+ }
+ if (FD_ISSET(0,&readfds)) {
+ r= read(0,rp,min(BUFFER-1-used,buf+BUFFER-rp));
+ if (!r) {
+/*fprintf(stderr,"\t eof detected\n");*/
+ seeneof=1; reading=0;
+ } else if (r<0) {
+ if (!(errno == EAGAIN || errno == EINTR)) { perror("read"); exit(1); }
+/*fprintf(stderr,"\t read transient error\n");*/
+ } else {
+/*fprintf(stderr,"\t read %d\n",r);*/
+ used+= r;
+ rp+= r;
+ if (rp == buf+BUFFER) rp=buf;
+ }
+ }
+ if (FD_ISSET(1,&writefds)) {
+ assert(used);
+ r= write(1,wp,min(used,buf+BUFFER-wp));
+ if (r<=0) {
+ if (!(errno == EAGAIN || errno == EINTR)) { perror("write"); exit(1); }
+/*fprintf(stderr,"\t write transient error\n");*/
+ } else {
+/*fprintf(stderr,"\t wrote %d\n",r);*/
+ used-= r;
+ wp+= r;
+ if (wp == buf+BUFFER) wp=buf;
+ }
+ if (used < WAITEMPTY && !seeneof) {
+/*fprintf(stderr,"\t starting writes\n");*/
+ reading=1;
+ }
+ }
+ }
+ exit(0);
+}
+
+--
+Ian Jackson personal email: <ijackson@chiark.greenend.org.uk>
+These opinions are my own. http://www.chiark.greenend.org.uk/~ijackson/
+PGP2 public key id 0x23f5addb, fingerprint 5906F687 BD03ACAD 0D8E602E FCF37657
+
+
#
+# Assorted useful functions used by the backup scripts.
+
sub printdate () {
print scalar(localtime),"\n";
}
+# Set status info -- we write the current status to a file
+# so if we hang or crash the last thing written to the file
+# will tell us where we were when things went pear-shaped.
sub setstatus ($) {
open S, ">this-status.new" or die $!;
print S $_[0],"\n" or die $!;
rename "this-status.new","this-status" or die $!;
}
+# startprocess, endprocesses, killprocesses are
+# used to implement the funky pipeline stuff.
sub startprocess ($$$) {
my ($i,$o,$c) = @_;
print LOG " $c\n" or die $!;
undef %processes;
}
+# Read a fsys.foo filesystem group definition file.
+# Syntax is: empty lines and those beginning with '#' are ignored.
+# Trailing whitespace is ignored. Lines of the form 'prefix foo bar'
+# are handled specially, as arex lines 'exclude regexp'; otherwise
+# we just shove the line into @fsys and let parsefsys deal with it.
sub readfsys ($) {
my ($fsnm) = @_;
open F, "$etc/fsys.$fsnm" or die "Filesystems $fsnm unknown ($!).\n";
for (;;) {
- $_= <F> or die; chomp; s/\s*$//;
+ $_= <F> or die "unexpected EOF in $etc/fsys.$fsnm\n"; chomp; s/\s*$//;
last if m/^end$/;
next unless m/\S/;
next if m/^\#/;
if (m/^prefix\s+(\w+)\s+(\S.*\S)$/) {
$prefix{$1}= $2;
next;
- } elsif (m/^prefix\-df\s+(\w+)\s+(\S.*\S)$/) {
- $prefixdf{$1}= $2;
- next;
}
+ if (m/^excludedir\s+(\S.*\S)$/) {
+ push @excldir,$1;
+ next;
+ }
+ if (m/^exclude\s+(\S.*\S)$/) {
+ push @excl,$1;
+ next;
+ }
push @fsys,$_;
}
close F or die $!;
}
+# Parse a line from a filesystem definition file. We expect the line
+# to be in $tf.
sub parsefsys () {
if ($tf =~ m,^(/\S*)\s+(\w+)$,) {
+ # Line of form '/file/system dumptype'
$atf= $1;
$tm= $2;
$prefix= '<local>';
-d _ or die "not a dir: $atf";
$rstr= '';
} elsif ($tf =~ m,^(/\S*)\s+(\w+)\s+(\w+)$,) {
+ # Line of form '/file/system dumptype prefix'
+ # (used for remote backups, I think)
$atf= $1;
$tm= $2;
$prefix= $3;
#!/bin/sh
+# Very simple: extract the default runlevel from /etc/inittab
+# and change to it with telinit.
runlevel=`sed -ne '/^id:/ s/.*:\([0-9]\):.*/\1/p' /etc/inittab`
telinit $runlevel
-chvt 11
+# This switches to virtual console 11, but I don't think I want that -- PMM
+#chvt 11
#!/usr/bin/perl
+# Read all the configuration files and check that all filesystems
+# are either backed up in both full and incremental dumps or
+# listed as exceptions.
+
BEGIN {
$etc= '/etc/backup';
require "$etc/settings.pl";
for $tf (@fsys) {
parsefsys();
$pstr= $prefix ne '<local>' ? "$prefix:$atf" : $atf;
- &e("dumped twice ($backed{$pstr}, $fsg): $pstr")
- if defined $backed{$pstr};
$backed{$pstr}= $fsg;
print " $pstr";
}
for $pfx ('', sort keys %prefix) {
$rstr= length($pfx) ? $prefix{$pfx}.' ' : '';
- $dfstr= exists($prefixdf{$pfx}) ? $prefixdf{$pfx} :
- 'df --no-sync -xiso9660 -xnfs -xproc';
- $cmd= "$rstr $dfstr";
- open X, "$cmd |" or die $!;
- $_= <X>; m/^Filesystem/ or die "$cmd => $_ ?";
+ open X, $rstr." df --no-sync -xnfs |" or die $!;
+ $_= <X>; m/^Filesystem/ or die "$_ ?";
$ppstr= length($pfx) ? $pfx : '<local>';
$pstr= length($pfx) ? "$pfx:" : '';
print "mount points: $ppstr:";
while (<X>) {
chomp;
- next if m,^procfs\s,;
m,^/dev/(\S+)\s.*\s(/\S*)\s*$, or die "$_ ?";
- ($dev,$mp) = ($1,$2);
- $mounted{"$pstr$mp"}="$pstr$dev"; print " $1-$2";
- if (defined($backto= $backed{"$pstr$mp"})) {
- if (m,^/dev/\S+\s+\d+\s+(\d+)\s,) {
- $usedkb{$backto} += $1;
- } else {
- $usedkb{$backto} += 0;
- $unkkb{$backto} .= " + $pstr$mp";
- }
- }
+ $mounted{"$pstr$2"}="$pstr$1"; print " $1-$2";
}
print "\n";
- $!=0; close(X); $? and die "$cmd $? $!";
}
+$!=0; close(X); $? and die "$? $!";
-foreach $fsg (keys %usedkb) {
- print "filesystem group $fsg: $usedkb{$fsg} 1K-blocks$unkkb{$fsg}\n";
-}
+# We check that all mounted filesystems are dumped and all
+# filesystems to be dumped are mounted. The expected-diffs
+# config file allows us to make exceptions.
+# eg:
+# #expect disk2 to be mounted but not dumped
+# !/disk2
+# # CD may or may not be mounted but should not be dumped in either case
+# ?/cdrom
open Z,"$etc/expected-diffs" or die $!;
for (;;) {
last if m/^end$/;
next unless m/^\S/;
next if m/^\#/;
- if (s/^\!//) {
+ if (s/^\?//) {
+ print "non-permanent filesystem expected not to be dumped: $_\n";
+ if (defined($mounted{$_})) {
+ delete $mounted{$_};
+ }
+ } elsif (s/^\!//) {
&e("expected not to be dumped, but not a mount point: $_")
unless defined($mounted{$_});
print "filesystem expected not to be dumped: $_\n";
#!/bin/sh
+# This is the driver script that actually runs backups.
cd /var/local/backup
PATH=/usr/local/lib/backup:$PATH export PATH
rm -f this-status p p2
echo 'FAILED to start dump script' >this-status
+# Here we go : run 'full', which (name notwithstanding) handles
+# both full and incremental backups, according to the ID of the
+# tape in the drive.
full 2>&1 | tee this-log
status=`cat this-status 2>/dev/null`
-cat <<END - this-log | /usr/lib/sendmail -oi -om -odq -t
-To: dump-reports
+# Mail a report to somewhere appropriate; -odq removed (means just
+# queue message, don't try to deliver) because it just delays the
+# message (you might want that if mail was one of the services turned
+# off for the duration of the backup, though).
+cat <<END - this-log | /usr/lib/sendmail -oi -om -t
+To: dump-reports@mnementh.local
Subject: Dump Report: $status
END
rm -f /TAPEID
+# Local hack for DAT drive on kuzuth: when done, eject the tape -- PMM
+mt rewoffl
+
if [ "x$1" != test ]; then
+ # Bring the system up as multiuser again
bringup
stty isig
fi
-#!sfere:/opt/bigdisc
-!davenant:/export/mirror
-davenant:/export/mirror/work
-!davenant:/export/mp3
-!kadath:/tmp
-#!elmyra:/mnt/fat/data
+?mnementh:/cdrom
end
-#
-
-prefix davenant ssh -o 'BatchMode yes' -c blowfish -o 'Compression yes' davenant 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/usr/sbin:$PATH'
-prefix xenophobe ssh -o 'BatchMode yes' -c blowfish -o 'Compression no' xenophobe 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/usr/sbin:$PATH'
-
-prefix khem ssh -o 'BatchMode yes' -c blowfish -o 'Compression yes' -o 'CompressionLevel 1' khem 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/usr/sbin:$PATH'
-
-prefix kadath ssh -o 'BatchMode yes' -c blowfish -o 'Compression yes' -o 'CompressionLevel 1' kadath -l ian 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/usr/sbin:$PATH really'
-prefix-df kadath /bin/df -t noprocfs,nfs
-#
-
-/usr/src dump davenant
-
-/var dump davenant
-/u dump davenant
-/ dump
-
-/ dump davenant
-
-/export/mirror/work cpio davenant
-
-/usr dump davenant
-/boot dump davenant
-#
-
-/ dump kadath
-
-/usr dump kadath
-/var dump kadath
-/home dump kadath
-
-/var dump khem
-/home dump khem
-/ dump khem
-
-/usr dump khem
-#
-
-/ dump xenophobe
-
-/dos/c cpio
-/dos/d cpio
+# Backup all filesystems on mnementh and kuzuth
+#
+# prefix for remote access to mnementh:
+prefix mnementh ssh mnementh
+excludedir '.*/tmp'
+excludedir '.*/nobackup'
+excludedir '.*netscape/cache'
+exclude '^/export/hp/swap'
+excludedir '^/var/spool/squid'
+excludedir '^/var/spool/news'
+/ cpio
+/disk2 cpio
+/ cpio mnementh
+/boot cpio mnementh
+/usr/local/src cpio mnementh
+/scratch cpio mnementh
end
$|=1;
+# Check to see whether the tape.nn and fsys.nn files are sane.
+# checkallused checks that all the filesystems mounted are in fact
+# dumped in both full and incremental dumps.
+setstatus "FAILED configuration check";
print "Configuration check ...\n" or die $!;
system 'checkallused'; $? and die $?;
printdate();
+setstatus "FAILED reading tape ID";
+# Try to read the tape ID from the tape into the file TAPEID
unlink 'TAPEID';
-system "mt -f $tape rewind"; $? and die $?;
system "mt -f $tape setblk $blocksizebytes"; $? and die $?;
system "dd if=$tape bs=${blocksize}b count=10 | tar -b$blocksize -vvxf - TAPEID";
setstatus "FAILED during startup";
+# We need some ID; if the tape has one already that takes precedence;
+# otherwise the user might have set a tape ID that this should be
+# by creating really-TAPEID.
if (open T, "TAPEID") {
unlink 'really-TAPEID';
} elsif (open T, "really-TAPEID") {
die "No TAPEID.\n";
}
+# read the ID; it had better be a non-empty string of alphanumeric chars.
chomp($tapeid= <T>);
$tapeid =~ m/[^0-9a-zA-Z]/ and die "Bad TAPEID ($&).\n";
$tapeid =~ m/[0-9a-zA-Z]/ or die "Empty TAPEID.\n";
setstatus "FAILED at tape identity check";
+# We don't let the user overwrite the tape used for the last backup.
if (open L, "last-tape") {
chomp($lasttape= <L>);
close L;
die "Tape $tapeid same as last time.\n" if $tapeid eq $lasttape;
-if (defined($tapedesc= readlink "$etc/tape.$tapeid")) {
- $tapedesc =~ s/^.*\.//;
- $tapedesc .= "($tapeid)";
-} else {
- $tapedesc = $tapeid;
-}
-
+# Parse the appropriate tape.nn file.
+# Format is: empty lines and lines starting '#' are ignored. Trailing
+# whitespace is ignored. File must end with 'end' on a line by itself.
+# Either there should be a line 'incremental' to indicate that this is
+# a tape for incremental backups, or a pair of lines 'filesystems fsg'
+# and 'next tapeid', indicating that this tape is part of a full
+# backup, containing the filesystem group fsg.
undef $fsys;
open D, "$etc/tape.$tapeid" or die "Unknown tape $tapeid ($!).\n";
for (;;) {
}
close D or die $!;
+# Incremental backups are handled by increm, not us.
if ($incremental) {
die "incremental tape $tapeid has next or filesystems\n"
if defined($next) || defined($fsys);
print STDERR "Incremental tape $tapeid.\n\n";
setstatus "FAILED during incremental startup";
- exec "increm",$tapeid,$tapedesc;
+ exec "increm $tapeid";
die $!;
}
+# Read the filesystem group definition (file fsys.nnn)
readfsys("$fsys");
openlog();
-$doing= "dump of $fsys to tape $tapedesc in drive $tape";
+$doing= "dump of $fsys to tape $tapeid in drive $tape";
print LOG "$doing:\n" or die $!;
+setstatus "FAILED writing tape ID";
+# First write the tape ID to this tape.
open T, ">TAPEID" or die $!;
-print T "$tapeid\n$tapedesc\n" or die $!;
+print T "$tapeid\n" or die $!;
close T or die $!;
system "tar -b$blocksize -vvcf TAPEID.tar TAPEID"; $? and die $?;
close(DUMPOW); close(TEEOW); close(BUFOW); close(FINDOW);
}
+# work out a find option string that will exclude the required files
+# Note that dump pays no attention to exclude options.
+$exclopt = '';
+foreach $exc (@excldir) {
+ $exclopt .= "-regex $exc -prune -o ";
+}
+foreach $exc (@excl) {
+ $exclopt .= "-regex $exc -o ";
+}
+
+# For each filesystem to be put on this tape:
for $tf (@fsys) {
printdate();
pipe(FINDOR,FINDOW) or die $!;
pipe(TEEOR,TEEOW) or die $!;
pipe(BUFOR,BUFOW) or die $!;
parsefsys();
+
+ # We can back up via dump or cpio
if ($tm eq 'dump') {
$dumpcmd= "dump 0bfu $softblocksizekb - $atf";
$dumpin= '</dev/null';
- } else {
+ } elsif ($tm eq 'cpio') {
startprocess '</dev/null','>&FINDOW',$rstr."find $atf -xdev -noleaf -print0";
$dumpcmd= "cpio -Hustar -o0C$softblocksizebytes";
$dumpin= '<&FINDOR';
+ } elsif ($tm eq 'zafio') {
+ # compress-each-file-then-archive using afio
+ startprocess '</dev/null','>&FINDOW',$rstr."find $atf -xdev -noleaf $exclopt -print";
+ # don't use verbose flag as this generates 2MB report emails :->
+ $dumpcmd = "afio -b $softblocksizebytes -Zo -";
+ $dumpin = '<&FINDOR';
}
+ # This is a funky way of doing a pipeline which pays attention
+ # to the exit status of all the commands in the pipeline.
+ # It is roughly equivalent to:
+ # md5sum <p >>this-md5sums
+ # dump <$dumpin | tee p | writebuffer | dd >/dev/null
startprocess '<p','>>this-md5sums','md5sum';
startprocess $dumpin,'>&DUMPOW',$rstr.$dumpcmd;
startprocess '<&DUMPOR','>&TEEOW','tee p';
endprocesses();
}
+# The backup should now be complete; verify it
+
setstatus "FAILED during check";
+# Rewind the tape and skip the TAPEID record
system "mt -f $tape rewind"; $? and die $?;
system "mt -f $ntape fsf 1"; $? and die $?;
+# Check the md5sums match for each filesystem on the tape
open S,"this-md5sums" or die $!;
for $tf (@fsys) {
printdate();
chomp($orgsum= <S>);
$orgsum =~ m/^[0-9a-fA-F]{32}$/i or die "orgsum \`$orgsum' ?";
- chomp($csum= `readbuffer <$ntape | md5sum`);
+ chomp($csum= `dd if=$ntape ibs=$blocksizebytes | readbuffer | md5sum`);
$orgsum eq $csum or die "MISMATCH $tf $csum $orgsum\n";
print "checksum ok $csum\t$tf\n" or die $!;
print LOG "checksum ok $csum\t$tf\n" or die $!;
setstatus "FAILED during cleanup";
+# Write to some status files to indicate what the backup system
+# ought to do when next invoked.
+# reset incremental backup count to 1.
open IAN,">increm-advance.new" or die $!;
print IAN "1\n" or die $!;
close IAN or die $!;
+# Next full backup is whatever the next link in the tape description
+# file says it ought to be.
open TN,">next-full.new" or die $!;
print TN "$next\n" or die $!;
close TN or die $!;
unlink 'last-tape','next-full';
+# We are the last tape to have been backed up
rename 'TAPEID','last-tape' or die $!;
rename 'this-md5sums',"md5sums.$fsys" or die $!;
rename 'log',"log.$fsys" or die $!;
print "$doing completed.\nNext dump tape is $next.\n" or die $!;
-setstatus "Successful: $tapedesc $fsys, next $next";
+setstatus "Successful ($tapeid $fsys, next $next)";
exit 0;
#!/usr/bin/perl
+# Do an incremental backup. We are invoked by full if the tape
+# description file says that it is for an incremental backup.
+# We expect a single commandline argument which is the ID string
+# of the tape to use.
+
BEGIN {
$etc= '/etc/backup';
require "$etc/settings.pl";
$|=1;
-@ARGV==2 or die;
-($tapeid,$tapedesc)= @ARGV;
+@ARGV==1 or die;
+$tapeid= $ARGV[0];
-print "Running incremental onto $tapedesc ...\n" or die $!;
+print "Running incremental (tape $tapeid) ...\n" or die $!;
+# Just check that we weren't passed a bogus ID (the tape description
+# file for incrementals is just 'incremental\nend\n')
open T,"$etc/tape.$tapeid" or die "Tape $tapeid not found: $!\n";
close T;
+# This is only used for the 'next FULL backup is X' message at the end.
open NF,"next-full" or die $!;
chomp($next= <NF>);
close NF or die $!;
setstatus "FAILED during incremental";
+# Read the number of the incremental involved
+# (ie, (how many files are already on the tape) - 1)
open A,"increm-advance" or die $!;
chomp($advance= <A>);
close A or die $!;
+# better be a decimal number
$advance =~ m/^\d+$/ or die "$advance ?";
+# Rewind the tape and if we are the first incremental on the tape then
+# write the TAPEID record, otherwise skip forward to the correct point.
+# (full will already have checked that this is the right tape before
+# it invoked us, so no need to read the existing TAPEID record first.)
system "mt -f $ntape rewind"; $? and die $?;
if ($advance == 1) {
open TI,">TAPEID" or die $!;
- print TI "$tapeid\n$tapedesc\n" or die $!;
+ print TI "$tapeid" or die $!;
close TI or die $!;
system "tar -b$blocksize -vvcf TAPEID.tar TAPEID"; $? and die $?;
system "mt -f $ntape fsf $advance"; $? and die $?;
}
+# Get a list of all filesystems
readfsys('all');
openlog();
setstatus "PROBLEMS during incremental dump";
for $tf (@fsys) {
+
pipe(DUMPOR,DUMPOW) or die $!;
pipe(BUFOR,BUFOW) or die $!;
parsefsys();
print LOG "Not dumping $atf ($prefix) - not \`dump'.\n" or die $!;
next;
}
+ # Same trick as full uses to do a pipeline whilst keeping track
+ # of all exit statuses:
+ # dump </dev/null | writebuffer | dd >/dev/null
startprocess '</dev/null','>&DUMPOW',$rstr."dump 1bfu $softblocksizekb - $atf";
startprocess '<&DUMPOR','>&BUFOW','writebuffer';
startprocess '<&BUFOR','>/dev/null'
,"dd ibs=$softblocksizebytes obs=$blocksizebytes of=$ntape";
closepipes();
endprocesses();
+ # advance is a file counter, so it needs to be updated for each
+ # dump we do to tape.
$advance++;
}
+# Rewind the tape, and increment the counter of incremental backups.
system "mt -f $tape rewind"; $? and die $?;
open IAN,">increm-advance.new" or die $!;
print IAN "$advance\n" or die $!;
print LOG "Next FULL dump tape is $next\n" or die $!;
print "Next FULL dump tape is $next\n" or die $!;
-setstatus "INCREMENTAL successful: $tapedesc, next full is $next";
+setstatus "INCREMENTAL successful (next full is $next)";
exit 0;
--- /dev/null
+This is a quick summary of IWJ's backup scripts and my config files:
+it's a bit patchy and might have the odd ommission. The canonical
+source is the sources, as always :->
+
+The three tarfiles in this directory should go in the following
+places (the paths can probably be configured/hacked, but this is
+where they are on my system and on chiark):
+
+etc.tgz : /etc/backup/
+lib.tgz : /usr/local/lib/backup/
+var.tgz : /var/local/backup/
+
+NOTE: these versions are not those used on chiark; they
+are somewhat modified by me (couple of extra features and
+lots of comments -- all errors in those are mine.)
+
+NB: you'll probably need to delete some of the files from
+var.tgz (in fact, I think you probably don't want any of
+them except maybe last-tape (which you want to change anyway).
+You'll also need to recompile readbuffer and writebuffer unless
+you're using SPARClinux :->
+
+Contents of /etc/backup/:
+warnings.* : files defining how many warnings you get as the
+system is brought down to do backups. The defaults are fine.
+settings.pl : generic config file: in particular, the name of
+the tape device is set here.
+tape.* : each tape you're going to use in the backup cycle
+has a name and a config file. Here the tapes are named 'a'-'e',
+although longer names should be OK. You need at least two
+tapes defined as the system won't write a backup on the same
+tape it wrote the last one to.
+
+Syntax of the tape.* files:
+filesystems X
+next N
+end
+
+where N is the next tape in the sequence (which should
+be circular; I have a->b->c->d->e->a...) and X is a
+filesystem-name (list of filesystems might work?).
+
+Each defined filesystem has a name and a config file
+fsys.<name>. These files define what is backed up and how.
+The filesystem 'all' must exist; it's used for incremental
+backups (and it must exist even if you don't do incrementals).
+I don't have any other filesystems defined as everything fits
+on one tape.
+
+Syntax of these files:
+Empty lines and lines starting '#' are comments and ignored.
+Lines starting 'excludedir' given regexps of things to exclude
+(temp dirs, Netscape's cache, etc).
+Lines starting 'prefix' give a command prefix necessary to
+run things on a remote machine:
+prefix <prefix-name> <command-part>
+Other lines should be of the form
+<directory name> <backup-type>
+for local backups, or
+<directory name> <backup-type> <prefix-name>
+for remote backups.
+The file must end with the word 'end' on a line of its own.
+
+Valid values for <backup-type> are 'cpio' (uses cpio to produce
+tar-format backups), 'dump' (uses dump to dump entire filesystems;
+<directory name> should be a mount-point for this), and [if you
+use my version of the scripts] 'zafio' (uses afio to compress
+each file as it is backed up). Only 'dump' type backups permit
+incremental backups.
+
+Finally, expected-diffs is a config file to indicate which
+filesystems should *not* be backed up. The scripts do a config
+check which involves checking that:
+ * all filesystems to be backed up are present
+ * all filesystems that are present are backed up
+expected-diffs allows you to make exceptions to this; backing
+up your CDROM drive is a bit pointless, frex.
+The format here is:
+<prefixchar><mountpoint>
+
+where <prefixchar> is ?, ! or nothing, and
+<mountpoint> is <prefix>:<mountpoint> for a remote fs or
+<mountpoint> for a local one
+(examples: "mnementh:/cdrom", "/cdrom").
+If <prefixchar> is nothing, the scripts will complain if the fs
+is mounted. If it is !, they will complain if it is not mounted.
+If ? they won't complain either way (useful for devices that are
+not always mounted, like /cdrom). '?' is an enhancement only
+present in my version of the scripts.
+
+Useful scripts: (all in /usr/local/lib/backup)
+checkallused: this only does a check of the configuration files.
+It should give a cryptic summary of the configuration and print
+'configuration ok'. If not, fix your config files :->
+
+loaded: this tells the scripts that a currently unlabelled tape
+should be treated as tape X: eg:
+loaded b
+will cause it to treat it as tape 'b'. [NB: this won't override
+the TAPEID label written on the tape; it's just for use with
+previously unused tapes.]
+
+driver : this is the script to actually run to do a backup.
+If run from the command line, give it the argument 'test'
+[otherwise it will attempt to run bringup to change runlevel,
+on the assumption that it was run from inittab (see below)].
+You'll need to edit this script to send the status report email
+to somewhere right for your system.
+
+takedown : Run this if you want to run a reduced level of system
+services during backups. Usage:
+takedown <freq>
+where <freq> can be 'now', 'soon' or nothing depending on number
+of warning messages desired. [configured via warnings.* files.]
+
+To use this you'll need to configure init:
+ * set up runlevel 5 to provide the level of services you want
+ (by tweaking the symlinks in /etc/rc5.d or equivalent)
+ * Add the following to /etc/inittab (tweak paths and VC number
+ if desired):
+
+# Runlevel 5 is set up to run a reduced level of services during
+# backups. (currently this means: no squid, no webserver, no newsserver)
+# We also run the backup script automatically on entering runlevel 5:
+# (I/O goes to VC 7, which is also the X server, but never mind -- we
+# very seldom run X anyway :->)
+dm:5:once:/usr/local/lib/backup/driver </dev/tty7 >/dev/tty7 2>&1
+
+ * takedown can be run from the command line or via cron.
+
+whatsthis: a simple script I hacked together to display the
+TAPEID of the current tape and optionally list its contents.
+Usage:
+whatsthis [--list [n]]
+
+WARNING: it's currently hardwired to assume 'cpio' type backups
+when listing; it could be trivially hardwired to assume 'zafio'
+or with slightly more effort it could be done properly :->
+
+That's all I can think of for now -- comments and
+questions to Peter Maydell <pmaydell@chiark.greenend.org.uk>
-/**/
+/*
+ * readbuffer.c
+ *
+ * Copyright (C) 1997,1998 Ian Jackson <ian@chiark.greenend.org.uk>
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2,
+ * or (at your option) any later version.
+ *
+ * This is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this file; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
#include <sys/time.h>
#include <sys/types.h>
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
-#include <sys/mman.h>
-#define BUFFER 16*1024*1024
+/* was 16MB -- PMM */
+#define BUFFER 6*1024*1024
#define WAITEMPTY ((BUFFER*1)/4)
static inline int min(int a, int b) { return a<=b ? a : b; }
if (fcntl(fd,F_SETFL,r) == -1) { perror("fcntl setfl"); exit(1); }
}
-int main(int argc, const char *const *argv) {
+int main(void) {
static unsigned char buf[BUFFER];
unsigned char *wp, *rp;
used=0; wp=rp=buf; reading=1; seeneof=0;
nonblock(0); nonblock(1);
- if (argv[1] && !strcmp(argv[1],"--mlock")) {
- if (mlock(buf,sizeof(buf))) { perror("mlock"); exit(1); }
- argv++; argc--;
- }
- if (argv[1]) { fputs("usage: readbuffer [--mlock]\n",stderr); exit(1); }
while (!seeneof || used) {
FD_ZERO(&readfds);
if (reading) {
#
chdir '/var/local/backup' or die $!;
push(@INC,'/usr/local/lib/backup');
+# huh? AFAICT this adds two entries to the PATH and
+# promptly removes them again??? -- PMM
$ENV{'PATH'}= '/usr/local/lib/backup:/usr/local/bin:'.$ENV{'PATH'};
$ENV{'PATH'} =~ s,^/usr/local/lib/backup:/usr/local/bin:,,;
-$blocksize= 1;
+# This sets both blocksizes to 10K. Note that both must be the
+# same if using the zftape floppy tape driver, since that requires
+# blocks to be the right size, but dd with the obs=10k option
+# doesn't pad the final block to the blocksize...
+$blocksize= 20; # was 1 -- PMM
$blocksizebytes= 512*$blocksize;
-$softblocksizekb= 1;
+$softblocksizekb= 10; # was 1 -- PMM
$softblocksizebytes= 1024*$softblocksizekb;
$tapename= 'st0';
$tape= "/dev/$tapename";
#!/bin/sh
#
# Take the system down for backups and then bring it back up.
+# Expects a single (possibly empty) argument X which is used to select
+# a file /etc/backup/warnings.X. This file will contain lines like:
+# T 300 "in 10 minutes"
+# T 240 "in 5 minutes"
+# T 45 "in 1 minute"
+# T 15 "in 15 seconds"
+# configuring the frequency of warning messages. If you call the
+# files 'warnings.soon', 'warnings.now' and 'warnings.' then
+# you can invoke this as:
+# takedown lots of warnings
+# takedown soon not so many warnings
+# takedown now no warning at all
set -e
cd /etc/backup
) &
sleep 1
+# We assume that runlevel 5 is set up suitably for doing backups
+# (ie non-essential services turned off in an effort to get the
+# tape to stream.)
telinit 5
--- /dev/null
+filesystems all
+next c
+end
--- /dev/null
+filesystems all
+next d
+end
--- /dev/null
+filesystems all
+next e
+end
--- /dev/null
+filesystems all
+next a
+end
--- /dev/null
+#!/usr/bin/perl
+
+# whatsthis : just read an ID off the tape and display it to the user.
+# Peter Maydell <pmaydell@chiark.greenend.org.uk>
+# First rough hack; mostly just code nabbed from full.
+# --list assumes the dump type was 'zafio', which is a bit bogus.
+
+# whatsthis : no args => just print tapeid
+# whatsthis --list [n] : print tapeid then list archive n (if n omitted,
+# 0 is assumed.) Note that archives are numbered from zero!
+
+sub rewind();
+sub stopandsay(@);
+
+$etc='/etc/backup';
+require "$etc/settings.pl";
+require 'backuplib.pl';
+
+$| = 1;
+
+# This isn't intended to be run automatically, so don't bother
+# with setting status.
+
+# If we are run with the argument --list then list the backup to
+# stdout. Otherwise just print the tape ID.
+$listing = 0; # default : don't list
+$listing = 1 if ($ARGV[0] eq '--list');
+$listarchive = 0;
+$listarchive = $ARGV[1] if defined $ARGV[1];
+
+print "Trying to read tape ID from currently inserted tape...\n";
+
+unlink 'TAPEID';
+system "mt -f $ntape setblk $blocksizebytes"; $? and die $?;
+system "dd if=$ntape bs=${blocksize}b count=10 | tar -b$blocksize -vvxf - TAPEID";
+$? and stopandsay "Failed to read TAPEID.\n";
+
+if (!open(T, "TAPEID"))
+{
+ stopandsay "Tape has no ID label.\n";
+}
+
+# OK, there's a TAPEID file, read the ID and check for sanity.
+chomp($tapeid= <T>);
+if ($tapeid =~ m/[^0-9a-zA-Z]/)
+{
+ stopandsay "Tape has a bad (non-alphanumeric) TAPEID ($&).\n";
+}
+elsif (! $tapeid =~ m/[0-9a-zA-Z]/)
+{
+ stopandsay "Empty TAPEID.\n";
+}
+
+print "TAPEID is $tapeid.\n";
+close T;
+
+# If we aren't listing the tape contents, we can just rewind the tape
+# and exit.
+if (!$listing)
+{
+ rewind();
+ exit;
+}
+
+# List the contents of archive $listarchive on the tape.
+# We are already at the right place for the first archive
+# (after the TAPEID).
+# For any other archive, we skip forwards to the start of that archive.
+if ($listarchive)
+{
+ system "mt -f $ntape fsf $listarchive";
+ $? and stopandsay "Couldn't skip forward to archive $listarchive.";
+}
+
+# Use file to figure out what the archive type is
+# This doesn't seem to work too well, so I'm disabling it -- PMM
+#$ftype = `dd if=$ntape ibs=$blocksizebytes | file -`;
+#$? and stopandsay "couldn't determine file type: $?";
+$ftype = 'POSIX tar';
+
+# What we want to do here is roughly:
+# dd if=$ntape ibs=$blocksizebytes | readbuffer | afio ???
+#
+# where the afio options are such as to list an archive created with
+# afio -b $softblocksizebytes -Zvo
+
+if ($ftype =~ /POSIX tar/) {
+ # POSIX tar archive; we read it with cpio
+ $reader = "cpio -it -C$softblocksizebytes -Hustar";
+} elsif ($ftype =~ /ASCII cpio/) {
+ $reader = "afio -b $softblocksizebytes -Zt -";
+} elsif ($ftype =~ /dump file/) {
+ stopandsay "sorry: can't list dump files yet";
+} else {
+ stopandsay "listing failed: unknown archive type";
+}
+
+# Now back up so we can read the file again properly
+#system "mt -f $ntape bsf 1"; $? and stopandsay "couldn't backspace tape: $?";
+
+system "dd if=$ntape ibs=$blocksizebytes | /usr/local/lib/backup/readbuffer | $reader";
+$? and stopandsay "listing failed: $?";
+
+# All's well, stop here.
+print "Listing complete.\n";
+rewind();
+exit;
+
+
+# Rewind the tape.
+sub rewind ()
+{
+ system "mt -f $tape rewind"; $? and die $?;
+}
+
+# Print the given message, rewind the tape and exit failure.
+sub stopandsay(@)
+{
+ print @_;
+ rewind();
+ exit(1);
+}
-/**/
+/*
+ * writebuffer.c
+ *
+ * Copyright (C) 1997,1998 Ian Jackson <ian@chiark.greenend.org.uk>
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2,
+ * or (at your option) any later version.
+ *
+ * This is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this file; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
#include <sys/time.h>
#include <sys/types.h>
#include <errno.h>
#include <unistd.h>
-#define BUFFER 16*1024*1024
+/* was 16MB -- PMM */
+#define BUFFER 6*1024*1024
#define WAITFILL ((BUFFER*3)/4)
static inline int min(int a, int b) { return a<=b ? a : b; }
~ian / chiark-utils.git / commitdiff