X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?p=chiark-utils.git;a=blobdiff_plain;f=scripts%2Fgit-cache-proxy;fp=scripts%2Fgit-cache-proxy;h=540188bc73797dc0e4308e8efa61e84e277ab155;hp=b4b23001cb3ed42686ccaf0c1392c7f4f0bc40c5;hb=2d867f1a7f27bd3ea2b59f2600f0477d51979aca;hpb=32c189107d19892ab4730757003c07576f139de5 diff --git a/scripts/git-cache-proxy b/scripts/git-cache-proxy index b4b2300..540188b 100755 --- a/scripts/git-cache-proxy +++ b/scripts/git-cache-proxy @@ -44,6 +44,14 @@ use File::Path qw(remove_tree); our $us = 'git-cache-proxy'; +our $debug = 0; +our $housekeepingeverydays = 1; +our $treeexpiredays = 21; +our $fetchtimeout = 1800; +our $maxfetchtimeout = 3600; +our $cachedir = '/var/cache/git-cache-proxy'; +our $housekeepingonly = 0; + #---------- error handling and logging ---------- # This is a bit fiddly, because we want to catch errors sent to stderr @@ -71,9 +79,10 @@ sub ensurelog () { sub logm ($$) { my ($pri, $msg) = @_; + return if $pri eq 'debug' && !$debug; if ($client eq '(local)') { print STDERR "$us: $pri: $msg\n" or die $!; - exit 1; + return; } ensurelog(); my $mainmsg = sprintf "%s-%s: %s", $server, $client, $msg; @@ -81,11 +90,11 @@ sub logm ($$) { my $wholemsg = sprintf("%s [%d] %s: %s\n", strftime("%Y-%m-%d %H:%M:%S Z", gmtime), $$, - $pri, + $pri eq 'err' ? 'error' : $pri, $mainmsg); print $log $wholemsg; } else { - syslog $pri, $mainmsg; + syslog $pri, "%s", "$pri $mainmsg"; } } @@ -127,26 +136,31 @@ sub gitfail ($) { #---------- argument parsing ---------- -our $housekeepingthreshdays = 1; -our $treeexpiredays = 21; -our $fetchtimeout = 1800; -our $maxfetchtimeout = 3600; -our $cachedir = '/var/cache/git-cache-proxy'; - for (;;) { last unless @ARGV; last unless $ARGV[0] =~ m/^-/; $_ = shift @ARGV; for (;;) { last unless m/^-./; - if (s/^-L(.*)$//) { + if (s/^-H/-/) { + $housekeepingonly++; + } elsif (s/^-D/-/) { + $debug++; + } elsif (s/^-L(.*)$//) { my $logfile = $_; open STDERR, ">>", $logfile or fail "open $logfile: $!"; $log = \*STDERR; } elsif (s/^-d(.*)$//) { $cachedir = $1; - } elsif (s/^--(maxfetchtimeout|fetchtimeout)=(\d+)$//) { - ${ $::{$1} } = $2; + } elsif (s/^--( max-fetch-timeout + | fetch-timeout + | tree-expire-days + | housekeeping-interval-days + )=(\d+)$//x) { + my $vn = $1; + $vn =~ y/-//d; + die $vn unless defined ${ $::{$vn} }; + ${ $::{$vn} } = $2; } else { fail "bad usage: unknown option `$_'"; } @@ -157,18 +171,18 @@ for (;;) { #---------- utility functions ---------- -sub lockfile ($$) { +sub lockfile ($$$) { my ($fh, $fn, $flockmode) = @_; + my $what = $fn.(($flockmode & ~LOCK_NB) == LOCK_SH ? " (shared)" : ""); for (;;) { close $fh; open $fh, '+>', $fn or fail "open/create $fn for lock: $!"; - if (!flock $fh, $lockmode) { - if ($lockmode & LOCK_NB and $! == EWOULDBLOCK) { + logm 'debug', "lock $what: acquiring"; + if (!flock $fh, $flockmode) { + if ($flockmode & LOCK_NB && $! == EWOULDBLOCK) { return 0; # ok then } - fail "lock $fn". - (($flockmode & ~LOCK_NB) == LOCK_SH ? " (shared)" : ""). - ": $!"; + fail "lock $what: $!"; } stat $fh or fail "stat opened $fn: $!"; my $fh_ino = ((stat _)[1]); @@ -177,7 +191,11 @@ sub lockfile ($$) { next; } my $fn_ino = ((stat _)[1]); - return 1 if $fn_ino == $fh_ino; + if ($fn_ino == $fh_ino) { + logm 'debug', "lock $what: acquired"; + return 1; + } + logm 'debug', "lock $what: deleted, need to loop again"; # oh dear } } @@ -238,7 +256,7 @@ sub readcommand () { $fetch = 0; } elsif (m/^fetch=must$/) { $fetch = 2; # the default - } elsif (m/^timeout=(\d+)$/) { + } elsif (m/^timeout=(\d+)$/ && $1 >= 1) { $fetchtimeout = $1 <= $maxfetchtimeout ? $1 : $maxfetchtimeout; } elsif ($must) { gitfail "unknown/unsupported option `$_'"; @@ -351,33 +369,50 @@ sub clonefetch () { return 0; } +sub hkfail ($) { my ($msg) = @_; fail "housekeeping: $msg"; } + sub housekeeping () { + logm 'info', "housekeeping started"; foreach $lock (<[a-z]*\\.lock>) { + my $subdir = $lock; $subdir =~ s/\\.lock$//; if (!lstat $lock) { - $! == ENOENT or fail "housekeeping: $lock: lstat: $!"; + $! == ENOENT or hkfail "$lock: lstat: $!"; next; } if (-M _ <= $treeexpiredays) { - logm 'debug', "housekeeping: $lock: not too old"; + logm 'debug', "housekeeping: subdirs $subdir: touched recently"; next; } - my $subdir = $lock; $subdir =~ s/\\.lock$//; + if (!lockfile \*LOCK, $lock, LOCK_EX|LOCK_NB) { + logm 'info', "housekeeping: subdirs $subdir: lock busy, skipping"; + next; + } + logm 'info', "housekeeping: subdirs $subdir: cleaning"; my $ok = 1; foreach my $suffix (qw(tmp git)) { my $dir = "${subdir}\\.$suffix"; my $errs; remove_tree($dir, { safe=>1, error=>\$errs }); - $ok = 0 if @$errs; - foreach my $err (@$errs) { - logm 'warning', "problem deleting: $err[0]: $err[1]"; + if (stat $dir) { + $ok = 0; + logm 'warning', "housekeeping: $dir: problems with". + "deletion prevent cleanup:"; + foreach my $err (@$errs) { + logm 'info', "problem deleting: $err->[0]: $err->[1]"; + } } } if ($ok) { - + unlink $lock or hkfail "remove $lock: $!"; + } + } + open HS, ">", "Housekeeping.stamp" or hkfail "touch Housekeeping.stamp: $!"; + close HS or hkfail "close Housekeeping.stamp: $!"; + logm 'info', "housekeeping finished"; +} sub housekeepingcheck ($$) { my ($dofork, $force) = @_; - or fail "open/create Housekeeping.lock: $!"; if (!$force) { if (!lockfile \*HLOCK, "Housekeeping.lock", LOCK_EX|LOCK_NB) { logm 'debug', "housekeeping lock taken, not running"; @@ -389,28 +424,28 @@ sub housekeepingcheck ($$) { logm 'info', "housekeeping forced"; } elsif (!lstat "Housekeeping.stamp") { $! == ENOENT or fail "lstat Housekeeping.stamp: $!"; - logm 'info', "housekeeping stamp missing, will run"; - } elsif (-M _ <= $housekeepingthreshdays) { + logm 'info', "housekeeping not done yet, will run"; + } elsif (-M _ <= $housekeepingeverydays) { logm 'debug', "housekeeping done recently"; close HLOCK; return 0; } if ($dofork) { my $child = fork; - defined $child or fail "fork for housekeeping: $!"; + defined $child or hkfail "fork: $!"; if (!$child) { housekeeping(); exit 0; } - return 1; } else { housekeeping(); - return 1; } + close HLOCK; + return 1; } sub runcommand () { - servinfo "servicing"; + servinfo "serving"; exec qw(git-upload-pack --strict --timeout=1000 .) or fail "exec git-upload-pack: $!"; } @@ -418,7 +453,12 @@ sub runcommand () { sub daemonservice () { readcommand(); while (!clonefetch()) { } + housekeepingcheck(1,0); runcommand(); } -daemonservice(); +if ($housekeepingonly) { + housekeepingcheck(0, $housekeepingonly>=2); +} else { + daemonservice(); +}