X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?a=blobdiff_plain;f=scripts%2Fgit-cache-proxy;h=52e01d0203b0e210200119cb42b57eea76ff7dbb;hb=refs%2Fheads%2Fmaster;hp=ffdf6eab52fe57bad3b276ad71a21002abc5011e;hpb=2b42677fc027af957b8622b38da698c4f50f41ae;p=chiark-utils.git diff --git a/scripts/git-cache-proxy b/scripts/git-cache-proxy index ffdf6ea..2d80cb1 100755 --- a/scripts/git-cache-proxy +++ b/scripts/git-cache-proxy @@ -19,6 +19,9 @@ # fetch=no just use what is in the cache # fetch=try use what is in the cache if the fetch/clone fails # timeout= length of time to allow for fetch/clone +# housekeeping-interval-days= } housekeeping tuning parameters +# tree-expire-days= } +# gc-interval-days= } # example inetd.conf line: # 9419 stream tcp nowait git-cache /usr/bin/git-cache-proxy git-cache-proxy @@ -64,10 +67,11 @@ our $us = 'git-cache-proxy'; our $debug = 0; our $housekeepingeverydays = 1; +our $gcintervaldays = 10; our $treeexpiredays = 21; -our $fetchtimeout = 1800; -our $maxfetchtimeout = 3600; -our $servetimeout = 1000; +our $fetchtimeout = 3600; +our $maxfetchtimeout = 7200; +our $servetimeout = 3600; our $cachedir = '/var/cache/git-cache-proxy'; our $housekeepingonly = 0; @@ -141,6 +145,8 @@ sub fail ($) { exit 0; } +$SIG{ALRM} = sub { fail "timeout" }; + sub gitfail ($) { my ($msg) = @_; close LOCK; @@ -176,6 +182,7 @@ for (;;) { | serve-timeout | tree-expire-days | housekeeping-interval-days + | gc-interval-days )=(\d+)$//x) { my $vn = $1; $vn =~ y/-//d; @@ -191,12 +198,13 @@ for (;;) { #---------- utility functions ---------- -sub lockfile ($$$) { - my ($fh, $fn, $flockmode) = @_; +sub lockfile ($$$$) { + my ($fh, $fn, $flockmode, $update_ts) = @_; my $what = $fn.(($flockmode & ~LOCK_NB) == LOCK_SH ? " (shared)" : ""); for (;;) { close $fh; - open $fh, '+>', $fn or fail "open/create $fn for lock: $!"; + open $fh, ($update_ts ? '+>' : '+>>'), $fn + or fail "open/create $fn for lock: $!"; logm 'debug', "lock $what: acquiring"; if (!flock $fh, $flockmode) { if ($flockmode & LOCK_NB && $! == EWOULDBLOCK) { @@ -246,14 +254,13 @@ sub servinfo ($) { } sub readcommand () { - $SIG{ALRM} = sub { fail "timeout" }; alarm 30; my $hex_len = xread 4; fail "Bad hex in packet length" unless $hex_len =~ m|^[0-9a-fA-F]{4}$|; my $line = xread -4 + hex $hex_len; unless (($service,$specpath,$spechost) = $line =~ - m|^(git-[a-z-]+) /*([!-~ ]+)\0host=([!-~]+)\0$|) { + m|^(git-[a-z-]+) /*([!-~ ]+)\0host=([!-~]+)\0|) { $line =~ s|[^ -~]+| |g; gitfail "unknown/unsupported instruction `$line'" } @@ -298,8 +305,18 @@ sub readcommand () { servinfo "locking"; } +sub update_gcstamp ($) { + my ($gitdir) = (@_); + my $gcdone = "$gitdir/cache-proxy-gc.stamp"; + if (open GCSTAMP, '>', $gcdone) { + close GCSTAMP; + } else { + $!==ENOENT or fail "create $gcdone: $!"; + } +} + sub clonefetch () { - lockfile \*LOCK, $lock, LOCK_EX; + lockfile \*LOCK, $lock, LOCK_EX, 1; my $exists = lstat $gitd; $exists or $!==ENOENT or fail "lstat $gitd: $!"; @@ -308,6 +325,19 @@ sub clonefetch () { if ($fetch) { + my $rbits = ''; + vec($rbits,0,1) = 1; + my $ebits = $rbits; + my $r=select $rbits,undef,$ebits,0; + $r>=0 or fail "select recheck STDOUT failed: $!"; + if ($r) { + servinfo 'client disconnected (stdin unexpectedly'. + (vec($rbits,0,1) ? ' readable' : ''). + (vec($ebits,0,1) ? ' exception' : ''). + ')'; + exit 0; + } + our @cmd; if (!$exists) { @@ -366,6 +396,7 @@ sub clonefetch () { alarm 0; if (!$exists) { + update_gcstamp($tmpd); rename $tmpd, $gitd or fail "rename fresh $tmpd to $gitd: $!"; $exists = 1; } @@ -378,7 +409,7 @@ sub clonefetch () { } servinfo "sharing"; - lockfile \*LOCK, $lock, LOCK_SH; # NB releases and relocks + lockfile \*LOCK, $lock, LOCK_SH, 1; # NB releases and relocks if (stat $gitd) { return 1; @@ -396,43 +427,94 @@ sub housekeeping () { logm 'info', "housekeeping started"; foreach $lock (<[a-z]*\\.lock>) { my $subdir = $lock; $subdir =~ s/\\.lock$//; + my $gcdone = "$subdir\\.git/cache-proxy-gc.stamp"; if (!lstat $lock) { $! == ENOENT or hkfail "$lock: lstat: $!"; next; } + my ($mode_what,$mode_locknb,$mode_action); if (-M _ <= $treeexpiredays) { - logm 'debug', "housekeeping: subdirs $subdir: touched recently"; - next; - } - if (!lockfile \*LOCK, $lock, LOCK_EX|LOCK_NB) { - logm 'info', "housekeeping: subdirs $subdir: lock busy, skipping"; - next; - } - logm 'info', "housekeeping: subdirs $subdir: cleaning"; - eval { - foreach my $suffix (qw(tmp git)) { - my $dir = "${subdir}\\.$suffix"; - my $tdir = "${subdir}\\.tmp"; - if ($dir ne $tdir) { - if (!rename $dir,$tdir) { - next if $! == ENOENT; - die "$dir: cannot rename to $tdir: $!\n"; + my $gccheck = sub { + if (!lstat "$gcdone") { + $! == ENOENT or hkfail "$gcdone: lstat: $!"; + return 1, "touched recently, never gc'd!"; + } elsif (-M _ <= $gcintervaldays) { + return 0, "touched recently, gc'd recently"; + } else { + return 1, "touched recently, needs gc"; + } + }; + my ($needsgc, $gcmsg) = $gccheck->(); + logm 'debug', "housekeeping: subdirs $subdir: $gcmsg"; + next unless $needsgc; + $mode_what = 'garbage collecting'; + $mode_locknb = 0; + $mode_action = sub { + my ($needsgc, $gcmsg) = $gccheck->(); + if (!$needsgc) { + logm 'info', + "housekeeping: subdirs $subdir: someone else has gc'd"; + return; + } + logm 'debug', "housekeeping: subdirs $subdir: $gcmsg (2)"; + my $gclog = "$subdir/gc.log"; + unlink $gclog or $!==ENOENT or hkfail "remove $gclog: $!"; + my $child = fork // hkfail "fork (for $subdir): $!"; + if (!$child) { + if (!chdir "$subdir\\.git") { + exit 0 if $!==ENOENT; + die "for gc: chdir $subdir: $!\n"; } + exec qw(git gc --quiet); + die "exec git gc (for $subdir): $!\n"; } - system qw(rm -rf --), $tdir; - if (stat $tdir) { - die "$dir: problem deleting file(s), rm exited $?\n"; - } elsif ($! != ENOENT) { - die "$tdir: cannot stat after deletion: $!\n"; + waitpid($child, 0) == $child or hkfail "waitpid failed! $!"; + if ($?) { + logm 'err', + "housekeeping: subdirs $subdir: gc failed (wait status $?)"; + } else { + update_gcstamp("$subdir\\.git"); + logm 'debug', + "housekeeping: subdirs $subdir: gc done"; } - } - }; - if (length $@) { - chomp $@; - logm 'warning', "housekeeping: $subdir: cleanup prevented: $@"; + }; } else { - unlink $lock or hkfail "remove $lock: $!"; + $mode_what = 'cleaning'; + $mode_locknb = LOCK_NB; + $mode_action = sub { + eval { + foreach my $suffix (qw(tmp git)) { + my $dir = "${subdir}\\.$suffix"; + my $tdir = "${subdir}\\.tmp"; + if ($dir ne $tdir) { + if (!rename $dir,$tdir) { + next if $! == ENOENT; + die "$dir: cannot rename to $tdir: $!\n"; + } + } + system qw(rm -rf --), $tdir; + if (stat $tdir) { + die "$dir: problem deleting file(s), rm exited $?\n"; + } elsif ($! != ENOENT) { + die "$tdir: cannot stat after deletion: $!\n"; + } + } + }; + if (length $@) { + chomp $@; + logm 'warning', "housekeeping: $subdir: cleanup prevented: $@"; + } else { + unlink $lock or hkfail "remove $lock: $!"; + } + }; + } + if (!lockfile \*LOCK, $lock, LOCK_EX|$mode_locknb, 0) { + die $! unless $mode_locknb; + logm 'info', "housekeeping: subdirs $subdir: lock busy, skipping"; + next; } + logm 'info', "housekeeping: subdirs $subdir: $mode_what"; + $mode_action->(); } open HS, ">", "Housekeeping.stamp" or hkfail "touch Housekeeping.stamp: $!"; close HS or hkfail "close Housekeeping.stamp: $!"; @@ -442,7 +524,7 @@ sub housekeeping () { sub housekeepingcheck ($$) { my ($dofork, $force) = @_; if (!$force) { - if (!lockfile \*HLOCK, "Housekeeping.lock", LOCK_EX|LOCK_NB) { + if (!lockfile \*HLOCK, "Housekeeping.lock", LOCK_EX|LOCK_NB, 1) { logm 'debug', "housekeeping lock taken, not running"; close HLOCK; return 0;