X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=dgit.git;a=blobdiff_plain;f=infra%2Fdgit-repos-server;h=f6ac5072d8849a1e3de83b60bffa67088b610ae6;hp=d2f94f1957e0bed922a0db052e2a81b921fa789a;hb=ab3be575e6a64ca698684995a02563b3cb82f68a;hpb=8afbacc6da4ad5ccaf931d113df175bef22cc7b1 diff --git a/infra/dgit-repos-server b/infra/dgit-repos-server index d2f94f19..f6ac5072 100755 --- a/infra/dgit-repos-server +++ b/infra/dgit-repos-server @@ -2,8 +2,14 @@ # dgit-repos-server # # usages: -# .../dgit-repos-server DISTRO SUITES KEYRING-AUTH-SPEC \ -# DGIT-REPOS-DIR POLICY-HOOK-SCRIPT --ssh +# dgit-repos-server DISTRO DISTRO-DIR AUTH-SPEC [] --ssh +# dgit-repos-server DISTRO DISTRO-DIR AUTH-SPEC [] --cron +# settings +# --repos=GIT-REPOS-DIR default DISTRO-DIR/repos/ +# --suites=SUITES-FILE default DISTRO-DIR/suites +# --policy-hook=POLICY-HOOK default DISTRO-DIR/policy-hook +# --dgit-live=DGIT-LIVE-DIR default DISTRO-DIR/dgit-live +# (DISTRO-DIR is not used other than as default) # internal usage: # .../dgit-repos-server --pre-receive-hook PACKAGE # @@ -14,15 +20,41 @@ # SUITES is the name of a file which lists the permissible suites # one per line (#-comments and blank lines ignored) # -# KEYRING-AUTH-SPEC is a :-separated list of +# AUTH-SPEC is a :-separated list of # KEYRING.GPG,AUTH-SPEC # where AUTH-SPEC is one of # a # mDM.TXT +# (With --cron AUTH-SPEC is not used and may be the empty string.) use strict; +$SIG{__WARN__} = sub { die $_[0]; }; -# What we do is this: +# DGIT-REPOS-DIR contains: +# git tree (or other object) lock (in acquisition order, outer first) +# +# _tmp/PACKAGE_prospective ! } SAME.lock, held during receive-pack +# +# _tmp/PACKAGE_incoming$$ ! } SAME.lock, held during receive-pack +# _tmp/PACKAGE_incoming$$_fresh ! } +# +# PACKAGE.git } PACKAGE.git.lock +# PACKAGE_garbage } (also covers executions of +# PACKAGE_garbage-old } policy hook script for PACKAGE) +# PACKAGE_garbage-tmp } +# policy* } (for policy hook script, covered by +# } lock only when invoked for a package) +# +# leaf locks, held during brief operaton only: +# +# _empty } SAME.lock +# _empty.new } +# +# _template } SAME.lock +# +# locks marked ! may be held during client data transfer + +# What we do on push is this: # - extract the destination repo name # - make a hardlink clone of the destination repo # - provide the destination with a stunt pre-receive hook @@ -44,13 +76,16 @@ use strict; # + verify the signature on the signed tag # and if necessary check that the keyid and package are listed in dm.txt # + check various correspondences: -# * the suite is one of those permitted # * the signed tag must refer to a commit # * the signed tag commit must be the refs/dgit value # * the name in the signed tag must correspond to its ref name # * the tag name must be debian/ (massaged as needed) +# * the suite is one of those permitted # * the signed tag has a suitable name -# * the commit is a fast forward +# * run the "push" policy hook +# * replay prevention for --deliberately-not-fast-forward +# * check the commit is a fast forward +# * handle a request from the policy hook for a fresh repo # + push the signed tag and new dgit branch to the actual repo # # If the destination repo does not already exist, we need to make @@ -63,8 +98,6 @@ use strict; # of _template # - we use the prospective new destination repo instead of the # actual new destination repo (since the latter doesn't exist) -# - we set up a post-receive hook as well, which -# + touches a stamp file # - after git-receive-pack exits, we # + check that the prospective repo contains a tag and head # + rename the prospective destination repo into place @@ -82,10 +115,58 @@ use strict; # the corresponding temporary tree, as the lockfile is also # a stampfile whose presence indicates that there may be # cleanup to do +# +# Policy hook script is invoked like this: +# POLICY-HOOK-SCRIPT DISTRO DGIT-REPOS-DIR DGIT-LIVE-DIR ACTION... +# ie. +# POLICY-HOOK-SCRIPT ... check-list [...] +# POLICY-HOOK-SCRIPT ... check-package PACKAGE [...] +# POLICY-HOOK-SCRIPT ... push PACKAGE \ +# VERSION SUITE TAGNAME DELIBERATELIES [...] +# POLICY-HOOK-SCRIPT ... push-confirm PACKAGE \ +# VERSION SUITE TAGNAME DELIBERATELIES FRESH-REPO|'' [...] +# +# Exit status is a bitmask. Bit weight constants are defined in Dgit.pm. +# NOFFCHECK (2) +# suppress dgit-repos-server's fast-forward check ("push" only) +# FRESHREPO (4) +# blow away repo right away (ie, as if before push or fetch) +# ("check-package" and "push" only) +# any unexpected bits mean failure, and then known set bits are ignored +# if no unexpected bits set, operation continues (subject to meaning +# of any expected bits set). So, eg, exit 0 means "continue normally" +# and would be appropriate for an unknown action. +# +# cwd for push and push-confirm is a temporary repo where the +# to-be-pushed objects have been received; TAGNAME is the +# version-based tag +# +# FRESH-REPO is '' iff the repo for this package already existed, or +# the pathname of the newly-created repo which will be renamed into +# place if everything goes well. (NB that this is generally not the +# same repo as the cwd, because the objects are first received into a +# temporary repo so they can be examined.) +# +# if push requested FRESHREPO, push-confirm happens in said fresh repo +# and FRESH-REPO is guaranteed not to be ''. +# +# policy hook for a particular package will be invoked only once at +# a time - (see comments about DGIT-REPOS-DIR, above) +# +# check-list and check-package are invoked via the --cron option. +# First, without any locking, check-list is called. It should produce +# a list of package names (one per line). Then check-package will be +# invoked for each named package, in each case after taking an +# appropriate lock. +# +# If policy hook wants to run dgit (or something else in the dgit +# package), it should use DGIT-LIVE-DIR/dgit (etc.) + use POSIX; use Fcntl qw(:flock); use File::Path qw(rmtree); +use File::Temp qw(tempfile); use Debian::Dgit qw(:DEFAULT :policyflags); @@ -94,15 +175,17 @@ open DEBUG, ">/dev/null" or die $!; our $func; our $dgitrepos; our $package; +our $distro; our $suitesfile; our $policyhook; -our $realdestrepo; +our $dgitlive; our $destrepo; our $workrepo; our $keyrings; our @lockfhs; our $debug=''; our @deliberatelies; +our %supersedes; our $policy; #----- utilities ----- @@ -111,6 +194,8 @@ sub debug { print DEBUG "$debug @_\n"; } +sub realdestrepo () { "$dgitrepos/$package.git"; } + sub acquirelock ($$) { my ($lock, $must) = @_; my $fh; @@ -133,7 +218,7 @@ sub acquirelock ($$) { return $fh; } -sub acquiretree ($$) { +sub acquirermtree ($$) { my ($tree, $must) = @_; my $fh = acquirelock("$tree.lock", $must); if ($fh) { @@ -143,6 +228,15 @@ sub acquiretree ($$) { return $fh; } +sub locksometree ($) { + my ($tree) = @_; + acquirelock("$tree.lock", 1); +} + +sub lockrealtree () { + locksometree(realdestrepo); +} + sub mkrepotmp () { my $tmpdir = "$dgitrepos/_tmp"; return if mkdir $tmpdir; @@ -187,13 +281,14 @@ sub runcmd { sub policyhook { my ($policyallowbits, @polargs) = @_; - # => ($exitstatuspolicybitmap, $policylockfh); + # => ($exitstatuspolicybitmap); die if $policyallowbits & ~0x3e; - my @cmd = ($policyhook,$distro,$repos,@polargs); - debugcmd @_; - my $r = system @_; + my @cmd = ($policyhook,$distro,$dgitrepos,$dgitlive,@polargs); + debugcmd @cmd; + my $r = system @cmd; die "system: $!" if $r < 0; die "hook (@cmd) failed ($?)" if $r & ~($policyallowbits << 8); + debug sprintf "hook (%s) => %#x", "@polargs", $r; return $r >> 8; } @@ -205,29 +300,41 @@ sub mkemptyrepo ($$) { sub mkrepo_fromtemplate ($) { my ($dir) = @_; my $template = "$dgitrepos/_template"; - debug "copy tempalate $template -> $dir"; + locksometree($template); + debug "copy template $template -> $dir"; my $r = system qw(cp -a --), $template, $dir; !$r or die "create new repo $dir failed: $r $!"; } sub movetogarbage () { - my $garbagerepo = "$dgitrepos/_tmp/${package}_garbage"; - acquiretree($garbagerepo,1); - rmtree $garbagerepo; - rename $realdestrepo, $garbagerepo + # realdestrepo must have been locked + my $garbagerepo = "$dgitrepos/${package}_garbage"; + # We arrange to always keep at least one old tree, for anti-rewind + # purposes (and, I guess, recovery from mistakes). This is either + # $garbage or $garbage-old. + if (stat_exists "$garbagerepo") { + rmtree "$garbagerepo-tmp"; + if (rename "$garbagerepo-old", "$garbagerepo-tmp") { + rmtree "$garbagerepo-tmp"; + } else { + die "$garbagerepo $!" unless $!==ENOENT; + } + rename "$garbagerepo", "$garbagerepo-old" or die "$garbagerepo $!"; + } + rename realdestrepo, $garbagerepo or $! == ENOENT - or die "rename repo $realdestrepo to $garbagerepo: $!"; + or die "$garbagerepo $!"; } -sub onwardpush () { - my @cmd = (qw(git send-pack), $destrepo); - push @cmd, qw(--force) if $policy & NOFFCHECK; - push @cmd, "$commit:refs/dgit/$suite", - "$tagval:refs/tags/$tagname"); - debugcmd @cmd; - $!=0; - my $r = system @cmd; - !$r or die "onward push to $destrepo failed: $r $!"; +sub policy_checkpackage () { + my $lfh = lockrealtree(); + + $policy = policyhook(FRESHREPO,'check-package',$package); + if ($policy & FRESHREPO) { + movetogarbage(); + } + + close $lfh; } #----- git-receive-pack ----- @@ -235,15 +342,17 @@ sub onwardpush () { sub fixmissing__git_receive_pack () { mkrepotmp(); $destrepo = "$dgitrepos/_tmp/${package}_prospective"; - acquiretree($destrepo, 1); + acquirermtree($destrepo, 1); mkrepo_fromtemplate($destrepo); } sub makeworkingclone () { mkrepotmp(); $workrepo = "$dgitrepos/_tmp/${package}_incoming$$"; - acquiretree($workrepo, 1); + acquirermtree($workrepo, 1); + my $lfh = lockrealtree(); runcmd qw(git clone -l -q --mirror), $destrepo, $workrepo; + close $lfh; rmtree "${workrepo}_fresh"; } @@ -269,7 +378,7 @@ sub dealwithfreshrepo () { } sub maybeinstallprospective () { - return if $destrepo eq $realdestrepo; + return if $destrepo eq realdestrepo; if (open REJ, "<", "$workrepo/drs-error") { local $/ = undef; @@ -307,10 +416,14 @@ sub maybeinstallprospective () { die Dumper(\%got)." -- missing refs in new repo" if grep { !$_ } values %got; - movetogarbage; # in case of FRESHREPO + lockrealtree(); - debug "install $destrepo => $realdestrepo"; - rename $destrepo, $realdestrepo or die $!; + if ($destrepo eq "${workrepo}_fresh") { + movetogarbage; + } + + debug "install $destrepo => ".realdestrepo; + rename $destrepo, realdestrepo or die $!; remove "$destrepo.lock" or die $!; } @@ -381,25 +494,31 @@ sub parsetag () { $version = $2; die "$3 != $suite " unless $3 eq $suite; + my $copyl = $_; for (;;) { - print PT or die $!; + print PT $copyl or die $!; $!=0; $_=; defined or die "missing signature? $!"; + $copyl = $_; if (m/^\[dgit ([^"].*)\]$/) { # [dgit "something"] is for future $_ = $1." "; - for (;;) { + while (length) { if (s/^distro\=(\S+) //) { die "$1 != $distro" unless $1 eq $distro; } elsif (s/^(--deliberately-$package_re) //) { push @deliberatelies, $1; + } elsif (s/^supersede:(\S+)=(\w+) //) { + die "supersede $1 twice" if defined $supersedes{$1}; + $supersedes{$1} = $2; } elsif (s/^[-+.=0-9a-z]\S* //) { } else { - die "unknown dgit info in tag"; + die "unknown dgit info in tag ($_)"; } } next; } last if m/^-----BEGIN PGP/; } + $_ = $copyl; for (;;) { print DS or die $!; $!=0; $_=; @@ -520,6 +639,74 @@ sub checksuite () { reject "unknown suite"; } +sub checktagnoreplay () { + # We check that the signed tag mentions the name and value of + # (a) in the case of FRESHREPO all tags in the repo; + # (b) in the case of just NOFFCHECK all tags referring to + # the current head for the suite (there must be at least one). + # This prevents a replay attack using an earlier signed tag. + return unless $policy & (FRESHREPO|NOFFCHECK); + + my $garbagerepo = "$dgitrepos/${package}_garbage"; + lockrealtree(); + + local $ENV{GIT_DIR}; + foreach my $garb ("$garbagerepo", "$garbagerepo-old") { + if (stat_exists $garb) { + $ENV{GIT_DIR} = $garb; + last; + } + } + if (!defined $ENV{GIT_DIR}) { + # Nothing to overwrite so the FRESHREPO and NOFFCHECK were + # pointless. Oh well. + debug "checktagnoreplay - no garbage, ok"; + return; + } + + my $onlyreferring; + if (!($policy & FRESHREPO)) { + my $branch = server_branch($suite); + $!=0; $?=0; $_ = + `git for-each-ref --format='%(objectname)' '[r]efs/$branch'`; + defined or die "$branch $? $!"; + $? and die "$branch $?"; + if (!length) { + # No such branch - NOFFCHECK was unnecessary. Oh well. + debug "checktagnoreplay - not FRESHREPO, new branch, ok"; + return; + } + m/^(\w+)\n$/ or die "$branch $_ ?"; + $onlyreferring = $1; + debug "checktagnoreplay - not FRESHREPO,". + " checking for overwriting refs/$branch=$onlyreferring"; + } + + my @problems; + + git_for_each_tag_referring($onlyreferring, sub { + my ($objid,$fullrefname,$tagname) = @_; + debug "checktagnoreplay - overwriting $fullrefname=$objid"; + my $supers = $supersedes{$fullrefname}; + if (!defined $supers) { + push @problems, "does not supersede $fullrefname"; + } elsif ($supers ne $objid) { + push @problems, + "supersedes $fullrefname=$supers but previously $fullrefname=$objid"; + } else { + # ok; + } + }); + + if (@problems) { + reject "replay attack prevention check failed:". + " signed tag for $version: ". + join("; ", @problems). + "\n"; + } + debug "checktagnoreply - all ok" +} + sub tagh1 ($) { my ($tag) = @_; my $vals = $tagh{$tag}; @@ -541,10 +728,13 @@ sub checks () { debug "translated version $v"; $tagname eq "debian/$v" or die; - $policy = policyhook(NOFFCHECK|FRESHREPO, 'push',$package, - $version,$suite,$tagname, - join(",",@delberatelies)); + lockrealtree(); + my @policy_args = ($package,$version,$suite,$tagname, + join(",",@deliberatelies)); + $policy = policyhook(NOFFCHECK|FRESHREPO, 'push', @policy_args); + + checktagnoreplay(); checksuite(); # check that our ref is being fast-forwarded @@ -571,6 +761,20 @@ sub checks () { $destrepo = "${workrepo}_fresh"; # workrepo lock covers mkrepo_fromtemplate $destrepo; } + + my $willinstall = ($destrepo eq realdestrepo ? '' : $destrepo); + policyhook(0, 'push-confirm', @policy_args, $willinstall); +} + +sub onwardpush () { + my @cmd = (qw(git send-pack), $destrepo); + push @cmd, qw(--force) if $policy & NOFFCHECK; + push @cmd, "$commit:refs/dgit/$suite", + "$tagval:refs/tags/$tagname"; + debugcmd @cmd; + $!=0; + my $r = system @cmd; + !$r or die "onward push to $destrepo failed: $r $!"; } sub stunthook () { @@ -589,7 +793,7 @@ sub stunthook () { sub fixmissing__git_upload_pack () { $destrepo = "$dgitrepos/_empty"; - my $lfh = acquiretree($destrepo,1); + my $lfh = locksometree($destrepo); return if stat_exists $destrepo; rmtree "$destrepo.new"; mkemptyrepo "$destrepo.new", "0644"; @@ -599,7 +803,10 @@ sub fixmissing__git_upload_pack () { } sub main__git_upload_pack () { - runcmd qw(git upload-pack), $destrepo; + my $lfh = locksometree($destrepo); + chdir $destrepo or die "$destrepo: $!"; + close $lfh; + runcmd qw(git upload-pack), "."; } #----- arg parsing and main program ----- @@ -611,6 +818,76 @@ sub argval () { return $v; } +our %indistrodir = ( + # keys are used for DGIT_DRS_XXX too + 'repos' => \$dgitrepos, + 'suites' => \$suitesfile, + 'policy-hook' => \$policyhook, + 'dgit-live' => \$dgitlive, + ); + +our @hookenvs = qw(distro suitesfile policyhook dgitlive keyrings dgitrepos); + +# workrepo and destrepo handled ad-hoc + +sub mode_ssh () { + die if @ARGV; + + my $cmd = $ENV{'SSH_ORIGINAL_COMMAND'}; + $cmd =~ m{ + ^ + (?: \S* / )? + ( [-0-9a-z]+ ) + \s+ + '? (?: \S* / )? + ($package_re) \.git + '?$ + }ox + or reject "command string not understood"; + my $method = $1; + $package = $2; + + my $funcn = $method; + $funcn =~ y/-/_/; + my $mainfunc = $main::{"main__$funcn"}; + + reject "unknown method" unless $mainfunc; + + policy_checkpackage(); + + if (stat_exists realdestrepo) { + $destrepo = realdestrepo; + } else { + debug " fixmissing $funcn"; + my $fixfunc = $main::{"fixmissing__$funcn"}; + &$fixfunc; + } + + debug " running main $funcn"; + &$mainfunc; +} + +sub mode_cron () { + die if @ARGV; + + my $listfh = tempfile(); + open STDOUT, ">&", $listfh or die $!; + policyhook(0,'check-list'); + open STDOUT, ">&STDERR" or die $!; + + seek $listfh, 0, 0 or die $!; + while (<$listfh>) { + chomp or die; + next if m/^\s*\#/; + next unless m/\S/; + die unless m/^($package_re)$/; + + $package = $1; + policy_checkpackage(); + } + die $! if $listfh->error; +} + sub parseargsdispatch () { die unless @ARGV; @@ -627,12 +904,9 @@ sub parseargsdispatch () { shift @ARGV; @ARGV == 1 or die; $package = shift @ARGV; - defined($distro = $ENV{'DGIT_DRS_DISTRO'}) or die; - defined($suitesfile = $ENV{'DGIT_DRS_SUITES'}) or die; + ${ $main::{$_} } = $ENV{"DGIT_DRS_\U$_"} foreach @hookenvs; defined($workrepo = $ENV{'DGIT_DRS_WORK'}) or die; defined($destrepo = $ENV{'DGIT_DRS_DEST'}) or die; - defined($keyrings = $ENV{'DGIT_DRS_KEYRINGS'}) or die $!; - defined($policyhook = $ENV{'DGIT_DRS_POLICYHOOK'}) or die $!; open STDOUT, ">&STDERR" or die $!; eval { stunthook(); @@ -644,51 +918,28 @@ sub parseargsdispatch () { exit 0; } - $ENV{'DGIT_DRS_DISTRO'} = argval(); - $ENV{'DGIT_DRS_SUITES'} = argval(); - $ENV{'DGIT_DRS_KEYRINGS'} = argval(); - $dgitrepos = argval(); - $ENV{'DGIT_DRS_POLICYHOOK'} = $policyhook = argval(); + $distro = $ENV{'DGIT_DRS_DISTRO'} = argval(); + my $distrodir = argval(); + $keyrings = $ENV{'DGIT_DRS_KEYRINGS'} = argval(); - die unless @ARGV==1 && $ARGV[0] eq '--ssh'; - - my $cmd = $ENV{'SSH_ORIGINAL_COMMAND'}; - $cmd =~ m{ - ^ - (?: \S* / )? - ( [-0-9a-z]+ ) - \s+ - '? (?: \S* / )? - ($package_re) \.git - '?$ - }ox - or reject "command string not understood"; - my $method = $1; - $package = $2; - $realdestrepo = "$dgitrepos/$package.git"; - - my $funcn = $method; - $funcn =~ y/-/_/; - my $mainfunc = $main::{"main__$funcn"}; - - reject "unknown method" unless $mainfunc; - - my ($policy, $pollock) = policyhook(FRESHREPO,'check-package',$package); - if ($policy & FRESHREPO) { - movetogarbage; + foreach my $dk (keys %indistrodir) { + ${ $indistrodir{$dk} } = "$distrodir/$dk"; } - close $pollock or die $!; - if (stat_exists $realdestrepo) { - $destrepo = $realdestrepo; - } else { - debug " fixmissing $funcn"; - my $fixfunc = $main::{"fixmissing__$funcn"}; - &$fixfunc; + while (@ARGV && $ARGV[0] =~ m/^--([-0-9a-z]+)=/ && $indistrodir{$1}) { + ${ $indistrodir{$1} } = $'; #'; + shift @ARGV; } - debug " running main $funcn"; - &$mainfunc; + $ENV{"DGIT_DRS_\U$_"} = ${ $main::{$_} } foreach @hookenvs; + + die unless @ARGV==1; + + my $mode = shift @ARGV; + die unless $mode =~ m/^--(\w+)$/; + my $fn = ${*::}{"mode_$1"}; + die unless $fn; + $fn->(); } sub unlockall () { @@ -704,7 +955,7 @@ sub cleanup () { foreach my $lf (<*.lock>) { my $tree = $lf; $tree =~ s/\.lock$//; - next unless acquiretree($tree, 0); + next unless acquirermtree($tree, 0); remove $lf or warn $!; unlockall(); }