X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=dgit.git;a=blobdiff_plain;f=infra%2Fdgit-repos-server;h=bbf1aa215a34e054b5b4532254865365c7f6e3b4;hp=9705a6420a628298c5c95857042ed3c74c94d941;hb=HEAD;hpb=d6be829bf56e699d27d388026d4af7ab710ba328 diff --git a/infra/dgit-repos-server b/infra/dgit-repos-server index 9705a642..bbf1aa21 100755 --- a/infra/dgit-repos-server +++ b/infra/dgit-repos-server @@ -1,9 +1,28 @@ #!/usr/bin/perl -w # dgit-repos-server # +# git protocol proxy to check dgit pushes etc. +# +# Copyright (C) 2014-2017,2019 Ian Jackson +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + # usages: # dgit-repos-server DISTRO DISTRO-DIR AUTH-SPEC [] --ssh # dgit-repos-server DISTRO DISTRO-DIR AUTH-SPEC [] --cron +# dgit-repos-server DISTRO DISTRO-DIR AUTH-SPEC [] \ +# --tag2upload URL TAGNAME # settings # --repos=GIT-REPOS-DIR default DISTRO-DIR/repos/ # --suites=SUITES-FILE default DISTRO-DIR/suites @@ -33,7 +52,10 @@ # (With --cron AUTH-SPEC is not used and may be the empty string.) use strict; +use Carp; +use IO::Handle; +use Debian::Dgit::Infra; # must precede Debian::Dgit; - can change @INC! use Debian::Dgit qw(:DEFAULT :policyflags); setup_sigwarn(); @@ -69,7 +91,7 @@ setup_sigwarn(); # as a result of this the stunt pre-receive hook runs; it does this: # + understand what refs we are allegedly updating and # check some correspondences: -# * we are updating only refs/tags/DISTRO/* and refs/dgit/* +# * we are updating only refs/tags/[archive/]DISTRO/* and refs/dgit/* # * and only one of each # * and the tag does not already exist # and @@ -86,7 +108,7 @@ setup_sigwarn(); # * the signed tag must refer to a commit # * the signed tag commit must be the refs/dgit value # * the name in the signed tag must correspond to its ref name -# * the tag name must be debian/ (massaged as needed) +# * the tag name must be [archive/]debian/ (massaged as needed) # * the suite is one of those permitted # * the signed tag has a suitable name # * run the "push" policy hook @@ -142,6 +164,10 @@ setup_sigwarn(); # FRESHREPO (4) # blow away repo right away (ie, as if before push or fetch) # ("check-package" and "push" only) +# NOCOMMITCHECK (8) +# suppress dgit-repos-server's check that commits do +# not lack "committer" info (eg as produced by #849041) +# ("push" only) # any unexpected bits mean failure, and then known set bits are ignored # if no unexpected bits set, operation continues (subject to meaning # of any expected bits set). So, eg, exit 0 means "continue normally" @@ -281,13 +307,6 @@ sub reject ($) { die "\ndgit-repos-server: reject: $why\n\n"; } -sub runcmd { - debugcmd '+',@_; - $!=0; $?=0; - my $r = system @_; - die (shellquote @_)." $? $!" if $r; -} - sub policyhook { my ($policyallowbits, @polargs) = @_; # => ($exitstatuspolicybitmap); @@ -342,7 +361,8 @@ sub movetogarbage () { ensuredir "$dgitrepos/_removed-tags"; open PREVIOUS, ">>", removedtagsfile or die removedtagsfile." $!"; - git_for_each_ref('refs/tags/'.debiantag('*',$distro), sub { + git_for_each_ref([ map { 'refs/tags/'.$_ } debiantags('*',$distro) ], + sub { my ($objid,$objtype,$fullrefname,$reftail) = @_; print PREVIOUS "\n$objid $reftail .\n" or die $!; }, $real); @@ -384,16 +404,21 @@ sub makeworkingclone () { rmtree "${workrepo}_fresh"; } +sub mkscript ($$) { + my ($path,$contents) = @_; + my $fh = new IO::File $path, O_WRONLY|O_CREAT|O_TRUNC, 0777 + or die "$path: $!"; + print $fh $contents or die "$path: $!"; + close $fh or die "$path: $!"; +} + sub setupstunthook () { my $prerecv = "$workrepo/hooks/pre-receive"; - my $fh = new IO::File $prerecv, O_WRONLY|O_CREAT|O_TRUNC, 0777 - or die "$prerecv: $!"; - print $fh <) { chomp or die; printdebug " show-refs| $_\n"; s/^\S*[1-9a-f]\S* (\S+)$/$1/ or die; next if m{^refs/heads/master$}; my $wh = - m{^refs/tags/} ? 'tag' : + m{^refs/tags/archive/} ? 'newtag' : + m{^refs/tags/} ? 'omtag' : m{^refs/dgit/} ? 'head' : die; + use Data::Dumper; die if $got{$wh}++; } $!=0; $?=0; close SR or $?==256 or die "$? $!"; printdebug "installprospective ?\n"; die Dumper(\%got)." -- missing refs in new repo" - if grep { !$_ } values %got; + unless $got{head} && grep { m/tag$/ && $got{$_} } keys %got; lockrealtree(); @@ -486,21 +513,22 @@ sub main__git_receive_pack () { our ($tagname, $tagval, $suite, $oldcommit, $commit); our ($version, %tagh); +our ($maint_tagname, $maint_tagval); our ($tagexists_error); sub readupdates () { printdebug " updates ...\n"; + my %tags; while () { chomp or die; printdebug " upd.| $_\n"; m/^(\S+) (\S+) (\S+)$/ or die "$_ ?"; my ($old, $sha1, $refname) = ($1, $2, $3); - if ($refname =~ m{^refs/tags/(?=$distro/)}) { - reject "pushing multiple tags!" if defined $tagname; - $tagname = $'; #'; - $tagval = $sha1; - $tagexists_error= "tag $tagname already exists -". + if ($refname =~ m{^refs/tags/(?=(?:archive/)?$distro/)}) { + my $tn = $'; #'; + $tags{$tn} = $sha1; + $tagexists_error= "tag $tn already exists -". " not replacing previously-pushed version" if $old =~ m/[^0]/; } elsif ($refname =~ m{^refs/dgit/}) { @@ -514,13 +542,23 @@ sub readupdates () { } STDIN->error and die $!; - reject "push is missing tag ref update" unless defined $tagname; + reject "push is missing tag ref update" unless %tags; + my @dtags = grep { m#^archive/# } keys %tags; + reject "need exactly one archive/* tag" if @dtags!=1; + my @mtags = grep { !m#^archive/# } keys %tags; + reject "pushing too many non-dgit tags" if @mtags>1; + ($tagname) = @dtags; + ($maint_tagname) = @mtags; + $tagval = $tags{$tagname}; + $maint_tagval = $tags{$maint_tagname // ''}; + reject "push is missing head ref update" unless defined $suite; printdebug " updates ok.\n"; } -sub parsetag () { - printdebug " parsetag...\n"; +sub readtag () { + printdebug " readtag...\n"; + open PT, ">dgit-tmp/plaintext" or die $!; open DS, ">dgit-tmp/plaintext.asc" or die $!; open T, "-|", qw(git cat-file tag), $tagval or die $!; @@ -536,12 +574,11 @@ sub parsetag () { } } $!=0; $_=; defined or die $!; - m/^($package_re) release (\S+) for \S+ \((\S+)\) \[dgit\]$/ or - reject "tag message not in expected format"; +} - die unless $1 eq $package; - $version = $2; - die "$3 != $suite " unless $3 eq $suite; +sub parsetag_general ($$) { + my ($dgititemfn, $distrofn) = @_; + printdebug " parsetag...\n"; my $copyl = $_; for (;;) { @@ -551,14 +588,11 @@ sub parsetag () { if (m/^\[dgit ([^"].*)\]$/) { # [dgit "something"] is for future $_ = $1." "; while (length) { - if (s/^distro\=(\S+) //) { - die "$1 != $distro" unless $1 eq $distro; - } elsif (s/^(--deliberately-$deliberately_re) //) { - push @deliberatelies, $1; - } elsif (s/^previously:(\S+)=(\w+) //) { - die "previously $1 twice" if defined $previously{$1}; - $previously{$1} = $2; - } elsif (s/^[-+.=0-9a-z]\S* //) { + if ($dgititemfn->()) { + } elsif (s/^distro\=(\S+) //) { + $distrofn->($1); + } elsif (s/^([-+.=0-9a-z]\S*) //) { + printdebug " parsetag ignoring unrecognised \`$1'\n"; } else { die "unknown dgit info in tag ($_)"; } @@ -567,6 +601,7 @@ sub parsetag () { } last if m/^-----BEGIN PGP/; } + $_ = $copyl; for (;;) { print DS or die $!; @@ -579,6 +614,30 @@ sub parsetag () { printdebug " parsetag ok.\n"; } +sub parsetag () { + readtag(); + m/^($package_re) release (\S+) for \S+ \((\S+)\) \[dgit\]$/ or + reject "tag message not in expected format"; + die unless $1 eq $package; + $version = $2; + die "$3 != $suite " unless $3 eq $suite; + + parsetag_general sub { + if (s/^(--deliberately-$deliberately_re) //) { + push @deliberatelies, $1; + } elsif (s/^previously:(\S+)=(\w+) //) { + die "previously $1 twice" if defined $previously{$1}; + $previously{$1} = $2; + } else { + return 0; + } + return 1; + }, sub { + my ($gotdistro) = @_; + die "$gotdistro != $distro" unless $gotdistro eq $distro; + }; +} + sub checksig_keyring ($) { my ($keyringfile) = @_; # returns primary-keyid if signed by a key in this keyring @@ -624,7 +683,7 @@ sub dm_txt_check ($$) { printdebug " dm_txt_check $keyid $dmtxtfn\n"; open DT, '<', $dmtxtfn or die "$dmtxtfn $!"; while (
) { - m/^fingerprint:\s+$keyid$/oi + m/^fingerprint:\s+\Q$keyid\E$/oi ..0 or next; if (s/^allow:/ /i..0) { } else { @@ -725,7 +784,7 @@ sub checktagnoreplay () { # current head for the suite (there must be at least one). # # This prevents any tag implying a NOFFCHECK push being - # replayed to rewind from a different head. + # replayed to overwrite a different head. # # The possibility of an earlier ff-only push being replayed is # eliminated as follows: the tag from such a push would still @@ -814,22 +873,32 @@ sub tagh1 ($) { return $vals->[0]; } -sub checks () { +sub basic_tag_checks() { printdebug "checks\n"; tagh1('type') eq 'commit' or reject "tag refers to wrong kind of object"; tagh1('object') eq $commit or reject "tag refers to wrong commit"; tagh1('tag') eq $tagname or reject "tag name in tag is wrong"; +} + +sub checks () { + basic_tag_checks(); - my $expecttagname = debiantag $version, $distro; - printdebug "expected tag $expecttagname\n"; - $tagname eq $expecttagname or die; + my @expecttagnames = debiantags($version, $distro); + printdebug "expected tag @expecttagnames\n"; + grep { $tagname eq $_ } @expecttagnames or die; + + foreach my $othertag (grep { $_ ne $tagname } @expecttagnames) { + reject "tag $othertag already exists -". + " not replacing previously-pushed version" + if git_get_ref "refs/tags/".$othertag; + } lockrealtree(); @policy_args = ($package,$version,$suite,$tagname, join(",",@deliberatelies)); - $policy = policyhook(NOFFCHECK|FRESHREPO, 'push', @policy_args); + $policy = policyhook(NOFFCHECK|FRESHREPO|NOCOMMITCHECK, 'push', @policy_args); if (defined $tagexists_error) { if ($policy & FRESHREPO) { @@ -849,6 +918,27 @@ sub checks () { chomp $mb; $mb eq $oldcommit or reject "not fast forward on dgit branch"; } + + # defend against commits generated by #849041 + if (!($policy & NOCOMMITCHECK)) { + my @checks = qw(%at + %ct); + my @chk = qw(git log -z); + push @chk, '--pretty=tformat:%H%n'. + (join "", map { $_, '%n' } @checks); + push @chk, "^$oldcommit" if $oldcommit =~ m/[^0]/; + push @chk, $commit;; + printdebug " ~NOCOMMITCHECK @chk\n"; + open CHK, "-|", @chk or die $!; + local $/ = "\0"; + while () { + next unless m/^$/m; + m/^\w+(?=\n)/ or die; + reject "corrupted object $& (missing metadata)"; + } + $!=0; $?=0; close CHK or $?==256 or die "$? $!"; + } + if ($policy & FRESHREPO) { # It's a bit late to be discovering this here, isn't it ? # @@ -872,9 +962,22 @@ sub onwardpush () { my @cmdbase = (qw(git send-pack), $destrepo); push @cmdbase, qw(--force) if $policy & NOFFCHECK; + if ($ENV{GIT_QUARANTINE_PATH}) { + my $recv_wrapper = "$ENV{GIT_QUARANTINE_PATH}/dgit-recv-wrapper"; + mkscript $recv_wrapper, <<'END'; +#!/bin/sh +set -e +unset GIT_QUARANTINE_PATH +exec git receive-pack "$@" +END + push @cmdbase, "--receive-pack=$recv_wrapper"; + } + my @cmd = @cmdbase; push @cmd, "$commit:refs/dgit/$suite", "$tagval:refs/tags/$tagname"; + push @cmd, "$maint_tagval:refs/tags/$maint_tagname" + if defined $maint_tagname; debugcmd '+',@cmd; $!=0; my $r = system @cmd; @@ -970,6 +1073,228 @@ our @hookenvs = qw(distro suitesfile suitesformasterfile policyhook # workrepo and destrepo handled ad-hoc +sub mode_tag2upload () { + # CALLER MUST PREVENT MULTIPLE CONCURRENT RUNS IN SAME CWD + # If we fail (exit nonzero), caller should capture our stderr, + # and retry some bounded number of times in some appropriate way + # Uses whatever ambient gpg key is available + @ARGV==2 or die; + + my $url; + ($url,$tagval) = @ARGV; + + $ENV{DGIT_DRS_EMAIL_NOREPLY} // die; + + my $start = time // die; + my @t = gmtime $start; + + die if $url =~ m/[^[:graph:]]/; + die if $tagval =~ m/[^[:graph:]]/; + + open OL, ">>overall.log" or die $!; + autoflush OL 1; + my $quit = sub { + printf OL "%04d-%02d-%02d %02d:%02d:%02d (%04ds): %s %s: %s\n", + $t[5] + 1900, @t[4,3,2,1,0], (time-$start), $url, $tagval, $_[0]; + exit 0; + }; + + $ENV{DGIT_DRS_ANY_URL} or $url =~ m{^https://}s + or $quit->("url scheme not as expected"); + + $tagval =~ m{^$distro/($versiontag_re)$}s + or $quit->("tag name not for us"); + + $version = $1; + $version =~ y/_\%\#/:~/d; + + my $work = 'work'; + + my $tagref = "refs/tags/$tagval"; + + rmtree $work; + rmtree 'bpd'; + mkdir $work or die $!; + mkdir 'bpd' or die $!; + unlink <*.orig*>; + dif $! if <*.orig*>; + changedir $work; + runcmd qw(git init -q); + runcmd qw(git remote add origin), $url; + runcmd qw(git fetch --depth=1 origin), "$tagref:$tagref"; + changedir ".git"; + mkdir 'dgit-tmp' or die $!; + + my $tagger; + open T, "-|", qw(git cat-file tag), $tagval or die $!; + { + local $/ = undef; + $!=0; $_=; defined or die $!; + + # quick and dirty check, will check properly later + m/^\[dgit[^"]* please-upload(?:\]| )/m or + $quit->("tag missing please-upload request"); + + m/^tagger (.*) \d+ [-+]\d+$/m or + $quit->("failed to fish tagger out of tag"); + $tagger = $1; + }; + + readtag(); + m/^($package_re) release (\S+) for ($suite_re)$/ or + $quit->("tag headline not for us"); + $package = $1; + my $tagmversion = $2; + $suite = $3; + + + # This is for us. From now on, we will capture errors to + # be emailed to the tagger. + + open H, ">>dgit-tmp/tagupl.email" or die $!; + print H <>dgit-tmp/tagupl.log" or die $!; + + my $child = fork() // die $!; + if ($child) { + # we are the parent + # if child exits 0, it has called $quit->() + $!=0; waitpid $child, 0 == $child or die $!; + printdebug "child $child ?=$?\n"; + exit 0 unless $?; + print L "execution child: ", waitstatusmsg(), "\n" or die $!; + close L or die $!; + print H <>tagupl.email + $ENV{DGIT_DRS_SENDMAIL} -oee -odb -oi -t \\ + -f$ENV{DGIT_DRS_EMAIL_NOREPLY} \\ + ("failed, emailed"); + } + + open STDERR, ">&L" or die $!; + open STDOUT, ">&STDERR" or die $!; + open DEBUG, ">&STDERR" if $debuglevel; + + reject "version mismatch $tagmversion != $version " + unless $tagmversion eq $version; + + my %need = map { $_ => 1 } qw(please-upload split); + my ($upstreamc, $upstreamt); + my $quilt; + my $distro_ok; + + confess if defined $upstreamt; + + parsetag_general sub { + if (m/^(\S+) / && exists $need{$1}) { + $_ = $'; + delete $need{$1}; + } elsif (s/^upstream=(\w+) //) { + $upstreamc = $1; + } elsif (s/^upstream-tag=(\S+) //) { + $upstreamt = $1; + } elsif (s/^--quilt=([-+0-9a-z]+) //) { + $quilt = $1; + } else { + return 0; + } + return 1; + }, sub { + my ($gotdistro) = @_; + $distro_ok ||= $gotdistro eq $distro; + }; + + $quit->("not for this distro") unless $distro_ok; + + reject "missing \"$_\"" foreach keys %need; + + verifytag(); + + reject "upstream tag and not commitish, or v-v" + unless defined $upstreamt == defined $upstreamc; + + my @dgit; + push @dgit, $ENV{DGIT_DRS_DGIT} // 'dgit'; + push @dgit, '-wn'; + push @dgit, "-p$package"; + push @dgit, '--build-products-dir=../bpd'; + + changedir ".."; + runcmd (@dgit, qw(setup-gitattributes)); + + my @fetch = qw(git fetch origin --unshallow); + if (defined $upstreamt) { + runcmd qw(git check-ref-format), "refs/tags/$upstreamt"; + my $utagref = "refs/tags/$upstreamt"; + push @fetch, "$utagref:$utagref"; + } + runcmd @fetch; + + runcmd qw(git checkout -q), "refs/tags/$tagval"; + + my $clogp = parsechangelog(); + my $clogf = sub { + my ($f, $exp) = @_; + my $got = getfield $clogp, $f; + return if $got eq $exp; + reject "mismatch: changelog $f $got != $exp"; + }; + $clogf->('Version', $version); + $clogf->('Source', $package); + + @fetch = (@dgit, qw(--for-push fetch), $suite); + debugcmd "+",@_; + $!=0; $?=-1; + if (system @fetch) { + failedcmd @fetch unless $? == 4*256; + } + # this is just to get the orig, so we don't really care about the ref + if (defined $upstreamc) { + my $need_upstreamc = git_rev_parse "refs/tags/$upstreamt"; + $upstreamc eq $need_upstreamc or reject + "upstream-commitish=$upstreamc but tag refers to $need_upstreamc"; + runcmd qw(git deborig), "$upstreamc"; + } + + my @dgitcmd; + push @dgitcmd, @dgit; + push @dgitcmd, qw(--force-uploading-source-only); + if (defined $quilt) { + push @dgitcmd, "--quilt=$quilt"; + if ($quilt =~ m/baredebian/) { + die "needed upstream commmitish with --quilt=baredebian" + unless defined $upstreamc; + push @dgitcmd, "--upstream-commitish=refs/tags/$upstreamt"; + } + } + push @dgitcmd, qw(push-source --new --overwrite), $suite; + + runcmd @dgitcmd; + + $quit->('done'); +} + sub mode_ssh () { die if @ARGV; @@ -1078,7 +1403,7 @@ sub parseargsdispatch () { $ENV{"DGIT_DRS_\U$_"} = ${ $main::{$_} } foreach @hookenvs; - die unless @ARGV==1; + die unless @ARGV>=1; my $mode = shift @ARGV; die unless $mode =~ m/^--(\w+)$/;