X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=dgit.git;a=blobdiff_plain;f=dgit;h=25afa6734dfebd9a692c524954a537e2b847dfcc;hp=c8907cf6eb48739a2f53854c46cf54cb641ba79c;hb=010a0e1ee137b4fc4d6e5cbfbc5551df8835ce7e;hpb=e4fb7e9b2592d0a80ca213c94f3a462abd31c6c8 diff --git a/dgit b/dgit index c8907cf6..25afa673 100755 --- a/dgit +++ b/dgit @@ -30,9 +30,13 @@ use Dpkg::Version; use POSIX; use IPC::Open2; use Digest::SHA; +use Digest::MD5; +use Config; our $our_version = 'UNRELEASED'; ###substituted### +our $rpushprotovsn = 2; + our $isuite = 'unstable'; our $idistro; our $package; @@ -116,9 +120,14 @@ sub stripepoch ($) { return $vsn; } +sub srcfn ($$) { + my ($vsn,$sfx) = @_; + return "${package}_".(stripepoch $vsn).$sfx +} + sub dscfn ($) { my ($vsn) = @_; - return "${package}_".(stripepoch $vsn).".dsc"; + return srcfn($vsn,".dsc"); } our $us = 'dgit'; @@ -133,6 +142,23 @@ END { } }; +our @signames = split / /, $Config{sig_name}; + +sub waitstatusmsg () { + if (!$?) { + return "terminated, reporting successful completion"; + } elsif (!($? & 255)) { + return "failed with error exit status ".WEXITSTATUS($?); + } elsif (WIFSIGNALED($?)) { + my $signum=WTERMSIG($?); + return "died due to fatal signal ". + ($signames[$signum] // "number $signum"). + ($? & 128 ? " (core dumped)" : ""); # POSIX(3pm) has no WCOREDUMP + } else { + return "failed with unknown wait status ".$?; + } +} + sub printdebug { print DEBUG $debugprefix, @_ or die $!; } sub fail { @@ -157,6 +183,13 @@ sub changedir ($) { chdir $newdir or die "chdir: $newdir: $!"; } +sub stat_exists ($) { + my ($f) = @_; + return 1 if stat $f; + return 0 if $!==&ENOENT; + die "stat $f: $!"; +} + #---------- remote protocol support, common ---------- # remote push initiator/responder protocol: @@ -194,17 +227,39 @@ sub changedir ($) { # # > complete +our $i_child_pid; + +sub i_child_report () { + # Sees if our child has died, and reap it if so. Returns a string + # describing how it died if it failed, or undef otherwise. + return undef unless $i_child_pid; + my $got = waitpid $i_child_pid, WNOHANG; + return undef if $got <= 0; + die unless $got == $i_child_pid; + $i_child_pid = undef; + return undef unless $?; + return "build host child ".waitstatusmsg(); +} + sub badproto ($$) { my ($fh, $m) = @_; fail "connection lost: $!" if $fh->error; fail "protocol violation; $m not expected"; } +sub badproto_badread ($$) { + my ($fh, $wh) = @_; + fail "connection lost: $!" if $!; + my $report = i_child_report(); + fail $report if defined $report; + badproto $fh, "eof (reading $wh)"; +} + sub protocol_expect (&$) { my ($match, $fh) = @_; local $_; $_ = <$fh>; - defined && chomp or badproto $fh, "eof"; + defined && chomp or badproto_badread $fh, "protocol message"; if (wantarray) { my @r = &$match; return @r if @r; @@ -236,7 +291,7 @@ sub protocol_read_bytes ($$) { $nbytes =~ m/^[1-9]\d{0,5}$/ or badproto \*RO, "bad byte count"; my $d; my $got = read $fh, $d, $nbytes; - $got==$nbytes or badproto $fh, "eof during data block"; + $got==$nbytes or badproto_badread $fh, "data block"; return $d; } @@ -350,10 +405,8 @@ sub failedcmd { { local ($!); printcmd \*STDERR, "$us: failed command:", @_ or die $!; }; if ($!) { fail "failed to fork/exec: $!"; - } elsif (!($? & 0xff)) { - fail "subprocess failed with error exit status ".($?>>8); } elsif ($?) { - fail "subprocess crashed (wait status $?)"; + fail "subprocess ".waitstatusmsg(); } else { fail "subprocess produced invalid output"; } @@ -463,13 +516,15 @@ our %defcfg = ('dgit.default.distro' => 'debian', 'dgit.default.username' => '', 'dgit.default.archive-query-default-component' => 'main', 'dgit.default.ssh' => 'ssh', + 'dgit.default.archive-query' => 'madison:', + 'dgit.default.sshpsql-dbname' => 'service=projectb', + 'dgit-distro.debian.archive-query' => 'sshpsql:', 'dgit-distro.debian.git-host' => 'git.debian.org', 'dgit-distro.debian.git-proto' => 'git+ssh://', 'dgit-distro.debian.git-path' => '/git/dgit-repos/repos', 'dgit-distro.debian.git-check' => 'ssh-cmd', 'dgit-distro.debian.git-create' => 'ssh-cmd', 'dgit-distro.debian.sshpsql-host' => 'mirror.ftp-master.debian.org', - 'dgit-distro.debian.sshpsql-dbname' => 'service=projectb', 'dgit-distro.debian.upload-host' => 'ftp-master', # for dput 'dgit-distro.debian.mirror' => 'http://ftp.debian.org/debian/', 'dgit-distro.debian.backports-quirk' => '(squeeze)-backports*', @@ -594,11 +649,29 @@ sub access_giturl () { return "$url/$package.git"; } -sub parsecontrolfh ($$@) { - my ($fh, $desc, @opts) = @_; - my %opts = ('name' => $desc, @opts); - my $c = Dpkg::Control::Hash->new(%opts); - $c->parse($fh) or die "parsing of $desc failed"; +sub parsecontrolfh ($$;$) { + my ($fh, $desc, $allowsigned) = @_; + our $dpkgcontrolhash_noissigned; + my $c; + for (;;) { + my %opts = ('name' => $desc); + $opts{allow_pgp}= $allowsigned || !$dpkgcontrolhash_noissigned; + $c = Dpkg::Control::Hash->new(%opts); + $c->parse($fh,$desc) or die "parsing of $desc failed"; + last if $allowsigned; + last if $dpkgcontrolhash_noissigned; + my $issigned= $c->get_option('is_pgp_signed'); + if (!defined $issigned) { + $dpkgcontrolhash_noissigned= 1; + seek $fh, 0,0 or die "seek $desc: $!"; + } elsif ($issigned) { + fail "control file $desc is (already) PGP-signed. ". + " Note that dgit push needs to modify the .dsc and then". + " do the signature itself"; + } else { + last; + } + } return $c; } @@ -657,16 +730,6 @@ our %rmad; sub archive_query ($) { my ($method) = @_; my $query = access_cfg('archive-query','RETURN-UNDEF'); - if (!defined $query) { - my $distro = access_basedistro(); - if ($distro eq 'debian') { - $query = "sshpsql:". - access_someuserhost('sshpsql').':'. - access_cfg('sshpsql-dbname'); - } else { - $query = "madison:$distro"; - } - } $query =~ s/^(\w+):// or badcfg "invalid archive-query method \`$query'"; my $proto = $1; my $data = $'; #'; @@ -682,6 +745,9 @@ sub pool_dsc_subpath ($$) { sub archive_query_madison ($$) { my ($proto,$data) = @_; die unless $proto eq 'madison'; + if (!length $data) { + $data= access_cfg('madison-distro',access_basedistro()); + } $rmad{$package} ||= cmdoutput qw(rmadison -asource),"-s$isuite","-u$data",$package; my $rmad = $rmad{$package}; @@ -708,7 +774,7 @@ sub madison_parse ($) { $5 eq 'source' or die "$rmad ?"; push @out, [$vsn,pool_dsc_subpath($vsn,$component),$newsuite]; } - return sort { -version_compare_string($a->[0],$b->[0]); } @out; + return sort { -version_compare($a->[0],$b->[0]); } @out; } sub canonicalise_suite_madison ($$) { @@ -723,6 +789,10 @@ sub canonicalise_suite_madison ($$) { sub sshpsql ($$) { my ($data,$sql) = @_; + if (!length $data) { + $data= access_someuserhost('sshpsql').':'. + access_cfg('sshpsql-dbname'); + } $data =~ m/:/ or badcfg "invalid sshpsql method string \`$data'"; my ($userhost,$dbname) = ($`,$'); #'; my @rows; @@ -766,7 +836,7 @@ sub archive_query_sshpsql ($$) { AND source.source='$package' AND files.filename LIKE '%.dsc'; END - @rows = sort { -version_compare_string($a->[0],$b->[0]) } @rows; + @rows = sort { -version_compare($a->[0],$b->[0]) } @rows; my $digester = Digest::SHA->new(256); @rows = map { my ($vsn,$component,$filename,$sha256sum) = @$_; @@ -824,7 +894,7 @@ sub archive_query_dummycat ($$) { } C->error and die "$dpath: $!"; close C; - return sort { -version_compare_string($a->[0],$b->[0]); } @rows; + return sort { -version_compare($a->[0],$b->[0]); } @rows; } sub canonicalise_suite () { @@ -857,7 +927,7 @@ sub get_archive_dsc () { } my $dscfh = new IO::File \$dscdata, '<' or die $!; printdebug Dumper($dscdata) if $debug>1; - $dsc = parsecontrolfh($dscfh,$dscurl, allow_pgp=>1); + $dsc = parsecontrolfh($dscfh,$dscurl,1); printdebug Dumper($dsc) if $debug>1; my $fmt = getfield $dsc, 'Format'; fail "unsupported source format $fmt, sorry" unless $format_ok{$fmt}; @@ -911,6 +981,12 @@ sub prep_ud () { mkdir $ud or die $!; } +sub mktree_in_ud_here () { + runcmd qw(git init -q); + rmtree('.git/objects'); + symlink '../../../../objects','.git/objects' or die $!; +} + sub mktree_in_ud_from_only_subdir () { # changes into the subdir my (@dirs) = <*/.>; @@ -918,11 +994,12 @@ sub mktree_in_ud_from_only_subdir () { $dirs[0] =~ m#^([^/]+)/\.$# or die; my $dir = $1; changedir $dir; - fail "source package contains .git directory" if stat '.git'; - die $! unless $!==&ENOENT; - runcmd qw(git init -q); - rmtree('.git/objects'); - symlink '../../../../objects','.git/objects' or die $!; + fail "source package contains .git directory" if stat_exists '.git'; + mktree_in_ud_here(); + my $format=get_source_format(); + if (madformat($format)) { + rmtree '.pc'; + } runcmd @git, qw(add -Af); my $tree = cmdoutput @git, qw(write-tree); $tree =~ m/^\w+$/ or die "$tree ?"; @@ -960,9 +1037,12 @@ sub dsc_files () { map { $_->{Filename} } dsc_files_info(); } -sub is_orig_file ($) { - local ($_) = @_; - m/\.orig(?:-\w+)?\.tar\.\w+$/; +sub is_orig_file ($;$) { + local ($_) = $_[0]; + my $base = $_[1]; + m/\.orig(?:-\w+)?\.tar\.\w+$/ or return 0; + defined $base or return 1; + return $` eq $base; } sub make_commit ($) { @@ -1038,7 +1118,7 @@ END my $oldclogp = parsecontrol('../changelogold.tmp','previous changelog'); my $oversion = getfield $oldclogp, 'Version'; my $vcmp = - version_compare_string($oversion, $cversion); + version_compare($oversion, $cversion); if ($vcmp < 0) { # git upload/ is earlier vsn than archive, use archive open C, ">../commit2.tmp" or die $!; @@ -1082,11 +1162,9 @@ sub complete_file_from_dsc ($$) { my $tf = "$dstdir/$f"; my $downloaded = 0; - if (stat $tf) { + if (stat_exists $tf) { progress "using existing $f"; } else { - die "$tf $!" unless $!==&ENOENT; - my $furl = $dscurl; $furl =~ s{/[^/]+$}{}; $furl .= "/$f"; @@ -1220,7 +1298,7 @@ END my $gotclogp = parsechangelog("-l$clogf"); my $got_vsn = getfield $gotclogp, 'Version'; printdebug "SKEW CHECK GOT $got_vsn\n"; - if (version_compare_string($got_vsn, $skew_warning_vsn) < 0) { + if (version_compare($got_vsn, $skew_warning_vsn) < 0) { print STDERR <; + F->error and die $!; + chomp; + return $_; } sub madformat ($) { my ($format) = @_; return 0 unless $format eq '3.0 (quilt)'; - progress "Format \`$format', urgh"; if ($noquilt) { progress "Not doing any fixup of \`$format' due to --no-quilt-fixup"; return 0; } + progress "Format \`$format', checking/updating patch stack"; return 1; } @@ -1432,7 +1526,7 @@ sub dopush () { push_parse_changelog("$clogpfn"); my $dscpath = "$buildproductsdir/$dscfn"; - stat $dscpath or + stat_exists $dscpath or fail "looked for .dsc $dscfn, but $!;". " maybe you forgot to build"; @@ -1477,10 +1571,9 @@ sub dopush () { if (!$changesfile) { my $multi = "$buildproductsdir/". "${package}_".(stripepoch $cversion)."_multi.changes"; - if (stat "$multi") { + if (stat_exists "$multi") { $changesfile = $multi; } else { - $!==&ENOENT or die "$multi: $!"; my $pat = "${package}_".(stripepoch $cversion)."_*.changes"; my @cs = glob "$buildproductsdir/$pat"; fail "failed to find unique changes file". @@ -1563,10 +1656,8 @@ sub cmd_clone { } $dstdir ||= "$package"; - if (stat $dstdir) { + if (stat_exists $dstdir) { fail "$dstdir already exists"; - } elsif ($! != &ENOENT) { - die "$dstdir: $!"; } my $cwd_remove; @@ -1670,12 +1761,15 @@ sub cmd_push { #---------- remote commands' implementation ---------- -sub cmd_remote_push_responder { +sub cmd_remote_push_build_host { my ($nrargs) = shift @ARGV; my (@rargs) = @ARGV[0..$nrargs-1]; @ARGV = @ARGV[$nrargs..$#ARGV]; die unless @rargs; - my ($dir) = @rargs; + my ($dir,$vsnwant) = @rargs; + # vsnwant is a comma-separated list; we report which we have + # chosen in our ready response (so other end can tell if they + # offered several) $debugprefix = ' '; $we_are_responder = 1; @@ -1686,19 +1780,30 @@ sub cmd_remote_push_responder { open STDOUT, ">&STDERR" or die $!; autoflush STDOUT 1; - responder_send_command("dgit-remote-push-ready"); + $vsnwant //= 1; + fail "build host has dgit rpush protocol version". + " $rpushprotovsn but invocation host has $vsnwant" + unless grep { $rpushprotovsn eq $_ } split /,/, $vsnwant; + + responder_send_command("dgit-remote-push-ready $rpushprotovsn"); changedir $dir; &cmd_push; } +sub cmd_remote_push_responder { cmd_remote_push_build_host(); } +# ... for compatibility with proto vsn.1 dgit (just so that user gets +# a good error message) + our $i_tmp; -our $i_child_pid; sub i_cleanup { - local ($@); - if ($i_child_pid) { - printdebug "(killing remote child $i_child_pid)\n"; + local ($@, $?); + my $report = i_child_report(); + if (defined $report) { + printdebug "($report)\n"; + } elsif ($i_child_pid) { + printdebug "(killing build host child $i_child_pid)\n"; kill 15, $i_child_pid; } if (defined $i_tmp && !defined $initiator_tempdir) { @@ -1725,11 +1830,11 @@ sub cmd_rpush { $dir = nextarg; } $dir =~ s{^-}{./-}; - my @rargs = ($dir); + my @rargs = ($dir,$rpushprotovsn); my @rdgit; push @rdgit, @dgit; push @rdgit, @ropts; - push @rdgit, qw(remote-push-responder), (scalar @rargs), @rargs; + push @rdgit, qw(remote-push-build-host), (scalar @rargs), @rargs; push @rdgit, @ARGV; my @cmd = (@ssh, $host, shellquote @rdgit); printcmd \*DEBUG,$debugprefix."+",@cmd; @@ -1762,10 +1867,10 @@ sub i_resp_progress ($) { sub i_resp_complete { my $pid = $i_child_pid; $i_child_pid = undef; # prevents killing some other process with same pid - printdebug "waiting for remote child $pid...\n"; + printdebug "waiting for build host child $pid...\n"; my $got = waitpid $pid, 0; die $! unless $got == $pid; - die "remote child failed $?" if $?; + die "build host child failed $?" if $?; i_cleanup(); printdebug "all done\n"; @@ -1776,7 +1881,8 @@ sub i_resp_file ($) { my ($keyword) = @_; my $localname = i_method "i_localname", $keyword; my $localpath = "$i_tmp/$localname"; - stat $localpath and badproto \*RO, "file $keyword ($localpath) twice"; + stat_exists $localpath and + badproto \*RO, "file $keyword ($localpath) twice"; protocol_receive_file \*RO, $localpath; i_method "i_file", $keyword; } @@ -1862,30 +1968,124 @@ our $dscfn; our $fakeeditorenv = 'DGIT_FAKE_EDITOR_QUILT'; sub build_maybe_quilt_fixup () { - if (!open F, "debian/source/format") { - die $! unless $!==&ENOENT; - return; - } - $_ = ; - F->error and die $!; - chomp; - return unless madformat($_); + my $format=get_source_format; + return unless madformat $format; # sigh - - my @cmd = (@git, qw(ls-files --exclude-standard -iodm)); - my $problems = cmdoutput @cmd; - if (length $problems) { - print STDERR "problematic files:\n"; - print STDERR " $_\n" foreach split /\n/, $problems; - fail "Cannot do quilt fixup in tree containing ignored files. ". - "Perhaps your package's clean target is broken, in which". - " case -wg (which says to use git-clean -xdf) may help."; - } + + # Our objective is: + # - honour any existing .pc in case it has any strangeness + # - determine the git commit corresponding to the tip of + # the patch stack (if there is one) + # - if there is such a git commit, convert each subsequent + # git commit into a quilt patch with dpkg-source --commit + # - otherwise convert all the differences in the tree into + # a single git commit + # + # To do this we: + + # Our git tree doesn't necessarily contain .pc. (Some versions of + # dgit would include the .pc in the git tree.) If there isn't + # one, we need to generate one by unpacking the patches that we + # have. + # + # We first look for a .pc in the git tree. If there is one, we + # will use it. (This is not the normal case.) + # + # Otherwise need to regenerate .pc so that dpkg-source --commit + # can work. We do this as follows: + # 1. Collect all relevant .orig from parent directory + # 2. Generate a debian.tar.gz out of + # debian/{patches,rules,source/format} + # 3. Generate a fake .dsc containing just these fields: + # Format Source Version Files + # 4. Extract the fake .dsc + # Now the fake .dsc has a .pc directory. + # (In fact we do this in every case, because in future we will + # want to search for a good base commit for generating patches.) + # + # Then we can actually do the dpkg-source --commit + # 1. Make a new working tree with the same object + # store as our main tree and check out the main + # tree's HEAD. + # 2. Copy .pc from the fake's extraction, if necessary + # 3. Run dpkg-source --commit + # 4. If the result has changes to debian/, then + # - git-add them them + # - git-add .pc if we had a .pc in-tree + # - git-commit + # 5. If we had a .pc in-tree, delete it, and git-commit + # 6. Back in the main tree, fast forward to the new HEAD my $clogp = parsechangelog(); - my $version = getfield $clogp, 'Version'; - my $author = getfield $clogp, 'Maintainer'; my $headref = rev_parse('HEAD'); + + prep_ud(); + changedir $ud; + + my $upstreamversion=$version; + $upstreamversion =~ s/-[^-]*$//; + + my $fakeversion="$upstreamversion-~~DGITFAKE"; + + my $fakedsc=new IO::File 'fake.dsc', '>' or die $!; + print $fakedsc <addfile($fh); + print $fakedsc " ".$md->hexdigest." $size $b\n" or die $!; + }; + + foreach my $f (<../../../../*>) { #/){ + my $b=$f; $b =~ s{.*/}{}; + next unless is_orig_file $b, srcfn $upstreamversion,''; + link $f, $b or die "$b $!"; + $dscaddfile->($b); + } + + my @files=qw(debian/source/format debian/rules); + if (stat_exists '../../../debian/patches') { + push @files, 'debian/patches'; + } + + my $debtar= srcfn $fakeversion,'.debian.tar.gz'; + runcmd qw(env GZIP=-1 tar -zcf), "./$debtar", qw(-C ../../..), @files; + + $dscaddfile->($debtar); + close $fakedsc or die $!; + + runcmd qw(sh -ec), 'exec dpkg-source --no-check -x fake.dsc >/dev/null'; + + my $fakexdir= $package.'-'.(stripepoch $upstreamversion); + rename $fakexdir, "fake" or die "$fakexdir $!"; + + mkdir "work" or die $!; + changedir "work"; + mktree_in_ud_here(); + runcmd @git, qw(reset --hard), $headref; + + my $mustdeletepc=0; + if (stat_exists ".pc") { + -d _ or die; + progress "Tree already contains .pc - will use it then delete it."; + $mustdeletepc=1; + } else { + rename '../fake/.pc','.pc' or die $!; + } + + my $author = getfield $clogp, 'Maintainer'; my $time = time; my $ncommits = 3; my $patchname = "auto-$version-$headref-$time"; @@ -1920,6 +2120,14 @@ END } commit_quilty_patch(); + + if ($mustdeletepc) { + runcmd @git, qw(rm -rq .pc); + commit_admin "Commit removal of .pc (quilt series tracking data)"; + } + + changedir '../../../..'; + runcmd @git, qw(pull --ff-only -q .git/dgit/unpack/work master); } sub quilt_fixup_editor () { @@ -1981,7 +2189,7 @@ sub changesopts () { } if (@vsns) { @vsns = map { $_->[0] } @vsns; - @vsns = sort { -version_compare_string($a, $b) } @vsns; + @vsns = sort { -version_compare($a, $b) } @vsns; $changes_since_version = $vsns[0]; progress "changelog will contain changes since $vsns[0]"; } else { @@ -2046,8 +2254,9 @@ sub cmd_sbuild { changedir ".."; my $pat = "${package}_".(stripepoch $version)."_*.changes"; if (act_local()) { - stat $dscfn or fail "$dscfn (in parent directory): $!"; - stat $sourcechanges or fail "$sourcechanges (in parent directory): $!"; + stat_exist $dscfn or fail "$dscfn (in parent directory): $!"; + stat_exists $sourcechanges + or fail "$sourcechanges (in parent directory): $!"; foreach my $cf (glob $pat) { next if $cf eq $sourcechanges; unlink $cf or fail "remove $cf: $!"; @@ -2064,7 +2273,7 @@ sub cmd_sbuild { runcmd_ordryrun_local @mergechanges, @changesfiles; my $multichanges = "${package}_".(stripepoch $version)."_multi.changes"; if (act_local()) { - stat $multichanges or fail "$multichanges: $!"; + stat_exists $multichanges or fail "$multichanges: $!"; } printdone "build successful, results in $multichanges\n" or die $!; } @@ -2073,6 +2282,7 @@ sub cmd_quilt_fixup { badusage "incorrect arguments to dgit quilt-fixup" if @ARGV; my $clogp = parsechangelog(); $version = getfield $clogp, 'Version'; + $package = getfield $clogp, 'Source'; build_maybe_quilt_fixup(); }