X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=dgit.git;a=blobdiff_plain;f=dgit;h=3afaf193a4127e320ce611162b9f5f896086350f;hp=c66c7a63a19162c86ccf71cc59513707d23e045e;hb=f0b46b0422f6ce3bb7c360ee3717eef95d621526;hpb=71437159c44af0328fc4bbc1826be18e4f057444 diff --git a/dgit b/dgit index c66c7a63..3afaf193 100755 --- a/dgit +++ b/dgit @@ -2,7 +2,8 @@ # dgit # Integration between git and Debian-style archives # -# Copyright (C)2013-2016 Ian Jackson +# Copyright (C)2013-2017 Ian Jackson +# Copyright (C)2017 Sean Whitton # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,9 +18,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +END { $? = $Debian::Dgit::ExitStatus::desired // -1; }; +use Debian::Dgit::ExitStatus; + use strict; -use Debian::Dgit; +use Debian::Dgit qw(:DEFAULT :playground); setup_sigwarn(); use IO::Handle; @@ -30,11 +34,12 @@ use File::Path; use File::Temp qw(tempdir); use File::Basename; use Dpkg::Version; +use Dpkg::Compression; +use Dpkg::Compression::Process; use POSIX; use IPC::Open2; use Digest::SHA; use Digest::MD5; -use List::Util qw(any); use List::MoreUtils qw(pairwise); use Text::Glob qw(match_glob); use Fcntl qw(:DEFAULT :flock); @@ -48,7 +53,9 @@ our $absurdity = undef; ###substituted### our @rpushprotovsn_support = qw(4 3 2); # 4 is new tag format our $protovsn; -our $isuite = 'unstable'; +our $cmd; +our $subcommand; +our $isuite; our $idistro; our $package; our @ropts; @@ -56,9 +63,10 @@ our @ropts; our $sign = 1; our $dryrun_level = 0; our $changesfile; -our $buildproductsdir = '..'; +our $buildproductsdir; +our $bpd_glob; our $new_package = 0; -our $ignoredirty = 0; +our $includedirty = 0; our $rmonerror = 1; our @deliberatelies; our %previously; @@ -70,18 +78,20 @@ our $overwrite_version; # undef: not specified; '': check changelog our $quilt_mode; our $quilt_modes_re = 'linear|smash|auto|nofix|nocheck|gbp|dpm|unapplied'; our $dodep14tag; -our $dodep14tag_re = 'want|no|always'; our $split_brain_save; our $we_are_responder; +our $we_are_initiator; our $initiator_tempdir; our $patches_applied_dirtily = 00; our $tagformat_want; our $tagformat; our $tagformatfn; +our $chase_dsc_distro=1; our %forceopts = map { $_=>0 } qw(unrepresentable unsupported-source-format dsc-changes-mismatch changes-origs-exactly + uploading-binaries uploading-source-only import-gitapply-absurd import-gitapply-no-absurd import-dsc-with-dgit-field); @@ -90,7 +100,7 @@ our %format_ok = map { $_=>1 } ("1.0","3.0 (native)","3.0 (quilt)"); our $suite_re = '[-+.0-9a-z]+'; our $cleanmode_re = 'dpkg-source(?:-d)?|git|git-ff|check|none'; -our $orig_f_comp_re = 'orig(?:-[-0-9a-z]+)?'; +our $orig_f_comp_re = qr{orig(?:-$extra_orig_namepart_re)?}; our $orig_f_sig_re = '\\.(?:asc|gpg|pgp)'; our $orig_f_tail_re = "$orig_f_comp_re\\.tar(?:\\.\\w+)?(?:$orig_f_sig_re)?"; @@ -98,19 +108,22 @@ our $git_authline_re = '^([^<>]+) \<(\S+)\> (\d+ [-+]\d+)$'; our $splitbraincache = 'dgit-intern/quilt-cache'; our $rewritemap = 'dgit-rewrite/map'; +our @dpkg_source_ignores = qw(-i(?:^|/)\.git(?:/|$) -I.git); + our (@git) = qw(git); our (@dget) = qw(dget); -our (@curl) = qw(curl); +our (@curl) = (qw(curl --proto-redir), '-all,http,https', qw(-L)); our (@dput) = qw(dput); our (@debsign) = qw(debsign); our (@gpg) = qw(gpg); our (@sbuild) = qw(sbuild); our (@ssh) = 'ssh'; our (@dgit) = qw(dgit); +our (@git_debrebase) = qw(git-debrebase); our (@aptget) = qw(apt-get); our (@aptcache) = qw(apt-cache); -our (@dpkgbuildpackage) = qw(dpkg-buildpackage -i\.git/ -I.git); -our (@dpkgsource) = qw(dpkg-source -i\.git/ -I.git); +our (@dpkgbuildpackage) = (qw(dpkg-buildpackage), @dpkg_source_ignores); +our (@dpkgsource) = (qw(dpkg-source), @dpkg_source_ignores); our (@dpkggenchanges) = qw(dpkg-genchanges); our (@mergechanges) = qw(mergechanges -f); our (@gbp_build) = (''); @@ -126,6 +139,7 @@ our %opts_opt_map = ('dget' => \@dget, # accept for compatibility 'ssh' => \@ssh, 'dgit' => \@dgit, 'git' => \@git, + 'git-debrebase' => \@git_debrebase, 'apt-get' => \@aptget, 'apt-cache' => \@aptcache, 'dpkg-source' => \@dpkgsource, @@ -143,17 +157,21 @@ our %opts_cfg_insertpos = map { } keys %opts_opt_map; sub parseopts_late_defaults(); +sub setup_gitattrs(;$); +sub check_gitattrs($$); +our $playground; our $keyid; autoflush STDOUT 1; our $supplementary_message = ''; -our $need_split_build_invocation = 0; +our $need_split_build_invocation = 1; our $split_brain = 0; END { local ($@, $?); + return unless forkcheck_mainprocess(); print STDERR "! $_\n" foreach $supplementary_message =~ m/^.+$/mg; } @@ -172,11 +190,6 @@ sub debiantag ($$) { return $tagformatfn->($v, $distro); } -sub debiantag_maintview ($$) { - my ($v,$distro) = @_; - return "$distro/".dep14_version_mangle $v; -} - sub madformat ($) { $_[0] eq '3.0 (quilt)' } sub lbranch () { return "$branchprefix/$csuite"; } @@ -218,13 +231,14 @@ initdebug(''); our @end; END { local ($?); + return unless forkcheck_mainprocess(); foreach my $f (@end) { eval { $f->(); }; print STDERR "$us: cleanup: $@" if length $@; } }; -sub badcfg { print STDERR "$us: invalid configuration: @_\n"; exit 12; } +sub badcfg { print STDERR "$us: invalid configuration: @_\n"; finish 12; } sub forceable_fail ($$) { my ($forceoptsl, $msg) = @_; @@ -242,13 +256,7 @@ sub forceing ($) { sub no_such_package () { print STDERR "$us: package $package does not exist in suite $isuite\n"; - exit 4; -} - -sub changedir ($) { - my ($newdir) = @_; - printdebug "CD $newdir\n"; - chdir $newdir or confess "chdir: $newdir: $!"; + finish 4; } sub deliberately ($) { @@ -277,6 +285,42 @@ sub gbp_pq { return opts_opt_multi_cmd @gbp_pq; } +sub dgit_privdir () { + our $dgit_privdir_made //= ensure_a_playground 'dgit'; +} + +sub bpd_abs () { + my $r = $buildproductsdir; + $r = "$maindir/$r" unless $r =~ m{^/}; + return $r; +} + +sub branch_gdr_info ($$) { + my ($symref, $head) = @_; + my ($status, $msg, $current, $ffq_prev, $gdrlast) = + gdr_ffq_prev_branchinfo($symref); + return () unless $status eq 'branch'; + $ffq_prev = git_get_ref $ffq_prev; + $gdrlast = git_get_ref $gdrlast; + $gdrlast &&= is_fast_fwd $gdrlast, $head; + return ($ffq_prev, $gdrlast); +} + +sub branch_is_gdr ($$) { + my ($symref, $head) = @_; + my ($ffq_prev, $gdrlast) = branch_gdr_info($symref, $head); + return 0 unless $ffq_prev || $gdrlast; + return 1; +} + +sub branch_is_gdr_unstitched_ff ($$$) { + my ($symref, $head, $ancestor) = @_; + my ($ffq_prev, $gdrlast) = branch_gdr_info($symref, $head); + return 0 unless $ffq_prev; + return 0 unless is_fast_fwd $ancestor, $ffq_prev; + return 1; +} + #---------- remote protocol support, common ---------- # remote push initiator/responder protocol: @@ -312,6 +356,9 @@ sub gbp_pq { # > param tagformat old|new # > param maint-view MAINT-VIEW-HEAD # +# > param buildinfo-filename P_V_X.buildinfo # zero or more times +# > file buildinfo # for buildinfos to sign +# # > previously REFNAME=OBJNAME # if --deliberately-not-fast-forward # # goes into tag, for replay prevention # @@ -328,6 +375,9 @@ sub gbp_pq { # [etc] # < data-block NBYTES [transfer of signed changes] # [etc] +# < data-block NBYTES [transfer of each signed buildinfo +# [etc] same number and order as "file buildinfo"] +# ... # < files-end # # > complete @@ -483,12 +533,6 @@ sub url_get { our ($dscdata,$dscurl,$dsc,$dsc_checked,$skew_warning_vsn); -sub runcmd { - debugcmd "+",@_; - $!=0; $?=-1; - failedcmd @_ if system @_; -} - sub act_local () { return $dryrun_level <= 1; } sub act_scary () { return !$dryrun_level; } @@ -520,11 +564,6 @@ sub runcmd_ordryrun_local { } } -sub shell_cmd { - my ($first_shell, @cmd) = @_; - return qw(sh -ec), $first_shell.'; exec "$@"', 'x', @cmd; -} - our $helpmsg = < sign tag and package with instead of default @@ -548,7 +588,7 @@ END sub badusage { print STDERR "$us: @_\n", $helpmsg or die $!; - exit 8; + finish 8; } sub nextarg { @@ -556,14 +596,19 @@ sub nextarg { return scalar shift @ARGV; } +sub pre_help () { + not_necessarily_a_tree(); +} sub cmd_help () { print $helpmsg or die $!; - exit 0; + finish 0; } our $td = $ENV{DGIT_TEST_DUMMY_DIR} || "DGIT_TEST_DUMMY_DIR-unset"; our %defcfg = ('dgit.default.distro' => 'debian', + 'dgit.default.default-suite' => 'unstable', + 'dgit.default.old-dsc-distro' => 'debian', 'dgit-suite.*-security.distro' => 'debian-security', 'dgit.default.username' => '', 'dgit.default.archive-query-default-component' => 'main', @@ -572,6 +617,12 @@ our %defcfg = ('dgit.default.distro' => 'debian', 'dgit.default.sshpsql-dbname' => 'service=projectb', 'dgit.default.aptget-components' => 'main', 'dgit.default.dgit-tag-format' => 'new,old,maint', + 'dgit.default.source-only-uploads' => 'ok', + 'dgit.dsc-url-proto-ok.http' => 'true', + 'dgit.dsc-url-proto-ok.https' => 'true', + 'dgit.dsc-url-proto-ok.git' => 'true', + 'dgit.vcs-git.suites', => 'sid', # ;-separated + 'dgit.default.dsc-url-proto-ok' => 'false', # old means "repo server accepts pushes with old dgit tags" # new means "repo server accepts pushes with new dgit tags" # maint means "repo server accepts split brain pushes" @@ -581,6 +632,7 @@ our %defcfg = ('dgit.default.distro' => 'debian', 'dgit-distro.debian.git-check' => 'url', 'dgit-distro.debian.git-check-suffix' => '/info/refs', 'dgit-distro.debian.new-private-pushers' => 't', + 'dgit-distro.debian.source-only-uploads' => 'not-wholly-new', 'dgit-distro.debian/push.git-url' => '', 'dgit-distro.debian/push.git-host' => 'push.dgit.debian.org', 'dgit-distro.debian/push.git-user-force' => 'dgit', @@ -626,32 +678,17 @@ our %defcfg = ('dgit.default.distro' => 'debian', our %gitcfgs; our @gitcfgsources = qw(cmdline local global system); +our $invoked_in_git_tree = 1; sub git_slurp_config () { - local ($debuglevel) = $debuglevel-2; - local $/="\0"; - # This algoritm is a bit subtle, but this is needed so that for # options which we want to be single-valued, we allow the # different config sources to override properly. See #835858. foreach my $src (@gitcfgsources) { next if $src eq 'cmdline'; # we do this ourselves since git doesn't handle it - - my @cmd = (@git, qw(config -z --get-regexp), "--$src", qw(.*)); - debugcmd "|",@cmd; - open GITS, "-|", @cmd or die $!; - while () { - chomp or die; - printdebug "=> ", (messagequote $_), "\n"; - m/\n/ or die "$_ ?"; - push @{ $gitcfgs{$src}{$`} }, $'; #'; - } - $!=0; $?=0; - close GITS - or ($!==0 && $?==256) - or failedcmd @cmd; + $gitcfgs{$src} = git_slurp_config_src $src; } } @@ -659,6 +696,7 @@ sub git_get_config ($) { my ($c) = @_; foreach my $src (@gitcfgsources) { my $l = $gitcfgs{$src}{$c}; + confess "internal error ($l $c)" if $l && !ref $l; printdebug"C $c ".(defined $l ? join " ", map { messagequote "'$_'" } @$l : "undef")."\n" @@ -674,6 +712,7 @@ sub git_get_config ($) { sub cfg { foreach my $c (@_) { return undef if $c =~ /RETURN-UNDEF/; + printdebug "C? $c\n" if $debuglevel >= 5; my $v = git_get_config($c); return $v if defined $v; my $dv = $defcfg{$c}; @@ -686,7 +725,13 @@ sub cfg { "$us: distro or suite appears not to be (properly) supported"; } -sub access_basedistro () { +sub not_necessarily_a_tree () { + # needs to be called from pre_* + @gitcfgsources = grep { $_ ne 'local' } @gitcfgsources; + $invoked_in_git_tree = 0; +} + +sub access_basedistro__noalias () { if (defined $idistro) { return $idistro; } else { @@ -706,6 +751,12 @@ sub access_basedistro () { } } +sub access_basedistro () { + my $noalias = access_basedistro__noalias(); + my $canon = cfg("dgit-distro.$noalias.alias-canon",'RETURN-UNDEF'); + return $canon // $noalias; +} + sub access_nomdistro () { my $base = access_basedistro(); my $r = cfg("dgit-distro.$base.nominal-distro",'RETURN-UNDEF') // $base; @@ -913,78 +964,19 @@ sub access_giturl (;$) { return "$url/$package$suffix"; } -sub parsecontrolfh ($$;$) { - my ($fh, $desc, $allowsigned) = @_; - our $dpkgcontrolhash_noissigned; - my $c; - for (;;) { - my %opts = ('name' => $desc); - $opts{allow_pgp}= $allowsigned || !$dpkgcontrolhash_noissigned; - $c = Dpkg::Control::Hash->new(%opts); - $c->parse($fh,$desc) or die "parsing of $desc failed"; - last if $allowsigned; - last if $dpkgcontrolhash_noissigned; - my $issigned= $c->get_option('is_pgp_signed'); - if (!defined $issigned) { - $dpkgcontrolhash_noissigned= 1; - seek $fh, 0,0 or die "seek $desc: $!"; - } elsif ($issigned) { - fail "control file $desc is (already) PGP-signed. ". - " Note that dgit push needs to modify the .dsc and then". - " do the signature itself"; - } else { - last; - } - } - return $c; -} - -sub parsecontrol { - my ($file, $desc, $allowsigned) = @_; - my $fh = new IO::Handle; - open $fh, '<', $file or die "$file: $!"; - my $c = parsecontrolfh($fh,$desc,$allowsigned); - $fh->error and die $!; - close $fh; - return $c; -} - -sub getfield ($$) { - my ($dctrl,$field) = @_; - my $v = $dctrl->{$field}; - return $v if defined $v; - fail "missing field $field in ".$dctrl->get_option('name'); -} - -sub parsechangelog { - my $c = Dpkg::Control::Hash->new(name => 'parsed changelog'); - my $p = new IO::Handle; - my @cmd = (qw(dpkg-parsechangelog), @_); - open $p, '-|', @cmd or die $!; - $c->parse($p); - $?=0; $!=0; close $p or failedcmd @cmd; - return $c; -} - sub commit_getclogp ($) { # Returns the parsed changelog hashref for a particular commit my ($objid) = @_; our %commit_getclogp_memo; my $memo = $commit_getclogp_memo{$objid}; return $memo if $memo; - mkpath '.git/dgit'; - my $mclog = ".git/dgit/clog-$objid"; + + my $mclog = dgit_privdir()."clog"; runcmd shell_cmd "exec >$mclog", @git, qw(cat-file blob), "$objid:debian/changelog"; $commit_getclogp_memo{$objid} = parsechangelog("-l$mclog"); } -sub must_getcwd () { - my $d = getcwd(); - defined $d or fail "getcwd failed: $!"; - return $d; -} - sub parse_dscdata () { my $dscfh = new IO::File \$dscdata, '<' or die $!; printdebug Dumper($dscdata) if $debuglevel>1; @@ -1139,6 +1131,12 @@ sub file_in_archive_ftpmasterapi { my $info = api_query($data, "file_in_archive/$pat", 1); } +sub package_not_wholly_new_ftpmasterapi { + my ($proto,$data,$pkg) = @_; + my $info = api_query($data,"madison?package=${pkg}&f=json"); + return !!@$info; +} + #---------- `aptget' archive query method ---------- our $aptget_base; @@ -1246,7 +1244,14 @@ END } my @inreleasefiles = grep { m#/InRelease$# } @releasefiles; @releasefiles = @inreleasefiles if @inreleasefiles; - die "apt updated wrong number of Release files (@releasefiles), erk" + if (!@releasefiles) { + fail <) { - chomp or die; - printdebug "| $_\n"; - m/^(\w+) (\S+)$/ or die "$_ ?"; - push @out, { sha256sum => $1, filename => $2 }; - } - close FIA or die failedcmd @cmd; + my $r = $fn->(); + close FIA or ($!==0 && $?==141) or die failedcmd @cmd; + return $r; +} + +sub file_in_archive_dummycatapi ($$$) { + my ($proto,$data,$filename) = @_; + my @out; + dummycatapi_run_in_mirror ' + find -name "$1" -print0 | + xargs -0r sha256sum + ', [$filename], sub { + while () { + chomp or die; + printdebug "| $_\n"; + m/^(\w+) (\S+)$/ or die "$_ ?"; + push @out, { sha256sum => $1, filename => $2 }; + } + }; return \@out; } +sub package_not_wholly_new_dummycatapi { + my ($proto,$data,$pkg) = @_; + dummycatapi_run_in_mirror " + find -name ${pkg}_*.dsc + ", [], sub { + local $/ = undef; + !!; + }; +} + #---------- `madison' archive query method ---------- sub archive_query_madison { @@ -1378,6 +1406,7 @@ sub canonicalise_suite_madison { } sub file_in_archive_madison { return undef; } +sub package_not_wholly_new_madison { return undef; } #---------- `sshpsql' archive query method ---------- @@ -1455,6 +1484,7 @@ END } sub file_in_archive_sshpsql ($$$) { return undef; } +sub package_not_wholly_new_sshpsql ($$$) { return undef; } #---------- `dummycat' archive query method ---------- @@ -1499,6 +1529,7 @@ sub archive_query_dummycat ($$) { } sub file_in_archive_dummycat () { return undef; } +sub package_not_wholly_new_dummycat () { return undef; } #---------- tag format handling ---------- @@ -1666,21 +1697,14 @@ sub create_remote_git_repo () { our ($dsc_hash,$lastpush_mergeinput); our ($dsc_distro, $dsc_hint_tag, $dsc_hint_url); -our $ud = '.git/dgit/unpack'; -sub prep_ud (;$) { - my ($d) = @_; - $d //= $ud; - rmtree($d); - mkpath '.git/dgit'; - mkdir $d or die $!; +sub prep_ud () { + dgit_privdir(); # ensures that $dgit_privdir_made is based on $maindir + $playground = fresh_playground 'dgit/unpack'; } sub mktree_in_ud_here () { - runcmd qw(git init -q); - runcmd qw(git config gc.auto 0); - rmtree('.git/objects'); - symlink '../../../../objects','.git/objects' or die $!; + playtree_setup $gitcfgs{local}; } sub git_write_tree () { @@ -1713,8 +1737,8 @@ sub remove_stray_gits ($) { sub mktree_in_ud_from_only_subdir ($;$) { my ($what,$raw) = @_; - # changes into the subdir + my (@dirs) = <*/.>; die "expected one subdir but found @dirs ?" unless @dirs==1; $dirs[0] =~ m#^([^/]+)/\.$# or die; @@ -1843,6 +1867,40 @@ sub is_orig_file_of_vsn ($$) { return 1; } +# This function determines whether a .changes file is source-only from +# the point of view of dak. Thus, it permits *_source.buildinfo +# files. +# +# It does not, however, permit any other buildinfo files. After a +# source-only upload, the buildds will try to upload files like +# foo_1.2.3_amd64.buildinfo. If the package maintainer included files +# named like this in their (otherwise) source-only upload, the uploads +# of the buildd can be rejected by dak. Fixing the resultant +# situation can require manual intervention. So we block such +# .buildinfo files when the user tells us to perform a source-only +# upload (such as when using the push-source subcommand with the -C +# option, which calls this function). +# +# Note, though, that when dgit is told to prepare a source-only +# upload, such as when subcommands like build-source and push-source +# without -C are used, dgit has a more restrictive notion of +# source-only .changes than dak: such uploads will never include +# *_source.buildinfo files. This is because there is no use for such +# files when using a tool like dgit to produce the source package, as +# dgit ensures the source is identical to git HEAD. +sub test_source_only_changes ($) { + my ($changes) = @_; + foreach my $l (split /\n/, getfield $changes, 'Files') { + $l =~ m/\S+$/ or next; + # \.tar\.[a-z0-9]+ covers orig.tar and the tarballs in native packages + unless ($& =~ m/(?:\.dsc|\.diff\.gz|\.tar\.[a-z0-9]+|_source\.buildinfo)$/) { + print "purportedly source-only changes polluted by $&\n"; + return 0; + } + } + return 1; +} + sub changes_update_origs_from_dsc ($$$$) { my ($dsc, $changes, $upstreamvsn, $changesfile) = @_; my %changes_f; @@ -1906,12 +1964,12 @@ END if ($found_same) { # in archive, delete from .changes if it's there $changed{$file} = "removed" if - $changes->{$fname} =~ s/^.* \Q$file\E$(?:)\n//m; - } elsif ($changes->{$fname} =~ m/^.* \Q$file\E$(?:)\n/m) { + $changes->{$fname} =~ s/\n.* \Q$file\E$(?:)$//m; + } elsif ($changes->{$fname} =~ m/^.* \Q$file\E$(?:)$/m) { # not in archive, but it's here in the .changes } else { my $dsc_data = getfield $dsc, $fname; - $dsc_data =~ m/^(.* \Q$file\E$)\n/m or die "$dsc_data $file ?"; + $dsc_data =~ m/^(.* \Q$file\E$)$/m or die "$dsc_data $file ?"; my $extra = $1; $extra =~ s/ \d+ /$&$placementinfo / or die "$fname $extra >$dsc_data< ?" @@ -1969,7 +2027,14 @@ sub make_commit_text ($) { sub clogp_authline ($) { my ($clogp) = @_; my $author = getfield $clogp, 'Maintainer'; - $author =~ s#,.*##ms; + if ($author =~ m/^[^"\@]+\,/) { + # single entry Maintainer field with unquoted comma + $author = ($& =~ y/,//rd).$'; # strip the comma + } + # git wants a single author; any remaining commas in $author + # are by now preceded by @ (or "). It seems safer to punt on + # "..." for now rather than attempting to dequote or something. + $author =~ s#,.*##ms unless $author =~ m/"/; my $date = cmdoutput qw(date), '+%s %z', qw(-d), getfield($clogp,'Date'); my $authline = "$author $date"; $authline =~ m/$git_authline_re/o or @@ -2048,29 +2113,50 @@ sub generate_commits_from_dsc () { # See big comment in fetch_from_archive, below. # See also README.dsc-import. prep_ud(); - changedir $ud; + changedir $playground; my @dfi = dsc_files_info(); foreach my $fi (@dfi) { my $f = $fi->{Filename}; die "$f ?" if $f =~ m#/|^\.|\.dsc$|\.tmp$#; + my $upper_f = (bpd_abs()."/$f"); + + printdebug "considering reusing $f: "; + + if (link_ltarget "$upper_f,fetch", $f) { + printdebug "linked (using ...,fetch).\n"; + } elsif ((printdebug "($!) "), + $! != ENOENT) { + fail "accessing $buildproductsdir/$f,fetch: $!"; + } elsif (link_ltarget $upper_f, $f) { + printdebug "linked.\n"; + } elsif ((printdebug "($!) "), + $! != ENOENT) { + fail "accessing $buildproductsdir/$f: $!"; + } else { + printdebug "absent.\n"; + } - printdebug "considering linking $f: "; - - link_ltarget "../../../../$f", $f - or ((printdebug "($!) "), 0) - or $!==&ENOENT - or die "$f $!"; - - printdebug "linked.\n"; - - complete_file_from_dsc('.', $fi) + my $refetched; + complete_file_from_dsc('.', $fi, \$refetched) or next; - if (is_orig_file_in_dsc($f, \@dfi)) { - link $f, "../../../../$f" - or $!==&EEXIST - or die "$f $!"; + printdebug "considering saving $f: "; + + if (link $f, $upper_f) { + printdebug "linked.\n"; + } elsif ((printdebug "($!) "), + $! != EEXIST) { + fail "saving $buildproductsdir/$f: $!"; + } elsif (!$refetched) { + printdebug "no need.\n"; + } elsif (link $f, "$upper_f,fetch") { + printdebug "linked (using ...,fetch).\n"; + } elsif ((printdebug "($!) "), + $! != EEXIST) { + fail "saving $buildproductsdir/$f,fetch: $!"; + } else { + printdebug "cannot.\n"; } } @@ -2113,7 +2199,7 @@ sub generate_commits_from_dsc () { if defined $compr_ext && !defined $cname; my $compr_proc = new Dpkg::Compression::Process compression => $cname; - my @compr_cmd = $compr_proc->get_uncompress_cmdline(); + @compr_cmd = $compr_proc->get_uncompress_cmdline(); my $compr_fh = new IO::Handle; my $compr_pid = open $compr_fh, "-|" // die $!; if (!$compr_pid) { @@ -2140,7 +2226,7 @@ sub generate_commits_from_dsc () { !$? or failedcmd @tarcmd; close $input or - (@compr_cmd ? failedcmd @compr_cmd + (@compr_cmd ? ($?==SIGPIPE || failedcmd @compr_cmd) : die $!); # finally, we have the results in "tarball", but maybe # with the wrong permissions @@ -2212,22 +2298,14 @@ sub generate_commits_from_dsc () { } my @clogcmd = qw(dpkg-parsechangelog --format rfc822 --all); - debugcmd "|",@clogcmd; - open CLOGS, "-|", @clogcmd or die $!; - my $clogp; my $r1clogp; printdebug "import clog search...\n"; + parsechangelog_loop \@clogcmd, "package changelog", sub { + my ($thisstanza, $desc) = @_; + no warnings qw(exiting); - for (;;) { - my $stanzatext = do { local $/=""; ; }; - printdebug "import clogp ".Dumper($stanzatext) if $debuglevel>1; - last if !defined $stanzatext; - - my $desc = "package changelog, entry no.$."; - open my $stanzafh, "<", \$stanzatext or die; - my $thisstanza = parsecontrolfh $stanzafh, $desc, 1; $clogp //= $thisstanza; printdebug "import clog $thisstanza->{version} $desc...\n"; @@ -2253,7 +2331,7 @@ sub generate_commits_from_dsc () { # version). Then it remains to choose between the physically # last entry in the file, and the one with the lowest version # number. If these are not the same, we guess that the - # versions were created in a non-monotic order rather than + # versions were created in a non-monotonic order rather than # that the changelog entries have been misordered. printdebug "import clog $thisstanza->{version} vs $upstreamv...\n"; @@ -2262,14 +2340,13 @@ sub generate_commits_from_dsc () { $r1clogp = $thisstanza; printdebug "import clog $r1clogp->{version} becomes r1\n"; - } - die $! if CLOGS->error; - close CLOGS or $?==SIGPIPE or failedcmd @clogcmd; + }; $clogp or fail "package changelog has no entries!"; my $authline = clogp_authline $clogp; my $changes = getfield $clogp, 'Changes'; + $changes =~ s/^\n//; # Changes: \n my $cversion = getfield $clogp, 'Version'; if (@tartrees) { @@ -2352,6 +2429,10 @@ END my $path = $ENV{PATH} or die; + # we use ../../gbp-pq-output, which (given that we are in + # $playground/PLAYTREE, and $playground is .git/dgit/unpack, + # is .git/dgit. + foreach my $use_absurd (qw(0 1)) { runcmd @git, qw(checkout -q unpa); runcmd @git, qw(update-ref -d refs/heads/patch-queue/unpa); @@ -2436,44 +2517,61 @@ END @output = $lastpush_mergeinput; } } - changedir '../../../..'; - rmtree($ud); + changedir $maindir; + rmtree $playground; return @output; } -sub complete_file_from_dsc ($$) { - our ($dstdir, $fi) = @_; - # Ensures that we have, in $dir, the file $fi, with the correct +sub complete_file_from_dsc ($$;$) { + our ($dstdir, $fi, $refetched) = @_; + # Ensures that we have, in $dstdir, the file $fi, with the correct # contents. (Downloading it from alongside $dscurl if necessary.) + # If $refetched is defined, can overwrite "$dstdir/$fi->{Filename}" + # and will set $$refetched=1 if it did so (or tried to). my $f = $fi->{Filename}; my $tf = "$dstdir/$f"; my $downloaded = 0; + my $got; + my $checkhash = sub { + open F, "<", "$tf" or die "$tf: $!"; + $fi->{Digester}->reset(); + $fi->{Digester}->addfile(*F); + F->error and die $!; + $got = $fi->{Digester}->hexdigest(); + return $got eq $fi->{Hash}; + }; + if (stat_exists $tf) { - progress "using existing $f"; + if ($checkhash->()) { + progress "using existing $f"; + return 1; + } + if (!$refetched) { + fail "file $f has hash $got but .dsc". + " demands hash $fi->{Hash} ". + "(perhaps you should delete this file?)"; + } + progress "need to fetch correct version of $f"; + unlink $tf or die "$tf $!"; + $$refetched = 1; } else { printdebug "$tf does not exist, need to fetch\n"; - my $furl = $dscurl; - $furl =~ s{/[^/]+$}{}; - $furl .= "/$f"; - die "$f ?" unless $f =~ m/^\Q${package}\E_/; - die "$f ?" if $f =~ m#/#; - runcmd_ordryrun_local @curl,qw(-f -o),$tf,'--',"$furl"; - return 0 if !act_local(); - $downloaded = 1; - } - - open F, "<", "$tf" or die "$tf: $!"; - $fi->{Digester}->reset(); - $fi->{Digester}->addfile(*F); - F->error and die $!; - my $got = $fi->{Digester}->hexdigest(); - $got eq $fi->{Hash} or + } + + my $furl = $dscurl; + $furl =~ s{/[^/]+$}{}; + $furl .= "/$f"; + die "$f ?" unless $f =~ m/^\Q${package}\E_/; + die "$f ?" if $f =~ m#/#; + runcmd_ordryrun_local @curl,qw(-f -o),$tf,'--',"$furl"; + return 0 if !act_local(); + + $checkhash->() or fail "file $f has hash $got but .dsc". " demands hash $fi->{Hash} ". - ($downloaded ? "(got wrong file from archive!)" - : "(perhaps you should delete this file?)"); + "(got wrong file from archive!)"; return 1; } @@ -2483,7 +2581,7 @@ sub ensure_we_have_orig () { foreach my $fi (@dfi) { my $f = $fi->{Filename}; next unless is_orig_file_in_dsc($f, \@dfi); - complete_file_from_dsc('..', $fi) + complete_file_from_dsc($buildproductsdir, $fi) or next; } } @@ -2501,7 +2599,7 @@ sub lrfetchref () { return lrfetchrefs.'/'.server_branch($csuite); } # (If we deleted them unconditionally, then we might end up # re-fetching the same git objects each time dgit fetch was run.) # -# So, leach use of lrfetchrefs needs to be accompanied by arrangements +# So, each use of lrfetchrefs needs to be accompanied by arrangements # in git_fetch_us to fetch the refs in question, and possibly a call # to lrfetchref_used. @@ -2515,7 +2613,7 @@ sub lrfetchref_used ($) { } sub git_lrfetch_sane { - my ($supplementary, @specs) = @_; + my ($url, $supplementary, @specs) = @_; # Make a 'refs/'.lrfetchrefs.'/*' be just like on server, # at least as regards @specs. Also leave the results in # %lrfetchrefs_f, and arrange for lrfetchref_used to be @@ -2546,8 +2644,6 @@ sub git_lrfetch_sane { # git fetch to try to generate it. If we don't manage to generate # the target state, we try again. - my $url = access_giturl(); - printdebug "git_lrfetch_sane suppl=$supplementary specs @specs\n"; my $specre = join '|', map { @@ -2641,6 +2737,11 @@ END my $want = $wantr{$rrefname}; next if $got eq $want; if (!defined $objgot{$want}) { + fail <{$field}; last if defined $f; } + if (!defined $f) { progress "$what: NO git hash"; + parse_dsc_field_def_dsc_distro(); } elsif (($dsc_hash, $dsc_distro, $dsc_hint_tag, $dsc_hint_url) - = $f =~ m/^(\w+) ($distro_re) ($versiontag_re) (\S+)(?:\s|$)/) { + = $f =~ m/^(\w+)\s+($distro_re)\s+($versiontag_re)\s+(\S+)(?:\s|$)/) { progress "$what: specified git info ($dsc_distro)"; $dsc_hint_tag = [ $dsc_hint_tag ]; } elsif ($f =~ m/^\w+\s*$/) { $dsc_hash = $&; - $dsc_distro //= 'debian'; + parse_dsc_field_def_dsc_distro(); $dsc_hint_tag = [ debiantags +(getfield $dsc, 'Version'), $dsc_distro ]; progress "$what: specified git hash"; @@ -2763,16 +2881,92 @@ sub resolve_dsc_field_commit ($$) { return unless defined $dsc_hash; - my $rewritemapdata = git_cat_file $already_mapref.':map'; - if (defined $rewritemapdata - && $rewritemapdata =~ m/^$dsc_hash(?:[ \t](\w+))/m) { - progress "server's git history rewrite map contains a relevant entry!"; + my $mapref = + defined $already_mapref && + ($already_distro eq $dsc_distro || !$chase_dsc_distro) + ? $already_mapref : undef; - $dsc_hash = $1; - if (defined $dsc_hash) { - progress "using rewritten git hash in place of .dsc value"; - } else { - progress "server data says .dsc hash is to be disregarded"; + my $do_fetch; + $do_fetch = sub { + my ($what, @fetch) = @_; + + local $idistro = $dsc_distro; + my $lrf = lrfetchrefs; + + if (!$chase_dsc_distro) { + progress + "not chasing .dsc distro $dsc_distro: not fetching $what"; + return 0; + } + + progress + ".dsc names distro $dsc_distro: fetching $what"; + + my $url = access_giturl(); + if (!defined $url) { + defined $dsc_hint_url or fail <("rewrite map", $rewritemap) or return; + $mapref = $lrf.'/'.$rewritemap; + } + my $rewritemapdata = git_cat_file $mapref.':map'; + if (defined $rewritemapdata + && $rewritemapdata =~ m/^$dsc_hash(?:[ \t](\w+))/m) { + progress + "server's git history rewrite map contains a relevant entry!"; + + $dsc_hash = $1; + if (defined $dsc_hash) { + progress "using rewritten git hash in place of .dsc value"; + } else { + progress "server data says .dsc hash is to be disregarded"; + } + } + } + + if (!defined git_cat_file $dsc_hash) { + my @tags = map { "tags/".$_ } @$dsc_hint_tag; + my $lrf = $do_fetch->("additional commits", @tags) && + defined git_cat_file $dsc_hash + or fail <", $mcf or die "$mcf $!"; print MC <", "$attrs.new" or die "$attrs.new $!"; if (!open ATTRS, "<", $attrs) { @@ -3168,11 +3364,109 @@ sub ensure_setup_existing_tree () { set_local_git_config $k, 'true'; } +sub open_main_gitattrs () { + confess 'internal error no maindir' unless defined $maindir; + my $gai = new IO::File "$maindir_gitcommon/info/attributes" + or $!==ENOENT + or die "open $maindir_gitcommon/info/attributes: $!"; + return $gai; +} + +our $gitattrs_ourmacro_re = qr{^\[attr\]dgit-defuse-attrs\s}; + +sub is_gitattrs_setup () { + # return values: + # trueish + # 1: gitattributes set up and should be left alone + # falseish + # 0: there is a dgit-defuse-attrs but it needs fixing + # undef: there is none + my $gai = open_main_gitattrs(); + return 0 unless $gai; + while (<$gai>) { + next unless m{$gitattrs_ourmacro_re}; + return 1 if m{\s-working-tree-encoding\s}; + printdebug "is_gitattrs_setup: found old macro\n"; + return 0; + } + $gai->error and die $!; + printdebug "is_gitattrs_setup: found nothing\n"; + return undef; +} + +sub setup_gitattrs (;$) { + my ($always) = @_; + return unless $always || access_cfg_bool(1, 'setup-gitattributes'); + + my $already = is_gitattrs_setup(); + if ($already) { + progress < $af.new" or die $!; + print GAO <) { + if (m{$gitattrs_ourmacro_re}) { + die unless defined $already; + $_ = $new; + } + chomp; + print GAO $_, "\n" or die $!; + } + $gai->error and die $!; + } + close GAO or die $!; + rename "$af.new", "$af" or die "install $af: $!"; +} + sub setup_new_tree () { setup_mergechangelogs(); setup_useremail(); + setup_gitattrs(); +} + +sub check_gitattrs ($$) { + my ($treeish, $what) = @_; + + return if is_gitattrs_setup; + + local $/="\0"; + my @cmd = (@git, qw(ls-tree -lrz --), "${treeish}:"); + debugcmd "|",@cmd; + my $gafl = new IO::File; + open $gafl, "-|", @cmd or die $!; + while (<$gafl>) { + chomp or die; + s/^\d+\s+\w+\s+\w+\s+(\d+)\t// or die; + next if $1 == 0; + next unless m{(?:^|/)\.gitattributes$}; + + # oh dear, found one + print STDERR <(), and returns undef @@ -3180,6 +3474,7 @@ sub multisuite_suite_child ($$$) { my $canonsuitefh = IO::File::new_tmpfile; my $pid = fork // die $!; if (!$pid) { + forkcheck_setup(); $isuite = $tsuite; $us .= " [$isuite]"; $debugprefix .= " "; @@ -3237,11 +3532,12 @@ sub fork_for_multisuite ($) { $before_fetch_merge->(); foreach my $tsuite (@suites[1..$#suites]) { + $tsuite =~ s/^-/$cbasesuite-/; my $csubsuite = multisuite_suite_child($tsuite, \@mergeinputs, sub { @end = (); - fetch(); - exit 0; + fetch_one(); + finish 0; }); # xxx collecte the ref here @@ -3351,18 +3647,22 @@ END } sub clone ($) { + # in multisuite, returns twice! + # once in parent after first suite fetched, + # and then again in child after everything is finished my ($dstdir) = @_; badusage "dry run makes no sense with clone" unless act_local(); my $multi_fetched = fork_for_multisuite(sub { printdebug "multi clone before fetch merge\n"; changedir $dstdir; + record_maindir(); }); if ($multi_fetched) { printdebug "multi clone after fetch merge\n"; clone_set_head(); clone_finish($dstdir); - exit 0; + return; } printdebug "clone main body\n"; @@ -3371,6 +3671,8 @@ sub clone ($) { mkdir $dstdir or fail "create \`$dstdir': $!"; changedir $dstdir; runcmd @git, qw(init -q); + record_maindir(); + setup_new_tree(); clone_set_head(); my $giturl = access_giturl(1); if (defined $giturl) { @@ -3389,23 +3691,40 @@ sub clone ($) { $vcsgiturl =~ s/\s+-b\s+\S+//g; runcmd @git, qw(remote add vcs-git), $vcsgiturl; } - setup_new_tree(); clone_finish($dstdir); } -sub fetch () { +sub fetch_one () { canonicalise_suite(); if (check_for_git()) { git_fetch_us(); } fetch_from_archive() or no_such_package(); + + my $vcsgiturl = $dsc && $dsc->{'Vcs-Git'}; + if (length $vcsgiturl and + (grep { $csuite eq $_ } + split /\;/, + cfg 'dgit.vcs-git.suites')) { + my $current = cfg 'remote.vcs-git.url', 'RETURN-UNDEF'; + if (defined $current && $current ne $vcsgiturl) { + print STDERR < message fragment "$saved" describing disposition of $dgitview return "commit id $dgitview" unless defined $split_brain_save; - my @cmd = (shell_cmd "cd ../../../..", - @git, qw(update-ref -m), + my @cmd = (shell_cmd 'cd "$1"; shift', $maindir, + git_update_ref_cmd "dgit --dgit-view-save $msg HEAD=$headref", $split_brain_save, $dgitview); runcmd @cmd; @@ -3568,18 +3889,28 @@ sub pseudomerge_version_check ($$) { } else { my $v = $i_arch_v->[0]; progress "Checking package changelog for archive version $v ..."; + my $cd; eval { my @xa = ("-f$v", "-t$v"); my $vclogp = parsechangelog @xa; - my $cv = [ (getfield $vclogp, 'Version'), - "Version field from dpkg-parsechangelog @xa" ]; + my $gf = sub { + my ($fn) = @_; + [ (getfield $vclogp, $fn), + "$fn field from dpkg-parsechangelog @xa" ]; + }; + my $cv = $gf->('Version'); infopair_cond_equal($i_arch_v, $cv); + $cd = $gf->('Distribution'); }; if ($@) { $@ =~ s/^dgit: //gm; fail "$@". "Perhaps debian/changelog does not mention $v ?"; } + fail <[0] =~ m/UNRELEASED/; +$cd->[1] is $cd->[0] +Your tree seems to based on earlier (not uploaded) $v. +END } } @@ -3601,8 +3932,9 @@ sub pseudomerge_make_commit ($$$$ $$) { : !length $overwrite_version ? " --overwrite" : " --overwrite=".$overwrite_version; - mkpath '.git/dgit'; - my $pmf = ".git/dgit/pseudomerge"; + # Contributing parent is the first parent - that makes + # git rev-list --first-parent DTRT. + my $pmf = dgit_privdir()."/pseudomerge"; open MC, ">", $pmf or die "$pmf $!"; print MC <[0] into your HEAD."; return $r; @@ -3716,8 +4051,12 @@ sub push_parse_changelog ($) { fail "-p specified $package but changelog specified $clogpackage" unless $package eq $clogpackage; my $cversion = getfield $clogp, 'Version'; - my $tag = debiantag($cversion, access_nomdistro); - runcmd @git, qw(check-ref-format), $tag; + + if (!$we_are_initiator) { + # rpush initiator can't do this because it doesn't have $isuite yet + my $tag = debiantag($cversion, access_nomdistro); + runcmd @git, qw(check-ref-format), $tag; + } my $dscfn = dscfn($cversion); @@ -3902,9 +4241,10 @@ END prep_ud(); access_giturl(); # check that success is vaguely likely + rpush_handle_protovsn_bothends() if $we_are_initiator; select_tagformat(); - my $clogpfn = ".git/dgit/changelog.822.tmp"; + my $clogpfn = dgit_privdir()."/changelog.822.tmp"; runcmd shell_cmd "exec >$clogpfn", qw(dpkg-parsechangelog); responder_send_file('parsed-changelog', $clogpfn); @@ -3924,7 +4264,14 @@ END my $format = getfield $dsc, 'Format'; printdebug "format $format\n"; + my $symref = git_get_symref(); my $actualhead = git_rev_parse('HEAD'); + + if (branch_is_gdr_unstitched_ff($symref, $actualhead, $archive_hash)) { + runcmd_ordryrun_local @git_debrebase, 'stitch'; + $actualhead = git_rev_parse('HEAD'); + } + my $dgithead = $actualhead; my $maintviewhead = undef; @@ -3933,27 +4280,28 @@ END if (madformat_wantfixup($format)) { # user might have not used dgit build, so maybe do this now: if (quiltmode_splitbrain()) { - changedir $ud; + changedir $playground; quilt_make_fake_dsc($upstreamversion); my $cachekey; ($dgithead, $cachekey) = quilt_check_splitbrain_cache($actualhead, $upstreamversion); $dgithead or fail "--quilt=$quilt_mode but no cached dgit view: - perhaps tree changed since dgit build[-source] ?"; + perhaps HEAD changed since dgit build[-source] ?"; $split_brain = 1; $dgithead = splitbrain_pseudomerge($clogp, $actualhead, $dgithead, $archive_hash); $maintviewhead = $actualhead; - changedir '../../../..'; + changedir $maindir; prep_ud(); # so _only_subdir() works, below } else { commit_quilty_patch(); } } - if (defined $overwrite_version && !defined $maintviewhead) { + if (defined $overwrite_version && !defined $maintviewhead + && $archive_hash) { $dgithead = plain_overwrite_pseudomerge($clogp, $dgithead, $archive_hash); @@ -3977,26 +4325,55 @@ END } } - changedir $ud; + changedir $playground; progress "checking that $dscfn corresponds to HEAD"; runcmd qw(dpkg-source -x --), - $dscpath =~ m#^/# ? $dscpath : "../../../$dscpath"; + $dscpath =~ m#^/# ? $dscpath : "$maindir/$dscpath"; my ($tree,$dir) = mktree_in_ud_from_only_subdir("source package"); check_for_vendor_patches() if madformat($dsc->{format}); - changedir '../../../..'; + changedir $maindir; my @diffcmd = (@git, qw(diff --quiet), $tree, $dgithead); debugcmd "+",@diffcmd; $!=0; $?=-1; my $r = system @diffcmd; if ($r) { if ($r==256) { + my $referent = $split_brain ? $dgithead : 'HEAD'; my $diffs = cmdoutput @git, qw(diff --stat), $tree, $dgithead; - fail <{Files} =~ m{\.deb$}m; + my $sourceonlypolicy = access_cfg 'source-only-uploads'; + if ($sourceonlypolicy eq 'ok') { + } elsif ($sourceonlypolicy eq 'always') { + forceable_fail [qw(uploading-binaries)], + "uploading binaries, although distroy policy is source only" + if $hasdebs; + } elsif ($sourceonlypolicy eq 'never') { + forceable_fail [qw(uploading-source-only)], + "source-only upload, although distroy policy requires .debs" + if !$hasdebs; + } elsif ($sourceonlypolicy eq 'not-wholly-new') { + forceable_fail [qw(uploading-source-only)], + "source-only upload, even though package is entirely NEW\n". + "(this is contrary to policy in ".(access_nomdistro()).")" + if !$hasdebs + && $new_package + && !(archive_query('package_not_wholly_new', $package) // 1); + } else { + badcfg "unknown source-only-uploads policy \`$sourceonlypolicy'"; + } + # Perhaps adjust .dsc to contain right set of origs changes_update_origs_from_dsc($dsc, $changes, $upstreamversion, $changesfile) @@ -4030,12 +4430,21 @@ END responder_send_file('changes',$changesfile); responder_send_command("param head $dgithead"); responder_send_command("param csuite $csuite"); + responder_send_command("param isuite $isuite"); responder_send_command("param tagformat $tagformat"); if (defined $maintviewhead) { die unless ($protovsn//4) >= 4; responder_send_command("param maint-view $maintviewhead"); } + # Perhaps send buildinfo(s) for signing + my $changes_files = getfield $changes, 'Files'; + my @buildinfos = ($changes_files =~ m/ .* (\S+\.buildinfo)$/mg); + foreach my $bi (@buildinfos) { + responder_send_command("param buildinfo-filename $bi"); + responder_send_file('buildinfo', "$buildproductsdir/$bi"); + } + if (deliberately_not_fast_forward) { git_for_each_ref(lrfetchrefs, sub { my ($objid,$objtype,$lrfetchrefname,$reftail) = @_; @@ -4046,7 +4455,7 @@ END } my @tagwants = push_tagwants($cversion, $dgithead, $maintviewhead, - ".git/dgit/tag"); + dgit_privdir()."/tag"); my @tagobjfns; supplementary_message(<<'END'); @@ -4094,7 +4503,7 @@ END runcmd_ordryrun @git, qw(-c push.followTags=false push), access_giturl(), @pushrefs; - runcmd_ordryrun @git, qw(update-ref -m), 'dgit push', lrref(), $dgithead; + runcmd_ordryrun git_update_ref_cmd 'dgit push', lrref(), $dgithead; supplementary_message(<<'END'); Push failed, while obtaining signatures on the .changes and .dsc. @@ -4106,9 +4515,10 @@ If you need to change the package, you must use a new version number. END if ($we_are_responder) { my $dryrunsuffix = act_local() ? "" : ".tmp"; + my @rfiles = ($dscpath, $changesfile); + push @rfiles, map { "$buildproductsdir/$_" } @buildinfos; responder_receive_files('signed-dsc-changes', - "$dscpath$dryrunsuffix", - "$changesfile$dryrunsuffix"); + map { "$_$dryrunsuffix" } @rfiles); } else { if (act_local()) { rename "$dscpath.tmp",$dscpath or die "$dscfn $!"; @@ -4134,6 +4544,9 @@ END responder_send_command("complete"); } +sub pre_clone () { + not_necessarily_a_tree(); +} sub cmd_clone { parseopts(); my $dstdir; @@ -4182,24 +4595,29 @@ sub cmd_clone { } sub branchsuite () { - my $branch = cmdoutput_errok @git, qw(symbolic-ref HEAD); - if ($branch =~ m#$lbranch_re#o) { + my $branch = git_get_symref(); + if (defined $branch && $branch =~ m#$lbranch_re#o) { return $1; } else { return undef; } } -sub fetchpullargs () { +sub package_from_d_control () { if (!defined $package) { my $sourcep = parsecontrol('debian/control','debian/control'); $package = getfield $sourcep, 'Source'; } +} + +sub fetchpullargs () { + package_from_d_control(); if (@ARGV==0) { $isuite = branchsuite(); if (!$isuite) { my $clogp = parsechangelog(); - $isuite = getfield $clogp, 'Distribution'; + my $clogsuite = getfield $clogp, 'Distribution'; + $isuite= $clogsuite if $clogsuite ne 'UNRELEASED'; } } elsif (@ARGV==1) { ($isuite) = @ARGV; @@ -4212,9 +4630,7 @@ sub fetchpullargs () { sub cmd_fetch { parseopts(); fetchpullargs(); - my $multi_fetched = fork_for_multisuite(sub { }); - exit 0 if $multi_fetched; - fetch(); + dofetch(); } sub cmd_pull { @@ -4229,21 +4645,98 @@ END pull(); } -sub cmd_push { +sub cmd_checkout { parseopts(); + package_from_d_control(); + @ARGV==1 or badusage "dgit checkout needs a suite argument"; + ($isuite) = @ARGV; + notpushing(); + + foreach my $canon (qw(0 1)) { + if (!$canon) { + $csuite= $isuite; + } else { + undef $csuite; + canonicalise_suite(); + } + if (length git_get_ref lref()) { + # local branch already exists, yay + last; + } + if (!length git_get_ref lrref()) { + if (!$canon) { + # nope + next; + } + dofetch(); + } + # now lrref exists + runcmd (@git, qw(update-ref), lref(), lrref(), ''); + last; + } + local $ENV{GIT_REFLOG_ACTION} = git_reflog_action_msg + "dgit checkout $isuite"; + runcmd (@git, qw(checkout), lbranch()); +} + +sub cmd_update_vcs_git () { + my $specsuite; + if (@ARGV==0 || $ARGV[0] =~ m/^-/) { + ($specsuite,) = split /\;/, cfg 'dgit.vcs-git.suites'; + } else { + ($specsuite) = (@ARGV); + shift @ARGV; + } + my $dofetch=1; + if (@ARGV) { + if ($ARGV[0] eq '-') { + $dofetch = 0; + } elsif ($ARGV[0] eq '-') { + shift; + } + } + + package_from_d_control(); + my $ctrl; + if ($specsuite eq '.') { + $ctrl = parsecontrol 'debian/control', 'debian/control'; + } else { + $isuite = $specsuite; + get_archive_dsc(); + $ctrl = $dsc; + } + my $url = getfield $ctrl, 'Vcs-Git'; + + my @cmd; + my $orgurl = cfg 'remote.vcs-git.url', 'RETURN-UNDEF'; + if (!defined $orgurl) { + print STDERR "setting up vcs-git: $url\n"; + @cmd = (@git, qw(remote add vcs-git), $url); + } elsif ($orgurl eq $url) { + print STDERR "vcs git already configured: $url\n"; + } else { + print STDERR "changing vcs-git url to: $url\n"; + @cmd = (@git, qw(remote set-url vcs-git), $url); + } + runcmd_ordryrun_local @cmd; + if ($dofetch) { + print "fetching (@ARGV)\n"; + runcmd_ordryrun_local @git, qw(fetch vcs-git), @ARGV; + } +} + +sub prep_push () { + parseopts(); + build_or_push_prep_early(); pushing(); - badusage "-p is not allowed with dgit push" if defined $package; check_not_dirty(); - my $clogp = parsechangelog(); - $package = getfield $clogp, 'Source'; my $specsuite; if (@ARGV==0) { } elsif (@ARGV==1) { ($specsuite) = (@ARGV); } else { - badusage "incorrect arguments to dgit push"; + badusage "incorrect arguments to dgit $subcommand"; } - $isuite = getfield $clogp, 'Distribution'; if ($new_package) { local ($package) = $existing_package; # this is a hack canonicalise_suite(); @@ -4253,15 +4746,36 @@ sub cmd_push { if (defined $specsuite && $specsuite ne $isuite && $specsuite ne $csuite) { - fail "dgit push: changelog specifies $isuite ($csuite)". + fail "dgit $subcommand: changelog specifies $isuite ($csuite)". " but command line specifies $specsuite"; } +} + +sub cmd_push { + prep_push(); + dopush(); +} + +sub cmd_push_source { + prep_push(); + fail "dgit push-source: --include-dirty/--ignore-dirty does not make". + "sense with push-source!" if $includedirty; + if ($changesfile) { + my $changes = parsecontrol("$buildproductsdir/$changesfile", + "source changes file"); + unless (test_source_only_changes($changes)) { + fail "user-specified changes file is not source-only"; + } + } else { + # Building a source package is very fast, so just do it + build_source_for_push(); + } dopush(); } #---------- remote commands' implementation ---------- -sub cmd_remote_push_build_host { +sub pre_remote_push_build_host { my ($nrargs) = shift @ARGV; my (@rargs) = @ARGV[0..$nrargs-1]; @ARGV = @ARGV[$nrargs..$#ARGV]; @@ -4274,8 +4788,6 @@ sub cmd_remote_push_build_host { $we_are_responder = 1; $us .= " (build host)"; - pushing(); - open PI, "<&STDIN" or die $!; open STDIN, "/dev/null" or die $!; open PO, ">&STDOUT" or die $!; @@ -4293,12 +4805,14 @@ sub cmd_remote_push_build_host { " but invocation host has $vsnwant" unless defined $protovsn; - responder_send_command("dgit-remote-push-ready $protovsn"); - rpush_handle_protovsn_bothends(); changedir $dir; +} +sub cmd_remote_push_build_host { + responder_send_command("dgit-remote-push-ready $protovsn"); &cmd_push; } +sub pre_remote_push_responder { pre_remote_push_build_host(); } sub cmd_remote_push_responder { cmd_remote_push_build_host(); } # ... for compatibility with proto vsn.1 dgit (just so that user gets # a good error message) @@ -4327,7 +4841,10 @@ sub i_cleanup { } } -END { i_cleanup(); } +END { + return unless forkcheck_mainprocess(); + i_cleanup(); +} sub i_method { my ($base,$selector,@args) = @_; @@ -4335,8 +4852,10 @@ sub i_method { { no strict qw(refs); &{"${base}_${selector}"}(@args); } } +sub pre_rpush () { + not_necessarily_a_tree(); +} sub cmd_rpush { - pushing(); my $host = nextarg; my $dir; if ($host =~ m/^((?:[^][]|\[[^][]*\])*)\:/) { @@ -4356,6 +4875,8 @@ sub cmd_rpush { my @cmd = (@ssh, $host, shellquote @rdgit); debugcmd "+",@cmd; + $we_are_initiator=1; + if (defined $initiator_tempdir) { rmtree $initiator_tempdir; mkdir $initiator_tempdir, 0700 or die "$initiator_tempdir: $!"; @@ -4369,11 +4890,6 @@ sub cmd_rpush { die "$protovsn ?" unless grep { $_ eq $protovsn } @rpushprotovsn_support; $supplementary_message = '' unless $protovsn >= 3; - fail "rpush negotiated protocol version $protovsn". - " which does not support quilt mode $quilt_mode" - if quiltmode_splitbrain; - - rpush_handle_protovsn_bothends(); for (;;) { my ($icmd,$iargs) = initiator_expect { m/^(\S+)(?: (.*))?$/; @@ -4404,7 +4920,7 @@ sub i_resp_complete { i_cleanup(); printdebug "all done\n"; - exit 0; + finish 0; } sub i_resp_file ($) { @@ -4437,6 +4953,18 @@ our %i_wanted; sub i_resp_want ($) { my ($keyword) = @_; die "$keyword ?" if $i_wanted{$keyword}++; + + defined $i_param{'csuite'} or badproto \*RO, "premature desire, no csuite"; + $isuite = $i_param{'isuite'} // $i_param{'csuite'}; + die unless $isuite =~ m/^$suite_re$/; + + pushing(); + rpush_handle_protovsn_bothends(); + + fail "rpush negotiated protocol version $protovsn". + " which does not support quilt mode $quilt_mode" + if quiltmode_splitbrain; + my @localpaths = i_method "i_want", $keyword; printdebug "[[ $keyword @localpaths\n"; foreach my $localpath (@localpaths) { @@ -4445,7 +4973,7 @@ sub i_resp_want ($) { print RI "files-end\n" or die $!; } -our ($i_clogp, $i_version, $i_dscfn, $i_changesfn); +our ($i_clogp, $i_version, $i_dscfn, $i_changesfn, @i_buildinfos); sub i_localname_parsed_changelog { return "remote-changelog.822"; @@ -4462,6 +4990,31 @@ sub i_localname_dsc { } sub i_file_dsc { } +sub i_localname_buildinfo ($) { + my $bi = $i_param{'buildinfo-filename'}; + defined $bi or badproto \*RO, "buildinfo before filename"; + defined $i_changesfn or badproto \*RO, "buildinfo before changes"; + $bi =~ m{^\Q$package\E_[!-.0-~]*\.buildinfo$}s + or badproto \*RO, "improper buildinfo filename"; + return $&; +} +sub i_file_buildinfo { + my $bi = $i_param{'buildinfo-filename'}; + my $bd = parsecontrol "$i_tmp/$bi", $bi; + my $ch = parsecontrol "$i_tmp/$i_changesfn", 'changes'; + if (!forceing [qw(buildinfo-changes-mismatch)]) { + files_compare_inputs($bd, $ch); + (getfield $bd, $_) eq (getfield $ch, $_) or + fail "buildinfo mismatch $_" + foreach qw(Source Version); + !defined $bd->{$_} or + fail "buildinfo contains $_" + foreach qw(Changes Changed-by Distribution); + } + push @i_buildinfos, $bi; + delete $i_param{'buildinfo-filename'}; +} + sub i_localname_changes { defined $i_dscfn or badproto \*RO, "dsc (before parsed-changelog)"; $i_changesfn = $i_dscfn; @@ -4503,7 +5056,7 @@ sub i_want_signed_tag { sub i_want_signed_dsc_changes { rename "$i_dscfn.tmp","$i_dscfn" or die "$i_dscfn $!"; sign_changes $i_changesfn; - return ($i_dscfn, $i_changesfn); + return ($i_dscfn, $i_changesfn, @i_buildinfos); } #---------- building etc. ---------- @@ -4520,7 +5073,7 @@ sub quiltify_dpkg_commit ($$$;$) { my ($patchname,$author,$msg, $xinfo) = @_; $xinfo //= ''; - mkpath '.git/dgit'; + mkpath '.git/dgit'; # we are in playtree my $descfn = ".git/dgit/quilt-description.tmp"; open O, '>', $descfn or die "$descfn: $!"; $msg =~ s/\n+/\n\n/; @@ -4552,7 +5105,7 @@ sub quiltify_trees_differ ($$;$$$) { # a list of unrepresentable changes (removals of upstream files # (as messages) local $/=undef; - my @cmd = (@git, qw(diff-tree -z)); + my @cmd = (@git, qw(diff-tree -z --no-renames)); push @cmd, qw(--name-only) unless $unrepres; push @cmd, qw(-r) if $finegrained || $unrepres; push @cmd, $x, $y; @@ -4571,16 +5124,23 @@ sub quiltify_trees_differ ($$;$$$) { if ($unrepres) { eval { - die "not a plain file\n" - unless $newmode =~ m/^10\d{4}$/ || - $oldmode =~ m/^10\d{4}$/; + die "not a plain file or symlink\n" + unless $newmode =~ m/^(?:10|12)\d{4}$/ || + $oldmode =~ m/^(?:10|12)\d{4}$/; if ($oldmode =~ m/[^0]/ && $newmode =~ m/[^0]/) { - die "mode changed\n" if $oldmode ne $newmode; + # both old and new files exist + die "mode or type changed\n" if $oldmode ne $newmode; + die "modified symlink\n" unless $newmode =~ m/^10/; + } elsif ($oldmode =~ m/[^0]/) { + # deletion + die "deletion of symlink\n" + unless $oldmode =~ m/^10/; } else { - die "non-default mode\n" - unless $newmode =~ m/^100644$/ || - $oldmode =~ m/^100644$/; + # creation + die "creation with non-default mode\n" + unless $newmode =~ m/^100644$/ or + $newmode =~ m/^120000$/; } }; if ($@) { @@ -4614,13 +5174,15 @@ sub quiltify_splitbrain_needed () { } } -sub quiltify_splitbrain ($$$$$$) { - my ($clogp, $unapplied, $headref, $diffbits, +sub quiltify_splitbrain ($$$$$$$) { + my ($clogp, $unapplied, $headref, $oldtiptree, $diffbits, $editedignores, $cachekey) = @_; + my $gitignore_special = 1; if ($quilt_mode !~ m/gbp|dpm/) { # treat .gitignore just like any other upstream file $diffbits = { %$diffbits }; $_ = !!$_ foreach values %$diffbits; + $gitignore_special = 0; } # We would like any commits we generate to be reproducible my @authline = clogp_authline($clogp); @@ -4631,11 +5193,19 @@ sub quiltify_splitbrain ($$$$$$) { local $ENV{GIT_AUTHOR_EMAIL} = $authline[1]; local $ENV{GIT_AUTHOR_DATE} = $authline[2]; + my $fulldiffhint = sub { + my ($x,$y) = @_; + my $cmd = "git diff $x $y -- :/ ':!debian'"; + $cmd .= " ':!/.gitignore' ':!*/.gitignore'" if $gitignore_special; + return "\nFor full diff showing the problem(s), type:\n $cmd\n"; + }; + if ($quilt_mode =~ m/gbp|unapplied/ && ($diffbits->{O2H} & 01)) { my $msg = "--quilt=$quilt_mode specified, implying patches-unapplied git tree\n". " but git tree differs from orig in upstream files."; + $msg .= $fulldiffhint->($unapplied, 'HEAD'); if (!stat_exists "debian/patches") { $msg .= "\n ... debian/patches is missing; perhaps this is a patch queue branch?"; @@ -4644,7 +5214,7 @@ sub quiltify_splitbrain ($$$$$$) { } if ($quilt_mode =~ m/dpm/ && ($diffbits->{H2A} & 01)) { - fail <($oldtiptree,'HEAD'); --quilt=$quilt_mode specified, implying patches-applied git tree but git tree differs from result of applying debian/patches to upstream END @@ -4660,7 +5230,7 @@ END } if ($quilt_mode =~ m/gbp|dpm/ && ($diffbits->{O2A} & 02)) { - fail <>' + ensuredir "$maindir_gitcommon/logs/refs/dgit-intern"; + my $makelogfh = new IO::File "$maindir_gitcommon/logs/refs/$splitbraincache", '>>' or die $!; my $oldcache = git_get_ref "refs/$splitbraincache"; @@ -4731,7 +5301,7 @@ END runcmd @git, qw(update-ref -m), $cachekey, "refs/$splitbraincache", $dgitview; - changedir '.git/dgit/unpack/work'; + changedir "$playground/work"; my $saved = maybe_split_brain_save $headref, $dgitview, "converted"; progress "dgit view: created ($saved)"; @@ -4810,11 +5380,7 @@ sub quiltify ($$$$) { last; } - if ($quilt_mode eq 'nofix') { - fail "quilt fixup required but quilt mode is \`nofix'\n". - "HEAD commit $c->{Commit} differs from tree implied by ". - " debian/patches (tree object $oldtiptree)"; - } + quiltify_nofix_bail " $c->{Commit}", " (tree object $oldtiptree)"; if ($quilt_mode eq 'smash') { printdebug " search quitting smash\n"; last; @@ -4872,12 +5438,13 @@ sub quiltify ($$$$) { return $s; }; if ($quilt_mode eq 'linear') { - print STDERR "$us: quilt fixup cannot be linear. Stopped at:\n"; + print STDERR "\n$us: error: quilt fixup cannot be linear. Stopped at:\n"; foreach my $notp (@nots) { print STDERR "$us: ", $reportnot->($notp), "\n"; } print STDERR "$us: $_\n" foreach @$failsuggestion; - fail "quilt fixup naive history linearisation failed.\n". + fail + "quilt history linearisation failed. Search \`quilt fixup' in dgit(7).\n". "Use dpkg-source --commit by hand; or, --quilt=smash for one ugly patch"; } elsif ($quilt_mode eq 'smash') { } elsif ($quilt_mode eq 'auto') { @@ -4935,6 +5502,7 @@ sub quiltify ($$$$) { die "contains unexpected slashes\n" if m{//} || m{/$}; die "contains leading punctuation\n" if m{^\W} || m{/\W}; die "contains bad character(s)\n" if m{[^-a-z0-9_.+=~/]}i; + die "is series file\n" if m{$series_filename_re}o; die "too long" if length > 200; }; return $_ unless $@; @@ -4973,6 +5541,7 @@ sub quiltify ($$$$) { $patchname =~ y/-a-z0-9_.+=~//cd; $patchname =~ s/^\W/x-$&/; $patchname = substr($patchname,0,40); + $patchname .= ".patch"; } if (!defined $patchdir) { $patchdir = ''; @@ -5026,9 +5595,36 @@ END my $clogp = parsechangelog(); my $headref = git_rev_parse('HEAD'); + my $symref = git_get_symref(); + + if ($quilt_mode eq 'linear' + && !$fopts->{'single-debian-patch'} + && branch_is_gdr($symref, $headref)) { + # This is much faster. It also makes patches that gdr + # likes better for future updates without laundering. + # + # However, it can fail in some casses where we would + # succeed: if there are existing patches, which correspond + # to a prefix of the branch, but are not in gbp/gdr + # format, gdr will fail (exiting status 7), but we might + # be able to figure out where to start linearising. That + # will be slower so hopefully there's not much to do. + my @cmd = (@git_debrebase, + qw(--noop-ok -funclean-mixed -funclean-ordering + make-patches --quiet-would-amend)); + # We tolerate soe snags that gdr wouldn't, by default. + if (act_local()) { + debugcmd "+",@cmd; + $!=0; $?=-1; + failedcmd @cmd if system @cmd and $?!=7*256; + } else { + dryrun_report @cmd; + } + $headref = git_rev_parse('HEAD'); + } prep_ud(); - changedir $ud; + changedir $playground; my $upstreamversion = upstreamversion $version; @@ -5040,12 +5636,12 @@ END die 'bug' if $split_brain && !$need_split_build_invocation; - changedir '../../../..'; + changedir $maindir; runcmd_ordryrun_local - @git, qw(pull --ff-only -q .git/dgit/unpack/work master); + @git, qw(pull --ff-only -q), "$playground/work", qw(master); } -sub quilt_fixup_mkwork ($) { +sub unpack_playtree_mkwork ($) { my ($headref) = @_; mkdir "work" or die $!; @@ -5054,12 +5650,14 @@ sub quilt_fixup_mkwork ($) { runcmd @git, qw(reset -q --hard), $headref; } -sub quilt_fixup_linkorigs ($$) { +sub unpack_playtree_linkorigs ($$) { my ($upstreamversion, $fn) = @_; # calls $fn->($leafname); - foreach my $f (<../../../../*>) { #/){ - my $b=$f; $b =~ s{.*/}{}; + my $bpd_abs = bpd_abs(); + opendir QFD, $bpd_abs or fail "buildproductsdir: $bpd_abs: $!"; + while ($!=0, defined(my $b = readdir QFD)) { + my $f = bpd_abs()."/".$b; { local ($debuglevel) = $debuglevel-1; printdebug "QF linkorigs $b, $f ?\n"; @@ -5069,6 +5667,8 @@ sub quilt_fixup_linkorigs ($$) { link_ltarget $f, $b or die "$b $!"; $fn->($b); } + die "$buildproductsdir: $!" if $!; + closedir QFD; } sub quilt_fixup_delete_pc () { @@ -5090,8 +5690,8 @@ sub quilt_fixup_singlepatch ($$$) { # get it to generate debian/patches/debian-changes, it is # necessary to build the source package. - quilt_fixup_linkorigs($upstreamversion, sub { }); - quilt_fixup_mkwork($headref); + unpack_playtree_linkorigs($upstreamversion, sub { }); + unpack_playtree_mkwork($headref); rmtree("debian/patches"); @@ -5131,18 +5731,18 @@ END print $fakedsc " ".$md->hexdigest." $size $b\n" or die $!; }; - quilt_fixup_linkorigs($upstreamversion, $dscaddfile); + unpack_playtree_linkorigs($upstreamversion, $dscaddfile); my @files=qw(debian/source/format debian/rules debian/control debian/changelog); foreach my $maybe (qw(debian/patches debian/source/options debian/tests/control)) { - next unless stat_exists "../../../$maybe"; + next unless stat_exists "$maindir/$maybe"; push @files, $maybe; } my $debtar= srcfn $fakeversion,'.debian.tar.gz'; - runcmd qw(env GZIP=-1n tar -zcf), "./$debtar", qw(-C ../../..), @files; + runcmd qw(env GZIP=-1n tar -zcf), "./$debtar", qw(-C), $maindir, @files; $dscaddfile->($debtar); close $fakedsc or die $!; @@ -5151,7 +5751,7 @@ END sub quilt_check_splitbrain_cache ($$) { my ($headref, $upstreamversion) = @_; # Called only if we are in (potentially) split brain mode. - # Called in $ud. + # Called in playground. # Computes the cache key and looks in the cache. # Returns ($dgit_view_commitid, $cachekey) or (undef, $cachekey) @@ -5185,11 +5785,11 @@ sub quilt_check_splitbrain_cache ($$) { debugcmd "|(probably)",@cmd; my $child = open GC, "-|"; defined $child or die $!; if (!$child) { - chdir '../../..' or die $!; - if (!stat ".git/logs/refs/$splitbraincache") { + chdir $maindir or die $!; + if (!stat "$maindir_gitcommon/logs/refs/$splitbraincache") { $! == ENOENT or die $!; printdebug ">(no reflog)\n"; - exit 0; + finish 0; } exec @cmd; die $!; } @@ -5199,7 +5799,7 @@ sub quilt_check_splitbrain_cache ($$) { next unless m/^(\w+) (\S.*\S)$/ && $2 eq $splitbrain_cachekey; my $cachehit = $1; - quilt_fixup_mkwork($headref); + unpack_playtree_mkwork($headref); my $saved = maybe_split_brain_save $headref, $cachehit, "cache-hit"; if ($cachehit ne $headref) { progress "dgit view: found cached ($saved)"; @@ -5315,6 +5915,8 @@ sub quilt_fixup_multipatch ($$$) { rmtree '.pc'; + rmtree 'debian'; # git checkout commitish paths does not delete! + runcmd @git, qw(checkout -f), $headref, qw(-- debian); my $unapplied=git_add_write_tree(); printdebug "fake orig tree object $unapplied\n"; @@ -5328,13 +5930,13 @@ sub quilt_fixup_multipatch ($$$) { failed to apply your git tree's patch stack (from debian/patches/) to the corresponding upstream tarball(s). Your source tree and .orig are probably too inconsistent. dgit can only fix up certain kinds of - anomaly (depending on the quilt mode). See --quilt= in dgit(1). + anomaly (depending on the quilt mode). Please see --quilt= in dgit(1). END } changedir '..'; - quilt_fixup_mkwork($headref); + unpack_playtree_mkwork($headref); my $mustdeletepc=0; if (stat_exists ".pc") { @@ -5402,7 +6004,7 @@ END " --[quilt=]gbp --[quilt=]dpm --quilt=unapplied ?"; if (quiltmode_splitbrain()) { - quiltify_splitbrain($clogp, $unapplied, $headref, + quiltify_splitbrain($clogp, $unapplied, $headref, $oldtiptree, $diffbits, \%editedignores, $splitbrain_cachekey); return; @@ -5440,7 +6042,7 @@ sub quilt_fixup_editor () { } I2->error and die $!; close O or die $1; - exit 0; + finish 0; } sub maybe_apply_patches_dirtily () { @@ -5506,15 +6108,23 @@ sub cmd_clean () { maybe_unapply_patches_again(); } -sub build_prep_early () { - our $build_prep_early_done //= 0; - return if $build_prep_early_done++; - notpushing(); - badusage "-p is not allowed when building" if defined $package; +# return values from massage_dbp_args are one or both of these flags +sub WANTSRC_SOURCE () { 01; } # caller should build source (separately) +sub WANTSRC_BUILDER () { 02; } # caller should run dpkg-buildpackage + +sub build_or_push_prep_early () { + our $build_or_push_prep_early_done //= 0; + return if $build_or_push_prep_early_done++; + badusage "-p is not allowed with dgit $subcommand" if defined $package; my $clogp = parsechangelog(); $isuite = getfield $clogp, 'Distribution'; $package = getfield $clogp, 'Source'; $version = getfield $clogp, 'Version'; +} + +sub build_prep_early () { + build_or_push_prep_early(); + notpushing(); check_not_dirty(); } @@ -5540,13 +6150,21 @@ sub changesopts_initial () { sub changesopts_version () { if (!defined $changes_since_version) { - my @vsns = archive_query('archive_query'); - my @quirk = access_quirk(); - if ($quirk[0] eq 'backports') { - local $isuite = $quirk[2]; - local $csuite; - canonicalise_suite(); - push @vsns, archive_query('archive_query'); + my @vsns; + unless (eval { + @vsns = archive_query('archive_query'); + my @quirk = access_quirk(); + if ($quirk[0] eq 'backports') { + local $isuite = $quirk[2]; + local $csuite; + canonicalise_suite(); + push @vsns, archive_query('archive_query'); + } + 1; + }) { + print STDERR $@; + fail + "archive query failed (queried because --since-version not specified)"; } if (@vsns) { @vsns = map { $_->[0] } @vsns; @@ -5582,16 +6200,11 @@ sub massage_dbp_args ($;$) { # unless we're not doing a split build and want dpkg-source # as cleanmode, in which case we can do nothing # - # return values: - # 0 - source will NOT need to be built separately by caller - # +1 - source will need to be built separately by caller - # +2 - source will need to be built separately by caller AND - # dpkg-buildpackage should not in fact be run at all! debugcmd '#massaging#', @$cmd if $debuglevel>1; #print STDERR "MASS0 ",Dumper($cmd, $xargs, $need_split_build_invocation); if ($cleanmode eq 'dpkg-source' && !$need_split_build_invocation) { $clean_using_builder = 1; - return 0; + return WANTSRC_BUILDER; } # -nc has the side effect of specifying -b if nothing else specified # and some combinations of -S, -b, et al, are errors, rather than @@ -5607,12 +6220,12 @@ sub massage_dbp_args ($;$) { } push @$cmd, '-nc'; #print STDERR "MASS1 ",Dumper($cmd, $xargs, $dmode); - my $r = 0; + my $r = WANTSRC_BUILDER; if ($need_split_build_invocation) { printdebug "massage split $dmode.\n"; - $r = $dmode =~ m/[S]/ ? +2 : - $dmode =~ y/gGF/ABb/ ? +1 : - $dmode =~ m/[ABb]/ ? 0 : + $r = $dmode =~ m/[S]/ ? WANTSRC_SOURCE : + $dmode =~ y/gGF/ABb/ ? WANTSRC_SOURCE | WANTSRC_BUILDER : + $dmode =~ m/[ABb]/ ? WANTSRC_BUILDER : die "$dmode ?"; } printdebug "massage done $r $dmode.\n"; @@ -5621,21 +6234,22 @@ sub massage_dbp_args ($;$) { return $r; } -sub in_parent (&) { +sub in_bpd (&) { my ($fn) = @_; my $wasdir = must_getcwd(); - changedir ".."; + changedir $buildproductsdir; $fn->(); changedir $wasdir; } -sub postbuild_mergechanges ($) { # must run with CWD=.. (eg in in_parent) +# this sub must run with CWD=$buildproductsdir (eg in in_bpd) +sub postbuild_mergechanges ($) { my ($msg_if_onlyone) = @_; # If there is only one .changes file, fail with $msg_if_onlyone, # or if that is undef, be a no-op. # Returns the changes file to report to the user. my $pat = changespat $version; - my @changesfiles = glob $pat; + my @changesfiles = grep { !m/_multi\.changes/ } glob $pat; @changesfiles = sort { ($b =~ m/_source\.changes$/ <=> $a =~ m/_source\.changes$/) or $a cmp $b @@ -5671,8 +6285,11 @@ END sub midbuild_checkchanges () { my $pat = changespat $version; return if $rmchanges; - my @unwanted = map { s#^\.\./##; $_; } glob "../$pat"; - @unwanted = grep { $_ ne changespat $version,'source' } @unwanted; + my @unwanted = map { s#.*/##; $_; } glob "$bpd_glob/$pat"; + @unwanted = grep { + $_ ne changespat $version,'source' and + $_ ne changespat $version,'multi' + } @unwanted; fail < 0) { + if ($wantsrc & WANTSRC_SOURCE) { build_source(); midbuild_checkchanges_vanilla $wantsrc; } else { build_prep(); } - if ($wantsrc < 2) { + if ($wantsrc & WANTSRC_BUILDER) { push @dbp, changesopts_version(); maybe_apply_patches_dirtily(); runcmd_ordryrun_local @dbp; @@ -5730,7 +6347,7 @@ sub cmd_gbp_build { # orig is absent. my $upstreamversion = upstreamversion $version; my $origfnpat = srcfn $upstreamversion, '.orig.tar.*'; - my $gbp_make_orig = $version =~ m/-/ && !(() = glob "../$origfnpat"); + my $gbp_make_orig = $version =~ m/-/ && !(() = glob "$bpd_glob/$origfnpat"); if ($gbp_make_orig) { clean_tree(); @@ -5751,15 +6368,17 @@ sub cmd_gbp_build { } my @cmd = opts_opt_multi_cmd @gbp_build; - push @cmd, (qw(-us -uc --git-no-sign-tags), "--git-builder=@dbp"); + push @cmd, (qw(-us -uc --git-no-sign-tags), + "--git-builder=".(shellquote @dbp)); if ($gbp_make_orig) { - ensuredir '.git/dgit'; - my $ok = '.git/dgit/origs-gen-ok'; + my $priv = dgit_privdir(); + my $ok = "$priv/origs-gen-ok"; unlink $ok or $!==&ENOENT or die $!; my @origs_cmd = @cmd; push @origs_cmd, qw(--git-cleaner=true); - push @origs_cmd, "--git-prebuild=touch $ok .git/dgit/no-such-dir/ok"; + push @origs_cmd, "--git-prebuild=". + "touch ".(shellquote $ok)." ".(shellquote "$priv/no-such-dir/ok"); push @origs_cmd, @ARGV; if (act_local()) { debugcmd @origs_cmd; @@ -5771,7 +6390,7 @@ sub cmd_gbp_build { } } - if ($wantsrc > 0) { + if ($wantsrc & WANTSRC_SOURCE) { build_source(); midbuild_checkchanges_vanilla $wantsrc; } else { @@ -5781,7 +6400,7 @@ sub cmd_gbp_build { build_prep(); } maybe_unapply_patches_again(); - if ($wantsrc < 2) { + if ($wantsrc & WANTSRC_BUILDER) { push @cmd, changesopts(); runcmd_ordryrun_local @cmd, @ARGV; } @@ -5789,65 +6408,48 @@ sub cmd_gbp_build { } sub cmd_git_build { cmd_gbp_build(); } # compatibility with <= 1.0 +sub build_source_for_push { + build_source(); + maybe_unapply_patches_again(); + $changesfile = $sourcechanges; +} + sub build_source { build_prep_early(); - my $our_cleanmode = $cleanmode; - if ($need_split_build_invocation) { - # Pretend that clean is being done some other way. This - # forces us not to try to use dpkg-buildpackage to clean and - # build source all in one go; and instead we run dpkg-source - # (and build_prep() will do the clean since $clean_using_builder - # is false). - $our_cleanmode = 'ELSEWHERE'; - } - if ($our_cleanmode =~ m/^dpkg-source/) { - # dpkg-source invocation (below) will clean, so build_prep shouldn't - $clean_using_builder = 1; - } build_prep(); $sourcechanges = changespat $version,'source'; if (act_local()) { - unlink "../$sourcechanges" or $!==ENOENT + unlink "$buildproductsdir/$sourcechanges" or $!==ENOENT or fail "remove $sourcechanges: $!"; } $dscfn = dscfn($version); - if ($our_cleanmode eq 'dpkg-source') { - maybe_apply_patches_dirtily(); - runcmd_ordryrun_local @dpkgbuildpackage, qw(-us -uc -S), - changesopts(); - } elsif ($our_cleanmode eq 'dpkg-source-d') { - maybe_apply_patches_dirtily(); - runcmd_ordryrun_local @dpkgbuildpackage, qw(-us -uc -S -d), - changesopts(); + my @cmd = (@dpkgsource, qw(-b --)); + if ($split_brain) { + changedir $playground; + runcmd_ordryrun_local @cmd, "work"; + my @udfiles = <${package}_*>; + changedir $maindir; + foreach my $f (@udfiles) { + printdebug "source copy, found $f\n"; + next unless + $f eq $dscfn or + ($f =~ m/\.debian\.tar(?:\.\w+)$/ && + $f eq srcfn($version, $&)); + printdebug "source copy, found $f - renaming\n"; + rename "$playground/$f", "$buildproductsdir/$f" or $!==ENOENT + or fail "put in place new source file ($f): $!"; + } } else { - my @cmd = (@dpkgsource, qw(-b --)); - if ($split_brain) { - changedir $ud; - runcmd_ordryrun_local @cmd, "work"; - my @udfiles = <${package}_*>; - changedir "../../.."; - foreach my $f (@udfiles) { - printdebug "source copy, found $f\n"; - next unless - $f eq $dscfn or - ($f =~ m/\.debian\.tar(?:\.\w+)$/ && - $f eq srcfn($version, $&)); - printdebug "source copy, found $f - renaming\n"; - rename "$ud/$f", "../$f" or $!==ENOENT - or fail "put in place new source file ($f): $!"; - } - } else { - my $pwd = must_getcwd(); - my $leafdir = basename $pwd; - changedir ".."; - runcmd_ordryrun_local @cmd, $leafdir; - changedir $pwd; - } - runcmd_ordryrun_local qw(sh -ec), - 'exec >$1; shift; exec "$@"','x', - "../$sourcechanges", - @dpkggenchanges, qw(-S), changesopts(); + my $pwd = must_getcwd(); + my $leafdir = basename $pwd; + changedir ".."; + runcmd_ordryrun_local @cmd, $leafdir; + changedir $pwd; } + runcmd_ordryrun_local qw(sh -ec), + 'exec >$1; shift; exec "$@"','x', + "$buildproductsdir/$sourcechanges", + @dpkggenchanges, qw(-S), changesopts(); } sub cmd_build_source { @@ -5861,7 +6463,7 @@ sub cmd_build_source { sub cmd_sbuild { build_source(); midbuild_checkchanges(); - in_parent { + in_bpd { if (act_local()) { stat_exists $dscfn or fail "$dscfn (in parent directory): $!"; stat_exists $sourcechanges @@ -5870,7 +6472,7 @@ sub cmd_sbuild { runcmd_ordryrun_local @sbuild, qw(-d), $isuite, @ARGV, $dscfn; }; maybe_unapply_patches_again(); - in_parent { + in_bpd { postbuild_mergechanges(<{Filename}; - my $here = "../$f"; - next if lstat $here; + my $here = "$buildproductsdir/$f"; + if (lstat $here) { + next if stat $here; + fail "lstat $here works but stat gives $! !"; + } fail "stat $here: $!" unless $! == ENOENT; my $there = $dscfn; if ($dscfn =~ m#^(?:\./+)?\.\./+#) { @@ -6000,8 +6623,10 @@ END fail "cannot import $dscfn which seems to be inside working tree!"; } $there =~ s#/+[^/]+$## or - fail "cannot import $dscfn which seems to not have a basename"; + fail "import $dscfn requires ../$f, but it does not exist"; $there .= "/$f"; + my $test = $there =~ m{^/} ? $there : "../$there"; + stat $test or fail "import $dscfn requires $test, but: $!"; symlink $there, $here or fail "symlink $there to $here: $!"; progress "made symlink $here -> $there"; # print STDERR Dumper($fi); @@ -6034,42 +6659,83 @@ END } } - my @cmd = (@git, qw(update-ref -m), "dgit import-dsc: $info", - $dstbranch, $newhash); - runcmd @cmd; - progress "dgit: import-dsc results are in in git ref $dstbranch"; + import_dsc_result $dstbranch, $newhash, + "dgit import-dsc: $info", + "results are in in git ref $dstbranch"; } +sub pre_archive_api_query () { + not_necessarily_a_tree(); +} sub cmd_archive_api_query { badusage "need only 1 subpath argument" unless @ARGV==1; my ($subpath) = @ARGV; + local $isuite = 'DGIT-API-QUERY-CMD'; my @cmd = archive_api_query_cmd($subpath); push @cmd, qw(-f); debugcmd ">",@cmd; exec @cmd or fail "exec curl: $!\n"; } +sub repos_server_url () { + $package = '_dgit-repos-server'; + local $access_forpush = 1; + local $isuite = 'DGIT-REPOS-SERVER'; + my $url = access_giturl(); +} + +sub pre_clone_dgit_repos_server () { + not_necessarily_a_tree(); +} sub cmd_clone_dgit_repos_server { badusage "need destination argument" unless @ARGV==1; my ($destdir) = @ARGV; - $package = '_dgit-repos-server'; - my @cmd = (@git, qw(clone), access_giturl(), $destdir); + my $url = repos_server_url(); + my @cmd = (@git, qw(clone), $url, $destdir); debugcmd ">",@cmd; exec @cmd or fail "exec git clone: $!\n"; } +sub pre_print_dgit_repos_server_source_url () { + not_necessarily_a_tree(); +} +sub cmd_print_dgit_repos_server_source_url { + badusage "no arguments allowed to dgit print-dgit-repos-server-source-url" + if @ARGV; + my $url = repos_server_url(); + print $url, "\n" or die $!; +} + +sub pre_print_dpkg_source_ignores { + not_necessarily_a_tree(); +} +sub cmd_print_dpkg_source_ignores { + badusage "no arguments allowed to dgit print-dpkg-source-ignores" + if @ARGV; + print "@dpkg_source_ignores\n" or die $!; +} + sub cmd_setup_mergechangelogs { badusage "no arguments allowed to dgit setup-mergechangelogs" if @ARGV; + local $isuite = 'DGIT-SETUP-TREE'; setup_mergechangelogs(1); } sub cmd_setup_useremail { - badusage "no arguments allowed to dgit setup-mergechangelogs" if @ARGV; + badusage "no arguments allowed to dgit setup-useremail" if @ARGV; + local $isuite = 'DGIT-SETUP-TREE'; setup_useremail(1); } +sub cmd_setup_gitattributes { + badusage "no arguments allowed to dgit setup-useremail" if @ARGV; + local $isuite = 'DGIT-SETUP-TREE'; + setup_gitattrs(1); +} + sub cmd_setup_new_tree { badusage "no arguments allowed to dgit setup-tree" if @ARGV; + local $isuite = 'DGIT-SETUP-TREE'; setup_new_tree(); } @@ -6077,11 +6743,13 @@ sub cmd_setup_new_tree { sub cmd_version { print "dgit version $our_version\n" or die $!; - exit 0; + finish 0; } our (%valopts_long, %valopts_short); +our (%funcopts_long); our @rvalopts; +our (@modeopt_cfgs); sub defvalopt ($$$$) { my ($long,$short,$val_re,$how) = @_; @@ -6117,6 +6785,26 @@ defvalopt '--initiator-tempdir','','.*', sub { " absolute, not relative, directory." }; +sub defoptmodes ($@) { + my ($varref, $cfgkey, $default, %optmap) = @_; + my %permit; + while (my ($opt,$val) = each %optmap) { + $funcopts_long{$opt} = sub { $$varref = $val; }; + $permit{$val} = $val; + } + push @modeopt_cfgs, { + Var => $varref, + Key => $cfgkey, + Default => $default, + Vals => \%permit + }; +} + +defoptmodes \$dodep14tag, qw( dep14tag want + --dep14tag want + --no-dep14tag no + --always-dep14tag always ); + sub parseopts () { my $om; @@ -6181,30 +6869,24 @@ sub parseopts () { } elsif (m/^--(gbp|dpm)$/s) { push @ropts, "--quilt=$1"; $quilt_mode = $1; - } elsif (m/^--ignore-dirty$/s) { + } elsif (m/^--(?:ignore|include)-dirty$/s) { push @ropts, $_; - $ignoredirty = 1; + $includedirty = 1; } elsif (m/^--no-quilt-fixup$/s) { push @ropts, $_; $quilt_mode = 'nocheck'; } elsif (m/^--no-rm-on-error$/s) { push @ropts, $_; $rmonerror = 0; + } elsif (m/^--no-chase-dsc-distro$/s) { + push @ropts, $_; + $chase_dsc_distro = 0; } elsif (m/^--overwrite$/s) { push @ropts, $_; $overwrite_version = ''; } elsif (m/^--overwrite=(.+)$/s) { push @ropts, $_; $overwrite_version = $1; - } elsif (m/^--dep14tag$/s) { - push @ropts, $_; - $dodep14tag= 'want'; - } elsif (m/^--no-dep14tag$/s) { - push @ropts, $_; - $dodep14tag= 'no'; - } elsif (m/^--always-dep14tag$/s) { - push @ropts, $_; - $dodep14tag= 'always'; } elsif (m/^--delayed=(\d+)$/s) { push @ropts, $_; push @dput, $_; @@ -6235,9 +6917,17 @@ sub parseopts () { # undocumented, for testing push @ropts, $_; $need_split_build_invocation = 1; + } elsif (m/^--config-lookup-explode=(.+)$/s) { + # undocumented, for testing + push @ropts, $_; + $gitcfgs{cmdline}{$1} = 'CONFIG-LOOKUP-EXPLODE'; + # ^ it's supposed to be an array ref } elsif (m/^(--[-0-9a-z]+)(=|$)/ && ($oi = $valopts_long{$1})) { $val = $2 ? $' : undef; #'; $valopt->($oi->{Long}); + } elsif ($funcopts_long{$_}) { + push @ropts, $_; + $funcopts_long{$_}(); } else { badusage "unknown long option \`$_'"; } @@ -6324,6 +7014,10 @@ END sub parseopts_late_defaults () { + $isuite //= cfg("dgit-distro.$idistro.default-suite", 'RETURN-UNDEF') + if defined $idistro; + $isuite //= cfg('dgit.default.default-suite'); + foreach my $k (keys %opts_opt_map) { my $om = $opts_opt_map{$k}; @@ -6366,16 +7060,21 @@ sub parseopts_late_defaults () { $quilt_mode = $1; } - if (!defined $dodep14tag) { + foreach my $moc (@modeopt_cfgs) { local $access_forpush; - $dodep14tag = access_cfg('dep14tag', 'RETURN-UNDEF') // 'want'; - $dodep14tag =~ m/^($dodep14tag_re)$/ - or badcfg "unknown dep14tag setting \`$dodep14tag'"; - $dodep14tag = $1; + my $vr = $moc->{Var}; + next if defined $$vr; + $$vr = access_cfg($moc->{Key}, 'RETURN-UNDEF') // $moc->{Default}; + my $v = $moc->{Vals}{$$vr}; + badcfg "unknown $moc->{Key} setting \`$$vr'" unless defined $v; + $$vr = $v; } $need_split_build_invocation ||= quiltmode_splitbrain(); + fail "dgit: --include-dirty is not supported in split view quilt mode" + if $split_brain && $includedirty; + if (!defined $cleanmode) { local $access_forpush; $cleanmode = access_cfg('clean-mode', 'RETURN-UNDEF'); @@ -6384,6 +7083,11 @@ sub parseopts_late_defaults () { badcfg "unknown clean-mode \`$cleanmode'" unless $cleanmode =~ m/^($cleanmode_re)$(?!\n)/s; } + + $buildproductsdir //= access_cfg('build-products-dir', 'RETURN-UNDEF'); + $buildproductsdir //= '..'; + $bpd_glob = $buildproductsdir; + $bpd_glob =~ s#[][\\{}*?~]#\\$&#g; } if ($ENV{$fakeeditorenv}) { @@ -6393,21 +7097,25 @@ if ($ENV{$fakeeditorenv}) { parseopts(); check_env_sanity(); -git_slurp_config(); print STDERR "DRY RUN ONLY\n" if $dryrun_level > 1; print STDERR "DAMP RUN - WILL MAKE LOCAL (UNSIGNED) CHANGES\n" if $dryrun_level == 1; if (!@ARGV) { print STDERR $helpmsg or die $!; - exit 8; + finish 8; } -my $cmd = shift @ARGV; +$cmd = $subcommand = shift @ARGV; $cmd =~ y/-/_/; my $pre_fn = ${*::}{"pre_$cmd"}; $pre_fn->() if $pre_fn; +record_maindir if $invoked_in_git_tree; +git_slurp_config(); + my $fn = ${*::}{"cmd_$cmd"}; $fn or badusage "unknown operation $cmd"; $fn->(); + +finish 0;