X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=dgit.git;a=blobdiff_plain;f=dgit;h=a568c579a00041813506a2425617304431126481;hp=4c80280523bd9f3506220622d61eef2a847a9cdb;hb=f260757c3cd326d1a424f862c0e97a1940dea2aa;hpb=7e481fcf4e2802d1743717a77270a01c76456559 diff --git a/dgit b/dgit index 4c802805..a568c579 100755 --- a/dgit +++ b/dgit @@ -2,7 +2,7 @@ # dgit # Integration between git and Debian-style archives # -# Copyright (C)2013-2015 Ian Jackson +# Copyright (C)2013-2016 Ian Jackson # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -97,7 +97,8 @@ our (@dpkgbuildpackage) = qw(dpkg-buildpackage -i\.git/ -I.git); our (@dpkgsource) = qw(dpkg-source -i\.git/ -I.git); our (@dpkggenchanges) = qw(dpkg-genchanges); our (@mergechanges) = qw(mergechanges -f); -our (@gbp) = qw(gbp); +our (@gbp_build) = (''); +our (@gbp_pq) = ('gbp pq'); our (@changesopts) = (''); our %opts_opt_map = ('dget' => \@dget, # accept for compatibility @@ -112,7 +113,8 @@ our %opts_opt_map = ('dget' => \@dget, # accept for compatibility 'dpkg-source' => \@dpkgsource, 'dpkg-buildpackage' => \@dpkgbuildpackage, 'dpkg-genchanges' => \@dpkggenchanges, - 'gbp' => \@gbp, + 'gbp-build' => \@gbp_build, + 'gbp-pq' => \@gbp_pq, 'ch' => \@changesopts, 'mergechanges' => \@mergechanges); @@ -246,6 +248,17 @@ sub quiltmode_splitbrain () { $quilt_mode =~ m/gbp|dpm|unapplied/; } +sub opts_opt_multi_cmd { + my @cmd; + push @cmd, split /\s+/, shift @_; + push @cmd, @_; + @cmd; +} + +sub gbp_pq { + return opts_opt_multi_cmd @gbp_pq; +} + #---------- remote protocol support, common ---------- # remote push initiator/responder protocol: @@ -587,36 +600,49 @@ our %defcfg = ('dgit.default.distro' => 'debian', 'dgit-distro.test-dummy.upload-host' => 'test-dummy', ); -our %gitcfg; +our %gitcfgs; +our @gitcfgsources = qw(cmdline local global system); sub git_slurp_config () { local ($debuglevel) = $debuglevel-2; local $/="\0"; - my @cmd = (@git, qw(config -z --get-regexp .*)); - debugcmd "|",@cmd; + # This algoritm is a bit subtle, but this is needed so that for + # options which we want to be single-valued, we allow the + # different config sources to override properly. See #835858. + foreach my $src (@gitcfgsources) { + next if $src eq 'cmdline'; + # we do this ourselves since git doesn't handle it + + my @cmd = (@git, qw(config -z --get-regexp), "--$src", qw(.*)); + debugcmd "|",@cmd; - open GITS, "-|", @cmd or die $!; - while () { - chomp or die; - printdebug "=> ", (messagequote $_), "\n"; - m/\n/ or die "$_ ?"; - push @{ $gitcfg{$`} }, $'; #'; + open GITS, "-|", @cmd or die $!; + while () { + chomp or die; + printdebug "=> ", (messagequote $_), "\n"; + m/\n/ or die "$_ ?"; + push @{ $gitcfgs{$src}{$`} }, $'; #'; + } + $!=0; $?=0; + close GITS + or ($!==0 && $?==256) + or failedcmd @cmd; } - $!=0; $?=0; - close GITS - or ($!==0 && $?==256) - or failedcmd @cmd; } sub git_get_config ($) { my ($c) = @_; - my $l = $gitcfg{$c}; - printdebug"C $c ".(defined $l ? messagequote "'$l'" : "undef")."\n" - if $debuglevel >= 4; - $l or return undef; - @$l==1 or badcfg "multiple values for $c" if @$l > 1; - return $l->[0]; + foreach my $src (@gitcfgsources) { + my $l = $gitcfgs{$src}{$c}; + printdebug"C $c ".(defined $l ? messagequote "'$l'" : "undef")."\n" + if $debuglevel >= 4; + $l or next; + @$l==1 or badcfg "multiple values for $c". + " (in $src git config)" if @$l > 1; + return $l->[0]; + } + return undef; } sub cfg { @@ -1401,10 +1427,13 @@ sub mktree_in_ud_from_only_subdir (;$) { return ($tree,$dir); } +our @files_csum_info_fields = + (['Checksums-Sha256','Digest::SHA', 'new(256)'], + ['Checksums-Sha1', 'Digest::SHA', 'new(1)'], + ['Files', 'Digest::MD5', 'new()']); + sub dsc_files_info () { - foreach my $csumi (['Checksums-Sha256','Digest::SHA', 'new(256)'], - ['Checksums-Sha1', 'Digest::SHA', 'new(1)'], - ['Files', 'Digest::MD5', 'new()']) { + foreach my $csumi (@files_csum_info_fields) { my ($fname, $module, $method) = @$csumi; my $field = $dsc->{$fname}; next unless defined $field; @@ -1432,6 +1461,65 @@ sub dsc_files () { map { $_->{Filename} } dsc_files_info(); } +sub files_compare_inputs (@) { + my $inputs = \@_; + my %record; + my %fchecked; + + my $showinputs = sub { + return join "; ", map { $_->get_option('name') } @$inputs; + }; + + foreach my $in (@$inputs) { + my $expected_files; + my $in_name = $in->get_option('name'); + + printdebug "files_compare_inputs $in_name\n"; + + foreach my $csumi (@files_csum_info_fields) { + my ($fname) = @$csumi; + printdebug "files_compare_inputs $in_name $fname\n"; + + my $field = $in->{$fname}; + next unless defined $field; + + my @files; + foreach (split /\n/, $field) { + next unless m/\S/; + + my ($info, $f) = m/^(\w+ \d+) (?:\S+ \S+ )?(\S+)$/ or + fail "could not parse $in_name $fname line \`$_'"; + + printdebug "files_compare_inputs $in_name $fname $f\n"; + + push @files, $f; + + my $re = \ $record{$f}{$fname}; + if (defined $$re) { + $fchecked{$f}{$in_name} = 1; + $$re eq $info or + fail "hash or size of $f varies in $fname fields". + " (between: ".$showinputs->().")"; + } else { + $$re = $info; + } + } + @files = sort @files; + $expected_files //= \@files; + "@$expected_files" eq "@files" or + fail "file list in $in_name varies between hash fields!"; + } + $expected_files or + fail "$in_name has no files list field(s)"; + } + printdebug "files_compare_inputs ".Dumper(\%fchecked, \%record) + if $debuglevel>=2; + + grep { keys %$_ == @$inputs-1 } values %fchecked + or fail "no file appears in all file lists". + " (looked in: ".$showinputs->().")"; +} + sub is_orig_file_in_dsc ($$) { my ($f, $dsc_files_info) = @_; return 0 if @$dsc_files_info <= 1; @@ -1553,6 +1641,7 @@ sub check_for_vendor_patches () { sub generate_commits_from_dsc () { # See big comment in fetch_from_archive, below. + # See also README.dsc-import. prep_ud(); changedir $ud; @@ -1757,7 +1846,7 @@ sub generate_commits_from_dsc () { printdebug "import clog $r1clogp->{version} becomes r1\n"; } die $! if CLOGS->error; - close CLOGS or $?==(SIGPIPE<<8) or failedcmd @clogcmd; + close CLOGS or $?==SIGPIPE or failedcmd @clogcmd; $clogp or fail "package changelog has no entries!"; @@ -1833,7 +1922,25 @@ END runcmd @git, qw(checkout -q -b dapplied), $dappliedcommit; runcmd @git, qw(checkout -q -b unpa), $rawimport_hash; - runcmd shell_cmd 'exec >/dev/null', @gbp, qw(pq import); + + # We need the answers to be reproducible + my @authline = clogp_authline($clogp); + local $ENV{GIT_COMMITTER_NAME} = $authline[0]; + local $ENV{GIT_COMMITTER_EMAIL} = $authline[1]; + local $ENV{GIT_COMMITTER_DATE} = $authline[2]; + local $ENV{GIT_AUTHOR_NAME} = $authline[0]; + local $ENV{GIT_AUTHOR_EMAIL} = $authline[1]; + local $ENV{GIT_AUTHOR_DATE} = $authline[2]; + + eval { + runcmd shell_cmd 'exec >/dev/null 2>../../gbp-pq-output', + gbp_pq, qw(import); + }; + if ($@) { + { local $@; eval { runcmd qw(cat ../../gbp-pq-output); }; } + die $@; + } + my $gapplied = git_rev_parse('HEAD'); my $gappliedtree = cmdoutput @git, qw(rev-parse HEAD:); $gappliedtree eq $dappliedtree or @@ -1944,7 +2051,7 @@ sub git_fetch_us () { push @specs, qw(heads/*) if deliberately_not_fast_forward; # This is rather miserable: - # When git-fetch --prune is passed a fetchspec ending with a *, + # When git fetch --prune is passed a fetchspec ending with a *, # it does a plausible thing. If there is no * then: # - it matches subpaths too, even if the supplied refspec # starts refs, and behaves completely madly if the source @@ -1954,15 +2061,15 @@ sub git_fetch_us () { # We want to fetch a fixed ref, and we don't know in advance # if it exists, so this is not suitable. # - # Our workaround is to use git-ls-remote. git-ls-remote has its + # Our workaround is to use git ls-remote. git ls-remote has its # own qairks. Notably, it has the absurd multi-tail-matching - # behaviour: git-ls-remote R refs/foo can report refs/foo AND + # behaviour: git ls-remote R refs/foo can report refs/foo AND # refs/refs/foo etc. # # Also, we want an idempotent snapshot, but we have to make two - # calls to the remote: one to git-ls-remote and to git-fetch. The - # solution is use git-ls-remote to obtain a target state, and - # git-fetch to try to generate it. If we don't manage to generate + # calls to the remote: one to git ls-remote and to git fetch. The + # solution is use git ls-remote to obtain a target state, and + # git fetch to try to generate it. If we don't manage to generate # the target state, we try again. my $specre = join '|', map { @@ -1996,7 +2103,7 @@ sub git_fetch_us () { my ($objid,$rrefname) = ($1,$2); if (!$wanted_rref->($rrefname)) { print STDERR <($rrefname)) { printdebug <('name', 'DEBFULLNAME'); } +sub ensure_setup_existing_tree () { + my $k = "remote.$remotename.skipdefaultupdate"; + my $c = git_get_config $k; + return if defined $c; + set_local_git_config $k, 'true'; +} + sub setup_new_tree () { setup_mergechangelogs(); setup_useremail(); @@ -2598,7 +2714,11 @@ sub commit_quilty_patch () { } my @adds = map { s/[][*?\\]/\\$&/g; $_; } sort keys %adds; runcmd_ordryrun_local @git, qw(add -f), @adds; - commit_admin "Commit Debian 3.0 (quilt) metadata"; + commit_admin <{format}); changedir '../../../..'; - my $diffopt = $debuglevel>0 ? '--exit-code' : '--quiet'; - my @diffcmd = (@git, qw(diff), $diffopt, $tree, $dgithead); + my @diffcmd = (@git, qw(diff --quiet), $tree, $dgithead); debugcmd "+",@diffcmd; $!=0; $?=-1; my $r = system @diffcmd; if ($r) { if ($r==256) { - fail "$dscfn specifies a different tree to your HEAD commit;". - " perhaps you forgot to build". - ($diffopt eq '--exit-code' ? "" : - " (run with -D to see full diff output)"); + my $diffs = cmdoutput @git, qw(diff --stat), $tree, $dgithead; + fail <{Tag}"; } - runcmd_ordryrun @git, qw(push),access_giturl(), @pushrefs; + runcmd_ordryrun @git, + qw(-c push.followTags=false push), access_giturl(), @pushrefs; runcmd_ordryrun @git, qw(update-ref -m), 'dgit push', lrref(), $dgithead; supplementary_message(<<'END'); @@ -3256,7 +3384,7 @@ sub cmd_clone { } if (stat $dstdir) { rmtree($dstdir) or die "remove $dstdir: $!\n"; - } elsif (!grep { $! == $_ } + } elsif (grep { $! == $_ } (ENOENT, ENOTDIR, EACCES, EPERM, ELOOP)) { } else { print STDERR "check whether to remove $dstdir: $!\n"; @@ -3605,13 +3733,10 @@ sub quiltify_dpkg_commit ($$$;$) { mkpath '.git/dgit'; my $descfn = ".git/dgit/quilt-description.tmp"; open O, '>', $descfn or die "$descfn: $!"; - $msg =~ s/\s+$//g; - $msg =~ s/\n/\n /g; - $msg =~ s/^\s+$/ ./mg; + $msg =~ s/\n+/\n\n/; print O <{$fn} # is set for each modified .gitignore filename $fn + # if $unrepres is defined, array ref to which is appeneded + # a list of unrepresentable changes (removals of upstream files + # (as messages) local $/=undef; - my @cmd = (@git, qw(diff-tree --name-only -z)); - push @cmd, qw(-r) if $finegrained; + my @cmd = (@git, qw(diff-tree -z)); + push @cmd, qw(--name-only) unless $unrepres; + push @cmd, qw(-r) if $finegrained || $unrepres; push @cmd, $x, $y; my $diffs= cmdoutput @cmd; my $r = 0; + my @lmodes; foreach my $f (split /\0/, $diffs) { + if ($unrepres && !@lmodes) { + @lmodes = $f =~ m/^\:(\w+) (\w+) \w+ \w+ / or die "$_ ?"; + next; + } + my ($oldmode,$newmode) = @lmodes; + @lmodes = (); + next if $f =~ m#^debian(?:/.*)?$#s; + + if ($unrepres) { + eval { + die "deleted\n" unless $newmode =~ m/[^0]/; + die "not a plain file\n" unless $newmode =~ m/^10\d{4}$/; + if ($oldmode =~ m/[^0]/) { + die "mode changed\n" if $oldmode ne $newmode; + } else { + die "non-default mode\n" unless $newmode =~ m/^100644$/; + } + }; + if ($@) { + local $/="\n"; chomp $@; + push @$unrepres, [ $f, $@ ]; + } + } + my $isignore = $f =~ m#^(?:.*/)?.gitignore$#s; $r |= $isignore ? 02 : 01; $ignorenamesr->{$f}=1 if $ignorenamesr && $isignore; @@ -3679,9 +3833,12 @@ sub quiltify_splitbrain ($$$$$$) { local $ENV{GIT_COMMITTER_NAME} = $authline[0]; local $ENV{GIT_COMMITTER_EMAIL} = $authline[1]; local $ENV{GIT_COMMITTER_DATE} = $authline[2]; - + local $ENV{GIT_AUTHOR_NAME} = $authline[0]; + local $ENV{GIT_AUTHOR_EMAIL} = $authline[1]; + local $ENV{GIT_AUTHOR_DATE} = $authline[2]; + if ($quilt_mode =~ m/gbp|unapplied/ && - ($diffbits->{H2O} & 01)) { + ($diffbits->{O2H} & 01)) { my $msg = "--quilt=$quilt_mode specified, implying patches-unapplied git tree\n". " but git tree differs from orig in upstream files."; @@ -3702,7 +3859,7 @@ END ($diffbits->{O2A} & 01)) { # some patches quiltify_splitbrain_needed(); progress "dgit view: creating patches-applied version using gbp pq"; - runcmd shell_cmd 'exec >/dev/null', @gbp, qw(pq import); + runcmd shell_cmd 'exec >/dev/null', gbp_pq, qw(import); # gbp pq import creates a fresh branch; push back to dgit-view runcmd @git, qw(update-ref refs/heads/dgit-view HEAD); runcmd @git, qw(checkout -q dgit-view); @@ -3715,7 +3872,7 @@ END .gitignores: but, such patches exist in debian/patches. END } - if (($diffbits->{H2O} & 02) && # user has modified .gitignore + if (($diffbits->{O2H} & 02) && # user has modified .gitignore !($diffbits->{O2A} & 02)) { # patches do not change .gitignore quiltify_splitbrain_needed(); progress "dgit view: creating patch to represent .gitignore changes"; @@ -3732,7 +3889,7 @@ The Debian packaging git branch contains these updates to the upstream .gitignore file(s). This patch is autogenerated, to provide these updates to users of the official Debian archive view of the package. -[dgit version $our_version] +[dgit ($our_version) update-gitignore] --- END close GIPATCH or die "$gipatch: $!"; @@ -3746,7 +3903,11 @@ END print SERIES "auto-gitignore\n" or die $!; close SERIES or die $!; runcmd @git, qw(add -- debian/patches/series), $gipatch; - commit_admin "Commit patch to update .gitignore"; + commit_admin <(); + my $title = $1; - my $patchname = $title; - $patchname =~ s/[.:]$//; - $patchname =~ y/ A-Z/-a-z/; - $patchname =~ y/-a-z0-9_.+=~//cd; - $patchname =~ s/^\W/x-$&/; - $patchname = substr($patchname,0,40); + my $patchname; + my $patchdir; + + my $gbp_check_suitable = sub { + $_ = shift; + my ($what) = @_; + + eval { + die "contains unexpected slashes\n" if m{//} || m{/$}; + die "contains leading punctuation\n" if m{^\W} || m{/\W}; + die "contains bad character(s)\n" if m{[^-a-z0-9_.+=~/]}i; + die "too long" if length > 200; + }; + return $_ unless $@; + print STDERR "quiltifying commit $cc:". + " ignoring/dropping Gbp-Pq $what: $@"; + return undef; + }; + + if ($msg =~ s/^ (?: gbp(?:-pq)? : \s* name \s+ | + gbp-pq-name: \s* ) + (\S+) \s* \n //ixm) { + $patchname = $gbp_check_suitable->($1, 'Name'); + } + if ($msg =~ s/^ (?: gbp(?:-pq)? : \s* topic \s+ | + gbp-pq-topic: \s* ) + (\S+) \s* \n //ixm) { + $patchdir = $gbp_check_suitable->($1, 'Topic'); + } + + $strip_nls->(); + + if (!defined $patchname) { + $patchname = $title; + $patchname =~ s/[.:]$//; + use Text::Iconv; + eval { + my $converter = new Text::Iconv qw(UTF-8 ASCII//TRANSLIT); + my $translitname = $converter->convert($patchname); + die unless defined $translitname; + $patchname = $translitname; + }; + print STDERR + "dgit: patch title transliteration error: $@" + if $@; + $patchname =~ y/ A-Z/-a-z/; + $patchname =~ y/-a-z0-9_.+=~//cd; + $patchname =~ s/^\W/x-$&/; + $patchname = substr($patchname,0,40); + } + if (!defined $patchdir) { + $patchdir = ''; + } + if (length $patchdir) { + $patchname = "$patchdir/$patchname"; + } + if ($patchname =~ m{^(.*)/}) { + mkpath "debian/patches/$1"; + } + my $index; for ($index=''; stat "debian/patches/$patchname$index"; @@ -3967,6 +4188,7 @@ sub quiltify ($$$$) { runcmd @git, qw(checkout -q), $target, qw(debian/changelog); quiltify_dpkg_commit "$patchname$index", $author, $msg, + "Date: $commitdate\n". "X-Dgit-Generated: $clogp->{Version} $cc\n"; runcmd @git, qw(checkout -q), $cc, qw(debian/changelog); @@ -4042,7 +4264,11 @@ sub quilt_fixup_linkorigs ($$) { sub quilt_fixup_delete_pc () { runcmd @git, qw(rm -rqf .pc); - commit_admin "Commit removal of .pc (quilt series tracking data)"; + commit_admin </dev/null'; + my @bbcmd = (qw(sh -ec), 'exec dpkg-source --before-build . >/dev/null'); + $!=0; $?=-1; + if (system @bbcmd) { + failedcmd @bbcmd if $? < 0; + fail < quiltify_trees_differ($headref, $unapplied, 1,\%editedignores), + O2H => quiltify_trees_differ($unapplied,$headref, 1, + \%editedignores, \@unrepres), H2A => quiltify_trees_differ($headref, $oldtiptree,1), O2A => quiltify_trees_differ($unapplied,$oldtiptree,1), }; my @dl; foreach my $b (qw(01 02)) { - foreach my $v (qw(H2O O2A H2A)) { + foreach my $v (qw(O2H O2A H2A)) { push @dl, ($diffbits->{$v} & $b) ? '##' : '=='; } } printdebug "differences \@dl @dl.\n"; progress sprintf +"$us: base trees orig=%.20s o+d/p=%.20s", + $unapplied, $oldtiptree; + progress sprintf "$us: quilt differences: src: %s orig %s gitignores: %s orig %s\n". "$us: quilt differences: HEAD %s o+d/p HEAD %s o+d/p", $dl[0], $dl[1], $dl[3], $dl[4], $dl[2], $dl[5]; + if (@unrepres) { + print STDERR "dgit: cannot represent change: $_->[1]: $_->[0]\n" + foreach @unrepres; + fail <{H2O} & $diffbits->{O2A})) { + if (!($diffbits->{O2H} & $diffbits->{O2A})) { push @failsuggestion, "This might be a patches-unapplied branch."; } elsif (!($diffbits->{H2A} & $diffbits->{O2A})) { push @failsuggestion, "This might be a patches-applied branch."; @@ -4576,17 +4824,24 @@ sub cmd_build { printdone "build successful\n"; } +sub pre_gbp_build { + $quilt_mode //= 'gbp'; +} + sub cmd_gbp_build { my @dbp = @dpkgbuildpackage; my $wantsrc = massage_dbp_args \@dbp, \@ARGV; - my @cmd; - if (length executable_on_path('git-buildpackage')) { - @cmd = qw(git-buildpackage); - } else { - @cmd = qw(gbp buildpackage); + if (!length $gbp_build[0]) { + if (length executable_on_path('git-buildpackage')) { + $gbp_build[0] = qw(git-buildpackage); + } else { + $gbp_build[0] = 'gbp buildpackage'; + } } + my @cmd = opts_opt_multi_cmd @gbp_build; + push @cmd, (qw(-us -uc --git-no-sign-tags), "--git-builder=@dbp"); if ($wantsrc > 0) { @@ -4683,9 +4938,10 @@ sub cmd_sbuild { if (!$rmchanges) { my @unwanted = map { s#^\.\./##; $_; } glob "../$pat"; @unwanted = grep { $_ ne changespat $version,'source' } @unwanted; - fail "changes files other than source matching $pat". - " already present (@unwanted);". - " building would result in ambiguity about the intended results" + fail < $a =~ m/_source\.changes$/) or $a cmp $b } @changesfiles; + fail <= 4; - next unless $vl; + next unless @vl; badcfg "cannot configure options for $k" if $opts_opt_cmdonly{$k}; my $insertpos = $opts_cfg_insertpos{$k}; @$om = ( @$om[0..$insertpos-1], - @$vl, + @vl, @$om[$insertpos..$#$om] ); } } @@ -5003,6 +5270,9 @@ if (!@ARGV) { my $cmd = shift @ARGV; $cmd =~ y/-/_/; +my $pre_fn = ${*::}{"pre_$cmd"}; +$pre_fn->() if $pre_fn; + if (!defined $rmchanges) { local $access_forpush; $rmchanges = access_cfg_bool(0, 'rm-old-changes');