X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=dgit.git;a=blobdiff_plain;f=dgit;h=d2f2c0fbf5e7a3b9f35980d9b08754bf5a66b636;hp=1bde3279b359c333cf7942fdaf15ccc1ef475836;hb=a0ebfd60790ce326fbde74744939588ee6f1d7d7;hpb=e23c0f9c068457c33e869f720d30e3c105976dba diff --git a/dgit b/dgit index 1bde3279..d2f2c0fb 100755 --- a/dgit +++ b/dgit @@ -2,8 +2,8 @@ # dgit # Integration between git and Debian-style archives # -# Copyright (C)2013-2017 Ian Jackson -# Copyright (C)2017 Sean Whitton +# Copyright (C)2013-2018 Ian Jackson +# Copyright (C)2017-2018 Sean Whitton # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -66,7 +66,7 @@ our $changesfile; our $buildproductsdir; our $bpd_glob; our $new_package = 0; -our $ignoredirty = 0; +our $includedirty = 0; our $rmonerror = 1; our @deliberatelies; our %previously; @@ -78,7 +78,7 @@ our $overwrite_version; # undef: not specified; '': check changelog our $quilt_mode; our $quilt_modes_re = 'linear|smash|auto|nofix|nocheck|gbp|dpm|unapplied'; our $dodep14tag; -our $split_brain_save; +our %internal_object_save; our $we_are_responder; our $we_are_initiator; our $initiator_tempdir; @@ -100,9 +100,6 @@ our %format_ok = map { $_=>1 } ("1.0","3.0 (native)","3.0 (quilt)"); our $suite_re = '[-+.0-9a-z]+'; our $cleanmode_re = 'dpkg-source(?:-d)?|git|git-ff|check|none'; -our $orig_f_comp_re = qr{orig(?:-$extra_orig_namepart_re)?}; -our $orig_f_sig_re = '\\.(?:asc|gpg|pgp)'; -our $orig_f_tail_re = "$orig_f_comp_re\\.tar(?:\\.\\w+)?(?:$orig_f_sig_re)?"; our $git_authline_re = '^([^<>]+) \<(\S+)\> (\d+ [-+]\d+)$'; our $splitbraincache = 'dgit-intern/quilt-cache'; @@ -129,6 +126,8 @@ our (@mergechanges) = qw(mergechanges -f); our (@gbp_build) = (''); our (@gbp_pq) = ('gbp pq'); our (@changesopts) = (''); +our (@pbuilder) = ("sudo -E pbuilder"); +our (@cowbuilder) = ("sudo -E cowbuilder"); our %opts_opt_map = ('dget' => \@dget, # accept for compatibility 'curl' => \@curl, @@ -148,7 +147,9 @@ our %opts_opt_map = ('dget' => \@dget, # accept for compatibility 'gbp-build' => \@gbp_build, 'gbp-pq' => \@gbp_pq, 'ch' => \@changesopts, - 'mergechanges' => \@mergechanges); + 'mergechanges' => \@mergechanges, + 'pbuilder' => \@pbuilder, + 'cowbuilder' => \@cowbuilder); our %opts_opt_cmdonly = ('gpg' => 1, 'git' => 1); our %opts_cfg_insertpos = map { @@ -166,7 +167,6 @@ our $keyid; autoflush STDOUT 1; our $supplementary_message = ''; -our $need_split_build_invocation = 0; our $split_brain = 0; END { @@ -198,15 +198,13 @@ sub lref () { return "refs/heads/".lbranch(); } sub lrref () { return "refs/remotes/$remotename/".server_branch($csuite); } sub rrref () { return server_ref($csuite); } -sub stripepoch ($) { - my ($vsn) = @_; - $vsn =~ s/^\d+\://; - return $vsn; -} - sub srcfn ($$) { - my ($vsn,$sfx) = @_; - return "${package}_".(stripepoch $vsn).$sfx + my ($vsn, $sfx) = @_; + return &source_file_leafname($package, $vsn, $sfx); +} +sub is_orig_file_of_vsn ($$) { + my ($f, $upstreamvsn) = @_; + return is_orig_file_of_p_v($f, $package, $upstreamvsn); } sub dscfn ($) { @@ -219,12 +217,6 @@ sub changespat ($;$) { return "${package}_".(stripepoch $vsn)."_".($arch//'*').".changes"; } -sub upstreamversion ($) { - my ($vsn) = @_; - $vsn =~ s/-[^-]+$//; - return $vsn; -} - our $us = 'dgit'; initdebug(''); @@ -275,14 +267,16 @@ sub quiltmode_splitbrain () { } sub opts_opt_multi_cmd { + my $extra = shift; my @cmd; push @cmd, split /\s+/, shift @_; + push @cmd, @$extra; push @cmd, @_; @cmd; } sub gbp_pq { - return opts_opt_multi_cmd @gbp_pq; + return opts_opt_multi_cmd [], @gbp_pq; } sub dgit_privdir () { @@ -292,6 +286,7 @@ sub dgit_privdir () { sub bpd_abs () { my $r = $buildproductsdir; $r = "$maindir/$r" unless $r =~ m{^/}; + return $r; } sub branch_gdr_info ($$) { @@ -569,6 +564,7 @@ main usages: dgit [dgit-opts] fetch|pull [dgit-opts] [suite] dgit [dgit-opts] build [dpkg-buildpackage-opts] dgit [dgit-opts] sbuild [sbuild-opts] + dgit [dgit-opts] pbuilder|cowbuilder [debbuildopts] dgit [dgit-opts] push [dgit-opts] [suite] dgit [dgit-opts] push-source [dgit-opts] [suite] dgit [dgit-opts] rpush build-host:build-dir ... @@ -817,7 +813,8 @@ sub access_forpush () { } sub pushing () { - die "$access_forpush ?" if ($access_forpush // 1) ne 1; + confess 'internal error '.Dumper($access_forpush)," ?" if + defined $access_forpush and !$access_forpush; badcfg "pushing but distro is configured readonly" if access_forpush_config() eq '0'; $access_forpush = 1; @@ -1081,7 +1078,7 @@ sub canonicalise_suite_ftpmasterapi { } qw(codename name); push @matched, $entry; } - fail "unknown suite $isuite" unless @matched; + fail "unknown suite $isuite, maybe -d would help" unless @matched; my $cn; eval { @matched==1 or die "multiple matches for suite $isuite\n"; @@ -1859,13 +1856,6 @@ sub is_orig_file_in_dsc ($$) { return 1; } -sub is_orig_file_of_vsn ($$) { - my ($f, $upstreamvsn) = @_; - my $base = srcfn $upstreamvsn, ''; - return 0 unless $f =~ m/^\Q$base\E\.$orig_f_tail_re$/; - return 1; -} - # This function determines whether a .changes file is source-only from # the point of view of dak. Thus, it permits *_source.buildinfo # files. @@ -3538,7 +3528,6 @@ sub fork_for_multisuite ($) { fetch_one(); finish 0; }); - # xxx collecte the ref here $csubsuite =~ s/^\Q$cbasesuite\E-/-/; push @csuites, $csubsuite; @@ -3736,7 +3725,7 @@ sub check_not_dirty () { } } - return if $ignoredirty; + return if $includedirty; git_check_unmodified(); } @@ -3829,13 +3818,14 @@ sub madformat_wantfixup ($) { sub maybe_split_brain_save ($$$) { my ($headref, $dgitview, $msg) = @_; # => message fragment "$saved" describing disposition of $dgitview - return "commit id $dgitview" unless defined $split_brain_save; + my $save = $internal_object_save{'dgit-view'}; + return "commit id $dgitview" unless defined $save; my @cmd = (shell_cmd 'cd "$1"; shift', $maindir, git_update_ref_cmd "dgit --dgit-view-save $msg HEAD=$headref", - $split_brain_save, $dgitview); + $save, $dgitview); runcmd @cmd; - return "and left in $split_brain_save"; + return "and left in $save"; } # An "infopair" is a tuple [ $thing, $what ] @@ -4432,7 +4422,8 @@ END responder_send_command("param isuite $isuite"); responder_send_command("param tagformat $tagformat"); if (defined $maintviewhead) { - die unless ($protovsn//4) >= 4; + confess "internal error (protovsn=$protovsn)" + if defined $protovsn and $protovsn < 4; responder_send_command("param maint-view $maintviewhead"); } @@ -4755,21 +4746,6 @@ sub cmd_push { dopush(); } -sub cmd_push_source { - prep_push(); - if ($changesfile) { - my $changes = parsecontrol("$buildproductsdir/$changesfile", - "source changes file"); - unless (test_source_only_changes($changes)) { - fail "user-specified changes file is not source-only"; - } - } else { - # Building a source package is very fast, so just do it - build_source_for_push(); - } - dopush(); -} - #---------- remote commands' implementation ---------- sub pre_remote_push_build_host { @@ -5631,14 +5607,12 @@ END quilt_fixup_multipatch($clogp, $headref, $upstreamversion); } - die 'bug' if $split_brain && !$need_split_build_invocation; - changedir $maindir; runcmd_ordryrun_local @git, qw(pull --ff-only -q), "$playground/work", qw(master); } -sub quilt_fixup_mkwork ($) { +sub unpack_playtree_mkwork ($) { my ($headref) = @_; mkdir "work" or die $!; @@ -5647,11 +5621,12 @@ sub quilt_fixup_mkwork ($) { runcmd @git, qw(reset -q --hard), $headref; } -sub quilt_fixup_linkorigs ($$) { +sub unpack_playtree_linkorigs ($$) { my ($upstreamversion, $fn) = @_; # calls $fn->($leafname); - opendir QFD, bpd_abs(); + my $bpd_abs = bpd_abs(); + opendir QFD, $bpd_abs or fail "buildproductsdir: $bpd_abs: $!"; while ($!=0, defined(my $b = readdir QFD)) { my $f = bpd_abs()."/".$b; { @@ -5686,8 +5661,8 @@ sub quilt_fixup_singlepatch ($$$) { # get it to generate debian/patches/debian-changes, it is # necessary to build the source package. - quilt_fixup_linkorigs($upstreamversion, sub { }); - quilt_fixup_mkwork($headref); + unpack_playtree_linkorigs($upstreamversion, sub { }); + unpack_playtree_mkwork($headref); rmtree("debian/patches"); @@ -5727,7 +5702,7 @@ END print $fakedsc " ".$md->hexdigest." $size $b\n" or die $!; }; - quilt_fixup_linkorigs($upstreamversion, $dscaddfile); + unpack_playtree_linkorigs($upstreamversion, $dscaddfile); my @files=qw(debian/source/format debian/rules debian/control debian/changelog); @@ -5744,6 +5719,31 @@ END close $fakedsc or die $!; } +sub quilt_fakedsc2unapplied ($$) { + my ($headref, $upstreamversion) = @_; + # must be run in the playground + # quilt_make_fake_dsc must have been called + + runcmd qw(sh -ec), + 'exec dpkg-source --no-check --skip-patches -x fake.dsc >/dev/null'; + + my $fakexdir= $package.'-'.(stripepoch $upstreamversion); + rename $fakexdir, "fake" or die "$fakexdir $!"; + + changedir 'fake'; + + remove_stray_gits("source package"); + mktree_in_ud_here(); + + rmtree '.pc'; + + rmtree 'debian'; # git checkout commitish paths does not delete! + runcmd @git, qw(checkout -f), $headref, qw(-- debian); + my $unapplied=git_add_write_tree(); + printdebug "fake orig tree object $unapplied\n"; + return $unapplied; +} + sub quilt_check_splitbrain_cache ($$) { my ($headref, $upstreamversion) = @_; # Called only if we are in (potentially) split brain mode. @@ -5795,7 +5795,7 @@ sub quilt_check_splitbrain_cache ($$) { next unless m/^(\w+) (\S.*\S)$/ && $2 eq $splitbrain_cachekey; my $cachehit = $1; - quilt_fixup_mkwork($headref); + unpack_playtree_mkwork($headref); my $saved = maybe_split_brain_save $headref, $cachehit, "cache-hit"; if ($cachehit ne $headref) { progress "dgit view: found cached ($saved)"; @@ -5897,24 +5897,7 @@ sub quilt_fixup_multipatch ($$$) { quilt_check_splitbrain_cache($headref, $upstreamversion); return if $cachehit; } - - runcmd qw(sh -ec), - 'exec dpkg-source --no-check --skip-patches -x fake.dsc >/dev/null'; - - my $fakexdir= $package.'-'.(stripepoch $upstreamversion); - rename $fakexdir, "fake" or die "$fakexdir $!"; - - changedir 'fake'; - - remove_stray_gits("source package"); - mktree_in_ud_here(); - - rmtree '.pc'; - - rmtree 'debian'; # git checkout commitish paths does not delete! - runcmd @git, qw(checkout -f), $headref, qw(-- debian); - my $unapplied=git_add_write_tree(); - printdebug "fake orig tree object $unapplied\n"; + my $unapplied=quilt_fakedsc2unapplied($headref, $upstreamversion); ensuredir '.pc'; @@ -5932,7 +5915,7 @@ END changedir '..'; - quilt_fixup_mkwork($headref); + unpack_playtree_mkwork($headref); my $mustdeletepc=0; if (stat_exists ".pc") { @@ -6104,6 +6087,10 @@ sub cmd_clean () { maybe_unapply_patches_again(); } +# return values from massage_dbp_args are one or both of these flags +sub WANTSRC_SOURCE () { 01; } # caller should build source (separately) +sub WANTSRC_BUILDER () { 02; } # caller should run dpkg-buildpackage + sub build_or_push_prep_early () { our $build_or_push_prep_early_done //= 0; return if $build_or_push_prep_early_done++; @@ -6112,6 +6099,7 @@ sub build_or_push_prep_early () { $isuite = getfield $clogp, 'Distribution'; $package = getfield $clogp, 'Source'; $version = getfield $clogp, 'Version'; + $dscfn = dscfn($version); } sub build_prep_early () { @@ -6120,9 +6108,12 @@ sub build_prep_early () { check_not_dirty(); } -sub build_prep () { +sub build_prep ($) { + my ($wantsrc) = @_; build_prep_early(); - clean_tree(); + # clean the tree if we're trying to include dirty changes in the + # source package, or we are running the builder in $maindir + clean_tree() if $includedirty || ($wantsrc & WANTSRC_BUILDER); build_maybe_quilt_fixup(); if ($rmchanges) { my $pat = changespat $version; @@ -6181,28 +6172,11 @@ sub changesopts () { sub massage_dbp_args ($;$) { my ($cmd,$xargs) = @_; - # We need to: - # - # - if we're going to split the source build out so we can - # do strange things to it, massage the arguments to dpkg-buildpackage - # so that the main build doessn't build source (or add an argument - # to stop it building source by default). - # - # - add -nc to stop dpkg-source cleaning the source tree, - # unless we're not doing a split build and want dpkg-source - # as cleanmode, in which case we can do nothing - # - # return values: - # 0 - source will NOT need to be built separately by caller - # +1 - source will need to be built separately by caller - # +2 - source will need to be built separately by caller AND - # dpkg-buildpackage should not in fact be run at all! + # Since we split the source build out so we can do strange things + # to it, massage the arguments to dpkg-buildpackage so that the + # main build doessn't build source (or add an argument to stop it + # building source by default). debugcmd '#massaging#', @$cmd if $debuglevel>1; -#print STDERR "MASS0 ",Dumper($cmd, $xargs, $need_split_build_invocation); - if ($cleanmode eq 'dpkg-source' && !$need_split_build_invocation) { - $clean_using_builder = 1; - return 0; - } # -nc has the side effect of specifying -b if nothing else specified # and some combinations of -S, -b, et al, are errors, rather than # later simply overriding earlie. So we need to: @@ -6217,14 +6191,12 @@ sub massage_dbp_args ($;$) { } push @$cmd, '-nc'; #print STDERR "MASS1 ",Dumper($cmd, $xargs, $dmode); - my $r = 0; - if ($need_split_build_invocation) { - printdebug "massage split $dmode.\n"; - $r = $dmode =~ m/[S]/ ? +2 : - $dmode =~ y/gGF/ABb/ ? +1 : - $dmode =~ m/[ABb]/ ? 0 : - die "$dmode ?"; - } + my $r = WANTSRC_BUILDER; + printdebug "massage split $dmode.\n"; + $r = $dmode =~ m/[S]/ ? WANTSRC_SOURCE : + $dmode =~ y/gGF/ABb/ ? WANTSRC_SOURCE | WANTSRC_BUILDER : + $dmode =~ m/[ABb]/ ? WANTSRC_BUILDER : + die "$dmode ?"; printdebug "massage done $r $dmode.\n"; push @$cmd, $dmode; #print STDERR "MASS2 ",Dumper($cmd, $xargs, $r); @@ -6246,7 +6218,7 @@ sub postbuild_mergechanges ($) { # or if that is undef, be a no-op. # Returns the changes file to report to the user. my $pat = changespat $version; - my @changesfiles = glob $pat; + my @changesfiles = grep { !m/_multi\.changes/ } glob $pat; @changesfiles = sort { ($b =~ m/_source\.changes$/ <=> $a =~ m/_source\.changes$/) or $a cmp $b @@ -6283,7 +6255,10 @@ sub midbuild_checkchanges () { my $pat = changespat $version; return if $rmchanges; my @unwanted = map { s#.*/##; $_; } glob "$bpd_glob/$pat"; - @unwanted = grep { $_ ne changespat $version,'source' } @unwanted; + @unwanted = grep { + $_ ne changespat $version,'source' and + $_ ne changespat $version,'multi' + } @unwanted; fail < 0) { + build_prep($wantsrc); + if ($wantsrc & WANTSRC_SOURCE) { build_source(); midbuild_checkchanges_vanilla $wantsrc; - } else { - build_prep(); } - if ($wantsrc < 2) { + if ($wantsrc & WANTSRC_BUILDER) { push @dbp, changesopts_version(); maybe_apply_patches_dirtily(); runcmd_ordryrun_local @dbp; @@ -6346,7 +6324,6 @@ sub cmd_gbp_build { if ($gbp_make_orig) { clean_tree(); $cleanmode = 'none'; # don't do it again - $need_split_build_invocation = 1; } my @dbp = @dpkgbuildpackage; @@ -6360,7 +6337,7 @@ sub cmd_gbp_build { $gbp_build[0] = 'gbp buildpackage'; } } - my @cmd = opts_opt_multi_cmd @gbp_build; + my @cmd = opts_opt_multi_cmd [], @gbp_build; push @cmd, (qw(-us -uc --git-no-sign-tags), "--git-builder=".(shellquote @dbp)); @@ -6384,17 +6361,17 @@ sub cmd_gbp_build { } } - if ($wantsrc > 0) { + build_prep($wantsrc); + if ($wantsrc & WANTSRC_SOURCE) { build_source(); midbuild_checkchanges_vanilla $wantsrc; } else { if (!$clean_using_builder) { push @cmd, '--git-cleaner=true'; } - build_prep(); } maybe_unapply_patches_again(); - if ($wantsrc < 2) { + if ($wantsrc & WANTSRC_BUILDER) { push @cmd, changesopts(); runcmd_ordryrun_local @cmd, @ARGV; } @@ -6402,77 +6379,158 @@ sub cmd_gbp_build { } sub cmd_git_build { cmd_gbp_build(); } # compatibility with <= 1.0 -sub build_source_for_push { - build_source(); - maybe_unapply_patches_again(); - $changesfile = $sourcechanges; +sub building_source_in_playtree { + # If $includedirty, we have to build the source package from the + # working tree, not a playtree, so that uncommitted changes are + # included (copying or hardlinking them into the playtree could + # cause trouble). + # + # Note that if we are building a source package in split brain + # mode we do not support including uncommitted changes, because + # that makes quilt fixup too hard. I.e. ($split_brain && (dgit is + # building a source package)) => !$includedirty + return !$includedirty; } sub build_source { - build_prep_early(); - build_prep(); $sourcechanges = changespat $version,'source'; if (act_local()) { unlink "$buildproductsdir/$sourcechanges" or $!==ENOENT or fail "remove $sourcechanges: $!"; } - $dscfn = dscfn($version); my @cmd = (@dpkgsource, qw(-b --)); - if ($split_brain) { + my $leafdir; + if (building_source_in_playtree()) { + $leafdir = 'work'; + my $headref = git_rev_parse('HEAD'); + # If we are in split brain, there is already a playtree with + # the thing we should package into a .dsc (thanks to quilt + # fixup). If not, make a playtree + prep_ud() unless $split_brain; changedir $playground; - runcmd_ordryrun_local @cmd, "work"; - my @udfiles = <${package}_*>; - changedir $maindir; - foreach my $f (@udfiles) { - printdebug "source copy, found $f\n"; - next unless - $f eq $dscfn or - ($f =~ m/\.debian\.tar(?:\.\w+)$/ && - $f eq srcfn($version, $&)); - printdebug "source copy, found $f - renaming\n"; - rename "$playground/$f", "$buildproductsdir/$f" or $!==ENOENT - or fail "put in place new source file ($f): $!"; + unless ($split_brain) { + my $upstreamversion = upstreamversion $version; + unpack_playtree_linkorigs($upstreamversion, sub { }); + unpack_playtree_mkwork($headref); + changedir '..'; } } else { - my $pwd = must_getcwd(); - my $leafdir = basename $pwd; - changedir ".."; - runcmd_ordryrun_local @cmd, $leafdir; - changedir $pwd; + $leafdir = basename $maindir; + changedir '..'; } + runcmd_ordryrun_local @cmd, $leafdir; + + changedir $leafdir; runcmd_ordryrun_local qw(sh -ec), - 'exec >$1; shift; exec "$@"','x', - "$buildproductsdir/$sourcechanges", + 'exec >../$1; shift; exec "$@"','x', $sourcechanges, @dpkggenchanges, qw(-S), changesopts(); + changedir '..'; + + printdebug "moving $dscfn, $sourcechanges, etc. to ".bpd_abs()."\n"; + $dsc = parsecontrol($dscfn, "source package"); + + my $mv = sub { + my ($why, $l) = @_; + printdebug " renaming ($why) $l\n"; + rename "$l", bpd_abs()."/$l" + or fail "put in place new built file ($l): $!"; + }; + foreach my $l (split /\n/, getfield $dsc, 'Files') { + $l =~ m/\S+$/ or next; + $mv->('Files', $&); + } + $mv->('dsc', $dscfn); + $mv->('changes', $sourcechanges); + + changedir $maindir; } sub cmd_build_source { - build_prep_early(); badusage "build-source takes no additional arguments" if @ARGV; + build_prep(WANTSRC_SOURCE); build_source(); maybe_unapply_patches_again(); printdone "source built, results in $dscfn and $sourcechanges"; } -sub cmd_sbuild { +sub cmd_push_source { + prep_push(); + fail "dgit push-source: --include-dirty/--ignore-dirty does not make". + "sense with push-source!" if $includedirty; + build_maybe_quilt_fixup(); + if ($changesfile) { + my $changes = parsecontrol("$buildproductsdir/$changesfile", + "source changes file"); + unless (test_source_only_changes($changes)) { + fail "user-specified changes file is not source-only"; + } + } else { + # Building a source package is very fast, so just do it + build_source(); + die "er, patches are applied dirtily but shouldn't be.." + if $patches_applied_dirtily; + $changesfile = $sourcechanges; + } + dopush(); +} + +sub binary_builder { + my ($bbuilder, $pbmc_msg, @args) = @_; + build_prep(WANTSRC_SOURCE); build_source(); midbuild_checkchanges(); in_bpd { if (act_local()) { - stat_exists $dscfn or fail "$dscfn (in parent directory): $!"; + stat_exists $dscfn or fail "$dscfn (in build products dir): $!"; stat_exists $sourcechanges - or fail "$sourcechanges (in parent directory): $!"; + or fail "$sourcechanges (in build products dir): $!"; } - runcmd_ordryrun_local @sbuild, qw(-d), $isuite, @ARGV, $dscfn; + runcmd_ordryrun_local @$bbuilder, @args; }; maybe_unapply_patches_again(); in_bpd { - postbuild_mergechanges(<ismember($signum) and die "$signame is blocked\n"; } @@ -7064,7 +7139,8 @@ sub parseopts_late_defaults () { $$vr = $v; } - $need_split_build_invocation ||= quiltmode_splitbrain(); + fail "dgit: --include-dirty is not supported in split view quilt mode" + if $split_brain && $includedirty; if (!defined $cleanmode) { local $access_forpush;