From: Ian Jackson Date: Sun, 30 Oct 2016 23:56:29 +0000 (+0000) Subject: Merge branch 'wip.tutorials' into wip X-Git-Tag: archive/debian/2.9~12 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=dgit.git;a=commitdiff_plain;h=cb809d942c58dea5dfba6ad8b7e3298338ebb24e;hp=6e7c6b8b8305f162cea9e3da85c24b3f21888a1a Merge branch 'wip.tutorials' into wip --- diff --git a/debian/changelog b/debian/changelog index dac3dbe4..a67a2051 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,31 @@ dgit (2.9~) unstable; urgency=low + * Split brain mode: Fix --new. Closes:#842354. + * During push, automatically calculate which .origs are required, + so user never needs [--ch:]-sa or [--ch:]-sd. Closes:#829116. + * Properly look for .origs etc. in .., fetching them less often. + Closes:#842386. + * New import-dsc feature. + * Test suite: Explicitly configure user.name and user.email, so + that tests work when environment doesn't have defaults. + Closes:#842279 (I hope). + * New option --dgit-view-save= for split view quilt modes. + In particular, means that the output of a split view quilt-fixup + is left somewhere useful. + * dgit clone: Set timestamps in cloned tree to a single unified time. + This makes it less likely that the user will trip over any + timestamp-dependent FTBFS bugs (eg #842452). + * Support dgit --delayed= push (with a warning in the manpage + about possible skew). + * dgit gbp-build will arrange to let gbp buildpackage generate + .orig tarballs if it seems applicable. Closes:#841094. + * Reject `dgit pull' in split view quilt modes, to avoid + creating unfortunate wreckage on non-dgit-view branches. + Closes:#842608. + * Cope when cloning suite which doesn't receive uploads, + like testing. Closes:#842621. + * Properly fetch all archive dgit view tags, as we intended. + * Actually provide a -p (--package=) option (!) * dgit-*(7). Many new tutorial manpages, several written and many improved by Sean Whitton. * dgit(7): Substantial updates, including documenting split view. diff --git a/debian/tests/control b/debian/tests/control index b4c278db..a9f66649 100644 --- a/debian/tests/control +++ b/debian/tests/control @@ -21,7 +21,7 @@ Tests-Directory: tests/tests Depends: dgit, dgit-infrastructure, devscripts, debhelper (>=8) Restrictions: x-dgit-git-only -Tests: absurd-gitapply build-modes build-modes-asplit build-modes-gbp-asplit clone-clogsigpipe clone-gitnosuite clone-nogit debpolicy-dbretry debpolicy-newreject debpolicy-quilt-gbp distropatches-reject drs-clone-nogit drs-push-masterupdate drs-push-rejects dsd-clone-nogit dsd-divert fetch-localgitonly fetch-somegit-notlast gitconfig import-native import-nonnative inarchivecopy mismatches-contents mismatches-dscchanges newtag-clone-nogit oldnewtagalt oldtag-clone-nogit overwrite-chkclog overwrite-junk overwrite-splitbrains overwrite-version push-buildproductsdir push-newpackage push-nextdgit quilt quilt-gbp quilt-gbp-build-modes quilt-singlepatch quilt-splitbrains rpush tag-updates test-list-uptodate trustingpolicy-replay unrepresentable version-opt +Tests: absurd-gitapply build-modes build-modes-asplit build-modes-gbp-asplit clone-clogsigpipe clone-gitnosuite clone-nogit debpolicy-dbretry debpolicy-newreject debpolicy-quilt-gbp distropatches-reject drs-clone-nogit drs-push-masterupdate drs-push-rejects dsd-clone-nogit dsd-divert fetch-localgitonly fetch-somegit-notlast gbp-orig gitconfig import-dsc import-native import-nonnative inarchivecopy mismatches-contents mismatches-dscchanges newtag-clone-nogit oldnewtagalt oldtag-clone-nogit orig-include-exclude orig-include-exclude-chkquery overwrite-chkclog overwrite-junk overwrite-splitbrains overwrite-version push-buildproductsdir push-newpackage push-nextdgit quilt quilt-gbp quilt-gbp-build-modes quilt-singlepatch quilt-splitbrains rpush tag-updates test-list-uptodate trustingpolicy-replay unrepresentable version-opt Tests-Directory: tests/tests Depends: dgit, dgit-infrastructure, devscripts, debhelper (>=8) diff --git a/dgit b/dgit index 7508fe15..8264f3e5 100755 --- a/dgit +++ b/dgit @@ -67,6 +67,7 @@ our $rmchanges; our $overwrite_version; # undef: not specified; '': check changelog our $quilt_mode; our $quilt_modes_re = 'linear|smash|auto|nofix|nocheck|gbp|dpm|unapplied'; +our $split_brain_save; our $we_are_responder; our $initiator_tempdir; our $patches_applied_dirtily = 00; @@ -76,9 +77,10 @@ our $tagformatfn; our %forceopts = map { $_=>0 } qw(unrepresentable unsupported-source-format - dsc-changes-mismatch + dsc-changes-mismatch changes-origs-exactly import-gitapply-absurd - import-gitapply-no-absurd); + import-gitapply-no-absurd + import-dsc-with-dgit-field); our %format_ok = map { $_=>1 } ("1.0","3.0 (native)","3.0 (quilt)"); @@ -220,6 +222,12 @@ sub changespat ($;$) { return "${package}_".(stripepoch $vsn)."_".($arch//'*').".changes"; } +sub upstreamversion ($) { + my ($vsn) = @_; + $vsn =~ s/-[^-]+$//; + return $vsn; +} + our $us = 'dgit'; initdebug(''); @@ -619,7 +627,7 @@ our %defcfg = ('dgit.default.distro' => 'debian', 'dgit-distro.test-dummy.git-url' => "$td/git", 'dgit-distro.test-dummy.git-host' => "git", 'dgit-distro.test-dummy.git-path' => "$td/git", - 'dgit-distro.test-dummy.archive-query' => "ftpmasterapi:", + 'dgit-distro.test-dummy.archive-query' => "dummycatapi:", 'dgit-distro.test-dummy.archive-query-url' => "file://$td/aq/", 'dgit-distro.test-dummy.mirror' => "file://$td/mirror/", 'dgit-distro.test-dummy.upload-host' => 'test-dummy', @@ -960,15 +968,22 @@ sub must_getcwd () { return $d; } +sub parse_dscdata () { + my $dscfh = new IO::File \$dscdata, '<' or die $!; + printdebug Dumper($dscdata) if $debuglevel>1; + $dsc = parsecontrolfh($dscfh,$dscurl,1); + printdebug Dumper($dsc) if $debuglevel>1; +} + our %rmad; -sub archive_query ($) { - my ($method) = @_; +sub archive_query ($;@) { + my ($method) = shift @_; my $query = access_cfg('archive-query','RETURN-UNDEF'); $query =~ s/^(\w+):// or badcfg "invalid archive-query method \`$query'"; my $proto = $1; my $data = $'; #'; - { no strict qw(refs); &{"${method}_${proto}"}($proto,$data); } + { no strict qw(refs); &{"${method}_${proto}"}($proto,$data,@_); } } sub pool_dsc_subpath ($$) { @@ -1009,9 +1024,9 @@ sub archive_api_query_cmd ($) { return @cmd; } -sub api_query ($$) { +sub api_query ($$;$) { use JSON; - my ($data, $subpath) = @_; + my ($data, $subpath, $ok404) = @_; badcfg "ftpmasterapi archive query method takes no data part" if length $data; my @cmd = archive_api_query_cmd($subpath); @@ -1023,12 +1038,13 @@ sub api_query ($$) { fail "curl failed to print 3-digit HTTP code"; } my $code = $&; + return undef if $code eq '404' && $ok404; fail "fetch of $url gave HTTP code $code" unless $url =~ m#^file://# or $code =~ m/^2/; return decode_json($json); } -sub canonicalise_suite_ftpmasterapi () { +sub canonicalise_suite_ftpmasterapi { my ($proto,$data) = @_; my $suites = api_query($data, 'suites'); my @matched; @@ -1052,7 +1068,7 @@ sub canonicalise_suite_ftpmasterapi () { return $cn; } -sub archive_query_ftpmasterapi () { +sub archive_query_ftpmasterapi { my ($proto,$data) = @_; my $info = api_query($data, "dsc_in_suite/$isuite/$package"); my @rows; @@ -1079,6 +1095,42 @@ sub archive_query_ftpmasterapi () { return @rows; } +sub file_in_archive_ftpmasterapi { + my ($proto,$data,$filename) = @_; + my $pat = $filename; + $pat =~ s/_/\\_/g; + $pat = "%/$pat"; + $pat =~ s#[^-+_.0-9a-z/]# sprintf '%%%02x', ord $& #ge; + my $info = api_query($data, "file_in_archive/$pat", 1); +} + +#---------- `dummyapicat' archive query method ---------- + +sub archive_query_dummycatapi { archive_query_ftpmasterapi @_; } +sub canonicalise_suite_dummycatapi { canonicalise_suite_ftpmasterapi @_; } + +sub file_in_archive_dummycatapi ($$$) { + my ($proto,$data,$filename) = @_; + my $mirror = access_cfg('mirror'); + $mirror =~ s#^file://#/# or die "$mirror ?"; + my @out; + my @cmd = (qw(sh -ec), ' + cd "$1" + find -name "$2" -print0 | + xargs -0r sha256sum + ', qw(x), $mirror, $filename); + debugcmd "-|", @cmd; + open FIA, "-|", @cmd or die $!; + while () { + chomp or die; + printdebug "| $_\n"; + m/^(\w+) (\S+)$/ or die "$_ ?"; + push @out, { sha256sum => $1, filename => $2 }; + } + close FIA or die failedcmd @cmd; + return \@out; +} + #---------- `madison' archive query method ---------- sub archive_query_madison { @@ -1127,6 +1179,8 @@ sub canonicalise_suite_madison { return $r[0][2]; } +sub file_in_archive_madison { return undef; } + #---------- `sshpsql' archive query method ---------- sub sshpsql ($$$) { @@ -1202,6 +1256,8 @@ END return $rows[0]; } +sub file_in_archive_sshpsql ($$$) { return undef; } + #---------- `dummycat' archive query method ---------- sub canonicalise_suite_dummycat ($$) { @@ -1243,6 +1299,8 @@ sub archive_query_dummycat ($$) { return sort { -version_compare($a->[0],$b->[0]); } @rows; } +sub file_in_archive_dummycat () { return undef; } + #---------- tag format handling ---------- sub access_cfg_tagformats () { @@ -1315,10 +1373,7 @@ sub get_archive_dsc () { fail "$dscurl has hash $got but". " archive told us to expect $digest"; } - my $dscfh = new IO::File \$dscdata, '<' or die $!; - printdebug Dumper($dscdata) if $debuglevel>1; - $dsc = parsecontrolfh($dscfh,$dscurl,1); - printdebug Dumper($dsc) if $debuglevel>1; + parse_dscdata(); my $fmt = getfield $dsc, 'Format'; $format_ok{$fmt} or forceable_fail [qw(unsupported-source-format)], "unsupported source format $fmt, sorry"; @@ -1464,9 +1519,9 @@ sub mktree_in_ud_from_only_subdir (;$) { } our @files_csum_info_fields = - (['Checksums-Sha256','Digest::SHA', 'new(256)'], - ['Checksums-Sha1', 'Digest::SHA', 'new(1)'], - ['Files', 'Digest::MD5', 'new()']); + (['Checksums-Sha256','Digest::SHA', 'new(256)', 'sha256sum'], + ['Checksums-Sha1', 'Digest::SHA', 'new(1)', 'sha1sum'], + ['Files', 'Digest::MD5', 'new()', 'md5sum']); sub dsc_files_info () { foreach my $csumi (@files_csum_info_fields) { @@ -1572,6 +1627,101 @@ sub is_orig_file_of_vsn ($$) { return 1; } +sub changes_update_origs_from_dsc ($$$$) { + my ($dsc, $changes, $upstreamvsn, $changesfile) = @_; + my %changes_f; + printdebug "checking origs needed ($upstreamvsn)...\n"; + $_ = getfield $changes, 'Files'; + m/^\w+ \d+ (\S+ \S+) \S+$/m or + fail "cannot find section/priority from .changes Files field"; + my $placementinfo = $1; + my %changed; + printdebug "checking origs needed placement '$placementinfo'...\n"; + foreach my $l (split /\n/, getfield $dsc, 'Files') { + $l =~ m/\S+$/ or next; + my $file = $&; + printdebug "origs $file | $l\n"; + next unless is_orig_file_of_vsn $file, $upstreamvsn; + printdebug "origs $file is_orig\n"; + my $have = archive_query('file_in_archive', $file); + if (!defined $have) { + print STDERR <{$archivefield}; + $_ = $dsc->{$fname}; + next unless defined; + m/^(\w+) .* \Q$file\E$/m or + fail ".dsc $fname missing entry for $file"; + if ($h->{$archivefield} eq $1) { + $same++; + } else { + push @differ, + "$archivefield: $h->{$archivefield} (archive) != $1 (local .dsc)"; + } + } + die "$file ".Dumper($h)." ?!" if $same && @differ; + $found_same++ + if $same; + push @found_differ, "archive $h->{filename}: ".join "; ", @differ + if @differ; + } + print "origs $file f.same=$found_same #f._differ=$#found_differ\n"; + if (@found_differ && !$found_same) { + fail join "\n", + "archive contains $file with different checksum", + @found_differ; + } + # Now we edit the changes file to add or remove it + foreach my $csumi (@files_csum_info_fields) { + my ($fname, $module, $method, $archivefield) = @$csumi; + next unless defined $changes->{$fname}; + if ($found_same) { + # in archive, delete from .changes if it's there + $changed{$file} = "removed" if + $changes->{$fname} =~ s/^.* \Q$file\E$(?:)\n//m; + } elsif ($changes->{$fname} =~ m/^.* \Q$file\E$(?:)\n/m) { + # not in archive, but it's here in the .changes + } else { + my $dsc_data = getfield $dsc, $fname; + $dsc_data =~ m/^(.* \Q$file\E$)\n/m or die "$dsc_data $file ?"; + my $extra = $1; + $extra =~ s/ \d+ /$&$placementinfo / + or die "$fname $extra >$dsc_data< ?" + if $fname eq 'Files'; + $changes->{$fname} .= "\n". $extra; + $changed{$file} = "added"; + } + } + } + if (%changed) { + foreach my $file (keys %changed) { + progress sprintf + "edited .changes for archive .orig contents: %s %s", + $changed{$file}, $file; + } + my $chtmp = "$changesfile.tmp"; + $changes->save($chtmp); + if (act_local()) { + rename $chtmp,$changesfile or die "$changesfile $!"; + } else { + progress "[new .changes left in $changesfile]"; + } + } else { + progress "$changesfile already has appropriate .orig(s) (if any)"; + } +} + sub make_commit ($) { my ($file) = @_; return cmdoutput @git, qw(hash-object -w -t commit), $file; @@ -1686,10 +1836,15 @@ sub generate_commits_from_dsc () { my $f = $fi->{Filename}; die "$f ?" if $f =~ m#/|^\.|\.dsc$|\.tmp$#; - link_ltarget "../../../$f", $f + printdebug "considering linking $f: "; + + link_ltarget "../../../../$f", $f + or ((printdebug "($!) "), 0) or $!==&ENOENT or die "$f $!"; + printdebug "linked.\n"; + complete_file_from_dsc('.', $fi) or next; @@ -1706,8 +1861,7 @@ sub generate_commits_from_dsc () { # from the debian/changelog, so we record the tree objects now and # make them into commits later. my @tartrees; - my $upstreamv = $dsc->{version}; - $upstreamv =~ s/-[^-]+$//; + my $upstreamv = upstreamversion $dsc->{version}; my $orig_f_base = srcfn $upstreamv, ''; foreach my $fi (@dfi) { @@ -2068,6 +2222,7 @@ sub complete_file_from_dsc ($$) { if (stat_exists $tf) { progress "using existing $f"; } else { + printdebug "$tf does not exist, need to fetch\n"; my $furl = $dscurl; $furl =~ s{/[^/]+$}{}; $furl .= "/$f"; @@ -2137,6 +2292,8 @@ sub git_fetch_us () { # git fetch to try to generate it. If we don't manage to generate # the target state, we try again. + printdebug "git_fetch_us specs @specs\n"; + my $specre = join '|', map { my $x = $_; $x =~ s/\W/\\$&/g; @@ -2152,6 +2309,7 @@ sub git_fetch_us () { my $fetch_iteration = 0; FETCH_ITERATION: for (;;) { + printdebug "git_fetch_us iteration $fetch_iteration\n"; if (++$fetch_iteration > 10) { fail "too many iterations trying to get sane fetch!"; } @@ -2179,10 +2337,12 @@ END # OK, now %want is exactly what we want for refs in @specs my @fspecs = map { - return () if !m/\*$/ && !exists $wantr{"refs/$_"}; + !m/\*$/ && !exists $wantr{"refs/$_"} ? () : "+refs/$_:".lrfetchrefs."/$_"; } @specs; + printdebug "git_fetch_us fspecs @fspecs\n"; + my @fcmd = (@git, qw(fetch -p -n -q), access_giturl(), @fspecs); runcmd_ordryrun_local @git, qw(fetch -p -n -q), access_giturl(), @fspecs; @@ -2416,11 +2576,8 @@ sub fetch_from_archive () { }; if (defined $dsc_hash) { - fail "missing remote git history even though dsc has hash -". - " could not find ref ".rref()." at ".access_giturl() - unless $lastpush_hash; ensure_we_have_orig(); - if ($dsc_hash eq $lastpush_hash) { + if (!$lastpush_hash || $dsc_hash eq $lastpush_hash) { @mergeinputs = $dsc_mergeinput } elsif (is_fast_fwd($dsc_hash,$lastpush_hash)) { print STDERR <($lastpush_hash, 'dgit repo server tip (last push)'); + $chkff->($lastpush_hash, 'dgit repo server tip (last push)') + if $lastpush_hash; $chkff->($lastfetch_hash, 'local tracking tip (last fetch)'); runcmd @git, qw(update-ref -m), "dgit fetch $csuite", @@ -2719,6 +2877,11 @@ sub clone ($) { } setup_new_tree(); runcmd @git, qw(reset --hard), lrref(); + runcmd qw(bash -ec), <<'END'; + set -o pipefail + git ls-tree -r --name-only -z HEAD | \ + xargs -0r touch -r . -- +END printdone "ready for work in $dstdir"; } @@ -2832,6 +2995,18 @@ sub madformat_wantfixup ($) { return 1; } +sub maybe_split_brain_save ($$$) { + my ($headref, $dgitview, $msg) = @_; + # => message fragment "$saved" describing disposition of $dgitview + return "commit id $dgitview" unless defined $split_brain_save; + my @cmd = (shell_cmd "cd ../../../..", + @git, qw(update-ref -m), + "dgit --dgit-view-save $msg HEAD=$headref", + $split_brain_save, $dgitview); + runcmd @cmd; + return "and left in $split_brain_save"; +} + # An "infopair" is a tuple [ $thing, $what ] # (often $thing is a commit hash; $what is a description) @@ -2950,12 +3125,12 @@ sub splitbrain_pseudomerge ($$$$) { # this: $dgitview' # + return $dgitview unless defined $archive_hash; + printdebug "splitbrain_pseudomerge...\n"; my $i_arch_v = pseudomerge_version_check($clogp, $archive_hash); - return $dgitview unless defined $archive_hash; - if (!defined $overwrite_version) { progress "Checking that HEAD inciudes all changes in archive..."; } @@ -2992,6 +3167,8 @@ END_OVERWR Make fast forward from $i_arch_v->[0] END_MAKEFF + maybe_split_brain_save $maintview, $r, "pseudomerge"; + progress "Made pseudo-merge of $i_arch_v->[0] into dgit view."; return $r; } @@ -3023,7 +3200,10 @@ sub push_parse_changelog ($) { my $clogp = Dpkg::Control::Hash->new(); $clogp->load($clogpfn) or die; - $package = getfield $clogp, 'Source'; + my $clogpackage = getfield $clogp, 'Source'; + $package //= $clogpackage; + fail "-p specified $package but changelog specified $clogpackage" + unless $package eq $clogpackage; my $cversion = getfield $clogp, 'Version'; my $tag = debiantag($cversion, access_basedistro); runcmd @git, qw(check-ref-format), $tag; @@ -3220,11 +3400,11 @@ END my $dgithead = $actualhead; my $maintviewhead = undef; + my $upstreamversion = upstreamversion $clogp->{Version}; + if (madformat_wantfixup($format)) { # user might have not used dgit build, so maybe do this now: if (quiltmode_splitbrain()) { - my $upstreamversion = $clogp->{Version}; - $upstreamversion =~ s/-[^-]*$//; changedir $ud; quilt_make_fake_dsc($upstreamversion); my $cachekey; @@ -3308,9 +3488,15 @@ END # Check that changes and .dsc agree enough $changesfile =~ m{[^/]*$}; - files_compare_inputs($dsc, parsecontrol($changesfile,$&)) + my $changes = parsecontrol($changesfile,$&); + files_compare_inputs($dsc, $changes) unless forceing [qw(dsc-changes-mismatch)]; + # Perhaps adjust .dsc to contain right set of origs + changes_update_origs_from_dsc($dsc, $changes, $upstreamversion, + $changesfile) + unless forceing [qw(changes-origs-exactly)]; + # Checks complete, we're going to try and go ahead: responder_send_file('changes',$changesfile); @@ -3502,6 +3688,12 @@ sub cmd_fetch { sub cmd_pull { parseopts(); fetchpullargs(); + if (quiltmode_splitbrain()) { + my ($format, $fopts) = get_source_format(); + madformat($format) and fail <{'single-debian-patch'}) { quilt_fixup_singlepatch($clogp, $headref, $upstreamversion); @@ -4475,8 +4667,9 @@ sub quilt_check_splitbrain_cache ($$) { my $cachehit = $1; quilt_fixup_mkwork($headref); + my $saved = maybe_split_brain_save $headref, $cachehit, "cache-hit"; if ($cachehit ne $headref) { - progress "dgit view: found cached (commit id $cachehit)"; + progress "dgit view: found cached ($saved)"; runcmd @git, qw(checkout -q -b dgit-view), $cachehit; $split_brain = 1; return ($cachehit, $splitbrain_cachekey); @@ -4782,15 +4975,21 @@ sub cmd_clean () { maybe_unapply_patches_again(); } -sub build_prep () { +sub build_prep_early () { + our $build_prep_early_done //= 0; + return if $build_prep_early_done++; notpushing(); badusage "-p is not allowed when building" if defined $package; - check_not_dirty(); - clean_tree(); my $clogp = parsechangelog(); $isuite = getfield $clogp, 'Distribution'; $package = getfield $clogp, 'Source'; $version = getfield $clogp, 'Version'; + check_not_dirty(); +} + +sub build_prep () { + build_prep_early(); + clean_tree(); build_maybe_quilt_fixup(); if ($rmchanges) { my $pat = changespat $version; @@ -4989,6 +5188,24 @@ sub pre_gbp_build { } sub cmd_gbp_build { + build_prep_early(); + + # gbp can make .origs out of thin air. In my tests it does this + # even for a 1.0 format package, with no origs present. So I + # guess it keys off just the version number. We don't know + # exactly what .origs ought to exist, but let's assume that we + # should run gbp if: the version has an upstream part and the main + # orig is absent. + my $upstreamversion = upstreamversion $version; + my $origfnpat = srcfn $upstreamversion, '.orig.tar.*'; + my $gbp_make_orig = $version =~ m/-/ && !(() = glob "../$origfnpat"); + + if ($gbp_make_orig) { + clean_tree(); + $cleanmode = 'none'; # don't do it again + $need_split_build_invocation = 1; + } + my @dbp = @dpkgbuildpackage; my $wantsrc = massage_dbp_args \@dbp, \@ARGV; @@ -5004,6 +5221,24 @@ sub cmd_gbp_build { push @cmd, (qw(-us -uc --git-no-sign-tags), "--git-builder=@dbp"); + if ($gbp_make_orig) { + ensuredir '.git/dgit'; + my $ok = '.git/dgit/origs-gen-ok'; + unlink $ok or $!==&ENOENT or die $!; + my @origs_cmd = @cmd; + push @origs_cmd, qw(--git-cleaner=true); + push @origs_cmd, "--git-prebuild=touch $ok .git/dgit/no-such-dir/ok"; + push @origs_cmd, @ARGV; + if (act_local()) { + debugcmd @origs_cmd; + system @origs_cmd; + do { local $!; stat_exists $ok; } + or failedcmd @origs_cmd; + } else { + dryrun_report @origs_cmd; + } + } + if ($wantsrc > 0) { build_source(); midbuild_checkchanges_vanilla $wantsrc; @@ -5119,6 +5354,157 @@ sub cmd_quilt_fixup { build_maybe_quilt_fixup(); } +sub cmd_import_dsc { + my $needsig = 0; + + while (@ARGV) { + last unless $ARGV[0] =~ m/^-/; + $_ = shift @ARGV; + last if m/^--?$/; + if (m/^--require-valid-signature$/) { + $needsig = 1; + } else { + badusage "unknown dgit import-dsc sub-option \`$_'"; + } + } + + badusage "usage: dgit import-dsc .../PATH/TO/.DSC BRANCH" unless @ARGV==2; + my ($dscfn, $dstbranch) = @ARGV; + + badusage "dry run makes no sense with import-dsc" unless act_local(); + + my $force = $dstbranch =~ s/^\+// ? +1 : + $dstbranch =~ s/^\.\.// ? -1 : + 0; + my $info = $force ? " $&" : ''; + $info = "$dscfn$info"; + + my $specbranch = $dstbranch; + $dstbranch = "refs/heads/$dstbranch" unless $dstbranch =~ m#^refs/#; + $dstbranch = cmdoutput @git, qw(check-ref-format --normalize), $dstbranch; + + my @symcmd = (@git, qw(symbolic-ref -q HEAD)); + my $chead = cmdoutput_errok @symcmd; + defined $chead or $?==256 or failedcmd @symcmd; + + fail "$dstbranch is checked out - will not update it" + if defined $chead and $chead eq $dstbranch; + + my $oldhash = git_get_ref $dstbranch; + + open D, "<", $dscfn or fail "open import .dsc ($dscfn): $!"; + $dscdata = do { local $/ = undef; ; }; + D->error and fail "read $dscfn: $!"; + close C; + + # we don't normally need this so import it here + use Dpkg::Source::Package; + my $dp = new Dpkg::Source::Package filename => $dscfn, + require_valid_signature => $needsig; + { + local $SIG{__WARN__} = sub { + print STDERR $_[0]; + return unless $needsig; + fail "import-dsc signature check failed"; + }; + if (!$dp->is_signed()) { + warn "$us: warning: importing unsigned .dsc\n"; + } else { + my $r = $dp->check_signature(); + die "->check_signature => $r" if $needsig && $r; + } + } + + parse_dscdata(); + + my $dgit_commit = $dsc->{$ourdscfield[0]}; + if (defined $dgit_commit && + !forceing [qw(import-dsc-with-dgit-field)]) { + $dgit_commit =~ m/\w+/ or fail "invalid hash in .dsc"; + progress "dgit: import-dsc of .dsc with Dgit field, using git hash"; + my @cmd = (qw(sh -ec), + "echo $dgit_commit | git cat-file --batch-check"); + my $objgot = cmdoutput @cmd; + if ($objgot =~ m#^\w+ missing\b#) { + fail < 0) { + progress "Not fast forward, forced update."; + } else { + fail "Not fast forward to $dgit_commit"; + } + } + @cmd = (@git, qw(update-ref -m), "dgit import-dsc (Dgit): $info", + $dstbranch, $dgit_commit); + runcmd @cmd; + progress "dgit: import-dsc updated git ref $dstbranch"; + return 0; + } + + fail <{Filename}; + my $here = "../$f"; + next if lstat $here; + fail "stat $here: $!" unless $! == ENOENT; + my $there = $dscfn; + if ($dscfn =~ m#^(?:\./+)?\.\./+#) { + $there = $'; + } elsif ($dscfn =~ m#^/#) { + $there = $dscfn; + } else { + fail "cannot import $dscfn which seems to be inside working tree!"; + } + $there =~ s#/+[^/]+$## or + fail "cannot import $dscfn which seems to not have a basename"; + $there .= "/$f"; + symlink $there, $here or fail "symlink $there to $here: $!"; + progress "made symlink $here -> $there"; + print STDERR Dumper($fi); + } + my @mergeinputs = generate_commits_from_dsc(); + die unless @mergeinputs == 1; + + my $newhash = $mergeinputs[0]{Commit}; + + if ($oldhash) { + if ($force > 0) { + progress "Import, forced update - synthetic orphan git history."; + } elsif ($force < 0) { + progress "Import, merging."; + my $tree = cmdoutput @git, qw(rev-parse), "$newhash:"; + my $version = getfield $dsc, 'Version'; + $newhash = make_commit_text <../before + +t-archive-process-incoming sid + +find $tmp/mirror -name '*orig*' -ls >../after +diff -u ../before ../after + +test-push-1 1.1-1.2 --ch:-sd + +test-push-2 + +t-archive-process-incoming sid + +cd .. +mkdir get +cd get + +t-dgit clone $p +# ^ checks that all the origs are there, ie that dgit added the origs + +cd .. diff --git a/tests/setup/examplegit b/tests/setup/examplegit index 924113fc..98900d09 100755 --- a/tests/setup/examplegit +++ b/tests/setup/examplegit @@ -2,7 +2,7 @@ set -e . tests/lib -suitespecs+=' stable' +suitespecs+=' stable testing' t-tstunt-parsechangelog diff --git a/tests/tests/gbp-orig b/tests/tests/gbp-orig new file mode 100755 index 00000000..ffc145f3 --- /dev/null +++ b/tests/tests/gbp-orig @@ -0,0 +1,75 @@ +#!/bin/bash +set -e +. tests/lib + +t-tstunt-parsechangelog + +t-archive-none example +t-git-none +t-worktree 1.0 + +cd $p + +: '----- construct an unpatched branch with patches -----' + +git checkout patch-queue/quilt-tip +gbp pq export +: 'now on quilt-tip' +git add debian/patches +git commit -m 'Commit patch queue' + +: '----- construct an upstream branch -----' + +git checkout --orphan upstream +git reset --hard +git clean -xdf + +tar --strip-components=1 -xf $troot/pkg-srcs/${p}_1.0.orig.tar.gz + +mkdir docs +cd docs +tar --strip-components=1 -xf $troot/pkg-srcs/${p}_1.0.orig-docs.tar.gz +cd .. + +git add -Af . +git commit -m 'Import 1.0' +git tag upstream/1.0 + +git checkout quilt-tip +t-git-pseudo-merge upstream + +v=1.0-1 + +: '----- let gbp build a .orig for comparison -----' + +gbp buildpackage --git-ignore-branch --git-no-sign-tags -us -uc + +mkdir ../gbp-output +mv ../*1.0* ../gbp-output/. +rm -f ../*.changes + +: '----- now do it ourselves -----' + +t-dgit -wgf --dgit-view-save=split.b gbp-build --git-ignore-branch + +t-dgit -wgf --quilt=gbp clean # gbp leaves dirty trees :-/ + +t-dgit -wgf --dgit-view-save=split.p --quilt=gbp push --new + +t-gbp-pushed-good + +: '----- check .origs are the same -----' + +# if gbp weren't weird about .gitignore we could just debdiff the .dscs + +for d in . gbp-output; do + cd $tmp/$d + mkdir tar-x + cd tar-x + tar zxf ../${p}_${v%-*}.orig.tar.gz +done + +cd $tmp +diff -ruN gbp-output/tar-x tar-x + +echo done. diff --git a/tests/tests/import-dsc b/tests/tests/import-dsc new file mode 100755 index 00000000..9d3d9b79 --- /dev/null +++ b/tests/tests/import-dsc @@ -0,0 +1,99 @@ +#!/bin/bash +set -e +. tests/lib + +t-setup-import examplegit + +p=example + +check-import () { + path=$1 + v=$2 + opts=$3 + branch=t.$v + + dsc=${path}/${p}_${v}.dsc + t-dgit $opts import-dsc $dsc $branch + + git checkout $branch + + check-imported $dsc +} + +check-imported () { + local dsc=$1 + ( + rm -rf ../t.unpack + mkdir ../t.unpack + cd ../t.unpack + dpkg-source -x $dsc + ) + + git checkout HEAD~0 + git branch -D u.$v ||: + git checkout -b u.$v $branch + git rm -rf . + git clean -xdf + cp -al ../t.unpack/*/. . + git add -Af . + + git diff --stat --exit-code +} + +cd $p + +check-import ../mirror/pool/main 1.2 + +dgit12=`git rev-parse HEAD` + +dsc2=../mirror/pool/main/${p}_2.0.dsc + +git checkout $branch +t-expect-fail 'is checked out - will not update' \ +t-dgit import-dsc $dsc2 $branch + +git checkout HEAD~0 + +t-expect-fail 'Not fast forward' \ +t-dgit import-dsc $dsc2 $branch + +t-expect-fail 'Not fast forward' \ +t-dgit import-dsc $dsc2 ..$branch + +t-dgit import-dsc $dsc2 +$branch +check-imported $dsc2 + +cd .. +mkdir $p.2 +cd $p.2 + +git init + +check-import ../../../pkg-srcs 1.0-1 + +t-expect-fail "Your git tree does not have that object" \ +check-import ../mirror/pool/main 1.2 + +check-import ../mirror/pool/main 1.2 --force-import-dsc-with-dgit-field + +v=1.0-1.100 +dsc2=../../../pkg-srcs/${p}_${v}.dsc + +t-expect-fail E:'Branch.*already exists' \ +t-dgit import-dsc $dsc2 $branch + +git branch merge-reset +t-dgit import-dsc $dsc2 ..$branch +t-has-ancestor merge-reset $branch + +git push . +merge-reset:$branch + +t-dgit import-dsc $dsc2 +$branch + +mb=$(t-git-merge-base merge-reset $branch) +test "x$mb" = x + +t-expect-fail 'signature check failed' \ +t-dgit import-dsc --require-valid-signature $dsc2 +$branch + +echo ok. diff --git a/tests/tests/inarchivecopy b/tests/tests/inarchivecopy index e87baaf3..632d4206 100755 --- a/tests/tests/inarchivecopy +++ b/tests/tests/inarchivecopy @@ -14,34 +14,34 @@ t-inarchive-copy () { local from=${2:-sid} local to=${3:-stable} egrep "^${vm//./\\.}" aq/package.$from.$p >>aq/package.$to.$p - t-archive-updated stable $p + t-archive-updated $to $p } copy-check-good () { git diff $vtag - t-refs-same refs/remotes/dgit/dgit/stable + t-refs-same refs/remotes/dgit/dgit/$tosuite t-ref-head + t-has-parent-or-is HEAD $vtag } copy-check () { local vm=$1 - t-inarchive-copy $vm + local tosuite=${2:-stable} + t-inarchive-copy $vm '' $tosuite vtag=$(v=$vm t-v-tag) cd $p t-refs-same-start - t-ref-same $vtag^1 - - t-dgit fetch stable - git merge --ff-only dgit/dgit/stable + t-dgit fetch $tosuite + git merge --ff-only dgit/dgit/$tosuite copy-check-good local fetched=$(t-sametree-parent HEAD) cd .. rm -rf example.cloned - t-dgit clone $p stable example.cloned + t-dgit clone $p $tosuite example.cloned cd example.cloned t-refs-same-start @@ -49,13 +49,31 @@ copy-check () { local cloned=$(t-sametree-parent HEAD) cd .. + rm -rf example.initd + mkdir example.initd + cd example.initd + git init + t-refs-same-start + t-dgit -p $p fetch $tosuite + git reset --hard refs/remotes/dgit/dgit/$tosuite + copy-check-good + local initd=$(t-sametree-parent HEAD) + cd .. + t-refs-same-start t-ref-same-val fetched $fetched t-ref-same-val cloned $cloned + t-ref-same-val initd $initd } copy-check 2.0 copy-check 2.1 +cd $p +git checkout -b dgit/testing $(v=1.1 t-v-tag) +cd .. + +copy-check 2.1 testing + echo ok. diff --git a/tests/tests/orig-include-exclude b/tests/tests/orig-include-exclude new file mode 100755 index 00000000..96bf4f21 --- /dev/null +++ b/tests/tests/orig-include-exclude @@ -0,0 +1,25 @@ +#!/bin/bash +set -e +. tests/lib + +suitespecs+=' stable' + +. $troot/lib-orig-include-exclude + +ofb=example_1.1.orig.tar +zcat $ofb.gz >$ofb.SPONG +gzip -1Nv $ofb.SPONG +mv $ofb.SPONG.gz $ofb.gz + +#suite=stable +#t-archive-none $p +#t-archive-updated stable $p + +cd $p + +test-push-1 1.1-1.3 '' stable + +t-expect-fail E:'archive contains .* with different checksum' \ +test-push-2 --new + +echo done. diff --git a/tests/tests/orig-include-exclude-chkquery b/tests/tests/orig-include-exclude-chkquery new file mode 100755 index 00000000..59fc80f0 --- /dev/null +++ b/tests/tests/orig-include-exclude-chkquery @@ -0,0 +1,33 @@ +#!/bin/bash +set -e +. tests/lib + +t-git-config dgit-distro.test-dummy.archive-query ftpmasterapi: +# ^ that will crash if it gets unexpected file_in_archive queries + +# orig-include-exclude will set origs and usvsns +update-files_in_archive () { + for o in $origs; do for usvsn in $usvsns; do \ + of=${p}_${v%-*}.${o}.tar.gz + pat="%/${of//_/\\_}" + # curl url-decodes these things so we have to have literals + find $tmp/mirror -name $of | \ + xargs -r sha256sum | \ + perl -pe ' + BEGIN { print "["; } + chomp; + s/^/{"sha256sum":"/; + s/ /","filename":"/; + s/$/"}$delim/; + $delim=","; + END { print "]\n"; } + ' \ + >$tmp/aq/"file_in_archive/$pat" + done; done +} + +test_push_2_hook=update-files_in_archive + +. $troot/lib-orig-include-exclude + +echo done. diff --git a/tests/tests/overwrite-splitbrains b/tests/tests/overwrite-splitbrains index c0c7470d..725357d0 100755 --- a/tests/tests/overwrite-splitbrains +++ b/tests/tests/overwrite-splitbrains @@ -7,7 +7,7 @@ t-tstunt-parsechangelog t-gbp-example-prep-no-ff t-newtag -t-dgit --quilt=gbp build-source +t-dgit --quilt=gbp --dgit-view-save=split.b build-source t-dgit fetch @@ -15,12 +15,12 @@ t-refs-same-start t-ref-head t-expect-fail 'check failed (maybe --overwrite is needed' \ -t-dgit --quilt=gbp push +t-dgit --quilt=gbp --dgit-view-save=split.p push t-refs-same-start t-ref-head -t-dgit --quilt=gbp --overwrite push +t-dgit --quilt=gbp --dgit-view-save=split.p --overwrite push t-gbp-pushed-good diff --git a/tests/tests/quilt-gbp b/tests/tests/quilt-gbp index 21e396bc..38d2489b 100755 --- a/tests/tests/quilt-gbp +++ b/tests/tests/quilt-gbp @@ -14,7 +14,8 @@ t-expect-fail 'requires split view so server needs to support' \ t-dgit -wgf --quilt=gbp build-source t-newtag -t-dgit --quilt=gbp build-source +t-dgit --quilt=gbp --dgit-view-save=split.b1 build-source +git rev-parse split.b1 t-dgit --quilt=gbp --gbp-pq=no-such-command-gbp build-source @@ -31,14 +32,14 @@ test-push-1 () { } test-push-2 () { - t-dgit --quilt=gbp push + t-dgit --quilt=gbp --dgit-view-save=split.p push t-gbp-pushed-good } test-push-1 -t-dgit --quilt=gbp --clean=git build-source +t-dgit --quilt=gbp --clean=git --dgit-view-save=split.b build-source t-expect-fail "HEAD specifies a different tree to $p" \ t-dgit push @@ -53,7 +54,7 @@ t-commit 'Check pseudomerge' 1.0-3 test-push-1 -t-dgit --quilt=gbp --clean=git build-source +t-dgit --quilt=gbp --clean=git --dgit-view-save=split.b build-source test-push-2 diff --git a/tests/tests/quilt-splitbrains b/tests/tests/quilt-splitbrains index 1e853f31..112bc39e 100755 --- a/tests/tests/quilt-splitbrains +++ b/tests/tests/quilt-splitbrains @@ -2,6 +2,8 @@ set -e . tests/lib +suitespecs+=' stable' + # This test script tests each of the split brain quilt modes, and # --quilt=linear, with a tree suitable for each of those, and pushes # them in sequence. The idea is to check that each tree is rejected @@ -24,9 +26,9 @@ want-success () { t-refs-same-start t-ref-head - t-dgit "$@" --quilt=$qmode build-source + t-dgit "$@" --quilt=$qmode --dgit-view-save=split.b build-source - t-dgit "$@" --quilt=$qmode push + t-dgit "$@" --quilt=$qmode --dgit-view-save=split.p push t-$qmode-pushed-good } @@ -130,5 +132,9 @@ t-dgit -wgf build-source want-success dpm +suite=stable +t-commit dpmish-stable 1.0-6 $suite + +want-success dpm --new echo ok.