X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=dgit.git;a=blobdiff_plain;f=dgit;h=909ac95f75ceaef2fd5c05a226416ce785d9d750;hp=88bd431ab37e277bcd273079a2f6fc42078e3030;hb=b0bc4d204e5900c7888e681404e2638ce7347cf9;hpb=81a31ed44401d58cff028c12db896fa3badfd4ee diff --git a/dgit b/dgit index 88bd431a..909ac95f 100755 --- a/dgit +++ b/dgit @@ -48,7 +48,7 @@ our $absurdity = undef; ###substituted### our @rpushprotovsn_support = qw(4 3 2); # 4 is new tag format our $protovsn; -our $isuite = 'unstable'; +our $isuite; our $idistro; our $package; our @ropts; @@ -78,6 +78,7 @@ our $patches_applied_dirtily = 00; our $tagformat_want; our $tagformat; our $tagformatfn; +our $chase_dsc_distro=1; our %forceopts = map { $_=>0 } qw(unrepresentable unsupported-source-format @@ -142,7 +143,6 @@ our %opts_cfg_insertpos = map { scalar @{ $opts_opt_map{$_} } } keys %opts_opt_map; -sub finalise_opts_opts(); sub parseopts_late_defaults(); our $keyid; @@ -186,30 +186,6 @@ sub lref () { return "refs/heads/".lbranch(); } sub lrref () { return "refs/remotes/$remotename/".server_branch($csuite); } sub rrref () { return server_ref($csuite); } -sub lrfetchrefs () { return "refs/dgit-fetch/$csuite"; } -sub lrfetchref () { return lrfetchrefs.'/'.server_branch($csuite); } - -# We fetch some parts of lrfetchrefs/*. Ideally we delete these -# locally fetched refs because they have unhelpful names and clutter -# up gitk etc. So we track whether we have "used up" head ref (ie, -# whether we have made another local ref which refers to this object). -# -# (If we deleted them unconditionally, then we might end up -# re-fetching the same git objects each time dgit fetch was run.) -# -# So, leach use of lrfetchrefs needs to be accompanied by arrangements -# in git_fetch_us to fetch the refs in question, and possibly a call -# to lrfetchref_used. - -our (%lrfetchrefs_f, %lrfetchrefs_d); -# $lrfetchrefs_X{lrfetchrefs."/heads/whatever"} = $objid - -sub lrfetchref_used ($) { - my ($fullrefname) = @_; - my $objid = $lrfetchrefs_f{$fullrefname}; - $lrfetchrefs_d{$fullrefname} = $objid if defined $objid; -} - sub stripepoch ($) { my ($vsn) = @_; $vsn =~ s/^\d+\://; @@ -589,6 +565,8 @@ sub cmd_help () { our $td = $ENV{DGIT_TEST_DUMMY_DIR} || "DGIT_TEST_DUMMY_DIR-unset"; our %defcfg = ('dgit.default.distro' => 'debian', + 'dgit.default.default-suite' => 'unstable', + 'dgit.default.old-dsc-distro' => 'debian', 'dgit-suite.*-security.distro' => 'debian-security', 'dgit.default.username' => '', 'dgit.default.archive-query-default-component' => 'main', @@ -597,6 +575,10 @@ our %defcfg = ('dgit.default.distro' => 'debian', 'dgit.default.sshpsql-dbname' => 'service=projectb', 'dgit.default.aptget-components' => 'main', 'dgit.default.dgit-tag-format' => 'new,old,maint', + 'dgit.dsc-url-proto-ok.http' => 'true', + 'dgit.dsc-url-proto-ok.https' => 'true', + 'dgit.dsc-url-proto-ok.git' => 'true', + 'dgit.default.dsc-url-proto-ok' => 'false', # old means "repo server accepts pushes with old dgit tags" # new means "repo server accepts pushes with new dgit tags" # maint means "repo server accepts split brain pushes" @@ -684,7 +666,10 @@ sub git_get_config ($) { my ($c) = @_; foreach my $src (@gitcfgsources) { my $l = $gitcfgs{$src}{$c}; - printdebug"C $c ".(defined $l ? messagequote "'$l'" : "undef")."\n" + croak "$l $c" if $l && !ref $l; + printdebug"C $c ".(defined $l ? + join " ", map { messagequote "'$_'" } @$l : + "undef")."\n" if $debuglevel >= 4; $l or next; @$l==1 or badcfg "multiple values for $c". @@ -697,16 +682,20 @@ sub git_get_config ($) { sub cfg { foreach my $c (@_) { return undef if $c =~ /RETURN-UNDEF/; + printdebug "C? $c\n" if $debuglevel >= 5; my $v = git_get_config($c); return $v if defined $v; my $dv = $defcfg{$c}; - return $dv if defined $dv; + if (defined $dv) { + printdebug "CD $c $dv\n" if $debuglevel >= 4; + return $dv; + } } badcfg "need value for one of: @_\n". "$us: distro or suite appears not to be (properly) supported"; } -sub access_basedistro () { +sub access_basedistro__noalias () { if (defined $idistro) { return $idistro; } else { @@ -726,9 +715,18 @@ sub access_basedistro () { } } +sub access_basedistro () { + my $noalias = access_basedistro__noalias(); + my $canon = cfg("dgit-distro.$noalias.alias-canon",'RETURN-UNDEF'); + return $canon // $noalias; +} + sub access_nomdistro () { my $base = access_basedistro(); - return cfg("dgit-distro.$base.nominal-distro",'RETURN-UNDEF') // $base; + my $r = cfg("dgit-distro.$base.nominal-distro",'RETURN-UNDEF') // $base; + $r =~ m/^$distro_re$/ or badcfg + "bad syntax for (nominal) distro \`$r' (does not match /^$distro_re$/)"; + return $r; } sub access_quirk () { @@ -792,11 +790,11 @@ sub pushing () { Push failed, before we got started. You can retry the push, after fixing the problem, if you like. END - finalise_opts_opts(); + parseopts_late_defaults(); } sub notpushing () { - finalise_opts_opts(); + parseopts_late_defaults(); } sub supplementary_message ($) { @@ -1681,6 +1679,7 @@ sub create_remote_git_repo () { } our ($dsc_hash,$lastpush_mergeinput); +our ($dsc_distro, $dsc_hint_tag, $dsc_hint_url); our $ud = '.git/dgit/unpack'; @@ -2504,19 +2503,41 @@ sub ensure_we_have_orig () { } } -sub git_fetch_us () { - # Want to fetch only what we are going to use, unless - # deliberately-not-ff, in which case we must fetch everything. +#---------- git fetch ---------- - my @specs = deliberately_not_fast_forward ? qw(tags/*) : - map { "tags/$_" } - (quiltmode_splitbrain - ? (map { $_->('*',access_nomdistro) } - \&debiantag_new, \&debiantag_maintview) - : debiantags('*',access_nomdistro)); - push @specs, server_branch($csuite); - push @specs, $rewritemap; - push @specs, qw(heads/*) if deliberately_not_fast_forward; +sub lrfetchrefs () { return "refs/dgit-fetch/".access_basedistro(); } +sub lrfetchref () { return lrfetchrefs.'/'.server_branch($csuite); } + +# We fetch some parts of lrfetchrefs/*. Ideally we delete these +# locally fetched refs because they have unhelpful names and clutter +# up gitk etc. So we track whether we have "used up" head ref (ie, +# whether we have made another local ref which refers to this object). +# +# (If we deleted them unconditionally, then we might end up +# re-fetching the same git objects each time dgit fetch was run.) +# +# So, leach use of lrfetchrefs needs to be accompanied by arrangements +# in git_fetch_us to fetch the refs in question, and possibly a call +# to lrfetchref_used. + +our (%lrfetchrefs_f, %lrfetchrefs_d); +# $lrfetchrefs_X{lrfetchrefs."/heads/whatever"} = $objid + +sub lrfetchref_used ($) { + my ($fullrefname) = @_; + my $objid = $lrfetchrefs_f{$fullrefname}; + $lrfetchrefs_d{$fullrefname} = $objid if defined $objid; +} + +sub git_lrfetch_sane { + my ($supplementary, @specs) = @_; + # Make a 'refs/'.lrfetchrefs.'/*' be just like on server, + # at least as regards @specs. Also leave the results in + # %lrfetchrefs_f, and arrange for lrfetchref_used to be + # able to clean these up. + # + # With $supplementary==1, @specs must not contain wildcards + # and we add to our previous fetches (non-atomically). # This is rather miserable: # When git fetch --prune is passed a fetchspec ending with a *, @@ -2540,30 +2561,33 @@ sub git_fetch_us () { # git fetch to try to generate it. If we don't manage to generate # the target state, we try again. - printdebug "git_fetch_us specs @specs\n"; + my $url = access_giturl(); + + printdebug "git_lrfetch_sane suppl=$supplementary specs @specs\n"; my $specre = join '|', map { my $x = $_; $x =~ s/\W/\\$&/g; - $x =~ s/\\\*$/.*/; + my $wildcard = $x =~ s/\\\*$/.*/; + die if $wildcard && $supplementary; "(?:refs/$x)"; } @specs; - printdebug "git_fetch_us specre=$specre\n"; + printdebug "git_lrfetch_sane specre=$specre\n"; my $wanted_rref = sub { local ($_) = @_; - return m/^(?:$specre)$/o; + return m/^(?:$specre)$/; }; my $fetch_iteration = 0; FETCH_ITERATION: for (;;) { - printdebug "git_fetch_us iteration $fetch_iteration\n"; + printdebug "git_lrfetch_sane iteration $fetch_iteration\n"; if (++$fetch_iteration > 10) { fail "too many iterations trying to get sane fetch!"; } my @look = map { "refs/$_" } @specs; - my @lcmd = (@git, qw(ls-remote -q --refs), access_giturl(), @look); + my @lcmd = (@git, qw(ls-remote -q --refs), $url, @look); debugcmd "|",@lcmd; my %wantr; @@ -2589,13 +2613,14 @@ END "+refs/$_:".lrfetchrefs."/$_"; } @specs; - printdebug "git_fetch_us fspecs @fspecs\n"; + printdebug "git_lrfetch_sane fspecs @fspecs\n"; - my @fcmd = (@git, qw(fetch -p -n -q), access_giturl(), @fspecs); - runcmd_ordryrun_local @git, qw(fetch -p -n -q), access_giturl(), - @fspecs; + my @fcmd = (@git, qw(fetch -p -n -q), $url, @fspecs); + runcmd_ordryrun_local @fcmd if @fspecs; - %lrfetchrefs_f = (); + if (!$supplementary) { + %lrfetchrefs_f = (); + } my %objgot; git_for_each_ref(lrfetchrefs, sub { @@ -2604,6 +2629,10 @@ END $objgot{$objid} = 1; }); + if ($supplementary) { + last; + } + foreach my $lrefname (sort keys %lrfetchrefs_f) { my $rrefname = 'refs'.substr($lrefname, length lrfetchrefs); if (!exists $wantr{$rrefname}) { @@ -2645,8 +2674,35 @@ END } last; } - printdebug "git_fetch_us: git fetch --no-insane emulation complete\n", + + if (defined $csuite) { + printdebug "git_lrfetch_sane: tidying any old suite lrfetchrefs\n"; + git_for_each_ref("refs/dgit-fetch/$csuite", sub { + my ($objid,$objtype,$lrefname,$reftail) = @_; + next if $lrfetchrefs_f{$lrefname}; # $csuite eq $distro ? + runcmd_ordryrun_local @git, qw(update-ref -d), $lrefname; + }); + } + + printdebug "git_lrfetch_sane: git fetch --no-insane emulation complete\n", Dumper(\%lrfetchrefs_f); +} + +sub git_fetch_us () { + # Want to fetch only what we are going to use, unless + # deliberately-not-ff, in which case we must fetch everything. + + my @specs = deliberately_not_fast_forward ? qw(tags/*) : + map { "tags/$_" } + (quiltmode_splitbrain + ? (map { $_->('*',access_nomdistro) } + \&debiantag_new, \&debiantag_maintview) + : debiantags('*',access_nomdistro)); + push @specs, server_branch($csuite); + push @specs, $rewritemap; + push @specs, qw(heads/*) if deliberately_not_fast_forward; + + git_lrfetch_sane 0, @specs; my %here; my @tagpats = debiantags('*',access_nomdistro); @@ -2673,6 +2729,8 @@ END }); } +#---------- dsc and archive handling ---------- + sub mergeinfo_getclogp ($) { # Ensures thit $mi->{Clogp} exists and returns it my ($mi) = @_; @@ -2701,6 +2759,124 @@ sub fetch_from_archive_record_2 ($) { } } +sub parse_dsc_field ($$) { + my ($dsc, $what) = @_; + my $f; + foreach my $field (@ourdscfield) { + $f = $dsc->{$field}; + last if defined $f; + } + if (!defined $f) { + progress "$what: NO git hash"; + } elsif (($dsc_hash, $dsc_distro, $dsc_hint_tag, $dsc_hint_url) + = $f =~ m/^(\w+)\s+($distro_re)\s+($versiontag_re)\s+(\S+)(?:\s|$)/) { + progress "$what: specified git info ($dsc_distro)"; + $dsc_hint_tag = [ $dsc_hint_tag ]; + } elsif ($f =~ m/^\w+\s*$/) { + $dsc_hash = $&; + $dsc_distro //= cfg qw(dgit.default.old-dsc-distro + dgit.default.distro); + $dsc_hint_tag = [ debiantags +(getfield $dsc, 'Version'), + $dsc_distro ]; + progress "$what: specified git hash"; + } else { + fail "$what: invalid Dgit info"; + } +} + +sub resolve_dsc_field_commit ($$) { + my ($already_distro, $already_mapref) = @_; + + return unless defined $dsc_hash; + + my $mapref = + defined $already_mapref && + ($already_distro eq $dsc_distro || !$chase_dsc_distro) + ? $already_mapref : undef; + + my $do_fetch; + $do_fetch = sub { + my ($what, @fetch) = @_; + + local $idistro = $dsc_distro; + my $lrf = lrfetchrefs; + + if (!$chase_dsc_distro) { + progress + "not chasing .dsc distro $dsc_distro: not fetching $what"; + return 0; + } + + progress + ".dsc names distro $dsc_distro: fetching $what"; + + my $url = access_giturl(); + if (!defined $url) { + defined $dsc_hint_url or fail <("rewrite map", $rewritemap) or return; + $mapref = $lrf.'/'.$rewritemap; + my $rewritemapdata = git_cat_file $mapref.':map'; + if (defined $rewritemapdata + && $rewritemapdata =~ m/^$dsc_hash(?:[ \t](\w+))/m) { + progress + "server's git history rewrite map contains a relevant entry!"; + + $dsc_hash = $1; + if (defined $dsc_hash) { + progress "using rewritten git hash in place of .dsc value"; + } else { + progress "server data says .dsc hash is to be disregarded"; + } + } + } + + if (!defined git_cat_file $dsc_hash) { + my @tags = map { "tags/".$_ } @$dsc_hint_tag; + my $lrf = $do_fetch->("additional commits", @tags) && + defined git_cat_file $dsc_hash + or fail <{$field}; - last if defined $dsc_hash; - } - if (defined $dsc_hash) { - $dsc_hash =~ m/\w+/ or fail "invalid hash in .dsc \`$dsc_hash'"; - $dsc_hash = $&; - progress "last upload to archive specified git hash"; - } else { - progress "last upload to archive has NO git hash"; - } + parse_dsc_field($dsc, 'last upload to archive'); + resolve_dsc_field_commit access_basedistro, + lrfetchrefs."/".$rewritemap } else { progress "no version available from the archive"; } - my $rewritemapdata = git_cat_file lrfetchrefs."/".$rewritemap.':map'; - if (defined $rewritemapdata - && $rewritemapdata =~ m/^$dsc_hash(?:[ \t](\w+))/m) { - progress "server's git history rewrite map contains a relevant entry!"; - $dsc_hash = $1; - if (defined $dsc_hash) { - progress "using rewritten git hash in place of .dsc value"; - } else { - progress "server data says .dsc hash is to be disregarded"; - } - } - # If the archive's .dsc has a Dgit field, there are three # relevant git commitids we need to choose between and/or merge # together: @@ -3724,7 +3880,11 @@ sub push_mktags ($$ $$ $) { die unless $tagwants->[0]{View} eq 'dgit'; - $dsc->{$ourdscfield[0]} = $tagwants->[0]{Objid}; + my $declaredistro = access_nomdistro(); + my $reader_giturl = do { local $access_forpush=0; access_giturl(); }; + $dsc->{$ourdscfield[0]} = join " ", + $tagwants->[0]{Objid}, $declaredistro, $tagwants->[0]{Tag}, + $reader_giturl; $dsc->save("$dscfn.tmp") or die $!; my $changes = parsecontrol($changesfile,$changesfilewhat); @@ -3741,7 +3901,6 @@ sub push_mktags ($$ $$ $) { # to control the "tagger" (b) we can do remote signing my $authline = clogp_authline $clogp; my $delibs = join(" ", "",@deliberatelies); - my $declaredistro = access_nomdistro(); my $mktag = sub { my ($tw) = @_; @@ -4132,7 +4291,6 @@ sub branchsuite () { } sub fetchpullargs () { - notpushing(); if (!defined $package) { my $sourcep = parsecontrol('debian/control','debian/control'); $package = getfield $sourcep, 'Source'; @@ -4141,13 +4299,15 @@ sub fetchpullargs () { $isuite = branchsuite(); if (!$isuite) { my $clogp = parsechangelog(); - $isuite = getfield $clogp, 'Distribution'; + my $clogsuite = getfield $clogp, 'Distribution'; + $isuite= $clogsuite if $clogsuite ne 'UNRELEASED'; } } elsif (@ARGV==1) { ($isuite) = @ARGV; } else { badusage "incorrect arguments to dgit fetch or dgit pull"; } + notpushing(); } sub cmd_fetch { @@ -5889,31 +6049,35 @@ sub cmd_import_dsc { parse_dscdata(); - my $dgit_commit = $dsc->{$ourdscfield[0]}; - if (defined $dgit_commit - && !forceing [qw(import-dsc-with-dgit-field)]) { - $dgit_commit =~ m/\w+/ or fail "invalid hash in .dsc"; - $dgit_commit = $&; + $package = getfield $dsc, 'Source'; + + parse_dsc_field($dsc, "Dgit metadata in .dsc") + unless forceing [qw(import-dsc-with-dgit-field)]; + + if (defined $dsc_hash) { progress "dgit: import-dsc of .dsc with Dgit field, using git hash"; + resolve_dsc_field_commit undef, undef; + } + if (defined $dsc_hash) { my @cmd = (qw(sh -ec), - "echo $dgit_commit | git cat-file --batch-check"); + "echo $dsc_hash | git cat-file --batch-check"); my $objgot = cmdoutput @cmd; if ($objgot =~ m#^\w+ missing\b#) { fail < 0) { progress "Not fast forward, forced update."; } else { - fail "Not fast forward to $dgit_commit"; + fail "Not fast forward to $dsc_hash"; } } @cmd = (@git, qw(update-ref -m), "dgit import-dsc (Dgit): $info", - $dstbranch, $dgit_commit); + $dstbranch, $dsc_hash); runcmd @cmd; progress "dgit: import-dsc updated git ref $dstbranch"; return 0; @@ -5926,7 +6090,6 @@ Specify +$specbranch to overwrite, discarding existing history END if $oldhash && !$force; - $package = getfield $dsc, 'Source'; my @dfi = dsc_files_info(); foreach my $fi (@dfi) { my $f = $fi->{Filename}; @@ -5995,18 +6158,28 @@ sub cmd_clone_dgit_repos_server { badusage "need destination argument" unless @ARGV==1; my ($destdir) = @ARGV; $package = '_dgit-repos-server'; + local $access_forpush = 0; my @cmd = (@git, qw(clone), access_giturl(), $destdir); debugcmd ">",@cmd; exec @cmd or fail "exec git clone: $!\n"; } +sub cmd_print_dgit_repos_server_source_url { + badusage "no arguments allowed to dgit print-dgit-repos-server-source-url" + if @ARGV; + $package = '_dgit-repos-server'; + local $access_forpush = 0; + my $url = access_giturl(); + print $url, "\n" or die $!; +} + sub cmd_setup_mergechangelogs { badusage "no arguments allowed to dgit setup-mergechangelogs" if @ARGV; setup_mergechangelogs(1); } sub cmd_setup_useremail { - badusage "no arguments allowed to dgit setup-mergechangelogs" if @ARGV; + badusage "no arguments allowed to dgit setup-useremail" if @ARGV; setup_useremail(1); } @@ -6132,6 +6305,9 @@ sub parseopts () { } elsif (m/^--no-rm-on-error$/s) { push @ropts, $_; $rmonerror = 0; + } elsif (m/^--no-chase-dsc-distro$/s) { + push @ropts, $_; + $chase_dsc_distro = 0; } elsif (m/^--overwrite$/s) { push @ropts, $_; $overwrite_version = ''; @@ -6177,6 +6353,11 @@ sub parseopts () { # undocumented, for testing push @ropts, $_; $need_split_build_invocation = 1; + } elsif (m/^--config-lookup-explode=(.+)$/s) { + # undocumented, for testing + push @ropts, $_; + $gitcfgs{cmdline}{$1} = 'CONFIG-LOOKUP-EXPLODE'; + # ^ it's supposed to be an array ref } elsif (m/^(--[-0-9a-z]+)(=|$)/ && ($oi = $valopts_long{$1})) { $val = $2 ? $' : undef; #'; $valopt->($oi->{Long}); @@ -6265,7 +6446,11 @@ END } -sub finalise_opts_opts () { +sub parseopts_late_defaults () { + $isuite //= cfg("dgit-distro.$idistro.default-suite", 'RETURN-UNDEF') + if defined $idistro; + $isuite //= cfg('dgit.default.default-suite'); + foreach my $k (keys %opts_opt_map) { my $om = $opts_opt_map{$k}; @@ -6293,32 +6478,6 @@ sub finalise_opts_opts () { } } - parseopts_late_defaults(); -} - -if ($ENV{$fakeeditorenv}) { - git_slurp_config(); - quilt_fixup_editor(); -} - -parseopts(); -check_env_sanity(); -git_slurp_config(); - -print STDERR "DRY RUN ONLY\n" if $dryrun_level > 1; -print STDERR "DAMP RUN - WILL MAKE LOCAL (UNSIGNED) CHANGES\n" - if $dryrun_level == 1; -if (!@ARGV) { - print STDERR $helpmsg or die $!; - exit 8; -} -my $cmd = shift @ARGV; -$cmd =~ y/-/_/; - -my $pre_fn = ${*::}{"pre_$cmd"}; -$pre_fn->() if $pre_fn; - -sub parseopts_late_defaults () { if (!defined $rmchanges) { local $access_forpush; $rmchanges = access_cfg_bool(0, 'rm-old-changes'); @@ -6354,6 +6513,28 @@ sub parseopts_late_defaults () { } } +if ($ENV{$fakeeditorenv}) { + git_slurp_config(); + quilt_fixup_editor(); +} + +parseopts(); +check_env_sanity(); +git_slurp_config(); + +print STDERR "DRY RUN ONLY\n" if $dryrun_level > 1; +print STDERR "DAMP RUN - WILL MAKE LOCAL (UNSIGNED) CHANGES\n" + if $dryrun_level == 1; +if (!@ARGV) { + print STDERR $helpmsg or die $!; + exit 8; +} +my $cmd = shift @ARGV; +$cmd =~ y/-/_/; + +my $pre_fn = ${*::}{"pre_$cmd"}; +$pre_fn->() if $pre_fn; + my $fn = ${*::}{"cmd_$cmd"}; $fn or badusage "unknown operation $cmd"; $fn->();