X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=dgit.git;a=blobdiff_plain;f=dgit;h=edc3c2276294c9e86b36a50c11924dcc840aff6e;hp=9411afc0cdc53e488131c3244778f101025cf8e9;hb=00d226d8688ae224d6b0b95e75c801fc9812195a;hpb=eeb4770ff4b9a82b4cad0388c6ca165044aaa5d8 diff --git a/dgit b/dgit index 9411afc0..edc3c227 100755 --- a/dgit +++ b/dgit @@ -1,4 +1,22 @@ #!/usr/bin/perl -w +# dgit +# Integration between git and Debian-style archives +# +# Copyright (C)2013 Ian Jackson +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + use strict; use IO::Handle; @@ -8,20 +26,39 @@ use Dpkg::Control::Hash; use File::Path; use POSIX; -open DEBUG, ">&STDERR" or die $!; - -our $pdo = 'http://packages.debian.org/'; -#our $mirror = 'http://mirror.relativity.greenend.org.uk/mirror/debian-ftp/'; +our $mirror = 'http://mirror.relativity.greenend.org.uk/mirror/debian-ftp/'; our $suite = 'sid'; -our $package = '2vcard'; +our $package; + +our $sign = 1; +our $dryrun = 0; +our $changesfile; our $aliothname = 'iwj@git.debian.org'; -our $aliothpath = '/git/dgit-test'; -our $alioth_git = 'git+ssh://$aliothname/$aliothpath'; +our $aliothpath = '/git/dgit-repos'; +our $alioth_git = "git+ssh://$aliothname/$aliothpath"; our $alioth_sshtestbodge = [$aliothname,$aliothpath]; -sub mainbranch () { return "$suite"; } -sub uploadingbranch () { return "uploading/$suite"; } +our (@git) = qw(git); +our (@dget) = qw(dget); +our (@dput) = qw(dput); +our (@debsign) = qw(debsign); +our $keyid; + +open DEBUG, ">/dev/null" or die $!; + +our %opts_opt_map = ('dget' => \@dget, + 'dput' => \@dput, + 'debsign' => \@debsign); + +our $remotename = 'dgit'; +our $ourdscfield = 'Vcs-Git-Master'; +our $branchprefix = 'dgit'; + +sub uploadbranch () { return "$branchprefix/$suite"; } +sub lref ($) { return "refs/heads/$_[0]"; } +sub rref ($) { return "refs/remotes/$remotename/$_[0]"; } +sub debiantag ($) { return "debian/$_[0]"; } our $ua; @@ -30,99 +67,195 @@ sub url_get { $ua = LWP::UserAgent->new(); $ua->env_proxy; } -print DEBUG "fetching @_...\n"; + print "fetching @_...\n"; my $r = $ua->get(@_) or die $!; die "$_[0]: ".$r->status_line."; failed.\n" unless $r->is_success; return $r->decoded_content(); } -our ($dscdata,$dscurl); +our ($dscdata,$dscurl,$dsc); + +sub printcmd { + my $fh = shift @_; + my $intro = shift @_; + print $fh $intro or die $!; + local $_; + foreach my $a (@_) { + $_ = $a; + if (s{['\\]}{\\$&}g || m{\s} || m{[^-_./0-9a-z]}i) { + print $fh " '$_'" or die $!; + } else { + print $fh " $_" or die $!; + } + } + print $fh "\n" or die $!; +} + +sub runcmd { + printcmd(\*DEBUG,"+",@_); + $!=0; $?=0; + die "@_ $! $?" if system @_; +} + +sub cmdoutput_errok { + printcmd(\*DEBUG,"|",@_); + open P, "-|", @_ or die $!; + my $d; + $!=0; $?=0; + { local $/ = undef; $d =

; } + die if P->error; + close P or return undef; + chomp $d; + return $d; +} + +sub cmdoutput { + my $d = cmdoutput_errok @_; + defined $d or die "@_ $? $!"; + return $d; +} + +sub dryrun_report { + printcmd(\*STDOUT,"#",@_); +} + +sub runcmd_ordryrun { + if (!$dryrun) { + runcmd @_; + } else { + dryrun_report @_; + } +} + +sub parsecontrol { + my $c = Dpkg::Control::Hash->new(); + $c->load(@_) or return undef; + return $c; +} + +sub parsechangelog { + my $c = Dpkg::Control::Hash->new(); + my $p = new IO::Handle; + open $p, '-|', qw(dpkg-parsechangelog) or die $!; + $c->parse($p); + $?=0; $!=0; close $p or die "$! $?"; + return $c; +} + +our $rmad; + +sub askmadison () { + $rmad ||= cmdoutput qw(rmadison -asource),"-s$suite",$package; + $rmad =~ m/^ \s*( [^ \t|]+ )\s* \| + \s*( [^ \t|]+ )\s* \| + \s*( [^ \t|]+ )\s* \| + \s*( [^ \t|]+ )\s* /x or die "$rmad $?"; + $1 eq $package or die "$rmad $package ?"; + my $vsn = $2; + if ($suite ne $3) { + # madison canonicalises for us + print "canonical suite name for $suite is $3\n"; + $suite = $3; + } + $4 eq 'source' or die "$rmad ?"; + return $vsn; +} + +sub canonicalise_suite () { + askmadison(); +} sub get_archive_dsc () { - my $pdourl = "$pdo/source/$suite/$package"; - my $pdodata = url_get($pdourl); - # FFS. The Debian archive has no sane way to find what - # version is currently the tip in any branch (aka, what - # is the current version in any suite). - $pdodata =~ m{ - Download\ \Q$package\E .* - \\2\ - }msx - or die "screenscraping of $pdourl failed :-(\n"; - $dscurl = $1; -print DEBUG Dumper($pdodata, $&, $dscurl); + my $vsn = askmadison(); + # fixme madison does not show us the component + my $prefix = substr($package, 0, $package =~ m/^l/ ? 4 : 1); + $dscurl = "$mirror/pool/main/$prefix/$package/${package}_$vsn.dsc"; $dscdata = url_get($dscurl); my $dscfh = new IO::File \$dscdata, '<' or die $!; -print DEBUG Dumper($dscdata, $dscfh); - my $dscp = Dpkg::Control::Hash->new(allow_pgp=>1); - $dscp->parse($dscfh, 'dsc') or die "parsing of $dscurl failed\n"; -# my $dscf = $dscp->{'fields'}; -my $dscf=$dscp; -print DEBUG Dumper($dscp,$dscf); - my $fmt = $dscf->{Format}; + print DEBUG Dumper($dscdata); + $dsc = Dpkg::Control::Hash->new(allow_pgp=>1); + $dsc->parse($dscfh, 'dsc') or die "parsing of $dscurl failed\n"; + print DEBUG Dumper($dsc); + my $fmt = $dsc->{Format}; die "unsupported format $fmt, sorry\n" unless $fmt eq '1.0'; - return $dscf; } sub check_for_git () { # returns 0 or 1 - open P, "ssh $alioth_sshtestbodge->[0] '". - "set -e; cd /git/dgit-test;". - "if test -d $package.git; then echo 1; else echo 0; fi". - "' |" - or die $!; + my $cmd= + "ssh $alioth_sshtestbodge->[0] '". + " set -e; cd $alioth_sshtestbodge->[1];". + " if test -d $package.git; then echo 1; else echo 0; fi". + "'"; + print DEBUG "$cmd\n"; + open P, "$cmd |" or die $!; $!=0; $?=0; my $r =

; close P; + print DEBUG ">$r<\n"; die "$r $! $?" unless $r =~ m/^[01]$/; - return !!$r; + return $r+0; } -sub runcmd { - $!=0; $?=0; - die "$! $?" if system @_; -} +our ($dsc_hash,$upload_hash); -our ($dsc,$dsc_hash,$lastupl_hash); +our $ud = '.git/dgit/unpack'; -sub generate_commit_from_dsc () { - my $ud = '.git/dgit/unpack'; - remove_tree($ud); +sub prep_ud () { + rmtree($ud); mkpath '.git/dgit'; mkdir $ud or die $!; +} + +sub mktree_in_ud_from_only_subdir () { + # changes into the subdir + my (@dirs) = <*/.>; + die unless @dirs==1; + $dirs[0] =~ m#^([^/]+)/\.$# or die; + my $dir = $1; + chdir $dir or die "$dir $!"; + die if stat '.git'; + die $! unless $!==&ENOENT; + runcmd qw(git init -q); + rmtree('.git/objects'); + symlink '../../../../objects','.git/objects' or die $!; + runcmd @git, qw(add -Af); + my $tree = cmdoutput @git, qw(write-tree); + chomp $tree; $tree =~ m/^\w+$/ or die "$tree ?"; + return ($tree,$dir); +} + +sub dsc_files () { + map { + m/^\w+ \d+ (\S+)$/ or die "$_ ?"; + $1; + } grep m/\S/, split /\n/, ($dsc->{'Checksums-Sha256'} || $dsc->{Files}); +} + +sub is_orig_file ($) { + local ($_) = @_; + m/\.orig\.tar\.\w+$/; +} + +sub generate_commit_from_dsc () { + prep_ud(); chdir $ud or die $!; my @files; - foreach (split /\n/, ($dsch->{Checksums-Sha256} || $dsch->{Files})) { - next unless m/\S/; - m/^\w+ \d+ (\S+)$/ or die "$_ ?"; - my $f = $1; + foreach my $f (dsc_files()) { die if $f =~ m#/|^\.|\.dsc$|\.tmp$#; push @files, $f; link "../../../$f", $f or $!==&ENOENT or die "$f $!"; } - runcmd qw(dget --), $dscurl; - foreach my $f (grep { m/\.tar\.gz$/ } @files) { - link $f, "../../../$f" + runcmd @dget, qw(--), $dscurl; + foreach my $f (grep { is_orig_file($_) } @files) { + link $f, "../../../../$f" or $!==&EEXIST or die "$f $!"; } - my (@dirs) = <*/.>; - die unless @dirs==1; - $dirs[0] =~ m#^([^/]+)/\.$# or die; - my $dir = $1; - chdir $dir or die "$dir $!"; - die if stat '.git'; - die $! unless $!==&ENOENT; - runcmd qw(git init); - remove_tree(.git/objects); - symlink '../../objects','.git/objects' or die $!; - runcmd qw(git add -Af); - my $tree = cmdoutput qw(git write-tree); - chomp $tree or die; + my ($tree,$dir) = mktree_in_ud_from_only_subdir(); runcmd qw(sh -ec), 'dpkg-parsechangelog >../changelog.tmp'; - my $clogp = Dpkg::Control::Hash->new(); - $clogp->parse('../changelog.tmp','changelog'); + my $clogp = parsecontrol('../changelog.tmp','changelog') or die; my $date = cmdoutput qw(date), '+%s %z', qw(-d),$clogp->{Date}; my $author = $clogp->{Maintainer}; $author =~ s#,.*##ms; @@ -130,82 +263,333 @@ sub generate_commit_from_dsc () { $authline =~ m/^[^<>]+ \<\S+\> \d+ [-+]\d+$/ or die $authline; open C, ">../commit.tmp" or die $!; print C "tree $tree\n" or die $!; - print C "parent $lastupl_hash\n" or die $! if defined $lastupl_hash; + print C "parent $upload_hash\n" or die $! if $upload_hash; print C <{Changes} -# generated by dgit + +# imported by dgit from the archive END close C or die $!; - my $commithash = runcmd qw(git hash-object -w -t commit ../commit.tmp); - chdir '../../..' or die $!; - remove_tree($ud); - cmdoutput qw(git log -n1), $commithash; - return $commithash; + print "synthesised git commit from .dsc $clogp->{Version}\n"; + my $commithash = cmdoutput @git, qw(hash-object -w -t commit ../commit.tmp); + chdir '../../../..' or die $!; + cmdoutput @git, qw(update-ref -m),"dgit synthesise $clogp->{Version}", + 'DGIT_ARCHIVE', $commithash; + cmdoutput @git, qw(log -n2), $commithash; + # ... gives git a chance to complain if our commit is malformed + my $outputhash = $commithash; + if ($upload_hash) { + chdir "$ud/$dir" or die $!; + runcmd @git, qw(reset --hard), $upload_hash; + runcmd qw(sh -ec), 'dpkg-parsechangelog >>../changelogold.tmp'; + my $oldclogp = Dpkg::Control::Hash->new(); + $oldclogp->parse('../changelogold.tmp','previous changelog') or die; + my $vcmp = + version_compare_string($oldclogp->{Version}, $clogp->{Version}); + if ($vcmp < 0) { + # git upload/ is earlier vsn than archive, use archive + } elsif ($vcmp >= 0) { + print STDERR <{Version} (older) +Last allegedly pushed/uploaded: $oldclogp->{Version} (newer or same) +Perhaps the upload is stuck in incoming. Using the version from git. +END + } else { + die "version in archive is same as version in git". + " to-be-uploaded (upload/) branch but archive". + " version hash no commit hash?!\n"; + } + chdir '../../../..' or die $!; + } + rmtree($ud); + return $outputhash; +} + +sub ensure_we_have_orig () { + foreach my $f (dsc_files()) { + next unless is_orig_file($f); + if (stat "../$f") { + die "$f ?" unless -f _; + } else { + die "$f $!" unless $!==&ENOENT; + } + my $origurl = $dscurl; + $origurl =~ s{/[^/]+$}{}; + $origurl .= "/$f"; + die "$f ?" unless $f =~ m/^${package}_/; + die "$f ?" if $f =~ m#/#; + runcmd_ordryrun qw(sh -ec),'cd ..; exec "$@"','x', + @dget,'--',$origurl; + } +} + +sub rev_parse ($) { + return cmdoutput @git, qw(rev-parse), "$_[0]~0"; +} + +sub is_fast_fwd ($$) { + my ($ancestor,$child) = @_; + my $mb = cmdoutput @git, qw(merge-base), $dsc_hash, $upload_hash; + return rev_parse($mb) eq rev_parse($ancestor); } sub fetch_from_archive () { + # ensures that rref(uploadbranch()) is what is actually in the archive, + # one way or another + my $upload_ref = rref(uploadbranch()); + $!=0; $upload_hash = + cmdoutput_errok @git, qw(show-ref --heads), $upload_ref; + die $! if $!; + die $? unless ($?==0 && chomp $upload_hash) + or ($?==256 && !length $upload_hash); + $upload_hash ||= ''; my $hash; if (defined $dsc_hash) { + die "missing git history even though dsc has hash" + unless $upload_hash; $hash = $dsc_hash; + ensure_we_have_orig(); } else { $hash = generate_commit_from_dsc(); } - cmdoutput qw(git update-ref FETCH_HEAD) $hash; + if ($upload_hash) { + die "not fast forward on last upload branch!". + " (archive's version left in DGIT_ARCHIVE)" + unless is_fast_fwd($dsc_hash, $upload_hash); + } + if ($upload_hash ne $hash) { + my @upd_cmd = (@git, qw(update-ref -m), 'dgit fetch', $upload_ref, $hash); + if (!$dryrun) { + cmdoutput @upd_cmd; + } else { + dryrun_report @upd_cmd; + } + } } -#sub combine () { -# if ( - -# runcmd qw(git write-tree - - - runcmd qw(mkdir -p ''); -# chdir '.git/dgit/unpack' or die $!; - - -# with_tmpdir($td,{ - -# }); - -# } - -# open P, "-|", qw(git rev-parse --), $dsc_hash; - -#} - sub clone () { - $dsc = get_archive_dsc(); - $dsc_hash = $dsc->{'Vcs-git-master'}; + die "dry run makes no sense with clone" if $dryrun; + get_archive_dsc(); + $dsc_hash = $dsc->{$ourdscfield}; if (defined $dsc_hash) { $dsc_hash =~ m/\w+/ or die "$dsc_hash $?"; $dsc_hash = $&; + print "last upload to archive specified git hash\n"; + } else { + print "last upload to archive has NO git hash\n"; } my $dstdir = "$package"; + my $branch = uploadbranch(); if (check_for_git()) { - runcmd qw(git clone --origin dgit -b), $suite, '--', - $alioth_git, $dstdir; + print "cloning existing git history\n"; + runcmd @git, qw(clone --origin),$remotename, qw(-b), $branch, '--', + "$alioth_git/$package.git", $dstdir; chdir "$dstdir" or die "$dstdir $!"; - update_from_archive(); + fetch_from_archive(); + runcmd @git, qw(reset --hard), rref(uploadbranch()); } else { + print "starting new git history\n"; mkdir $dstdir or die "$dstdir $!"; chdir "$dstdir" or die "$dstdir $!"; - runcmd qw(git init); - open H, "> .git/refs/HEAD" or die $!; - print H "ref: refs/heads/$suite\n" or die $!; + runcmd @git, qw(init -q); + open H, "> .git/HEAD" or die $!; + print H "ref: ".lref(uploadbranch())."\n" or die $!; close H or die $!; - runcmd qw(git remote add dgit), $alioth_git; - runcmd "git config branch.$suite.remote dgit"; - runcmd "git config branch.$suite.merge refs/heads/$suite"; - update_from_archive(); + runcmd @git, qw(remote add), $remotename, "$alioth_git/$package.git"; + runcmd "git config branch.$branch.remote $remotename"; + runcmd "git config branch.$branch.merge ".lref(uploadbranch()); + my $newhash = generate_commit_from_dsc(); + runcmd @git, qw(reset --hard), $newhash; } + print "ready for work in $dstdir\n"; } sub fetch () { - my ($archive_or_mirror, $suite, $package) = @_; - my $dsc = get_archive_dsc(); + get_archive_dsc(); + if (check_for_git()) { + runcmd_ordryrun @git, qw(fetch -p),$remotename, + '+refs/heads/*:refs/remotes/origin/*'; + } + fetch_from_archive(); } - -print Dumper(get_archive_dsc()); + +sub pull () { + fetch(); + runcmd_ordryrun @git, qw(merge -m),"Merge from $suite [dgit]", + lref(uploadbranch()); +} + +sub dopush () { + runcmd @git, qw(diff --quiet HEAD); + my $clogp = parsechangelog(); + $package = $clogp->{Source}; + my $dscfn = "${package}_$clogp->{Version}.dsc"; + stat "../$dscfn" or die "$dscfn $!"; + $dsc = parsecontrol("../$dscfn"); + prep_ud(); + chdir $ud or die $!; + print "checking that $dscfn corresponds to HEAD\n"; + runcmd qw(dpkg-source -x --), "../../../../$dscfn"; + my ($tree,$dir) = mktree_in_ud_from_only_subdir(); + chdir '../../../..' or die $!; + runcmd @git, qw(diff --exit-code), $tree; +#fetch from alioth +#do fast forward check and maybe fake merge +# if (!is_fast_fwd(mainbranch +# runcmd @git, qw(fetch -p ), "$alioth_git/$package.git", +# map { lref($_).":".rref($_) } +# (uploadbranch()); + $dsc->{$ourdscfield} = rev_parse('HEAD'); + $dsc->save("../$dscfn.tmp") or die $!; + if (!$dryrun) { + rename "../$dscfn.tmp","../$dscfn" or die "$dscfn $!"; + } else { + print "[new .dsc left in $dscfn.tmp]\n"; + } + if (!$changesfile) { + my $pat = "../${package}_$clogp->{Version}_*.changes"; + my @cs = glob $pat; + die "$pat ?" unless @cs==1; + ($changesfile) = @cs; + } + my $tag = debiantag($dsc->{Version}); + if (!check_for_git()) { + runcmd_ordryrun qw(ssh),$alioth_sshtestbodge->[0], + "set -e; cd $alioth_sshtestbodge->[1];". + " mkdir -p $package.git;". + " cd $package.git;". + " if ! test -d objects; then git init --bare; fi"; + } + runcmd_ordryrun @git, qw(push),$remotename,"HEAD:".lref(uploadbranch()); + if ($sign) { + my @tag_cmd = (@git, qw(tag -s -m), + "Release $dsc->{Version} for $suite [dgit]"); + push @tag_cmd, qw(-u),$keyid if defined $keyid; + push @tag_cmd, $tag; + runcmd_ordryrun @tag_cmd; + my @debsign_cmd = @debsign; + push @debsign_cmd, "-k$keyid" if defined $keyid; + push @debsign_cmd, $changesfile; + runcmd_ordryrun @debsign_cmd; + } + runcmd_ordryrun @git, qw(push),$remotename,"refs/tags/$tag"; + runcmd_ordryrun @dput, $changesfile; +} + +sub cmd_clone { + if (@ARGV==1) { + ($package) = @ARGV; + } elsif (@ARGV==2) { + ($package,$suite) = @ARGV; + } else { + die; + } + clone(); +} + +sub branchsuite () { + my $branch = cmdoutput_errok @git, qw(symbolic-ref HEAD); + chomp $branch; + if ($branch =~ m#^refs/heads/$branchprefix/([^/.]+)$#o) { + return $1; + } else { + return undef; + } +} + +sub fetchpullargs () { + my $clogp = parsechangelog(); + $package = $clogp->{Source}; + if (@ARGV==0) { + $suite = branchsuite(); + $suite ||= $clogp->{Distribution}; + canonicalise_suite(); + print "fetching from suite $suite\n"; + } elsif (@ARGV==1) { + ($suite) = @ARGV; + canonicalise_suite(); + } else { + die; + } +} + +sub cmd_fetch { + fetchpullargs(); + fetch(); +} + +sub cmd_pull { + fetchpullargs(); + pull(); +} + +sub cmd_push { + my $clogp = parsechangelog(); + $package = $clogp->{Source}; + if (@ARGV==0) { + $suite = $clogp->{Distribution}; + canonicalise_suite(); + } else { + die; + } + dopush(); +} + +sub cmd_build { + my $clogp = parsechangelog(); + $suite = $clogp->{Distribution}; + $package = $clogp->{Source}; + canonicalise_suite(); + runcmd_ordryrun + qw(git-buildpackage -us -uc --git-no-sign-tags), + "--git-debian-branch=".uploadbranch(), + @ARGV; +} + +sub parseopts () { + my $om; + while (@ARGV) { + last unless $ARGV[0] =~ m/^-/; + $_ = shift @ARGV; + last if m/^--?$/; + if (m/^--/) { + if (m/^--dry-run$/) { + $dryrun=1; + } elsif (m/^--no-sign$/) { + $sign=0; + } elsif (m/^--(\w+)=(.*)/s && ($om = $opts_opt_map{$1})) { + $om->[0] = $2; + } elsif (m/^--(\w+):(.*)/s && ($om = $opts_opt_map{$1})) { + push @$om, $2; + } else { + die "$_ ?"; + } + } else { + while (m/^-./s) { + if (s/^-n/-/) { + $dryrun=1; + } elsif (s/^-D/-/) { + open DEBUG, ">&STDERR" or die $!; + } elsif (s/^-c(.*=.*)//s) { + push @git, $1; + } elsif (s/^-C(.*)//s) { + $changesfile = $1; + } elsif (s/^-k(.*)//s) { + $keyid=$1; + } else { + die "$_ ?"; + } + } + } + } +} + +parseopts(); +die unless @ARGV; +my $cmd = shift @ARGV; +parseopts(); + +{ no strict qw(refs); &{"cmd_$cmd"}(); }