From 3940b368db5ffb18b61b6857c22498b46094fca1 Mon Sep 17 00:00:00 2001 From: Ian Jackson Date: Fri, 25 Oct 2013 17:03:20 +0100 Subject: [PATCH] Abolish the sshdakls method and replace it with sshpsql: that is, ssh (to coccia, by default) and run sql commands on the ftpmaster database. This is faster and has fewer bugs but is vulnerable to db schema changes. Closes:#726955. Closes:#720170. Closes:#720176. --- debian/changelog | 4 ++ dgit | 101 +++++++++++++++++++++++++++++++++-------------- dgit.1 | 16 ++++---- 3 files changed, 84 insertions(+), 37 deletions(-) diff --git a/debian/changelog b/debian/changelog index 41374bb0..fb187093 100644 --- a/debian/changelog +++ b/debian/changelog @@ -6,6 +6,10 @@ dgit (0.17~experimental4) experimental; urgency=low order so that we use the correct Description (the _source one, not the one from sbuild which didn't get e.g. -v). * Clarify the manpage's comments about orig tarballs. Closes: #723605. + * Abolish the sshdakls method and replace it with sshpsql: that is, ssh + (to coccia, by default) and run sql commands on the ftpmaster + database. This is faster and has fewer bugs but is vulnerable to db + schema changes. Closes:#726955. Closes:#720170. Closes:#720176. * Fix the "shellquote" command to work properly. The bugs in it ought not to have caused any real trouble in previous versions of dgit. diff --git a/dgit b/dgit index e5dbabba..a75a0711 100755 --- a/dgit +++ b/dgit @@ -450,9 +450,8 @@ our %defcfg = ('dgit.default.distro' => 'debian', 'dgit-distro.debian.git-path' => '/git/dgit-repos/repos', 'dgit-distro.debian.git-check' => 'ssh-cmd', 'dgit-distro.debian.git-create' => 'ssh-cmd', - 'dgit-distro.debian.sshdakls-host' => 'coccia.debian.org', - 'dgit-distro.debian.sshdakls-dir' => - '/srv/ftp-master.debian.org/ftp/dists', + 'dgit-distro.debian.sshpsql-host' => 'coccia.debian.org', + 'dgit-distro.debian.sshpsql-dbname' => 'service=projectb', 'dgit-distro.debian.upload-host' => 'ftp-master', # for dput 'dgit-distro.debian.mirror' => 'http://ftp.debian.org/debian/'); @@ -572,9 +571,9 @@ sub archive_query ($) { if (!defined $query) { my $distro = access_distro(); if ($distro eq 'debian') { - $query = "sshdakls:". - access_someuserhost('sshdakls').':'. - access_cfg('sshdakls-dir'); + $query = "sshpsql:". + access_someuserhost('sshpsql').':'. + access_cfg('sshpsql-dbname'); } else { $query = "madison:$distro"; } @@ -600,28 +599,6 @@ sub archive_query_madison ($$) { return madison_parse($rmad); } -sub archive_query_sshdakls ($$) { - my ($proto,$data) = @_; - $data =~ s/:.*// or badcfg "invalid sshdakls method string \`$data'"; - my $dakls = cmdoutput - access_cfg_ssh, $data, qw(dak ls -asource),"-s$isuite",$package; - return madison_parse($dakls); -} - -sub canonicalise_suite_sshdakls ($$) { - my ($proto,$data) = @_; - $data =~ m/:/ or badcfg "invalid sshdakls method string \`$data'"; - my @cmd = - (access_cfg_ssh, $`, - "set -e; cd $';". - " if test -h $isuite; then readlink $isuite; exit 0; fi;". - " if test -d $isuite; then echo $isuite; exit 0; fi;". - " exit 1"); - my $dakls = cmdoutput @cmd; - failedcmd @cmd unless $dakls =~ m/^\w/; - return $dakls; -} - sub madison_parse ($) { my ($rmad) = @_; my @out; @@ -646,6 +623,7 @@ sub madison_parse ($) { } sub canonicalise_suite_madison ($$) { + # madison canonicalises for us my @r = archive_query_madison($_[0],$_[1]); @r or fail "unable to canonicalise suite using package $package". @@ -654,12 +632,77 @@ sub canonicalise_suite_madison ($$) { return $r[0][2]; } +sub sshpsql ($$) { + my ($data,$sql) = @_; + $data =~ m/:/ or badcfg "invalid sshpsql method string \`$data'"; + my ($userhost,$dbname) = ($`,$'); #'; + my @rows; + my @cmd = (access_cfg_ssh, $userhost, + shellquote qw(psql -A), $dbname, qw(-c), $sql); + printcmd(\*DEBUG,$debugprefix."|",@cmd) if $debug>0; + open P, "-|", @cmd or die $!; + while (

) { + chomp or die; + printdebug("$debugprefix>|$_|\n"); + push @rows, $_; + } + $!=0; $?=0; close P or die "$! $?"; + @rows or die; + my $nrows = pop @rows; + $nrows =~ s/^\((\d+) rows?\)$/$1/ or die "$nrows ?"; + @rows == $nrows+1 or die "$nrows ".(scalar @rows)." ?"; + @rows = map { [ split /\|/, $_ ] } @rows; + my $ncols = scalar @{ shift @rows }; + die if grep { scalar @$_ != $ncols } @rows; + return @rows; +} + +sub sql_injection_check { + foreach (@_) { die "$_ $& ?" if m/[']/; } +} + +sub archive_query_sshpsql ($$) { + my ($proto,$data) = @_; + sql_injection_check $isuite, $package; + my @rows = sshpsql($data, <[0],$b->[0]) } @rows; + @rows = map { + my ($vsn,$component,$filename) = @$_; + [ $vsn, "/pool/$component/$filename" ]; + } @rows; + return @rows; +} + +sub canonicalise_suite_sshpsql ($$) { + my ($proto,$data) = @_; + sql_injection_check $isuite; + my @rows = sshpsql($data, <[0] } @rows; + fail "unknown suite $isuite" unless @rows; + die "ambiguous $isuite: @rows ?" if @rows>1; + return $rows[0]; +} + sub canonicalise_suite () { return if defined $csuite; fail "cannot operate on $isuite suite" if $isuite eq 'UNRELEASED'; $csuite = archive_query('canonicalise_suite'); if ($isuite ne $csuite) { - # madison canonicalises for us progress "canonical suite name for $isuite is $csuite"; } } diff --git a/dgit.1 b/dgit.1 index c3caa496..75a43a39 100644 --- a/dgit.1 +++ b/dgit.1 @@ -613,11 +613,11 @@ on the dgit command line. .TP .BI dgit-distro. distro .archive-query-default-component .TP -.BI dgit-distro. distro .sshdakls-user +.BI dgit-distro. distro .sshpsql-user .TP -.BI dgit-distro. distro .sshdakls-host +.BI dgit-distro. distro .sshpsql-host .TP -.BI dgit-distro. distro .sshdakls-dir +.BI dgit-distro. distro .sshpsql-dbname .TP .BI dgit-distro. distro .ssh .TP @@ -659,7 +659,8 @@ by default. Debian does not have a working rmadison server, so to find out what version of a package is in the archive, or to canonicalise suite -names, we ssh directly into the ftpmaster server. +names, we ssh directly into the ftpmaster server and run psql there to +access the database. The mechanism for checking for and creating per-package repos on alioth is a hideous bodge. One consequence is that dgit currently @@ -705,10 +706,9 @@ as the build host. The option parser requires values to be cuddled to the option name. -dgit assumes knowledge of the archive layout. There appears to be no -sane way to find the path in the archive pool of the .dsc for a -particular suite. I'm assured that the archive layout is a -`well known algorithm' by now. +dgit assumes knowledge of the archive database. (The information dgit +needs is not currently available via any public online service with a +well-defined interface, let alone a secure one.) --dry-run does not always work properly, as not doing some of the git fetches may result in subsequent actions being different. Doing a -- 2.30.2