From 1c640671a7ee8d30ee80c393822fa02ef3da0031 Mon Sep 17 00:00:00 2001 From: Ian Jackson Date: Wed, 29 Jul 2009 20:06:45 +0100 Subject: [PATCH 1/1] timestamp processing --- yarrg/Commods.pm | 11 +++++++++-- yarrg/commod-results-processor | 9 +++------ yarrg/commod-update-receiver | 18 ++++++++++++++---- yarrg/convert.c | 29 +++++++++++++++++++++++++++++ yarrg/database-info-fetch | 9 +++++++++ 5 files changed, 64 insertions(+), 12 deletions(-) diff --git a/yarrg/Commods.pm b/yarrg/Commods.pm index 9b190c4..67277c2 100644 --- a/yarrg/Commods.pm +++ b/yarrg/Commods.pm @@ -32,7 +32,7 @@ BEGIN { our ($VERSION, @ISA, @EXPORT, @EXPORT_OK, %EXPORT_TAGS); $VERSION = 1.00; @ISA = qw(Exporter); - @EXPORT = qw(&parse_info1 &rsync_master + @EXPORT = qw(&parse_info_clientside &parse_info_serverside &parse_info_serverside_ocean %oceans %commods %clients %routes %route_mysteries &parse_pctb_commodmap %pctb_commodmap @pctb_commodmap @@ -40,7 +40,7 @@ BEGIN { &pipethrough_prep &pipethrough_run &pipethrough_run_along &pipethrough_run_finish &pipethrough_run_gzip - &cgipostform); + &cgipostform &yarrgpostform); %EXPORT_TAGS = ( ); @EXPORT_OK = qw(); @@ -252,6 +252,13 @@ sub pipethrough_run_gzip ($) { pipethrough_run($_[0],undef,'gzip','gzip'); } +sub yarrgpostform ($$) { + my ($ua, $form) = @_; + my $dest= $ENV{'YPPSC_YARRG_YARRG'}; + get_our_version($form, 'client'); + return cgipostform($ua, "$dest/commod-update-receiver", $form); +} + sub cgipostform ($$$) { my ($ua, $url, $form) = @_; my $req= HTTP::Request::Common::POST($url, diff --git a/yarrg/commod-results-processor b/yarrg/commod-results-processor index 7cd4e04..e53bbb5 100755 --- a/yarrg/commod-results-processor +++ b/yarrg/commod-results-processor @@ -429,15 +429,14 @@ sub gzip ($) { return pipethrough_run($tf,undef,'gzip','gzip'); } -sub main__uploadyaarg () { +sub main__uploadyarrg () { my %o; parse_info_clientside(); $o{'ocean'}= $ENV{'YPPSC_OCEAN'} or die; $o{'island'}= $ENV{'YPPSC_ISLAND'} or die; - $o{'timestamp'}= 10; - get_our_version(\%o, 'client'); + $o{'timestamp'}= $ENV{'YPPSC_DATA_TIMESTAMP'} or die; my $tf= pipethrough_prep(); write_tsv($tf); @@ -446,9 +445,7 @@ sub main__uploadyaarg () { Content_Type => 'application/octet-stream', Content => $oz ]; - my $dest= $ENV{'YPPSC_YARRG_YARRG'}; - - my $respcontent= cgipostform($ua, "$dest/commod-update-receiver", \%o); + my $respcontent= yarrgpostform($ua, \%o); $respcontent =~ m/^OK\b/ or die "$respcontent ?"; } diff --git a/yarrg/commod-update-receiver b/yarrg/commod-update-receiver index 70cf36a..16a9ac6 100755 --- a/yarrg/commod-update-receiver +++ b/yarrg/commod-update-receiver @@ -25,13 +25,17 @@ # are used without permission. This program is not endorsed or # sponsored by Three Rings. - -# Uploads contain: -# ocean canonical mixed case -# island canonical mixed case +# All calls contain: # clientname "ypp-sc-tools" # clientversion 2.1-g2e06a26 [from git-describe --tags HEAD] # clientfixes "lastpage" [space separated list] +# +# Timestamp requests contain: +# requesttimestamp +# +# Uploads contain: +# ocean canonical mixed case +# island canonical mixed case # data filename=deduped.tsv.gz output of yarrg --tsv @@ -84,6 +88,12 @@ foreach my $bug (@$clientinfo) { $o{'clientversion'}= must_param('clientversion', "^(\\d[-+._0-9a-zA-Z]+)\$"); +if (param('requesttimestamp')) { + my $now= time; defined $now or die; + print header(-type=>'text/plain', -charset=>'us-ascii'), "OK $now.\n"; + exit(0); +} + $o{'ocean'}= must_param('ocean', $re_any); $o{'island'}= must_param('island', $re_any); diff --git a/yarrg/convert.c b/yarrg/convert.c index 42467f2..49fc3ed 100644 --- a/yarrg/convert.c +++ b/yarrg/convert.c @@ -152,6 +152,32 @@ static void rsync_core(const char *stem, const char *suffix, void fetch_with_rsync_gz(const char *stem) { rsync_core(stem,".gz",""); } void fetch_with_rsync(const char *stem) { rsync_core(stem,"","z"); } +static void get_timestamp(void) { + FILE *tf; + pid_t child; + sysassert( tf= tmpfile() ); + + sysassert( (child= fork()) != -1 ); + if (!child) { + sysassert( dup2(fileno(tf),1)==1 ); + EXECLP_HELPER("database-info-fetch","timestamp",(char*)0); + } + waitpid_check_exitstatus(child,"timestamp request",0); + + sysassert( fseek(tf,0,SEEK_SET) == 0 ); + static char lbuf[30]; + int l= fread(lbuf,1,sizeof(lbuf),tf); + sysassert( !ferror(tf) ); + assert( feof(tf) ); + assert( l>1 ); + l--; + assert( lbuf[l]=='\n' ); + lbuf[l]= 0; + + sysassert(! setenv("YPPSC_DATA_TIMESTAMP",lbuf,1) ); + fclose(tf); +} + static void set_server(const char *envname, const char *defprotocol, const char *defvalue, const char *defvalue_test, const char *userspecified, @@ -323,6 +349,9 @@ int main(int argc, char **argv) { o_screenshot_fn= masprintf("%s/_pages.ppm.gz", get_vardir()); /* Actually do the work */ + + if ((o_flags & ff_upload) && (o_flags & ff_use_yarrg)) + get_timestamp(); canon_colour_prepare(); diff --git a/yarrg/database-info-fetch b/yarrg/database-info-fetch index c56fda4..1e789b8 100755 --- a/yarrg/database-info-fetch +++ b/yarrg/database-info-fetch @@ -184,6 +184,15 @@ sub main__island () { }); } +sub main__timestamp () { + my %o; + $o{'requesttimestamp'}= '1'; + my $respcontent= yarrgpostform($ua, \%o); + $respcontent =~ m/^OK ([1-9]\d{1,20})\./ or die "$respcontent ?"; + print "$1\n"; + exit(0); +} + sub main__sunshinewidget () { print <