From 877b0ba6b2d72d7957e2a8015520b87371cba6c9 Mon Sep 17 00:00:00 2001 From: Ian Jackson Date: Wed, 29 Jul 2009 19:47:08 +0100 Subject: [PATCH] WIP Much rework for new upload arrangements and new name --- yarrg/Commods.pm | 43 +++++++++++++++----- yarrg/commod-results-processor | 18 +++----- yarrg/commod-update-receiver | 2 +- yarrg/database-info-fetch | 4 +- yarrg/db-idempotent-populate | 3 +- yarrg/{master-master.txt => master-info.txt} | 0 yarrg/update-master-info | 22 ++++++++++ 7 files changed, 66 insertions(+), 26 deletions(-) rename yarrg/{master-master.txt => master-info.txt} (100%) create mode 100755 yarrg/update-master-info diff --git a/yarrg/Commods.pm b/yarrg/Commods.pm index 955446f..9b190c4 100644 --- a/yarrg/Commods.pm +++ b/yarrg/Commods.pm @@ -32,7 +32,8 @@ BEGIN { our ($VERSION, @ISA, @EXPORT, @EXPORT_OK, %EXPORT_TAGS); $VERSION = 1.00; @ISA = qw(Exporter); - @EXPORT = qw(&parse_masters &parse_masters_ocean + @EXPORT = qw(&parse_info1 &rsync_master + &parse_info_serverside &parse_info_serverside_ocean %oceans %commods %clients %routes %route_mysteries &parse_pctb_commodmap %pctb_commodmap @pctb_commodmap &get_our_version &check_tsv_line @@ -58,7 +59,7 @@ our (%pctb_commodmap,@pctb_commodmap); my %colours; # eg $colours{'c'}{'black'}= $sources my @rawcm; # eg $rawcm[0]='fine rum'; $rawcm[1]='fine %c cloth' -sub parse_master_master1 ($$) { +sub parse_info1 ($$) { my ($mmfn,$src)= @_; my $mm= new IO::File $mmfn, 'r' or die "$mmfn $!"; my @ctx= (); @@ -131,14 +132,37 @@ sub parse_master_master1 ($$) { my $arch= $ocean->{$an}; push @allislands, sort keys %$arch; } - parse_master_map_route_islands($on, \@allislands, $routes); + parse_info_maproutes($on, \@allislands, $routes); foreach my $route (values %$routes) { - parse_master_map_route_islands($on, \@allislands, $route); + parse_info_maproutes($on, \@allislands, $route); } } } -sub parse_master_map_route_islands ($$$) { +sub parse_info_clientside () { + my $yarrg= $ENV{'YPPSC_YARRG_DICT_UPDATE'}; + return unless $yarrg; + my $master= fetch_with_rsync('info'); + parse_info1($master,'s'); + parse_info1('_local-info.txt','s'); +} + +sub fetch_with_rsync ($) { + my ($stem) = @_; + + my $rsync= $ENV{'YPPSC_YARRG_RSYNC'}; + $rsync= 'rsync' if !defined $rsync; + + my $local= "_master-$stem.txt"; + my $src= $ENV{'YPPSC_YARRG_DICT_UPDATE'}; + if ($src) { + my $remote= "$src/master-$stem.txt"; + $!=0; system 'rsync','-Lt','--',$remote,$local; + die "$? $!" if $! or $?; + } +} + +sub parse_info_maproutes ($$$) { my ($on, $allislands, $routemap) = @_;; foreach my $k (sort keys %$routemap) { my @ok= grep { index($_,$k) >= 0 } @$allislands; @@ -153,14 +177,13 @@ sub parse_master_map_route_islands ($$$) { } } -sub parse_masters () { - parse_master_master1('master-master.txt','s'); +sub parse_info_serverside () { + parse_info1('master-info.txt','s'); } -sub parse_masters_ocean ($) { +sub parse_info_serverside_ocean ($) { my ($oceanname) = @_; - parse_master_master1('master-master.txt','s'); die "unknown ocean $oceanname ?" unless exists $oceans{$oceanname}; - parse_master_master1("ocean-".(lc $oceanname).".txt",'s'); + parse_info1("ocean-".(lc $oceanname).".txt",'s'); } sub parse_pctb_commodmap () { diff --git a/yarrg/commod-results-processor b/yarrg/commod-results-processor index 827cac4..7cd4e04 100755 --- a/yarrg/commod-results-processor +++ b/yarrg/commod-results-processor @@ -310,17 +310,8 @@ sub read_newcommods ($) { } sub refresh_newcommods() { - my $rsync= $ENV{'YPPSC_YARRG_RSYNC'}; - $rsync= 'rsync' if !defined $rsync; - - my $local= "_master-newcommods.txt"; - my $src= $ENV{'YPPSC_YARRG_DICT_UPDATE'}; - if ($src) { - my $remote= "$src/master-newcommods.txt"; - $!=0; system 'rsync','-Lt','--',$remote,$local; - die "$? $!" if $! or $?; - } - read_newcommods($local); + my $master= fetch_with_rsync('newcommods'); + read_newcommods($master); read_newcommods('_local-newcommods.txt'); } @@ -440,6 +431,9 @@ sub gzip ($) { sub main__uploadyaarg () { my %o; + + parse_info_clientside(); + $o{'ocean'}= $ENV{'YPPSC_OCEAN'} or die; $o{'island'}= $ENV{'YPPSC_ISLAND'} or die; $o{'timestamp'}= 10; @@ -452,7 +446,7 @@ sub main__uploadyaarg () { Content_Type => 'application/octet-stream', Content => $oz ]; - my $dest= $ENV{'YPPSC_YAARG_SUBMIT'}; + my $dest= $ENV{'YPPSC_YARRG_YARRG'}; my $respcontent= cgipostform($ua, "$dest/commod-update-receiver", \%o); $respcontent =~ m/^OK\b/ or die "$respcontent ?"; diff --git a/yarrg/commod-update-receiver b/yarrg/commod-update-receiver index 42bd45d..70cf36a 100755 --- a/yarrg/commod-update-receiver +++ b/yarrg/commod-update-receiver @@ -49,7 +49,7 @@ setlocale(LC_CTYPE, "en_GB.UTF-8"); my $re_any= "^(.*)\$"; -parse_masters(); +parse_info_serverside(); sub fail ($) { my ($msg) = @_; diff --git a/yarrg/database-info-fetch b/yarrg/database-info-fetch index 86e3d5f..c56fda4 100755 --- a/yarrg/database-info-fetch +++ b/yarrg/database-info-fetch @@ -145,7 +145,7 @@ sub compare_sources_one ($$) { sub main__comparesources () { my $ocean= get_ocean(); - parse_masters(); + parse_info_clientside(); get_arches_islands_pctb($ocean); parse_pctb_commodmap() or die; @@ -167,7 +167,7 @@ sub main__comparesources () { sub main__island () { my $ocean= get_ocean(); - parse_masters(); + parse_info_clientside(); get_arches_islands_pctb($ocean); for_islands($ocean, diff --git a/yarrg/db-idempotent-populate b/yarrg/db-idempotent-populate index b7743ed..ea73f60 100755 --- a/yarrg/db-idempotent-populate +++ b/yarrg/db-idempotent-populate @@ -38,7 +38,8 @@ my ($oceanname) = @ARGV; #---------- setup ---------- -parse_masters_ocean($oceanname); +parse_info_serverside(); +parse_info_serverside_ocean($oceanname); our $ocean= $oceans{$oceanname}; db_setocean($oceanname); diff --git a/yarrg/master-master.txt b/yarrg/master-info.txt similarity index 100% rename from yarrg/master-master.txt rename to yarrg/master-info.txt diff --git a/yarrg/update-master-info b/yarrg/update-master-info new file mode 100755 index 0000000..a52b212 --- /dev/null +++ b/yarrg/update-master-info @@ -0,0 +1,22 @@ +#!/usr/bin/perl -w + +use strict (qw(vars)); + +use DBI; + +use Commods; + +@ARGV==1 or die; +my ($rsyncdir) = @ARGV; + +parse_info_serverside(); + +foreach my $oceanname (sort keys %oceans) { + print STDERR "updating ocean $oceanname...\n"; + system('./db-idempotent-populate',$oceanname); die $? if $?; +} + +print STDERR "installing new master-info...\n"; +my $df= "$rsyncdir/master-info.txt"; +system('cp','--','master-info.txt',"$df.tmp"); die $? if $?; +system('mv','--',"$df.tmp",$df); die $? if $? -- 2.30.2