WIP Much rework for new upload arrangements and new name

diff --git a/yarrg/Commods.pm b/yarrg/Commods.pm
index 955446f402f48719943af31844cc17e9742649a1..9b190c4b3a88d5236cc1e9ff9f41058a12118be2 100644 (file)
--- a/yarrg/Commods.pm
+++ b/yarrg/Commods.pm
@@ -32,7 +32,8 @@ BEGIN {
     our ($VERSION, @ISA, @EXPORT, @EXPORT_OK, %EXPORT_TAGS);
     $VERSION     = 1.00;
     @ISA         = qw(Exporter);
     our ($VERSION, @ISA, @EXPORT, @EXPORT_OK, %EXPORT_TAGS);
     $VERSION     = 1.00;
     @ISA         = qw(Exporter);

-    @EXPORT      = qw(&parse_masters &parse_masters_ocean
+    @EXPORT      = qw(&parse_info1 &rsync_master
+                     &parse_info_serverside &parse_info_serverside_ocean

                      %oceans %commods %clients %routes %route_mysteries
                      &parse_pctb_commodmap %pctb_commodmap @pctb_commodmap
                      &get_our_version &check_tsv_line
                      %oceans %commods %clients %routes %route_mysteries
                      &parse_pctb_commodmap %pctb_commodmap @pctb_commodmap
                      &get_our_version &check_tsv_line


@@ -58,7 +59,7 @@ our (%pctb_commodmap,@pctb_commodmap);
 my %colours; # eg $colours{'c'}{'black'}= $sources
 my @rawcm; # eg $rawcm[0]='fine rum'; $rawcm[1]='fine %c cloth'
 
 my %colours; # eg $colours{'c'}{'black'}= $sources
 my @rawcm; # eg $rawcm[0]='fine rum'; $rawcm[1]='fine %c cloth'
 

-sub parse_master_master1 ($$) {
+sub parse_info1 ($$) {

     my ($mmfn,$src)= @_;
     my $mm= new IO::File $mmfn, 'r' or die "$mmfn $!";
     my @ctx= ();
     my ($mmfn,$src)= @_;
     my $mm= new IO::File $mmfn, 'r' or die "$mmfn $!";
     my @ctx= ();


@@ -131,14 +132,37 @@ sub parse_master_master1 ($$) {
            my $arch= $ocean->{$an};
            push @allislands, sort keys %$arch;
        }
            my $arch= $ocean->{$an};
            push @allislands, sort keys %$arch;
        }

-       parse_master_map_route_islands($on, \@allislands, $routes);
+       parse_info_maproutes($on, \@allislands, $routes);

        foreach my $route (values %$routes) {
        foreach my $route (values %$routes) {

-           parse_master_map_route_islands($on, \@allislands, $route);
+           parse_info_maproutes($on, \@allislands, $route);

        }
     }
 }
 
        }
     }
 }
 

-sub parse_master_map_route_islands ($$$) {
+sub parse_info_clientside () {
+    my $yarrg= $ENV{'YPPSC_YARRG_DICT_UPDATE'};
+    return unless $yarrg;
+    my $master= fetch_with_rsync('info');
+    parse_info1($master,'s');
+    parse_info1('_local-info.txt','s');
+}
+
+sub fetch_with_rsync ($) {
+    my ($stem) = @_;
+
+    my $rsync= $ENV{'YPPSC_YARRG_RSYNC'};
+    $rsync= 'rsync' if !defined $rsync;
+
+    my $local= "_master-$stem.txt";
+    my $src= $ENV{'YPPSC_YARRG_DICT_UPDATE'};
+    if ($src) {
+       my $remote= "$src/master-$stem.txt";
+       $!=0; system 'rsync','-Lt','--',$remote,$local;
+       die "$? $!" if $! or $?;
+    }
+}
+
+sub parse_info_maproutes ($$$) {

     my ($on, $allislands, $routemap) = @_;;
     foreach my $k (sort keys %$routemap) {
        my @ok= grep { index($_,$k) >= 0 } @$allislands;
     my ($on, $allislands, $routemap) = @_;;
     foreach my $k (sort keys %$routemap) {
        my @ok= grep { index($_,$k) >= 0 } @$allislands;


@@ -153,14 +177,13 @@ sub parse_master_map_route_islands ($$$) {
     }
 }
 
     }
 }
 

-sub parse_masters () {
-    parse_master_master1('master-master.txt','s');
+sub parse_info_serverside () {
+    parse_info1('master-info.txt','s');

 }
 }

-sub parse_masters_ocean ($) {
+sub parse_info_serverside_ocean ($) {

     my ($oceanname) = @_;
     my ($oceanname) = @_;

-    parse_master_master1('master-master.txt','s');

     die "unknown ocean $oceanname ?" unless exists $oceans{$oceanname};
     die "unknown ocean $oceanname ?" unless exists $oceans{$oceanname};

-    parse_master_master1("ocean-".(lc $oceanname).".txt",'s');
+    parse_info1("ocean-".(lc $oceanname).".txt",'s');

 }
 
 sub parse_pctb_commodmap () {
 }
 
 sub parse_pctb_commodmap () {

diff --git a/yarrg/commod-results-processor b/yarrg/commod-results-processor
index 827cac45ebfc05ac9a5a628d15437d758a93de07..7cd4e0457b6f438398ffa55f46dcdb035d834ed7 100755 (executable)
--- a/yarrg/commod-results-processor
+++ b/yarrg/commod-results-processor
@@ -310,17 +310,8 @@ sub read_newcommods ($) {
 }
 
 sub refresh_newcommods() {
 }
 
 sub refresh_newcommods() {

-    my $rsync= $ENV{'YPPSC_YARRG_RSYNC'};
-    $rsync= 'rsync' if !defined $rsync;
-
-    my $local= "_master-newcommods.txt";
-    my $src= $ENV{'YPPSC_YARRG_DICT_UPDATE'};
-    if ($src) {
-       my $remote= "$src/master-newcommods.txt";
-       $!=0; system 'rsync','-Lt','--',$remote,$local;
-       die "$? $!" if $! or $?;
-    }
-    read_newcommods($local);
+    my $master= fetch_with_rsync('newcommods');
+    read_newcommods($master);

     read_newcommods('_local-newcommods.txt');
 }
 
     read_newcommods('_local-newcommods.txt');
 }
 


@@ -440,6 +431,9 @@ sub gzip ($) {
 
 sub main__uploadyaarg () {
     my %o;
 
 sub main__uploadyaarg () {
     my %o;

+
+    parse_info_clientside();
+

     $o{'ocean'}= $ENV{'YPPSC_OCEAN'} or die;
     $o{'island'}= $ENV{'YPPSC_ISLAND'} or die;
     $o{'timestamp'}= 10;
     $o{'ocean'}= $ENV{'YPPSC_OCEAN'} or die;
     $o{'island'}= $ENV{'YPPSC_ISLAND'} or die;
     $o{'timestamp'}= 10;


@@ -452,7 +446,7 @@ sub main__uploadyaarg () {
                    Content_Type => 'application/octet-stream',
                    Content => $oz ];
 
                    Content_Type => 'application/octet-stream',
                    Content => $oz ];
 

-    my $dest= $ENV{'YPPSC_YAARG_SUBMIT'};
+    my $dest= $ENV{'YPPSC_YARRG_YARRG'};

 
     my $respcontent= cgipostform($ua, "$dest/commod-update-receiver", \%o);
     $respcontent =~ m/^OK\b/ or die "$respcontent ?";
 
     my $respcontent= cgipostform($ua, "$dest/commod-update-receiver", \%o);
     $respcontent =~ m/^OK\b/ or die "$respcontent ?";

diff --git a/yarrg/commod-update-receiver b/yarrg/commod-update-receiver
index 42bd45dc29539cca3b9edc4b48af7f86d13ce636..70cf36a93bdfe96aa51446172f2b53349786439f 100755 (executable)
--- a/yarrg/commod-update-receiver
+++ b/yarrg/commod-update-receiver
@@ -49,7 +49,7 @@ setlocale(LC_CTYPE, "en_GB.UTF-8");
 
 my $re_any= "^(.*)\$";
 
 
 my $re_any= "^(.*)\$";
 

-parse_masters();
+parse_info_serverside();

 
 sub fail ($) {
     my ($msg) = @_;
 
 sub fail ($) {
     my ($msg) = @_;

diff --git a/yarrg/database-info-fetch b/yarrg/database-info-fetch
index 86e3d5f2baa55843ba5a3a77cbc5643c7809d59e..c56fda42e2d4f871fea5f256d46c6ad858657432 100755 (executable)
--- a/yarrg/database-info-fetch
+++ b/yarrg/database-info-fetch
@@ -145,7 +145,7 @@ sub compare_sources_one ($$) {
 sub main__comparesources () {
     my $ocean= get_ocean();
     
 sub main__comparesources () {
     my $ocean= get_ocean();
     

-    parse_masters();
+    parse_info_clientside();

     get_arches_islands_pctb($ocean);
     parse_pctb_commodmap() or die;
 
     get_arches_islands_pctb($ocean);
     parse_pctb_commodmap() or die;
 


@@ -167,7 +167,7 @@ sub main__comparesources () {
 sub main__island () {
     my $ocean= get_ocean();
     
 sub main__island () {
     my $ocean= get_ocean();
     

-    parse_masters();
+    parse_info_clientside();

     get_arches_islands_pctb($ocean);
 
     for_islands($ocean,
     get_arches_islands_pctb($ocean);
 
     for_islands($ocean,

diff --git a/yarrg/db-idempotent-populate b/yarrg/db-idempotent-populate
index b7743ed9cb1ff3f78ef9b6be379d4635ec467e1e..ea73f607aa6a1fd6f05c384697400406b26b6b83 100755 (executable)
--- a/yarrg/db-idempotent-populate
+++ b/yarrg/db-idempotent-populate
@@ -38,7 +38,8 @@ my ($oceanname) = @ARGV;
 
 #---------- setup ----------
 
 
 #---------- setup ----------
 

-parse_masters_ocean($oceanname);
+parse_info_serverside();
+parse_info_serverside_ocean($oceanname);

 our $ocean= $oceans{$oceanname};
 
 db_setocean($oceanname);
 our $ocean= $oceans{$oceanname};
 
 db_setocean($oceanname);

diff --git a/yarrg/master-master.txt b/yarrg/master-info.txt
similarity index 100%
rename from yarrg/master-master.txt
rename to yarrg/master-info.txt

diff --git a/yarrg/update-master-info b/yarrg/update-master-info
new file mode 100755 (executable)
index 0000000..a52b212
--- /dev/null
+++ b/yarrg/update-master-info
@@ -0,0 +1,22 @@
+#!/usr/bin/perl -w
+
+use strict (qw(vars));
+
+use DBI;
+
+use Commods;
+
+@ARGV==1 or die;
+my ($rsyncdir) = @ARGV;
+
+parse_info_serverside();
+
+foreach my $oceanname (sort keys %oceans) {
+    print STDERR "updating ocean $oceanname...\n";
+    system('./db-idempotent-populate',$oceanname); die $? if $?;
+}
+
+print STDERR "installing new master-info...\n";
+my $df= "$rsyncdir/master-info.txt";
+system('cp','--','master-info.txt',"$df.tmp"); die $? if $?;
+system('mv','--',"$df.tmp",$df); die $? if $?