yarrg/commod-results-processor: remove obsolete $setisland

[ypp-sc-tools.db-test.git] / yarrg / commod-results-processor

diff --git a/yarrg/commod-results-processor b/yarrg/commod-results-processor
index 350ef366d0baa6fda394e4105a4ca624c6008681..3482381224748026b3741bdf233031d61a08e8a8 100755 (executable)
--- a/yarrg/commod-results-processor
+++ b/yarrg/commod-results-processor
@@ -29,7 +29,6 @@ use strict (qw(vars));
 use HTTP::Request;
 use IO::File;
 use POSIX;
-use LWP::UserAgent;
 use XML::Parser;
 
 use Commods;
@@ -40,7 +39,7 @@ use Commods;
 # $commod{'Hemp'}{Hold}
 
 our @v;
-our ($commod,$stall,%commod);
+our ($commod,$stall,%commod,@commods_inorder);
 
 @ARGV==1 or die "You probably don't want to run this program directly.\n";
 our ($mode) = shift @ARGV;
@@ -52,6 +51,7 @@ sub bs_read ($$) {
     return if @v <= $c;
     my ($price,$qty) = @v[$c..$c+1];
     return if !length($price) && !length($qty);
+    die "$price ?" unless $price =~ m/^\d/;
     die "$_ ?" unless length($price) && length($qty);
     $commod{$commod}{$bs}{$stall}= {
        Stall => $stall,
@@ -64,7 +64,12 @@ while (<>) {
     chomp;
     @v= split /\t/;
 #print STDERR "[".join("|",@v)."]\n";
+    foreach (@v[2..$#v]) {
+       s/\,//g;
+       die "$_ ?" if m/.\D/;
+    }
     ($commod,$stall) = @v;
+    push @commods_inorder, $commod unless exists $commod{$commod};
     bs_read(Buy,  2);
     bs_read(Sell, 4);
     $commod{$commod}{Hold}= $v[6]+0 if @v>6;
@@ -129,16 +134,18 @@ sub main__arbitrage () {
            $info.=
                sprintf("%-13.13s| %-19.19s %4d| %-19.19s %4d|%3d x%3d =%3d\n",
                       $commod,
-                      $buys[0]{Stall},$buys[0]{Price},
                       $sells[0]{Stall},$sells[0]{Price},
+                      $buys[0]{Stall},$buys[0]{Price},
                       $qty, $pricediff, $tprofit);
            sub arb_subtract_qty (\@) {
                my ($verbs) = @_;
                my $verb= shift @$verbs;
+               my $vqty= $verb->{Qty};
+               $vqty =~ s/^\>//;
                unshift @$verbs, {
                     Stall => $verb->{Stall},
                     Price => $verb->{Price},
-                    Qty => $verb->{Qty} - $qty
+                    Qty => $vqty - $qty
                };
            }
            arb_subtract_qty(@buys);
@@ -203,8 +210,8 @@ sub bs_p_tsv ($$) {
     }
 }
 
-sub write_tsv ($) {
-    my ($f) = @_;
+sub write_tsv ($$) {
+    my ($f,$showhold) = @_;
     foreach $commod (sort keys %commod) {
        $current= $commod{$commod};
        my %stalls;
@@ -214,6 +221,9 @@ sub write_tsv ($) {
            printf($f "%s\t%s", $commod, $stall) or die $!;
            bs_p_tsv($f, Buy);
            bs_p_tsv($f, Sell);
+           if ($showhold && $commod{$commod}{Hold}) {
+               printf($f "\t%s", $commod{$commod}{Hold}) or die $!;
+           }
            print($f "\n") or die $!;
        }
     }
@@ -222,13 +232,48 @@ sub write_tsv ($) {
 }
 
 sub main__tsv () {
-    write_tsv(\*STDOUT);
+    write_tsv(\*STDOUT,1);
+}
+
+sub undef_printable { my ($ov)= @_; defined $ov ? $ov : '?'; };
+
+sub commodsinorder_print1 ($$) {
+    my ($keyword,$commod) = @_;
+    printf("%s\t%-40s %10s %s",
+          $keyword,
+          $commod,
+          undef_printable($commods{$commod}{Ordval}),
+          undef_printable($commods{$commod}{Class}))
+       or die $!;
 }
 
+sub main__commodsinorder () {
+    parse_info_serverside();
+    my $last_ov;
+    foreach my $commod (@commods_inorder) {
+       my $ov= $commods{$commod}{Ordval};
+       commodsinorder_print1('found',$commod);
+       if (defined $ov) {
+           if (defined $last_ov && $ov <= $last_ov) {
+               print " out-of-order" or die $!;
+           }
+           $last_ov= $ov;
+       }
+       print "\n" or die $!;
+    }
+    foreach my $commod (sort {
+           undef_printable($commods{$a}{Ordval}) cmp
+           undef_printable($commods{$b}{Ordval})
+       } keys %commods) {
+       next if exists $commod{$commod};
+       commodsinorder_print1('none',$commod);
+       print "\n" or die $!;
+    }
+}
 
-our ($pctb) = $ENV{'YPPSC_PCTB_PCTB'};
+our ($pctb) = $ENV{'YPPSC_YARRG_PCTB'};
 
-our ($ua)= LWP::UserAgent->new;
+our ($ua)= http_useragent("commod-results-processor $mode");
 
 sub refresh_commodmap() {
     die unless $pctb;
@@ -286,7 +331,16 @@ sub refresh_commodmap() {
     my $content= $resp->content;
 
 #    print STDERR "[[[$content]]]\n";
-    $xp->parse($content);
+    my $commodmapxmltmp= '_commodmap.xml';
+    if (!eval {
+       $xp->parse($content); 1;
+    }) {
+       open R, ">./$commodmapxmltmp" or die $!;
+       print R $content or die $!;
+       close R or die $!;
+       die "$@ parsing commodmap";
+    }
+    unlink $commodmapxmltmp or $!==&ENOENT or die $!;
     close $o or die $!;
     rename "_commodmap.tsv.tmp","_commodmap.tsv" or die $!;
 }
@@ -310,17 +364,8 @@ sub read_newcommods ($) {
 }
 
 sub refresh_newcommods() {
-    my $rsync= $ENV{'YPPSC_PCTB_RSYNC'};
-    $rsync= 'rsync' if !defined $rsync;
-
-    my $local= "_master-newcommods.txt";
-    my $src= $ENV{'YPPSC_PCTB_DICT_UPDATE'};
-    if ($src) {
-       my $remote= "$src/master-newcommods.txt";
-       $!=0; system 'rsync','-Lt','--',$remote,$local;
-       die "$? $!" if $! or $?;
-    }
-    read_newcommods($local);
+    my $master= fetch_with_rsync('newcommods');
+    read_newcommods($master);
     read_newcommods('_local-newcommods.txt');
 }
 
@@ -438,24 +483,26 @@ sub gzip ($) {
     return pipethrough_run($tf,undef,'gzip','gzip');
 }
 
-sub main__uploadyaarg () {
+sub main__uploadyarrg () {
     my %o;
+
+    parse_info_clientside();
+
     $o{'ocean'}= $ENV{'YPPSC_OCEAN'} or die;
     $o{'island'}= $ENV{'YPPSC_ISLAND'} or die;
-    $o{'timestamp'}= 10;
-    get_our_version(\%o, 'client');
+    $o{'timestamp'}= $ENV{'YPPSC_DATA_TIMESTAMP'} or die;
 
     my $tf= pipethrough_prep();
-    write_tsv($tf);
+    write_tsv($tf,0);
     my $oz= pipethrough_run_gzip($tf);
     $o{'data'}=  [ undef, 'deduped.tsv.gz',
                    Content_Type => 'application/octet-stream',
                    Content => $oz ];
 
-    my $dest= $ENV{'YPPSC_YAARG_SUBMIT'};
-
-    my $respcontent= cgipostform($ua, "$dest/commod-update-receiver", \%o);
+    my $respcontent= yarrgpostform($ua, \%o);
     $respcontent =~ m/^OK\b/ or die "$respcontent ?";
+    $respcontent =~ s/^/ /mg;
+    print $respcontent,"\n";
 }
 
 sub main__uploadpctb () {
@@ -513,9 +560,6 @@ sub main__uploadpctb () {
     die "@filenames ?" if grep { $_ ne $filename } @filenames;
     die "@forcerls ?" if grep { $_ ne $forcerl } @forcerls;
 
-    my $setisland= {
-    };
-
     print STDERR "Setting ocean and island...\n";
 
     my $siurl= ($url . "?action=setisland".