Correct bugs in new AA table cell structure parsing

[ypp-sc-tools.db-test.git] / pctb / commod-results-processor

diff --git a/pctb/commod-results-processor b/pctb/commod-results-processor
index c3cbe8e1b479447245540556754789863109f453..3ea53be79f49b73a36b9be2f592a307a05b2833e 100755 (executable)
--- a/pctb/commod-results-processor
+++ b/pctb/commod-results-processor
@@ -244,7 +244,7 @@ sub refresh_commodmap() {
     my $incommodmap=0;
     my $intag='';
     my %got;
-    my $o= new IO::File "#commodmap#.tsv.new",'w' or die $!;
+    my $o= new IO::File "#commodmap#.tsv.tmp",'w' or die $!;
     undef %commodmap;
 
     my $xp= new XML::Parser
@@ -292,7 +292,40 @@ sub refresh_commodmap() {
 #    print STDERR "[[[$content]]]\n";
     $xp->parse($content);
     close $o or die $!;
-    rename "#commodmap#.tsv.new","#commodmap#.tsv" or die $!;
+    rename "#commodmap#.tsv.tmp","#commodmap#.tsv" or die $!;
+}
+
+our %newcommods;
+
+sub read_newcommods ($) {
+    my ($file) = @_;
+    if (!open NC, "< $file") {
+       $!==&ENOENT or die $!;
+       return;
+    }
+    while (<NC>) {
+       chomp; s/^\s*//; s/\s+$//;
+       next if m/^\#/;
+       next unless m/\S/;
+       $newcommods{$_}= 1;
+    }
+    NC->error and die $!;
+    close NC or die $!;
+}
+
+sub refresh_newcommods() {
+    my $rsync= $ENV{'YPPSC_PCTB_RSYNC'};
+    $rsync= 'rsync' if !defined $rsync;
+
+    my $local= "#master-newcommods#.txt";
+    my $src= $ENV{'YPPSC_PCTB_DICT_UPDATE'};
+    if ($src) {
+       my $remote= "$src/master-newcommods.txt";
+       $!=0; system 'rsync','-Lt','--',$remote,$local;
+       die "$? $!" if $! or $?;
+    }
+    read_newcommods($local);
+    read_newcommods('#local-newcommods#.txt');
 }
 
 our (%stallmap, @stallmap);
@@ -302,9 +335,11 @@ sub bs_gen_md ($$) {
     my $count= 0;
     my $o= '';
     
-    foreach $commod (sort {
-       $commodmap{$a} <=> $commodmap{$b}
-    } keys %commod) {
+    foreach $commod (
+                    sort { $commodmap{$a} <=> $commodmap{$b} }
+                    grep { exists $commodmap{$_} }
+                    keys %commod
+                    ) {
 #print STDERR "COMMOD $commod\n";
         $current= $commod{$commod};
        my $l= bs_p($commod,$bs,$sortmul);
@@ -350,13 +385,19 @@ sub genmarketdata () {
     my @missing= grep { !exists $commodmap{$_} } keys %commod;
     if (@missing) {
        refresh_commodmap();
+       refresh_newcommods();
        my $missing=0;
        foreach $commod (sort keys %commod) {
            next if exists $commodmap{$commod};
-           printf STDERR "Unknown commodity \`%s'!\n", $commod;
-           $missing++;
+           if (exists $newcommods{$commod}) {
+               printf STDERR "Ignoring new commodity \`%s'!\n", $commod;
+           } else {
+               printf STDERR "Unknown commodity \`%s'!\n", $commod;
+               $missing++;
+           }
        }
-       die "$missing unknown commoditi(es).  OCR failure?\n"
+       die "$missing unknown commoditi(es).".
+           "  See README (search for \`newcommods').\n"
            if $missing;
     }    
 
@@ -450,29 +491,32 @@ sub main__upload () {
     die "@forcerls ?" if grep { $_ ne $forcerl } @forcerls;
 
     my $setisland= {
-       'action' => 'setisland',
-       'filename' => $filename,
-       'forcereload' => $forcerl,
-       'ocean' => $oceanids[0],
-       'island' => $islandid,
     };
 
     print STDERR "Setting ocean and island...\n";
 
-    $resp= $ua->post("${url}IWJTEST", $setisland);
+    my $siurl= ($url . "?action=setisland".
+               "&filename=$filename".
+               "&forcereload=$forcerl".
+               "&ocean=$oceanids[0]".
+               "&island=$islandid");
+    $resp= $ua->get($siurl);
+
     die $resp->status_line unless $resp->is_success;
 
     $resptxt= $resp->content();
     save_upload_html('2', $resptxt);
 
-    
-    
-#
-#http://pctb.ilk.org/upload.php?&ocean=2&island=48&action=setisland&forcereload=1244748679&filename=PFayDW' >v.html 
-#    
-#    print ">$filename|$forcerl|@oceanids|$islandid<\n";
+    die unless $resptxt =~ m/your uploaded data has been processed/i;
+    die unless $resptxt =~ m/your data has been integrated into the database/i;
 
-    system 'printenv|grep YPP|sort';
+    $resptxt =~ s/\<a href=\"about:\w+\"\>[^<>]+\<\/a\>//g;
+    save_upload_html('3', $resptxt);
+
+    print "\n" or die $!;
+    system('w3m -T text/html -dump < \#upload-3#.html');
+    
+    print "\n" or die $!;
 }