X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=ypp-sc-tools.db-test.git;a=blobdiff_plain;f=pctb%2Fcommod-results-processor;h=3ea53be79f49b73a36b9be2f592a307a05b2833e;hp=36473186a80f242a7a0fcece99f7d8db6c2ba72a;hb=d5514a07ad8470e0c00f158ef0a02ff6a90adb22;hpb=8d6cf0f224b5df9866eba9350343067edcee78dd

diff --git a/pctb/commod-results-processor b/pctb/commod-results-processor
index 3647318..3ea53be 100755
--- a/pctb/commod-results-processor
+++ b/pctb/commod-results-processor
@@ -236,6 +236,7 @@ sub load_commodmap() {
 
 sub refresh_commodmap() {
     die unless $pctb;
+    $pctb =~ s,/*$,,;
     my $resp= $ua->get("$pctb/commodmap.php?version=2");
     die $resp->status_line unless $resp->is_success;
 
@@ -243,7 +244,7 @@ sub refresh_commodmap() {
     my $incommodmap=0;
     my $intag='';
     my %got;
-    my $o= new IO::File "#commodmap#.tsv.new",'w' or die $!;
+    my $o= new IO::File "#commodmap#.tsv.tmp",'w' or die $!;
     undef %commodmap;
 
     my $xp= new XML::Parser
@@ -291,7 +292,40 @@ sub refresh_commodmap() {
 #    print STDERR "[[[$content]]]\n";
     $xp->parse($content);
     close $o or die $!;
-    rename "#commodmap#.tsv.new","#commodmap#.tsv" or die $!;
+    rename "#commodmap#.tsv.tmp","#commodmap#.tsv" or die $!;
+}
+
+our %newcommods;
+
+sub read_newcommods ($) {
+    my ($file) = @_;
+    if (!open NC, "< $file") {
+	$!==&ENOENT or die $!;
+	return;
+    }
+    while (<NC>) {
+	chomp; s/^\s*//; s/\s+$//;
+	next if m/^\#/;
+	next unless m/\S/;
+	$newcommods{$_}= 1;
+    }
+    NC->error and die $!;
+    close NC or die $!;
+}
+
+sub refresh_newcommods() {
+    my $rsync= $ENV{'YPPSC_PCTB_RSYNC'};
+    $rsync= 'rsync' if !defined $rsync;
+
+    my $local= "#master-newcommods#.txt";
+    my $src= $ENV{'YPPSC_PCTB_DICT_UPDATE'};
+    if ($src) {
+	my $remote= "$src/master-newcommods.txt";
+	$!=0; system 'rsync','-Lt','--',$remote,$local;
+	die "$? $!" if $! or $?;
+    }
+    read_newcommods($local);
+    read_newcommods('#local-newcommods#.txt');
 }
 
 our (%stallmap, @stallmap);
@@ -301,9 +335,11 @@ sub bs_gen_md ($$) {
     my $count= 0;
     my $o= '';
     
-    foreach $commod (sort {
-	$commodmap{$a} <=> $commodmap{$b}
-    } keys %commod) {
+    foreach $commod (
+		     sort { $commodmap{$a} <=> $commodmap{$b} }
+		     grep { exists $commodmap{$_} }
+		     keys %commod
+		     ) {
 #print STDERR "COMMOD $commod\n";
         $current= $commod{$commod};
 	my $l= bs_p($commod,$bs,$sortmul);
@@ -349,13 +385,19 @@ sub genmarketdata () {
     my @missing= grep { !exists $commodmap{$_} } keys %commod;
     if (@missing) {
 	refresh_commodmap();
+	refresh_newcommods();
 	my $missing=0;
 	foreach $commod (sort keys %commod) {
 	    next if exists $commodmap{$commod};
-	    printf STDERR "Unknown commodity \`%s'!\n", $commod;
-	    $missing++;
+	    if (exists $newcommods{$commod}) {
+		printf STDERR "Ignoring new commodity \`%s'!\n", $commod;
+	    } else {
+		printf STDERR "Unknown commodity \`%s'!\n", $commod;
+		$missing++;
+	    }
 	}
-	die "$missing unknown commoditi(es).  OCR failure?\n"
+	die "$missing unknown commoditi(es).".
+	    "  See README (search for \`newcommods').\n"
 	    if $missing;
     }    
 
@@ -386,9 +428,19 @@ sub main__genmarketdata () {
     print $o or die $!;
 }
 
+sub save_upload_html ($$) {
+    my ($which, $resptxt) = @_;
+    open R, ">./#upload-$which#.html" or die $!;
+    print R $resptxt or die $!;
+    close R or die $!;
+}
+
 sub main__upload () {
+    my $ocean= $ENV{'YPPSC_OCEAN'};  die unless $ocean;
+    my $island= $ENV{'YPPSC_ISLAND'};  die unless $island;
     die unless $pctb;
     my $o= genmarketdata();
+    $pctb =~ s,/*$,,;
     my $url= "$pctb/upload.php";
     my $content= {
 	'marketdata' => [ undef, "marketdata.gz",
@@ -396,37 +448,75 @@ sub main__upload () {
 			  Content => $o
 			  ]
 		      };
+
+    print STDERR "Uploading data to server...\n";
+
     my $resp= $ua->post("$url", Content => $content,
 			Content_Type => 'form-data');
     die $resp->status_line unless $resp->is_success;
 
     my $resptxt= $resp->content();
+    save_upload_html('1', $resptxt);
+
+    open R, ">./#upload-1#.html" or die $!;
+    print R $resptxt or die $!;
+    close R or die $!;
 
     my @filenames= $resptxt =~
- m/input\s+type="hidden"\s+name="filename"\s+value=\"([_.0-9a-z]+)\"/i;
-    @filenames or die "$resptxt ?";
+ m/input\s+type="hidden"\s+name="filename"\s+value=\"([_.0-9a-z]+)\"/ig;
+    @filenames or die;
 
     my @forcerls= $resptxt =~
- m/input\s+type="hidden"\s+name="forcereload"\s+value=\"([1-9]\d+)\"/i;
-    @forcerls or die "$resptxt ?";
+ m/input\s+type="hidden"\s+name="forcereload"\s+value=\"([1-9]\d+)\"/ig;
+    @forcerls or die;
 
     my $filename= $filenames[0];
     my $forcerl= $forcerls[0];
 
-    die "$resptxt | @filenames ?" if grep { $_ ne $filename } @filenames;
-    die "$resptxt | @forcerls ?" if grep { $_ ne $forcerl } @forcerls;
+    $ocean= ucfirst lc $ocean;
+    my @oceanids= $resptxt =~
+ m/\<option value\=\"(\d+)\"\>$ocean\<\/option\>/;
+    @oceanids==1 or die "@oceanids ?";
+
+    my $islandid;
+    while ($resptxt =~
+ m/^islands\[\d+\]\[\d+\]\=new\s+option\(\"(.*)\"\,(\d+)\)\s*$/mig
+	   ) {
+	next unless $1 eq $island;
+	$islandid= $2;
+    }
+    defined $islandid or die;
 
-#    my $setsland= {
-#	'action' => 'setisland'
-#	'filename' => $filename,
-#	'forcereload' => $forcerl,
-#    my $url_setisland= "$url
-#
-#http://pctb.ilk.org/upload.php?topisland=560&ocean=2&island=48&action=setisland&forcereload=1244748679&filename=PFayDW' >v.html 
-#    
-    print ">$filename|$forcerl<\n";
+    die "@filenames ?" if grep { $_ ne $filename } @filenames;
+    die "@forcerls ?" if grep { $_ ne $forcerl } @forcerls;
+
+    my $setisland= {
+    };
+
+    print STDERR "Setting ocean and island...\n";
 
-    system 'printenv|grep YPP|sort';
+    my $siurl= ($url . "?action=setisland".
+		"&filename=$filename".
+		"&forcereload=$forcerl".
+		"&ocean=$oceanids[0]".
+		"&island=$islandid");
+    $resp= $ua->get($siurl);
+
+    die $resp->status_line unless $resp->is_success;
+
+    $resptxt= $resp->content();
+    save_upload_html('2', $resptxt);
+
+    die unless $resptxt =~ m/your uploaded data has been processed/i;
+    die unless $resptxt =~ m/your data has been integrated into the database/i;
+
+    $resptxt =~ s/\<a href=\"about:\w+\"\>[^<>]+\<\/a\>//g;
+    save_upload_html('3', $resptxt);
+
+    print "\n" or die $!;
+    system('w3m -T text/html -dump < \#upload-3#.html');
+    
+    print "\n" or die $!;
 }