X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?a=blobdiff_plain;f=pctb%2Fcommod-results-processor;h=4a70ddf7767376ab8b9db5406750c257460e9c3c;hb=64063c86071bbfc2de0fe032e372e1321c4b34e5;hp=32763293ac705f6128cd88a7312000ce296501ae;hpb=e888c1dd3476ca49bccf82b93b4a3633587d400d;p=ypp-sc-tools.db-live.git diff --git a/pctb/commod-results-processor b/pctb/commod-results-processor index 3276329..4a70ddf 100755 --- a/pctb/commod-results-processor +++ b/pctb/commod-results-processor @@ -218,13 +218,13 @@ sub main__tsv () { our (%commodmap); -our ($pctb) = $ENV{'YPPSC_PCTB_PCTB'}; die unless $pctb; +our ($pctb) = $ENV{'YPPSC_PCTB_PCTB'}; our ($ua)= LWP::UserAgent->new; sub load_commodmap() { undef %commodmap; - my $c= new IO::File "#commodmap#.tsv"; + my $c= new IO::File "_commodmap.tsv"; if (!$c) { $!==&ENOENT or die $!; return; } while (<$c>) { m/^(\S.*\S)\t(\d+)\n$/ or die "$_"; @@ -235,6 +235,8 @@ sub load_commodmap() { } sub refresh_commodmap() { + die unless $pctb; + $pctb =~ s,/*$,,; my $resp= $ua->get("$pctb/commodmap.php?version=2"); die $resp->status_line unless $resp->is_success; @@ -242,7 +244,7 @@ sub refresh_commodmap() { my $incommodmap=0; my $intag=''; my %got; - my $o= new IO::File "#commodmap#.tsv.new",'w' or die $!; + my $o= new IO::File "_commodmap.tsv.tmp",'w' or die $!; undef %commodmap; my $xp= new XML::Parser @@ -290,7 +292,40 @@ sub refresh_commodmap() { # print STDERR "[[[$content]]]\n"; $xp->parse($content); close $o or die $!; - rename "#commodmap#.tsv.new","#commodmap#.tsv" or die $!; + rename "_commodmap.tsv.tmp","_commodmap.tsv" or die $!; +} + +our %newcommods; + +sub read_newcommods ($) { + my ($file) = @_; + if (!open NC, "< $file") { + $!==&ENOENT or die $!; + return; + } + while () { + chomp; s/^\s*//; s/\s+$//; + next if m/^\#/; + next unless m/\S/; + $newcommods{$_}= 1; + } + NC->error and die $!; + close NC or die $!; +} + +sub refresh_newcommods() { + my $rsync= $ENV{'YPPSC_PCTB_RSYNC'}; + $rsync= 'rsync' if !defined $rsync; + + my $local= "_master-newcommods.txt"; + my $src= $ENV{'YPPSC_PCTB_DICT_UPDATE'}; + if ($src) { + my $remote= "$src/master-newcommods.txt"; + $!=0; system 'rsync','-Lt','--',$remote,$local; + die "$? $!" if $! or $?; + } + read_newcommods($local); + read_newcommods('_local-newcommods.txt'); } our (%stallmap, @stallmap); @@ -300,9 +335,11 @@ sub bs_gen_md ($$) { my $count= 0; my $o= ''; - foreach $commod (sort { - $commodmap{$a} <=> $commodmap{$b} - } keys %commod) { + foreach $commod ( + sort { $commodmap{$a} <=> $commodmap{$b} } + grep { exists $commodmap{$_} } + keys %commod + ) { #print STDERR "COMMOD $commod\n"; $current= $commod{$commod}; my $l= bs_p($commod,$bs,$sortmul); @@ -331,6 +368,16 @@ sub bs_gen_md ($$) { sub writeint { return pack 'v*', @_; } +our (%stalltypetoabbrevmap)= qw( + Apothecary A + Distilling D + Furnishing F + Ironworking I + Shipbuilding S + Tailoring T + Weaving W + ); + sub genmarketdata () { our $version= '005b'; @@ -338,13 +385,19 @@ sub genmarketdata () { my @missing= grep { !exists $commodmap{$_} } keys %commod; if (@missing) { refresh_commodmap(); + refresh_newcommods(); my $missing=0; foreach $commod (sort keys %commod) { next if exists $commodmap{$commod}; - printf STDERR "Unknown commodity \`%s'!\n", $commod; - $missing++; + if (exists $newcommods{$commod}) { + printf STDERR "Ignoring new commodity \`%s'!\n", $commod; + } else { + printf STDERR "Unknown commodity \`%s'!\n", $commod; + $missing++; + } } - die "$missing unknown commoditi(es). OCR failure?\n" + die "$missing unknown commoditi(es).". + " See README (search for \`newcommods').\n" if $missing; } @@ -355,7 +408,18 @@ sub genmarketdata () { my $ot= sprintf("$version\n". "%d\n", scalar(@stallmap)); - foreach $stall (@stallmap) { $ot .= "$stall\n"; } + foreach $stall (@stallmap) { + my $st= $stall; + if ($st =~ m/^(\S+)\'s (\S+) Stall$/) { + my $stkind= $stalltypetoabbrevmap{$2}; + if (defined $stkind) { + $st= "$1^$stkind"; + } else { + warn "unknown stall type $2 in $st\n"; + } + } + $ot .= "$st\n"; + } return $ot.$ob; } @@ -364,21 +428,95 @@ sub main__genmarketdata () { print $o or die $!; } +sub save_upload_html ($$) { + my ($which, $resptxt) = @_; + open R, ">./_upload-$which.html" or die $!; + print R $resptxt or die $!; + close R or die $!; +} + sub main__upload () { + my $ocean= $ENV{'YPPSC_OCEAN'}; die unless $ocean; + my $island= $ENV{'YPPSC_ISLAND'}; die unless $island; + die unless $pctb; my $o= genmarketdata(); + $pctb =~ s,/*$,,; my $url= "$pctb/upload.php"; - $url= "http://www.chiark.greenend.org.uk/ucgi/~ijackson/check/upload.php"; my $content= { 'marketdata' => [ undef, "marketdata.gz", Content_Type => 'application/gzip', Content => $o ] }; + + print STDERR "Uploading data to $pctb...\n"; + my $resp= $ua->post("$url", Content => $content, Content_Type => 'form-data'); die $resp->status_line unless $resp->is_success; - print "[[ ",$resp->content," ]]\n"; + my $resptxt= $resp->content(); + save_upload_html('1', $resptxt); + + open R, ">./_upload-1.html" or die $!; + print R $resptxt or die $!; + close R or die $!; + + my @filenames= $resptxt =~ + m/input\s+type="hidden"\s+name="filename"\s+value=\"([_.0-9a-z]+)\"/ig; + @filenames or die; + + my @forcerls= $resptxt =~ + m/input\s+type="hidden"\s+name="forcereload"\s+value=\"([1-9]\d+)\"/ig; + @forcerls or die; + + my $filename= $filenames[0]; + my $forcerl= $forcerls[0]; + + $ocean= ucfirst lc $ocean; + my @oceanids= $resptxt =~ + m/\