X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=ypp-sc-tools.main.git;a=blobdiff_plain;f=pctb%2Fcommod-results-processor;h=4beb6d066a59af51b46591b70d7158941f26a9e4;hp=36473186a80f242a7a0fcece99f7d8db6c2ba72a;hb=dc85c07a4cf546e987221571a20269ea659efa91;hpb=8d6cf0f224b5df9866eba9350343067edcee78dd diff --git a/pctb/commod-results-processor b/pctb/commod-results-processor index 3647318..4beb6d0 100755 --- a/pctb/commod-results-processor +++ b/pctb/commod-results-processor @@ -32,6 +32,8 @@ use POSIX; use LWP::UserAgent; use XML::Parser; +use Commods; + # $commod{'Hemp'}{Buy|Sell}{'stall'}{Stall} # $commod{'Hemp'}{Buy|Sell}{'stall'}{Price} # $commod{'Hemp'}{Buy|Sell}{'stall'}{Qty} @@ -217,25 +219,13 @@ sub main__tsv () { } -our (%commodmap); our ($pctb) = $ENV{'YPPSC_PCTB_PCTB'}; our ($ua)= LWP::UserAgent->new; -sub load_commodmap() { - undef %commodmap; - my $c= new IO::File "#commodmap#.tsv"; - if (!$c) { $!==&ENOENT or die $!; return; } - while (<$c>) { - m/^(\S.*\S)\t(\d+)\n$/ or die "$_"; - $commodmap{$1}= $2; - } - $c->error and die $!; - close $c; -} - sub refresh_commodmap() { die unless $pctb; + $pctb =~ s,/*$,,; my $resp= $ua->get("$pctb/commodmap.php?version=2"); die $resp->status_line unless $resp->is_success; @@ -243,8 +233,8 @@ sub refresh_commodmap() { my $incommodmap=0; my $intag=''; my %got; - my $o= new IO::File "#commodmap#.tsv.new",'w' or die $!; - undef %commodmap; + my $o= new IO::File "_commodmap.tsv.tmp",'w' or die $!; + undef %pctb_commodmap; my $xp= new XML::Parser (Handlers => @@ -274,8 +264,8 @@ sub refresh_commodmap() { my $index= $1; $_= $got{'name'}; s/^\s+//; s/\s+$//; s/\n/ /g; s/\s+/ /; - die "$_ ?" if exists $commodmap{$_}; - $commodmap{$_}= $index; + die "$_ ?" if exists $pctb_commodmap{$_}; + $pctb_commodmap{$_}= $index; print $o "$_\t$index\n" or die $!; } elsif (lc $_ eq $intag) { $got{$intag}= $cdata; @@ -291,7 +281,40 @@ sub refresh_commodmap() { # print STDERR "[[[$content]]]\n"; $xp->parse($content); close $o or die $!; - rename "#commodmap#.tsv.new","#commodmap#.tsv" or die $!; + rename "_commodmap.tsv.tmp","_commodmap.tsv" or die $!; +} + +our %newcommods; + +sub read_newcommods ($) { + my ($file) = @_; + if (!open NC, "< $file") { + $!==&ENOENT or die $!; + return; + } + while () { + chomp; s/^\s*//; s/\s+$//; + next if m/^\#/; + next unless m/\S/; + $newcommods{$_}= 1; + } + NC->error and die $!; + close NC or die $!; +} + +sub refresh_newcommods() { + my $rsync= $ENV{'YPPSC_PCTB_RSYNC'}; + $rsync= 'rsync' if !defined $rsync; + + my $local= "_master-newcommods.txt"; + my $src= $ENV{'YPPSC_PCTB_DICT_UPDATE'}; + if ($src) { + my $remote= "$src/master-newcommods.txt"; + $!=0; system 'rsync','-Lt','--',$remote,$local; + die "$? $!" if $! or $?; + } + read_newcommods($local); + read_newcommods('_local-newcommods.txt'); } our (%stallmap, @stallmap); @@ -301,15 +324,17 @@ sub bs_gen_md ($$) { my $count= 0; my $o= ''; - foreach $commod (sort { - $commodmap{$a} <=> $commodmap{$b} - } keys %commod) { + foreach $commod ( + sort { $pctb_commodmap{$a} <=> $pctb_commodmap{$b} } + grep { exists $pctb_commodmap{$_} } + keys %commod + ) { #print STDERR "COMMOD $commod\n"; $current= $commod{$commod}; my $l= bs_p($commod,$bs,$sortmul); next unless @$l; #print STDERR "COMMOD $commod has ".scalar(@$l)."\n"; - $o .= writeint($commodmap{$commod}); + $o .= writeint($pctb_commodmap{$commod}); $o .= writeint(scalar @$l); foreach my $cs (@$l) { $stall= $cs->{Stall}; @@ -345,17 +370,23 @@ our (%stalltypetoabbrevmap)= qw( sub genmarketdata () { our $version= '005b'; - load_commodmap(); - my @missing= grep { !exists $commodmap{$_} } keys %commod; + parse_pctb_commodmap(); + my @missing= grep { !exists $pctb_commodmap{$_} } keys %commod; if (@missing) { refresh_commodmap(); + refresh_newcommods(); my $missing=0; foreach $commod (sort keys %commod) { - next if exists $commodmap{$commod}; - printf STDERR "Unknown commodity \`%s'!\n", $commod; - $missing++; + next if exists $pctb_commodmap{$commod}; + if (exists $newcommods{$commod}) { + printf STDERR "Ignoring new commodity \`%s'!\n", $commod; + } else { + printf STDERR "Unknown commodity \`%s'!\n", $commod; + $missing++; + } } - die "$missing unknown commoditi(es). OCR failure?\n" + die "$missing unknown commoditi(es).". + " See README (search for \`newcommods').\n" if $missing; } @@ -386,9 +417,29 @@ sub main__genmarketdata () { print $o or die $!; } -sub main__upload () { +sub save_upload_html ($$) { + my ($which, $resptxt) = @_; + open R, ">./_upload-$which.html" or die $!; + print R $resptxt or die $!; + close R or die $!; +} + +sub main__upload_yaarg () { + my $ocean= $ENV{'YPPSC_OCEAN'}; die unless $ocean; + my $island= $ENV{'YPPSC_ISLAND'}; die unless $island; + my $content= { + 'data' => [ undef, 'deduped.tsv.gz', + Content_Type => 'application/octet-stream', + Content => '' +]}; +} + +sub main__upload_pctb () { + my $ocean= $ENV{'YPPSC_OCEAN'}; die unless $ocean; + my $island= $ENV{'YPPSC_ISLAND'}; die unless $island; die unless $pctb; my $o= genmarketdata(); + $pctb =~ s,/*$,,; my $url= "$pctb/upload.php"; my $content= { 'marketdata' => [ undef, "marketdata.gz", @@ -396,37 +447,75 @@ sub main__upload () { Content => $o ] }; + + print STDERR "Uploading data to $pctb...\n"; + my $resp= $ua->post("$url", Content => $content, Content_Type => 'form-data'); die $resp->status_line unless $resp->is_success; my $resptxt= $resp->content(); + save_upload_html('1', $resptxt); + + open R, ">./_upload-1.html" or die $!; + print R $resptxt or die $!; + close R or die $!; my @filenames= $resptxt =~ - m/input\s+type="hidden"\s+name="filename"\s+value=\"([_.0-9a-z]+)\"/i; - @filenames or die "$resptxt ?"; + m/input\s+type="hidden"\s+name="filename"\s+value=\"([_.0-9a-z]+)\"/ig; + @filenames or die; my @forcerls= $resptxt =~ - m/input\s+type="hidden"\s+name="forcereload"\s+value=\"([1-9]\d+)\"/i; - @forcerls or die "$resptxt ?"; + m/input\s+type="hidden"\s+name="forcereload"\s+value=\"([1-9]\d+)\"/ig; + @forcerls or die; my $filename= $filenames[0]; my $forcerl= $forcerls[0]; - die "$resptxt | @filenames ?" if grep { $_ ne $filename } @filenames; - die "$resptxt | @forcerls ?" if grep { $_ ne $forcerl } @forcerls; + $ocean= ucfirst lc $ocean; + my @oceanids= $resptxt =~ + m/\