X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=ypp-sc-tools.web-live.git;a=blobdiff_plain;f=pctb%2Fcommod-results-processor;h=350ef366d0baa6fda394e4105a4ca624c6008681;hp=b604419db6cf7c3e43531fda9350fbd0eb9affea;hb=a5ec88cc0c003b52afa18af3a97242fe891501b8;hpb=12b118fd416fb96309d9e4ef65336e08441594c1 diff --git a/pctb/commod-results-processor b/pctb/commod-results-processor index b604419..350ef36 100755 --- a/pctb/commod-results-processor +++ b/pctb/commod-results-processor @@ -32,6 +32,8 @@ use POSIX; use LWP::UserAgent; use XML::Parser; +use Commods; + # $commod{'Hemp'}{Buy|Sell}{'stall'}{Stall} # $commod{'Hemp'}{Buy|Sell}{'stall'}{Price} # $commod{'Hemp'}{Buy|Sell}{'stall'}{Qty} @@ -191,51 +193,46 @@ sub main__bestprices () { } } -sub bs_p_tsv ($) { - my ($bs) = @_; +sub bs_p_tsv ($$) { + my ($f, $bs) = @_; if (exists $current->{$bs}{$stall}) { my $si= $current->{$bs}{$stall}; - printf("\t%d\t%s", $si->{Price}, $si->{Qty}) or die $!; + printf($f "\t%d\t%s", $si->{Price}, $si->{Qty}) or die $!; } else { - printf("\t\t") or die $!; + printf($f "\t\t") or die $!; } } -sub main__tsv () { +sub write_tsv ($) { + my ($f) = @_; foreach $commod (sort keys %commod) { $current= $commod{$commod}; my %stalls; map { $stalls{$_}=1; } keys %{ $current->{Buy} }; map { $stalls{$_}=1; } keys %{ $current->{Sell} }; foreach $stall (sort keys %stalls) { - printf("%s\t%s", $commod, $stall) or die $!; - bs_p_tsv(Buy); - bs_p_tsv(Sell); - print("\n") or die $!; + printf($f "%s\t%s", $commod, $stall) or die $!; + bs_p_tsv($f, Buy); + bs_p_tsv($f, Sell); + print($f "\n") or die $!; } } + $f->error and die $!; + $f->flush or die $!; +} + +sub main__tsv () { + write_tsv(\*STDOUT); } -our (%commodmap); our ($pctb) = $ENV{'YPPSC_PCTB_PCTB'}; our ($ua)= LWP::UserAgent->new; -sub load_commodmap() { - undef %commodmap; - my $c= new IO::File "#commodmap#.tsv"; - if (!$c) { $!==&ENOENT or die $!; return; } - while (<$c>) { - m/^(\S.*\S)\t(\d+)\n$/ or die "$_"; - $commodmap{$1}= $2; - } - $c->error and die $!; - close $c; -} - sub refresh_commodmap() { die unless $pctb; + $pctb =~ s,/*$,,; my $resp= $ua->get("$pctb/commodmap.php?version=2"); die $resp->status_line unless $resp->is_success; @@ -243,8 +240,8 @@ sub refresh_commodmap() { my $incommodmap=0; my $intag=''; my %got; - my $o= new IO::File "#commodmap#.tsv.new",'w' or die $!; - undef %commodmap; + my $o= new IO::File "_commodmap.tsv.tmp",'w' or die $!; + undef %pctb_commodmap; my $xp= new XML::Parser (Handlers => @@ -274,8 +271,8 @@ sub refresh_commodmap() { my $index= $1; $_= $got{'name'}; s/^\s+//; s/\s+$//; s/\n/ /g; s/\s+/ /; - die "$_ ?" if exists $commodmap{$_}; - $commodmap{$_}= $index; + die "$_ ?" if exists $pctb_commodmap{$_}; + $pctb_commodmap{$_}= $index; print $o "$_\t$index\n" or die $!; } elsif (lc $_ eq $intag) { $got{$intag}= $cdata; @@ -291,7 +288,40 @@ sub refresh_commodmap() { # print STDERR "[[[$content]]]\n"; $xp->parse($content); close $o or die $!; - rename "#commodmap#.tsv.new","#commodmap#.tsv" or die $!; + rename "_commodmap.tsv.tmp","_commodmap.tsv" or die $!; +} + +our %newcommods; + +sub read_newcommods ($) { + my ($file) = @_; + if (!open NC, "< $file") { + $!==&ENOENT or die $!; + return; + } + while () { + chomp; s/^\s*//; s/\s+$//; + next if m/^\#/; + next unless m/\S/; + $newcommods{$_}= 1; + } + NC->error and die $!; + close NC or die $!; +} + +sub refresh_newcommods() { + my $rsync= $ENV{'YPPSC_PCTB_RSYNC'}; + $rsync= 'rsync' if !defined $rsync; + + my $local= "_master-newcommods.txt"; + my $src= $ENV{'YPPSC_PCTB_DICT_UPDATE'}; + if ($src) { + my $remote= "$src/master-newcommods.txt"; + $!=0; system 'rsync','-Lt','--',$remote,$local; + die "$? $!" if $! or $?; + } + read_newcommods($local); + read_newcommods('_local-newcommods.txt'); } our (%stallmap, @stallmap); @@ -301,15 +331,17 @@ sub bs_gen_md ($$) { my $count= 0; my $o= ''; - foreach $commod (sort { - $commodmap{$a} <=> $commodmap{$b} - } keys %commod) { + foreach $commod ( + sort { $pctb_commodmap{$a} <=> $pctb_commodmap{$b} } + grep { exists $pctb_commodmap{$_} } + keys %commod + ) { #print STDERR "COMMOD $commod\n"; $current= $commod{$commod}; my $l= bs_p($commod,$bs,$sortmul); next unless @$l; #print STDERR "COMMOD $commod has ".scalar(@$l)."\n"; - $o .= writeint($commodmap{$commod}); + $o .= writeint($pctb_commodmap{$commod}); $o .= writeint(scalar @$l); foreach my $cs (@$l) { $stall= $cs->{Stall}; @@ -345,17 +377,23 @@ our (%stalltypetoabbrevmap)= qw( sub genmarketdata () { our $version= '005b'; - load_commodmap(); - my @missing= grep { !exists $commodmap{$_} } keys %commod; + parse_pctb_commodmap(); + my @missing= grep { !exists $pctb_commodmap{$_} } keys %commod; if (@missing) { refresh_commodmap(); + refresh_newcommods(); my $missing=0; foreach $commod (sort keys %commod) { - next if exists $commodmap{$commod}; - printf STDERR "Unknown commodity \`%s'!\n", $commod; - $missing++; + next if exists $pctb_commodmap{$commod}; + if (exists $newcommods{$commod}) { + printf STDERR "Ignoring new commodity \`%s'!\n", $commod; + } else { + printf STDERR "Unknown commodity \`%s'!\n", $commod; + $missing++; + } } - die "$missing unknown commoditi(es). OCR failure?\n" + die "$missing unknown commoditi(es).". + " See README (search for \`newcommods').\n" if $missing; } @@ -386,22 +424,122 @@ sub main__genmarketdata () { print $o or die $!; } -sub main__upload () { +sub save_upload_html ($$) { + my ($which, $resptxt) = @_; + open R, ">./_upload-$which.html" or die $!; + print R $resptxt or die $!; + close R or die $!; +} + +sub gzip ($) { + my ($raw) = @_; + my $tf= pipethrough_prep(); + print $tf $raw or die $!; + return pipethrough_run($tf,undef,'gzip','gzip'); +} + +sub main__uploadyaarg () { + my %o; + $o{'ocean'}= $ENV{'YPPSC_OCEAN'} or die; + $o{'island'}= $ENV{'YPPSC_ISLAND'} or die; + $o{'timestamp'}= 10; + get_our_version(\%o, 'client'); + + my $tf= pipethrough_prep(); + write_tsv($tf); + my $oz= pipethrough_run_gzip($tf); + $o{'data'}= [ undef, 'deduped.tsv.gz', + Content_Type => 'application/octet-stream', + Content => $oz ]; + + my $dest= $ENV{'YPPSC_YAARG_SUBMIT'}; + + my $respcontent= cgipostform($ua, "$dest/commod-update-receiver", \%o); + $respcontent =~ m/^OK\b/ or die "$respcontent ?"; +} + +sub main__uploadpctb () { + my $ocean= $ENV{'YPPSC_OCEAN'}; die unless $ocean; + my $island= $ENV{'YPPSC_ISLAND'}; die unless $island; die unless $pctb; my $o= genmarketdata(); + $pctb =~ s,/*$,,; my $url= "$pctb/upload.php"; - $url= "http://www.chiark.greenend.org.uk/ucgi/~ijackson/check/upload.php"; my $content= { 'marketdata' => [ undef, "marketdata.gz", Content_Type => 'application/gzip', - Content => $o + Content => gzip($o), ] }; + + print STDERR "Uploading data to $pctb...\n"; + my $resp= $ua->post("$url", Content => $content, Content_Type => 'form-data'); die $resp->status_line unless $resp->is_success; - print "[[ ",$resp->content," ]]\n"; + my $resptxt= $resp->content(); + save_upload_html('1', $resptxt); + + open R, ">./_upload-1.html" or die $!; + print R $resptxt or die $!; + close R or die $!; + + my @filenames= $resptxt =~ + m/input\s+type="hidden"\s+name="filename"\s+value=\"([_.0-9a-z]+)\"/ig; + @filenames or die; + + my @forcerls= $resptxt =~ + m/input\s+type="hidden"\s+name="forcereload"\s+value=\"([1-9]\d+)\"/ig; + @forcerls or die; + + my $filename= $filenames[0]; + my $forcerl= $forcerls[0]; + + $ocean= ucfirst lc $ocean; + my @oceanids= $resptxt =~ + m/\