WIP yppsc-parsedb-updatereceiver; pipeval works

[ypp-sc-tools.db-test.git] / pctb / yppsc-commod-processor

diff --git a/pctb/yppsc-commod-processor b/pctb/yppsc-commod-processor
index 911242c82a5f3a6963e22a7079ce5e2ab86808d4..01db528889af367558bb6add9f82f08e7ade862e 100755 (executable)
--- a/pctb/yppsc-commod-processor
+++ b/pctb/yppsc-commod-processor
@@ -92,8 +92,6 @@ sub bs_p_bestprice ($) {
     }
 }
 
-our $arbitrage_only= 0;
-
 sub main__arbitrage () {
     my @arbs= ();
     foreach $commod (sort keys %commod) {
@@ -186,8 +184,6 @@ sub main__bestprices () {
        $current= $commod{$commod};
        my $buys=  bs_p($commod,Buy, -1);
        my $sells= bs_p($commod,Sell,+1);
-       if ($arbitrage_only) {
-       }
        printf("%-15.15s", $commod) or die $!;
        bs_p_bestprice($buys);
        bs_p_bestprice($sells);
@@ -223,7 +219,7 @@ sub main__tsv () {
 
 our (%commodmap);
 our ($pctb) = 'http://pctb.ilk.org/';
-our ($ua);
+our ($ua)= LWP::UserAgent->new;
 
 sub load_commodmap() {
     undef %commodmap;
@@ -238,8 +234,7 @@ sub load_commodmap() {
 }
 
 sub refresh_commodmap() {
-    my $ua= LWP::UserAgent->new;
-    my $resp= $ua->get("$pctb/commodmap.php");
+    my $resp= $ua->get("$pctb/commodmap.php?version=2");
     die $resp->status_line unless $resp->is_success;
 
     my $cdata='';
@@ -247,27 +242,28 @@ sub refresh_commodmap() {
     my $intag='';
     my %got;
     my $o= new IO::File "#commodmap#.tsv.new",'w' or die $!;
+    undef %commodmap;
 
     my $xp= new XML::Parser
        (Handlers =>
         {
             Start => sub {
                 $_=$_[1];
-print STDERR "START [$_] intag=$intag icm=$incommodmap\n";
+#print STDERR "START [$_] intag=$intag icm=$incommodmap\n";
                 if (m/^commodmap$/i) {
                     $incommodmap++;
                     undef %got;
                 } elsif (m/^(?:name|index)$/i) {
                     $cdata='';
                     $intag=lc($_) if $incommodmap;
-                    print STDERR "START RECOGNISED $intag icm=$incommodmap\n";
-                } else {
-                    print STDERR "START UNRECOGNISED\n";
+#print STDERR "START RECOGNISED $intag icm=$incommodmap\n";
+#               } else {
+#print STDERR "START UNRECOGNISED\n";
                 }
             },
             End => sub {
                 $_=$_[1];
-print STDERR "END [$_] intag=$intag icm=$incommodmap\n";
+#print STDERR "END [$_] intag=$intag icm=$incommodmap\n";
                 if (m/^commodmap$/i) {
                     $incommodmap--;
                     die unless exists $got{'name'};
@@ -276,7 +272,7 @@ print STDERR "END [$_] intag=$intag icm=$incommodmap\n";
                     my $index= $1;
                     $_= $got{'name'};
                     s/^\s+//; s/\s+$//; s/\n/ /g; s/\s+/ /;
-                    die if exists $commodmap{$_};
+                    die "$_ ?" if exists $commodmap{$_};
                     $commodmap{$_}= $index;
                     print $o "$_\t$index\n" or die $!;
                 } elsif (lc $_ eq $intag) {
@@ -284,27 +280,57 @@ print STDERR "END [$_] intag=$intag icm=$incommodmap\n";
                 }
             },
             Char => sub {
-print STDERR "CHAR [$_[1]] intag=$intag icm=$incommodmap\n";
+#print STDERR "CHAR [$_[1]] intag=$intag icm=$incommodmap\n";
                 $cdata .= $_[1];
             }
         }) or die;
     my $content= $resp->content;
 
-    # hacks to strip off drivel that seems to have been added!
-    $content =~ s/^.*\n(\<\?xml)/$1/s;
-    $content =~ s/\<\/body\>.*//s;
-    print STDERR "[[[$content]]]\n";
+#    print STDERR "[[[$content]]]\n";
     $xp->parse($content);
     close $o or die $!;
     rename "#commodmap#.tsv.new","#commodmap#.tsv" or die $!;
 }
-       
-#
-#sub bs_gen_md ($) {
-#    my ($bs) = @_;
-       
 
-sub main__genmarketdata () {
+our (%stallmap, @stallmap);
+
+sub bs_gen_md ($$) {
+    my ($bs,$sortmul) = @_;
+    my $count= 0;
+    my $o= '';
+    
+    foreach $commod (sort {
+       $commodmap{$a} <=> $commodmap{$b}
+    } keys %commod) {
+#print STDERR "COMMOD $commod\n";
+        $current= $commod{$commod};
+       my $l= bs_p($commod,$bs,$sortmul);
+       next unless @$l;
+#print STDERR "COMMOD $commod  has ".scalar(@$l)."\n";
+       $o .= writeint($commodmap{$commod});
+       $o .= writeint(scalar @$l);
+       foreach my $cs (@$l) {
+           $stall= $cs->{Stall};
+           my $stallix= $stallmap{$stall};
+           if (!defined $stallix) {
+               push @stallmap, $stall;
+               $stallmap{$stall}= $stallix= @stallmap;
+#print STDERR "STALL DEF $stallix $stall\n";
+           }
+           my $qty= $cs->{Qty};
+           $qty =~ s/^\>\s*//;
+           $o .= writeint($stallix, $cs->{Price}, $qty+0);
+           $count++;
+       }
+    }
+#print STDERR "COMMOD $commod COUNT WAS $count\n";
+    return
+       writeint($count).$o;
+}
+
+sub writeint { return pack 'v*', @_; }
+
+sub genmarketdata () {
     our $version= '005b';
 
     load_commodmap();
@@ -314,24 +340,44 @@ sub main__genmarketdata () {
        my $missing=0;
        foreach $commod (sort keys %commod) {
            next if exists $commodmap{$commod};
-           print STDERR "Unknown commodity \`%s'!\n";
+           printf STDERR "Unknown commodity \`%s'!\n", $commod;
            $missing++;
        }
-       die "$missing unknown commodities.  OCR failure?\n"
+       die "$missing unknown commoditi(es).  OCR failure?\n"
            if $missing;
     }    
-    
- #   foreach $commod (sort keys %commod) {
-#      next if 
-       
 
-#    bs_gen_md(Buy);
-#    bs_gen_md(Sell);
+    my $ob='';
+    $ob .= bs_gen_md(Buy, -1);
+    $ob .= bs_gen_md(Sell,+1);
+
+    my $ot= sprintf("$version\n".
+                   "%d\n",
+                   scalar(@stallmap));
+    foreach $stall (@stallmap) { $ot .= "$stall\n"; }
+    return $ot.$ob;
 }
 
+sub main__genmarketdata () {
+    my $o= genmarketdata();
+    print $o or die $!;
+}
 
 sub main__upload () {
-    die "\nUploading not yet implemented, sorry.\n";
+    my $o= genmarketdata();
+    my $url= "$pctb/upload.php";
+    $url= "http://www.chiark.greenend.org.uk/ucgi/~ijackson/check/upload.php";
+    my $content= {
+       'marketdata' => [ undef, "marketdata.gz",
+                         Content_Type => 'application/gzip',
+                         Content => $o
+                         ]
+                     };
+    my $resp= $ua->post("$url", Content => $content,
+                       Content_Type => 'form-data');
+    die $resp->status_line unless $resp->is_success;
+
+    print "[[ ",$resp->content," ]]\n";
 }