Merge branch 'master' into ourdb

Conflicts:

	pctb/commod-results-processor

diff --combined pctb/commod-results-processor
index 4beb6d066a59af51b46591b70d7158941f26a9e4,fe26f8847348aea3dbaf7e076f05b2ef52269744..cba01d9ac732a3713a3f658bf1910a456330f4dd
--- 1/pctb/commod-results-processor
--- 2/pctb/commod-results-processor
+++ b/pctb/commod-results-processor
@@@ -32,8 -32,6 +32,8 @@@ use POSIX
  use LWP::UserAgent;
  use XML::Parser;
  
 +use Commods;
 +
  # $commod{'Hemp'}{Buy|Sell}{'stall'}{Stall}
  # $commod{'Hemp'}{Buy|Sell}{'stall'}{Price}
  # $commod{'Hemp'}{Buy|Sell}{'stall'}{Qty}
@@@ -219,10 -217,23 +219,10 @@@ sub main__tsv () 
  }
  
  
 -our (%commodmap);
  our ($pctb) = $ENV{'YPPSC_PCTB_PCTB'};
  
  our ($ua)= LWP::UserAgent->new;
  
 -sub load_commodmap() {
 -    undef %commodmap;
 -    my $c= new IO::File "_commodmap.tsv";
 -    if (!$c) { $!==&ENOENT or die $!; return; }
 -    while (<$c>) {
 -      m/^(\S.*\S)\t(\d+)\n$/ or die "$_";
 -      $commodmap{$1}= $2;
 -    }
 -    $c->error and die $!;
 -    close $c;
 -}
 -
  sub refresh_commodmap() {
      die unless $pctb;
      $pctb =~ s,/*$,,;
@@@ -234,7 -245,7 +234,7 @@@
      my $intag='';
      my %got;
      my $o= new IO::File "_commodmap.tsv.tmp",'w' or die $!;
 -    undef %commodmap;
 +    undef %pctb_commodmap;
  
      my $xp= new XML::Parser
        (Handlers =>
@@@ -264,8 -275,8 +264,8 @@@
                     my $index= $1;
                     $_= $got{'name'};
                     s/^\s+//; s/\s+$//; s/\n/ /g; s/\s+/ /;
 -                   die "$_ ?" if exists $commodmap{$_};
 -                   $commodmap{$_}= $index;
 +                   die "$_ ?" if exists $pctb_commodmap{$_};
 +                   $pctb_commodmap{$_}= $index;
                     print $o "$_\t$index\n" or die $!;
                 } elsif (lc $_ eq $intag) {
                     $got{$intag}= $cdata;
@@@ -325,8 -336,8 +325,8 @@@ sub bs_gen_md ($$) 
      my $o= '';
      
      foreach $commod (
 -                   sort { $commodmap{$a} <=> $commodmap{$b} }
 -                   grep { exists $commodmap{$_} }
 +                   sort { $pctb_commodmap{$a} <=> $pctb_commodmap{$b} }
 +                   grep { exists $pctb_commodmap{$_} }
                     keys %commod
                     ) {
  #print STDERR "COMMOD $commod\n";
@@@ -334,7 -345,7 +334,7 @@@
        my $l= bs_p($commod,$bs,$sortmul);
        next unless @$l;
  #print STDERR "COMMOD $commod  has ".scalar(@$l)."\n";
 -      $o .= writeint($commodmap{$commod});
 +      $o .= writeint($pctb_commodmap{$commod});
        $o .= writeint(scalar @$l);
        foreach my $cs (@$l) {
            $stall= $cs->{Stall};
@@@ -370,14 -381,14 +370,14 @@@ our (%stalltypetoabbrevmap)= qw
  sub genmarketdata () {
      our $version= '005b';
  
 -    load_commodmap();
 -    my @missing= grep { !exists $commodmap{$_} } keys %commod;
 +    parse_pctb_commodmap();
 +    my @missing= grep { !exists $pctb_commodmap{$_} } keys %commod;
      if (@missing) {
        refresh_commodmap();
        refresh_newcommods();
        my $missing=0;
        foreach $commod (sort keys %commod) {
 -          next if exists $commodmap{$commod};
 +          next if exists $pctb_commodmap{$commod};
            if (exists $newcommods{$commod}) {
                printf STDERR "Ignoring new commodity \`%s'!\n", $commod;
            } else {
@@@ -424,17 -435,25 +424,35 @@@ sub save_upload_html ($$) 
      close R or die $!;
  }
  
- sub main__upload_yaarg () {
+ sub gzip ($) {
+     my ($raw) = @_;
+     my $tf= IO::File::new_tmpfile() or die $!;
+     print $tf $raw or die $!;
+     $tf->flush or die $!;
+     $tf->seek(0,0) or die $!;
+     my $child= open GZ, "-|"; defined $child or die $!;
+     if (!$child) {
+       open STDIN, "<&", $tf;
+       exec 'gzip'; die $!;
+     }
+     my $r;
+     { undef $/; $!=0; $r= <GZ>; }
+     defined $r or die $!;
+     close GZ or die "$! $?";  die $? if $?;
+     return $r;
+ }
+ 
 -sub main__upload () {
++sub main__uploadyaarg () {
 +    my $ocean= $ENV{'YPPSC_OCEAN'};  die unless $ocean;
 +    my $island= $ENV{'YPPSC_ISLAND'};  die unless $island;
 +    my $content= {
 +      'data' => [ undef, 'deduped.tsv.gz',
 +                  Content_Type => 'application/octet-stream',
 +                  Content => '' 
 +]};
 +}
 +
- sub main__upload_pctb () {
++sub main__uploadpctb () {
      my $ocean= $ENV{'YPPSC_OCEAN'};  die unless $ocean;
      my $island= $ENV{'YPPSC_ISLAND'};  die unless $island;
      die unless $pctb;
@@@ -444,7 -463,7 +462,7 @@@
      my $content= {
        'marketdata' => [ undef, "marketdata.gz",
                          Content_Type => 'application/gzip',
-                         Content => $o
+                         Content => gzip($o),
                          ]
                      };