New newcommods machinery for ignoring new commodities

diff --git a/pctb/README b/pctb/README
index e9f4f097b06adb1e6fe73a9c4a5a5b34766170a7..f20ebb89fde6fc899151b4a62aa17af0cf9eb836 100644 (file)
--- a/pctb/README
+++ b/pctb/README
@@ -75,6 +75,24 @@ The program reads and writes the following files:
    it.  Don't try `display vid:#pages#.pnm' as this will consume
    truly stupendous quantities of RAM - it wedged my laptop.
 
    it.  Don't try `display vid:#pages#.pnm' as this will consume
    truly stupendous quantities of RAM - it wedged my laptop.
 

+ * #master-newcommods#.txt #local-newcommods#.txt
+
+   Dictionary of newly introduced commodities.  When a new commodity
+   appears in Puzzle Pirates, the PCTB server operators need to add it
+   to their database for us to be able to upload data about it.
+
+   It can sometimes take a few days to do this.  In the meantime, it
+   is possible to upload partial data - data just omitting that
+   commodity.  This is controlled by these files: they list
+   commodities which should be automatically ignored if the PCTB
+   server doesn't know about them.  The master file is downloaded and
+   updated automatically from my server.  You may create the local
+   file yourself.  The format is simple: one commodity per line.
+
+   Unrecognised commodities can also be due to OCR failure so
+   double-check what you're doing before overriding the uploader by
+   telling it to ignore an unrecognised commodity.
+

  * #master-char*#.txt  #local-char*#.txt
    #master-pixmap#.txt #local-pixmap#.txt
 
  * #master-char*#.txt  #local-char*#.txt
    #master-pixmap#.txt #local-pixmap#.txt
 

diff --git a/pctb/README.privacy b/pctb/README.privacy
index a59a2ada9910883119100ca0fc5581575241fd90..fa63dea55c84df25f472c5bf3f57c75011591eb4 100644 (file)
--- a/pctb/README.privacy
+++ b/pctb/README.privacy
@@ -29,9 +29,10 @@ won't happen.
 2. YPP SC PCTB client dictionary server
 =======================================
 
 2. YPP SC PCTB client dictionary server
 =======================================
 

-This server maintains the master database of character and island name
-images, which is used for the commodity screen OCR and also for
-determining your island name.
+This server maintains the master databases of character and island
+name images, and of new commodity names, which is used for the
+commodity screen OCR, for determining your island name, and for
+checking whether commodities not found on the PCTB server are real.

 
 By default, we ask the server for an updated set of dictionaries every
 time we run; this is done with the rsync protocol (indeed, by invoking
 
 By default, we ask the server for an updated set of dictionaries every
 time we run; this is done with the rsync protocol (indeed, by invoking

diff --git a/pctb/commod-results-processor b/pctb/commod-results-processor
index c3cbe8e1b479447245540556754789863109f453..e3f45d177b190c00ee7451fdf73f66cd99bb8bca 100755 (executable)
--- a/pctb/commod-results-processor
+++ b/pctb/commod-results-processor
@@ -295,6 +295,39 @@ sub refresh_commodmap() {
     rename "#commodmap#.tsv.new","#commodmap#.tsv" or die $!;
 }
 
     rename "#commodmap#.tsv.new","#commodmap#.tsv" or die $!;
 }
 

+our %newcommods;
+
+sub read_newcommods ($) {
+    my ($file) = @_;
+    if (!open NC, "< $file") {
+       $!==&ENOENT or die $!;
+       return;
+    }
+    while (<NC>) {
+       chomp; s/^\s*//; s/\s+$//;
+       next if m/^\#/;
+       next unless m/\S/;
+       $newcommods{$_}= 1;
+    }
+    NC->error and die $!;
+    close NC or die $!;
+}
+
+sub refresh_newcommods() {
+    my $rsync= $ENV{'YPPSC_PCTB_RSYNC'};
+    $rsync= 'rsync' if !defined $rsync;
+
+    my $local= "#master-newcommods#.txt";
+    my $src= $ENV{'YPPSC_PCTB_DICT_UPDATE'};
+    if ($src) {
+       my $remote= "$src/master-newcommods.txt";
+       $!=0; system 'rsync','-Lt','--',$remote,$local;
+       die "$? $!" if $! or $?;
+    }
+    read_newcommods($local);
+    read_newcommods('#local-newcommods#.txt');
+}
+

 our (%stallmap, @stallmap);
 
 sub bs_gen_md ($$) {
 our (%stallmap, @stallmap);
 
 sub bs_gen_md ($$) {


@@ -302,9 +335,11 @@ sub bs_gen_md ($$) {
     my $count= 0;
     my $o= '';
     
     my $count= 0;
     my $o= '';
     

-    foreach $commod (sort {
-       $commodmap{$a} <=> $commodmap{$b}
-    } keys %commod) {
+    foreach $commod (
+                    sort { $commodmap{$a} <=> $commodmap{$b} }
+                    grep { exists $commodmap{$_} }
+                    keys %commod
+                    ) {

 #print STDERR "COMMOD $commod\n";
         $current= $commod{$commod};
        my $l= bs_p($commod,$bs,$sortmul);
 #print STDERR "COMMOD $commod\n";
         $current= $commod{$commod};
        my $l= bs_p($commod,$bs,$sortmul);


@@ -350,13 +385,19 @@ sub genmarketdata () {
     my @missing= grep { !exists $commodmap{$_} } keys %commod;
     if (@missing) {
        refresh_commodmap();
     my @missing= grep { !exists $commodmap{$_} } keys %commod;
     if (@missing) {
        refresh_commodmap();

+       refresh_newcommods();

        my $missing=0;
        foreach $commod (sort keys %commod) {
            next if exists $commodmap{$commod};
        my $missing=0;
        foreach $commod (sort keys %commod) {
            next if exists $commodmap{$commod};

-           printf STDERR "Unknown commodity \`%s'!\n", $commod;
-           $missing++;
+           if (exists $newcommods{$commod}) {
+               printf STDERR "Ignoring new commodity \`%s'!\n", $commod;
+           } else {
+               printf STDERR "Unknown commodity \`%s'!\n", $commod;
+               $missing++;
+           }

        }
        }

-       die "$missing unknown commoditi(es).  OCR failure?\n"
+       die "$missing unknown commoditi(es).".
+           "  See README (search for \`newcommods').\n"

            if $missing;
     }    
 
            if $missing;
     }