Merge branch 'stable-3.x'

diff --git a/yarrg/Commods.pm b/yarrg/Commods.pm
index 9c8281876f334b99578e73622cf95f00e9ec6259..372fe16f51544081e9378d68563a91a5b38868df 100644 (file)
--- a/yarrg/Commods.pm
+++ b/yarrg/Commods.pm
@@ -24,6 +24,7 @@ package Commods;
 use IO::File;
 use HTTP::Request::Common ();
 use POSIX;
+use LWP::UserAgent;
 
 use strict;
 use warnings;
@@ -41,7 +42,8 @@ BEGIN {
                      &get_our_version &check_tsv_line
                      &pipethrough_prep &pipethrough_run
                      &pipethrough_run_along &pipethrough_run_finish
-                     &pipethrough_run_gzip
+                     &pipethrough_run_gzip &http_useragent &version_core
+                     &http_useragent_string_map
                      &cgipostform &yarrgpostform &cgi_get_caller
                      &set_ctype_utf8 $masterinfoversion);
     %EXPORT_TAGS = ( );
@@ -232,7 +234,14 @@ sub get_our_version ($$) {
     my ($aref,$prefix) = @_;
     $aref->{"${prefix}name"}= 'ypp-sc-tools yarrg';
     $aref->{"${prefix}fixes"}= 'lastpage checkpager';
+    $aref->{"${prefix}version"}= version_core();
+    return $aref;
+    # clientname       "ypp-sc-tools"
+    # clientversion    2.1-g2e06a26  [from git-describe --tags HEAD]
+    # clientfixes      "lastpage"  [space separated list]
+}
 
+sub version_core () {
     my $version= `
        if type -p git-describe >/dev/null 2>&1; then
                gd=git-describe
@@ -242,11 +251,7 @@ sub get_our_version ($$) {
        \$gd --tags HEAD || echo 0unknown
     `; $? and die $?;
     chomp($version);
-    $aref->{"${prefix}version"}= $version;
-    return $aref;
-    # clientname       "ypp-sc-tools"
-    # clientversion    2.1-g2e06a26  [from git-describe --tags HEAD]
-    # clientfixes      "lastpage"  [space separated list]
+    return $version;
 }
 
 sub pipethrough_prep () {
@@ -386,4 +391,22 @@ sub set_ctype_utf8 () {
     setlocale(LC_CTYPE, "en.UTF-8");
 }
 
+sub http_useragent_string_map ($$) {
+    my ($caller_lib_agent, $reason_style_or_caller) = @_;
+    $caller_lib_agent =~ y/A-Za-z/N-ZA-Mn-za-m/;
+    $caller_lib_agent =~ s/\s/_/g;
+    my $version= version_core();
+    return "yarrg/$version ($reason_style_or_caller)".
+          " $caller_lib_agent".
+          " (http://yarrg.chiark.net/intro)";
+}
+
+sub http_useragent ($) {
+    my ($who) = @_;
+    my $ua= LWP::UserAgent->new;
+    my $base= $ua->_agent();
+    $ua->agent(http_useragent_string_map($base, $who));
+    return $ua;
+}
+
 1;

diff --git a/yarrg/README b/yarrg/README
index 9b594591c40721d4bbb3d013619140cf20c89d1b..b542a1cca8e33e543853972df9f6845e05eb7ecd 100644 (file)
--- a/yarrg/README
+++ b/yarrg/README
@@ -4,7 +4,7 @@ YARRG - Yet Another Revenue Research Gatherer
 Overview
 --------
 
-This tool can:
+This tool will:
   - screenscrape the commodities trading screen
   - produce the results as a tab separated values file
   - upload the results to the YARRG and PCTB servers
@@ -14,9 +14,8 @@ To build, install the dependencies, cd to this directory and type `make'.
 To screenscrape and upload to both servers, select `trade
 commodities' from the hold of a vessel or building, and run:
    ./yarrg
-Currently we upload to the dedicated yarrg server yarrg.chiark.net,
-and also to pctb.ilk.org (the testing instance of the PCTB database,
-pending approval from the operators of the main server).
+We upload to the dedicated yarrg server http://yarrg.chiark.net/,
+and also to the PCTB server http://pctb.crabdance.com/.
 
 Or, for example, for a tab-separated values dump:
    ./yarrg --tsv >commods.tsv

diff --git a/yarrg/README.privacy b/yarrg/README.privacy
index d9f8b60d3c93368c6e501561e0eb60486496f189..769134f376783cca6bb0f3b263646c71e1141e44 100644 (file)
--- a/yarrg/README.privacy
+++ b/yarrg/README.privacy
@@ -15,7 +15,6 @@ helpers, you'll have to read the source.  Sorry.)
 
 These servers records everyone's uploads of commodity prices and
 allows you to search for routes using the PCTB and YARRG websites.
-(*NB* currently the YARRG website is still under development.)
 
 The YARRG data upload server may also distribute the uploaded data to
 other searching site operators for inclusion in their databases; your

diff --git a/yarrg/commod-results-processor b/yarrg/commod-results-processor
index bb0724d0d81e0c26204152879d45f605a7b64c1e..36804d35d64c34f25f07fbb22a0de06ad8ee9454 100755 (executable)
--- a/yarrg/commod-results-processor
+++ b/yarrg/commod-results-processor
@@ -29,7 +29,6 @@ use strict (qw(vars));
 use HTTP::Request;
 use IO::File;
 use POSIX;
-use LWP::UserAgent;
 use XML::Parser;
 
 use Commods;
@@ -129,8 +128,8 @@ sub main__arbitrage () {
            $info.=
                sprintf("%-13.13s| %-19.19s %4d| %-19.19s %4d|%3d x%3d =%3d\n",
                       $commod,
-                      $buys[0]{Stall},$buys[0]{Price},
                       $sells[0]{Stall},$sells[0]{Price},
+                      $buys[0]{Stall},$buys[0]{Price},
                       $qty, $pricediff, $tprofit);
            sub arb_subtract_qty (\@) {
                my ($verbs) = @_;
@@ -230,7 +229,7 @@ sub main__tsv () {
 
 our ($pctb) = $ENV{'YPPSC_YARRG_PCTB'};
 
-our ($ua)= LWP::UserAgent->new;
+our ($ua)= http_useragent("commod-results-processor $mode");
 
 sub refresh_commodmap() {
     die unless $pctb;
@@ -288,7 +287,16 @@ sub refresh_commodmap() {
     my $content= $resp->content;
 
 #    print STDERR "[[[$content]]]\n";
-    $xp->parse($content);
+    my $commodmapxmltmp= '_commodmap.xml';
+    if (!eval {
+       $xp->parse($content); 1;
+    }) {
+       open R, ">./$commodmapxmltmp" or die $!;
+       print R $content or die $!;
+       close R or die $!;
+       die "$@ parsing commodmap";
+    }
+    unlink $commodmapxmltmp or $!==&ENOENT or die $!;
     close $o or die $!;
     rename "_commodmap.tsv.tmp","_commodmap.tsv" or die $!;
 }

diff --git a/yarrg/convert.c b/yarrg/convert.c
index 76a9a7af96ac18c97d0e941496ff2b495551d182..4ba23b514d45a6363ab6336c9a37dcd3daed63a6 100644 (file)
--- a/yarrg/convert.c
+++ b/yarrg/convert.c
@@ -332,7 +332,7 @@ int main(int argc, char **argv) {
             o_serv_yarrg,       o_flags & ff_use_yarrg);
 
   set_server("YPPSC_YARRG_PCTB",
-            "http://",          "pctb.ilk.org" /*pctb.crabdance.com*/,
+            "http://",          "pctb.crabdance.com",
                                 "pctb.ilk.org",
             o_serv_pctb,        o_flags & ff_use_pctb);
             

diff --git a/yarrg/database-info-fetch b/yarrg/database-info-fetch
index 90db002aa4bddd79dd320c61eabaafece677e577..041e008f4220e0a775c8ce4aeab18f5c7d4535ff 100755 (executable)
--- a/yarrg/database-info-fetch
+++ b/yarrg/database-info-fetch
@@ -25,7 +25,6 @@
 # sponsored by Three Rings.
 
 use strict (qw(vars));
-use LWP::UserAgent;
 use JSON;
 #use Data::Dumper;
 use IO::File;
@@ -38,7 +37,7 @@ our ($which) = shift @ARGV;
 $which =~ s/\W//g;
 
 our ($pctb) = $ENV{'YPPSC_YARRG_PCTB'};
-our ($ua)= LWP::UserAgent->new;
+our ($ua)= http_useragent("database_info_fetch $which");
 our $jsonresp;
 
 sub jparsetable ($$) {
@@ -193,6 +192,14 @@ sub main__timestamp () {
     exit(0);
 }
 
+sub main__yarrgversion () {
+    printf "%s\n", version_core();
+}
+
+sub main__useragentstringmap ($$) {
+    printf "%s\n", http_useragent_string_map($_[0], $_[1]);
+}
+
 sub main__sunshinewidget () {
     print <<END
 Land {On land} {

diff --git a/yarrg/dictionary-manager b/yarrg/dictionary-manager
index 65fc977b505fd9b8c3b3dd343e37e891d853edff..2231266edc8f0e0ef441067a7e5ffd276ba52ba6 100755 (executable)
--- a/yarrg/dictionary-manager
+++ b/yarrg/dictionary-manager
@@ -509,6 +509,12 @@ proc upload_init {} {
     if {$privacy_setting} {
        package require http
        ::http::config -urlencoding utf-8
+       set ua [::http::config -useragent]
+       debug "USERAGENT OLD \"$ua\""
+       set ua [exec ./database-info-fetch useragentstringmap $ua \
+                   dictionary-manager 2>@ stderr]
+       ::http::config -useragent $ua
+       debug "USERAGENT NEW \"$ua\""
     }
 }
 

diff --git a/yarrg/structure.c b/yarrg/structure.c
index a49fe0fd7e91782d6d1f68254ac6974e74097e87..46b6853339580cb6df798fc3b28c835b95b7c59b 100644 (file)
--- a/yarrg/structure.c
+++ b/yarrg/structure.c
@@ -444,11 +444,16 @@ void find_structure(const CanonImage *im,
 }                  
 
 void check_correct_commodities(void) {
-  Rect search= { { 50,39 }, { 130,59 } };
+  Rect search;
 
+  search.tl= s.mr.tl;  search.tl.x +=  34;  search.tl.y -= 44;
+  search.br= s.mr.tl;  search.br.x += 114;  search.br.y -= 23;
+  MUST(search.br.x < cim->w-1 && search.tl.y > 0,
+       MR(search);MI(cim->w);MI(cim->h));
+  
+  debug_rect("commodselr",0, search);
   ADJUST_BOX(search,"_",>=,10, cim->h, MUST, tl,y,+1);
   ADJUST_BOX(search,"_",>=,10, 0,      MUST, br,y,-1);
-
   debug_rect("commodselr",1, search);
 
   static const char *all_small[]= {

diff --git a/yarrg/yppedia-ocean-scraper b/yarrg/yppedia-ocean-scraper
index 476c1cd9b381bb0d5d5ec1979cc4928bff9fa9f4..68efa38f6781980c63308a55b55b4334195873d5 100755 (executable)
--- a/yarrg/yppedia-ocean-scraper
+++ b/yarrg/yppedia-ocean-scraper
@@ -37,8 +37,8 @@ signal.signal(signal.SIGINT, signal.SIG_DFL)
 import sys
 import os
 import urllib
-import urllib2
 import re as regexp
+import subprocess
 from optparse import OptionParser
 from BeautifulSoup import BeautifulSoup
 
@@ -59,6 +59,20 @@ def fix_stdout():
 fix_stdout()
 
 
+# User agent:
+class YarrgURLopener(urllib.FancyURLopener):
+       base_version= urllib.URLopener().version
+       proc= subprocess.Popen(
+               ["./database-info-fetch", "useragentstringmap",
+                base_version, "manual islands/topology fetch"],
+               shell=False,
+               stderr=None,
+               stdout=subprocess.PIPE,
+               )
+       version = proc.communicate()[0].rstrip('\n');
+       assert(proc.returncode is not None and proc.returncode == 0)
+urllib._urlopener = YarrgURLopener()
+
 ocean = None
 soup = None
 opts = None
@@ -77,7 +91,7 @@ def fetch():
        url = ('http://yppedia.puzzlepirates.com/' +
                        (url_base % urllib.quote(ocean,'')))
        debug('fetching',url)
-       dataf = urllib2.urlopen(url)
+       dataf = urllib.urlopen(url)
        debug('fetched',dataf)
        soup = BeautifulSoup(dataf)