use IO::File;
use HTTP::Request::Common ();
use POSIX;
+use LWP::UserAgent;
use strict;
use warnings;
&get_our_version &check_tsv_line
&pipethrough_prep &pipethrough_run
&pipethrough_run_along &pipethrough_run_finish
- &pipethrough_run_gzip
+ &pipethrough_run_gzip &http_useragent &version_core
+ &http_useragent_string_map
&cgipostform &yarrgpostform &cgi_get_caller
&set_ctype_utf8 $masterinfoversion);
%EXPORT_TAGS = ( );
my ($aref,$prefix) = @_;
$aref->{"${prefix}name"}= 'ypp-sc-tools yarrg';
$aref->{"${prefix}fixes"}= 'lastpage checkpager';
+ $aref->{"${prefix}version"}= version_core();
+ return $aref;
+ # clientname "ypp-sc-tools"
+ # clientversion 2.1-g2e06a26 [from git-describe --tags HEAD]
+ # clientfixes "lastpage" [space separated list]
+}
+sub version_core () {
my $version= `
if type -p git-describe >/dev/null 2>&1; then
gd=git-describe
\$gd --tags HEAD || echo 0unknown
`; $? and die $?;
chomp($version);
- $aref->{"${prefix}version"}= $version;
- return $aref;
- # clientname "ypp-sc-tools"
- # clientversion 2.1-g2e06a26 [from git-describe --tags HEAD]
- # clientfixes "lastpage" [space separated list]
+ return $version;
}
sub pipethrough_prep () {
setlocale(LC_CTYPE, "en.UTF-8");
}
+sub http_useragent_string_map ($$) {
+ my ($caller_lib_agent, $reason_style_or_caller) = @_;
+ $caller_lib_agent =~ y/A-Za-z/N-ZA-Mn-za-m/;
+ $caller_lib_agent =~ s/\s/_/g;
+ my $version= version_core();
+ return "yarrg/$version ($reason_style_or_caller)".
+ " $caller_lib_agent".
+ " (http://yarrg.chiark.net/intro)";
+}
+
+sub http_useragent ($) {
+ my ($who) = @_;
+ my $ua= LWP::UserAgent->new;
+ my $base= $ua->_agent();
+ $ua->agent(http_useragent_string_map($base, $who));
+ return $ua;
+}
+
1;
Overview
--------
-This tool can:
+This tool will:
- screenscrape the commodities trading screen
- produce the results as a tab separated values file
- upload the results to the YARRG and PCTB servers
To screenscrape and upload to both servers, select `trade
commodities' from the hold of a vessel or building, and run:
./yarrg
-Currently we upload to the dedicated yarrg server yarrg.chiark.net,
-and also to pctb.ilk.org (the testing instance of the PCTB database,
-pending approval from the operators of the main server).
+We upload to the dedicated yarrg server http://yarrg.chiark.net/,
+and also to the PCTB server http://pctb.crabdance.com/.
Or, for example, for a tab-separated values dump:
./yarrg --tsv >commods.tsv
These servers records everyone's uploads of commodity prices and
allows you to search for routes using the PCTB and YARRG websites.
-(*NB* currently the YARRG website is still under development.)
The YARRG data upload server may also distribute the uploaded data to
other searching site operators for inclusion in their databases; your
use HTTP::Request;
use IO::File;
use POSIX;
-use LWP::UserAgent;
use XML::Parser;
use Commods;
$info.=
sprintf("%-13.13s| %-19.19s %4d| %-19.19s %4d|%3d x%3d =%3d\n",
$commod,
- $buys[0]{Stall},$buys[0]{Price},
$sells[0]{Stall},$sells[0]{Price},
+ $buys[0]{Stall},$buys[0]{Price},
$qty, $pricediff, $tprofit);
sub arb_subtract_qty (\@) {
my ($verbs) = @_;
our ($pctb) = $ENV{'YPPSC_YARRG_PCTB'};
-our ($ua)= LWP::UserAgent->new;
+our ($ua)= http_useragent("commod-results-processor $mode");
sub refresh_commodmap() {
die unless $pctb;
my $content= $resp->content;
# print STDERR "[[[$content]]]\n";
- $xp->parse($content);
+ my $commodmapxmltmp= '_commodmap.xml';
+ if (!eval {
+ $xp->parse($content); 1;
+ }) {
+ open R, ">./$commodmapxmltmp" or die $!;
+ print R $content or die $!;
+ close R or die $!;
+ die "$@ parsing commodmap";
+ }
+ unlink $commodmapxmltmp or $!==&ENOENT or die $!;
close $o or die $!;
rename "_commodmap.tsv.tmp","_commodmap.tsv" or die $!;
}
o_serv_yarrg, o_flags & ff_use_yarrg);
set_server("YPPSC_YARRG_PCTB",
- "http://", "pctb.ilk.org" /*pctb.crabdance.com*/,
+ "http://", "pctb.crabdance.com",
"pctb.ilk.org",
o_serv_pctb, o_flags & ff_use_pctb);
# sponsored by Three Rings.
use strict (qw(vars));
-use LWP::UserAgent;
use JSON;
#use Data::Dumper;
use IO::File;
$which =~ s/\W//g;
our ($pctb) = $ENV{'YPPSC_YARRG_PCTB'};
-our ($ua)= LWP::UserAgent->new;
+our ($ua)= http_useragent("database_info_fetch $which");
our $jsonresp;
sub jparsetable ($$) {
exit(0);
}
+sub main__yarrgversion () {
+ printf "%s\n", version_core();
+}
+
+sub main__useragentstringmap ($$) {
+ printf "%s\n", http_useragent_string_map($_[0], $_[1]);
+}
+
sub main__sunshinewidget () {
print <<END
Land {On land} {
if {$privacy_setting} {
package require http
::http::config -urlencoding utf-8
+ set ua [::http::config -useragent]
+ debug "USERAGENT OLD \"$ua\""
+ set ua [exec ./database-info-fetch useragentstringmap $ua \
+ dictionary-manager 2>@ stderr]
+ ::http::config -useragent $ua
+ debug "USERAGENT NEW \"$ua\""
}
}
}
void check_correct_commodities(void) {
- Rect search= { { 50,39 }, { 130,59 } };
+ Rect search;
+ search.tl= s.mr.tl; search.tl.x += 34; search.tl.y -= 44;
+ search.br= s.mr.tl; search.br.x += 114; search.br.y -= 23;
+ MUST(search.br.x < cim->w-1 && search.tl.y > 0,
+ MR(search);MI(cim->w);MI(cim->h));
+
+ debug_rect("commodselr",0, search);
ADJUST_BOX(search,"_",>=,10, cim->h, MUST, tl,y,+1);
ADJUST_BOX(search,"_",>=,10, 0, MUST, br,y,-1);
-
debug_rect("commodselr",1, search);
static const char *all_small[]= {
import sys
import os
import urllib
-import urllib2
import re as regexp
+import subprocess
from optparse import OptionParser
from BeautifulSoup import BeautifulSoup
fix_stdout()
+# User agent:
+class YarrgURLopener(urllib.FancyURLopener):
+ base_version= urllib.URLopener().version
+ proc= subprocess.Popen(
+ ["./database-info-fetch", "useragentstringmap",
+ base_version, "manual islands/topology fetch"],
+ shell=False,
+ stderr=None,
+ stdout=subprocess.PIPE,
+ )
+ version = proc.communicate()[0].rstrip('\n');
+ assert(proc.returncode is not None and proc.returncode == 0)
+urllib._urlopener = YarrgURLopener()
+
ocean = None
soup = None
opts = None
url = ('http://yppedia.puzzlepirates.com/' +
(url_base % urllib.quote(ocean,'')))
debug('fetching',url)
- dataf = urllib2.urlopen(url)
+ dataf = urllib.urlopen(url)
debug('fetched',dataf)
soup = BeautifulSoup(dataf)