our (%commodmap);
-our ($pctb) = $ENV{'YPPSC_PCTB_PCTB'}; die unless $pctb;
+our ($pctb) = $ENV{'YPPSC_PCTB_PCTB'};
our ($ua)= LWP::UserAgent->new;
sub load_commodmap() {
undef %commodmap;
- my $c= new IO::File "#commodmap#.tsv";
+ my $c= new IO::File "_commodmap.tsv";
if (!$c) { $!==&ENOENT or die $!; return; }
while (<$c>) {
m/^(\S.*\S)\t(\d+)\n$/ or die "$_";
}
sub refresh_commodmap() {
+ die unless $pctb;
+ $pctb =~ s,/*$,,;
my $resp= $ua->get("$pctb/commodmap.php?version=2");
die $resp->status_line unless $resp->is_success;
my $incommodmap=0;
my $intag='';
my %got;
- my $o= new IO::File "#commodmap#.tsv.new",'w' or die $!;
+ my $o= new IO::File "_commodmap.tsv.tmp",'w' or die $!;
undef %commodmap;
my $xp= new XML::Parser
# print STDERR "[[[$content]]]\n";
$xp->parse($content);
close $o or die $!;
- rename "#commodmap#.tsv.new","#commodmap#.tsv" or die $!;
+ rename "_commodmap.tsv.tmp","_commodmap.tsv" or die $!;
+}
+
+our %newcommods;
+
+sub read_newcommods ($) {
+ my ($file) = @_;
+ if (!open NC, "< $file") {
+ $!==&ENOENT or die $!;
+ return;
+ }
+ while (<NC>) {
+ chomp; s/^\s*//; s/\s+$//;
+ next if m/^\#/;
+ next unless m/\S/;
+ $newcommods{$_}= 1;
+ }
+ NC->error and die $!;
+ close NC or die $!;
+}
+
+sub refresh_newcommods() {
+ my $rsync= $ENV{'YPPSC_PCTB_RSYNC'};
+ $rsync= 'rsync' if !defined $rsync;
+
+ my $local= "_master-newcommods.txt";
+ my $src= $ENV{'YPPSC_PCTB_DICT_UPDATE'};
+ if ($src) {
+ my $remote= "$src/master-newcommods.txt";
+ $!=0; system 'rsync','-Lt','--',$remote,$local;
+ die "$? $!" if $! or $?;
+ }
+ read_newcommods($local);
+ read_newcommods('_local-newcommods.txt');
}
our (%stallmap, @stallmap);
my $count= 0;
my $o= '';
- foreach $commod (sort {
- $commodmap{$a} <=> $commodmap{$b}
- } keys %commod) {
+ foreach $commod (
+ sort { $commodmap{$a} <=> $commodmap{$b} }
+ grep { exists $commodmap{$_} }
+ keys %commod
+ ) {
#print STDERR "COMMOD $commod\n";
$current= $commod{$commod};
my $l= bs_p($commod,$bs,$sortmul);
sub writeint { return pack 'v*', @_; }
+our (%stalltypetoabbrevmap)= qw(
+ Apothecary A
+ Distilling D
+ Furnishing F
+ Ironworking I
+ Shipbuilding S
+ Tailoring T
+ Weaving W
+ );
+
sub genmarketdata () {
our $version= '005b';
my @missing= grep { !exists $commodmap{$_} } keys %commod;
if (@missing) {
refresh_commodmap();
+ refresh_newcommods();
my $missing=0;
foreach $commod (sort keys %commod) {
next if exists $commodmap{$commod};
- printf STDERR "Unknown commodity \`%s'!\n", $commod;
- $missing++;
+ if (exists $newcommods{$commod}) {
+ printf STDERR "Ignoring new commodity \`%s'!\n", $commod;
+ } else {
+ printf STDERR "Unknown commodity \`%s'!\n", $commod;
+ $missing++;
+ }
}
- die "$missing unknown commoditi(es). OCR failure?\n"
+ die "$missing unknown commoditi(es).".
+ " See README (search for \`newcommods').\n"
if $missing;
}
my $ot= sprintf("$version\n".
"%d\n",
scalar(@stallmap));
- foreach $stall (@stallmap) { $ot .= "$stall\n"; }
+ foreach $stall (@stallmap) {
+ my $st= $stall;
+ if ($st =~ m/^(\S+)\'s (\S+) Stall$/) {
+ my $stkind= $stalltypetoabbrevmap{$2};
+ if (defined $stkind) {
+ $st= "$1^$stkind";
+ } else {
+ warn "unknown stall type $2 in $st\n";
+ }
+ }
+ $ot .= "$st\n";
+ }
return $ot.$ob;
}
print $o or die $!;
}
+sub save_upload_html ($$) {
+ my ($which, $resptxt) = @_;
+ open R, ">./_upload-$which.html" or die $!;
+ print R $resptxt or die $!;
+ close R or die $!;
+}
+
+sub gzip ($) {
+ my ($raw) = @_;
+ my $tf= IO::File::new_tmpfile() or die $!;
+ print $tf $raw or die $!;
+ $tf->flush or die $!;
+ $tf->seek(0,0) or die $!;
+ my $child= open GZ, "-|"; defined $child or die $!;
+ if (!$child) {
+ open STDIN, "<&", $tf;
+ exec 'gzip'; die $!;
+ }
+ my $r;
+ { undef $/; $!=0; $r= <GZ>; }
+ defined $r or die $!;
+ close GZ or die "$! $?"; die $? if $?;
+ return $r;
+}
+
sub main__upload () {
+ my $ocean= $ENV{'YPPSC_OCEAN'}; die unless $ocean;
+ my $island= $ENV{'YPPSC_ISLAND'}; die unless $island;
+ die unless $pctb;
my $o= genmarketdata();
+ $pctb =~ s,/*$,,;
my $url= "$pctb/upload.php";
- $url= "http://www.chiark.greenend.org.uk/ucgi/~ijackson/check/upload.php";
my $content= {
'marketdata' => [ undef, "marketdata.gz",
Content_Type => 'application/gzip',
- Content => $o
+ Content => gzip($o),
]
};
+
+ print STDERR "Uploading data to $pctb...\n";
+
my $resp= $ua->post("$url", Content => $content,
Content_Type => 'form-data');
die $resp->status_line unless $resp->is_success;
- print "[[ ",$resp->content," ]]\n";
+ my $resptxt= $resp->content();
+ save_upload_html('1', $resptxt);
+
+ open R, ">./_upload-1.html" or die $!;
+ print R $resptxt or die $!;
+ close R or die $!;
+
+ my @filenames= $resptxt =~
+ m/input\s+type="hidden"\s+name="filename"\s+value=\"([_.0-9a-z]+)\"/ig;
+ @filenames or die;
+
+ my @forcerls= $resptxt =~
+ m/input\s+type="hidden"\s+name="forcereload"\s+value=\"([1-9]\d+)\"/ig;
+ @forcerls or die;
+
+ my $filename= $filenames[0];
+ my $forcerl= $forcerls[0];
+
+ $ocean= ucfirst lc $ocean;
+ my @oceanids= $resptxt =~
+ m/\<option value\=\"(\d+)\"\>$ocean\<\/option\>/;
+ @oceanids==1 or die "@oceanids ?";
+
+ my $islandid;
+ while ($resptxt =~
+ m/^islands\[\d+\]\[\d+\]\=new\s+option\(\"(.*)\"\,(\d+)\)\s*$/mig
+ ) {
+ next unless $1 eq $island;
+ $islandid= $2;
+ }
+ defined $islandid or die;
+
+ die "@filenames ?" if grep { $_ ne $filename } @filenames;
+ die "@forcerls ?" if grep { $_ ne $forcerl } @forcerls;
+
+ my $setisland= {
+ };
+
+ print STDERR "Setting ocean and island...\n";
+
+ my $siurl= ($url . "?action=setisland".
+ "&filename=$filename".
+ "&forcereload=$forcerl".
+ "&ocean=$oceanids[0]".
+ "&island=$islandid");
+ $resp= $ua->get($siurl);
+
+ die $resp->status_line unless $resp->is_success;
+
+ $resptxt= $resp->content();
+ save_upload_html('2', $resptxt);
+
+ die unless $resptxt =~ m/your uploaded data has been processed/i;
+ die unless $resptxt =~ m/your data has been integrated into the database/i;
+
+ $resptxt =~ s/\<a href=\"about:\w+\"\>[^<>]+\<\/a\>//g;
+ save_upload_html('3', $resptxt);
+
+ print "\n" or die $!;
+ system('w3m -T text/html -dump < _upload-3.html');
+
+ print "\n" or die $!;
}
~yarrgweb / ypp-sc-tools.db-test.git / blobdiff