add robots.txt to stop crawlers delving into lookup/ and doing searches etc.

[ypp-sc-tools.db-live.git] / yarrg / CommodsScrape.pm

diff --git a/yarrg/CommodsScrape.pm b/yarrg/CommodsScrape.pm
index 3e6f1c68b339924e1b631562c341bd76e9a649c9..d6951cf6b905e99c792a280013cd94395b1ba521 100644 (file)
--- a/yarrg/CommodsScrape.pm
+++ b/yarrg/CommodsScrape.pm
@@ -46,7 +46,7 @@ sub yppedia_chart_parse ($$ $$$$ $) {
        $conv_nxy, $on_archlabel, $on_island, $on_league,
        $on_incomprehensible) = @_;
 
-    my ($x,$y, $arch,$island,$solid,$dirn);
+    my ($x,$y, $arch,$island,$sizecol,$solid,$dirn);
     my $nn= sub { return $conv_nxy->($x,$y) };
     
     # We don't even bother with tag soup; instead we do line-oriented parsing.
@@ -54,7 +54,7 @@ sub yppedia_chart_parse ($$ $$$$ $) {
        s/\<--.*--\>//g;
        s/^\s*//; chomp; s/\s+$//; s/\s+/ /g;
        s/\<\/?(?:b|em)\>//g;
-       s/\{\{chart\ style\|[^{}]*\}\}//gi;
+       s/\{\{(?:chart\ style|Chart league difficulty)\|[^{}]*\}\}//gi;
        next unless m/\{\{/; # only interested in chart template stuff
 
        if (($x,$y,$arch) =
@@ -68,15 +68,15 @@ sub yppedia_chart_parse ($$ $$$$ $) {
            $on_archlabel->($x,$y,$arch);
        } elsif (m/^\{\{ chart\ label \|\d+\|\d+\|
                 \<big\> \'+ \[\[ .* \b ocean \]\]/xi) {
-       } elsif (($x,$y,$island) =
+       } elsif (($x,$y,$island,$sizecol) =
            m/^\{\{ chart\ island\ icon \|(\d+)\|(\d+)\|
-                   ([^| ][^|]*[^| ]) \| .*\}\}$/xi) {
+                   ([^| ][^|]*[^| ]) \| [^|]* \| (\w+) \| .*\}\}$/xi) {
            my $n= $nn->();
            printf $debugfh "%2d,%-2d island %s\n", $x,$y,$island;
-           $on_island->($n, $island);
+           $on_island->($n, $island, $sizecol);
        } elsif (($solid,$x,$y,$dirn) =
            m/^\{\{ chart\ league((?:\ solid)?) \|(\d+)\|(\d+)\|
-                   ([-\/\\o]) \| .*\}\}$/xi) {
+                   \.?([-\/\\o])\.? \| .*\}\}$/xi) {
            next if $dirn eq 'o';
 
            printf $debugfh "%2d,%-2d league %-6s %s\n", $x,$y,
@@ -86,7 +86,7 @@ sub yppedia_chart_parse ($$ $$$$ $) {
            if ($dirn eq '-') { $bx+=2; }
            elsif ($dirn eq '\\') { $bx++; $by++; }
            elsif ($dirn eq '/') { $x++; $by++; }
-           else { die; }
+           else { die "$dirn ?"; }
 
            my $na= $nn->();
            my $nb= $conv_nxy->($bx,$by);