Better HTTP User-Agent strings (rot13 the library, say who we are)

diff --git a/yarrg/Commods.pm b/yarrg/Commods.pm
index b58471ce43a9370a74aa8a6ec8eb11f51822ff62..59ad3e1384cf9dcab67d350a775a617ce9613ba6 100644 (file)
--- a/yarrg/Commods.pm
+++ b/yarrg/Commods.pm
@@ -24,6 +24,7 @@ package Commods;
 use IO::File;
 use HTTP::Request::Common ();
 use POSIX;
 use IO::File;
 use HTTP::Request::Common ();
 use POSIX;

+use LWP::UserAgent;

 
 use strict;
 use warnings;
 
 use strict;
 use warnings;


@@ -40,7 +41,8 @@ BEGIN {
                      &get_our_version &check_tsv_line
                      &pipethrough_prep &pipethrough_run
                      &pipethrough_run_along &pipethrough_run_finish
                      &get_our_version &check_tsv_line
                      &pipethrough_prep &pipethrough_run
                      &pipethrough_run_along &pipethrough_run_finish

-                     &pipethrough_run_gzip
+                     &pipethrough_run_gzip &http_useragent &version_core
+                     &http_useragent_string_map

                      &cgipostform &yarrgpostform &cgi_get_caller
                      &set_ctype_utf8 $masterinfoversion);
     %EXPORT_TAGS = ( );
                      &cgipostform &yarrgpostform &cgi_get_caller
                      &set_ctype_utf8 $masterinfoversion);
     %EXPORT_TAGS = ( );


@@ -247,7 +249,14 @@ sub get_our_version ($$) {
     my ($aref,$prefix) = @_;
     $aref->{"${prefix}name"}= 'ypp-sc-tools yarrg';
     $aref->{"${prefix}fixes"}= 'lastpage checkpager';
     my ($aref,$prefix) = @_;
     $aref->{"${prefix}name"}= 'ypp-sc-tools yarrg';
     $aref->{"${prefix}fixes"}= 'lastpage checkpager';

+    $aref->{"${prefix}version"}= version_core();
+    return $aref;
+    # clientname       "ypp-sc-tools"
+    # clientversion    2.1-g2e06a26  [from git-describe --tags HEAD]
+    # clientfixes      "lastpage"  [space separated list]
+}

 
 

+sub version_core () {

     my $version= `
        if type -p git-describe >/dev/null 2>&1; then
                gd=git-describe
     my $version= `
        if type -p git-describe >/dev/null 2>&1; then
                gd=git-describe


@@ -257,11 +266,7 @@ sub get_our_version ($$) {
        \$gd --tags HEAD || echo 0unknown
     `; $? and die $?;
     chomp($version);
        \$gd --tags HEAD || echo 0unknown
     `; $? and die $?;
     chomp($version);

-    $aref->{"${prefix}version"}= $version;
-    return $aref;
-    # clientname       "ypp-sc-tools"
-    # clientversion    2.1-g2e06a26  [from git-describe --tags HEAD]
-    # clientfixes      "lastpage"  [space separated list]
+    return $version;

 }
 
 sub pipethrough_prep () {
 }
 
 sub pipethrough_prep () {


@@ -401,4 +406,22 @@ sub set_ctype_utf8 () {
     setlocale(LC_CTYPE, "en.UTF-8");
 }
 
     setlocale(LC_CTYPE, "en.UTF-8");
 }
 

+sub http_useragent_string_map ($$) {
+    my ($caller_lib_agent, $reason_style_or_caller) = @_;
+    $caller_lib_agent =~ y/A-Za-z/N-ZA-Mn-za-m/;
+    $caller_lib_agent =~ s/\s/_/g;
+    my $version= version_core();
+    return "yarrg/$version ($reason_style_or_caller)".
+          " $caller_lib_agent".
+          " (http://yarrg.chiark.net/intro)";
+}
+
+sub http_useragent ($) {
+    my ($who) = @_;
+    my $ua= LWP::UserAgent->new;
+    my $base= $ua->_agent();
+    $ua->agent(http_useragent_string_map($base, $who));
+    return $ua;
+}
+

 1;
 1;

diff --git a/yarrg/commod-results-processor b/yarrg/commod-results-processor
index d322c1c7b682638d14c861b573386e97d81cc279..99c6924739391838729b35ea97e17463b83122ed 100755 (executable)
--- a/yarrg/commod-results-processor
+++ b/yarrg/commod-results-processor
@@ -29,7 +29,6 @@ use strict (qw(vars));
 use HTTP::Request;
 use IO::File;
 use POSIX;
 use HTTP::Request;
 use IO::File;
 use POSIX;

-use LWP::UserAgent;

 use XML::Parser;
 
 use Commods;
 use XML::Parser;
 
 use Commods;


@@ -230,7 +229,7 @@ sub main__tsv () {
 
 our ($pctb) = $ENV{'YPPSC_YARRG_PCTB'};
 
 
 our ($pctb) = $ENV{'YPPSC_YARRG_PCTB'};
 

-our ($ua)= LWP::UserAgent->new;
+our ($ua)= http_useragent("commod-results-processor $mode");

 
 sub refresh_commodmap() {
     die unless $pctb;
 
 sub refresh_commodmap() {
     die unless $pctb;

diff --git a/yarrg/database-info-fetch b/yarrg/database-info-fetch
index 90db002aa4bddd79dd320c61eabaafece677e577..041e008f4220e0a775c8ce4aeab18f5c7d4535ff 100755 (executable)
--- a/yarrg/database-info-fetch
+++ b/yarrg/database-info-fetch
@@ -25,7 +25,6 @@
 # sponsored by Three Rings.
 
 use strict (qw(vars));
 # sponsored by Three Rings.
 
 use strict (qw(vars));

-use LWP::UserAgent;

 use JSON;
 #use Data::Dumper;
 use IO::File;
 use JSON;
 #use Data::Dumper;
 use IO::File;


@@ -38,7 +37,7 @@ our ($which) = shift @ARGV;
 $which =~ s/\W//g;
 
 our ($pctb) = $ENV{'YPPSC_YARRG_PCTB'};
 $which =~ s/\W//g;
 
 our ($pctb) = $ENV{'YPPSC_YARRG_PCTB'};

-our ($ua)= LWP::UserAgent->new;
+our ($ua)= http_useragent("database_info_fetch $which");

 our $jsonresp;
 
 sub jparsetable ($$) {
 our $jsonresp;
 
 sub jparsetable ($$) {


@@ -193,6 +192,14 @@ sub main__timestamp () {
     exit(0);
 }
 
     exit(0);
 }
 

+sub main__yarrgversion () {
+    printf "%s\n", version_core();
+}
+
+sub main__useragentstringmap ($$) {
+    printf "%s\n", http_useragent_string_map($_[0], $_[1]);
+}
+

 sub main__sunshinewidget () {
     print <<END
 Land {On land} {
 sub main__sunshinewidget () {
     print <<END
 Land {On land} {

diff --git a/yarrg/dictionary-manager b/yarrg/dictionary-manager
index 65fc977b505fd9b8c3b3dd343e37e891d853edff..2231266edc8f0e0ef441067a7e5ffd276ba52ba6 100755 (executable)
--- a/yarrg/dictionary-manager
+++ b/yarrg/dictionary-manager
@@ -509,6 +509,12 @@ proc upload_init {} {
     if {$privacy_setting} {
        package require http
        ::http::config -urlencoding utf-8
     if {$privacy_setting} {
        package require http
        ::http::config -urlencoding utf-8

+       set ua [::http::config -useragent]
+       debug "USERAGENT OLD \"$ua\""
+       set ua [exec ./database-info-fetch useragentstringmap $ua \
+                   dictionary-manager 2>@ stderr]
+       ::http::config -useragent $ua
+       debug "USERAGENT NEW \"$ua\""

     }
 }
 
     }
 }
 

diff --git a/yarrg/yppedia-ocean-scraper b/yarrg/yppedia-ocean-scraper
index 476c1cd9b381bb0d5d5ec1979cc4928bff9fa9f4..68efa38f6781980c63308a55b55b4334195873d5 100755 (executable)
--- a/yarrg/yppedia-ocean-scraper
+++ b/yarrg/yppedia-ocean-scraper
@@ -37,8 +37,8 @@ signal.signal(signal.SIGINT, signal.SIG_DFL)
 import sys
 import os
 import urllib
 import sys
 import os
 import urllib

-import urllib2

 import re as regexp
 import re as regexp

+import subprocess

 from optparse import OptionParser
 from BeautifulSoup import BeautifulSoup
 
 from optparse import OptionParser
 from BeautifulSoup import BeautifulSoup
 


@@ -59,6 +59,20 @@ def fix_stdout():
 fix_stdout()
 
 
 fix_stdout()
 
 

+# User agent:
+class YarrgURLopener(urllib.FancyURLopener):
+       base_version= urllib.URLopener().version
+       proc= subprocess.Popen(
+               ["./database-info-fetch", "useragentstringmap",
+                base_version, "manual islands/topology fetch"],
+               shell=False,
+               stderr=None,
+               stdout=subprocess.PIPE,
+               )
+       version = proc.communicate()[0].rstrip('\n');
+       assert(proc.returncode is not None and proc.returncode == 0)
+urllib._urlopener = YarrgURLopener()
+

 ocean = None
 soup = None
 opts = None
 ocean = None
 soup = None
 opts = None


@@ -77,7 +91,7 @@ def fetch():
        url = ('http://yppedia.puzzlepirates.com/' +
                        (url_base % urllib.quote(ocean,'')))
        debug('fetching',url)
        url = ('http://yppedia.puzzlepirates.com/' +
                        (url_base % urllib.quote(ocean,'')))
        debug('fetching',url)

-       dataf = urllib2.urlopen(url)
+       dataf = urllib.urlopen(url)

        debug('fetched',dataf)
        soup = BeautifulSoup(dataf)
 
        debug('fetched',dataf)
        soup = BeautifulSoup(dataf)