From: Ian Jackson <ijackson@chiark.greenend.org.uk>
Date: Tue, 1 Sep 2009 16:52:10 +0000 (+0100)
Subject: Fix charset problems; support Opal and Jade
X-Git-Tag: 3.4~31
X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=ypp-sc-tools.db-test.git;a=commitdiff_plain;h=03e5036282e2fc11ca05928c383c3c3c5b2d6355

Fix charset problems; support Opal and Jade
---

diff --git a/yarrg/TODO b/yarrg/TODO
index c824e38..6d1412d 100644
--- a/yarrg/TODO
+++ b/yarrg/TODO
@@ -14,16 +14,13 @@ DATABASE/DICTIONARY MANAGER
 	eliminate black dye from live database
 
 O N	yppedia distance scraper finish
-	BUGS	- Sage fails with a `no possible target' (line 321)
-		- copyright and usage info about update-master-info needed
+	copyright and usage info about update-master-info needed
 
 	when update rejected print better error message including
 	 broken commodity name
 
 	notice commodities deleted from source-info and warn about them
 
-	support Jade and Opal (UTF-8 trouble with Python ?!)
-
 WEBSITE
 -------
 
diff --git a/yarrg/source-info.txt b/yarrg/source-info.txt
index f39e21b..e63eee7 100644
--- a/yarrg/source-info.txt
+++ b/yarrg/source-info.txt
@@ -123,6 +123,11 @@ commods
  wood			175kg 250l
 
 
+client ypp-sc-tools yarrg
+ lastpage
+
+
+#---------- OCEANS ----------
 # subscriber oceans
 
 ocean Midnight
@@ -158,9 +163,16 @@ ocean Viridian
 ocean Crimson
 
 # International oceans (doubloon oceans)
-# broken due to charset problems
-#ocean Jade
-#ocean Opal
 
-client ypp-sc-tools yarrg
- lastpage
+ocean Jade
+ Cigüeña
+  Isla Scrimshaw
+ Ibis
+  Isla Kiwara
+ Águila
+  Cayo Escorbuto
+
+ocean Opal
+ Canis
+  Atchafalaya-Insel
+
diff --git a/yarrg/update-master-info b/yarrg/update-master-info
index 3f570a6..3515bff 100755
--- a/yarrg/update-master-info
+++ b/yarrg/update-master-info
@@ -4,6 +4,7 @@ use strict (qw(vars));
 use DBI;
 use Commods;
 
+$ENV{'LC_CTYPE'}= 'en_GB.UTF-8';
 
 sub full ($) {
     my ($ocean) = @_;
diff --git a/yarrg/yppedia-ocean-scraper b/yarrg/yppedia-ocean-scraper
index a854035..476c1cd 100755
--- a/yarrg/yppedia-ocean-scraper
+++ b/yarrg/yppedia-ocean-scraper
@@ -42,6 +42,23 @@ import re as regexp
 from optparse import OptionParser
 from BeautifulSoup import BeautifulSoup
 
+
+# For fuck's sake!
+import codecs
+import locale
+def fix_stdout():
+    sys.stdout = codecs.EncodedFile(sys.stdout, locale.getpreferredencoding())
+    def null_decode(input, errors='strict'):
+        return input, len(input)
+    sys.stdout.decode = null_decode
+# From
+#  http://ewx.livejournal.com/457086.html?thread=3016574
+#  http://ewx.livejournal.com/457086.html?thread=3016574
+# lightly modified.
+# See also Debian #415968.
+fix_stdout()
+
+
 ocean = None
 soup = None
 opts = None