X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?a=blobdiff_plain;f=src%2Fcom%2Ftedpearson%2Fypp%2Fmarket%2FMarketUploader.java;h=98ef25e955239db0c402a706ded93dcd6555332d;hb=813ffd281d1dff106d6c04ef48925a45d552f223;hp=325d7cc27e133107cf9541142e8848bbceeccb3b;hpb=51eeef799e34499922a74027c64622be2cc11a6d;p=jarrg-ian.git

diff --git a/src/com/tedpearson/ypp/market/MarketUploader.java b/src/com/tedpearson/ypp/market/MarketUploader.java
index 325d7cc..98ef25e 100644
--- a/src/com/tedpearson/ypp/market/MarketUploader.java
+++ b/src/com/tedpearson/ypp/market/MarketUploader.java
@@ -16,11 +16,10 @@ import org.w3c.dom.*;
 import javax.xml.parsers.DocumentBuilderFactory;
 import org.xml.sax.InputSource;
 import java.util.zip.GZIPOutputStream;
-import com.myjavatools.web.ClientHttpRequest;
+import net.chiark.yarrg.ClientHttpRequest;
 import java.util.regex.*;
 import java.util.prefs.Preferences;
 import java.beans.*;
-import com.tedpearson.util.update.*;
 
 /**
 *	MarketUploader is a class that handles the uploading of market
@@ -39,6 +38,9 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	private JFrame frame = null;
 	private Window window = null;
 	private JButton findMarket = null;
+	private JLabel resultSummary = null;
+	private JLabel arbitrageResult = null;
+        private long startTime = 0;
 
 	private final static String PCTB_LIVE_HOST_URL = "http://pctb.crabdance.com/";
 	private final static String PCTB_TEST_HOST_URL = "http://pctb.ilk.org/";
@@ -46,14 +48,16 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 
 	// Yarrg protocol parameters
 	private final static String YARRG_CLIENTNAME = "jpctb greenend";
-	private final static String YARRG_CLIENTVERSION = "0.1";
-	private final static String YARRG_CLIENTFIXES = "";
+	private final static String YARRG_CLIENTVERSION =
+	    com.tedpearson.ypp.market.Version.version;
+	private final static String YARRG_CLIENTFIXES = "bug-094";
 	private final static String YARRG_LIVE_URL = "http://upload.yarrg.chiark.net/commod-update-receiver";
 	private final static String YARRG_TEST_URL = "http://upload.yarrg.chiark.net/test/commod-update-receiver";
 	private String YARRG_URL;
 
 	private boolean uploadToYarrg;
 	private boolean uploadToPCTB;
+	private boolean showArbitrage;
 
 	private String islandName = null;
 	private String oceanName = null;
@@ -61,14 +65,8 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 
 	private AccessibleContext sidePanel;
 	private HashMap<String,Integer> commodMap;
-	private HashMap<String,String> islandNumbers = new HashMap<String,String>();
-	{
-		String[] nums = new String[]
-			{"","Viridian","Midnight","Hunter","Cobalt","Sage","Ice","Malachite","Crimson","Opal"};
-		for(int i=1;i<nums.length;i++) {
-			islandNumbers.put(nums[i],""+i);
-		}
-	}
+        public PrintStream dtxt = null;
+
 	private PropertyChangeListener changeListener = new PropertyChangeListener() {
 		public void propertyChange(PropertyChangeEvent e) {
 			if(e.getNewValue() != null && 
@@ -78,12 +76,42 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 				int index = text.indexOf(":");
 				String name = text.substring(0,index);
 				islandName = name;
-				//System.out.println(islandName);
+				// if (dtxt!=null) dtxt.println(islandName);
 				sidePanel.removePropertyChangeListener(this);
 				latch.countDown();
 			}
 		}
 	    };
+
+        private int parseQty(String str) {
+		if (str.equals(">1000")) {
+			return 1001;
+		} else {
+			return Integer.parseInt(str);
+		}
+	}
+
+        private void progresslog(String s) {
+	    if (dtxt == null) return;
+	    long now = new Date().getTime();
+	    dtxt.println("progress "+(now - startTime)+"ms "+s);
+	}
+
+        private void debug_write_stringdata(String what, String data) throws FileNotFoundException,IOException {
+	    if (dtxt==null) return;
+	    PrintStream strm = new PrintStream(new File("jarrg-debug-"+what));
+	    strm.print(data);
+	    strm.close();
+	}
+
+        private void progressNote(ProgressMonitor pm, String s) {
+	    String arb = null;
+	    if (arbitrageResult != null)
+		arb = arbitrageResult.getText();
+	    if (arb != null && arb.length() != 0)
+		s = "<html>" + arb + "<br>" + s;
+	    pm.setNote(s);
+	}
 	
 	/**
 	*	An abstract market offer, entailing a commodity being bought or sold by
@@ -114,11 +142,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 			commodity = commodId.intValue();
 			price = Integer.parseInt(record.get(priceIndex));
 			String qty = record.get(priceIndex+1);
-			if(qty.equals(">1000")) {
-				quantity = 1001;
-			} else {
-				quantity = Integer.parseInt(record.get(priceIndex+1));
-			}
+			quantity = parseQty(qty);
 			shoppe = stallMap.get(record.get(1)).intValue();
 		}
 		
@@ -218,9 +242,16 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	*	Entry point.  Read our preferences.
 	*/
 	public MarketUploader() {
-		// check if we've been turned off in the control panel
 		Preferences prefs = Preferences.userNodeForPackage(getClass());
 
+		if (prefs.getBoolean("writeDebugFiles", false)) {
+		    try {
+			dtxt = new PrintStream(new File("jarrg-debug-log.txt"));
+		    } catch (java.io.FileNotFoundException e) {
+			System.err.println("JARRG: Error opening debug log: "+e);
+		    }
+		}
+
 		if (prefs.getBoolean("useLiveServers", false)) {
 			YARRG_URL = YARRG_LIVE_URL;
 			PCTB_HOST_URL = PCTB_LIVE_HOST_URL;
@@ -231,6 +262,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 		
 		uploadToYarrg=prefs.getBoolean("uploadToYarrg", true);
 		uploadToPCTB=prefs.getBoolean("uploadToPCTB", true);
+		showArbitrage=prefs.getBoolean("showArbitrage", true);
 
 		EventQueueMonitor.addTopLevelWindowListener(this);
 		if (EventQueueMonitor.isGUIInitialized()) {
@@ -250,9 +282,10 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 			if (window.getAccessibleContext().getAccessibleName().equals("Puzzle Pirates")) frame.setVisible(true);
 			return;
 		}
-		frame = new JFrame("MarketUploader");
+		frame = new JFrame("Jarrg Uploader");
 		frame.setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE);
-		frame.getContentPane().setLayout(new FlowLayout());
+		GridLayout layout = new GridLayout(2,1);
+		frame.getContentPane().setLayout(layout);
 		//frame.setPreferredSize(new Dimension(200, 60));
 		
 		findMarket = new JButton("Upload Market Data");
@@ -261,24 +294,38 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 				findMarket.setEnabled(false);
 				new Thread() {
 					public void run() {
+					        startTime = new Date().getTime();
+						resultSummary.setText("");
+						arbitrageResult.setText("");
 						try {
-							runPCTB();
+							runUpload();
 						} catch(Exception e) {
 							error(e.toString());
 							e.printStackTrace();
+							resultSummary.setText("failed");
 						} finally {
 							if(sidePanel != null) {
 								// remove it if it's still attached
 								sidePanel.removePropertyChangeListener(changeListener);
 							}
 						}
-						//findMarketTable();
 						findMarket.setEnabled(true);
 					}
 				}.start();
 			}
 		});
 		frame.add(findMarket);
+
+		resultSummary = new JLabel("ready");
+		frame.add(resultSummary);
+		
+		arbitrageResult = new JLabel("");
+
+		if (showArbitrage) {
+		    layout.setRows(layout.getRows() + 1);
+		    frame.add(arbitrageResult);
+		}
+
 		frame.pack();
 	}
 	
@@ -318,7 +365,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 				}
 			}
 		}
-		// if we don't find the island name, hopefully the server will
 	}
 
 	/**
@@ -340,6 +386,23 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 		JOptionPane.showMessageDialog(frame,msg,"Error",JOptionPane.ERROR_MESSAGE);
 	}
 	
+	private void error_html(String msg, String html) {
+		Pattern body = Pattern.compile("<body>(.*)</body>", Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
+		Matcher m = body.matcher(html);
+		if (m.find()) {
+			html = m.group(1);
+			Pattern fixup = Pattern.compile("<(\\w+) */>");;
+			m = fixup.matcher(html);
+			html = m.replaceAll("<$1>");
+			m = Pattern.compile("[\\r\\n]+").matcher(html);
+			html = m.replaceAll(" ");
+		}
+		String whole_msg = "<html><h1>Error</h1>"+msg+"<h1>PCTB Server said:</h1><blockquote>"+html+"</blockquote>";
+		if (dtxt!=null) dtxt.println("###" + whole_msg + "###");
+
+		JOptionPane.showMessageDialog(frame,whole_msg,"Error",JOptionPane.ERROR_MESSAGE);
+	}
+	
 	/**
 	*	Run the data collection process, and upload the results. This is the method
 	*	that calls most of the other worker methods for the process. If an error occurs,
@@ -348,22 +411,39 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	*
 	*	@exception Exception if an error we didn't expect occured
 	*/
-	private void runPCTB() throws Exception {
-		String yarrgts = "";
+        private class YarrgTimestampFetcher extends Thread {
+	    public String ts = null;
+	    public void run() {
+		try {
+		    ts = getYarrgTimestamp();
+		    progresslog("(async) yarrg timestamp ready.");
+		} catch(Exception e) {
+		    error("Error getting YARRG timestamp: "+e);
+		}
+	    }
+	};
+
+	private void runUpload() throws Exception {
+		progresslog("starting");
+
 		ProgressMonitor pm = new ProgressMonitor(frame,"Processing Market Data","Getting table data",0,100);
 		pm.setMillisToDecideToPopup(0);
 		pm.setMillisToPopup(0);
+		boolean doneyarrg = false, donepctb = false;
+		YarrgTimestampFetcher yarrgts_thread = null;
 
 		if (uploadToYarrg) {
-			yarrgts = getYarrgTimestamp();
+		        progresslog("(async) yarrg timestamp...");
+			yarrgts_thread = new YarrgTimestampFetcher();
+			yarrgts_thread.start();
 		}
 
-		AccessibleTable t = findMarketTable();
-		if(t == null) {
+		AccessibleTable accesstable = findMarketTable();
+		if(accesstable == null) {
 			error("Market table not found! Please open the Buy/Sell Commodities interface.");
 			return;
 		}
-		if(t.getAccessibleRowCount() == 0) {
+		if(accesstable.getAccessibleRowCount() == 0) {
 			error("No data found, please wait for the table to have data first!");
 			return;
 		}
@@ -372,17 +452,67 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 			return;
 		}
 
+		progresslog("(async) getisland...");
 	        getIsland();
+		progresslog("getocean...");
 		getOcean();
+		progresslog("getocean done");
 
 		if (latch != null) {
 		    latch.await(2, java.util.concurrent.TimeUnit.SECONDS);
 		}
+		progresslog("(async) getisland done");
+
+		String yarrgts = null;
+		if (yarrgts_thread != null) {
+		    progresslog("(async) yarrg timestamp join...");
+		    yarrgts_thread.join();
+		    progresslog("(async) yarrg timestamp joined.");
+		    yarrgts = yarrgts_thread.ts;
+		}
+
+		if (islandName == null) {
+		    error("Could not find island name in YPP user interface.");
+		    return;
+		}
 
-		ArrayList<ArrayList<String>> data = getData(t);
+		progresslog("table check...");
 
-		if (uploadToYarrg) {
-			pm.setNote("Preparing data for Yarrg");
+		String headings_expected[] = new String[]
+		    { "Commodity", "Trading outlet", "Buy price", "Will buy", "Sell price", "Will sell" };
+		ArrayList<ArrayList<String>> headers = getData(accesstable.getAccessibleColumnHeader());
+		if (headers.size() != 1) {
+		    error("Table headings not one row! " + headers.toString());
+		    return;
+		}
+		if (headers.get(0).size() != 6) {
+		    error("Table headings not six columns! " + headers.toString());
+		    return;
+		}
+		for (int col=0; col<headings_expected.length; col++) {
+		    String expd = headings_expected[col];
+		    String got = headers.get(0).get(col);
+		    if (expd.compareTo(got) != 0) {
+			error("Table heading for column "+col
+			      +" is not \""+expd+"\" but \""+got+"\".\n\n"
+			      +"Please do not reorder the table when using this tool.");
+			return;
+		    }
+		}
+
+		progresslog("table read...");
+
+		ArrayList<ArrayList<String>> data = getData(accesstable);
+
+		if (showArbitrage) {
+		        progresslog("arbitrage...");
+			calculateArbitrage(data);
+		        progresslog("arbitrage done.");
+		}
+
+		if (uploadToYarrg && yarrgts != null) {
+		        progresslog("yarrg prepare...");
+			progressNote(pm, "Yarrg: Preparing data");
 			pm.setProgress(10);
 
 			StringBuilder yarrgsb = new StringBuilder();
@@ -402,63 +532,87 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 
 			yarrgdata = yarrgsb.toString();
 
-			pm.setNote("Uploading to Yarrg");
+			progressNote(pm, "Yarrg: Uploading");
+			progresslog("yarrg upload...");
 
-			if (islandName != null) {
-				runYarrg(yarrgts, oceanName, islandName, yarrgdata);
-			} else {
-				System.out.println("Couldn't upload to Yarrg - no island name found");
-			}
+			debug_write_stringdata("yarrg-deduped.tsv", yarrgdata);
+
+			doneyarrg = runYarrg(yarrgts, oceanName, islandName, yarrgdata);
+			progresslog("yarrg done.");
 		}
 
-		pm.setNote("Getting stall names");
-		pm.setProgress(20);
-		if(pm.isCanceled()) {
+		if (uploadToPCTB) {
+		    progresslog("pctb prepare...");
+		    progressNote(pm, "PCTB: Getting stall names");
+		    pm.setProgress(20);
+		    if(pm.isCanceled()) {
 			return;
-		}
-		TreeSet<Offer> buys = new TreeSet<Offer>();
-		TreeSet<Offer> sells = new TreeSet<Offer>();
-		LinkedHashMap<String,Integer> stallMap = getStallMap(data);
-		pm.setProgress(40);
-		pm.setNote("Sorting offers");
-		if(pm.isCanceled()) {
+		    }
+		    TreeSet<Offer> buys = new TreeSet<Offer>();
+		    TreeSet<Offer> sells = new TreeSet<Offer>();
+		    LinkedHashMap<String,Integer> stallMap = getStallMap(data);
+		    pm.setProgress(40);
+		    progressNote(pm, "PCTB: Sorting offers");
+		    if(pm.isCanceled()) {
 			return;
-		}
-		// get commod map
+		    }
+		    // get commod map
 		
-		HashMap<String,Integer> commodMap = getCommodMap();
-		if(commodMap == null) {
+		    progresslog("pctb commodmap...");
+		    HashMap<String,Integer> commodMap = getCommodMap();
+		    if(commodMap == null) {
 			return;
-		}
-		int[] offerCount = getBuySellMaps(data,buys,sells,stallMap,commodMap);
-		//println(buys.toString());
-		//System.out.println(sells);
-		//System.out.println("\n\n\n"+buys);
+		    }
+		    progresslog("pctb commodmap done.");
+		    int[] offerCount = getBuySellMaps(data,buys,sells,stallMap,commodMap);
+		    // if (dtxt!=null) dtxt.println(sells);
+		    // if (dtxt!=null) dtxt.println("\n\n\n"+buys);
 
-		if (uploadToPCTB) {
-			ByteArrayOutputStream outStream = new ByteArrayOutputStream();
-			pm.setProgress(60);
-			pm.setNote("Sending data");
-			if(pm.isCanceled()) {
-				return;
-			}
-			GZIPOutputStream out = new GZIPOutputStream(outStream);
-			//FileOutputStream out = new FileOutputStream(new File("output.text"));
-			DataOutputStream dos = new DataOutputStream(out);
-			dos.writeBytes("005\n");
-			dos.writeBytes(stallMap.size()+"\n");
-			dos.writeBytes(getAbbrevStallList(stallMap));
-			writeBuySellOffers(buys,sells,offerCount,out);
-			out.finish();
-			InputStream in = sendInitialData(new ByteArrayInputStream(outStream.toByteArray()));
-			pm.setProgress(80);
-			if(pm.isCanceled()) {
-				return;
-			}
-			pm.setNote("Waiting for PCTB...");
-			finishUpload(in);
+		    ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+		    pm.setProgress(60);
+		    progressNote(pm, "PCTB: Sending data");
+		    if(pm.isCanceled()) {
+			return;
+		    }
+		    GZIPOutputStream out = new GZIPOutputStream(outStream);
+		    DataOutputStream dos = new DataOutputStream(out);
+		    dos.writeBytes("005y\n");
+		    dos.writeBytes(stallMap.size()+"\n");
+		    dos.writeBytes(getAbbrevStallList(stallMap));
+		    writeBuySellOffers(buys,sells,offerCount,out);
+		    out.finish();
+		    progresslog("pctb send...");
+
+		    byte[] ba = outStream.toByteArray();
+		    if (dtxt!=null) {
+			FileOutputStream dbgdata = new FileOutputStream(new File("jarrg-debug-pctb-marketdata.gz"));
+			dbgdata.write(ba);
+			dbgdata.close();
+		    }
+
+		    InputStream in = sendInitialData(new ByteArrayInputStream(ba));
+		    progresslog("pctb sent.");
+		    if (in == null) return;
+		    pm.setProgress(80);
+		    if(pm.isCanceled()) {
+			return;
+		    }
+		    progressNote(pm, "PCTB: Waiting ...");
+		    progresslog("pctb finish...");
+		    donepctb = finishUpload(in);
+		    progresslog("pctb done.");
 		}
 		pm.setProgress(100);
+
+		if ((uploadToPCTB && !donepctb) ||
+		    (uploadToYarrg && !doneyarrg)) {
+		        resultSummary.setText("trouble");
+		} else if (donepctb || doneyarrg) {
+		        resultSummary.setText("Done " + islandName);
+		} else {
+		        resultSummary.setText("uploaded nowhere!");
+		}
+		progresslog("done.");
 	}
 	
 	/**
@@ -486,13 +640,13 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 		Accessible node1 = window;
 		Accessible node = descendNodes(node1,new int[] {0,1,0,0,0,0,1,0,0,1,0,0}); // commod market
 		// commod market: {0,1,0,0,0,0,1,0,0,1,0}  {0,1,0,0,0,0,1,0,1,0,0,1,0,0})
-		//System.out.println(node);
+		// if (dtxt!=null) dtxt.println(node);
 		if (!(node instanceof JTable)) {
 			node = descendNodes(node1,new int[] {0,1,0,0,0,0,1,0,1,0,0,1,0,0}); // commod market
 		}
 		if (!(node instanceof JTable)) return null;
 		AccessibleTable table = node.getAccessibleContext().getAccessibleTable();
-		//System.out.println(table);
+		// if (dtxt!=null) dtxt.println(table);
 		return table;
 	}
 	
@@ -509,7 +663,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	private Accessible descendNodes(Accessible parent, int[] path) {
 		for(int i=0;i<path.length;i++) {
 			if (null == (parent = descend(parent, path[i]))) return null;
-			// System.out.println(parent.getClass());
+			// if (dtxt!=null) dtxt.println(parent.getClass());
 		}
 		return parent;
 	}
@@ -566,13 +720,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	
 	/**
 	*	Gets the list of commodities and their associated commodity ids.
-	*	On the first run, the data is downloaded from the PCTB server. 
-	*	After the first run, the data is cached using <code>Preferences</code>.
-	*	<p>
-	*	Potential issues: When more commodities are added to the server, this
-	*	program will currently break unless the user deletes the preferences
-	*	file or we give them a new release with a slighly different storage
-	*	location for the data.
 	*
 	*	@return a map where the key is the commodity and the value is the commodity id.
 	*/
@@ -581,7 +728,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 			return commodMap;
 		}
 		HashMap<String,Integer> map = new HashMap<String,Integer>();
-		Preferences prefs = Preferences.userNodeForPackage(getClass());
 		String xml;
 		try {
 			URL host = new URL(PCTB_HOST_URL + "commodmap.php");
@@ -594,10 +740,9 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 			int first = sb.indexOf("<pre>") + 5;
 			int last = sb.indexOf("</body>");
 			xml = sb.substring(first,last);
-			//System.out.println(xml);
 			Reader reader = new CharArrayReader(xml.toCharArray());
 			Document d = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new InputSource(reader));
-			NodeList maps = d.getElementsByTagName("c");
+			NodeList maps = d.getElementsByTagName("CommodMap");
 			for(int i=0;i<maps.getLength();i++) {
 				NodeList content = maps.item(i).getChildNodes();
 				Integer num = Integer.parseInt(content.item(1).getTextContent());
@@ -664,9 +809,13 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 					buySellCount[1]++;
 				}
 			} catch(IllegalArgumentException e) {
-				// System.err.println("Error: Unsupported Commodity \"" + offer.get(0) + "\"");
+				if (dtxt!=null) dtxt.println("Error: Unsupported Commodity \"" + offer.get(0) + "\"");
 			}
 		}
+		if (buySellCount[0]==0 && buySellCount[1]==0) {
+		    error("No (valid) offers for PCTB?!");
+		    throw new IllegalArgumentException();
+		}
 		return buySellCount;
 	}
 	
@@ -781,6 +930,16 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 		writeOffers(out,sells);
 	}
 	
+	private String readstreamstring(InputStream in) throws IOException {
+		StringBuilder sb = new StringBuilder();
+		BufferedReader br = new BufferedReader(new InputStreamReader(in));
+		String str;
+		while((str = br.readLine()) != null) {
+			sb.append(str+"\n");
+		}
+		return sb.toString();
+	}
+
 	/**
 	*	Sends the data to the server via multipart-formdata POST,
 	*	with the gzipped data as a file upload.
@@ -790,7 +949,12 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	private InputStream sendInitialData(InputStream file) throws IOException {
 		ClientHttpRequest http = new ClientHttpRequest(PCTB_HOST_URL + "upload.php");
 		http.setParameter("marketdata","marketdata.gz",file,"application/gzip");
-		return http.post();
+		if (!http.post()) {
+			String err = readstreamstring(http.resultstream());
+			error("Error sending initial data:\n"+err);
+			return null;
+		}
+		return http.resultstream();
 	}
 	
 	/**
@@ -813,123 +977,74 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	*
 	*	@param in stream of data from the server to read
 	*/
-	private void finishUpload(InputStream in) throws IOException {
-		StringBuilder sb = new StringBuilder();
-		BufferedReader br = new BufferedReader(new InputStreamReader(in));
-		String str;
-		while((str = br.readLine()) != null) {
-			sb.append(str+"\n");
-		}
-		String html = sb.toString();
-		//System.out.println(html);
-		String topIsland = "0", ocean, islandNum, action, forceReload, filename;
+	private boolean finishUpload(InputStream in) throws IOException {
+		String html = readstreamstring(in);
+		debug_write_stringdata("pctb-initial.html", html);
 		Matcher m;
-		Pattern whoIsland = Pattern.compile("<option value=\"\\d+\">" + islandName + ", ([^<]+)</ocean>");
-		m = whoIsland.matcher(html);
-		if(m.find()) {
-			// the server agrees with us
-			ocean = islandNumbers.get(m.group(1));
-		} else {
-			// if the server doesn't agree with us:
-			Pattern island = Pattern.compile("<option value=\"(\\d+)\">([^,]+), ([^<]+)</ocean>");
-			m = island.matcher(html);
-			if(!m.find()) {
-				// server doesn't know what island. if we do, let's select it.
-				if(islandName != null && !islandName.equals("")) {
-					// find the island name in the list as many times as it occurs
-					// if more than once, present a dialog
-					// set the ocean, we have the islandname, topIsland = 0
-					Pattern myIsland = Pattern.compile("islands\\[(\\d+)\\]\\[\\d+\\]=new Option\\(\"" + islandName +
-						"\",\\d+");
-					Matcher m1 = myIsland.matcher(html);
-					ArrayList<Integer> myOceanNums = new ArrayList<Integer>();
-					while(m1.find()) {
-						myOceanNums.add(new Integer(m1.group(1)));
-					}
-					if(myOceanNums.size() > 0) {
-						if(myOceanNums.size() > 1) {
-							String[] myOceansList = new String[myOceanNums.size()];
-							int i = 0;
-							for(int myOcean : myOceanNums) {
-								Pattern oceanNumPat = Pattern.compile("<option value=\"\\" + myOcean + "\">([^<]+)</option>");
-								m1 = oceanNumPat.matcher(html);
-								if(m1.find()) {
-									myOceansList[i++] = m1.group(1);
-								}
-							}
-							Object option = JOptionPane.showInputDialog(null,"We found islands named \"" +
-								islandName +"\" on " + myOceansList.length + " oceans:","Choose Ocean",
-								JOptionPane.QUESTION_MESSAGE, null, myOceansList, null);
-							if(option == null) {
-								error("Unable to determine the current island!");
-								return;
-							}
-							ocean = islandNumbers.get(option).toString();
-						} else {
-							ocean = myOceanNums.get(0).toString();
-						}
-					} else {
-						error("Unknown island!");
-						return;
-					}
-				} else {
-					error("Unable to determine island name from the client!");
-					return;
-				}
-			} else {
-				topIsland = m.group(1);
-				islandName = m.group(2);
-				ocean = islandNumbers.get(m.group(3));
-			}
-		}
-		Pattern oceanIslandNum = Pattern.compile("islands\\[" + ocean + "\\]\\[\\d+\\]=new Option\\(\"" + islandName + "\",(\\d+)");
-		m = oceanIslandNum.matcher(html);
-		if(!m.find()) {
-			error("This does not seem to be a valid island! Unable to upload.");
-			return;
-		}
-		islandNum = m.group(1);
-		Pattern params = Pattern.compile("(?s)<input type=\"hidden\" name=\"action\" value=\"([^\"]+)\" />.+?<input type=\"hidden\" name=\"forcereload\" value=\"([^\"]+)\" />.+?<input type=\"hidden\" name=\"filename\" value=\"([^\"]+)\" />");
+
+		Pattern params = Pattern.compile("(?s)<input type=\"hidden\" name=\"action\" value=\"setisland\" />.+?<input type=\"hidden\" name=\"forcereload\" value=\"([^\"]+)\" />.+?<input type=\"hidden\" name=\"filename\" value=\"([^\"]+)\" />");
 		m = params.matcher(html);
 		if(!m.find()) {
-			error("The PCTB server returned unusual data. Maybe you're using an old version of the uploader?");
-			return;
+			error_html("The PCTB server returned unusual data. Maybe you're using an old version of the uploader?",
+				   html);
+			return false;
 		}
-		action = m.group(1);
-		forceReload = m.group(2);
-		filename = m.group(3);
-		URL get = new URL(PCTB_HOST_URL + "upload.php?topisland=" + topIsland + "&ocean=" + ocean + "&island="
-			+ islandNum + "&action=" + action + "&forcereload=" + forceReload + "&filename=" + filename);
-		// System.out.println(get);
-		BufferedReader br2 = new BufferedReader(new InputStreamReader(get.openStream()));
-		sb = new StringBuilder();
-		while((str = br2.readLine()) != null) {
-			sb.append(str+"\n");
+		String forceReload = m.group(1);
+		String filename = m.group(2);
+
+		Pattern oceanNumPat = Pattern.compile("<option value=\"(\\d+)\">"+oceanName+"</option>");
+		m = oceanNumPat.matcher(html);
+		if (!m.find()) {
+			error_html("Unable to find the ocean in the server's list of oceans!", html);
+			return false;
 		}
+		String oceanNum = m.group(1);
+
+		Pattern oceanIslandNum = Pattern.compile("islands\\[" + oceanNum + "\\]\\[\\d+\\]=new Option\\(\"" + islandName + "\",(\\d+)");
+		m = oceanIslandNum.matcher(html);
+		if(!m.find()) {
+			error_html("This does not seem to be a valid island! Unable to upload.", html);
+			return false;
+		}
+		String islandNum = m.group(1);
+
+		URL get = new URL(PCTB_HOST_URL + "upload.php?action=setisland&ocean=" + oceanNum + "&island="
+			+ islandNum + "&forcereload=" + forceReload + "&filename=" + filename);
+		String complete = readstreamstring(get.openStream());
+		debug_write_stringdata("pctb-final.html", complete);
 		Pattern done = Pattern.compile("Your data has been integrated into the database. Thank you!");
-		m = done.matcher(sb.toString());
+		m = done.matcher(complete);
 		if(m.find()) {
-			//System.out.println("FILE upload successful!!!");
+			return true;
 		} else {
-			error("Something was wrong with the final upload parameters!");
-			System.err.println(sb.toString());
-			System.err.println(html);
+			error_html("Something was wrong with the final upload parameters!", complete);
+			return false;
 		}
 	}
 
+    private InputStream post_for_yarrg(ClientHttpRequest http) throws IOException {
+	if (!http.post()) {
+	    String err = readstreamstring(http.resultstream());
+	    error("<html><h1>Error reported by YARRG server</h1>\n" + err);
+	    return null;
+	}
+	return http.resultstream();
+    }
+
     private String getYarrgTimestamp() throws IOException {
 	ClientHttpRequest http = new ClientHttpRequest (YARRG_URL);
 	http.setParameter("clientname", YARRG_CLIENTNAME);
 	http.setParameter("clientversion", YARRG_CLIENTVERSION);
 	http.setParameter("clientfixes", YARRG_CLIENTFIXES);
 	http.setParameter("requesttimestamp", "y");
-	InputStream in = http.post();
+	InputStream in = post_for_yarrg(http);
+	if (in == null) return null;
 	BufferedReader br = new BufferedReader(new InputStreamReader(in));
 	String tsresult = br.readLine();
 	return tsresult.substring(3, tsresult.length()-1);
     }
 
-    private void runYarrg(String timestamp, String ocean, String island, String yarrgdata) throws IOException {
+    private boolean runYarrg(String timestamp, String ocean, String island, String yarrgdata) throws IOException {
 	ByteArrayOutputStream bos = new ByteArrayOutputStream();
 	BufferedOutputStream bufos = new BufferedOutputStream(new GZIPOutputStream(bos));
 	bufos.write(yarrgdata.getBytes() );
@@ -944,11 +1059,94 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	http.setParameter("ocean", ocean);
 	http.setParameter("island", island);
 	http.setParameter("data", "deduped.tsv.gz", file, "application/octet-stream");
-	InputStream in = http.post();
-	BufferedReader br = new BufferedReader(new InputStreamReader(in));
-	String yarrgresult; 
-	while((yarrgresult = br.readLine()) != null) {
-	    System.out.println(yarrgresult);
+	InputStream in = post_for_yarrg(http);
+	if (in == null) return false;
+	String output = readstreamstring(in);
+	if (!output.startsWith("OK")) {
+	    error("<html><h1>Unexpected output from YARRG server</h1>\n" + output);
+	    return false;
+	}
+	debug_write_stringdata("yarrg-result.txt", output);
+	return true;
+    }
+
+    private int calculateArbitrageCommodity(ArrayList<SortedSet<int[]>> arb_bs) {
+	// if (dtxt!=null) dtxt.println("ARBITRAGE?");
+	int profit = 0;
+	SortedSet<int[]> buys = arb_bs.get(0);
+	SortedSet<int[]> sells = arb_bs.get(1);
+	while (true) {
+	    int[] buy, sell;
+	    try {
+		// NB "sell" means they sell, ie we buy
+		sell = sells.last();
+		buy = buys.first();
+	    } catch (NoSuchElementException e) {
+		break;
+	    }
+
+	    int unitprofit = buy[0] - sell[0];
+	    int count = buy[1] < sell[1] ? buy[1] : sell[1];
+	    // if (dtxt!=null) dtxt.println(" sell @"+sell[0]+" x"+sell[1]+" buy @"+buy[0]+" x"+buy[1]
+	    //		       +" => x"+count+" @"+unitprofit);
+
+	    if (unitprofit <= 0)
+		break;
+	    
+	    profit += count * unitprofit;
+	    buy[1] -= count;
+	    sell[1] -= count;
+	    if (buy[1]==0) buys.remove(buy);
+	    if (sell[1]==0) sells.remove(sell);
+	}
+	// if (dtxt!=null) dtxt.println(" PROFIT "+profit);
+	return profit;
+    }
+
+    private class arbitrageOfferComparator implements Comparator {
+	public int compare(Object o1, Object o2) {
+	    int p1 = ((int[])o1)[0];
+	    int p2 = ((int[])o2)[0];
+	    return p2 - p1;
+	}
+    }
+
+    private @SuppressWarnings("unchecked")
+    void calculateArbitrage(ArrayList<ArrayList<String>> data) {
+	int arbitrage = 0;
+	ArrayList<SortedSet<int[]>> arb_bs = null;
+	String lastcommod = null;
+	Comparator compar = new arbitrageOfferComparator();
+
+	for (ArrayList<String> row : data) {
+	    String thiscommod = row.get(0);
+	    // if (dtxt!=null) dtxt.println("ROW "+row.toString());
+	    if (lastcommod == null || !thiscommod.equals(lastcommod)) {
+		if (lastcommod != null)
+		    arbitrage += calculateArbitrageCommodity(arb_bs);
+		// if (dtxt!=null) dtxt.println("ROW rdy");
+		arb_bs = new ArrayList<SortedSet<int[]>>(2);
+		arb_bs.add(0, new TreeSet<int[]>(compar));
+		arb_bs.add(1, new TreeSet<int[]>(compar));
+		// if (dtxt!=null) dtxt.println("ROW init");
+		lastcommod = thiscommod;
+	    }
+	    for (int bs = 0; bs < 2; bs++) {
+		String pricestr = row.get(bs*2 + 2);
+		if (pricestr == null)
+		    continue;
+		int[] entry = new int[2];
+		// if (dtxt!=null) dtxt.println("ROW BS "+bs);
+		entry[0] = parseQty(pricestr);
+		entry[1] = parseQty(row.get(bs*2 + 3));
+		arb_bs.get(bs).add(entry);
+	    }
+	}
+	arbitrage += calculateArbitrageCommodity(arb_bs);
+	if (arbitrage != 0) {
+	    arbitrageResult.setText("<html><strong>arbitrage: "+arbitrage+" poe</strong>");
+	} else {
+	    arbitrageResult.setText("no arbitrage");
 	}
     }