X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=jarrg-ian.git;a=blobdiff_plain;f=src%2Fcom%2Ftedpearson%2Fypp%2Fmarket%2FMarketUploader.java;h=90a920a44442f321f0bcec4ec63a61e2d3eae2d7;hp=2466b44a8b3fc35fa5c78de039fe022c9c47d276;hb=b1b517d0db9bcb15127dc1415238bfe1a9366b50;hpb=e057e83033ea929281de2171df40657650959ef6

diff --git a/src/com/tedpearson/ypp/market/MarketUploader.java b/src/com/tedpearson/ypp/market/MarketUploader.java
index 2466b44..90a920a 100644
--- a/src/com/tedpearson/ypp/market/MarketUploader.java
+++ b/src/com/tedpearson/ypp/market/MarketUploader.java
@@ -40,6 +40,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	private JButton findMarket = null;
 	private JLabel resultSummary = null;
 	private JLabel arbitrageResult = null;
+        private int unknownPCTBcommods = 0;
         private long startTime = 0;
 
 	private final static String PCTB_LIVE_HOST_URL = "http://pctb.crabdance.com/";
@@ -65,6 +66,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 
 	private AccessibleContext sidePanel;
 	private HashMap<String,Integer> commodMap;
+        public PrintStream dtxt = null;
 
 	private PropertyChangeListener changeListener = new PropertyChangeListener() {
 		public void propertyChange(PropertyChangeEvent e) {
@@ -75,7 +77,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 				int index = text.indexOf(":");
 				String name = text.substring(0,index);
 				islandName = name;
-				//System.out.println(islandName);
+				// if (dtxt!=null) dtxt.println(islandName);
 				sidePanel.removePropertyChangeListener(this);
 				latch.countDown();
 			}
@@ -91,8 +93,23 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	}
 
         private void progresslog(String s) {
+	    if (dtxt == null) return;
 	    long now = new Date().getTime();
-	    System.out.println("progress "+(now - startTime)+"ms "+s);
+	    dtxt.println("progress "+(now - startTime)+"ms "+s);
+	}
+
+        private void debug_write_stringdata(String what, String data) throws FileNotFoundException,IOException {
+	    if (dtxt==null) return;
+	    PrintStream strm = new PrintStream(new File("jarrg-debug-"+what));
+	    strm.print(data);
+	    strm.close();
+	}
+
+        private void debug_write_bytes(String what, byte[] data) throws FileNotFoundException,IOException {
+	    if (dtxt==null) return;
+	    FileOutputStream strm = new FileOutputStream(new File("jarrg-debug-"+what));
+	    strm.write(data);
+	    strm.close();
 	}
 
         private void progressNote(ProgressMonitor pm, String s) {
@@ -233,9 +250,16 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	*	Entry point.  Read our preferences.
 	*/
 	public MarketUploader() {
-		// check if we've been turned off in the control panel
 		Preferences prefs = Preferences.userNodeForPackage(getClass());
 
+		if (prefs.getBoolean("writeDebugFiles", false)) {
+		    try {
+			dtxt = new PrintStream(new File("jarrg-debug-log.txt"));
+		    } catch (java.io.FileNotFoundException e) {
+			System.err.println("JARRG: Error opening debug log: "+e);
+		    }
+		}
+
 		if (prefs.getBoolean("useLiveServers", false)) {
 			YARRG_URL = YARRG_LIVE_URL;
 			PCTB_HOST_URL = PCTB_LIVE_HOST_URL;
@@ -281,6 +305,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 					        startTime = new Date().getTime();
 						resultSummary.setText("");
 						arbitrageResult.setText("");
+						unknownPCTBcommods = 0;
 						try {
 							runUpload();
 						} catch(Exception e) {
@@ -293,7 +318,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 								sidePanel.removePropertyChangeListener(changeListener);
 							}
 						}
-						//findMarketTable();
 						findMarket.setEnabled(true);
 					}
 				}.start();
@@ -350,7 +374,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 				}
 			}
 		}
-		// if we don't find the island name, hopefully the server will
 	}
 
 	/**
@@ -373,8 +396,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	}
 	
 	private void error_html(String msg, String html) {
-		//System.err.println("===" + html + "===");
-
 		Pattern body = Pattern.compile("<body>(.*)</body>", Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
 		Matcher m = body.matcher(html);
 		if (m.find()) {
@@ -386,7 +407,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 			html = m.replaceAll(" ");
 		}
 		String whole_msg = "<html><h1>Error</h1>"+msg+"<h1>PCTB Server said:</h1><blockquote>"+html+"</blockquote>";
-		System.err.println("###" + whole_msg + "###");
+		if (dtxt!=null) dtxt.println("###" + whole_msg + "###");
 
 		JOptionPane.showMessageDialog(frame,whole_msg,"Error",JOptionPane.ERROR_MESSAGE);
 	}
@@ -399,19 +420,31 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	*
 	*	@exception Exception if an error we didn't expect occured
 	*/
+        private class YarrgTimestampFetcher extends Thread {
+	    public String ts = null;
+	    public void run() {
+		try {
+		    ts = getYarrgTimestamp();
+		    progresslog("(async) yarrg timestamp ready.");
+		} catch(Exception e) {
+		    error("Error getting YARRG timestamp: "+e);
+		}
+	    }
+	};
+
 	private void runUpload() throws Exception {
 		progresslog("starting");
 
-		String yarrgts = "";
 		ProgressMonitor pm = new ProgressMonitor(frame,"Processing Market Data","Getting table data",0,100);
 		pm.setMillisToDecideToPopup(0);
 		pm.setMillisToPopup(0);
 		boolean doneyarrg = false, donepctb = false;
+		YarrgTimestampFetcher yarrgts_thread = null;
 
 		if (uploadToYarrg) {
-		        progresslog("yarrg timestamp...");
-			yarrgts = getYarrgTimestamp();
-		        progresslog("yarrg timestamp done.");
+		        progresslog("(async) yarrg timestamp...");
+			yarrgts_thread = new YarrgTimestampFetcher();
+			yarrgts_thread.start();
 		}
 
 		AccessibleTable accesstable = findMarketTable();
@@ -428,7 +461,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 			return;
 		}
 
-		progresslog("getisland...");
+		progresslog("(async) getisland...");
 	        getIsland();
 		progresslog("getocean...");
 		getOcean();
@@ -437,6 +470,15 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 		if (latch != null) {
 		    latch.await(2, java.util.concurrent.TimeUnit.SECONDS);
 		}
+		progresslog("(async) getisland done");
+
+		String yarrgts = null;
+		if (yarrgts_thread != null) {
+		    progresslog("(async) yarrg timestamp join...");
+		    yarrgts_thread.join();
+		    progresslog("(async) yarrg timestamp joined.");
+		    yarrgts = yarrgts_thread.ts;
+		}
 
 		if (islandName == null) {
 		    error("Could not find island name in YPP user interface.");
@@ -523,14 +565,15 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 		    }
 		    // get commod map
 		
+		    progresslog("pctb commodmap...");
 		    HashMap<String,Integer> commodMap = getCommodMap();
 		    if(commodMap == null) {
 			return;
 		    }
+		    progresslog("pctb commodmap done.");
 		    int[] offerCount = getBuySellMaps(data,buys,sells,stallMap,commodMap);
-		    //println(buys.toString());
-		    //System.out.println(sells);
-		    //System.out.println("\n\n\n"+buys);
+		    // if (dtxt!=null) dtxt.println(sells);
+		    // if (dtxt!=null) dtxt.println("\n\n\n"+buys);
 
 		    ByteArrayOutputStream outStream = new ByteArrayOutputStream();
 		    pm.setProgress(60);
@@ -539,7 +582,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 			return;
 		    }
 		    GZIPOutputStream out = new GZIPOutputStream(outStream);
-		    //FileOutputStream out = new FileOutputStream(new File("output.text"));
 		    DataOutputStream dos = new DataOutputStream(out);
 		    dos.writeBytes("005y\n");
 		    dos.writeBytes(stallMap.size()+"\n");
@@ -547,7 +589,11 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 		    writeBuySellOffers(buys,sells,offerCount,out);
 		    out.finish();
 		    progresslog("pctb send...");
-		    InputStream in = sendInitialData(new ByteArrayInputStream(outStream.toByteArray()));
+
+		    byte[] ba = outStream.toByteArray();
+		    debug_write_bytes("pctb-marketdata.gz", ba);
+
+		    InputStream in = sendInitialData(new ByteArrayInputStream(ba));
 		    progresslog("pctb sent.");
 		    if (in == null) return;
 		    pm.setProgress(80);
@@ -564,6 +610,8 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 		if ((uploadToPCTB && !donepctb) ||
 		    (uploadToYarrg && !doneyarrg)) {
 		        resultSummary.setText("trouble");
+		} else if (unknownPCTBcommods != 0) {
+		        resultSummary.setText("PCTB lacks "+unknownPCTBcommods+" commod");
 		} else if (donepctb || doneyarrg) {
 		        resultSummary.setText("Done " + islandName);
 		} else {
@@ -597,13 +645,13 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 		Accessible node1 = window;
 		Accessible node = descendNodes(node1,new int[] {0,1,0,0,0,0,1,0,0,1,0,0}); // commod market
 		// commod market: {0,1,0,0,0,0,1,0,0,1,0}  {0,1,0,0,0,0,1,0,1,0,0,1,0,0})
-		//System.out.println(node);
+		// if (dtxt!=null) dtxt.println(node);
 		if (!(node instanceof JTable)) {
 			node = descendNodes(node1,new int[] {0,1,0,0,0,0,1,0,1,0,0,1,0,0}); // commod market
 		}
 		if (!(node instanceof JTable)) return null;
 		AccessibleTable table = node.getAccessibleContext().getAccessibleTable();
-		//System.out.println(table);
+		// if (dtxt!=null) dtxt.println(table);
 		return table;
 	}
 	
@@ -620,7 +668,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	private Accessible descendNodes(Accessible parent, int[] path) {
 		for(int i=0;i<path.length;i++) {
 			if (null == (parent = descend(parent, path[i]))) return null;
-			// System.out.println(parent.getClass());
+			// if (dtxt!=null) dtxt.println(parent.getClass());
 		}
 		return parent;
 	}
@@ -677,13 +725,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	
 	/**
 	*	Gets the list of commodities and their associated commodity ids.
-	*	On the first run, the data is downloaded from the PCTB server. 
-	*	After the first run, the data is cached using <code>Preferences</code>.
-	*	<p>
-	*	Potential issues: When more commodities are added to the server, this
-	*	program will currently break unless the user deletes the preferences
-	*	file or we give them a new release with a slighly different storage
-	*	location for the data.
 	*
 	*	@return a map where the key is the commodity and the value is the commodity id.
 	*/
@@ -692,7 +733,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 			return commodMap;
 		}
 		HashMap<String,Integer> map = new HashMap<String,Integer>();
-		Preferences prefs = Preferences.userNodeForPackage(getClass());
 		String xml;
 		try {
 			URL host = new URL(PCTB_HOST_URL + "commodmap.php");
@@ -702,10 +742,10 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 			while((str = br.readLine()) != null) {
 				sb.append(str);
 			}
+			if (dtxt != null) debug_write_stringdata("pctb-commodmap.xmlish", sb.toString());
 			int first = sb.indexOf("<pre>") + 5;
 			int last = sb.indexOf("</body>");
 			xml = sb.substring(first,last);
-			//System.out.println(xml);
 			Reader reader = new CharArrayReader(xml.toCharArray());
 			Document d = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new InputSource(reader));
 			NodeList maps = d.getElementsByTagName("CommodMap");
@@ -775,7 +815,8 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 					buySellCount[1]++;
 				}
 			} catch(IllegalArgumentException e) {
-				System.err.println("Error: Unsupported Commodity \"" + offer.get(0) + "\"");
+			        unknownPCTBcommods++;
+				if (dtxt!=null) dtxt.println("Error: Unsupported Commodity \"" + offer.get(0) + "\"");
 			}
 		}
 		if (buySellCount[0]==0 && buySellCount[1]==0) {
@@ -945,7 +986,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	*/
 	private boolean finishUpload(InputStream in) throws IOException {
 		String html = readstreamstring(in);
-		//System.out.println(html);
+		debug_write_stringdata("pctb-initial.html", html);
 		Matcher m;
 
 		Pattern params = Pattern.compile("(?s)<input type=\"hidden\" name=\"action\" value=\"setisland\" />.+?<input type=\"hidden\" name=\"forcereload\" value=\"([^\"]+)\" />.+?<input type=\"hidden\" name=\"filename\" value=\"([^\"]+)\" />");
@@ -977,10 +1018,10 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 		URL get = new URL(PCTB_HOST_URL + "upload.php?action=setisland&ocean=" + oceanNum + "&island="
 			+ islandNum + "&forcereload=" + forceReload + "&filename=" + filename);
 		String complete = readstreamstring(get.openStream());
+		debug_write_stringdata("pctb-final.html", complete);
 		Pattern done = Pattern.compile("Your data has been integrated into the database. Thank you!");
 		m = done.matcher(complete);
 		if(m.find()) {
-			System.out.println("FILE upload successful!!!");
 			return true;
 		} else {
 			error_html("Something was wrong with the final upload parameters!", complete);
@@ -1015,7 +1056,9 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	BufferedOutputStream bufos = new BufferedOutputStream(new GZIPOutputStream(bos));
 	bufos.write(yarrgdata.getBytes() );
 	bufos.close();
-	ByteArrayInputStream file = new ByteArrayInputStream(bos.toByteArray());
+	byte[] compressed = bos.toByteArray();
+	debug_write_bytes("yarrg-deduped.tsv.gz", compressed);
+	ByteArrayInputStream file = new ByteArrayInputStream(compressed);
 
 	ClientHttpRequest http = new ClientHttpRequest (YARRG_URL);
 	http.setParameter("clientname", YARRG_CLIENTNAME);
@@ -1027,16 +1070,17 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	http.setParameter("data", "deduped.tsv.gz", file, "application/octet-stream");
 	InputStream in = post_for_yarrg(http);
 	if (in == null) return false;
-	BufferedReader br = new BufferedReader(new InputStreamReader(in));
-	String yarrgresult; 
-	while((yarrgresult = br.readLine()) != null) {
-	    System.out.println(yarrgresult);
+	String output = readstreamstring(in);
+	if (!output.startsWith("OK")) {
+	    error("<html><h1>Unexpected output from YARRG server</h1>\n" + output);
+	    return false;
 	}
+	debug_write_stringdata("yarrg-result.txt", output);
 	return true;
     }
 
     private int calculateArbitrageCommodity(ArrayList<SortedSet<int[]>> arb_bs) {
-	//System.out.println("ARBITRAGE?");
+	// if (dtxt!=null) dtxt.println("ARBITRAGE?");
 	int profit = 0;
 	SortedSet<int[]> buys = arb_bs.get(0);
 	SortedSet<int[]> sells = arb_bs.get(1);
@@ -1052,7 +1096,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 
 	    int unitprofit = buy[0] - sell[0];
 	    int count = buy[1] < sell[1] ? buy[1] : sell[1];
-	    //System.out.println(" sell @"+sell[0]+" x"+sell[1]+" buy @"+buy[0]+" x"+buy[1]
+	    // if (dtxt!=null) dtxt.println(" sell @"+sell[0]+" x"+sell[1]+" buy @"+buy[0]+" x"+buy[1]
 	    //		       +" => x"+count+" @"+unitprofit);
 
 	    if (unitprofit <= 0)
@@ -1064,7 +1108,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	    if (buy[1]==0) buys.remove(buy);
 	    if (sell[1]==0) sells.remove(sell);
 	}
-	//System.out.println(" PROFIT "+profit);
+	// if (dtxt!=null) dtxt.println(" PROFIT "+profit);
 	return profit;
     }
 
@@ -1085,15 +1129,15 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 
 	for (ArrayList<String> row : data) {
 	    String thiscommod = row.get(0);
-	    //System.out.println("ROW "+row.toString());
+	    // if (dtxt!=null) dtxt.println("ROW "+row.toString());
 	    if (lastcommod == null || !thiscommod.equals(lastcommod)) {
 		if (lastcommod != null)
 		    arbitrage += calculateArbitrageCommodity(arb_bs);
-		//System.out.println("ROW rdy");
+		// if (dtxt!=null) dtxt.println("ROW rdy");
 		arb_bs = new ArrayList<SortedSet<int[]>>(2);
 		arb_bs.add(0, new TreeSet<int[]>(compar));
 		arb_bs.add(1, new TreeSet<int[]>(compar));
-		//System.out.println("ROW init");
+		// if (dtxt!=null) dtxt.println("ROW init");
 		lastcommod = thiscommod;
 	    }
 	    for (int bs = 0; bs < 2; bs++) {
@@ -1101,7 +1145,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 		if (pricestr == null)
 		    continue;
 		int[] entry = new int[2];
-		//System.out.println("ROW BS "+bs);
+		// if (dtxt!=null) dtxt.println("ROW BS "+bs);
 		entry[0] = parseQty(pricestr);
 		entry[1] = parseQty(row.get(bs*2 + 3));
 		arb_bs.get(bs).add(entry);