X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=jarrg-ian.git;a=blobdiff_plain;f=src%2Fcom%2Ftedpearson%2Fypp%2Fmarket%2FMarketUploader.java;h=90a920a44442f321f0bcec4ec63a61e2d3eae2d7;hp=f105f1aaa0e21e7d990bf782c2d2cb2e19f3a114;hb=b1b517d0db9bcb15127dc1415238bfe1a9366b50;hpb=dc197e5a18540055d76a1e33c7c8ac7a2f462337 diff --git a/src/com/tedpearson/ypp/market/MarketUploader.java b/src/com/tedpearson/ypp/market/MarketUploader.java index f105f1a..90a920a 100644 --- a/src/com/tedpearson/ypp/market/MarketUploader.java +++ b/src/com/tedpearson/ypp/market/MarketUploader.java @@ -40,6 +40,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis private JButton findMarket = null; private JLabel resultSummary = null; private JLabel arbitrageResult = null; + private int unknownPCTBcommods = 0; private long startTime = 0; private final static String PCTB_LIVE_HOST_URL = "http://pctb.crabdance.com/"; @@ -97,6 +98,20 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis dtxt.println("progress "+(now - startTime)+"ms "+s); } + private void debug_write_stringdata(String what, String data) throws FileNotFoundException,IOException { + if (dtxt==null) return; + PrintStream strm = new PrintStream(new File("jarrg-debug-"+what)); + strm.print(data); + strm.close(); + } + + private void debug_write_bytes(String what, byte[] data) throws FileNotFoundException,IOException { + if (dtxt==null) return; + FileOutputStream strm = new FileOutputStream(new File("jarrg-debug-"+what)); + strm.write(data); + strm.close(); + } + private void progressNote(ProgressMonitor pm, String s) { String arb = null; if (arbitrageResult != null) @@ -235,7 +250,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis * Entry point. Read our preferences. */ public MarketUploader() { - // check if we've been turned off in the control panel Preferences prefs = Preferences.userNodeForPackage(getClass()); if (prefs.getBoolean("writeDebugFiles", false)) { @@ -291,6 +305,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis startTime = new Date().getTime(); resultSummary.setText(""); arbitrageResult.setText(""); + unknownPCTBcommods = 0; try { runUpload(); } catch(Exception e) { @@ -303,7 +318,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis sidePanel.removePropertyChangeListener(changeListener); } } - //findMarketTable(); findMarket.setEnabled(true); } }.start(); @@ -360,7 +374,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis } } } - // if we don't find the island name, hopefully the server will } /** @@ -383,8 +396,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis } private void error_html(String msg, String html) { - //System.err.println("===" + html + "==="); - Pattern body = Pattern.compile("(.*)", Pattern.DOTALL | Pattern.CASE_INSENSITIVE); Matcher m = body.matcher(html); if (m.find()) { @@ -409,19 +420,31 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis * * @exception Exception if an error we didn't expect occured */ + private class YarrgTimestampFetcher extends Thread { + public String ts = null; + public void run() { + try { + ts = getYarrgTimestamp(); + progresslog("(async) yarrg timestamp ready."); + } catch(Exception e) { + error("Error getting YARRG timestamp: "+e); + } + } + }; + private void runUpload() throws Exception { progresslog("starting"); - String yarrgts = ""; ProgressMonitor pm = new ProgressMonitor(frame,"Processing Market Data","Getting table data",0,100); pm.setMillisToDecideToPopup(0); pm.setMillisToPopup(0); boolean doneyarrg = false, donepctb = false; + YarrgTimestampFetcher yarrgts_thread = null; if (uploadToYarrg) { - progresslog("yarrg timestamp..."); - yarrgts = getYarrgTimestamp(); - progresslog("yarrg timestamp done."); + progresslog("(async) yarrg timestamp..."); + yarrgts_thread = new YarrgTimestampFetcher(); + yarrgts_thread.start(); } AccessibleTable accesstable = findMarketTable(); @@ -438,7 +461,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis return; } - progresslog("getisland..."); + progresslog("(async) getisland..."); getIsland(); progresslog("getocean..."); getOcean(); @@ -447,6 +470,15 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis if (latch != null) { latch.await(2, java.util.concurrent.TimeUnit.SECONDS); } + progresslog("(async) getisland done"); + + String yarrgts = null; + if (yarrgts_thread != null) { + progresslog("(async) yarrg timestamp join..."); + yarrgts_thread.join(); + progresslog("(async) yarrg timestamp joined."); + yarrgts = yarrgts_thread.ts; + } if (islandName == null) { error("Could not find island name in YPP user interface."); @@ -533,12 +565,13 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis } // get commod map + progresslog("pctb commodmap..."); HashMap commodMap = getCommodMap(); if(commodMap == null) { return; } + progresslog("pctb commodmap done."); int[] offerCount = getBuySellMaps(data,buys,sells,stallMap,commodMap); - //println(buys.toString()); // if (dtxt!=null) dtxt.println(sells); // if (dtxt!=null) dtxt.println("\n\n\n"+buys); @@ -549,7 +582,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis return; } GZIPOutputStream out = new GZIPOutputStream(outStream); - //FileOutputStream out = new FileOutputStream(new File("output.text")); DataOutputStream dos = new DataOutputStream(out); dos.writeBytes("005y\n"); dos.writeBytes(stallMap.size()+"\n"); @@ -559,11 +591,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis progresslog("pctb send..."); byte[] ba = outStream.toByteArray(); - if (dtxt!=null) { - FileOutputStream dbgdata = new FileOutputStream(new File("jarrg-debug-pctb-marketdata.gz")); - dbgdata.write(ba); - dbgdata.close(); - } + debug_write_bytes("pctb-marketdata.gz", ba); InputStream in = sendInitialData(new ByteArrayInputStream(ba)); progresslog("pctb sent."); @@ -582,6 +610,8 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis if ((uploadToPCTB && !donepctb) || (uploadToYarrg && !doneyarrg)) { resultSummary.setText("trouble"); + } else if (unknownPCTBcommods != 0) { + resultSummary.setText("PCTB lacks "+unknownPCTBcommods+" commod"); } else if (donepctb || doneyarrg) { resultSummary.setText("Done " + islandName); } else { @@ -695,13 +725,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis /** * Gets the list of commodities and their associated commodity ids. - * On the first run, the data is downloaded from the PCTB server. - * After the first run, the data is cached using Preferences. - *

- * Potential issues: When more commodities are added to the server, this - * program will currently break unless the user deletes the preferences - * file or we give them a new release with a slighly different storage - * location for the data. * * @return a map where the key is the commodity and the value is the commodity id. */ @@ -710,7 +733,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis return commodMap; } HashMap map = new HashMap(); - Preferences prefs = Preferences.userNodeForPackage(getClass()); String xml; try { URL host = new URL(PCTB_HOST_URL + "commodmap.php"); @@ -720,17 +742,10 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis while((str = br.readLine()) != null) { sb.append(str); } + if (dtxt != null) debug_write_stringdata("pctb-commodmap.xmlish", sb.toString()); int first = sb.indexOf("

") + 5;
 			int last = sb.indexOf("");
 			xml = sb.substring(first,last);
-			//if (dtxt!=null) {
-			//        dtxt.println(">>");
-			//	  dtxt.println(sb);;
-			//	  dtxt.println("||");
-			//	  dtxt.println(xml);
-			//	  dtxt.println("<<");
-			//}
-			// if (dtxt!=null) dtxt.println(xml);
 			Reader reader = new CharArrayReader(xml.toCharArray());
 			Document d = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new InputSource(reader));
 			NodeList maps = d.getElementsByTagName("CommodMap");
@@ -800,6 +815,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 					buySellCount[1]++;
 				}
 			} catch(IllegalArgumentException e) {
+			        unknownPCTBcommods++;
 				if (dtxt!=null) dtxt.println("Error: Unsupported Commodity \"" + offer.get(0) + "\"");
 			}
 		}
@@ -970,7 +986,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	*/
 	private boolean finishUpload(InputStream in) throws IOException {
 		String html = readstreamstring(in);
-		// if (dtxt!=null) dtxt.println(html);
+		debug_write_stringdata("pctb-initial.html", html);
 		Matcher m;
 
 		Pattern params = Pattern.compile("(?s).+?.+?");
@@ -1002,10 +1018,10 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 		URL get = new URL(PCTB_HOST_URL + "upload.php?action=setisland&ocean=" + oceanNum + "&island="
 			+ islandNum + "&forcereload=" + forceReload + "&filename=" + filename);
 		String complete = readstreamstring(get.openStream());
+		debug_write_stringdata("pctb-final.html", complete);
 		Pattern done = Pattern.compile("Your data has been integrated into the database. Thank you!");
 		m = done.matcher(complete);
 		if(m.find()) {
-			if (dtxt!=null) dtxt.println("FILE upload successful!!!");
 			return true;
 		} else {
 			error_html("Something was wrong with the final upload parameters!", complete);
@@ -1040,7 +1056,9 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	BufferedOutputStream bufos = new BufferedOutputStream(new GZIPOutputStream(bos));
 	bufos.write(yarrgdata.getBytes() );
 	bufos.close();
-	ByteArrayInputStream file = new ByteArrayInputStream(bos.toByteArray());
+	byte[] compressed = bos.toByteArray();
+	debug_write_bytes("yarrg-deduped.tsv.gz", compressed);
+	ByteArrayInputStream file = new ByteArrayInputStream(compressed);
 
 	ClientHttpRequest http = new ClientHttpRequest (YARRG_URL);
 	http.setParameter("clientname", YARRG_CLIENTNAME);
@@ -1052,11 +1070,12 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	http.setParameter("data", "deduped.tsv.gz", file, "application/octet-stream");
 	InputStream in = post_for_yarrg(http);
 	if (in == null) return false;
-	BufferedReader br = new BufferedReader(new InputStreamReader(in));
-	String yarrgresult; 
-	while((yarrgresult = br.readLine()) != null) {
-	    if (dtxt!=null) dtxt.println(yarrgresult);
+	String output = readstreamstring(in);
+	if (!output.startsWith("OK")) {
+	    error("
Unexpected output from YARRG server
\n" + output);
+	    return false;
 	}
+	debug_write_stringdata("yarrg-result.txt", output);
 	return true;
     }