X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?a=blobdiff_plain;ds=sidebyside;f=src%2Fcom%2Ftedpearson%2Fypp%2Fmarket%2FMarketUploader.java;h=90a920a44442f321f0bcec4ec63a61e2d3eae2d7;hb=b1b517d0db9bcb15127dc1415238bfe1a9366b50;hp=7632c01f1221bd8ac3b27fd0c1242bcaeca3fbe6;hpb=0b0a3e3376419ae1321e349764092c0210cda5eb;p=jarrg-ian.git diff --git a/src/com/tedpearson/ypp/market/MarketUploader.java b/src/com/tedpearson/ypp/market/MarketUploader.java index 7632c01..90a920a 100644 --- a/src/com/tedpearson/ypp/market/MarketUploader.java +++ b/src/com/tedpearson/ypp/market/MarketUploader.java @@ -39,6 +39,9 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis private Window window = null; private JButton findMarket = null; private JLabel resultSummary = null; + private JLabel arbitrageResult = null; + private int unknownPCTBcommods = 0; + private long startTime = 0; private final static String PCTB_LIVE_HOST_URL = "http://pctb.crabdance.com/"; private final static String PCTB_TEST_HOST_URL = "http://pctb.ilk.org/"; @@ -55,6 +58,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis private boolean uploadToYarrg; private boolean uploadToPCTB; + private boolean showArbitrage; private String islandName = null; private String oceanName = null; @@ -62,6 +66,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis private AccessibleContext sidePanel; private HashMap commodMap; + public PrintStream dtxt = null; private PropertyChangeListener changeListener = new PropertyChangeListener() { public void propertyChange(PropertyChangeEvent e) { @@ -72,12 +77,49 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis int index = text.indexOf(":"); String name = text.substring(0,index); islandName = name; - //System.out.println(islandName); + // if (dtxt!=null) dtxt.println(islandName); sidePanel.removePropertyChangeListener(this); latch.countDown(); } } }; + + private int parseQty(String str) { + if (str.equals(">1000")) { + return 1001; + } else { + return Integer.parseInt(str); + } + } + + private void progresslog(String s) { + if (dtxt == null) return; + long now = new Date().getTime(); + dtxt.println("progress "+(now - startTime)+"ms "+s); + } + + private void debug_write_stringdata(String what, String data) throws FileNotFoundException,IOException { + if (dtxt==null) return; + PrintStream strm = new PrintStream(new File("jarrg-debug-"+what)); + strm.print(data); + strm.close(); + } + + private void debug_write_bytes(String what, byte[] data) throws FileNotFoundException,IOException { + if (dtxt==null) return; + FileOutputStream strm = new FileOutputStream(new File("jarrg-debug-"+what)); + strm.write(data); + strm.close(); + } + + private void progressNote(ProgressMonitor pm, String s) { + String arb = null; + if (arbitrageResult != null) + arb = arbitrageResult.getText(); + if (arb != null && arb.length() != 0) + s = "" + arb + "
" + s; + pm.setNote(s); + } /** * An abstract market offer, entailing a commodity being bought or sold by @@ -108,11 +150,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis commodity = commodId.intValue(); price = Integer.parseInt(record.get(priceIndex)); String qty = record.get(priceIndex+1); - if(qty.equals(">1000")) { - quantity = 1001; - } else { - quantity = Integer.parseInt(record.get(priceIndex+1)); - } + quantity = parseQty(qty); shoppe = stallMap.get(record.get(1)).intValue(); } @@ -212,9 +250,16 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis * Entry point. Read our preferences. */ public MarketUploader() { - // check if we've been turned off in the control panel Preferences prefs = Preferences.userNodeForPackage(getClass()); + if (prefs.getBoolean("writeDebugFiles", false)) { + try { + dtxt = new PrintStream(new File("jarrg-debug-log.txt")); + } catch (java.io.FileNotFoundException e) { + System.err.println("JARRG: Error opening debug log: "+e); + } + } + if (prefs.getBoolean("useLiveServers", false)) { YARRG_URL = YARRG_LIVE_URL; PCTB_HOST_URL = PCTB_LIVE_HOST_URL; @@ -225,6 +270,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis uploadToYarrg=prefs.getBoolean("uploadToYarrg", true); uploadToPCTB=prefs.getBoolean("uploadToPCTB", true); + showArbitrage=prefs.getBoolean("showArbitrage", true); EventQueueMonitor.addTopLevelWindowListener(this); if (EventQueueMonitor.isGUIInitialized()) { @@ -244,9 +290,10 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis if (window.getAccessibleContext().getAccessibleName().equals("Puzzle Pirates")) frame.setVisible(true); return; } - frame = new JFrame("MarketUploader"); + frame = new JFrame("Jarrg Uploader"); frame.setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE); - frame.getContentPane().setLayout(new GridLayout(2,1)); + GridLayout layout = new GridLayout(2,1); + frame.getContentPane().setLayout(layout); //frame.setPreferredSize(new Dimension(200, 60)); findMarket = new JButton("Upload Market Data"); @@ -255,9 +302,12 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis findMarket.setEnabled(false); new Thread() { public void run() { + startTime = new Date().getTime(); resultSummary.setText(""); + arbitrageResult.setText(""); + unknownPCTBcommods = 0; try { - runPCTB(); + runUpload(); } catch(Exception e) { error(e.toString()); e.printStackTrace(); @@ -268,7 +318,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis sidePanel.removePropertyChangeListener(changeListener); } } - //findMarketTable(); findMarket.setEnabled(true); } }.start(); @@ -279,6 +328,13 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis resultSummary = new JLabel("ready"); frame.add(resultSummary); + arbitrageResult = new JLabel(""); + + if (showArbitrage) { + layout.setRows(layout.getRows() + 1); + frame.add(arbitrageResult); + } + frame.pack(); } @@ -318,7 +374,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis } } } - // if we don't find the island name, hopefully the server will } /** @@ -341,8 +396,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis } private void error_html(String msg, String html) { - //System.err.println("===" + html + "==="); - Pattern body = Pattern.compile("(.*)", Pattern.DOTALL | Pattern.CASE_INSENSITIVE); Matcher m = body.matcher(html); if (m.find()) { @@ -354,7 +407,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis html = m.replaceAll(" "); } String whole_msg = "

Error

"+msg+"

PCTB Server said:

"+html+"

"; - System.err.println("###" + whole_msg + "###"); + if (dtxt!=null) dtxt.println("###" + whole_msg + "###"); JOptionPane.showMessageDialog(frame,whole_msg,"Error",JOptionPane.ERROR_MESSAGE); } @@ -367,23 +420,39 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis * * @exception Exception if an error we didn't expect occured */ - private void runPCTB() throws Exception { - String yarrgts = ""; + private class YarrgTimestampFetcher extends Thread { + public String ts = null; + public void run() { + try { + ts = getYarrgTimestamp(); + progresslog("(async) yarrg timestamp ready."); + } catch(Exception e) { + error("Error getting YARRG timestamp: "+e); + } + } + }; + + private void runUpload() throws Exception { + progresslog("starting"); + ProgressMonitor pm = new ProgressMonitor(frame,"Processing Market Data","Getting table data",0,100); pm.setMillisToDecideToPopup(0); pm.setMillisToPopup(0); boolean doneyarrg = false, donepctb = false; + YarrgTimestampFetcher yarrgts_thread = null; if (uploadToYarrg) { - yarrgts = getYarrgTimestamp(); + progresslog("(async) yarrg timestamp..."); + yarrgts_thread = new YarrgTimestampFetcher(); + yarrgts_thread.start(); } - AccessibleTable t = findMarketTable(); - if(t == null) { + AccessibleTable accesstable = findMarketTable(); + if(accesstable == null) { error("Market table not found! Please open the Buy/Sell Commodities interface."); return; } - if(t.getAccessibleRowCount() == 0) { + if(accesstable.getAccessibleRowCount() == 0) { error("No data found, please wait for the table to have data first!"); return; } @@ -392,17 +461,67 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis return; } + progresslog("(async) getisland..."); getIsland(); + progresslog("getocean..."); getOcean(); + progresslog("getocean done"); if (latch != null) { latch.await(2, java.util.concurrent.TimeUnit.SECONDS); } + progresslog("(async) getisland done"); + + String yarrgts = null; + if (yarrgts_thread != null) { + progresslog("(async) yarrg timestamp join..."); + yarrgts_thread.join(); + progresslog("(async) yarrg timestamp joined."); + yarrgts = yarrgts_thread.ts; + } + + if (islandName == null) { + error("Could not find island name in YPP user interface."); + return; + } - ArrayList> data = getData(t); + progresslog("table check..."); + + String headings_expected[] = new String[] + { "Commodity", "Trading outlet", "Buy price", "Will buy", "Sell price", "Will sell" }; + ArrayList> headers = getData(accesstable.getAccessibleColumnHeader()); + if (headers.size() != 1) { + error("Table headings not one row! " + headers.toString()); + return; + } + if (headers.get(0).size() != 6) { + error("Table headings not six columns! " + headers.toString()); + return; + } + for (int col=0; col> data = getData(accesstable); + + if (showArbitrage) { + progresslog("arbitrage..."); + calculateArbitrage(data); + progresslog("arbitrage done."); + } if (uploadToYarrg && yarrgts != null) { - pm.setNote("Yarrg: Preparing data"); + progresslog("yarrg prepare..."); + progressNote(pm, "Yarrg: Preparing data"); pm.setProgress(10); StringBuilder yarrgsb = new StringBuilder(); @@ -422,17 +541,16 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis yarrgdata = yarrgsb.toString(); - pm.setNote("Yarrg: Uploading"); + progressNote(pm, "Yarrg: Uploading"); + progresslog("yarrg upload..."); - if (islandName != null) { - doneyarrg = runYarrg(yarrgts, oceanName, islandName, yarrgdata); - } else { - System.out.println("Couldn't upload to Yarrg - no island name found"); - } + doneyarrg = runYarrg(yarrgts, oceanName, islandName, yarrgdata); + progresslog("yarrg done."); } if (uploadToPCTB) { - pm.setNote("PCTB: Getting stall names"); + progresslog("pctb prepare..."); + progressNote(pm, "PCTB: Getting stall names"); pm.setProgress(20); if(pm.isCanceled()) { return; @@ -441,54 +559,65 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis TreeSet sells = new TreeSet(); LinkedHashMap stallMap = getStallMap(data); pm.setProgress(40); - pm.setNote("PCTB: Sorting offers"); + progressNote(pm, "PCTB: Sorting offers"); if(pm.isCanceled()) { return; } // get commod map + progresslog("pctb commodmap..."); HashMap commodMap = getCommodMap(); if(commodMap == null) { return; } + progresslog("pctb commodmap done."); int[] offerCount = getBuySellMaps(data,buys,sells,stallMap,commodMap); - //println(buys.toString()); - //System.out.println(sells); - //System.out.println("\n\n\n"+buys); + // if (dtxt!=null) dtxt.println(sells); + // if (dtxt!=null) dtxt.println("\n\n\n"+buys); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); pm.setProgress(60); - pm.setNote("PCTB: Sending data"); + progressNote(pm, "PCTB: Sending data"); if(pm.isCanceled()) { return; } GZIPOutputStream out = new GZIPOutputStream(outStream); - //FileOutputStream out = new FileOutputStream(new File("output.text")); DataOutputStream dos = new DataOutputStream(out); dos.writeBytes("005y\n"); dos.writeBytes(stallMap.size()+"\n"); dos.writeBytes(getAbbrevStallList(stallMap)); writeBuySellOffers(buys,sells,offerCount,out); out.finish(); - InputStream in = sendInitialData(new ByteArrayInputStream(outStream.toByteArray())); + progresslog("pctb send..."); + + byte[] ba = outStream.toByteArray(); + debug_write_bytes("pctb-marketdata.gz", ba); + + InputStream in = sendInitialData(new ByteArrayInputStream(ba)); + progresslog("pctb sent."); if (in == null) return; pm.setProgress(80); if(pm.isCanceled()) { return; } - pm.setNote("PCTB: Waiting ..."); + progressNote(pm, "PCTB: Waiting ..."); + progresslog("pctb finish..."); donepctb = finishUpload(in); + progresslog("pctb done."); } pm.setProgress(100); if ((uploadToPCTB && !donepctb) || (uploadToYarrg && !doneyarrg)) { resultSummary.setText("trouble"); + } else if (unknownPCTBcommods != 0) { + resultSummary.setText("PCTB lacks "+unknownPCTBcommods+" commod"); } else if (donepctb || doneyarrg) { resultSummary.setText("Done " + islandName); } else { resultSummary.setText("uploaded nowhere!"); } + progresslog("done."); } /** @@ -516,13 +645,13 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis Accessible node1 = window; Accessible node = descendNodes(node1,new int[] {0,1,0,0,0,0,1,0,0,1,0,0}); // commod market // commod market: {0,1,0,0,0,0,1,0,0,1,0} {0,1,0,0,0,0,1,0,1,0,0,1,0,0}) - //System.out.println(node); + // if (dtxt!=null) dtxt.println(node); if (!(node instanceof JTable)) { node = descendNodes(node1,new int[] {0,1,0,0,0,0,1,0,1,0,0,1,0,0}); // commod market } if (!(node instanceof JTable)) return null; AccessibleTable table = node.getAccessibleContext().getAccessibleTable(); - //System.out.println(table); + // if (dtxt!=null) dtxt.println(table); return table; } @@ -539,7 +668,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis private Accessible descendNodes(Accessible parent, int[] path) { for(int i=0;iPreferences. - *

- * Potential issues: When more commodities are added to the server, this - * program will currently break unless the user deletes the preferences - * file or we give them a new release with a slighly different storage - * location for the data. * * @return a map where the key is the commodity and the value is the commodity id. */ @@ -611,7 +733,6 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis return commodMap; } HashMap map = new HashMap(); - Preferences prefs = Preferences.userNodeForPackage(getClass()); String xml; try { URL host = new URL(PCTB_HOST_URL + "commodmap.php"); @@ -621,10 +742,10 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis while((str = br.readLine()) != null) { sb.append(str); } + if (dtxt != null) debug_write_stringdata("pctb-commodmap.xmlish", sb.toString()); int first = sb.indexOf("

") + 5;
 			int last = sb.indexOf("");
 			xml = sb.substring(first,last);
-			//System.out.println(xml);
 			Reader reader = new CharArrayReader(xml.toCharArray());
 			Document d = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new InputSource(reader));
 			NodeList maps = d.getElementsByTagName("CommodMap");
@@ -694,7 +815,8 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 					buySellCount[1]++;
 				}
 			} catch(IllegalArgumentException e) {
-				System.err.println("Error: Unsupported Commodity \"" + offer.get(0) + "\"");
+			        unknownPCTBcommods++;
+				if (dtxt!=null) dtxt.println("Error: Unsupported Commodity \"" + offer.get(0) + "\"");
 			}
 		}
 		if (buySellCount[0]==0 && buySellCount[1]==0) {
@@ -864,7 +986,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	*/
 	private boolean finishUpload(InputStream in) throws IOException {
 		String html = readstreamstring(in);
-		//System.out.println(html);
+		debug_write_stringdata("pctb-initial.html", html);
 		Matcher m;
 
 		Pattern params = Pattern.compile("(?s).+?.+?");
@@ -896,10 +1018,10 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 		URL get = new URL(PCTB_HOST_URL + "upload.php?action=setisland&ocean=" + oceanNum + "&island="
 			+ islandNum + "&forcereload=" + forceReload + "&filename=" + filename);
 		String complete = readstreamstring(get.openStream());
+		debug_write_stringdata("pctb-final.html", complete);
 		Pattern done = Pattern.compile("Your data has been integrated into the database. Thank you!");
 		m = done.matcher(complete);
 		if(m.find()) {
-			System.out.println("FILE upload successful!!!");
 			return true;
 		} else {
 			error_html("Something was wrong with the final upload parameters!", complete);
@@ -934,7 +1056,9 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	BufferedOutputStream bufos = new BufferedOutputStream(new GZIPOutputStream(bos));
 	bufos.write(yarrgdata.getBytes() );
 	bufos.close();
-	ByteArrayInputStream file = new ByteArrayInputStream(bos.toByteArray());
+	byte[] compressed = bos.toByteArray();
+	debug_write_bytes("yarrg-deduped.tsv.gz", compressed);
+	ByteArrayInputStream file = new ByteArrayInputStream(compressed);
 
 	ClientHttpRequest http = new ClientHttpRequest (YARRG_URL);
 	http.setParameter("clientname", YARRG_CLIENTNAME);
@@ -946,12 +1070,93 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 	http.setParameter("data", "deduped.tsv.gz", file, "application/octet-stream");
 	InputStream in = post_for_yarrg(http);
 	if (in == null) return false;
-	BufferedReader br = new BufferedReader(new InputStreamReader(in));
-	String yarrgresult; 
-	while((yarrgresult = br.readLine()) != null) {
-	    System.out.println(yarrgresult);
+	String output = readstreamstring(in);
+	if (!output.startsWith("OK")) {
+	    error("Unexpected output from YARRG server\n" + output);
+	    return false;
 	}
+	debug_write_stringdata("yarrg-result.txt", output);
 	return true;
     }
+
+    private int calculateArbitrageCommodity(ArrayList> arb_bs) {
+	// if (dtxt!=null) dtxt.println("ARBITRAGE?");
+	int profit = 0;
+	SortedSet buys = arb_bs.get(0);
+	SortedSet sells = arb_bs.get(1);
+	while (true) {
+	    int[] buy, sell;
+	    try {
+		// NB "sell" means they sell, ie we buy
+		sell = sells.last();
+		buy = buys.first();
+	    } catch (NoSuchElementException e) {
+		break;
+	    }
+
+	    int unitprofit = buy[0] - sell[0];
+	    int count = buy[1] < sell[1] ? buy[1] : sell[1];
+	    // if (dtxt!=null) dtxt.println(" sell @"+sell[0]+" x"+sell[1]+" buy @"+buy[0]+" x"+buy[1]
+	    //		       +" => x"+count+" @"+unitprofit);
+
+	    if (unitprofit <= 0)
+		break;
+	    
+	    profit += count * unitprofit;
+	    buy[1] -= count;
+	    sell[1] -= count;
+	    if (buy[1]==0) buys.remove(buy);
+	    if (sell[1]==0) sells.remove(sell);
+	}
+	// if (dtxt!=null) dtxt.println(" PROFIT "+profit);
+	return profit;
+    }
+
+    private class arbitrageOfferComparator implements Comparator {
+	public int compare(Object o1, Object o2) {
+	    int p1 = ((int[])o1)[0];
+	    int p2 = ((int[])o2)[0];
+	    return p2 - p1;
+	}
+    }
+
+    private @SuppressWarnings("unchecked")
+    void calculateArbitrage(ArrayList> data) {
+	int arbitrage = 0;
+	ArrayList> arb_bs = null;
+	String lastcommod = null;
+	Comparator compar = new arbitrageOfferComparator();
+
+	for (ArrayList row : data) {
+	    String thiscommod = row.get(0);
+	    // if (dtxt!=null) dtxt.println("ROW "+row.toString());
+	    if (lastcommod == null || !thiscommod.equals(lastcommod)) {
+		if (lastcommod != null)
+		    arbitrage += calculateArbitrageCommodity(arb_bs);
+		// if (dtxt!=null) dtxt.println("ROW rdy");
+		arb_bs = new ArrayList>(2);
+		arb_bs.add(0, new TreeSet(compar));
+		arb_bs.add(1, new TreeSet(compar));
+		// if (dtxt!=null) dtxt.println("ROW init");
+		lastcommod = thiscommod;
+	    }
+	    for (int bs = 0; bs < 2; bs++) {
+		String pricestr = row.get(bs*2 + 2);
+		if (pricestr == null)
+		    continue;
+		int[] entry = new int[2];
+		// if (dtxt!=null) dtxt.println("ROW BS "+bs);
+		entry[0] = parseQty(pricestr);
+		entry[1] = parseQty(row.get(bs*2 + 3));
+		arb_bs.get(bs).add(entry);
+	    }
+	}
+	arbitrage += calculateArbitrageCommodity(arb_bs);
+	if (arbitrage != 0) {
+	    arbitrageResult.setText("arbitrage: "+arbitrage+" poe");
+	} else {
+	    arbitrageResult.setText("no arbitrage");
+	}
+    }
     
 }