chiark / gitweb /
Show progress of various things in log, with milliseconds
[jarrg-ian.git] / src / com / tedpearson / ypp / market / MarketUploader.java
index 5318b28..98c5dd8 100644 (file)
@@ -16,7 +16,7 @@ import org.w3c.dom.*;
 import javax.xml.parsers.DocumentBuilderFactory;
 import org.xml.sax.InputSource;
 import java.util.zip.GZIPOutputStream;
-import com.myjavatools.web.ClientHttpRequest;
+import net.chiark.yarrg.ClientHttpRequest;
 import java.util.regex.*;
 import java.util.prefs.Preferences;
 import java.beans.*;
@@ -38,6 +38,9 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
        private JFrame frame = null;
        private Window window = null;
        private JButton findMarket = null;
+       private JLabel resultSummary = null;
+       private JLabel arbitrageResult = null;
+        private long startTime = 0;
 
        private final static String PCTB_LIVE_HOST_URL = "http://pctb.crabdance.com/";
        private final static String PCTB_TEST_HOST_URL = "http://pctb.ilk.org/";
@@ -45,14 +48,16 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 
        // Yarrg protocol parameters
        private final static String YARRG_CLIENTNAME = "jpctb greenend";
-       private final static String YARRG_CLIENTVERSION = "0.1";
-       private final static String YARRG_CLIENTFIXES = "";
+       private final static String YARRG_CLIENTVERSION =
+           com.tedpearson.ypp.market.Version.version;
+       private final static String YARRG_CLIENTFIXES = "bug-094";
        private final static String YARRG_LIVE_URL = "http://upload.yarrg.chiark.net/commod-update-receiver";
        private final static String YARRG_TEST_URL = "http://upload.yarrg.chiark.net/test/commod-update-receiver";
        private String YARRG_URL;
 
        private boolean uploadToYarrg;
        private boolean uploadToPCTB;
+       private boolean showArbitrage;
 
        private String islandName = null;
        private String oceanName = null;
@@ -60,14 +65,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 
        private AccessibleContext sidePanel;
        private HashMap<String,Integer> commodMap;
-       private HashMap<String,String> islandNumbers = new HashMap<String,String>();
-       {
-               String[] nums = new String[]
-                       {"","Viridian","Midnight","Hunter","Cobalt","Sage","Ice","Malachite","Crimson","Opal"};
-               for(int i=1;i<nums.length;i++) {
-                       islandNumbers.put(nums[i],""+i);
-               }
-       }
+
        private PropertyChangeListener changeListener = new PropertyChangeListener() {
                public void propertyChange(PropertyChangeEvent e) {
                        if(e.getNewValue() != null && 
@@ -83,6 +81,28 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
                        }
                }
            };
+
+        private int parseQty(String str) {
+               if (str.equals(">1000")) {
+                       return 1001;
+               } else {
+                       return Integer.parseInt(str);
+               }
+       }
+
+        private void progresslog(String s) {
+           long now = new Date().getTime();
+           System.out.println("progress "+(now - startTime)+"ms "+s);
+       }
+
+        private void progressNote(ProgressMonitor pm, String s) {
+           String arb = null;
+           if (arbitrageResult != null)
+               arb = arbitrageResult.getText();
+           if (arb != null && arb.length() != 0)
+               s = "<html>" + arb + "<br>" + s;
+           pm.setNote(s);
+       }
        
        /**
        *       An abstract market offer, entailing a commodity being bought or sold by
@@ -113,11 +133,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
                        commodity = commodId.intValue();
                        price = Integer.parseInt(record.get(priceIndex));
                        String qty = record.get(priceIndex+1);
-                       if(qty.equals(">1000")) {
-                               quantity = 1001;
-                       } else {
-                               quantity = Integer.parseInt(record.get(priceIndex+1));
-                       }
+                       quantity = parseQty(qty);
                        shoppe = stallMap.get(record.get(1)).intValue();
                }
                
@@ -230,6 +246,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
                
                uploadToYarrg=prefs.getBoolean("uploadToYarrg", true);
                uploadToPCTB=prefs.getBoolean("uploadToPCTB", true);
+               showArbitrage=prefs.getBoolean("showArbitrage", true);
 
                EventQueueMonitor.addTopLevelWindowListener(this);
                if (EventQueueMonitor.isGUIInitialized()) {
@@ -249,9 +266,10 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
                        if (window.getAccessibleContext().getAccessibleName().equals("Puzzle Pirates")) frame.setVisible(true);
                        return;
                }
-               frame = new JFrame("MarketUploader");
+               frame = new JFrame("Jarrg Uploader");
                frame.setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE);
-               frame.getContentPane().setLayout(new FlowLayout());
+               GridLayout layout = new GridLayout(2,1);
+               frame.getContentPane().setLayout(layout);
                //frame.setPreferredSize(new Dimension(200, 60));
                
                findMarket = new JButton("Upload Market Data");
@@ -260,11 +278,15 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
                                findMarket.setEnabled(false);
                                new Thread() {
                                        public void run() {
+                                               startTime = new Date().getTime();
+                                               resultSummary.setText("");
+                                               arbitrageResult.setText("");
                                                try {
                                                        runPCTB();
                                                } catch(Exception e) {
                                                        error(e.toString());
                                                        e.printStackTrace();
+                                                       resultSummary.setText("failed");
                                                } finally {
                                                        if(sidePanel != null) {
                                                                // remove it if it's still attached
@@ -278,6 +300,17 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
                        }
                });
                frame.add(findMarket);
+
+               resultSummary = new JLabel("ready");
+               frame.add(resultSummary);
+               
+               arbitrageResult = new JLabel("");
+
+               if (showArbitrage) {
+                   layout.setRows(layout.getRows() + 1);
+                   frame.add(arbitrageResult);
+               }
+
                frame.pack();
        }
        
@@ -339,6 +372,25 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
                JOptionPane.showMessageDialog(frame,msg,"Error",JOptionPane.ERROR_MESSAGE);
        }
        
+       private void error_html(String msg, String html) {
+               //System.err.println("===" + html + "===");
+
+               Pattern body = Pattern.compile("<body>(.*)</body>", Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
+               Matcher m = body.matcher(html);
+               if (m.find()) {
+                       html = m.group(1);
+                       Pattern fixup = Pattern.compile("<(\\w+) */>");;
+                       m = fixup.matcher(html);
+                       html = m.replaceAll("<$1>");
+                       m = Pattern.compile("[\\r\\n]+").matcher(html);
+                       html = m.replaceAll(" ");
+               }
+               String whole_msg = "<html><h1>Error</h1>"+msg+"<h1>PCTB Server said:</h1><blockquote>"+html+"</blockquote>";
+               System.err.println("###" + whole_msg + "###");
+
+               JOptionPane.showMessageDialog(frame,whole_msg,"Error",JOptionPane.ERROR_MESSAGE);
+       }
+       
        /**
        *       Run the data collection process, and upload the results. This is the method
        *       that calls most of the other worker methods for the process. If an error occurs,
@@ -348,21 +400,26 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
        *       @exception Exception if an error we didn't expect occured
        */
        private void runPCTB() throws Exception {
+               progresslog("starting");
+
                String yarrgts = "";
                ProgressMonitor pm = new ProgressMonitor(frame,"Processing Market Data","Getting table data",0,100);
                pm.setMillisToDecideToPopup(0);
                pm.setMillisToPopup(0);
+               boolean doneyarrg = false, donepctb = false;
 
                if (uploadToYarrg) {
+                       progresslog("yarrg timestamp...");
                        yarrgts = getYarrgTimestamp();
+                       progresslog("yarrg timestamp done.");
                }
 
-               AccessibleTable t = findMarketTable();
-               if(t == null) {
+               AccessibleTable accesstable = findMarketTable();
+               if(accesstable == null) {
                        error("Market table not found! Please open the Buy/Sell Commodities interface.");
                        return;
                }
-               if(t.getAccessibleRowCount() == 0) {
+               if(accesstable.getAccessibleRowCount() == 0) {
                        error("No data found, please wait for the table to have data first!");
                        return;
                }
@@ -371,17 +428,53 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
                        return;
                }
 
+               progresslog("getisland...");
                getIsland();
+               progresslog("getocean...");
                getOcean();
+               progresslog("getocean done");
 
                if (latch != null) {
                    latch.await(2, java.util.concurrent.TimeUnit.SECONDS);
                }
 
-               ArrayList<ArrayList<String>> data = getData(t);
+               progresslog("table check...");
 
-               if (uploadToYarrg) {
-                       pm.setNote("Preparing data for Yarrg");
+               String headings_expected[] = new String[]
+                   { "Commodity", "Trading outlet", "Buy price", "Will buy", "Sell price", "Will sell" };
+               ArrayList<ArrayList<String>> headers = getData(accesstable.getAccessibleColumnHeader());
+               if (headers.size() != 1) {
+                   error("Table headings not one row! " + headers.toString());
+                   return;
+               }
+               if (headers.get(0).size() != 6) {
+                   error("Table headings not six columns! " + headers.toString());
+                   return;
+               }
+               for (int col=0; col<headings_expected.length; col++) {
+                   String expd = headings_expected[col];
+                   String got = headers.get(0).get(col);
+                   if (expd.compareTo(got) != 0) {
+                       error("Table heading for column "+col
+                             +" is not \""+expd+"\" but \""+got+"\".\n\n"
+                             +"Please do not reorder the table when using this tool.");
+                       return;
+                   }
+               }
+
+               progresslog("table read...");
+
+               ArrayList<ArrayList<String>> data = getData(accesstable);
+
+               if (showArbitrage) {
+                       progresslog("arbitrage...");
+                       calculateArbitrage(data);
+                       progresslog("arbitrage done.");
+               }
+
+               if (uploadToYarrg && yarrgts != null) {
+                       progresslog("yarrg prepare...");
+                       progressNote(pm, "Yarrg: Preparing data");
                        pm.setProgress(10);
 
                        StringBuilder yarrgsb = new StringBuilder();
@@ -401,63 +494,81 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
 
                        yarrgdata = yarrgsb.toString();
 
-                       pm.setNote("Uploading to Yarrg");
+                       progressNote(pm, "Yarrg: Uploading");
+                       progresslog("yarrg upload...");
 
                        if (islandName != null) {
-                               runYarrg(yarrgts, oceanName, islandName, yarrgdata);
+                               doneyarrg = runYarrg(yarrgts, oceanName, islandName, yarrgdata);
                        } else {
                                System.out.println("Couldn't upload to Yarrg - no island name found");
                        }
+                       progresslog("yarrg done.");
                }
 
-               pm.setNote("Getting stall names");
-               pm.setProgress(20);
-               if(pm.isCanceled()) {
+               if (uploadToPCTB) {
+                   progresslog("pctb prepare...");
+                   progressNote(pm, "PCTB: Getting stall names");
+                   pm.setProgress(20);
+                   if(pm.isCanceled()) {
                        return;
-               }
-               TreeSet<Offer> buys = new TreeSet<Offer>();
-               TreeSet<Offer> sells = new TreeSet<Offer>();
-               LinkedHashMap<String,Integer> stallMap = getStallMap(data);
-               pm.setProgress(40);
-               pm.setNote("Sorting offers");
-               if(pm.isCanceled()) {
+                   }
+                   TreeSet<Offer> buys = new TreeSet<Offer>();
+                   TreeSet<Offer> sells = new TreeSet<Offer>();
+                   LinkedHashMap<String,Integer> stallMap = getStallMap(data);
+                   pm.setProgress(40);
+                   progressNote(pm, "PCTB: Sorting offers");
+                   if(pm.isCanceled()) {
                        return;
-               }
-               // get commod map
+                   }
+                   // get commod map
                
-               HashMap<String,Integer> commodMap = getCommodMap();
-               if(commodMap == null) {
+                   HashMap<String,Integer> commodMap = getCommodMap();
+                   if(commodMap == null) {
                        return;
-               }
-               int[] offerCount = getBuySellMaps(data,buys,sells,stallMap,commodMap);
-               //println(buys.toString());
-               //System.out.println(sells);
-               //System.out.println("\n\n\n"+buys);
+                   }
+                   int[] offerCount = getBuySellMaps(data,buys,sells,stallMap,commodMap);
+                   //println(buys.toString());
+                   //System.out.println(sells);
+                   //System.out.println("\n\n\n"+buys);
 
-               if (uploadToPCTB) {
-                       ByteArrayOutputStream outStream = new ByteArrayOutputStream();
-                       pm.setProgress(60);
-                       pm.setNote("Sending data");
-                       if(pm.isCanceled()) {
-                               return;
-                       }
-                       GZIPOutputStream out = new GZIPOutputStream(outStream);
-                       //FileOutputStream out = new FileOutputStream(new File("output.text"));
-                       DataOutputStream dos = new DataOutputStream(out);
-                       dos.writeBytes("005\n");
-                       dos.writeBytes(stallMap.size()+"\n");
-                       dos.writeBytes(getAbbrevStallList(stallMap));
-                       writeBuySellOffers(buys,sells,offerCount,out);
-                       out.finish();
-                       InputStream in = sendInitialData(new ByteArrayInputStream(outStream.toByteArray()));
-                       pm.setProgress(80);
-                       if(pm.isCanceled()) {
-                               return;
-                       }
-                       pm.setNote("Waiting for PCTB...");
-                       finishUpload(in);
+                   ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+                   pm.setProgress(60);
+                   progressNote(pm, "PCTB: Sending data");
+                   if(pm.isCanceled()) {
+                       return;
+                   }
+                   GZIPOutputStream out = new GZIPOutputStream(outStream);
+                   //FileOutputStream out = new FileOutputStream(new File("output.text"));
+                   DataOutputStream dos = new DataOutputStream(out);
+                   dos.writeBytes("005y\n");
+                   dos.writeBytes(stallMap.size()+"\n");
+                   dos.writeBytes(getAbbrevStallList(stallMap));
+                   writeBuySellOffers(buys,sells,offerCount,out);
+                   out.finish();
+                   progresslog("pctb send...");
+                   InputStream in = sendInitialData(new ByteArrayInputStream(outStream.toByteArray()));
+                   progresslog("pctb sent.");
+                   if (in == null) return;
+                   pm.setProgress(80);
+                   if(pm.isCanceled()) {
+                       return;
+                   }
+                   progressNote(pm, "PCTB: Waiting ...");
+                   progresslog("pctb finish...");
+                   donepctb = finishUpload(in);
+                   progresslog("pctb done.");
                }
                pm.setProgress(100);
+
+               if ((uploadToPCTB && !donepctb) ||
+                   (uploadToYarrg && !doneyarrg)) {
+                       resultSummary.setText("trouble");
+               } else if (donepctb || doneyarrg) {
+                       resultSummary.setText("Done " + islandName);
+               } else {
+                       resultSummary.setText("uploaded nowhere!");
+               }
+               progresslog("done.");
        }
        
        /**
@@ -596,7 +707,7 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
                        //System.out.println(xml);
                        Reader reader = new CharArrayReader(xml.toCharArray());
                        Document d = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new InputSource(reader));
-                       NodeList maps = d.getElementsByTagName("c");
+                       NodeList maps = d.getElementsByTagName("CommodMap");
                        for(int i=0;i<maps.getLength();i++) {
                                NodeList content = maps.item(i).getChildNodes();
                                Integer num = Integer.parseInt(content.item(1).getTextContent());
@@ -663,9 +774,13 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
                                        buySellCount[1]++;
                                }
                        } catch(IllegalArgumentException e) {
-                               // System.err.println("Error: Unsupported Commodity \"" + offer.get(0) + "\"");
+                               System.err.println("Error: Unsupported Commodity \"" + offer.get(0) + "\"");
                        }
                }
+               if (buySellCount[0]==0 && buySellCount[1]==0) {
+                   error("No (valid) offers for PCTB?!");
+                   throw new IllegalArgumentException();
+               }
                return buySellCount;
        }
        
@@ -780,6 +895,16 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
                writeOffers(out,sells);
        }
        
+       private String readstreamstring(InputStream in) throws IOException {
+               StringBuilder sb = new StringBuilder();
+               BufferedReader br = new BufferedReader(new InputStreamReader(in));
+               String str;
+               while((str = br.readLine()) != null) {
+                       sb.append(str+"\n");
+               }
+               return sb.toString();
+       }
+
        /**
        *       Sends the data to the server via multipart-formdata POST,
        *       with the gzipped data as a file upload.
@@ -789,7 +914,12 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
        private InputStream sendInitialData(InputStream file) throws IOException {
                ClientHttpRequest http = new ClientHttpRequest(PCTB_HOST_URL + "upload.php");
                http.setParameter("marketdata","marketdata.gz",file,"application/gzip");
-               return http.post();
+               if (!http.post()) {
+                       String err = readstreamstring(http.resultstream());
+                       error("Error sending initial data:\n"+err);
+                       return null;
+               }
+               return http.resultstream();
        }
        
        /**
@@ -812,123 +942,74 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
        *
        *       @param in stream of data from the server to read
        */
-       private void finishUpload(InputStream in) throws IOException {
-               StringBuilder sb = new StringBuilder();
-               BufferedReader br = new BufferedReader(new InputStreamReader(in));
-               String str;
-               while((str = br.readLine()) != null) {
-                       sb.append(str+"\n");
-               }
-               String html = sb.toString();
+       private boolean finishUpload(InputStream in) throws IOException {
+               String html = readstreamstring(in);
                //System.out.println(html);
-               String topIsland = "0", ocean, islandNum, action, forceReload, filename;
                Matcher m;
-               Pattern whoIsland = Pattern.compile("<option value=\"\\d+\">" + islandName + ", ([^<]+)</ocean>");
-               m = whoIsland.matcher(html);
-               if(m.find()) {
-                       // the server agrees with us
-                       ocean = islandNumbers.get(m.group(1));
-               } else {
-                       // if the server doesn't agree with us:
-                       Pattern island = Pattern.compile("<option value=\"(\\d+)\">([^,]+), ([^<]+)</ocean>");
-                       m = island.matcher(html);
-                       if(!m.find()) {
-                               // server doesn't know what island. if we do, let's select it.
-                               if(islandName != null && !islandName.equals("")) {
-                                       // find the island name in the list as many times as it occurs
-                                       // if more than once, present a dialog
-                                       // set the ocean, we have the islandname, topIsland = 0
-                                       Pattern myIsland = Pattern.compile("islands\\[(\\d+)\\]\\[\\d+\\]=new Option\\(\"" + islandName +
-                                               "\",\\d+");
-                                       Matcher m1 = myIsland.matcher(html);
-                                       ArrayList<Integer> myOceanNums = new ArrayList<Integer>();
-                                       while(m1.find()) {
-                                               myOceanNums.add(new Integer(m1.group(1)));
-                                       }
-                                       if(myOceanNums.size() > 0) {
-                                               if(myOceanNums.size() > 1) {
-                                                       String[] myOceansList = new String[myOceanNums.size()];
-                                                       int i = 0;
-                                                       for(int myOcean : myOceanNums) {
-                                                               Pattern oceanNumPat = Pattern.compile("<option value=\"\\" + myOcean + "\">([^<]+)</option>");
-                                                               m1 = oceanNumPat.matcher(html);
-                                                               if(m1.find()) {
-                                                                       myOceansList[i++] = m1.group(1);
-                                                               }
-                                                       }
-                                                       Object option = JOptionPane.showInputDialog(null,"We found islands named \"" +
-                                                               islandName +"\" on " + myOceansList.length + " oceans:","Choose Ocean",
-                                                               JOptionPane.QUESTION_MESSAGE, null, myOceansList, null);
-                                                       if(option == null) {
-                                                               error("Unable to determine the current island!");
-                                                               return;
-                                                       }
-                                                       ocean = islandNumbers.get(option).toString();
-                                               } else {
-                                                       ocean = myOceanNums.get(0).toString();
-                                               }
-                                       } else {
-                                               error("Unknown island!");
-                                               return;
-                                       }
-                               } else {
-                                       error("Unable to determine island name from the client!");
-                                       return;
-                               }
-                       } else {
-                               topIsland = m.group(1);
-                               islandName = m.group(2);
-                               ocean = islandNumbers.get(m.group(3));
-                       }
-               }
-               Pattern oceanIslandNum = Pattern.compile("islands\\[" + ocean + "\\]\\[\\d+\\]=new Option\\(\"" + islandName + "\",(\\d+)");
-               m = oceanIslandNum.matcher(html);
-               if(!m.find()) {
-                       error("This does not seem to be a valid island! Unable to upload.");
-                       return;
-               }
-               islandNum = m.group(1);
-               Pattern params = Pattern.compile("(?s)<input type=\"hidden\" name=\"action\" value=\"([^\"]+)\" />.+?<input type=\"hidden\" name=\"forcereload\" value=\"([^\"]+)\" />.+?<input type=\"hidden\" name=\"filename\" value=\"([^\"]+)\" />");
+
+               Pattern params = Pattern.compile("(?s)<input type=\"hidden\" name=\"action\" value=\"setisland\" />.+?<input type=\"hidden\" name=\"forcereload\" value=\"([^\"]+)\" />.+?<input type=\"hidden\" name=\"filename\" value=\"([^\"]+)\" />");
                m = params.matcher(html);
                if(!m.find()) {
-                       error("The PCTB server returned unusual data. Maybe you're using an old version of the uploader?");
-                       return;
+                       error_html("The PCTB server returned unusual data. Maybe you're using an old version of the uploader?",
+                                  html);
+                       return false;
                }
-               action = m.group(1);
-               forceReload = m.group(2);
-               filename = m.group(3);
-               URL get = new URL(PCTB_HOST_URL + "upload.php?topisland=" + topIsland + "&ocean=" + ocean + "&island="
-                       + islandNum + "&action=" + action + "&forcereload=" + forceReload + "&filename=" + filename);
-               // System.out.println(get);
-               BufferedReader br2 = new BufferedReader(new InputStreamReader(get.openStream()));
-               sb = new StringBuilder();
-               while((str = br2.readLine()) != null) {
-                       sb.append(str+"\n");
+               String forceReload = m.group(1);
+               String filename = m.group(2);
+
+               Pattern oceanNumPat = Pattern.compile("<option value=\"(\\d+)\">"+oceanName+"</option>");
+               m = oceanNumPat.matcher(html);
+               if (!m.find()) {
+                       error_html("Unable to find the ocean in the server's list of oceans!", html);
+                       return false;
+               }
+               String oceanNum = m.group(1);
+
+               Pattern oceanIslandNum = Pattern.compile("islands\\[" + oceanNum + "\\]\\[\\d+\\]=new Option\\(\"" + islandName + "\",(\\d+)");
+               m = oceanIslandNum.matcher(html);
+               if(!m.find()) {
+                       error_html("This does not seem to be a valid island! Unable to upload.", html);
+                       return false;
                }
+               String islandNum = m.group(1);
+
+               URL get = new URL(PCTB_HOST_URL + "upload.php?action=setisland&ocean=" + oceanNum + "&island="
+                       + islandNum + "&forcereload=" + forceReload + "&filename=" + filename);
+               String complete = readstreamstring(get.openStream());
                Pattern done = Pattern.compile("Your data has been integrated into the database. Thank you!");
-               m = done.matcher(sb.toString());
+               m = done.matcher(complete);
                if(m.find()) {
-                       //System.out.println("FILE upload successful!!!");
+                       System.out.println("FILE upload successful!!!");
+                       return true;
                } else {
-                       error("Something was wrong with the final upload parameters!");
-                       System.err.println(sb.toString());
-                       System.err.println(html);
+                       error_html("Something was wrong with the final upload parameters!", complete);
+                       return false;
                }
        }
 
+    private InputStream post_for_yarrg(ClientHttpRequest http) throws IOException {
+       if (!http.post()) {
+           String err = readstreamstring(http.resultstream());
+           error("<html><h1>Error reported by YARRG server</h1>\n" + err);
+           return null;
+       }
+       return http.resultstream();
+    }
+
     private String getYarrgTimestamp() throws IOException {
        ClientHttpRequest http = new ClientHttpRequest (YARRG_URL);
        http.setParameter("clientname", YARRG_CLIENTNAME);
        http.setParameter("clientversion", YARRG_CLIENTVERSION);
        http.setParameter("clientfixes", YARRG_CLIENTFIXES);
        http.setParameter("requesttimestamp", "y");
-       InputStream in = http.post();
+       InputStream in = post_for_yarrg(http);
+       if (in == null) return null;
        BufferedReader br = new BufferedReader(new InputStreamReader(in));
        String tsresult = br.readLine();
        return tsresult.substring(3, tsresult.length()-1);
     }
 
-    private void runYarrg(String timestamp, String ocean, String island, String yarrgdata) throws IOException {
+    private boolean runYarrg(String timestamp, String ocean, String island, String yarrgdata) throws IOException {
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        BufferedOutputStream bufos = new BufferedOutputStream(new GZIPOutputStream(bos));
        bufos.write(yarrgdata.getBytes() );
@@ -943,12 +1024,94 @@ public class MarketUploader implements TopLevelWindowListener, GUIInitializedLis
        http.setParameter("ocean", ocean);
        http.setParameter("island", island);
        http.setParameter("data", "deduped.tsv.gz", file, "application/octet-stream");
-       InputStream in = http.post();
+       InputStream in = post_for_yarrg(http);
+       if (in == null) return false;
        BufferedReader br = new BufferedReader(new InputStreamReader(in));
        String yarrgresult; 
        while((yarrgresult = br.readLine()) != null) {
            System.out.println(yarrgresult);
        }
+       return true;
+    }
+
+    private int calculateArbitrageCommodity(ArrayList<SortedSet<int[]>> arb_bs) {
+       //System.out.println("ARBITRAGE?");
+       int profit = 0;
+       SortedSet<int[]> buys = arb_bs.get(0);
+       SortedSet<int[]> sells = arb_bs.get(1);
+       while (true) {
+           int[] buy, sell;
+           try {
+               // NB "sell" means they sell, ie we buy
+               sell = sells.last();
+               buy = buys.first();
+           } catch (NoSuchElementException e) {
+               break;
+           }
+
+           int unitprofit = buy[0] - sell[0];
+           int count = buy[1] < sell[1] ? buy[1] : sell[1];
+           //System.out.println(" sell @"+sell[0]+" x"+sell[1]+" buy @"+buy[0]+" x"+buy[1]
+           //                 +" => x"+count+" @"+unitprofit);
+
+           if (unitprofit <= 0)
+               break;
+           
+           profit += count * unitprofit;
+           buy[1] -= count;
+           sell[1] -= count;
+           if (buy[1]==0) buys.remove(buy);
+           if (sell[1]==0) sells.remove(sell);
+       }
+       //System.out.println(" PROFIT "+profit);
+       return profit;
+    }
+
+    private class arbitrageOfferComparator implements Comparator {
+       public int compare(Object o1, Object o2) {
+           int p1 = ((int[])o1)[0];
+           int p2 = ((int[])o2)[0];
+           return p2 - p1;
+       }
+    }
+
+    private @SuppressWarnings("unchecked")
+    void calculateArbitrage(ArrayList<ArrayList<String>> data) {
+       int arbitrage = 0;
+       ArrayList<SortedSet<int[]>> arb_bs = null;
+       String lastcommod = null;
+       Comparator compar = new arbitrageOfferComparator();
+
+       for (ArrayList<String> row : data) {
+           String thiscommod = row.get(0);
+           //System.out.println("ROW "+row.toString());
+           if (lastcommod == null || !thiscommod.equals(lastcommod)) {
+               if (lastcommod != null)
+                   arbitrage += calculateArbitrageCommodity(arb_bs);
+               //System.out.println("ROW rdy");
+               arb_bs = new ArrayList<SortedSet<int[]>>(2);
+               arb_bs.add(0, new TreeSet<int[]>(compar));
+               arb_bs.add(1, new TreeSet<int[]>(compar));
+               //System.out.println("ROW init");
+               lastcommod = thiscommod;
+           }
+           for (int bs = 0; bs < 2; bs++) {
+               String pricestr = row.get(bs*2 + 2);
+               if (pricestr == null)
+                   continue;
+               int[] entry = new int[2];
+               //System.out.println("ROW BS "+bs);
+               entry[0] = parseQty(pricestr);
+               entry[1] = parseQty(row.get(bs*2 + 3));
+               arb_bs.get(bs).add(entry);
+           }
+       }
+       arbitrage += calculateArbitrageCommodity(arb_bs);
+       if (arbitrage != 0) {
+           arbitrageResult.setText("<html><strong>arbitrage: "+arbitrage+" poe</strong>");
+       } else {
+           arbitrageResult.setText("no arbitrage");
+       }
     }
     
 }