chiark / gitweb /
WIP routesearch; Cache relevant bits of database in memory
authorIan Jackson <ian@liberator.relativity.greenend.org.uk>
Sun, 4 Oct 2009 21:15:45 +0000 (22:15 +0100)
committerIan Jackson <ian@liberator.relativity.greenend.org.uk>
Sun, 4 Oct 2009 23:21:42 +0000 (00:21 +0100)
yarrg/TODO
yarrg/rscommon.h
yarrg/rsmain.c
yarrg/rsvalue.c
yarrg/x.gdb

index 28df53f9f430c8e93bdc3be5f5fa5471fa152aa4..bed30adc750a1cf9ec98803c54fdc95224a37d06 100644 (file)
@@ -1,9 +1,5 @@
 route optimiser
  performance improvements
- - select * from buy, sell where
-       buy.commodid==sell.commodid and buy.price > sell.price;
-    and cache the results in core
- - same for ite
  - fast version of rsvalue.c ?
        time ./routesearch -DN 13460 20210 -1 0.005 search 30 7
        46.478s without -DN
index 5cf25f667601c0b250c1bfb8b61587c77a0cff60..d91765e83b4b8ed0d5426897a69fc784d499dc9a 100644 (file)
@@ -77,7 +77,6 @@ void setup_sql(void);
 
 typedef struct {
   double distance_loss_factor;
-  int ntrades;
   struct TradesBlock *trades;
   double route_tail_value;
 } IslandPair;
index 0982889c8b3f8fc34d2c25e9fb4fe314f3d79a84..ed21fe88e441cba93ce1c940b99a7fe5585166e5 100644 (file)
@@ -28,10 +28,6 @@ int main(int argc, const char **argv) {
     }
   }
 
-  setup_sql();
-  setup_value();
-  setup_search();
-
   max_mass= atof(*argv++);
   max_volu= atof(*argv++);
   max_capi= atof(*argv++);
@@ -39,6 +35,10 @@ int main(int argc, const char **argv) {
 
   if (!loss_per_league) loss_per_league= 1e-7;
   distance_loss_factor_per_league= 1.0 - loss_per_league;
+
+  setup_sql();
+  setup_value();
+  setup_search();
   
   arg= *argv++;
   if (!strcmp(arg,"specific")) {
index eb4d3f053480f82ef5b04ab710488054d1568c2c..c3fac1a01f9260c0812ea58c0f0bffe23f395fe0 100644 (file)
@@ -7,10 +7,10 @@
 DEBUG_DEFINE_DEBUGF(value);
 
 typedef struct { int mass, volu; } CommodInfo;
-static int commodstablesz;
-static CommodInfo *commodstable;
+static int commodstabsz;
+static CommodInfo *commodstab;
 
-static sqlite3_stmt *ss_ipair_dist, *ss_ipair_trades;
+static sqlite3_stmt *ss_ipair_dist;
 static sqlite3_stmt *ss_ite_buy, *ss_ite_sell;
 
 #define MAX_LEGS (MAX_ROUTELEN-1)
@@ -23,6 +23,7 @@ typedef struct {
 
 typedef struct TradesBlock {
   struct TradesBlock *next;
+  int ntrades;
   Trade t[TRADES_PER_BLOCK];
 } TradesBlock;
 
@@ -30,7 +31,9 @@ static IslandPair ***ipairs; /* ipairs[sislandid][dislandid] */
 
 typedef struct IslandTradeEnd {
   struct IslandTradeEnd *next;
+  /* key: */
   int commodid, price;
+  /* values: */
   int qty;
   unsigned long generation;
   int rownum;
@@ -66,22 +69,7 @@ static void avail_c(const Trade *t, IslandTradeEnd **trades,
   for (search= *trades; search; search=search->next)
     if (search->commodid==t->commodid && search->price==price)
       goto found;
-  /* not found, add new row */
-
-  search= mmalloc(sizeof(*search));
-  search->commodid= t->commodid;
-  search->price= price;
-  search->next= *trades;
-  search->generation= 0;
-
-  SQL_BIND(ss_ite, 1, islandid);
-  SQL_BIND(ss_ite, 2, t->commodid);
-  SQL_BIND(ss_ite, 3, price);
-  assert(SQL_STEP(ss_ite));
-  search->qty= sqlite3_column_int(ss_ite, 0);
-  SQL_MUST( sqlite3_reset(ss_ite) );
-  
-  *trades= search;
+  abort();
   
  found:;
   if (search->generation != generation) {
@@ -143,41 +131,16 @@ IslandPair *ipair_get(int si, int di) {
     return ip;
 
   ipa[di]= ip= mmalloc(sizeof(*ip));
-  ip->ntrades= 0;
   ip->trades= 0;
   ip->route_tail_value= -1;
-  int inblock= TRADES_PER_BLOCK;
-  TradesBlock *block=0, **tail=&ip->trades;
 
-  debugf("VALUE ipair_get(%d,%d) running...\n", si,di);
+  debugf("VALUE ipair_get(i%d,i%d) running...\n", si,di);
   SQL_MUST( sqlite3_bind_int(ss_ipair_dist, 1, si) );
   SQL_MUST( sqlite3_bind_int(ss_ipair_dist, 2, di) );
   assert(SQL_STEP(ss_ipair_dist));
   int dist= sqlite3_column_int(ss_ipair_dist, 0);
   ip->distance_loss_factor= pow(distance_loss_factor_per_league, dist);
   sqlite3_reset(ss_ipair_dist);
-
-  SQL_MUST( sqlite3_bind_int(ss_ipair_trades, 1, si) );
-  SQL_MUST( sqlite3_bind_int(ss_ipair_trades, 2, di) );
-
-  while (SQL_STEP(ss_ipair_trades)) {
-    if (inblock == TRADES_PER_BLOCK) {
-      block= mmalloc(sizeof(*block));
-      block->next= 0;
-      *tail= block;
-      tail= &block->next;
-      inblock= 0;
-    }
-    int *irp, i;
-    for (i=0, irp=&block->t[inblock].commodid; i<3; i++, irp++)
-      *irp= sqlite3_column_int(ss_ipair_trades, i);
-    ip->ntrades++;
-    inblock++;
-  }
-  if (inblock < TRADES_PER_BLOCK)
-    block->t[inblock].commodid= -1;
-
-  sqlite3_reset(ss_ipair_trades);
   
   return ip;
 }
@@ -256,58 +219,54 @@ double value_route(int nislands, const int *islands, int exclude_arbitrage) {
       /*----- actually add these trades to the LP problem -----*/
       
       IslandPair *ip= ipair_get(islands[s], islands[d]);
-      TradesBlock *block= ip->trades;
-      int tradestodo= ip->ntrades;
-      if (!tradestodo)
-       goto next_d;
 
-      int inblock= 0;
-      int col= lpx_add_cols(lp,ip->ntrades);
+      if (!ip->trades)
+       goto next_d;
 
       double loss_factor= delay_slot_loss_factor * ip->distance_loss_factor;
       debugf(" SOME   i%d#%d..i%d#%d  dslf=%g dlf=%g  lf=%g\n",
             si,s, di,d,
             delay_slot_loss_factor, ip->distance_loss_factor, loss_factor);
 
-      while (tradestodo-- >0) {
-       if (inblock >= TRADES_PER_BLOCK) {
-         block= block->next;
-         inblock= 0;
-       }
-       Trade *t= &block->t[inblock++];
-
-       debugf("  TRADE i%d#%d..i%d#%d c%d %d-%d  ",
-              si,s, di,d, t->commodid, t->src_price, t->dst_price);
-
-       nconstraint_rows=0;
-
-       avail_c(t, &itradeends[si].src, t->src_price, "src", si, ss_ite_sell);
-       avail_c(t, &itradeends[di].dst, t->dst_price, "dst", di, ss_ite_buy);
-
-       int leg;
-       for (leg=s; leg<d; leg++) {
-         add_leg_c(mass_constraints,leg, commodstable[t->commodid].mass*1e-3);
-         add_leg_c(volu_constraints,leg, commodstable[t->commodid].volu*1e-3);
-         add_leg_c(capi_constraints,leg, t->src_price);
-       }
-
-       double unit_profit= t->dst_price * loss_factor - t->src_price;
-       debugf("    unit profit %f\n", unit_profit);
-
-       lpx_set_col_bnds(lp, col, LPX_LO, 0, 0);
-       lpx_set_obj_coef(lp, col, unit_profit);
-       lpx_set_mat_col(lp, col, nconstraint_rows,
-                       constraint_rows, constraint_coeffs);
-
-       if (DEBUGP(value)) {
-         char *name= masprintf("c%d_p%d_%d_p%d_%d",
-                               t->commodid, s, t->src_price, d, t->dst_price);
-         lpx_set_col_name(lp, col, name);
-         free(name);
-       }
-
-       col++;
-      } /* while (tradestodo-- >0) */
+      TradesBlock *block;
+      for (block=ip->trades; block; block=block->next) {
+       int inblock;
+       for (inblock=0; inblock<block->ntrades; inblock++) {
+         Trade *t= &block->t[inblock];
+
+         debugf("  TRADE i%d#%d..i%d#%d c%d %d-%d  ",
+                si,s, di,d, t->commodid, t->src_price, t->dst_price);
+
+         nconstraint_rows=0;
+
+         avail_c(t, &itradeends[si].src, t->src_price, "src", si,ss_ite_sell);
+         avail_c(t, &itradeends[di].dst, t->dst_price, "dst", di,ss_ite_buy);
+
+         int leg;
+         for (leg=s; leg<d; leg++) {
+           add_leg_c(mass_constraints,leg, commodstab[t->commodid].mass*1e-3);
+           add_leg_c(volu_constraints,leg, commodstab[t->commodid].volu*1e-3);
+           add_leg_c(capi_constraints,leg, t->src_price);
+         }
+
+         double unit_profit= t->dst_price * loss_factor - t->src_price;
+         debugf("    unit profit %f\n", unit_profit);
+         if (unit_profit <= 0) continue;
+
+         int col= lpx_add_cols(lp,1);
+         lpx_set_col_bnds(lp, col, LPX_LO, 0, 0);
+         lpx_set_obj_coef(lp, col, unit_profit);
+         lpx_set_mat_col(lp, col, nconstraint_rows,
+                         constraint_rows, constraint_coeffs);
+
+         if (DEBUGP(value)) {
+           char *name= masprintf("c%d_p%d_%d_p%d_%d", t->commodid,
+                                 s, t->src_price, d, t->dst_price);
+           lpx_set_col_name(lp, col, name);
+           free(name);
+         }
+       } /* inblock */
+      } /* block */
       
       /*----- that's done adding these trades to the LP problem -----*/
       
@@ -339,53 +298,110 @@ double value_route(int nislands, const int *islands, int exclude_arbitrage) {
   return profit;
 }
 
+#define TRADE_FROM                                                     \
+    "  FROM sell, buy\n"                                               \
+    "  WHERE sell.commodid=buy.commodid AND sell.price < buy.price\n"
+             
+static void read_trades(void) {
+  /* We would like to use DISTINCT but sqlite3 is too stupid
+   * to notice that it could use the index to do the DISTINCT
+   * which makes it rather slow. */
+  sqlite3_stmt *ss_trades;
+
+#define TRADE_COLS \
+    "sell.commodid, sell.islandid, sell.price, buy.islandid, buy.price"
+  SQL_PREPARE(ss_trades,
+             " SELECT " TRADE_COLS "\n"
+             TRADE_FROM
+             "  ORDER BY " TRADE_COLS);
+
+  SQL_DISTINCT_DECL(cols,5);
+  while (SQL_DISTINCT_STEP(ss_trades,cols,5)) {    
+    IslandPair *ip= ipair_get(cols[1], cols[3]);
+    TradesBlock *block= ip->trades;
+    if (!block || ip->trades->ntrades >= TRADES_PER_BLOCK) {
+      block= mmalloc(sizeof(*block));
+      block->next= ip->trades;
+      ip->trades= block;
+      block->ntrades= 0;
+    }
+    Trade *trade= &block->t[block->ntrades];
+    trade->commodid=  cols[0];
+    trade->src_price= cols[2];
+    trade->dst_price= cols[4];
+    block->ntrades++;
+  }
+  sqlite3_finalize(ss_trades);
+}
+
+static void read_islandtradeends(const char *bs, int srcdstoff) {
+
+#define TRADEEND_KEYCOLS "%s.commodid, %s.islandid, %s.stallid"
+  char *stmt= masprintf(" SELECT " TRADEEND_KEYCOLS ", %s.price, %s.qty\n"
+                       TRADE_FROM
+                       "  ORDER BY "  TRADEEND_KEYCOLS,
+                       bs,bs,bs,bs,bs, bs,bs,bs);
+  char *stmt_id= masprintf("qtys (%s)",bs);
+  sqlite3_stmt *ss= sql_prepare(stmt, stmt_id);
+  free(stmt); free(stmt_id);
+
+  SQL_DISTINCT_DECL(cols,5);
+  while (SQL_DISTINCT_STEP(ss,cols,3)) {
+    IslandTradeEnd *search;
+
+    int commodid= cols[0];
+    int islandid= cols[1];
+    int price= cols[3];
+    int qty= cols[4];
+
+    IslandTradeEnd **trades= (void*)((char*)&itradeends[islandid] + srcdstoff);
+
+    for (search= *trades; search; search=search->next)
+      if (search->commodid==commodid && search->price==price)
+       goto found;
+    /* not found, add new end */
+
+    search= mmalloc(sizeof(*search));
+    search->commodid= commodid;
+    search->price= price;
+    search->next= *trades;
+    search->generation= 0;
+    search->qty= 0;
+    *trades= search;
+
+  found:
+    search->qty += qty;
+  }
+  sqlite3_finalize(ss);
+}
+
 void setup_value(void) {
   sqlite3_stmt *sst;
   int i;
 
-  commodstablesz= sql_single_int("SELECT max(commodid) FROM commods") + 1;
-  commodstable= mmalloc(sizeof(*commodstable)*commodstablesz);
-  for (i=0; i<commodstablesz; i++)
-    commodstable[i].mass= commodstable[i].volu= -1;
-
-  itradeends= mcalloc(sizeof(*itradeends) * islandtablesz);
+  commodstabsz= sql_single_int("SELECT max(commodid) FROM commods") + 1;
+  commodstab= mmalloc(sizeof(*commodstab)*commodstabsz);
+  for (i=0; i<commodstabsz; i++)
+    commodstab[i].mass= commodstab[i].volu= -1;
 
   SQL_PREPARE(sst,
              "SELECT commodid,unitmass,unitvolume FROM commods");
   while (SQL_STEP(sst)) {
     int id= sqlite3_column_int(sst,0);
-    assert(id>=0 && id<commodstablesz);
-    commodstable[id].mass= sqlite3_column_int(sst,1);
-    commodstable[id].volu= sqlite3_column_int(sst,2);
+    assert(id>=0 && id<commodstabsz);
+    commodstab[id].mass= sqlite3_column_int(sst,1);
+    commodstab[id].volu= sqlite3_column_int(sst,2);
   }
   sqlite3_finalize(sst);
 
+  ipairs= mcalloc(sizeof(*ipairs) * islandtablesz);
+  itradeends= mcalloc(sizeof(*itradeends) * islandtablesz);
+
   SQL_PREPARE(ss_ipair_dist,
              " SELECT dist FROM dists\n"
              "  WHERE aiid=? and biid=?");
 
-  SQL_PREPARE(ss_ipair_trades,
-             "SELECT DISTINCT\n"
-             " sell.commodid           commodid,\n"
-             " sell.price              src_price,\n"
-             " buy.price               dst_price\n"
-             " FROM sell JOIN buy\n"
-             "   ON sell.commodid = buy.commodid\n"
-             "  AND buy.price > sell.price\n"
-             " WHERE sell.islandid=?\n"
-             "   AND buy.islandid=?");
-
-#define BS(bs)                                 \
-  SQL_PREPARE(ss_ite_##bs,                     \
-             "SELECT\n"                        \
-             " sum(qty)\n"                     \
-             " FROM " #bs "\n"                 \
-             " WHERE islandid=?\n"             \
-             "   AND commodid=?\n"             \
-             "   AND price=?");
-  BS(buy)
-  BS(sell)
-#undef BS
-
-  ipairs= mcalloc(sizeof(*ipairs) * islandtablesz);
+  read_trades();
+  read_islandtradeends("sell", offsetof(IslandTradeEndHeads, src));
+  read_islandtradeends("buy",  offsetof(IslandTradeEndHeads, dst));
 }
index c3fc6388a4b353fb1da2adb52465e5f9bdb97037..06e18f8b0021441f73ccb72f05baac862652b4aa 100644 (file)
@@ -1,5 +1,4 @@
-file routesearch
-set args -1 -1 -1 0 34 36 34 36
-break value_route
+file ./routesearch
+set args 13460 20210 -1 0.0005 search 20 7
+break ipair_get if si==7 && di==5
 run
-finish