Merge branch 'ijackson'

diff --git a/pctb/Makefile b/pctb/Makefile
index 8223583a4cbc8fe8aff377be57eedbf51c18bfea..4da46770a04a54ef481e75bdf62418d30e6e1727 100644 (file)
--- a/pctb/Makefile
+++ b/pctb/Makefile
@@ -50,6 +50,6 @@ realclean: clean
        rm -f $(TARGETS)
        rm -f raw.tsv
        rm -f _pages.ppm _upload-*.html _commodmap.tsv
-       rm -f _master-*.txt _local-*.txt
+       rm -f _master-*.txt _master-*.txt.gz _local-*.txt
        rm -f ./#pages#.ppm ./#upload-*#.html ./#commodmap#.tsv
        rm -f ./#master-*#.txt ./#local-*#.txt raw.tsv

diff --git a/pctb/README.files b/pctb/README.files
index b2384a1125b572defc47823dfa6eab3a3b550748..c8d49288e50f8f9c0046ee967f5d205c47dd45ed 100644 (file)
--- a/pctb/README.files
+++ b/pctb/README.files
@@ -43,8 +43,8 @@ The program reads and writes the following files:
    character set dictionary is missing the lowercase `y ' glyph.
    See README.charset.
 
- * _master-char*.txt  _local-char*.txt
-   _master-pixmap.txt _local-pixmap.txt
+ * _master-char*.txt      _local-char*.txt
+   _master-pixmap.txt.gz  _local-pixmap.txt
 
    Character set and image dictionaries.  For the semantics of the
    char* files README.charset.  There is not currently any accurate
@@ -52,7 +52,7 @@ The program reads and writes the following files:
 
    _master-*.txt contain the centrally defined and approved data.
    They are downloaded automatically from the SC PCTB server and
-   updated each run.  You can safely delete this file, if everything
+   updated each run.  You can safely delete these files, if everything
    is online, if you want to fetch a fresh copy.
 
    _local-*.txt are a local copy of your submissions, so that they

diff --git a/pctb/common.c b/pctb/common.c
index bc1cb8df749cb63ec2ef392300551c37262c6e16..2c2a085a30e9fdaa7ded64ffa7b10c82f31208ac 100644 (file)
--- a/pctb/common.c
+++ b/pctb/common.c
@@ -42,17 +42,68 @@ void *mrealloc(void *p, size_t sz) {
 
 
 FILE *dbfile;
-static const char *path;
+static const char *basepath; /* as passed in by caller */
+static pid_t dbzcat;
+
+int dbfile_gzopen(const char *basepath_spec) {
+  assert(!dbfile);
+
+  basepath= basepath_spec;
+  //uncomppath= masprintf("%s (uncompressed)", basepath);
+
+  char *zpath= masprintf("%s.gz", basepath);
+  int zfd= open(zpath, O_RDONLY);
+  free(zpath);
+
+  if (zfd<0) { sysassert(errno==ENOENT); return 0; }
+
+  int pipefds[2];
+  sysassert(! pipe(pipefds) );
+
+  sysassert( (dbzcat=fork()) != -1 );
+  if (!dbzcat) {
+    sysassert( dup2(zfd,0)==0 );
+    sysassert( dup2(pipefds[1],1)==1 );
+    sysassert(! close(zfd) );
+    sysassert(! close(pipefds[0]) );
+    sysassert(! close(pipefds[1]) );
+    execlp("zcat","zcat",(char*)0);
+    sysassert(!"execlp zcat");
+  }
+  sysassert(! close(zfd) );
+  sysassert(! close(pipefds[1]) );
+  sysassert( dbfile= fdopen(pipefds[0], "r") );
+
+  return 1;
+}  
 
 int dbfile_open(const char *tpath) {
   assert(!dbfile);
-  path= tpath;
-  dbfile= fopen(path,"r");
-  if (dbfile) return 1;
-  sysassert(errno==ENOENT);
-  return 0;
+
+  basepath= tpath;
+
+  dbzcat= -1;
+  dbfile= fopen(tpath,"r");
+  if (!dbfile) { sysassert(errno==ENOENT); return 0; }
+  return 1;
 }  
 
+void dbfile_close(void) {
+  if (!dbfile) return;
+
+  sysassert(!ferror(dbfile));
+  sysassert(!fclose(dbfile));
+
+  if (dbzcat != -1) {
+    char *zcatstr= masprintf("zcat %s.gz", basepath);
+    waitpid_check_exitstatus(dbzcat,zcatstr,1);
+    free(zcatstr);
+    dbzcat= -1;
+  }
+
+  dbfile= 0;
+}
+
 #define dbassertgl(x) ((x) ? (void)0 : dbfile_assertfail(file,line,#x))
 
 void dbfile_getsline(char *lbuf, size_t lbufsz, const char *file, int line) {
@@ -66,13 +117,6 @@ void dbfile_getsline(char *lbuf, size_t lbufsz, const char *file, int line) {
   lbuf[l]= 0;
 }
 
-void dbfile_close(void) {
-  if (!dbfile) return;
-  sysassert(!ferror(dbfile));
-  sysassert(!fclose(dbfile));
-  dbfile= 0;
-}
-
 int dbfile_vscanf(const char *fmt, va_list al) {
   int r= vfscanf(dbfile,fmt,al);
   sysassert(!ferror(dbfile));
@@ -88,11 +132,16 @@ int dbfile_scanf(const char *fmt, ...) {
 }
 
 void dbfile_assertfail(const char *file, int line, const char *m) {
-  if (dbfile)
+  if (dbzcat)
+    fatal("Error in dictionary file %s.gz:\n"
+         " Requirement not met at %s:%d:\n"
+         " %s",
+         basepath, file,line, m);
+  else if (dbfile)
     fatal("Error in dictionary file %s at byte %ld:\n"
          " Requirement not met at %s:%d:\n"
          " %s",
-         path,(long)ftell(dbfile), file,line, m);
+         basepath,(long)ftell(dbfile), file,line, m);
   else
     fatal("Semantic error in dictionaries:\n"
          " Requirement not met at %s:%d:\n"

diff --git a/pctb/common.h b/pctb/common.h
index 210f23ac4834baa516d8b707025ca956e93e3083..92ba2daa13d65f27cdf97fe332c48c8ee3ef5108 100644 (file)
--- a/pctb/common.h
+++ b/pctb/common.h
@@ -41,9 +41,12 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <dirent.h>
-#include <pcre.h>
 #include <inttypes.h>
 
+#include <pcre.h>
+
+#include <fcntl.h>
+#include <unistd.h>
 #include <sys/wait.h>
 #include <sys/types.h>
 
@@ -133,7 +136,7 @@ void fatal(const char *fmt, ...)       FMT(1,2) NORET;
 void sysassert_fail(const char *file, int line, const char *what)
    __attribute__((noreturn));
 
-void waitpid_check_exitstatus(pid_t pid, const char *what);
+void waitpid_check_exitstatus(pid_t pid, const char *what, int sigpipeok);
 
 
 void *mmalloc(size_t sz);
@@ -145,7 +148,8 @@ void dbfile_assertfail(const char *file, int line, const char *m) NORET;
 
 FILE *dbfile;
 void dbfile_getsline(char *lbuf, size_t lbufsz, const char *file, int line);
-int dbfile_open(const char *tpath); /* 0: ENOENT; 1: worked */
+int dbfile_open(const char *tpath);   /* 0: ENOENT; 1: worked */
+int dbfile_gzopen(const char *tpath); /* 0: ENOENT; 1: worked */
 void dbfile_close(void); /* idempotent */
 
 int dbfile_scanf(const char *fmt, ...) SCANFMT(1,2);

diff --git a/pctb/convert.c b/pctb/convert.c
index 8386c1609a0f68f85a0936a0df953b26c120f2c2..e6e0102e6b02b69fb7031d52de2a43a80dc03e18 100644 (file)
--- a/pctb/convert.c
+++ b/pctb/convert.c
@@ -110,33 +110,40 @@ static void run_analysis(void) {
     EXECLP_HELPER("commod-results-processor", o_outmode_str, (char*)0);
   }
 
-  waitpid_check_exitstatus(processor, "output processor/uploader");
+  waitpid_check_exitstatus(processor, "output processor/uploader", 0);
   fclose(tf);
   progress_log("all complete.");
 }
 
-void fetch_with_rsync(const char *stem) {
+static void rsync_core(const char *stem, const char *suffix,
+                      const char *zopt) {
   pid_t fetcher;
 
+  progress("Updating dictionary %s...",stem);
+
   sysassert( (fetcher= fork()) != -1 );
   if (!fetcher) {
     const char *rsync= getenv("YPPSC_PCTB_RSYNC");
     if (!rsync) rsync= "rsync";
   
     const char *src= getenv("YPPSC_PCTB_DICT_UPDATE");
-    char *remote= masprintf("%s/master-%s.txt", src, stem);
-    char *local= masprintf("_master-%s.txt", stem);
+    char *remote= masprintf("%s/master-%s.txt%s", src, stem, suffix);
+    char *local= masprintf("_master-%s.txt%s", stem, suffix);
     if (DEBUGP(rsync))
       fprintf(stderr,"executing rsync to fetch %s to %s\n",remote,local);
-    execlp(rsync, "rsync",
-          DEBUGP(rsync) ? "-zvLt" : "-zLt",
-          "--",remote,local,(char*)0);
+    char *opts= masprintf("-Lt%s%s",
+                         zopt,
+                         DEBUGP(rsync) ? "v" : "");
+    execlp(rsync, "rsync",opts,"--",remote,local,(char*)0);
     sysassert(!"exec rsync failed");
   }
 
-  waitpid_check_exitstatus(fetcher, "rsync");
+  waitpid_check_exitstatus(fetcher, "rsync", 0);
 }
 
+void fetch_with_rsync_gz(const char *stem) { rsync_core(stem,".gz",""); }
+void fetch_with_rsync(const char *stem) { rsync_core(stem,"","z"); }
+
 static void set_server(const char *envname, const char *defprotocol,
                       const char *defvalue, const char *defvalue_test,
                       const char *userspecified,
@@ -211,7 +218,8 @@ int main(int argc, char **argv) {
             IS("--best-prices"))      o_outmode_kind=omk_str,
                                       o_outmode_str=arg+2;
 
-    else if (IS("--screenshot-file"))  o_screenshot_fn= ARGVAL;
+    else if (IS("--screenshot-file")||
+            IS("--screenshots-file")) o_screenshot_fn= ARGVAL;
     else if (IS("--pctb-server"))         o_serv_pctb=        ARGVAL;
     else if (IS("--dict-submit-server"))  o_serv_dict_submit= ARGVAL;
     else if (IS("--dict-update-server"))  o_serv_dict_fetch=  ARGVAL;
@@ -411,7 +419,7 @@ void sysassert_fail(const char *file, int line, const char *what) {
   _exit(16);
 }
 
-void waitpid_check_exitstatus(pid_t pid, const char *what) { 
+void waitpid_check_exitstatus(pid_t pid, const char *what, int sigpipeok) { 
   pid_t got;
   int st;
   for (;;) {
@@ -426,8 +434,9 @@ void waitpid_check_exitstatus(pid_t pid, const char *what) {
       fatal("%s failed with nonzero exit status %d",
            what, WEXITSTATUS(st));
   } else if (WIFSIGNALED(st)) {
-    fatal("%s died due to signal %s%s", what,
-         strsignal(WTERMSIG(st)), WCOREDUMP(st)?" (core dumped)":"");
+    if (!sigpipeok || WTERMSIG(st) != SIGPIPE)
+      fatal("%s died due to signal %s%s", what,
+           strsignal(WTERMSIG(st)), WCOREDUMP(st)?" (core dumped)":"");
   } else {
     fatal("%s gave strange wait status %d", what, st);
   }

diff --git a/pctb/convert.h b/pctb/convert.h
index ba13c433f3e4340d02b8b0c0522551587750481a..0dc2255ddaea9c9a1cc214e04c3487c5255a145a 100644 (file)
--- a/pctb/convert.h
+++ b/pctb/convert.h
@@ -69,7 +69,10 @@ static inline Rgb ri_rgb(const RgbImage *ri, int x, int y) {
 
 /*----- from structure.c -----*/
 
-void find_structure(CanonImage *im, int *max_relevant_y_r);
+void find_structure(CanonImage *im, int *max_relevant_y_r,
+                   Point *commod_focus_point_r,
+                   Point *commod_page_point_r,
+                   Point *commod_focuslast_point_r);
 Rect find_sunshine_widget(void);
 
 void canon_colour_prepare(void);
@@ -82,7 +85,8 @@ void analyse(FILE *tsv_output);
 /*----- from convert.c -----*/
 
 extern FILE *screenshot_file;
-extern void fetch_with_rsync(const char *stem);
+void fetch_with_rsync(const char *stem);
+void fetch_with_rsync_gz(const char *stem);
 
 void vwarning(const char *fmt, va_list) FMT(1,0);
 void warning(const char *fmt, ...)      FMT(1,2);

diff --git a/pctb/ocr.c b/pctb/ocr.c
index 37ca7d28ed284de9b70c1cc29b1f528a20d24a12..f7cb4504224583c48f4796a11991b48b81366e22 100644 (file)
--- a/pctb/ocr.c
+++ b/pctb/ocr.c
@@ -597,7 +597,6 @@ OcrReader *ocr_init(int h) {
 
   if (o_flags & ff_dict_fetch) {
     char *fetchfile= masprintf("char%d",h);
-    progress("Updating %s...",fetchfile);
     fetch_with_rsync(fetchfile);
     free(fetchfile);
   }

diff --git a/pctb/pages.c b/pctb/pages.c
index 010299948d929d3f1ce247d683de11b900bb6697..72f9bb478a982f5b0206af12c2cc0b6209c6df83 100644 (file)
--- a/pctb/pages.c
+++ b/pctb/pages.c
@@ -52,6 +52,7 @@ static Display *disp;
 static struct timeval tv_startup;
 static unsigned wwidth, wheight;
 static int max_relevant_y= -1;
+static Point commod_focus_point, commod_page_point, commod_focuslast_point;
 
 static XImage *shmim;
 static XShmSegmentInfo shminfo;
@@ -104,6 +105,9 @@ static void translate_coords_toroot(int wx, int wy, int *rx, int *ry) {
   Window dummy;
   xassert( XTranslateCoordinates(disp, id,attr.root, wx,wy, rx,ry, &dummy) );
 }
+static void translate_coords_toroot_p(Point w, int *rx, int *ry) {
+  translate_coords_toroot(w.x, w.y, rx, ry);
+}
 
 static void check_client_window_all_on_screen(void) {
   Rect onroot;
@@ -160,8 +164,8 @@ static void send_key(KeySym sym) {
 }
 static void mouse_1_updown_here(void) {
   check_not_disturbed();
-  XTestFakeButtonEvent(disp,1,1, 50);
-  XTestFakeButtonEvent(disp,1,0, 50);
+  XTestFakeButtonEvent(disp,1,1, 10);
+  XTestFakeButtonEvent(disp,1,0, 10);
 }
 static void mouse_1_updown(int x, int y) {
   check_not_disturbed();
@@ -171,6 +175,12 @@ static void mouse_1_updown(int x, int y) {
   XTestFakeMotionEvent(disp, screen, xpos,ypos, 0);
   mouse_1_updown_here();
 }
+static void pgdown_by_mouse(void) {
+  check_not_disturbed();
+  debugf("PAGING   Mouse\n");
+  mouse_1_updown_here();
+  sync_after_input();
+}
 
 static int pgupdown;
 
@@ -183,12 +193,6 @@ static void send_pgup_many(void) {
   debugf("PAGING   PageUp x %d\n",i);
   sync_after_input();
 }
-static void send_pgdown(void) {
-  send_key(XK_Next);
-  pgupdown++;
-  debugf("PAGING   PageDown\n");
-  sync_after_input();
-}
 static void send_pgdown_torestore(void) {
   debugf("PAGING   PageDown x %d\n", -pgupdown);
   while (pgupdown < 0) {
@@ -497,7 +501,7 @@ static void set_focus_commodity(void) {
 
   debugf("PAGING set_focus\n");
 
-  mouse_1_updown(160,160);
+  mouse_1_updown(commod_focus_point.x, commod_focus_point.y);
   sync_after_input();
 
   delay(0.5);
@@ -508,8 +512,8 @@ static void set_focus_commodity(void) {
                        ) );
 
   int xpos,ypos;
-  translate_coords_toroot(10,10, &xpos,&ypos);
-  XTestFakeMotionEvent(disp,screen, xpos,ypos, 0);
+  translate_coords_toroot_p(commod_page_point, &xpos,&ypos);
+  XTestFakeMotionEvent(disp, screen, xpos,ypos, 0);
 
   sync_after_input();
 
@@ -543,7 +547,10 @@ static void prepare_ypp_client(void) {
   wait_for_stability(&current,0,0, "checking current YPP client screen...");
 
   test= convert_page(current);
-  find_structure(test, &max_relevant_y);
+  find_structure(test, &max_relevant_y,
+                &commod_focus_point,
+                &commod_page_point,
+                &commod_focuslast_point);
   check_correct_commodities();
   Rect sunshine= find_sunshine_widget();
 
@@ -603,11 +610,14 @@ void take_screenshots(void) {
       break;
     }
 
-    send_pgdown();
+    pgdown_by_mouse();
     npages++;
   }
   progress("finishing with the YPP client...");
+  mouse_1_updown(commod_focuslast_point.x, commod_focuslast_point.y);
+  sync_after_input();
   send_pgdown_torestore();
+  sync_after_input();
 
   debugf("PAGING all done.\n");
   progress_log("collected %d screenshots.",npages);

diff --git a/pctb/resolve.c b/pctb/resolve.c
index 19615f3e4ea251ab301cb496d28b09b7181203c3..e89d2054e91a0e076c685a03b6f1a2273d8415e2 100644 (file)
--- a/pctb/resolve.c
+++ b/pctb/resolve.c
@@ -83,7 +83,7 @@ void resolve_finish(void) {
   }
 
   if (r==0) {
-    waitpid_check_exitstatus(resolver_pid, "dictionary-manager");
+    waitpid_check_exitstatus(resolver_pid, "dictionary-manager", 0);
     fclose(resolver);
     close(resolver_done);
     resolver= 0;

diff --git a/pctb/rgbimage.c b/pctb/rgbimage.c
index dfcd1808f25421ceb9223343c1d24618bbae2eef..81eddf9898e83427a83f7376bef93868e8a4dc56 100644 (file)
--- a/pctb/rgbimage.c
+++ b/pctb/rgbimage.c
@@ -50,9 +50,9 @@
 
 static int identify1(const RgbImage *base, Rect portion,
                    char result[MAXIMGIDENT], const char *what,
-                   const char *which) {
+                   const char *which, int (*opener)(const char *fn)) {
   char *dbfile_name= masprintf("_%s-pixmap.txt",which);
-  if (!dbfile_open(dbfile_name))
+  if (!opener(dbfile_name))
     goto not_found;
 
 #define FGETSLINE (dbfile_getsline(result,MAXIMGIDENT,__FILE__,__LINE__))
@@ -111,8 +111,8 @@ static int identify1(const RgbImage *base, Rect portion,
 
 static int identify(const RgbImage *base, Rect portion,
                    char result[MAXIMGIDENT], const char *what) {
-  return identify1(base,portion,result,what, "master") ||
-         identify1(base,portion,result,what, "local");
+  return identify1(base,portion,result,what, "master", dbfile_gzopen) ||
+         identify1(base,portion,result,what, "local",  dbfile_open);
 }
 
 void fwrite_ppmraw(FILE *f, const RgbImage *ri) {
@@ -147,7 +147,7 @@ void identify_rgbimage(const RgbImage *base, Rect portion,
 
   if (!synced) {
     if (o_flags & ff_dict_fetch)
-      fetch_with_rsync("pixmap");
+      fetch_with_rsync_gz("pixmap");
     synced++;
   }
   

diff --git a/pctb/structure.c b/pctb/structure.c
index d777d5713697d6f9c32ddfc998c7f97ed8416c6c..ad95cb508791b5213e64bd8170258b302ff0d7cb 100644 (file)
--- a/pctb/structure.c
+++ b/pctb/structure.c
@@ -40,7 +40,7 @@ DEBUG_DEFINE_DEBUGF(struct)
 #define TEXT_COLUMNS        2
 #define MAX_COLUMNS         7
 
-static Rect mainr = { START_MAIN,START_MAIN };
+static Rect mainr;
 static int commbasey, comminty;
 static int colrightx[INTERESTING_COLUMNS];
 static int text_h=-1, columns=-1;
@@ -242,7 +242,7 @@ static int commod_selector_matches(Rect search, const char *const *all,
 #define ADJUST_BOX(search,insidechrs,OP,want, lim,LIMIT_MUST, TLBR,XY,increm) \
   for (;;) {                                                                 \
     LIMIT_MUST( (search).tl.XY != (search).br.XY &&                          \
-               (search).tl.XY != (lim),                                      \
+               (search).TLBR.XY != (lim),                                    \
                MR((search));MSB(#TLBR);MSB(#XY) );                           \
     int got=0;                                                               \
     Point p=(search).tl;                                                     \
@@ -255,7 +255,10 @@ static int commod_selector_matches(Rect search, const char *const *all,
     (search).TLBR.XY += increm;                                                      \
   }
 
-void find_structure(CanonImage *im, int *max_relevant_y_r) {
+void find_structure(CanonImage *im, int *max_relevant_y_r,
+                   Point *commod_focus_point_r,
+                   Point *commod_page_point_r,
+                   Point *commod_focuslast_point_r) {
   cim= im;
   
   Rect whole = { {0,0}, {cim->w-1,cim->h-1} };
@@ -272,6 +275,11 @@ void find_structure(CanonImage *im, int *max_relevant_y_r) {
     }
   }
 
+  Point mainr_tl= START_MAIN;
+  mainr.tl= mainr_tl;
+  WALK_UNTIL_MUST(mainr.tl, y,-1, whole.tl.y, ' ');
+  mainr.br= mainr.tl;
+
   WALK_UNTIL_MUST(mainr.tl, x,-1, whole.tl.x, '*');
   WALK_UNTIL_MUST(mainr.tl, y,-1, whole.tl.y, '*');
   WALK_UNTIL_MUST(mainr.br, x,+1, whole.br.x, '*');
@@ -287,6 +295,7 @@ void find_structure(CanonImage *im, int *max_relevant_y_r) {
     Point csb_p;                                       \
     Rect csb_r;                                                \
     csb_p= mainr.tl;                                   \
+    csb_p.x++; csb_p.y++;                              \
     csb_p.xy= mainr.tlbr.xy;                           \
     if (get_p(csb_p)=='+') {                           \
       csb_r= mainr;                                    \
@@ -306,37 +315,65 @@ void find_structure(CanonImage *im, int *max_relevant_y_r) {
 
   debug_rect("mainr",1, mainr);
 
-  Point up = START_MAIN;
-  WALK_UNTIL_MUST(up, y,-1, mainr.tl.y, '+');
+  Rect updown= {START_MAIN,START_MAIN};
+  const int chkw= 100;
+  updown.br.x += chkw-1;
+  updown.br.y++;
+  debug_rect("updown",__LINE__,updown);
+
+  ADJUST_BOX(updown, "+", >=,chkw, mainr.tl.y,   MUST, tl,y,-1);
+  debug_rect("updown",__LINE__,updown);
+  updown.br.y= updown.tl.y;
+  updown.tl.y= updown.tl.y-1;
 
-  Point down = START_MAIN;
-  down.y++;
-  WALK_UNTIL_MUST(down, y,+1, mainr.br.y, '+');
+  ADJUST_BOX(updown, "+*",>=,chkw, mainr.tl.y-1, MUST, tl,y,-1);
+  debug_rect("updown",__LINE__,updown);
 
-  commbasey= up.y;
-  comminty= down.y - up.y + 2;
+  commbasey= updown.tl.y + 1;
+  comminty= updown.br.y - updown.tl.y;
 
-  Point across= { mainr.tl.x, commbasey };
+  Rect across= {{ mainr.tl.x - 1, commbasey              },
+               { mainr.tl.x,     commbasey + comminty-2 }};
   int colno=0;
   for (;;) {
-    MUST( get_p(across) != '+', MI(colno);MP(across);MR(mainr);MI(commbasey) );
-    WALK_UNTIL(across, x,+1, mainr.br.x, '+');
-    MUST( colno < MAX_COLUMNS, MP(across);MR(mainr);MI(commbasey); );
-    int colrx= across.x;
-    if (colrx > mainr.br.x) colrx= mainr.br.x;
+
+#define LIMIT_QUITEQ(cond,mp) { if (!(cond)) break; }
+    debug_rect("across",colno*1000000+__LINE__, across);
+    ADJUST_BOX(across, "+",>=,comminty-1, mainr.br.x, LIMIT_QUITEQ, br,x,+1);
+    debug_rect("across",colno*1000000+__LINE__, across);
+
+    MUST( colno < MAX_COLUMNS, MI(colno);MR(across);MR(mainr);MI(commbasey); );
+    int colrx= across.br.x-1;
+    if (colrx >= mainr.br.x) colrx= mainr.br.x;
     if (colno < INTERESTING_COLUMNS)
       colrightx[colno]= colrx;
       
     colno++;
     
-    if (across.x >= mainr.br.x-1)
+    if (across.br.x >= mainr.br.x)
       break;
 
-    across.x++;
-    REQUIRE_RECTANGLE(across.x,mainr.tl.y, across.x,mainr.br.y, "+");
-    across.x++;
+    REQUIRE_RECTANGLE(across.br.x,mainr.tl.y, across.br.x,mainr.br.y, "+");
+    across.br.x++;
   }
-  MUST( colno >= MIN_COLUMNS, MI(colno);MR(mainr);MP(across); );
+  MUST( colno >= MIN_COLUMNS, MI(colno);MR(mainr);MR(across); );
+
+  const int pagerh= 6;
+  Rect pager= {{ mainr.br.x,     mainr.br.y - (pagerh-1) },
+              { mainr.br.x + 1, mainr.br.y              }};
+
+  debug_rect("pager",__LINE__,pager);
+  ADJUST_BOX(pager, "o",>=,pagerh-2, whole.br.x,MUST, br,x,+1);
+  debug_rect("pager",__LINE__,pager);
+
+  pager.tl.x= pager.br.x;
+  pager.br.x= pager.br.x + 1;
+  debug_rect("pager",__LINE__,pager);
+  ADJUST_BOX(pager, "o",>=,pagerh-2, whole.br.x,MUST, br,x,+1);
+  debug_rect("pager",__LINE__,pager);
+
+  ADJUST_BOX(pager, "o",>=,RECT_W(pager)-2, mainr.tl.y,LIMIT_QUITEQ, tl,y,-1);
+  debug_rect("pager",__LINE__,pager);
 
 #define SET_ONCE(var,val) do{                                          \
     int v= (val);                                                      \
@@ -349,6 +386,21 @@ void find_structure(CanonImage *im, int *max_relevant_y_r) {
   if (max_relevant_y_r)
     SET_ONCE(*max_relevant_y_r, mainr.br.y + 10);
 
+  if (commod_focus_point_r) {
+    *commod_focus_point_r= mainr.tl;
+    commod_focus_point_r->x += 10;
+    commod_focus_point_r->y += comminty/3;
+  }
+  if (commod_focuslast_point_r) {
+    *commod_focuslast_point_r= mainr.br;
+    commod_focuslast_point_r->x -= 10;
+    commod_focuslast_point_r->y -= comminty/3;
+  }
+  if (commod_page_point_r) {
+    commod_page_point_r->x= (pager.tl.x + pager.br.x) / 2;
+    commod_page_point_r->y=  pager.tl.y - 1;
+  }
+
   MUST( text_h <= OCR_MAX_H, MI(text_h) );
 }                  
 
@@ -616,7 +668,7 @@ void analyse(FILE *tsv_output) {
   int page, tryrect, colno;
 
   for (page=0; page<npages; page++) {
-    find_structure(page_images[page], 0);
+    find_structure(page_images[page], 0,0,0,0);
 
     if (!page)
       check_correct_commodities();
@@ -677,7 +729,7 @@ Rect find_sunshine_widget(void) {
 }
 
 void find_islandname(void) {
-  find_structure(page_images[0], 0);
+  find_structure(page_images[0], 0,0,0,0);
 
   RgbImage *ri= alloc_rgb_image(page_images[0]->rgb->w,
                                page_images[0]->rgb->h);
@@ -803,7 +855,7 @@ void find_islandname(void) {
        nspaces++;
        might_be_colon=0;
       } else {
-       if (!(pattern & 1UL<<31) && nruns==2 && runs[1]==runs[0]) {
+       if (nruns==2 && runs[1]==runs[0]) {
          if (!nspaces) {
            if (pattern==colon_pattern)
              goto ok_might_be_colon;
@@ -812,6 +864,8 @@ void find_islandname(void) {
            might_be_colon=1;
            goto ok_might_be_colon;
          }
+       } else if (nruns==1 && runs[0]==1 && might_be_colon) {
+         goto colon_found;
        }
        might_be_colon=0;
       ok_might_be_colon: