rm -f $(TARGETS)
rm -f raw.tsv
rm -f _pages.ppm _upload-*.html _commodmap.tsv
- rm -f _master-*.txt _local-*.txt
+ rm -f _master-*.txt _master-*.txt.gz _local-*.txt
rm -f ./#pages#.ppm ./#upload-*#.html ./#commodmap#.tsv
rm -f ./#master-*#.txt ./#local-*#.txt raw.tsv
character set dictionary is missing the lowercase `y ' glyph.
See README.charset.
- * _master-char*.txt _local-char*.txt
- _master-pixmap.txt _local-pixmap.txt
+ * _master-char*.txt _local-char*.txt
+ _master-pixmap.txt.gz _local-pixmap.txt
Character set and image dictionaries. For the semantics of the
char* files README.charset. There is not currently any accurate
_master-*.txt contain the centrally defined and approved data.
They are downloaded automatically from the SC PCTB server and
- updated each run. You can safely delete this file, if everything
+ updated each run. You can safely delete these files, if everything
is online, if you want to fetch a fresh copy.
_local-*.txt are a local copy of your submissions, so that they
FILE *dbfile;
-static const char *path;
+static const char *basepath; /* as passed in by caller */
+static pid_t dbzcat;
+
+int dbfile_gzopen(const char *basepath_spec) {
+ assert(!dbfile);
+
+ basepath= basepath_spec;
+ //uncomppath= masprintf("%s (uncompressed)", basepath);
+
+ char *zpath= masprintf("%s.gz", basepath);
+ int zfd= open(zpath, O_RDONLY);
+ free(zpath);
+
+ if (zfd<0) { sysassert(errno==ENOENT); return 0; }
+
+ int pipefds[2];
+ sysassert(! pipe(pipefds) );
+
+ sysassert( (dbzcat=fork()) != -1 );
+ if (!dbzcat) {
+ sysassert( dup2(zfd,0)==0 );
+ sysassert( dup2(pipefds[1],1)==1 );
+ sysassert(! close(zfd) );
+ sysassert(! close(pipefds[0]) );
+ sysassert(! close(pipefds[1]) );
+ execlp("zcat","zcat",(char*)0);
+ sysassert(!"execlp zcat");
+ }
+ sysassert(! close(zfd) );
+ sysassert(! close(pipefds[1]) );
+ sysassert( dbfile= fdopen(pipefds[0], "r") );
+
+ return 1;
+}
int dbfile_open(const char *tpath) {
assert(!dbfile);
- path= tpath;
- dbfile= fopen(path,"r");
- if (dbfile) return 1;
- sysassert(errno==ENOENT);
- return 0;
+
+ basepath= tpath;
+
+ dbzcat= -1;
+ dbfile= fopen(tpath,"r");
+ if (!dbfile) { sysassert(errno==ENOENT); return 0; }
+ return 1;
}
+void dbfile_close(void) {
+ if (!dbfile) return;
+
+ sysassert(!ferror(dbfile));
+ sysassert(!fclose(dbfile));
+
+ if (dbzcat != -1) {
+ char *zcatstr= masprintf("zcat %s.gz", basepath);
+ waitpid_check_exitstatus(dbzcat,zcatstr,1);
+ free(zcatstr);
+ dbzcat= -1;
+ }
+
+ dbfile= 0;
+}
+
#define dbassertgl(x) ((x) ? (void)0 : dbfile_assertfail(file,line,#x))
void dbfile_getsline(char *lbuf, size_t lbufsz, const char *file, int line) {
lbuf[l]= 0;
}
-void dbfile_close(void) {
- if (!dbfile) return;
- sysassert(!ferror(dbfile));
- sysassert(!fclose(dbfile));
- dbfile= 0;
-}
-
int dbfile_vscanf(const char *fmt, va_list al) {
int r= vfscanf(dbfile,fmt,al);
sysassert(!ferror(dbfile));
}
void dbfile_assertfail(const char *file, int line, const char *m) {
- if (dbfile)
+ if (dbzcat)
+ fatal("Error in dictionary file %s.gz:\n"
+ " Requirement not met at %s:%d:\n"
+ " %s",
+ basepath, file,line, m);
+ else if (dbfile)
fatal("Error in dictionary file %s at byte %ld:\n"
" Requirement not met at %s:%d:\n"
" %s",
- path,(long)ftell(dbfile), file,line, m);
+ basepath,(long)ftell(dbfile), file,line, m);
else
fatal("Semantic error in dictionaries:\n"
" Requirement not met at %s:%d:\n"
#include <stdlib.h>
#include <unistd.h>
#include <dirent.h>
-#include <pcre.h>
#include <inttypes.h>
+#include <pcre.h>
+
+#include <fcntl.h>
+#include <unistd.h>
#include <sys/wait.h>
#include <sys/types.h>
void sysassert_fail(const char *file, int line, const char *what)
__attribute__((noreturn));
-void waitpid_check_exitstatus(pid_t pid, const char *what);
+void waitpid_check_exitstatus(pid_t pid, const char *what, int sigpipeok);
void *mmalloc(size_t sz);
FILE *dbfile;
void dbfile_getsline(char *lbuf, size_t lbufsz, const char *file, int line);
-int dbfile_open(const char *tpath); /* 0: ENOENT; 1: worked */
+int dbfile_open(const char *tpath); /* 0: ENOENT; 1: worked */
+int dbfile_gzopen(const char *tpath); /* 0: ENOENT; 1: worked */
void dbfile_close(void); /* idempotent */
int dbfile_scanf(const char *fmt, ...) SCANFMT(1,2);
EXECLP_HELPER("commod-results-processor", o_outmode_str, (char*)0);
}
- waitpid_check_exitstatus(processor, "output processor/uploader");
+ waitpid_check_exitstatus(processor, "output processor/uploader", 0);
fclose(tf);
progress_log("all complete.");
}
-void fetch_with_rsync(const char *stem) {
+static void rsync_core(const char *stem, const char *suffix,
+ const char *zopt) {
pid_t fetcher;
+ progress("Updating dictionary %s...",stem);
+
sysassert( (fetcher= fork()) != -1 );
if (!fetcher) {
const char *rsync= getenv("YPPSC_PCTB_RSYNC");
if (!rsync) rsync= "rsync";
const char *src= getenv("YPPSC_PCTB_DICT_UPDATE");
- char *remote= masprintf("%s/master-%s.txt", src, stem);
- char *local= masprintf("_master-%s.txt", stem);
+ char *remote= masprintf("%s/master-%s.txt%s", src, stem, suffix);
+ char *local= masprintf("_master-%s.txt%s", stem, suffix);
if (DEBUGP(rsync))
fprintf(stderr,"executing rsync to fetch %s to %s\n",remote,local);
- execlp(rsync, "rsync",
- DEBUGP(rsync) ? "-zvLt" : "-zLt",
- "--",remote,local,(char*)0);
+ char *opts= masprintf("-Lt%s%s",
+ zopt,
+ DEBUGP(rsync) ? "v" : "");
+ execlp(rsync, "rsync",opts,"--",remote,local,(char*)0);
sysassert(!"exec rsync failed");
}
- waitpid_check_exitstatus(fetcher, "rsync");
+ waitpid_check_exitstatus(fetcher, "rsync", 0);
}
+void fetch_with_rsync_gz(const char *stem) { rsync_core(stem,".gz",""); }
+void fetch_with_rsync(const char *stem) { rsync_core(stem,"","z"); }
+
static void set_server(const char *envname, const char *defprotocol,
const char *defvalue, const char *defvalue_test,
const char *userspecified,
IS("--best-prices")) o_outmode_kind=omk_str,
o_outmode_str=arg+2;
- else if (IS("--screenshot-file")) o_screenshot_fn= ARGVAL;
+ else if (IS("--screenshot-file")||
+ IS("--screenshots-file")) o_screenshot_fn= ARGVAL;
else if (IS("--pctb-server")) o_serv_pctb= ARGVAL;
else if (IS("--dict-submit-server")) o_serv_dict_submit= ARGVAL;
else if (IS("--dict-update-server")) o_serv_dict_fetch= ARGVAL;
_exit(16);
}
-void waitpid_check_exitstatus(pid_t pid, const char *what) {
+void waitpid_check_exitstatus(pid_t pid, const char *what, int sigpipeok) {
pid_t got;
int st;
for (;;) {
fatal("%s failed with nonzero exit status %d",
what, WEXITSTATUS(st));
} else if (WIFSIGNALED(st)) {
- fatal("%s died due to signal %s%s", what,
- strsignal(WTERMSIG(st)), WCOREDUMP(st)?" (core dumped)":"");
+ if (!sigpipeok || WTERMSIG(st) != SIGPIPE)
+ fatal("%s died due to signal %s%s", what,
+ strsignal(WTERMSIG(st)), WCOREDUMP(st)?" (core dumped)":"");
} else {
fatal("%s gave strange wait status %d", what, st);
}
/*----- from structure.c -----*/
-void find_structure(CanonImage *im, int *max_relevant_y_r);
+void find_structure(CanonImage *im, int *max_relevant_y_r,
+ Point *commod_focus_point_r,
+ Point *commod_page_point_r,
+ Point *commod_focuslast_point_r);
Rect find_sunshine_widget(void);
void canon_colour_prepare(void);
/*----- from convert.c -----*/
extern FILE *screenshot_file;
-extern void fetch_with_rsync(const char *stem);
+void fetch_with_rsync(const char *stem);
+void fetch_with_rsync_gz(const char *stem);
void vwarning(const char *fmt, va_list) FMT(1,0);
void warning(const char *fmt, ...) FMT(1,2);
if (o_flags & ff_dict_fetch) {
char *fetchfile= masprintf("char%d",h);
- progress("Updating %s...",fetchfile);
fetch_with_rsync(fetchfile);
free(fetchfile);
}
static struct timeval tv_startup;
static unsigned wwidth, wheight;
static int max_relevant_y= -1;
+static Point commod_focus_point, commod_page_point, commod_focuslast_point;
static XImage *shmim;
static XShmSegmentInfo shminfo;
Window dummy;
xassert( XTranslateCoordinates(disp, id,attr.root, wx,wy, rx,ry, &dummy) );
}
+static void translate_coords_toroot_p(Point w, int *rx, int *ry) {
+ translate_coords_toroot(w.x, w.y, rx, ry);
+}
static void check_client_window_all_on_screen(void) {
Rect onroot;
}
static void mouse_1_updown_here(void) {
check_not_disturbed();
- XTestFakeButtonEvent(disp,1,1, 50);
- XTestFakeButtonEvent(disp,1,0, 50);
+ XTestFakeButtonEvent(disp,1,1, 10);
+ XTestFakeButtonEvent(disp,1,0, 10);
}
static void mouse_1_updown(int x, int y) {
check_not_disturbed();
XTestFakeMotionEvent(disp, screen, xpos,ypos, 0);
mouse_1_updown_here();
}
+static void pgdown_by_mouse(void) {
+ check_not_disturbed();
+ debugf("PAGING Mouse\n");
+ mouse_1_updown_here();
+ sync_after_input();
+}
static int pgupdown;
debugf("PAGING PageUp x %d\n",i);
sync_after_input();
}
-static void send_pgdown(void) {
- send_key(XK_Next);
- pgupdown++;
- debugf("PAGING PageDown\n");
- sync_after_input();
-}
static void send_pgdown_torestore(void) {
debugf("PAGING PageDown x %d\n", -pgupdown);
while (pgupdown < 0) {
debugf("PAGING set_focus\n");
- mouse_1_updown(160,160);
+ mouse_1_updown(commod_focus_point.x, commod_focus_point.y);
sync_after_input();
delay(0.5);
) );
int xpos,ypos;
- translate_coords_toroot(10,10, &xpos,&ypos);
- XTestFakeMotionEvent(disp,screen, xpos,ypos, 0);
+ translate_coords_toroot_p(commod_page_point, &xpos,&ypos);
+ XTestFakeMotionEvent(disp, screen, xpos,ypos, 0);
sync_after_input();
wait_for_stability(¤t,0,0, "checking current YPP client screen...");
test= convert_page(current);
- find_structure(test, &max_relevant_y);
+ find_structure(test, &max_relevant_y,
+ &commod_focus_point,
+ &commod_page_point,
+ &commod_focuslast_point);
check_correct_commodities();
Rect sunshine= find_sunshine_widget();
break;
}
- send_pgdown();
+ pgdown_by_mouse();
npages++;
}
progress("finishing with the YPP client...");
+ mouse_1_updown(commod_focuslast_point.x, commod_focuslast_point.y);
+ sync_after_input();
send_pgdown_torestore();
+ sync_after_input();
debugf("PAGING all done.\n");
progress_log("collected %d screenshots.",npages);
}
if (r==0) {
- waitpid_check_exitstatus(resolver_pid, "dictionary-manager");
+ waitpid_check_exitstatus(resolver_pid, "dictionary-manager", 0);
fclose(resolver);
close(resolver_done);
resolver= 0;
static int identify1(const RgbImage *base, Rect portion,
char result[MAXIMGIDENT], const char *what,
- const char *which) {
+ const char *which, int (*opener)(const char *fn)) {
char *dbfile_name= masprintf("_%s-pixmap.txt",which);
- if (!dbfile_open(dbfile_name))
+ if (!opener(dbfile_name))
goto not_found;
#define FGETSLINE (dbfile_getsline(result,MAXIMGIDENT,__FILE__,__LINE__))
static int identify(const RgbImage *base, Rect portion,
char result[MAXIMGIDENT], const char *what) {
- return identify1(base,portion,result,what, "master") ||
- identify1(base,portion,result,what, "local");
+ return identify1(base,portion,result,what, "master", dbfile_gzopen) ||
+ identify1(base,portion,result,what, "local", dbfile_open);
}
void fwrite_ppmraw(FILE *f, const RgbImage *ri) {
if (!synced) {
if (o_flags & ff_dict_fetch)
- fetch_with_rsync("pixmap");
+ fetch_with_rsync_gz("pixmap");
synced++;
}
#define TEXT_COLUMNS 2
#define MAX_COLUMNS 7
-static Rect mainr = { START_MAIN,START_MAIN };
+static Rect mainr;
static int commbasey, comminty;
static int colrightx[INTERESTING_COLUMNS];
static int text_h=-1, columns=-1;
#define ADJUST_BOX(search,insidechrs,OP,want, lim,LIMIT_MUST, TLBR,XY,increm) \
for (;;) { \
LIMIT_MUST( (search).tl.XY != (search).br.XY && \
- (search).tl.XY != (lim), \
+ (search).TLBR.XY != (lim), \
MR((search));MSB(#TLBR);MSB(#XY) ); \
int got=0; \
Point p=(search).tl; \
(search).TLBR.XY += increm; \
}
-void find_structure(CanonImage *im, int *max_relevant_y_r) {
+void find_structure(CanonImage *im, int *max_relevant_y_r,
+ Point *commod_focus_point_r,
+ Point *commod_page_point_r,
+ Point *commod_focuslast_point_r) {
cim= im;
Rect whole = { {0,0}, {cim->w-1,cim->h-1} };
}
}
+ Point mainr_tl= START_MAIN;
+ mainr.tl= mainr_tl;
+ WALK_UNTIL_MUST(mainr.tl, y,-1, whole.tl.y, ' ');
+ mainr.br= mainr.tl;
+
WALK_UNTIL_MUST(mainr.tl, x,-1, whole.tl.x, '*');
WALK_UNTIL_MUST(mainr.tl, y,-1, whole.tl.y, '*');
WALK_UNTIL_MUST(mainr.br, x,+1, whole.br.x, '*');
Point csb_p; \
Rect csb_r; \
csb_p= mainr.tl; \
+ csb_p.x++; csb_p.y++; \
csb_p.xy= mainr.tlbr.xy; \
if (get_p(csb_p)=='+') { \
csb_r= mainr; \
debug_rect("mainr",1, mainr);
- Point up = START_MAIN;
- WALK_UNTIL_MUST(up, y,-1, mainr.tl.y, '+');
+ Rect updown= {START_MAIN,START_MAIN};
+ const int chkw= 100;
+ updown.br.x += chkw-1;
+ updown.br.y++;
+ debug_rect("updown",__LINE__,updown);
+
+ ADJUST_BOX(updown, "+", >=,chkw, mainr.tl.y, MUST, tl,y,-1);
+ debug_rect("updown",__LINE__,updown);
+ updown.br.y= updown.tl.y;
+ updown.tl.y= updown.tl.y-1;
- Point down = START_MAIN;
- down.y++;
- WALK_UNTIL_MUST(down, y,+1, mainr.br.y, '+');
+ ADJUST_BOX(updown, "+*",>=,chkw, mainr.tl.y-1, MUST, tl,y,-1);
+ debug_rect("updown",__LINE__,updown);
- commbasey= up.y;
- comminty= down.y - up.y + 2;
+ commbasey= updown.tl.y + 1;
+ comminty= updown.br.y - updown.tl.y;
- Point across= { mainr.tl.x, commbasey };
+ Rect across= {{ mainr.tl.x - 1, commbasey },
+ { mainr.tl.x, commbasey + comminty-2 }};
int colno=0;
for (;;) {
- MUST( get_p(across) != '+', MI(colno);MP(across);MR(mainr);MI(commbasey) );
- WALK_UNTIL(across, x,+1, mainr.br.x, '+');
- MUST( colno < MAX_COLUMNS, MP(across);MR(mainr);MI(commbasey); );
- int colrx= across.x;
- if (colrx > mainr.br.x) colrx= mainr.br.x;
+
+#define LIMIT_QUITEQ(cond,mp) { if (!(cond)) break; }
+ debug_rect("across",colno*1000000+__LINE__, across);
+ ADJUST_BOX(across, "+",>=,comminty-1, mainr.br.x, LIMIT_QUITEQ, br,x,+1);
+ debug_rect("across",colno*1000000+__LINE__, across);
+
+ MUST( colno < MAX_COLUMNS, MI(colno);MR(across);MR(mainr);MI(commbasey); );
+ int colrx= across.br.x-1;
+ if (colrx >= mainr.br.x) colrx= mainr.br.x;
if (colno < INTERESTING_COLUMNS)
colrightx[colno]= colrx;
colno++;
- if (across.x >= mainr.br.x-1)
+ if (across.br.x >= mainr.br.x)
break;
- across.x++;
- REQUIRE_RECTANGLE(across.x,mainr.tl.y, across.x,mainr.br.y, "+");
- across.x++;
+ REQUIRE_RECTANGLE(across.br.x,mainr.tl.y, across.br.x,mainr.br.y, "+");
+ across.br.x++;
}
- MUST( colno >= MIN_COLUMNS, MI(colno);MR(mainr);MP(across); );
+ MUST( colno >= MIN_COLUMNS, MI(colno);MR(mainr);MR(across); );
+
+ const int pagerh= 6;
+ Rect pager= {{ mainr.br.x, mainr.br.y - (pagerh-1) },
+ { mainr.br.x + 1, mainr.br.y }};
+
+ debug_rect("pager",__LINE__,pager);
+ ADJUST_BOX(pager, "o",>=,pagerh-2, whole.br.x,MUST, br,x,+1);
+ debug_rect("pager",__LINE__,pager);
+
+ pager.tl.x= pager.br.x;
+ pager.br.x= pager.br.x + 1;
+ debug_rect("pager",__LINE__,pager);
+ ADJUST_BOX(pager, "o",>=,pagerh-2, whole.br.x,MUST, br,x,+1);
+ debug_rect("pager",__LINE__,pager);
+
+ ADJUST_BOX(pager, "o",>=,RECT_W(pager)-2, mainr.tl.y,LIMIT_QUITEQ, tl,y,-1);
+ debug_rect("pager",__LINE__,pager);
#define SET_ONCE(var,val) do{ \
int v= (val); \
if (max_relevant_y_r)
SET_ONCE(*max_relevant_y_r, mainr.br.y + 10);
+ if (commod_focus_point_r) {
+ *commod_focus_point_r= mainr.tl;
+ commod_focus_point_r->x += 10;
+ commod_focus_point_r->y += comminty/3;
+ }
+ if (commod_focuslast_point_r) {
+ *commod_focuslast_point_r= mainr.br;
+ commod_focuslast_point_r->x -= 10;
+ commod_focuslast_point_r->y -= comminty/3;
+ }
+ if (commod_page_point_r) {
+ commod_page_point_r->x= (pager.tl.x + pager.br.x) / 2;
+ commod_page_point_r->y= pager.tl.y - 1;
+ }
+
MUST( text_h <= OCR_MAX_H, MI(text_h) );
}
int page, tryrect, colno;
for (page=0; page<npages; page++) {
- find_structure(page_images[page], 0);
+ find_structure(page_images[page], 0,0,0,0);
if (!page)
check_correct_commodities();
}
void find_islandname(void) {
- find_structure(page_images[0], 0);
+ find_structure(page_images[0], 0,0,0,0);
RgbImage *ri= alloc_rgb_image(page_images[0]->rgb->w,
page_images[0]->rgb->h);
nspaces++;
might_be_colon=0;
} else {
- if (!(pattern & 1UL<<31) && nruns==2 && runs[1]==runs[0]) {
+ if (nruns==2 && runs[1]==runs[0]) {
if (!nspaces) {
if (pattern==colon_pattern)
goto ok_might_be_colon;
might_be_colon=1;
goto ok_might_be_colon;
}
+ } else if (nruns==1 && runs[0]==1 && might_be_colon) {
+ goto colon_found;
}
might_be_colon=0;
ok_might_be_colon: