From: Ian Jackson Date: Sat, 4 Jul 2009 17:21:11 +0000 (+0100) Subject: OCR rejection infrastructure X-Git-Tag: 1.9.2~33 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=ypp-sc-tools.db-live.git;a=commitdiff_plain;h=c7280e7f26a6fd3685fe96d94c1d76ec77812e57 OCR rejection infrastructure --- diff --git a/pctb/README b/pctb/README index a8dc8d7..2a085b2 100644 --- a/pctb/README +++ b/pctb/README @@ -96,6 +96,14 @@ The program reads and writes the following files: double-check what you're doing before overriding the uploader by telling it to ignore an unrecognised commodity. + * #master-reject#.txt #local-reject#.txt + + Dictionary of regexps which, when the OCR appears to match, we + reject instead. At the moment this is used to stop us thinking + that `Butterfly weed' is `Butterflyweed'. This happens if the + character set dictionary is missing the lowercase `y ' glyph. + See README.charset. + * #master-char*#.txt #local-char*#.txt #master-pixmap#.txt #local-pixmap#.txt diff --git a/pctb/README.charset b/pctb/README.charset index 31e1221..25eb5d8 100644 --- a/pctb/README.charset +++ b/pctb/README.charset @@ -144,6 +144,12 @@ example, the recognised data is wrong), you should delete the file only use the centrally provided (and vetted) master file (which is automatically updated when you run the PCTB client, by default). +It is also possible to have the OCR system reject particular strings. +If you put a regexp in #local-reject#.txt, any OCR result which +matches this string will instead cause an OCR failure, invoking the +OCR dictionary editor if appropriate. #master-reject#.txt is the +centrally maintained version of this file. + Alternatively you can edit #local-char15#.txt with a text editor. The format is not documented at the moment. diff --git a/pctb/README.privacy b/pctb/README.privacy index fa63dea..957f680 100644 --- a/pctb/README.privacy +++ b/pctb/README.privacy @@ -29,14 +29,17 @@ won't happen. 2. YPP SC PCTB client dictionary server ======================================= -This server maintains the master databases of character and island -name images, and of new commodity names, which is used for the -commodity screen OCR, for determining your island name, and for -checking whether commodities not found on the PCTB server are real. - -By default, we ask the server for an updated set of dictionaries every -time we run; this is done with the rsync protocol (indeed, by invoking -rsync). You can disable this with --dict-local-only. +This server maintains the master copies of various databases which are +used to assist the OCR process. Collectively I call these +`dictionaries'. There is a dictionary of character and island name +images, of new commodity names, and of notable known OCR +misrecognitions. These are used for the commodity screen OCR, for +determining your island name, and for checking whether commodities not +found on the PCTB server are real. + +By default, we ask the server for appropriate updated dictionaries +every time we run; this is done with the rsync protocol (indeed, by +invoking rsync). You can disable this with --dict-local-only. If we find a screen display we don't understand, we will ask you about it by popping up a window which allows you to select the island (or diff --git a/pctb/ocr.c b/pctb/ocr.c index 4759782..4b9fa50 100644 --- a/pctb/ocr.c +++ b/pctb/ocr.c @@ -83,6 +83,8 @@ struct OcrCellTypeInfo { struct OcrReader { int h; DatabaseNode contexts[NCONTEXTS]; + char *result; + int lresult, aresult; OcrResultGlyph *results; int aresults, nresults; }; @@ -224,6 +226,84 @@ static void readdb1(OcrReader *rd, const char *which, int local) { free(dbfname); } +typedef struct Rejection Rejection; +struct Rejection { + struct Rejection *next; + const char *fname; + int lno; + pcre *re; +}; + +Rejection *rejections; + +static void load_rejections(const char *which) { + char lbuf[1000]; + char *fname= masprintf("%s/#%s-reject#.txt", get_vardir(), which); + int c, lno=0; + Rejection *rej; + + if (!dbfile_open(fname)) { free(fname); return; } + + while ((c= fgetc(dbfile))!=EOF) { + ungetc(c,dbfile); + lno++; + dbfile_getsline(lbuf,sizeof(lbuf),fname,lno); + + if (!lbuf[0] || isspace(lbuf[0] || lbuf[0]=='#')) + continue; + + rej= mmalloc(sizeof(*rej)); + rej->next= rejections; + rej->fname= fname; + rej->lno= lno; + + const char *err; + int erroffset; + rej->re= pcre_compile(lbuf, PCRE_NO_AUTO_CAPTURE|PCRE_UTF8, + &err, &erroffset, 0); + if (!rej->re) { + char *what= masprintf("invalid regexp at offset %d: %s\n", + erroffset, err); + dbfile_assertfail(fname, lno, what); + } + debugf("OCR LOADED REJECTION %s:%d `%s' %p\n", fname,lno, lbuf, rej->re); + + rejections= rej; + } + sysassert(feof(dbfile)); + dbfile_close(); +} + +static int should_reject(OcrReader *rd) { + static int rejections_loaded; + int ovector[30]; + + if (!rejections_loaded) { + fetch_with_rsync("reject"); + load_rejections("master"); + load_rejections("local"); + rejections_loaded=1; + } + + debugf("[OCR REJECTION `%s'%d", rd->result, rd->lresult); + Rejection *rej; + for (rej=rejections; rej; rej=rej->next) { + debugf(" (%p)", rej); + + int res= pcre_exec(rej->re, 0, rd->result,rd->lresult, 0, + 0, ovector, ARRAYSIZE(ovector)); + if (res==PCRE_ERROR_NOMATCH) continue; + sysassert(res>0); + + debugf(" MATCH]\n"); + fprintf(stderr,"Rejecting OCR result `%s' (due to %s:%d)\n", + rd->result, rej->fname,rej->lno); + return 1; + } + debugf(" OK]"); + return 0; +} + static void cu_pr_ctxmap(FILE *resolver, unsigned ctxmap) { fprintf(resolver,"{"); const char *spc=""; @@ -243,7 +323,10 @@ static void callout_unknown(OcrReader *rd, int w, const Pixcol cols[], const char *p; FILE *resolver= resolve_start(); - if (!resolver || !(o_flags & ff_editcharset)) + if (!resolver || + !((o_flags & ff_charset_edit) || + ((o_flags & ff_charset_allowedit) && + (o_flags & ff_charset_havelocal)))) fatal("OCR failed - unrecognised characters or ligatures.\n" "Character set database needs to be updated or augmented.\n" "See README.charset.\n"); @@ -291,6 +374,18 @@ static void add_result(OcrReader *rd, const char *s, int l, int r, rd->results[rd->nresults].match= match; rd->results[rd->nresults].ctxmap= ctxmap; rd->nresults++; + + if (!s) return; /* just the sentinel for the caller */ + + int sl= strlen(s); + int newlresult= rd->lresult + sl; + if (newlresult >= rd->aresult) { + rd->aresult= (newlresult << 1) + 1; + rd->result= mrealloc(rd->result, rd->aresult); + } + memcpy(rd->result + rd->lresult, s, sl); + rd->lresult= newlresult; + rd->result[rd->lresult]= 0; } @@ -427,6 +522,8 @@ OcrResultGlyph *ocr(OcrReader *rd, OcrCellType ct, int w, nspaces=- w; fca.ctxmap= ct->initial; rd->nresults=0; + rd->lresult=0; + rd->result[0]=0; debugf("OCR h=%d w=%d",rd->h,w); for (x=0; xstr, x, match_rx, match_ctxi, fca.ctxmap); + if (should_reject(rd)) { + callout_unknown(rd, w,cols, match_rx+1,match_rx, 0); + goto restart; + } x= match_rx+1; if (match->match) fca.ctxmap= ct->midword; else debugf(" (empty)"); @@ -503,7 +604,12 @@ OcrReader *ocr_init(int h) { rd= mmalloc(sizeof(*rd)); memset(rd,0,sizeof(*rd)); + rd->h= h; + + rd->aresult= 10; + rd->result= mmalloc(rd->aresult); + readdb(rd); return rd; }