X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=ypp-sc-tools.main.git;a=blobdiff_plain;f=pctb%2Focr.c;h=6482b81430292af3a7c4da942088ce0e0ce00ace;hp=7655659f902546278529397e0fecc225ea6da4e6;hb=ad71c2a5bfcc29289b2f94324544c3d1a186751b;hpb=af1d80a949cfb4d17084f7b2fe52ac0b75c71473 diff --git a/pctb/ocr.c b/pctb/ocr.c index 7655659..6482b81 100644 --- a/pctb/ocr.c +++ b/pctb/ocr.c @@ -36,7 +36,7 @@ typedef struct { typedef struct DatabaseNode { char *str; int nlinks, alinks; - unsigned match:1, defined:1, endsword:1; + unsigned match:1, defined:1, endsword:1, local:1; DatabaseLink *links; } DatabaseNode; @@ -53,13 +53,22 @@ typedef struct { int rx; } FindCharResults; -enum { ct_Lower, ct_Upper, ct_Word, ct_Digit }; -static const char *context_names[]= { - "Lower", /* bit 0, value 001 */ - "Upper", /* bit 1, value 002 */ - "Word", /* bit 2, value 004 */ - "Digit", /* bit 3, value 010 */ +#define FOR_EACH_CONTEXT(EACH) \ + EACH(Word) \ + EACH(Lower) \ + EACH(Upper) \ + EACH(Digit) + +#define FEC_ENUM(Context) ct_##Context, +#define FEC_BIT(Context) ctf_##Context = 1 << ct_##Context, +enum { + FOR_EACH_CONTEXT(FEC_ENUM) + FOR_EACH_CONTEXT(FEC_BIT) }; + +#define FEC_STRINGS(Context) #Context, +static const char *context_names[]= { FOR_EACH_CONTEXT(FEC_STRINGS) }; + struct OcrCellTypeInfo { /* bitmaps of indices into context_names: */ unsigned initial, nextword, midword; @@ -90,7 +99,7 @@ static void cleardb_node(DatabaseNode *n) { cleardb_node(n->links[i].then); } -static void readdb1(OcrReader *rd, const char *which); +static void readdb1(OcrReader *rd, const char *which, int local); static void readdb(OcrReader *rd) { int ctxi; @@ -98,11 +107,11 @@ static void readdb(OcrReader *rd) { for (ctxi=0; ctxicontexts[ctxi]); - readdb1(rd, "master"); - readdb1(rd, "local"); + readdb1(rd, "master", 0); + readdb1(rd, "local", 1); } -static void readdb1(OcrReader *rd, const char *which) { +static void readdb1(OcrReader *rd, const char *which, int local) { int nchrs; DatabaseNode *current, *additional; char chrs[100]; @@ -188,6 +197,7 @@ static void readdb1(OcrReader *rd, const char *which) { current->str= 0; current->defined= 1; current->match= 0; + current->local= local; if (nchrs) { current->str= mmalloc(nchrs+1); @@ -223,7 +233,7 @@ static void callout_unknown(OcrReader *rd, int w, Pixcol cols[], Pixcol pv; FILE *resolver= resolve_start(); - if (!resolver) + if (!resolver || !(o_flags & ff_editcharset)) fatal("OCR failed - unrecognised characters or ligatures.\n" "Character set database needs to be updated or augmented.\n" "See README.charset.\n"); @@ -235,7 +245,9 @@ static void callout_unknown(OcrReader *rd, int w, Pixcol cols[], for (i=0, s=rd->results; inresults; i++, s++) { if (!strcmp(s->s," ")) continue; fprintf(resolver," %d %d ",s->l,s->r); - cu_pr_ctxmap(resolver, 1u << s->ctxi); + cu_pr_ctxmap(resolver, 1u << s->match); + fprintf(resolver," "); + cu_pr_ctxmap(resolver, s->ctxmap); fprintf(resolver," "); for (p=s->s; (c= *p); p++) { if (c=='\\') fprintf(resolver,"\\%c",c); @@ -265,7 +277,8 @@ static void callout_unknown(OcrReader *rd, int w, Pixcol cols[], readdb(rd); } -static void add_result(OcrReader *rd, const char *s, int l, int r, int ctxi) { +static void add_result(OcrReader *rd, const char *s, int l, int r, + int match, unsigned ctxmap) { if (rd->nresults >= rd->aresults) { rd->aresults++; rd->aresults<<=1; rd->results= mrealloc(rd->results, sizeof(*rd->results)*rd->aresults); @@ -273,7 +286,8 @@ static void add_result(OcrReader *rd, const char *s, int l, int r, int ctxi) { rd->results[rd->nresults].s= s; rd->results[rd->nresults].l= l; rd->results[rd->nresults].r= r; - rd->results[rd->nresults].ctxi= ctxi; + rd->results[rd->nresults].match= match; + rd->results[rd->nresults].ctxmap= ctxmap; rd->nresults++; } @@ -349,25 +363,43 @@ static DatabaseNode *findchar(const FindCharArgs *fca, static int findchar_select_text(const FindCharArgs *fca, const FindCharResults results[]) { - if (fca->ctxmap != 017) return -1; + + dbassert(! results[ct_Digit].match ); /* digits are supposedly unambiguous */ + + switch (fca->ctxmap) { - dbassert(! results[ct_Digit].match ); - if (results[ct_Word].match) return ct_Word; - if (results[ct_Lower].rx > results[ct_Upper].rx) return ct_Lower; - if (results[ct_Upper].rx > results[ct_Lower].rx) return ct_Upper; +#define RETURN_IF_LONGER(this,that) do{ \ + if (results[ct_##this].rx > results[ct_##that].rx) \ + return ct_##this; \ + }while(0) + + case ctf_Digit | ctf_Upper | ctf_Lower | ctf_Word: + /* Start of word. Prefer Word match; failing that, take the longest */ + if (results[ct_Word].match) return ct_Word; + RETURN_IF_LONGER(Lower,Upper); + RETURN_IF_LONGER(Upper,Lower); + break; + + case ctf_Digit | ctf_Upper | ctf_Lower: + /* Mid-word. Prefer longer match; failing that, match lower. */ + RETURN_IF_LONGER(Upper,Lower); + return ct_Lower; + } + + /* oh well */ return -1; } const struct OcrCellTypeInfo ocr_celltype_number= { - 010,010,010, + ctf_Digit, ctf_Digit, ctf_Digit, .space_spaces= 5, .name= "number", .findchar_select= 0 }; const struct OcrCellTypeInfo ocr_celltype_text= { - .initial= 012, /* Digit|Upper */ - .nextword= 017, /* Digit|Upper|Lower|Word */ - .midword= 011, /* Digit|Lower */ + .initial= ctf_Digit | ctf_Upper, + .nextword= ctf_Digit | ctf_Upper | ctf_Lower | ctf_Word, + .midword= ctf_Digit | ctf_Upper | ctf_Lower, .space_spaces= 4, .name= "text", .findchar_select= findchar_select_text @@ -416,7 +448,7 @@ OcrResultGlyph *ocr(OcrReader *rd, OcrCellType ct, int w, Pixcol cols[]) { /* something here, so we need to add the spaces */ if (nspaces >= ct->space_spaces) - add_result(rd," ",x-nspaces,x+1,-1); + add_result(rd," ",x-nspaces,x+1,-1,0); nspaces=0; fca.x= x; @@ -427,7 +459,7 @@ OcrResultGlyph *ocr(OcrReader *rd, OcrCellType ct, int w, Pixcol cols[]) { if (match) { debugf(" || YES"); - add_result(rd, match->str, x, match_rx, match_ctxi); + add_result(rd, match->str, x, match_rx, match_ctxi, fca.ctxmap); x= match_rx+1; if (match->match) fca.ctxmap= ct->midword; else debugf(" (empty)"); @@ -448,7 +480,7 @@ OcrResultGlyph *ocr(OcrReader *rd, OcrCellType ct, int w, Pixcol cols[]) { } } - add_result(rd, 0,-1,-1,-1); + add_result(rd, 0,-1,-1,-1,0); debugf("OCR finished %d glyphs\n",rd->nresults); debug_flush(); return rd->results; @@ -470,3 +502,119 @@ OcrReader *ocr_init(int h) { readdb(rd); return rd; } + +/*---------- character set dump ----------*/ + +static void show_recurse(const DatabaseNode *t, int *count, + const DatabaseNode **store_ary) { + if (t->defined) { + if (store_ary) store_ary[*count]= t; + (*count)++; + } + int l; + for (l=0; lnlinks; l++) + show_recurse(t->links[l].then, count,store_ary); +} + +static int show_char_compar(const void *av, const void *bv) { + const DatabaseNode *const *ap= av; const DatabaseNode *a= *ap; + const DatabaseNode *const *bp= bv; const DatabaseNode *b= *bp; + return strcmp(a->str, b->str) ?: + ((int)a->match - (int)b->match) ?: + ((int)a->endsword - (int)b->endsword) ?: + ((int)a->local - (int)b->local) ?: + 0; +} + +void ocr_showcharsets(void) { + DIR *d; + struct dirent *de; + char found[32]; + pcre *fnpat; + int matchvec[10]; + char hbuf[10]; + const char *pcre_err; + int pcre_erroffset; + + memset(found,0,sizeof(found)); + + fnpat= pcre_compile("\\#(?:master|local)\\-char([1-9]\\d{0,2})\\#\\.txt$", + PCRE_ANCHORED|PCRE_DOLLAR_ENDONLY, + &pcre_err,&pcre_erroffset, 0); + debugf("pcre_compile %p %s\n",fnpat,pcre_err); + assert(fnpat); + + sysassert( d= opendir(get_vardir()) ); + for (;;) { + errno=0; de= readdir(d); if (!de) break; + + int rer= pcre_exec(fnpat,0, de->d_name,strlen(de->d_name), 0,0, + matchvec,ARRAYSIZE(matchvec)); + debugf("pcre_exec `%s' => %d\n", de->d_name,rer); + + if (rer==PCRE_ERROR_NOMATCH || rer==PCRE_ERROR_BADUTF8) continue; + assert(rer==2); + + rer= pcre_copy_substring(de->d_name,matchvec,rer, 1, hbuf,sizeof(hbuf)); + debugf("pcre_copy_substring => %d\n", rer); + assert(rer>0); + + int h= atoi(hbuf); + if (h >= ARRAYSIZE(found)) continue; + + found[h]= 1; + } + + int h; + for (h=0; hcontexts[ctxi], &nchars, 0); + const DatabaseNode **chars= mmalloc(sizeof(*chars) * nchars); + int chari= 0; + show_recurse(&rd->contexts[ctxi], &chari, chars); + assert(chari==nchars); + qsort(chars, nchars, sizeof(*chars), show_char_compar); + + int local; + for (local=0; local<2; local++) { + printf("%2d %-6s %-6s ", h, context_names[ctxi], + local?"local":"master"); + for (chari=0; chari"; + + if (t->local != local) continue; + + if (!t->match) + printf(" [nomatch]"); + else if (!t->endsword && strspn(t->str, accept) == strlen(t->str)) + printf(" %s",t->str); + else { + printf(" \""); + char *p= t->str; + int c; + while ((c=*p++)) { + if (c=='"' || c=='\\') printf("\\%c",c); + else if (c>=' ' && c<=126) putchar(c); + else printf("\\x%02x", (unsigned char)c); + } + if (t->endsword) putchar(' '); + putchar('"'); + } + } + putchar('\n'); + } + free(chars); + } + } +}