OCR rejection infrastructure

diff --git a/pctb/README b/pctb/README
index a8dc8d7b8f59cbb58b7330b99cc117f812fd8d2f..2a085b2011c9d0938e5ee37cb19fd9ed6420625c 100644 (file)
--- a/pctb/README
+++ b/pctb/README
@@ -96,6 +96,14 @@ The program reads and writes the following files:
    double-check what you're doing before overriding the uploader by
    telling it to ignore an unrecognised commodity.
 
+ * #master-reject#.txt #local-reject#.txt
+
+   Dictionary of regexps which, when the OCR appears to match, we
+   reject instead.  At the moment this is used to stop us thinking
+   that `Butterfly weed' is `Butterflyweed'.  This happens if the
+   character set dictionary is missing the lowercase `y ' glyph.
+   See README.charset.
+
  * #master-char*#.txt  #local-char*#.txt
    #master-pixmap#.txt #local-pixmap#.txt
 

diff --git a/pctb/README.charset b/pctb/README.charset
index 31e1221d7b4340c1f2bb1ed68fb0366f69c0478c..25eb5d88feb9e6d8ad603533610219eb34c682b5 100644 (file)
--- a/pctb/README.charset
+++ b/pctb/README.charset
@@ -144,6 +144,12 @@ example, the recognised data is wrong), you should delete the file
 only use the centrally provided (and vetted) master file (which is
 automatically updated when you run the PCTB client, by default).
 
+It is also possible to have the OCR system reject particular strings.
+If you put a regexp in #local-reject#.txt, any OCR result which
+matches this string will instead cause an OCR failure, invoking the
+OCR dictionary editor if appropriate.  #master-reject#.txt is the
+centrally maintained version of this file.
+
 Alternatively you can edit #local-char15#.txt with a text editor.  The
 format is not documented at the moment.
 

diff --git a/pctb/README.privacy b/pctb/README.privacy
index fa63dea55c84df25f472c5bf3f57c75011591eb4..957f68002c327b6fa44f33d38c1c94d51e290ac4 100644 (file)
--- a/pctb/README.privacy
+++ b/pctb/README.privacy
@@ -29,14 +29,17 @@ won't happen.
 2. YPP SC PCTB client dictionary server
 =======================================
 
-This server maintains the master databases of character and island
-name images, and of new commodity names, which is used for the
-commodity screen OCR, for determining your island name, and for
-checking whether commodities not found on the PCTB server are real.
-
-By default, we ask the server for an updated set of dictionaries every
-time we run; this is done with the rsync protocol (indeed, by invoking
-rsync).  You can disable this with --dict-local-only.
+This server maintains the master copies of various databases which are
+used to assist the OCR process.  Collectively I call these
+`dictionaries'.  There is a dictionary of character and island name
+images, of new commodity names, and of notable known OCR
+misrecognitions.  These are used for the commodity screen OCR, for
+determining your island name, and for checking whether commodities not
+found on the PCTB server are real.
+
+By default, we ask the server for appropriate updated dictionaries
+every time we run; this is done with the rsync protocol (indeed, by
+invoking rsync).  You can disable this with --dict-local-only.
 
 If we find a screen display we don't understand, we will ask you about
 it by popping up a window which allows you to select the island (or

diff --git a/pctb/ocr.c b/pctb/ocr.c
index 47597828c5f0091f1b5dce2e1592bb2e82aa4da6..4b9fa505b156afa3a001f3635ad0efeca5de2c84 100644 (file)
--- a/pctb/ocr.c
+++ b/pctb/ocr.c
@@ -83,6 +83,8 @@ struct OcrCellTypeInfo {
 struct OcrReader {
   int h;
   DatabaseNode contexts[NCONTEXTS];
+  char *result;
+  int lresult, aresult;
   OcrResultGlyph *results;
   int aresults, nresults;
 };
@@ -224,6 +226,84 @@ static void readdb1(OcrReader *rd, const char *which, int local) {
   free(dbfname);
 }
 
+typedef struct Rejection Rejection;
+struct Rejection {
+  struct Rejection *next;
+  const char *fname;
+  int lno;
+  pcre *re;
+};
+
+Rejection *rejections;
+
+static void load_rejections(const char *which) {
+  char lbuf[1000];
+  char *fname= masprintf("%s/#%s-reject#.txt", get_vardir(), which);
+  int c, lno=0;
+  Rejection *rej;
+  
+  if (!dbfile_open(fname)) { free(fname); return; }
+  
+  while ((c= fgetc(dbfile))!=EOF) {
+    ungetc(c,dbfile);
+    lno++;
+    dbfile_getsline(lbuf,sizeof(lbuf),fname,lno);
+
+    if (!lbuf[0] || isspace(lbuf[0] || lbuf[0]=='#'))
+      continue;
+
+    rej= mmalloc(sizeof(*rej));
+    rej->next= rejections;
+    rej->fname= fname;
+    rej->lno= lno;
+
+    const char *err;
+    int erroffset;
+    rej->re= pcre_compile(lbuf, PCRE_NO_AUTO_CAPTURE|PCRE_UTF8,
+                         &err, &erroffset, 0);
+    if (!rej->re) {
+      char *what= masprintf("invalid regexp at offset %d: %s\n",
+                           erroffset, err);
+      dbfile_assertfail(fname, lno, what);
+    }
+    debugf("OCR LOADED REJECTION %s:%d `%s' %p\n", fname,lno, lbuf, rej->re);
+
+    rejections= rej;
+  }
+  sysassert(feof(dbfile));
+  dbfile_close();
+}
+  
+static int should_reject(OcrReader *rd) {
+  static int rejections_loaded;
+  int ovector[30];
+
+  if (!rejections_loaded) {
+    fetch_with_rsync("reject");
+    load_rejections("master");
+    load_rejections("local");
+    rejections_loaded=1;
+  }
+
+  debugf("[OCR REJECTION `%s'%d", rd->result, rd->lresult);
+  Rejection *rej;
+  for (rej=rejections; rej; rej=rej->next) {
+    debugf(" (%p)", rej);
+
+    int res= pcre_exec(rej->re, 0, rd->result,rd->lresult, 0,
+                      0, ovector, ARRAYSIZE(ovector));
+    if (res==PCRE_ERROR_NOMATCH) continue;
+    sysassert(res>0);
+
+    debugf(" MATCH]\n");
+    fprintf(stderr,"Rejecting OCR result `%s' (due to %s:%d)\n",
+           rd->result, rej->fname,rej->lno);
+    return 1;
+  }
+  debugf(" OK]");
+  return 0;
+}
+
 static void cu_pr_ctxmap(FILE *resolver, unsigned ctxmap) {
   fprintf(resolver,"{");
   const char *spc="";
@@ -243,7 +323,10 @@ static void callout_unknown(OcrReader *rd, int w, const Pixcol cols[],
   const char *p;
 
   FILE *resolver= resolve_start();
-  if (!resolver || !(o_flags & ff_editcharset))
+  if (!resolver ||
+      !((o_flags & ff_charset_edit) ||
+       ((o_flags & ff_charset_allowedit) &&
+        (o_flags & ff_charset_havelocal))))
     fatal("OCR failed - unrecognised characters or ligatures.\n"
          "Character set database needs to be updated or augmented.\n"
          "See README.charset.\n");
@@ -291,6 +374,18 @@ static void add_result(OcrReader *rd, const char *s, int l, int r,
   rd->results[rd->nresults].match= match;
   rd->results[rd->nresults].ctxmap= ctxmap;
   rd->nresults++;
+
+  if (!s) return; /* just the sentinel for the caller */
+
+  int sl= strlen(s);
+  int newlresult= rd->lresult + sl;
+  if (newlresult >= rd->aresult) {
+    rd->aresult= (newlresult << 1) + 1;
+    rd->result= mrealloc(rd->result, rd->aresult);
+  }
+  memcpy(rd->result + rd->lresult, s, sl);
+  rd->lresult= newlresult;
+  rd->result[rd->lresult]= 0;
 }
 
 
@@ -427,6 +522,8 @@ OcrResultGlyph *ocr(OcrReader *rd, OcrCellType ct, int w,
   nspaces=- w;
   fca.ctxmap= ct->initial;
   rd->nresults=0;
+  rd->lresult=0;
+  rd->result[0]=0;
   debugf("OCR h=%d w=%d",rd->h,w);
   for (x=0; x<w; x++) debugf(" "PIXCOL_PRFMT, PIXCOL_PRVAL(cols[x]));
   debugf("\n");
@@ -463,6 +560,10 @@ OcrResultGlyph *ocr(OcrReader *rd, OcrCellType ct, int w,
     if (match) {
       debugf(" || YES");
       add_result(rd, match->str, x, match_rx, match_ctxi, fca.ctxmap);
+      if (should_reject(rd)) {
+       callout_unknown(rd, w,cols, match_rx+1,match_rx, 0);
+       goto restart;
+      }
       x= match_rx+1;
       if (match->match) fca.ctxmap= ct->midword;
       else debugf(" (empty)");
@@ -503,7 +604,12 @@ OcrReader *ocr_init(int h) {
 
   rd= mmalloc(sizeof(*rd));
   memset(rd,0,sizeof(*rd));
+
   rd->h= h;
+
+  rd->aresult= 10;
+  rd->result= mmalloc(rd->aresult);
+
   readdb(rd);
   return rd;
 }