#include "convert.h"
-void debug_flush(void) {
- sysassert(!ferror(debug));
- sysassert(!fflush(debug));
-}
-
const char *get_vardir(void) { return "."; }
const char *get_libdir(void) { return "."; }
enum outmodekind {
- omk_unset, omk_upload, omk_str, omk_raw, omk_none
+ omk_unset, omk_raw, omk_str, omk_upload_yarrg, omk_upload_pctb
};
-static enum outmodekind o_outmode_kind;
-static const char *o_outmode_str= 0;
+
+typedef struct {
+ enum outmodekind kind; /* unset is sentinel */
+ const char *str; /* non-0 iff not unset or raw */
+} OutputMode;
+
+static OutputMode o_outmodes[10];
static char *o_screenshot_fn;
-static const char *o_serv_pctb, *o_serv_dict_fetch, *o_serv_dict_submit;
+static const char *o_serv_pctb, *o_serv_yarrg;
+static const char *o_serv_dict_fetch, *o_serv_dict_submit;
const char *o_resolver= "./dictionary-manager";
FILE *screenshot_file;
static void run_analysis(void) {
FILE *tf;
+ OutputMode *om;
sysassert( tf= tmpfile() );
progress("running recognition...");
analyse(tf);
if (o_flags & ff_upload) {
- if (npages<=1)
+ if (o_flags & ff_singlepage)
fatal("Recognition successful, but refusing to upload partial data\n"
" (--single-page specified). Specify an output mode?");
}
- sysassert( fseek(tf,0,SEEK_SET) == 0);
+ for (om=o_outmodes; om->kind != omk_unset; om++) {
+ sysassert( fseek(tf,0,SEEK_SET) == 0);
- progress_log("processing results (--%s)...", o_outmode_str);
- pid_t processor;
- sysassert( (processor= fork()) != -1 );
+ progress_log("processing results (--%s)...", om->str);
+ pid_t processor;
+ sysassert( (processor= fork()) != -1 );
- if (!processor) {
- sysassert( dup2(fileno(tf),0) ==0 );
- EXECLP_HELPER("commod-results-processor", o_outmode_str, (char*)0);
- }
+ if (!processor) {
+ sysassert( dup2(fileno(tf),0) ==0 );
+ if (om->kind==omk_raw) {
+ execlp("cat","cat",(char*)0);
+ sysassert(!"execute cat");
+ } else {
+ EXECLP_HELPER("commod-results-processor", om->str, (char*)0);
+ }
+ }
- waitpid_check_exitstatus(processor, "output processor/uploader", 0);
+ waitpid_check_exitstatus(processor, "output processor/uploader", 0);
+ }
+
fclose(tf);
progress_log("all complete.");
}
void fetch_with_rsync_gz(const char *stem) { rsync_core(stem,".gz",""); }
void fetch_with_rsync(const char *stem) { rsync_core(stem,"","z"); }
+static void get_timestamp(void) {
+ FILE *tf;
+ pid_t child;
+ sysassert( tf= tmpfile() );
+
+ sysassert( (child= fork()) != -1 );
+ if (!child) {
+ sysassert( dup2(fileno(tf),1)==1 );
+ EXECLP_HELPER("database-info-fetch","timestamp",(char*)0);
+ }
+ waitpid_check_exitstatus(child,"timestamp request",0);
+
+ sysassert( fseek(tf,0,SEEK_SET) == 0 );
+ static char lbuf[30];
+ int l= fread(lbuf,1,sizeof(lbuf),tf);
+ sysassert( !ferror(tf) );
+ assert( feof(tf) );
+ assert( l>1 );
+ l--;
+ assert( lbuf[l]=='\n' );
+ lbuf[l]= 0;
+
+ sysassert(! setenv("YPPSC_DATA_TIMESTAMP",lbuf,1) );
+ fclose(tf);
+}
+
static void set_server(const char *envname, const char *defprotocol,
const char *defvalue, const char *defvalue_test,
const char *userspecified,
sysassert(! setenv(envname,value,1) );
}
+static void outputmode(enum outmodekind kind, const char *str) {
+ OutputMode *om= o_outmodes;
+ OutputMode *sentinel= o_outmodes + ARRAYSIZE(o_outmodes) - 1;
+ for (;;) {
+ if (om==sentinel) badusage("too many output modes specified");
+ if (!om->kind) break;
+ om++;
+ }
+ om->kind= kind;
+ om->str= str;
+}
+
+static void outputmode_uploads(void) {
+ outputmode(omk_upload_yarrg, "upload-yarrg");
+ outputmode(omk_upload_pctb, "upload-pctb");
+}
+
int main(int argc, char **argv) {
const char *arg;
sysassert( setlocale(LC_MESSAGES,"") );
sysassert( setlocale(LC_CTYPE,"en_GB.UTF-8") ||
+ setlocale(LC_CTYPE,"en_US.UTF-8") ||
setlocale(LC_CTYPE,"en.UTF-8") );
#define ARGVAL ((*++argv) ? *argv : \
else if (IS("--dict-read-only")) o_flags &= (~ffs_dict | ff_dict_fetch);
else if (IS("--dict-anon")) o_flags &= ~ff_dict_pirate;
else if (IS("--dict-submit")) o_flags |= ff_dict_fetch|ff_dict_submit;
- else if (IS("--raw-tsv")) o_outmode_kind= omk_raw;
- else if (IS("--upload")) o_outmode_kind= omk_upload;
+ else if (IS("--dict-no-update")) o_flags &= ~ff_dict_fetch; // testing
+ else if (IS("--raw-tsv")) outputmode(omk_raw,0);
+ else if (IS("--upload")) outputmode_uploads();
+ else if (IS("--upload-yarrg")) outputmode(omk_upload_yarrg,arg+2);
+ else if (IS("--upload-pctb")) outputmode(omk_upload_pctb,arg+2);
else if (IS("--arbitrage") ||
IS("--tsv") ||
- IS("--best-prices")) o_outmode_kind=omk_str,
- o_outmode_str=arg+2;
-
+ IS("--best-prices")) outputmode(omk_str,arg+2);
else if (IS("--screenshot-file")||
IS("--screenshots-file")) o_screenshot_fn= ARGVAL;
+ else if (IS("--yarrg-server")) o_serv_yarrg= ARGVAL;
else if (IS("--pctb-server")) o_serv_pctb= ARGVAL;
else if (IS("--dict-submit-server")) o_serv_dict_submit= ARGVAL;
else if (IS("--dict-update-server")) o_serv_dict_fetch= ARGVAL;
/* Consequential changes to options */
if (o_mode & mf_analyse) {
- if (!o_outmode_kind) {
+ if (!o_outmodes[0].kind) {
if (o_flags & ff_printisland) {
- o_outmode_kind= omk_none;
o_flags |= ff_singlepage;
} else {
- o_outmode_kind= omk_upload;
+ outputmode_uploads();
}
}
+ } else {
+ if (o_outmodes[0].kind)
+ badusage("overall mode does not include analysis but output option(s)"
+ " (eg `--%s') specified", o_outmodes[0].str);
+ }
- if (o_outmode_kind==omk_upload) {
- o_flags |= ffs_upload;
- o_outmode_str= "upload";
+ OutputMode *om;
+ for (om=o_outmodes; om->kind; om++) {
+ switch (om->kind) {
+ case omk_upload_yarrg: o_flags |= ffs_upload | ff_use_yarrg; break;
+ case omk_upload_pctb: o_flags |= ffs_upload | ff_use_pctb; break;
+ default: ;
}
}
- /* Defaults */
+ if ((o_flags & (ff_needisland|ff_upload)) &&
+ !(o_flags & (ffm_use)))
+ o_flags |= ffm_use; /* all */
+
+ if (o_serv_yarrg && !o_serv_dict_submit)
+ o_serv_dict_submit= o_serv_yarrg;
+ /* Defaults */
+
+ set_server("YPPSC_YARRG_YARRG",
+ "http://", "upload.yarrg.chiark.net",
+ "upload.yarrg.chiark.net/test",
+ o_serv_yarrg, o_flags & ff_use_yarrg);
+
set_server("YPPSC_YARRG_PCTB",
- "http://", "pctb.ilk.org" /*pctb.crabdance.com*/,
+ "http://", "pctb.crabdance.com",
"pctb.ilk.org",
- o_serv_pctb, o_flags & (ff_needisland|ff_upload));
+ o_serv_pctb, o_flags & ff_use_pctb);
set_server("YPPSC_YARRG_DICT_UPDATE",
- "rsync://", "rsync.pctb.chiark.greenend.org.uk/pctb",
- "rsync.pctb.chiark.greenend.org.uk/pctb/test",
+ "rsync://", "rsync.yarrg.chiark.net/yarrg",
+ "rsync.yarrg.chiark.net/yarrg/test",
o_serv_dict_fetch, o_flags & ff_dict_fetch);
set_server("YPPSC_YARRG_DICT_SUBMIT",
- "http://", "dictup.pctb.chiark.greenend.org.uk",
- "dictup.pctb.chiark.greenend.org.uk/test",
+ "http://", "upload.yarrg.chiark.net",
+ "upload.yarrg.chiark.net/test",
o_serv_dict_submit, o_flags & ff_dict_submit);
if (!o_screenshot_fn)
o_screenshot_fn= masprintf("%s/_pages.ppm.gz", get_vardir());
/* Actually do the work */
+
+ if ((o_flags & ff_upload) && (o_flags & ff_use_yarrg))
+ get_timestamp();
canon_colour_prepare();
printf("%s, %s\n", archipelago, island);
sysassert(! setenv("YPPSC_ISLAND",island,1) );
}
- switch (o_outmode_kind) {
- case omk_upload: case omk_str: run_analysis(); break;
- case omk_raw: analyse(stdout); break;
- case omk_none: break;
- default: abort();
- }
+ if (o_outmodes[0].kind==omk_raw && o_outmodes[1].kind==omk_unset)
+ analyse(stdout);
+ else
+ run_analysis();
}
progress_log("Finished.");
return 0;
}
+FILE *dbfile;
+static const char *basepath; /* as passed in by caller */
+static pid_t dbzcat;
+int dbfile_gzopen(const char *basepath_spec) {
+ assert(!dbfile);
-DEFINE_VWRAPPERF(, progress, )
-DEFINE_VWRAPPERF(, progress_log, )
-DEFINE_VWRAPPERF(, progress_spinner, )
-DEFINE_VWRAPPERF(, warning, )
-DEFINE_VWRAPPERF(, fatal, NORET)
+ basepath= basepath_spec;
-static int last_progress_len;
-
-static void vprogress_core(int spinner, const char *fmt, va_list al) {
- int r;
+ char *zpath= masprintf("%s.gz", basepath);
+ int e= gzopen(zpath, O_RDONLY, &dbfile, &dbzcat, 0);
+ free(zpath);
+ if (e) { errno=e; sysassert(errno==ENOENT); return 0; }
- if (o_quiet) return;
- if (!debug_flags && !isatty(2)) return;
-
- if (last_progress_len)
- putc('\r',stderr);
+ return 1;
+}
- r= vfprintf(stderr,fmt,al);
+int dbfile_open(const char *tpath) {
+ assert(!dbfile);
- if (spinner) {
- putc(spinner,stderr);
- r++;
- }
+ basepath= tpath;
- if (r < last_progress_len) {
- fprintf(stderr,"%*s", last_progress_len - r, "");
- if (!r) putc('\r', stderr);
- else while (last_progress_len-- > r) putc('\b',stderr);
- }
- last_progress_len= r;
+ dbzcat= -1;
+ dbfile= fopen(tpath,"r");
+ if (!dbfile) { sysassert(errno==ENOENT); return 0; }
+ return 1;
+}
- if (ferror(stderr) || fflush(stderr)) _exit(16);
-}
-
-void vprogress(const char *fmt, va_list al) { vprogress_core(0,fmt,al); }
-void vprogress_spinner(const char *fmt, va_list al) {
- static const char spinchars[]="/-\\";
- static int spinner;
-
- vprogress_core(spinchars[spinner],fmt,al);
- spinner++;
- spinner %= (sizeof(spinchars)-1);
+void dbfile_close(void) {
+ gzclose(&dbfile, &dbzcat, basepath);
}
-void vprogress_log(const char *fmt, va_list al) {
- if (o_quiet) return;
-
- progress("");
- vfprintf(stderr,fmt,al);
- putc('\n',stderr);
- fflush(stderr);
+#define dbassertgl(x) ((x) ? (void)0 : dbfile_assertfail(file,line,#x))
+
+void dbfile_getsline(char *lbuf, size_t lbufsz, const char *file, int line) {
+ errno=0;
+ char *s= fgets(lbuf,lbufsz,dbfile);
+ sysassert(!ferror(dbfile));
+ dbassertgl(!feof(dbfile));
+ assert(s);
+ int l= strlen(lbuf);
+ dbassertgl(l>0); dbassertgl(lbuf[--l]=='\n');
+ lbuf[l]= 0;
}
-void vwarning(const char *fmt, va_list al) {
- progress("");
- fputs("Warning: ",stderr);
- vfprintf(stderr,fmt,al);
- fputs("\n",stderr);
- fflush(stderr);
+int dbfile_vscanf(const char *fmt, va_list al) {
+ int r= vfscanf(dbfile,fmt,al);
+ sysassert(!ferror(dbfile));
+ return r;
}
-void vfatal(const char *fmt, va_list al) {
- progress("");
- fputs("\n\nFatal error: ",stderr);
- vfprintf(stderr,fmt,al);
- fflush(stderr);
- fputs("\n\n",stderr);
- _exit(4);
+int dbfile_scanf(const char *fmt, ...) {
+ va_list al;
+ va_start(al,fmt);
+ int r= dbfile_vscanf(fmt,al);
+ va_end(al);
+ return r;
}
-void sysassert_fail(const char *file, int line, const char *what) {
- int e= errno;
- progress("");
- fprintf(stderr,
- "\nfatal operational error:\n"
- " unsuccessful execution of: %s\n"
- " %s:%d: %s\n\n",
- what, file,line, strerror(e));
- _exit(16);
+void dbfile_assertfail(const char *file, int line, const char *m) {
+ if (dbzcat)
+ fatal("Error in dictionary file %s.gz:\n"
+ " Requirement not met at %s:%d:\n"
+ " %s",
+ basepath, file,line, m);
+ else if (dbfile)
+ fatal("Error in dictionary file %s at byte %ld:\n"
+ " Requirement not met at %s:%d:\n"
+ " %s",
+ basepath,(long)ftell(dbfile), file,line, m);
+ else
+ fatal("Semantic error in dictionaries:\n"
+ " Requirement not met at %s:%d:\n"
+ " %s",
+ file,line, m);
}
-void waitpid_check_exitstatus(pid_t pid, const char *what, int sigpipeok) {
- pid_t got;
- int st;
- for (;;) {
- got= waitpid(pid, &st, 0);
- if (pid==-1) { sysassert(errno==EINTR); continue; }
- break;
+int gzopen(const char *zpath, int oflags, FILE **f_r, pid_t *pid_r,
+ const char *gziplevel /* 0 for read; may be 0, or "-1" etc. */) {
+
+ int zfd= open(zpath, oflags, 0666);
+ if (zfd<0) return errno;
+
+ int pipefds[2];
+ sysassert(! pipe(pipefds) );
+
+ int oi,io; const char *cmd; const char *stdiomode;
+ switch ((oflags & O_ACCMODE)) {
+ case O_RDONLY: oi=0; io=1; cmd="gunzip"; stdiomode="r"; break;
+ case O_WRONLY: oi=1; io=0; cmd="gzip"; stdiomode="w"; break;
+ default: abort();
}
- sysassert( got==pid );
-
- if (WIFEXITED(st)) {
- if (WEXITSTATUS(st))
- fatal("%s failed with nonzero exit status %d",
- what, WEXITSTATUS(st));
- } else if (WIFSIGNALED(st)) {
- if (!sigpipeok || WTERMSIG(st) != SIGPIPE)
- fatal("%s died due to signal %s%s", what,
- strsignal(WTERMSIG(st)), WCOREDUMP(st)?" (core dumped)":"");
- } else {
- fatal("%s gave strange wait status %d", what, st);
+
+ sysassert( (*pid_r=fork()) != -1 );
+ if (!*pid_r) {
+ sysassert( dup2(zfd,oi)==oi );
+ sysassert( dup2(pipefds[io],io)==io );
+ sysassert(! close(zfd) );
+ sysassert(! close(pipefds[0]) );
+ sysassert(! close(pipefds[1]) );
+ execlp(cmd,cmd,gziplevel,(char*)0);
+ sysassert(!"execlp gzip/gunzip");
}
+ sysassert(! close(zfd) );
+ sysassert(! close(pipefds[io]) );
+ sysassert( *f_r= fdopen(pipefds[oi], stdiomode) );
+
+ return 0;
}
-char *masprintf(const char *fmt, ...) {
- char *r;
- va_list al;
- va_start(al,fmt);
- sysassert( vasprintf(&r,fmt,al) >= 0);
- sysassert(r);
- va_end(al);
- return r;
+void gzclose(FILE **f, pid_t *p, const char *what) {
+ if (!*f) return;
+
+ sysassert(!ferror(*f));
+ sysassert(!fclose(*f));
+
+ if (*p != -1) {
+ char *process= masprintf("%s (de)compressor",what);
+ waitpid_check_exitstatus(*p,process,1);
+ free(process);
+ *p= -1;
+ }
+
+ *f= 0;
}
~yarrgweb / ypp-sc-tools.db-test.git / blobdiff