From 60cd4954d1d3c11c46251f68e2d56f24661a6d67 Mon Sep 17 00:00:00 2001 From: Ian Jackson Date: Thu, 9 Jul 2009 17:38:52 +0100 Subject: [PATCH] Use compressed _master-pixmap.txt.gz --- pctb/Makefile | 2 +- pctb/README.files | 6 ++-- pctb/common.c | 79 ++++++++++++++++++++++++++++++++++++++--------- pctb/common.h | 10 ++++-- pctb/convert.c | 28 ++++++++++------- pctb/convert.h | 3 +- pctb/resolve.c | 2 +- pctb/rgbimage.c | 10 +++--- 8 files changed, 100 insertions(+), 40 deletions(-) diff --git a/pctb/Makefile b/pctb/Makefile index 8223583..4da4677 100644 --- a/pctb/Makefile +++ b/pctb/Makefile @@ -50,6 +50,6 @@ realclean: clean rm -f $(TARGETS) rm -f raw.tsv rm -f _pages.ppm _upload-*.html _commodmap.tsv - rm -f _master-*.txt _local-*.txt + rm -f _master-*.txt _master-*.txt.gz _local-*.txt rm -f ./#pages#.ppm ./#upload-*#.html ./#commodmap#.tsv rm -f ./#master-*#.txt ./#local-*#.txt raw.tsv diff --git a/pctb/README.files b/pctb/README.files index b2384a1..c8d4928 100644 --- a/pctb/README.files +++ b/pctb/README.files @@ -43,8 +43,8 @@ The program reads and writes the following files: character set dictionary is missing the lowercase `y ' glyph. See README.charset. - * _master-char*.txt _local-char*.txt - _master-pixmap.txt _local-pixmap.txt + * _master-char*.txt _local-char*.txt + _master-pixmap.txt.gz _local-pixmap.txt Character set and image dictionaries. For the semantics of the char* files README.charset. There is not currently any accurate @@ -52,7 +52,7 @@ The program reads and writes the following files: _master-*.txt contain the centrally defined and approved data. They are downloaded automatically from the SC PCTB server and - updated each run. You can safely delete this file, if everything + updated each run. You can safely delete these files, if everything is online, if you want to fetch a fresh copy. _local-*.txt are a local copy of your submissions, so that they diff --git a/pctb/common.c b/pctb/common.c index bc1cb8d..2c2a085 100644 --- a/pctb/common.c +++ b/pctb/common.c @@ -42,17 +42,68 @@ void *mrealloc(void *p, size_t sz) { FILE *dbfile; -static const char *path; +static const char *basepath; /* as passed in by caller */ +static pid_t dbzcat; + +int dbfile_gzopen(const char *basepath_spec) { + assert(!dbfile); + + basepath= basepath_spec; + //uncomppath= masprintf("%s (uncompressed)", basepath); + + char *zpath= masprintf("%s.gz", basepath); + int zfd= open(zpath, O_RDONLY); + free(zpath); + + if (zfd<0) { sysassert(errno==ENOENT); return 0; } + + int pipefds[2]; + sysassert(! pipe(pipefds) ); + + sysassert( (dbzcat=fork()) != -1 ); + if (!dbzcat) { + sysassert( dup2(zfd,0)==0 ); + sysassert( dup2(pipefds[1],1)==1 ); + sysassert(! close(zfd) ); + sysassert(! close(pipefds[0]) ); + sysassert(! close(pipefds[1]) ); + execlp("zcat","zcat",(char*)0); + sysassert(!"execlp zcat"); + } + sysassert(! close(zfd) ); + sysassert(! close(pipefds[1]) ); + sysassert( dbfile= fdopen(pipefds[0], "r") ); + + return 1; +} int dbfile_open(const char *tpath) { assert(!dbfile); - path= tpath; - dbfile= fopen(path,"r"); - if (dbfile) return 1; - sysassert(errno==ENOENT); - return 0; + + basepath= tpath; + + dbzcat= -1; + dbfile= fopen(tpath,"r"); + if (!dbfile) { sysassert(errno==ENOENT); return 0; } + return 1; } +void dbfile_close(void) { + if (!dbfile) return; + + sysassert(!ferror(dbfile)); + sysassert(!fclose(dbfile)); + + if (dbzcat != -1) { + char *zcatstr= masprintf("zcat %s.gz", basepath); + waitpid_check_exitstatus(dbzcat,zcatstr,1); + free(zcatstr); + dbzcat= -1; + } + + dbfile= 0; +} + #define dbassertgl(x) ((x) ? (void)0 : dbfile_assertfail(file,line,#x)) void dbfile_getsline(char *lbuf, size_t lbufsz, const char *file, int line) { @@ -66,13 +117,6 @@ void dbfile_getsline(char *lbuf, size_t lbufsz, const char *file, int line) { lbuf[l]= 0; } -void dbfile_close(void) { - if (!dbfile) return; - sysassert(!ferror(dbfile)); - sysassert(!fclose(dbfile)); - dbfile= 0; -} - int dbfile_vscanf(const char *fmt, va_list al) { int r= vfscanf(dbfile,fmt,al); sysassert(!ferror(dbfile)); @@ -88,11 +132,16 @@ int dbfile_scanf(const char *fmt, ...) { } void dbfile_assertfail(const char *file, int line, const char *m) { - if (dbfile) + if (dbzcat) + fatal("Error in dictionary file %s.gz:\n" + " Requirement not met at %s:%d:\n" + " %s", + basepath, file,line, m); + else if (dbfile) fatal("Error in dictionary file %s at byte %ld:\n" " Requirement not met at %s:%d:\n" " %s", - path,(long)ftell(dbfile), file,line, m); + basepath,(long)ftell(dbfile), file,line, m); else fatal("Semantic error in dictionaries:\n" " Requirement not met at %s:%d:\n" diff --git a/pctb/common.h b/pctb/common.h index 210f23a..92ba2da 100644 --- a/pctb/common.h +++ b/pctb/common.h @@ -41,9 +41,12 @@ #include #include #include -#include #include +#include + +#include +#include #include #include @@ -133,7 +136,7 @@ void fatal(const char *fmt, ...) FMT(1,2) NORET; void sysassert_fail(const char *file, int line, const char *what) __attribute__((noreturn)); -void waitpid_check_exitstatus(pid_t pid, const char *what); +void waitpid_check_exitstatus(pid_t pid, const char *what, int sigpipeok); void *mmalloc(size_t sz); @@ -145,7 +148,8 @@ void dbfile_assertfail(const char *file, int line, const char *m) NORET; FILE *dbfile; void dbfile_getsline(char *lbuf, size_t lbufsz, const char *file, int line); -int dbfile_open(const char *tpath); /* 0: ENOENT; 1: worked */ +int dbfile_open(const char *tpath); /* 0: ENOENT; 1: worked */ +int dbfile_gzopen(const char *tpath); /* 0: ENOENT; 1: worked */ void dbfile_close(void); /* idempotent */ int dbfile_scanf(const char *fmt, ...) SCANFMT(1,2); diff --git a/pctb/convert.c b/pctb/convert.c index 8386c16..6bc9b2c 100644 --- a/pctb/convert.c +++ b/pctb/convert.c @@ -110,12 +110,13 @@ static void run_analysis(void) { EXECLP_HELPER("commod-results-processor", o_outmode_str, (char*)0); } - waitpid_check_exitstatus(processor, "output processor/uploader"); + waitpid_check_exitstatus(processor, "output processor/uploader", 0); fclose(tf); progress_log("all complete."); } -void fetch_with_rsync(const char *stem) { +static void rsync_core(const char *stem, const char *suffix, + const char *zopt) { pid_t fetcher; sysassert( (fetcher= fork()) != -1 ); @@ -124,19 +125,23 @@ void fetch_with_rsync(const char *stem) { if (!rsync) rsync= "rsync"; const char *src= getenv("YPPSC_PCTB_DICT_UPDATE"); - char *remote= masprintf("%s/master-%s.txt", src, stem); - char *local= masprintf("_master-%s.txt", stem); + char *remote= masprintf("%s/master-%s.txt%s", src, stem, suffix); + char *local= masprintf("_master-%s.txt%s", stem, suffix); if (DEBUGP(rsync)) fprintf(stderr,"executing rsync to fetch %s to %s\n",remote,local); - execlp(rsync, "rsync", - DEBUGP(rsync) ? "-zvLt" : "-zLt", - "--",remote,local,(char*)0); + char *opts= masprintf("-Lt%s%s", + zopt, + DEBUGP(rsync) ? "v" : ""); + execlp(rsync, "rsync",opts,"--",remote,local,(char*)0); sysassert(!"exec rsync failed"); } - waitpid_check_exitstatus(fetcher, "rsync"); + waitpid_check_exitstatus(fetcher, "rsync", 0); } +void fetch_with_rsync_gz(const char *stem) { rsync_core(stem,".gz",""); } +void fetch_with_rsync(const char *stem) { rsync_core(stem,"","z"); } + static void set_server(const char *envname, const char *defprotocol, const char *defvalue, const char *defvalue_test, const char *userspecified, @@ -411,7 +416,7 @@ void sysassert_fail(const char *file, int line, const char *what) { _exit(16); } -void waitpid_check_exitstatus(pid_t pid, const char *what) { +void waitpid_check_exitstatus(pid_t pid, const char *what, int sigpipeok) { pid_t got; int st; for (;;) { @@ -426,8 +431,9 @@ void waitpid_check_exitstatus(pid_t pid, const char *what) { fatal("%s failed with nonzero exit status %d", what, WEXITSTATUS(st)); } else if (WIFSIGNALED(st)) { - fatal("%s died due to signal %s%s", what, - strsignal(WTERMSIG(st)), WCOREDUMP(st)?" (core dumped)":""); + if (!sigpipeok || WTERMSIG(st) != SIGPIPE) + fatal("%s died due to signal %s%s", what, + strsignal(WTERMSIG(st)), WCOREDUMP(st)?" (core dumped)":""); } else { fatal("%s gave strange wait status %d", what, st); } diff --git a/pctb/convert.h b/pctb/convert.h index ffa80eb..0dc2255 100644 --- a/pctb/convert.h +++ b/pctb/convert.h @@ -85,7 +85,8 @@ void analyse(FILE *tsv_output); /*----- from convert.c -----*/ extern FILE *screenshot_file; -extern void fetch_with_rsync(const char *stem); +void fetch_with_rsync(const char *stem); +void fetch_with_rsync_gz(const char *stem); void vwarning(const char *fmt, va_list) FMT(1,0); void warning(const char *fmt, ...) FMT(1,2); diff --git a/pctb/resolve.c b/pctb/resolve.c index 19615f3..e89d205 100644 --- a/pctb/resolve.c +++ b/pctb/resolve.c @@ -83,7 +83,7 @@ void resolve_finish(void) { } if (r==0) { - waitpid_check_exitstatus(resolver_pid, "dictionary-manager"); + waitpid_check_exitstatus(resolver_pid, "dictionary-manager", 0); fclose(resolver); close(resolver_done); resolver= 0; diff --git a/pctb/rgbimage.c b/pctb/rgbimage.c index dfcd180..81eddf9 100644 --- a/pctb/rgbimage.c +++ b/pctb/rgbimage.c @@ -50,9 +50,9 @@ static int identify1(const RgbImage *base, Rect portion, char result[MAXIMGIDENT], const char *what, - const char *which) { + const char *which, int (*opener)(const char *fn)) { char *dbfile_name= masprintf("_%s-pixmap.txt",which); - if (!dbfile_open(dbfile_name)) + if (!opener(dbfile_name)) goto not_found; #define FGETSLINE (dbfile_getsline(result,MAXIMGIDENT,__FILE__,__LINE__)) @@ -111,8 +111,8 @@ static int identify1(const RgbImage *base, Rect portion, static int identify(const RgbImage *base, Rect portion, char result[MAXIMGIDENT], const char *what) { - return identify1(base,portion,result,what, "master") || - identify1(base,portion,result,what, "local"); + return identify1(base,portion,result,what, "master", dbfile_gzopen) || + identify1(base,portion,result,what, "local", dbfile_open); } void fwrite_ppmraw(FILE *f, const RgbImage *ri) { @@ -147,7 +147,7 @@ void identify_rgbimage(const RgbImage *base, Rect portion, if (!synced) { if (o_flags & ff_dict_fetch) - fetch_with_rsync("pixmap"); + fetch_with_rsync_gz("pixmap"); synced++; } -- 2.30.2