From ebd7e30dd682685e1f4390718ec2797b8e902768 Mon Sep 17 00:00:00 2001 From: ian Date: Mon, 16 Jan 2006 01:06:32 +0000 Subject: [PATCH] wip is good --- cdb/cdb.tct | 16 ++- cdb/chiark_tcl_cdb.h | 4 + cdb/writeable.c | 242 +++++++++++++++++++++++++------------------ 3 files changed, 156 insertions(+), 106 deletions(-) diff --git a/cdb/cdb.tct b/cdb/cdb.tct index 0999af9..05d818b 100644 --- a/cdb/cdb.tct +++ b/cdb/cdb.tct @@ -58,10 +58,15 @@ Table cdbwr Cdbwr_SubCommand open 0 pathb string on_info obj + ?maxage int + # >0 means entries start with a 16-hex-digit + # time_t and a single space + # this time_t _is_ part of the value as seen ! => iddata(&cdbtcl_rwdatabases) open-okjunk RWSCF_OKJUNK pathb string on_info obj + ?maxage int => iddata(&cdbtcl_rwdatabases) # on_info ...: # on_info open-clean @@ -73,21 +78,24 @@ Table cdbwr Cdbwr_SubCommand # on_info close lookup 0 db iddata(&cdbtcl_rwdatabases) - key obj + key string ?def obj => obj lookup-hb 0 db iddata(&cdbtcl_rwdatabases) - key hb + key string ?def hb => hb + delete 0 + db iddata(&cdbtcl_rwdatabases) + key string update 0 db iddata(&cdbtcl_rwdatabases) - key obj + key string value obj update-hb 0 db iddata(&cdbtcl_rwdatabases) - key hb + key string value hb compact-force 0 db iddata(&cdbtcl_rwdatabases) diff --git a/cdb/chiark_tcl_cdb.h b/cdb/chiark_tcl_cdb.h index ff5424b..99bcc03 100644 --- a/cdb/chiark_tcl_cdb.h +++ b/cdb/chiark_tcl_cdb.h @@ -5,9 +5,13 @@ #define CHIARK_TCL_CDB_H #include +#include #include #include +#include +#include + #include #include "hbytes.h" diff --git a/cdb/writeable.c b/cdb/writeable.c index 15ffed2..9f692d5 100644 --- a/cdb/writeable.c +++ b/cdb/writeable.c @@ -1,26 +1,27 @@ /**/ -#include - #include "chiark_tcl_cdb.h" +#define ftello ftell +#define fseeko fseek +/*---------- Forward declarations ----------*/ struct ht_forall_ctx; - +/*---------- Useful routines ----------*/ static void maybe_close(int fd) { if (fd>=0) close(fd); } -#define PE do{ \ - rc= cht_posixerr(ip, errno, "failed to " PE); goto x_rc; \ +#define PE(m) do{ \ + rc= cht_posixerr(ip, errno, "failed to " m); goto x_rc; \ }while(0) /*---------- Pathbuf ----------*/ -typedef struct { +typedef struct Pathbuf { char *buf, *sfx; } Pathbuf; @@ -47,75 +48,64 @@ static void pathbuf_free(Pathbuf *pb) { typedef struct HashTable { Tcl_HashTable t; + Byte padding[128]; /* allow for expansion by Tcl, urgh */ Byte confound[16]; } HashTable; -typedef struct HashDatum { +typedef struct HashValue { int len; Byte data[1]; -} HashDatum; +} HashValue; -static unsigned int hkt_hash(Tcl_HashTable *tht, void *key_v) { - HashTable *ht= (void*)tht; - const HashDatum *key= key_v; - - struct md4_ctx mdc; - Byte mdr[MD4_DIGEST_SIZE]; - - md4_init(&mdc); - md4_update(&mdc, sizeof(ht->confound), ht->confound); - md4_update(&mdc, key->len, key->data); - md4_digest(&mdc, sizeof(mdr), mdr); - assert(sizeof(int) <= MD4_DIGEST_SIZE); - return *(int*)md4; -} - -static const struct Tcl_HashKeyType ht_keytype= { - TCL_HASH_KEY_TYPE_VERSION, 0, - htk_hash, htk_compare, 0, 0 -} - -static HashDatum *htd_prep(int len) { - HashDatum *hd; - hd= TALLOC(offsetof(HashDatum, data) + len); +static HashValue *htv_prep(int len) { + HashValue *hd; + hd= TALLOC((hd->data - (Byte*)hd) + len); hd->len= len; } -static Byte *htd_fillptr(HashDatum *hd) { +static Byte *htv_fillptr(HashValue *hd) { return hd->data; } -static void htd_fill(HashDatum *hd, const Byte *data) { +static void htv_fill(HashValue *hd, const Byte *data) { memcpy(hd->data, data, hd->len); } -static int ht_setup(HashTable *ht) { - rc= cht_get_urandom(ip, ht->confound, sizeof(ht->confound)); - if (rc) return rc; - - Tcl_InitCustomHashTable(&ht->t, TCL_CUSTOM_PTR_KEYS, &ht_keytype); - return TCL_OK; +static void ht_setup(HashTable *ht) { + Tcl_InitHashTable(&ht->t, TCL_STRING_KEYS); } -static void ht_update(HashTable *ht, HashDatum *key_eat, HashDatum *val_eat) { +static void ht_update(HashTable *ht, const char *key, HashValue *val_eat) { Tcl_HashEntry *he; int new; - he= Tcl_CreateHashEntry(&ht->t, key_eat, &new); - /* eats the key! (since the data structure owns the memory) */ + he= Tcl_CreateHashEntry(&ht->t, (char*)key, &new); if (!new) TFREE(Tcl_GetHashValue(he)); + Tcl_SetHashValue(he, val_eat); + /* eats the value since the data structure owns the memory */ +} +static void ht_maybeupdate(HashTable *ht, const char *key, + HashValue *val_eat) { + /* like ht_update except does not overwrite existing values */ + Tcl_HashEntry *he; + int new; + + he= Tcl_CreateHashEntry(&ht->t, (char*)key, &new); + if (!new) { TFREE(val_eat); return; } Tcl_SetHashValue(he, val_eat); } static int ht_forall(HashTable *ht, - int (*fn)(const HashDatum *key, const HashDatum *val, + int (*fn)(const char *key, const HashValue *val, struct ht_forall_ctx *ctx), struct ht_forall_ctx *ctx) { /* Returns first nonzero value returned by any call to fn, or 0. */ Tcl_HashSearch sp; Tcl_HashEntry *he; - HashDatum *key, *val; + const char *key; + HashValue *val; + int r; for (he= Tcl_FirstHashEntry(&ht->t, &sp); he; - he= Tcl_NextHashEntry(&ht->t, &sp)) { + he= Tcl_NextHashEntry(&sp)) { val= Tcl_GetHashValue(he); if (!val->len) continue; @@ -127,10 +117,23 @@ static int ht_forall(HashTable *ht, return 0; } +static void ht_destroy(HashTable *ht) { + Tcl_HashSearch sp; + Tcl_HashEntry *he; + + for (he= Tcl_FirstHashEntry(&ht->t, &sp); + he; + he= Tcl_NextHashEntry(&sp)) { + TFREE(Tcl_GetHashValue(he)); + } + Tcl_DeleteHashTable(&ht->t); +} + /*---------- Rw data structure ----------*/ typedef struct { - int ix, autocompact; + int ix, autocompact, maxage; +fixme implement maxage during compact; int cdb_fd, lock_fd; struct cdb cdb; /* valid iff cdb_fd >= 0 */ off_t cdb_bytes; /* valid iff cdb_fd >= 0 */ @@ -140,13 +143,11 @@ typedef struct { off_t mainsz; } Rw; -static void destroy_cdbrw_idtabcb(Tcl_Interp *ip, void *val) { abort(); } - -int rw_close(Interp *ip, Rw *rw) { +static int rw_close(Tcl_Interp *ip, Rw *rw) { int rc, r; rc= TCL_OK; - ht_destroy(rw); + ht_destroy(&rw->logincore); maybe_close(rw->cdb_fd); maybe_close(rw->lock_fd); @@ -159,36 +160,37 @@ int rw_close(Interp *ip, Rw *rw) { pathbuf_free(&rw->pbsome); pathbuf_free(&rw->pbother); TFREE(rw); return rc; -}; -void destroy_cdbrw_idtabcb(Interp *ip, Rw *rw) { rw_close(0,rw); } +} +static void destroy_cdbrw_idtabcb(Tcl_Interp *ip, void *rw) { rw_close(0,rw); } const IdDataSpec cdbtcl_rwdatabases= { "cdb-rwdb", "cdb-openrwdatabases-table", destroy_cdbrw_idtabcb }; /*---------- File handling ----------*/ -int acquire_lock(Tcl_Interp *ip, PathBuf *pb, int *lockfd_r) { +static int acquire_lock(Tcl_Interp *ip, Pathbuf *pb, int *lockfd_r) { /* *lockfd_r must be -1 on entry. If may be set to >=0 even * on error, and must be closed by the caller. */ - mode_t umask, lockmode; + mode_t um, lockmode; struct flock fl; + int r; - umask= umask(~(mode_t)0); - umask(umask); + um= umask(~(mode_t)0); + umask(um); - lockmode= 0666 & ~((umask & 0444)>>1); + lockmode= 0666 & ~((um & 0444)>>1); /* Remove r where umask would remove w; * eg umask intending 0664 here gives 0660 */ - *lockfd_r= open(pathbuf_sfx(".lock"), O_RDONLY|O_CREAT, lockmode); + *lockfd_r= open(pathbuf_sfx(pb,".lock"), O_RDONLY|O_CREAT, lockmode); if (*lockfd_r < 0) return cht_posixerr(ip, errno, "could not open/create lockfile"); fl.l_type= F_WRLCK; fl.l_whence= SEEK_SET; fl.l_start= 0; - fl.l_end= 0; + fl.l_len= 0; fl.l_pid= getpid(); r= fcntl(*lockfd_r, F_SETLK, &fl); @@ -225,8 +227,8 @@ static int readlognum(FILE *f, int delim, int *num_r) { } static int readstorelogrecord(FILE *f, HashTable *ht, - void (*updatefn)(HashTable*, HashDatum*, - HashDatum*)) { + void (*updatefn)(HashTable*, const char*, + HashValue*)) { /* returns: * 0 for OK * -1 eof @@ -234,7 +236,9 @@ static int readstorelogrecord(FILE *f, HashTable *ht, * -3 got newline indicating end */ int keylen, vallen; - HashDatum *key, *val; + char *key; + HashValue *val; + int c, rc, r; c= getc(f); if (c==EOF) { if (feof(f)) return -1; return -2; } @@ -244,22 +248,25 @@ static int readstorelogrecord(FILE *f, HashTable *ht, rc= readlognum(f, ',', &keylen); if (rc) return rc; rc= readlognum(f, ':', &vallen); if (rc) return rc; - key= htd_prep(keylen); - val= htd_prep(vallen); + key= TALLOC(keylen+1); + val= htv_prep(vallen); - r= fread(htd_fillptr(key), 1,keylen, f); + r= fread(key, 1,keylen, f); if (r!=keylen) goto x2_free_keyval; + if (memchr(key,0,keylen)) goto x2_free_keyval; + key[keylen]= 0; c= getc(f); if (c!='-') goto x2_free_keyval; c= getc(f); if (c!='>') goto x2_free_keyval; - r= fread(htd_fillptr(val), 1,vallen, f); + r= fread(htv_fillptr(val), 1,vallen, f); if (r!=vallen) goto x2_free_keyval; updatefn(ht, key, val); + TFREE(key); return TCL_OK; -x2_free_keyval; + x2_free_keyval: TFREE(val); TFREE(key); return -2; @@ -271,19 +278,19 @@ int cht_do_cdbwr_create_empty(ClientData cd, Tcl_Interp *ip, const char *pathb) { static const char *const toremoves[]= { ".main", ".cdb", ".log", ".tmp", 0 - } + }; Pathbuf pb; - int lock_fd=-1, fd=-1, rc; + int lock_fd=-1, fd=-1, rc, r; const char *const *toremove; pathbuf_init(&pb, pathb); rc= acquire_lock(ip, &pb, &lock_fd); if (rc) goto x_rc; - fd= open(pathbuf_sfx(".main"), O_RDWR|O_CREAT|O_EXCL, 0666); + fd= open(pathbuf_sfx(&pb, ".main"), O_RDWR|O_CREAT|O_EXCL, 0666); if (fd <= 0) PE("create new database file"); - for (toremoves=toremove; *toremove; toremove++) { + for (toremove=toremoves; *toremove; toremove++) { r= remove(*toremove); if (r && errno != ENOENT) PE("delete possible spurious file during creation"); @@ -298,32 +305,60 @@ int cht_do_cdbwr_create_empty(ClientData cd, Tcl_Interp *ip, return rc; } +/*---------- Info callbacks ----------*/ + +static int infocbv3(Tcl_Interp *ip, Rw *rw, const char *arg1, + const char *arg2fmt, const char *arg3, va_list al) { + abort(); +} + +static int infocb3(Tcl_Interp *ip, Rw *rw, const char *arg1, + const char *arg2fmt, const char *arg3, ...) { + int rc; + va_list al; + va_start(al, arg3); + rc= infocbv3(ip,rw,arg1,arg2fmt,arg3,al); + va_end(al); + return rc; +} + +static int infocb(Tcl_Interp *ip, Rw *rw, const char *arg1, + const char *arg2fmt, ...) { + int rc; + va_list al; + va_start(al, arg2fmt); + rc= infocbv3(ip,rw,arg1,arg2fmt,0,al); + va_end(al); + return rc; +} + /*---------- Opening ----------*/ -int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, - const char *pathb, Tcl_Obj *on_info, void **result) { +int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, const char *pathb, + Tcl_Obj *on_info, int maxage, void **result) { const Cdbwr_SubCommand *subcmd= cd; - int rc, mainfd=-1; + int r, rc, mainfd=-1; Rw *rw; struct stat stab; - fpos_t logrecstart; + off_t logrecstart, logjunkpos; rw= TALLOC(sizeof(*rw)); - rc= ht_setup(&rw->logincore); if (rc) { TFREE(rw); return rc; } + ht_setup(&rw->logincore); rw->cdb_fd= rw->lock_fd= -1; rw->logfile= 0; - pathbuf_init(&rw->pbsome, &pathb); - pathbuf_init(&rw->pbother, &pathb); + rw->maxage= maxage; + pathbuf_init(&rw->pbsome, pathb); + pathbuf_init(&rw->pbother, pathb); rw->autocompact= 1; mainfd= open(pathbuf_sfx(&rw->pbsome,".main"), O_RDONLY); if (mainfd<0) PE("open exist3ing database file .main"); rc= acquire_lock(ip, &rw->pbsome, &rw->lock_fd); if (rc) goto x_rc; - r= stat(mainfd, &stab); if (r) PE("fstat .main"); + r= fstat(mainfd, &stab); if (r) PE("fstat .main"); rw->mainsz= stab.st_size; rw->cdb_fd= open(pathbuf_sfx(&rw->pbsome,".cdb"), O_RDONLY); - if (fd>=0) { + if (rw->cdb_fd >=0) { r= cdb_init(&rw->cdb, rw->cdb_fd); if (r) { rc= cht_posixerr(ip, errno, "failed to initialise cdb reader"); @@ -349,13 +384,13 @@ int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, } else { /* rw->logfile */ r= fstat(fileno(rw->logfile), &stab); if (r==-1) PE("fstat .log during open"); - rc= infocb(rw, "open-dirty-start", "log=%luby", + rc= infocb(ip, rw, "open-dirty-start", "log=%luby", (unsigned long)stab.st_size); if (rc) goto x_rc; for (;;) { - r= fgetpos(rw->logfile, &logrecstart); - if (r) PE("fgetpos .log during (dirty) open"); + logrecstart= ftello(rw->logfile); + if (logrecstart < 0) PE("ftello .log during (dirty) open"); r= readstorelogrecord(rw->logfile, &rw->logincore, ht_update); if (ferror(rw->logfile)) { rc= cht_posixerr(ip, errno, "error reading .log during (dirty) open"); @@ -365,27 +400,27 @@ int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, break; } else if (r==-2 || r==-3) { char buf[100]; - r= fgetpos(rw->logfile, &logjunkpos); - if (r) PE("fgetpos .log during report of junk in dirty open"); + logjunkpos= ftello(rw->logfile); + if(logjunkpos<0) PE("ftello .log during report of junk in dirty open"); snprintf(buf,sizeof(buf), "CDB SYNTAX LOG %lu %lu", - (unsigned long)junkpos, (unsigned long)logrecstart); + (unsigned long)logjunkpos, (unsigned long)logrecstart); if (!(subcmd->flags & RWSCF_OKJUNK)) { Tcl_SetObjErrorCode(ip, Tcl_NewStringObj(buf,-1)); - snprintf(buf,sizeof(buf),"%lu",(unsigned long)junkpos); + snprintf(buf,sizeof(buf),"%lu",(unsigned long)logjunkpos); Tcl_ResetResult(ip); Tcl_AppendResult(ip, "syntax error (junk) in .log during" " (dirty) open, at file position ", buf, (char*)0); rc= TCL_ERROR; goto x_rc; } - rc= infocb(rw, "open-dirty-junk", "errorfpos=%luby {%s}", - (unsigned long)logjunkpos, buf); + rc= infocb3(ip, rw, "open-dirty-junk", "errorfpos=%luby", buf, + (unsigned long)logjunkpos); if (rc) goto x_rc; - r= fsetpos(rw->logfile, logrecstart); - if (r) PE("failed to fsetpos .log before junk during dirty open"); + r= fseeko(rw->logfile, logrecstart, SEEK_SET); + if (r) PE("failed to fseeko .log before junk during dirty open"); r= ftruncate(fileno(rw->logfile), logrecstart); if (r) PE("ftruncate .log to chop junk during dirty open"); @@ -408,20 +443,23 @@ int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, /*---------- Compacting ----------*/ -static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsz) { +static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsize) { /* creates new .cdb and .main * closes logfile * leaves .log with old data * leaves cdb fd open onto old db * leaves logincore full of crap */ - int rc; + int r, rc; int cdbfd, cdbmaking; + off_t errpos; + char buf[100]; + struct ht_forall_ctx { struct cdb_make cdbm; FILE *mainfile; int count; - } addctx; + } a; a.mainfile= 0; cdbfd= -1; @@ -438,7 +476,7 @@ static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsz) { /* merge unsuperseded records from main into hash table */ - a.mainfile= fopen(pathbuf_sfx(&rw_pbsome,".main"), "r"); + a.mainfile= fopen(pathbuf_sfx(&rw->pbsome,".main"), "r"); if (!a.mainfile) PE("failed to open .main for reading during compact"); for (;;) { @@ -449,11 +487,11 @@ static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsz) { if (r==-3) { break; } else if (r==-1 || r==-2) { - r= fgetpos(a.mainfile, &errpos); - if (r) PE("fgetpos .main during report of syntax error"); - snprintf(buf,sizeof(buf), "CDB SYNTAX MAIN %lu", (uunsigned long)errpos); + errpos= ftello(a.mainfile); + if (errpos<0) PE("ftello .main during report of syntax error"); + snprintf(buf,sizeof(buf), "CDB SYNTAX MAIN %lu", (unsigned long)errpos); Tcl_SetObjErrorCode(ip, Tcl_NewStringObj(buf,-1)); - snprintf(buf,sizeof(buf), "%lu", (uunsigned long)errpos); + snprintf(buf,sizeof(buf), "%lu", (unsigned long)errpos); Tcl_ResetResult(ip); Tcl_AppendResult(ip, "syntax error in .main during" " compact, at file position ", buf, (char*)0); @@ -524,11 +562,11 @@ x_rc: } static void compact_forclose(Tcl_Interp *ip, Rw *rw) { - long logsz; + off_t logsz; int rc; - logsz= ftell(rw->logfile); - if (logsz < 0) PE("ftell logfile (during tidy close)"); + logsz= ftello(rw->logfile); + if (logsz < 0) PE("ftello logfile (during tidy close)"); rc= compact_core(ip, rw, logsz); if (rc) goto x_rc; -- 2.30.2