X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?p=chiark-tcl.git;a=blobdiff_plain;f=cdb%2Fwriteable.c;h=3613a7b8084642d31c807fde215590f20caf00b1;hp=0074a6126181e2d76378c1ca21099346223ccbaa;hb=486592e95c37e209166ca15f49878a8a8428bcdb;hpb=fad73e0340b64f1a7da862d8f511ce9094e14375 diff --git a/cdb/writeable.c b/cdb/writeable.c index 0074a61..3613a7b 100644 --- a/cdb/writeable.c +++ b/cdb/writeable.c @@ -19,6 +19,8 @@ static void maybe_close(int fd) { rc= cht_posixerr(ip, errno, "failed to " m); goto x_rc; \ }while(0) +/*==================== Subsystems and subtypes ====================*/ + /*---------- Pathbuf ----------*/ typedef struct Pathbuf { @@ -61,13 +63,11 @@ static HashValue *htv_prep(int len) { HashValue *hd; hd= TALLOC((hd->data - (Byte*)hd) + len); hd->len= len; + return hd; } static Byte *htv_fillptr(HashValue *hd) { return hd->data; } -static void htv_fill(HashValue *hd, const Byte *data) { - memcpy(hd->data, data, hd->len); -} static void ht_setup(HashTable *ht) { Tcl_InitHashTable(&ht->t, TCL_STRING_KEYS); @@ -92,12 +92,22 @@ static void ht_maybeupdate(HashTable *ht, const char *key, Tcl_SetHashValue(he, val_eat); } +static fixme ht_lookup(HashTable *ht, const char *key) { + Tcl_HashEntry *he; + const HashValue *htv; + + he= Tcl_FindHashEntry(ht, key); + if (!he) return notfound; + + htv= Tcl_GetHashValue(he); + +} + static int ht_forall(HashTable *ht, - int (*fn)(const char *key, const HashValue *val, + int (*fn)(const char *key, HashValue *val, struct ht_forall_ctx *ctx), struct ht_forall_ctx *ctx) { - /* Returns first positive value returned by any call to fn, or 0. - * If fn returns -1, key is deleted. */ + /* Returns first positive value returned by any call to fn, or 0. */ Tcl_HashSearch sp; Tcl_HashEntry *he; const char *key; @@ -125,23 +135,25 @@ static void ht_destroy(HashTable *ht) { for (he= Tcl_FirstHashEntry(&ht->t, &sp); he; he= Tcl_NextHashEntry(&sp)) { + /* ht_forall skips empty (deleted) entries so is no good for this */ TFREE(Tcl_GetHashValue(he)); } Tcl_DeleteHashTable(&ht->t); } +/*==================== Existential ====================*/ + /*---------- Rw data structure ----------*/ -typedef struct { - int ix, autocompact, maxage; -fixme implement maxage during compact; +typedef struct Rw { + int ix, autocompact; int cdb_fd, lock_fd; struct cdb cdb; /* valid iff cdb_fd >= 0 */ - off_t cdb_bytes; /* valid iff cdb_fd >= 0 */ FILE *logfile; HashTable logincore; Pathbuf pbsome, pbother; off_t mainsz; + ScriptToInvoke on_info, on_lexminval; } Rw; static int rw_close(Tcl_Interp *ip, Rw *rw) { @@ -149,6 +161,7 @@ static int rw_close(Tcl_Interp *ip, Rw *rw) { rc= TCL_OK; ht_destroy(&rw->logincore); + if (rw->cdb_fd >= 0) cdb_free(&rw->cdb); maybe_close(rw->cdb_fd); maybe_close(rw->lock_fd); @@ -201,9 +214,11 @@ static int acquire_lock(Tcl_Interp *ip, Pathbuf *pb, int *lockfd_r) { else return cht_posixerr(ip, errno, "unexpected error from fcntl while" " acquiring lock"); } + + return TCL_OK; } -/*---------- Log reading ----------*/ +/*---------- Log reading and writing ----------*/ static int readlognum(FILE *f, int delim, int *num_r) { int c; @@ -228,13 +243,17 @@ static int readlognum(FILE *f, int delim, int *num_r) { } static int readstorelogrecord(FILE *f, HashTable *ht, + int (*omitfn)(const HashValue*, + struct ht_forall_ctx *ctx), + struct ht_forall_ctx *ctx, void (*updatefn)(HashTable*, const char*, HashValue*)) { /* returns: - * 0 for OK - * -1 eof - * -2 corrupt or error - * -3 got newline indicating end + * 0 for OK + * -1 eof + * -2 corrupt or error + * -3 got newline indicating end + * >0 value from omitfn */ int keylen, vallen; char *key; @@ -263,9 +282,12 @@ static int readstorelogrecord(FILE *f, HashTable *ht, r= fread(htv_fillptr(val), 1,vallen, f); if (r!=vallen) goto x2_free_keyval; - updatefn(ht, key, val); + rc= omitfn ? omitfn(val, ctx) : TCL_OK; + if (rc) { assert(rc>0); TFREE(val); } + else updatefn(ht, key, val); + TFREE(key); - return TCL_OK; + return rc; x2_free_keyval: TFREE(val); @@ -273,6 +295,18 @@ static int readstorelogrecord(FILE *f, HashTable *ht, return -2; } +static int writerecord(FILE *f, const char *key, const HashValue *val) { + int r; + + r= fprintf(f, "+%d,%d:%s->", strlen(key), val->len, key); + if (r<0) return -1; + + r= fwrite(val->data, 1, val->len, f); + if (r != val->len) return -1; + + return 0; +} + /*---------- Creating ----------*/ int cht_do_cdbwr_create_empty(ClientData cd, Tcl_Interp *ip, @@ -310,7 +344,17 @@ int cht_do_cdbwr_create_empty(ClientData cd, Tcl_Interp *ip, static int infocbv3(Tcl_Interp *ip, Rw *rw, const char *arg1, const char *arg2fmt, const char *arg3, va_list al) { - abort(); + Tcl_Obj *aa[3]; + int na; + char buf[200]; + vsnprintf(buf, sizeof(buf), arg2fmt, al); + + na= 0; + aa[na++]= cht_ret_string(ip, arg1); + aa[na++]= cht_ret_string(ip, buf); + if (arg3) aa[na++]= cht_ret_string(ip, arg3); + + return cht_scriptinv_invoke_fg(&rw->on_info, na, aa); } static int infocb3(Tcl_Interp *ip, Rw *rw, const char *arg1, @@ -335,8 +379,22 @@ static int infocb(Tcl_Interp *ip, Rw *rw, const char *arg1, /*---------- Opening ----------*/ +static int cdbinit(Tcl_Interp *ip, Rw *rw) { + /* On entry, cdb_fd >=0 but cdb is _undefined_/ + * On exit, either cdb_fd<0 or cdb is initialised */ + int r, rc; + + r= cdb_init(&rw->cdb, rw->cdb_fd); + if (r) { + rc= cht_posixerr(ip, errno, "failed to initialise cdb reader"); + close(rw->cdb_fd); rw->cdb_fd= -1; return rc; + } + return TCL_OK; +} + int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, const char *pathb, - Tcl_Obj *on_info, int maxage, void **result) { + Tcl_Obj *on_info, Tcl_Obj *on_lexminval, + void **result) { const Cdbwr_SubCommand *subcmd= cd; int r, rc, mainfd=-1; Rw *rw; @@ -345,12 +403,20 @@ int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, const char *pathb, rw= TALLOC(sizeof(*rw)); ht_setup(&rw->logincore); + cht_scriptinv_init(&rw->on_info); + cht_scriptinv_init(&rw->on_lexminval); rw->cdb_fd= rw->lock_fd= -1; rw->logfile= 0; - rw->maxage= maxage; pathbuf_init(&rw->pbsome, pathb); pathbuf_init(&rw->pbother, pathb); rw->autocompact= 1; + if (on_lexminval) { + rc= cht_scriptinv_set(&rw->on_lexminval, ip, on_lexminval, 0); + if (rc) goto x_rc; + } else { + rw->on_lexminval.llength= 0; + } + mainfd= open(pathbuf_sfx(&rw->pbsome,".main"), O_RDONLY); if (mainfd<0) PE("open exist3ing database file .main"); rc= acquire_lock(ip, &rw->pbsome, &rw->lock_fd); if (rc) goto x_rc; @@ -360,11 +426,7 @@ int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, const char *pathb, rw->cdb_fd= open(pathbuf_sfx(&rw->pbsome,".cdb"), O_RDONLY); if (rw->cdb_fd >=0) { - r= cdb_init(&rw->cdb, rw->cdb_fd); - if (r) { - rc= cht_posixerr(ip, errno, "failed to initialise cdb reader"); - close(rw->cdb_fd); rw->cdb_fd= -1; goto x_rc; - } + rc= cdbinit(ip, rw); if (rc) goto x_rc; } else if (errno == ENOENT) { if (rw->mainsz) { rc= cht_staticerr(ip, ".cdb does not exist but .main is nonempty -" @@ -392,7 +454,7 @@ int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, const char *pathb, for (;;) { logrecstart= ftello(rw->logfile); if (logrecstart < 0) PE("ftello .log during (dirty) open"); - r= readstorelogrecord(rw->logfile, &rw->logincore, ht_update); + r= readstorelogrecord(rw->logfile, &rw->logincore, 0,0, ht_update); if (ferror(rw->logfile)) { rc= cht_posixerr(ip, errno, "error reading .log during (dirty) open"); goto x_rc; @@ -442,9 +504,58 @@ int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, const char *pathb, return rc; } -/*---------- Compacting ----------*/ +int cht_do_cdbwr_open_okjunk(ClientData cd, Tcl_Interp *ip, const char *pathb, + Tcl_Obj *on_info, Tcl_Obj *on_lexminval, + void **result) { + return cht_do_cdbwr_open(cd,ip,pathb,on_info,on_lexminval,result); +} + +/*==================== COMPACTION ====================*/ + +struct ht_forall_ctx { + struct cdb_make cdbm; + FILE *mainfile; + int lexminvall; + long *reccount; + const char *lexminval; +}; + +/*---------- helper functions ----------*/ -static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsize) { +static int expiredp(const HashValue *val, struct ht_forall_ctx *a) { + int r, l; + if (!val->len) return 0; + l= val->len < a->lexminvall ? val->len : a->lexminvall; + r= memcmp(val->data, a->lexminval, l); + if (r>0) return 0; + if (r<0) return 1; + return val->len < a->lexminvall; +} + +static int delete_ifexpired(const char *key, HashValue *val, + struct ht_forall_ctx *a) { + if (!expiredp(val, a)) return 0; + val->len= 0; + /* we don't actually need to realloc it to free the memory because + * this will shortly all be deleted as part of the compaction */ + return 0; +} + +static int addto_cdb(const char *key, HashValue *val, + struct ht_forall_ctx *a) { + return cdb_make_add(&a->cdbm, key, strlen(key), val->data, val->len); +} + +static int addto_main(const char *key, HashValue *val, + struct ht_forall_ctx *a) { + (*a->reccount)++; + return writerecord(a->mainfile, key, val); +} + +/*---------- compact main entrypoint ----------*/ + +static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsz, + long *reccount_r) { /* creates new .cdb and .main * closes logfile * leaves .log with old data @@ -453,18 +564,16 @@ static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsize) { */ int r, rc; int cdbfd, cdbmaking; - off_t errpos; + off_t errpos, newmainsz; char buf[100]; - - struct ht_forall_ctx { - struct cdb_make cdbm; - FILE *mainfile; - int count; - } a; + Tcl_Obj *res; + struct ht_forall_ctx a; a.mainfile= 0; cdbfd= -1; cdbmaking= 0; + *reccount_r= 0; + a.reccount= reccount_r; r= fclose(rw->logfile); if (r) { rc= cht_posixerr(ip, errno, "probable data loss! failed to fclose" @@ -472,16 +581,35 @@ static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsize) { rw->logfile= 0; rc= infocb(ip, rw, "compact-start", "log=%luby main=%luby", - logsize, (unsigned long)rw->mainsz); + logsz, (unsigned long)rw->mainsz); if (rc) goto x_rc; + if (rw->on_lexminval.llength) { + rc= cht_scriptinv_invoke_fg(&rw->on_lexminval, 0,0); + if (rc) goto x_rc; + + res= Tcl_GetObjResult(ip); assert(res); + a.lexminval= Tcl_GetStringFromObj(res, &a.lexminvall); + assert(a.lexminval); + + /* we rely not calling Tcl_Eval during the actual compaction; + * if we did Tcl_Eval then the interp result would be trashed. + */ + rc= ht_forall(&rw->logincore, delete_ifexpired, &a); + + } else { + a.lexminval= ""; + } + /* merge unsuperseded records from main into hash table */ a.mainfile= fopen(pathbuf_sfx(&rw->pbsome,".main"), "r"); if (!a.mainfile) PE("failed to open .main for reading during compact"); for (;;) { - r= readstorelogrecord(a.mainfile, &rw->logincore, ht_maybeupdate); + r= readstorelogrecord(a.mainfile, &rw->logincore, + expiredp, &a, + ht_maybeupdate); if (ferror(a.mainfile)) { rc= cht_posixerr(ip, errno, "error reading" " .main during compact"); goto x_rc; } @@ -514,10 +642,10 @@ static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsize) { if (r) PE("cdb_make_start during compact"); cdbmaking= 1; - r= ht_forall(&rw->logincore, addto_cdb, &addctx); + r= ht_forall(&rw->logincore, addto_cdb, &a); if (r) PE("cdb_make_add during compact"); - r= cdb_make_finish(&a.cdbm, cdbfd); + r= cdb_make_finish(&a.cdbm); if(r) PE("cdb_make_finish during compact"); cdbmaking= 0; @@ -533,14 +661,16 @@ static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsize) { a.mainfile= fopen(pathbuf_sfx(&rw->pbsome,".tmp"), "w"); if (!a.mainfile) PE("create .tmp for new main during compact"); - a.count= 0; - r= ht_forall(&rw->logincore, addto_main, a.mainfile); + r= ht_forall(&rw->logincore, addto_main, &a); if (r) { rc= cht_posixerr(ip, r, "error writing to new .main" " during compact"); goto x_rc; } r= fflush(a.mainfile); if (r) PE("fflush new main during compact"); r= fdatasync(fileno(a.mainfile)); if (r) PE("fdatasync new main during compact"); + + newmainsz= ftello(a.mainfile); + if (newmainsz<0) PE("ftello new main during compact"); r= fclose(a.mainfile); if (r) PE("fclose new main during compact"); a.mainfile= 0; @@ -548,53 +678,182 @@ static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsize) { r= rename(rw->pbsome.buf, pathbuf_sfx(&rw->pbother,".main")); if (r) PE("install new .main during compact"); + rw->mainsz= newmainsz; + /* done! */ - rc= infocb(ip, rw, "compact-end", "log=%luby main=%luby", - logsize, (unsigned long)rw->mainsz); + rc= infocb(ip, rw, "compact-end", "main=%luby nrecs=%l", + (unsigned long)rw->mainsz, *a.reccount); if (rc) goto x_rc; - rc= TCL_OK; + return rc; + x_rc: - if (mainfile) fclose(mainfile); - if (cdbmaking) cdb_make_finish(&a.cdbm, cdbfd); + if (a.mainfile) fclose(a.mainfile); + if (cdbmaking) cdb_make_finish(&a.cdbm); maybe_close(cdbfd); remove(pathbuf_sfx(&rw->pbsome,".tmp")); /* for tidyness */ + return rc; } - -static void compact_forclose(Tcl_Interp *ip, Rw *rw) { + +/*---------- Closing ----------*/ + +static int compact_forclose(Tcl_Interp *ip, Rw *rw, long *reccount_r) { off_t logsz; - int rc; + int r, rc; logsz= ftello(rw->logfile); if (logsz < 0) PE("ftello logfile (during tidy close)"); - rc= compact_core(ip, rw, logsz); if (rc) goto x_rc; + rc= compact_core(ip, rw, logsz, reccount_r); if (rc) goto x_rc; r= remove(pathbuf_sfx(&rw->pbsome,".log")); if (r) PE("remove .log (during tidy close)"); + + return TCL_OK; + +x_rc: return rc; } int cht_do_cdbwr_close(ClientData cd, Tcl_Interp *ip, void *rw_v) { Rw *rw= rw_v; - int rc, compact_rc, infocb_rc; - - if (rw->autocompact) compact_rc= compact_forclose(ip, rw); - else compact_rc= TCL_OK; + int rc, rc_close; + long reccount= -1; + off_t logsz; - rc= rw_close(ip,rw); - infocb_rc= infocb_close(rw); + if (rw->autocompact) rc= compact_forclose(ip, rw, &reccount); + else rc= TCL_OK; + + if (!rc) { + if (!rw->logfile) { + logsz= ftello(rw->logfile); + if (logsz < 0) + rc= cht_posixerr(ip, errno, "ftell logfile during close info"); + else + rc= infocb(ip, rw, "close", "main=%luby log=%luby", + rw->mainsz, logsz); + } else if (reccount>=0) { + rc= infocb(ip, rw, "close", "main=%luby nrecs=%l", rw->mainsz, reccount); + } else { + rc= infocb(ip, rw, "close", "main=%luby", rw->mainsz); + } + } + rc_close= rw_close(ip,rw); + if (rc_close) rc= rc_close; cht_tabledataid_disposing(ip, rw_v, &cdbtcl_rwdatabases); - if (!rc) rc= compact_rc; - if (!rc) rc= infocb_rc; return rc; } +/*---------- Other compaction-related entrypoints ----------*/ + +static int compact_keepopen(Tcl_Interp *ip, Rw *rw, int force) { + off_t logsz; + long reccount; + int rc, r; + + logsz= ftello(rw->logfile); + if (logsz < 0) return cht_posixerr(ip, errno, "ftell .log" + " during compact check or force"); + + if (!force && logsz < rw->mainsz / 10 + 1000) return TCL_OK; + + rc= compact_core(ip, rw, logsz, &reccount); if (rc) goto x_rc; + + maybe_close(rw->cdb_fd); + rw->cdb_fd= -1; + ht_destroy(&rw->logincore); + ht_setup(&rw->logincore); + + rw->cdb_fd= open(pathbuf_sfx(&rw->pbsome,".cdb"), O_RDONLY); + if (rw->cdb_fd < 0) PE("reopen .cdb after compact"); + + rc= cdbinit(ip, rw); if (rc) goto x_rc; + + rw->logfile= fopen(pathbuf_sfx(&rw->pbsome,".log"), "w"); + if (!rw->logfile) PE("reopen .log after compact"); + + r= fsync(fileno(rw->logfile)); if (r) PE("fsync .log after compact reopen"); + + return TCL_OK; + +x_rc: + /* doom! all updates fail after this (because rw->logfile is 0), and + * we may be using a lot more RAM than would be ideal. Program will + * have to reopen if it really wants sanity. */ + return rc; +} + +int cht_do_cdbwr_compact_force(ClientData cd, Tcl_Interp *ip, void *rw_v) { + return compact_keepopen(ip, rw_v, 1); +} +int cht_do_cdbwr_compact_check(ClientData cd, Tcl_Interp *ip, void *rw_v) { + return compact_keepopen(ip, rw_v, 0); +} + +int cht_do_cdbwr_compact_explicit(ClientData cd, Tcl_Interp *ip, void *rw_v) { + Rw *rw= rw_v; + rw->autocompact= 0; + return TCL_OK; +} +int cht_do_cdbwr_compact_auto(ClientData cd, Tcl_Interp *ip, void *rw_v) { + Rw *rw= rw_v; + rw->autocompact= 1; + return TCL_OK; +} + +/*---------- Updateing ----------*/ + +static int update(Tcl_Interp *ip, Rw *rw, const char *key, + const Byte *data, int dlen) { + HashValue *val; + int rc, r; + + if (!rw->logfile) return cht_staticerr + (ip, "previous compact failed; cdbwr must be closed and reopened " + "before any further updates", "CDB BROKEN"); + + val= htv_prep(dlen); assert(val); + memcpy(htv_fillptr(val), data, dlen); + + r= writerecord(rw->logfile, key, val); + if (!r) r= fflush(rw->logfile); + if (r) PE("write update to logfile"); + + ht_update(&rw->logincore, key, val); + return compact_keepopen(ip, rw, 0); + + x_rc: + TFREE(val); + return rc; +} + +int cht_do_cdbwr_update(ClientData cd, Tcl_Interp *ip, + void *rw_v, const char *key, Tcl_Obj *value) { + int dlen; + const char *data; + data= Tcl_GetStringFromObj(value, &dlen); assert(data); + return update(ip, rw_v, key, data, dlen); +} + +int cht_do_cdbwr_update_hb(ClientData cd, Tcl_Interp *ip, + void *rw_v, const char *key, HBytes_Value value) { + return update(ip, rw_v, key, cht_hb_data(&value), cht_hb_len(&value)); +} + +int cht_do_cdbwr_delete(ClientData cd, Tcl_Interp *ip, void *rw_v, + const char *key) { + return update(ip, rw_v, key, 0, 0); +} + +/*---------- Lookups ----------*/ + +static int lookup(Tcl_Interp *ip, Rw *rw, const char *key, ) { + ht_lookup( + +int cht_do_cdbwr_lookup(ClientData cd, Tcl_Interp *ip, void *db, + const char *key, Tcl_Obj *def, Tcl_Obj **result) { -int cht_do_cdbwr_lookup(ClientData cd, Tcl_Interp *ip, void *db, Tcl_Obj *key, Tcl_Obj **result); -int cht_do_cdbwr_lookup_hb(ClientData cd, Tcl_Interp *ip, void *db, HBytes_Value key, HBytes_Value *result); -int cht_do_cdbwr_update(ClientData cd, Tcl_Interp *ip, void *db, Tcl_Obj *key, Tcl_Obj *value); -int cht_do_cdbwr_update_hb(ClientData cd, Tcl_Interp *ip, void *db, HBytes_Value key, HBytes_Value value); -int cht_do_cdbwr_update_quick(ClientData cd, Tcl_Interp *ip, void *db, Tcl_Obj *key, Tcl_Obj *value); -int cht_do_cdbwr_update_quick_hb(ClientData cd, Tcl_Interp *ip, void *db, HBytes_Value key, HBytes_Value value); +} + +int cht_do_cdbwr_lookup_hb(ClientData cd, Tcl_Interp *ip, void *db, const char *key, HBytes_Value def, HBytes_Value *result);