3 #include <nettle/md4.h>
5 #include "chiark_tcl_cdb.h"
13 static void maybe_close(int fd) {
18 rc= cht_posixerr(ip, errno, "failed to " PE); goto x_rc; \
21 /*---------- Pathbuf ----------*/
29 static void pathbuf_init(Pathbuf *pb, const char *pathb) {
31 pb->buf= TALLOC(l + 4);
32 memcpy(pb->buf, pathb, l);
36 static const char *pathbuf_sfx(Pathbuf *pb, const char *suffix) {
37 assert(strlen(suffix) <= MAX_SUFFIX);
38 strcpy(pb->sfx, suffix);
41 static void pathbuf_free(Pathbuf *pb) {
46 /*---------- Our hash table ----------*/
48 typedef struct HashTable {
53 typedef struct HashDatum {
58 static unsigned int hkt_hash(Tcl_HashTable *tht, void *key_v) {
59 HashTable *ht= (void*)tht;
60 const HashDatum *key= key_v;
63 Byte mdr[MD4_DIGEST_SIZE];
66 md4_update(&mdc, sizeof(ht->confound), ht->confound);
67 md4_update(&mdc, key->len, key->data);
68 md4_digest(&mdc, sizeof(mdr), mdr);
69 assert(sizeof(int) <= MD4_DIGEST_SIZE);
73 static const struct Tcl_HashKeyType ht_keytype= {
74 TCL_HASH_KEY_TYPE_VERSION, 0,
75 htk_hash, htk_compare, 0, 0
78 static HashDatum *htd_prep(int len) {
80 hd= TALLOC(offsetof(HashDatum, data) + len);
83 static Byte *htd_fillptr(HashDatum *hd) {
86 static void htd_fill(HashDatum *hd, const Byte *data) {
87 memcpy(hd->data, data, hd->len);
90 static int ht_setup(HashTable *ht) {
91 rc= cht_get_urandom(ip, ht->confound, sizeof(ht->confound));
94 Tcl_InitCustomHashTable(&ht->t, TCL_CUSTOM_PTR_KEYS, &ht_keytype);
97 static void ht_update(HashTable *ht, HashDatum *key_eat, HashDatum *val_eat) {
101 he= Tcl_CreateHashEntry(&ht->t, key_eat, &new);
102 /* eats the key! (since the data structure owns the memory) */
103 if (!new) TFREE(Tcl_GetHashValue(he));
104 Tcl_SetHashValue(he, val_eat);
107 static int ht_forall(HashTable *ht,
108 int (*fn)(const HashDatum *key, const HashDatum *val,
109 struct ht_forall_ctx *ctx),
110 struct ht_forall_ctx *ctx) {
111 /* Returns first nonzero value returned by any call to fn, or 0. */
114 HashDatum *key, *val;
116 for (he= Tcl_FirstHashEntry(&ht->t, &sp);
118 he= Tcl_NextHashEntry(&ht->t, &sp)) {
119 val= Tcl_GetHashValue(he);
120 if (!val->len) continue;
122 key= Tcl_GetHashKey(&ht->t, he);
124 r= fn(key, val, ctx);
130 /*---------- Rw data structure ----------*/
135 struct cdb cdb; /* valid iff cdb_fd >= 0 */
136 off_t cdb_bytes; /* valid iff cdb_fd >= 0 */
139 Pathbuf pbsome, pbother;
143 static void destroy_cdbrw_idtabcb(Tcl_Interp *ip, void *val) { abort(); }
145 int rw_close(Interp *ip, Rw *rw) {
150 maybe_close(rw->cdb_fd);
151 maybe_close(rw->lock_fd);
154 r= fclose(rw->logfile);
155 if (r && ip) { rc= cht_posixerr(ip, errno, "data loss! failed to"
156 " fclose logfile during untidy close"); }
159 pathbuf_free(&rw->pbsome); pathbuf_free(&rw->pbother);
163 void destroy_cdbrw_idtabcb(Interp *ip, Rw *rw) { rw_close(0,rw); }
165 const IdDataSpec cdbtcl_rwdatabases= {
166 "cdb-rwdb", "cdb-openrwdatabases-table", destroy_cdbrw_idtabcb
169 /*---------- File handling ----------*/
171 int acquire_lock(Tcl_Interp *ip, PathBuf *pb, int *lockfd_r) {
172 /* *lockfd_r must be -1 on entry. If may be set to >=0 even
173 * on error, and must be closed by the caller. */
174 mode_t umask, lockmode;
177 umask= umask(~(mode_t)0);
180 lockmode= 0666 & ~((umask & 0444)>>1);
181 /* Remove r where umask would remove w;
182 * eg umask intending 0664 here gives 0660 */
184 *lockfd_r= open(pathbuf_sfx(".lock"), O_RDONLY|O_CREAT, lockmode);
186 return cht_posixerr(ip, errno, "could not open/create lockfile");
189 fl.l_whence= SEEK_SET;
194 r= fcntl(*lockfd_r, F_SETLK, &fl);
196 if (errno == EACCES || errno == EAGAIN)
197 return cht_staticerr(ip, "lock held by another process", "CDB LOCKED");
198 else return cht_posixerr(ip, errno, "unexpected error from fcntl while"
203 /*---------- Log reading ----------*/
205 static int readlognum(FILE *f, int delim, int *num_r) {
207 char numbuf[20], *p, *ep;
212 c= getc(f); if (c==EOF) return -2;
213 if (c == delim) break;
214 if (!isdigit((unsigned char)c)) return -2;
216 if (p == numbuf+sizeof(numbuf)) return -2;
218 if (p == numbuf) return -2;
221 errno=0; ul= strtoul(numbuf, &ep, 10);
222 if (*ep || errno || ul >= INT_MAX/2) return -2;
227 static int readstorelogrecord(FILE *f, HashTable *ht,
228 void (*updatefn)(HashTable*, HashDatum*,
233 * -2 corrupt or error
234 * -3 got newline indicating end
237 HashDatum *key, *val;
240 if (c==EOF) { if (feof(f)) return -1; return -2; }
241 if (c=='\n') return -3;
242 if (c!='+') return -2;
244 rc= readlognum(f, ',', &keylen); if (rc) return rc;
245 rc= readlognum(f, ':', &vallen); if (rc) return rc;
247 key= htd_prep(keylen);
248 val= htd_prep(vallen);
250 r= fread(htd_fillptr(key), 1,keylen, f);
251 if (r!=keylen) goto x2_free_keyval;
253 c= getc(f); if (c!='-') goto x2_free_keyval;
254 c= getc(f); if (c!='>') goto x2_free_keyval;
256 r= fread(htd_fillptr(val), 1,vallen, f);
257 if (r!=vallen) goto x2_free_keyval;
259 updatefn(ht, key, val);
268 /*---------- Creating ----------*/
270 int cht_do_cdbwr_create_empty(ClientData cd, Tcl_Interp *ip,
272 static const char *const toremoves[]= {
273 ".main", ".cdb", ".log", ".tmp", 0
277 int lock_fd=-1, fd=-1, rc;
278 const char *const *toremove;
280 pathbuf_init(&pb, pathb);
281 rc= acquire_lock(ip, &pb, &lock_fd); if (rc) goto x_rc;
283 fd= open(pathbuf_sfx(".main"), O_RDWR|O_CREAT|O_EXCL, 0666);
284 if (fd <= 0) PE("create new database file");
286 for (toremoves=toremove; *toremove; toremove++) {
287 r= remove(*toremove);
288 if (r && errno != ENOENT)
289 PE("delete possible spurious file during creation");
296 maybe_close(lock_fd);
301 /*---------- Opening ----------*/
303 int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip,
304 const char *pathb, Tcl_Obj *on_info, void **result) {
305 const Cdbwr_SubCommand *subcmd= cd;
311 rw= TALLOC(sizeof(*rw));
312 rc= ht_setup(&rw->logincore); if (rc) { TFREE(rw); return rc; }
313 rw->cdb_fd= rw->lock_fd= -1; rw->logfile= 0;
314 pathbuf_init(&rw->pbsome, &pathb);
315 pathbuf_init(&rw->pbother, &pathb);
318 mainfd= open(pathbuf_sfx(&rw->pbsome,".main"), O_RDONLY);
319 if (mainfd<0) PE("open exist3ing database file .main");
320 rc= acquire_lock(ip, &rw->pbsome, &rw->lock_fd); if (rc) goto x_rc;
322 r= stat(mainfd, &stab); if (r) PE("fstat .main");
323 rw->mainsz= stab.st_size;
325 rw->cdb_fd= open(pathbuf_sfx(&rw->pbsome,".cdb"), O_RDONLY);
327 r= cdb_init(&rw->cdb, rw->cdb_fd);
329 rc= cht_posixerr(ip, errno, "failed to initialise cdb reader");
330 close(rw->cdb_fd); rw->cdb_fd= -1; goto x_rc;
332 } else if (errno == ENOENT) {
334 rc= cht_staticerr(ip, ".cdb does not exist but .main is nonempty -"
335 " .cdb must have been accidentally deleted!",
344 rw->logfile= fopen(pathbuf_sfx(&rw->pbsome,".log"), "r+");
346 if (errno != ENOENT) PE("failed to open .log during open");
347 rw->logfile= fopen(rw->pbsome.buf, "w");
348 if (!rw->logfile) PE("create .log during (clean) open");
349 } else { /* rw->logfile */
350 r= fstat(fileno(rw->logfile), &stab);
351 if (r==-1) PE("fstat .log during open");
352 rc= infocb(rw, "open-dirty-start", "log=%luby",
353 (unsigned long)stab.st_size);
357 r= fgetpos(rw->logfile, &logrecstart);
358 if (r) PE("fgetpos .log during (dirty) open");
359 r= readstorelogrecord(rw->logfile, &rw->logincore, ht_update);
360 if (ferror(rw->logfile)) {
361 rc= cht_posixerr(ip, errno, "error reading .log during (dirty) open");
366 } else if (r==-2 || r==-3) {
368 r= fgetpos(rw->logfile, &logjunkpos);
369 if (r) PE("fgetpos .log during report of junk in dirty open");
371 snprintf(buf,sizeof(buf), "CDB SYNTAX LOG %lu %lu",
372 (unsigned long)junkpos, (unsigned long)logrecstart);
374 if (!(subcmd->flags & RWSCF_OKJUNK)) {
375 Tcl_SetObjErrorCode(ip, Tcl_NewStringObj(buf,-1));
376 snprintf(buf,sizeof(buf),"%lu",(unsigned long)junkpos);
378 Tcl_AppendResult(ip, "syntax error (junk) in .log during"
379 " (dirty) open, at file position ", buf, (char*)0);
383 rc= infocb(rw, "open-dirty-junk", "errorfpos=%luby {%s}",
384 (unsigned long)logjunkpos, buf);
387 r= fsetpos(rw->logfile, logrecstart);
388 if (r) PE("failed to fsetpos .log before junk during dirty open");
390 r= ftruncate(fileno(rw->logfile), logrecstart);
391 if (r) PE("ftruncate .log to chop junk during dirty open");
397 /* now log is positioned for appending and everything is read */
409 /*---------- Compacting ----------*/
411 static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsz) {
412 /* creates new .cdb and .main
414 * leaves .log with old data
415 * leaves cdb fd open onto old db
416 * leaves logincore full of crap
419 int cdbfd, cdbmaking;
420 struct ht_forall_ctx {
421 struct cdb_make cdbm;
430 r= fclose(rw->logfile);
431 if (r) { rc= cht_posixerr(ip, errno, "data loss! failed to fclose"
432 " logfile during compact"); goto x_rc; }
435 rc= infocb(ip, rw, "compact-start", "log=%luby main=%luby",
436 logsize, (unsigned long)rw->mainsz);
439 /* merge unsuperseded records from main into hash table */
441 a.mainfile= fopen(pathbuf_sfx(&rw_pbsome,".main"), "r");
442 if (!a.mainfile) PE("failed to open .main for reading during compact");
445 r= readstorelogrecord(a.mainfile, &rw->logincore, ht_maybeupdate);
446 if (ferror(a.mainfile)) { rc= cht_posixerr(ip, errno, "error reading"
447 " .main during compact"); goto x_rc;
451 } else if (r==-1 || r==-2) {
452 r= fgetpos(a.mainfile, &errpos);
453 if (r) PE("fgetpos .main during report of syntax error");
454 snprintf(buf,sizeof(buf), "CDB SYNTAX MAIN %lu", (uunsigned long)errpos);
455 Tcl_SetObjErrorCode(ip, Tcl_NewStringObj(buf,-1));
456 snprintf(buf,sizeof(buf), "%lu", (uunsigned long)errpos);
458 Tcl_AppendResult(ip, "syntax error in .main during"
459 " compact, at file position ", buf, (char*)0);
471 cdbfd= open(pathbuf_sfx(&rw->pbsome,".tmp"), O_WRONLY|O_CREAT|O_TRUNC, 0666);
472 if (cdbfd<0) PE("create .tmp for new cdb during compact");
474 r= cdb_make_start(&a.cdbm, cdbfd);
475 if (r) PE("cdb_make_start during compact");
478 r= ht_forall(&rw->logincore, addto_cdb, &addctx);
479 if (r) PE("cdb_make_add during compact");
481 r= cdb_make_finish(&a.cdbm, cdbfd);
482 if(r) PE("cdb_make_finish during compact");
485 r= fdatasync(cdbfd); if (r) PE("fdatasync new cdb during compact");
486 r= close(cdbfd); if (r) PE("close new cdb during compact");
489 r= rename(rw->pbsome.buf, pathbuf_sfx(&rw->pbother,".cdb"));
490 if (r) PE("install new .cdb during compact");
492 /* create new main */
494 a.mainfile= fopen(pathbuf_sfx(&rw->pbsome,".tmp"), "w");
495 if (!a.mainfile) PE("create .tmp for new main during compact");
498 r= ht_forall(&rw->logincore, addto_main, a.mainfile);
499 if (r) { rc= cht_posixerr(ip, r, "error writing to new .main"
500 " during compact"); goto x_rc; }
502 r= fflush(a.mainfile); if (r) PE("fflush new main during compact");
503 r= fdatasync(fileno(a.mainfile));
504 if (r) PE("fdatasync new main during compact");
506 r= fclose(a.mainfile); if (r) PE("fclose new main during compact");
509 r= rename(rw->pbsome.buf, pathbuf_sfx(&rw->pbother,".main"));
510 if (r) PE("install new .main during compact");
514 rc= infocb(ip, rw, "compact-end", "log=%luby main=%luby",
515 logsize, (unsigned long)rw->mainsz);
520 if (mainfile) fclose(mainfile);
521 if (cdbmaking) cdb_make_finish(&a.cdbm, cdbfd);
523 remove(pathbuf_sfx(&rw->pbsome,".tmp")); /* for tidyness */
526 static void compact_forclose(Tcl_Interp *ip, Rw *rw) {
530 logsz= ftell(rw->logfile);
531 if (logsz < 0) PE("ftell logfile (during tidy close)");
533 rc= compact_core(ip, rw, logsz); if (rc) goto x_rc;
535 r= remove(pathbuf_sfx(&rw->pbsome,".log"));
536 if (r) PE("remove .log (during tidy close)");
539 int cht_do_cdbwr_close(ClientData cd, Tcl_Interp *ip, void *rw_v) {
541 int rc, compact_rc, infocb_rc;
543 if (rw->autocompact) compact_rc= compact_forclose(ip, rw);
544 else compact_rc= TCL_OK;
547 infocb_rc= infocb_close(rw);
549 cht_tabledataid_disposing(ip, rw_v, &cdbtcl_rwdatabases);
550 if (!rc) rc= compact_rc;
551 if (!rc) rc= infocb_rc;
556 int cht_do_cdbwr_lookup(ClientData cd, Tcl_Interp *ip, void *db, Tcl_Obj *key, Tcl_Obj **result);
557 int cht_do_cdbwr_lookup_hb(ClientData cd, Tcl_Interp *ip, void *db, HBytes_Value key, HBytes_Value *result);
558 int cht_do_cdbwr_update(ClientData cd, Tcl_Interp *ip, void *db, Tcl_Obj *key, Tcl_Obj *value);
559 int cht_do_cdbwr_update_hb(ClientData cd, Tcl_Interp *ip, void *db, HBytes_Value key, HBytes_Value value);
560 int cht_do_cdbwr_update_quick(ClientData cd, Tcl_Interp *ip, void *db, Tcl_Obj *key, Tcl_Obj *value);
561 int cht_do_cdbwr_update_quick_hb(ClientData cd, Tcl_Interp *ip, void *db, HBytes_Value key, HBytes_Value value);