X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?p=chiark-tcl.git;a=blobdiff_plain;f=cdb%2Fwriteable.c;h=3ad0c1e5499ab01cc994e7d54dbf352299823595;hp=a73be821a001a894b0114920a06e4b045f1ab985;hb=62f8f2f13b024b49dfbed2b2839ec510ad37eb85;hpb=f16e3e0cf0a2b965643df01e24ef59e1393da78d

diff --git a/cdb/writeable.c b/cdb/writeable.c
index a73be82..3ad0c1e 100644
--- a/cdb/writeable.c
+++ b/cdb/writeable.c
@@ -1,7 +1,25 @@
-/**/
+/*
+ * cdb, cdb-wr - Tcl bindings for tinycdb and a journalling write extension
+ * Copyright 2006-2012 Ian Jackson
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
 
 #include "chiark_tcl_cdb.h"
 
+#define KEYLEN_MAX (INT_MAX/2)
+
 #define ftello ftell
 #define fseeko fseek
 
@@ -26,7 +44,8 @@ typedef struct Pathbuf {
 #define MAX_SUFFIX 5
 
 static void pathbuf_init(Pathbuf *pb, const char *pathb) {
-  int l= strlen(pathb);
+  size_t l= strlen(pathb);
+  assert(l < INT_MAX);
   pb->buf= TALLOC(l + MAX_SUFFIX + 1);
   memcpy(pb->buf, pathb, l);
   pb->sfx= pb->buf + l;
@@ -56,7 +75,7 @@ typedef struct HashValue {
 
 static HashValue *htv_prep(int len) {
   HashValue *hd;
-  hd= TALLOC((hd->data - (Byte*)hd) + len);
+  hd= TALLOC(offsetof(typeof(*hd), data) + len);
   hd->len= len;
   return hd;
 }  
@@ -142,20 +161,24 @@ typedef struct Rw {
   int ix, autocompact;
   int cdb_fd, lock_fd;
   struct cdb cdb; /* valid iff cdb_fd >= 0 */
-  FILE *logfile;
+  FILE *logfile; /* may be 0; if so, is broken */
   HashTable logincore;
   Pathbuf pbsome, pbother;
   off_t mainsz;
   ScriptToInvoke on_info, on_lexminval;
 } Rw;
 
+static void rw_cdb_close(Tcl_Interp *ip, Rw *rw) {
+  if (rw->cdb_fd >= 0) cdb_free(&rw->cdb);
+  maybe_close(rw->cdb_fd);
+}
+
 static int rw_close(Tcl_Interp *ip, Rw *rw) {
   int rc, r;
 
   rc= TCL_OK;
   ht_destroy(&rw->logincore);
-  if (rw->cdb_fd >= 0) cdb_free(&rw->cdb);
-  maybe_close(rw->cdb_fd);
+  rw_cdb_close(ip,rw);
   maybe_close(rw->lock_fd);
 
   if (rw->logfile) {
@@ -168,9 +191,9 @@ static int rw_close(Tcl_Interp *ip, Rw *rw) {
   return rc;
 }
 
-static void destroy_cdbrw_idtabcb(Tcl_Interp *ip, void *rw) {
-  rw_close(0,rw);
-  TFREE(rw);
+static void destroy_cdbrw_idtabcb(Tcl_Interp *ip, void *rw_v) {
+  rw_close(0,rw_v);
+  TFREE(rw_v);
 }
 const IdDataSpec cdbtcl_rwdatabases= {
   "cdb-rwdb", "cdb-openrwdatabases-table", destroy_cdbrw_idtabcb
@@ -232,7 +255,7 @@ static int readlognum(FILE *f, int delim, int *num_r) {
   *p= 0;
 
   errno=0; ul= strtoul(numbuf, &ep, 10);
-  if (*ep || errno || ul >= INT_MAX/2) return -2;
+  if (*ep || errno || ul >= KEYLEN_MAX) return -2;
   *num_r= ul;
   return 0;
 }
@@ -295,7 +318,7 @@ static int readstorelogrecord(FILE *f, HashTable *ht,
 static int writerecord(FILE *f, const char *key, const HashValue *val) {
   int r;
 
-  r= fprintf(f, "+%d,%d:%s->", strlen(key), val->len, key);
+  r= fprintf(f, "+%d,%d:%s->", (int)strlen(key), val->len, key);
   if (r<0) return -1;
   
   r= fwrite(val->data, 1, val->len, f);
@@ -311,7 +334,7 @@ static int writerecord(FILE *f, const char *key, const HashValue *val) {
 
 int cht_do_cdbwr_create_empty(ClientData cd, Tcl_Interp *ip,
 			      const char *pathb) {
-  static const char *const toremoves[]= { ".cdb", ".log", ".tmp", 0 };
+  static const char *const toremoves[]= { ".cdb", ".jrn", ".tmp", 0 };
 
   Pathbuf pb, pbmain;
   int lock_fd=-1, rc, r;
@@ -393,7 +416,7 @@ static int infocb(Tcl_Interp *ip, Rw *rw, const char *arg1,
 /*---------- Opening ----------*/
 
 static int cdbinit(Tcl_Interp *ip, Rw *rw) {
-  /* On entry, cdb_fd >=0 but cdb is _undefined_/
+  /* On entry, cdb_fd >=0 but cdb is _undefined_
    * On exit, either cdb_fd<0 or cdb is initialised */
   int r, rc;
   
@@ -452,24 +475,24 @@ int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, const char *pathb,
     PE("open .cdb");
   }
 
-  rw->logfile= fopen(pathbuf_sfx(&rw->pbsome,".log"), "r+");
+  rw->logfile= fopen(pathbuf_sfx(&rw->pbsome,".jrn"), "r+");
   if (!rw->logfile) {
-    if (errno != ENOENT) PE("failed to open .log during open");
+    if (errno != ENOENT) PE("failed to open .jrn during open");
     rw->logfile= fopen(rw->pbsome.buf, "w");
-    if (!rw->logfile) PE("create .log during (clean) open");
+    if (!rw->logfile) PE("create .jrn during (clean) open");
   } else { /* rw->logfile */
     r= fstat(fileno(rw->logfile), &stab);
-    if (r==-1) PE("fstat .log during open");
+    if (r==-1) PE("fstat .jrn during open");
     rc= infocb(ip, rw, "open-dirty-start", "log=%luby",
 	       (unsigned long)stab.st_size);
     if (rc) goto x_rc;
 
     for (;;) {
       logrecstart= ftello(rw->logfile);
-      if (logrecstart < 0) PE("ftello .log during (dirty) open");
+      if (logrecstart < 0) PE("ftello .jrn during (dirty) open");
       r= readstorelogrecord(rw->logfile, &rw->logincore, 0,0, ht_update);
       if (ferror(rw->logfile)) {
-	rc= cht_posixerr(ip, errno, "error reading .log during (dirty) open");
+	rc= cht_posixerr(ip, errno, "error reading .jrn during (dirty) open");
 	goto x_rc;
       }
       if (r==-1) {
@@ -477,7 +500,7 @@ int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, const char *pathb,
       } else if (r==-2 || r==-3) {
 	char buf[100];
 	logjunkpos= ftello(rw->logfile);
-	if(logjunkpos<0) PE("ftello .log during report of junk in dirty open");
+	if(logjunkpos<0) PE("ftello .jrn during report of junk in dirty open");
 
 	snprintf(buf,sizeof(buf), "CDB SYNTAX LOG %lu %lu",
 		 (unsigned long)logjunkpos, (unsigned long)logrecstart);
@@ -486,7 +509,7 @@ int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, const char *pathb,
 	  Tcl_SetObjErrorCode(ip, Tcl_NewStringObj(buf,-1));
 	  snprintf(buf,sizeof(buf),"%lu",(unsigned long)logjunkpos);
 	  Tcl_ResetResult(ip);
-	  Tcl_AppendResult(ip, "syntax error (junk) in .log during"
+	  Tcl_AppendResult(ip, "syntax error (junk) in .jrn during"
 			   " (dirty) open, at file position ", buf, (char*)0);
 	  rc= TCL_ERROR;
 	  goto x_rc;
@@ -496,10 +519,10 @@ int cht_do_cdbwr_open(ClientData cd, Tcl_Interp *ip, const char *pathb,
 	if (rc) goto x_rc;
 
 	r= fseeko(rw->logfile, logrecstart, SEEK_SET);
-	if (r) PE("failed to fseeko .log before junk during dirty open");
+	if (r) PE("failed to fseeko .jrn before junk during dirty open");
 
 	r= ftruncate(fileno(rw->logfile), logrecstart);
-	if (r) PE("ftruncate .log to chop junk during dirty open");
+	if (r) PE("ftruncate .jrn to chop junk during dirty open");
       } else {
 	assert(!r);
       }
@@ -572,7 +595,7 @@ static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsz,
 			long *reccount_r) {
   /* creates new .cdb and .main
    * closes logfile
-   * leaves .log with old data
+   * leaves .jrn with old data
    * leaves cdb fd open onto old db
    * leaves logincore full of crap
    */
@@ -590,9 +613,9 @@ static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsz,
   a.reccount= reccount_r;
 
   r= fclose(rw->logfile);
+  rw->logfile= 0;
   if (r) { rc= cht_posixerr(ip, errno, "probable data loss!  failed to fclose"
 			    " logfile during compact");  goto x_rc; }
-  rw->logfile= 0;
   
   rc= infocb(ip, rw, "compact-start", "log=%luby main=%luby",
 	     logsz, (unsigned long)rw->mainsz);
@@ -680,7 +703,7 @@ static int compact_core(Tcl_Interp *ip, Rw *rw, unsigned long logsz,
   if (!a.mainfile) PE("create .tmp for new main during compact");
 
   r= ht_forall(&rw->logincore, addto_main, &a);
-  if (r) { rc= cht_posixerr(ip, r, "error writing to new .main"
+  if (r) { rc= cht_posixerr(ip, errno, "error writing to new .main"
 			    " during compact");  goto x_rc; }
 
   r= putc('\n', a.mainfile);
@@ -728,8 +751,8 @@ static int compact_forclose(Tcl_Interp *ip, Rw *rw, long *reccount_r) {
 
   rc= compact_core(ip, rw, logsz, reccount_r);  if (rc) goto x_rc;
 
-  r= remove(pathbuf_sfx(&rw->pbsome,".log"));
-  if (r) PE("remove .log (during tidy close)");
+  r= remove(pathbuf_sfx(&rw->pbsome,".jrn"));
+  if (r) PE("remove .jrn (during tidy close)");
 
   return TCL_OK;
 
@@ -746,7 +769,7 @@ int cht_do_cdbwr_close(ClientData cd, Tcl_Interp *ip, void *rw_v) {
   else rc= TCL_OK;
 
   if (!rc) {
-    if (!rw->logfile) {
+    if (rw->logfile) {
       logsz= ftello(rw->logfile);
       if (logsz < 0)
 	rc= cht_posixerr(ip, errno, "ftell logfile during close info");
@@ -754,7 +777,8 @@ int cht_do_cdbwr_close(ClientData cd, Tcl_Interp *ip, void *rw_v) {
 	rc= infocb(ip, rw, "close", "main=%luby log=%luby",
 		   rw->mainsz, logsz);
     } else if (reccount>=0) {
-      rc= infocb(ip, rw, "close", "main=%luby nrecs=%l", rw->mainsz, reccount);
+      rc= infocb(ip, rw, "close", "main=%luby nrecs=%ld",
+		 rw->mainsz, reccount);
     } else {
       rc= infocb(ip, rw, "close", "main=%luby", rw->mainsz);
     }
@@ -775,15 +799,23 @@ static int compact_keepopen(Tcl_Interp *ip, Rw *rw, int force) {
   int rc, r;
 
   logsz= ftello(rw->logfile);
-  if (logsz < 0) return cht_posixerr(ip, errno, "ftell .log"
+  if (logsz < 0) return cht_posixerr(ip, errno, "ftell .jrn"
 				       " during compact check or force");
 
-  if (!force && logsz < rw->mainsz / 2 + 1000) return TCL_OK;
+  if (!force && logsz < rw->mainsz / 3 + 1000) return TCL_OK;
+  /* Test case:                    ^^^ testing best value for this
+   *   main=9690434by nrecs=122803  read all in one go
+   *  no autocompact, :    6.96user 0.68system 0:08.93elapsed
+   *  auto, mulitplier 2:  7.10user 0.79system 0:09.54elapsed
+   *  auto, unity:         7.80user 0.98system 0:11.84elapsed
+   *  auto, divisor  2:    8.23user 1.05system 0:13.30elapsed
+   *  auto, divisor  3:    8.55user 1.12system 0:12.88elapsed
+   *  auto, divisor  5:    9.95user 1.43system 0:15.72elapsed
+   */
 
   rc= compact_core(ip, rw, logsz, &reccount);  if (rc) goto x_rc;
 
-  maybe_close(rw->cdb_fd);
-  rw->cdb_fd= -1;
+  rw_cdb_close(ip,rw);
   ht_destroy(&rw->logincore);
   ht_setup(&rw->logincore);
 
@@ -792,10 +824,10 @@ static int compact_keepopen(Tcl_Interp *ip, Rw *rw, int force) {
 
   rc= cdbinit(ip, rw);  if (rc) goto x_rc;
 
-  rw->logfile= fopen(pathbuf_sfx(&rw->pbsome,".log"), "w");
-  if (!rw->logfile) PE("reopen .log after compact");
+  rw->logfile= fopen(pathbuf_sfx(&rw->pbsome,".jrn"), "w");
+  if (!rw->logfile) PE("reopen .jrn after compact");
 
-  r= fsync(fileno(rw->logfile));  if (r) PE("fsync .log after compact reopen");
+  r= fsync(fileno(rw->logfile));  if (r) PE("fsync .jrn after compact reopen");
 
   return TCL_OK;
 
@@ -829,12 +861,22 @@ int cht_do_cdbwr_compact_auto(ClientData cd, Tcl_Interp *ip, void *rw_v) {
 static int update(Tcl_Interp *ip, Rw *rw, const char *key,
 		  const Byte *data, int dlen) {
   HashValue *val;
+  const char *failed;
   int rc, r;
+  off_t recstart;
+
+  if (strlen(key) >= KEYLEN_MAX)
+    return cht_staticerr(ip, "key too long", "CDB KEYOVERFLOW");
 
   if (!rw->logfile) return cht_staticerr
-    (ip, "previous compact failed; cdbwr must be closed and reopened "
-     "before any further updates", "CDB BROKEN");
+    (ip, "failure during previous compact or error recovery;"
+     " cdbwr must be closed and reopened before any further updates",
+     "CDB BROKEN");
   
+  recstart= ftello(rw->logfile);
+  if (recstart < 0)
+    return cht_posixerr(ip, errno, "failed to ftello .jrn during update");
+
   val= htv_prep(dlen);  assert(val);
   memcpy(htv_fillptr(val), data, dlen);
 
@@ -849,6 +891,33 @@ static int update(Tcl_Interp *ip, Rw *rw, const char *key,
 
  x_rc:
   TFREE(val);
+  assert(rc);
+
+  /* Now, we have to try to sort out the journal so that it's
+   * truncated and positioned to where this abortively-written record
+   * started, with no buffered output and the error indicator clear.
+   *
+   * There seems to be no portable way to ensure the buffered unwritten
+   * output is discarded, so we close and reopen the stream.
+   */
+  fclose(rw->logfile);
+
+  rw->logfile= fopen(pathbuf_sfx(&rw->pbsome,".jrn"), "r+");
+  if (!rw->logfile) { failed= "fopen"; goto reset_fail; }
+
+  r= ftruncate(fileno(rw->logfile), recstart);
+  if (r) { failed= "ftruncate"; goto reset_fail; }
+
+  r= fseeko(rw->logfile, recstart, SEEK_SET);
+  if (r) { failed= "fseeko"; goto reset_fail; }
+
+  return rc;
+
+ reset_fail:
+  Tcl_AppendResult(ip, " (additionally, ", failed, " failed"
+		   " in error recovery: ", strerror(errno), ")", (char*)0);
+  if (rw->logfile) { fclose(rw->logfile); rw->logfile= 0; }
+
   return rc;
 }  
 
@@ -880,11 +949,18 @@ static int lookup_rw(Tcl_Interp *ip, void *rw_v, const char *key,
   val= ht_lookup(&rw->logincore, key);
   if (val) {
     if (val->len) { *data_r= val->data; *len_r= val->len; return TCL_OK; }
-    else { *data_r= 0; *len_r= -1; return TCL_OK; }
+    else goto not_found;
   }
 
+  if (rw->cdb_fd<0) goto not_found;
+
   return cht_cdb_lookup_cdb(ip, &rw->cdb, key, strlen(key), data_r, len_r);
-} 
+
+ not_found:
+  *data_r= 0;
+  *len_r= -1;
+  return TCL_OK;
+}
 
 int cht_do_cdbwr_lookup(ClientData cd, Tcl_Interp *ip, void *rw_v,
 			const char *key, Tcl_Obj *def,
@@ -907,9 +983,3 @@ int cht_do_cdbwr_lookup_hb(ClientData cd, Tcl_Interp *ip, void *rw_v,
   return cht_cdb_donesomelookup(ip, rw_v, def, result, data, dlen,
 				cht_cdb_storeanswer_hb);
 }
-
-int cht_do_cdbtoplevel_cdb_wr(ClientData cd, Tcl_Interp *ip,
-			      const Cdbwr_SubCommand* subcmd,
-			      int objc, Tcl_Obj *const *objv) {
-  return subcmd->func((void*)subcmd,ip,objc,objv);
-}