Fixes for disorder-dbpugrade:

   - increased verbosity
   - bad (non-UTF-8) keys are now logged but ignored by default
   - ...but they can be deleted or abort the upgrade
   - aliases that sort later than the track they alias no longer
     abort the upgrade

diff --git a/doc/disorder-dbupgrade.8.in b/doc/disorder-dbupgrade.8.in
index 3d8a3b006945bc1130d8a25437ad248ae2fe458e..7da152b465bf84d4ab4909f6036066dd16a9ce58 100644 (file)
--- a/doc/disorder-dbupgrade.8.in
+++ b/doc/disorder-dbupgrade.8.in
@@ -29,6 +29,16 @@ is DisOrder's database upgrader.  It is invoked by DisOrder when
 necessary and does not need to be invoked manually.
 .SH OPTIONS
 .TP
 necessary and does not need to be invoked manually.
 .SH OPTIONS
 .TP

+.B --delete-bad-keys\fR, -x
+If invalid keys are found in the database then delete them.  See
+.B "INVALID KEYS"
+below.
+.TP
+.B --fail-bad-keys\fR, -X
+If invalid keys are found in the database then fail.  See
+.B "INVALID KEYS"
+below.
+.TP

 .B --config \fIPATH\fR, \fB-c \fIPATH
 Set the configuration file.
 .TP
 .B --config \fIPATH\fR, \fB-c \fIPATH
 Set the configuration file.
 .TP


@@ -40,6 +50,13 @@ Display a usage message.
 .TP
 .B --version\fR, \fB-V
 Display version number.
 .TP
 .B --version\fR, \fB-V
 Display version number.

+.SH "INVALID KEYS"
+An invalid key is one that is not valid UTF-8 or cannot be converted
+to NFC for some reason.  By default a warning message is issued and
+they are left in the database (if doing so will not compromise its
+integrity).  The
+.B -x
+option can be used to delete them if they are known to be harmles.

 .SH "SEE ALSO"
 \fBdisorderd\fR(8), \fBdisorder_config\fR(5)
 .\" Local Variables:
 .SH "SEE ALSO"
 \fBdisorderd\fR(8), \fBdisorder_config\fR(5)
 .\" Local Variables:

diff --git a/server/dbupgrade.c b/server/dbupgrade.c
index 8b5299111b19af1b7141c9ace9d1e8445d579bc9..b856ae5eb36ba312915b5afde9f218da4b27ca93 100644 (file)
--- a/server/dbupgrade.c
+++ b/server/dbupgrade.c
@@ -42,12 +42,24 @@
 
 static DB_TXN *global_tid;
 
 
 static DB_TXN *global_tid;
 

+#define BADKEY_WARN 0
+#define BADKEY_FAIL 1
+#define BADKEY_DELETE 2
+
+/** @brief Bad key behavior */
+static int badkey = BADKEY_WARN;
+
+static long aliases_removed, keys_normalized, values_normalized, renoticed;
+static long keys_already_ok, values_already_ok;
+

 static const struct option options[] = {
   { "help", no_argument, 0, 'h' },
   { "version", no_argument, 0, 'V' },
   { "config", required_argument, 0, 'c' },
   { "debug", no_argument, 0, 'd' },
   { "no-debug", no_argument, 0, 'D' },
 static const struct option options[] = {
   { "help", no_argument, 0, 'h' },
   { "version", no_argument, 0, 'V' },
   { "config", required_argument, 0, 'c' },
   { "debug", no_argument, 0, 'd' },
   { "no-debug", no_argument, 0, 'D' },

+  { "delete-bad-keys", no_argument, 0, 'x' },
+  { "fail-bad-keys", no_argument, 0, 'X' },

   { "syslog", no_argument, 0, 's' },
   { "no-syslog", no_argument, 0, 'S' },
   { 0, 0, 0, 0 }
   { "syslog", no_argument, 0, 's' },
   { "no-syslog", no_argument, 0, 'S' },
   { 0, 0, 0, 0 }


@@ -63,6 +75,8 @@ static void help(void) {
          "  --config PATH, -c PATH  Set configuration file\n"
          "  --debug, -d             Turn on debugging\n"
           "  --[no-]syslog           Force logging\n"
          "  --config PATH, -c PATH  Set configuration file\n"
          "  --debug, -d             Turn on debugging\n"
           "  --[no-]syslog           Force logging\n"

+          "  --delete-bad-keys, -x   Delete unconvertible keys\n"
+          "  --fail-bad-keys, -X     Fail if bad keys are found\n"

           "\n"
           "Database upgrader for DisOrder.  Not intended to be run\n"
           "directly.\n");
           "\n"
           "Database upgrader for DisOrder.  Not intended to be run\n"
           "directly.\n");


@@ -90,6 +104,12 @@ static int scan_core(const char *name, DB *db,
   int err, r = 0;
   DBT k[1], d[1];
 
   int err, r = 0;
   DBT k[1], d[1];
 

+  values_normalized = 0;
+  keys_normalized = 0;
+  aliases_removed = 0;
+  renoticed = 0;
+  keys_already_ok = 0;
+  values_already_ok = 0;

   memset(k, 0, sizeof k);
   memset(d, 0, sizeof d);
   while((err = c->c_get(c, k, d, DB_NEXT)) == 0) {
   memset(k, 0, sizeof k);
   memset(d, 0, sizeof d);
   while((err = c->c_get(c, k, d, DB_NEXT)) == 0) {


@@ -104,6 +124,17 @@ static int scan_core(const char *name, DB *db,
   r = (err == DB_LOCK_DEADLOCK ? err : 0);
   if((err = c->c_close(c)))
     fatal(0, "%s: error closing cursor: %s", name, db_strerror(err));
   r = (err == DB_LOCK_DEADLOCK ? err : 0);
   if((err = c->c_close(c)))
     fatal(0, "%s: error closing cursor: %s", name, db_strerror(err));

+  info("%s: %ld entries scanned", name, count);
+  if(values_normalized || values_already_ok)
+    info("%s: %ld values converted, %ld already ok", name,
+         values_normalized, values_already_ok);
+  if(keys_normalized || keys_already_ok)
+    info("%s: %ld keys converted, %ld already OK", name,
+         keys_normalized, keys_already_ok);
+  if(aliases_removed)
+    info("%s: %ld aliases removed", name, aliases_removed);
+  if(renoticed)
+    info("%s: %ld tracks re-noticed", name, renoticed);

   return r;
 }
 
   return r;
 }
 


@@ -153,12 +184,33 @@ static int normalize_keys(const char *name, DB *db, DBC *c,
 
   /* Find the normalized form of the key */
   knfc = utf8_compose_canon(k->data, k->size, &nknfc);
 
   /* Find the normalized form of the key */
   knfc = utf8_compose_canon(k->data, k->size, &nknfc);

-  if(!knfc)
-    fatal(0, "%s: cannot convert key to NFC: %.*s", name,
-          (int)k->size, (const char *)k->data);
+  if(!knfc) {
+    switch(badkey) {
+    case BADKEY_WARN:
+      error(0, "%s: invalid key: %.*s", name,
+            (int)k->size, (const char *)k->data);
+      break;
+    case BADKEY_DELETE:
+      error(0, "%s: deleting invalid key: %.*s", name,
+            (int)k->size, (const char *)k->data);
+      if((err = c->c_del(c, 0))) {
+        if(err != DB_LOCK_DEADLOCK)
+          fatal(0, "%s: error removing denormalized key: %s",
+                name, db_strerror(err));
+        return err;
+      }
+      break;
+    case BADKEY_FAIL:
+      fatal(0, "%s: invalid key: %.*s", name,
+            (int)k->size, (const char *)k->data);
+    }
+    return 0;
+  }

   /* If the key is already in NFC then do nothing */
   /* If the key is already in NFC then do nothing */

-  if(nknfc == k->size && !memcmp(k->data, knfc, nknfc))
+  if(nknfc == k->size && !memcmp(k->data, knfc, nknfc)) {
+    ++keys_already_ok;

     return 0;
     return 0;

+  }

   /* To rename the key we must delete the old one and insert a new one */
   if((err = c->c_del(c, 0))) {
     if(err != DB_LOCK_DEADLOCK)
   /* To rename the key we must delete the old one and insert a new one */
   if((err = c->c_del(c, 0))) {
     if(err != DB_LOCK_DEADLOCK)


@@ -173,6 +225,7 @@ static int normalize_keys(const char *name, DB *db, DBC *c,
       fatal(0, "%s: error storing normalized key: %s", name, db_strerror(err));
     return err;
   }
       fatal(0, "%s: error storing normalized key: %s", name, db_strerror(err));
     return err;
   }

+  ++keys_normalized;

   return 0;
 }
 
   return 0;
 }
 


@@ -189,8 +242,10 @@ static int normalize_values(const char *name, DB *db,
     fatal(0, "%s: cannot convert data to NFC: %.*s", name,
           (int)d->size, (const char *)d->data);
   /* If the key is already in NFC then do nothing */
     fatal(0, "%s: cannot convert data to NFC: %.*s", name,
           (int)d->size, (const char *)d->data);
   /* If the key is already in NFC then do nothing */

-  if(ndnfc == d->size && !memcmp(d->data, dnfc, ndnfc))
+  if(ndnfc == d->size && !memcmp(d->data, dnfc, ndnfc)) {
+    ++values_already_ok;

     return 0;
     return 0;

+  }

   d->size = ndnfc;
   d->data = dnfc;
   if((err = db->put(db, global_tid, k, d, 0))) {
   d->size = ndnfc;
   d->data = dnfc;
   if((err = db->put(db, global_tid, k, d, 0))) {


@@ -198,6 +253,7 @@ static int normalize_values(const char *name, DB *db,
       fatal(0, "%s: error storing normalized data: %s", name, db_strerror(err));
     return err;
   }
       fatal(0, "%s: error storing normalized data: %s", name, db_strerror(err));
     return err;
   }

+  ++values_normalized;

   return 0;
 }
 
   return 0;
 }
 


@@ -209,11 +265,17 @@ static int renotice(const char *name, DB attribute((unused)) *db,
   const char *const path = kvp_get(t, "_path");
   int err;
 
   const char *const path = kvp_get(t, "_path");
   int err;
 

-  if(!path)
+  if(!path) {
+    /* If an alias sorts later than the actual filename then it'll appear
+     * in the scan. */
+    if(kvp_get(t, "_alias_for"))
+      return 0;

     fatal(0, "%s: no '_path' for %.*s", name,
           (int)k->size, (const char *)k->data);
     fatal(0, "%s: no '_path' for %.*s", name,
           (int)k->size, (const char *)k->data);

+  }

   switch(err = trackdb_notice_tid(track, path, global_tid)) {
   case 0:
   switch(err = trackdb_notice_tid(track, path, global_tid)) {
   case 0:

+    ++renoticed;

     return 0;
   case DB_LOCK_DEADLOCK:
     return err;
     return 0;
   case DB_LOCK_DEADLOCK:
     return err;


@@ -235,8 +297,11 @@ static int remove_aliases_normalize_keys(const char *name, DB *db, DBC *c,
         fatal(0, "%s: error removing alias: %s", name, db_strerror(err));
       return err;
     }
         fatal(0, "%s: error removing alias: %s", name, db_strerror(err));
       return err;
     }

+    ++aliases_removed;

     return 0;
     return 0;

-  }
+  } else if(!kvp_get(t, "_path"))
+    error(0, "%s: %.*s has neither _alias_for nor _path", name,
+          (int)k->size, (const char *)k->data);

   return normalize_keys(name, db, c, k, d);
 }
 
   return normalize_keys(name, db, c, k, d);
 }
 


@@ -274,7 +339,7 @@ int main(int argc, char **argv) {
   set_progname(argv);
   mem_init();
   if(!setlocale(LC_CTYPE, "")) fatal(errno, "error calling setlocale");
   set_progname(argv);
   mem_init();
   if(!setlocale(LC_CTYPE, "")) fatal(errno, "error calling setlocale");

-  while((n = getopt_long(argc, argv, "hVc:dDSs", options, 0)) >= 0) {
+  while((n = getopt_long(argc, argv, "hVc:dDSsxX", options, 0)) >= 0) {

     switch(n) {
     case 'h': help();
     case 'V': version();
     switch(n) {
     case 'h': help();
     case 'V': version();


@@ -283,6 +348,8 @@ int main(int argc, char **argv) {
     case 'D': debugging = 0; break;
     case 'S': logsyslog = 0; break;
     case 's': logsyslog = 1; break;
     case 'D': debugging = 0; break;
     case 'S': logsyslog = 0; break;
     case 's': logsyslog = 1; break;

+    case 'x': badkey = BADKEY_DELETE; break;
+    case 'X': badkey = BADKEY_FAIL; break;

     default: fatal(0, "invalid option");
     }
   }
     default: fatal(0, "invalid option");
     }
   }