X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~mdw/git/disorder/blobdiff_plain/ad2f8275a5fc0dba29ca97bc02342d1f1627e590..d8b957853160200fe6b00d8c0e8c61a3f62ecd7c:/server/dbupgrade.c diff --git a/server/dbupgrade.c b/server/dbupgrade.c index 8b52991..74354b7 100644 --- a/server/dbupgrade.c +++ b/server/dbupgrade.c @@ -1,6 +1,6 @@ /* * This file is part of DisOrder - * Copyright (C) 2007 Richard Kettlewell + * Copyright (C) 2007, 2008 Richard Kettlewell * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -17,30 +17,19 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * USA */ +#include "disorder-server.h" -#include -#include "types.h" +static DB_TXN *global_tid; -#include -#include -#include -#include -#include -#include -#include -#include +#define BADKEY_WARN 0 +#define BADKEY_FAIL 1 +#define BADKEY_DELETE 2 -#include "syscalls.h" -#include "log.h" -#include "defs.h" -#include "kvp.h" -#include "trackdb.h" -#include "trackdb-int.h" -#include "mem.h" -#include "configuration.h" -#include "unicode.h" +/** @brief Bad key behavior */ +static int badkey = BADKEY_WARN; -static DB_TXN *global_tid; +static long aliases_removed, keys_normalized, values_normalized, renoticed; +static long keys_already_ok, values_already_ok; static const struct option options[] = { { "help", no_argument, 0, 'h' }, @@ -48,6 +37,8 @@ static const struct option options[] = { { "config", required_argument, 0, 'c' }, { "debug", no_argument, 0, 'd' }, { "no-debug", no_argument, 0, 'D' }, + { "delete-bad-keys", no_argument, 0, 'x' }, + { "fail-bad-keys", no_argument, 0, 'X' }, { "syslog", no_argument, 0, 's' }, { "no-syslog", no_argument, 0, 'S' }, { 0, 0, 0, 0 } @@ -63,6 +54,8 @@ static void help(void) { " --config PATH, -c PATH Set configuration file\n" " --debug, -d Turn on debugging\n" " --[no-]syslog Force logging\n" + " --delete-bad-keys, -x Delete unconvertible keys\n" + " --fail-bad-keys, -X Fail if bad keys are found\n" "\n" "Database upgrader for DisOrder. Not intended to be run\n" "directly.\n"); @@ -70,13 +63,6 @@ static void help(void) { exit(0); } -/* display version number and terminate */ -static void version(void) { - xprintf("disorder-dbupgrade version %s\n", disorder_version_string); - xfclose(stdout); - exit(0); -} - /** @brief Visit each key in a database and call @p callback * @return 0 or DB_LOCK_DEADLOCK * @@ -90,6 +76,12 @@ static int scan_core(const char *name, DB *db, int err, r = 0; DBT k[1], d[1]; + values_normalized = 0; + keys_normalized = 0; + aliases_removed = 0; + renoticed = 0; + keys_already_ok = 0; + values_already_ok = 0; memset(k, 0, sizeof k); memset(d, 0, sizeof d); while((err = c->c_get(c, k, d, DB_NEXT)) == 0) { @@ -104,6 +96,17 @@ static int scan_core(const char *name, DB *db, r = (err == DB_LOCK_DEADLOCK ? err : 0); if((err = c->c_close(c))) fatal(0, "%s: error closing cursor: %s", name, db_strerror(err)); + info("%s: %ld entries scanned", name, count); + if(values_normalized || values_already_ok) + info("%s: %ld values converted, %ld already ok", name, + values_normalized, values_already_ok); + if(keys_normalized || keys_already_ok) + info("%s: %ld keys converted, %ld already OK", name, + keys_normalized, keys_already_ok); + if(aliases_removed) + info("%s: %ld aliases removed", name, aliases_removed); + if(renoticed) + info("%s: %ld tracks re-noticed", name, renoticed); return r; } @@ -112,9 +115,9 @@ static int scan_core(const char *name, DB *db, * Everything happens inside the @p global_tid tranasction. @p callback * should return 0 or DB_LOCK_DEADLOCK. */ -static void scan(const char *name, DB *db, - int (*callback)(const char *name, DB *db, DBC *c, - DBT *k, DBT *d)) { +static void scandb(const char *name, DB *db, + int (*callback)(const char *name, DB *db, DBC *c, + DBT *k, DBT *d)) { info("scanning %s", name); for(;;) { global_tid = trackdb_begin_transaction(); @@ -153,12 +156,33 @@ static int normalize_keys(const char *name, DB *db, DBC *c, /* Find the normalized form of the key */ knfc = utf8_compose_canon(k->data, k->size, &nknfc); - if(!knfc) - fatal(0, "%s: cannot convert key to NFC: %.*s", name, - (int)k->size, (const char *)k->data); + if(!knfc) { + switch(badkey) { + case BADKEY_WARN: + error(0, "%s: invalid key: %.*s", name, + (int)k->size, (const char *)k->data); + break; + case BADKEY_DELETE: + error(0, "%s: deleting invalid key: %.*s", name, + (int)k->size, (const char *)k->data); + if((err = c->c_del(c, 0))) { + if(err != DB_LOCK_DEADLOCK) + fatal(0, "%s: error removing denormalized key: %s", + name, db_strerror(err)); + return err; + } + break; + case BADKEY_FAIL: + fatal(0, "%s: invalid key: %.*s", name, + (int)k->size, (const char *)k->data); + } + return 0; + } /* If the key is already in NFC then do nothing */ - if(nknfc == k->size && !memcmp(k->data, knfc, nknfc)) + if(nknfc == k->size && !memcmp(k->data, knfc, nknfc)) { + ++keys_already_ok; return 0; + } /* To rename the key we must delete the old one and insert a new one */ if((err = c->c_del(c, 0))) { if(err != DB_LOCK_DEADLOCK) @@ -173,6 +197,7 @@ static int normalize_keys(const char *name, DB *db, DBC *c, fatal(0, "%s: error storing normalized key: %s", name, db_strerror(err)); return err; } + ++keys_normalized; return 0; } @@ -189,8 +214,10 @@ static int normalize_values(const char *name, DB *db, fatal(0, "%s: cannot convert data to NFC: %.*s", name, (int)d->size, (const char *)d->data); /* If the key is already in NFC then do nothing */ - if(ndnfc == d->size && !memcmp(d->data, dnfc, ndnfc)) + if(ndnfc == d->size && !memcmp(d->data, dnfc, ndnfc)) { + ++values_already_ok; return 0; + } d->size = ndnfc; d->data = dnfc; if((err = db->put(db, global_tid, k, d, 0))) { @@ -198,6 +225,7 @@ static int normalize_values(const char *name, DB *db, fatal(0, "%s: error storing normalized data: %s", name, db_strerror(err)); return err; } + ++values_normalized; return 0; } @@ -209,11 +237,17 @@ static int renotice(const char *name, DB attribute((unused)) *db, const char *const path = kvp_get(t, "_path"); int err; - if(!path) + if(!path) { + /* If an alias sorts later than the actual filename then it'll appear + * in the scan. */ + if(kvp_get(t, "_alias_for")) + return 0; fatal(0, "%s: no '_path' for %.*s", name, (int)k->size, (const char *)k->data); + } switch(err = trackdb_notice_tid(track, path, global_tid)) { case 0: + ++renoticed; return 0; case DB_LOCK_DEADLOCK: return err; @@ -235,8 +269,11 @@ static int remove_aliases_normalize_keys(const char *name, DB *db, DBC *c, fatal(0, "%s: error removing alias: %s", name, db_strerror(err)); return err; } + ++aliases_removed; return 0; - } + } else if(!kvp_get(t, "_path")) + error(0, "%s: %.*s has neither _alias_for nor _path", name, + (int)k->size, (const char *)k->data); return normalize_keys(name, db, c, k, d); } @@ -252,16 +289,16 @@ static void upgrade(void) { /* Normalize keys and values as required. We will also remove aliases as * they will be regenerated when we re-noticed the tracks. */ info("renormalizing keys"); - scan("tracks.db", trackdb_tracksdb, remove_aliases_normalize_keys); - scan("prefs.db", trackdb_prefsdb, normalize_keys); - scan("global.db", trackdb_globaldb, normalize_keys); - scan("noticed.db", trackdb_noticeddb, normalize_values); + scandb("tracks.db", trackdb_tracksdb, remove_aliases_normalize_keys); + scandb("prefs.db", trackdb_prefsdb, normalize_keys); + scandb("global.db", trackdb_globaldb, normalize_keys); + scandb("noticed.db", trackdb_noticeddb, normalize_values); /* search.db and tags.db we will rebuild */ info("regenerating search database and aliases"); truncate_database("search.db", trackdb_searchdb); truncate_database("tags.db", trackdb_tagsdb); /* Regenerate the search database and aliases */ - scan("tracks.db", trackdb_tracksdb, renotice); + scandb("tracks.db", trackdb_tracksdb, renotice); /* Finally update the database version */ snprintf(buf, sizeof buf, "%ld", config->dbversion); trackdb_set_global("_dbversion", buf, 0); @@ -274,15 +311,17 @@ int main(int argc, char **argv) { set_progname(argv); mem_init(); if(!setlocale(LC_CTYPE, "")) fatal(errno, "error calling setlocale"); - while((n = getopt_long(argc, argv, "hVc:dDSs", options, 0)) >= 0) { + while((n = getopt_long(argc, argv, "hVc:dDSsxX", options, 0)) >= 0) { switch(n) { case 'h': help(); - case 'V': version(); + case 'V': version("disorder-dbupgrade"); case 'c': configfile = optarg; break; case 'd': debugging = 1; break; case 'D': debugging = 0; break; case 'S': logsyslog = 0; break; case 's': logsyslog = 1; break; + case 'x': badkey = BADKEY_DELETE; break; + case 'X': badkey = BADKEY_FAIL; break; default: fatal(0, "invalid option"); } }