From 3dc3d7dbde4b666c7e829a27d5838e5ed49bdbe6 Mon Sep 17 00:00:00 2001 Message-Id: <3dc3d7dbde4b666c7e829a27d5838e5ed49bdbe6.1714618811.git.mdw@distorted.org.uk> From: Mark Wooding Date: Wed, 21 Nov 2007 19:15:50 +0000 Subject: [PATCH] database upgrade tool. needs to be run manually. Organization: Straylight/Edgeware From: Richard Kettlewell --- .bzrignore | 3 + doc/Makefile.am | 6 +- doc/disorder-dbupgrade.8.in | 47 ++++++ server/Makefile.am | 7 +- server/dbupgrade.c | 308 ++++++++++++++++++++++++++++++++++++ server/dump.c | 2 +- server/rescan.c | 2 +- server/state.c | 2 +- server/stats.c | 2 +- server/trackdb-int.h | 2 + server/trackdb.c | 25 ++- server/trackdb.h | 2 +- tests/dbversion.py | 9 +- 13 files changed, 402 insertions(+), 15 deletions(-) create mode 100644 doc/disorder-dbupgrade.8.in create mode 100644 server/dbupgrade.c mode change 100644 => 100755 tests/dbversion.py diff --git a/.bzrignore b/.bzrignore index 259f5aa..1b45fb1 100644 --- a/.bzrignore +++ b/.bzrignore @@ -127,3 +127,6 @@ lib/SentenceBreakProperty.txt lib/WordBreakTest.txt lib/DerivedNormalizationProps.txt tests/*.log +server/disorder-dbupgrade +doc/disorder-dbupgrade.8.html +doc/disorder-dbupgrade.8 diff --git a/doc/Makefile.am b/doc/Makefile.am index d7e9187..3e8078c 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -21,7 +21,7 @@ SEDFILES=disorder.1 disorderd.8 disorder_config.5 \ disorder-dump.8 disorder_protocol.5 disorder-deadlock.8 \ disorder-rescan.8 disobedience.1 disorderfm.1 disorder-playrtp.1 \ - disorder-decode.8 disorder-stats.8 + disorder-decode.8 disorder-stats.8 disorder-dbupgrade.8 include ${top_srcdir}/scripts/sedfiles.make @@ -29,7 +29,7 @@ man_MANS=disorderd.8 disorder.1 disorder.3 disorder_config.5 disorder-dump.8 \ disorder_protocol.5 disorder-deadlock.8 \ disorder-rescan.8 disobedience.1 disorderfm.1 disorder-speaker.8 \ disorder-playrtp.1 disorder-normalize.8 disorder-decode.8 \ - disorder-stats.8 + disorder-stats.8 disorder-dbupgrade.8 noinst_MANS=tkdisorder.1 @@ -48,6 +48,6 @@ EXTRA_DIST=disorderd.8.in disorder.1.in disorder_config.5.in \ tkdisorder.1 disorder-deadlock.8.in disorder-rescan.8.in \ disobedience.1.in disorderfm.1.in disorder-speaker.8 \ disorder-playrtp.1.in disorder-decode.8.in disorder-normalize.8 \ - disorder-stats.8.in + disorder-stats.8.in disorder-dbupgrade.8.in CLEANFILES=$(SEDFILES) $(HTMLMAN) diff --git a/doc/disorder-dbupgrade.8.in b/doc/disorder-dbupgrade.8.in new file mode 100644 index 0000000..3d8a3b0 --- /dev/null +++ b/doc/disorder-dbupgrade.8.in @@ -0,0 +1,47 @@ +.\" +.\" Copyright (C) 2007 Richard Kettlewell +.\" +.\" This program is free software; you can redistribute it and/or modify +.\" it under the terms of the GNU General Public License as published by +.\" the Free Software Foundation; either version 2 of the License, or +.\" (at your option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, but +.\" WITHOUT ANY WARRANTY; without even the implied warranty of +.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +.\" General Public License for more details. +.\" +.\" You should have received a copy of the GNU General Public License +.\" along with this program; if not, write to the Free Software +.\" Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 +.\" USA +.\" +.TH disorder-dbupgrade 8 +.SH NAME +disorder-dbupgrade \- DisOrder Database Upgrader +.SH SYNOPSIS +.B disorder-dbupgrade +.RI [ OPTIONS ] +.RI [ PATH ...] +.SH DESCRIPTION +.B disorder-dbupgrade +is DisOrder's database upgrader. It is invoked by DisOrder when +necessary and does not need to be invoked manually. +.SH OPTIONS +.TP +.B --config \fIPATH\fR, \fB-c \fIPATH +Set the configuration file. +.TP +.B --debug\fR, \fB-d +Enable debugging. +.TP +.B --help\fR, \fB-h +Display a usage message. +.TP +.B --version\fR, \fB-V +Display version number. +.SH "SEE ALSO" +\fBdisorderd\fR(8), \fBdisorder_config\fR(5) +.\" Local Variables: +.\" mode:nroff +.\" End: diff --git a/server/Makefile.am b/server/Makefile.am index bdb5071..d1f2fd7 100644 --- a/server/Makefile.am +++ b/server/Makefile.am @@ -20,7 +20,7 @@ sbin_PROGRAMS=disorderd disorder-deadlock disorder-rescan disorder-dump \ disorder-speaker disorder-decode disorder-normalize \ - disorder-stats + disorder-stats disorder-dbupgrade noinst_PROGRAMS=disorder.cgi trackname noinst_DATA=uk.org.greenend.rjk.disorder.plist @@ -88,6 +88,11 @@ disorder_dump_LDADD=$(LIBOBJS) ../lib/libdisorder.a \ $(LIBPCRE) $(LIBDB) $(LIBICONV) $(LIBGC) $(LIBGCRYPT) disorder_dump_DEPENDENCIES=$(LIBOBJS) ../lib/libdisorder.a +disorder_dbupgrade_SOURCES=dbupgrade.c trackdb.c trackdb.h ../lib/memgc.c +disorder_dbupgrade_LDADD=$(LIBOBJS) ../lib/libdisorder.a \ + $(LIBDB) $(LIBGC) $(LIBPCRE) $(LIBICONV) $(LIBGCRYPT) +disorder_dbupgrade_DEPENDENCIES=../lib/libdisorder.a + disorder_cgi_SOURCES=dcgi.c dcgi.h \ api.c api-client.c api-client.h \ cgi.c cgi.h cgimain.c exports.c diff --git a/server/dbupgrade.c b/server/dbupgrade.c new file mode 100644 index 0000000..492cf64 --- /dev/null +++ b/server/dbupgrade.c @@ -0,0 +1,308 @@ +/* + * This file is part of DisOrder + * Copyright (C) 2007 Richard Kettlewell + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + */ + +#include +#include "types.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "syscalls.h" +#include "log.h" +#include "defs.h" +#include "kvp.h" +#include "trackdb.h" +#include "trackdb-int.h" +#include "mem.h" +#include "configuration.h" +#include "unicode.h" + +static DB_TXN *global_tid; + +static const struct option options[] = { + { "help", no_argument, 0, 'h' }, + { "version", no_argument, 0, 'V' }, + { "config", required_argument, 0, 'c' }, + { "debug", no_argument, 0, 'd' }, + { "no-debug", no_argument, 0, 'D' }, + { "syslog", no_argument, 0, 's' }, + { "no-syslog", no_argument, 0, 'S' }, + { 0, 0, 0, 0 } +}; + +/* display usage message and terminate */ +static void help(void) { + xprintf("Usage:\n" + " disorder-dbupgrade [OPTIONS]\n" + "Options:\n" + " --help, -h Display usage message\n" + " --version, -V Display version number\n" + " --config PATH, -c PATH Set configuration file\n" + " --debug, -d Turn on debugging\n" + " --[no-]syslog Force logging\n" + "\n" + "Database upgrader for DisOrder. Not intended to be run\n" + "directly.\n"); + xfclose(stdout); + exit(0); +} + +/* display version number and terminate */ +static void version(void) { + xprintf("disorder-dbupgrade version %s\n", disorder_version_string); + xfclose(stdout); + exit(0); +} + +/** @brief Visit each key in a database and call @p callback + * @return 0 or DB_LOCK_DEADLOCK + * + * @p global_tid must be set. @p callback should return 0 or DB_LOCK_DEADLOCK. + */ +static int scan_core(const char *name, DB *db, + int (*callback)(const char *name, DB *db, DBC *c, + DBT *k, DBT *d)) { + long count = 0; + DBC *c = trackdb_opencursor(db, global_tid); + int err, r = 0; + DBT k[1], d[1]; + + memset(k, 0, sizeof k); + memset(d, 0, sizeof d); + while((err = c->c_get(c, k, d, DB_NEXT)) == 0) { + if((err = callback(name, db, c, k, d))) + break; + ++count; + if(count % 1000 == 0) + info("scanning %s, %ld so far", name, count); + } + if(err && err != DB_NOTFOUND && err != DB_LOCK_DEADLOCK) + fatal(0, "%s: error scanning database: %s", name, db_strerror(err)); + r = (err == DB_LOCK_DEADLOCK ? err : 0); + if((err = c->c_close(c))) + fatal(0, "%s: error closing cursor: %s", name, db_strerror(err)); + return r; +} + +/** @brief Visit each key in a database and call @p callback + * + * Everything happens inside the @p global_tid tranasction. @p callback + * should return 0 or DB_LOCK_DEADLOCK. + */ +static void scan(const char *name, DB *db, + int (*callback)(const char *name, DB *db, DBC *c, + DBT *k, DBT *d)) { + info("scanning %s", name); + for(;;) { + global_tid = trackdb_begin_transaction(); + if(scan_core(name, db, callback)) { + trackdb_abort_transaction(global_tid); + global_tid = 0; + error(0, "detected deadlock, restarting scan"); + continue; + } else { + trackdb_commit_transaction(global_tid); + global_tid = 0; + break; + } + } +} + +/** @brief Truncate database @p db */ +static void truncate_database(const char *name, DB *db) { + u_int32_t count; + int err; + + do { + err = db->truncate(db, 0, &count, DB_AUTO_COMMIT); + } while(err == DB_LOCK_DEADLOCK); + if(err) + fatal(0, "error truncating %s: %s", name, db_strerror(err)); +} + +/* scan callbacks */ + +static int normalize_keys(const char *name, DB *db, DBC *c, + DBT *k, DBT *d) { + char *knfc; + size_t nknfc; + int err; + + /* Find the normalized form of the key */ + knfc = utf8_compose_canon(k->data, k->size, &nknfc); + if(!knfc) + fatal(0, "%s: cannot convert key to NFC: %.*s", name, + (int)k->size, (const char *)k->data); + /* If the key is already in NFC then do nothing */ + if(nknfc == k->size && !memcmp(k->data, knfc, nknfc)) + return 0; + /* To rename the key we must delete the old one and insert a new one */ + if((err = c->c_del(c, 0))) { + if(err != DB_LOCK_DEADLOCK) + fatal(0, "%s: error removing denormalized key: %s", + name, db_strerror(err)); + return err; + } + k->size = nknfc; + k->data = knfc; + if((err = db->put(db, global_tid, k, d, DB_NOOVERWRITE))) { + if(err != DB_LOCK_DEADLOCK) + fatal(0, "%s: error storing normalized key: %s", name, db_strerror(err)); + return err; + } + return 0; +} + +static int normalize_values(const char *name, DB *db, + DBC attribute((unused)) *c, + DBT *k, DBT *d) { + char *dnfc; + size_t ndnfc; + int err; + + /* Find the normalized form of the value */ + dnfc = utf8_compose_canon(d->data, d->size, &ndnfc); + if(!dnfc) + fatal(0, "%s: cannot convert data to NFC: %.*s", name, + (int)d->size, (const char *)d->data); + /* If the key is already in NFC then do nothing */ + if(ndnfc == d->size && !memcmp(d->data, dnfc, ndnfc)) + return 0; + d->size = ndnfc; + d->data = dnfc; + if((err = db->put(db, global_tid, k, d, 0))) { + if(err != DB_LOCK_DEADLOCK) + fatal(0, "%s: error storing normalized data: %s", name, db_strerror(err)); + return err; + } + return 0; +} + +static int renotice(const char *name, DB attribute((unused)) *db, + DBC attribute((unused)) *c, + DBT *k, DBT *d) { + const struct kvp *const t = kvp_urldecode(d->data, d->size); + const char *const track = xstrndup(k->data, k->size); + const char *const path = kvp_get(t, "_path"); + int err; + + if(!path) + fatal(0, "%s: no '_path' for %.*s", name, + (int)k->size, (const char *)k->data); + switch(err = trackdb_notice_tid(track, path, global_tid)) { + case 0: + return 0; + case DB_LOCK_DEADLOCK: + return err; + default: + fatal(0, "%s: unexpected return from trackdb_notice_tid: %s", + name, db_strerror(err)); + } +} + +static int remove_aliases_normalize_keys(const char *name, DB *db, DBC *c, + DBT *k, DBT *d) { + const struct kvp *const t = kvp_urldecode(d->data, d->size); + int err; + + if(kvp_get(t, "_alias_for")) { + /* This is an alias. We remove all the alias entries. */ + if((err = c->c_del(c, 0))) { + if(err != DB_LOCK_DEADLOCK) + fatal(0, "%s: error removing alias: %s", name, db_strerror(err)); + return err; + } + return 0; + } + return normalize_keys(name, db, c, k, d); +} + +/** @brief Upgrade the database to the current version + * + * This function is supposed to be idempotent, so if it is interrupted + * half way through it is safe to restart. + */ +static void upgrade(void) { + char buf[32]; + + info("upgrading database to dbversion %ld", config->dbversion); + /* Normalize keys and values as required. We will also remove aliases as + * they will be regenerated when we re-noticed the tracks. */ + info("renormalizing keys"); + scan("tracks.db", trackdb_tracksdb, remove_aliases_normalize_keys); + scan("prefs.db", trackdb_prefsdb, normalize_keys); + scan("global.db", trackdb_globaldb, normalize_keys); + scan("noticed.db", trackdb_noticeddb, normalize_values); + /* search.db and tags.db we will rebuild */ + info("regenerating search database and aliases"); + truncate_database("search.db", trackdb_searchdb); + truncate_database("tags.db", trackdb_tagsdb); + /* Regenerate the search database and aliases */ + scan("tracks.db", trackdb_tracksdb, renotice); + /* Finally update the database version */ + snprintf(buf, sizeof buf, "%ld", config->dbversion); + trackdb_set_global("_dbversion", buf, 0); + info("completed database upgrade"); +} + +int main(int argc, char **argv) { + int n, logsyslog = !isatty(2); + + set_progname(argv); + mem_init(); + if(!setlocale(LC_CTYPE, "")) fatal(errno, "error calling setlocale"); + while((n = getopt_long(argc, argv, "hVc:dDSs", options, 0)) >= 0) { + switch(n) { + case 'h': help(); + case 'V': version(); + case 'c': configfile = optarg; break; + case 'd': debugging = 1; break; + case 'D': debugging = 0; break; + case 'S': logsyslog = 0; break; + case 's': logsyslog = 1; break; + default: fatal(0, "invalid option"); + } + } + /* If stderr is a TTY then log there, otherwise to syslog. */ + if(logsyslog) { + openlog(progname, LOG_PID, LOG_DAEMON); + log_default = &log_syslog; + } + if(config_read(0)) fatal(0, "cannot read configuration"); + /* Open the database */ + trackdb_init(0); + trackdb_open(1/*dbupgrade*/); + upgrade(); + return 0; +} + +/* +Local Variables: +c-basic-offset:2 +comment-column:40 +fill-column:79 +indent-tabs-mode:nil +End: +*/ diff --git a/server/dump.c b/server/dump.c index 25dd01e..cedcc73 100644 --- a/server/dump.c +++ b/server/dump.c @@ -425,7 +425,7 @@ int main(int argc, char **argv) { } if(config_read(0)) fatal(0, "cannot read configuration"); trackdb_init(recover); - trackdb_open(); + trackdb_open(0); if(dump) { /* we write to a temporary file and rename into place */ byte_xasprintf(&tmp, "%s.%lx.tmp", path, (unsigned long)getpid()); diff --git a/server/rescan.c b/server/rescan.c index cc31888..b2beac8 100644 --- a/server/rescan.c +++ b/server/rescan.c @@ -362,7 +362,7 @@ int main(int argc, char **argv) { xsigaction(SIGINT, &sa, 0); info("started"); trackdb_init(0); - trackdb_open(); + trackdb_open(0); if(optind == argc) { /* Rescan all collections */ do_all(rescan_collection); diff --git a/server/state.c b/server/state.c index 4ff2654..91bc078 100644 --- a/server/state.c +++ b/server/state.c @@ -152,7 +152,7 @@ int reconfigure(ev_source *ev, int reload) { info("%s: installed new configuration", configfile); } } - trackdb_open(); + trackdb_open(0); if(need_another_rescan) trackdb_rescan(ev); if(!ret) { diff --git a/server/stats.c b/server/stats.c index f23ecf9..0c354e6 100644 --- a/server/stats.c +++ b/server/stats.c @@ -96,7 +96,7 @@ int main(int argc, char **argv) { if(config_read(0)) fatal(0, "cannot read configuration"); trackdb_init(0); - trackdb_open(); + trackdb_open(0); stats = trackdb_stats(0); while(*stats) xprintf("%s\n", *stats++); diff --git a/server/trackdb-int.h b/server/trackdb-int.h index f728e0a..02b07b9 100644 --- a/server/trackdb-int.h +++ b/server/trackdb-int.h @@ -26,7 +26,9 @@ extern DB_ENV *trackdb_env; extern DB *trackdb_tracksdb; extern DB *trackdb_prefsdb; extern DB *trackdb_searchdb; +extern DB *trackdb_tagsdb; extern DB *trackdb_noticeddb; +extern DB *trackdb_globaldb; DBC *trackdb_opencursor(DB *db, DB_TXN *tid); /* open a transaction */ diff --git a/server/trackdb.c b/server/trackdb.c index b4e0de9..2318bd9 100644 --- a/server/trackdb.c +++ b/server/trackdb.c @@ -285,8 +285,10 @@ static DB *open_db(const char *path, return db; } -/* open track databases */ -void trackdb_open(void) { +/** @brief Open track databases + * @param dbupgrade Non-0 to allow non-current database versions + */ +void trackdb_open(int dbupgrade) { int newdb, err; /* sanity checks */ @@ -301,18 +303,31 @@ void trackdb_open(void) { s = trackdb_get_global("_dbversion"); oldversion = s ? atol(s) : 1; - if(oldversion != config->dbversion) { + if(oldversion > config->dbversion) { + /* Database is from the future */ + fatal(0, "this version of DisOrder is too old for database version %ld", + oldversion); + } + if(oldversion < config->dbversion && !dbupgrade) { /* This database needs upgrading. This isn't implemented yet so we just * fail. */ fatal(0, "database needs upgrading from %ld to %ld", oldversion, config->dbversion); } + if(oldversion == config->dbversion && dbupgrade) { + /* This doesn't make any sense */ + fatal(0, "database is already at current version"); + } newdb = 0; /* Close the database again, we'll open it property below */ if((err = trackdb_globaldb->close(trackdb_globaldb, 0))) fatal(0, "error closing global.db: %s", db_strerror(err)); trackdb_globaldb = 0; } else { + if(dbupgrade) { + /* Cannot upgrade a new database */ + fatal(0, "cannot upgrade a database that does not exist"); + } /* This is a brand new database */ newdb = 1; } @@ -327,8 +342,8 @@ void trackdb_open(void) { trackdb_globaldb = open_db("global.db", 0, DB_HASH, DB_CREATE, 0666); trackdb_noticeddb = open_db("noticed.db", DB_DUPSORT, DB_BTREE, DB_CREATE, 0666); - /* Stash the database version */ - if(newdb) { + if(newdb && !dbupgrade) { + /* Stash the database version */ char buf[32]; snprintf(buf, sizeof buf, "%ld", config->dbversion); diff --git a/server/trackdb.h b/server/trackdb.h index fe43474..9fca722 100644 --- a/server/trackdb.h +++ b/server/trackdb.h @@ -37,7 +37,7 @@ void trackdb_deinit(void); void trackdb_master(struct ev_source *ev); /* start deadlock manager */ -void trackdb_open(void); +void trackdb_open(int dbupgrade); void trackdb_close(void); /* open/close track databases */ diff --git a/tests/dbversion.py b/tests/dbversion.py old mode 100644 new mode 100755 index 9849bf3..945d7ad --- a/tests/dbversion.py +++ b/tests/dbversion.py @@ -18,7 +18,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # -import dtest,time,disorder,sys,re +import dtest,time,disorder,sys,re,subprocess def test(): """Database version tests""" @@ -48,6 +48,13 @@ def test(): dtest.stop_daemon() if not ok: sys.exit(1) + # Try running the upgrade tool + print "Attempting an upgrade..." + rc = subprocess.call(["disorder-dbupgrade", + "--config", "%s/config" % dtest.testroot]) + if rc != 0: + print "disorder-dbupgrade: FAILED: exit code %s" % rc + sys.exit(1) if __name__ == '__main__': dtest.run() -- [mdw]