From: Richard Kettlewell Date: Thu, 22 Nov 2007 13:17:54 +0000 (+0000) Subject: sync with trunk X-Git-Tag: debian-1_5_99dev9~1^2~9 X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~mdw/git/disorder/commitdiff_plain/f0c20432f8c9dffd049c785f21cc85d98ff628fd?ds=sidebyside;hp=-c sync with trunk --- f0c20432f8c9dffd049c785f21cc85d98ff628fd diff --combined server/trackdb.c index c977b6a,432d212..cb6b45a --- a/server/trackdb.c +++ b/server/trackdb.c @@@ -17,8 -17,6 +17,8 @@@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * USA */ +/** @file server/trackdb.c + * @brief Track database */ #include #include "types.h" @@@ -47,6 -45,7 +47,6 @@@ #include "configuration.h" #include "syscalls.h" #include "wstat.h" -#include "words.h" #include "printf.h" #include "filepart.h" #include "trackname.h" @@@ -55,8 -54,6 +55,8 @@@ #include "cache.h" #include "eventlog.h" #include "hash.h" +#include "unicode.h" +#include "unidata.h" #define RESCAN "disorder-rescan" #define DEADLOCK "disorder-deadlock" @@@ -67,6 -64,9 +67,6 @@@ static const char *getpart(const char * const struct kvp *p, int *used_db); static int trackdb_alltags_tid(DB_TXN *tid, char ***taglistp); -static int trackdb_get_global_tid(const char *name, - DB_TXN *tid, - const char **rp); static char **trackdb_new_tid(int *ntracksp, int maxtracks, DB_TXN *tid); @@@ -79,59 -79,11 +79,59 @@@ unsigned long cache_files_hits, cache_f static const char *home; /* home had better not change */ DB_ENV *trackdb_env; /* db environment */ -DB *trackdb_tracksdb; /* the db itself */ -DB *trackdb_prefsdb; /* preferences */ -DB *trackdb_searchdb; /* the search database */ + +/** @brief The tracks database + * - Keys are UTF-8(NFC(unicode(path name))) + * - Values are encoded key-value pairs + * - Data is reconstructable data about tracks that currently exist + */ +DB *trackdb_tracksdb; + +/** @brief The preferences database + * + * - Keys are UTF-8(NFC(unicode(path name))) + * - Values are encoded key-value pairs + * - Data is user data about tracks (that might not exist any more) + * and cannot be reconstructed + */ +DB *trackdb_prefsdb; + +/** @brief The search database + * + * - Keys are UTF-8(NFKC(casefold(search term))) + * - Values are UTF-8(NFC(unicode(path name))) + * - There can be more than one value per key + * - Presence of key,value means that path matches the search terms + * - Only tracks fond in @ref tracks_tracksdb are represented here + * - This database can be reconstructed, it contains no user data + */ +DB *trackdb_searchdb; + +/** @brief The tags database + * + * - Keys are UTF-8(NFKC(casefold(tag))) + * - Values are UTF-8(NFC(unicode(path name))) + * - There can be more than one value per key + * - Presence of key,value means that path matches the tag + * - This is always in sync with the tags preference + * - This database can be reconstructed, it contains no user data + */ DB *trackdb_tagsdb; /* the tags database */ + +/** @brief The global preferences database + * - Keys are UTF-8(NFC(preference)) + * - Values are global preference values + * - Data is user data and cannot be reconstructed + */ DB *trackdb_globaldb; /* global preferences */ + +/** @brief The noticed database + * - Keys are 64-bit big-endian timestamps + * - Values are UTF-8(NFC(unicode(path name))) + * - There can be more than one value per key + * - Presence of key,value means that path was added at the given time + * - Data cannot be reconstructed (but isn't THAT important) + */ DB *trackdb_noticeddb; /* when track noticed */ static pid_t db_deadlock_pid = -1; /* deadlock manager PID */ static pid_t rescan_pid = -1; /* rescanner PID */ @@@ -147,17 -99,9 +147,17 @@@ static int compare(DB attribute((unused return compare_path_raw(a->data, a->size, b->data, b->size); } -/* open environment */ -void trackdb_init(int recover) { +/** @brief Open database environment + * @param flags Flags word + * + * Flags should be one of: + * - @ref TRACKDB_NO_RECOVER + * - @ref TRACKDB_NORMAL_RECOVER + * - @ref TRACKDB_FATAL_RECOVER + */ +void trackdb_init(int flags) { int err; + const int recover = flags & TRACKDB_RECOVER_MASK; static int recover_type[] = { 0, DB_RECOVER, DB_RECOVER_FATAL }; /* sanity checks */ @@@ -288,86 -232,16 +288,86 @@@ static DB *open_db(const char *path if((err = db->set_bt_compare(db, compare))) fatal(0, "db->set_bt_compare %s: %s", path, db_strerror(err)); if((err = db->open(db, 0, path, 0, dbtype, - openflags | DB_AUTO_COMMIT, mode))) - fatal(0, "db->open %s: %s", path, db_strerror(err)); + openflags | DB_AUTO_COMMIT, mode))) { + if((openflags & DB_CREATE) || errno != ENOENT) + fatal(0, "db->open %s: %s", path, db_strerror(err)); + db->close(db, 0); + db = 0; + } return db; } -/* open track databases */ -void trackdb_open(void) { +/** @brief Open track databases + * @param Flags flags word + * + * @p flags should be one of: + * - @p TRACKDB_NO_UPGRADE, if no upgrade should be attempted + * - @p TRACKDB_CAN_UPGRADE, if an upgrade may be attempted + * - @p TRACKDB_OPEN_FOR_UPGRADE, if this is disorder-dbupgrade + */ +void trackdb_open(int flags) { + int newdb, err; + pid_t pid; + /* sanity checks */ assert(opened == 0); ++opened; + /* check the database version first */ + trackdb_globaldb = open_db("global.db", 0, DB_HASH, 0, 0666); + if(trackdb_globaldb) { + /* This is an existing database */ + const char *s; + long oldversion; + + s = trackdb_get_global("_dbversion"); + /* Close the database again, we'll open it property below */ + if((err = trackdb_globaldb->close(trackdb_globaldb, 0))) + fatal(0, "error closing global.db: %s", db_strerror(err)); + trackdb_globaldb = 0; + /* Convert version string to an integer */ + oldversion = s ? atol(s) : 1; + if(oldversion > config->dbversion) { + /* Database is from the future; we never allow this. */ + fatal(0, "this version of DisOrder is too old for database version %ld", + oldversion); + } + if(oldversion < config->dbversion) { + /* Database version is out of date */ + switch(flags & TRACKDB_UPGRADE_MASK) { + case TRACKDB_NO_UPGRADE: + /* This database needs upgrading but this is not permitted */ + fatal(0, "database needs upgrading from %ld to %ld", + oldversion, config->dbversion); + case TRACKDB_CAN_UPGRADE: + /* This database needs upgrading */ + info("invoking disorder-dbupgrade to upgrade from %ld to %ld", + oldversion, config->dbversion); + pid = subprogram(0, "disorder-dbupgrade", -1); + while(waitpid(pid, &err, 0) == -1 && errno == EINTR) + ; + if(err) + fatal(0, "disorder-dbupgrade %s", wstat(err)); + info("disorder-dbupgrade succeeded"); + break; + case TRACKDB_OPEN_FOR_UPGRADE: + break; + default: + abort(); + } + } + if(oldversion == config->dbversion && (flags & TRACKDB_OPEN_FOR_UPGRADE)) { + /* This doesn't make any sense */ + fatal(0, "database is already at current version"); + } + newdb = 0; + } else { + if(flags & TRACKDB_OPEN_FOR_UPGRADE) { + /* Cannot upgrade a new database */ + fatal(0, "cannot upgrade a database that does not exist"); + } + /* This is a brand new database */ + newdb = 1; + } /* open the databases */ trackdb_tracksdb = open_db("tracks.db", DB_RECNUM, DB_BTREE, DB_CREATE, 0666); @@@ -379,14 -253,6 +379,14 @@@ trackdb_globaldb = open_db("global.db", 0, DB_HASH, DB_CREATE, 0666); trackdb_noticeddb = open_db("noticed.db", DB_DUPSORT, DB_BTREE, DB_CREATE, 0666); + if(newdb) { + /* Stash the database version */ + char buf[32]; + + assert(!(flags & TRACKDB_OPEN_FOR_UPGRADE)); + snprintf(buf, sizeof buf, "%ld", config->dbversion); + trackdb_set_global("_dbversion", buf, 0); + } D(("opened databases")); } @@@ -627,50 -493,24 +627,50 @@@ static int is_display_pref(const char * return !strncmp(name, prefix, (sizeof prefix) - 1); } +/** @brief Word_Break property tailor that treats underscores as spaces */ +static int tailor_underscore_Word_Break_Other(uint32_t c) { + switch(c) { + default: + return -1; + case 0x005F: /* LOW LINE (SPACING UNDERSCORE) */ + return unicode_Word_Break_Other; + } +} + +/** @brief Normalize and split a string using a given tailoring */ +static void word_split(struct vector *v, + const char *s, + unicode_property_tailor *pt) { + size_t nw, nt32, i; + uint32_t *t32, **w32; + + /* Convert to UTF-32 */ + if(!(t32 = utf8_to_utf32(s, strlen(s), &nt32))) + return; + /* Erase case distinctions */ + if(!(t32 = utf32_casefold_compat(t32, nt32, &nt32))) + return; + /* Split into words, treating _ as a space */ + w32 = utf32_word_split(t32, nt32, &nw, pt); + /* Convert words back to UTF-8 and append to result */ + for(i = 0; i < nw; ++i) + vector_append(v, utf32_to_utf8(w32[i], utf32_len(w32[i]), 0)); +} + /* compute the words of a track name */ static char **track_to_words(const char *track, const struct kvp *p) { struct vector v; - char **w; - int nw; const char *rootless = track_rootless(track); if(!rootless) rootless = track; /* bodge */ vector_init(&v); - if((w = words(casefold(strip_extension(rootless)), &nw))) - vector_append_many(&v, w, nw); - + rootless = strip_extension(rootless); + word_split(&v, strip_extension(rootless), tailor_underscore_Word_Break_Other); for(; p; p = p->next) if(is_display_pref(p->name)) - if((w = words(casefold(p->value), &nw))) - vector_append_many(&v, w, nw); + word_split(&v, p->value, 0); vector_terminate(&v); return dedupe(v.vec, v.nvec); } @@@ -862,9 -702,6 +862,9 @@@ int trackdb_notice(const char *track } /** @brief notice a possibly new track + * @param track NFC UTF-8 track name + * @param path Raw path name + * @param tid Transaction ID * @return @c DB_NOTFOUND if new, 0 if already known, @c DB_LOCK_DEADLOCK also */ int trackdb_notice_tid(const char *track, @@@ -1819,7 -1656,7 +1819,7 @@@ char **trackdb_search(char **wordlist, /* casefold all the words */ w = xmalloc(nwordlist * sizeof (char *)); for(n = 0; n < nwordlist; ++n) { - w[n] = casefold(wordlist[n]); + w[n] = utf8_casefold_compat(wordlist[n], strlen(wordlist[n]), 0); if(checktag(w[n])) ++ntags; /* count up tags */ } /* find the longest non-stopword */ @@@ -1981,9 -1818,9 +1981,9 @@@ static int reap_rescan(ev_source attrib void attribute((unused)) *u) { if(pid == rescan_pid) rescan_pid = -1; if(status) - error(0, "disorderd-rescan: %s", wstat(status)); + error(0, RESCAN": %s", wstat(status)); else - D(("disorderd-rescan terminate: %s", wstat(status))); + D((RESCAN" terminated: %s", wstat(status))); /* Our cache of file lookups is out of date now */ cache_clean(&cache_files_type); eventlog("rescanned", (char *)0); @@@ -1991,14 -1828,22 +1991,22 @@@ } void trackdb_rescan(ev_source *ev) { + int w; + if(rescan_pid != -1) { error(0, "rescan already underway"); return; } rescan_pid = subprogram(ev, RESCAN, -1); - ev_child(ev, rescan_pid, 0, reap_rescan, 0); - D(("started rescanner")); - + if(ev) { + ev_child(ev, rescan_pid, 0, reap_rescan, 0); + D(("started rescanner")); + } else { + /* This is the first rescan, we block until it is complete */ + while(waitpid(rescan_pid, &w, 0) < 0 && errno == EINTR) + ; + reap_rescan(0, rescan_pid, w, 0, 0); + } } int trackdb_rescan_cancel(void) { @@@ -2015,13 -1860,27 +2023,13 @@@ void trackdb_set_global(const char *nam const char *value, const char *who) { DB_TXN *tid; - DBT k, d; int err; int state; - memset(&k, 0, sizeof k); - memset(&d, 0, sizeof d); - k.data = (void *)name; - k.size = strlen(name); - if(value) { - d.data = (void *)value; - d.size = strlen(value); - } for(;;) { tid = trackdb_begin_transaction(); - if(value) - err = trackdb_globaldb->put(trackdb_globaldb, tid, &k, &d, 0); - else - err = trackdb_globaldb->del(trackdb_globaldb, tid, &k, 0); - if(!err || err == DB_NOTFOUND) break; - if(err != DB_LOCK_DEADLOCK) - fatal(0, "error updating database: %s", db_strerror(err)); + if(!(err = trackdb_set_global_tid(name, value, tid))) + break; trackdb_abort_transaction(tid); } trackdb_commit_transaction(tid); @@@ -2044,30 -1903,6 +2052,30 @@@ reqtracks = 0; } +int trackdb_set_global_tid(const char *name, + const char *value, + DB_TXN *tid) { + DBT k, d; + int err; + + memset(&k, 0, sizeof k); + memset(&d, 0, sizeof d); + k.data = (void *)name; + k.size = strlen(name); + if(value) { + d.data = (void *)value; + d.size = strlen(value); + } + if(value) + err = trackdb_globaldb->put(trackdb_globaldb, tid, &k, &d, 0); + else + err = trackdb_globaldb->del(trackdb_globaldb, tid, &k, 0); + if(err == DB_LOCK_DEADLOCK) return err; + if(err) + fatal(0, "error updating database: %s", db_strerror(err)); + return 0; +} + const char *trackdb_get_global(const char *name) { DB_TXN *tid; int err; @@@ -2083,9 -1918,9 +2091,9 @@@ return r; } -static int trackdb_get_global_tid(const char *name, - DB_TXN *tid, - const char **rp) { +int trackdb_get_global_tid(const char *name, + DB_TXN *tid, + const char **rp) { DBT k, d; int err; @@@ -2103,7 -1938,7 +2111,7 @@@ case DB_LOCK_DEADLOCK: return err; default: - fatal(0, "error updating database: %s", db_strerror(err)); + fatal(0, "error reading database: %s", db_strerror(err)); } } diff --combined tests/dbversion.py index 78788b1,0000000..d7f58a5 mode 100755,000000..100755 --- a/tests/dbversion.py +++ b/tests/dbversion.py @@@ -1,41 -1,0 +1,39 @@@ +#! /usr/bin/env python +# +# This file is part of DisOrder. +# Copyright (C) 2007 Richard Kettlewell +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 +# USA +# +import dtest,time,disorder,sys,re,subprocess + +def test(): + """Database version tests""" + # Start up with dbversion 1 + config = "%s/config" % dtest.testroot + configsave = "%s.save" % config + dtest.copyfile(config, configsave) + open(config, "a").write("dbversion 1\n") + dtest.start_daemon() - time.sleep(2) + dtest.stop_daemon() + # Revert to default configuration + dtest.copyfile(configsave, config) + print "Testing daemon manages to upgrade..." + dtest.start_daemon() - time.sleep(4) + assert dtest.check_files() == 0 + +if __name__ == '__main__': + dtest.run() diff --combined tests/dtest.py index 00c5ead,6ae9fd7..ab76991 --- a/tests/dtest.py +++ b/tests/dtest.py @@@ -21,7 -21,7 +21,7 @@@ """Utility module used by tests""" - import os,os.path,subprocess,sys,re,unicodedata -import os,os.path,subprocess,sys,re,time ++import os,os.path,subprocess,sys,re,time,unicodedata def fatal(s): """Write an error message and exit""" @@@ -75,51 -75,11 +75,51 @@@ Make track with relative path S exist"" if not os.path.exists(trackdir): os.makedirs(trackdir) copyfile("%s/sounds/slap.ogg" % top_srcdir, trackpath) + # We record the tracks we created so they can be tested against + # server responses. We put them into NFC since that's what the server + # uses internally. + bits = unicodedata.normalize("NFC", + unicode(s, "UTF-8")).split('/') + dp = tracks + for d in bits [0:-1]: + dd = "%s/%s" % (dp, d) + if dp not in dirs_by_dir: + dirs_by_dir[dp] = [] + if dd not in dirs_by_dir[dp]: + dirs_by_dir[dp].append(dd) + dp = "%s/%s" % (dp, d) + if dp not in files_by_dir: + files_by_dir[dp] = [] + files_by_dir[dp].append("%s/%s" % (dp, bits[-1])) def stdtracks(): - maketrack("Joe Bloggs/First Album/01:First track.ogg") + # We create some tracks with non-ASCII characters in the name and + # we (currently) force UTF-8. + # + # On a traditional UNIX filesystem, that treats filenames as byte strings + # with special significant for '/', this should just work, though the + # names will look wrong to ls(1) in a non UTF-8 locale. + # + # On Apple HFS+ filenames normalized to a decomposed form that isn't quite + # NFD, so our attempts to have both normalized and denormalized filenames + # is frustrated. Provided we test on traditional filesytsems too this + # shouldn't be a problem. + # (See http://developer.apple.com/qa/qa2001/qa1173.html) + + global dirs_by_dir, files_by_dir + dirs_by_dir={} + files_by_dir={} + + # C3 8C = 00CC LATIN CAPITAL LETTER I WITH GRAVE + # (in NFC) + maketrack("Joe Bloggs/First Album/01:F\xC3\x8Crst track.ogg") + maketrack("Joe Bloggs/First Album/02:Second track.ogg") - maketrack("Joe Bloggs/First Album/03:Third track.ogg") + + # CC 81 = 0301 COMBINING ACUTE ACCENT + # (giving an NFD i-acute) + maketrack("Joe Bloggs/First Album/03:ThI\xCC\x81rd track.ogg") + # ...hopefuly giving C3 8D = 00CD LATIN CAPITAL LETTER I WITH ACUTE maketrack("Joe Bloggs/First Album/04:Fourth track.ogg") maketrack("Joe Bloggs/First Album/05:Fifth track.ogg") maketrack("Joe Bloggs/Second Album/01:First track.ogg") @@@ -139,7 -99,7 +139,7 @@@ maketrack("misc/blahblahblah.ogg") maketrack("Various/Greatest Hits/01:Jim Whatever - Spong.ogg") maketrack("Various/Greatest Hits/02:Joe Bloggs - Yadda.ogg") - + def common_setup(): remove_dir(testroot) os.mkdir(testroot) @@@ -177,10 -137,34 +177,32 @@@ Start the daemon."" global daemon, errs assert daemon == None print " starting daemon" + # remove the socket if it exists + socket = "%s/socket" % testroot + try: + os.remove(socket) + except: + pass daemon = subprocess.Popen(["disorderd", "--foreground", "--config", "%s/config" % testroot], stderr=errs) - disorder._configfile = "%s/config" % testroot - disorder._userconf = False + # Wait for the socket to be created + waited = 0 + while not os.path.exists(socket): + rc = daemon.poll() + if rc is not None: + print "FATAL: daemon failed to start up" + sys.exit(1) + waited += 1 + if waited == 1: + print " waiting for socket..." + elif waited >= 60: + print "FATAL: took too long for socket to appear" + sys.exit(1) + time.sleep(1) + if waited > 0: + print " took about %ds for socket to appear" % waited def stop_daemon(): """stop_daemon() @@@ -193,8 -177,11 +215,11 @@@ Stop the daemon if it has not stopped a if rc == None: print " stopping daemon" os.kill(daemon.pid, 15) + print " waiting for daemon" rc = daemon.wait() - print " daemon has stopped" + print " daemon has stopped" + else: + print " daemon already stopped" daemon = None def run(module=None, report=True): @@@ -253,33 -240,6 +278,33 @@@ Recursively delete directory D"" else: os.remove(d) +def check_files(): + c = disorder.client() + failures = 0 + for d in dirs_by_dir: + xdirs = dirs_by_dir[d] + dirs = c.directories(d) + xdirs.sort() + dirs.sort() + if dirs != xdirs: + print + print "directory: %s" % d + print "expected: %s" % xdirs + print "got: %s" % dirs + failures += 1 + for d in files_by_dir: + xfiles = files_by_dir[d] + files = c.files(d) + xfiles.sort() + files.sort() + if files != xfiles: + print + print "directory: %s" % d + print "expected: %s" % xfiles + print "got: %s" % files + failures += 1 + return failures + # ----------------------------------------------------------------------------- # Common setup diff --combined tests/files.py index 24d4ea4,0000000..3fc19f4 mode 100755,000000..100755 --- a/tests/files.py +++ b/tests/files.py @@@ -1,30 -1,0 +1,29 @@@ +#! /usr/bin/env python +# +# This file is part of DisOrder. +# Copyright (C) 2007 Richard Kettlewell +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 +# USA +# +import dtest,time,disorder,sys + +def test(): + """Check that the file listing comes out right""" + dtest.start_daemon() - time.sleep(2) # give rescan a chance + assert dtest.check_files() == 0 + +if __name__ == '__main__': + dtest.run()