chiark / gitweb /
disorder-choose rejects tracks in queue/recent list
[disorder] / server / dbupgrade.c
CommitLineData
3dc3d7db
RK
1/*
2 * This file is part of DisOrder
5aff007d 3 * Copyright (C) 2007, 2008 Richard Kettlewell
3dc3d7db
RK
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
18 * USA
19 */
20
21#include <config.h>
22#include "types.h"
23
24#include <string.h>
25#include <getopt.h>
26#include <db.h>
27#include <locale.h>
28#include <errno.h>
29#include <syslog.h>
30#include <pcre.h>
ad2f8275 31#include <unistd.h>
3dc3d7db
RK
32
33#include "syscalls.h"
34#include "log.h"
35#include "defs.h"
36#include "kvp.h"
5df73aeb 37#include "rights.h"
3dc3d7db
RK
38#include "trackdb.h"
39#include "trackdb-int.h"
40#include "mem.h"
41#include "configuration.h"
42#include "unicode.h"
3fbdc96d 43#include "version.h"
3dc3d7db
RK
44
45static DB_TXN *global_tid;
46
de3bd7ad 47#define BADKEY_WARN 0
48#define BADKEY_FAIL 1
49#define BADKEY_DELETE 2
50
51/** @brief Bad key behavior */
52static int badkey = BADKEY_WARN;
53
54static long aliases_removed, keys_normalized, values_normalized, renoticed;
55static long keys_already_ok, values_already_ok;
56
3dc3d7db
RK
57static const struct option options[] = {
58 { "help", no_argument, 0, 'h' },
59 { "version", no_argument, 0, 'V' },
60 { "config", required_argument, 0, 'c' },
61 { "debug", no_argument, 0, 'd' },
62 { "no-debug", no_argument, 0, 'D' },
de3bd7ad 63 { "delete-bad-keys", no_argument, 0, 'x' },
64 { "fail-bad-keys", no_argument, 0, 'X' },
3dc3d7db
RK
65 { "syslog", no_argument, 0, 's' },
66 { "no-syslog", no_argument, 0, 'S' },
67 { 0, 0, 0, 0 }
68};
69
70/* display usage message and terminate */
71static void help(void) {
72 xprintf("Usage:\n"
73 " disorder-dbupgrade [OPTIONS]\n"
74 "Options:\n"
75 " --help, -h Display usage message\n"
76 " --version, -V Display version number\n"
77 " --config PATH, -c PATH Set configuration file\n"
78 " --debug, -d Turn on debugging\n"
79 " --[no-]syslog Force logging\n"
de3bd7ad 80 " --delete-bad-keys, -x Delete unconvertible keys\n"
81 " --fail-bad-keys, -X Fail if bad keys are found\n"
3dc3d7db
RK
82 "\n"
83 "Database upgrader for DisOrder. Not intended to be run\n"
84 "directly.\n");
85 xfclose(stdout);
86 exit(0);
87}
88
3dc3d7db
RK
89/** @brief Visit each key in a database and call @p callback
90 * @return 0 or DB_LOCK_DEADLOCK
91 *
92 * @p global_tid must be set. @p callback should return 0 or DB_LOCK_DEADLOCK.
93 */
94static int scan_core(const char *name, DB *db,
95 int (*callback)(const char *name, DB *db, DBC *c,
96 DBT *k, DBT *d)) {
97 long count = 0;
98 DBC *c = trackdb_opencursor(db, global_tid);
99 int err, r = 0;
100 DBT k[1], d[1];
101
de3bd7ad 102 values_normalized = 0;
103 keys_normalized = 0;
104 aliases_removed = 0;
105 renoticed = 0;
106 keys_already_ok = 0;
107 values_already_ok = 0;
3dc3d7db
RK
108 memset(k, 0, sizeof k);
109 memset(d, 0, sizeof d);
110 while((err = c->c_get(c, k, d, DB_NEXT)) == 0) {
111 if((err = callback(name, db, c, k, d)))
112 break;
113 ++count;
114 if(count % 1000 == 0)
115 info("scanning %s, %ld so far", name, count);
116 }
117 if(err && err != DB_NOTFOUND && err != DB_LOCK_DEADLOCK)
118 fatal(0, "%s: error scanning database: %s", name, db_strerror(err));
119 r = (err == DB_LOCK_DEADLOCK ? err : 0);
120 if((err = c->c_close(c)))
121 fatal(0, "%s: error closing cursor: %s", name, db_strerror(err));
de3bd7ad 122 info("%s: %ld entries scanned", name, count);
123 if(values_normalized || values_already_ok)
124 info("%s: %ld values converted, %ld already ok", name,
125 values_normalized, values_already_ok);
126 if(keys_normalized || keys_already_ok)
127 info("%s: %ld keys converted, %ld already OK", name,
128 keys_normalized, keys_already_ok);
129 if(aliases_removed)
130 info("%s: %ld aliases removed", name, aliases_removed);
131 if(renoticed)
132 info("%s: %ld tracks re-noticed", name, renoticed);
3dc3d7db
RK
133 return r;
134}
135
136/** @brief Visit each key in a database and call @p callback
137 *
138 * Everything happens inside the @p global_tid tranasction. @p callback
139 * should return 0 or DB_LOCK_DEADLOCK.
140 */
141static void scan(const char *name, DB *db,
142 int (*callback)(const char *name, DB *db, DBC *c,
143 DBT *k, DBT *d)) {
144 info("scanning %s", name);
145 for(;;) {
146 global_tid = trackdb_begin_transaction();
147 if(scan_core(name, db, callback)) {
148 trackdb_abort_transaction(global_tid);
149 global_tid = 0;
150 error(0, "detected deadlock, restarting scan");
151 continue;
152 } else {
153 trackdb_commit_transaction(global_tid);
154 global_tid = 0;
155 break;
156 }
157 }
158}
159
160/** @brief Truncate database @p db */
161static void truncate_database(const char *name, DB *db) {
162 u_int32_t count;
163 int err;
164
165 do {
166 err = db->truncate(db, 0, &count, DB_AUTO_COMMIT);
167 } while(err == DB_LOCK_DEADLOCK);
168 if(err)
169 fatal(0, "error truncating %s: %s", name, db_strerror(err));
170}
171
172/* scan callbacks */
173
174static int normalize_keys(const char *name, DB *db, DBC *c,
175 DBT *k, DBT *d) {
176 char *knfc;
177 size_t nknfc;
178 int err;
179
180 /* Find the normalized form of the key */
181 knfc = utf8_compose_canon(k->data, k->size, &nknfc);
de3bd7ad 182 if(!knfc) {
183 switch(badkey) {
184 case BADKEY_WARN:
185 error(0, "%s: invalid key: %.*s", name,
186 (int)k->size, (const char *)k->data);
187 break;
188 case BADKEY_DELETE:
189 error(0, "%s: deleting invalid key: %.*s", name,
190 (int)k->size, (const char *)k->data);
191 if((err = c->c_del(c, 0))) {
192 if(err != DB_LOCK_DEADLOCK)
193 fatal(0, "%s: error removing denormalized key: %s",
194 name, db_strerror(err));
195 return err;
196 }
197 break;
198 case BADKEY_FAIL:
199 fatal(0, "%s: invalid key: %.*s", name,
200 (int)k->size, (const char *)k->data);
201 }
202 return 0;
203 }
3dc3d7db 204 /* If the key is already in NFC then do nothing */
de3bd7ad 205 if(nknfc == k->size && !memcmp(k->data, knfc, nknfc)) {
206 ++keys_already_ok;
3dc3d7db 207 return 0;
de3bd7ad 208 }
3dc3d7db
RK
209 /* To rename the key we must delete the old one and insert a new one */
210 if((err = c->c_del(c, 0))) {
211 if(err != DB_LOCK_DEADLOCK)
212 fatal(0, "%s: error removing denormalized key: %s",
213 name, db_strerror(err));
214 return err;
215 }
216 k->size = nknfc;
217 k->data = knfc;
218 if((err = db->put(db, global_tid, k, d, DB_NOOVERWRITE))) {
219 if(err != DB_LOCK_DEADLOCK)
220 fatal(0, "%s: error storing normalized key: %s", name, db_strerror(err));
221 return err;
222 }
de3bd7ad 223 ++keys_normalized;
3dc3d7db
RK
224 return 0;
225}
226
227static int normalize_values(const char *name, DB *db,
228 DBC attribute((unused)) *c,
229 DBT *k, DBT *d) {
230 char *dnfc;
231 size_t ndnfc;
232 int err;
233
234 /* Find the normalized form of the value */
235 dnfc = utf8_compose_canon(d->data, d->size, &ndnfc);
236 if(!dnfc)
237 fatal(0, "%s: cannot convert data to NFC: %.*s", name,
238 (int)d->size, (const char *)d->data);
239 /* If the key is already in NFC then do nothing */
de3bd7ad 240 if(ndnfc == d->size && !memcmp(d->data, dnfc, ndnfc)) {
241 ++values_already_ok;
3dc3d7db 242 return 0;
de3bd7ad 243 }
3dc3d7db
RK
244 d->size = ndnfc;
245 d->data = dnfc;
246 if((err = db->put(db, global_tid, k, d, 0))) {
247 if(err != DB_LOCK_DEADLOCK)
248 fatal(0, "%s: error storing normalized data: %s", name, db_strerror(err));
249 return err;
250 }
de3bd7ad 251 ++values_normalized;
3dc3d7db
RK
252 return 0;
253}
254
255static int renotice(const char *name, DB attribute((unused)) *db,
256 DBC attribute((unused)) *c,
257 DBT *k, DBT *d) {
258 const struct kvp *const t = kvp_urldecode(d->data, d->size);
259 const char *const track = xstrndup(k->data, k->size);
260 const char *const path = kvp_get(t, "_path");
261 int err;
262
de3bd7ad 263 if(!path) {
264 /* If an alias sorts later than the actual filename then it'll appear
265 * in the scan. */
266 if(kvp_get(t, "_alias_for"))
267 return 0;
3dc3d7db
RK
268 fatal(0, "%s: no '_path' for %.*s", name,
269 (int)k->size, (const char *)k->data);
de3bd7ad 270 }
3dc3d7db
RK
271 switch(err = trackdb_notice_tid(track, path, global_tid)) {
272 case 0:
de3bd7ad 273 ++renoticed;
3dc3d7db
RK
274 return 0;
275 case DB_LOCK_DEADLOCK:
276 return err;
277 default:
278 fatal(0, "%s: unexpected return from trackdb_notice_tid: %s",
279 name, db_strerror(err));
280 }
281}
282
283static int remove_aliases_normalize_keys(const char *name, DB *db, DBC *c,
284 DBT *k, DBT *d) {
285 const struct kvp *const t = kvp_urldecode(d->data, d->size);
286 int err;
287
288 if(kvp_get(t, "_alias_for")) {
289 /* This is an alias. We remove all the alias entries. */
290 if((err = c->c_del(c, 0))) {
291 if(err != DB_LOCK_DEADLOCK)
292 fatal(0, "%s: error removing alias: %s", name, db_strerror(err));
293 return err;
294 }
de3bd7ad 295 ++aliases_removed;
3dc3d7db 296 return 0;
de3bd7ad 297 } else if(!kvp_get(t, "_path"))
298 error(0, "%s: %.*s has neither _alias_for nor _path", name,
299 (int)k->size, (const char *)k->data);
3dc3d7db
RK
300 return normalize_keys(name, db, c, k, d);
301}
302
303/** @brief Upgrade the database to the current version
304 *
305 * This function is supposed to be idempotent, so if it is interrupted
306 * half way through it is safe to restart.
307 */
308static void upgrade(void) {
309 char buf[32];
310
311 info("upgrading database to dbversion %ld", config->dbversion);
312 /* Normalize keys and values as required. We will also remove aliases as
313 * they will be regenerated when we re-noticed the tracks. */
314 info("renormalizing keys");
315 scan("tracks.db", trackdb_tracksdb, remove_aliases_normalize_keys);
316 scan("prefs.db", trackdb_prefsdb, normalize_keys);
317 scan("global.db", trackdb_globaldb, normalize_keys);
318 scan("noticed.db", trackdb_noticeddb, normalize_values);
319 /* search.db and tags.db we will rebuild */
320 info("regenerating search database and aliases");
321 truncate_database("search.db", trackdb_searchdb);
322 truncate_database("tags.db", trackdb_tagsdb);
323 /* Regenerate the search database and aliases */
324 scan("tracks.db", trackdb_tracksdb, renotice);
325 /* Finally update the database version */
326 snprintf(buf, sizeof buf, "%ld", config->dbversion);
327 trackdb_set_global("_dbversion", buf, 0);
328 info("completed database upgrade");
329}
330
331int main(int argc, char **argv) {
332 int n, logsyslog = !isatty(2);
333
334 set_progname(argv);
335 mem_init();
336 if(!setlocale(LC_CTYPE, "")) fatal(errno, "error calling setlocale");
de3bd7ad 337 while((n = getopt_long(argc, argv, "hVc:dDSsxX", options, 0)) >= 0) {
3dc3d7db
RK
338 switch(n) {
339 case 'h': help();
3fbdc96d 340 case 'V': version("disorder-dbupgrade");
3dc3d7db
RK
341 case 'c': configfile = optarg; break;
342 case 'd': debugging = 1; break;
343 case 'D': debugging = 0; break;
344 case 'S': logsyslog = 0; break;
345 case 's': logsyslog = 1; break;
de3bd7ad 346 case 'x': badkey = BADKEY_DELETE; break;
347 case 'X': badkey = BADKEY_FAIL; break;
3dc3d7db
RK
348 default: fatal(0, "invalid option");
349 }
350 }
351 /* If stderr is a TTY then log there, otherwise to syslog. */
352 if(logsyslog) {
353 openlog(progname, LOG_PID, LOG_DAEMON);
354 log_default = &log_syslog;
355 }
356 if(config_read(0)) fatal(0, "cannot read configuration");
357 /* Open the database */
d25c4615
RK
358 trackdb_init(TRACKDB_NO_RECOVER);
359 trackdb_open(TRACKDB_OPEN_FOR_UPGRADE);
3dc3d7db
RK
360 upgrade();
361 return 0;
362}
363
364/*
365Local Variables:
366c-basic-offset:2
367comment-column:40
368fill-column:79
369indent-tabs-mode:nil
370End:
371*/