chiark / gitweb /
update CHANGES.html
[disorder] / server / dbupgrade.c
CommitLineData
3dc3d7db
RK
1/*
2 * This file is part of DisOrder
5aff007d 3 * Copyright (C) 2007, 2008 Richard Kettlewell
3dc3d7db
RK
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
18 * USA
19 */
05b75f8d 20#include "disorder-server.h"
3dc3d7db
RK
21
22static DB_TXN *global_tid;
23
de3bd7ad 24#define BADKEY_WARN 0
25#define BADKEY_FAIL 1
26#define BADKEY_DELETE 2
27
28/** @brief Bad key behavior */
29static int badkey = BADKEY_WARN;
30
31static long aliases_removed, keys_normalized, values_normalized, renoticed;
32static long keys_already_ok, values_already_ok;
33
3dc3d7db
RK
34static const struct option options[] = {
35 { "help", no_argument, 0, 'h' },
36 { "version", no_argument, 0, 'V' },
37 { "config", required_argument, 0, 'c' },
38 { "debug", no_argument, 0, 'd' },
39 { "no-debug", no_argument, 0, 'D' },
de3bd7ad 40 { "delete-bad-keys", no_argument, 0, 'x' },
41 { "fail-bad-keys", no_argument, 0, 'X' },
3dc3d7db
RK
42 { "syslog", no_argument, 0, 's' },
43 { "no-syslog", no_argument, 0, 'S' },
44 { 0, 0, 0, 0 }
45};
46
47/* display usage message and terminate */
48static void help(void) {
49 xprintf("Usage:\n"
50 " disorder-dbupgrade [OPTIONS]\n"
51 "Options:\n"
52 " --help, -h Display usage message\n"
53 " --version, -V Display version number\n"
54 " --config PATH, -c PATH Set configuration file\n"
55 " --debug, -d Turn on debugging\n"
56 " --[no-]syslog Force logging\n"
de3bd7ad 57 " --delete-bad-keys, -x Delete unconvertible keys\n"
58 " --fail-bad-keys, -X Fail if bad keys are found\n"
3dc3d7db
RK
59 "\n"
60 "Database upgrader for DisOrder. Not intended to be run\n"
61 "directly.\n");
62 xfclose(stdout);
63 exit(0);
64}
65
3dc3d7db
RK
66/** @brief Visit each key in a database and call @p callback
67 * @return 0 or DB_LOCK_DEADLOCK
68 *
69 * @p global_tid must be set. @p callback should return 0 or DB_LOCK_DEADLOCK.
70 */
71static int scan_core(const char *name, DB *db,
72 int (*callback)(const char *name, DB *db, DBC *c,
73 DBT *k, DBT *d)) {
74 long count = 0;
75 DBC *c = trackdb_opencursor(db, global_tid);
76 int err, r = 0;
77 DBT k[1], d[1];
78
de3bd7ad 79 values_normalized = 0;
80 keys_normalized = 0;
81 aliases_removed = 0;
82 renoticed = 0;
83 keys_already_ok = 0;
84 values_already_ok = 0;
3dc3d7db
RK
85 memset(k, 0, sizeof k);
86 memset(d, 0, sizeof d);
87 while((err = c->c_get(c, k, d, DB_NEXT)) == 0) {
88 if((err = callback(name, db, c, k, d)))
89 break;
90 ++count;
91 if(count % 1000 == 0)
92 info("scanning %s, %ld so far", name, count);
93 }
94 if(err && err != DB_NOTFOUND && err != DB_LOCK_DEADLOCK)
95 fatal(0, "%s: error scanning database: %s", name, db_strerror(err));
96 r = (err == DB_LOCK_DEADLOCK ? err : 0);
97 if((err = c->c_close(c)))
98 fatal(0, "%s: error closing cursor: %s", name, db_strerror(err));
de3bd7ad 99 info("%s: %ld entries scanned", name, count);
100 if(values_normalized || values_already_ok)
101 info("%s: %ld values converted, %ld already ok", name,
102 values_normalized, values_already_ok);
103 if(keys_normalized || keys_already_ok)
104 info("%s: %ld keys converted, %ld already OK", name,
105 keys_normalized, keys_already_ok);
106 if(aliases_removed)
107 info("%s: %ld aliases removed", name, aliases_removed);
108 if(renoticed)
109 info("%s: %ld tracks re-noticed", name, renoticed);
3dc3d7db
RK
110 return r;
111}
112
113/** @brief Visit each key in a database and call @p callback
114 *
115 * Everything happens inside the @p global_tid tranasction. @p callback
116 * should return 0 or DB_LOCK_DEADLOCK.
117 */
05b75f8d
RK
118static void scandb(const char *name, DB *db,
119 int (*callback)(const char *name, DB *db, DBC *c,
120 DBT *k, DBT *d)) {
3dc3d7db
RK
121 info("scanning %s", name);
122 for(;;) {
123 global_tid = trackdb_begin_transaction();
124 if(scan_core(name, db, callback)) {
125 trackdb_abort_transaction(global_tid);
126 global_tid = 0;
127 error(0, "detected deadlock, restarting scan");
128 continue;
129 } else {
130 trackdb_commit_transaction(global_tid);
131 global_tid = 0;
132 break;
133 }
134 }
135}
136
137/** @brief Truncate database @p db */
138static void truncate_database(const char *name, DB *db) {
139 u_int32_t count;
140 int err;
141
142 do {
143 err = db->truncate(db, 0, &count, DB_AUTO_COMMIT);
144 } while(err == DB_LOCK_DEADLOCK);
145 if(err)
146 fatal(0, "error truncating %s: %s", name, db_strerror(err));
147}
148
149/* scan callbacks */
150
151static int normalize_keys(const char *name, DB *db, DBC *c,
152 DBT *k, DBT *d) {
153 char *knfc;
154 size_t nknfc;
155 int err;
156
157 /* Find the normalized form of the key */
158 knfc = utf8_compose_canon(k->data, k->size, &nknfc);
de3bd7ad 159 if(!knfc) {
160 switch(badkey) {
161 case BADKEY_WARN:
162 error(0, "%s: invalid key: %.*s", name,
163 (int)k->size, (const char *)k->data);
164 break;
165 case BADKEY_DELETE:
166 error(0, "%s: deleting invalid key: %.*s", name,
167 (int)k->size, (const char *)k->data);
168 if((err = c->c_del(c, 0))) {
169 if(err != DB_LOCK_DEADLOCK)
170 fatal(0, "%s: error removing denormalized key: %s",
171 name, db_strerror(err));
172 return err;
173 }
174 break;
175 case BADKEY_FAIL:
176 fatal(0, "%s: invalid key: %.*s", name,
177 (int)k->size, (const char *)k->data);
178 }
179 return 0;
180 }
3dc3d7db 181 /* If the key is already in NFC then do nothing */
de3bd7ad 182 if(nknfc == k->size && !memcmp(k->data, knfc, nknfc)) {
183 ++keys_already_ok;
3dc3d7db 184 return 0;
de3bd7ad 185 }
3dc3d7db
RK
186 /* To rename the key we must delete the old one and insert a new one */
187 if((err = c->c_del(c, 0))) {
188 if(err != DB_LOCK_DEADLOCK)
189 fatal(0, "%s: error removing denormalized key: %s",
190 name, db_strerror(err));
191 return err;
192 }
193 k->size = nknfc;
194 k->data = knfc;
195 if((err = db->put(db, global_tid, k, d, DB_NOOVERWRITE))) {
196 if(err != DB_LOCK_DEADLOCK)
197 fatal(0, "%s: error storing normalized key: %s", name, db_strerror(err));
198 return err;
199 }
de3bd7ad 200 ++keys_normalized;
3dc3d7db
RK
201 return 0;
202}
203
204static int normalize_values(const char *name, DB *db,
205 DBC attribute((unused)) *c,
206 DBT *k, DBT *d) {
207 char *dnfc;
208 size_t ndnfc;
209 int err;
210
211 /* Find the normalized form of the value */
212 dnfc = utf8_compose_canon(d->data, d->size, &ndnfc);
213 if(!dnfc)
214 fatal(0, "%s: cannot convert data to NFC: %.*s", name,
215 (int)d->size, (const char *)d->data);
216 /* If the key is already in NFC then do nothing */
de3bd7ad 217 if(ndnfc == d->size && !memcmp(d->data, dnfc, ndnfc)) {
218 ++values_already_ok;
3dc3d7db 219 return 0;
de3bd7ad 220 }
3dc3d7db
RK
221 d->size = ndnfc;
222 d->data = dnfc;
223 if((err = db->put(db, global_tid, k, d, 0))) {
224 if(err != DB_LOCK_DEADLOCK)
225 fatal(0, "%s: error storing normalized data: %s", name, db_strerror(err));
226 return err;
227 }
de3bd7ad 228 ++values_normalized;
3dc3d7db
RK
229 return 0;
230}
231
232static int renotice(const char *name, DB attribute((unused)) *db,
233 DBC attribute((unused)) *c,
234 DBT *k, DBT *d) {
235 const struct kvp *const t = kvp_urldecode(d->data, d->size);
236 const char *const track = xstrndup(k->data, k->size);
237 const char *const path = kvp_get(t, "_path");
238 int err;
239
de3bd7ad 240 if(!path) {
241 /* If an alias sorts later than the actual filename then it'll appear
242 * in the scan. */
243 if(kvp_get(t, "_alias_for"))
244 return 0;
3dc3d7db
RK
245 fatal(0, "%s: no '_path' for %.*s", name,
246 (int)k->size, (const char *)k->data);
de3bd7ad 247 }
3dc3d7db
RK
248 switch(err = trackdb_notice_tid(track, path, global_tid)) {
249 case 0:
de3bd7ad 250 ++renoticed;
3dc3d7db
RK
251 return 0;
252 case DB_LOCK_DEADLOCK:
253 return err;
254 default:
255 fatal(0, "%s: unexpected return from trackdb_notice_tid: %s",
256 name, db_strerror(err));
257 }
258}
259
260static int remove_aliases_normalize_keys(const char *name, DB *db, DBC *c,
261 DBT *k, DBT *d) {
262 const struct kvp *const t = kvp_urldecode(d->data, d->size);
263 int err;
264
265 if(kvp_get(t, "_alias_for")) {
266 /* This is an alias. We remove all the alias entries. */
267 if((err = c->c_del(c, 0))) {
268 if(err != DB_LOCK_DEADLOCK)
269 fatal(0, "%s: error removing alias: %s", name, db_strerror(err));
270 return err;
271 }
de3bd7ad 272 ++aliases_removed;
3dc3d7db 273 return 0;
de3bd7ad 274 } else if(!kvp_get(t, "_path"))
275 error(0, "%s: %.*s has neither _alias_for nor _path", name,
276 (int)k->size, (const char *)k->data);
3dc3d7db
RK
277 return normalize_keys(name, db, c, k, d);
278}
279
280/** @brief Upgrade the database to the current version
281 *
282 * This function is supposed to be idempotent, so if it is interrupted
283 * half way through it is safe to restart.
284 */
285static void upgrade(void) {
286 char buf[32];
287
288 info("upgrading database to dbversion %ld", config->dbversion);
289 /* Normalize keys and values as required. We will also remove aliases as
290 * they will be regenerated when we re-noticed the tracks. */
291 info("renormalizing keys");
05b75f8d
RK
292 scandb("tracks.db", trackdb_tracksdb, remove_aliases_normalize_keys);
293 scandb("prefs.db", trackdb_prefsdb, normalize_keys);
294 scandb("global.db", trackdb_globaldb, normalize_keys);
295 scandb("noticed.db", trackdb_noticeddb, normalize_values);
3dc3d7db
RK
296 /* search.db and tags.db we will rebuild */
297 info("regenerating search database and aliases");
298 truncate_database("search.db", trackdb_searchdb);
299 truncate_database("tags.db", trackdb_tagsdb);
300 /* Regenerate the search database and aliases */
05b75f8d 301 scandb("tracks.db", trackdb_tracksdb, renotice);
3dc3d7db
RK
302 /* Finally update the database version */
303 snprintf(buf, sizeof buf, "%ld", config->dbversion);
304 trackdb_set_global("_dbversion", buf, 0);
305 info("completed database upgrade");
306}
307
308int main(int argc, char **argv) {
309 int n, logsyslog = !isatty(2);
310
311 set_progname(argv);
312 mem_init();
313 if(!setlocale(LC_CTYPE, "")) fatal(errno, "error calling setlocale");
de3bd7ad 314 while((n = getopt_long(argc, argv, "hVc:dDSsxX", options, 0)) >= 0) {
3dc3d7db
RK
315 switch(n) {
316 case 'h': help();
3fbdc96d 317 case 'V': version("disorder-dbupgrade");
3dc3d7db
RK
318 case 'c': configfile = optarg; break;
319 case 'd': debugging = 1; break;
320 case 'D': debugging = 0; break;
321 case 'S': logsyslog = 0; break;
322 case 's': logsyslog = 1; break;
de3bd7ad 323 case 'x': badkey = BADKEY_DELETE; break;
324 case 'X': badkey = BADKEY_FAIL; break;
3dc3d7db
RK
325 default: fatal(0, "invalid option");
326 }
327 }
328 /* If stderr is a TTY then log there, otherwise to syslog. */
329 if(logsyslog) {
330 openlog(progname, LOG_PID, LOG_DAEMON);
331 log_default = &log_syslog;
332 }
333 if(config_read(0)) fatal(0, "cannot read configuration");
334 /* Open the database */
d25c4615
RK
335 trackdb_init(TRACKDB_NO_RECOVER);
336 trackdb_open(TRACKDB_OPEN_FOR_UPGRADE);
3dc3d7db
RK
337 upgrade();
338 return 0;
339}
340
341/*
342Local Variables:
343c-basic-offset:2
344comment-column:40
345fill-column:79
346indent-tabs-mode:nil
347End:
348*/