chiark / gitweb /
Consistency check for finished tracks.
[disorder] / server / dbupgrade.c
CommitLineData
3dc3d7db
RK
1/*
2 * This file is part of DisOrder
5aff007d 3 * Copyright (C) 2007, 2008 Richard Kettlewell
3dc3d7db 4 *
e7eb3a27 5 * This program is free software: you can redistribute it and/or modify
3dc3d7db 6 * it under the terms of the GNU General Public License as published by
e7eb3a27 7 * the Free Software Foundation, either version 3 of the License, or
3dc3d7db
RK
8 * (at your option) any later version.
9 *
e7eb3a27
RK
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
3dc3d7db 15 * You should have received a copy of the GNU General Public License
e7eb3a27 16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
3dc3d7db 17 */
132a5a4a
RK
18/** @file server/dbupgrade.c
19 * @brief Database upgrade utility
20 *
21 * Invoked by the server when necessary.
22 */
05b75f8d 23#include "disorder-server.h"
3dc3d7db
RK
24
25static DB_TXN *global_tid;
26
de3bd7ad 27#define BADKEY_WARN 0
28#define BADKEY_FAIL 1
29#define BADKEY_DELETE 2
30
31/** @brief Bad key behavior */
32static int badkey = BADKEY_WARN;
33
34static long aliases_removed, keys_normalized, values_normalized, renoticed;
35static long keys_already_ok, values_already_ok;
36
3dc3d7db
RK
37static const struct option options[] = {
38 { "help", no_argument, 0, 'h' },
39 { "version", no_argument, 0, 'V' },
40 { "config", required_argument, 0, 'c' },
41 { "debug", no_argument, 0, 'd' },
42 { "no-debug", no_argument, 0, 'D' },
de3bd7ad 43 { "delete-bad-keys", no_argument, 0, 'x' },
44 { "fail-bad-keys", no_argument, 0, 'X' },
3dc3d7db
RK
45 { "syslog", no_argument, 0, 's' },
46 { "no-syslog", no_argument, 0, 'S' },
47 { 0, 0, 0, 0 }
48};
49
50/* display usage message and terminate */
51static void help(void) {
52 xprintf("Usage:\n"
53 " disorder-dbupgrade [OPTIONS]\n"
54 "Options:\n"
55 " --help, -h Display usage message\n"
56 " --version, -V Display version number\n"
57 " --config PATH, -c PATH Set configuration file\n"
58 " --debug, -d Turn on debugging\n"
59 " --[no-]syslog Force logging\n"
de3bd7ad 60 " --delete-bad-keys, -x Delete unconvertible keys\n"
61 " --fail-bad-keys, -X Fail if bad keys are found\n"
3dc3d7db
RK
62 "\n"
63 "Database upgrader for DisOrder. Not intended to be run\n"
64 "directly.\n");
65 xfclose(stdout);
66 exit(0);
67}
68
3dc3d7db
RK
69/** @brief Visit each key in a database and call @p callback
70 * @return 0 or DB_LOCK_DEADLOCK
71 *
72 * @p global_tid must be set. @p callback should return 0 or DB_LOCK_DEADLOCK.
73 */
74static int scan_core(const char *name, DB *db,
75 int (*callback)(const char *name, DB *db, DBC *c,
76 DBT *k, DBT *d)) {
77 long count = 0;
78 DBC *c = trackdb_opencursor(db, global_tid);
79 int err, r = 0;
80 DBT k[1], d[1];
81
de3bd7ad 82 values_normalized = 0;
83 keys_normalized = 0;
84 aliases_removed = 0;
85 renoticed = 0;
86 keys_already_ok = 0;
87 values_already_ok = 0;
3dc3d7db
RK
88 memset(k, 0, sizeof k);
89 memset(d, 0, sizeof d);
90 while((err = c->c_get(c, k, d, DB_NEXT)) == 0) {
91 if((err = callback(name, db, c, k, d)))
92 break;
93 ++count;
94 if(count % 1000 == 0)
2e9ba080 95 disorder_info("scanning %s, %ld so far", name, count);
3dc3d7db
RK
96 }
97 if(err && err != DB_NOTFOUND && err != DB_LOCK_DEADLOCK)
2e9ba080 98 disorder_fatal(0, "%s: error scanning database: %s", name, db_strerror(err));
3dc3d7db
RK
99 r = (err == DB_LOCK_DEADLOCK ? err : 0);
100 if((err = c->c_close(c)))
2e9ba080
RK
101 disorder_fatal(0, "%s: error closing cursor: %s", name, db_strerror(err));
102 disorder_info("%s: %ld entries scanned", name, count);
de3bd7ad 103 if(values_normalized || values_already_ok)
2e9ba080
RK
104 disorder_info("%s: %ld values converted, %ld already ok", name,
105 values_normalized, values_already_ok);
de3bd7ad 106 if(keys_normalized || keys_already_ok)
2e9ba080
RK
107 disorder_info("%s: %ld keys converted, %ld already OK", name,
108 keys_normalized, keys_already_ok);
de3bd7ad 109 if(aliases_removed)
2e9ba080 110 disorder_info("%s: %ld aliases removed", name, aliases_removed);
de3bd7ad 111 if(renoticed)
2e9ba080 112 disorder_info("%s: %ld tracks re-noticed", name, renoticed);
3dc3d7db
RK
113 return r;
114}
115
116/** @brief Visit each key in a database and call @p callback
117 *
118 * Everything happens inside the @p global_tid tranasction. @p callback
119 * should return 0 or DB_LOCK_DEADLOCK.
120 */
05b75f8d
RK
121static void scandb(const char *name, DB *db,
122 int (*callback)(const char *name, DB *db, DBC *c,
123 DBT *k, DBT *d)) {
2e9ba080 124 disorder_info("scanning %s", name);
3dc3d7db
RK
125 for(;;) {
126 global_tid = trackdb_begin_transaction();
127 if(scan_core(name, db, callback)) {
128 trackdb_abort_transaction(global_tid);
129 global_tid = 0;
2e9ba080 130 disorder_error(0, "detected deadlock, restarting scan");
3dc3d7db
RK
131 continue;
132 } else {
133 trackdb_commit_transaction(global_tid);
134 global_tid = 0;
135 break;
136 }
137 }
138}
139
140/** @brief Truncate database @p db */
141static void truncate_database(const char *name, DB *db) {
142 u_int32_t count;
143 int err;
144
145 do {
146 err = db->truncate(db, 0, &count, DB_AUTO_COMMIT);
147 } while(err == DB_LOCK_DEADLOCK);
148 if(err)
2e9ba080 149 disorder_fatal(0, "error truncating %s: %s", name, db_strerror(err));
3dc3d7db
RK
150}
151
152/* scan callbacks */
153
154static int normalize_keys(const char *name, DB *db, DBC *c,
155 DBT *k, DBT *d) {
156 char *knfc;
157 size_t nknfc;
158 int err;
159
160 /* Find the normalized form of the key */
161 knfc = utf8_compose_canon(k->data, k->size, &nknfc);
de3bd7ad 162 if(!knfc) {
163 switch(badkey) {
164 case BADKEY_WARN:
2e9ba080
RK
165 disorder_error(0, "%s: invalid key: %.*s", name,
166 (int)k->size, (const char *)k->data);
de3bd7ad 167 break;
168 case BADKEY_DELETE:
2e9ba080
RK
169 disorder_error(0, "%s: deleting invalid key: %.*s", name,
170 (int)k->size, (const char *)k->data);
de3bd7ad 171 if((err = c->c_del(c, 0))) {
172 if(err != DB_LOCK_DEADLOCK)
2e9ba080
RK
173 disorder_fatal(0, "%s: error removing denormalized key: %s",
174 name, db_strerror(err));
de3bd7ad 175 return err;
176 }
177 break;
178 case BADKEY_FAIL:
2e9ba080
RK
179 disorder_fatal(0, "%s: invalid key: %.*s", name,
180 (int)k->size, (const char *)k->data);
de3bd7ad 181 }
182 return 0;
183 }
3dc3d7db 184 /* If the key is already in NFC then do nothing */
de3bd7ad 185 if(nknfc == k->size && !memcmp(k->data, knfc, nknfc)) {
186 ++keys_already_ok;
3dc3d7db 187 return 0;
de3bd7ad 188 }
3dc3d7db
RK
189 /* To rename the key we must delete the old one and insert a new one */
190 if((err = c->c_del(c, 0))) {
191 if(err != DB_LOCK_DEADLOCK)
2e9ba080
RK
192 disorder_fatal(0, "%s: error removing denormalized key: %s",
193 name, db_strerror(err));
3dc3d7db
RK
194 return err;
195 }
196 k->size = nknfc;
197 k->data = knfc;
198 if((err = db->put(db, global_tid, k, d, DB_NOOVERWRITE))) {
199 if(err != DB_LOCK_DEADLOCK)
2e9ba080
RK
200 disorder_fatal(0, "%s: error storing normalized key: %s",
201 name, db_strerror(err));
3dc3d7db
RK
202 return err;
203 }
de3bd7ad 204 ++keys_normalized;
3dc3d7db
RK
205 return 0;
206}
207
208static int normalize_values(const char *name, DB *db,
209 DBC attribute((unused)) *c,
210 DBT *k, DBT *d) {
211 char *dnfc;
212 size_t ndnfc;
213 int err;
214
215 /* Find the normalized form of the value */
216 dnfc = utf8_compose_canon(d->data, d->size, &ndnfc);
217 if(!dnfc)
2e9ba080
RK
218 disorder_fatal(0, "%s: cannot convert data to NFC: %.*s", name,
219 (int)d->size, (const char *)d->data);
3dc3d7db 220 /* If the key is already in NFC then do nothing */
de3bd7ad 221 if(ndnfc == d->size && !memcmp(d->data, dnfc, ndnfc)) {
222 ++values_already_ok;
3dc3d7db 223 return 0;
de3bd7ad 224 }
3dc3d7db
RK
225 d->size = ndnfc;
226 d->data = dnfc;
227 if((err = db->put(db, global_tid, k, d, 0))) {
228 if(err != DB_LOCK_DEADLOCK)
2e9ba080
RK
229 disorder_fatal(0, "%s: error storing normalized data: %s",
230 name, db_strerror(err));
3dc3d7db
RK
231 return err;
232 }
de3bd7ad 233 ++values_normalized;
3dc3d7db
RK
234 return 0;
235}
236
237static int renotice(const char *name, DB attribute((unused)) *db,
238 DBC attribute((unused)) *c,
239 DBT *k, DBT *d) {
240 const struct kvp *const t = kvp_urldecode(d->data, d->size);
241 const char *const track = xstrndup(k->data, k->size);
242 const char *const path = kvp_get(t, "_path");
243 int err;
244
de3bd7ad 245 if(!path) {
246 /* If an alias sorts later than the actual filename then it'll appear
247 * in the scan. */
248 if(kvp_get(t, "_alias_for"))
249 return 0;
2e9ba080
RK
250 disorder_fatal(0, "%s: no '_path' for %.*s", name,
251 (int)k->size, (const char *)k->data);
de3bd7ad 252 }
3dc3d7db
RK
253 switch(err = trackdb_notice_tid(track, path, global_tid)) {
254 case 0:
de3bd7ad 255 ++renoticed;
3dc3d7db
RK
256 return 0;
257 case DB_LOCK_DEADLOCK:
258 return err;
259 default:
2e9ba080
RK
260 disorder_fatal(0, "%s: unexpected return from trackdb_notice_tid: %s",
261 name, db_strerror(err));
3dc3d7db
RK
262 }
263}
264
265static int remove_aliases_normalize_keys(const char *name, DB *db, DBC *c,
266 DBT *k, DBT *d) {
267 const struct kvp *const t = kvp_urldecode(d->data, d->size);
268 int err;
269
270 if(kvp_get(t, "_alias_for")) {
271 /* This is an alias. We remove all the alias entries. */
272 if((err = c->c_del(c, 0))) {
273 if(err != DB_LOCK_DEADLOCK)
2e9ba080 274 disorder_fatal(0, "%s: error removing alias: %s", name, db_strerror(err));
3dc3d7db
RK
275 return err;
276 }
de3bd7ad 277 ++aliases_removed;
3dc3d7db 278 return 0;
de3bd7ad 279 } else if(!kvp_get(t, "_path"))
2e9ba080
RK
280 disorder_error(0, "%s: %.*s has neither _alias_for nor _path", name,
281 (int)k->size, (const char *)k->data);
3dc3d7db
RK
282 return normalize_keys(name, db, c, k, d);
283}
284
285/** @brief Upgrade the database to the current version
286 *
287 * This function is supposed to be idempotent, so if it is interrupted
288 * half way through it is safe to restart.
289 */
290static void upgrade(void) {
291 char buf[32];
292
2e9ba080 293 disorder_info("upgrading database to dbversion %ld", config->dbversion);
3dc3d7db
RK
294 /* Normalize keys and values as required. We will also remove aliases as
295 * they will be regenerated when we re-noticed the tracks. */
2e9ba080 296 disorder_info("renormalizing keys");
05b75f8d
RK
297 scandb("tracks.db", trackdb_tracksdb, remove_aliases_normalize_keys);
298 scandb("prefs.db", trackdb_prefsdb, normalize_keys);
299 scandb("global.db", trackdb_globaldb, normalize_keys);
300 scandb("noticed.db", trackdb_noticeddb, normalize_values);
3dc3d7db 301 /* search.db and tags.db we will rebuild */
2e9ba080 302 disorder_info("regenerating search database and aliases");
3dc3d7db
RK
303 truncate_database("search.db", trackdb_searchdb);
304 truncate_database("tags.db", trackdb_tagsdb);
305 /* Regenerate the search database and aliases */
05b75f8d 306 scandb("tracks.db", trackdb_tracksdb, renotice);
3dc3d7db
RK
307 /* Finally update the database version */
308 snprintf(buf, sizeof buf, "%ld", config->dbversion);
309 trackdb_set_global("_dbversion", buf, 0);
2e9ba080 310 disorder_info("completed database upgrade");
3dc3d7db
RK
311}
312
313int main(int argc, char **argv) {
314 int n, logsyslog = !isatty(2);
315
316 set_progname(argv);
317 mem_init();
2e9ba080 318 if(!setlocale(LC_CTYPE, "")) disorder_fatal(errno, "error calling setlocale");
de3bd7ad 319 while((n = getopt_long(argc, argv, "hVc:dDSsxX", options, 0)) >= 0) {
3dc3d7db
RK
320 switch(n) {
321 case 'h': help();
3fbdc96d 322 case 'V': version("disorder-dbupgrade");
3dc3d7db
RK
323 case 'c': configfile = optarg; break;
324 case 'd': debugging = 1; break;
325 case 'D': debugging = 0; break;
326 case 'S': logsyslog = 0; break;
327 case 's': logsyslog = 1; break;
de3bd7ad 328 case 'x': badkey = BADKEY_DELETE; break;
329 case 'X': badkey = BADKEY_FAIL; break;
2e9ba080 330 default: disorder_fatal(0, "invalid option");
3dc3d7db
RK
331 }
332 }
333 /* If stderr is a TTY then log there, otherwise to syslog. */
334 if(logsyslog) {
335 openlog(progname, LOG_PID, LOG_DAEMON);
336 log_default = &log_syslog;
337 }
2e9ba080 338 if(config_read(0, NULL)) disorder_fatal(0, "cannot read configuration");
3dc3d7db 339 /* Open the database */
d25c4615
RK
340 trackdb_init(TRACKDB_NO_RECOVER);
341 trackdb_open(TRACKDB_OPEN_FOR_UPGRADE);
3dc3d7db
RK
342 upgrade();
343 return 0;
344}
345
346/*
347Local Variables:
348c-basic-offset:2
349comment-column:40
350fill-column:79
351indent-tabs-mode:nil
352End:
353*/