[disorder] / server / dbupgrade.c

/*
 * This file is part of DisOrder
 * Copyright (C) 2007 Richard Kettlewell
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
 * USA
 */

#include <config.h>
#include "types.h"

#include <string.h>
#include <getopt.h>
#include <db.h>
#include <locale.h>
#include <errno.h>
#include <syslog.h>
#include <pcre.h>
#include <unistd.h>

#include "syscalls.h"
#include "log.h"
#include "defs.h"
#include "kvp.h"
#include "trackdb.h"
#include "trackdb-int.h"
#include "mem.h"
#include "configuration.h"
#include "unicode.h"

static DB_TXN *global_tid;

static const struct option options[] = {
  { "help", no_argument, 0, 'h' },
  { "version", no_argument, 0, 'V' },
  { "config", required_argument, 0, 'c' },
  { "debug", no_argument, 0, 'd' },
  { "no-debug", no_argument, 0, 'D' },
  { "syslog", no_argument, 0, 's' },
  { "no-syslog", no_argument, 0, 'S' },
  { 0, 0, 0, 0 }
};

/* display usage message and terminate */
static void help(void) {
  xprintf("Usage:\n"
	  "  disorder-dbupgrade [OPTIONS]\n"
	  "Options:\n"
	  "  --help, -h              Display usage message\n"
	  "  --version, -V           Display version number\n"
	  "  --config PATH, -c PATH  Set configuration file\n"
	  "  --debug, -d             Turn on debugging\n"
          "  --[no-]syslog           Force logging\n"
          "\n"
          "Database upgrader for DisOrder.  Not intended to be run\n"
          "directly.\n");
  xfclose(stdout);
  exit(0);
}

/* display version number and terminate */
static void version(void) {
  xprintf("disorder-dbupgrade version %s\n", disorder_version_string);
  xfclose(stdout);
  exit(0);
}

/** @brief Visit each key in a database and call @p callback
 * @return 0 or DB_LOCK_DEADLOCK
 *
 * @p global_tid must be set.  @p callback should return 0 or DB_LOCK_DEADLOCK.
 */
static int scan_core(const char *name, DB *db,
                     int (*callback)(const char *name, DB *db, DBC *c,
                                     DBT *k, DBT *d)) {
  long count = 0;
  DBC *c = trackdb_opencursor(db, global_tid);
  int err, r = 0;
  DBT k[1], d[1];

  memset(k, 0, sizeof k);
  memset(d, 0, sizeof d);
  while((err = c->c_get(c, k, d, DB_NEXT)) == 0) {
    if((err = callback(name, db, c, k, d)))
      break;
    ++count;
    if(count % 1000 == 0)
      info("scanning %s, %ld so far", name, count);
  }
  if(err && err != DB_NOTFOUND && err != DB_LOCK_DEADLOCK)
    fatal(0, "%s: error scanning database: %s", name, db_strerror(err));
  r = (err == DB_LOCK_DEADLOCK ? err : 0);
  if((err = c->c_close(c)))
    fatal(0, "%s: error closing cursor: %s", name, db_strerror(err));
  return r;
}

/** @brief Visit each key in a database and call @p callback
 *
 * Everything happens inside the @p global_tid tranasction.  @p callback
 * should return 0 or DB_LOCK_DEADLOCK.
 */
static void scan(const char *name, DB *db,
                 int (*callback)(const char *name, DB *db, DBC *c,
                                 DBT *k, DBT *d)) {
  info("scanning %s", name);
  for(;;) {
    global_tid = trackdb_begin_transaction();
    if(scan_core(name, db, callback)) {
      trackdb_abort_transaction(global_tid);
      global_tid = 0;
      error(0, "detected deadlock, restarting scan");
      continue;
    } else {
      trackdb_commit_transaction(global_tid);
      global_tid = 0;
      break;
    }
  }
}

/** @brief Truncate database @p db */
static void truncate_database(const char *name, DB *db) {
  u_int32_t count;
  int err;
  
  do {
    err = db->truncate(db, 0, &count, DB_AUTO_COMMIT);
  } while(err == DB_LOCK_DEADLOCK);
  if(err)
    fatal(0, "error truncating %s: %s", name, db_strerror(err));
}

/* scan callbacks */

static int normalize_keys(const char *name, DB *db, DBC *c,
                          DBT *k, DBT *d) {
  char *knfc;
  size_t nknfc;
  int err;

  /* Find the normalized form of the key */
  knfc = utf8_compose_canon(k->data, k->size, &nknfc);
  if(!knfc)
    fatal(0, "%s: cannot convert key to NFC: %.*s", name,
          (int)k->size, (const char *)k->data);
  /* If the key is already in NFC then do nothing */
  if(nknfc == k->size && !memcmp(k->data, knfc, nknfc))
    return 0;
  /* To rename the key we must delete the old one and insert a new one */
  if((err = c->c_del(c, 0))) {
    if(err != DB_LOCK_DEADLOCK)
      fatal(0, "%s: error removing denormalized key: %s",
            name, db_strerror(err));
    return err;
  }
  k->size = nknfc;
  k->data = knfc;
  if((err = db->put(db, global_tid, k, d, DB_NOOVERWRITE))) {
    if(err != DB_LOCK_DEADLOCK)
      fatal(0, "%s: error storing normalized key: %s", name, db_strerror(err));
    return err;
  }
  return 0;
}

static int normalize_values(const char *name, DB *db,
                            DBC attribute((unused)) *c,
                            DBT *k, DBT *d) {
  char *dnfc;
  size_t ndnfc;
  int err;

  /* Find the normalized form of the value */
  dnfc = utf8_compose_canon(d->data, d->size, &ndnfc);
  if(!dnfc)
    fatal(0, "%s: cannot convert data to NFC: %.*s", name,
          (int)d->size, (const char *)d->data);
  /* If the key is already in NFC then do nothing */
  if(ndnfc == d->size && !memcmp(d->data, dnfc, ndnfc))
    return 0;
  d->size = ndnfc;
  d->data = dnfc;
  if((err = db->put(db, global_tid, k, d, 0))) {
    if(err != DB_LOCK_DEADLOCK)
      fatal(0, "%s: error storing normalized data: %s", name, db_strerror(err));
    return err;
  }
  return 0;
}

static int renotice(const char *name, DB attribute((unused)) *db,
                    DBC attribute((unused)) *c,
                    DBT *k, DBT *d) {
  const struct kvp *const t = kvp_urldecode(d->data, d->size);
  const char *const track = xstrndup(k->data, k->size);
  const char *const path = kvp_get(t, "_path");
  int err;

  if(!path)
    fatal(0, "%s: no '_path' for %.*s", name,
          (int)k->size, (const char *)k->data);
  switch(err = trackdb_notice_tid(track, path, global_tid)) {
  case 0:
    return 0;
  case DB_LOCK_DEADLOCK:
    return err;
  default:
    fatal(0, "%s: unexpected return from trackdb_notice_tid: %s",
          name, db_strerror(err));
  }
}
 
static int remove_aliases_normalize_keys(const char *name, DB *db, DBC *c,
                                         DBT *k, DBT *d) {
  const struct kvp *const t = kvp_urldecode(d->data, d->size);
  int err;

  if(kvp_get(t, "_alias_for")) {
    /* This is an alias.  We remove all the alias entries. */
    if((err = c->c_del(c, 0))) {
      if(err != DB_LOCK_DEADLOCK)
        fatal(0, "%s: error removing alias: %s", name, db_strerror(err));
      return err;
    }
    return 0;
  }
  return normalize_keys(name, db, c, k, d);
}

/** @brief Upgrade the database to the current version
 *
 * This function is supposed to be idempotent, so if it is interrupted
 * half way through it is safe to restart.
 */
static void upgrade(void) {
  char buf[32];

  info("upgrading database to dbversion %ld", config->dbversion);
  /* Normalize keys and values as required.  We will also remove aliases as
   * they will be regenerated when we re-noticed the tracks. */
  info("renormalizing keys");
  scan("tracks.db", trackdb_tracksdb, remove_aliases_normalize_keys);
  scan("prefs.db", trackdb_prefsdb, normalize_keys);
  scan("global.db", trackdb_globaldb, normalize_keys);
  scan("noticed.db", trackdb_noticeddb, normalize_values);
  /* search.db and tags.db we will rebuild */
  info("regenerating search database and aliases");
  truncate_database("search.db", trackdb_searchdb);
  truncate_database("tags.db", trackdb_tagsdb);
  /* Regenerate the search database and aliases */
  scan("tracks.db", trackdb_tracksdb, renotice);
  /* Finally update the database version */
  snprintf(buf, sizeof buf, "%ld", config->dbversion);
  trackdb_set_global("_dbversion", buf, 0);
  info("completed database upgrade");
}

int main(int argc, char **argv) {
  int n, logsyslog = !isatty(2);
  
  set_progname(argv);
  mem_init();
  if(!setlocale(LC_CTYPE, "")) fatal(errno, "error calling setlocale");
  while((n = getopt_long(argc, argv, "hVc:dDSs", options, 0)) >= 0) {
    switch(n) {
    case 'h': help();
    case 'V': version();
    case 'c': configfile = optarg; break;
    case 'd': debugging = 1; break;
    case 'D': debugging = 0; break;
    case 'S': logsyslog = 0; break;
    case 's': logsyslog = 1; break;
    default: fatal(0, "invalid option");
    }
  }
  /* If stderr is a TTY then log there, otherwise to syslog. */
  if(logsyslog) {
    openlog(progname, LOG_PID, LOG_DAEMON);
    log_default = &log_syslog;
  }
  if(config_read(0)) fatal(0, "cannot read configuration");
  /* Open the database */
  trackdb_init(TRACKDB_NO_RECOVER);
  trackdb_open(TRACKDB_OPEN_FOR_UPGRADE);
  upgrade();
  return 0;
}

/*
Local Variables:
c-basic-offset:2
comment-column:40
fill-column:79
indent-tabs-mode:nil
End:
*/
Commit	Line	Data
3dc3d7db RK	1	/*
	2	* This file is part of DisOrder
	3	* Copyright (C) 2007 Richard Kettlewell
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License as published by
	7	* the Free Software Foundation; either version 2 of the License, or
	8	* (at your option) any later version.
	9	*
	10	* This program is distributed in the hope that it will be useful, but
	11	* WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	* General Public License for more details.
	14	*
	15	* You should have received a copy of the GNU General Public License
	16	* along with this program; if not, write to the Free Software
	17	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
	18	* USA
	19	*/
	20
	21	#include <config.h>
	22	#include "types.h"
	23
	24	#include <string.h>
	25	#include <getopt.h>
	26	#include <db.h>
	27	#include <locale.h>
	28	#include <errno.h>
	29	#include <syslog.h>
	30	#include <pcre.h>
ad2f8275	31	#include <unistd.h>
3dc3d7db RK	32
	33	#include "syscalls.h"
	34	#include "log.h"
	35	#include "defs.h"
	36	#include "kvp.h"
	37	#include "trackdb.h"
	38	#include "trackdb-int.h"
	39	#include "mem.h"
	40	#include "configuration.h"
	41	#include "unicode.h"
	42
	43	static DB_TXN *global_tid;
	44
	45	static const struct option options[] = {
	46	{ "help", no_argument, 0, 'h' },
	47	{ "version", no_argument, 0, 'V' },
	48	{ "config", required_argument, 0, 'c' },
	49	{ "debug", no_argument, 0, 'd' },
	50	{ "no-debug", no_argument, 0, 'D' },
	51	{ "syslog", no_argument, 0, 's' },
	52	{ "no-syslog", no_argument, 0, 'S' },
	53	{ 0, 0, 0, 0 }
	54	};
	55
	56	/* display usage message and terminate */
	57	static void help(void) {
	58	xprintf("Usage:\n"
	59	" disorder-dbupgrade [OPTIONS]\n"
	60	"Options:\n"
	61	" --help, -h Display usage message\n"
	62	" --version, -V Display version number\n"
	63	" --config PATH, -c PATH Set configuration file\n"
	64	" --debug, -d Turn on debugging\n"
	65	" --[no-]syslog Force logging\n"
	66	"\n"
	67	"Database upgrader for DisOrder. Not intended to be run\n"
	68	"directly.\n");
	69	xfclose(stdout);
	70	exit(0);
	71	}
	72
	73	/* display version number and terminate */
	74	static void version(void) {
	75	xprintf("disorder-dbupgrade version %s\n", disorder_version_string);
	76	xfclose(stdout);
	77	exit(0);
	78	}
	79
	80	/** @brief Visit each key in a database and call @p callback
	81	* @return 0 or DB_LOCK_DEADLOCK
	82	*
	83	* @p global_tid must be set. @p callback should return 0 or DB_LOCK_DEADLOCK.
	84	*/
	85	static int scan_core(const char name, DB db,
	86	int (callback)(const char name, DB db, DBC c,
	87	DBT k, DBT d)) {
	88	long count = 0;
	89	DBC *c = trackdb_opencursor(db, global_tid);
	90	int err, r = 0;
	91	DBT k[1], d[1];
	92
	93	memset(k, 0, sizeof k);
	94	memset(d, 0, sizeof d);
	95	while((err = c->c_get(c, k, d, DB_NEXT)) == 0) {
96	if((err = callback(name, db, c, k, d)))
97	break;
98	++count;
99	if(count % 1000 == 0)
100	info("scanning %s, %ld so far", name, count);
101	}
102	if(err && err != DB_NOTFOUND && err != DB_LOCK_DEADLOCK)
103	fatal(0, "%s: error scanning database: %s", name, db_strerror(err));
104	r = (err == DB_LOCK_DEADLOCK ? err : 0);
105	if((err = c->c_close(c)))
106	fatal(0, "%s: error closing cursor: %s", name, db_strerror(err));
107	return r;
108	}
109
110	/** @brief Visit each key in a database and call @p callback
111	*
112	* Everything happens inside the @p global_tid tranasction. @p callback
113	* should return 0 or DB_LOCK_DEADLOCK.
114	*/
115	static void scan(const char name, DB db,
116	int (callback)(const char name, DB db, DBC c,
117	DBT k, DBT d)) {
118	info("scanning %s", name);
119	for(;;) {
120	global_tid = trackdb_begin_transaction();
121	if(scan_core(name, db, callback)) {
122	trackdb_abort_transaction(global_tid);
123	global_tid = 0;
124	error(0, "detected deadlock, restarting scan");
125	continue;
126	} else {
127	trackdb_commit_transaction(global_tid);
128	global_tid = 0;
129	break;
130	}
131	}
132	}
133
134	/** @brief Truncate database @p db */
135	static void truncate_database(const char name, DB db) {
136	u_int32_t count;
137	int err;
138
139	do {
140	err = db->truncate(db, 0, &count, DB_AUTO_COMMIT);
141	} while(err == DB_LOCK_DEADLOCK);
142	if(err)
143	fatal(0, "error truncating %s: %s", name, db_strerror(err));
144	}
145
146	/* scan callbacks */
147
148	static int normalize_keys(const char name, DB db, DBC *c,
149	DBT k, DBT d) {
150	char *knfc;
151	size_t nknfc;
152	int err;
153
154	/* Find the normalized form of the key */
155	knfc = utf8_compose_canon(k->data, k->size, &nknfc);
156	if(!knfc)
157	fatal(0, "%s: cannot convert key to NFC: %.*s", name,
158	(int)k->size, (const char *)k->data);
159	/* If the key is already in NFC then do nothing */
160	if(nknfc == k->size && !memcmp(k->data, knfc, nknfc))
161	return 0;
162	/* To rename the key we must delete the old one and insert a new one */
163	if((err = c->c_del(c, 0))) {
164	if(err != DB_LOCK_DEADLOCK)
165	fatal(0, "%s: error removing denormalized key: %s",
166	name, db_strerror(err));
167	return err;
168	}
169	k->size = nknfc;
170	k->data = knfc;
171	if((err = db->put(db, global_tid, k, d, DB_NOOVERWRITE))) {
172	if(err != DB_LOCK_DEADLOCK)
173	fatal(0, "%s: error storing normalized key: %s", name, db_strerror(err));
174	return err;
175	}
176	return 0;
177	}
178
179	static int normalize_values(const char name, DB db,
180	DBC attribute((unused)) *c,
181	DBT k, DBT d) {
182	char *dnfc;
183	size_t ndnfc;
184	int err;
185
186	/* Find the normalized form of the value */
187	dnfc = utf8_compose_canon(d->data, d->size, &ndnfc);
188	if(!dnfc)
189	fatal(0, "%s: cannot convert data to NFC: %.*s", name,
190	(int)d->size, (const char *)d->data);
191	/* If the key is already in NFC then do nothing */
192	if(ndnfc == d->size && !memcmp(d->data, dnfc, ndnfc))
193	return 0;
194	d->size = ndnfc;
195	d->data = dnfc;
196	if((err = db->put(db, global_tid, k, d, 0))) {
197	if(err != DB_LOCK_DEADLOCK)
198	fatal(0, "%s: error storing normalized data: %s", name, db_strerror(err));
199	return err;
200	}
201	return 0;
202	}
203
204	static int renotice(const char name, DB attribute((unused)) db,
205	DBC attribute((unused)) *c,
206	DBT k, DBT d) {
207	const struct kvp *const t = kvp_urldecode(d->data, d->size);
208	const char *const track = xstrndup(k->data, k->size);
209	const char *const path = kvp_get(t, "_path");
210	int err;
211
212	if(!path)
213	fatal(0, "%s: no '_path' for %.*s", name,
214	(int)k->size, (const char *)k->data);
215	switch(err = trackdb_notice_tid(track, path, global_tid)) {
216	case 0:
217	return 0;
218	case DB_LOCK_DEADLOCK:
219	return err;
220	default:
221	fatal(0, "%s: unexpected return from trackdb_notice_tid: %s",
222	name, db_strerror(err));
223	}
224	}
225
226	static int remove_aliases_normalize_keys(const char name, DB db, DBC *c,
227	DBT k, DBT d) {
228	const struct kvp *const t = kvp_urldecode(d->data, d->size);
229	int err;
230
231	if(kvp_get(t, "_alias_for")) {
232	/* This is an alias. We remove all the alias entries. */
233	if((err = c->c_del(c, 0))) {
234	if(err != DB_LOCK_DEADLOCK)
235	fatal(0, "%s: error removing alias: %s", name, db_strerror(err));
236	return err;
237	}
238	return 0;
239	}
240	return normalize_keys(name, db, c, k, d);
241	}
242
243	/** @brief Upgrade the database to the current version
244	*
245	* This function is supposed to be idempotent, so if it is interrupted
246	* half way through it is safe to restart.
247	*/
248	static void upgrade(void) {
249	char buf[32];
250
251	info("upgrading database to dbversion %ld", config->dbversion);
252	/* Normalize keys and values as required. We will also remove aliases as
253	* they will be regenerated when we re-noticed the tracks. */
254	info("renormalizing keys");
255	scan("tracks.db", trackdb_tracksdb, remove_aliases_normalize_keys);
256	scan("prefs.db", trackdb_prefsdb, normalize_keys);
257	scan("global.db", trackdb_globaldb, normalize_keys);
258	scan("noticed.db", trackdb_noticeddb, normalize_values);
259	/* search.db and tags.db we will rebuild */
260	info("regenerating search database and aliases");
261	truncate_database("search.db", trackdb_searchdb);
262	truncate_database("tags.db", trackdb_tagsdb);
263	/* Regenerate the search database and aliases */
264	scan("tracks.db", trackdb_tracksdb, renotice);
265	/* Finally update the database version */
266	snprintf(buf, sizeof buf, "%ld", config->dbversion);
267	trackdb_set_global("_dbversion", buf, 0);
268	info("completed database upgrade");
269	}
270
271	int main(int argc, char **argv) {
272	int n, logsyslog = !isatty(2);
273
274	set_progname(argv);
275	mem_init();
276	if(!setlocale(LC_CTYPE, "")) fatal(errno, "error calling setlocale");
277	while((n = getopt_long(argc, argv, "hVc:dDSs", options, 0)) >= 0) {
278	switch(n) {
279	case 'h': help();
280	case 'V': version();
281	case 'c': configfile = optarg; break;
282	case 'd': debugging = 1; break;
283	case 'D': debugging = 0; break;
284	case 'S': logsyslog = 0; break;
285	case 's': logsyslog = 1; break;
286	default: fatal(0, "invalid option");
287	}
288	}
289	/* If stderr is a TTY then log there, otherwise to syslog. */
290	if(logsyslog) {
291	openlog(progname, LOG_PID, LOG_DAEMON);
292	log_default = &log_syslog;
293	}
294	if(config_read(0)) fatal(0, "cannot read configuration");
295	/* Open the database */
d25c4615 RK	296	trackdb_init(TRACKDB_NO_RECOVER);
d25c4615 RK	297	trackdb_open(TRACKDB_OPEN_FOR_UPGRADE);
3dc3d7db RK	298	upgrade();
	299	return 0;
	300	}
	301
	302	/*
	303	Local Variables:
	304	c-basic-offset:2
	305	comment-column:40
	306	fill-column:79
	307	indent-tabs-mode:nil
	308	End:
	309	*/