[disorder] / server / rescan.c

/*
 * This file is part of DisOrder 
 * Copyright (C) 2005, 2006, 2007 Richard Kettlewell
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
 * USA
 */

#include <config.h>
#include "types.h"

#include <getopt.h>
#include <stdio.h>
#include <stdlib.h>
#include <db.h>
#include <locale.h>
#include <errno.h>
#include <sys/types.h>
#include <unistd.h>
#include <pcre.h>
#include <fnmatch.h>
#include <sys/wait.h>
#include <string.h>
#include <syslog.h>

#include "configuration.h"
#include "syscalls.h"
#include "log.h"
#include "defs.h"
#include "mem.h"
#include "plugin.h"
#include "inputline.h"
#include "charset.h"
#include "wstat.h"
#include "kvp.h"
#include "printf.h"
#include "trackdb.h"
#include "trackdb-int.h"
#include "trackname.h"

static DB_TXN *global_tid;

static const struct option options[] = {
  { "help", no_argument, 0, 'h' },
  { "version", no_argument, 0, 'V' },
  { "config", required_argument, 0, 'c' },
  { "debug", no_argument, 0, 'd' },
  { "no-debug", no_argument, 0, 'D' },
  { 0, 0, 0, 0 }
};

/* display usage message and terminate */
static void help(void) {
  xprintf("Usage:\n"
	  "  disorder-rescan [OPTIONS] [PATH...]\n"
	  "Options:\n"
	  "  --help, -h              Display usage message\n"
	  "  --version, -V           Display version number\n"
	  "  --config PATH, -c PATH  Set configuration file\n"
	  "  --debug, -d             Turn on debugging\n"
          "\n"
          "Rescanner for DisOrder.  Not intended to be run\n"
          "directly.\n");
  xfclose(stdout);
  exit(0);
}

/* display version number and terminate */
static void version(void) {
  xprintf("disorder-rescan version %s\n", disorder_version_string);
  xfclose(stdout);
  exit(0);
}

static volatile sig_atomic_t signalled;

static void signal_handler(int sig) {
  if(sig == 0) _exit(-1);               /* "Cannot happen" */
  signalled = sig;
}

static int aborted(void) {
  return signalled || getppid() == 1;
}

/* Exit if our parent has gone away or we have been told to stop. */
static void checkabort(void) {
  if(getppid() == 1) {
    info("parent has terminated");
    trackdb_abort_transaction(global_tid);
    exit(0);
  }
  if(signalled) {
    info("received signal %d", signalled);
    trackdb_abort_transaction(global_tid);
    exit(0);
  }
}

/* rescan a collection */
static void rescan_collection(const struct collection *c) {
  pid_t pid, r;
  int p[2], n, w;
  FILE *fp = 0;
  char *path, *track;
  long ntracks = 0, nnew = 0;
  
  checkabort();
  info("rescanning %s with %s", c->root, c->module);
  /* plugin runs in a subprocess */
  xpipe(p);
  if(!(pid = xfork())) {
    exitfn = _exit;
    xclose(p[0]);
    xdup2(p[1], 1);
    xclose(p[1]);
    scan(c->module, c->root);
    if(fflush(stdout) < 0)
      fatal(errno, "error writing to scanner pipe");
    _exit(0);
  }
  xclose(p[1]);
  if(!(fp = fdopen(p[0], "r")))
    fatal(errno, "error calling fdopen");
  /* read tracks from the plugin */
  while(!inputline("rescanner", fp, &path, 0)) {
    checkabort();
    /* actually we can cope relatively well within the server, but they'll go
     * wrong in track listings */
    if(strchr(path, '\n')) {
      error(0, "cannot cope with tracks with newlines in the name");
      continue;
    }
    if(!(track = any2utf8(c->encoding, path))) {
      error(0, "cannot convert track path to UTF-8: %s", path);
      continue;
    }
    D(("track %s", track));
    /* only tracks with a known player are admitted */
    for(n = 0; (n < config->player.n
		&& fnmatch(config->player.s[n].s[0], track, 0) != 0); ++n)
      ;
    if(n < config->player.n) {
      nnew += !!trackdb_notice(track, path);
      ++ntracks;
      if(ntracks % 1000 == 0)
        info("rescanning %s, %ld tracks so far", c->root, ntracks);
    }
  }
  /* tidy up */
  if(ferror(fp)) {
    error(errno, "error reading from scanner pipe");
    goto done;
  }
  xfclose(fp);
  fp = 0;
  while((r = waitpid(pid, &w, 0)) == -1 && errno == EINTR)
    ;
  if(r < 0) fatal(errno, "error calling waitpid");
  pid = 0;
  if(w) {
    error(0, "scanner subprocess: %s", wstat(w));
    goto done;
  }
  info("rescanned %s, %ld tracks, %ld new", c->root, ntracks, nnew);
done:
  if(fp)
    xfclose(fp);
  if(pid)
    while((r = waitpid(pid, &w, 0)) == -1 && errno == EINTR)
      ;
}

struct recheck_state {
  const struct collection *c;
  long nobsolete, nnocollection, nlength;
};

/* called for each non-alias track */
static int recheck_callback(const char *track,
                            struct kvp *data,
                            void *u,
                            DB_TXN *tid) {
  struct recheck_state *cs = u;
  const struct collection *c = cs->c;
  const char *path = kvp_get(data, "_path");
  char buffer[20];
  int err;
  long n;

  if(aborted()) return EINTR;
  D(("rechecking %s", track));
  /* if we're not checking a specific collection, find the right collection */
  if(!c) {
    if(!(c = find_track_collection(track))) {
      D(("obsoleting %s", track));
      if((err = trackdb_obsolete(track, tid))) return err;
      ++cs->nnocollection;
      return 0;
    }
  }
  /* see if the track has evaporated */
  if(check(c->module, c->root, path) == 0) {
    D(("obsoleting %s", track));
    if((err = trackdb_obsolete(track, tid))) return err;
    ++cs->nobsolete;
    return 0;
  }
  /* make sure we know the length */
  if(!kvp_get(data, "_length")) {
    D(("recalculating length of %s", track));
    n = tracklength(track, path);
    if(n > 0) {
      byte_snprintf(buffer, sizeof buffer, "%ld", n);
      kvp_set(&data, "_length", buffer);
      if((err = trackdb_putdata(trackdb_tracksdb, track, data, tid, 0)))
        return err;
      ++cs->nlength;
    }
  }
  return 0;
}

/* recheck a collection */
static void recheck_collection(const struct collection *c) {
  struct recheck_state cs;

  if(c)
    info("rechecking %s", c->root);
  else
    info("rechecking all tracks");
  for(;;) {
    checkabort();
    global_tid = trackdb_begin_transaction();
    memset(&cs, 0, sizeof cs);
    cs.c = c;
    if(trackdb_scan(c ? c->root : 0, recheck_callback, &cs, global_tid))
      goto fail;
    break;
  fail:
    /* Maybe we need to shut down */
    checkabort();
    /* Abort the transaction and try again in a bit. */
    trackdb_abort_transaction(global_tid);
    global_tid = 0;
    /* Let anything else that is going on get out of the way. */
    sleep(10);
    checkabort();
    if(c)
      info("resuming recheck of %s", c->root);
    else
      info("resuming global recheck");
  }
  trackdb_commit_transaction(global_tid);
  global_tid = 0;
  if(c)
    info("rechecked %s, %ld obsoleted, %ld lengths calculated",
         c->root, cs.nobsolete, cs.nlength);
  else
    info("rechecked all tracks, %ld no collection, %ld obsoleted, %ld lengths calculated",
         cs.nnocollection, cs.nobsolete, cs.nlength);
}

/* rescan/recheck a collection by name */
static void do_directory(const char *s,
			 void (*fn)(const struct collection *c)) {
  int n;
  
  for(n = 0; (n < config->collection.n
	      && strcmp(config->collection.s[n].root, s)); ++n)
    ;
  if(n < config->collection.n)
    fn(&config->collection.s[n]);
  else
    error(0, "no collection has root '%s'", s);
}

/* rescan/recheck all collections */
static void do_all(void (*fn)(const struct collection *c)) {
  int n;

  for(n = 0; n < config->collection.n; ++n)
    fn(&config->collection.s[n]);
  /* TODO: we need to tidy up tracks from collections now removed.  We could do
   * this two ways: either remember collections we think there are and spot
   * their disappearance, or iterate over all tracks and gc any that don't fit
   * into some collection.
   *
   * Having a way to rename collections would be rather convenient too but
   * that's another kettle of monkeys.
   */
}

int main(int argc, char **argv) {
  int n;
  struct sigaction sa;
  
  set_progname(argv);
  mem_init();
  if(!setlocale(LC_CTYPE, "")) fatal(errno, "error calling setlocale");
  while((n = getopt_long(argc, argv, "hVc:dD", options, 0)) >= 0) {
    switch(n) {
    case 'h': help();
    case 'V': version();
    case 'c': configfile = optarg; break;
    case 'd': debugging = 1; break;
    case 'D': debugging = 0; break;
    default: fatal(0, "invalid option");
    }
  }
  /* If stderr is a TTY then log there, otherwise to syslog. */
  if(!isatty(2)) {
    openlog(progname, LOG_PID, LOG_DAEMON);
    log_default = &log_syslog;
  }
  if(config_read()) fatal(0, "cannot read configuration");
  xnice(config->nice_rescan);
  sa.sa_handler = signal_handler;
  sa.sa_flags = SA_RESTART;
  sigemptyset(&sa.sa_mask);
  xsigaction(SIGTERM, &sa, 0);
  xsigaction(SIGINT, &sa, 0);
  info("started");
  trackdb_init(0);
  trackdb_open();
  if(optind == argc) {
    /* Rescan all collections */
    do_all(rescan_collection);
    /* Check that every track still exists */
    recheck_collection(0);
  }
  else {
    /* Rescan specified collections */
    for(n = optind; n < argc; ++n)
      do_directory(argv[n], rescan_collection);
    /* Check specified collections for tracks that have gone */
    for(n = optind; n < argc; ++n)
      do_directory(argv[n], recheck_collection);
  }
  trackdb_close();
  trackdb_deinit();
  info("completed");
  return 0;
}

/*
Local Variables:
c-basic-offset:2
comment-column:40
fill-column:79
indent-tabs-mode:nil
End:
*/
Commit	Line	Data
460b9539	1	/*
460b9539	2	* This file is part of DisOrder
eb525fcd	3	* Copyright (C) 2005, 2006, 2007 Richard Kettlewell
460b9539	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License as published by
	7	* the Free Software Foundation; either version 2 of the License, or
	8	* (at your option) any later version.
	9	*
	10	* This program is distributed in the hope that it will be useful, but
	11	* WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	* General Public License for more details.
	14	*
	15	* You should have received a copy of the GNU General Public License
	16	* along with this program; if not, write to the Free Software
	17	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
	18	* USA
	19	*/
	20
	21	#include <config.h>
	22	#include "types.h"
	23
	24	#include <getopt.h>
	25	#include <stdio.h>
	26	#include <stdlib.h>
	27	#include <db.h>
	28	#include <locale.h>
	29	#include <errno.h>
	30	#include <sys/types.h>
	31	#include <unistd.h>
	32	#include <pcre.h>
	33	#include <fnmatch.h>
	34	#include <sys/wait.h>
	35	#include <string.h>
	36	#include <syslog.h>
	37
	38	#include "configuration.h"
	39	#include "syscalls.h"
	40	#include "log.h"
	41	#include "defs.h"
	42	#include "mem.h"
	43	#include "plugin.h"
	44	#include "inputline.h"
	45	#include "charset.h"
	46	#include "wstat.h"
	47	#include "kvp.h"
	48	#include "printf.h"
	49	#include "trackdb.h"
	50	#include "trackdb-int.h"
d1694464	51	#include "trackname.h"
460b9539	52
	53	static DB_TXN *global_tid;
	54
	55	static const struct option options[] = {
	56	{ "help", no_argument, 0, 'h' },
	57	{ "version", no_argument, 0, 'V' },
	58	{ "config", required_argument, 0, 'c' },
	59	{ "debug", no_argument, 0, 'd' },
	60	{ "no-debug", no_argument, 0, 'D' },
	61	{ 0, 0, 0, 0 }
	62	};
	63
	64	/* display usage message and terminate */
	65	static void help(void) {
	66	xprintf("Usage:\n"
	67	" disorder-rescan [OPTIONS] [PATH...]\n"
	68	"Options:\n"
	69	" --help, -h Display usage message\n"
	70	" --version, -V Display version number\n"
	71	" --config PATH, -c PATH Set configuration file\n"
	72	" --debug, -d Turn on debugging\n"
	73	"\n"
	74	"Rescanner for DisOrder. Not intended to be run\n"
	75	"directly.\n");
	76	xfclose(stdout);
	77	exit(0);
	78	}
	79
	80	/* display version number and terminate */
	81	static void version(void) {
	82	xprintf("disorder-rescan version %s\n", disorder_version_string);
	83	xfclose(stdout);
	84	exit(0);
	85	}
	86
	87	static volatile sig_atomic_t signalled;
	88
	89	static void signal_handler(int sig) {
	90	if(sig == 0) _exit(-1); /* "Cannot happen" */
	91	signalled = sig;
	92	}
	93
	94	static int aborted(void) {
	95	return signalled \|\| getppid() == 1;
	96	}
	97
	98	/* Exit if our parent has gone away or we have been told to stop. */
	99	static void checkabort(void) {
	100	if(getppid() == 1) {
	101	info("parent has terminated");
	102	trackdb_abort_transaction(global_tid);
	103	exit(0);
	104	}
	105	if(signalled) {
	106	info("received signal %d", signalled);
	107	trackdb_abort_transaction(global_tid);
	108	exit(0);
	109	}
	110	}
	111
	112	/* rescan a collection */
	113	static void rescan_collection(const struct collection *c) {
	114	pid_t pid, r;
	115	int p[2], n, w;
116	FILE *fp = 0;
117	char path, track;
118	long ntracks = 0, nnew = 0;
119
120	checkabort();
121	info("rescanning %s with %s", c->root, c->module);
122	/* plugin runs in a subprocess */
123	xpipe(p);
124	if(!(pid = xfork())) {
125	exitfn = _exit;
126	xclose(p[0]);
127	xdup2(p[1], 1);
128	xclose(p[1]);
129	scan(c->module, c->root);
130	if(fflush(stdout) < 0)
131	fatal(errno, "error writing to scanner pipe");
132	_exit(0);
133	}
134	xclose(p[1]);
135	if(!(fp = fdopen(p[0], "r")))
136	fatal(errno, "error calling fdopen");
137	/* read tracks from the plugin */
138	while(!inputline("rescanner", fp, &path, 0)) {
139	checkabort();
140	/* actually we can cope relatively well within the server, but they'll go
141	* wrong in track listings */
142	if(strchr(path, '\n')) {
143	error(0, "cannot cope with tracks with newlines in the name");
144	continue;
145	}
146	if(!(track = any2utf8(c->encoding, path))) {
147	error(0, "cannot convert track path to UTF-8: %s", path);
148	continue;
149	}
150	D(("track %s", track));
151	/* only tracks with a known player are admitted */
152	for(n = 0; (n < config->player.n
153	&& fnmatch(config->player.s[n].s[0], track, 0) != 0); ++n)
154	;
155	if(n < config->player.n) {
156	nnew += !!trackdb_notice(track, path);
157	++ntracks;
e4ba53fd RK	158	if(ntracks % 1000 == 0)
e4ba53fd RK	159	info("rescanning %s, %ld tracks so far", c->root, ntracks);
460b9539	160	}
	161	}
	162	/* tidy up */
	163	if(ferror(fp)) {
	164	error(errno, "error reading from scanner pipe");
	165	goto done;
	166	}
	167	xfclose(fp);
	168	fp = 0;
	169	while((r = waitpid(pid, &w, 0)) == -1 && errno == EINTR)
	170	;
	171	if(r < 0) fatal(errno, "error calling waitpid");
	172	pid = 0;
	173	if(w) {
	174	error(0, "scanner subprocess: %s", wstat(w));
	175	goto done;
	176	}
	177	info("rescanned %s, %ld tracks, %ld new", c->root, ntracks, nnew);
	178	done:
	179	if(fp)
	180	xfclose(fp);
	181	if(pid)
	182	while((r = waitpid(pid, &w, 0)) == -1 && errno == EINTR)
	183	;
	184	}
	185
	186	struct recheck_state {
	187	const struct collection *c;
d1694464	188	long nobsolete, nnocollection, nlength;
460b9539	189	};
	190
	191	/* called for each non-alias track */
	192	static int recheck_callback(const char *track,
	193	struct kvp *data,
	194	void *u,
	195	DB_TXN *tid) {
	196	struct recheck_state *cs = u;
	197	const struct collection *c = cs->c;
	198	const char *path = kvp_get(data, "_path");
	199	char buffer[20];
	200	int err;
	201	long n;
	202
	203	if(aborted()) return EINTR;
	204	D(("rechecking %s", track));
d1694464	205	/* if we're not checking a specific collection, find the right collection */
	206	if(!c) {
	207	if(!(c = find_track_collection(track))) {
	208	D(("obsoleting %s", track));
	209	if((err = trackdb_obsolete(track, tid))) return err;
	210	++cs->nnocollection;
	211	return 0;
	212	}
	213	}
460b9539	214	/* see if the track has evaporated */
	215	if(check(c->module, c->root, path) == 0) {
	216	D(("obsoleting %s", track));
	217	if((err = trackdb_obsolete(track, tid))) return err;
	218	++cs->nobsolete;
	219	return 0;
	220	}
	221	/* make sure we know the length */
	222	if(!kvp_get(data, "_length")) {
	223	D(("recalculating length of %s", track));
	224	n = tracklength(track, path);
	225	if(n > 0) {
	226	byte_snprintf(buffer, sizeof buffer, "%ld", n);
	227	kvp_set(&data, "_length", buffer);
	228	if((err = trackdb_putdata(trackdb_tracksdb, track, data, tid, 0)))
	229	return err;
	230	++cs->nlength;
	231	}
	232	}
	233	return 0;
	234	}
	235
	236	/* recheck a collection */
	237	static void recheck_collection(const struct collection *c) {
	238	struct recheck_state cs;
	239
d1694464	240	if(c)
	241	info("rechecking %s", c->root);
	242	else
	243	info("rechecking all tracks");
460b9539	244	for(;;) {
	245	checkabort();
	246	global_tid = trackdb_begin_transaction();
	247	memset(&cs, 0, sizeof cs);
	248	cs.c = c;
d1694464	249	if(trackdb_scan(c ? c->root : 0, recheck_callback, &cs, global_tid))
d1694464	250	goto fail;
460b9539	251	break;
	252	fail:
	253	/* Maybe we need to shut down */
	254	checkabort();
	255	/* Abort the transaction and try again in a bit. */
	256	trackdb_abort_transaction(global_tid);
	257	global_tid = 0;
	258	/* Let anything else that is going on get out of the way. */
	259	sleep(10);
	260	checkabort();
d1694464	261	if(c)
	262	info("resuming recheck of %s", c->root);
	263	else
	264	info("resuming global recheck");
460b9539	265	}
	266	trackdb_commit_transaction(global_tid);
	267	global_tid = 0;
d1694464	268	if(c)
	269	info("rechecked %s, %ld obsoleted, %ld lengths calculated",
	270	c->root, cs.nobsolete, cs.nlength);
	271	else
	272	info("rechecked all tracks, %ld no collection, %ld obsoleted, %ld lengths calculated",
	273	cs.nnocollection, cs.nobsolete, cs.nlength);
460b9539	274	}
	275
	276	/* rescan/recheck a collection by name */
	277	static void do_directory(const char *s,
	278	void (fn)(const struct collection c)) {
	279	int n;
	280
	281	for(n = 0; (n < config->collection.n
	282	&& strcmp(config->collection.s[n].root, s)); ++n)
	283	;
	284	if(n < config->collection.n)
	285	fn(&config->collection.s[n]);
	286	else
	287	error(0, "no collection has root '%s'", s);
	288	}
	289
	290	/* rescan/recheck all collections */
	291	static void do_all(void (fn)(const struct collection c)) {
	292	int n;
	293
	294	for(n = 0; n < config->collection.n; ++n)
	295	fn(&config->collection.s[n]);
6aba3f6c RK	296	/* TODO: we need to tidy up tracks from collections now removed. We could do
	297	* this two ways: either remember collections we think there are and spot
	298	* their disappearance, or iterate over all tracks and gc any that don't fit
	299	* into some collection.
	300	*
	301	* Having a way to rename collections would be rather convenient too but
	302	* that's another kettle of monkeys.
	303	*/
460b9539	304	}
	305
	306	int main(int argc, char **argv) {
	307	int n;
	308	struct sigaction sa;
	309
	310	set_progname(argv);
320598d4	311	mem_init();
460b9539	312	if(!setlocale(LC_CTYPE, "")) fatal(errno, "error calling setlocale");
	313	while((n = getopt_long(argc, argv, "hVc:dD", options, 0)) >= 0) {
	314	switch(n) {
	315	case 'h': help();
	316	case 'V': version();
	317	case 'c': configfile = optarg; break;
	318	case 'd': debugging = 1; break;
	319	case 'D': debugging = 0; break;
	320	default: fatal(0, "invalid option");
	321	}
	322	}
	323	/* If stderr is a TTY then log there, otherwise to syslog. */
	324	if(!isatty(2)) {
	325	openlog(progname, LOG_PID, LOG_DAEMON);
	326	log_default = &log_syslog;
	327	}
	328	if(config_read()) fatal(0, "cannot read configuration");
	329	xnice(config->nice_rescan);
	330	sa.sa_handler = signal_handler;
	331	sa.sa_flags = SA_RESTART;
	332	sigemptyset(&sa.sa_mask);
	333	xsigaction(SIGTERM, &sa, 0);
	334	xsigaction(SIGINT, &sa, 0);
	335	info("started");
	336	trackdb_init(0);
	337	trackdb_open();
	338	if(optind == argc) {
d1694464	339	/* Rescan all collections */
460b9539	340	do_all(rescan_collection);
d1694464	341	/* Check that every track still exists */
d1694464	342	recheck_collection(0);
460b9539	343	}
460b9539	344	else {
d1694464	345	/* Rescan specified collections */
460b9539	346	for(n = optind; n < argc; ++n)
460b9539	347	do_directory(argv[n], rescan_collection);
d1694464	348	/* Check specified collections for tracks that have gone */
460b9539	349	for(n = optind; n < argc; ++n)
	350	do_directory(argv[n], recheck_collection);
	351	}
	352	trackdb_close();
	353	trackdb_deinit();
	354	info("completed");
	355	return 0;
	356	}
	357
	358	/*
	359	Local Variables:
	360	c-basic-offset:2
	361	comment-column:40
	362	fill-column:79
	363	indent-tabs-mode:nil
	364	End:
	365	*/