X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~mdw/git/disorder/blobdiff_plain/62dc37489c39c8d20046b4d5e323707239491df6..4942ee7d61bf22ba38bf026c7d05028cb7db0d54:/server/rescan.c

diff --git a/server/rescan.c b/server/rescan.c
index aaa5f93..dfe855a 100644
--- a/server/rescan.c
+++ b/server/rescan.c
@@ -1,56 +1,28 @@
 /*
  * This file is part of DisOrder 
- * Copyright (C) 2005, 2006, 2007 Richard Kettlewell
+ * Copyright (C) 2005-2008 Richard Kettlewell
  *
- * This program is free software; you can redistribute it and/or modify
+ * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
+ * the Free Software Foundation, either version 3 of the License, or
  * (at your option) any later version.
  *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
  * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- * USA
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+/** @file server/rescan.c
+ * @brief Rescanning utility
+ *
+ * Invoked by the server.
+ */
+#include "disorder-server.h"
 
-#include <config.h>
-#include "types.h"
-
-#include <getopt.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <db.h>
-#include <locale.h>
-#include <errno.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <pcre.h>
-#include <fnmatch.h>
-#include <sys/wait.h>
-#include <string.h>
-#include <syslog.h>
-#include <time.h>
-
-#include "configuration.h"
-#include "syscalls.h"
-#include "log.h"
-#include "defs.h"
-#include "mem.h"
-#include "plugin.h"
-#include "inputline.h"
-#include "charset.h"
-#include "wstat.h"
-#include "kvp.h"
-#include "printf.h"
-#include "trackdb.h"
-#include "trackdb-int.h"
-#include "trackname.h"
-
+static time_t last_report;
 static DB_TXN *global_tid;
 
 static const struct option options[] = {
@@ -59,6 +31,10 @@ static const struct option options[] = {
   { "config", required_argument, 0, 'c' },
   { "debug", no_argument, 0, 'd' },
   { "no-debug", no_argument, 0, 'D' },
+  { "syslog", no_argument, 0, 's' },
+  { "no-syslog", no_argument, 0, 'S' },
+  { "check", no_argument, 0, 'K' },
+  { "no-check", no_argument, 0, 'C' },
   { 0, 0, 0, 0 }
 };
 
@@ -71,6 +47,8 @@ static void help(void) {
 	  "  --version, -V           Display version number\n"
 	  "  --config PATH, -c PATH  Set configuration file\n"
 	  "  --debug, -d             Turn on debugging\n"
+          "  --[no-]syslog           Enable/disable logging to syslog\n"
+          "  --[no-]check            Enable/disable track length check\n"
           "\n"
           "Rescanner for DisOrder.  Not intended to be run\n"
           "directly.\n");
@@ -78,13 +56,6 @@ static void help(void) {
   exit(0);
 }
 
-/* display version number and terminate */
-static void version(void) {
-  xprintf("disorder-rescan version %s\n", disorder_version_string);
-  xfclose(stdout);
-  exit(0);
-}
-
 static volatile sig_atomic_t signalled;
 
 static void signal_handler(int sig) {
@@ -148,6 +119,13 @@ static void rescan_collection(const struct collection *c) {
       error(0, "cannot convert track path to UTF-8: %s", path);
       continue;
     }
+    if(config->dbversion > 1) {
+      /* We use NFC track names */
+      if(!(track = utf8_compose_canon(track, strlen(track), 0))) {
+        error(0, "cannot convert track path to NFC: %s", path);
+        continue;
+      }
+    }
     D(("track %s", track));
     /* only tracks with a known player are admitted */
     for(n = 0; (n < config->player.n
@@ -156,8 +134,10 @@ static void rescan_collection(const struct collection *c) {
     if(n < config->player.n) {
       nnew += !!trackdb_notice(track, path);
       ++ntracks;
-      if(ntracks % 1000 == 0)
+      if(ntracks % 100 == 0 && xtime(0) > last_report + 10) {
         info("rescanning %s, %ld tracks so far", c->root, ntracks);
+        xtime(&last_report);
+      }
     }
   }
   /* tidy up */
@@ -187,52 +167,75 @@ done:
 struct recheck_state {
   const struct collection *c;
   long nobsolete, nnocollection, nlength;
+  struct recheck_track *tracks;
+};
+
+struct recheck_track {
+  struct recheck_track *next;
+  const char *track;
 };
 
 /* called for each non-alias track */
-static int recheck_callback(const char *track,
-                            struct kvp *data,
-                            void *u,
-                            DB_TXN *tid) {
+static int recheck_list_callback(const char *track,
+                                 struct kvp attribute((unused)) *data,
+                                 struct kvp attribute((unused)) *prefs,
+                                 void *u,
+                                 DB_TXN attribute((unused)) *tid) {
   struct recheck_state *cs = u;
+  struct recheck_track *t = xmalloc(sizeof *t);
+
+  t->next = cs->tracks;
+  t->track = track;
+  cs->tracks = t;
+  return 0;
+}
+
+static int recheck_track_tid(struct recheck_state *cs,
+                             const struct recheck_track *t,
+                             DB_TXN *tid) {
   const struct collection *c = cs->c;
-  const char *path = kvp_get(data, "_path");
+  const char *path;
   char buffer[20];
   int err, n;
   long length;
+  struct kvp *data;
 
-  if(aborted()) return EINTR;
-  D(("rechecking %s", track));
+  if((err = trackdb_getdata(trackdb_tracksdb, t->track, &data, tid)))
+    return err;
+  path = kvp_get(data, "_path");
+  D(("rechecking %s", t->track));
   /* if we're not checking a specific collection, find the right collection */
   if(!c) {
-    if(!(c = find_track_collection(track))) {
-      D(("obsoleting %s", track));
-      if((err = trackdb_obsolete(track, tid))) return err;
+    if(!(c = find_track_collection(t->track))) {
+      D(("obsoleting %s", t->track));
+      if((err = trackdb_obsolete(t->track, tid)))
+        return err;
       ++cs->nnocollection;
       return 0;
     }
   }
   /* see if the track has evaporated */
   if(check(c->module, c->root, path) == 0) {
-    D(("obsoleting %s", track));
-    if((err = trackdb_obsolete(track, tid))) return err;
+    D(("obsoleting %s", t->track));
+    if((err = trackdb_obsolete(t->track, tid)))
+      return err;
     ++cs->nobsolete;
     return 0;
   }
   /* make sure we know the length */
   if(!kvp_get(data, "_length")) {
-    D(("recalculating length of %s", track));
+    D(("recalculating length of %s", t->track));
     for(n = 0; n < config->tracklength.n; ++n)
-      if(fnmatch(config->tracklength.s[n].s[0], track, 0) == 0)
+      if(fnmatch(config->tracklength.s[n].s[0], t->track, 0) == 0)
         break;
     if(n >= config->tracklength.n)
-      error(0, "no tracklength plugin found for %s", track);
+      error(0, "no tracklength plugin found for %s", t->track);
     else {
-      length = tracklength(config->tracklength.s[n].s[1], track, path);
+      length = tracklength(config->tracklength.s[n].s[1], t->track, path);
       if(length > 0) {
         byte_snprintf(buffer, sizeof buffer, "%ld", length);
         kvp_set(&data, "_length", buffer);
-        if((err = trackdb_putdata(trackdb_tracksdb, track, data, tid, 0)))
+        if((err = trackdb_putdata(trackdb_tracksdb, t->track, data, tid, 0)))
           return err;
         ++cs->nlength;
       }
@@ -241,20 +244,38 @@ static int recheck_callback(const char *track,
   return 0;
 }
 
+static int recheck_track(struct recheck_state *cs,
+                         const struct recheck_track *t) {
+  int e;
+
+  WITH_TRANSACTION(recheck_track_tid(cs, t, tid));
+  return e;
+}
+
 /* recheck a collection */
 static void recheck_collection(const struct collection *c) {
   struct recheck_state cs;
+  const struct recheck_track *t;
+  long nrc;
 
   if(c)
     info("rechecking %s", c->root);
   else
     info("rechecking all tracks");
+  /* Doing the checking inside a transaction locks up the server for much too
+   * long (because it spends lots of time thinking about each track).  So we
+   * pull the full track list into memory and work from that.
+   *
+   * 100,000 tracks at, say, 80 bytes per track name, gives 8MB, which is quite
+   * reasonable.
+   */
   for(;;) {
     checkabort();
+    info("getting track list");
     global_tid = trackdb_begin_transaction();
     memset(&cs, 0, sizeof cs);
     cs.c = c;
-    if(trackdb_scan(c ? c->root : 0, recheck_callback, &cs, global_tid))
+    if(trackdb_scan(c ? c->root : 0, recheck_list_callback, &cs, global_tid))
       goto fail;
     break;
   fail:
@@ -273,6 +294,20 @@ static void recheck_collection(const struct collection *c) {
   }
   trackdb_commit_transaction(global_tid);
   global_tid = 0;
+  nrc = 0;
+  for(t = cs.tracks; t; t = t->next) {
+    if(aborted())
+      return;
+    recheck_track(&cs, t);
+    ++nrc;
+    if(nrc % 100 == 0 && xtime(0) > last_report + 10) {
+      if(c)
+        info("rechecking %s, %ld tracks so far", c->root, nrc);
+      else
+        info("rechecking all tracks, %ld tracks so far", nrc);
+      xtime(&last_report);
+    }
+  }
   if(c)
     info("rechecked %s, %ld obsoleted, %ld lengths calculated",
          c->root, cs.nobsolete, cs.nlength);
@@ -315,33 +350,37 @@ static void do_all(void (*fn)(const struct collection *c)) {
 static void expire_noticed(void) {
   time_t now;
 
-  time(&now);
+  xtime(&now);
   trackdb_expire_noticed(now - config->noticed_history * 86400);
 }
 
 int main(int argc, char **argv) {
-  int n;
+  int n, logsyslog = !isatty(2);
   struct sigaction sa;
+  int do_check = 1;
   
   set_progname(argv);
   mem_init();
   if(!setlocale(LC_CTYPE, "")) fatal(errno, "error calling setlocale");
-  while((n = getopt_long(argc, argv, "hVc:dD", options, 0)) >= 0) {
+  while((n = getopt_long(argc, argv, "hVc:dDSsKC", options, 0)) >= 0) {
     switch(n) {
     case 'h': help();
-    case 'V': version();
+    case 'V': version("disorder-rescan");
     case 'c': configfile = optarg; break;
     case 'd': debugging = 1; break;
     case 'D': debugging = 0; break;
+    case 'S': logsyslog = 0; break;
+    case 's': logsyslog = 1; break;
+    case 'K': do_check = 1; break;
+    case 'C': do_check = 0; break;
     default: fatal(0, "invalid option");
     }
   }
-  /* If stderr is a TTY then log there, otherwise to syslog. */
-  if(!isatty(2)) {
+  if(logsyslog) {
     openlog(progname, LOG_PID, LOG_DAEMON);
     log_default = &log_syslog;
   }
-  if(config_read(0)) fatal(0, "cannot read configuration");
+  if(config_read(0, NULL)) fatal(0, "cannot read configuration");
   xnice(config->nice_rescan);
   sa.sa_handler = signal_handler;
   sa.sa_flags = SA_RESTART;
@@ -349,13 +388,14 @@ int main(int argc, char **argv) {
   xsigaction(SIGTERM, &sa, 0);
   xsigaction(SIGINT, &sa, 0);
   info("started");
-  trackdb_init(0);
-  trackdb_open();
+  trackdb_init(TRACKDB_NO_RECOVER);
+  trackdb_open(TRACKDB_NO_UPGRADE);
   if(optind == argc) {
     /* Rescan all collections */
     do_all(rescan_collection);
     /* Check that every track still exists */
-    recheck_collection(0);
+    if(do_check)
+      recheck_collection(0);
     /* Expire noticed.db */
     expire_noticed();
   }
@@ -364,8 +404,9 @@ int main(int argc, char **argv) {
     for(n = optind; n < argc; ++n)
       do_directory(argv[n], rescan_collection);
     /* Check specified collections for tracks that have gone */
-    for(n = optind; n < argc; ++n)
-      do_directory(argv[n], recheck_collection);
+    if(do_check)
+      for(n = optind; n < argc; ++n)
+        do_directory(argv[n], recheck_collection);
   }
   trackdb_close();
   trackdb_deinit();