From d6dde5a3f49c7036a73f835fac1ece3144397fb7 Mon Sep 17 00:00:00 2001 Message-Id: From: Mark Wooding Date: Thu, 1 Nov 2007 19:07:31 +0000 Subject: [PATCH] compute stats in a subprocess to avoid wedging the server if it takes ages Organization: Straylight/Edgeware From: Richard Kettlewell --- .bzrignore | 3 ++ doc/Makefile.am | 8 ++-- doc/disorder-stats.8.in | 46 ++++++++++++++++++ server/Makefile.am | 8 +++- server/server.c | 22 ++++----- server/stats.c | 104 ++++++++++++++++++++++++++++++++++++++++ server/trackdb.c | 104 ++++++++++++++++++++++++++++++++++++---- server/trackdb.h | 5 ++ 8 files changed, 274 insertions(+), 26 deletions(-) create mode 100644 doc/disorder-stats.8.in create mode 100644 server/stats.c diff --git a/.bzrignore b/.bzrignore index 978ce4d..4fd0909 100644 --- a/.bzrignore +++ b/.bzrignore @@ -113,3 +113,6 @@ disobedience/manual.h disobedience/manual.html disobedience/images.h debian/disorder-server +doc/disorder-stats.8 +doc/disorder-stats.8.html +server/disorder-stats diff --git a/doc/Makefile.am b/doc/Makefile.am index e773f20..d7e9187 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -21,14 +21,15 @@ SEDFILES=disorder.1 disorderd.8 disorder_config.5 \ disorder-dump.8 disorder_protocol.5 disorder-deadlock.8 \ disorder-rescan.8 disobedience.1 disorderfm.1 disorder-playrtp.1 \ - disorder-decode.8 + disorder-decode.8 disorder-stats.8 include ${top_srcdir}/scripts/sedfiles.make man_MANS=disorderd.8 disorder.1 disorder.3 disorder_config.5 disorder-dump.8 \ disorder_protocol.5 disorder-deadlock.8 \ disorder-rescan.8 disobedience.1 disorderfm.1 disorder-speaker.8 \ - disorder-playrtp.1 disorder-normalize.8 disorder-decode.8 + disorder-playrtp.1 disorder-normalize.8 disorder-decode.8 \ + disorder-stats.8 noinst_MANS=tkdisorder.1 @@ -46,6 +47,7 @@ EXTRA_DIST=disorderd.8.in disorder.1.in disorder_config.5.in \ disorder.3 disorder-dump.8.in disorder_protocol.5.in \ tkdisorder.1 disorder-deadlock.8.in disorder-rescan.8.in \ disobedience.1.in disorderfm.1.in disorder-speaker.8 \ - disorder-playrtp.1.in disorder-decode.8.in disorder-normalize.8 + disorder-playrtp.1.in disorder-decode.8.in disorder-normalize.8 \ + disorder-stats.8.in CLEANFILES=$(SEDFILES) $(HTMLMAN) diff --git a/doc/disorder-stats.8.in b/doc/disorder-stats.8.in new file mode 100644 index 0000000..4ceda15 --- /dev/null +++ b/doc/disorder-stats.8.in @@ -0,0 +1,46 @@ +.\" +.\" Copyright (C) 2007 Richard Kettlewell +.\" +.\" This program is free software; you can redistribute it and/or modify +.\" it under the terms of the GNU General Public License as published by +.\" the Free Software Foundation; either version 2 of the License, or +.\" (at your option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, but +.\" WITHOUT ANY WARRANTY; without even the implied warranty of +.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +.\" General Public License for more details. +.\" +.\" You should have received a copy of the GNU General Public License +.\" along with this program; if not, write to the Free Software +.\" Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 +.\" USA +.\" +.TH disorder-stats 8 +.SH NAME +disorder-stats \- DisOrder statistics +.SH SYNOPSIS +.B disorder-stats +.RI [ OPTIONS ] +.SH DESCRIPTION +.B disorder-stats +reports server statistics. It is used by the server and would not +normally be invoked manually. +.SH OPTIONS +.TP +.B --config \fIPATH\fR, \fB-c \fIPATH +Set the configuration file. +.TP +.B --debug\fR, \fB-d +Enable debugging. +.TP +.B --help\fR, \fB-h +Display a usage message. +.TP +.B --version\fR, \fB-V +Display version number. +.SH "SEE ALSO" +\fBdisorderd\fR(8), \fBdisorder_config\fR(5) +.\" Local Variables: +.\" mode:nroff +.\" End: diff --git a/server/Makefile.am b/server/Makefile.am index 6ef505b..e3a1a0a 100644 --- a/server/Makefile.am +++ b/server/Makefile.am @@ -19,7 +19,8 @@ # sbin_PROGRAMS=disorderd disorder-deadlock disorder-rescan disorder-dump \ - disorder-speaker disorder-decode disorder-normalize + disorder-speaker disorder-decode disorder-normalize \ + disorder-stats noinst_PROGRAMS=disorder.cgi trackname noinst_DATA=uk.org.greenend.rjk.disorder.plist @@ -75,6 +76,11 @@ disorder_rescan_LDADD=$(LIBOBJS) ../lib/libdisorder.a \ disorder_rescan_LDFLAGS=-export-dynamic disorder_rescan_DEPENDENCIES=../lib/libdisorder.a +disorder_stats_SOURCES=stats.c trackdb.c trackdb.h +disorder_stats_LDADD=$(LIBOBJS) ../lib/libdisorder.a \ + $(LIBDB) $(LIBPCRE) $(LIBICONV) $(LIBGCRYPT) +disorder_stats_DEPENDENCIES=../lib/libdisorder.a + disorder_dump_SOURCES=dump.c \ trackdb.c trackdb.h \ ../lib/memgc.c diff --git a/server/server.c b/server/server.c index 5a11b8c..a112f78 100644 --- a/server/server.c +++ b/server/server.c @@ -665,21 +665,19 @@ static int c_random_enabled(struct conn *c, return 1; /* completed */ } +static void got_stats(char *stats, void *u) { + struct conn *const c = u; + + sink_printf(ev_writer_sink(c->w), "253 stats\n%s\n.\n", stats); + /* Now we can start processing commands again */ + ev_reader_enable(c->r); +} + static int c_stats(struct conn *c, char attribute((unused)) **vec, int attribute((unused)) nvec) { - char **v; - int nv, n; - - v = trackdb_stats(&nv); - sink_printf(ev_writer_sink(c->w), "253 stats\n"); - for(n = 0; n < nv; ++n) { - if(v[n][0] == '.') - sink_writes(ev_writer_sink(c->w), "."); - sink_printf(ev_writer_sink(c->w), "%s\n", v[n]); - } - sink_writes(ev_writer_sink(c->w), ".\n"); - return 1; + trackdb_stats_subprocess(c->ev, got_stats, c); + return 0; /* not yet complete */ } static int c_volume(struct conn *c, diff --git a/server/stats.c b/server/stats.c new file mode 100644 index 0000000..d125caf --- /dev/null +++ b/server/stats.c @@ -0,0 +1,104 @@ +/* + * This file is part of DisOrder + * Copyright (C) 2007 Richard Kettlewell + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + */ + +#include +#include "types.h" + +#include +#include +#include +#include +#include + +#include "defs.h" +#include "mem.h" +#include "log.h" +#include "syscalls.h" +#include "configuration.h" +#include "trackdb.h" + +static const struct option options[] = { + { "help", no_argument, 0, 'h' }, + { "version", no_argument, 0, 'V' }, + { "config", required_argument, 0, 'c' }, + { "debug", no_argument, 0, 'd' }, + { "no-debug", no_argument, 0, 'D' }, + { 0, 0, 0, 0 } +}; + +/* display usage message and terminate */ +static void help(void) { + xprintf("Usage:\n" + " disorder-stats [OPTIONS]\n" + "Options:\n" + " --help, -h Display usage message\n" + " --version, -V Display version number\n" + " --config PATH, -c PATH Set configuration file\n" + " --[no-]debug, -d Turn on (off) debugging\n" + "\n" + "Generate DisOrder database statistics.\n"); + xfclose(stdout); + exit(0); +} + +/* display version number and terminate */ +static void version(void) { + xprintf("disorder-stats version %s\n", disorder_version_string); + xfclose(stdout); + exit(0); +} + +int main(int argc, char **argv) { + int n; + char **stats; + + set_progname(argv); + mem_init(); + if(!setlocale(LC_CTYPE, "")) fatal(errno, "error calling setlocale"); + while((n = getopt_long(argc, argv, "hVc:dD", options, 0)) >= 0) { + switch(n) { + case 'h': help(); + case 'V': version(); + case 'c': configfile = optarg; break; + case 'd': debugging = 1; break; + case 'D': debugging = 0; break; + default: fatal(0, "invalid option"); + } + } + if(config_read(0)) + fatal(0, "cannot read configuration"); + trackdb_init(0); + trackdb_open(); + stats = trackdb_stats(0); + while(*stats) + xprintf("%s\n", *stats++); + xfclose(stdout); + return 0; +} + + +/* +Local Variables: +c-basic-offset:2 +comment-column:40 +fill-column:79 +indent-tabs-mode:nil +End: +*/ diff --git a/server/trackdb.c b/server/trackdb.c index 2d369fe..59f755f 100644 --- a/server/trackdb.c +++ b/server/trackdb.c @@ -154,7 +154,8 @@ static int reap_db_deadlock(ev_source attribute((unused)) *ev, return 0; } -static pid_t subprogram(ev_source *ev, const char *prog) { +static pid_t subprogram(ev_source *ev, const char *prog, + int outputfd) { pid_t pid; int lfd; @@ -170,6 +171,11 @@ static pid_t subprogram(ev_source *ev, const char *prog) { if(lfd != -1) { xdup2(lfd, 1); xdup2(lfd, 2); + xclose(lfd); + } + if(outputfd != -1) { + xdup2(outputfd, 1); + xclose(outputfd); } /* If we were negatively niced, undo it. We don't bother checking for * error, it's not that important. */ @@ -186,7 +192,7 @@ static pid_t subprogram(ev_source *ev, const char *prog) { /* start deadlock manager */ void trackdb_master(ev_source *ev) { assert(db_deadlock_pid == -1); - db_deadlock_pid = subprogram(ev, DEADLOCK); + db_deadlock_pid = subprogram(ev, DEADLOCK, -1); ev_child(ev, db_deadlock_pid, 0, reap_db_deadlock, 0); D(("started deadlock manager")); } @@ -952,7 +958,6 @@ static int search_league(struct vector *v, int count, DB_TXN *tid) { char **trackdb_stats(int *nstatsp) { DB_TXN *tid; struct vector v; - char *s; vector_init(&v); for(;;) { @@ -968,12 +973,6 @@ char **trackdb_stats(int *nstatsp) { if(get_stats(&v, trackdb_prefsdb, SI(hash), tid)) goto fail; vector_append(&v, (char *)""); if(search_league(&v, 10, tid)) goto fail; - vector_append(&v, (char *)""); - vector_append(&v, (char *)"Server stats:"); - byte_xasprintf(&s, "track lookup cache hits: %lu", cache_files_hits); - vector_append(&v, (char *)s); - byte_xasprintf(&s, "track lookup cache misses: %lu", cache_files_misses); - vector_append(&v, (char *)s); vector_terminate(&v); break; fail: @@ -984,6 +983,91 @@ fail: return v.vec; } +struct stats_details { + void (*done)(char *data, void *u); + void *u; + int exited; /* subprocess exited */ + int closed; /* pipe close */ + int wstat; /* wait status from subprocess */ + struct dynstr data[1]; /* data read from pipe */ +}; + +static void stats_complete(struct stats_details *d) { + char *s; + + if(!(d->exited && d->closed)) + return; + byte_xasprintf(&s, "\n" + "Server stats:\n" + "track lookup cache hits: %lu\n" + "track lookup cache misses: %lu\n", + cache_files_hits, + cache_files_misses); + dynstr_append_string(d->data, s); + dynstr_terminate(d->data); + d->done(d->data->vec, d->u); +} + +static int stats_finished(ev_source attribute((unused)) *ev, + pid_t attribute((unused)) pid, + int status, + const struct rusage attribute((unused)) *rusage, + void *u) { + struct stats_details *const d = u; + + d->exited = 1; + if(status) + error(0, "disorder-stats %s", wstat(status)); + stats_complete(d); + return 0; +} + +static int stats_read(ev_source attribute((unused)) *ev, + ev_reader *reader, + int attribute((unused)) fd, + void *ptr, + size_t bytes, + int eof, + void *u) { + struct stats_details *const d = u; + + dynstr_append_bytes(d->data, ptr, bytes); + ev_reader_consume(reader, bytes); + if(eof) + d->closed = 1; + stats_complete(d); + return 0; +} + +static int stats_error(ev_source attribute((unused)) *ev, + int attribute((unused)) fd, + int errno_value, + void *u) { + struct stats_details *const d = u; + + error(errno_value, "error reading from pipe to disorder-stats"); + d->closed = 1; + stats_complete(d); + return 0; +} + +void trackdb_stats_subprocess(ev_source *ev, + void (*done)(char *data, void *u), + void *u) { + int p[2]; + pid_t pid; + struct stats_details *d = xmalloc(sizeof *d); + + dynstr_init(d->data); + d->done = done; + d->u = u; + xpipe(p); + pid = subprogram(ev, "disorder-stats", p[1]); + xclose(p[1]); + ev_child(ev, pid, 0, stats_finished, d); + ev_reader_new(ev, p[0], stats_read, stats_error, d, "disorder-stats reader"); +} + /* set a pref (remove if value=0) */ int trackdb_set(const char *track, const char *name, @@ -1758,7 +1842,7 @@ void trackdb_rescan(ev_source *ev) { error(0, "rescan already underway"); return; } - rescan_pid = subprogram(ev, RESCAN); + rescan_pid = subprogram(ev, RESCAN, -1); ev_child(ev, rescan_pid, 0, reap_rescan, 0); D(("started rescanner")); diff --git a/server/trackdb.h b/server/trackdb.h index cb9e639..fe43474 100644 --- a/server/trackdb.h +++ b/server/trackdb.h @@ -44,6 +44,11 @@ void trackdb_close(void); char **trackdb_stats(int *nstatsp); /* return a list of database stats */ +void trackdb_stats_subprocess(struct ev_source *ev, + void (*done)(char *data, void *u), + void *u); +/* collect stats in background and call done() with results */ + int trackdb_set(const char *track, const char *name, const char *value); -- [mdw]