X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?p=inn-innduct.git;a=blobdiff_plain;f=backends%2Finnduct.c;h=0ed8f2efce237b777d73dd645c6cdb84c7c2d099;hp=b829841613d0aa010d7ce43bdd0e0934479a5fe6;hb=38f146351e4aab76fbdfe3fae4c8b76f7c6b338f;hpb=df2fee3940b22c3b173e077bbd0c83007b23e794 diff --git a/backends/innduct.c b/backends/innduct.c index b829841..0ed8f2e 100644 --- a/backends/innduct.c +++ b/backends/innduct.c @@ -1,27 +1,34 @@ /* - * debugging rune: - * build-lfs/backends/innduct --connection-timeout=30 --no-daemon -C ../inn.conf -f `pwd`/fee sit localhost + * innduct + * tailing reliable realtime streaming feeder for inn + * + * Copyright (C) 2010 Ian Jackson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * (I believe that when you compile and link this as part of the inn2 + * build, with the Makefile runes I have provided, all the libraries + * and files which end up included in innduct are licence-compatible + * with GPLv3. If not then please let me know. -Ian Jackson.) */ -/*-- -flow control notes -to ensure articles go away eventually -separate queue for each input file - queue expiry - every period, check head of backlog queue for expiry with SMretrieve - if too old: discard, and check next article - also check every backlog article as we read it - flush expiry - after too long in SEPARATED/DROPPING ie Separated/Finishing/Dropping - one-off: eat queued articles from flushing and write them to defer - one-off: connfail all connections which have any articles from flushing - newly read articles from flushing go straight to defer - this should take care of it and get us out of this state -to avoid filling up ram needlessly - input control - limit number of queued articles for each ipf - pause/resume inputfile tailing ---*/ +/* + * todo + * + * don't mind reconnecting if we just disconnected due to idle + * some weird disconnection event still investigating + */ /* * Newsfeeds file entries should look like this: @@ -217,7 +224,7 @@ perl -ne 'print if m/-8\<-/..m/-\>8-/; print "\f" if m/-\^L-/' backends/innduct. #define INNDCOMMCHILD_ESTATUS_NONESUCH 27 #define MAX_LINE_FEEDFILE (NNTP_MSGID_MAXLEN + sizeof(TOKEN)*2 + 10) -#define MAX_CONTROL_COMMAND 1000 +#define MAX_CLI_COMMAND 1000 #define VA va_list al; va_start(al,fmt) #define PRINTF(f,a) __attribute__((__format__(printf,f,a))) @@ -293,7 +300,7 @@ typedef struct InputFile InputFile; typedef struct XmitDetails XmitDetails; typedef struct Filemon_Perfile Filemon_Perfile; typedef enum StateMachineState StateMachineState; -typedef struct ControlCommand ControlCommand; +typedef struct CliCommand CliCommand; DEFLIST(Conn); DEFLIST(Article); @@ -313,7 +320,8 @@ static void statemc_setstate(StateMachineState newsms, int periods, static void statemc_start_flush(const char *why); /* Normal => Flushing */ static void spawn_inndcomm_flush(const char *why); /* Moved => Flushing */ -static int trigger_flush_ok(void); /* => Flushing,FLUSHING, ret 1; or ret 0 */ +static int trigger_flush_ok(const char *why /* 0 means timeout */); + /* => Flushing,FLUSHING, ret 1; or ret 0 */ static void article_done(Article *art, int whichcount); @@ -342,7 +350,7 @@ static void inputfile_reading_resume(InputFile *ipf); static void filemon_start(InputFile *ipf); static void filemon_stop(InputFile *ipf); -static void filemon_callback(InputFile *ipf); +static void tailing_make_readable(InputFile *ipf); static void vconnfail(Conn *conn, const char *fmt, va_list al) PRINTF(2,0); static void connfail(Conn *conn, const char *fmt, ...) PRINTF(2,3); @@ -350,11 +358,12 @@ static void connfail(Conn *conn, const char *fmt, ...) PRINTF(2,3); static const oop_rd_style peer_rd_style; static oop_rd_call peer_rd_err, peer_rd_ok; + /*----- configuration options -----*/ /* when changing defaults, remember to update the manpage */ static const char *sitename, *remote_host; -static const char *feedfile, *realsockdir="/tmp/innduct.control"; +static const char *feedfile, *path_run, *path_cli, *path_cli_dir; static int quiet_multiple=0; static int become_daemon=1, try_filemon=1; static int try_stream=1; @@ -377,7 +386,7 @@ static double nocheck_decay= 100; /* conv'd from articles to lambda by main */ /* all these are initialised to seconds, and converted to periods in main */ static int reconnect_delay_periods=1000; static int flushfail_retry_periods=1000; -static int backlog_retry_minperiods=50; +static int backlog_retry_minperiods=100; static int backlog_spontrescan_periods=300; static int spontaneous_flush_periods=100000; static int max_separated_periods=2000; @@ -455,7 +464,7 @@ struct InputFile { oop_read *rd; /* non-0: reading; 0: constructing, or had EOF */ off_t offset; - int skippinglong, paused; + int skippinglong, paused, fake_readable; ArticleList queue; long inprogress; /* includes queue.count and also articles in conns */ @@ -516,8 +525,7 @@ struct Conn { /* main initialises */ static oop_source *loop; static ConnList conns; -static char *path_lock, *path_flushing, *path_defer; -static char *path_control, *path_dump; +static char *path_lock, *path_flushing, *path_defer, *path_dump; static char *globpat_backlog; static pid_t self_pid; @@ -554,7 +562,7 @@ static void logv(int sysloglevel, const char *pfx, int errnoval, const char *fmt, va_list al) PRINTF(5,0); static void logv(int sysloglevel, const char *pfx, int errnoval, const char *fmt, va_list al) { - char msgbuf[256]; /* NB do not call xvasprintf here or you'll recurse */ + char msgbuf[1024]; /* NB do not call xvasprintf here or you'll recurse */ vsnprintf(msgbuf,sizeof(msgbuf), fmt,al); msgbuf[sizeof(msgbuf)-1]= 0; @@ -782,13 +790,13 @@ static int isewouldblock(int errnoval) { return errnoval==EWOULDBLOCK || errnoval==EAGAIN; } -/*========== command and control connections ==========*/ +/*========== command and control (CLI) connections ==========*/ -static int control_master; +static int cli_master; -typedef struct ControlConn ControlConn; -struct ControlConn { - void (*destroy)(ControlConn*); +typedef struct CliConn CliConn; +struct CliConn { + void (*destroy)(CliConn*); int fd; oop_read *rd; FILE *out; @@ -799,53 +807,53 @@ struct ControlConn { socklen_t salen; }; -static const oop_rd_style control_rd_style= { +static const oop_rd_style cli_rd_style= { OOP_RD_DELIM_STRIP, '\n', OOP_RD_NUL_FORBID, OOP_RD_SHORTREC_FORBID }; -static void control_destroy(ControlConn *cc) { +static void cli_destroy(CliConn *cc) { cc->destroy(cc); } -static void control_checkouterr(ControlConn *cc /* may destroy*/) { +static void cli_checkouterr(CliConn *cc /* may destroy*/) { if (ferror(cc->out) | fflush(cc->out)) { info("CTRL%d write error %s", cc->fd, strerror(errno)); - control_destroy(cc); + cli_destroy(cc); } } -static void control_prompt(ControlConn *cc /* may destroy*/) { +static void cli_prompt(CliConn *cc /* may destroy*/) { fprintf(cc->out, "%s| ", sitename); - control_checkouterr(cc); + cli_checkouterr(cc); } -struct ControlCommand { +struct CliCommand { const char *cmd; - void (*f)(ControlConn *cc, const ControlCommand *ccmd, + void (*f)(CliConn *cc, const CliCommand *ccmd, const char *arg, size_t argsz); void *xdata; int xval; }; -static const ControlCommand control_commands[]; +static const CliCommand cli_commands[]; -#define CCMD(wh) \ - static void ccmd_##wh(ControlConn *cc, const ControlCommand *c, \ +#define CCMD(wh) \ + static void ccmd_##wh(CliConn *cc, const CliCommand *c, \ const char *arg, size_t argsz) CCMD(help) { fputs("commands:\n", cc->out); - const ControlCommand *ccmd; - for (ccmd=control_commands; ccmd->cmd; ccmd++) + const CliCommand *ccmd; + for (ccmd=cli_commands; ccmd->cmd; ccmd++) fprintf(cc->out, " %s\n", ccmd->cmd); fputs("NB: permissible arguments are not shown above." " Not all commands listed are safe. See innduct(8).\n", cc->out); } CCMD(flush) { - int ok= trigger_flush_ok(); + int ok= trigger_flush_ok("manual request"); if (!ok) fprintf(cc->out,"already flushing (state is %s)\n", sms_names[sms]); } @@ -864,7 +872,7 @@ CCMD(setintarg) { *(int*)c->xdata= atoi(arg); } CCMD(setint) { *(int*)c->xdata= c->xval; } CCMD(setint_period) { *(int*)c->xdata= c->xval; period(); } -static const ControlCommand control_commands[]= { +static const CliCommand cli_commands[]= { { "h", ccmd_help }, { "flush", ccmd_flush }, { "stop", ccmd_stop }, @@ -885,10 +893,10 @@ POKES("prod ", ccmd_setint_period) { 0 } }; -static void *control_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, - const char *errmsg, int errnoval, - const char *data, size_t recsz, void *cc_v) { - ControlConn *cc= cc_v; +static void *cli_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, + const char *errmsg, int errnoval, + const char *data, size_t recsz, void *cc_v) { + CliConn *cc= cc_v; if (!data) { info("CTRL%d closed", cc->fd); @@ -898,8 +906,8 @@ static void *control_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, if (recsz == 0) goto prompt; - const ControlCommand *ccmd; - for (ccmd=control_commands; ccmd->cmd; ccmd++) { + const CliCommand *ccmd; + for (ccmd=cli_commands; ccmd->cmd; ccmd++) { int l= strlen(ccmd->cmd); if (recsz < l) continue; if (recsz > l && data[l] != ' ') continue; @@ -913,36 +921,36 @@ static void *control_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, fputs("unknown command; h for help\n", cc->out); prompt: - control_prompt(cc); + cli_prompt(cc); return OOP_CONTINUE; } -static void *control_rd_err(oop_source *lp, oop_read *oread, oop_rd_event ev, - const char *errmsg, int errnoval, - const char *data, size_t recsz, void *cc_v) { - ControlConn *cc= cc_v; +static void *cli_rd_err(oop_source *lp, oop_read *oread, oop_rd_event ev, + const char *errmsg, int errnoval, + const char *data, size_t recsz, void *cc_v) { + CliConn *cc= cc_v; info("CTRL%d read error %s", cc->fd, errmsg); cc->destroy(cc); return OOP_CONTINUE; } -static int control_conn_startup(ControlConn *cc /* may destroy*/, +static int cli_conn_startup(CliConn *cc /* may destroy*/, const char *how) { cc->rd= oop_rd_new_fd(loop, cc->fd, 0,0); - if (!cc->rd) { warn("oop_rd_new_fd control failed"); return -1; } + if (!cc->rd) { warn("oop_rd_new_fd cli failed"); return -1; } - int er= oop_rd_read(cc->rd, &control_rd_style, MAX_CONTROL_COMMAND, - control_rd_ok, cc, - control_rd_err, cc); - if (er) { errno= er; syswarn("oop_rd_read control failed"); return -1; } + int er= oop_rd_read(cc->rd, &cli_rd_style, MAX_CLI_COMMAND, + cli_rd_ok, cc, + cli_rd_err, cc); + if (er) { errno= er; syswarn("oop_rd_read cli failed"); return -1; } info("CTRL%d %s ready", cc->fd, how); - control_prompt(cc); + cli_prompt(cc); return 0; } -static void control_stdio_destroy(ControlConn *cc) { +static void cli_stdio_destroy(CliConn *cc) { if (cc->rd) { oop_rd_cancel(cc->rd); errno= oop_rd_delete_tidy(cc->rd); @@ -951,17 +959,17 @@ static void control_stdio_destroy(ControlConn *cc) { free(cc); } -static void control_stdio(void) { - NEW_DECL(ControlConn *,cc); - cc->destroy= control_stdio_destroy; +static void cli_stdio(void) { + NEW_DECL(CliConn *,cc); + cc->destroy= cli_stdio_destroy; cc->fd= 0; cc->out= stdout; - int r= control_conn_startup(cc,"stdio"); + int r= cli_conn_startup(cc,"stdio"); if (r) cc->destroy(cc); } -static void control_accepted_destroy(ControlConn *cc) { +static void cli_accepted_destroy(CliConn *cc) { if (cc->rd) { oop_rd_cancel(cc->rd); oop_rd_delete_kill(cc->rd); @@ -971,19 +979,19 @@ static void control_accepted_destroy(ControlConn *cc) { free(cc); } -static void *control_master_readable(oop_source *lp, int master, - oop_event ev, void *u) { - NEW_DECL(ControlConn *,cc); - cc->destroy= control_accepted_destroy; +static void *cli_master_readable(oop_source *lp, int master, + oop_event ev, void *u) { + NEW_DECL(CliConn *,cc); + cc->destroy= cli_accepted_destroy; cc->salen= sizeof(cc->sa); cc->fd= accept(master, &cc->sa.sa, &cc->salen); - if (cc->fd<0) { syswarn("error accepting control connection"); goto x; } + if (cc->fd<0) { syswarn("error accepting cli connection"); goto x; } cc->out= fdopen(cc->fd, "w"); - if (!cc->out) { syswarn("error fdopening accepted control conn"); goto x; } + if (!cc->out) { syswarn("error fdopening accepted cli connection"); goto x; } - int r= control_conn_startup(cc, "accepted"); + int r= cli_conn_startup(cc, "accepted"); if (r) goto x; return OOP_CONTINUE; @@ -993,14 +1001,12 @@ static void *control_master_readable(oop_source *lp, int master, return OOP_CONTINUE; } -#define NOCONTROL(...) do{ \ - syswarn("no control socket, because failed to " __VA_ARGS__); \ - goto nocontrol; \ +#define NOCLI(...) do{ \ + syswarn("no cli listener, because failed to " __VA_ARGS__); \ + goto nocli; \ }while(0) -static void control_init(void) { - char *real=0; - +static void cli_init(void) { union { struct sockaddr sa; struct sockaddr_un un; @@ -1009,80 +1015,50 @@ static void control_init(void) { memset(&sa,0,sizeof(sa)); int maxlen= sizeof(sa.un.sun_path); - int reallen= readlink(path_control, sa.un.sun_path, maxlen); - if (reallen<0) { - if (errno != ENOENT) - NOCONTROL("readlink control socket symlink path %s", path_control); - } - if (reallen >= maxlen) { - debug("control socket symlink path too long (r=%d)",reallen); - xunlink(path_control, "old (overlong) control socket symlink"); - reallen= -1; + if (!path_cli) { + info("control command line disabled"); + return; } - - if (reallen<0) { - struct stat stab; - int r= lstat(realsockdir,&stab); - if (r) { - if (errno != ENOENT) NOCONTROL("lstat real socket dir %s", realsockdir); - r= mkdir(realsockdir, 0700); - if (r) NOCONTROL("mkdir real socket dir %s", realsockdir); - - } else { - uid_t self= geteuid(); - if (!S_ISDIR(stab.st_mode) || - stab.st_uid != self || - stab.st_mode & 0007) { - warn("no control socket, because real socket directory" - " is somehow wrong (ISDIR=%d, uid=%lu (exp.%lu), mode %lo)", - !!S_ISDIR(stab.st_mode), - (unsigned long)stab.st_uid, (unsigned long)self, - (unsigned long)stab.st_mode & 0777UL); - goto nocontrol; - } - } - - real= xasprintf("%s/s%lx.%lx", realsockdir, - (unsigned long)xtime(), (unsigned long)self_pid); - int reallen= strlen(real); + int pathlen= strlen(path_cli); + if (pathlen > maxlen) { + warn("no cli listener, because cli socket path %s too long (%d>%d)", + path_cli, pathlen, maxlen); + return; + } - if (reallen >= maxlen) { - warn("no control socket, because tmpnam gave overly-long path" - " %s", real); - goto nocontrol; - } - r= symlink(real, path_control); - if (r) NOCONTROL("make control socket path %s a symlink to real" - " socket path %s", path_control, real); - memcpy(sa.un.sun_path, real, reallen); + if (path_cli_dir) { + int r= mkdir(path_cli_dir, 0700); + if (r && errno!=EEXIST) + NOCLI("create cli socket directory %s", path_cli_dir); } - int r= unlink(sa.un.sun_path); + int r= unlink(path_cli); if (r && errno!=ENOENT) - NOCONTROL("remove old real socket %s", sa.un.sun_path); + NOCLI("remove old cli socket %s", path_cli); - control_master= socket(PF_UNIX, SOCK_STREAM, 0); - if (control_master<0) NOCONTROL("create new control socket"); + cli_master= socket(PF_UNIX, SOCK_STREAM, 0); + if (cli_master<0) NOCLI("create new cli master socket"); + int sl= pathlen + offsetof(struct sockaddr_un, sun_path); sa.un.sun_family= AF_UNIX; - int sl= strlen(sa.un.sun_path) + offsetof(struct sockaddr_un, sun_path); - r= bind(control_master, &sa.sa, sl); - if (r) NOCONTROL("bind to real socket path %s", sa.un.sun_path); + memcpy(sa.un.sun_path, path_cli, pathlen); + + r= bind(cli_master, &sa.sa, sl); + if (r) NOCLI("bind to cli socket path %s", sa.un.sun_path); - r= listen(control_master, 5); - if (r) NOCONTROL("listen"); + r= listen(cli_master, 5); + if (r) NOCLI("listen to cli master socket"); - xsetnonblock(control_master, 1); + xsetnonblock(cli_master, 1); - loop->on_fd(loop, control_master, OOP_READ, control_master_readable, 0); - info("control socket ok, real path %s", sa.un.sun_path); + loop->on_fd(loop, cli_master, OOP_READ, cli_master_readable, 0); + info("cli ready, listening on %s", path_cli); return; - nocontrol: - free(real); - xclose_perhaps(&control_master, "control master",0); + nocli: + xclose_perhaps(&cli_master, "cli master",0); return; } @@ -1148,7 +1124,7 @@ static void vconnfail(Conn *conn, const char *fmt, va_list al) { xmit_free(d); char *m= xvasprintf(fmt,al); - warn("C%d connection failed (requeueing " RCI_TRIPLE_FMT_BASE "): %s", + warn("C%d connection failed requeueing " RCI_TRIPLE_FMT_BASE ": %s", conn->fd, RCI_TRIPLE_VALS_BASE(requeue, /*nothing*/), m); free(m); @@ -1539,6 +1515,26 @@ static void conn_maybe_write(Conn *conn) { /*---------- expiry, flow control and deferral ----------*/ +/* + * flow control notes + * to ensure articles go away eventually + * separate queue for each input file + * queue expiry + * every period, check head of backlog queue for expiry with SMretrieve + * if too old: discard, and check next article + * also check every backlog article as we read it + * flush expiry + * after too long in SEPARATED/DROPPING ie Separated/Finishing/Dropping + * one-off: eat queued articles from flushing and write them to defer + * one-off: connfail all connections which have any articles from flushing + * newly read articles from flushing go straight to defer + * this should take care of it and get us out of this state + * to avoid filling up ram needlessly + * input control + * limit number of queued articles for each ipf + * pause/resume inputfile tailing + */ + static void check_reading_pause_resume(InputFile *ipf) { if (ipf->queue.count >= max_queue_per_ipf) inputfile_reading_pause(ipf); @@ -2136,7 +2132,16 @@ static void *feedfile_got_article(oop_source *lp, oop_read *rd, static void *tailing_rable_call_time(oop_source *loop, struct timeval tv, void *user) { + /* lifetime of ipf here is OK because destruction will cause + * on_cancel which will cancel this callback */ InputFile *ipf= user; + + if (!ipf->fake_readable) return OOP_CONTINUE; + + /* we just keep calling readable until our caller (oop_rd) + * has called try_read, and try_read has found EOF so given EAGAIN */ + loop->on_time(loop, OOP_TIME_NOW, tailing_rable_call_time, ipf); + return ipf->readable_callback(loop, &ipf->readable, ipf->readable_callback_user); } @@ -2149,9 +2154,10 @@ static void tailing_on_cancel(struct oop_readable *rable) { ipf->readable_callback= 0; } -static void tailing_queue_readable(InputFile *ipf) { - /* lifetime of ipf here is OK because destruction will cause - * on_cancel which will cancel this callback */ +static void tailing_make_readable(InputFile *ipf) { + if (!ipf || !ipf->readable_callback) /* so callers can be naive */ + return; + ipf->fake_readable= 1; loop->on_time(loop, OOP_TIME_NOW, tailing_rable_call_time, ipf); } @@ -2163,8 +2169,7 @@ static int tailing_on_readable(struct oop_readable *rable, ipf->readable_callback= cb; ipf->readable_callback_user= user; filemon_start(ipf); - - tailing_queue_readable(ipf); + tailing_make_readable(ipf); return 0; } @@ -2175,11 +2180,13 @@ static ssize_t tailing_try_read(struct oop_readable *rable, void *buffer, ssize_t r= read(ipf->fd, buffer, length); if (r==-1) { if (errno==EINTR) continue; + ipf->fake_readable= 0; return r; } if (!r) { if (ipf==main_input_file) { errno=EAGAIN; + ipf->fake_readable= 0; return -1; } else if (ipf==flushing_input_file) { assert(ipf->rd); @@ -2190,7 +2197,6 @@ static ssize_t tailing_try_read(struct oop_readable *rable, void *buffer, abort(); } } - tailing_queue_readable(ipf); return r; } } @@ -2254,8 +2260,8 @@ static void *filemon_inotify_readable(oop_source *lp, int fd, die("inotify read %d bytes wanted struct of %d", r, (int)sizeof(iev)); } InputFile *ipf= filemon_inotify_wd2ipf[iev.wd]; - debug("filemon inotify readable read %p wd=%d", ipf, iev.wd); - filemon_callback(ipf); + /*debug("filemon inotify readable read %p wd=%d", ipf, iev.wd);*/ + tailing_make_readable(ipf); } return OOP_CONTINUE; } @@ -2316,11 +2322,6 @@ static void filemon_stop(InputFile *ipf) { ipf->filemon= 0; } -static void filemon_callback(InputFile *ipf) { - if (ipf && ipf->readable_callback) /* so filepoll() can be naive */ - ipf->readable_callback(loop, &ipf->readable, ipf->readable_callback_user); -} - /*---------- interface to start and stop an input file ----------*/ static const oop_rd_style feedfile_rdstyle= { @@ -2610,20 +2611,21 @@ static void statemc_start_flush(const char *why) { /* Normal => Flushing */ spawn_inndcomm_flush(why); /* => Flushing FLUSHING */ } -static int trigger_flush_ok(void) { /* => Flushing,FLUSHING, ret 1; or ret 0 */ +static int trigger_flush_ok(const char *why) { switch (sms) { case sm_NORMAL: - statemc_start_flush("periodic"); /* Normal => Flushing; => FLUSHING */ - return 1; + statemc_start_flush(why ? why : "periodic"); + return 1; /* Normal => Flushing; => FLUSHING */ case sm_FLUSHFAILED: - spawn_inndcomm_flush("retry"); /* Moved => Flushing; => FLUSHING */ - return 1; + spawn_inndcomm_flush(why ? why : "retry"); + return 1; /* Moved => Flushing; => FLUSHING */ case sm_SEPARATED: case sm_DROPPING: - warn("took too long to complete old feedfile after flush, autodeferring"); + warn("abandoning old feedfile after flush (%s), autodeferring", + why ? why : "took too long to complete"); assert(flushing_input_file); autodefer_input_file(flushing_input_file); return 1; @@ -2639,7 +2641,7 @@ static void statemc_period_poll(void) { assert(until_flush>=0); if (until_flush) return; - int ok= trigger_flush_ok(); + int ok= trigger_flush_ok(0); assert(ok); } @@ -2922,11 +2924,11 @@ static void search_backlog_file(void) { preterminate(); notice("feed dropped and our work is complete"); - int r= unlink(path_control); + int r= unlink(path_cli); if (r && errno!=ENOENT) - syswarn("failed to remove control symlink for old feed"); + syswarn("failed to unlink cli socket for old feed"); - xunlink(path_lock, "lockfile for old feed"); + xunlink(path_lock, "lockfile for old feed"); exit(4); } until_backlog_nextscan= backlog_spontrescan_periods; @@ -3058,7 +3060,7 @@ static void *inndcomm_event(oop_source *lp, int fd, oop_event e, void *u) { case INNDCOMMCHILD_ESTATUS_NONESUCH: notice("feed has been dropped by innd, finishing up"); flushing_input_file= main_input_file; - tailing_queue_readable(flushing_input_file); + tailing_make_readable(flushing_input_file); /* we probably previously returned EAGAIN from our fake read method * when in fact we were at EOF, so signal another readable event * so we actually see the EOF */ @@ -3078,11 +3080,13 @@ static void *inndcomm_event(oop_source *lp, int fd, oop_event e, void *u) { case 0: /* as above */ flushing_input_file= main_input_file; - tailing_queue_readable(flushing_input_file); + tailing_make_readable(flushing_input_file); main_input_file= open_input_file(feedfile); if (!main_input_file) - die("flush succeeded but feedfile %s does not exist!", feedfile); + die("flush succeeded but feedfile %s does not exist!" + " (this probably means feedfile does not correspond" + " to site %s in newsfeeds)", feedfile, sitename); if (flushing_input_file) { SMS(SEPARATED, max_separated_periods, "recovery flush complete"); @@ -3227,8 +3231,8 @@ static void every(int interval, int fixed_rate, void (*f)(void)) { } static void filepoll(void) { - filemon_callback(main_input_file); - filemon_callback(flushing_input_file); + tailing_make_readable(main_input_file); + tailing_make_readable(flushing_input_file); } static char *debug_report_ipf(InputFile *ipf) { @@ -3278,7 +3282,7 @@ static void period(void) { /*========== dumping state ==========*/ -static void dump_article_list(FILE *f, const ControlCommand *c, +static void dump_article_list(FILE *f, const CliCommand *c, const ArticleList *al) { fprintf(f, " count=%d\n", al->count); if (!c->xval) return; @@ -3295,7 +3299,7 @@ static void dump_article_list(FILE *f, const ControlCommand *c, } } -static void dump_input_file(FILE *f, const ControlCommand *c, +static void dump_input_file(FILE *f, const CliCommand *c, InputFile *ipf, const char *wh) { char *dipf= debug_report_ipf(ipf); fprintf(f,"input %s %s", wh, dipf); @@ -3305,6 +3309,7 @@ static void dump_input_file(FILE *f, const ControlCommand *c, DUMPV("%d", ipf->,readcount_ok); DUMPV("%d", ipf->,readcount_blank); DUMPV("%d", ipf->,readcount_err); + DUMPV("%d", ipf->,count_nooffer_missing); } fprintf(f,"\n"); if (ipf) { @@ -3345,7 +3350,7 @@ CCMD(dump) { fprintf(f,"special"); DUMPV("%ld", (long),connecting_child); DUMPV("%d", , connecting_fdpass_sock); - DUMPV("%d", , control_master); + DUMPV("%d", , cli_master); fprintf(f,"\n"); fprintf(f,"filemon "); @@ -3388,10 +3393,11 @@ CCMD(dump) { } fprintf(f,"paths"); + DUMPV("%s", , feedfile); + DUMPV("%s", , path_cli); DUMPV("%s", , path_lock); DUMPV("%s", , path_flushing); DUMPV("%s", , path_defer); - DUMPV("%s", , path_control); DUMPV("%s", , path_dump); DUMPV("%s", , globpat_backlog); fprintf(f,"\n"); @@ -3575,7 +3581,8 @@ static const Option innduct_options[]= { {0,"no-filemon", 0, &try_filemon, op_setint, 0 }, {'C',"inndconf", "F", &inndconffile, op_string }, {'P',"port", "PORT", &port, op_integer }, -{0,"ctrl-sock-dir", 0, &realsockdir, op_string }, +{0,"chdir", "DIR", &path_run, op_string }, +{0,"cli", "DIR/|PATH", &path_cli, op_string }, {0,"help", 0, 0, help }, {0,"max-connections", "N", &max_connections, op_integer }, @@ -3625,6 +3632,12 @@ static void convert_to_periods_rndup(int *store) { *store /= period_seconds; } +static int path_ends_slash(const char *specified) { + int l= strlen(specified); + assert(l); + return specified[l-1] == '/'; +} + int main(int argc, char **argv) { if (!argv[1]) { printusage(stderr); @@ -3637,7 +3650,10 @@ int main(int argc, char **argv) { sitename= *argv++; if (!sitename) badusage("need site name argument"); - remote_host= *argv++; + + if (*argv) remote_host= *argv++; + else remote_host= sitename; + if (*argv) badusage("too many non-option arguments"); /* defaults */ @@ -3667,13 +3683,25 @@ int main(int argc, char **argv) { badusage("bad input data ratio must be between 0..100"); max_bad_data_ratio *= 0.01; - if (!feedfile) { - feedfile= xasprintf("%s/%s",innconf->pathoutgoing,sitename); - } else if (!feedfile[0]) { - badusage("feed filename must be nonempty"); - } else if (feedfile[strlen(feedfile)-1]=='/') { - feedfile= xasprintf("%s%s",feedfile,sitename); + if (!path_run) + path_run= innconf->pathrun; + + if (!feedfile) feedfile= sitename; + if (!feedfile[0]) badusage("feed filename, if specified, must be nonempty"); + if (path_ends_slash(feedfile)) + feedfile= xasprintf("%s%s", feedfile, sitename); + if (feedfile[0] != '/') + feedfile= xasprintf("%s/%s", innconf->pathoutgoing, feedfile); + + if (!path_cli) { + path_cli_dir= "innduct"; + } else if (!path_cli[0] || !strcmp(path_cli,"none")) { + path_cli= 0; /* ok, don't then */ + } else if (path_ends_slash(path_cli)) { + path_cli_dir= xasprintf("%.*s", strlen(path_cli)-1, path_cli); } + if (path_cli_dir) + path_cli= xasprintf("%s/%s", path_cli_dir, sitename); if (max_queue_per_ipf<0) max_queue_per_ipf= max_queue_per_conn * 2; @@ -3689,7 +3717,6 @@ int main(int argc, char **argv) { path_lock= xasprintf("%s_lock", feedfile); path_flushing= xasprintf("%s_flushing", feedfile); path_defer= xasprintf("%s_defer", feedfile); - path_control= xasprintf("%s_control", feedfile); path_dump= xasprintf("%s_dump", feedfile); globpat_backlog= xasprintf("%s_backlog*", feedfile); @@ -3717,7 +3744,7 @@ int main(int argc, char **argv) { if (child1) _exit(0); pid_t sid= setsid(); - if (sid != child1) sysfatal("setsid failed"); + if (sid == -1) sysfatal("setsid failed"); pid_t child2= xfork("daemonise second fork"); if (child2) _exit(0); @@ -3726,16 +3753,23 @@ int main(int argc, char **argv) { self_pid= getpid(); if (self_pid==-1) sysdie("getpid"); + r= chdir(path_run); + if (r) sysdie("could not chdir to pathrun %s", path_run); + statemc_lock(); init_signals(); notice("starting"); + int val= 1; + r= SMsetup(SM_PREOPEN, &val); if (!r) warn("SMsetup SM_PREOPEN failed"); + r= SMinit(); if (!r) die("storage manager initialisation (SMinit) failed"); + if (!become_daemon) - control_stdio(); + cli_stdio(); - control_init(); + cli_init(); int filemon_ok= 0; if (!try_filemon) {