X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?p=inn-innduct.git;a=blobdiff_plain;f=backends%2Finnduct.c;h=6e9dd262dbfd5bd4c573dc5b3c858df6fb949a91;hp=005df921d90ec2ecf0fabd4fbb51ee6f86fd70a0;hb=c07a9d0aee10d37d55c8a6f60cf261da9e2d44cd;hpb=55f4f72edbae3d5d9b96ca0fabdd745300373a51 diff --git a/backends/innduct.c b/backends/innduct.c index 005df92..6e9dd26 100644 --- a/backends/innduct.c +++ b/backends/innduct.c @@ -23,10 +23,11 @@ * with GPLv3. If not then please let me know. -Ian Jackson.) */ - /* - * debugging rune: - * build-lfs/backends/innduct --connection-timeout=30 --no-daemon -C ../inn.conf -f `pwd`/fee sit localhost + * todo + * + * don't mind reconnecting if we just disconnected due to idle + * some weird disconnection event still investigating */ /* @@ -223,7 +224,7 @@ perl -ne 'print if m/-8\<-/..m/-\>8-/; print "\f" if m/-\^L-/' backends/innduct. #define INNDCOMMCHILD_ESTATUS_NONESUCH 27 #define MAX_LINE_FEEDFILE (NNTP_MSGID_MAXLEN + sizeof(TOKEN)*2 + 10) -#define MAX_CONTROL_COMMAND 1000 +#define MAX_CLI_COMMAND 1000 #define VA va_list al; va_start(al,fmt) #define PRINTF(f,a) __attribute__((__format__(printf,f,a))) @@ -299,7 +300,7 @@ typedef struct InputFile InputFile; typedef struct XmitDetails XmitDetails; typedef struct Filemon_Perfile Filemon_Perfile; typedef enum StateMachineState StateMachineState; -typedef struct ControlCommand ControlCommand; +typedef struct CliCommand CliCommand; DEFLIST(Conn); DEFLIST(Article); @@ -319,7 +320,8 @@ static void statemc_setstate(StateMachineState newsms, int periods, static void statemc_start_flush(const char *why); /* Normal => Flushing */ static void spawn_inndcomm_flush(const char *why); /* Moved => Flushing */ -static int trigger_flush_ok(void); /* => Flushing,FLUSHING, ret 1; or ret 0 */ +static int trigger_flush_ok(const char *why /* 0 means timeout */); + /* => Flushing,FLUSHING, ret 1; or ret 0 */ static void article_done(Article *art, int whichcount); @@ -348,7 +350,7 @@ static void inputfile_reading_resume(InputFile *ipf); static void filemon_start(InputFile *ipf); static void filemon_stop(InputFile *ipf); -static void filemon_callback(InputFile *ipf); +static void tailing_make_readable(InputFile *ipf); static void vconnfail(Conn *conn, const char *fmt, va_list al) PRINTF(2,0); static void connfail(Conn *conn, const char *fmt, ...) PRINTF(2,3); @@ -356,11 +358,12 @@ static void connfail(Conn *conn, const char *fmt, ...) PRINTF(2,3); static const oop_rd_style peer_rd_style; static oop_rd_call peer_rd_err, peer_rd_ok; + /*----- configuration options -----*/ /* when changing defaults, remember to update the manpage */ static const char *sitename, *remote_host; -static const char *feedfile, *realsockdir="/tmp/innduct.control"; +static const char *feedfile, *path_run, *path_cli, *path_cli_dir; static int quiet_multiple=0; static int become_daemon=1, try_filemon=1; static int try_stream=1; @@ -383,11 +386,13 @@ static double nocheck_decay= 100; /* conv'd from articles to lambda by main */ /* all these are initialised to seconds, and converted to periods in main */ static int reconnect_delay_periods=1000; static int flushfail_retry_periods=1000; -static int backlog_retry_minperiods=50; +static int backlog_retry_minperiods=100; static int backlog_spontrescan_periods=300; static int spontaneous_flush_periods=100000; static int max_separated_periods=2000; static int need_activity_periods=1000; +static int lowvol_thresh=3; +static int lowvol_periods=1000; static double max_bad_data_ratio= 1; /* conv'd from percentage by main */ static int max_bad_data_initial= 30; @@ -461,7 +466,7 @@ struct InputFile { oop_read *rd; /* non-0: reading; 0: constructing, or had EOF */ off_t offset; - int skippinglong, paused; + int skippinglong, paused, fake_readable; ArticleList queue; long inprogress; /* includes queue.count and also articles in conns */ @@ -506,7 +511,8 @@ struct Conn { ISNODE(Conn); int fd; /* may be 0, meaning closed (during construction/destruction) */ oop_read *rd; /* likewise */ - int max_queue, stream, quitting; + int max_queue, stream; + const char *quitting; int since_activity; /* periods */ ArticleList waiting; /* not yet told peer */ ArticleList priority; /* peer says send it now */ @@ -522,10 +528,12 @@ struct Conn { /* main initialises */ static oop_source *loop; static ConnList conns; -static char *path_lock, *path_flushing, *path_defer; -static char *path_control, *path_dump; +static char *path_lock, *path_flushing, *path_defer, *path_dump; static char *globpat_backlog; static pid_t self_pid; +static int *lowvol_perperiod; +static int lowvol_circptr; +static int lowvol_total; /* does not include current period */ /* statemc_init initialises */ static StateMachineState sms; @@ -560,7 +568,7 @@ static void logv(int sysloglevel, const char *pfx, int errnoval, const char *fmt, va_list al) PRINTF(5,0); static void logv(int sysloglevel, const char *pfx, int errnoval, const char *fmt, va_list al) { - char msgbuf[256]; /* NB do not call xvasprintf here or you'll recurse */ + char msgbuf[1024]; /* NB do not call xvasprintf here or you'll recurse */ vsnprintf(msgbuf,sizeof(msgbuf), fmt,al); msgbuf[sizeof(msgbuf)-1]= 0; @@ -788,13 +796,13 @@ static int isewouldblock(int errnoval) { return errnoval==EWOULDBLOCK || errnoval==EAGAIN; } -/*========== command and control connections ==========*/ +/*========== command and control (CLI) connections ==========*/ -static int control_master; +static int cli_master; -typedef struct ControlConn ControlConn; -struct ControlConn { - void (*destroy)(ControlConn*); +typedef struct CliConn CliConn; +struct CliConn { + void (*destroy)(CliConn*); int fd; oop_read *rd; FILE *out; @@ -805,53 +813,53 @@ struct ControlConn { socklen_t salen; }; -static const oop_rd_style control_rd_style= { +static const oop_rd_style cli_rd_style= { OOP_RD_DELIM_STRIP, '\n', OOP_RD_NUL_FORBID, OOP_RD_SHORTREC_FORBID }; -static void control_destroy(ControlConn *cc) { +static void cli_destroy(CliConn *cc) { cc->destroy(cc); } -static void control_checkouterr(ControlConn *cc /* may destroy*/) { +static void cli_checkouterr(CliConn *cc /* may destroy*/) { if (ferror(cc->out) | fflush(cc->out)) { info("CTRL%d write error %s", cc->fd, strerror(errno)); - control_destroy(cc); + cli_destroy(cc); } } -static void control_prompt(ControlConn *cc /* may destroy*/) { +static void cli_prompt(CliConn *cc /* may destroy*/) { fprintf(cc->out, "%s| ", sitename); - control_checkouterr(cc); + cli_checkouterr(cc); } -struct ControlCommand { +struct CliCommand { const char *cmd; - void (*f)(ControlConn *cc, const ControlCommand *ccmd, + void (*f)(CliConn *cc, const CliCommand *ccmd, const char *arg, size_t argsz); void *xdata; int xval; }; -static const ControlCommand control_commands[]; +static const CliCommand cli_commands[]; -#define CCMD(wh) \ - static void ccmd_##wh(ControlConn *cc, const ControlCommand *c, \ +#define CCMD(wh) \ + static void ccmd_##wh(CliConn *cc, const CliCommand *c, \ const char *arg, size_t argsz) CCMD(help) { fputs("commands:\n", cc->out); - const ControlCommand *ccmd; - for (ccmd=control_commands; ccmd->cmd; ccmd++) + const CliCommand *ccmd; + for (ccmd=cli_commands; ccmd->cmd; ccmd++) fprintf(cc->out, " %s\n", ccmd->cmd); fputs("NB: permissible arguments are not shown above." " Not all commands listed are safe. See innduct(8).\n", cc->out); } CCMD(flush) { - int ok= trigger_flush_ok(); + int ok= trigger_flush_ok("manual request"); if (!ok) fprintf(cc->out,"already flushing (state is %s)\n", sms_names[sms]); } @@ -870,7 +878,7 @@ CCMD(setintarg) { *(int*)c->xdata= atoi(arg); } CCMD(setint) { *(int*)c->xdata= c->xval; } CCMD(setint_period) { *(int*)c->xdata= c->xval; period(); } -static const ControlCommand control_commands[]= { +static const CliCommand cli_commands[]= { { "h", ccmd_help }, { "flush", ccmd_flush }, { "stop", ccmd_stop }, @@ -891,10 +899,10 @@ POKES("prod ", ccmd_setint_period) { 0 } }; -static void *control_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, - const char *errmsg, int errnoval, - const char *data, size_t recsz, void *cc_v) { - ControlConn *cc= cc_v; +static void *cli_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, + const char *errmsg, int errnoval, + const char *data, size_t recsz, void *cc_v) { + CliConn *cc= cc_v; if (!data) { info("CTRL%d closed", cc->fd); @@ -904,8 +912,8 @@ static void *control_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, if (recsz == 0) goto prompt; - const ControlCommand *ccmd; - for (ccmd=control_commands; ccmd->cmd; ccmd++) { + const CliCommand *ccmd; + for (ccmd=cli_commands; ccmd->cmd; ccmd++) { int l= strlen(ccmd->cmd); if (recsz < l) continue; if (recsz > l && data[l] != ' ') continue; @@ -919,36 +927,36 @@ static void *control_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, fputs("unknown command; h for help\n", cc->out); prompt: - control_prompt(cc); + cli_prompt(cc); return OOP_CONTINUE; } -static void *control_rd_err(oop_source *lp, oop_read *oread, oop_rd_event ev, - const char *errmsg, int errnoval, - const char *data, size_t recsz, void *cc_v) { - ControlConn *cc= cc_v; +static void *cli_rd_err(oop_source *lp, oop_read *oread, oop_rd_event ev, + const char *errmsg, int errnoval, + const char *data, size_t recsz, void *cc_v) { + CliConn *cc= cc_v; info("CTRL%d read error %s", cc->fd, errmsg); cc->destroy(cc); return OOP_CONTINUE; } -static int control_conn_startup(ControlConn *cc /* may destroy*/, +static int cli_conn_startup(CliConn *cc /* may destroy*/, const char *how) { cc->rd= oop_rd_new_fd(loop, cc->fd, 0,0); - if (!cc->rd) { warn("oop_rd_new_fd control failed"); return -1; } + if (!cc->rd) { warn("oop_rd_new_fd cli failed"); return -1; } - int er= oop_rd_read(cc->rd, &control_rd_style, MAX_CONTROL_COMMAND, - control_rd_ok, cc, - control_rd_err, cc); - if (er) { errno= er; syswarn("oop_rd_read control failed"); return -1; } + int er= oop_rd_read(cc->rd, &cli_rd_style, MAX_CLI_COMMAND, + cli_rd_ok, cc, + cli_rd_err, cc); + if (er) { errno= er; syswarn("oop_rd_read cli failed"); return -1; } info("CTRL%d %s ready", cc->fd, how); - control_prompt(cc); + cli_prompt(cc); return 0; } -static void control_stdio_destroy(ControlConn *cc) { +static void cli_stdio_destroy(CliConn *cc) { if (cc->rd) { oop_rd_cancel(cc->rd); errno= oop_rd_delete_tidy(cc->rd); @@ -957,17 +965,17 @@ static void control_stdio_destroy(ControlConn *cc) { free(cc); } -static void control_stdio(void) { - NEW_DECL(ControlConn *,cc); - cc->destroy= control_stdio_destroy; +static void cli_stdio(void) { + NEW_DECL(CliConn *,cc); + cc->destroy= cli_stdio_destroy; cc->fd= 0; cc->out= stdout; - int r= control_conn_startup(cc,"stdio"); + int r= cli_conn_startup(cc,"stdio"); if (r) cc->destroy(cc); } -static void control_accepted_destroy(ControlConn *cc) { +static void cli_accepted_destroy(CliConn *cc) { if (cc->rd) { oop_rd_cancel(cc->rd); oop_rd_delete_kill(cc->rd); @@ -977,19 +985,19 @@ static void control_accepted_destroy(ControlConn *cc) { free(cc); } -static void *control_master_readable(oop_source *lp, int master, - oop_event ev, void *u) { - NEW_DECL(ControlConn *,cc); - cc->destroy= control_accepted_destroy; +static void *cli_master_readable(oop_source *lp, int master, + oop_event ev, void *u) { + NEW_DECL(CliConn *,cc); + cc->destroy= cli_accepted_destroy; cc->salen= sizeof(cc->sa); cc->fd= accept(master, &cc->sa.sa, &cc->salen); - if (cc->fd<0) { syswarn("error accepting control connection"); goto x; } + if (cc->fd<0) { syswarn("error accepting cli connection"); goto x; } cc->out= fdopen(cc->fd, "w"); - if (!cc->out) { syswarn("error fdopening accepted control conn"); goto x; } + if (!cc->out) { syswarn("error fdopening accepted cli connection"); goto x; } - int r= control_conn_startup(cc, "accepted"); + int r= cli_conn_startup(cc, "accepted"); if (r) goto x; return OOP_CONTINUE; @@ -999,14 +1007,12 @@ static void *control_master_readable(oop_source *lp, int master, return OOP_CONTINUE; } -#define NOCONTROL(...) do{ \ - syswarn("no control socket, because failed to " __VA_ARGS__); \ - goto nocontrol; \ +#define NOCLI(...) do{ \ + syswarn("no cli listener, because failed to " __VA_ARGS__); \ + goto nocli; \ }while(0) -static void control_init(void) { - char *real=0; - +static void cli_init(void) { union { struct sockaddr sa; struct sockaddr_un un; @@ -1015,91 +1021,73 @@ static void control_init(void) { memset(&sa,0,sizeof(sa)); int maxlen= sizeof(sa.un.sun_path); - int reallen= readlink(path_control, sa.un.sun_path, maxlen); - if (reallen<0) { - if (errno != ENOENT) - NOCONTROL("readlink control socket symlink path %s", path_control); - } - if (reallen >= maxlen) { - debug("control socket symlink path too long (r=%d)",reallen); - xunlink(path_control, "old (overlong) control socket symlink"); - reallen= -1; + if (!path_cli) { + info("control command line disabled"); + return; } - - if (reallen<0) { - struct stat stab; - int r= lstat(realsockdir,&stab); - if (r) { - if (errno != ENOENT) NOCONTROL("lstat real socket dir %s", realsockdir); - r= mkdir(realsockdir, 0700); - if (r) NOCONTROL("mkdir real socket dir %s", realsockdir); - - } else { - uid_t self= geteuid(); - if (!S_ISDIR(stab.st_mode) || - stab.st_uid != self || - stab.st_mode & 0007) { - warn("no control socket, because real socket directory" - " is somehow wrong (ISDIR=%d, uid=%lu (exp.%lu), mode %lo)", - !!S_ISDIR(stab.st_mode), - (unsigned long)stab.st_uid, (unsigned long)self, - (unsigned long)stab.st_mode & 0777UL); - goto nocontrol; - } - } - - real= xasprintf("%s/s%lx.%lx", realsockdir, - (unsigned long)xtime(), (unsigned long)self_pid); - int reallen= strlen(real); + int pathlen= strlen(path_cli); + if (pathlen > maxlen) { + warn("no cli listener, because cli socket path %s too long (%d>%d)", + path_cli, pathlen, maxlen); + return; + } - if (reallen >= maxlen) { - warn("no control socket, because tmpnam gave overly-long path" - " %s", real); - goto nocontrol; - } - r= symlink(real, path_control); - if (r) NOCONTROL("make control socket path %s a symlink to real" - " socket path %s", path_control, real); - memcpy(sa.un.sun_path, real, reallen); + if (path_cli_dir) { + int r= mkdir(path_cli_dir, 0700); + if (r && errno!=EEXIST) + NOCLI("create cli socket directory %s", path_cli_dir); } - int r= unlink(sa.un.sun_path); + int r= unlink(path_cli); if (r && errno!=ENOENT) - NOCONTROL("remove old real socket %s", sa.un.sun_path); + NOCLI("remove old cli socket %s", path_cli); - control_master= socket(PF_UNIX, SOCK_STREAM, 0); - if (control_master<0) NOCONTROL("create new control socket"); + cli_master= socket(PF_UNIX, SOCK_STREAM, 0); + if (cli_master<0) NOCLI("create new cli master socket"); + int sl= pathlen + offsetof(struct sockaddr_un, sun_path); sa.un.sun_family= AF_UNIX; - int sl= strlen(sa.un.sun_path) + offsetof(struct sockaddr_un, sun_path); - r= bind(control_master, &sa.sa, sl); - if (r) NOCONTROL("bind to real socket path %s", sa.un.sun_path); + memcpy(sa.un.sun_path, path_cli, pathlen); + + r= bind(cli_master, &sa.sa, sl); + if (r) NOCLI("bind to cli socket path %s", sa.un.sun_path); - r= listen(control_master, 5); - if (r) NOCONTROL("listen"); + r= listen(cli_master, 5); + if (r) NOCLI("listen to cli master socket"); - xsetnonblock(control_master, 1); + xsetnonblock(cli_master, 1); - loop->on_fd(loop, control_master, OOP_READ, control_master_readable, 0); - info("control socket ok, real path %s", sa.un.sun_path); + loop->on_fd(loop, cli_master, OOP_READ, cli_master_readable, 0); + info("cli ready, listening on %s", path_cli); return; - nocontrol: - free(real); - xclose_perhaps(&control_master, "control master",0); + nocli: + xclose_perhaps(&cli_master, "cli master",0); return; } /*========== management of connections ==========*/ +static void reconnect_blocking_event(void) { + until_connect= reconnect_delay_periods; +} + static void conn_closefd(Conn *conn, const char *msgprefix) { int r= close_perhaps(&conn->fd); if (r) info("C%d %serror closing socket: %s", conn->fd, msgprefix, strerror(errno)); } +static int conn_busy(Conn *conn) { + return + conn->waiting.count || + conn->priority.count || + conn->sent.count || + conn->xmitu; +} + static void conn_dispose(Conn *conn) { if (!conn) return; if (conn->rd) { @@ -1113,7 +1101,6 @@ static void conn_dispose(Conn *conn) { } conn_closefd(conn,""); free(conn); - until_connect= reconnect_delay_periods; } static void *conn_exception(oop_source *lp, int fd, @@ -1153,12 +1140,14 @@ static void vconnfail(Conn *conn, const char *fmt, va_list al) { for (i=0, d=conn->xmitd; ixmitu; i++, d++) xmit_free(d); + LIST_REMOVE(conns,conn); + char *m= xvasprintf(fmt,al); - warn("C%d connection failed (requeueing " RCI_TRIPLE_FMT_BASE "): %s", - conn->fd, RCI_TRIPLE_VALS_BASE(requeue, /*nothing*/), m); + warn("C%d (now %d) connection failed requeueing " RCI_TRIPLE_FMT_BASE ": %s", + conn->fd, conns.count, RCI_TRIPLE_VALS_BASE(requeue, /*nothing*/), m); free(m); - LIST_REMOVE(conns,conn); + reconnect_blocking_event(); conn_dispose(conn); check_assign_articles(); } @@ -1170,17 +1159,56 @@ static void connfail(Conn *conn, const char *fmt, ...) { va_end(al); } +static void conn_idle_close(Conn *conn, const char *why) { + static const char quitcmd[]= "QUIT\r\n"; + int todo= sizeof(quitcmd)-1; + const char *p= quitcmd; + for (;;) { + int r= write(conn->fd, p, todo); + if (r<0) { + if (isewouldblock(errno)) + connfail(conn, "blocked writing QUIT to idle connection"); + else + connfail(conn, "failed to write QUIT to idle connection: %s", + strerror(errno)); + break; + } + assert(r<=todo); + todo -= r; + if (!todo) { + conn->quitting= why; + conn->since_activity= 0; + debug("C%d is idle (%s), quitting", conn->fd, why); + break; + } + } +} + +/* + * For our last connection, we also shut it down if we have had + * less than K in the last L + */ static void check_idle_conns(void) { Conn *conn; + + int volthisperiod= lowvol_perperiod[lowvol_circptr]; + lowvol_circptr++; + lowvol_circptr %= lowvol_periods; + lowvol_total += volthisperiod; + lowvol_total -= lowvol_perperiod[lowvol_circptr]; + lowvol_perperiod[lowvol_circptr]= 0; + FOR_CONN(conn) conn->since_activity++; + search_again: FOR_CONN(conn) { if (conn->since_activity <= need_activity_periods) continue; /* We need to shut this down */ if (conn->quitting) - connfail(conn,"timed out waiting for response to QUIT"); + connfail(conn,"timed out waiting for response to QUIT (%s)", + conn->quitting); else if (conn->sent.count) connfail(conn,"timed out waiting for responses"); else if (conn->waiting.count || conn->priority.count) @@ -1188,32 +1216,18 @@ static void check_idle_conns(void) { else if (conn->xmitu) connfail(conn,"peer has been sending responses" " before receiving our commands!"); - else { - static const char quitcmd[]= "QUIT\r\n"; - int todo= sizeof(quitcmd)-1; - const char *p= quitcmd; - for (;;) { - int r= write(conn->fd, p, todo); - if (r<0) { - if (isewouldblock(errno)) - connfail(conn, "blocked writing QUIT to idle connection"); - else - connfail(conn, "failed to write QUIT to idle connection: %s", - strerror(errno)); - break; - } - assert(r<=todo); - todo -= r; - if (!todo) { - conn->quitting= 1; - conn->since_activity= 0; - debug("C%d is idle, quitting", conn->fd); - break; - } - } - } + else + conn_idle_close(conn, "no activity"); + goto search_again; } + + conn= LIST_HEAD(conns); + if (!volthisperiod && + conns.count==1 && + lowvol_total < lowvol_thresh && + !conn_busy(conn)) + conn_idle_close(conn, "low volume"); } /*---------- making new connections ----------*/ @@ -1335,8 +1349,9 @@ static void *connchild_event(oop_source *lp, int fd, oop_event e, void *u) { &peer_rd_err, conn); if (r) sysdie("oop_rd_read for peer (fd=%d)",conn->fd); - notice("C%d connected %s", conn->fd, conn->stream ? "streaming" : "plain"); LIST_ADDHEAD(conns, conn); + notice("C%d (now %d) connected %s", + conn->fd, conns.count, conn->stream ? "streaming" : "plain"); connect_attempt_discard(); check_assign_articles(); @@ -1345,6 +1360,7 @@ static void *connchild_event(oop_source *lp, int fd, oop_event e, void *u) { x: conn_dispose(conn); connect_attempt_discard(); + reconnect_blocking_event(); return OOP_CONTINUE; } @@ -1359,6 +1375,8 @@ static void connect_start(void) { assert(!connecting_fdpass_sock); info("starting connection attempt"); + int ok_until_connect= until_connect; + reconnect_blocking_event(); int socks[2]; int r= socketpair(AF_UNIX, SOCK_STREAM, 0, socks); @@ -1451,6 +1469,9 @@ static void connect_start(void) { connecting_fdpass_sock= socks[0]; xsetnonblock(connecting_fdpass_sock, 1); on_fd_read_except(connecting_fdpass_sock, connchild_event); + + if (!conns.count) + until_connect= ok_until_connect; } /*---------- assigning articles to conns, and transmitting ----------*/ @@ -1503,11 +1524,11 @@ static void check_assign_articles(void) { Article *art= dequeue(0); if (!art) break; LIST_ADDTAIL(use->waiting, art); + lowvol_perperiod[lowvol_circptr]++; spare--; } conn_maybe_write(use); } else if (allow_connect_start()) { - until_connect= reconnect_delay_periods; connect_start(); break; } else { @@ -1621,6 +1642,7 @@ static void autodefer_input_file_articles(InputFile *ipf) { } static void autodefer_input_file(InputFile *ipf) { + static const char *const abandon= "stuck"; ipf->autodefer= 0; autodefer_input_file_articles(ipf); @@ -1631,11 +1653,11 @@ static void autodefer_input_file(InputFile *ipf) { if (has_article_in(&walk->waiting, ipf) || has_article_in(&walk->priority, ipf) || has_article_in(&walk->sent, ipf)) - walk->quitting= -1; + walk->quitting= abandon; } while (ipf->inprogress) { FOR_CONN(walk) - if (walk->quitting < 0) goto found; + if (walk->quitting == abandon) goto found; abort(); /* where are they ?? */ found: @@ -1698,15 +1720,18 @@ static void *conn_write_some_xmits(Conn *conn) { assert(rs > 0); int done; - for (done=0; rs && donexmitu; done++) { + for (done=0; rs; ) { + assert(donexmitu); struct iovec *vp= &conn->xmit[done]; XmitDetails *dp= &conn->xmitd[done]; - if (rs > vp->iov_len) { + if (rs >= vp->iov_len) { rs -= vp->iov_len; - xmit_free(dp); + xmit_free(dp); /* vp->iov_len -= vp->iov_len, etc. */ + done++; } else { vp->iov_base= (char*)vp->iov_base + rs; vp->iov_len -= rs; + break; /* rs -= rs */ } } int newu= conn->xmitu - done; @@ -1926,19 +1951,17 @@ static void *peer_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, return OOP_CONTINUE; } - int conn_busy= - conn->waiting.count || - conn->priority.count || - conn->sent.count || - conn->xmitu; + int busy= conn_busy(conn); if (conn->quitting) { - if (code!=205 && code!=503) { - connfail(conn, "peer gave unexpected response to QUIT: %s", sani); + if (code!=205 && code!=400) { + connfail(conn, "peer gave unexpected response to QUIT (%s): %s", + conn->quitting, sani); } else { - notice("C%d idle connection closed by us", conn->fd); - assert(!conn_busy); LIST_REMOVE(conns,conn); + notice("C%d (now %d) idle connection closed (%s)", + conn->fd, conns.count, conn->quitting); + assert(!busy); conn_dispose(conn); } return OOP_CONTINUE; @@ -1970,11 +1993,12 @@ static void *peer_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, default: PEERBADMSG("peer sent unexpected message"); case 400: - if (conn_busy) + if (busy) PEERBADMSG("peer timed us out or stopped accepting articles"); - notice("C%d idle connection closed by peer", conn->fd); LIST_REMOVE(conns,conn); + notice("C%d (now %d) idle connection closed by peer", + conns.count, conn->fd); conn_dispose(conn); return OOP_CONTINUE; @@ -2162,7 +2186,16 @@ static void *feedfile_got_article(oop_source *lp, oop_read *rd, static void *tailing_rable_call_time(oop_source *loop, struct timeval tv, void *user) { + /* lifetime of ipf here is OK because destruction will cause + * on_cancel which will cancel this callback */ InputFile *ipf= user; + + if (!ipf->fake_readable) return OOP_CONTINUE; + + /* we just keep calling readable until our caller (oop_rd) + * has called try_read, and try_read has found EOF so given EAGAIN */ + loop->on_time(loop, OOP_TIME_NOW, tailing_rable_call_time, ipf); + return ipf->readable_callback(loop, &ipf->readable, ipf->readable_callback_user); } @@ -2175,9 +2208,10 @@ static void tailing_on_cancel(struct oop_readable *rable) { ipf->readable_callback= 0; } -static void tailing_queue_readable(InputFile *ipf) { - /* lifetime of ipf here is OK because destruction will cause - * on_cancel which will cancel this callback */ +static void tailing_make_readable(InputFile *ipf) { + if (!ipf || !ipf->readable_callback) /* so callers can be naive */ + return; + ipf->fake_readable= 1; loop->on_time(loop, OOP_TIME_NOW, tailing_rable_call_time, ipf); } @@ -2189,8 +2223,7 @@ static int tailing_on_readable(struct oop_readable *rable, ipf->readable_callback= cb; ipf->readable_callback_user= user; filemon_start(ipf); - - tailing_queue_readable(ipf); + tailing_make_readable(ipf); return 0; } @@ -2201,11 +2234,13 @@ static ssize_t tailing_try_read(struct oop_readable *rable, void *buffer, ssize_t r= read(ipf->fd, buffer, length); if (r==-1) { if (errno==EINTR) continue; + ipf->fake_readable= 0; return r; } if (!r) { if (ipf==main_input_file) { errno=EAGAIN; + ipf->fake_readable= 0; return -1; } else if (ipf==flushing_input_file) { assert(ipf->rd); @@ -2216,7 +2251,6 @@ static ssize_t tailing_try_read(struct oop_readable *rable, void *buffer, abort(); } } - tailing_queue_readable(ipf); return r; } } @@ -2280,8 +2314,8 @@ static void *filemon_inotify_readable(oop_source *lp, int fd, die("inotify read %d bytes wanted struct of %d", r, (int)sizeof(iev)); } InputFile *ipf= filemon_inotify_wd2ipf[iev.wd]; - debug("filemon inotify readable read %p wd=%d", ipf, iev.wd); - filemon_callback(ipf); + /*debug("filemon inotify readable read %p wd=%d", ipf, iev.wd);*/ + tailing_make_readable(ipf); } return OOP_CONTINUE; } @@ -2342,11 +2376,6 @@ static void filemon_stop(InputFile *ipf) { ipf->filemon= 0; } -static void filemon_callback(InputFile *ipf) { - if (ipf && ipf->readable_callback) /* so filepoll() can be naive */ - ipf->readable_callback(loop, &ipf->readable, ipf->readable_callback_user); -} - /*---------- interface to start and stop an input file ----------*/ static const oop_rd_style feedfile_rdstyle= { @@ -2636,20 +2665,21 @@ static void statemc_start_flush(const char *why) { /* Normal => Flushing */ spawn_inndcomm_flush(why); /* => Flushing FLUSHING */ } -static int trigger_flush_ok(void) { /* => Flushing,FLUSHING, ret 1; or ret 0 */ +static int trigger_flush_ok(const char *why) { switch (sms) { case sm_NORMAL: - statemc_start_flush("periodic"); /* Normal => Flushing; => FLUSHING */ - return 1; + statemc_start_flush(why ? why : "periodic"); + return 1; /* Normal => Flushing; => FLUSHING */ case sm_FLUSHFAILED: - spawn_inndcomm_flush("retry"); /* Moved => Flushing; => FLUSHING */ - return 1; + spawn_inndcomm_flush(why ? why : "retry"); + return 1; /* Moved => Flushing; => FLUSHING */ case sm_SEPARATED: case sm_DROPPING: - warn("took too long to complete old feedfile after flush, autodeferring"); + warn("abandoning old feedfile after flush (%s), autodeferring", + why ? why : "took too long to complete"); assert(flushing_input_file); autodefer_input_file(flushing_input_file); return 1; @@ -2665,7 +2695,7 @@ static void statemc_period_poll(void) { assert(until_flush>=0); if (until_flush) return; - int ok= trigger_flush_ok(); + int ok= trigger_flush_ok(0); assert(ok); } @@ -2948,11 +2978,11 @@ static void search_backlog_file(void) { preterminate(); notice("feed dropped and our work is complete"); - int r= unlink(path_control); + int r= unlink(path_cli); if (r && errno!=ENOENT) - syswarn("failed to remove control symlink for old feed"); + syswarn("failed to unlink cli socket for old feed"); - xunlink(path_lock, "lockfile for old feed"); + xunlink(path_lock, "lockfile for old feed"); exit(4); } until_backlog_nextscan= backlog_spontrescan_periods; @@ -3084,7 +3114,7 @@ static void *inndcomm_event(oop_source *lp, int fd, oop_event e, void *u) { case INNDCOMMCHILD_ESTATUS_NONESUCH: notice("feed has been dropped by innd, finishing up"); flushing_input_file= main_input_file; - tailing_queue_readable(flushing_input_file); + tailing_make_readable(flushing_input_file); /* we probably previously returned EAGAIN from our fake read method * when in fact we were at EOF, so signal another readable event * so we actually see the EOF */ @@ -3104,17 +3134,19 @@ static void *inndcomm_event(oop_source *lp, int fd, oop_event e, void *u) { case 0: /* as above */ flushing_input_file= main_input_file; - tailing_queue_readable(flushing_input_file); + tailing_make_readable(flushing_input_file); main_input_file= open_input_file(feedfile); if (!main_input_file) - die("flush succeeded but feedfile %s does not exist!", feedfile); + die("flush succeeded but feedfile %s does not exist!" + " (this probably means feedfile does not correspond" + " to site %s in newsfeeds)", feedfile, sitename); if (flushing_input_file) { - SMS(SEPARATED, max_separated_periods, "recovery flush complete"); + SMS(SEPARATED, max_separated_periods, "flush complete"); } else { close_defer(); - SMS(NORMAL, spontaneous_flush_periods, "flush complete"); + SMS(NORMAL, spontaneous_flush_periods, "recovery flush complete"); } return OOP_CONTINUE; @@ -3253,8 +3285,8 @@ static void every(int interval, int fixed_rate, void (*f)(void)) { } static void filepoll(void) { - filemon_callback(main_input_file); - filemon_callback(flushing_input_file); + tailing_make_readable(main_input_file); + tailing_make_readable(flushing_input_file); } static char *debug_report_ipf(InputFile *ipf) { @@ -3280,11 +3312,11 @@ static void period(void) { debug("PERIOD" " sms=%s[%d] conns=%d until_connect=%d" " input_files main:%s flushing:%s backlog:%s[%d]" - " children connecting=%ld inndcomm=%ld" + " children connecting=%ld inndcomm=%ld lowvol_total=%d" , sms_names[sms], until_flush, conns.count, until_connect, dipf_main, dipf_flushing, dipf_backlog, until_backlog_nextscan, - (long)connecting_child, (long)inndcomm_child + (long)connecting_child, (long)inndcomm_child, lowvol_total ); free(dipf_main); @@ -3304,7 +3336,7 @@ static void period(void) { /*========== dumping state ==========*/ -static void dump_article_list(FILE *f, const ControlCommand *c, +static void dump_article_list(FILE *f, const CliCommand *c, const ArticleList *al) { fprintf(f, " count=%d\n", al->count); if (!c->xval) return; @@ -3321,7 +3353,7 @@ static void dump_article_list(FILE *f, const ControlCommand *c, } } -static void dump_input_file(FILE *f, const ControlCommand *c, +static void dump_input_file(FILE *f, const CliCommand *c, InputFile *ipf, const char *wh) { char *dipf= debug_report_ipf(ipf); fprintf(f,"input %s %s", wh, dipf); @@ -3331,6 +3363,7 @@ static void dump_input_file(FILE *f, const ControlCommand *c, DUMPV("%d", ipf->,readcount_ok); DUMPV("%d", ipf->,readcount_blank); DUMPV("%d", ipf->,readcount_err); + DUMPV("%d", ipf->,count_nooffer_missing); } fprintf(f,"\n"); if (ipf) { @@ -3371,7 +3404,18 @@ CCMD(dump) { fprintf(f,"special"); DUMPV("%ld", (long),connecting_child); DUMPV("%d", , connecting_fdpass_sock); - DUMPV("%d", , control_master); + DUMPV("%d", , cli_master); + fprintf(f,"\n"); + + fprintf(f,"lowvol"); + DUMPV("%d", , lowvol_circptr); + DUMPV("%d", , lowvol_total); + fprintf(f,":"); + for (i=0; ifd); DUMPV("%p",conn->,rd); DUMPV("%d",conn->,max_queue); - DUMPV("%d",conn->,stream); DUMPV("%d",conn->,quitting); + DUMPV("%d",conn->,stream); DUMPV("\"%s\"",conn->,quitting); DUMPV("%d",conn->,since_activity); fprintf(f,"\n"); @@ -3414,10 +3458,11 @@ CCMD(dump) { } fprintf(f,"paths"); + DUMPV("%s", , feedfile); + DUMPV("%s", , path_cli); DUMPV("%s", , path_lock); DUMPV("%s", , path_flushing); DUMPV("%s", , path_defer); - DUMPV("%s", , path_control); DUMPV("%s", , path_dump); DUMPV("%s", , globpat_backlog); fprintf(f,"\n"); @@ -3601,7 +3646,8 @@ static const Option innduct_options[]= { {0,"no-filemon", 0, &try_filemon, op_setint, 0 }, {'C',"inndconf", "F", &inndconffile, op_string }, {'P',"port", "PORT", &port, op_integer }, -{0,"ctrl-sock-dir", 0, &realsockdir, op_string }, +{0,"chdir", "DIR", &path_run, op_string }, +{0,"cli", "DIR/|PATH", &path_cli, op_string }, {0,"help", 0, 0, help }, {0,"max-connections", "N", &max_connections, op_integer }, @@ -3624,6 +3670,8 @@ static const Option innduct_options[]= { {0,"max-flush-interval", "PERIOD", &spontaneous_flush_periods,op_seconds }, {0,"flush-finish-timeout", "PERIOD", &max_separated_periods, op_seconds }, {0,"idle-timeout", "PERIOD", &need_activity_periods, op_seconds }, +{0,"low-volume-thresh", "PERIOD", &lowvol_thresh, op_integer }, +{0,"low-volume-window", "PERIOD", &lowvol_periods, op_seconds }, {0,"max-bad-input-data-ratio","PERCENT", &max_bad_data_ratio, op_double }, {0,"max-bad-input-data-init", "PERCENT", &max_bad_data_initial, op_integer }, @@ -3651,6 +3699,12 @@ static void convert_to_periods_rndup(int *store) { *store /= period_seconds; } +static int path_ends_slash(const char *specified) { + int l= strlen(specified); + assert(l); + return specified[l-1] == '/'; +} + int main(int argc, char **argv) { if (!argv[1]) { printusage(stderr); @@ -3663,7 +3717,10 @@ int main(int argc, char **argv) { sitename= *argv++; if (!sitename) badusage("need site name argument"); - remote_host= *argv++; + + if (*argv) remote_host= *argv++; + else remote_host= sitename; + if (*argv) badusage("too many non-option arguments"); /* defaults */ @@ -3688,18 +3745,31 @@ int main(int argc, char **argv) { convert_to_periods_rndup(&spontaneous_flush_periods); convert_to_periods_rndup(&max_separated_periods); convert_to_periods_rndup(&need_activity_periods); + convert_to_periods_rndup(&lowvol_periods); if (max_bad_data_ratio < 0 || max_bad_data_ratio > 100) badusage("bad input data ratio must be between 0..100"); max_bad_data_ratio *= 0.01; - if (!feedfile) { - feedfile= xasprintf("%s/%s",innconf->pathoutgoing,sitename); - } else if (!feedfile[0]) { - badusage("feed filename must be nonempty"); - } else if (feedfile[strlen(feedfile)-1]=='/') { - feedfile= xasprintf("%s%s",feedfile,sitename); + if (!path_run) + path_run= innconf->pathrun; + + if (!feedfile) feedfile= sitename; + if (!feedfile[0]) badusage("feed filename, if specified, must be nonempty"); + if (path_ends_slash(feedfile)) + feedfile= xasprintf("%s%s", feedfile, sitename); + if (feedfile[0] != '/') + feedfile= xasprintf("%s/%s", innconf->pathoutgoing, feedfile); + + if (!path_cli) { + path_cli_dir= "innduct"; + } else if (!path_cli[0] || !strcmp(path_cli,"none")) { + path_cli= 0; /* ok, don't then */ + } else if (path_ends_slash(path_cli)) { + path_cli_dir= xasprintf("%.*s", strlen(path_cli)-1, path_cli); } + if (path_cli_dir) + path_cli= xasprintf("%s/%s", path_cli_dir, sitename); if (max_queue_per_ipf<0) max_queue_per_ipf= max_queue_per_conn * 2; @@ -3710,12 +3780,19 @@ int main(int argc, char **argv) { if (strchr(feedfile, c)) badusage("feed filename may not contain metacharacter %c",c); + int i; + lowvol_perperiod= xcalloc(sizeof(*lowvol_perperiod), lowvol_periods); + for (i=0; i