X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?p=inn-innduct.git;a=blobdiff_plain;f=backends%2Finnduct.c;h=772fc89c0079e99be2146d40a53fd7e65bcf344c;hp=8c9d769fee9a04f5f35d5773033b527b4191feea;hb=3fca16a4ee8d432329ce49383132b0de5137a70f;hpb=d2ca385313c2f8994f8ea7f6c62e975d3c331467 diff --git a/backends/innduct.c b/backends/innduct.c index 8c9d769..772fc89 100644 --- a/backends/innduct.c +++ b/backends/innduct.c @@ -1,10 +1,33 @@ +/* + * innduct + * tailing reliable realtime streaming feeder for inn + * + * Copyright (C) 2010 Ian Jackson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * (I believe that when you compile and link this as part of the inn2 + * build, with the Makefile runes I have provided, all the libraries + * and files which end up included in innduct are licence-compatible + * with GPLv3. If not then please let me know. -Ian Jackson.) + */ + /* * todo - * - rename sm_period_counter as it's just about flushes - * - manpage: document control master stuff * - * debugging rune: - * build-lfs/backends/innduct --connection-timeout=30 --no-daemon -C ../inn.conf -f `pwd`/fee sit localhost + * don't mind reconnecting if we just disconnected due to idle + * some weird disconnection event still investigating */ /* @@ -201,7 +224,7 @@ perl -ne 'print if m/-8\<-/..m/-\>8-/; print "\f" if m/-\^L-/' backends/innduct. #define INNDCOMMCHILD_ESTATUS_NONESUCH 27 #define MAX_LINE_FEEDFILE (NNTP_MSGID_MAXLEN + sizeof(TOKEN)*2 + 10) -#define MAX_CONTROL_COMMAND 1000 +#define MAX_CLI_COMMAND 1000 #define VA va_list al; va_start(al,fmt) #define PRINTF(f,a) __attribute__((__format__(printf,f,a))) @@ -277,7 +300,7 @@ typedef struct InputFile InputFile; typedef struct XmitDetails XmitDetails; typedef struct Filemon_Perfile Filemon_Perfile; typedef enum StateMachineState StateMachineState; -typedef struct ControlCommand ControlCommand; +typedef struct CliCommand CliCommand; DEFLIST(Conn); DEFLIST(Article); @@ -297,12 +320,14 @@ static void statemc_setstate(StateMachineState newsms, int periods, static void statemc_start_flush(const char *why); /* Normal => Flushing */ static void spawn_inndcomm_flush(const char *why); /* Moved => Flushing */ -static int trigger_flush_ok(void); /* => Flushing,FLUSHING, ret 1; or ret 0 */ +static int trigger_flush_ok(const char *why /* 0 means timeout */); + /* => Flushing,FLUSHING, ret 1; or ret 0 */ -static void article_done(Conn *conn, Article *art, int whichcount); +static void article_done(Article *art, int whichcount); static void check_assign_articles(void); static void queue_check_input_done(void); +static void check_reading_pause_resume(InputFile *ipf); static void statemc_check_flushing_done(void); static void statemc_check_backlog_done(void); @@ -319,10 +344,13 @@ static char *debug_report_ipf(InputFile *ipf); static void inputfile_reading_start(InputFile *ipf); static void inputfile_reading_stop(InputFile *ipf); +static void inputfile_reading_pause(InputFile *ipf); +static void inputfile_reading_resume(InputFile *ipf); + /* pause and resume are idempotent, and no-op if not done _reading_start */ static void filemon_start(InputFile *ipf); static void filemon_stop(InputFile *ipf); -static void filemon_callback(InputFile *ipf); +static void tailing_make_readable(InputFile *ipf); static void vconnfail(Conn *conn, const char *fmt, va_list al) PRINTF(2,0); static void connfail(Conn *conn, const char *fmt, ...) PRINTF(2,3); @@ -330,11 +358,12 @@ static void connfail(Conn *conn, const char *fmt, ...) PRINTF(2,3); static const oop_rd_style peer_rd_style; static oop_rd_call peer_rd_err, peer_rd_ok; + /*----- configuration options -----*/ /* when changing defaults, remember to update the manpage */ static const char *sitename, *remote_host; -static const char *feedfile, *realsockdir="/tmp/innduct.control"; +static const char *feedfile, *path_run, *path_cli, *path_cli_dir; static int quiet_multiple=0; static int become_daemon=1, try_filemon=1; static int try_stream=1; @@ -346,6 +375,7 @@ static int max_queue_per_conn=200; static int target_max_feedfile_size=100000; static int period_seconds=60; static int filepoll_seconds=5; +static int max_queue_per_ipf=-1; static int connection_setup_timeout=200; static int inndcomm_flush_timeout=100; @@ -356,10 +386,13 @@ static double nocheck_decay= 100; /* conv'd from articles to lambda by main */ /* all these are initialised to seconds, and converted to periods in main */ static int reconnect_delay_periods=1000; static int flushfail_retry_periods=1000; -static int backlog_retry_minperiods=50; +static int backlog_retry_minperiods=100; static int backlog_spontrescan_periods=300; static int spontaneous_flush_periods=100000; +static int max_separated_periods=2000; static int need_activity_periods=1000; +static int recentact_thresh=3; +static int recentact_periods=1000; static double max_bad_data_ratio= 1; /* conv'd from percentage by main */ static int max_bad_data_initial= 30; @@ -432,12 +465,15 @@ struct InputFile { Filemon_Perfile *filemon; oop_read *rd; /* non-0: reading; 0: constructing, or had EOF */ - long inprogress; /* no. of articles read but not processed */ off_t offset; - int skippinglong; + int skippinglong, paused, fake_readable; + + ArticleList queue; + long inprogress; /* includes queue.count and also articles in conns */ + long autodefer; /* -1 means not doing autodefer */ int counts[art_MaxState][RCI_max]; - int readcount_ok, readcount_blank, readcount_err; + int readcount_ok, readcount_blank, readcount_err, count_nooffer_missing; char path[]; }; @@ -491,15 +527,16 @@ struct Conn { /* main initialises */ static oop_source *loop; static ConnList conns; -static ArticleList queue; -static char *path_lock, *path_flushing, *path_defer; -static char *path_control, *path_dump; +static char *path_lock, *path_flushing, *path_defer, *path_dump; static char *globpat_backlog; static pid_t self_pid; +static int *recentact_perperiod; +static int recentact_circptr; +static int recentact_total; /* statemc_init initialises */ static StateMachineState sms; -static int sm_period_counter; +static int until_flush; static InputFile *main_input_file, *flushing_input_file, *backlog_input_file; static FILE *defer; @@ -530,7 +567,7 @@ static void logv(int sysloglevel, const char *pfx, int errnoval, const char *fmt, va_list al) PRINTF(5,0); static void logv(int sysloglevel, const char *pfx, int errnoval, const char *fmt, va_list al) { - char msgbuf[256]; /* NB do not call xvasprintf here or you'll recurse */ + char msgbuf[1024]; /* NB do not call xvasprintf here or you'll recurse */ vsnprintf(msgbuf,sizeof(msgbuf), fmt,al); msgbuf[sizeof(msgbuf)-1]= 0; @@ -758,13 +795,13 @@ static int isewouldblock(int errnoval) { return errnoval==EWOULDBLOCK || errnoval==EAGAIN; } -/*========== command and control connections ==========*/ +/*========== command and control (CLI) connections ==========*/ -static int control_master; +static int cli_master; -typedef struct ControlConn ControlConn; -struct ControlConn { - void (*destroy)(ControlConn*); +typedef struct CliConn CliConn; +struct CliConn { + void (*destroy)(CliConn*); int fd; oop_read *rd; FILE *out; @@ -775,51 +812,53 @@ struct ControlConn { socklen_t salen; }; -static const oop_rd_style control_rd_style= { +static const oop_rd_style cli_rd_style= { OOP_RD_DELIM_STRIP, '\n', OOP_RD_NUL_FORBID, OOP_RD_SHORTREC_FORBID }; -static void control_destroy(ControlConn *cc) { +static void cli_destroy(CliConn *cc) { cc->destroy(cc); } -static void control_checkouterr(ControlConn *cc /* may destroy*/) { +static void cli_checkouterr(CliConn *cc /* may destroy*/) { if (ferror(cc->out) | fflush(cc->out)) { info("CTRL%d write error %s", cc->fd, strerror(errno)); - control_destroy(cc); + cli_destroy(cc); } } -static void control_prompt(ControlConn *cc /* may destroy*/) { +static void cli_prompt(CliConn *cc /* may destroy*/) { fprintf(cc->out, "%s| ", sitename); - control_checkouterr(cc); + cli_checkouterr(cc); } -struct ControlCommand { +struct CliCommand { const char *cmd; - void (*f)(ControlConn *cc, const ControlCommand *ccmd, + void (*f)(CliConn *cc, const CliCommand *ccmd, const char *arg, size_t argsz); void *xdata; int xval; }; -static const ControlCommand control_commands[]; +static const CliCommand cli_commands[]; -#define CCMD(wh) \ - static void ccmd_##wh(ControlConn *cc, const ControlCommand *c, \ +#define CCMD(wh) \ + static void ccmd_##wh(CliConn *cc, const CliCommand *c, \ const char *arg, size_t argsz) CCMD(help) { fputs("commands:\n", cc->out); - const ControlCommand *ccmd; - for (ccmd=control_commands; ccmd->cmd; ccmd++) + const CliCommand *ccmd; + for (ccmd=cli_commands; ccmd->cmd; ccmd++) fprintf(cc->out, " %s\n", ccmd->cmd); + fputs("NB: permissible arguments are not shown above." + " Not all commands listed are safe. See innduct(8).\n", cc->out); } CCMD(flush) { - int ok= trigger_flush_ok(); + int ok= trigger_flush_ok("manual request"); if (!ok) fprintf(cc->out,"already flushing (state is %s)\n", sms_names[sms]); } @@ -838,7 +877,7 @@ CCMD(setintarg) { *(int*)c->xdata= atoi(arg); } CCMD(setint) { *(int*)c->xdata= c->xval; } CCMD(setint_period) { *(int*)c->xdata= c->xval; period(); } -static const ControlCommand control_commands[]= { +static const CliCommand cli_commands[]= { { "h", ccmd_help }, { "flush", ccmd_flush }, { "stop", ccmd_stop }, @@ -848,21 +887,21 @@ static const ControlCommand control_commands[]= { { "p", ccmd_period }, #define POKES(cmd,func) \ - { cmd "sm", func, &sm_period_counter, 1 }, \ + { cmd "flush", func, &until_flush, 1 }, \ { cmd "conn", func, &until_connect, 0 }, \ { cmd "blscan", func, &until_backlog_nextscan, 0 }, -POKES("prod ", ccmd_setint_period) POKES("next ", ccmd_setint) +POKES("prod ", ccmd_setint_period) { "pretend flush", ccmd_setintarg, &simulate_flush }, { "wedge blscan", ccmd_setint, &until_backlog_nextscan, -1 }, { 0 } }; -static void *control_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, - const char *errmsg, int errnoval, - const char *data, size_t recsz, void *cc_v) { - ControlConn *cc= cc_v; +static void *cli_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, + const char *errmsg, int errnoval, + const char *data, size_t recsz, void *cc_v) { + CliConn *cc= cc_v; if (!data) { info("CTRL%d closed", cc->fd); @@ -872,8 +911,8 @@ static void *control_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, if (recsz == 0) goto prompt; - const ControlCommand *ccmd; - for (ccmd=control_commands; ccmd->cmd; ccmd++) { + const CliCommand *ccmd; + for (ccmd=cli_commands; ccmd->cmd; ccmd++) { int l= strlen(ccmd->cmd); if (recsz < l) continue; if (recsz > l && data[l] != ' ') continue; @@ -887,36 +926,36 @@ static void *control_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, fputs("unknown command; h for help\n", cc->out); prompt: - control_prompt(cc); + cli_prompt(cc); return OOP_CONTINUE; } -static void *control_rd_err(oop_source *lp, oop_read *oread, oop_rd_event ev, - const char *errmsg, int errnoval, - const char *data, size_t recsz, void *cc_v) { - ControlConn *cc= cc_v; +static void *cli_rd_err(oop_source *lp, oop_read *oread, oop_rd_event ev, + const char *errmsg, int errnoval, + const char *data, size_t recsz, void *cc_v) { + CliConn *cc= cc_v; info("CTRL%d read error %s", cc->fd, errmsg); cc->destroy(cc); return OOP_CONTINUE; } -static int control_conn_startup(ControlConn *cc /* may destroy*/, +static int cli_conn_startup(CliConn *cc /* may destroy*/, const char *how) { cc->rd= oop_rd_new_fd(loop, cc->fd, 0,0); - if (!cc->rd) { warn("oop_rd_new_fd control failed"); return -1; } + if (!cc->rd) { warn("oop_rd_new_fd cli failed"); return -1; } - int er= oop_rd_read(cc->rd, &control_rd_style, MAX_CONTROL_COMMAND, - control_rd_ok, cc, - control_rd_err, cc); - if (er) { errno= er; syswarn("oop_rd_read control failed"); return -1; } + int er= oop_rd_read(cc->rd, &cli_rd_style, MAX_CLI_COMMAND, + cli_rd_ok, cc, + cli_rd_err, cc); + if (er) { errno= er; syswarn("oop_rd_read cli failed"); return -1; } info("CTRL%d %s ready", cc->fd, how); - control_prompt(cc); + cli_prompt(cc); return 0; } -static void control_stdio_destroy(ControlConn *cc) { +static void cli_stdio_destroy(CliConn *cc) { if (cc->rd) { oop_rd_cancel(cc->rd); errno= oop_rd_delete_tidy(cc->rd); @@ -925,17 +964,17 @@ static void control_stdio_destroy(ControlConn *cc) { free(cc); } -static void control_stdio(void) { - NEW_DECL(ControlConn *,cc); - cc->destroy= control_stdio_destroy; +static void cli_stdio(void) { + NEW_DECL(CliConn *,cc); + cc->destroy= cli_stdio_destroy; cc->fd= 0; cc->out= stdout; - int r= control_conn_startup(cc,"stdio"); + int r= cli_conn_startup(cc,"stdio"); if (r) cc->destroy(cc); } -static void control_accepted_destroy(ControlConn *cc) { +static void cli_accepted_destroy(CliConn *cc) { if (cc->rd) { oop_rd_cancel(cc->rd); oop_rd_delete_kill(cc->rd); @@ -945,19 +984,19 @@ static void control_accepted_destroy(ControlConn *cc) { free(cc); } -static void *control_master_readable(oop_source *lp, int master, - oop_event ev, void *u) { - NEW_DECL(ControlConn *,cc); - cc->destroy= control_accepted_destroy; +static void *cli_master_readable(oop_source *lp, int master, + oop_event ev, void *u) { + NEW_DECL(CliConn *,cc); + cc->destroy= cli_accepted_destroy; cc->salen= sizeof(cc->sa); cc->fd= accept(master, &cc->sa.sa, &cc->salen); - if (cc->fd<0) { syswarn("error accepting control connection"); goto x; } + if (cc->fd<0) { syswarn("error accepting cli connection"); goto x; } cc->out= fdopen(cc->fd, "w"); - if (!cc->out) { syswarn("error fdopening accepted control conn"); goto x; } + if (!cc->out) { syswarn("error fdopening accepted cli connection"); goto x; } - int r= control_conn_startup(cc, "accepted"); + int r= cli_conn_startup(cc, "accepted"); if (r) goto x; return OOP_CONTINUE; @@ -967,14 +1006,12 @@ static void *control_master_readable(oop_source *lp, int master, return OOP_CONTINUE; } -#define NOCONTROL(...) do{ \ - syswarn("no control socket, because failed to " __VA_ARGS__); \ - goto nocontrol; \ +#define NOCLI(...) do{ \ + syswarn("no cli listener, because failed to " __VA_ARGS__); \ + goto nocli; \ }while(0) -static void control_init(void) { - char *real=0; - +static void cli_init(void) { union { struct sockaddr sa; struct sockaddr_un un; @@ -983,91 +1020,73 @@ static void control_init(void) { memset(&sa,0,sizeof(sa)); int maxlen= sizeof(sa.un.sun_path); - int reallen= readlink(path_control, sa.un.sun_path, maxlen); - if (reallen<0) { - if (errno != ENOENT) - NOCONTROL("readlink control socket symlink path %s", path_control); - } - if (reallen >= maxlen) { - debug("control socket symlink path too long (r=%d)",reallen); - xunlink(path_control, "old (overlong) control socket symlink"); - reallen= -1; + if (!path_cli) { + info("control command line disabled"); + return; } - - if (reallen<0) { - struct stat stab; - int r= lstat(realsockdir,&stab); - if (r) { - if (errno != ENOENT) NOCONTROL("lstat real socket dir %s", realsockdir); - - r= mkdir(realsockdir, 0700); - if (r) NOCONTROL("mkdir real socket dir %s", realsockdir); - - } else { - uid_t self= geteuid(); - if (!S_ISDIR(stab.st_mode) || - stab.st_uid != self || - stab.st_mode & 0007) { - warn("no control socket, because real socket directory" - " is somehow wrong (ISDIR=%d, uid=%lu (exp.%lu), mode %lo)", - !!S_ISDIR(stab.st_mode), - (unsigned long)stab.st_uid, (unsigned long)self, - (unsigned long)stab.st_mode & 0777UL); - goto nocontrol; - } - } - real= xasprintf("%s/s%lx.%lx", realsockdir, - (unsigned long)xtime(), (unsigned long)self_pid); - int reallen= strlen(real); + int pathlen= strlen(path_cli); + if (pathlen > maxlen) { + warn("no cli listener, because cli socket path %s too long (%d>%d)", + path_cli, pathlen, maxlen); + return; + } - if (reallen >= maxlen) { - warn("no control socket, because tmpnam gave overly-long path" - " %s", real); - goto nocontrol; - } - r= symlink(real, path_control); - if (r) NOCONTROL("make control socket path %s a symlink to real" - " socket path %s", path_control, real); - memcpy(sa.un.sun_path, real, reallen); + if (path_cli_dir) { + int r= mkdir(path_cli_dir, 0700); + if (r && errno!=EEXIST) + NOCLI("create cli socket directory %s", path_cli_dir); } - int r= unlink(sa.un.sun_path); + int r= unlink(path_cli); if (r && errno!=ENOENT) - NOCONTROL("remove old real socket %s", sa.un.sun_path); + NOCLI("remove old cli socket %s", path_cli); - control_master= socket(PF_UNIX, SOCK_STREAM, 0); - if (control_master<0) NOCONTROL("create new control socket"); + cli_master= socket(PF_UNIX, SOCK_STREAM, 0); + if (cli_master<0) NOCLI("create new cli master socket"); + int sl= pathlen + offsetof(struct sockaddr_un, sun_path); sa.un.sun_family= AF_UNIX; - int sl= strlen(sa.un.sun_path) + offsetof(struct sockaddr_un, sun_path); - r= bind(control_master, &sa.sa, sl); - if (r) NOCONTROL("bind to real socket path %s", sa.un.sun_path); + memcpy(sa.un.sun_path, path_cli, pathlen); - r= listen(control_master, 5); - if (r) NOCONTROL("listen"); + r= bind(cli_master, &sa.sa, sl); + if (r) NOCLI("bind to cli socket path %s", sa.un.sun_path); - xsetnonblock(control_master, 1); + r= listen(cli_master, 5); + if (r) NOCLI("listen to cli master socket"); - loop->on_fd(loop, control_master, OOP_READ, control_master_readable, 0); - info("control socket ok, real path %s", sa.un.sun_path); + xsetnonblock(cli_master, 1); + + loop->on_fd(loop, cli_master, OOP_READ, cli_master_readable, 0); + info("cli ready, listening on %s", path_cli); return; - nocontrol: - free(real); - xclose_perhaps(&control_master, "control master",0); + nocli: + xclose_perhaps(&cli_master, "cli master",0); return; } /*========== management of connections ==========*/ +static void reconnect_blocking_event(void) { + until_connect= reconnect_delay_periods; +} + static void conn_closefd(Conn *conn, const char *msgprefix) { int r= close_perhaps(&conn->fd); if (r) info("C%d %serror closing socket: %s", conn->fd, msgprefix, strerror(errno)); } +static int conn_busy(Conn *conn) { + return + conn->waiting.count || + conn->priority.count || + conn->sent.count || + conn->xmitu; +} + static void conn_dispose(Conn *conn) { if (!conn) return; if (conn->rd) { @@ -1081,7 +1100,6 @@ static void conn_dispose(Conn *conn) { } conn_closefd(conn,""); free(conn); - until_connect= reconnect_delay_periods; } static void *conn_exception(oop_source *lp, int fd, @@ -1102,12 +1120,18 @@ static void vconnfail(Conn *conn, const char *fmt, va_list al) { memset(requeue,0,sizeof(requeue)); Article *art; - while ((art= LIST_REMHEAD(conn->priority))) LIST_ADDTAIL(queue, art); - while ((art= LIST_REMHEAD(conn->waiting))) LIST_ADDTAIL(queue, art); + + while ((art= LIST_REMHEAD(conn->priority))) + LIST_ADDTAIL(art->ipf->queue, art); + + while ((art= LIST_REMHEAD(conn->waiting))) + LIST_ADDTAIL(art->ipf->queue, art); + while ((art= LIST_REMHEAD(conn->sent))) { requeue[art->state]++; if (art->state==art_Unsolicited) art->state= art_Unchecked; - LIST_ADDTAIL(queue,art); + LIST_ADDTAIL(art->ipf->queue,art); + check_reading_pause_resume(art->ipf); } int i; @@ -1115,12 +1139,14 @@ static void vconnfail(Conn *conn, const char *fmt, va_list al) { for (i=0, d=conn->xmitd; ixmitu; i++, d++) xmit_free(d); + LIST_REMOVE(conns,conn); + char *m= xvasprintf(fmt,al); - warn("C%d connection failed (requeueing " RCI_TRIPLE_FMT_BASE "): %s", - conn->fd, RCI_TRIPLE_VALS_BASE(requeue, /*nothing*/), m); + warn("C%d (now %d) connection failed requeueing " RCI_TRIPLE_FMT_BASE ": %s", + conn->fd, conns.count, RCI_TRIPLE_VALS_BASE(requeue, /*nothing*/), m); free(m); - LIST_REMOVE(conns,conn); + reconnect_blocking_event(); conn_dispose(conn); check_assign_articles(); } @@ -1132,10 +1158,47 @@ static void connfail(Conn *conn, const char *fmt, ...) { va_end(al); } +static void conn_idle_close(Conn *conn, const char *why) { + static const char quitcmd[]= "QUIT\r\n"; + int todo= sizeof(quitcmd)-1; + const char *p= quitcmd; + for (;;) { + int r= write(conn->fd, p, todo); + if (r<0) { + if (isewouldblock(errno)) + connfail(conn, "blocked writing QUIT to idle connection"); + else + connfail(conn, "failed to write QUIT to idle connection: %s", + strerror(errno)); + break; + } + assert(r<=todo); + todo -= r; + if (!todo) { + conn->quitting= 1; + conn->since_activity= 0; + debug("C%d is idle (%s), quitting", conn->fd, why); + break; + } + } +} + +/* + * For our last connection, we also shut it down if we have had + * less than K in the last L + */ static void check_idle_conns(void) { Conn *conn; + + int veryrecentact= recentact_perperiod[recentact_circptr]; + recentact_circptr++; + recentact_circptr %= recentact_periods; + recentact_total -= recentact_perperiod[recentact_circptr]; + recentact_perperiod[recentact_circptr]= 0; + FOR_CONN(conn) conn->since_activity++; + search_again: FOR_CONN(conn) { if (conn->since_activity <= need_activity_periods) continue; @@ -1150,32 +1213,18 @@ static void check_idle_conns(void) { else if (conn->xmitu) connfail(conn,"peer has been sending responses" " before receiving our commands!"); - else { - static const char quitcmd[]= "QUIT\r\n"; - int todo= sizeof(quitcmd)-1; - const char *p= quitcmd; - for (;;) { - int r= write(conn->fd, p, todo); - if (r<0) { - if (isewouldblock(errno)) - connfail(conn, "blocked writing QUIT to idle connection"); - else - connfail(conn, "failed to write QUIT to idle connection: %s", - strerror(errno)); - break; - } - assert(r<=todo); - todo -= r; - if (!todo) { - conn->quitting= 1; - conn->since_activity= 0; - debug("C%d is idle, quitting", conn->fd); - break; - } - } - } + else + conn_idle_close(conn, "no activity"); + goto search_again; } + + conn= LIST_HEAD(conns); + if (!veryrecentact && + conns.count==1 && + recentact_total+veryrecentact < recentact_thresh && + !conn_busy(conn)) + conn_idle_close(conn, "low volume"); } /*---------- making new connections ----------*/ @@ -1184,6 +1233,7 @@ static pid_t connecting_child; static int connecting_fdpass_sock; static void connect_attempt_discard(void) { + reconnect_blocking_event(); if (connecting_child) { int status= xwaitpid(&connecting_child, "connect"); if (!(WIFEXITED(status) || @@ -1297,8 +1347,9 @@ static void *connchild_event(oop_source *lp, int fd, oop_event e, void *u) { &peer_rd_err, conn); if (r) sysdie("oop_rd_read for peer (fd=%d)",conn->fd); - notice("C%d connected %s", conn->fd, conn->stream ? "streaming" : "plain"); LIST_ADDHEAD(conns, conn); + notice("C%d (now %d) connected %s", + conn->fd, conns.count, conn->stream ? "streaming" : "plain"); connect_attempt_discard(); check_assign_articles(); @@ -1321,6 +1372,7 @@ static void connect_start(void) { assert(!connecting_fdpass_sock); info("starting connection attempt"); + reconnect_blocking_event(); int socks[2]; int r= socketpair(AF_UNIX, SOCK_STREAM, 0, socks); @@ -1417,9 +1469,27 @@ static void connect_start(void) { /*---------- assigning articles to conns, and transmitting ----------*/ +static Article *dequeue_from(int peek, InputFile *ipf) { + if (!ipf) return 0; + if (peek) return LIST_HEAD(ipf->queue); + + Article *art= LIST_REMHEAD(ipf->queue); + if (!art) return 0; + check_reading_pause_resume(ipf); + return art; +} + +static Article *dequeue(int peek) { + Article *art; + art= dequeue_from(peek, flushing_input_file); if (art) return art; + art= dequeue_from(peek, backlog_input_file); if (art) return art; + art= dequeue_from(peek, main_input_file); if (art) return art; + return 0; +} + static void check_assign_articles(void) { for (;;) { - if (!queue.count) + if (!dequeue(1)) break; Conn *walk, *use=0; @@ -1444,14 +1514,14 @@ static void check_assign_articles(void) { if (use) { if (!inqueue) use->since_activity= 0; /* reset idle counter */ while (spare>0) { - Article *art= LIST_REMHEAD(queue); + Article *art= dequeue(0); if (!art) break; LIST_ADDTAIL(use->waiting, art); + recentact_perperiod[recentact_circptr]++; spare--; } conn_maybe_write(use); } else if (allow_connect_start()) { - until_connect= reconnect_delay_periods; connect_start(); break; } else { @@ -1487,6 +1557,111 @@ static void conn_maybe_write(Conn *conn) { } } +/*---------- expiry, flow control and deferral ----------*/ + +/* + * flow control notes + * to ensure articles go away eventually + * separate queue for each input file + * queue expiry + * every period, check head of backlog queue for expiry with SMretrieve + * if too old: discard, and check next article + * also check every backlog article as we read it + * flush expiry + * after too long in SEPARATED/DROPPING ie Separated/Finishing/Dropping + * one-off: eat queued articles from flushing and write them to defer + * one-off: connfail all connections which have any articles from flushing + * newly read articles from flushing go straight to defer + * this should take care of it and get us out of this state + * to avoid filling up ram needlessly + * input control + * limit number of queued articles for each ipf + * pause/resume inputfile tailing + */ + +static void check_reading_pause_resume(InputFile *ipf) { + if (ipf->queue.count >= max_queue_per_ipf) + inputfile_reading_pause(ipf); + else + inputfile_reading_resume(ipf); +} + +static void article_defer(Article *art /* not on a queue */, int whichcount) { + open_defer(); + if (fprintf(defer, "%s %s\n", TokenToText(art->token), art->messageid) <0 + || fflush(defer)) + sysfatal("write to defer file %s",path_defer); + article_done(art, whichcount); +} + +static int article_check_expired(Article *art /* must be queued, not conn */) { + ARTHANDLE *artdata= SMretrieve(art->token, RETR_STAT); + if (artdata) { SMfreearticle(artdata); return 0; } + + LIST_REMOVE(art->ipf->queue, art); + art->missing= 1; + art->ipf->count_nooffer_missing++; + article_done(art,-1); + return 1; +} + +static void inputfile_queue_check_expired(InputFile *ipf) { + if (!ipf) return; + + for (;;) { + Article *art= LIST_HEAD(ipf->queue); + int exp= article_check_expired(art); + if (!exp) break; + } + check_reading_pause_resume(ipf); +} + +static void article_autodefer(InputFile *ipf, Article *art) { + ipf->autodefer++; + article_defer(art,-1); +} + +static int has_article_in(const ArticleList *al, InputFile *ipf) { + Article *art; + for (art=LIST_HEAD(*al); art; art=LIST_NEXT(art)) + if (art->ipf == ipf) return 1; + return 0; +} + +static void autodefer_input_file_articles(InputFile *ipf) { + Article *art; + while ((art= LIST_REMHEAD(ipf->queue))) + article_autodefer(ipf, art); +} + +static void autodefer_input_file(InputFile *ipf) { + ipf->autodefer= 0; + + autodefer_input_file_articles(ipf); + + if (ipf->inprogress) { + Conn *walk; + FOR_CONN(walk) { + if (has_article_in(&walk->waiting, ipf) || + has_article_in(&walk->priority, ipf) || + has_article_in(&walk->sent, ipf)) + walk->quitting= -1; + } + while (ipf->inprogress) { + FOR_CONN(walk) + if (walk->quitting < 0) goto found; + abort(); /* where are they ?? */ + + found: + connfail(walk, "connection is stuck or crawling," + " and we need to finish flush"); + autodefer_input_file_articles(ipf); + } + } + + check_reading_pause_resume(ipf); +} + /*========== article transmission ==========*/ static XmitDetails *xmit_core(Conn *conn, const char *data, int len, @@ -1537,15 +1712,18 @@ static void *conn_write_some_xmits(Conn *conn) { assert(rs > 0); int done; - for (done=0; rs && donexmitu; done++) { + for (done=0; rs; ) { + assert(donexmitu); struct iovec *vp= &conn->xmit[done]; XmitDetails *dp= &conn->xmitd[done]; - if (rs > vp->iov_len) { + if (rs >= vp->iov_len) { rs -= vp->iov_len; - xmit_free(dp); + xmit_free(dp); /* vp->iov_len -= vp->iov_len, etc. */ + done++; } else { vp->iov_base= (char*)vp->iov_base + rs; vp->iov_len -= rs; + break; /* rs -= rs */ } } int newu= conn->xmitu - done; @@ -1584,7 +1762,7 @@ static void conn_make_some_xmits(Conn *conn) { XMIT_LITERAL("\r\n"); xmit_artbody(conn, artdata); } else { - article_done(conn, art, -1); + article_done(art, -1); continue; } } else { @@ -1615,7 +1793,6 @@ static void conn_make_some_xmits(Conn *conn) { } } - /*========== handling responses from peer ==========*/ static const oop_rd_style peer_rd_style= { @@ -1705,8 +1882,9 @@ static void update_nocheck(int accepted) { nocheck= new_nocheck; } -static void article_done(Conn *conn, Article *art, int whichcount) { - if (!art->missing) art->ipf->counts[art->state][whichcount]++; +static void article_done(Article *art, int whichcount) { + if (whichcount>=0 && !art->missing) + art->ipf->counts[art->state][whichcount]++; if (whichcount == RC_accepted) update_nocheck(1); else if (whichcount == RC_unwanted) update_nocheck(0); @@ -1765,19 +1943,16 @@ static void *peer_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, return OOP_CONTINUE; } - int conn_busy= - conn->waiting.count || - conn->priority.count || - conn->sent.count || - conn->xmitu; + int busy= conn_busy(conn); if (conn->quitting) { - if (code!=205 && code!=503) { + if (code!=205 && code!=400) { connfail(conn, "peer gave unexpected response to QUIT: %s", sani); } else { - notice("C%d idle connection closed by us", conn->fd); - assert(!conn_busy); LIST_REMOVE(conns,conn); + notice("C%d (now %d) idle connection closed by us", + conn->fd, conns.count); + assert(!busy); conn_dispose(conn); } return OOP_CONTINUE; @@ -1794,7 +1969,7 @@ static void *peer_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, #define ARTICLE_DEALTWITH(streaming,musthavesent,how) do{ \ code_streaming= (streaming); \ GET_ARTICLE(musthavesent); \ - article_done(conn, art, RC_##how); \ + article_done(art, RC_##how); \ goto dealtwith; \ }while(0) @@ -1806,13 +1981,15 @@ static void *peer_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, switch (code) { - case 400: PEERBADMSG("peer stopped accepting articles"); default: PEERBADMSG("peer sent unexpected message"); - case 503: - if (conn_busy) PEERBADMSG("peer timed us out"); - notice("C%d idle connection closed by peer", conn->fd); + case 400: + if (busy) + PEERBADMSG("peer timed us out or stopped accepting articles"); + LIST_REMOVE(conns,conn); + notice("C%d (now %d) idle connection closed by peer", + conns.count, conn->fd); conn_dispose(conn); return OOP_CONTINUE; @@ -1839,11 +2016,7 @@ static void *peer_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, code_streaming= 1; case 436: /* IHAVE says try later */ GET_ARTICLE(0); - open_defer(); - if (fprintf(defer, "%s %s\n", TokenToText(art->token), art->messageid) <0 - || fflush(defer)) - sysfatal("write to defer file %s",path_defer); - article_done(conn, art, RC_deferred); + article_defer(art, RC_deferred); break; } @@ -1884,6 +2057,8 @@ static InputFile *open_input_file(const char *path) { memset(ipf,0,sizeof(*ipf)); ipf->fd= fd; + ipf->autodefer= -1; + LIST_INIT(ipf->queue); strcpy(ipf->path, path); return ipf; @@ -1982,13 +2157,19 @@ static void *feedfile_got_article(oop_source *lp, oop_read *rd, art->offset= old_offset; art->blanklen= recsz; strcpy(art->messageid, space+1); - LIST_ADDTAIL(queue, art); + LIST_ADDTAIL(ipf->queue, art); + + if (ipf->autodefer >= 0) + article_autodefer(ipf, art); + else if (ipf==backlog_input_file) + article_check_expired(art); if (sms==sm_NORMAL && ipf==main_input_file && ipf->offset >= target_max_feedfile_size) statemc_start_flush("feed file size"); - check_assign_articles(); + check_assign_articles(); /* may destroy conn but that's OK */ + check_reading_pause_resume(ipf); return OOP_CONTINUE; } @@ -1996,7 +2177,16 @@ static void *feedfile_got_article(oop_source *lp, oop_read *rd, static void *tailing_rable_call_time(oop_source *loop, struct timeval tv, void *user) { + /* lifetime of ipf here is OK because destruction will cause + * on_cancel which will cancel this callback */ InputFile *ipf= user; + + if (!ipf->fake_readable) return OOP_CONTINUE; + + /* we just keep calling readable until our caller (oop_rd) + * has called try_read, and try_read has found EOF so given EAGAIN */ + loop->on_time(loop, OOP_TIME_NOW, tailing_rable_call_time, ipf); + return ipf->readable_callback(loop, &ipf->readable, ipf->readable_callback_user); } @@ -2009,9 +2199,10 @@ static void tailing_on_cancel(struct oop_readable *rable) { ipf->readable_callback= 0; } -static void tailing_queue_readable(InputFile *ipf) { - /* lifetime of ipf here is OK because destruction will cause - * on_cancel which will cancel this callback */ +static void tailing_make_readable(InputFile *ipf) { + if (!ipf || !ipf->readable_callback) /* so callers can be naive */ + return; + ipf->fake_readable= 1; loop->on_time(loop, OOP_TIME_NOW, tailing_rable_call_time, ipf); } @@ -2023,8 +2214,7 @@ static int tailing_on_readable(struct oop_readable *rable, ipf->readable_callback= cb; ipf->readable_callback_user= user; filemon_start(ipf); - - tailing_queue_readable(ipf); + tailing_make_readable(ipf); return 0; } @@ -2035,11 +2225,13 @@ static ssize_t tailing_try_read(struct oop_readable *rable, void *buffer, ssize_t r= read(ipf->fd, buffer, length); if (r==-1) { if (errno==EINTR) continue; + ipf->fake_readable= 0; return r; } if (!r) { if (ipf==main_input_file) { errno=EAGAIN; + ipf->fake_readable= 0; return -1; } else if (ipf==flushing_input_file) { assert(ipf->rd); @@ -2050,7 +2242,6 @@ static ssize_t tailing_try_read(struct oop_readable *rable, void *buffer, abort(); } } - tailing_queue_readable(ipf); return r; } } @@ -2114,8 +2305,8 @@ static void *filemon_inotify_readable(oop_source *lp, int fd, die("inotify read %d bytes wanted struct of %d", r, (int)sizeof(iev)); } InputFile *ipf= filemon_inotify_wd2ipf[iev.wd]; - debug("filemon inotify readable read %p wd=%d", ipf, iev.wd); - filemon_callback(ipf); + /*debug("filemon inotify readable read %p wd=%d", ipf, iev.wd);*/ + tailing_make_readable(ipf); } return OOP_CONTINUE; } @@ -2139,7 +2330,7 @@ static void filemon_method_dump_info(FILE *f) { DUMPV("%d",,filemon_inotify_fd); DUMPV("%d",,filemon_inotify_wdmax); for (i=0; ifilemon= 0; } -static void filemon_callback(InputFile *ipf) { - if (ipf && ipf->readable_callback) /* so filepoll() can be naive */ - ipf->readable_callback(loop, &ipf->readable, ipf->readable_callback_user); -} - /*---------- interface to start and stop an input file ----------*/ static const oop_rd_style feedfile_rdstyle= { @@ -2189,6 +2375,24 @@ static const oop_rd_style feedfile_rdstyle= { OOP_RD_SHORTREC_LONG, }; +static void inputfile_reading_resume(InputFile *ipf) { + if (!ipf->rd) return; + if (!ipf->paused) return; + + int r= oop_rd_read(ipf->rd, &feedfile_rdstyle, MAX_LINE_FEEDFILE, + feedfile_got_article,ipf, feedfile_read_err, ipf); + if (r) sysdie("unable start reading feedfile %s",ipf->path); + + ipf->paused= 0; +} + +static void inputfile_reading_pause(InputFile *ipf) { + if (!ipf->rd) return; + if (ipf->paused) return; + oop_rd_cancel(ipf->rd); + ipf->paused= 1; +} + static void inputfile_reading_start(InputFile *ipf) { assert(!ipf->rd); ipf->readable.on_readable= tailing_on_readable; @@ -2203,14 +2407,13 @@ static void inputfile_reading_start(InputFile *ipf) { ipf->rd= oop_rd_new(loop, &ipf->readable, 0,0); assert(ipf->rd); - int r= oop_rd_read(ipf->rd, &feedfile_rdstyle, MAX_LINE_FEEDFILE, - feedfile_got_article,ipf, feedfile_read_err, ipf); - if (r) sysdie("unable start reading feedfile %s",ipf->path); + ipf->paused= 1; + inputfile_reading_resume(ipf); } static void inputfile_reading_stop(InputFile *ipf) { assert(ipf->rd); - oop_rd_cancel(ipf->rd); + inputfile_reading_pause(ipf); oop_rd_delete(ipf->rd); ipf->rd= 0; assert(!ipf->filemon); /* we shouldn't be monitoring it now */ @@ -2421,7 +2624,8 @@ static void statemc_init(void) { flushing_input_file= main_input_file; main_input_file= open_input_file(feedfile); if (!main_input_file) die("feedfile vanished during startup"); - SMS(SEPARATED, 0, "found both old and current feed files"); + SMS(SEPARATED, max_separated_periods, + "found both old and current feed files"); } else { debug("startup: F exists, D ENOENT => Normal"); InputFile *file_f= open_input_file(feedfile); @@ -2439,7 +2643,7 @@ static void statemc_start_flush(const char *why) { /* Normal => Flushing */ why, (unsigned long)(main_input_file ? main_input_file->offset : 0), (unsigned long)target_max_feedfile_size, - sm_period_counter); + until_flush); int r= link(feedfile, path_flushing); if (r) sysfatal("link feedfile %s to flushing file %s", @@ -2452,26 +2656,37 @@ static void statemc_start_flush(const char *why) { /* Normal => Flushing */ spawn_inndcomm_flush(why); /* => Flushing FLUSHING */ } -static int trigger_flush_ok(void) { /* => Flushing,FLUSHING, ret 1; or ret 0 */ +static int trigger_flush_ok(const char *why) { switch (sms) { + case sm_NORMAL: - statemc_start_flush("periodic"); /* Normal => Flushing; => FLUSHING */ - return 1; + statemc_start_flush(why ? why : "periodic"); + return 1; /* Normal => Flushing; => FLUSHING */ + case sm_FLUSHFAILED: - spawn_inndcomm_flush("retry"); /* Moved => Flushing; => FLUSHING */ + spawn_inndcomm_flush(why ? why : "retry"); + return 1; /* Moved => Flushing; => FLUSHING */ + + case sm_SEPARATED: + case sm_DROPPING: + warn("abandoning old feedfile after flush (%s), autodeferring", + why ? why : "took too long to complete"); + assert(flushing_input_file); + autodefer_input_file(flushing_input_file); return 1; + default: return 0; } } static void statemc_period_poll(void) { - if (!sm_period_counter) return; - sm_period_counter--; - assert(sm_period_counter>=0); + if (!until_flush) return; + until_flush--; + assert(until_flush>=0); - if (sm_period_counter) return; - int ok= trigger_flush_ok(); + if (until_flush) return; + int ok= trigger_flush_ok(0); assert(ok); } @@ -2495,13 +2710,17 @@ static void notice_processed(InputFile *ipf, int completed, char *inprog= completed ? xasprintf("%s","") /* GCC produces a stupid warning for printf("") ! */ : xasprintf(" inprogress=%ld", ipf->inprogress); + char *autodefer= ipf->autodefer >= 0 + ? xasprintf(" autodeferred=%ld", ipf->autodefer) + : xasprintf("%s",""); - info("%s %s%s read=%d (+bl=%d,+err=%d)%s" - " offered=%d (ch=%d,nc=%d) accepted=%d (ch=%d,nc=%d)" + info("%s %s%s read=%d (+bl=%d,+err=%d)%s%s" + " missing=%d offered=%d (ch=%d,nc=%d) accepted=%d (ch=%d,nc=%d)" RESULT_COUNTS(RCI_NOTHING, RCI_TRIPLE_FMT) , completed?"completed":"processed", what, spec, - ipf->readcount_ok, ipf->readcount_blank, ipf->readcount_err, inprog, + ipf->readcount_ok, ipf->readcount_blank, ipf->readcount_err, + inprog, autodefer, ipf->count_nooffer_missing, CNT(Unchecked,sent) + CNT(Unsolicited,sent) , CNT(Unchecked,sent), CNT(Unsolicited,sent), CNT(Wanted,accepted) + CNT(Unsolicited,accepted) @@ -2510,6 +2729,7 @@ static void notice_processed(InputFile *ipf, int completed, ); free(inprog); + free(autodefer); #undef CNT } @@ -2559,7 +2779,7 @@ static void statemc_check_flushing_done(void) { notice("flush complete"); SMS(NORMAL, spontaneous_flush_periods, "flush complete"); } else if (sms==sm_DROPPING) { - SMS(DROPPED, 0, "old flush complete"); + SMS(DROPPED, max_separated_periods, "old flush complete"); search_backlog_file(); notice("feed dropped, but will continue until backlog is finished"); } @@ -2580,7 +2800,7 @@ static void queue_check_input_done(void) { static void statemc_setstate(StateMachineState newsms, int periods, const char *forlog, const char *why) { sms= newsms; - sm_period_counter= periods; + until_flush= periods; const char *xtra= ""; switch (sms) { @@ -2749,11 +2969,11 @@ static void search_backlog_file(void) { preterminate(); notice("feed dropped and our work is complete"); - int r= unlink(path_control); + int r= unlink(path_cli); if (r && errno!=ENOENT) - syswarn("failed to remove control symlink for old feed"); + syswarn("failed to unlink cli socket for old feed"); - xunlink(path_lock, "lockfile for old feed"); + xunlink(path_lock, "lockfile for old feed"); exit(4); } until_backlog_nextscan= backlog_spontrescan_periods; @@ -2885,7 +3105,7 @@ static void *inndcomm_event(oop_source *lp, int fd, oop_event e, void *u) { case INNDCOMMCHILD_ESTATUS_NONESUCH: notice("feed has been dropped by innd, finishing up"); flushing_input_file= main_input_file; - tailing_queue_readable(flushing_input_file); + tailing_make_readable(flushing_input_file); /* we probably previously returned EAGAIN from our fake read method * when in fact we were at EOF, so signal another readable event * so we actually see the EOF */ @@ -2893,7 +3113,8 @@ static void *inndcomm_event(oop_source *lp, int fd, oop_event e, void *u) { main_input_file= 0; if (flushing_input_file) { - SMS(DROPPING, 0, "feed dropped by innd, but must finish last flush"); + SMS(DROPPING, max_separated_periods, + "feed dropped by innd, but must finish last flush"); } else { close_defer(); SMS(DROPPED, 0, "feed dropped by innd"); @@ -2904,17 +3125,19 @@ static void *inndcomm_event(oop_source *lp, int fd, oop_event e, void *u) { case 0: /* as above */ flushing_input_file= main_input_file; - tailing_queue_readable(flushing_input_file); + tailing_make_readable(flushing_input_file); main_input_file= open_input_file(feedfile); if (!main_input_file) - die("flush succeeded but feedfile %s does not exist!", feedfile); + die("flush succeeded but feedfile %s does not exist!" + " (this probably means feedfile does not correspond" + " to site %s in newsfeeds)", feedfile, sitename); if (flushing_input_file) { - SMS(SEPARATED, spontaneous_flush_periods, "recovery flush complete"); + SMS(SEPARATED, max_separated_periods, "flush complete"); } else { close_defer(); - SMS(NORMAL, spontaneous_flush_periods, "flush complete"); + SMS(NORMAL, spontaneous_flush_periods, "recovery flush complete"); } return OOP_CONTINUE; @@ -3053,8 +3276,8 @@ static void every(int interval, int fixed_rate, void (*f)(void)) { } static void filepoll(void) { - filemon_callback(main_input_file); - filemon_callback(flushing_input_file); + tailing_make_readable(main_input_file); + tailing_make_readable(flushing_input_file); } static char *debug_report_ipf(InputFile *ipf) { @@ -3063,12 +3286,13 @@ static char *debug_report_ipf(InputFile *ipf) { const char *slash= strrchr(ipf->path,'/'); const char *path= slash ? slash+1 : ipf->path; - return xasprintf("%p/%s:ip=%ld,off=%ld,fd=%d%s%s", + return xasprintf("%p/%s:queue=%d,ip=%ld,autodef=%ld,off=%ld,fd=%d%s%s%s", ipf, path, - ipf->inprogress, (long)ipf->offset, - ipf->fd, + ipf->queue.count, ipf->inprogress, ipf->autodefer, + (long)ipf->offset, ipf->fd, ipf->rd ? "" : ",!rd", - ipf->skippinglong ? "*skiplong" : ""); + ipf->skippinglong ? "*skiplong" : "", + ipf->rd && ipf->paused ? "*paused" : ""); } static void period(void) { @@ -3077,13 +3301,12 @@ static void period(void) { char *dipf_backlog= debug_report_ipf(backlog_input_file); debug("PERIOD" - " sms=%s[%d] conns=%d queue=%d until_connect=%d" - " input_files main:%s flushing:%s backlog:%s" + " sms=%s[%d] conns=%d until_connect=%d" + " input_files main:%s flushing:%s backlog:%s[%d]" " children connecting=%ld inndcomm=%ld" , - sms_names[sms], sm_period_counter, - conns.count, queue.count, until_connect, - dipf_main, dipf_flushing, dipf_backlog, + sms_names[sms], until_flush, conns.count, until_connect, + dipf_main, dipf_flushing, dipf_backlog, until_backlog_nextscan, (long)connecting_child, (long)inndcomm_child ); @@ -3093,6 +3316,7 @@ static void period(void) { if (until_connect) until_connect--; + inputfile_queue_check_expired(backlog_input_file); poll_backlog_file(); if (!backlog_input_file) close_defer(); /* want to start on a new backlog */ statemc_period_poll(); @@ -3103,7 +3327,7 @@ static void period(void) { /*========== dumping state ==========*/ -static void dump_article_list(FILE *f, const ControlCommand *c, +static void dump_article_list(FILE *f, const CliCommand *c, const ArticleList *al) { fprintf(f, " count=%d\n", al->count); if (!c->xval) return; @@ -3120,7 +3344,8 @@ static void dump_article_list(FILE *f, const ControlCommand *c, } } -static void dump_input_file(FILE *f, InputFile *ipf, const char *wh) { +static void dump_input_file(FILE *f, const CliCommand *c, + InputFile *ipf, const char *wh) { char *dipf= debug_report_ipf(ipf); fprintf(f,"input %s %s", wh, dipf); free(dipf); @@ -3129,6 +3354,7 @@ static void dump_input_file(FILE *f, InputFile *ipf, const char *wh) { DUMPV("%d", ipf->,readcount_ok); DUMPV("%d", ipf->,readcount_blank); DUMPV("%d", ipf->,readcount_err); + DUMPV("%d", ipf->,count_nooffer_missing); } fprintf(f,"\n"); if (ipf) { @@ -3141,6 +3367,8 @@ static void dump_input_file(FILE *f, InputFile *ipf, const char *wh) { wh, *statename RESULT_COUNTS(RC_DUMP_VAL,RC_DUMP_VAL)); } + fprintf(f,"input %s queue", wh); + dump_article_list(f,c,&ipf->queue); } } @@ -3152,7 +3380,7 @@ CCMD(dump) { fprintf(f,"general"); DUMPV("%s", sms_names,[sms]); - DUMPV("%d", ,sm_period_counter); + DUMPV("%d", ,until_flush); DUMPV("%ld", (long),self_pid); DUMPV("%p", , defer); DUMPV("%d", , until_connect); @@ -3167,15 +3395,15 @@ CCMD(dump) { fprintf(f,"special"); DUMPV("%ld", (long),connecting_child); DUMPV("%d", , connecting_fdpass_sock); - DUMPV("%d", , control_master); + DUMPV("%d", , cli_master); fprintf(f,"\n"); fprintf(f,"filemon "); filemon_method_dump_info(f); - dump_input_file(f, main_input_file, "main" ); - dump_input_file(f, flushing_input_file, "flushing"); - dump_input_file(f, backlog_input_file, "backlog" ); + dump_input_file(f,c, main_input_file, "main" ); + dump_input_file(f,c, flushing_input_file, "flushing"); + dump_input_file(f,c, backlog_input_file, "backlog" ); fprintf(f,"conns count=%d\n", conns.count); @@ -3197,7 +3425,6 @@ CCMD(dump) { const struct iovec *iv= &conn->xmit[i]; const XmitDetails *xd= &conn->xmitd[i]; char *dinfo; - long diff; switch (xd->kind) { case xk_Const: dinfo= xasprintf("Const"); break; case xk_Artdata: dinfo= xasprintf("A%p", xd->info.sm_art); break; @@ -3210,13 +3437,12 @@ CCMD(dump) { } } - fprintf(f,"queue"); dump_article_list(f,c,&queue); - fprintf(f,"paths"); + DUMPV("%s", , feedfile); + DUMPV("%s", , path_cli); DUMPV("%s", , path_lock); DUMPV("%s", , path_flushing); DUMPV("%s", , path_defer); - DUMPV("%s", , path_control); DUMPV("%s", , path_dump); DUMPV("%s", , globpat_backlog); fprintf(f,"\n"); @@ -3400,11 +3626,13 @@ static const Option innduct_options[]= { {0,"no-filemon", 0, &try_filemon, op_setint, 0 }, {'C',"inndconf", "F", &inndconffile, op_string }, {'P',"port", "PORT", &port, op_integer }, -{0,"ctrl-sock-dir", 0, &realsockdir, op_string }, +{0,"chdir", "DIR", &path_run, op_string }, +{0,"cli", "DIR/|PATH", &path_cli, op_string }, {0,"help", 0, 0, help }, {0,"max-connections", "N", &max_connections, op_integer }, {0,"max-queue-per-conn", "N", &max_queue_per_conn, op_integer }, +{0,"max-queue-per-file", "N", &max_queue_per_ipf, op_integer }, {0,"feedfile-flush-size","BYTES", &target_max_feedfile_size, op_integer }, {0,"period-interval", "TIME", &period_seconds, op_seconds }, @@ -3420,7 +3648,10 @@ static const Option innduct_options[]= { {0,"earliest-deferred-retry","PERIOD", &backlog_retry_minperiods, op_seconds }, {0,"backlog-rescan-interval","PERIOD",&backlog_spontrescan_periods,op_seconds}, {0,"max-flush-interval", "PERIOD", &spontaneous_flush_periods,op_seconds }, +{0,"flush-finish-timeout", "PERIOD", &max_separated_periods, op_seconds }, {0,"idle-timeout", "PERIOD", &need_activity_periods, op_seconds }, +{0,"low-volume-thresh", "PERIOD", &recentact_thresh, op_integer }, +{0,"low-volume-window", "PERIOD", &recentact_periods, op_seconds }, {0,"max-bad-input-data-ratio","PERCENT", &max_bad_data_ratio, op_double }, {0,"max-bad-input-data-init", "PERCENT", &max_bad_data_initial, op_integer }, @@ -3448,6 +3679,12 @@ static void convert_to_periods_rndup(int *store) { *store /= period_seconds; } +static int path_ends_slash(const char *specified) { + int l= strlen(specified); + assert(l); + return specified[l-1] == '/'; +} + int main(int argc, char **argv) { if (!argv[1]) { printusage(stderr); @@ -3460,7 +3697,10 @@ int main(int argc, char **argv) { sitename= *argv++; if (!sitename) badusage("need site name argument"); - remote_host= *argv++; + + if (*argv) remote_host= *argv++; + else remote_host= sitename; + if (*argv) badusage("too many non-option arguments"); /* defaults */ @@ -3483,19 +3723,36 @@ int main(int argc, char **argv) { convert_to_periods_rndup(&backlog_retry_minperiods); convert_to_periods_rndup(&backlog_spontrescan_periods); convert_to_periods_rndup(&spontaneous_flush_periods); + convert_to_periods_rndup(&max_separated_periods); convert_to_periods_rndup(&need_activity_periods); + convert_to_periods_rndup(&recentact_periods); if (max_bad_data_ratio < 0 || max_bad_data_ratio > 100) badusage("bad input data ratio must be between 0..100"); max_bad_data_ratio *= 0.01; - if (!feedfile) { - feedfile= xasprintf("%s/%s",innconf->pathoutgoing,sitename); - } else if (!feedfile[0]) { - badusage("feed filename must be nonempty"); - } else if (feedfile[strlen(feedfile)-1]=='/') { - feedfile= xasprintf("%s%s",feedfile,sitename); + if (!path_run) + path_run= innconf->pathrun; + + if (!feedfile) feedfile= sitename; + if (!feedfile[0]) badusage("feed filename, if specified, must be nonempty"); + if (path_ends_slash(feedfile)) + feedfile= xasprintf("%s%s", feedfile, sitename); + if (feedfile[0] != '/') + feedfile= xasprintf("%s/%s", innconf->pathoutgoing, feedfile); + + if (!path_cli) { + path_cli_dir= "innduct"; + } else if (!path_cli[0] || !strcmp(path_cli,"none")) { + path_cli= 0; /* ok, don't then */ + } else if (path_ends_slash(path_cli)) { + path_cli_dir= xasprintf("%.*s", strlen(path_cli)-1, path_cli); } + if (path_cli_dir) + path_cli= xasprintf("%s/%s", path_cli_dir, sitename); + + if (max_queue_per_ipf<0) + max_queue_per_ipf= max_queue_per_conn * 2; const char *feedfile_forbidden= "?*[~#"; int c; @@ -3503,12 +3760,18 @@ int main(int argc, char **argv) { if (strchr(feedfile, c)) badusage("feed filename may not contain metacharacter %c",c); + int i; + recentact_perperiod= xcalloc(sizeof(*recentact_perperiod),recentact_periods); + for (i=0; i