X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?a=blobdiff_plain;f=backends%2Finnduct.c;h=c0404fc011b9a0b0715055664c635170ee6812d7;hb=8637044e5d6c13c4ddf8dc83440a59b060b25a3d;hp=f7ef3acddbd990896415674212923224a97358c5;hpb=e579bdbfd0a065fc822a5353b5d89bf289228efb;p=inn-innduct.git diff --git a/backends/innduct.c b/backends/innduct.c index f7ef3ac..c0404fc 100644 --- a/backends/innduct.c +++ b/backends/innduct.c @@ -1,13 +1,26 @@ /* - * todo - * - skipping_long offset calculation is wrong - * - reset signals TERM and INT (and HUP) in children + * innduct + * tailing reliable realtime streaming feeder for inn * - * - manpage: document control master stuff - * - admin-initiated flush + * Copyright (C) 2010 Ian Jackson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . * - * debugging rune: - * build-lfs/backends/innduct --connection-timeout=30 --no-daemon -C ../inn.conf -f `pwd`/fee sit localhost + * (I believe that when you compile and link this as part of the inn2 + * build, with the Makefile runes I have provided, all the libraries + * and files which end up included in innduct are licence-compatible + * with GPLv3. If not then please let me know. -Ian Jackson.) */ /* @@ -209,6 +222,7 @@ perl -ne 'print if m/-8\<-/..m/-\>8-/; print "\f" if m/-\^L-/' backends/innduct. #define VA va_list al; va_start(al,fmt) #define PRINTF(f,a) __attribute__((__format__(printf,f,a))) #define NORET_PRINTF(f,a) __attribute__((__noreturn__,__format__(printf,f,a))) +#define NORET __attribute__((__noreturn__)) #define NEW(ptr) ((ptr)= zxmalloc(sizeof(*(ptr)))) #define NEW_DECL(type,ptr) type ptr = zxmalloc(sizeof(*(ptr))) @@ -299,11 +313,13 @@ static void statemc_setstate(StateMachineState newsms, int periods, static void statemc_start_flush(const char *why); /* Normal => Flushing */ static void spawn_inndcomm_flush(const char *why); /* Moved => Flushing */ +static int trigger_flush_ok(void); /* => Flushing,FLUSHING, ret 1; or ret 0 */ -static void article_done(Conn *conn, Article *art, int whichcount); +static void article_done(Article *art, int whichcount); static void check_assign_articles(void); static void queue_check_input_done(void); +static void check_reading_pause_resume(InputFile *ipf); static void statemc_check_flushing_done(void); static void statemc_check_backlog_done(void); @@ -315,11 +331,14 @@ static void open_defer(void); static void close_defer(void); static void search_backlog_file(void); static void preterminate(void); -static void raise_default(int signo); +static void raise_default(int signo) NORET; static char *debug_report_ipf(InputFile *ipf); static void inputfile_reading_start(InputFile *ipf); static void inputfile_reading_stop(InputFile *ipf); +static void inputfile_reading_pause(InputFile *ipf); +static void inputfile_reading_resume(InputFile *ipf); + /* pause and resume are idempotent, and no-op if not done _reading_start */ static void filemon_start(InputFile *ipf); static void filemon_stop(InputFile *ipf); @@ -331,11 +350,12 @@ static void connfail(Conn *conn, const char *fmt, ...) PRINTF(2,3); static const oop_rd_style peer_rd_style; static oop_rd_call peer_rd_err, peer_rd_ok; + /*----- configuration options -----*/ /* when changing defaults, remember to update the manpage */ static const char *sitename, *remote_host; -static const char *feedfile, *realsockdir="/tmp/innduct.control"; +static const char *feedfile, *path_control; static int quiet_multiple=0; static int become_daemon=1, try_filemon=1; static int try_stream=1; @@ -347,6 +367,7 @@ static int max_queue_per_conn=200; static int target_max_feedfile_size=100000; static int period_seconds=60; static int filepoll_seconds=5; +static int max_queue_per_ipf=-1; static int connection_setup_timeout=200; static int inndcomm_flush_timeout=100; @@ -360,6 +381,7 @@ static int flushfail_retry_periods=1000; static int backlog_retry_minperiods=50; static int backlog_spontrescan_periods=300; static int spontaneous_flush_periods=100000; +static int max_separated_periods=2000; static int need_activity_periods=1000; static double max_bad_data_ratio= 1; /* conv'd from percentage by main */ @@ -410,13 +432,12 @@ typedef enum { #define CONNIOVS 128 typedef enum { - xk_Malloc, xk_Const, xk_Artdata + xk_Const, xk_Artdata } XmitKind; struct XmitDetails { XmitKind kind; union { - char *malloc_tofree; ARTHANDLE *sm_art; } info; }; @@ -434,12 +455,15 @@ struct InputFile { Filemon_Perfile *filemon; oop_read *rd; /* non-0: reading; 0: constructing, or had EOF */ - long inprogress; /* no. of articles read but not processed */ off_t offset; - int skippinglong; + int skippinglong, paused; + + ArticleList queue; + long inprogress; /* includes queue.count and also articles in conns */ + long autodefer; /* -1 means not doing autodefer */ int counts[art_MaxState][RCI_max]; - int readcount_ok, readcount_blank, readcount_err; + int readcount_ok, readcount_blank, readcount_err, count_nooffer_missing; char path[]; }; @@ -493,15 +517,13 @@ struct Conn { /* main initialises */ static oop_source *loop; static ConnList conns; -static ArticleList queue; -static char *path_lock, *path_flushing, *path_defer; -static char *path_control, *path_dump; +static char *path_lock, *path_flushing, *path_defer, *path_dump; static char *globpat_backlog; static pid_t self_pid; /* statemc_init initialises */ static StateMachineState sms; -static int sm_period_counter; +static int until_flush; static InputFile *main_input_file, *flushing_input_file, *backlog_input_file; static FILE *defer; @@ -679,9 +701,16 @@ static time_t xtime(void) { return now; } -static void xsigaction(int s, const struct sigaction *sa) { - int r= sigaction(s,sa,0); - if (r) sysdie("sigaction failed for \"%s\"", strsignal(s)); +static void xsigaction(int signo, const struct sigaction *sa) { + int r= sigaction(signo,sa,0); + if (r) sysdie("sigaction failed for \"%s\"", strsignal(signo)); +} + +static void xsigsetdefault(int signo) { + struct sigaction sa; + memset(&sa,0,sizeof(sa)); + sa.sa_handler= SIG_DFL; + xsigaction(signo,&sa); } static void xgettimeofday(struct timeval *tv_r) { @@ -811,13 +840,14 @@ CCMD(help) { const ControlCommand *ccmd; for (ccmd=control_commands; ccmd->cmd; ccmd++) fprintf(cc->out, " %s\n", ccmd->cmd); + fputs("NB: permissible arguments are not shown above." + " Not all commands listed are safe. See innduct(8).\n", cc->out); } -CCMD(period) { period(); } -CCMD(setintarg) { *(int*)c->xdata= atoi(arg); } -CCMD(setint) { *(int*)c->xdata= c->xval; } -CCMD(setint_period) { *(int*)c->xdata= c->xval; period(); } -CCMD(dump); +CCMD(flush) { + int ok= trigger_flush_ok(); + if (!ok) fprintf(cc->out,"already flushing (state is %s)\n", sms_names[sms]); +} CCMD(stop) { preterminate(); @@ -826,19 +856,29 @@ CCMD(stop) { abort(); } +CCMD(dump); + +/* messing with our head: */ +CCMD(period) { period(); } +CCMD(setintarg) { *(int*)c->xdata= atoi(arg); } +CCMD(setint) { *(int*)c->xdata= c->xval; } +CCMD(setint_period) { *(int*)c->xdata= c->xval; period(); } + static const ControlCommand control_commands[]= { { "h", ccmd_help }, - { "p", ccmd_period }, + { "flush", ccmd_flush }, { "stop", ccmd_stop }, { "dump q", ccmd_dump, 0,0 }, { "dump a", ccmd_dump, 0,1 }, + { "p", ccmd_period }, + #define POKES(cmd,func) \ - { cmd "sm", func, &sm_period_counter, 1 }, \ + { cmd "flush", func, &until_flush, 1 }, \ { cmd "conn", func, &until_connect, 0 }, \ { cmd "blscan", func, &until_backlog_nextscan, 0 }, -POKES("prod ", ccmd_setint_period) POKES("next ", ccmd_setint) +POKES("prod ", ccmd_setint_period) { "pretend flush", ccmd_setintarg, &simulate_flush }, { "wedge blscan", ccmd_setint, &until_backlog_nextscan, -1 }, @@ -959,8 +999,6 @@ static void *control_master_readable(oop_source *lp, int master, }while(0) static void control_init(void) { - char *real=0; - union { struct sockaddr sa; struct sockaddr_un un; @@ -969,66 +1007,24 @@ static void control_init(void) { memset(&sa,0,sizeof(sa)); int maxlen= sizeof(sa.un.sun_path); - int reallen= readlink(path_control, sa.un.sun_path, maxlen); - if (reallen<0) { - if (errno != ENOENT) - NOCONTROL("readlink control socket symlink path %s", path_control); - } - if (reallen >= maxlen) { - debug("control socket symlink path too long (r=%d)",reallen); - xunlink(path_control, "old (overlong) control socket symlink"); - reallen= -1; - } - - if (reallen<0) { - struct stat stab; - int r= lstat(realsockdir,&stab); - if (r) { - if (errno != ENOENT) NOCONTROL("lstat real socket dir %s", realsockdir); - - r= mkdir(realsockdir, 0700); - if (r) NOCONTROL("mkdir real socket dir %s", realsockdir); - - } else { - uid_t self= geteuid(); - if (!S_ISDIR(stab.st_mode) || - stab.st_uid != self || - stab.st_mode & 0007) { - warn("no control socket, because real socket directory" - " is somehow wrong (ISDIR=%d, uid=%lu (exp.%lu), mode %lo)", - !!S_ISDIR(stab.st_mode), - (unsigned long)stab.st_uid, (unsigned long)self, - (unsigned long)stab.st_mode & 0777UL); - goto nocontrol; - } - } - - real= xasprintf("%s/s%lx.%lx", realsockdir, - (unsigned long)xtime(), (unsigned long)self_pid); - int reallen= strlen(real); + int pathlen= strlen(path_control); + if (pathlen > maxlen) + NOCONTROL("control socket path %s too long (%d>%d)", + path_control, pathlen, maxlen); - if (reallen >= maxlen) { - warn("no control socket, because tmpnam gave overly-long path" - " %s", real); - goto nocontrol; - } - r= symlink(real, path_control); - if (r) NOCONTROL("make control socket path %s a symlink to real" - " socket path %s", path_control, real); - memcpy(sa.un.sun_path, real, reallen); - } - - int r= unlink(sa.un.sun_path); + int r= unlink(path_control); if (r && errno!=ENOENT) - NOCONTROL("remove old real socket %s", sa.un.sun_path); + NOCONTROL("remove old control socket %s", path_control); control_master= socket(PF_UNIX, SOCK_STREAM, 0); if (control_master<0) NOCONTROL("create new control socket"); - sa.un.sun_family= AF_UNIX; int sl= strlen(sa.un.sun_path) + offsetof(struct sockaddr_un, sun_path); + sa.un.sun_family= AF_UNIX; + memcpy(sa.un.sun_path, path_control, pathlen); + r= bind(control_master, &sa.sa, sl); - if (r) NOCONTROL("bind to real socket path %s", sa.un.sun_path); + if (r) NOCONTROL("bind to control socket path %s", sa.un.sun_path); r= listen(control_master, 5); if (r) NOCONTROL("listen"); @@ -1036,12 +1032,11 @@ static void control_init(void) { xsetnonblock(control_master, 1); loop->on_fd(loop, control_master, OOP_READ, control_master_readable, 0); - info("control socket ok, real path %s", sa.un.sun_path); + info("control socket ok, listening on %s", path_control); return; nocontrol: - free(real); xclose_perhaps(&control_master, "control master",0); return; } @@ -1088,12 +1083,18 @@ static void vconnfail(Conn *conn, const char *fmt, va_list al) { memset(requeue,0,sizeof(requeue)); Article *art; - while ((art= LIST_REMHEAD(conn->priority))) LIST_ADDTAIL(queue, art); - while ((art= LIST_REMHEAD(conn->waiting))) LIST_ADDTAIL(queue, art); + + while ((art= LIST_REMHEAD(conn->priority))) + LIST_ADDTAIL(art->ipf->queue, art); + + while ((art= LIST_REMHEAD(conn->waiting))) + LIST_ADDTAIL(art->ipf->queue, art); + while ((art= LIST_REMHEAD(conn->sent))) { requeue[art->state]++; if (art->state==art_Unsolicited) art->state= art_Unchecked; - LIST_ADDTAIL(queue,art); + LIST_ADDTAIL(art->ipf->queue,art); + check_reading_pause_resume(art->ipf); } int i; @@ -1403,9 +1404,27 @@ static void connect_start(void) { /*---------- assigning articles to conns, and transmitting ----------*/ +static Article *dequeue_from(int peek, InputFile *ipf) { + if (!ipf) return 0; + if (peek) return LIST_HEAD(ipf->queue); + + Article *art= LIST_REMHEAD(ipf->queue); + if (!art) return 0; + check_reading_pause_resume(ipf); + return art; +} + +static Article *dequeue(int peek) { + Article *art; + art= dequeue_from(peek, flushing_input_file); if (art) return art; + art= dequeue_from(peek, backlog_input_file); if (art) return art; + art= dequeue_from(peek, main_input_file); if (art) return art; + return 0; +} + static void check_assign_articles(void) { for (;;) { - if (!queue.count) + if (!dequeue(1)) break; Conn *walk, *use=0; @@ -1430,7 +1449,7 @@ static void check_assign_articles(void) { if (use) { if (!inqueue) use->since_activity= 0; /* reset idle counter */ while (spare>0) { - Article *art= LIST_REMHEAD(queue); + Article *art= dequeue(0); if (!art) break; LIST_ADDTAIL(use->waiting, art); spare--; @@ -1473,6 +1492,111 @@ static void conn_maybe_write(Conn *conn) { } } +/*---------- expiry, flow control and deferral ----------*/ + +/* + * flow control notes + * to ensure articles go away eventually + * separate queue for each input file + * queue expiry + * every period, check head of backlog queue for expiry with SMretrieve + * if too old: discard, and check next article + * also check every backlog article as we read it + * flush expiry + * after too long in SEPARATED/DROPPING ie Separated/Finishing/Dropping + * one-off: eat queued articles from flushing and write them to defer + * one-off: connfail all connections which have any articles from flushing + * newly read articles from flushing go straight to defer + * this should take care of it and get us out of this state + * to avoid filling up ram needlessly + * input control + * limit number of queued articles for each ipf + * pause/resume inputfile tailing + */ + +static void check_reading_pause_resume(InputFile *ipf) { + if (ipf->queue.count >= max_queue_per_ipf) + inputfile_reading_pause(ipf); + else + inputfile_reading_resume(ipf); +} + +static void article_defer(Article *art /* not on a queue */, int whichcount) { + open_defer(); + if (fprintf(defer, "%s %s\n", TokenToText(art->token), art->messageid) <0 + || fflush(defer)) + sysfatal("write to defer file %s",path_defer); + article_done(art, whichcount); +} + +static int article_check_expired(Article *art /* must be queued, not conn */) { + ARTHANDLE *artdata= SMretrieve(art->token, RETR_STAT); + if (artdata) { SMfreearticle(artdata); return 0; } + + LIST_REMOVE(art->ipf->queue, art); + art->missing= 1; + art->ipf->count_nooffer_missing++; + article_done(art,-1); + return 1; +} + +static void inputfile_queue_check_expired(InputFile *ipf) { + if (!ipf) return; + + for (;;) { + Article *art= LIST_HEAD(ipf->queue); + int exp= article_check_expired(art); + if (!exp) break; + } + check_reading_pause_resume(ipf); +} + +static void article_autodefer(InputFile *ipf, Article *art) { + ipf->autodefer++; + article_defer(art,-1); +} + +static int has_article_in(const ArticleList *al, InputFile *ipf) { + Article *art; + for (art=LIST_HEAD(*al); art; art=LIST_NEXT(art)) + if (art->ipf == ipf) return 1; + return 0; +} + +static void autodefer_input_file_articles(InputFile *ipf) { + Article *art; + while ((art= LIST_REMHEAD(ipf->queue))) + article_autodefer(ipf, art); +} + +static void autodefer_input_file(InputFile *ipf) { + ipf->autodefer= 0; + + autodefer_input_file_articles(ipf); + + if (ipf->inprogress) { + Conn *walk; + FOR_CONN(walk) { + if (has_article_in(&walk->waiting, ipf) || + has_article_in(&walk->priority, ipf) || + has_article_in(&walk->sent, ipf)) + walk->quitting= -1; + } + while (ipf->inprogress) { + FOR_CONN(walk) + if (walk->quitting < 0) goto found; + abort(); /* where are they ?? */ + + found: + connfail(walk, "connection is stuck or crawling," + " and we need to finish flush"); + autodefer_input_file_articles(ipf); + } + } + + check_reading_pause_resume(ipf); +} + /*========== article transmission ==========*/ static XmitDetails *xmit_core(Conn *conn, const char *data, int len, @@ -1497,7 +1621,6 @@ static void xmit_artbody(Conn *conn, ARTHANDLE *ah /* consumed */) { static void xmit_free(XmitDetails *d) { switch (d->kind) { - case xk_Malloc: free(d->info.malloc_tofree); break; case xk_Artdata: SMfreearticle(d->info.sm_art); break; case xk_Const: break; default: abort(); @@ -1571,7 +1694,7 @@ static void conn_make_some_xmits(Conn *conn) { XMIT_LITERAL("\r\n"); xmit_artbody(conn, artdata); } else { - article_done(conn, art, -1); + article_done(art, -1); continue; } } else { @@ -1602,7 +1725,6 @@ static void conn_make_some_xmits(Conn *conn) { } } - /*========== handling responses from peer ==========*/ static const oop_rd_style peer_rd_style= { @@ -1692,8 +1814,9 @@ static void update_nocheck(int accepted) { nocheck= new_nocheck; } -static void article_done(Conn *conn, Article *art, int whichcount) { - if (!art->missing) art->ipf->counts[art->state][whichcount]++; +static void article_done(Article *art, int whichcount) { + if (whichcount>=0 && !art->missing) + art->ipf->counts[art->state][whichcount]++; if (whichcount == RC_accepted) update_nocheck(1); else if (whichcount == RC_unwanted) update_nocheck(0); @@ -1781,7 +1904,7 @@ static void *peer_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, #define ARTICLE_DEALTWITH(streaming,musthavesent,how) do{ \ code_streaming= (streaming); \ GET_ARTICLE(musthavesent); \ - article_done(conn, art, RC_##how); \ + article_done(art, RC_##how); \ goto dealtwith; \ }while(0) @@ -1793,11 +1916,12 @@ static void *peer_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, switch (code) { - case 400: PEERBADMSG("peer stopped accepting articles"); default: PEERBADMSG("peer sent unexpected message"); - case 503: - if (conn_busy) PEERBADMSG("peer timed us out"); + case 400: + if (conn_busy) + PEERBADMSG("peer timed us out or stopped accepting articles"); + notice("C%d idle connection closed by peer", conn->fd); LIST_REMOVE(conns,conn); conn_dispose(conn); @@ -1826,11 +1950,7 @@ static void *peer_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev, code_streaming= 1; case 436: /* IHAVE says try later */ GET_ARTICLE(0); - open_defer(); - if (fprintf(defer, "%s %s\n", TokenToText(art->token), art->messageid) <0 - || fflush(defer)) - sysfatal("write to defer file %s",path_defer); - article_done(conn, art, RC_deferred); + article_defer(art, RC_deferred); break; } @@ -1871,6 +1991,8 @@ static InputFile *open_input_file(const char *path) { memset(ipf,0,sizeof(*ipf)); ipf->fd= fd; + ipf->autodefer= -1; + LIST_INIT(ipf->queue); strcpy(ipf->path, path); return ipf; @@ -1921,7 +2043,7 @@ static void *feedfile_got_article(oop_source *lp, oop_read *rd, if (!data) { feedfile_eof(ipf); return OOP_CONTINUE; } off_t old_offset= ipf->offset; - ipf->offset += recsz + 1; + ipf->offset += recsz + !!(ev == OOP_RD_OK); #define X_BAD_DATA(m) return feedfile_got_bad_data(ipf,old_offset,data,m); @@ -1961,7 +2083,7 @@ static void *feedfile_got_article(oop_source *lp, oop_read *rd, ipf->readcount_ok++; art= xmalloc(sizeof(*art) - 1 + midlen + 1); - memset(art,0,sizeof(art)); + memset(art,0,sizeof(*art)); art->state= art_Unchecked; art->midlen= midlen; art->ipf= ipf; ipf->inprogress++; @@ -1969,13 +2091,19 @@ static void *feedfile_got_article(oop_source *lp, oop_read *rd, art->offset= old_offset; art->blanklen= recsz; strcpy(art->messageid, space+1); - LIST_ADDTAIL(queue, art); + LIST_ADDTAIL(ipf->queue, art); + + if (ipf->autodefer >= 0) + article_autodefer(ipf, art); + else if (ipf==backlog_input_file) + article_check_expired(art); if (sms==sm_NORMAL && ipf==main_input_file && ipf->offset >= target_max_feedfile_size) statemc_start_flush("feed file size"); - check_assign_articles(); + check_assign_articles(); /* may destroy conn but that's OK */ + check_reading_pause_resume(ipf); return OOP_CONTINUE; } @@ -2126,7 +2254,7 @@ static void filemon_method_dump_info(FILE *f) { DUMPV("%d",,filemon_inotify_fd); DUMPV("%d",,filemon_inotify_wdmax); for (i=0; ird) return; + if (!ipf->paused) return; + + int r= oop_rd_read(ipf->rd, &feedfile_rdstyle, MAX_LINE_FEEDFILE, + feedfile_got_article,ipf, feedfile_read_err, ipf); + if (r) sysdie("unable start reading feedfile %s",ipf->path); + + ipf->paused= 0; +} + +static void inputfile_reading_pause(InputFile *ipf) { + if (!ipf->rd) return; + if (ipf->paused) return; + oop_rd_cancel(ipf->rd); + ipf->paused= 1; +} + static void inputfile_reading_start(InputFile *ipf) { assert(!ipf->rd); ipf->readable.on_readable= tailing_on_readable; @@ -2190,14 +2336,13 @@ static void inputfile_reading_start(InputFile *ipf) { ipf->rd= oop_rd_new(loop, &ipf->readable, 0,0); assert(ipf->rd); - int r= oop_rd_read(ipf->rd, &feedfile_rdstyle, MAX_LINE_FEEDFILE, - feedfile_got_article,ipf, feedfile_read_err, ipf); - if (r) sysdie("unable start reading feedfile %s",ipf->path); + ipf->paused= 1; + inputfile_reading_resume(ipf); } static void inputfile_reading_stop(InputFile *ipf) { assert(ipf->rd); - oop_rd_cancel(ipf->rd); + inputfile_reading_pause(ipf); oop_rd_delete(ipf->rd); ipf->rd= 0; assert(!ipf->filemon); /* we shouldn't be monitoring it now */ @@ -2266,7 +2411,7 @@ static void inputfile_reading_stop(InputFile *ipf) { | flsh->rd!=0 | | flsh->rd!=0 | [Separated] | | [Dropping] | main F idle | | main none - | old D tail | | old D tail + | flsh D tail | | flsh D tail | ============= | | ============ | | | | install | ^ | EOF ON D | | defer | EOF ON D @@ -2276,7 +2421,7 @@ static void inputfile_reading_stop(InputFile *ipf) { | flsh->rd==0 | V flsh->rd==0 | [Finishing] | | [Dropping] | main F tail | `. main none - | old D closed | `. old D closed + | flsh D closed | `. flsh D closed | =============== V `. =============== | | `. | | | ALL D PROCESSED `. | ALL D PROCESSED @@ -2289,7 +2434,7 @@ static void inputfile_reading_stop(InputFile *ipf) { DROPPED [Dropped] main none - old none + flsh none some backlog ============== | @@ -2405,7 +2550,11 @@ static void statemc_init(void) { if (file_d) { debug("startup: F!=D => Separated"); startup_set_input_file(file_d); - SMS(SEPARATED, 0, "found both old and current feed files"); + flushing_input_file= main_input_file; + main_input_file= open_input_file(feedfile); + if (!main_input_file) die("feedfile vanished during startup"); + SMS(SEPARATED, max_separated_periods, + "found both old and current feed files"); } else { debug("startup: F exists, D ENOENT => Normal"); InputFile *file_f= open_input_file(feedfile); @@ -2423,7 +2572,7 @@ static void statemc_start_flush(const char *why) { /* Normal => Flushing */ why, (unsigned long)(main_input_file ? main_input_file->offset : 0), (unsigned long)target_max_feedfile_size, - sm_period_counter); + until_flush); int r= link(feedfile, path_flushing); if (r) sysfatal("link feedfile %s to flushing file %s", @@ -2436,24 +2585,39 @@ static void statemc_start_flush(const char *why) { /* Normal => Flushing */ spawn_inndcomm_flush(why); /* => Flushing FLUSHING */ } -static void statemc_period_poll(void) { - if (!sm_period_counter) return; - sm_period_counter--; - assert(sm_period_counter>=0); - - if (sm_period_counter) return; +static int trigger_flush_ok(void) { /* => Flushing,FLUSHING, ret 1; or ret 0 */ switch (sms) { + case sm_NORMAL: statemc_start_flush("periodic"); /* Normal => Flushing; => FLUSHING */ - break; + return 1; + case sm_FLUSHFAILED: spawn_inndcomm_flush("retry"); /* Moved => Flushing; => FLUSHING */ - break; + return 1; + + case sm_SEPARATED: + case sm_DROPPING: + warn("took too long to complete old feedfile after flush, autodeferring"); + assert(flushing_input_file); + autodefer_input_file(flushing_input_file); + return 1; + default: - abort(); + return 0; } } +static void statemc_period_poll(void) { + if (!until_flush) return; + until_flush--; + assert(until_flush>=0); + + if (until_flush) return; + int ok= trigger_flush_ok(); + assert(ok); +} + static int inputfile_is_done(InputFile *ipf) { if (!ipf) return 0; if (ipf->inprogress) return 0; /* new article in the meantime */ @@ -2474,13 +2638,17 @@ static void notice_processed(InputFile *ipf, int completed, char *inprog= completed ? xasprintf("%s","") /* GCC produces a stupid warning for printf("") ! */ : xasprintf(" inprogress=%ld", ipf->inprogress); + char *autodefer= ipf->autodefer >= 0 + ? xasprintf(" autodeferred=%ld", ipf->autodefer) + : xasprintf("%s",""); - info("%s %s%s read=%d (+bl=%d,+err=%d)%s" - " offered=%d (ch=%d,nc=%d) accepted=%d (ch=%d,nc=%d)" + info("%s %s%s read=%d (+bl=%d,+err=%d)%s%s" + " missing=%d offered=%d (ch=%d,nc=%d) accepted=%d (ch=%d,nc=%d)" RESULT_COUNTS(RCI_NOTHING, RCI_TRIPLE_FMT) , completed?"completed":"processed", what, spec, - ipf->readcount_ok, ipf->readcount_blank, ipf->readcount_err, inprog, + ipf->readcount_ok, ipf->readcount_blank, ipf->readcount_err, + inprog, autodefer, ipf->count_nooffer_missing, CNT(Unchecked,sent) + CNT(Unsolicited,sent) , CNT(Unchecked,sent), CNT(Unsolicited,sent), CNT(Wanted,accepted) + CNT(Unsolicited,accepted) @@ -2489,6 +2657,7 @@ static void notice_processed(InputFile *ipf, int completed, ); free(inprog); + free(autodefer); #undef CNT } @@ -2538,7 +2707,7 @@ static void statemc_check_flushing_done(void) { notice("flush complete"); SMS(NORMAL, spontaneous_flush_periods, "flush complete"); } else if (sms==sm_DROPPING) { - SMS(DROPPED, 0, "old flush complete"); + SMS(DROPPED, max_separated_periods, "old flush complete"); search_backlog_file(); notice("feed dropped, but will continue until backlog is finished"); } @@ -2559,7 +2728,7 @@ static void queue_check_input_done(void) { static void statemc_setstate(StateMachineState newsms, int periods, const char *forlog, const char *why) { sms= newsms; - sm_period_counter= periods; + until_flush= periods; const char *xtra= ""; switch (sms) { @@ -2777,7 +2946,7 @@ static void search_backlog_file(void) { static void preterminate(void) { if (in_child) return; notice_processed(main_input_file,0,"feedfile",""); - notice_processed(flushing_input_file,0,"flushing file",""); + notice_processed(flushing_input_file,0,"flushing",""); if (backlog_input_file) notice_processed(backlog_input_file,0, "backlog file ", backlog_input_file->path); @@ -2787,11 +2956,9 @@ static int signal_self_pipe[2]; static sig_atomic_t terminate_sig_flag; static void raise_default(int signo) { - struct sigaction sa; - memset(&sa,0,sizeof(sa)); - sa.sa_handler= SIG_DFL; - xsigaction(signo,&sa); + xsigsetdefault(signo); raise(signo); + abort(); } static void *sigarrived_event(oop_source *lp, int fd, oop_event e, void *u) { @@ -2804,7 +2971,6 @@ static void *sigarrived_event(oop_source *lp, int fd, oop_event e, void *u) { preterminate(); notice("terminating (%s)", strsignal(terminate_sig_flag)); raise_default(terminate_sig_flag); - abort(); } return OOP_CONTINUE; } @@ -2812,7 +2978,8 @@ static void *sigarrived_event(oop_source *lp, int fd, oop_event e, void *u) { static void sigarrived_handler(int signum) { static char x; switch (signum) { - case SIGINT: case SIGTERM: + case SIGTERM: + case SIGINT: if (!terminate_sig_flag) terminate_sig_flag= signum; break; default: @@ -2874,7 +3041,8 @@ static void *inndcomm_event(oop_source *lp, int fd, oop_event e, void *u) { main_input_file= 0; if (flushing_input_file) { - SMS(DROPPING, 0, "feed dropped by innd, but must finish last flush"); + SMS(DROPPING, max_separated_periods, + "feed dropped by innd, but must finish last flush"); } else { close_defer(); SMS(DROPPED, 0, "feed dropped by innd"); @@ -2892,7 +3060,7 @@ static void *inndcomm_event(oop_source *lp, int fd, oop_event e, void *u) { die("flush succeeded but feedfile %s does not exist!", feedfile); if (flushing_input_file) { - SMS(SEPARATED, spontaneous_flush_periods, "recovery flush complete"); + SMS(SEPARATED, max_separated_periods, "recovery flush complete"); } else { close_defer(); SMS(NORMAL, spontaneous_flush_periods, "flush complete"); @@ -2984,8 +3152,10 @@ static void postfork_stdio(FILE *f, const char *what, const char *what2) { static void postfork(void) { in_child= 1; - if (signal(SIGPIPE, SIG_DFL) == SIG_ERR) - sysdie("(in child) failed to reset SIGPIPE"); + xsigsetdefault(SIGTERM); + xsigsetdefault(SIGINT); + xsigsetdefault(SIGPIPE); + if (terminate_sig_flag) raise(terminate_sig_flag); postfork_inputfile(main_input_file); postfork_inputfile(flushing_input_file); @@ -3042,12 +3212,13 @@ static char *debug_report_ipf(InputFile *ipf) { const char *slash= strrchr(ipf->path,'/'); const char *path= slash ? slash+1 : ipf->path; - return xasprintf("%p/%s:ip=%ld,off=%ld,fd=%d%s%s", + return xasprintf("%p/%s:queue=%d,ip=%ld,autodef=%ld,off=%ld,fd=%d%s%s%s", ipf, path, - ipf->inprogress, (long)ipf->offset, - ipf->fd, + ipf->queue.count, ipf->inprogress, ipf->autodefer, + (long)ipf->offset, ipf->fd, ipf->rd ? "" : ",!rd", - ipf->skippinglong ? "*skiplong" : ""); + ipf->skippinglong ? "*skiplong" : "", + ipf->rd && ipf->paused ? "*paused" : ""); } static void period(void) { @@ -3056,13 +3227,12 @@ static void period(void) { char *dipf_backlog= debug_report_ipf(backlog_input_file); debug("PERIOD" - " sms=%s[%d] conns=%d queue=%d until_connect=%d" - " input_files main:%s flushing:%s backlog:%s" + " sms=%s[%d] conns=%d until_connect=%d" + " input_files main:%s flushing:%s backlog:%s[%d]" " children connecting=%ld inndcomm=%ld" , - sms_names[sms], sm_period_counter, - conns.count, queue.count, until_connect, - dipf_main, dipf_flushing, dipf_backlog, + sms_names[sms], until_flush, conns.count, until_connect, + dipf_main, dipf_flushing, dipf_backlog, until_backlog_nextscan, (long)connecting_child, (long)inndcomm_child ); @@ -3072,6 +3242,7 @@ static void period(void) { if (until_connect) until_connect--; + inputfile_queue_check_expired(backlog_input_file); poll_backlog_file(); if (!backlog_input_file) close_defer(); /* want to start on a new backlog */ statemc_period_poll(); @@ -3099,7 +3270,8 @@ static void dump_article_list(FILE *f, const ControlCommand *c, } } -static void dump_input_file(FILE *f, InputFile *ipf, const char *wh) { +static void dump_input_file(FILE *f, const ControlCommand *c, + InputFile *ipf, const char *wh) { char *dipf= debug_report_ipf(ipf); fprintf(f,"input %s %s", wh, dipf); free(dipf); @@ -3108,6 +3280,7 @@ static void dump_input_file(FILE *f, InputFile *ipf, const char *wh) { DUMPV("%d", ipf->,readcount_ok); DUMPV("%d", ipf->,readcount_blank); DUMPV("%d", ipf->,readcount_err); + DUMPV("%d", ipf->,count_nooffer_missing); } fprintf(f,"\n"); if (ipf) { @@ -3120,6 +3293,8 @@ static void dump_input_file(FILE *f, InputFile *ipf, const char *wh) { wh, *statename RESULT_COUNTS(RC_DUMP_VAL,RC_DUMP_VAL)); } + fprintf(f,"input %s queue", wh); + dump_article_list(f,c,&ipf->queue); } } @@ -3131,7 +3306,7 @@ CCMD(dump) { fprintf(f,"general"); DUMPV("%s", sms_names,[sms]); - DUMPV("%d", ,sm_period_counter); + DUMPV("%d", ,until_flush); DUMPV("%ld", (long),self_pid); DUMPV("%p", , defer); DUMPV("%d", , until_connect); @@ -3152,9 +3327,9 @@ CCMD(dump) { fprintf(f,"filemon "); filemon_method_dump_info(f); - dump_input_file(f, main_input_file, "main" ); - dump_input_file(f, flushing_input_file, "flushing"); - dump_input_file(f, backlog_input_file, "backlog" ); + dump_input_file(f,c, main_input_file, "main" ); + dump_input_file(f,c, flushing_input_file, "flushing"); + dump_input_file(f,c, backlog_input_file, "backlog" ); fprintf(f,"conns count=%d\n", conns.count); @@ -3176,18 +3351,9 @@ CCMD(dump) { const struct iovec *iv= &conn->xmit[i]; const XmitDetails *xd= &conn->xmitd[i]; char *dinfo; - long diff; switch (xd->kind) { - case xk_Malloc: - diff= xd->info.malloc_tofree - (char*)iv->iov_base; - dinfo= xasprintf("M%5ld", diff); - break; - case xk_Const: - dinfo= xasprintf("Const"); - break; - case xk_Artdata: - dinfo= xasprintf("A%p", xd->info.sm_art); - break; + case xk_Const: dinfo= xasprintf("Const"); break; + case xk_Artdata: dinfo= xasprintf("A%p", xd->info.sm_art); break; default: abort(); } @@ -3197,13 +3363,12 @@ CCMD(dump) { } } - fprintf(f,"queue"); dump_article_list(f,c,&queue); - fprintf(f,"paths"); + DUMPV("%s", , feedfile); + DUMPV("%s", , path_control); DUMPV("%s", , path_lock); DUMPV("%s", , path_flushing); DUMPV("%s", , path_defer); - DUMPV("%s", , path_control); DUMPV("%s", , path_dump); DUMPV("%s", , globpat_backlog); fprintf(f,"\n"); @@ -3387,11 +3552,12 @@ static const Option innduct_options[]= { {0,"no-filemon", 0, &try_filemon, op_setint, 0 }, {'C',"inndconf", "F", &inndconffile, op_string }, {'P',"port", "PORT", &port, op_integer }, -{0,"ctrl-sock-dir", 0, &realsockdir, op_string }, +{0,"cli", 0, &path_control, op_string }, {0,"help", 0, 0, help }, {0,"max-connections", "N", &max_connections, op_integer }, {0,"max-queue-per-conn", "N", &max_queue_per_conn, op_integer }, +{0,"max-queue-per-file", "N", &max_queue_per_ipf, op_integer }, {0,"feedfile-flush-size","BYTES", &target_max_feedfile_size, op_integer }, {0,"period-interval", "TIME", &period_seconds, op_seconds }, @@ -3407,6 +3573,7 @@ static const Option innduct_options[]= { {0,"earliest-deferred-retry","PERIOD", &backlog_retry_minperiods, op_seconds }, {0,"backlog-rescan-interval","PERIOD",&backlog_spontrescan_periods,op_seconds}, {0,"max-flush-interval", "PERIOD", &spontaneous_flush_periods,op_seconds }, +{0,"flush-finish-timeout", "PERIOD", &max_separated_periods, op_seconds }, {0,"idle-timeout", "PERIOD", &need_activity_periods, op_seconds }, {0,"max-bad-input-data-ratio","PERCENT", &max_bad_data_ratio, op_double }, @@ -3435,6 +3602,14 @@ static void convert_to_periods_rndup(int *store) { *store /= period_seconds; } +static void assemble_path(const char **path_io, const char *suffix, + const char *what) { + const char *const specified= *path_io; + if (!specified[0]) badusage("%s, if specified, must be nonempty", what); + if (specified[strlen(specified)-1]=='/') + *path_io= xasprintf("%s%s%s", specified, sitename, suffix); +} + int main(int argc, char **argv) { if (!argv[1]) { printusage(stderr); @@ -3470,19 +3645,21 @@ int main(int argc, char **argv) { convert_to_periods_rndup(&backlog_retry_minperiods); convert_to_periods_rndup(&backlog_spontrescan_periods); convert_to_periods_rndup(&spontaneous_flush_periods); + convert_to_periods_rndup(&max_separated_periods); convert_to_periods_rndup(&need_activity_periods); if (max_bad_data_ratio < 0 || max_bad_data_ratio > 100) badusage("bad input data ratio must be between 0..100"); max_bad_data_ratio *= 0.01; + + if (!feedfile) feedfile= xasprintf("%s/%s",innconf->pathoutgoing,sitename); + else assemble_path(&feedfile, "", "feed filename"); - if (!feedfile) { - feedfile= xasprintf("%s/%s",innconf->pathoutgoing,sitename); - } else if (!feedfile[0]) { - badusage("feed filename must be nonempty"); - } else if (feedfile[strlen(feedfile)-1]=='/') { - feedfile= xasprintf("%s%s",feedfile,sitename); - } + if (path_control) path_control= xasprintf("%s_cli", feedfile); + else assemble_path(&path_control, "%s_cli", "control socket path"); + + if (max_queue_per_ipf<0) + max_queue_per_ipf= max_queue_per_conn * 2; const char *feedfile_forbidden= "?*[~#"; int c; @@ -3495,7 +3672,6 @@ int main(int argc, char **argv) { path_lock= xasprintf("%s_lock", feedfile); path_flushing= xasprintf("%s_flushing", feedfile); path_defer= xasprintf("%s_defer", feedfile); - path_control= xasprintf("%s_control", feedfile); path_dump= xasprintf("%s_dump", feedfile); globpat_backlog= xasprintf("%s_backlog*", feedfile); @@ -3504,7 +3680,6 @@ int main(int argc, char **argv) { loop= (oop_source*)sysloop; LIST_INIT(conns); - LIST_INIT(queue); if (become_daemon) { int i; @@ -3539,6 +3714,10 @@ int main(int argc, char **argv) { notice("starting"); + int val= 1; + r= SMsetup(SM_PREOPEN, &val); if (!r) warn("SMsetup SM_PREOPEN failed"); +// r= SMinit(); if (!r) die("storage manager initialisation (SMinit) failed"); + if (!become_daemon) control_stdio();