X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?p=inn-innduct.git;a=blobdiff_plain;f=backends%2Finnduct.c;h=34f7e3768582b29f7e85db08ae09ee71878b8aab;hp=c2ca557455c4f5927f54f027a98b6b7155f2b5a1;hb=1def4a401dd87f14dd7f9e04a9fbfab83cac68c8;hpb=17fba8701e3313e552be7a230b55c3c85622d915 diff --git a/backends/innduct.c b/backends/innduct.c index c2ca557..34f7e37 100644 --- a/backends/innduct.c +++ b/backends/innduct.c @@ -1,6 +1,15 @@ /* * TODO * - close idle connections + * - cope better with garbage in feed file + * - cope better with NULs in feed file + * - check all structs initialised + * - check all fd watches properly undone + * - check all init functions called + * - actually implement badusage + * - options for all options + * - manpage + * - pid, sitename, hostname in lockfile * - -k kill mode ? */ @@ -155,11 +164,14 @@ perl -ne 'print if m/-8\<-/..m/-\>8-/; print "\f" if m/-\^L-/' backends/innduct. #define _GNU_SOURCE -#include "inn/list.h" #include "config.h" #include "storage.h" #include "nntp.h" #include "libinn.h" +#include "inndcomm.h" + +#include "inn/list.h" +#include "inn/innconf.h" #include #include @@ -176,6 +188,9 @@ perl -ne 'print if m/-8\<-/..m/-\>8-/; print "\f" if m/-\^L-/' backends/innduct. #include #include #include +#include +#include +#include #include #include @@ -190,6 +205,8 @@ perl -ne 'print if m/-8\<-/..m/-\>8-/; print "\f" if m/-\^L-/' backends/innduct. #define INNDCOMMCHILD_ESTATUS_FAIL 6 #define INNDCOMMCHILD_ESTATUS_NONESUCH 7 +#define MAX_LINE_FEEDFILE (NNTP_MSGID_MAXLEN + sizeof(TOKEN)*2 + 10) + /*----- doubly linked lists -----*/ #define ISNODE(T) struct { T *succ, *pred; } node /* must be at start */ @@ -245,6 +262,7 @@ typedef struct Conn Conn; typedef struct Article Article; typedef struct InputFile InputFile; typedef struct XmitDetails XmitDetails; +typedef struct Filemon_Perfile Filemon_Perfile; typedef enum StateMachineState StateMachineState; DEFLIST(Conn); @@ -258,38 +276,57 @@ static void *conn_write_some_xmits(Conn *conn); static void xmit_free(XmitDetails *d); -static int filemon_init(void); -static void filemon_setfile(int mainfeed_fd, const char *mainfeed_path); -static void filemon_callback(void); - static void statemc_setstate(StateMachineState newsms, int periods, const char *forlog, const char *why); +static void statemc_start_flush(const char *why); /* Normal => Flushing */ +static void spawn_inndcomm_flush(const char *why); /* Moved => Flushing */ + static void check_master_queue(void); static void queue_check_input_done(void); +static void statemc_check_flushing_done(void); +static void statemc_check_backlog_done(void); + static void postfork(const char *what); static void postfork_inputfile(InputFile *ipf); +static void open_defer(void); +static void close_defer(void); +static void search_backlog_file(void); + +static void inputfile_tailing_start(InputFile *ipf); +static void inputfile_tailing_stop(InputFile *ipf); + +static int filemon_init(void); +static void filemon_start(InputFile *ipf); +static void filemon_stop(InputFile *ipf); +static void filemon_callback(InputFile *ipf); + /*----- configuration options -----*/ -static char *sitename, *feedfile; +static const char *sitename, *feedfile, *pathoutgoing; static const char *remote_host; static int quiet_multiple=0, become_daemon=1; static int max_connections=10, max_queue_per_conn=200; +static int target_max_feedfile_size=100000; + +static double max_bad_data_ratio= 0.01; +static int max_bad_data_initial= 30; + /* in one corrupt 4096-byte block the number of newlines has + * mean 16 and standard deviation 3.99. 30 corresponds to z=+3.5 */ static int connection_setup_timeout=200, port=119, try_stream=1; static int inndcomm_flush_timeout=100; static int reconnect_delay_periods, flushfail_retry_periods, open_wait_periods; static int backlog_retry_minperiods, backlog_spontaneous_rescan_periods; +static int spontaneous_flush_periods; static const char *inndconffile; -static double accept_proportion; static double nocheck_thresh_pct= 95.0; static double nocheck_thresh; /* computed in main from _pct */ static double nocheck_decay_articles= 100; /* converted to _decay */ static double nocheck_decay; /* computed in main from _articles */ -static int nocheck, nocheck_reported; /*----- statistics -----*/ @@ -351,13 +388,15 @@ struct InputFile { void *readable_callback_user; int fd; - struct Filemon_Perfile *filemon; + Filemon_Perfile *filemon; oop_read *rd; long inprogress; /* no. of articles read but not processed */ off_t offset; + int skippinglong; int counts[art_MaxState][RCI_max]; + int readcount_ok, readcount_blank, readcount_err; char path[]; }; @@ -394,7 +433,8 @@ static const char *sms_names[]= { struct Conn { ISNODE(Conn); int fd, max_queue, stream, quitting; - ArticleList queue; /* not yet told peer, or CHECK said send it */ + ArticleList waiting; /* not yet told peer */ + ArticleList priority; /* peer says send it now */ ArticleList sent; /* offered/transmitted - in xmit or waiting reply */ struct iovec xmit[CONNIOVS]; XmitDetails xmitd[CONNIOVS]; @@ -410,7 +450,7 @@ static int until_connect; static ConnList conns; static ArticleList queue; -static char *path_lock, *path_flushing, *path_defer; +static char *path_lock, *path_flushing, *path_defer, *globpat_backlog; #define SMS(newstate, periods, why) \ (statemc_setstate(sm_##newstate,(periods),#newstate,(why))) @@ -418,16 +458,20 @@ static char *path_lock, *path_flushing, *path_defer; static StateMachineState sms; static FILE *defer; static InputFile *main_input_file, *flushing_input_file, *backlog_input_file; -static int sm_period_counter; +static int sm_period_counter, until_backlog_nextscan; +static double accept_proportion; +static int nocheck, nocheck_reported; /*========== logging ==========*/ -static void logcore(int sysloglevel, const char *fmt, ...) - __attribute__((__format__(printf,2,3))); +#define VA va_list al; va_start(al,fmt) +#define PRINTF(f,a) __attribute__((__format__(printf,f,a))) +#define NORET_PRINTF(f,a) __attribute__((__noreturn__,__format__(printf,f,a))) + +static void logcore(int sysloglevel, const char *fmt, ...) PRINTF(2,3); static void logcore(int sysloglevel, const char *fmt, ...) { - va_list al; - va_start(al,fmt); + VA; if (become_daemon) { vsyslog(sysloglevel,fmt,al); } else { @@ -438,10 +482,9 @@ static void logcore(int sysloglevel, const char *fmt, ...) { } static void logv(int sysloglevel, const char *pfx, int errnoval, - int exitstatus, const char *fmt, va_list al) - __attribute__((__format__(printf,5,0))); + const char *fmt, va_list al) PRINTF(5,0); static void logv(int sysloglevel, const char *pfx, int errnoval, - int exitstatus, const char *fmt, va_list al) { + const char *fmt, va_list al) { char msgbuf[256]; /* NB do not call xvasprintf here or you'll recurse */ vsnprintf(msgbuf,sizeof(msgbuf), fmt,al); msgbuf[sizeof(msgbuf)-1]= 0; @@ -455,44 +498,48 @@ static void logv(int sysloglevel, const char *pfx, int errnoval, errnoval>=0 ? strerror(errnoval) : ""); } -#define logwrap(fn, pfx, sysloglevel, err, estatus) \ - static void fn(const char *fmt, ...) \ - __attribute__((__format__(printf,1,2))); \ +#define diewrap(fn, pfx, sysloglevel, err, estatus) \ + static void fn(const char *fmt, ...) NORET_PRINTF(1,2); \ + static void fn(const char *fmt, ...) { \ + VA; \ + logv(sysloglevel, pfx, err, fmt, al); \ + exit(estatus); \ + } + +#define logwrap(fn, pfx, sysloglevel, err) \ + static void fn(const char *fmt, ...) PRINTF(1,2); \ static void fn(const char *fmt, ...) { \ - va_list al; \ - va_start(al,fmt); \ - logv(sysloglevel, pfx, err, estatus, fmt, al); \ + VA; \ + logv(sysloglevel, pfx, err, fmt, al); \ + va_end(al); \ } -logwrap(sysdie, " critical", LOG_CRIT, errno, 16); -logwrap(die, " critical", LOG_CRIT, -1, 16); +diewrap(sysdie, " critical", LOG_CRIT, errno, 16); +diewrap(die, " critical", LOG_CRIT, -1, 16); -logwrap(sysfatal, " fatal", LOG_ERR, errno, 12); -logwrap(fatal, " fatal", LOG_ERR, -1, 12); +diewrap(sysfatal, " fatal", LOG_ERR, errno, 12); +diewrap(fatal, " fatal", LOG_ERR, -1, 12); -logwrap(syswarn, " warning", LOG_WARNING, errno, 0); -logwrap(warn, " warning", LOG_WARNING, -1, 0); +logwrap(syswarn, " warning", LOG_WARNING, errno); +logwrap(warn, " warning", LOG_WARNING, -1); -logwrap(notice, "", LOG_NOTICE, -1, 0); -logwrap(info, " info", LOG_INFO, -1, 0); -logwrap(debug, " debug", LOG_DEBUG, -1, 0); +logwrap(notice, "", LOG_NOTICE, -1); +logwrap(info, " info", LOG_INFO, -1); +logwrap(debug, " debug", LOG_DEBUG, -1); /*========== utility functions etc. ==========*/ -static char *xvasprintf(const char *fmt, va_list al) - __attribute__((__format__(printf,1,0))); +static char *xvasprintf(const char *fmt, va_list al) PRINTF(1,0); static char *xvasprintf(const char *fmt, va_list al) { char *str; int rc= vasprintf(&str,fmt,al); if (rc<0) sysdie("vasprintf(\"%s\",...) failed", fmt); return str; } -static char *xasprintf(const char *fmt, ...) - __attribute__((__format__(printf,1,2))); +static char *xasprintf(const char *fmt, ...) PRINTF(1,2); static char *xasprintf(const char *fmt, ...) { - va_list al; - va_start(al,fmt); + VA; char *str= xvasprintf(fmt,al); va_end(al); return str; @@ -557,6 +604,11 @@ static void xunlink(const char *path, const char *what) { if (r) sysdie("can't unlink %s %s", path, what); } +static time_t xtime(void) { + time_t now= time(0); + if (now==-1) sysdie("time(2) failed"); +} + static void check_isreg(const struct stat *stab, const char *path, const char *what) { if (!S_ISREG(stab->st_mode)) @@ -727,7 +779,7 @@ static void *connchild_event(oop_source *lp, int fd, oop_event e, void *u) { setnonblock(conn->fd, 1); conn->max_queue= conn->stream ? max_queue_per_conn : 1; LIST_ADDHEAD(conns, conn); - notice("#%d connected %s", conn->fd, conn->stream ? "streaming" : "plain"); + notice("C%d connected %s", conn->fd, conn->stream ? "streaming" : "plain"); connect_attempt_discard(); check_master_queue(); return 0; @@ -737,6 +789,12 @@ static void *connchild_event(oop_source *lp, int fd, oop_event e, void *u) { connect_attempt_discard(); } +static int allow_connect_start(void) { + return conns.count < max_connections + && !connecting_child + && !until_connect; +} + static void connect_start(void) { assert(!connecting_sockets[0]); assert(!connecting_sockets[1]); @@ -845,7 +903,8 @@ static void check_master_queue(void) { * connections, the spare ones will go away eventually. */ for (walk=LIST_HEAD(conns); walk; walk=LIST_NEXT(walk)) { - int inqueue= walk->sent.count + walk->queue.count; + int inqueue= walk->sent.count + walk->priority.count + + walk->waiting.count; spare= walk->max_queue - inqueue; assert(inqueue <= max_queue_per_conn); assert(spare >= 0); @@ -855,12 +914,11 @@ static void check_master_queue(void) { if (use) { while (spare>0) { Article *art= LIST_REMHEAD(queue); - LIST_ADDTAIL(use->queue, art); + LIST_ADDTAIL(use->waiting, art); spare--; } conn_maybe_write(use); - } else if (conns.count < max_connections && - !connecting_child && !until_connect) { + } else if (allow_connect_start()) { until_connect= reconnect_delay_periods; connect_start(); break; @@ -898,14 +956,13 @@ static void conn_maybe_write(Conn *conn) { } } -static void vconnfail(Conn *conn, const char *fmt, va_list al) - __attribute__((__format__(printf,2,0))); - +static void vconnfail(Conn *conn, const char *fmt, va_list al) PRINTF(2,0); static void vconnfail(Conn *conn, const char *fmt, va_list al) { int requeue[art_MaxState]; Article *art; - while ((art= LIST_REMHEAD(conn->queue))) LIST_ADDTAIL(queue, art); + while ((art= LIST_REMHEAD(conn->priority))) LIST_ADDTAIL(queue, art); + while ((art= LIST_REMHEAD(conn->waiting))) LIST_ADDTAIL(queue, art); while ((art= LIST_REMHEAD(conn->sent))) { requeue[art->state]++; if (art->state==art_Unsolicited) art->state= art_Unchecked; @@ -918,7 +975,7 @@ static void vconnfail(Conn *conn, const char *fmt, va_list al) { xmit_free(d); char *m= xvasprintf(fmt,al); - warn("#%d connection failed, requeueing " RCI_TRIPLE_FMT_BASE ": %s", + warn("C%d connection failed (requeueing " RCI_TRIPLE_FMT_BASE "): %s", conn->fd, RCI_TRIPLE_VALS_BASE(requeue, /*nothing*/), m); free(m); @@ -927,8 +984,7 @@ static void vconnfail(Conn *conn, const char *fmt, va_list al) { check_master_queue(); } -static void connfail(Conn *conn, const char *fmt, ...) - __attribute__((__format__(printf,2,3))); +static void connfail(Conn *conn, const char *fmt, ...) PRINTF(2,3); static void connfail(Conn *conn, const char *fmt, ...) { va_list al; va_start(al,fmt); @@ -1010,7 +1066,8 @@ static void conn_make_some_xmits(Conn *conn) { if (conn->xmitu+5 > CONNIOVS) break; - Article *art= LIST_REMHEAD(queue); + Article *art= LIST_REMHEAD(conn->priority); + if (!art) art= LIST_REMHEAD(conn->waiting); if (!art) break; if (art->state >= art_Wanted || (conn->stream && nocheck)) { @@ -1052,7 +1109,7 @@ static void conn_make_some_xmits(Conn *conn) { XMIT_LITERAL("\r\n"); assert(art->state == art_Unchecked); - art->ipf->counts[art->state][RCI_sent]++; + art->ipf->counts[art->state][RC_sent]++; LIST_ADDTAIL(conn->sent, art); } } @@ -1205,8 +1262,9 @@ static void *peer_rd_ok(oop_source *lp, oop_read *oread, oop_event ev, if (code!=205 && code!=503) { connfail(conn, "peer gave unexpected response to QUIT: %s", sani); } else { - notice("#%d idle connection closed\n"); - assert(!conn->queue.count); + notice("C%d idle connection closed\n"); + assert(!conn->waiting.count); + assert(!conn->priority.count); assert(!conn->sent.count); assert(!conn->xmitu); LIST_REMOVE(conns,conn); @@ -1252,7 +1310,7 @@ static void *peer_rd_ok(oop_source *lp, oop_read *oread, oop_event ev, assert(art->state == art_Unchecked); art->ipf->counts[art->state][RC_accepted]++; art->state= art_Wanted; - LIST_ADDTAIL(conn->queue, art); + LIST_ADDTAIL(conn->priority, art); break; case 431: /* CHECK or TAKETHIS says try later */ @@ -1268,7 +1326,7 @@ static void *peer_rd_ok(oop_source *lp, oop_read *oread, oop_event ev, } - conn_maybe_write(sendon); + conn_maybe_write(conn); check_master_queue(); return OOP_CONTINUE; } @@ -1305,10 +1363,6 @@ static InputFile *open_input_file(const char *path) { InputFile *ipf= xmalloc(sizeof(*ipf) + strlen(path) + 1); memset(ipf,0,sizeof(*ipf)); - ipf->readable.on_readable= tailing_on_readable; - ipf->readable.on_cancel= tailing_on_cancel; - ipf->readable.try_read= tailing_try_read; - ipf->fd= fd; strcpy(ipf->path, path); @@ -1328,97 +1382,110 @@ static void close_input_file(InputFile *ipf) { /*---------- dealing with articles read in the input file ----------*/ -typedef void *feedfile_got_article(oop_source *lp, oop_read *rd, - oop_rd_event ev, const char *errmsg, - int errnoval, - const char *data, size_t recsz, - void *ipf_v) { +static void *feedfile_got_bad_data(InputFile *ipf, off_t offset, + const char *data, const char *how) { + warn("corrupted file: %s, offset %lu: %s: %s", + ipf->path, (unsigned long)offset, how, sanitise(data)); + ipf->readcount_err++; + if (ipf->readcount_err > max_bad_data_initial + + (ipf->readcount_ok+ipf->readcount_blank) / max_bad_data_ratio) + die("too much garbage in input file! (%d errs, %d ok, %d blank)", + ipf->readcount_err, ipf->readcount_ok, ipf->readcount_blank); + return OOP_CONTINUE; +} + +static void *feedfile_read_err(oop_source *lp, oop_read *rd, + oop_rd_event ev, const char *errmsg, + int errnoval, const char *data, size_t recsz, + void *ipf_v) { + InputFile *ipf= ipf_v; + assert(ev == OOP_RD_SYSTEM); + errno= errnoval; + sysdie("error reading input file: %s, offset %lu", + ipf->path, (unsigned long)ipf->offset); +} + +static void *feedfile_got_article(oop_source *lp, oop_read *rd, + oop_rd_event ev, const char *errmsg, + int errnoval, const char *data, size_t recsz, + void *ipf_v) { InputFile *ipf= ipf_v; Article *art; char tokentextbuf[sizeof(TOKEN)*2+3]; if (!data) { feedfile_eof(ipf); return OOP_CONTINUE; } - if (data[0] && data[0]!=' ') { - char *space= strchr(data,' '); - int tokenlen= space-data; - int midlen= (int)recsz-tokenlen-1; - if (midlen < 0) goto bad_data; - - if (tokenlen != sizeof(TOKEN)*2+2) goto bad_data; - memcpy(tokentextbuf, data, tokenlen); - tokentextbuf[tokenlen]= 0; - if (!IsToken(tokentextbuf)) goto bad_data; - - art= xmalloc(sizeof(*art) - 1 + midlen + 1); - art->offset= ipf->offset; - art->blanklen= recsz; - art->midlen= midlen; - art->state= art_Unchecked; - art->ipf= ipf; ipf->inprogress++; - art->token= TextToToken(tokentextbuf); - strcpy(art->messageid, space+1); - LIST_ADDTAIL(queue, art); - } + off_t old_offset= ipf->offset; ipf->offset += recsz + 1; - if (sms==sm_NORMAL && ipf==main_input_file && - ipf->offset >= flush_threshold) - statemc_start_flush("feed file size"); +#define X_BAD_DATA(m) return feedfile_got_bad_data(ipf,old_offset,data,m); - check_master_queue(); -} + if (ev==OOP_RD_PARTREC) + feedfile_got_bad_data(ipf,old_offset,data,"missing final newline"); + /* but process it anyway */ -static void statemc_start_flush(const char *why) { /* Normal => Flushing */ - assert(sms == sm_NORMAL); + if (ipf->skippinglong) { + if (ev==OOP_RD_OK) ipf->skippinglong= 0; /* fine now */ + return; + } + if (ev==OOP_RD_LONG) { + ipf->skippinglong= 1; + X_BAD_DATA("overly long line"); + } - debug("starting flush (%s) (%lu >= %lu) (%d)", - why, - (unsigned long)ipf->offset, (unsigned long)flush_threshold, - sm_period_counter); + if (memchr(data,'\0',recsz)) X_BAD_DATA("nul byte"); + if (!recsz) X_BAD_DATA("empty line"); - int r= link(feedfile, duct_path); - if (r) sysdie("link feedfile %s to flushing file %s", feedfile, - path_duct); - /* => Hardlinked */ + if (data[0]==' ') { + if (strspn(data," ") != recsz) X_BAD_DATA("line partially blanked"); + ipf->readcount_blank++; + return OOP_CONTINUE; + } + + char *space= strchr(data,' '); + int tokenlen= space-data; + int midlen= (int)recsz-tokenlen-1; + if (midlen <= 2) X_BAD_DATA("no room for messageid"); + if (space[1]!='<' || space[midlen]!='>') X_BAD_DATA("invalid messageid"); + + if (tokenlen != sizeof(TOKEN)*2+2) X_BAD_DATA("token wrong length"); + memcpy(tokentextbuf, data, tokenlen); + tokentextbuf[tokenlen]= 0; + if (!IsToken(tokentextbuf)) X_BAD_DATA("token wrong syntax"); + + ipf->readcount_ok++; + + art= xmalloc(sizeof(*art) - 1 + midlen + 1); + art->offset= ipf->offset; + art->blanklen= recsz; + art->midlen= midlen; + art->state= art_Unchecked; + art->ipf= ipf; ipf->inprogress++; + art->token= TextToToken(tokentextbuf); + strcpy(art->messageid, space+1); + LIST_ADDTAIL(queue, art); - xunlink(feedfile, "old feedfile link"); - /* => Moved */ + if (sms==sm_NORMAL && ipf==main_input_file && + ipf->offset >= target_max_feedfile_size) + statemc_start_flush("feed file size"); - spawn_inndcomm_flush(why); /* => Flushing FLUSHING */ + check_master_queue(); + return OOP_CONTINUE; } /*========== tailing input file ==========*/ -static void filemon_start(InputFile *ipf) { - assert(!ipf->filemon); - - ipf->filemon= xmalloc(sizeof(*ipf->filemon)); - memset(ipf->filemon, 0, sizeof(*ipf->filemon)); - filemon_method_startfile(ipf, ipf->filemon); -} - -static void filemon_stop(InputFile *ipf) { - if (!ipf->filemon) return; - filemon_method_stopfile(ipf, ipf->filemon); - free(ipf->filemon); - ipf->filemon= 0; -} - -static void filemon_callback(InputFile *ipf) { - ipf->readable_callback(ipf->readable_callback_user); -} - static void *tailing_rable_call_time(oop_source *loop, struct timeval tv, void *user) { InputFile *ipf= user; - return ipf->readable_callback(ipf->readable_callback_user); + return ipf->readable_callback(loop, &ipf->readable, + ipf->readable_callback_user); } -static void on_cancel(struct oop_readable *rable) { +static void tailing_on_cancel(struct oop_readable *rable) { InputFile *ipf= (void*)rable; - if (ipf->filemon) filemon_stopfile(ipf); + if (ipf->filemon) filemon_stop(ipf); loop->cancel_time(loop, OOP_TIME_NOW, tailing_rable_call_time, ipf); ipf->readable_callback= 0; } @@ -1436,7 +1503,7 @@ static int tailing_on_readable(struct oop_readable *rable, tailing_on_cancel(rable); ipf->readable_callback= cb; ipf->readable_callback_user= user; - filemon_startfile(ipf); + filemon_start(ipf); tailing_queue_readable(ipf); return 0; @@ -1479,9 +1546,9 @@ static int filemon_inotify_fd; static int filemon_inotify_wdmax; static InputFile **filemon_inotify_wd2ipf; -typedef struct Filemon_Perfile { +struct Filemon_Perfile { int wd; -} Filemon_Inotify_Perfile; +}; static void filemon_method_startfile(InputFile *ipf, Filemon_Perfile *pf) { int wd= inotify_add_watch(filemon_inotify_fd, ipf->path, IN_MODIFY); @@ -1546,13 +1613,13 @@ static int filemon_method_init(void) { return 1; } -#endif /* HAVE_INOTIFY && !HAVE_FILEMON *// +#endif /* HAVE_INOTIFY && !HAVE_FILEMON */ /*---------- filemon dummy implementation ----------*/ #if !defined(HAVE_FILEMON) -typedef struct Filemon_Perfile { int dummy; } Filemon_Dummy_Perfile; +struct Filemon_Perfile { int dummy; }; static int filemon_method_init(void) { return 0; } static void filemon_method_startfile(InputFile *ipf, Filemon_Perfile *pf) { } @@ -1560,21 +1627,42 @@ static void filemon_method_stopfile(InputFile *ipf, Filemon_Perfile *pf) { } #endif /* !HAVE_FILEMON */ +/*---------- filemon generic interface ----------*/ + +static void filemon_start(InputFile *ipf) { + assert(!ipf->filemon); + + ipf->filemon= xmalloc(sizeof(*ipf->filemon)); + memset(ipf->filemon, 0, sizeof(*ipf->filemon)); + filemon_method_startfile(ipf, ipf->filemon); +} + +static void filemon_stop(InputFile *ipf) { + if (!ipf->filemon) return; + filemon_method_stopfile(ipf, ipf->filemon); + free(ipf->filemon); + ipf->filemon= 0; +} + +static void filemon_callback(InputFile *ipf) { + ipf->readable_callback(loop, &ipf->readable, ipf->readable_callback_user); +} + /*---------- interface to start and stop an input file ----------*/ static const oop_rd_style feedfile_rdstyle= { OOP_RD_DELIM_STRIP, '\n', - OOP_RD_NUL_FORBID, - OOP_RD_SHORTREC_EOF, + OOP_RD_NUL_PERMIT, + OOP_RD_SHORTREC_LONG, }; static void inputfile_tailing_start(InputFile *ipf) { assert(!ipf->fd); - ipf->readable->on_readable= tailing_on_readable; - ipf->readable->on_cancel= tailing_on_cancel; - ipf->readable->try_read= tailing_try_read; - ipf->readable->delete_tidy= 0; /* we never call oop_rd_delete_{tidy,kill} */ - ipf->readable->delete_kill= 0; + ipf->readable.on_readable= tailing_on_readable; + ipf->readable.on_cancel= tailing_on_cancel; + ipf->readable.try_read= tailing_try_read; + ipf->readable.delete_tidy= 0; /* we never call oop_rd_delete_{tidy,kill} */ + ipf->readable.delete_kill= 0; ipf->readable_callback= 0; ipf->readable_callback_user= 0; @@ -1583,7 +1671,7 @@ static void inputfile_tailing_start(InputFile *ipf) { assert(ipf->fd); int r= oop_rd_read(ipf->rd, &feedfile_rdstyle, MAX_LINE_FEEDFILE, - feedfile_got_article,ipf, feedfile_problem,ipf); + feedfile_got_article,ipf, feedfile_read_err, ipf); if (r) sysdie("unable start reading feedfile %s",ipf->path); } @@ -1697,6 +1785,12 @@ static void inputfile_tailing_stop(InputFile *ipf) { * ->8- */ +static void startup_set_input_file(InputFile *f) { + assert(!main_input_file); + main_input_file= f; + inputfile_tailing_start(f); +} + static void statemc_init(void) { struct stat stab, stabf; @@ -1713,7 +1807,7 @@ static void statemc_init(void) { memset(&fl,0,sizeof(fl)); fl.l_type= F_WRLCK; fl.l_whence= SEEK_SET; - r= fcntl(lockfd, F_SETLK, &fl); + int r= fcntl(lockfd, F_SETLK, &fl); if (r==-1) { if (errno==EACCES || errno==EAGAIN) { if (quiet_multiple) exit(0); @@ -1722,7 +1816,7 @@ static void statemc_init(void) { sysdie("fcntl F_SETLK lockfile %s", path_lock); } - xfstat_isreg(lockfd, &stabf, "lockfile"); + xfstat_isreg(lockfd, &stabf, path_lock, "lockfile"); int lock_noent; xlstat_isreg(path_lock, &stab, &lock_noent, "lockfile"); @@ -1760,7 +1854,7 @@ static void statemc_init(void) { int noent_f; InputFile *file_d= open_input_file(path_flushing); - if (file_d) xfstat_isreg(file_d->fd, &stab_d, "flushing file"); + if (file_d) xfstat_isreg(file_d->fd, &stab_d, path_flushing,"flushing file"); xlstat_isreg(feedfile, &stab_f, &noent_f, "feedfile"); @@ -1782,7 +1876,7 @@ static void statemc_init(void) { SMS(SEPARATED, 0, "found both old and current feed files"); } else { debug("startup: F exists, D ENOENT => Normal"); - FILE *file_f= open_input_file(feedfile); + InputFile *file_f= open_input_file(feedfile); if (!file_f) die("feed file vanished during startup"); startup_set_input_file(file_f); SMS(NORMAL, flushfail_retry_periods, "normal startup"); @@ -1790,6 +1884,26 @@ static void statemc_init(void) { } } +static void statemc_start_flush(const char *why) { /* Normal => Flushing */ + assert(sms == sm_NORMAL); + + debug("starting flush (%s) (%lu >?= %lu) (%d)", + why, + (unsigned long)(main_input_file ? main_input_file->offset : 0), + (unsigned long)target_max_feedfile_size, + sm_period_counter); + + int r= link(feedfile, path_flushing); + if (r) sysdie("link feedfile %s to flushing file %s", + feedfile, path_flushing); + /* => Hardlinked */ + + xunlink(feedfile, "old feedfile link"); + /* => Moved */ + + spawn_inndcomm_flush(why); /* => Flushing FLUSHING */ +} + static void statemc_period_poll(void) { if (!sm_period_counter) return; sm_period_counter--; @@ -1808,12 +1922,6 @@ static void statemc_period_poll(void) { } } -static void startup_set_input_file(InputFile *f) { - assert(!main_input_file); - main_input_file= f; - inputfile_tailing_start(f); -} - static int inputfile_is_done(InputFile *ipf) { if (!ipf) return 0; if (ipf->inprogress) return 0; /* new article in the meantime */ @@ -1825,27 +1933,33 @@ static void notice_processed(InputFile *ipf, const char *what, const char *spec) { #define RCI_NOTHING(x) /* nothing */ #define RCI_TRIPLE_FMT(x) " " #x "=" RCI_TRIPLE_FMT_BASE -#define RCI_TRIPLE_VALS(x) , RCI_TRIPLE_VALS_BASE(ipf->counts, .x) +#define RCI_TRIPLE_VALS(x) , RCI_TRIPLE_VALS_BASE(ipf->counts, [RC_##x]) + +#define CNT(art,rc) (ipf->counts[art_##art][RC_##rc]) - info("processed %s%s offered=%d(ch%d,nc%d) accepted=%d(ch%d+nc%d)" + info("processed %s%s read=%d(+%dbl,+%derr)" + " offered=%d(ch%d,nc%d) accepted=%d(ch%d+nc%d)" RESULT_COUNTS(RCI_NOTHING, RCI_TRIPLE_FMT) , - what,spec, - ipf->counts[art_Unchecked].sent + ipf->counts[art_Unsolicited].sent - , ipf->counts[art_Unchecked].sent, ipf->counts[art_Unsolicited].sent, - ipf->counts[art_Wanted].accepted + ipf->counts[art_Unsolicited].accepted - ,ipf->counts[art_Wanted].accepted,ipf->counts[art_Unsolicited].accepted + what, spec, + ipf->readcount_ok, ipf->readcount_blank, ipf->readcount_err, + CNT(Unchecked,sent) + CNT(Unsolicited,sent) + , CNT(Unchecked,sent), CNT(Unsolicited,sent), + CNT(Wanted,accepted) + CNT(Unsolicited,accepted) + , CNT(Wanted,accepted), CNT(Unsolicited,accepted) RESULT_COUNTS(RCI_NOTHING, RCI_TRIPLE_VALS) ); + +#undef CNT } static void statemc_check_backlog_done(void) { - InputFile *ipf= backlog_input_file(); + InputFile *ipf= backlog_input_file; if (!inputfile_is_done(ipf)) return; - const char *slash= strrchr(ipf->path, "/"); + const char *slash= strrchr(ipf->path, '/'); const char *leaf= slash ? slash+1 : ipf->path; - const char *under= strchr(slash, "_"); + const char *under= strchr(slash, '_'); const char *rest= under ? under+1 : leaf; if (!strncmp(rest,"backlog",7)) rest += 7; notice_processed(ipf,"backlog:",rest); @@ -1937,7 +2051,7 @@ static void open_defer(void) { /* truncate away any half-written records */ - xfstat_isreg(fileno(defer), &stab, "newly opened defer file"); + xfstat_isreg(fileno(defer), &stab, path_defer, "newly opened defer file"); if (stab.st_size > LONG_MAX) die("defer file %s size is far too large", path_defer); @@ -1952,7 +2066,7 @@ static void open_defer(void) { if (fseek(defer, truncto-1, SEEK_SET) < 0) sysdie("seek in defer file %s while truncating partial", path_defer); - r= getc(defer); + int r= getc(defer); if (r==EOF) { if (ferror(defer)) sysdie("failed read from defer file %s", path_defer); @@ -1985,21 +2099,26 @@ static void close_defer(void) { if (!defer) return; - xfstat(fileno(defer), &stab, "defer file"); + struct stat stab; + xfstat_isreg(fileno(defer), &stab, path_defer, "defer file"); if (fclose(defer)) sysfatal("could not close defer file %s", path_defer); defer= 0; + time_t now= xtime(); + char *backlog= xasprintf("%s_backlog_%lu.%lu", feedfile, - (unsigned long)now.tv_sec, + (unsigned long)now, (unsigned long)stab.st_ino); - if (link(path_defer, path_backlog)) + if (link(path_defer, backlog)) sysfatal("could not install defer file %s as backlog file %s", path_defer, backlog); if (unlink(path_defer)) sysdie("could not unlink old defer link %s to backlog file %s", path_defer, backlog); + free(backlog); + if (until_backlog_nextscan < 0 || until_backlog_nextscan > backlog_retry_minperiods + 1) until_backlog_nextscan= backlog_retry_minperiods + 1; @@ -2015,12 +2134,12 @@ static void search_backlog_file(void) { /* returns non-0 iff there are any backlog files */ glob_t gl; - int r; + int r, i; struct stat stab; const char *oldest_path=0; time_t oldest_mtime, now; - if (backlog_input_file) return 3; + if (backlog_input_file) return; try_again: @@ -2071,10 +2190,10 @@ static void search_backlog_file(void) { exit(0); } until_backlog_nextscan= backlog_spontaneous_rescan_periods; - return 0; + return; } - now= time(); if (now==-1) sysdie("time(2) failed"); + now= xtime(); double age= difftime(now, oldest_mtime); long age_deficiency= (backlog_retry_minperiods * PERIOD_SECONDS) - age; @@ -2089,7 +2208,7 @@ static void search_backlog_file(void) { } inputfile_tailing_start(backlog_input_file); until_backlog_nextscan= -1; - return 1; + return; } until_backlog_nextscan= age_deficiency / PERIOD_SECONDS; @@ -2100,7 +2219,7 @@ static void search_backlog_file(void) { debug("backlog scan: young age=%f deficiency=%ld nextscan=%d oldest=%s", age, age_deficiency, until_backlog_nextscan, oldest_path); - return 2; + return; } /*========== flushing the feed ==========*/ @@ -2110,7 +2229,7 @@ static pid_t inndcomm_child; static void *inndcomm_event(oop_source *lp, int fd, oop_event e, void *u) { assert(inndcomm_child); int status= xwaitpid(&inndcomm_child, "inndcomm"); - loop->cancel_fd(fd); + cancel_fd_read_except(fd); close(fd); assert(!flushing_input_file); @@ -2188,11 +2307,12 @@ void spawn_inndcomm_flush(const char *why) { /* Moved => Flushing */ if (pipe(pipefds)) sysdie("create pipe for inndcomm child sentinel"); - inndcomm_child= xfork(); + inndcomm_child= xfork("inndcomm child"); if (!inndcomm_child) { - static char flushargv[2]= { sitename, 0 }; + const char *flushargv[2]= { sitename, 0 }; char *reply; + int r; close(pipefds[0]); @@ -2241,8 +2361,9 @@ static void postfork(const char *what) { postfork_stdio(defer); } -#define EVERY(what, interval, body) \ - static const struct timeval what##_timeout = { 5, 0 }; \ +#define EVERY(what, interval_sec, interval_usec, body) \ + static const struct timeval what##_timeout = \ + { interval_sec, interval_usec }; \ static void what##_schedule(void); \ static void *what##_timedout(oop_source *lp, struct timeval tv, void *u) { \ { body } \ @@ -2252,13 +2373,13 @@ static void postfork(const char *what) { loop->on_time(loop, what##_timeout, what##_timedout, 0); \ } -EVERY(filepoll, {5,0}, { +EVERY(filepoll, 5,0, { if (main_input_file && main_input_file->readable_callback) filemon_callback(main_input_file); }); -#define DEBUGF_IPF(wh) " " #wh "=%p/%s:ip=%ld,off=%ld,fd=%d%s" \ -#define DEBUG_IPF(sh) \ +#define DEBUGF_IPF(wh) " " #wh "=%p/%s:ip=%ld,off=%ld,fd=%d%s" +#define DEBUG_IPF(wh) \ wh##_input_file, debug_ipf_path(wh##_input_file), \ wh##_input_file->inprogress, (long)wh##_input_file->offset, \ wh##_input_file->fd, wh##_input_file->rd ? "+" : "" @@ -2267,10 +2388,10 @@ static const char *debug_ipf_path(InputFile *ipf) { return slash ? slash+1 : ipf->path; } -EVERY(period, {PERIOD_SECONDS,0}, { +EVERY(period, PERIOD_SECONDS,0, { debug("PERIOD" " sms=%s[%d] conns=%d queue=%d until_connect=%d" - " input_files" DEBUGF_IPF(main) DEBUGF_IPF(old) DEBUGF_FMT(flushing) + " input_files" DEBUGF_IPF(main) DEBUGF_IPF(old) DEBUGF_IPF(flushing) " children connecting=%ld inndcomm_child" , sms_names[sms], sm_period_counter, @@ -2290,103 +2411,115 @@ EVERY(period, {PERIOD_SECONDS,0}, { /*========== option parsing ==========*/ +/*---------- generic option parser and logging ----------*/ + +static void vbadusage(const char *fmt, va_list al) NORET_PRINTF(1,0); +static void vbadusage(const char *fmt, va_list al) { + abort(); +} +static void badusage(const char *fmt, ...) NORET_PRINTF(1,2); +static void badusage(const char *fmt, ...) { + va_list al; + va_start(al,fmt); + vbadusage(fmt,al); +} + enum OptFlags { - of_seconds= 001000u; - of_boolean= 002000u; + of_seconds= 001000u, + of_boolean= 002000u, }; typedef struct Option Option; typedef void OptionParser(const Option*, const char *val); struct Option { - int short; - const char *long; + int shrt; + const char *lng; void *store; OptionParser *fn; - int noarg; + int noarg, intval; }; -void op_integer(const Option *o, const char *val) { +/*---------- specific option types ----------*/ + +static void op_integer(const Option *o, const char *val) { char *ep; errno= 0; unsigned long ul= strtoul(val,&ep,10); if (*ep || ep==val || errno || ul>INT_MAX) - badusage("bad integer value for %s",o->long); + badusage("bad integer value for %s",o->lng); int *store= o->store; *store= ul; } -void op_double(const Option *o, const char *val) { +static void op_double(const Option *o, const char *val) { int *store= o->store; char *ep; errno= 0; *store= strtod(val, &ep); if (*ep || ep==val || errno) - badusage("bad floating point value for %s",o->long); + badusage("bad floating point value for %s",o->lng); } -void op_string(const Option *o, const char *val) { - char **store= o->store; - free(*store); +static void op_string(const Option *o, const char *val) { + const char **store= o->store; *store= val; } -void op_seconds(const Option *o, const char *val) { +static void op_seconds(const Option *o, const char *val) { int *store= o->store; char *ep; + int unit; double v= strtod(val,&ep); - if (ep==val) badusage("bad time/duration value for %s",o->long); + if (ep==val) badusage("bad time/duration value for %s",o->lng); if (!*ep || !strcmp(ep,"s")) unit= 1; else if (!strcmp(ep,"m")) unit= 60; else if (!strcmp(ep,"h")) unit= 3600; else if (!strcmp(ep,"d")) unit= 86400; - else badusage("bad units %s for time/duration value for %s",ep,o->long); + else badusage("bad units %s for time/duration value for %s",ep,o->lng); v *= unit; v= ceil(v); - if (v > INT_MAX) badusage("time/duration value for %s out of range",o->long); + if (v > INT_MAX) badusage("time/duration value for %s out of range",o->lng); *store= v; } -void op_periods_rndup(const Option *o, const char *val) { +static void op_periods_rndup(const Option *o, const char *val) { int *store= o->store; op_seconds(o,val); *store += PERIOD_SECONDS-1; *store /= PERIOD_SECONDS; } -void op_periods_booltrue(const Option *o, const char *val) { - int *store= o->store; - *store= 1; -} -void op_periods_boolfalse(const Option *o, const char *val) { +static void op_setint(const Option *o, const char *val) { int *store= o->store; - *store= 0; + *store= o->intval; } +/*---------- specific options ----------*/ + static const Option options[]= { {'f',"feedfile", &feedfile, op_string }, -{'q',"quiet-multiple", &quiet_multiple, op_booltrue, 1 }, +{'q',"quiet-multiple", &quiet_multiple, op_setint, 1,1 }, -{ 0, "max-connections", &max_connections op_integer }, -{ 0, "max-queue-per-conn", &max_queue_per_conn op_integer }, +{ 0, "max-connections", &max_connections, op_integer }, +{ 0, "max-queue-per-conn", &max_queue_per_conn, op_integer }, -{ 0, "streaming", &try_stream, op_booltrue, 1 }, -{ 0, "no-streaming", &try_stream, op_boolfalse, 1 }, -{'P',"port", &port op_integer }, +{ 0, "streaming", &try_stream, op_setint, 1,1 }, +{ 0, "no-streaming", &try_stream, op_setint, 1,0 }, +{'P',"port", &port, op_integer }, { 0, "inndconf", &inndconffile, op_string }, -{'d',"daemon", &become_daemon, op_booltrue, 1 }, -{ 0, "no-daemon", &become_daemon, op_boolfalse, 1 }, +{'d',"daemon", &become_daemon, op_setint, 1,1 }, +{ 0, "no-daemon", &become_daemon, op_setint, 1,0 }, { 0, "no-check-proportion", &nocheck_thresh_pct, op_double }, { 0, "no-check-filter", &nocheck_decay_articles, op_double }, { 0, "reconnect-interval", &reconnect_delay_periods, op_periods_rndup }, { 0, "flush-retry-interval", &flushfail_retry_periods, op_periods_rndup }, -{ 0, "connection-timeout", &connection_timeout, op_seconds }, { 0, "inndcomm-timeout", &inndcomm_flush_timeout, op_seconds }, }; @@ -2405,25 +2538,25 @@ int main(int argc, char **argv) { arg++; char *equals= strchr(arg,'='); int len= equals ? (equals - arg) : strlen(arg); - for (o=options; o->long; o++) - if (strlen(o->long) == len && !memcmp(o->long,arg,len)) + for (o=options; o->lng; o++) + if (strlen(o->lng) == len && !memcmp(o->lng,arg,len)) goto found_long; badusage("unknown long option --%s",arg); found_long: if (o->noarg) { - if (equals) badusage("option --%s does not take a value",o->long); + if (equals) badusage("option --%s does not take a value",o->lng); arg= 0; } else if (equals) { arg= equals+1; } else { arg= *++argv; - if (!arg) badusage("option --%s needs a value",o->long); + if (!arg) badusage("option --%s needs a value",o->lng); } o->fn(o, arg); break; /* eaten the whole argument now */ } - for (o=options; o->long; o++) - if (a == o->short) + for (o=options; o->lng; o++) + if (a == o->shrt) goto found_short; badusage("unknown short option -%c",a); found_short: @@ -2432,7 +2565,7 @@ int main(int argc, char **argv) { } else { if (!*++arg) { arg= *++argv; - if (!arg) badusage("option -%c needs a value",o->short); + if (!arg) badusage("option -%c needs a value",o->shrt); } o->fn(o,arg); break; /* eaten the whole argument now */ @@ -2473,13 +2606,15 @@ int main(int argc, char **argv) { if (strchr(feedfile, c)) badusage("feed filename may not contain metacharacter %c",c); - loop= oop_sys_new(); - if (!loop) sysdie("could not create liboop event loop"); + oop_source_sys *sysloop= oop_sys_new(); + if (!sysloop) sysdie("could not create liboop event loop"); + loop= (oop_source*)sysloop; if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) sysdie("could not ignore SIGPIPE"); if (become_daemon) { + int i; for (i=3; i<255; i++) /* do this now before we open syslog, etc. */ close(i); @@ -2504,7 +2639,7 @@ int main(int argc, char **argv) { notice("starting"); - if (!filemon_init()) { + if (!filemon_method_init()) { warn("no file monitoring available, polling"); filepoll_schedule(); } @@ -2513,5 +2648,7 @@ int main(int argc, char **argv) { statemc_init(); - loop->execute. + void *r= oop_sys_run(sysloop); + assert(r == OOP_ERROR); + sysdie("event loop failed"); }