/*
* TODO
* - close idle connections
+ * - cope better with garbage in feed file
+ * - cope better with NULs in feed file
* - -k kill mode ?
*/
typedef struct Article Article;
typedef struct InputFile InputFile;
typedef struct XmitDetails XmitDetails;
+typedef struct Filemon_Perfile Filemon_Perfile;
typedef enum StateMachineState StateMachineState;
DEFLIST(Conn);
/*----- function predeclarations -----*/
-static void conn_check_work(Conn *conn);
+static void conn_maybe_write(Conn *conn);
static void conn_make_some_xmits(Conn *conn);
static void *conn_write_some_xmits(Conn *conn);
static void xmit_free(XmitDetails *d);
-static int filemon_init(void);
-static void filemon_setfile(int mainfeed_fd, const char *mainfeed_path);
-static void filemon_callback(void);
-
static void statemc_setstate(StateMachineState newsms, int periods,
const char *forlog, const char *why);
static void check_master_queue(void);
static void queue_check_input_done(void);
+static void statemc_start_flush(const char *why); /* Normal => Flushing */
+static void statemc_check_flushing_done(void);
+static void statemc_check_backlog_done(void);
+
static void postfork(const char *what);
static void postfork_inputfile(InputFile *ipf);
+static void open_defer(void);
+
+static void inputfile_tailing_start(InputFile *ipf);
+static void inputfile_tailing_stop(InputFile *ipf);
+
+static int filemon_init(void);
+static void filemon_start(InputFile *ipf);
+static void filemon_stop(InputFile *ipf);
+static void filemon_callback(InputFile *ipf);
+
/*----- configuration options -----*/
static char *sitename, *feedfile;
static int quiet_multiple=0, become_daemon=1;
static int max_connections=10, max_queue_per_conn=200;
+static int target_max_feedfile_size=100000;
+
+static double max_bad_data_ratio= 0.01;
+static int max_bad_data_initial= 30;
+ /* in one corrupt 4096-byte block the number of newlines has
+ * mean 16 and standard deviation 3.99. 30 corresponds to z=+3.5 */
static int connection_setup_timeout=200, port=119, try_stream=1;
static int inndcomm_flush_timeout=100;
void *readable_callback_user;
int fd;
- struct Filemon_Perfile *filemon;
+ Filemon_Perfile *filemon;
oop_read *rd;
long inprogress; /* no. of articles read but not processed */
off_t offset;
int counts[art_MaxState][RCI_max];
+ int readcount_ok, readcount_blank, readcount_err;
char path[];
};
struct Conn {
ISNODE(Conn);
int fd, max_queue, stream, quitting;
- ArticleList queue; /* not yet told peer, or CHECK said send it */
+ ArticleList waiting; /* not yet told peer */
+ ArticleList priority; /* peer says send it now */
ArticleList sent; /* offered/transmitted - in xmit or waiting reply */
struct iovec xmit[CONNIOVS];
XmitDetails xmitd[CONNIOVS];
setnonblock(conn->fd, 1);
conn->max_queue= conn->stream ? max_queue_per_conn : 1;
LIST_ADDHEAD(conns, conn);
- notice("#%d connected %s", conn->fd, conn->stream ? "streaming" : "plain");
+ notice("C%d connected %s", conn->fd, conn->stream ? "streaming" : "plain");
connect_attempt_discard();
check_master_queue();
return 0;
connect_attempt_discard();
}
+static int allow_connect_start(void) {
+ return conns.count < max_connections
+ && !connecting_child
+ && !until_connect;
+}
+
static void connect_start(void) {
assert(!connecting_sockets[0]);
assert(!connecting_sockets[1]);
Conn *walk, *use=0;
int spare;
+
+ /* Find a connection to offer this article. We prefer a busy
+ * connection to an idle one, provided it's not full. We take the
+ * first (oldest) and since that's stable, it will mean we fill up
+ * connections in order. That way if we have too many
+ * connections, the spare ones will go away eventually.
+ */
for (walk=LIST_HEAD(conns); walk; walk=LIST_NEXT(walk)) {
- int inqueue= walk->sent.count + walk->queue.count;
+ int inqueue= walk->sent.count + walk->priority.count
+ + walk->waiting.count;
spare= walk->max_queue - inqueue;
assert(inqueue <= max_queue_per_conn);
assert(spare >= 0);
if (use) {
while (spare>0) {
Article *art= LIST_REMHEAD(queue);
- LIST_ADDTAIL(use->queue, art);
+ LIST_ADDTAIL(use->waiting, art);
spare--;
}
- conn_check_work(use);
- } else if (conns.count < max_connections &&
- !connecting_child && !until_connect) {
+ conn_maybe_write(use);
+ } else if (allow_connect_start()) {
until_connect= reconnect_delay_periods;
connect_start();
break;
}
static void *conn_writeable(oop_source *l, int fd, oop_event ev, void *u) {
- conn_check_work(u);
+ conn_maybe_write(u);
return OOP_CONTINUE;
}
-static void conn_check_work(Conn *conn) {
+static void conn_maybe_write(Conn *conn) {
void *rp= 0;
for (;;) {
conn_make_some_xmits(conn);
int requeue[art_MaxState];
Article *art;
- while ((art= LIST_REMHEAD(conn->queue))) LIST_ADDTAIL(queue, art);
+ while ((art= LIST_REMHEAD(conn->priority))) LIST_ADDTAIL(queue, art);
+ while ((art= LIST_REMHEAD(conn->waiting))) LIST_ADDTAIL(queue, art);
while ((art= LIST_REMHEAD(conn->sent))) {
requeue[art->state]++;
if (art->state==art_Unsolicited) art->state= art_Unchecked;
xmit_free(d);
char *m= xvasprintf(fmt,al);
- warn("#%d connection failed, requeueing " RCI_TRIPLE_FMT_BASE ": %s",
+ warn("C%d connection failed (requeueing " RCI_TRIPLE_FMT_BASE "): %s",
conn->fd, RCI_TRIPLE_VALS_BASE(requeue, /*nothing*/), m);
free(m);
if (conn->xmitu+5 > CONNIOVS)
break;
- Article *art= LIST_REMHEAD(queue);
+ Article *art= LIST_REMHEAD(conn->priority);
+ if (!art) art= LIST_REMHEAD(conn->waiting);
if (!art) break;
if (art->state >= art_Wanted || (conn->stream && nocheck)) {
XMIT_LITERAL("\r\n");
assert(art->state == art_Unchecked);
- art->ipf->counts[art->state][RCI_sent]++;
+ art->ipf->counts[art->state][RC_sent]++;
LIST_ADDTAIL(conn->sent, art);
}
}
if (code!=205 && code!=503) {
connfail(conn, "peer gave unexpected response to QUIT: %s", sani);
} else {
- notice("#%d idle connection closed\n");
- assert(!conn->queue.count);
+ notice("C%d idle connection closed\n");
+ assert(!conn->waiting.count);
+ assert(!conn->priority.count);
assert(!conn->sent.count);
assert(!conn->xmitu);
LIST_REMOVE(conns,conn);
assert(art->state == art_Unchecked);
art->ipf->counts[art->state][RC_accepted]++;
art->state= art_Wanted;
- LIST_ADDTAIL(conn->queue, art);
+ LIST_ADDTAIL(conn->priority, art);
break;
case 431: /* CHECK or TAKETHIS says try later */
}
- check_check_work(conn);
+ conn_maybe_write(conn);
+ check_master_queue();
return OOP_CONTINUE;
}
InputFile *ipf= xmalloc(sizeof(*ipf) + strlen(path) + 1);
memset(ipf,0,sizeof(*ipf));
- ipf->readable.on_readable= tailing_on_readable;
- ipf->readable.on_cancel= tailing_on_cancel;
- ipf->readable.try_read= tailing_try_read;
-
ipf->fd= fd;
strcpy(ipf->path, path);
/*---------- dealing with articles read in the input file ----------*/
-typedef void *feedfile_got_article(oop_source *lp, oop_read *rd,
- oop_rd_event ev, const char *errmsg,
- int errnoval,
+static void *feedfile_got_bad_data(InputFile *ipf, off_t offset,
const char *data, size_t recsz,
- void *ipf_v) {
+ const char *how) {
+ warn("corrupted file: %s, offset %lu: %s: %s",
+ ipf->path, (unsigned long)offset, how, sanitise(data));
+ ipf->readcount_err++;
+ if (ipf->readcount_err > max_bad_data_initial +
+ (ipf->readcount_ok+ipf->readcount_blank) / max_bad_data_ratio)
+ die("too much garbage in input file! (%d errs, %d ok, %d blank)",
+ ipf->readcount_err, ipf->readcount_ok, ipf->readcount_blank);
+ return OOP_CONTINUE;
+}
+
+static void *feedfile_got_article(oop_source *lp, oop_read *rd,
+ oop_rd_event ev, const char *errmsg,
+ int errnoval, const char *data, size_t recsz,
+ void *ipf_v) {
InputFile *ipf= ipf_v;
Article *art;
char tokentextbuf[sizeof(TOKEN)*2+3];
if (!data) { feedfile_eof(ipf); return OOP_CONTINUE; }
- if (data[0] && data[0]!=' ') {
- char *space= strchr(data,' ');
- int tokenlen= space-data;
- int midlen= (int)recsz-tokenlen-1;
- if (midlen < 0) goto bad_data;
-
- if (tokenlen != sizeof(TOKEN)*2+2) goto bad_data;
- memcpy(tokentextbuf, data, tokenlen);
- tokentextbuf[tokenlen]= 0;
- if (!IsToken(tokentextbuf)) goto bad_data;
-
- art= xmalloc(sizeof(*art) - 1 + midlen + 1);
- art->offset= ipf->offset;
- art->blanklen= recsz;
- art->midlen= midlen;
- art->state= art_Unchecked;
- art->ipf= ipf; ipf->inprogress++;
- art->token= TextToToken(tokentextbuf);
- strcpy(art->messageid, space+1);
- LIST_ADDTAIL(queue, art);
- }
+ off_t old_offset= ipf->offset;
ipf->offset += recsz + 1;
- if (sms==sm_NORMAL && ipf==main_input_file &&
- ipf->offset >= flush_threshold)
- statemc_start_flush("feed file size");
-
- check_master_queue();
-}
+#define BAD_DATA(m) return feedfile_got_bad_data(ipf,old_offset,data,recsz,m);
-static void statemc_start_flush(const char *why) { /* Normal => Flushing */
- assert(sms == sm_NORMAL);
+ if (memchr(data,'\0',recsz)) BAD_DATA("nul byte");
+ if (!recsz) BAD_DATA("empty line");
- debug("starting flush (%s) (%lu >= %lu) (%d)",
- why,
- (unsigned long)ipf->offset, (unsigned long)flush_threshold,
- sm_period_counter);
-
- int r= link(feedfile, duct_path);
- if (r) sysdie("link feedfile %s to flushing file %s", feedfile,
- path_duct);
- /* => Hardlinked */
+ if (data[0]==' ') {
+ if (strspn(data," ") != recsz) BAD_DATA("line partially blanked");
+ ipf->readcount_blank++;
+ return OOP_CONTINUE;
+ }
+
+ char *space= strchr(data,' ');
+ int tokenlen= space-data;
+ int midlen= (int)recsz-tokenlen-1;
+ if (midlen <= 0) BAD_DATA("no room for messageid");
+
+ if (tokenlen != sizeof(TOKEN)*2+2) BAD_DATA("token wrong length");
+ memcpy(tokentextbuf, data, tokenlen);
+ tokentextbuf[tokenlen]= 0;
+ if (!IsToken(tokentextbuf)) BAD_DATA("token wrong syntax");
+
+ ipf->readcount_ok++;
+
+ art= xmalloc(sizeof(*art) - 1 + midlen + 1);
+ art->offset= ipf->offset;
+ art->blanklen= recsz;
+ art->midlen= midlen;
+ art->state= art_Unchecked;
+ art->ipf= ipf; ipf->inprogress++;
+ art->token= TextToToken(tokentextbuf);
+ strcpy(art->messageid, space+1);
+ LIST_ADDTAIL(queue, art);
- xunlink(feedfile, "old feedfile link");
- /* => Moved */
+ if (sms==sm_NORMAL && ipf==main_input_file &&
+ ipf->offset >= target_max_feedfile_size)
+ statemc_start_flush("feed file size");
- spawn_inndcomm_flush(why); /* => Flushing FLUSHING */
+ check_master_queue();
+ return OOP_CONTINUE;
}
/*========== tailing input file ==========*/
-static void filemon_start(InputFile *ipf) {
- assert(!ipf->filemon);
-
- ipf->filemon= xmalloc(sizeof(*ipf->filemon));
- memset(ipf->filemon, 0, sizeof(*ipf->filemon));
- filemon_method_startfile(ipf, ipf->filemon);
-}
-
-static void filemon_stop(InputFile *ipf) {
- if (!ipf->filemon) return;
- filemon_method_stopfile(ipf, ipf->filemon);
- free(ipf->filemon);
- ipf->filemon= 0;
-}
-
-static void filemon_callback(InputFile *ipf) {
- ipf->readable_callback(ipf->readable_callback_user);
-}
-
static void *tailing_rable_call_time(oop_source *loop, struct timeval tv,
void *user) {
InputFile *ipf= user;
- return ipf->readable_callback(ipf->readable_callback_user);
+ return ipf->readable_callback(loop, &ipf->readable,
+ ipf->readable_callback_user);
}
-static void on_cancel(struct oop_readable *rable) {
+static void tailing_on_cancel(struct oop_readable *rable) {
InputFile *ipf= (void*)rable;
- if (ipf->filemon) filemon_stopfile(ipf);
+ if (ipf->filemon) filemon_stop(ipf);
loop->cancel_time(loop, OOP_TIME_NOW, tailing_rable_call_time, ipf);
ipf->readable_callback= 0;
}
tailing_on_cancel(rable);
ipf->readable_callback= cb;
ipf->readable_callback_user= user;
- filemon_startfile(ipf);
+ filemon_start(ipf);
tailing_queue_readable(ipf);
return 0;
static int filemon_inotify_wdmax;
static InputFile **filemon_inotify_wd2ipf;
-typedef struct Filemon_Perfile {
+struct Filemon_Perfile {
int wd;
-} Filemon_Inotify_Perfile;
+};
static void filemon_method_startfile(InputFile *ipf, Filemon_Perfile *pf) {
int wd= inotify_add_watch(filemon_inotify_fd, ipf->path, IN_MODIFY);
return 1;
}
-#endif /* HAVE_INOTIFY && !HAVE_FILEMON *//
+#endif /* HAVE_INOTIFY && !HAVE_FILEMON */
/*---------- filemon dummy implementation ----------*/
#if !defined(HAVE_FILEMON)
-typedef struct Filemon_Perfile { int dummy; } Filemon_Dummy_Perfile;
+struct Filemon_Perfile { int dummy; };
static int filemon_method_init(void) { return 0; }
static void filemon_method_startfile(InputFile *ipf, Filemon_Perfile *pf) { }
#endif /* !HAVE_FILEMON */
+/*---------- filemon generic interface ----------*/
+
+static void filemon_start(InputFile *ipf) {
+ assert(!ipf->filemon);
+
+ ipf->filemon= xmalloc(sizeof(*ipf->filemon));
+ memset(ipf->filemon, 0, sizeof(*ipf->filemon));
+ filemon_method_startfile(ipf, ipf->filemon);
+}
+
+static void filemon_stop(InputFile *ipf) {
+ if (!ipf->filemon) return;
+ filemon_method_stopfile(ipf, ipf->filemon);
+ free(ipf->filemon);
+ ipf->filemon= 0;
+}
+
+static void filemon_callback(InputFile *ipf) {
+ ipf->readable_callback(loop, &ipf->readable, ipf->readable_callback_user);
+}
+
/*---------- interface to start and stop an input file ----------*/
static const oop_rd_style feedfile_rdstyle= {
OOP_RD_DELIM_STRIP, '\n',
- OOP_RD_NUL_FORBID,
+ OOP_RD_NUL_PERMIT,
OOP_RD_SHORTREC_EOF,
};
static void inputfile_tailing_start(InputFile *ipf) {
assert(!ipf->fd);
ipf->readable->on_readable= tailing_on_readable;
- ipf->readable->on_cancel= tailing_on_cancel;
- ipf->readable->try_read= tailing_try_read;
- ipf->readable->delete_tidy= 0; /* we never call oop_rd_delete_{tidy,kill} */
- ipf->readable->delete_kill= 0;
+ ipf->readable.on_cancel= tailing_on_cancel;
+ ipf->readable.try_read= tailing_try_read;
+ ipf->readable.delete_tidy= 0; /* we never call oop_rd_delete_{tidy,kill} */
+ ipf->readable.delete_kill= 0;
ipf->readable_callback= 0;
ipf->readable_callback_user= 0;
}
}
+static void statemc_start_flush(const char *why) { /* Normal => Flushing */
+ assert(sms == sm_NORMAL);
+
+ debug("starting flush (%s) (%lu >= %lu) (%d)",
+ why,
+ (unsigned long)ipf->offset, (unsigned long)flush_threshold,
+ sm_period_counter);
+
+ int r= link(feedfile, duct_path);
+ if (r) sysdie("link feedfile %s to flushing file %s", feedfile,
+ path_duct);
+ /* => Hardlinked */
+
+ xunlink(feedfile, "old feedfile link");
+ /* => Moved */
+
+ spawn_inndcomm_flush(why); /* => Flushing FLUSHING */
+}
+
static void statemc_period_poll(void) {
if (!sm_period_counter) return;
sm_period_counter--;
#define RCI_TRIPLE_FMT(x) " " #x "=" RCI_TRIPLE_FMT_BASE
#define RCI_TRIPLE_VALS(x) , RCI_TRIPLE_VALS_BASE(ipf->counts, .x)
- info("processed %s%s offered=%d(ch%d,nc%d) accepted=%d(ch%d+nc%d)"
+ info("processed %s%s read=%d(+%dbl,+%derr)"
+ " offered=%d(ch%d,nc%d) accepted=%d(ch%d+nc%d)"
RESULT_COUNTS(RCI_NOTHING, RCI_TRIPLE_FMT)
,
- what,spec,
+ what, spec,
+ ipf->readcount_ok, ipf->readcount_blank, ipf->readcount_err,
ipf->counts[art_Unchecked].sent + ipf->counts[art_Unsolicited].sent
, ipf->counts[art_Unchecked].sent, ipf->counts[art_Unsolicited].sent,
ipf->counts[art_Wanted].accepted + ipf->counts[art_Unsolicited].accepted
~ian / inn-innduct.git / blobdiff