static int backlog_retry_minperiods=50;
static int backlog_spontrescan_periods=300;
static int spontaneous_flush_periods=100000;
+static int max_separated_periods=2000;
static int need_activity_periods=1000;
static double max_bad_data_ratio= 1; /* conv'd from percentage by main */
ArticleList queue;
long inprogress; /* includes queue.count and also articles in conns */
+ long autodefer; /* -1 means not doing autodefer */
int counts[art_MaxState][RCI_max];
int readcount_ok, readcount_blank, readcount_err;
}
}
+/*---------- expiry and deferral ----------*/
+
+static void article_defer(Article *art /* not on a queue */, int whichcount) {
+ open_defer();
+ if (fprintf(defer, "%s %s\n", TokenToText(art->token), art->messageid) <0
+ || fflush(defer))
+ sysfatal("write to defer file %s",path_defer);
+ article_done(art, whichcount);
+}
+
static int article_check_expired(Article *art /* must be queued, not conn */) {
ARTHANDLE *artdata= SMretrieve(art->token, RETR_STAT);
if (artdata) { SMfreearticle(artdata); return 0; }
}
}
+static void article_autodefer(InputFile *ipf, Article *art) {
+ ipf->autodefer++;
+ article_defer(art,-1);
+}
+
+static int has_article_in(const ArticleList *al, InputFile *ipf) {
+ Article *art;
+ for (art=LIST_HEAD(*al); art; art=LIST_NEXT(art))
+ if (art->ipf == ipf) return 1;
+ return 0;
+}
+
+static void autodefer_input_file(InputFile *ipf) {
+ ipf->autodefer= 0;
+
+ Article *art;
+ while ((art= LIST_REMHEAD(ipf->queue)))
+ article_autodefer(ipf, art);
+
+ if (ipf->inprogress) {
+ Conn *walk;
+ FOR_CONN(walk) {
+ if (has_article_in(&walk->waiting, ipf) ||
+ has_article_in(&walk->priority, ipf) ||
+ has_article_in(&walk->sent, ipf))
+ walk->quitting= -1;
+ }
+ while (ipf->inprogress) {
+ FOR_CONN(walk)
+ if (walk->quitting < 0) goto found;
+ abort(); /* where are they ?? */
+
+ found:
+ connfail(walk, "connection is stuck or crawling,"
+ " and we need to finish flush");
+ }
+ }
+}
+
/*========== article transmission ==========*/
static XmitDetails *xmit_core(Conn *conn, const char *data, int len,
}
}
-
/*========== handling responses from peer ==========*/
static const oop_rd_style peer_rd_style= {
code_streaming= 1;
case 436: /* IHAVE says try later */
GET_ARTICLE(0);
- open_defer();
- if (fprintf(defer, "%s %s\n", TokenToText(art->token), art->messageid) <0
- || fflush(defer))
- sysfatal("write to defer file %s",path_defer);
- article_done(art, RC_deferred);
+ article_defer(art, RC_deferred);
break;
}
memset(ipf,0,sizeof(*ipf));
ipf->fd= fd;
- strcpy(ipf->path, path);
+ ipf->autodefer= -1;
LIST_INIT(ipf->queue);
+ strcpy(ipf->path, path);
return ipf;
}
strcpy(art->messageid, space+1);
LIST_ADDTAIL(ipf->queue, art);
- if (ipf==backlog_input_file)
+ if (ipf->autodefer >= 0)
+ article_autodefer(ipf, art);
+ else if (ipf==backlog_input_file)
article_check_expired(art);
if (sms==sm_NORMAL && ipf==main_input_file &&
flushing_input_file= main_input_file;
main_input_file= open_input_file(feedfile);
if (!main_input_file) die("feedfile vanished during startup");
- SMS(SEPARATED, 0, "found both old and current feed files");
+ SMS(SEPARATED, max_separated_periods,
+ "found both old and current feed files");
} else {
debug("startup: F exists, D ENOENT => Normal");
InputFile *file_f= open_input_file(feedfile);
static int trigger_flush_ok(void) { /* => Flushing,FLUSHING, ret 1; or ret 0 */
switch (sms) {
+
case sm_NORMAL:
statemc_start_flush("periodic"); /* Normal => Flushing; => FLUSHING */
return 1;
+
case sm_FLUSHFAILED:
spawn_inndcomm_flush("retry"); /* Moved => Flushing; => FLUSHING */
return 1;
+
+ case sm_SEPARATED:
+ case sm_DROPPING:
+ warn("took too long to complete old feedfile after flush, autodeferring");
+ assert(flushing_input_file);
+ autodefer_input_file(flushing_input_file);
+ return 1;
+
default:
return 0;
}
char *inprog= completed
? xasprintf("%s","") /* GCC produces a stupid warning for printf("") ! */
: xasprintf(" inprogress=%ld", ipf->inprogress);
+ char *autodefer= ipf->autodefer >= 0
+ ? xasprintf(" autodeferred=%ld", ipf->autodefer)
+ : xasprintf("%s","");
- info("%s %s%s read=%d (+bl=%d,+err=%d)%s"
+ info("%s %s%s read=%d (+bl=%d,+err=%d)%s%s"
" offered=%d (ch=%d,nc=%d) accepted=%d (ch=%d,nc=%d)"
RESULT_COUNTS(RCI_NOTHING, RCI_TRIPLE_FMT)
,
completed?"completed":"processed", what, spec,
- ipf->readcount_ok, ipf->readcount_blank, ipf->readcount_err, inprog,
+ ipf->readcount_ok, ipf->readcount_blank, ipf->readcount_err,
+ inprog, autodefer,
CNT(Unchecked,sent) + CNT(Unsolicited,sent)
, CNT(Unchecked,sent), CNT(Unsolicited,sent),
CNT(Wanted,accepted) + CNT(Unsolicited,accepted)
);
free(inprog);
+ free(autodefer);
#undef CNT
}
notice("flush complete");
SMS(NORMAL, spontaneous_flush_periods, "flush complete");
} else if (sms==sm_DROPPING) {
- SMS(DROPPED, 0, "old flush complete");
+ SMS(DROPPED, max_separated_periods, "old flush complete");
search_backlog_file();
notice("feed dropped, but will continue until backlog is finished");
}
main_input_file= 0;
if (flushing_input_file) {
- SMS(DROPPING, 0, "feed dropped by innd, but must finish last flush");
+ SMS(DROPPING, max_separated_periods,
+ "feed dropped by innd, but must finish last flush");
} else {
close_defer();
SMS(DROPPED, 0, "feed dropped by innd");
die("flush succeeded but feedfile %s does not exist!", feedfile);
if (flushing_input_file) {
- SMS(SEPARATED, spontaneous_flush_periods, "recovery flush complete");
+ SMS(SEPARATED, max_separated_periods, "recovery flush complete");
} else {
close_defer();
SMS(NORMAL, spontaneous_flush_periods, "flush complete");
const char *slash= strrchr(ipf->path,'/');
const char *path= slash ? slash+1 : ipf->path;
- return xasprintf("%p/%s:queue=%d,ip=%ld,off=%ld,fd=%d%s%s",
+ return xasprintf("%p/%s:queue=%d,ip=%ld,autodef=%ld,off=%ld,fd=%d%s%s",
ipf, path,
- ipf->queue.count, ipf->inprogress, (long)ipf->offset,
- ipf->fd,
+ ipf->queue.count, ipf->inprogress, ipf->autodefer,
+ (long)ipf->offset, ipf->fd,
ipf->rd ? "" : ",!rd",
ipf->skippinglong ? "*skiplong" : "");
}
{0,"earliest-deferred-retry","PERIOD", &backlog_retry_minperiods, op_seconds },
{0,"backlog-rescan-interval","PERIOD",&backlog_spontrescan_periods,op_seconds},
{0,"max-flush-interval", "PERIOD", &spontaneous_flush_periods,op_seconds },
+{0,"flush-finish-timeout", "PERIOD", &max_separated_periods, op_seconds },
{0,"idle-timeout", "PERIOD", &need_activity_periods, op_seconds },
{0,"max-bad-input-data-ratio","PERCENT", &max_bad_data_ratio, op_double },
convert_to_periods_rndup(&backlog_retry_minperiods);
convert_to_periods_rndup(&backlog_spontrescan_periods);
convert_to_periods_rndup(&spontaneous_flush_periods);
+ convert_to_periods_rndup(&max_separated_periods);
convert_to_periods_rndup(&need_activity_periods);
if (max_bad_data_ratio < 0 || max_bad_data_ratio > 100)