Forcibly deal with stuck articles from flushing_file after flush

diff --git a/backends/innduct.c b/backends/innduct.c
index c4d42e1300e2f5101005b37b6cc9e37924a98b2c..047d99e6f9f3563903ac17e2310fa1470f93066e 100644 (file)
--- a/backends/innduct.c
+++ b/backends/innduct.c
@@ -375,6 +375,7 @@ static int flushfail_retry_periods=1000;
 static int backlog_retry_minperiods=50;
 static int backlog_spontrescan_periods=300;
 static int spontaneous_flush_periods=100000;
+static int max_separated_periods=2000;
 static int need_activity_periods=1000;
 
 static double max_bad_data_ratio= 1; /* conv'd from percentage by main */
@@ -453,6 +454,7 @@ struct InputFile {
 
   ArticleList queue;
   long inprogress; /* includes queue.count and also articles in conns */
+  long autodefer; /* -1 means not doing autodefer */
 
   int counts[art_MaxState][RCI_max];
   int readcount_ok, readcount_blank, readcount_err;
@@ -1525,6 +1527,16 @@ static void conn_maybe_write(Conn *conn)  {
   }
 }
 
+/*---------- expiry and deferral ----------*/
+
+static void article_defer(Article *art /* not on a queue */, int whichcount) {
+  open_defer();
+  if (fprintf(defer, "%s %s\n", TokenToText(art->token), art->messageid) <0
+      || fflush(defer))
+    sysfatal("write to defer file %s",path_defer);
+  article_done(art, whichcount);
+}
+
 static int article_check_expired(Article *art /* must be queued, not conn */) {
   ARTHANDLE *artdata= SMretrieve(art->token, RETR_STAT);
   if (artdata) { SMfreearticle(artdata); return 0; }
@@ -1546,6 +1558,45 @@ static void inputfile_queue_check_expired(InputFile *ipf) {
   }
 }
 
+static void article_autodefer(InputFile *ipf, Article *art) {
+  ipf->autodefer++;
+  article_defer(art,-1);
+}
+
+static int has_article_in(const ArticleList *al, InputFile *ipf) {
+  Article *art;
+  for (art=LIST_HEAD(*al); art; art=LIST_NEXT(art))
+    if (art->ipf == ipf) return 1;
+  return 0;
+}
+
+static void autodefer_input_file(InputFile *ipf) {
+  ipf->autodefer= 0;
+
+  Article *art;
+  while ((art= LIST_REMHEAD(ipf->queue)))
+    article_autodefer(ipf, art);
+
+  if (ipf->inprogress) {
+    Conn *walk;
+    FOR_CONN(walk) {
+      if (has_article_in(&walk->waiting,  ipf) ||
+         has_article_in(&walk->priority, ipf) ||
+         has_article_in(&walk->sent,     ipf))
+       walk->quitting= -1;
+    }
+    while (ipf->inprogress) {
+      FOR_CONN(walk)
+       if (walk->quitting < 0) goto found;
+      abort(); /* where are they ?? */
+
+    found:
+      connfail(walk, "connection is stuck or crawling,"
+              " and we need to finish flush");
+    }
+  }
+}
+
 /*========== article transmission ==========*/
 
 static XmitDetails *xmit_core(Conn *conn, const char *data, int len,
@@ -1674,7 +1725,6 @@ static void conn_make_some_xmits(Conn *conn) {
   }
 }
 
-
 /*========== handling responses from peer ==========*/
 
 static const oop_rd_style peer_rd_style= {
@@ -1898,11 +1948,7 @@ static void *peer_rd_ok(oop_source *lp, oop_read *oread, oop_rd_event ev,
     code_streaming= 1;
   case 436: /* IHAVE says try later */
     GET_ARTICLE(0);
-    open_defer();
-    if (fprintf(defer, "%s %s\n", TokenToText(art->token), art->messageid) <0
-       || fflush(defer))
-      sysfatal("write to defer file %s",path_defer);
-    article_done(art, RC_deferred);
+    article_defer(art, RC_deferred);
     break;
 
   }
@@ -1943,8 +1989,9 @@ static InputFile *open_input_file(const char *path) {
   memset(ipf,0,sizeof(*ipf));
 
   ipf->fd= fd;
-  strcpy(ipf->path, path);
+  ipf->autodefer= -1;
   LIST_INIT(ipf->queue);
+  strcpy(ipf->path, path);
 
   return ipf;
 }
@@ -2044,7 +2091,9 @@ static void *feedfile_got_article(oop_source *lp, oop_read *rd,
   strcpy(art->messageid, space+1);
   LIST_ADDTAIL(ipf->queue, art);
 
-  if (ipf==backlog_input_file)
+  if (ipf->autodefer >= 0)
+    article_autodefer(ipf, art);
+  else if (ipf==backlog_input_file)
     article_check_expired(art);
 
   if (sms==sm_NORMAL && ipf==main_input_file &&
@@ -2484,7 +2533,8 @@ static void statemc_init(void) {
       flushing_input_file= main_input_file;
       main_input_file= open_input_file(feedfile);
       if (!main_input_file) die("feedfile vanished during startup");
-      SMS(SEPARATED, 0, "found both old and current feed files");
+      SMS(SEPARATED, max_separated_periods,
+         "found both old and current feed files");
     } else {
       debug("startup: F exists, D ENOENT => Normal");
       InputFile *file_f= open_input_file(feedfile);
@@ -2517,12 +2567,22 @@ static void statemc_start_flush(const char *why) { /* Normal => Flushing */
 
 static int trigger_flush_ok(void) { /* => Flushing,FLUSHING, ret 1; or ret 0 */
   switch (sms) {
+
   case sm_NORMAL:
     statemc_start_flush("periodic"); /* Normal => Flushing; => FLUSHING */
     return 1;
+
   case sm_FLUSHFAILED:
     spawn_inndcomm_flush("retry"); /* Moved => Flushing; => FLUSHING */
     return 1;
+
+  case sm_SEPARATED:
+  case sm_DROPPING:
+    warn("took too long to complete old feedfile after flush, autodeferring");
+    assert(flushing_input_file);
+    autodefer_input_file(flushing_input_file);
+    return 1;
+
   default:
     return 0;
   }
@@ -2558,13 +2618,17 @@ static void notice_processed(InputFile *ipf, int completed,
   char *inprog= completed
     ? xasprintf("%s","") /* GCC produces a stupid warning for printf("") ! */
     : xasprintf(" inprogress=%ld", ipf->inprogress);
+  char *autodefer= ipf->autodefer >= 0
+    ? xasprintf(" autodeferred=%ld", ipf->autodefer)
+    : xasprintf("%s","");
 
-  info("%s %s%s read=%d (+bl=%d,+err=%d)%s"
+  info("%s %s%s read=%d (+bl=%d,+err=%d)%s%s"
        " offered=%d (ch=%d,nc=%d) accepted=%d (ch=%d,nc=%d)"
        RESULT_COUNTS(RCI_NOTHING, RCI_TRIPLE_FMT)
        ,
        completed?"completed":"processed", what, spec,
-       ipf->readcount_ok, ipf->readcount_blank, ipf->readcount_err, inprog,
+       ipf->readcount_ok, ipf->readcount_blank, ipf->readcount_err,
+       inprog, autodefer,
        CNT(Unchecked,sent) + CNT(Unsolicited,sent)
        , CNT(Unchecked,sent), CNT(Unsolicited,sent),
        CNT(Wanted,accepted) + CNT(Unsolicited,accepted)
@@ -2573,6 +2637,7 @@ static void notice_processed(InputFile *ipf, int completed,
        );
 
   free(inprog);
+  free(autodefer);
 
 #undef CNT
 }
@@ -2622,7 +2687,7 @@ static void statemc_check_flushing_done(void) {
     notice("flush complete");
     SMS(NORMAL, spontaneous_flush_periods, "flush complete");
   } else if (sms==sm_DROPPING) {
-    SMS(DROPPED, 0, "old flush complete");
+    SMS(DROPPED, max_separated_periods, "old flush complete");
     search_backlog_file();
     notice("feed dropped, but will continue until backlog is finished");
   }
@@ -2956,7 +3021,8 @@ static void *inndcomm_event(oop_source *lp, int fd, oop_event e, void *u) {
       main_input_file= 0;
 
       if (flushing_input_file) {
-       SMS(DROPPING, 0, "feed dropped by innd, but must finish last flush");
+       SMS(DROPPING, max_separated_periods,
+           "feed dropped by innd, but must finish last flush");
       } else {
        close_defer();
        SMS(DROPPED, 0, "feed dropped by innd");
@@ -2974,7 +3040,7 @@ static void *inndcomm_event(oop_source *lp, int fd, oop_event e, void *u) {
        die("flush succeeded but feedfile %s does not exist!", feedfile);
 
       if (flushing_input_file) {
-       SMS(SEPARATED, spontaneous_flush_periods, "recovery flush complete");
+       SMS(SEPARATED, max_separated_periods, "recovery flush complete");
       } else {
        close_defer();
        SMS(NORMAL, spontaneous_flush_periods, "flush complete");
@@ -3126,10 +3192,10 @@ static char *debug_report_ipf(InputFile *ipf) {
   const char *slash= strrchr(ipf->path,'/');
   const char *path= slash ? slash+1 : ipf->path;
 
-  return xasprintf("%p/%s:queue=%d,ip=%ld,off=%ld,fd=%d%s%s",
+  return xasprintf("%p/%s:queue=%d,ip=%ld,autodef=%ld,off=%ld,fd=%d%s%s",
                   ipf, path,
-                  ipf->queue.count, ipf->inprogress, (long)ipf->offset,
-                  ipf->fd,
+                  ipf->queue.count, ipf->inprogress, ipf->autodefer,
+                  (long)ipf->offset, ipf->fd,
                   ipf->rd ? "" : ",!rd",
                   ipf->skippinglong ? "*skiplong" : "");
 }
@@ -3483,6 +3549,7 @@ static const Option innduct_options[]= {
 {0,"earliest-deferred-retry","PERIOD", &backlog_retry_minperiods, op_seconds },
 {0,"backlog-rescan-interval","PERIOD",&backlog_spontrescan_periods,op_seconds},
 {0,"max-flush-interval",     "PERIOD", &spontaneous_flush_periods,op_seconds },
+{0,"flush-finish-timeout",   "PERIOD", &max_separated_periods,    op_seconds },
 {0,"idle-timeout",           "PERIOD", &need_activity_periods,    op_seconds },
 
 {0,"max-bad-input-data-ratio","PERCENT", &max_bad_data_ratio,   op_double    },
@@ -3546,6 +3613,7 @@ int main(int argc, char **argv) {
   convert_to_periods_rndup(&backlog_retry_minperiods);
   convert_to_periods_rndup(&backlog_spontrescan_periods);
   convert_to_periods_rndup(&spontaneous_flush_periods);
+  convert_to_periods_rndup(&max_separated_periods);
   convert_to_periods_rndup(&need_activity_periods);
 
   if (max_bad_data_ratio < 0 || max_bad_data_ratio > 100)

diff --git a/doc/man/innduct.8 b/doc/man/innduct.8
index c6f8b701974f8bb7208a14e150d1a39059a2b3d2..21705a870f543ab6155731627e3eb2f998704db8 100644 (file)
--- a/doc/man/innduct.8
+++ b/doc/man/innduct.8
@@ -255,6 +255,18 @@ threshold.
 The default is
 .BR 100000s .
 .TP
+.BI \-\-flush-finish-timeout= FLUSH-FINISH-PERIOD
+If we flushed
+.IR FLUSH-FINISH-PERIOD
+ago, and are still trying to finish processing articles that were
+written to the old feed file, we forcibly and violently make sure that
+we do by abandoning and deferring all the work (which includes
+unceremoniously dropping any connections on which we've sent some of
+those articles but not yet had replies, as they're probably stuck
+somehow).
+The default is
+.BR 2000s .
+.TP
 .BI \-\-idle-timeout= PERIOD
 Connections which have had no activity for
 .IR PERIOD
@@ -331,7 +343,10 @@ should only be used by a developer in conjuction with the innduct
 source code.
 .TP
 .B flush
-Start a new feed file and trigger a flush of the feed.
+Start a new feed file and trigger a flush of the feed.  (Or, cause
+the
+.I FLUSH-FINISH-PERIOD
+to expire early, forcibly completing a previously started flush.)
 .TP
 .B stop
 Log statistics and exit.  (Same effect as SIGTERM or SIGINT.)