+static void poll_backlog_file(void) {
+ if (backlog_nextscan_periods < 0) return;
+ if (backlog_nextscan_periods-- > 0) return;
+ search_backlog_file();
+}
+
+static int search_backlog_file(void) {
+ /* returns non-0 iff there are any backlog files */
+
+ glob_t gl;
+ int r;
+ struct stat stab;
+ const char *oldest_path=0;
+ time_t oldest_mtime, now;
+
+ if (backlog_input_file) return 3;
+
+ try_again:
+
+ r= glob(globpat_backlog, GLOB_ERR|GLOB_MARK|GLOB_NOSORT, 0, &gl);
+
+ switch (r) {
+ case GLOB_ABORTED:
+ sysdie("failed to expand backlog pattern %s", globpat_backlog);
+ case GLOB_NOSPACE:
+ die("out of memory expanding backlog pattern %s", globpat_backlog);
+ case 0:
+ for (i=0; i<gl.gl_pathc; i++) {
+ const char *path= gl.gl_pathv[i];
+
+ if (strchr(path,'#') || strchr(path,'~')) {
+ debug("backlog file search skipping %s", path);
+ continue;
+ }
+ r= stat(path, &stab);
+ if (r) {
+ syswarn("failed to stat backlog file %s", path);
+ continue;
+ }
+ if (!S_ISREG(stab.st_mode)) {
+ warn("backlog file %s is not a plain file (or link to one)", path);
+ continue;
+ }
+ if (!oldest_path || stab.st_mtime < oldest_mtime) {
+ oldest_path= path;
+ oldest_mtime= stab.st_mtime;
+ }
+ }
+ case GLOB_NOMATCH: /* fall through */
+ break;
+ default:
+ sysdie("glob expansion of backlog pattern %s gave unexpected"
+ " nonzero (error?) return value %d", globpat_backlog, r);
+ }
+
+ globfree(&gl);
+
+ if (!oldest_path) {
+ debug("backlog scan: none");
+ backlog_nextscan_periods= backlog_spontaneous_rescan_periods;
+ return 0;
+ }
+
+ now= time(); if (now==-1) sysdie("time(2) failed");
+ double age= difftime(now, oldest_mtime);
+ long age_deficiency= (backlog_retry_minperiods * PERIOD_SECONDS) - age;
+
+ if (age_deficiency <= 0) {
+ debug("backlog scan: found age=%f deficiency=%ld oldest=%s",
+ age, age_deficiency, oldest_path);
+
+ backlog_input_file= open_input_file(oldest_path);
+ if (!backlog_input_file) {
+ warn("backlog file %s vanished as we opened it", backlog_input_file);
+ goto try_again;
+ }
+ inputfile_tailing_start(backlog_input_file);
+ backlog_nextscan_periods= -1;
+ return 1;
+ }
+
+ backlog_nextscan_periods= age_deficiency / PERIOD_SECONDS;
+
+ if (backlog_spontaneous_rescan_periods >= 0 &&
+ backlog_nextscan_periods > backlog_spontaneous_rescan_periods)
+ backlog_nextscan_periods= backlog_spontaneous_rescan_periods;
+
+ debug("backlog scan: young age=%f deficiency=%ld nextscan=%d oldest=%s",
+ age, age_deficiency, backlog_nextscan_periods, oldest_path);
+ return 2;
+}
+
+/*========== flushing the feed ==========*/
+
+static pid_t inndcomm_child;
+
+static void *inndcomm_event(oop_source *lp, int fd, oop_event e, void *u) {
+ assert(inndcomm_child);
+ int status= xwaitpid(&inndcomm_child, "inndcomm");
+ loop->cancel_fd(fd);
+ close(fd);
+
+ assert(!flushing_input_file);
+
+ if (WIFEXITED(status)) {
+ switch (WEXITSTATUS(status)) {
+
+ case INNDCOMMCHILD_ESTATUS_FAIL:
+ goto failed;
+
+ case INNDCOMMCHILD_ESTATUS_NONESUCH:
+ warn("feed has been dropped by innd, finishing up");
+ flushing_input_file= main_input_file;
+ main_input_file= 0;
+ SMS(DROPPING, 0, "dropped by innd");
+ return OOP_CONTINUE;
+
+ case 0:
+ flushing_input_file= main_input_file;
+ main_input_file= open_input_file(feedfile);
+ if (!main_input_file)
+ die("flush succeeded but feedfile %s does not exist!", feedfile);
+ until_spontaneous_flush= spontaneous_flush_periods;
+ SMS(SEPARATED, 0, "feed file missing");
+ return OOP_CONTINUE;
+
+ default:
+ goto unexpected_exitstatus;
+
+ }
+ } else if (WIFSIGNALED(status) && WTERMSIG(status) == SIGALRM) {
+ warn("flush timed out trying to talk to innd");
+ goto failed;
+ } else {
+ unexpected_exitstatus:
+ report_child_status("inndcomm child", status);
+ }
+
+ failed:
+ SMS(FLUSHFAIL, flushfail_retry_periods, "flush failed, will retry");
+}
+
+static void inndcommfail(const char *what) {
+ syswarn("error communicating with innd: %s failed: %s", what, ICCfailure);
+ exit(INNDCOMMCHILD_ESTATUS_FAIL);
+}
+
+void spawn_inndcomm_flush(void) {
+ int pipefds[2];
+
+ assert(sms==sm_NORMAL || sms==sm_FLUSHFAIL);
+ assert(!inndcomm_child);
+
+ if (pipe(pipefds)) sysdie("create pipe for inndcomm child sentinel");
+
+ inndcomm_child= xfork();
+
+ if (!inndcomm_child) {
+ static char flushargv[2]= { sitename, 0 };
+ char *reply;
+
+ close(pipefds[0]);
+
+ alarm(inndcomm_flush_timeout);
+ r= ICCopen(); if (r) inndcommfail("connect");
+ r= ICCcommand('f',flushargv,&reply); if (r<0) inndcommfail("transmit");
+ if (!r) exit(0); /* yay! */
+
+ if (!strcmp(reply, "1 No such site")) exit(INNDCOMMCHILD_ESTATUS_NONESUCH);
+ syswarn("innd ctlinnd flush failed: innd said %s", reply);
+ exit(INNDCOMMCHILD_ESTATUS_FAIL);
+ }
+
+ close(pipefds[1]);
+ int sentinel_fd= pipefds[0];
+ on_fd_read_except(sentinel_fd, inndcomm_event);
+
+ SMS(FLUSHING, 0, "flush is in progress");
+}
+
+/*========== main program ==========*/
+
+static void postfork_inputfile(InputFile *ipf) {
+ if (!ipf) return;
+ assert(ipf->fd >= 0);
+ close(ipf->fd);
+ ipf->fd= -1;
+}
+
+static void postfork_conns(Connection *conn) {
+ while (conn) {
+ close(conn->fd);
+ conn= conn->next;
+ }
+}
+
+static void postfork_stdio(FILE *f) {
+ /* we have no stdio streams that are buffered long-term */
+ if (f) fclose(f);
+}
+
+static void postfork(const char *what) {
+ if (signal(SIGPIPE, SIG_DFL) == SIG_ERR)
+ sysdie("%s child: failed to reset SIGPIPE");
+
+ postfork_inputfile(main_input_file);
+ postfork_inputfile(flushing_input_file);
+ postfork_conns(idle.head);
+ postfork_conns(working.head);
+ postfork_conns(full.head);
+ postfork_stdio(defer);
+}
+
+#define EVERY(what, interval, body) \
+ static const struct timeval what##_timeout = { 5, 0 }; \
+ static void what##_schedule(void); \
+ static void *what##_timedout(oop_source *lp, struct timeval tv, void *u) { \
+ { body } \
+ what##_schedule(); \
+ } \
+ static void what##_schedule(void) { \
+ loop->on_time(loop, what##_timeout, what##_timedout, 0); \
+ }
+
+EVERY(filepoll, {5,0}, {
+ if (main_input_file && main_input_file->readable_callback)
+ filemon_callback(main_input_file);
+});
+
+#define DEBUGF_IPF(wh) " " #wh "=%p/%s:ip=%ld,off=%ld,fd=%d%s" \
+#define DEBUG_IPF(sh) \
+ wh##_input_file, debug_ipf_path(wh##_input_file), \
+ wh##_input_file->inprogress, (long)wh##_input_file->offset, \
+ wh##_input_file->fd, wh##_input_file->rd ? "+" : ""
+static const char *debug_ipf_path(InputFile *ipf) {
+ char *slash= strrchr(ipf->path,'/');
+ return slash ? slash+1 : ipf->path;
+}
+
+EVERY(period, {PERIOD_SECONDS,0}, {
+ debug("PERIOD"
+ " sms=%s queue=%d sm_period_counter=%d"
+ " connect_delay=%d until_spontaneous_flush=%d"
+ " input_files" DEBUGF_IPF(main) DEBUGF_IPF(old) DEBUGF_FMT(flushing)
+ " conns idle=%d working=%d full=%d"
+ " children connecting=%ld inndcomm_child"
+ ,
+ sms_names[sms], queue.count, sm_period_counter,
+ connect_delay, until_spontaneous_flush,
+ DEBUG_IPF(main), DEBUG_IPF(flushing), DEBUG_IPF(flushing),
+ idle.count, working.count, full.count,
+ (long)connecting_child, (long)inndcomm_child
+ );
+ if (connect_delay) connect_delay--;
+ if (until_spontaneous_flush) until_spontaneous_flush--;
+ poll_backlog_file();
+ if (!backlog_input_file) close_defer(); /* want to start on a new backlog */
+ statemc_poll();
+ check_master_queue();
+});
+
+
+/*========== option parsing ==========*/
+
+enum OptFlags {
+ of_seconds= 001000u;
+ of_boolean= 002000u;
+};
+
+typedef struct Option Option;
+typedef void OptionParser(const Option*, const char *val);
+
+struct Option {
+ int short;
+ const char *long;
+ void *store;
+ OptionParser *fn;
+ int noarg;
+};
+
+void op_integer(const Option *o, const char *val) {
+ char *ep;
+ errno= 0;
+ unsigned long ul= strtoul(val,&ep,10);
+ if (*ep || ep==val || errno || ul>INT_MAX)
+ badusage("bad integer value for %s",o->long);
+ int *store= o->store;
+ *store= ul;
+}
+
+void op_double(const Option *o, const char *val) {
+ int *store= o->store;
+ char *ep;
+ errno= 0;
+ *store= strtod(val, &ep);
+ if (*ep || ep==val || errno)
+ badusage("bad floating point value for %s",o->long);
+}
+
+void op_string(const Option *o, const char *val) {
+ char **store= o->store;
+ free(*store);
+ *store= val;
+}
+
+void op_seconds(const Option *o, const char *val) {
+ int *store= o->store;
+ char *ep;
+
+ double v= strtod(val,&ep);
+ if (ep==val) badusage("bad time/duration value for %s",o->long);
+
+ if (!*ep || !strcmp(ep,"s")) unit= 1;
+ else if (!strcmp(ep,"m")) unit= 60;
+ else if (!strcmp(ep,"h")) unit= 3600;
+ else if (!strcmp(ep,"d")) unit= 86400;
+ else badusage("bad units %s for time/duration value for %s",ep,o->long);
+
+ v *= unit;
+ v= ceil(v);
+ if (v > INT_MAX) badusage("time/duration value for %s out of range",o->long);
+ *store= v;
+}
+
+void op_periods_rndup(const Option *o, const char *val) {
+ int *store= o->store;
+ op_seconds(o,val);
+ *store += PERIOD_SECONDS-1;
+ *store /= PERIOD_SECONDS;
+}
+
+void op_periods_booltrue(const Option *o, const char *val) {
+ int *store= o->store;
+ *store= 1;
+}
+void op_periods_boolfalse(const Option *o, const char *val) {
+ int *store= o->store;
+ *store= 0;
+}
+
+static const Option options[]= {
+{ 0, "max-connections", &max_connections op_integer },
+{ 0, "streaming", &try_stream, op_booltrue, 1 },
+{ 0, "no-streaming", &try_stream, op_boolfalse, 1 },
+{'h',"host", &remote_host, op_string },
+{'P',"port", &port op_integer },
+{ 0, "inndconf", &inndconffile, op_string },
+{'f',"feedfile", &feedfile, op_string },
+{'q',"quiet-multiple", &quiet_if_locked, op_booltrue, 1 },
+{ 0, "no-quiet-multiple", &quiet_if_locked, op_boolfalse, 1 },
+{'d',"daemon", &become_daemon, op_booltrue, 1 },
+{ 0, "no-daemon", &become_daemon, op_boolfalse, 1 },
+
+{ 0, "no-check-proportion", &nocheck_thresh_pct, op_double },
+{ 0, "no-check-filter", &nocheck_decay_articles, op_double },
+
+{ 0, "max-queue-size", &max_queue_per_conn op_integer },
+{ 0, "reconnect-interval", &reconnect_delay_periods, op_periods_rndup },
+{ 0, "flush-retry-interval", &flushfail_retry_periods, op_periods_rndup },
+{ 0, "feedfile-open-timeout", &open_wait_periods, op_periods_rndup },
+{ 0, "connection-timeout", &connection_timeout, op_seconds },
+{ 0, "inndcomm-timeout", &inndcomm_flush_timeout, op_seconds },