/*
* TODO
- * - actually implement badusage
- * - options for all options
- * - manpage
* - pid, sitename, hostname in lockfile
- * - -k kill mode ?
*/
/*
/*----- general definitions, probably best not changed -----*/
-#define CONNCHILD_ESTATUS_STREAM 4
-#define CONNCHILD_ESTATUS_NOSTREAM 5
+#define CONNCHILD_ESTATUS_STREAM 24
+#define CONNCHILD_ESTATUS_NOSTREAM 25
-#define INNDCOMMCHILD_ESTATUS_FAIL 6
-#define INNDCOMMCHILD_ESTATUS_NONESUCH 7
+#define INNDCOMMCHILD_ESTATUS_FAIL 26
+#define INNDCOMMCHILD_ESTATUS_NONESUCH 27
#define MAX_LINE_FEEDFILE (NNTP_MSGID_MAXLEN + sizeof(TOKEN)*2 + 10)
static oop_rd_call peer_rd_err, peer_rd_ok;
/*----- configuration options -----*/
+/* when changing defaults, remember to update the manpage */
static const char *sitename, *remote_host;
static const char *feedfile;
pid_t child;
child= fork();
- if (child==-1) sysdie("cannot fork for %s",what);
+ if (child==-1) sysfatal("cannot fork for %s",what);
debug("forked %s %ld", what, (unsigned long)child);
if (!child) postfork();
return child;
static void filemon_method_startfile(InputFile *ipf, Filemon_Perfile *pf) {
int wd= inotify_add_watch(filemon_inotify_fd, ipf->path, IN_MODIFY);
- if (wd < 0) sysdie("inotify_add_watch %s", ipf->path);
+ if (wd < 0) sysfatal("inotify_add_watch %s", ipf->path);
if (wd >= filemon_inotify_wdmax) {
int newmax= wd+2;
if (quiet_multiple) exit(0);
fatal("another duct holds the lockfile");
}
- sysdie("fcntl F_SETLK lockfile %s", path_lock);
+ sysfatal("fcntl F_SETLK lockfile %s", path_lock);
}
xfstat_isreg(lockfd, &stabf, path_lock, "lockfile");
sm_period_counter);
int r= link(feedfile, path_flushing);
- if (r) sysdie("link feedfile %s to flushing file %s",
- feedfile, path_flushing);
+ if (r) sysfatal("link feedfile %s to flushing file %s",
+ feedfile, path_flushing);
/* => Hardlinked */
xunlink(feedfile, "old feedfile link");
switch (r) {
case GLOB_ABORTED:
- sysdie("failed to expand backlog pattern %s", globpat_backlog);
+ sysfatal("failed to expand backlog pattern %s", globpat_backlog);
case GLOB_NOSPACE:
- die("out of memory expanding backlog pattern %s", globpat_backlog);
+ fatal("out of memory expanding backlog pattern %s", globpat_backlog);
case 0:
for (i=0; i<gl.gl_pathc; i++) {
const char *path= gl.gl_pathv[i];
if (sms==sm_DROPPED) {
notice("feed dropped and our work is complete");
xunlink(path_lock, "lockfile for old feed");
- exit(0);
+ exit(4);
}
until_backlog_nextscan= backlog_spontrescan_periods;
goto xfree;
assert(!inndcomm_child);
assert(!inndcomm_sentinel_fd);
- if (pipe(pipefds)) sysdie("create pipe for inndcomm child sentinel");
+ if (pipe(pipefds)) sysfatal("create pipe for inndcomm child sentinel");
inndcomm_child= xfork("inndcomm child");
double v= strtod(val,&ep);
if (ep==val) badusage("bad time/duration value for %s",o->lng);
- if (!*ep || !strcmp(ep,"s")) unit= 1;
- else if (!strcmp(ep,"m")) unit= 60;
- else if (!strcmp(ep,"h")) unit= 3600;
- else if (!strcmp(ep,"d")) unit= 86400;
+ if (!*ep || !strcmp(ep,"s") || !strcmp(ep,"sec")) unit= 1;
+ else if (!strcmp(ep,"m") || !strcmp(ep,"min")) unit= 60;
+ else if (!strcmp(ep,"h") || !strcmp(ep,"hour")) unit= 3600;
+ else if (!strcmp(ep,"d") || !strcmp(ep,"day")) unit= 86400;
+ else if (!strcmp(ep,"das")) unit= 10;
+ else if (!strcmp(ep,"hs")) unit= 100;
+ else if (!strcmp(ep,"ks")) unit= 1000;
+ else if (!strcmp(ep,"Ms")) unit= 1000000;
else badusage("bad units %s for time/duration value for %s",ep,o->lng);
v *= unit;
int main(int argc, char **argv) {
if (!argv[1]) {
printusage(stderr);
- exit(12);
+ exit(8);
}
parse_options(innduct_options, &argv);
if (nocheck_decay < 0.1)
badusage("nocheck decay articles must be at least 0.1");
- nocheck_decay= 1 - 1.0/nocheck_decay;
+ nocheck_decay= pow(0.5, 1.0/nocheck_decay);
convert_to_periods_rndup(&reconnect_delay_periods);
convert_to_periods_rndup(&flushfail_retry_periods);
openlog("innduct",LOG_NDELAY|LOG_PID,LOG_NEWS);
int null= open("/dev/null",O_RDWR);
- if (null<0) sysdie("failed to open /dev/null");
+ if (null<0) sysfatal("failed to open /dev/null");
dup2(null,0);
dup2(null,1);
dup2(null,2);
if (child1) _exit(0);
pid_t sid= setsid();
- if (sid != child1) sysdie("setsid failed");
+ if (sid != child1) sysfatal("setsid failed");
pid_t child2= xfork("daemonise second fork");
if (child2) _exit(0);
is not specified, it defaults to
.IR site .
-Logging is sent to syslog.
+innduct daemonises after argument parsing, and all logging (including
+error messages) are sent to syslog (facility
+.BR news ).
+
+The best way to run innduct is probably to periodically invoke innduct
+for each feed (e.g. from cron), passing innduct it the
+.B \-q
+option to arrange that it silently exits if an innduct is already
+running for that site.
+.SH INNDUCT VS INNFEED/NNTPSEND/INNXMIT
+.TP
+.B innfeed
+does roughly the same thing as innduct. However, the way it receives
+information from innd can result in articles being lost (not offered
+to peers) if innfeed crashes for any reason. This is an inherent
+defect in the innd channel feed protocol. innduct uses a file feed,
+constantly "tailing" the feed file, and where implemented uses
+.BR inotify (2)
+to reduce the latency which would come from having to constantly poll
+the feed file. innfeed is capable of feeding multiple peers from a
+single innfeed instance, whereas each innduct process handles exactly
+one peer. innduct is much smaller and simpler, at 3kloc to innfeed's
+25kloc. innfeed needs a separate wrapper script or similar
+infrastructure (of which there is an example in its manpage), whereas
+innduct can be run directly and doesn't need help from shell scripts.
+.TP
+.B nntpsend
+processes feed files in batch mode. That is, you have to periodically
+invoke nntpsend, and when you do, the feed is flushed and articles
+which arrived before the flush are sent to the peer. This introduces
+a batching delay, and also means that the NNTP connection to the peer
+needs to be remade at each batch. nntpsend (which uses innxmit)
+cannot make use of multiple connections to a single peer site.
+However, nntpsend can be left to find automatically which sites need
+feeding by looking in
+.IR pathoutgoing .
+.TP
+.B innxmit
+is the actual NNTP feeder program used by nntpsend.
.SH GENERAL OPTIONS
.TP
.BR \-f | \-\-feedfile= \fIfeedfile\fR
(including all debug messages) are written to stderr.
.TP
.BI \-\-no-streaming
-Do not try to use the streaming extensions to NNTP (eg if the peer
-can't cope when we send MODE STREAM).
+Do not try to use the streaming extensions to NNTP (for use eg if the
+peer can't cope when we send MODE STREAM).
.TP
.BI \-\-inndconf= FILE
Read
instead of the default
.BR inn.conf .
This is currently only used if
-.BI \-f feedfile
+.I feedfile
is not specified, to find the value
.I pathoutgoing
for constructing the feedfile name from the site name.
.BI \-\-help
Just print a brief usage message and list of the options to stdout.
.SH TUNING OPTIONS
-You should not normally need to adjust these.
+You should not normally need to adjust these. Time intervals may
+specified in seconds, or as a number followed by one of the following
+units:
+.BR "s m h d" ,
+.BR "sec min hour day" ,
+.BR "das hs ks Ms" .
.TP
.BI \-\-max-connections= max
Restricts the maximum number of simultaneous NNTP connections used by
for each site to
.IR max .
-The default is 10. There is no global limit on the number of
-connections.
+The default is
+.BR 10 .
+There is no global limit on the number of connections.
.TP
.BI \-\-max-queue-per-conn= max
Restricts the maximum number of outstanding articles queued on any
particular connection
.IR max .
-The default is 200. (Non-streaming connections can only handle one
-article at a time.)
+(Non-streaming connections can only handle one article at a time.)
+The default is
+.BR 200 .
.TP
.BI \-\-feedfile-flush-size= bytes
-Specifies that innduct should flush when the feedfile size exceeds
+Specifies that innduct should flush the feed and start a new feedfile
+when the existing feedfile size exceeds
.IR bytes ;
the effect is that the innduct will try to avoid the various
batchfiles growing much beyond this size while the link to the peer is
-working. The default is 100,000 bytes.
+working. The default is
+.BR 100000 .
.TP
.BI \-\-period-interval= PERIOD-INTERVAL
Specifies wakup interval and period granularity.
innduct wakes up every PERIOD-INTERVAL to do various housekeeping
checks. Also, many of the timeout and rescan intervals (those
specified in this manual as
-.IR PERIOD s)
-are rounded up to the next multiple of PERIOD-INTERVAL.
-The default is one minute.
+.TP
+.BI \-\-connection-timeout= TIME
+How long to allow for a connection setup attempt before giving up.
+The default is
+.BR 200s .
+.TP
+.BI \-\-stuck-flush-timeout= TIME
+How long to wait for innd to respond to a flush request before giving
+up. The default is
+.BR 100s .
+.TP
+.BI \-\-no-check-proportion= PERCENT
+If the moving average of the proportion of articles being accepted
+(rather than declined) by the peer exceeds this value, innduct uses
+"no check mode" - ie it just sends the peer the articles with TAKETHIS
+rather than checking first with CHECK whether the article is wanted.
+This only affects streaming connections. The default is
+.B 95
+(ie, 95%).
+.TP
+.BI \-\-no-check-response-time= ARTICLES
+The moving average mentioned above is an alpha-smoothed value with a
+half-life of
+.IR ARTICLES .
+The default is
+.BR 100 .
+.TP
+.BI \-\-reconnect-interval= PERIOD
+Limits initiation of new connections to one each
+.IR PERIOD .
+This applies to reconnections if the peer has been down, and also to
+ramping up the number of connections we are using after startup or in
+response to an article flood. The default is
+.BR 1000s .
+.TP
+.BI \-\-flush-retry-interval= PERIOD
+If our attempt to flush the feed failed (usually this will be because
+innd is not running), try again after
+.IR PERIOD .
+The default is
+.BR 1000s .
+.TP
+.BI \-\-earliest-deferred-retry= PERIOD
+When the peer responds to our offer of an article with a 431 or 436
+NNTP response code, indicating that the article has already been
+offered to it by another of its peers, and that we should try again,
+we wait at least
+.IR PERIOD .
+before offering the article again. The default is
+.BR 50s .
+.TP
+.BI \-\-backlog-rescan-interval= PERIOD
+We scan the directory containing
+.I feedfile
+for backlog files at least every
+.IR PERIOD ,
+in case the administrator has manually dropped in a file there for
+processing.
+The default is
+.TP
+.BI \-\-max-flush-interval= PERIOD
+We flush the feed at least every
+.IR PERIOD
+even if the current instance of the feedfile has not reached the size
+threshold.
+The default is
+.BR 100000s .
+.TP
+.BI \-\-max-flush-interval= PERIOD
+We flush the feed and start a new feedfile at least every
+.IR PERIOD
+even if the current instance of the feedfile has not reached the size
+threshold.
+The default is
+.BR 100000s .
+.TP
+.BI \-\-idle-timeout= PERIOD
+Connections which have had no activity for
+.IR PERIOD
+will be closed. This includes connections where we have sent commands
+or articles but have not yet had the responses, so this same value
+doubles as the timeout after which we conclude that the peer is
+unresponsive or the connection has become broken.
+The default is
+.BR 1000s .
+.TP
+.BI \-\-max-bad-input-data-ratio= PERCENT
+We tolerate up to this proportion of badly-formatted lines in the
+feedfile and other input files. Every badly-formatted line is logged,
+but if there are too many we conclude that the corruption to our
+on-disk data is too severe, and crash; to successfully restart,
+administrator intervention will be required. This avoids flooding the
+logs with warnings and also arranges to abort earlyish if an attempt
+is made to process a file in the wrong format.
+The default is
+.BR 1
+(ie, 1%).
+.TP
+.BI \-\-max-bad-input-data-init= LINES
+Additionally, we tolerate this number of additional badly-formatted
+lines, so that if the badly-formatted lines are a few but at the start
+of the file, we don't crash immediately.
+The default is
+.BR 30
+(which would suffice to ignore one whole corrupt 4096-byte disk block
+filled with random data, or one corrupt 1024-byte disk block filled
+with an inappropriate text file with a mean line length of at least
+35).
+.SH INTERACTING WITH INNDUCT
+innduct dances a somewhat complicated dance with innd to make sure
+that everything goes smoothly and that there are no races. (See the
+two ascii-art diagrams in innduct.c for details of the protocol.) Do
+not mess with the feedfile and other associated files, other than as
+explained below in the section
+.BR FILES .
+.LP
+If you tell innd to drop the feed, innduct will (when it notices,
+which will normally be the next time it decides flushes) finish up the
+articles it has in hand now, and then exit. It is harmless to cause
+innd to flush the feed (but innduct won't notice and flushing won't
+start a new feedfile; you have to leave that to innduct).
+.LP
+There are no signals that can usefully be sent to innduct to give it
+complicated instructions. If you need to kill innduct, feel free to
+send it a
+.B SIGTERM
+or
+.B SIGKILL
+and nothing will be broken or corrupted.
.SH EXIT STATUS
-0 ok
-4 one is already running
-8 bad usage
-12 trouble
+.TP
+.B 0
+An instance of innduct is already running for this
+.I feedfile
+and
+.B -q
+was specified.
+.TP
+.B 4
+The feed has been dropped by innd, and we (or previous innducts) have
+successfully offered all the old articles to the peer site. Our work
+is done.
+.TP
+.B 8
+innduct was invoked with bad options or command line arguments. The
+error message will be printed to stderr, and also (if any options or
+arguments were passed at all) to syslog with severity
+.BR crit .
+.TP
+.B 12
+Things are going wrong, hopefully shortage of memory, system file
+table entries; disk IO problems; disk full; etc. The specifics of the
+error will be logged to syslog with severity
+.B err
+(if syslog is working!)
+.TP
+.B 16
+Things are going badly wrong in an unexpected way: system calls which
+are not expected to fail are doing so, or the protocol for
+communicating with innd is being violated, or some such. Details will
+be logged with severity
+.B crit
+(if syslog is working!)
+.TP
+.BR 24 - 27
+These exit statuses are used by children forked by innduct to
+communicate to the parent. You should not see them. If you do, it is
+a bug.
.SH FILES
.IX Header "FILES"
.IP \fIpathoutgoing\fR/\fIsite\fR
.IX Item "backlog file"
Batch file containing details of articles whose transmission has less
recently been deferred at the request of the recipient site. Created
-by innduct, and will also be read and removed by innduct. However the
-administrator may also safely remove backlog files.
+by innduct, and will also be read and removed by innduct. However you
+(the administrator) may also safely remove backlog files.
.IP \fIfeedfile\fR_backlog\fIsomething\fR
.IX Item "manual backlog file"
Batch file manually provided by the administrator. The file should be
complete and ready to process at the time it is renamed or hardlinked
to this name. innduct will then automatically find and read and
process it and eventually remove it. The administrator may also
-safely remove backlog files. \fIsomething\fR may not contain
-\fB#\fR \fB~\fR or \fB/\fR.
+safely remove backlog files. \fIsomething\fR may not contain \fB#\fR
+\fB~\fR or \fB/\fR. Be sure to have finished writing the file before
+you rename it to match the pattern \fIfeedfile\fR\fB_backlog\fR*, as
+otherwise innduct may find and process the file and read it to EOF
+before you have finished creating it.
.IP /etc/news/inn.conf
.IX Item inn.conf
Used to find
~ian / inn-innduct.git / commitdiff