--- /dev/null
+/*
+ * innduct
+ * tailing reliable realtime streaming feeder for inn
+ *
+ * Copyright (C) 2010 Ian Jackson <ijackson@chiark.greenend.org.uk>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * (I believe that when you compile and link this as part of the inn2
+ * build, with the Makefile runes I have provided, all the libraries
+ * and files which end up included in innduct are licence-compatible
+ * with GPLv3. If not then please let me know. -Ian Jackson.)
+ */
+
+#ifndef INNDUCT_H
+#define INNDUCT_H
+
+#define _GNU_SOURCE 1
+
+#include "config.h"
+#include "storage.h"
+#include "nntp.h"
+#include "libinn.h"
+#include "inndcomm.h"
+
+#include "inn/list.h"
+#include "inn/innconf.h"
+#include "inn/messages.h"
+
+#include <sys/uio.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <unistd.h>
+#include <string.h>
+#include <signal.h>
+#include <stdio.h>
+#include <errno.h>
+#include <syslog.h>
+#include <fcntl.h>
+#include <stdarg.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <glob.h>
+#include <time.h>
+#include <math.h>
+#include <ctype.h>
+
+#include <oop.h>
+#include <oop-read.h>
+
+/*----- general definitions, probably best not changed -----*/
+
+#define CONNCHILD_ESTATUS_STREAM 24
+#define CONNCHILD_ESTATUS_NOSTREAM 25
+
+#define INNDCOMMCHILD_ESTATUS_FAIL 26
+#define INNDCOMMCHILD_ESTATUS_NONESUCH 27
+
+#define MAX_LINE_FEEDFILE (NNTP_MSGID_MAXLEN + sizeof(TOKEN)*2 + 10)
+#define MAX_CLI_COMMAND 1000
+
+#define VA va_list al; va_start(al,fmt)
+#define PRINTF(f,a) __attribute__((__format__(printf,f,a)))
+#define NORET_PRINTF(f,a) __attribute__((__noreturn__,__format__(printf,f,a)))
+#define NORET __attribute__((__noreturn__))
+
+#define NEW(ptr) ((ptr)= zxmalloc(sizeof(*(ptr))))
+#define NEW_DECL(type,ptr) type ptr = zxmalloc(sizeof(*(ptr)))
+
+#define DUMPV(fmt,pfx,v) fprintf(f, " " #v "=" fmt, pfx v);
+
+#define FOR_CONN(conn) \
+ for ((conn)=LIST_HEAD(conns); (conn); (conn)=LIST_NEXT((conn)))
+
+/*----- doubly linked lists -----*/
+
+#define ISNODE(T) struct node list_node
+#define DEFLIST(T) \
+ typedef struct { \
+ union { struct list li; T *for_type; } u; \
+ int count; \
+ } T##List
+
+#define NODE(n) (assert((void*)&(n)->list_node == (n)), &(n)->list_node)
+
+#define LIST_CHECKCANHAVENODE(l,n) \
+ ((void)((n) == ((l).u.for_type))) /* just for the type check */
+
+#define LIST_ADDSOMEHOW(l,n,list_addsomehow) \
+ ( LIST_CHECKCANHAVENODE(l,n), \
+ list_addsomehow(&(l).u.li, NODE((n))), \
+ (void)(l).count++ \
+ )
+
+#define LIST_REMSOMEHOW(l,list_remsomehow) \
+ ( (typeof((l).u.for_type)) \
+ ( (l).count \
+ ? ( (l).count--, \
+ list_remsomehow(&(l).u.li) ) \
+ : 0 \
+ ) \
+ )
+
+
+#define LIST_ADDHEAD(l,n) LIST_ADDSOMEHOW((l),(n),list_addhead)
+#define LIST_ADDTAIL(l,n) LIST_ADDSOMEHOW((l),(n),list_addtail)
+#define LIST_REMHEAD(l) LIST_REMSOMEHOW((l),list_remhead)
+#define LIST_REMTAIL(l) LIST_REMSOMEHOW((l),list_remtail)
+
+#define LIST_INIT(l) ((l).count=0, list_new(&(l).u.li))
+#define LIST_HEAD(l) ((typeof((l).u.for_type))(list_head((struct list*)&(l))))
+#define LIST_NEXT(n) ((typeof(n))list_succ(NODE((n))))
+#define LIST_BACK(n) ((typeof(n))list_pred(NODE((n))))
+
+#define LIST_REMOVE(l,n) \
+ ( LIST_CHECKCANHAVENODE(l,n), \
+ list_remove(NODE((n))), \
+ (void)(l).count-- \
+ )
+
+#define LIST_INSERT(l,n,pred) \
+ ( LIST_CHECKCANHAVENODE(l,n), \
+ LIST_CHECKCANHAVENODE(l,pred), \
+ list_insert((struct list*)&(l), NODE((n)), NODE((pred))), \
+ (void)(l).count++ \
+ )
+
+/*----- type predeclarations -----*/
+
+typedef struct Conn Conn;
+typedef struct Article Article;
+typedef struct InputFile InputFile;
+typedef struct XmitDetails XmitDetails;
+typedef struct Filemon_Perfile Filemon_Perfile;
+typedef enum StateMachineState StateMachineState;
+typedef struct CliCommand CliCommand;
+
+DEFLIST(Conn);
+DEFLIST(Article);
+
+
+/*----- configuration options -----*/
+/* when changing defaults, remember to update the manpage */
+
+static const char *sitename, *remote_host;
+static const char *feedfile, *path_run, *path_cli, *path_cli_dir;
+static int quiet_multiple=0;
+static int interactive=0, try_filemon=1;
+static int try_stream=1;
+static int port=119;
+static const char *inndconffile;
+
+static int max_connections=10;
+static int max_queue_per_conn=200;
+static int target_max_feedfile_size=100000;
+static int period_seconds=30;
+static int filepoll_seconds=5;
+static int max_queue_per_ipf=-1;
+
+static int connection_setup_timeout=200;
+static int inndcomm_flush_timeout=100;
+
+static double nocheck_thresh= 95.0; /* converted from percentage by main */
+static double nocheck_decay= 100; /* conv'd from articles to lambda by main */
+
+/* all these are initialised to seconds, and converted to periods in main */
+static int reconnect_delay_periods=1000;
+static int flushfail_retry_periods=1000;
+static int backlog_retry_minperiods=100;
+static int backlog_spontrescan_periods=300;
+static int spontaneous_flush_periods=100000;
+static int max_separated_periods=2000;
+static int need_activity_periods=1000;
+static int lowvol_thresh=3;
+static int lowvol_periods=1000;
+
+static double max_bad_data_ratio= 1; /* conv'd from percentage by main */
+static int max_bad_data_initial= 30;
+ /* in one corrupt 4096-byte block the number of newlines has
+ * mean 16 and standard deviation 3.99. 30 corresponds to z=+3.5 */
+
+
+/*----- statistics -----*/
+
+typedef enum { /* in queue in conn->sent */
+ art_Unchecked, /* not checked, not sent checking */
+ art_Wanted, /* checked, wanted sent body as requested */
+ art_Unsolicited, /* - sent body without check */
+ art_MaxState,
+} ArtState;
+
+static const char *const artstate_names[]=
+ { "Unchecked", "Wanted", "Unsolicited", 0 };
+
+#define RESULT_COUNTS(RCS,RCN) \
+ RCS(sent) \
+ RCS(accepted) \
+ RCN(unwanted) \
+ RCN(rejected) \
+ RCN(deferred) \
+ RCN(missing) \
+ RCN(connretry)
+
+#define RCI_TRIPLE_FMT_BASE "%d (id=%d,bod=%d,nc=%d)"
+#define RCI_TRIPLE_VALS_BASE(counts,x) \
+ counts[art_Unchecked] x \
+ + counts[art_Wanted] x \
+ + counts[art_Unsolicited] x, \
+ counts[art_Unchecked] x \
+ , counts[art_Wanted] x \
+ , counts[art_Unsolicited] x
+
+typedef enum {
+#define RC_INDEX(x) RC_##x,
+ RESULT_COUNTS(RC_INDEX, RC_INDEX)
+ RCI_max
+} ResultCountIndex;
+
+
+/*----- transmission buffers -----*/
+
+#define CONNIOVS 128
+
+typedef enum {
+ xk_Const, xk_Artdata
+} XmitKind;
+
+struct XmitDetails {
+ XmitKind kind;
+ union {
+ ARTHANDLE *sm_art;
+ } info;
+};
+
+
+/*----- core operational data structure types -----*/
+
+struct InputFile {
+ /* This is also an instance of struct oop_readable */
+ struct oop_readable readable; /* first */
+ oop_readable_call *readable_callback;
+ void *readable_callback_user;
+
+ int fd;
+ Filemon_Perfile *filemon;
+
+ oop_read *rd; /* non-0: reading; 0: constructing, or had EOF */
+ off_t offset;
+ int skippinglong, paused, fake_readable;
+
+ ArticleList queue;
+ long inprogress; /* includes queue.count and also articles in conns */
+ long autodefer; /* -1 means not doing autodefer */
+
+ int counts[art_MaxState][RCI_max];
+ int readcount_ok, readcount_blank, readcount_err, count_nooffer_missing;
+ char path[];
+};
+
+struct Article {
+ ISNODE(Article);
+ ArtState state;
+ int midlen, missing;
+ InputFile *ipf;
+ TOKEN token;
+ off_t offset;
+ int blanklen;
+ char messageid[1];
+};
+
+#define SMS_LIST(X) \
+ X(NORMAL) \
+ X(FLUSHING) \
+ X(FLUSHFAILED) \
+ X(SEPARATED) \
+ X(DROPPING) \
+ X(DROPPED)
+
+enum StateMachineState {
+#define SMS_DEF_ENUM(s) sm_##s,
+ SMS_LIST(SMS_DEF_ENUM)
+};
+
+extern const char *sms_names[];
+
+/*========== function declarations ==========*/
+
+/*----- help.c -----*/
+
+static void syscrash(const char *fmt, ...) NORET_PRINTF(1,2);
+static void crash(const char *fmt, ...) NORET_PRINTF(1,2);
+static void info(const char *fmt, ...) PRINTF(1,2);
+static void dbg(const char *fmt, ...) PRINTF(1,2);
+
+void logv(int sysloglevel, const char *pfx, int errnoval,
+ const char *fmt, va_list al) PRINTF(5,0);
+
+char *xvasprintf(const char *fmt, va_list al) PRINTF(1,0);
+char *xasprintf(const char *fmt, ...) PRINTF(1,2);
+
+int close_perhaps(int *fd);
+void xclose(int fd, const char *what, const char *what2);
+void xclose_perhaps(int *fd, const char *what, const char *what2);
+pid_t xfork(const char *what);
+
+static void on_fd_read_except(int fd, oop_call_fd callback);
+void cancel_fd_read_except(int fd);
+
+void report_child_status(const char *what, int status);
+int xwaitpid(pid_t *pid, const char *what);
+
+void *zxmalloc(size_t sz);
+void xunlink(const char *path, const char *what);
+time_t xtime(void);
+void xsigaction(int signo, const struct sigaction *sa);
+void xsigsetdefault(int signo);
+
+void xgettimeofday(struct timeval *tv_r);
+void xsetnonblock(int fd, int nonb);
+
+void check_isreg(const struct stat *stab, const char *path,
+ const char *what);
+void xfstat(int fd, struct stat *stab_r, const char *what);
+void xfstat_isreg(int fd, struct stat *stab_r,
+ const char *path, const char *what);
+void xlstat_isreg(const char *path, struct stat *stab,
+ int *enoent_r /* 0 means ENOENT is fatal */,
+ const char *what);
+int samefile(const struct stat *a, const struct stat *b);
+
+char *sanitise(const char *input, int len);
+
+static inline int isewouldblock(int errnoval) {
+ return errnoval==EWOULDBLOCK || errnoval==EAGAIN;
+}
+
+/*----- innduct.c -----*/
+
+void postfork(void);
+
+/*----- conn.c -----*/
+
+void conn_closefd(Conn *conn, const char *msgprefix);
+
+/*----- defer.c -----*/
+
+void poll_backlog_file(void);
+
+/*----- infile.c -----*/
+
+void filepoll(void);
+
+/*----- statemc.c -----*/
+
+sig_atomic_t terminate_sig_flag;
+
+/*----- xmit.c -----*/
+
+void inputfile_queue_check_expired(InputFile *ipf);
+
+/*----- external linkage for debug/dump only -----*/
+
+pid_t connecting_child;
+pid_t inndcomm_child;
+
+/*========== general operational variables ==========*/
+
+/* innduct.c */
+extern oop_source *loop;
+extern ConnList conns;
+extern char *path_lock, *path_flushing, *path_defer, *path_dump;
+extern char *globpat_backlog;
+extern pid_t self_pid;
+extern int *lowvol_perperiod;
+extern int lowvol_circptr;
+extern int lowvol_total; /* does not include current period */
+
+/* statemc.c */
+extern StateMachineState sms;
+extern int until_flush;
+extern InputFile *main_input_file, *flushing_input_file, *backlog_input_file;
+extern FILE *defer;
+extern int until_connect, until_backlog_nextscan;
+extern double accept_proportion;
+extern int nocheck, nocheck_reported, in_child;
+
+/* help.c */
+extern int simulate_flush;
+extern int logv_use_syslog;
+extern const char *logv_prefix;
+
+
+#endif /*INNDUCT_H*/
~ian / inn-innduct.git / blobdiff