X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?a=blobdiff_plain;f=backends%2Finnduct.c;h=85a27995b0b7ae641134c222717fabb1c43e9618;hb=216ae9cf99b4ac7018dfcaf362a60926e0310bcb;hp=8b4b2859359150848e8cccd79a842ed1ddb38ba2;hpb=bd976de6f0a8f28a87c4261be5b3df3f59bdb661;p=innduct.git diff --git a/backends/innduct.c b/backends/innduct.c index 8b4b285..85a2799 100644 --- a/backends/innduct.c +++ b/backends/innduct.c @@ -62,27 +62,13 @@ START | - check D, F - | - <--------------------------------------'| - Nothing F, D both ENOENT | - F: ENOENT | - D: ENOENT | - duct: not not reading anything | - | | - |`---------------------. | - | | duct times out waiting for F | - V innd creates F | duct exits | - | V | - Noduct GO TO Dropped | - F: innd writing | - D: ENOENT | - duct: not running or not reading anything | - | | - | | - ,-->--+ <---------------------------------'| - | | duct opens F F exists | - | | D ENOENT | + ,-->--. check F, D + | | | + | | | + | | <----------------<---------------------------------'| + | | F exists | + | | D ENOENT | + | | duct opens F | | V | | Normal | | F: innd writing, duct reading | @@ -96,31 +82,35 @@ | F == D: innd writing, duct reading both exist | ^ | | | | duct unlinks F | - | V | - | Moved <----+------------<--'| - | F: ENOENT | F ENOENT | - | D: innd writing, duct reading | D exists | - | | | | - | | duct requests flush of feed | | - | | (others can too, harmlessly) | | - | V | | - | Flushing | | - | F: ENOENT | | - | D: innd flushing, duct reading | | - | | | | - | | inndcomm flush fails | | - | |`-------------------------->---------' | - | | | + | | <-----------<-------------<--'| + | | open D F ENOENT | + | | if exists | + | | | + | V <---------------------. | + | Moved | | + | F: ENOENT | | + | D: innd writing, duct reading; or ENOENT | | + | | | | + | | duct requests flush of feed | | + | | (others can too, harmlessly) | | + | V | | + | Flushing | | + | F: ENOENT | | + | D: innd flushing, duct; or ENOENT | | + | | | | + | | inndcomm flush fails | | + | |`-------------------------->------------------' | + | | | | | inndcomm reports no such site | | |`---------------------------------------------------- | -. - | | | | + | | | | | | innd finishes writing D, creates F | | | | inndcomm reports flush successful | | | | | | | V | | | Separated <----------------' | | F: innd writing F!=D / - | D: duct reading both exist / + | D: duct reading; or ENOENT both exist / | | / | | duct gets to the end of D / | | duct opens F too / @@ -382,6 +372,11 @@ static int xwaitpid(pid_t *pid, const char *what) { return status; } +static void xunlink(const char *path, const char *what) { + int r= unlink(path); + if (r) sysdie("can't unlink %s %s", path, what); +} + static void check_isreg(const struct stat *stab, const char *path, const char *what) { if (!S_ISREG(stab->st_mode)) @@ -948,11 +943,14 @@ static Article *article_reply_check(Connection *conn, const char *response, static void update_nocheck(int accepted) { accept_proportion *= accept_decay; accept_proportion += accepted; - nocheck= accept_proportion >= nocheck_thresh; - if (nocheck && !nocheck_reported) { + int new_nocheck= accept_proportion >= nocheck_thresh; + if (new_nocheck && !nocheck_reported) { notice("entering nocheck mode for the first time"); nocheck_reported= 1; + } else if (new_nocheck != nockech) { + debug("nocheck mode %s", new_nocheck ? "start" : "stop"); } + nocheck= new_nocheck; } static void article_done(Connection *conn, Article *art, int whichcount) { @@ -1195,8 +1193,7 @@ typedef void *feedfile_got_article(oop_source *lp, oop_read *rd, if (r) sysdie("link feedfile %s to flushing file %s", feedfile, dut_path); /* => Hardlinked */ - r= unlink(feedfile); - if (r) sysdie("unlink old feedfile link %s", feedfile); + xunlink(feedfile, "old feedfile link"); /* => Moved */ spawn_inndcomm_flush(); /* => Flushing, sets sms to sm_FLUSHING */ @@ -1413,78 +1410,83 @@ static void inputfile_tailing_stop(InputFile *ipf) { /* See official state diagram at top of file. We implement * this as follows: - ================ - WAITING - [Nothing/Noduct] - poll for F - ================ - | - | TIMEOUT and no defer, no backlog - |`--------------------------. - | | - | OPEN F SUCCEEDS | exit - ,--------->| V - | V ========= - | ======== (ESRCH) - | NORMAL [Dropped] - | [Normal] ========= - | main F tail - | ======== - | | - | | F IS SO BIG WE SHOULD FLUSH - ^ | hardlink F to D - | [Hardlinked] - | | unlink F - | | our handle onto F is now onto D - | [Moved] - | | - | |<---------------------------------------------------. + .=======. + ||START|| + `=======' + | + | open F + | + | F ENOENT + |`---------------------------------------------------. + F OPEN OK | | + |`---------------- - - - | + D ENOENT | D EXISTS see OVERALL STATES diagram | + | for full startup logic | + ,--------->| | + | V | + | ============ try to | + | NORMAL open D | + | [Normal] | + | main F tail | + | ============ V + | | | + | | F IS SO BIG WE SHOULD FLUSH, OR TIMEOUT | + ^ | hardlink F to D | + | [Hardlinked] | + | | unlink F | + | | our handle onto F is now onto D | + | [Moved] | + | | | + | |<-------------------<---------------------<---------+ | | | | | spawn inndcomm flush | | V | - | ========== | + | ================ | | FLUSHING | | [Flushing] | - | main D tail | - | ========== | + | main D tail/none | + | ================ | | | | | | INNDCOMM FLUSH FAILS ^ | |`----------------------->----------. | | | | | | | NO SUCH SITE V | - ^ |`--------------->----. =========== | - | | \ FLUSHFAIL | - | | \ [Moved] | - | | \ main D tail | - | | \ =========== | + ^ |`--------------->----. ================ | + | | \ FLUSHFAIL | + | | \ [Moved] | + | | FLUSH OK \ main D tail/none | + | | open F \ ================ | | | \ | | | | \ | TIME TO RETRY | - | | \ `----------------' - | | FLUSH OK \ - | | open F \ - | V V - | ============= ============ - | SEPARATED/ DROPPING/ - | old->fd>=0 old->fd>=0 - | [Separated] [Dropping] - | main F idle main none - | old D tail old D tail - | ============= ============ - | | | - ^ | EOF ON D | EOF ON D - | V V - | =============== =============== - | SEPARATED/ DROPPING/ - | old->fd==-1 old->fd==-1 - | [Finishing] [Dropping] - | main F tail main none - | old D closed old D closed - | =============== =============== - | | | - | | ALL D PROCESSED | ALL D PROCESSED - | V install defer as backlog V install defer as backlog - ^ | close D | close D - | | unlink D | unlink D + |<--------'| ,---<---'\ `----------------' + | D NONE | | D NONE \ + | | | \ + | | | | + | | V | + | V | V + | ============= | ============ + | SEPARATED/ | DROPPING/ + | flsh->fd>=0 | flsh->fd>=0 + | [Separated] | [Dropping] + | main F idle | main none + | old D tail | old D tail + | ============= | ============ + | | | | + ^ | EOF ON D | | EOF ON D + | V | V + | =============== | =============== + | SEPARATED/ | DROPPING/ + | flsh->fd==-1 V flsh->fd==-1 + | [Finishing] | [Dropping] + | main F tail `. main none + | old D closed `. old D closed + | =============== `. =============== + | | `. | + | | ALL D PROCESSED `. | ALL D PROCESSED + | V install defer as backlog `. V install defer as backlog + ^ | close D \| close D + | | unlink D | unlink D + | | | unlink lock | | | exit `----------' V ========== @@ -1542,9 +1544,8 @@ static void statemc_init(void) { open_defer(); /* so that we will later close it and rename it */ break; case 2: - if (unlink(path_defer)) - sysdie("could not unlink stale defer file link %s (presumably" - " hardlink to backlog file)", path_defer); + xunlink(path_defer, "stale defer file link" + " (presumably hardlink to backlog file)"); break; default: die("defer file %s has unexpected link count %d", @@ -1569,8 +1570,7 @@ static void statemc_init(void) { if (samefile(&stab_d, &stab_f)) { debug("startup: F==D => Hardlinked"); - r= unlink(path_flushing); - if (r) sysdie("unlink feed file %s during startup", feedfile); + xunlink(path_flushing, "feed file (during startup)"); found_moved: debug(" => Moved"); startup_set_input_file(file_d); @@ -1582,7 +1582,6 @@ static void statemc_init(void) { } } else { debug("startup: D ENOENT => Nothing"); - fixme need to try flushing innd here - needs state diagram changes; SMS(WAITING, open_wait_periods, "no feed file currently exists"); } } @@ -1650,8 +1649,7 @@ static void *statemc_check_input_done(oop_source *lp, close_defer(); - if (unlink(path_flushing)) - sysdie("could not unlink old flushing file %s", path_flushing); + xunlink(path_flushing, "old flushing file"); if (sms==sm_DROPPING) { if (search_backlog_file()) { @@ -1659,8 +1657,7 @@ static void *statemc_check_input_done(oop_source *lp, return; } notice("feed dropped and our work is complete"); - r= unlink(path_lock); - if (r) sysdie("unlink lockfile for old feed %s", path_lock); + xunlink(path_lock, "lockfile for old feed"); exit(0); } @@ -1759,14 +1756,14 @@ static void close_defer(void) { sysdie("could not unlink old defer link %s to backlog file %s", path_defer, backlog); - if (backlog_nextscan_periods < 0 || - backlog_nextscan_periods > backlog_retry_minperiods + 1) - backlog_nextscan_periods= backlog_retry_minperiods + 1; + if (until_backlog_nextscan < 0 || + until_backlog_nextscan > backlog_retry_minperiods + 1) + until_backlog_nextscan= backlog_retry_minperiods + 1; } static void poll_backlog_file(void) { - if (backlog_nextscan_periods < 0) return; - if (backlog_nextscan_periods-- > 0) return; + if (until_backlog_nextscan < 0) return; + if (until_backlog_nextscan-- > 0) return; search_backlog_file(); } @@ -1823,7 +1820,7 @@ static int search_backlog_file(void) { if (!oldest_path) { debug("backlog scan: none"); - backlog_nextscan_periods= backlog_spontaneous_rescan_periods; + until_backlog_nextscan= backlog_spontaneous_rescan_periods; return 0; } @@ -1841,18 +1838,18 @@ static int search_backlog_file(void) { goto try_again; } inputfile_tailing_start(backlog_input_file); - backlog_nextscan_periods= -1; + until_backlog_nextscan= -1; return 1; } - backlog_nextscan_periods= age_deficiency / PERIOD_SECONDS; + until_backlog_nextscan= age_deficiency / PERIOD_SECONDS; if (backlog_spontaneous_rescan_periods >= 0 && - backlog_nextscan_periods > backlog_spontaneous_rescan_periods) - backlog_nextscan_periods= backlog_spontaneous_rescan_periods; + until_backlog_nextscan > backlog_spontaneous_rescan_periods) + until_backlog_nextscan= backlog_spontaneous_rescan_periods; debug("backlog scan: young age=%f deficiency=%ld nextscan=%d oldest=%s", - age, age_deficiency, backlog_nextscan_periods, oldest_path); + age, age_deficiency, until_backlog_nextscan, oldest_path); return 2; } @@ -1992,7 +1989,7 @@ EVERY(filepoll, {5,0}, { if (main_input_file && main_input_file->readable_callback) filemon_callback(main_input_file); }); - + #define DEBUGF_IPF(wh) " " #wh "=%p/%s:ip=%ld,off=%ld,fd=%d%s" \ #define DEBUG_IPF(sh) \ wh##_input_file, debug_ipf_path(wh##_input_file), \ @@ -2179,9 +2176,15 @@ int main(int argc, char **argv) { } if (!arg) badusage("need site name argument"); - if (*++argv) badusage("too many non-option arguments"); sitename= arg; + if ((arg= *++argv)) { + if (remote_host) badusage("must not specify -h and also host as argument"); + remote_host= arg; + } + + if (*++argv) badusage("too many non-option arguments"); + if (nocheck_thresh_pct < 0 || nocheck_thresh_pct > 100) badusage("nocheck threshold percentage must be between 0..100"); nocheck_thresh= nocheck_thresh_pct * 0.01;