* prefork-interp [<option>,..],<interpreter> <script> [<args> ...]
* prefork-interp '[<option> ..] <interpreter>' <script> [<args> ...]
*
- * Options must specify argument laundering mode.
- * Currently the only mode supported is:
+ * Options must specify argument mediation approach.
+ * Currently the only argument mediation supported is:
+ *
* -U unlaundered: setup and executor both get all arguments and env vars
* ident covers only env vars specified with -E
* ident covers only arguments interpreter and (if present) script
+ *
+ * Options for setting the operation mode:
+ *
+ * (none) Default: start new server if needed, then run service
+ * -f Force a fresh service (old one is terminated)
+ * --kill Kill any existing service; do not actually run anything
+ *
+ * Options for controlling whether different invocations share a server:
+ *
+ * -E VAR ident includes env var VAR (or its absence)
+ * -G STRING ident includes string STRING
+ * -g IDENT use IDENT rather than hex(SHA256(... identity things ...))
+ *
+ * (Ordering of -E and -G options is relevant; invocations with different
+ * -E -G options are different even if the env var settings are the same)
*/
/*
-
-
-
+***************************************************************************
+\f
State during service execution, process parentage and key fds
- ---- pipes, sockets
- 012 descriptors
- -==- fds shared
- || process parentage
- && session leader (daemon)
- & process group leader
-
- CALLER
- ||
- ||
+ CALLER
+ ||
+ ||
|| listen watch-err/in
- || call (accept) \ ,------2
- || ,-----------------------------. SERVER -----0 WATCHER(C)
+ || call (accept) \ ,------2
+ || ,-----------------------------. SERVER -----0 WATCHER(C)
CLIENT 2--=fdpassed>=---------. \ || && | &&
- (C) 1--=fdpassed>=---------. \ \ || inotify
- 0--=fdpassed>=---------. \ \ \ || sockpath
- \ \ \ \ ||
- | | |\ | ||
- | | | \ | ||
+ (C) 1--=fdpassed>=---------. \ \ || inotify
+ 0--=fdpassed>=---------. \ \ \ || sockpath
+ \ \ \ \ ||
+ | | |\ | ||
+ | | | \ | ||
| \ | \ \ ||
\ \ \ \ MONITOR &
- \ \ \ `12 || |
- \ \ \ || |
+ \ \ \ `12 || |
+ \ \ \ || |
\ \ \ || |execterm
- \ \ \ || |
- \ \ \ || |
- \ \ 2 || |
- \ 1 EXECUTOR
- 0
-
- Control flow and causality
-
- | - \ / process control flow
- ... causes mediated by fds or other IPC etc.
+ \ \ \ || |
+ \ \ \ || |
+ \ \ 2 || |
+ \ 1 EXECUTOR
+ 0
+ ---- pipes, sockets
+ 012 descriptors
+ -==- fds shared
+ || process parentage
+ && session leader (daemon)
+ & process group leader
+
+***************************************************************************
+\f
+ Control flow and causality
+
+ CALLER
+ |
+ |fork/exec
+ |
+ CLIENT
+ |
+ attempt to connect, and read greeting
+ |failure? \success?
+ | \
+ tidy up stale /run entries *1 (continue from send_fds, below)
+ acquire lock
+ |
+ retry attempt to connect, and read greeting
+ |failure? \success?
+ | \
+ create listening socket release lock
+ | \
+ fork/daemonise *1
+ | `------------------.
+ | WATCHER(C) &&
+ |
+ make "fake" initial call socketpair (C)
+ | prefork-interp
+ fork/exec #########################################################
+ | `-------------. application
+ | # SCRIPT (setup)
+ | # |
+ | # script initialisation
+ | # | application
+ | ###########|#############################################
+ | # | prefork-interp
+ | # identify fds from envirnment (Perl)
+ | # open syslog
+ | # |
+ | # dzemonize
+ | ,.....<....../ |
+ waitpid # fork for initial service
+ | # |child? |parent?
+ | # | |
+ | # | SCRIPT [server] &&
+ | # | |
+ | # | ** accept / event loop **
+ | # | accepted? \ \ \
+ | # | / \ watch\ \idle
+ | # | fork child \stderr\ \timeout?
+ | # | _________/ | | |
+ | # |/ |read? | |
+ | # SCRIPT [monitor] | eof?| |
+ | # setpgrpt & | | |
+ | # | log msg | |
+ read ,....<.....send greeting | | |
+ greeting # | ___________________
+ | # |
+ release # |
+ lock *1 # |
+ | / # |
+ send fds.....>.... |
+ | # \receive fds
+ | # |
+ | # fork for executor (Perl)
+ | # |parent? \child? prefork-interp
+ | # | ######\############################
+ | # | # SCRIPT (executor) application
+ | # | # execute service
+ | # wait for read # |
+ | # (select) # terminates
+ | # | | # |
+ | # | # kernel closes execterm
+ | # | ,......<....../|
+ | # execterm? # |
+ | # | # zombie
+ | # | | ,......<...../
+ | # waitpid # _______________
+ | # | #
+ | ,....<....,..send status #
+ read status # ________________ #
+ _____________ #
+
+
+ ********** Or, if client is killed **********
+
+ | # | # execute service
+ terminates # wait for read # |
+ | # (select) # |
+ kernel # | | # |
+ closes call # | # |
+ \..>......_ | # |
+ _____________ # \|call? # |
+ # | # |
+ # kill whole pgrp... # killled
+ # | # zombie
+ # | | ,......<....../
+ # waitpid # _______________
+ # | #
+ # send exit status #
+ # _____SIGPIPE______ #
+
+ | - \ / process control flow
+ ... < > causes mediated by fds or other IPC etc.
&& session leader (daemon)
& process group leader
-
- CALLER
- |
- |fork/exec
- |
- CLIENT(C)
- tidy up stale sockets etc.
- acquire lock
- create listening socket
- |
- fork/daemonise
- | `------------------.
- | WATCHER(C) &&
- |
- make "fake" initial call socketpair
- |
- fork/exec
- | `------.
- | SCRIPT (setup)
- | |
- | script initialisation
- | |
- | identify fds from envirnment
- | open syslog
- | |
- | dzemonize
- | ,...../ |
- waitpid |
- | fork for initial service
- | |child |parent
- | | |
- | | SCRIPT [server] &&
- | | |
- | | ** accept / event loop **
- | | / \ \ \
- | | accepted? \ \ \idle timeout?
- | | | \ \ \
- | | fork child \ \ \
- | | _________/ watch | |watch stderr eof?
- | |/ stderr| | |
- | SCRIPT [monitor] | | |
- | | log msg | |
- | send greeting | | |
- read greeting | | | |
- | | exit
- release lock |
- | |
- send fds.... |
- | receive fds
- | |
- | fork for executor
- | | \
- | | \child
- | | \
- | | setpgrp &
- | | execute service
- | | |
- | wait for read |
- | (select) |
- | | | |
- | | exits
- | | kernel closes execterm
- | | ,......./|
- | execterm? |
- | | zombie
- | | ,....../
- | waitpid
- | |
- | send exit status
- read exit status
-
-
- Or, if client is killed
-
- | | |
- | wait for read |
- | (select) |
- | | | |
- exits | |
- | | |
- kernel closes | |
- \....| |
- call? |
- | |
- kill whole pgrp... |
- | killled
- | zombie
- | ,....../
- waitpid
- |
- send exit status
- die due to SIGPIPE
-
-
- */
-/*
- * Process structure:
- * client (C wrapper) connects to server
- * (including reading ack byte)
- * if fails or garbage
- * === acquires lock ===
- * makes new listening socket
- * makes watcher pipes
- * forks watcher and awaits
- * makes first-instance socketpair
- * forks setup (script, sock fds indicated in env)
- * fd0, fd1, fd2: from-outer
- * other fd: call(client-end)(fake)
- * reaps setup (and reports error)
- * (implicitly releases lock)
- *
- * watcher fd[012]: watcher pipes
- * starts watch on socket path
- * sets stderr to line buffered
- * sets stdin to nonblocking
- * daemonises (one fork, becomes session leader)
- * when socket stat changes, quit
- *
- * setup (pre-exec) fd0: null,
- * fd[12]: fd2-from-outer
- * env fds: listener, call(server-end)(fake),
- * watcher read, watcher write
- * close fd: lockfile
- * possibly clean env, argv
- *
- * setup (script) runs initialisation parts of the script
- * at prefork establishment point:
- * setup (pm) [1] opens syslog
- * forks for server
- * [2] exits
- *
- * server (pm) [1] [fd0: null],
- * [fd[12]: fd2-from-outer]
- * setsid
- * right away, forks init monitor
- * [2] closes outer caller fds and call(fake)
- * [server (pm)] fd[012]: null
- * other fds: listener, syslog
- * runs in loop accepting and forking,
- * reaping and limiting children (incl init monitor)
- * reports failures of monitors to syslog
- *
- * [client (C wrapper)] if client connect succeeds:
- * now fd: call(client-end)
- * sends message with: cmdline, env
- * sends fds
- *
- * [server (script)] accepts, forks subseq monitor
- *
- * monitor [1] [fd0: null]
- * (init [fd[12]: init: fd2-from-outer; subseq: null]
- * or errors: init: fd2; subseq: syslog
- * subseq) other fds: syslog, call(server-end)
- * sends ack byte
- * receives args, env, fds
- * forks executor
- *
- * executor sorts out fds:
- * fd0, fd1, fd2: from-outer
- * close fds: call(server-end)
- * retained fds: syslog
- *
- * sets cmdline, env
- * runs main part of script
- * exits normally
- *
- * [monitor] [fd[012]: null]
- * [fd[12]: init: fd2-from-outer; subseq: null]
- * [errors: init: fd2; subseq: syslog]
- * reaps executor
- * reports status via socket
- *
- * [client (C wrapper)] [fd0, fd1, fd2: from-outer]
- * [other fd: call(client-end)]
- * receives status, exits appropriately
- * (if was bad signal, reports to stderr, exits 127)
- */
+ # language/implementation boundary
+ *1 line continued elsewhere
+ event? condition
+ ______ process termination (after reaping, if shown)
+
+***************************************************************************
+\f
+ Sequence of events and fd pluming.
+ NB INCOMPLETE - does not cover execterm, cleanup
+
+ client (C wrapper) connects to server
+ (including reading ack byte)
+ if fails or garbage
+ === acquires lock ===
+ makes new listening socket
+ makes watcher pipes
+ forks watcher and awaits
+ makes first-instance socketpair
+ forks setup (script, sock fds indicated in env)
+ fd0, fd1, fd2: from-outer
+ other fd: call(client-end)(fake)
+ reaps setup (and reports error)
+ (implicitly releases lock)
+
+ watcher fd[012]: watcher pipes
+ starts watch on socket path
+ sets stderr to line buffered
+ sets stdin to nonblocking
+ daemonises (one fork, becomes session leader)
+ when socket stat changes, quit
+
+ setup (pre-exec) fd0: null,
+ fd[12]: fd2-from-outer
+ env fds: listener, call(server-end)(fake),
+ watcher read, watcher write
+ close fd: lockfile
+ possibly clean env, argv
+
+ setup (script) runs initialisation parts of the script
+ at prefork establishment point:
+ setup (pm) [1] opens syslog
+ forks for server
+ [2] exits
+
+ server (pm) [1] [fd0: null],
+ [fd[12]: fd2-from-outer]
+ setsid
+ right away, forks init monitor
+ [2] closes outer caller fds and call(fake)
+ [server (pm)] fd[012]: null
+ other fds: listener, syslog
+ runs in loop accepting and forking,
+ reaping and limiting children (incl init monitor)
+ reports failures of monitors to syslog
+
+ [client (C wrapper)] if client connect succeeds:
+ now fd: call(client-end)
+ sends message with: cmdline, env
+ sends fds
+
+ [server (script)] accepts, forks subseq monitor
+
+ monitor [1] [fd0: null]
+ (init [fd[12]: init: fd2-from-outer; subseq: null]
+ or errors: init: fd2; subseq: syslog
+ subseq) other fds: syslog, call(server-end)
+ sends ack byte
+ receives args, env, fds
+ forks executor
+
+ executor sorts out fds:
+ fd0, fd1, fd2: from-outer
+ close fds: call(server-end)
+ retained fds: syslog
+
+ sets cmdline, env
+ runs main part of script
+ exits normally
+
+ [monitor] [fd[012]: null]
+ [fd[12]: init: fd2-from-outer; subseq: null]
+ [errors: init: fd2; subseq: syslog]
+ reaps executor
+ reports status via socket
+
+ [client (C wrapper)] [fd0, fd1, fd2: from-outer]
+ [other fd: call(client-end)]
+ receives status, exits appropriately
+ (if was bad signal, reports to stderr, exits 127)
+
+***************************************************************************
+\f
+ Protocol, and functions of the script
+
+ 1. Script interpreter will be spawned apparently as normal;
+ should run synchronously in the normal way until
+ "initialisation complete" point. At initialisation complete:
+
+ 2. Env var PREFORK_INTERP contains:
+
+ v1,SECS.NSECS[,...] LISTEN,CALL,WATCHE,WATCHI[,...][ ???]
+
+ To parse it: treat as bytes and split on ASCII space, taking
+ the first two words. (There may or may not be
+ further "words"; and if there are they might be binary data.)
+ Then split each of the first two words (which will contain only
+ ASCII printing characters) on comma. Take the first two items:
+
+ v1 Protocol version indicator - literal. If something else,
+ fail (means installation is incompatible somehow).
+
+ SECS.NSECS
+ timestamp just before script started running, as a
+ decimal time_t. NSECS is exactly 9 digits.
+ To be used for auto reloading (see below).
+
+ The 2nd word's items are file descriptors:
+
+ LISTEN listening socket nonblocking
+ CALL call socket for initial call blocking
+ WATCHE liveness watcher stderr nonblocking
+ WATCHI liveness sentinel unspecified
+
+ (any further descriptors should be ignored, not closed)
+
+ 3. Library should do the following:
+
+ 1. Read and understand the PREFORK_INTERP env var.
+ If it is not set, initialisation complete should simply return.
+ (This allows simple synchronous operation.)
+
+ 2. Open syslog
+ 3. fork/exit (fork and have parent exit) (to make server)
+ 4. setsid (to become session leader)
+ 5. fork initial service (monitor) child, using CALL (see below)
+ 6. Replace stdin/stdout/stderr with /dev/null,
+ and make a note to send all error messages to syslog
+ 7. Enter select loop, looking for the following:
+
+ A. accept on LISTEN:
+ i. see if we need to reload: is any file forming part
+ of the program newer than the SECS.NSECS ?
+ If so, log at LOG_INFO, and exit immediately
+ (dropping CALL, LISTEN, WATCHI, etc.)
+ ii. see if we can reap any children, possibly waiting
+ for children if we are at our concurrency limit
+ (limit should be configured through library, default 4)
+ Report child exit status if not zero or SIGPIPE.
+ iii. fork service (monitor) child, using accepted fd
+
+ B. WATCHE is readable:
+ * EOF: log at LOG_INFO, and exit
+ * data to read: read what is available immediately;
+ it will be an error message: log it at LOG_ERR, and exit
+
+ 4. service (monitor) child does the following:
+
+ 1. close all of LISTEN, WATCHI, WATCHE
+ 2. setpgrp
+ 3. send a greeting (on CALL) "PFI\n\0\0\0\0" (8 bytes)
+ 4. read a single byte, fail if it's not zero
+ 5. three times, receive a single byte with a file descriptor
+ attached as ancillary data. (These descriptors will be
+ service stdin, stdout, stderr.)
+ 6. read a 4-byte big-endian length
+ 7. read that many bytes, the initial service request message,
+ which contains the following nul-terminated strings:
+ * environment variable settings in the format NAME=value
+ * an empty string
+ * arguments NOT INCLUDING argv[0] or script filename
+ (not that this means the service request must end in a nul)
+ 8. make a new pipe EXECTERM
+ 9. fork for the service executor; in the child
+ i. redirect stdin/stdout/stderr to the recevied fds
+ ii. replace environment and arguments with those received,
+ iii. close descriptors: close the original received descriptors;
+ close CALL; keep only the writing end of EXECTERM
+ iv. if the script programming language does things with SIGINT,
+ set it set back to default handling (immediate termination).
+ v. return back to script, now in the grandchild
+
+ 10. in the parent, close EXECTERM writing end, and
+ 11. select, looking for one of the following:
+ * CALL is readable
+ * EXECTERM reading end is readable
+ No need to actually read, since these shouldn't produce
+ spurious wakeups (but do loop on EINTR).
+ 12. set SIGINT to ignored
+ 13. send SIGINT to the entire process group
+ 14. wait, blocking, for the executor child
+ 15. write the wait status, in 32-bit big-endian, to CAL
+ 16. exit 0
+
+ Errors detected in the service monitor should be sent to
+ syslog, or stderr, depending on whether this is the initial
+ service monitor (from part 3 step 5) or an accepted socket
+ service monitor (from part 4 step 9); this can be achieved
+ easily by having a global flag (set at part 3 step 6),
+ or perhaps using logger(8) and redirecting stderr (but
+ then be careful to ensure everyone gets only the necessary fds).
+
+ EOF on CALL, or EPIPE/SIGPIPE writing to it, are not errors.
+ In this case, exit zero or die with SIGPIPE, so parent
+ won't report error either (part 3 step 7(A)(ii)).
+
+***************************************************************************
+\f
+*/
#include <arpa/inet.h>
+#include <sys/utsname.h>
#include <uv.h>
fprintf(f, "usage: #!/usr/bin/prefork-interp [<options>]\n");
}
-static int laundering;
+#define MODE_NORMAL 0
+#define MODE_KILL 'k'
+#define MODE_FRESH 'f'
+
+#define MEDIATION_UNSPECIFIED 0
+#define MEDIATION_UNLAUNDERED 'U'
+
+static int mediation = MEDIATION_UNSPECIFIED;
+static int mode = MODE_NORMAL;
static int max_sockets = 100; // maximum entries in the run dir is 2x this
static struct stat initial_stab;
const struct cmdinfo cmdinfos[]= {
PREFORK_CMDINFOS
- { 0, 'U', 0, .iassignto= &laundering, .arg= 'U' },
+ { 0, 'U', 0, .iassignto= &mediation, .arg= MEDIATION_UNLAUNDERED },
+ { "kill", 0, 0, .iassignto= &mode, .arg= MODE_KILL },
+ { 0, 'f', 0, .iassignto= &mode, .arg= MODE_FRESH },
{ 0 }
};
+static void ident_add_stat(const char *path) {
+ struct stat stab;
+ int r = stat(path, &stab);
+ if (r) diee("failed to stat %s", path);
+
+ IDENT_ADD_OBJ(path[0], stab.st_dev);
+ IDENT_ADD_OBJ('i', stab.st_ino);
+}
+
void ident_addinit(void) {
- char ident_magic[1] = { 0 };
- sha256_update(&identsc, sizeof(ident_magic), ident_magic);
+ ident_add_key_byte(1);
+
+ struct utsname uts = { };
+ size_t utslen = sizeof(uts);
+ int r = uname(&uts);
+ if (r) diee("uname failed!");
+ IDENT_ADD_OBJ('u', utslen);
+ IDENT_ADD_OBJ('u', uts);
+
+ ident_add_stat(".");
+ ident_add_stat("/");
}
static void propagate_exit_status(int status, const char *what) {
int r;
if (WIFEXITED(status)) {
- _exit(status);
+ _exit(WEXITSTATUS(status));
}
if (WIFSIGNALED(status)) {
if (! WCOREDUMP(status) &&
(sig == SIGINT ||
+ sig == SIGTERM ||
sig == SIGHUP ||
sig == SIGPIPE ||
sig == SIGKILL)) {
struct sigaction sa;
FILLZERO(sa);
sa.sa_handler = SIG_DFL;
- r = sigaction(sig, &sa, 0);
- if (r) diee("failed to reset signal handler while propagating %s",
- signame);
-
- sigset_t sset;
- sigemptyset(&sset);
- sigaddset(&sset, sig);
- r = sigprocmask(SIG_UNBLOCK, &sset, 0);
- if (r) diee("failed to reset signal block while propagating %s",
- signame);
+ if (sig != SIGKILL) {
+ r = sigaction(sig, &sa, 0);
+ if (r) diee("failed to reset signal handler while propagating %s",
+ signame);
+
+ sigset_t sset;
+ sigemptyset(&sset);
+ sigaddset(&sset, sig);
+ r = sigprocmask(SIG_UNBLOCK, &sset, 0);
+ if (r) diee("failed to reset signal block while propagating %s",
+ signame);
+ }
raise(sig);
die("unexpectedly kept running after raising (to propagate) %s",
}
static void send_request(void) {
- // Sending these first makes it easier for the script to
+ char ibyte= 0;
+ ssize_t sr = fwrite(&ibyte, 1, 1, call_sock);
+ if (sr != 1) diee("write signalling byte");
+
+ // Sending these before the big message makes it easier for the script to
// use buffered IO for the message.
send_fd(0);
send_fd(1);
prepare_message(0, &p);
assert(p == m + tlen);
- ssize_t sr = fwrite(m, tlen, 1, call_sock);
+ sr = fwrite(m, tlen, 1, call_sock);
if (sr != 1) diee("write request (buffer)");
if (fflush(call_sock)) diee("write request");
char got_magic[sizeof(header_magic)];
if (protocol_read_maybe(&got_magic, sizeof(got_magic)) < 0)
- return "initial monitor process quit";
+ return "initial monitor process quit"
+ " (maybe script didn't call preform_initialisation_complete?)";
if (memcmp(got_magic, header_magic, sizeof(header_magic)))
die("got unexpected protocol magic 0x%02x%02x%02x%02x",
int r;
int fd = -1;
+ if (mode != MODE_NORMAL) return 0;
+
fd = socket(AF_UNIX, SOCK_STREAM, 0);
if (fd==-1) diee("socket() for client");
}
static __attribute__((noreturn))
-void become_setup(int sfd, int fake_pair[2],
+void become_setup(int sfd, int lockfd, int fake_pair[2],
int watcher_stdin, int watcher_stderr) {
+ close(lockfd);
close(fake_pair[0]);
int call_fd = fake_pair[1];
// Extension could work like this:
//
- // We advertise a new protocol (perhaps one which is nearly entirely
+ // We could advertise a new protocol (perhaps one which is nearly entirely
// different after the connect) by putting a name for it comma-separated
// next to "v1". Simple extension can be done by having the script
// side say something about it in the ack xdata, which we currently ignore.
- putenv(m_asprintf("PREFORK_INTERP=v1 %d,%d,%d,%d",
+ // Or we could add other extra data after v1.
+ putenv(m_asprintf("PREFORK_INTERP=v1,%jd.%09ld %d,%d,%d,%d",
+ (intmax_t)initial_stab.st_mtim.tv_sec,
+ (long)initial_stab.st_mtim.tv_nsec,
sfd, call_fd, watcher_stdin, watcher_stderr));
execvp(executor_argv[0], (char**)executor_argv);
preclean();
int lockfd = acquire_lock();
+
+ if (mode == MODE_KILL) {
+ r= unlink(socket_path);
+ if (r && errno != ENOENT) diee("remove socket %s", socket_path);
+
+ r= unlink(lock_path);
+ if (r) diee("rmeove lock %s", lock_path);
+ _exit(0);
+ }
+
call_sock = connect_existing();
if (call_sock) { close(lockfd); return; }
pid_t setup_pid = fork();
if (setup_pid == (pid_t)-1) diee("fork for spawn setup");
- if (!setup_pid) become_setup(sfd, fake_pair,
+ if (!setup_pid) become_setup(sfd, lockfd, fake_pair,
watcher_stdin[1], watcher_stderr[0]);
close(fake_pair[1]);
close(sfd);
}
static void make_executor_argv(const char *const *argv) {
- switch (laundering) {
- case 'U': break;
+ switch (mediation) {
+ case MEDIATION_UNLAUNDERED: break;
default: die("need -U (specifying unlaundered argument handling)");
}
~ian / chiark-utils.git / blobdiff