prefork-interp: diagrams

[chiark-utils.git] / cprogs / prefork-interp.c

/*
 * "Interpreter" that you can put in #! like this
 *   #!/usr/bin/prefork-interp [<options>] <interpreter>
 *
 * Usages:
 *   prefork-interp  [<option> ..] <interpreter>  [<script> [<args> ...]]
 *   prefork-interp  [<option>,..],<interpreter>   <script> [<args> ...]
 *   prefork-interp '[<option> ..] <interpreter>'  <script> [<args> ...]
 *
 * Options must specify argument laundering mode.
 * Currently the only mode supported is:
 *   -U    unlaundered: setup and executor both get all arguments and env vars
 *         ident covers only env vars specified  with -E
 *         ident covers only arguments interpreter and (if present) script
 */

/*
                                   
                                                 
                                                 
  State during service execution, process parentage and key fds

    ---- pipes, sockets   
    012  descriptors      
    -==- fds shared       
    ||   process parentage
    &&   session leader (daemon) 
    &    process group leader    
                             
      CALLER                 
        ||                   
        ||                   
        ||                               listen     watch-err/in
        ||       call                 (accept) \     ,------2          
        || ,-----------------------------.     SERVER -----0 WATCHER(C) 
      CLIENT 2--=fdpassed>=---------.     \      || &&          |      &&
       (C)  1--=fdpassed>=---------. \     \     ||           inotify    
           0--=fdpassed>=---------. \ \     \    ||           sockpath   
                                   \ \ \     \   ||                      
                                   | | |\     |  || 
                                   | | | \    |  || 
                                   | \ |  \   \  ||
                                    \ \ \  \   MONITOR &
                                     \ \ \  `12  ||  |  
                                      \ \ \      ||  |  
                                       \ \ \     ||  |execterm
                                        \ \ \    ||  |  
                                         \ \ \   ||  |  
                                          \ \ 2  ||  |  
                                           \ 1 EXECUTOR 
                                            0           
                                                        
 Control flow and causality                             
                                                        
    | - \ /    process control flow                     
    ...        causes mediated by fds or other IPC etc. 
    &&         session leader (daemon)
    &          process group leader
                                             
     CALLER                                  
         |                                   
         |fork/exec                          
         |                                   
     CLIENT(C)                               
         |                                   
      attempt to connect, and read greeting
         |failure?                \success?  
         |                         \         
      tidy up stale /run entries    continue from send_fds, below  
      acquire lock                                                 
         |                                                         
      retry attempt to connect, and read greeting                  
         |failure?                \success?                        
         |                         \                               
      create listening socket     release lock                     
         |                           \                             
      fork/daemonise                  continue from send_fds, below                    
         |    `------------------.           
         |                      WATCHER(C) &&
         |                                  
       make "fake" initial call socketpair                       
         |                                           C prefork-interp               
       fork/exec  + + + + + + + + + + + + + + + + + + + + + + + + + + + +           
         |      `------.                             Perl, application              
         |      +    SCRIPT (setup)                                                 
         |      +      |                                                            
         |      +   script initialisation                                           
         |      +      |                             Perl, application                     
         |      + + + +| + + + + + + + + + + + + + + + + + + + + + + + +                   
         |      +      |                             Perl, prefork-interp                      
         |      +   identify fds from envirnment                        
         |      +   open syslog                                         
         |      +      |                                                
         |      +   dzemonize                            
         |   ,...../   |                                 
      waitpid   +      |                                 
         |      +    fork for initial service            
         |      +      |child        |parent                 
         |      +      |             |                       
         |      +      |         SCRIPT [server] &&      
         |      +      |             |                       
         |      +      |         ** accept / event loop **   
         |      +      |            /            \ \ \                   
         |      +      |        accepted?         \ \ \idle timeout?     
         |      +      |           |               \ \ \                 
         |      +      |        fork child          \ \ \                
         |      +      | _________/            watch | |watch stderr eof?
         |      +      |/                      stderr| | |          
         |      + SCRIPT [monitor]                   | | |   
         |      +      |                      log msg  | |   
         |      +   setpgrpt &                       | | |   
         |      +      |                             | | |   
         |      + send greeting                      | | |   
      read greeting    |                             exit    
         |      +      |                                     
      release lock     |          from read greeting success, above 
         |    ________ | ________/                       
         |   /  +      |                                     
      send fds....     |                                     
         |      + receive fds                                
         |      +         |                                  
         |      +     fork for executor                      
         |      +   p  |parent        \child         Perl, prefork-interp                                         
         |      +      |        + + + + \+ + + + + + + + + + + + + + + +                            
         |      +      |        +   execute service  Perl, application                                 
         |      +      |        +       |                               
         |      +wait for read  +       |                               
         |      +  (select)     +       |                               
         |      +    |   |      +       |                               
         |      +        |      +     exits                             
         |      +        |      +     kernel closes execterm            
         |      +        | ,.........../|                               
         |      +  execterm?    +       |                               
         |      +        |      +     zombie                            
         |      +        | ,........../             
         |      +   waitpid     +
         |      +        |      +                                         
         |    ,..,..send status +
    read status +               +                                         
                +               +
                                         
  Or, if client is killed                
                                         
         |             |                |      
         |       wait for read          |    
         |         (select)             |    
         |           |   |              |      
       exits         |                  |    
         |           |                  |    
     kernel closes   |                  |    
                \....|                  |    
                     call?              |    
                     |                  |    
               kill whole pgrp...       |    
                         |           killled 
                         |            zombie 
                         | ,.....    ./      
                    waitpid
                         |  
                    send exit status
                    die due to SIGPIPE
         
                
 */                 
/*
 * Process structure:
 *  client (C wrapper)        connects to server
 *                              (including reading ack byte)
 *                            if fails or garbage
 *                            === acquires lock ===
 *                            makes new listening socket
 *                            makes watcher pipes
 *                            forks watcher and awaits
 *                            makes first-instance socketpair
 *                            forks setup (script, sock fds indicated in env)
 *                            fd0, fd1, fd2: from-outer
 *                            other fd: call(client-end)(fake)
 *                            reaps setup (and reports error)
 *                            (implicitly releases lock)
 *
 *     watcher                fd[012]: watcher pipes
 *                            starts watch on socket path
 *                            sets stderr to line buffered
 *                            sets stdin to nonblocking
 *                            daemonises (one fork, becomes session leader)
 *                            when socket stat changes, quit
 *
 *     setup (pre-exec)       fd0: null,
 *                            fd[12]: fd2-from-outer
 *                            env fds: listener, call(server-end)(fake),
 *                                      watcher read, watcher write
 *                            close fd: lockfile
 *                            possibly clean env, argv
 *
 *     setup (script)         runs initialisation parts of the script
 *                            at prefork establishment point:
 *     setup (pm) [1]         opens syslog
 *                            forks for server
 *                [2]         exits
 *
 *        server (pm) [1]     [fd0: null],
 *                            [fd[12]: fd2-from-outer]
 *                            setsid
 *                            right away, forks init monitor
 *                    [2]     closes outer caller fds and call(fake)
 *        [server (pm)]       fd[012]: null
 *                            other fds: listener, syslog
 *                            runs in loop accepting and forking,
 *                            reaping and limiting children (incl init monitor)
 *                            reports failures of monitors to syslog
 *
 *  [client (C wrapper)]      if client connect succeeds:
 *                            now fd: call(client-end)
 *                               sends message with: cmdline, env
 *                               sends fds
 *
 *        [server (script)]   accepts, forks subseq monitor
 *
 *          monitor [1]       [fd0: null]
 *           (init            [fd[12]: init: fd2-from-outer; subseq: null]
 *             or             errors: init: fd2; subseq: syslog
 *            subseq)         other fds: syslog, call(server-end)
 *                            sends ack byte
 *                            receives args, env, fds
 *                            forks executor
 *
 *            executor        sorts out fds:
 *                            fd0, fd1, fd2: from-outer
 *                            close fds: call(server-end)
 *                            retained fds: syslog
 *
 *                            sets cmdline, env
 *                            runs main part of script
 *                            exits normally
 *
 *          [monitor]         [fd[012]: null]
 *                            [fd[12]: init: fd2-from-outer; subseq: null]
 *                            [errors: init: fd2; subseq: syslog]
 *                            reaps executor
 *                            reports status via socket
 *
 *    [client (C wrapper)]    [fd0, fd1, fd2: from-outer]
 *                            [other fd: call(client-end)]
 *                            receives status, exits appropriately
 *                            (if was bad signal, reports to stderr, exits 127)
 */

#include <arpa/inet.h>

#include <uv.h>

#include "prefork.h"

const char our_name[] = "prefork-interp";

static struct sockaddr_un sockaddr_sun;
static FILE *call_sock;

#define ACK_BYTE '\n'

static const char *const *executor_argv;

static const char header_magic[4] = "PFI\n";

void fusagemessage(FILE *f) {
  fprintf(f, "usage: #!/usr/bin/prefork-interp [<options>]\n");
}

static int laundering;
static int max_sockets = 100; // maximum entries in the run dir is 2x this

static struct stat initial_stab;

const struct cmdinfo cmdinfos[]= {
  PREFORK_CMDINFOS
  { 0, 'U',   0,                    .iassignto= &laundering,    .arg= 'U' },
  { 0 }
};

void ident_addinit(void) {
  char ident_magic[1] = { 0 };
  sha256_update(&identsc, sizeof(ident_magic), ident_magic);
}

static void propagate_exit_status(int status, const char *what) {
  int r;

  if (WIFEXITED(status)) {
    _exit(status);
  }

  if (WIFSIGNALED(status)) {
    int sig = WTERMSIG(status);
    const char *signame = strsignal(sig);
    if (signame == 0) signame = "unknown signal";

    if (! WCOREDUMP(status) &&
        (sig == SIGINT ||
         sig == SIGHUP ||
         sig == SIGPIPE ||
         sig == SIGKILL)) {
      struct sigaction sa;
      FILLZERO(sa);
      sa.sa_handler = SIG_DFL;
      r = sigaction(sig, &sa, 0);
      if (r) diee("failed to reset signal handler while propagating %s",
                  signame);

      sigset_t sset;
      sigemptyset(&sset);
      sigaddset(&sset, sig);
      r = sigprocmask(SIG_UNBLOCK, &sset, 0);
      if (r) diee("failed to reset signal block while propagating %s",
                  signame);

      raise(sig);
      die("unexpectedly kept running after raising (to propagate) %s",
          signame);
    }

    die("%s failed due to signal %d %s%s", what, sig, signame,
        WCOREDUMP(status) ? " (core dumped)" : "");
  }

  die("%s failed with weird wait status %d 0x%x", what, status, status);
}

typedef struct {
  char *name_hash;
  time_t atime;
} PrecleanEntry;

static int preclean_entry_compar_name(const void *av, const void *bv) {
  const PrecleanEntry *a = av;
  const PrecleanEntry *b = bv;
  return strcmp(a->name_hash, b->name_hash);
}

static int preclean_entry_compar_atime(const void *av, const void *bv) {
  const PrecleanEntry *ae = av;  time_t a = ae->atime;
  const PrecleanEntry *be = bv;  time_t b = be->atime;
  return (a > b ? +1 :
          a < b ? -1 : 0);
}

static time_t preclean_stat_atime(const char *s_path) {
  struct stat stab;
  int r= lstat(s_path, &stab);
  if (r) {
    if (errno!=ENOENT) diee("pre-cleanup: stat socket (%s)", s_path);
    return 0;
  }
  return stab.st_atime;
}

static void preclean(void) {
  DIR *dir = opendir(run_base);
  if (!dir) {
    if (errno == ENOENT) return;
    diee("pre-cleanup: open run dir (%s)", run_base);
  }

  PrecleanEntry *entries=0;
  size_t avail_entries=0;
  size_t used_entries=0;

  struct dirent *de;
  while ((errno = 0, de = readdir(dir))) {
    char c0 = de->d_name[0];
    if (!(c0 == 'l' || c0 == 's')) continue;
    char *name_hash = m_asprintf("%s", de->d_name+1);
    char *s_path = m_asprintf("%s/s%s", run_base, name_hash);
    time_t atime = preclean_stat_atime(s_path);

    if (avail_entries == used_entries) {
      assert(avail_entries < INT_MAX / 4 / sizeof(PrecleanEntry));
      avail_entries <<= 1;
      avail_entries += 10;
      entries = realloc(entries, avail_entries * sizeof(PrecleanEntry));
    }
    entries[used_entries].name_hash = name_hash;
    entries[used_entries].atime = atime;
    used_entries++;
  }
  if (errno) diee("pre-cleanup: read run dir (%s)", run_base);

  // First we dedupe (after sorting by path)
  qsort(entries, used_entries, sizeof(PrecleanEntry),
        preclean_entry_compar_name);
  PrecleanEntry *p, *q;
  for (p=entries, q=entries; p < entries + used_entries; p++) {
    if (q > entries && !strcmp(p->name_hash, (q-1)->name_hash))
      continue;
    *q++ = *p;
  }
  used_entries = q - entries;

  // Now maybe delete some things
  //
  // Actually this has an off-by-one error since we are about
  // to create a socket, so the actual number of sockets is one more.
  // But, *actually*, since there might be multiple of us running at once,
  // we might have even more than that.  This doesn't really matter.
  if (used_entries > max_sockets) {
    qsort(entries, used_entries, sizeof(PrecleanEntry),
          preclean_entry_compar_atime);
    for (p=entries; p < entries + max_sockets; p++) {
      char *l_path = m_asprintf("%s/l%s", run_base, p->name_hash);
      char *s_path = m_asprintf("%s/s%s", run_base, p->name_hash);
      int lock_fd = flock_file(l_path);
      // Recheck atime - we might have raced!
      time_t atime = preclean_stat_atime(s_path);
      if (atime != p->atime) {
        // Raced.  This will leave use deleting too few things.  Whatever.
      } else {
        int r= unlink(s_path);
        if (r && errno!=ENOENT) diee("preclean: delete stale (%s)", s_path);
        r= unlink(l_path);
        if (r) diee("preclean: delete stale lock (%s)", s_path);
        // NB we don't hold the lock any more now.
      }
      close(lock_fd);
      free(l_path);
      free(s_path);
    }
  }

  for (p=entries; p < entries + used_entries; p++)
    free(p->name_hash);
  free(entries);
}

static __attribute((noreturn)) void die_data_overflow(void) {
  die("cannot handle data with length >2^32");
}

static void prepare_data(size_t *len, char **buf,
                         const void *data, size_t dl) {
  if (len) {
    if (dl >= SIZE_MAX - *len)
      die_data_overflow();
    *len += dl;
  }
  if (buf) {
    memcpy(*buf, data, dl);
    *buf += dl;
  }
}

static void prepare_length(size_t *len, char **buf, size_t dl_sz) {
  if (dl_sz > UINT32_MAX) die_data_overflow();
  uint32_t dl = htonl(dl_sz);
  prepare_data(len, buf, &dl, sizeof(dl));
}

static void prepare_string(size_t *len, char **buf, const char *s) {
  size_t sl = strlen(s);
  prepare_data(len, buf, s, sl+1);
}

static void prepare_message(size_t *len, char **buf) {
  const char *s;

  const char *const *p = (void*)environ;
  while ((s = *p++)) {
    if (strchr(s, '='))
      prepare_string(len, buf, s);
  }

  prepare_string(len, buf, "");

  p = executor_argv;
  while ((s = *p++))
    prepare_string(len, buf, s);
}

static void send_fd(int payload_fd) {
  int via_fd = fileno(call_sock);

  union {
    struct cmsghdr align;
    char buf[CMSG_SPACE(sizeof(payload_fd))];
  } cmsg_buf;

  struct msghdr msg;
  FILLZERO(msg);
  FILLZERO(cmsg_buf);

  char dummy_byte = 0;
  struct iovec iov;
  FILLZERO(iov);
  iov.iov_base = &dummy_byte;
  iov.iov_len = 1;

  msg.msg_name = 0;
  msg.msg_iov = &iov;
  msg.msg_iovlen = 1;
  msg.msg_control = cmsg_buf.buf;
  msg.msg_controllen = sizeof(cmsg_buf.buf);

  struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
  cmsg->cmsg_level = SOL_SOCKET;
  cmsg->cmsg_type = SCM_RIGHTS;
  cmsg->cmsg_len = CMSG_LEN(sizeof(payload_fd));
  *(int*)CMSG_DATA(cmsg) = payload_fd;

  msg.msg_controllen = sizeof(cmsg_buf.buf);

  for (;;) {
    ssize_t r = sendmsg(via_fd, &msg, 0);
    if (r == -1) {
      if (errno == EINTR) continue;
      diee("send fd");
    }
    assert(r == 1);
    break;
  }
}

static void send_request(void) {
  // Sending these first makes it easier for the script to
  // use buffered IO for the message.
  send_fd(0);
  send_fd(1);
  send_fd(2);

  size_t len = 0;
  prepare_message(&len, 0);

  size_t tlen = len + 4;
  char *m = xmalloc(tlen);
  char *p = m;
  prepare_length(0, &p, len);
  prepare_message(0, &p);
  assert(p == m + tlen);

  ssize_t sr = fwrite(m, tlen, 1, call_sock);
  if (sr != 1) diee("write request (buffer)");

  if (fflush(call_sock)) diee("write request");
}

static FILE *call_sock_from_fd(int fd) {
  int r;

  FILE *call_sock = fdopen(fd, "r+");
  if (!call_sock) diee("fdopen socket");

  r = setvbuf(call_sock, 0, _IONBF, 0);
  if (r) die("setvbuf socket");

  return call_sock;
}

static bool was_eof(FILE *call_sock) {
  return feof(call_sock) || errno==ECONNRESET;
}

// Returns -1 on EOF
static int protocol_read_maybe(void *data, size_t sz) {
  if (!sz) return 0;
  size_t sr = fread(data, sz, 1, call_sock);
  if (sr != 1) {
    if (was_eof(call_sock)) return -1;
    diee("read() on monitor call socket (%zd)", sz);
  }
  return 0;
}

static void protocol_read(void *data, size_t sz) {
  if (protocol_read_maybe(data, sz) < 0)
    die("monitor process quit unexpectedly");
}

// Returns 0 if OK, error msg if peer was garbage.
static const char *read_greeting(void) {
  char got_magic[sizeof(header_magic)];

  if (protocol_read_maybe(&got_magic, sizeof(got_magic)) < 0)
    return "initial monitor process quit";

  if (memcmp(got_magic, header_magic, sizeof(header_magic)))
    die("got unexpected protocol magic 0x%02x%02x%02x%02x",
        got_magic[0], got_magic[1], got_magic[2], got_magic[3]);

  uint32_t xdata_len;
  protocol_read(&xdata_len, sizeof(xdata_len));
  void *xdata = xmalloc(xdata_len);
  protocol_read(xdata, xdata_len);

  return 0;
}

// Returns: call(client-end), or 0 to mean "is garbage"
// find_socket_path must have been called
static FILE *connect_existing(void) {
  int r;
  int fd = -1;

  fd = socket(AF_UNIX, SOCK_STREAM, 0);
  if (fd==-1) diee("socket() for client");

  socklen_t salen = sizeof(sockaddr_sun);
  r = connect(fd, (const struct sockaddr*)&sockaddr_sun, salen);
  if (r==-1) {
    if (errno==ECONNREFUSED || errno==ENOENT) goto x_garbage;
    diee("connect() %s", socket_path);
  }

  call_sock = call_sock_from_fd(fd);
  fd = -1;

  if (read_greeting())
    goto x_garbage;

  return call_sock;

 x_garbage:
  if (call_sock) { fclose(call_sock); call_sock=0; }
  if (fd >= 0) close(fd);
  return 0;
}

static void watcher_cb_stdin(uv_poll_t *handle, int status, int events) {
  char c;
  int r;

  if ((errno = -status)) diee("watcher: poll stdin");
  for (;;) {
    r= read(0, &c, 1);
    if (r!=-1) _exit(0);
    if (!(errno==EINTR || errno==EWOULDBLOCK || errno==EAGAIN))
      diee("watcher: read sentinel stdin");
  }
}

static void watcher_cb_sockpath(uv_fs_event_t *handle, const char *filename,
                                int events, int status) {
  int r;
  struct stat now_stab;

  if ((errno = -status)) diee("watcher: poll stdin");
  for (;;) {
    r= stat(socket_path, &now_stab);
    if (r==-1) {
      if (errno==ENOENT) _exit(0);
      if (errno==EINTR) continue;
      diee("stat socket: %s", socket_path);
    }
    if (!stabs_same_inode(&now_stab, &initial_stab))
      _exit(0);
  }
}

// On entry, stderr is still inherited, but 0 and 1 are the pipes
static __attribute__((noreturn))
void become_watcher(void) {
  uv_loop_t loop;
  uv_poll_t uvhandle_stdin;
  uv_fs_event_t uvhandle_sockpath;
  int r;

  nonblock(0);

  errno= -uv_loop_init(&loop);
  if (errno) diee("watcher: uv_loop_init");

  errno= -uv_poll_init(&loop, &uvhandle_stdin, 0);
  if (errno) diee("watcher: uv_poll_init");
  errno= -uv_poll_start(&uvhandle_stdin,
                        UV_READABLE | UV_WRITABLE | UV_DISCONNECT,
                        watcher_cb_stdin);
  if (errno) diee("watcher: uv_poll_start");

  errno= -uv_fs_event_init(&loop, &uvhandle_sockpath);
  if (errno) diee("watcher: uv_fs_event_init");

  errno= -uv_fs_event_start(&uvhandle_sockpath, watcher_cb_sockpath,
                            socket_path, 0);
  if (errno) diee("watcher: uv_fs_event_start");

  // OK everything is set up, let us daemonise
  if (dup2(1,2) != 2) diee("watcher: set daemonised stderr");
  r= setvbuf(stderr, 0, _IOLBF, BUFSIZ);
  if (r) diee("watcher: setvbuf stderr");

  pid_t child = fork();
  if (child == (pid_t)-1) diee("watcher: fork");
  if (child) _exit(0);

  if (setsid() == (pid_t)-1) diee("watcher: setsid");

  r= uv_run(&loop, UV_RUN_DEFAULT);
  die("uv_run returned (%d)", r);
}

static __attribute__((noreturn))
void become_setup(int sfd, int fake_pair[2],
                  int watcher_stdin, int watcher_stderr) {
  close(fake_pair[0]);
  int call_fd = fake_pair[1];

  int null_0 = open("/dev/null", O_RDONLY);  if (null_0 < 0) diee("open null");
  if (dup2(null_0, 0)) diee("dup2 /dev/null onto stdin");
  close(null_0);
  if (dup2(2, 1) != 1) die("dup2 stderr onto stdout");

  nonblock(sfd);

  // Extension could work like this:
  //
  // We advertise a new protocol (perhaps one which is nearly entirely
  // different after the connect) by putting a name for it comma-separated
  // next to "v1".  Simple extension can be done by having the script
  // side say something about it in the ack xdata, which we currently ignore.
  putenv(m_asprintf("PREFORK_INTERP=v1 %d,%d,%d,%d",
                    sfd, call_fd, watcher_stdin, watcher_stderr));

  execvp(executor_argv[0], (char**)executor_argv);
  diee("execute %s", executor_argv[0]);
}

static void connect_or_spawn(void) {
  int r;

  call_sock = connect_existing();
  if (call_sock) return;

  // We're going to make a new one, so clean out old ones
  preclean();

  int lockfd = acquire_lock();
  call_sock = connect_existing();
  if (call_sock) { close(lockfd); return; }

  // We must start a fresh one, and we hold the lock

  r = unlink(socket_path);
  if (r<0 && errno!=ENOENT)
    diee("failed to remove stale socket %s", socket_path);

  int sfd = socket(AF_UNIX, SOCK_STREAM, 0);
  if (sfd<0) diee("socket() for new listener");

  socklen_t salen = sizeof(sockaddr_sun);
  r= bind(sfd, (const struct sockaddr*)&sockaddr_sun, salen);
  if (r<0) diee("bind() on new listener");

  r= stat(socket_path, &initial_stab);
  if (r<0) diee("stat() fresh socket");

  // We never want callers to get ECONNREFUSED.  But:
  // There is a race here: from my RTFM they may get ECONNREFUSED
  // if they try between our bind() and listen().  But if they do, they'll
  // acquire the lock (serialising with us) and retry, and then it will work.
  r = listen(sfd, INT_MAX);
  if (r<0) diee("listen() for new listener");

  // Fork watcher

  int watcher_stdin[2];
  int watcher_stderr[2];
  if (pipe(watcher_stdin) || pipe(watcher_stderr))
    diee("pipe() for socket inode watcher");

  pid_t watcher = fork();
  if (watcher == (pid_t)-1) diee("fork for watcher");
  if (!watcher) {
    close(sfd);
    close(lockfd);
    close(watcher_stdin[1]);
    close(watcher_stderr[0]);
    if (dup2(watcher_stdin[0], 0) != 0 ||
        dup2(watcher_stderr[1], 1) != 1)
      diee("initial dup2() for watcher");
    close(watcher_stdin[0]);
    close(watcher_stderr[1]);
    become_watcher();
  }

  close(watcher_stdin[0]);
  close(watcher_stderr[1]);
  nonblock(watcher_stderr[0]);

  // Fork setup

  int fake_pair[2];
  r = socketpair(AF_UNIX, SOCK_STREAM, 0, fake_pair);
  if (r<0) diee("socketpair() for fake initial connection");

  pid_t setup_pid = fork();
  if (setup_pid == (pid_t)-1) diee("fork for spawn setup");
  if (!setup_pid) become_setup(sfd, fake_pair,
                               watcher_stdin[1], watcher_stderr[0]);
  close(fake_pair[1]);
  close(sfd);

  call_sock = call_sock_from_fd(fake_pair[0]);

  int status;
  pid_t got = waitpid(setup_pid, &status, 0);
  if (got == (pid_t)-1) diee("waitpid setup [%ld]", (long)setup_pid);
  if (got != setup_pid) diee("waitpid setup [%ld] gave [%ld]!",
                             (long)setup_pid, (long)got);
  if (status != 0) propagate_exit_status(status, "setup");

  const char *emsg = read_greeting();
  if (emsg) die("setup failed: %s", emsg);

  close(lockfd);
  return;
}

static void make_executor_argv(const char *const *argv) {
  switch (laundering) {
  case 'U': break;
  default: die("need -U (specifying unlaundered argument handling)");
  }

  const char *arg;
  #define EACH_NEW_ARG(EACH) {                  \
    arg = interp; { EACH }                      \
    if ((arg = script)) { EACH }                \
    const char *const *walk = argv;             \
    while ((arg = *walk++)) { EACH }            \
  }

  size_t count = 1;
  EACH_NEW_ARG( (void)arg; count++; );

  const char **out = calloc(count, sizeof(char*));
  executor_argv = (const char* const*)out;
  if (!executor_argv) diee("allocate for arguments");

  EACH_NEW_ARG( *out++ = arg; );
  *out++ = 0;
}

int main(int argc_unused, const char *const *argv) {
  process_opts(&argv);

  // Now we have
  //  - possibly interp
  //  - possibly script
  //  - remaining args
  // which ought to be passed on to the actual executor.
  make_executor_argv(argv);

  find_socket_path();
  FILLZERO(sockaddr_sun);
  sockaddr_sun.sun_family = AF_UNIX;
  assert(strlen(socket_path) <= sizeof(sockaddr_sun.sun_path));
  strncpy(sockaddr_sun.sun_path, socket_path, sizeof(sockaddr_sun.sun_path));

  connect_or_spawn();

  // We're committed now, send the request (or bail out)
  send_request();

  uint32_t status;
  protocol_read(&status, sizeof(status));

  status = ntohl(status);
  if (status > INT_MAX) die("status 0x%lx does not fit in an int",
                            (unsigned long)status);

  propagate_exit_status(status, "invocation");
}