chiark / gitweb /
ptyforward: rework PTY forwarder logic used by nspawn to utilize the normal event...
[elogind.git] / src / nspawn / nspawn.c
index 0a8dc0cf301cd7bf047487e75fed5cc33db06c44..e199eb665aaff63dcb986f7b0b846558d31a31bd 100644 (file)
@@ -40,7 +40,6 @@
 #include <sys/un.h>
 #include <sys/socket.h>
 #include <linux/netlink.h>
-#include <sys/eventfd.h>
 #include <net/if.h>
 #include <linux/veth.h>
 #include <sys/personality.h>
 #include "def.h"
 #include "rtnl-util.h"
 #include "udev-util.h"
-#include "eventfd-util.h"
 #include "blkid-util.h"
 #include "gpt.h"
 #include "siphash24.h"
 #include "copy.h"
 #include "base-filesystem.h"
+#include "barrier.h"
+#include "event-util.h"
 
 #ifdef HAVE_SECCOMP
 #include "seccomp-util.h"
@@ -107,6 +107,12 @@ typedef enum LinkJournal {
         LINK_GUEST
 } LinkJournal;
 
+typedef enum Volatile {
+        VOLATILE_NO,
+        VOLATILE_YES,
+        VOLATILE_STATE,
+} Volatile;
+
 static char *arg_directory = NULL;
 static char *arg_user = NULL;
 static sd_id128_t arg_uuid = {};
@@ -159,9 +165,9 @@ static bool arg_network_veth = false;
 static const char *arg_network_bridge = NULL;
 static unsigned long arg_personality = 0xffffffffLU;
 static const char *arg_image = NULL;
+static Volatile arg_volatile = VOLATILE_NO;
 
-static int help(void) {
-
+static void help(void) {
         printf("%s [OPTIONS...] [PATH] [ARGUMENTS...]\n\n"
                "Spawn a minimal namespace container for debugging, testing and building.\n\n"
                "  -h --help                 Show this help\n"
@@ -207,10 +213,9 @@ static int help(void) {
                "     --share-system         Share system namespaces with host\n"
                "     --register=BOOLEAN     Register container as machine\n"
                "     --keep-unit            Do not register a scope for the machine, reuse\n"
-               "                            the service unit nspawn is running in\n",
+               "                            the service unit nspawn is running in\n"
+               "     --volatile[=MODE]      Run the system in volatile mode\n",
                program_invocation_short_name);
-
-        return 0;
 }
 
 static int parse_argv(int argc, char *argv[]) {
@@ -235,6 +240,7 @@ static int parse_argv(int argc, char *argv[]) {
                 ARG_NETWORK_VETH,
                 ARG_NETWORK_BRIDGE,
                 ARG_PERSONALITY,
+                ARG_VOLATILE,
         };
 
         static const struct option options[] = {
@@ -267,6 +273,7 @@ static int parse_argv(int argc, char *argv[]) {
                 { "network-bridge",        required_argument, NULL, ARG_NETWORK_BRIDGE    },
                 { "personality",           required_argument, NULL, ARG_PERSONALITY       },
                 { "image",                 required_argument, NULL, 'i'                   },
+                { "volatile",              optional_argument, NULL, ARG_VOLATILE          },
                 {}
         };
 
@@ -276,12 +283,13 @@ static int parse_argv(int argc, char *argv[]) {
         assert(argc >= 0);
         assert(argv);
 
-        while ((c = getopt_long(argc, argv, "+hD:u:bL:M:jS:Z:qi:", options, NULL)) >= 0) {
+        while ((c = getopt_long(argc, argv, "+hD:u:bL:M:jS:Z:qi:", options, NULL)) >= 0)
 
                 switch (c) {
 
                 case 'h':
-                        return help();
+                        help();
+                        return 0;
 
                 case ARG_VERSION:
                         puts(PACKAGE_STRING);
@@ -386,7 +394,7 @@ static int parse_argv(int argc, char *argv[]) {
 
                 case ARG_CAPABILITY:
                 case ARG_DROP_CAPABILITY: {
-                        char *state, *word;
+                        const char *state, *word;
                         size_t length;
 
                         FOREACH_WORD_SEPARATOR(word, length, optarg, ",", state) {
@@ -559,13 +567,31 @@ static int parse_argv(int argc, char *argv[]) {
 
                         break;
 
+                case ARG_VOLATILE:
+
+                        if (!optarg)
+                                arg_volatile = VOLATILE_YES;
+                        else {
+                                r = parse_boolean(optarg);
+                                if (r < 0) {
+                                        if (streq(optarg, "state"))
+                                                arg_volatile = VOLATILE_STATE;
+                                        else {
+                                                log_error("Failed to parse --volatile= argument: %s", optarg);
+                                                return r;
+                                        }
+                                } else
+                                        arg_volatile = r ? VOLATILE_YES : VOLATILE_NO;
+                        }
+
+                        break;
+
                 case '?':
                         return -EINVAL;
 
                 default:
                         assert_not_reached("Unhandled option");
                 }
-        }
 
         if (arg_share_system)
                 arg_register = false;
@@ -585,6 +611,11 @@ static int parse_argv(int argc, char *argv[]) {
                 return -EINVAL;
         }
 
+        if (arg_volatile != VOLATILE_NO && arg_read_only) {
+                log_error("Cannot combine --read-only with --volatile. Note that --volatile already implies a read-only base hierarchy.");
+                return -EINVAL;
+        }
+
         arg_retain = (arg_retain | plus | (arg_private_network ? 1ULL << CAP_NET_ADMIN : 0)) & ~minus;
 
         return 1;
@@ -645,7 +676,18 @@ static int mount_all(const char *dest) {
                 if (mount_table[k].what && t > 0)
                         continue;
 
-                mkdir_p(where, 0755);
+                t = mkdir_p(where, 0755);
+                if (t < 0) {
+                        if (mount_table[k].fatal) {
+                               log_error("Failed to create directory %s: %s", where, strerror(-t));
+
+                                if (r == 0)
+                                        r = t;
+                        } else
+                               log_warning("Failed to create directory %s: %s", where, strerror(-t));
+
+                        continue;
+                }
 
 #ifdef HAVE_SELINUX
                 if (arg_selinux_apifs_context &&
@@ -664,13 +706,15 @@ static int mount_all(const char *dest) {
                           where,
                           mount_table[k].type,
                           mount_table[k].flags,
-                          o) < 0 &&
-                    mount_table[k].fatal) {
+                          o) < 0) {
 
-                        log_error("mount(%s) failed: %m", where);
+                        if (mount_table[k].fatal) {
+                                log_error("mount(%s) failed: %m", where);
 
-                        if (r == 0)
-                                r = -errno;
+                                if (r == 0)
+                                        r = -errno;
+                        } else
+                                log_warning("mount(%s) failed: %m", where);
                 }
         }
 
@@ -712,16 +756,36 @@ static int mount_binds(const char *dest, char **l, bool ro) {
                 }
 
                 /* Create the mount point, but be conservative -- refuse to create block
-                * and char devices. */
-                if (S_ISDIR(source_st.st_mode))
-                        mkdir_label(where, 0755);
-                else if (S_ISFIFO(source_st.st_mode))
-                        mkfifo(where, 0644);
-                else if (S_ISSOCK(source_st.st_mode))
-                        mknod(where, 0644 | S_IFSOCK, 0);
-                else if (S_ISREG(source_st.st_mode))
-                        touch(where);
-                else {
+                 * and char devices. */
+                if (S_ISDIR(source_st.st_mode)) {
+                        r = mkdir_label(where, 0755);
+                        if (r < 0 && errno != EEXIST) {
+                                log_error("Failed to create mount point %s: %s", where, strerror(-r));
+
+                                return r;
+                        }
+                } else if (S_ISFIFO(source_st.st_mode)) {
+                        r = mkfifo(where, 0644);
+                        if (r < 0 && errno != EEXIST) {
+                                log_error("Failed to create mount point %s: %m", where);
+
+                                return -errno;
+                        }
+                } else if (S_ISSOCK(source_st.st_mode)) {
+                        r = mknod(where, 0644 | S_IFSOCK, 0);
+                        if (r < 0 && errno != EEXIST) {
+                                log_error("Failed to create mount point %s: %m", where);
+
+                                return -errno;
+                        }
+                } else if (S_ISREG(source_st.st_mode)) {
+                        r = touch(where);
+                        if (r < 0) {
+                                log_error("Failed to create mount point %s: %s", where, strerror(-r));
+
+                                return r;
+                        }
+                } else {
                         log_error("Refusing to create mountpoint for file: %s", *x);
                         return -ENOTSUP;
                 }
@@ -748,12 +812,18 @@ static int mount_tmpfs(const char *dest) {
 
         STRV_FOREACH_PAIR(i, o, arg_tmpfs) {
                 _cleanup_free_ char *where = NULL;
+                int r;
 
                 where = strappend(dest, *i);
                 if (!where)
                         return log_oom();
 
-                mkdir_label(where, 0755);
+                r = mkdir_label(where, 0755);
+                if (r < 0) {
+                        log_error("creating mount point for tmpfs %s failed: %s", where, strerror(-r));
+
+                        return r;
+                }
 
                 if (mount("tmpfs", where, "tmpfs", MS_NODEV|MS_STRICTATIME, *o) < 0) {
                         log_error("tmpfs mount to %s failed: %m", where);
@@ -796,7 +866,6 @@ static int setup_timezone(const char *dest) {
                 if (!y)
                         y = path_startswith(q, "/usr/share/zoneinfo/");
 
-
                 /* Already pointing to the right place? Then do nothing .. */
                 if (y && streq(y, z))
                         return 0;
@@ -815,7 +884,20 @@ static int setup_timezone(const char *dest) {
         if (!what)
                 return log_oom();
 
-        unlink(where);
+        r = mkdir_parents(where, 0755);
+        if (r < 0) {
+                log_error("Failed to create directory for timezone info %s in container: %s", where, strerror(-r));
+
+                return 0;
+        }
+
+        r = unlink(where);
+        if (r < 0 && errno != ENOENT) {
+                log_error("Failed to remove existing timezone info %s in container: %m", where);
+
+                return 0;
+        }
+
         if (symlink(what, where) < 0) {
                 log_error("Failed to correct timezone of container: %m");
                 return 0;
@@ -826,6 +908,7 @@ static int setup_timezone(const char *dest) {
 
 static int setup_resolv_conf(const char *dest) {
         _cleanup_free_ char *where = NULL;
+        int r;
 
         assert(dest);
 
@@ -839,11 +922,126 @@ static int setup_resolv_conf(const char *dest) {
 
         /* We don't really care for the results of this really. If it
          * fails, it fails, but meh... */
-        copy_file("/etc/resolv.conf", where, O_TRUNC|O_NOFOLLOW, 0644);
+        r = mkdir_parents(where, 0755);
+        if (r < 0) {
+                log_warning("Failed to create parent directory for resolv.conf %s: %s", where, strerror(-r));
+
+                return 0;
+        }
+
+        r = copy_file("/etc/resolv.conf", where, O_TRUNC|O_NOFOLLOW, 0644);
+        if (r < 0) {
+                log_warning("Failed to copy /etc/resolv.conf to %s: %s", where, strerror(-r));
+
+                return 0;
+        }
 
         return 0;
 }
 
+static int setup_volatile_state(const char *directory) {
+        const char *p;
+        int r;
+
+        assert(directory);
+
+        if (arg_volatile != VOLATILE_STATE)
+                return 0;
+
+        /* --volatile=state means we simply overmount /var
+           with a tmpfs, and the rest read-only. */
+
+        r = bind_remount_recursive(directory, true);
+        if (r < 0) {
+                log_error("Failed to remount %s read-only: %s", directory, strerror(-r));
+                return r;
+        }
+
+        p = strappenda(directory, "/var");
+        r = mkdir(p, 0755);
+        if (r < 0 && errno != EEXIST) {
+                log_error("Failed to create %s: %m", directory);
+                return -errno;
+        }
+
+        if (mount("tmpfs", p, "tmpfs", MS_STRICTATIME, "mode=755") < 0) {
+                log_error("Failed to mount tmpfs to /var: %m");
+                return -errno;
+        }
+
+        return 0;
+}
+
+static int setup_volatile(const char *directory) {
+        bool tmpfs_mounted = false, bind_mounted = false;
+        char template[] = "/tmp/nspawn-volatile-XXXXXX";
+        const char *f, *t;
+        int r;
+
+        assert(directory);
+
+        if (arg_volatile != VOLATILE_YES)
+                return 0;
+
+        /* --volatile=yes means we mount a tmpfs to the root dir, and
+           the original /usr to use inside it, and that read-only. */
+
+        if (!mkdtemp(template)) {
+                log_error("Failed to create temporary directory: %m");
+                return -errno;
+        }
+
+        if (mount("tmpfs", template, "tmpfs", MS_STRICTATIME, "mode=755") < 0) {
+                log_error("Failed to mount tmpfs for root directory: %m");
+                r = -errno;
+                goto fail;
+        }
+
+        tmpfs_mounted = true;
+
+        f = strappenda(directory, "/usr");
+        t = strappenda(template, "/usr");
+
+        r = mkdir(t, 0755);
+        if (r < 0 && errno != EEXIST) {
+                log_error("Failed to create %s: %m", t);
+                r = -errno;
+                goto fail;
+        }
+
+        if (mount(f, t, "bind", MS_BIND|MS_REC, NULL) < 0) {
+                log_error("Failed to create /usr bind mount: %m");
+                r = -errno;
+                goto fail;
+        }
+
+        bind_mounted = true;
+
+        r = bind_remount_recursive(t, true);
+        if (r < 0) {
+                log_error("Failed to remount %s read-only: %s", t, strerror(-r));
+                goto fail;
+        }
+
+        if (mount(template, directory, NULL, MS_MOVE, NULL) < 0) {
+                log_error("Failed to move root mount: %m");
+                r = -errno;
+                goto fail;
+        }
+
+        rmdir(template);
+
+        return 0;
+
+fail:
+        if (bind_mounted)
+                umount(t);
+        if (tmpfs_mounted)
+                umount(template);
+        rmdir(template);
+        return r;
+}
+
 static char* id128_format_as_uuid(sd_id128_t id, char s[37]) {
 
         snprintf(s, 37,
@@ -904,7 +1102,8 @@ static int copy_devnodes(const char *dest) {
                 "full\0"
                 "random\0"
                 "urandom\0"
-                "tty\0";
+                "tty\0"
+                "net/tun\0";
 
         const char *d;
         int r = 0;
@@ -935,10 +1134,17 @@ static int copy_devnodes(const char *dest) {
                         log_error("%s is not a char or block device, cannot copy", from);
                         return -EIO;
 
-                } else if (mknod(to, st.st_mode, st.st_rdev) < 0) {
+                } else {
+                        r = mkdir_parents(to, 0775);
+                        if (r < 0) {
+                                log_error("Failed to create parent directory of %s: %s", to, strerror(-r));
+                                return -r;
+                        }
 
-                        log_error("mknod(%s) failed: %m", dest);
-                        return  -errno;
+                        if (mknod(to, st.st_mode, st.st_rdev) < 0) {
+                                log_error("mknod(%s) failed: %m", dest);
+                                return  -errno;
+                        }
                 }
         }
 
@@ -1084,7 +1290,7 @@ static int setup_hostname(void) {
         if (arg_share_system)
                 return 0;
 
-        if (sethostname(arg_machine, strlen(arg_machine)) < 0)
+        if (sethostname_idempotent(arg_machine) < 0)
                 return -errno;
 
         return 0;
@@ -1169,7 +1375,7 @@ static int setup_journal(const char *directory) {
 
                         r = mkdir_p(q, 0755);
                         if (r < 0)
-                                log_warning("failed to create directory %s: %m", q);
+                                log_warning("Failed to create directory %s: %m", q);
                         return 0;
                 }
 
@@ -1204,7 +1410,7 @@ static int setup_journal(const char *directory) {
 
                 r = mkdir_p(q, 0755);
                 if (r < 0)
-                        log_warning("failed to create directory %s: %m", q);
+                        log_warning("Failed to create directory %s: %m", q);
                 return 0;
         }
 
@@ -1259,9 +1465,9 @@ static int drop_capabilities(void) {
         return capability_bounding_set_drop(~arg_retain, false);
 }
 
-static int register_machine(pid_t pid) {
+static int register_machine(pid_t pid, int local_ifindex) {
         _cleanup_bus_error_free_ sd_bus_error error = SD_BUS_ERROR_NULL;
-        _cleanup_bus_unref_ sd_bus *bus = NULL;
+        _cleanup_bus_close_unref_ sd_bus *bus = NULL;
         int r;
 
         if (!arg_register)
@@ -1279,16 +1485,17 @@ static int register_machine(pid_t pid) {
                                 "org.freedesktop.machine1",
                                 "/org/freedesktop/machine1",
                                 "org.freedesktop.machine1.Manager",
-                                "RegisterMachine",
+                                "RegisterMachineWithNetwork",
                                 &error,
                                 NULL,
-                                "sayssus",
+                                "sayssusai",
                                 arg_machine,
                                 SD_BUS_MESSAGE_APPEND_ID128(arg_uuid),
                                 "nspawn",
                                 "container",
                                 (uint32_t) pid,
-                                strempty(arg_directory));
+                                strempty(arg_directory),
+                                local_ifindex > 0 ? 1 : 0, local_ifindex);
         } else {
                 _cleanup_bus_message_unref_ sd_bus_message *m = NULL;
 
@@ -1298,7 +1505,7 @@ static int register_machine(pid_t pid) {
                                 "org.freedesktop.machine1",
                                 "/org/freedesktop/machine1",
                                 "org.freedesktop.machine1.Manager",
-                                "CreateMachine");
+                                "CreateMachineWithNetwork");
                 if (r < 0) {
                         log_error("Failed to create message: %s", strerror(-r));
                         return r;
@@ -1306,13 +1513,14 @@ static int register_machine(pid_t pid) {
 
                 r = sd_bus_message_append(
                                 m,
-                                "sayssus",
+                                "sayssusai",
                                 arg_machine,
                                 SD_BUS_MESSAGE_APPEND_ID128(arg_uuid),
                                 "nspawn",
                                 "container",
                                 (uint32_t) pid,
-                                strempty(arg_directory));
+                                strempty(arg_directory),
+                                local_ifindex > 0 ? 1 : 0, local_ifindex);
                 if (r < 0) {
                         log_error("Failed to append message arguments: %s", strerror(-r));
                         return r;
@@ -1338,7 +1546,7 @@ static int register_machine(pid_t pid) {
                         return r;
                 }
 
-                r = sd_bus_message_append(m, "(sv)", "DeviceAllow", "a(ss)", 10,
+                r = sd_bus_message_append(m, "(sv)", "DeviceAllow", "a(ss)", 11,
                                           /* Allow the container to
                                            * access and create the API
                                            * device nodes, so that
@@ -1351,6 +1559,7 @@ static int register_machine(pid_t pid) {
                                           "/dev/random", "rwm",
                                           "/dev/urandom", "rwm",
                                           "/dev/tty", "rwm",
+                                          "/dev/net/tun", "rwm",
                                           /* Allow the container
                                            * access to ptys. However,
                                            * do not permit the
@@ -1394,7 +1603,7 @@ static int register_machine(pid_t pid) {
 static int terminate_machine(pid_t pid) {
         _cleanup_bus_error_free_ sd_bus_error error = SD_BUS_ERROR_NULL;
         _cleanup_bus_message_unref_ sd_bus_message *reply = NULL;
-        _cleanup_bus_unref_ sd_bus *bus = NULL;
+        _cleanup_bus_close_unref_ sd_bus *bus = NULL;
         const char *path;
         int r;
 
@@ -1479,9 +1688,10 @@ static int reset_audit_loginuid(void) {
         return 0;
 }
 
-#define HASH_KEY SD_ID128_MAKE(c3,c4,f9,19,b5,57,b2,1c,e6,cf,14,27,03,9c,ee,a2)
+#define HOST_HASH_KEY SD_ID128_MAKE(1a,37,6f,c7,46,ec,45,0b,ad,a3,d5,31,06,60,5d,b1)
+#define CONTAINER_HASH_KEY SD_ID128_MAKE(c3,c4,f9,19,b5,57,b2,1c,e6,cf,14,27,03,9c,ee,a2)
 
-static int get_mac(struct ether_addr *mac) {
+static int generate_mac(struct ether_addr *mac, sd_id128_t hash_key) {
         int r;
 
         uint8_t result[8];
@@ -1503,7 +1713,7 @@ static int get_mac(struct ether_addr *mac) {
 
         /* Let's hash the host machine ID plus the container name. We
          * use a fixed, but originally randomly created hash key here. */
-        siphash24(result, v, sz, HASH_KEY.bytes);
+        siphash24(result, v, sz, hash_key.bytes);
 
         assert_cc(ETH_ALEN <= sizeof(result));
         memcpy(mac->ether_addr_octet, result, ETH_ALEN);
@@ -1515,11 +1725,11 @@ static int get_mac(struct ether_addr *mac) {
         return 0;
 }
 
-static int setup_veth(pid_t pid, char iface_name[IFNAMSIZ]) {
+static int setup_veth(pid_t pid, char iface_name[IFNAMSIZ], int *ifi) {
         _cleanup_rtnl_message_unref_ sd_rtnl_message *m = NULL;
         _cleanup_rtnl_unref_ sd_rtnl *rtnl = NULL;
-        struct ether_addr mac;
-        int r;
+        struct ether_addr mac_host, mac_container;
+        int r, i;
 
         if (!arg_private_network)
                 return 0;
@@ -1529,15 +1739,18 @@ static int setup_veth(pid_t pid, char iface_name[IFNAMSIZ]) {
 
         /* Use two different interface name prefixes depending whether
          * we are in bridge mode or not. */
-        if (arg_network_bridge)
-                memcpy(iface_name, "vb-", 3);
-        else
-                memcpy(iface_name, "ve-", 3);
-        strncpy(iface_name+3, arg_machine, IFNAMSIZ - 3);
+        snprintf(iface_name, IFNAMSIZ - 1, "%s-%s",
+                 arg_network_bridge ? "vb" : "ve", arg_machine);
 
-        r = get_mac(&mac);
+        r = generate_mac(&mac_container, CONTAINER_HASH_KEY);
         if (r < 0) {
-                log_error("Failed to generate predictable MAC address for host0");
+                log_error("Failed to generate predictable MAC address for container side");
+                return r;
+        }
+
+        r = generate_mac(&mac_host, HOST_HASH_KEY);
+        if (r < 0) {
+                log_error("Failed to generate predictable MAC address for host side");
                 return r;
         }
 
@@ -1559,6 +1772,12 @@ static int setup_veth(pid_t pid, char iface_name[IFNAMSIZ]) {
                 return r;
         }
 
+        r = sd_rtnl_message_append_ether_addr(m, IFLA_ADDRESS, &mac_host);
+        if (r < 0) {
+                log_error("Failed to add netlink MAC address: %s", strerror(-r));
+                return r;
+        }
+
         r = sd_rtnl_message_open_container(m, IFLA_LINKINFO);
         if (r < 0) {
                 log_error("Failed to open netlink container: %s", strerror(-r));
@@ -1583,7 +1802,7 @@ static int setup_veth(pid_t pid, char iface_name[IFNAMSIZ]) {
                 return r;
         }
 
-        r = sd_rtnl_message_append_ether_addr(m, IFLA_ADDRESS, &mac);
+        r = sd_rtnl_message_append_ether_addr(m, IFLA_ADDRESS, &mac_container);
         if (r < 0) {
                 log_error("Failed to add netlink MAC address: %s", strerror(-r));
                 return r;
@@ -1619,10 +1838,18 @@ static int setup_veth(pid_t pid, char iface_name[IFNAMSIZ]) {
                 return r;
         }
 
+        i = (int) if_nametoindex(iface_name);
+        if (i <= 0) {
+                log_error("Failed to resolve interface %s: %m", iface_name);
+                return -errno;
+        }
+
+        *ifi = i;
+
         return 0;
 }
 
-static int setup_bridge(const char veth_name[]) {
+static int setup_bridge(const char veth_name[], int *ifi) {
         _cleanup_rtnl_message_unref_ sd_rtnl_message *m = NULL;
         _cleanup_rtnl_unref_ sd_rtnl *rtnl = NULL;
         int r, bridge;
@@ -1642,6 +1869,8 @@ static int setup_bridge(const char veth_name[]) {
                 return -errno;
         }
 
+        *ifi = bridge;
+
         r = sd_rtnl_open(&rtnl, 0);
         if (r < 0) {
                 log_error("Failed to connect to netlink: %s", strerror(-r));
@@ -1739,7 +1968,7 @@ static int move_network_interfaces(pid_t pid) {
                 if (ifi < 0)
                         return ifi;
 
-                r = sd_rtnl_message_new_link(rtnl, &m, RTM_NEWLINK, ifi);
+                r = sd_rtnl_message_new_link(rtnl, &m, RTM_SETLINK, ifi);
                 if (r < 0) {
                         log_error("Failed to allocate netlink message: %s", strerror(-r));
                         return r;
@@ -1864,22 +2093,25 @@ static int setup_macvlan(pid_t pid) {
         return 0;
 }
 
-static int audit_still_doesnt_work_in_containers(void) {
+static int setup_seccomp(void) {
 
 #ifdef HAVE_SECCOMP
+        static const int blacklist[] = {
+                SCMP_SYS(kexec_load),
+                SCMP_SYS(open_by_handle_at),
+                SCMP_SYS(init_module),
+                SCMP_SYS(finit_module),
+                SCMP_SYS(delete_module),
+                SCMP_SYS(iopl),
+                SCMP_SYS(ioperm),
+                SCMP_SYS(swapon),
+                SCMP_SYS(swapoff),
+        };
+
         scmp_filter_ctx seccomp;
+        unsigned i;
         int r;
 
-        /*
-           Audit is broken in containers, much of the userspace audit
-           hookup will fail if running inside a container. We don't
-           care and just turn off creation of audit sockets.
-
-           This will make socket(AF_NETLINK, *, NETLINK_AUDIT) fail
-           with EAFNOSUPPORT which audit userspace uses as indication
-           that audit is disabled in the kernel.
-         */
-
         seccomp = seccomp_init(SCMP_ACT_ALLOW);
         if (!seccomp)
                 return log_oom();
@@ -1890,6 +2122,26 @@ static int audit_still_doesnt_work_in_containers(void) {
                 goto finish;
         }
 
+        for (i = 0; i < ELEMENTSOF(blacklist); i++) {
+                r = seccomp_rule_add(seccomp, SCMP_ACT_ERRNO(EPERM), blacklist[i], 0);
+                if (r == -EFAULT)
+                        continue; /* unknown syscall */
+                if (r < 0) {
+                        log_error("Failed to block syscall: %s", strerror(-r));
+                        goto finish;
+                }
+        }
+
+        /*
+           Audit is broken in containers, much of the userspace audit
+           hookup will fail if running inside a container. We don't
+           care and just turn off creation of audit sockets.
+
+           This will make socket(AF_NETLINK, *, NETLINK_AUDIT) fail
+           with EAFNOSUPPORT which audit userspace uses as indication
+           that audit is disabled in the kernel.
+         */
+
         r = seccomp_rule_add(
                         seccomp,
                         SCMP_ACT_ERRNO(EAFNOSUPPORT),
@@ -2365,20 +2617,27 @@ static int mount_devices(
 
 static void loop_remove(int nr, int *image_fd) {
         _cleanup_close_ int control = -1;
+        int r;
 
         if (nr < 0)
                 return;
 
         if (image_fd && *image_fd >= 0) {
-                ioctl(*image_fd, LOOP_CLR_FD);
+                r = ioctl(*image_fd, LOOP_CLR_FD);
+                if (r < 0)
+                        log_warning("Failed to close loop image: %m");
                 *image_fd = safe_close(*image_fd);
         }
 
         control = open("/dev/loop-control", O_RDWR|O_CLOEXEC|O_NOCTTY|O_NONBLOCK);
-        if (control < 0)
+        if (control < 0) {
+                log_warning("Failed to open /dev/loop-control: %m");
                 return;
+        }
 
-        ioctl(control, LOOP_CTL_REMOVE, nr);
+        r = ioctl(control, LOOP_CTL_REMOVE, nr);
+        if (r < 0)
+                log_warning("Failed to remove loop %d: %m", nr);
 }
 
 static int spawn_getent(const char *database, const char *key, pid_t *rpid) {
@@ -2439,7 +2698,8 @@ static int spawn_getent(const char *database, const char *key, pid_t *rpid) {
 }
 
 static int change_uid_gid(char **_home) {
-        char line[LINE_MAX], *w, *x, *state, *u, *g, *h;
+        char line[LINE_MAX], *x, *u, *g, *h;
+        const char *word, *state;
         _cleanup_free_ uid_t *uids = NULL;
         _cleanup_free_ char *home = NULL;
         _cleanup_fclose_ FILE *f = NULL;
@@ -2589,10 +2849,10 @@ static int change_uid_gid(char **_home) {
         x += strcspn(x, WHITESPACE);
         x += strspn(x, WHITESPACE);
 
-        FOREACH_WORD(w, l, x, state) {
+        FOREACH_WORD(word, l, x, state) {
                 char c[l+1];
 
-                memcpy(c, w, l);
+                memcpy(c, word, l);
                 c[l] = 0;
 
                 if (!GREEDY_REALLOC(uids, sz, n_uids+1))
@@ -2645,68 +2905,67 @@ static int change_uid_gid(char **_home) {
 }
 
 /*
- * Return 0 in case the container is being rebooted, has been shut
- * down or exited successfully. On failures a negative value is
- * returned.
+ * Return values:
+ * < 0 : wait_for_terminate() failed to get the state of the
+ *       container, the container was terminated by a signal, or
+ *       failed for an unknown reason.  No change is made to the
+ *       container argument.
+ * > 0 : The program executed in the container terminated with an
+ *       error.  The exit code of the program executed in the
+ *       container is returned.  The container argument has been set
+ *       to CONTAINER_TERMINATED.
+ *   0 : The container is being rebooted, has been shut down or exited
+ *       successfully.  The container argument has been set to either
+ *       CONTAINER_TERMINATED or CONTAINER_REBOOTED.
  *
- * The status of the container "CONTAINER_TERMINATED" or
- * "CONTAINER_REBOOTED" will be saved in the container argument
+ * That is, success is indicated by a return value of zero, and an
+ * error is indicated by a non-zero value.
  */
 static int wait_for_container(pid_t pid, ContainerStatus *container) {
-        int r;
         siginfo_t status;
+        int r;
 
         r = wait_for_terminate(pid, &status);
-        if (r < 0)
+        if (r < 0) {
+                log_warning("Failed to wait for container: %s", strerror(-r));
                 return r;
+        }
 
         switch (status.si_code) {
+
         case CLD_EXITED:
-                r = status.si_status;
-                if (r == 0) {
-                        if (!arg_quiet)
-                                log_debug("Container %s exited successfully.",
-                                          arg_machine);
+                if (status.si_status == 0) {
+                        log_full(arg_quiet ? LOG_DEBUG : LOG_INFO, "Container %s exited successfully.", arg_machine);
 
-                        *container = CONTAINER_TERMINATED;
-                } else {
-                        log_error("Container %s failed with error code %i.",
-                                  arg_machine, status.si_status);
-                        r = -1;
-                }
-                break;
+                } else
+                        log_full(arg_quiet ? LOG_DEBUG : LOG_INFO, "Container %s failed with error code %i.", arg_machine, status.si_status);
+
+                *container = CONTAINER_TERMINATED;
+                return status.si_status;
 
         case CLD_KILLED:
                 if (status.si_status == SIGINT) {
-                        if (!arg_quiet)
-                                log_info("Container %s has been shut down.",
-                                         arg_machine);
 
+                        log_full(arg_quiet ? LOG_DEBUG : LOG_INFO, "Container %s has been shut down.", arg_machine);
                         *container = CONTAINER_TERMINATED;
-                        r = 0;
-                        break;
+                        return 0;
+
                 } else if (status.si_status == SIGHUP) {
-                        if (!arg_quiet)
-                                log_info("Container %s is being rebooted.",
-                                         arg_machine);
 
+                        log_full(arg_quiet ? LOG_DEBUG : LOG_INFO, "Container %s is being rebooted.", arg_machine);
                         *container = CONTAINER_REBOOTED;
-                        r = 0;
-                        break;
+                        return 0;
                 }
+
                 /* CLD_KILLED fallthrough */
 
         case CLD_DUMPED:
-                log_error("Container %s terminated by signal %s.",
-                          arg_machine, signal_to_string(status.si_status));
-                r = -1;
-                break;
+                log_error("Container %s terminated by signal %s.", arg_machine, signal_to_string(status.si_status));
+                return -EIO;
 
         default:
-                log_error("Container %s failed due to unknown reason.",
-                          arg_machine);
-                r = -1;
-                break;
+                log_error("Container %s failed due to unknown reason.", arg_machine);
+                return -EIO;
         }
 
         return r;
@@ -2714,6 +2973,22 @@ static int wait_for_container(pid_t pid, ContainerStatus *container) {
 
 static void nop_handler(int sig) {}
 
+static int on_orderly_shutdown(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) {
+        pid_t pid;
+
+        pid = PTR_TO_UINT32(userdata);
+        if (pid > 0) {
+                if (kill(pid, SIGRTMIN+3) >= 0) {
+                        log_info("Trying to halt container. Send SIGTERM again to trigger immediate termination.");
+                        sd_event_source_set_userdata(s, NULL);
+                        return 0;
+                }
+        }
+
+        sd_event_exit(sd_event_source_get_event(s), 0);
+        return 0;
+}
+
 int main(int argc, char *argv[]) {
 
         _cleanup_free_ char *kdbus_domain = NULL, *device_path = NULL, *root_device = NULL, *home_device = NULL, *srv_device = NULL;
@@ -2835,7 +3110,11 @@ int main(int argc, char *argv[]) {
                         goto finish;
                 }
 
-                r = dissect_image(image_fd, &root_device, &root_device_rw, &home_device, &home_device_rw, &srv_device, &srv_device_rw, &secondary);
+                r = dissect_image(image_fd,
+                                  &root_device, &root_device_rw,
+                                  &home_device, &home_device_rw,
+                                  &srv_device, &srv_device_rw,
+                                  &secondary);
                 if (r < 0)
                         goto finish;
         }
@@ -2886,22 +3165,31 @@ int main(int argc, char *argv[]) {
                 goto finish;
         }
 
-        sd_notify(0, "READY=1");
+        sd_notify(false,
+                  "READY=1\n"
+                  "STATUS=Container running.");
 
         assert_se(sigemptyset(&mask) == 0);
-        assert_se(sigemptyset(&mask_chld) == 0);
-        sigaddset(&mask_chld, SIGCHLD);
         sigset_add_many(&mask, SIGCHLD, SIGWINCH, SIGTERM, SIGINT, -1);
         assert_se(sigprocmask(SIG_BLOCK, &mask, NULL) == 0);
 
+        assert_se(sigemptyset(&mask_chld) == 0);
+        assert_se(sigaddset(&mask_chld, SIGCHLD) == 0);
+
         for (;;) {
                 ContainerStatus container_status;
-                int eventfds[2] = { -1, -1 };
+                _cleanup_(barrier_destroy) Barrier barrier = BARRIER_NULL;
                 struct sigaction sa = {
                         .sa_handler = nop_handler,
                         .sa_flags = SA_NOCLDSTOP,
                 };
 
+                r = barrier_create(&barrier);
+                if (r < 0) {
+                        log_error("Cannot initialize IPC barrier: %s", strerror(-r));
+                        goto finish;
+                }
+
                 /* Child can be killed before execv(), so handle SIGCHLD
                  * in order to interrupt parent's blocking calls and
                  * give it a chance to call wait() and terminate. */
@@ -2917,9 +3205,9 @@ int main(int argc, char *argv[]) {
                         goto finish;
                 }
 
-                pid = clone_with_eventfd(SIGCHLD|CLONE_NEWNS|
-                                         (arg_share_system ? 0 : CLONE_NEWIPC|CLONE_NEWPID|CLONE_NEWUTS)|
-                                         (arg_private_network ? CLONE_NEWNET : 0), eventfds);
+                pid = syscall(__NR_clone, SIGCHLD|CLONE_NEWNS|
+                                          (arg_share_system ? 0 : CLONE_NEWIPC|CLONE_NEWPID|CLONE_NEWUTS)|
+                                          (arg_private_network ? CLONE_NEWNET : 0), NULL);
                 if (pid < 0) {
                         if (errno == EINVAL)
                                 log_error("clone() failed, do you have namespace support enabled in your kernel? (You need UTS, IPC, PID and NET namespacing built in): %m");
@@ -2948,6 +3236,8 @@ int main(int argc, char *argv[]) {
                         };
                         char **env_use;
 
+                        barrier_set_role(&barrier, BARRIER_CHILD);
+
                         envp[n_env] = strv_find_prefix(environ, "TERM=");
                         if (envp[n_env])
                                 n_env ++;
@@ -2961,9 +3251,7 @@ int main(int argc, char *argv[]) {
                         kmsg_socket_pair[0] = safe_close(kmsg_socket_pair[0]);
 
                         reset_all_signal_handlers();
-
-                        assert_se(sigemptyset(&mask) == 0);
-                        assert_se(sigprocmask(SIG_SETMASK, &mask, NULL) == 0);
+                        reset_signal_mask();
 
                         k = open_terminal(console, O_RDWR);
                         if (k != STDIN_FILENO) {
@@ -2973,26 +3261,26 @@ int main(int argc, char *argv[]) {
                                 }
 
                                 log_error("Failed to open console: %s", strerror(-k));
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
                         }
 
                         if (dup2(STDIN_FILENO, STDOUT_FILENO) != STDOUT_FILENO ||
                             dup2(STDIN_FILENO, STDERR_FILENO) != STDERR_FILENO) {
                                 log_error("Failed to duplicate console: %m");
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
                         }
 
                         if (setsid() < 0) {
                                 log_error("setsid() failed: %m");
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
                         }
 
                         if (reset_audit_loginuid() < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) {
                                 log_error("PR_SET_PDEATHSIG failed: %m");
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
                         }
 
                         /* Mark everything as slave, so that we still
@@ -3000,108 +3288,109 @@ int main(int argc, char *argv[]) {
                          * propagate mounts to the real root. */
                         if (mount(NULL, "/", NULL, MS_SLAVE|MS_REC, NULL) < 0) {
                                 log_error("MS_SLAVE|MS_REC failed: %m");
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
                         }
 
                         if (mount_devices(arg_directory,
                                           root_device, root_device_rw,
                                           home_device, home_device_rw,
                                           srv_device, srv_device_rw) < 0)
-                                goto child_fail;
-
-                        r = base_filesystem_create(arg_directory);
-                        if (r < 0) {
-                                log_error("Failed to create the base filesystem: %s", strerror(-r));
-                                goto child_fail;
-                        }
+                                _exit(EXIT_FAILURE);
 
                         /* Turn directory into bind mount */
                         if (mount(arg_directory, arg_directory, "bind", MS_BIND|MS_REC, NULL) < 0) {
                                 log_error("Failed to make bind mount: %m");
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
                         }
 
+                        r = setup_volatile(arg_directory);
+                        if (r < 0)
+                                _exit(EXIT_FAILURE);
+
+                        if (setup_volatile_state(arg_directory) < 0)
+                                _exit(EXIT_FAILURE);
+
+                        r = base_filesystem_create(arg_directory);
+                        if (r < 0)
+                                _exit(EXIT_FAILURE);
+
                         if (arg_read_only) {
                                 k = bind_remount_recursive(arg_directory, true);
                                 if (k < 0) {
                                         log_error("Failed to make tree read-only: %s", strerror(-k));
-                                        goto child_fail;
+                                        _exit(EXIT_FAILURE);
                                 }
                         }
 
                         if (mount_all(arg_directory) < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         if (copy_devnodes(arg_directory) < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         if (setup_ptmx(arg_directory) < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         dev_setup(arg_directory);
 
-                        if (audit_still_doesnt_work_in_containers() < 0)
-                                goto child_fail;
+                        if (setup_seccomp() < 0)
+                                _exit(EXIT_FAILURE);
 
                         if (setup_dev_console(arg_directory, console) < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         if (setup_kmsg(arg_directory, kmsg_socket_pair[1]) < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         kmsg_socket_pair[1] = safe_close(kmsg_socket_pair[1]);
 
                         if (setup_boot_id(arg_directory) < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         if (setup_timezone(arg_directory) < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         if (setup_resolv_conf(arg_directory) < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         if (setup_journal(arg_directory) < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         if (mount_binds(arg_directory, arg_bind, false) < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         if (mount_binds(arg_directory, arg_bind_ro, true) < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         if (mount_tmpfs(arg_directory) < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         if (setup_kdbus(arg_directory, kdbus_domain) < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         /* Tell the parent that we are ready, and that
                          * it can cgroupify us to that we lack access
                          * to certain devices and resources. */
-                        r = eventfd_send_state(eventfds[1],
-                                               EVENTFD_CHILD_SUCCEEDED);
-                        eventfds[1] = safe_close(eventfds[1]);
-                        if (r < 0)
-                                goto child_fail;
+                        barrier_place(&barrier);
 
                         if (chdir(arg_directory) < 0) {
                                 log_error("chdir(%s) failed: %m", arg_directory);
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
                         }
 
                         if (mount(arg_directory, "/", NULL, MS_MOVE, NULL) < 0) {
                                 log_error("mount(MS_MOVE) failed: %m");
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
                         }
 
                         if (chroot(".") < 0) {
                                 log_error("chroot() failed: %m");
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
                         }
 
                         if (chdir("/") < 0) {
                                 log_error("chdir() failed: %m");
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
                         }
 
                         umask(0022);
@@ -3111,18 +3400,18 @@ int main(int argc, char *argv[]) {
 
                         if (drop_capabilities() < 0) {
                                 log_error("drop_capabilities() failed: %m");
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
                         }
 
                         r = change_uid_gid(&home);
                         if (r < 0)
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
 
                         if ((asprintf((char**)(envp + n_env++), "HOME=%s", home ? home: "/root") < 0) ||
                             (asprintf((char**)(envp + n_env++), "USER=%s", arg_user ? arg_user : "root") < 0) ||
                             (asprintf((char**)(envp + n_env++), "LOGNAME=%s", arg_user ? arg_user : "root") < 0)) {
                                 log_oom();
-                                goto child_fail;
+                                _exit(EXIT_FAILURE);
                         }
 
                         if (!sd_id128_equal(arg_uuid, SD_ID128_NULL)) {
@@ -3130,7 +3419,7 @@ int main(int argc, char *argv[]) {
 
                                 if (asprintf((char**)(envp + n_env++), "container_uuid=%s", id128_format_as_uuid(arg_uuid, as_uuid)) < 0) {
                                         log_oom();
-                                        goto child_fail;
+                                        _exit(EXIT_FAILURE);
                                 }
                         }
 
@@ -3138,13 +3427,13 @@ int main(int argc, char *argv[]) {
                                 k = fdset_cloexec(fds, false);
                                 if (k < 0) {
                                         log_error("Failed to unset O_CLOEXEC for file descriptors.");
-                                        goto child_fail;
+                                        _exit(EXIT_FAILURE);
                                 }
 
                                 if ((asprintf((char **)(envp + n_env++), "LISTEN_FDS=%u", n_fd_passed) < 0) ||
                                     (asprintf((char **)(envp + n_env++), "LISTEN_PID=1") < 0)) {
                                         log_oom();
-                                        goto child_fail;
+                                        _exit(EXIT_FAILURE);
                                 }
                         }
 
@@ -3153,12 +3442,12 @@ int main(int argc, char *argv[]) {
                         if (arg_personality != 0xffffffffLU) {
                                 if (personality(arg_personality) < 0) {
                                         log_error("personality() failed: %m");
-                                        goto child_fail;
+                                        _exit(EXIT_FAILURE);
                                 }
                         } else if (secondary) {
                                 if (personality(PER_LINUX32) < 0) {
                                         log_error("personality() failed: %m");
-                                        goto child_fail;
+                                        _exit(EXIT_FAILURE);
                                 }
                         }
 
@@ -3166,7 +3455,7 @@ int main(int argc, char *argv[]) {
                         if (arg_selinux_context)
                                 if (setexeccon((security_context_t) arg_selinux_context) < 0) {
                                         log_error("setexeccon(\"%s\") failed: %m", arg_selinux_context);
-                                        goto child_fail;
+                                        _exit(EXIT_FAILURE);
                                 }
 #endif
 
@@ -3176,7 +3465,7 @@ int main(int argc, char *argv[]) {
                                 n = strv_env_merge(2, envp, arg_setenv);
                                 if (!n) {
                                         log_oom();
-                                        goto child_fail;
+                                        _exit(EXIT_FAILURE);
                                 }
 
                                 env_use = n;
@@ -3184,10 +3473,8 @@ int main(int argc, char *argv[]) {
                                 env_use = (char**) envp;
 
                         /* Wait until the parent is ready with the setup, too... */
-                        r = eventfd_parent_succeeded(eventfds[0]);
-                        eventfds[0] = safe_close(eventfds[0]);
-                        if (r < 0)
-                                goto child_fail;
+                        if (!barrier_place_and_sync(&barrier))
+                                _exit(EXIT_FAILURE);
 
                         if (arg_boot) {
                                 char **a;
@@ -3216,99 +3503,135 @@ int main(int argc, char *argv[]) {
                         }
 
                         log_error("execv() failed: %m");
-
-                child_fail:
-                        /* Tell the parent that the setup failed, so he
-                         * can clean up resources and terminate. */
-                        if (eventfds[1] != -1)
-                                eventfd_send_state(eventfds[1],
-                                                   EVENTFD_CHILD_FAILED);
                         _exit(EXIT_FAILURE);
                 }
 
+                barrier_set_role(&barrier, BARRIER_PARENT);
                 fdset_free(fds);
                 fds = NULL;
 
-                /* Wait for the child event:
-                 * If EVENTFD_CHILD_FAILED, the child will terminate soon.
-                 * If EVENTFD_CHILD_SUCCEEDED, the child is reporting that
-                 * it is ready with all it needs to do with priviliges.
-                 * After we got the notification we can make the process
-                 * join its cgroup which might limit what it can do */
-                r = eventfd_child_succeeded(eventfds[1]);
-                eventfds[1] = safe_close(eventfds[1]);
-                if (r < 0)
-                        goto check_container_status;
+                /* wait for child-setup to be done */
+                if (barrier_place_and_sync(&barrier)) {
+                        _cleanup_event_unref_ sd_event *event = NULL;
+                        _cleanup_(pty_forward_freep) PTYForward *forward = NULL;
+                        int ifi = 0;
 
-                r = register_machine(pid);
-                if (r < 0)
-                        goto finish;
+                        r = move_network_interfaces(pid);
+                        if (r < 0)
+                                goto finish;
 
-                r = move_network_interfaces(pid);
-                if (r < 0)
-                        goto finish;
+                        r = setup_veth(pid, veth_name, &ifi);
+                        if (r < 0)
+                                goto finish;
 
-                r = setup_veth(pid, veth_name);
-                if (r < 0)
-                        goto finish;
+                        r = setup_bridge(veth_name, &ifi);
+                        if (r < 0)
+                                goto finish;
 
-                r = setup_bridge(veth_name);
-                if (r < 0)
-                        goto finish;
+                        r = setup_macvlan(pid);
+                        if (r < 0)
+                                goto finish;
 
-                r = setup_macvlan(pid);
-                if (r < 0)
-                        goto finish;
+                        r = register_machine(pid, ifi);
+                        if (r < 0)
+                                goto finish;
 
-                /* Block SIGCHLD here, before notifying child.
-                 * process_pty() will handle it with the other signals. */
-                r = sigprocmask(SIG_BLOCK, &mask_chld, NULL);
-                if (r < 0)
-                        goto finish;
+                        /* Block SIGCHLD here, before notifying child.
+                         * process_pty() will handle it with the other signals. */
+                        r = sigprocmask(SIG_BLOCK, &mask_chld, NULL);
+                        if (r < 0)
+                                goto finish;
 
-                /* Reset signal to default */
-                r = default_signals(SIGCHLD, -1);
-                if (r < 0)
-                        goto finish;
+                        /* Reset signal to default */
+                        r = default_signals(SIGCHLD, -1);
+                        if (r < 0)
+                                goto finish;
 
-                /* Notify the child that the parent is ready with all
-                 * its setup, and that the child can now hand over
-                 * control to the code to run inside the container. */
-                r = eventfd_send_state(eventfds[0],
-                                       EVENTFD_PARENT_SUCCEEDED);
-                eventfds[0] = safe_close(eventfds[0]);
-                if (r < 0)
-                        goto finish;
+                        /* Notify the child that the parent is ready with all
+                         * its setup, and that the child can now hand over
+                         * control to the code to run inside the container. */
+                        barrier_place(&barrier);
 
-                k = process_pty(master, &mask, arg_boot ? pid : 0, SIGRTMIN+3);
-                if (k < 0) {
-                        r = EXIT_FAILURE;
-                        break;
-                }
+                        r = sd_event_new(&event);
+                        if (r < 0) {
+                                log_error("Failed to get default event source: %s", strerror(-r));
+                                goto finish;
+                        }
 
-                if (!arg_quiet)
-                        putc('\n', stdout);
+                        if (arg_boot) {
+                                /* Try to kill the init system on SIGINT or SIGTERM */
+                                sd_event_add_signal(event, NULL, SIGINT, on_orderly_shutdown, UINT32_TO_PTR(pid));
+                                sd_event_add_signal(event, NULL, SIGTERM, on_orderly_shutdown, UINT32_TO_PTR(pid));
+                        } else {
+                                /* Immediately exit */
+                                sd_event_add_signal(event, NULL, SIGINT, NULL, NULL);
+                                sd_event_add_signal(event, NULL, SIGTERM, NULL, NULL);
+                        }
 
-                /* Kill if it is not dead yet anyway */
-                terminate_machine(pid);
+                        /* simply exit on sigchld */
+                        sd_event_add_signal(event, NULL, SIGCHLD, NULL, NULL);
 
-check_container_status:
-                /* Redundant, but better safe than sorry */
+                        r = pty_forward_new(event, master, &forward);
+                        if (r < 0) {
+                                log_error("Failed to create PTY forwarder: %s", strerror(-r));
+                                goto finish;
+                        }
+
+                        r = sd_event_loop(event);
+                        if (r < 0) {
+                                log_error("Failed to run event loop: %s", strerror(-r));
+                                return r;
+                        }
+
+                        forward = pty_forward_free(forward);
+
+                        if (!arg_quiet)
+                                putc('\n', stdout);
+
+                        /* Kill if it is not dead yet anyway */
+                        terminate_machine(pid);
+                }
+
+                /* Normally redundant, but better safe than sorry */
                 kill(pid, SIGKILL);
 
                 r = wait_for_container(pid, &container_status);
                 pid = 0;
 
                 if (r < 0) {
+                        /* We failed to wait for the container, or the
+                         * container exited abnormally */
                         r = EXIT_FAILURE;
                         break;
-                } else if (container_status == CONTAINER_TERMINATED)
+                } else if (r > 0 || container_status == CONTAINER_TERMINATED)
+                        /* The container exited with a non-zero
+                         * status, or with zero status and no reboot
+                         * was requested. */
                         break;
 
                 /* CONTAINER_REBOOTED, loop again */
+
+                if (arg_keep_unit) {
+                        /* Special handling if we are running as a
+                         * service: instead of simply restarting the
+                         * machine we want to restart the entire
+                         * service, so let's inform systemd about this
+                         * with the special exit code 133. The service
+                         * file uses RestartForceExitStatus=133 so
+                         * that this results in a full nspawn
+                         * restart. This is necessary since we might
+                         * have cgroup parameters set we want to have
+                         * flushed out. */
+                        r = 133;
+                        break;
+                }
         }
 
 finish:
+        sd_notify(false,
+                  "STOPPING=1\n"
+                  "STATUS=Terminating...");
+
         loop_remove(loop_nr, &image_fd);
 
         if (pid > 0)