From b6e676ce41508e2aeea22202fc8f234126177f52 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 5 Mar 2015 00:56:08 +0100 Subject: [PATCH] importd: add new bus calls for importing local tar and raw images This also adds "machinectl import-raw" and "machinectl import-tar" to wrap these new bus calls. THe commands basically do for local files that "machinectl pull-raw" and friends do for remote files. --- Makefile.am | 34 +- src/import/import-common.c | 150 ++++++ src/import/import-common.h | 27 ++ src/import/import-raw.c | 454 +++++++++++++++++++ src/import/import-raw.h | 37 ++ src/import/import-tar.c | 374 +++++++++++++++ src/import/import-tar.h | 37 ++ src/import/import.c | 336 ++++++++++++++ src/import/importd.c | 185 ++++++-- src/import/org.freedesktop.import1.policy.in | 10 + src/import/pull-common.c | 121 ----- src/import/pull-common.h | 5 - src/import/pull-dkr.c | 3 +- src/import/pull-job.c | 14 +- src/import/pull-raw.c | 16 +- src/import/pull-tar.c | 7 +- src/import/pull.c | 2 + src/machine/machinectl.c | 144 +++++- src/shared/machine-pool.h | 3 + 19 files changed, 1770 insertions(+), 189 deletions(-) create mode 100644 src/import/import-common.c create mode 100644 src/import/import-common.h create mode 100644 src/import/import-raw.c create mode 100644 src/import/import-raw.h create mode 100644 src/import/import-tar.c create mode 100644 src/import/import-tar.h create mode 100644 src/import/import.c diff --git a/Makefile.am b/Makefile.am index d430fff97..843a7df9e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5430,14 +5430,16 @@ if HAVE_GCRYPT rootlibexec_PROGRAMS += \ systemd-importd \ - systemd-pull + systemd-pull \ + systemd-import systemd_importd_SOURCES = \ src/import/importd.c systemd_importd_CFLAGS = \ $(AM_CFLAGS) \ - -D SYSTEMD_PULL_PATH=\"$(rootlibexecdir)/systemd-pull\" + -D SYSTEMD_PULL_PATH=\"$(rootlibexecdir)/systemd-pull\" \ + -D SYSTEMD_IMPORT_PATH=\"$(rootlibexecdir)/systemd-import\" systemd_importd_LDADD = \ libsystemd-internal.la \ @@ -5456,6 +5458,8 @@ systemd_pull_SOURCES = \ src/import/pull-job.h \ src/import/pull-common.c \ src/import/pull-common.h \ + src/import/import-common.c \ + src/import/import-common.h \ src/import/import-compress.c \ src/import/import-compress.h \ src/import/curl-util.c \ @@ -5484,6 +5488,32 @@ systemd_pull_LDADD = \ -lbz2 \ $(GCRYPT_LIBS) +systemd_import_SOURCES = \ + src/import/import.c \ + src/import/import-raw.c \ + src/import/import-raw.h \ + src/import/import-tar.c \ + src/import/import-tar.h \ + src/import/import-common.c \ + src/import/import-common.h \ + src/import/import-compress.c \ + src/import/import-compress.h \ + src/import/qcow2-util.c \ + src/import/qcow2-util.h + +systemd_import_CFLAGS = \ + $(AM_CFLAGS) \ + $(XZ_CFLAGS) \ + $(ZLIB_CFLAGS) + +systemd_import_LDADD = \ + libsystemd-internal.la \ + libsystemd-label.la \ + libsystemd-shared.la \ + $(XZ_LIBS) \ + $(ZLIB_LIBS) \ + -lbz2 + dist_rootlibexec_DATA = \ src/import/import-pubring.gpg diff --git a/src/import/import-common.c b/src/import/import-common.c new file mode 100644 index 000000000..6c3f347e7 --- /dev/null +++ b/src/import/import-common.c @@ -0,0 +1,150 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include +#include +#include + +#include "util.h" +#include "btrfs-util.h" +#include "capability.h" +#include "import-common.h" + +int import_make_read_only_fd(int fd) { + int r; + + assert(fd >= 0); + + /* First, let's make this a read-only subvolume if it refers + * to a subvolume */ + r = btrfs_subvol_set_read_only_fd(fd, true); + if (r == -ENOTTY || r == -ENOTDIR || r == -EINVAL) { + struct stat st; + + /* This doesn't refer to a subvolume, or the file + * system isn't even btrfs. In that, case fall back to + * chmod()ing */ + + r = fstat(fd, &st); + if (r < 0) + return log_error_errno(errno, "Failed to stat temporary image: %m"); + + /* Drop "w" flag */ + if (fchmod(fd, st.st_mode & 07555) < 0) + return log_error_errno(errno, "Failed to chmod() final image: %m"); + + return 0; + + } else if (r < 0) + return log_error_errno(r, "Failed to make subvolume read-only: %m"); + + return 0; +} + +int import_make_read_only(const char *path) { + _cleanup_close_ int fd = 1; + + fd = open(path, O_RDONLY|O_NOCTTY|O_CLOEXEC); + if (fd < 0) + return log_error_errno(errno, "Failed to open %s: %m", path); + + return import_make_read_only_fd(fd); +} + +int import_fork_tar(const char *path, pid_t *ret) { + _cleanup_close_pair_ int pipefd[2] = { -1, -1 }; + pid_t pid; + int r; + + assert(path); + assert(ret); + + if (pipe2(pipefd, O_CLOEXEC) < 0) + return log_error_errno(errno, "Failed to create pipe for tar: %m"); + + pid = fork(); + if (pid < 0) + return log_error_errno(errno, "Failed to fork off tar: %m"); + + if (pid == 0) { + int null_fd; + uint64_t retain = + (1ULL << CAP_CHOWN) | + (1ULL << CAP_FOWNER) | + (1ULL << CAP_FSETID) | + (1ULL << CAP_MKNOD) | + (1ULL << CAP_SETFCAP) | + (1ULL << CAP_DAC_OVERRIDE); + + /* Child */ + + reset_all_signal_handlers(); + reset_signal_mask(); + assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0); + + pipefd[1] = safe_close(pipefd[1]); + + if (dup2(pipefd[0], STDIN_FILENO) != STDIN_FILENO) { + log_error_errno(errno, "Failed to dup2() fd: %m"); + _exit(EXIT_FAILURE); + } + + if (pipefd[0] != STDIN_FILENO) + pipefd[0] = safe_close(pipefd[0]); + + null_fd = open("/dev/null", O_WRONLY|O_NOCTTY); + if (null_fd < 0) { + log_error_errno(errno, "Failed to open /dev/null: %m"); + _exit(EXIT_FAILURE); + } + + if (dup2(null_fd, STDOUT_FILENO) != STDOUT_FILENO) { + log_error_errno(errno, "Failed to dup2() fd: %m"); + _exit(EXIT_FAILURE); + } + + if (null_fd != STDOUT_FILENO) + null_fd = safe_close(null_fd); + + fd_cloexec(STDIN_FILENO, false); + fd_cloexec(STDOUT_FILENO, false); + fd_cloexec(STDERR_FILENO, false); + + if (unshare(CLONE_NEWNET) < 0) + log_error_errno(errno, "Failed to lock tar into network namespace, ignoring: %m"); + + r = capability_bounding_set_drop(~retain, true); + if (r < 0) + log_error_errno(r, "Failed to drop capabilities, ignoring: %m"); + + execlp("tar", "tar", "--numeric-owner", "-C", path, "-px", NULL); + log_error_errno(errno, "Failed to execute tar: %m"); + _exit(EXIT_FAILURE); + } + + pipefd[0] = safe_close(pipefd[0]); + r = pipefd[1]; + pipefd[1] = -1; + + *ret = pid; + + return r; +} diff --git a/src/import/import-common.h b/src/import/import-common.h new file mode 100644 index 000000000..639ea1799 --- /dev/null +++ b/src/import/import-common.h @@ -0,0 +1,27 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +#pragma once + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +int import_make_read_only_fd(int fd); +int import_make_read_only(const char *path); + +int import_fork_tar(const char *path, pid_t *ret); diff --git a/src/import/import-raw.c b/src/import/import-raw.c new file mode 100644 index 000000000..15e5eb2ca --- /dev/null +++ b/src/import/import-raw.c @@ -0,0 +1,454 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include + +#include "sd-daemon.h" +#include "sd-event.h" +#include "util.h" +#include "path-util.h" +#include "btrfs-util.h" +#include "copy.h" +#include "mkdir.h" +#include "ratelimit.h" +#include "machine-pool.h" +#include "qcow2-util.h" +#include "import-compress.h" +#include "import-common.h" +#include "import-raw.h" + +struct RawImport { + sd_event *event; + + char *image_root; + + RawImportFinished on_finished; + void *userdata; + + char *local; + bool force_local; + bool read_only; + bool grow_machine_directory; + + char *temp_path; + char *final_path; + + int input_fd; + int output_fd; + + ImportCompress compress; + + uint64_t written_since_last_grow; + + sd_event_source *input_event_source; + + uint8_t buffer[16*1024]; + size_t buffer_size; + + uint64_t written_compressed; + uint64_t written_uncompressed; + + struct stat st; + + unsigned last_percent; + RateLimit progress_rate_limit; +}; + +RawImport* raw_import_unref(RawImport *i) { + if (!i) + return NULL; + + sd_event_unref(i->event); + + if (i->temp_path) { + (void) unlink(i->temp_path); + free(i->temp_path); + } + + import_compress_free(&i->compress); + + sd_event_source_unref(i->input_event_source); + + safe_close(i->output_fd); + + free(i->final_path); + free(i->image_root); + free(i->local); + free(i); + + return NULL; +} + +int raw_import_new( + RawImport **ret, + sd_event *event, + const char *image_root, + RawImportFinished on_finished, + void *userdata) { + + _cleanup_(raw_import_unrefp) RawImport *i = NULL; + int r; + + assert(ret); + + i = new0(RawImport, 1); + if (!i) + return -ENOMEM; + + i->input_fd = i->output_fd = -1; + i->on_finished = on_finished; + i->userdata = userdata; + + RATELIMIT_INIT(i->progress_rate_limit, 500 * USEC_PER_MSEC, 1); + i->last_percent = (unsigned) -1; + + i->image_root = strdup(image_root ?: "/var/lib/machines"); + if (!i->image_root) + return -ENOMEM; + + i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines"); + + if (event) + i->event = sd_event_ref(event); + else { + r = sd_event_default(&i->event); + if (r < 0) + return r; + } + + *ret = i; + i = NULL; + + return 0; +} + +static void raw_import_report_progress(RawImport *i) { + unsigned percent; + assert(i); + + /* We have no size information, unless the source is a regular file */ + if (!S_ISREG(i->st.st_mode)) + return; + + if (i->written_compressed >= (uint64_t) i->st.st_size) + percent = 100; + else + percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->st.st_size); + + if (percent == i->last_percent) + return; + + if (!ratelimit_test(&i->progress_rate_limit)) + return; + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_info("Imported %u%%.", percent); + + i->last_percent = percent; +} + +static int raw_import_maybe_convert_qcow2(RawImport *i) { + _cleanup_close_ int converted_fd = -1; + _cleanup_free_ char *t = NULL; + int r; + + assert(i); + + r = qcow2_detect(i->output_fd); + if (r < 0) + return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m"); + if (r == 0) + return 0; + + /* This is a QCOW2 image, let's convert it */ + r = tempfn_random(i->final_path, &t); + if (r < 0) + return log_oom(); + + converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (converted_fd < 0) + return log_error_errno(errno, "Failed to create %s: %m", t); + + r = chattr_fd(converted_fd, true, FS_NOCOW_FL); + if (r < 0) + log_warning_errno(errno, "Failed to set file attributes on %s: %m", t); + + log_info("Unpacking QCOW2 file."); + + r = qcow2_convert(i->output_fd, converted_fd); + if (r < 0) { + unlink(t); + return log_error_errno(r, "Failed to convert qcow2 image: %m"); + } + + (void) unlink(i->temp_path); + free(i->temp_path); + i->temp_path = t; + t = NULL; + + safe_close(i->output_fd); + i->output_fd = converted_fd; + converted_fd = -1; + + return 1; +} + +static int raw_import_finish(RawImport *i) { + int r; + + assert(i); + assert(i->output_fd >= 0); + assert(i->temp_path); + assert(i->final_path); + + /* In case this was a sparse file, make sure the file system is right */ + if (i->written_uncompressed > 0) { + if (ftruncate(i->output_fd, i->written_uncompressed) < 0) + return log_error_errno(errno, "Failed to truncate file: %m"); + } + + r = raw_import_maybe_convert_qcow2(i); + if (r < 0) + return r; + + if (S_ISREG(i->st.st_mode)) { + (void) copy_times(i->input_fd, i->output_fd); + (void) copy_xattr(i->input_fd, i->output_fd); + } + + if (i->read_only) { + r = import_make_read_only_fd(i->output_fd); + if (r < 0) + return r; + } + + if (i->force_local) { + (void) btrfs_subvol_remove(i->final_path); + (void) rm_rf_dangerous(i->final_path, false, true, false); + } + + if (renameat2(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path, RENAME_NOREPLACE) < 0) + return log_error_errno(errno, "Failed to move image into place: %m"); + + free(i->temp_path); + i->temp_path = NULL; + + return 0; +} + +static int raw_import_open_disk(RawImport *i) { + int r; + + assert(i); + + assert(!i->final_path); + assert(!i->temp_path); + assert(i->output_fd < 0); + + i->final_path = strjoin(i->image_root, "/", i->local, ".raw", NULL); + if (!i->final_path) + return log_oom(); + + r = tempfn_random(i->final_path, &i->temp_path); + if (r < 0) + return log_oom(); + + (void) mkdir_parents_label(i->temp_path, 0700); + + i->output_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (i->output_fd < 0) + return log_error_errno(errno, "Failed to open destination %s: %m", i->temp_path); + + r = chattr_fd(i->output_fd, true, FS_NOCOW_FL); + if (r < 0) + log_warning_errno(errno, "Failed to set file attributes on %s: %m", i->temp_path); + + return 0; +} + +static int raw_import_try_reflink(RawImport *i) { + off_t p; + int r; + + assert(i); + assert(i->input_fd >= 0); + assert(i->output_fd >= 0); + + if (i->compress.type != IMPORT_COMPRESS_UNCOMPRESSED) + return 0; + + if (!S_ISREG(i->st.st_mode)) + return 0; + + p = lseek(i->input_fd, 0, SEEK_CUR); + if (p == (off_t) -1) + return log_error_errno(errno, "Failed to read file offset of input file: %m"); + + /* Let's only try a btrfs reflink, if we are reading from the beginning of the file */ + if ((uint64_t) p != (uint64_t) i->buffer_size) + return 0; + + r = btrfs_reflink(i->input_fd, i->output_fd); + if (r >= 0) + return 1; + + return 0; +} + +static int raw_import_write(const void *p, size_t sz, void *userdata) { + RawImport *i = userdata; + ssize_t n; + + if (i->grow_machine_directory && i->written_since_last_grow >= GROW_INTERVAL_BYTES) { + i->written_since_last_grow = 0; + grow_machine_directory(); + } + + n = sparse_write(i->output_fd, p, sz, 64); + if (n < 0) + return -errno; + if ((size_t) n < sz) + return -EIO; + + i->written_uncompressed += sz; + i->written_since_last_grow += sz; + + return 0; +} + +static int raw_import_process(RawImport *i) { + ssize_t l; + int r; + + assert(i); + assert(i->buffer_size < sizeof(i->buffer)); + + l = read(i->input_fd, i->buffer + i->buffer_size, sizeof(i->buffer) - i->buffer_size); + if (l < 0) { + r = log_error_errno(errno, "Failed to read input file: %m"); + goto finish; + } + if (l == 0) { + if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) { + log_error("Premature end of file: %m"); + r = -EIO; + goto finish; + } + + r = raw_import_finish(i); + goto finish; + } + + i->buffer_size += l; + + if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) { + r = import_uncompress_detect(&i->compress, i->buffer, i->buffer_size); + if (r < 0) { + log_error("Failed to detect file compression: %m"); + goto finish; + } + if (r == 0) /* Need more data */ + return 0; + + r = raw_import_open_disk(i); + if (r < 0) + goto finish; + + r = raw_import_try_reflink(i); + if (r < 0) + goto finish; + if (r > 0) { + r = raw_import_finish(i); + goto finish; + } + } + + r = import_uncompress(&i->compress, i->buffer, i->buffer_size, raw_import_write, i); + if (r < 0) { + log_error_errno(r, "Failed to decode and write: %m"); + goto finish; + } + + i->written_compressed += i->buffer_size; + i->buffer_size = 0; + + raw_import_report_progress(i); + + return 0; + +finish: + if (i->on_finished) + i->on_finished(i, r, i->userdata); + else + sd_event_exit(i->event, r); + + return 0; +} + +static int raw_import_on_input(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + RawImport *i = userdata; + + return raw_import_process(i); +} + +static int raw_import_on_defer(sd_event_source *s, void *userdata) { + RawImport *i = userdata; + + return raw_import_process(i); +} + +int raw_import_start(RawImport *i, int fd, const char *local, bool force_local, bool read_only) { + int r; + + assert(i); + assert(fd >= 0); + assert(local); + + if (!machine_name_is_valid(local)) + return -EINVAL; + + if (i->input_fd >= 0) + return -EBUSY; + + r = free_and_strdup(&i->local, local); + if (r < 0) + return r; + i->force_local = force_local; + i->read_only = read_only; + + if (fstat(fd, &i->st) < 0) + return -errno; + + r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, raw_import_on_input, i); + if (r == -EPERM) { + /* This fd does not support epoll, for example because it is a regular file. Busy read in that case */ + r = sd_event_add_defer(i->event, &i->input_event_source, raw_import_on_defer, i); + if (r < 0) + return r; + + r = sd_event_source_set_enabled(i->input_event_source, SD_EVENT_ON); + } + if (r < 0) + return r; + + i->input_fd = fd; + return r; +} diff --git a/src/import/import-raw.h b/src/import/import-raw.h new file mode 100644 index 000000000..bf7c77034 --- /dev/null +++ b/src/import/import-raw.h @@ -0,0 +1,37 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +#pragma once + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include "sd-event.h" +#include "macro.h" +#include "import-util.h" + +typedef struct RawImport RawImport; + +typedef void (*RawImportFinished)(RawImport *import, int error, void *userdata); + +int raw_import_new(RawImport **import, sd_event *event, const char *image_root, RawImportFinished on_finished, void *userdata); +RawImport* raw_import_unref(RawImport *import); + +DEFINE_TRIVIAL_CLEANUP_FUNC(RawImport*, raw_import_unref); + +int raw_import_start(RawImport *i, int fd, const char *local, bool force_local, bool read_only); diff --git a/src/import/import-tar.c b/src/import/import-tar.c new file mode 100644 index 000000000..d5b6daddd --- /dev/null +++ b/src/import/import-tar.c @@ -0,0 +1,374 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include + +#include "sd-daemon.h" +#include "sd-event.h" +#include "util.h" +#include "path-util.h" +#include "btrfs-util.h" +#include "copy.h" +#include "mkdir.h" +#include "ratelimit.h" +#include "machine-pool.h" +#include "qcow2-util.h" +#include "import-compress.h" +#include "import-common.h" +#include "import-tar.h" + +struct TarImport { + sd_event *event; + + char *image_root; + + TarImportFinished on_finished; + void *userdata; + + char *local; + bool force_local; + bool read_only; + bool grow_machine_directory; + + char *temp_path; + char *final_path; + + int input_fd; + int tar_fd; + + ImportCompress compress; + + uint64_t written_since_last_grow; + + sd_event_source *input_event_source; + + uint8_t buffer[16*1024]; + size_t buffer_size; + + uint64_t written_compressed; + uint64_t written_uncompressed; + + struct stat st; + + pid_t tar_pid; + + unsigned last_percent; + RateLimit progress_rate_limit; +}; + +TarImport* tar_import_unref(TarImport *i) { + if (!i) + return NULL; + + sd_event_unref(i->event); + + if (i->tar_pid > 1) { + (void) kill_and_sigcont(i->tar_pid, SIGKILL); + (void) wait_for_terminate(i->tar_pid, NULL); + } + + if (i->temp_path) { + (void) btrfs_subvol_remove(i->temp_path); + (void) rm_rf_dangerous(i->temp_path, false, true, false); + free(i->temp_path); + } + + import_compress_free(&i->compress); + + sd_event_source_unref(i->input_event_source); + + safe_close(i->tar_fd); + + free(i->final_path); + free(i->image_root); + free(i->local); + free(i); + + return NULL; +} + +int tar_import_new( + TarImport **ret, + sd_event *event, + const char *image_root, + TarImportFinished on_finished, + void *userdata) { + + _cleanup_(tar_import_unrefp) TarImport *i = NULL; + int r; + + assert(ret); + + i = new0(TarImport, 1); + if (!i) + return -ENOMEM; + + i->input_fd = i->tar_fd = -1; + i->on_finished = on_finished; + i->userdata = userdata; + + RATELIMIT_INIT(i->progress_rate_limit, 500 * USEC_PER_MSEC, 1); + i->last_percent = (unsigned) -1; + + i->image_root = strdup(image_root ?: "/var/lib/machines"); + if (!i->image_root) + return -ENOMEM; + + i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines"); + + if (event) + i->event = sd_event_ref(event); + else { + r = sd_event_default(&i->event); + if (r < 0) + return r; + } + + *ret = i; + i = NULL; + + return 0; +} + +static void tar_import_report_progress(TarImport *i) { + unsigned percent; + assert(i); + + /* We have no size information, unless the source is a regular file */ + if (!S_ISREG(i->st.st_mode)) + return; + + if (i->written_compressed >= (uint64_t) i->st.st_size) + percent = 100; + else + percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->st.st_size); + + if (percent == i->last_percent) + return; + + if (!ratelimit_test(&i->progress_rate_limit)) + return; + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_info("Imported %u%%.", percent); + + i->last_percent = percent; +} + +static int tar_import_finish(TarImport *i) { + int r; + + assert(i); + assert(i->tar_fd >= 0); + assert(i->temp_path); + assert(i->final_path); + + i->tar_fd = safe_close(i->tar_fd); + + if (i->tar_pid > 0) { + r = wait_for_terminate_and_warn("tar", i->tar_pid, true); + i->tar_pid = 0; + if (r < 0) + return r; + } + + if (i->read_only) { + r = import_make_read_only(i->temp_path); + if (r < 0) + return r; + } + + if (i->force_local) { + (void) btrfs_subvol_remove(i->final_path); + (void) rm_rf_dangerous(i->final_path, false, true, false); + } + + if (renameat2(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path, RENAME_NOREPLACE) < 0) + return log_error_errno(errno, "Failed to move image into place: %m"); + + free(i->temp_path); + i->temp_path = NULL; + + return 0; +} + +static int tar_import_fork_tar(TarImport *i) { + int r; + + assert(i); + + assert(!i->final_path); + assert(!i->temp_path); + assert(i->tar_fd < 0); + + i->final_path = strjoin(i->image_root, "/", i->local, NULL); + if (!i->final_path) + return log_oom(); + + r = tempfn_random(i->final_path, &i->temp_path); + if (r < 0) + return log_oom(); + + (void) mkdir_parents_label(i->temp_path, 0700); + + r = btrfs_subvol_make(i->temp_path); + if (r == -ENOTTY) { + if (mkdir(i->temp_path, 0755) < 0) + return log_error_errno(errno, "Failed to create directory %s: %m", i->temp_path); + } else if (r < 0) + return log_error_errno(errno, "Failed to create subvolume %s: %m", i->temp_path); + + i->tar_fd = import_fork_tar(i->temp_path, &i->tar_pid); + if (i->tar_fd < 0) + return i->tar_fd; + + return 0; +} + +static int tar_import_write(const void *p, size_t sz, void *userdata) { + TarImport *i = userdata; + int r; + + if (i->grow_machine_directory && i->written_since_last_grow >= GROW_INTERVAL_BYTES) { + i->written_since_last_grow = 0; + grow_machine_directory(); + } + + r = loop_write(i->tar_fd, p, sz, false); + if (r < 0) + return r; + + i->written_uncompressed += sz; + i->written_since_last_grow += sz; + + return 0; +} + +static int tar_import_process(TarImport *i) { + ssize_t l; + int r; + + assert(i); + assert(i->buffer_size < sizeof(i->buffer)); + + l = read(i->input_fd, i->buffer + i->buffer_size, sizeof(i->buffer) - i->buffer_size); + if (l < 0) { + r = log_error_errno(errno, "Failed to read input file: %m"); + goto finish; + } + if (l == 0) { + if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) { + log_error("Premature end of file: %m"); + r = -EIO; + goto finish; + } + + r = tar_import_finish(i); + goto finish; + } + + i->buffer_size += l; + + if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) { + r = import_uncompress_detect(&i->compress, i->buffer, i->buffer_size); + if (r < 0) { + log_error("Failed to detect file compression: %m"); + goto finish; + } + if (r == 0) /* Need more data */ + return 0; + + r = tar_import_fork_tar(i); + if (r < 0) + goto finish; + } + + r = import_uncompress(&i->compress, i->buffer, i->buffer_size, tar_import_write, i); + if (r < 0) { + log_error_errno(r, "Failed to decode and write: %m"); + goto finish; + } + + i->written_compressed += i->buffer_size; + i->buffer_size = 0; + + tar_import_report_progress(i); + + return 0; + +finish: + if (i->on_finished) + i->on_finished(i, r, i->userdata); + else + sd_event_exit(i->event, r); + + return 0; +} + +static int tar_import_on_input(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + TarImport *i = userdata; + + return tar_import_process(i); +} + +static int tar_import_on_defer(sd_event_source *s, void *userdata) { + TarImport *i = userdata; + + return tar_import_process(i); +} + +int tar_import_start(TarImport *i, int fd, const char *local, bool force_local, bool read_only) { + int r; + + assert(i); + assert(fd >= 0); + assert(local); + + if (!machine_name_is_valid(local)) + return -EINVAL; + + if (i->input_fd >= 0) + return -EBUSY; + + r = free_and_strdup(&i->local, local); + if (r < 0) + return r; + i->force_local = force_local; + i->read_only = read_only; + + if (fstat(fd, &i->st) < 0) + return -errno; + + r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, tar_import_on_input, i); + if (r == -EPERM) { + /* This fd does not support epoll, for example because it is a regular file. Busy read in that case */ + r = sd_event_add_defer(i->event, &i->input_event_source, tar_import_on_defer, i); + if (r < 0) + return r; + + r = sd_event_source_set_enabled(i->input_event_source, SD_EVENT_ON); + } + if (r < 0) + return r; + + i->input_fd = fd; + return r; +} diff --git a/src/import/import-tar.h b/src/import/import-tar.h new file mode 100644 index 000000000..aaecb5139 --- /dev/null +++ b/src/import/import-tar.h @@ -0,0 +1,37 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +#pragma once + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include "sd-event.h" +#include "macro.h" +#include "import-util.h" + +typedef struct TarImport TarImport; + +typedef void (*TarImportFinished)(TarImport *import, int error, void *userdata); + +int tar_import_new(TarImport **import, sd_event *event, const char *image_root, TarImportFinished on_finished, void *userdata); +TarImport* tar_import_unref(TarImport *import); + +DEFINE_TRIVIAL_CLEANUP_FUNC(TarImport*, tar_import_unref); + +int tar_import_start(TarImport *import, int fd, const char *local, bool force_local, bool read_only); diff --git a/src/import/import.c b/src/import/import.c new file mode 100644 index 000000000..762c425e6 --- /dev/null +++ b/src/import/import.c @@ -0,0 +1,336 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include + +#include "sd-event.h" +#include "event-util.h" +#include "verbs.h" +#include "build.h" +#include "machine-image.h" +#include "import-util.h" +#include "import-tar.h" +#include "import-raw.h" + +static bool arg_force = false; +static bool arg_read_only = false; +static const char *arg_image_root = "/var/lib/machines"; + +static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { + log_notice("Transfer aborted."); + sd_event_exit(sd_event_source_get_event(s), EINTR); + return 0; +} + +static void on_tar_finished(TarImport *import, int error, void *userdata) { + sd_event *event = userdata; + assert(import); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int import_tar(int argc, char *argv[], void *userdata) { + _cleanup_(tar_import_unrefp) TarImport *import = NULL; + _cleanup_event_unref_ sd_event *event = NULL; + const char *path = NULL, *local = NULL; + _cleanup_free_ char *ll = NULL; + _cleanup_close_ int open_fd = -1; + int r, fd; + + if (argc >= 2) + path = argv[1]; + if (isempty(path) || streq(path, "-")) + path = NULL; + + if (argc >= 3) + local = argv[2]; + else if (path) + local = basename(path); + if (isempty(local) || streq(local, "-")) + local = NULL; + + if (local) { + r = tar_strip_suffixes(local, &ll); + if (r < 0) + return log_oom(); + + local = ll; + + if (!machine_name_is_valid(local)) { + log_error("Local image name '%s' is not valid.", local); + return -EINVAL; + } + + if (!arg_force) { + r = image_find(local, NULL); + if (r < 0) + return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); + else if (r > 0) { + log_error_errno(EEXIST, "Image '%s' already exists.", local); + return -EEXIST; + } + } + } else + local = "imported"; + + if (path) { + open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open tar image to import: %m"); + + fd = open_fd; + + log_info("Importing '%s', saving as '%s'.", path, local); + } else { + _cleanup_free_ char *pretty = NULL; + + fd = STDIN_FILENO; + + (void) readlink_malloc("/proc/self/fd/0", &pretty); + log_info("Importing '%s', saving as '%s'.", strna(pretty), local); + } + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, SIGTERM, SIGINT, -1) == 0); + sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + r = tar_import_new(&import, event, arg_image_root, on_tar_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate importer: %m"); + + r = tar_import_start(import, fd, local, arg_force, arg_read_only); + if (r < 0) + return log_error_errno(r, "Failed to import image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static void on_raw_finished(RawImport *import, int error, void *userdata) { + sd_event *event = userdata; + assert(import); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int import_raw(int argc, char *argv[], void *userdata) { + _cleanup_(raw_import_unrefp) RawImport *import = NULL; + _cleanup_event_unref_ sd_event *event = NULL; + const char *path = NULL, *local = NULL; + _cleanup_free_ char *ll = NULL; + _cleanup_close_ int open_fd = -1; + int r, fd; + + if (argc >= 2) + path = argv[1]; + if (isempty(path) || streq(path, "-")) + path = NULL; + + if (argc >= 3) + local = argv[2]; + else if (path) + local = basename(path); + if (isempty(local) || streq(local, "-")) + local = NULL; + + if (local) { + r = raw_strip_suffixes(local, &ll); + if (r < 0) + return log_oom(); + + local = ll; + + if (!machine_name_is_valid(local)) { + log_error("Local image name '%s' is not valid.", local); + return -EINVAL; + } + + if (!arg_force) { + r = image_find(local, NULL); + if (r < 0) + return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); + else if (r > 0) { + log_error_errno(EEXIST, "Image '%s' already exists.", local); + return -EEXIST; + } + } + } else + local = "imported"; + + if (path) { + open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open raw image to import: %m"); + + fd = open_fd; + + log_info("Importing '%s', saving as '%s'.", path, local); + } else { + _cleanup_free_ char *pretty = NULL; + + fd = STDIN_FILENO; + + (void) readlink_malloc("/proc/self/fd/0", &pretty); + log_info("Importing '%s', saving as '%s'.", pretty, local); + } + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, SIGTERM, SIGINT, -1) == 0); + sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + r = raw_import_new(&import, event, arg_image_root, on_raw_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate importer: %m"); + + r = raw_import_start(import, fd, local, arg_force, arg_read_only); + if (r < 0) + return log_error_errno(r, "Failed to import image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static int help(int argc, char *argv[], void *userdata) { + + printf("%s [OPTIONS...] {COMMAND} ...\n\n" + "Import container or virtual machine image.\n\n" + " -h --help Show this help\n" + " --version Show package version\n" + " --force Force creation of image\n" + " --image-root=PATH Image root directory\n" + " --read-only Create a read-only image\n\n" + "Commands:\n" + " tar FILE [NAME] Download a TAR image\n" + " raw FILE [NAME] Download a RAW image\n", + program_invocation_short_name); + + return 0; +} + +static int parse_argv(int argc, char *argv[]) { + + enum { + ARG_VERSION = 0x100, + ARG_FORCE, + ARG_IMAGE_ROOT, + ARG_READ_ONLY, + }; + + static const struct option options[] = { + { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, ARG_VERSION }, + { "force", no_argument, NULL, ARG_FORCE }, + { "image-root", required_argument, NULL, ARG_IMAGE_ROOT }, + { "read-only", no_argument, NULL, ARG_READ_ONLY }, + {} + }; + + int c; + + assert(argc >= 0); + assert(argv); + + while ((c = getopt_long(argc, argv, "h", options, NULL)) >= 0) + + switch (c) { + + case 'h': + return help(0, NULL, NULL); + + case ARG_VERSION: + puts(PACKAGE_STRING); + puts(SYSTEMD_FEATURES); + return 0; + + case ARG_FORCE: + arg_force = true; + break; + + case ARG_IMAGE_ROOT: + arg_image_root = optarg; + break; + + case ARG_READ_ONLY: + arg_read_only = true; + break; + + case '?': + return -EINVAL; + + default: + assert_not_reached("Unhandled option"); + } + + return 1; +} + +static int import_main(int argc, char *argv[]) { + + static const Verb verbs[] = { + { "help", VERB_ANY, VERB_ANY, 0, help }, + { "tar", 2, 3, 0, import_tar }, + { "raw", 2, 3, 0, import_raw }, + {} + }; + + return dispatch_verb(argc, argv, verbs, NULL); +} + +int main(int argc, char *argv[]) { + int r; + + setlocale(LC_ALL, ""); + log_parse_environment(); + log_open(); + + r = parse_argv(argc, argv); + if (r <= 0) + goto finish; + + ignore_signals(SIGPIPE, -1); + + r = import_main(argc, argv); + +finish: + return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/src/import/importd.c b/src/import/importd.c index 8a6a8c8ed..e2ff9f716 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -39,9 +39,11 @@ typedef struct Transfer Transfer; typedef struct Manager Manager; typedef enum TransferType { - TRANSFER_TAR, - TRANSFER_RAW, - TRANSFER_DKR, + TRANSFER_IMPORT_TAR, + TRANSFER_IMPORT_RAW, + TRANSFER_PULL_TAR, + TRANSFER_PULL_RAW, + TRANSFER_PULL_DKR, _TRANSFER_TYPE_MAX, _TRANSFER_TYPE_INVALID = -1, } TransferType; @@ -58,6 +60,7 @@ struct Transfer { char *remote; char *local; bool force_local; + bool read_only; char *dkr_index_url; @@ -73,6 +76,8 @@ struct Transfer { unsigned n_canceled; unsigned progress_percent; + + int stdin_fd; }; struct Manager { @@ -92,9 +97,11 @@ struct Manager { #define TRANSFERS_MAX 64 static const char* const transfer_type_table[_TRANSFER_TYPE_MAX] = { - [TRANSFER_TAR] = "tar", - [TRANSFER_RAW] = "raw", - [TRANSFER_DKR] = "dkr", + [TRANSFER_IMPORT_TAR] = "import-tar", + [TRANSFER_IMPORT_RAW] = "import-raw", + [TRANSFER_PULL_TAR] = "pull-tar", + [TRANSFER_PULL_RAW] = "pull-raw", + [TRANSFER_PULL_DKR] = "pull-dkr", }; DEFINE_PRIVATE_STRING_TABLE_LOOKUP_TO_STRING(transfer_type, TransferType); @@ -120,6 +127,7 @@ static Transfer *transfer_unref(Transfer *t) { } safe_close(t->log_fd); + safe_close(t->stdin_fd); free(t); return NULL; @@ -148,6 +156,8 @@ static int transfer_new(Manager *m, Transfer **ret) { t->type = _TRANSFER_TYPE_INVALID; t->log_fd = -1; + t->stdin_fd = -1; + t->verify = _IMPORT_VERIFY_INVALID; id = m->current_transfer_id + 1; @@ -353,19 +363,19 @@ static int transfer_start(Transfer *t) { return -errno; if (t->pid == 0) { const char *cmd[] = { - "systemd-pull", - transfer_type_to_string(t->type), - "--verify", + NULL, /* systemd-import or systemd-pull */ + NULL, /* tar, raw, dkr */ + NULL, /* --verify= */ NULL, /* verify argument */ NULL, /* maybe --force */ + NULL, /* maybe --read-only */ NULL, /* maybe --dkr-index-url */ NULL, /* the actual URL */ NULL, /* remote */ NULL, /* local */ NULL }; - int null_fd; - unsigned k = 3; + unsigned k = 0; /* Child */ @@ -388,20 +398,32 @@ static int transfer_start(Transfer *t) { if (pipefd[1] != STDOUT_FILENO && pipefd[1] != STDERR_FILENO) pipefd[1] = safe_close(pipefd[1]); - null_fd = open("/dev/null", O_RDONLY|O_NOCTTY); - if (null_fd < 0) { - log_error_errno(errno, "Failed to open /dev/null: %m"); - _exit(EXIT_FAILURE); + if (t->stdin_fd >= 0) { + if (dup2(t->stdin_fd, STDIN_FILENO) != STDIN_FILENO) { + log_error_errno(errno, "Failed to dup2() fd: %m"); + _exit(EXIT_FAILURE); + } + + if (t->stdin_fd != STDIN_FILENO) + safe_close(t->stdin_fd); + } else { + int null_fd; + + null_fd = open("/dev/null", O_RDONLY|O_NOCTTY); + if (null_fd < 0) { + log_error_errno(errno, "Failed to open /dev/null: %m"); + _exit(EXIT_FAILURE); + } + + if (dup2(null_fd, STDIN_FILENO) != STDIN_FILENO) { + log_error_errno(errno, "Failed to dup2() fd: %m"); + _exit(EXIT_FAILURE); + } + + if (null_fd != STDIN_FILENO) + safe_close(null_fd); } - if (dup2(null_fd, STDIN_FILENO) != STDIN_FILENO) { - log_error_errno(errno, "Failed to dup2() fd: %m"); - _exit(EXIT_FAILURE); - } - - if (null_fd != STDIN_FILENO) - safe_close(null_fd); - fd_cloexec(STDIN_FILENO, false); fd_cloexec(STDOUT_FILENO, false); fd_cloexec(STDERR_FILENO, false); @@ -409,22 +431,44 @@ static int transfer_start(Transfer *t) { setenv("SYSTEMD_LOG_TARGET", "console-prefixed", 1); setenv("NOTIFY_SOCKET", "/run/systemd/import/notify", 1); - cmd[k++] = import_verify_to_string(t->verify); + if (IN_SET(t->type, TRANSFER_IMPORT_TAR, TRANSFER_IMPORT_RAW)) + cmd[k++] = SYSTEMD_IMPORT_PATH; + else + cmd[k++] = SYSTEMD_PULL_PATH; + + if (IN_SET(t->type, TRANSFER_IMPORT_TAR, TRANSFER_PULL_TAR)) + cmd[k++] = "tar"; + else if (IN_SET(t->type, TRANSFER_IMPORT_RAW, TRANSFER_PULL_RAW)) + cmd[k++] = "raw"; + else + cmd[k++] = "dkr"; + + if (t->verify != _IMPORT_VERIFY_INVALID) { + cmd[k++] = "--verify"; + cmd[k++] = import_verify_to_string(t->verify); + } + if (t->force_local) cmd[k++] = "--force"; + if (t->read_only) + cmd[k++] = "--read-only"; if (t->dkr_index_url) { cmd[k++] = "--dkr-index-url"; cmd[k++] = t->dkr_index_url; } - cmd[k++] = t->remote; + if (t->remote) + cmd[k++] = t->remote; + else + cmd[k++] = "-"; + if (t->local) cmd[k++] = t->local; cmd[k] = NULL; - execv(SYSTEMD_PULL_PATH, (char * const *) cmd); - log_error_errno(errno, "Failed to execute import tool: %m"); + execv(cmd[0], (char * const *) cmd); + log_error_errno(errno, "Failed to execute %s tool: %m", cmd[0]); _exit(EXIT_FAILURE); } @@ -432,6 +476,8 @@ static int transfer_start(Transfer *t) { t->log_fd = pipefd[0]; pipefd[0] = -1; + t->stdin_fd = safe_close(t->stdin_fd); + r = sd_event_add_child(t->manager->event, &t->pid_event_source, t->pid, WEXITED, transfer_on_pid, t); if (r < 0) return r; @@ -644,6 +690,67 @@ static Transfer *manager_find(Manager *m, TransferType type, const char *dkr_ind return NULL; } +static int method_import_tar_or_raw(sd_bus *bus, sd_bus_message *msg, void *userdata, sd_bus_error *error) { + _cleanup_(transfer_unrefp) Transfer *t = NULL; + int fd, force, read_only, r; + const char *local, *object; + Manager *m = userdata; + TransferType type; + uint32_t id; + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.import", + false, + UID_INVALID, + &m->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = sd_bus_message_read(msg, "hsbb", &fd, &local, &force, &read_only); + if (r < 0) + return r; + + if (!machine_name_is_valid(local)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Local name %s is invalid", local); + + r = setup_machine_directory((uint64_t) -1, error); + if (r < 0) + return r; + + type = streq_ptr(sd_bus_message_get_member(msg), "ImportTar") ? TRANSFER_IMPORT_TAR : TRANSFER_IMPORT_RAW; + + r = transfer_new(m, &t); + if (r < 0) + return r; + + t->type = type; + t->force_local = force; + t->read_only = read_only; + + t->local = strdup(local); + if (!t->local) + return -ENOMEM; + + t->stdin_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); + if (t->stdin_fd < 0) + return -errno; + + r = transfer_start(t); + if (r < 0) + return r; + + object = t->object_path; + id = t->id; + t = NULL; + + return sd_bus_reply_method_return(msg, "uo", id, object); +} + static int method_pull_tar_or_raw(sd_bus *bus, sd_bus_message *msg, void *userdata, sd_bus_error *error) { _cleanup_(transfer_unrefp) Transfer *t = NULL; const char *remote, *local, *verify, *object; @@ -693,7 +800,7 @@ static int method_pull_tar_or_raw(sd_bus *bus, sd_bus_message *msg, void *userda if (r < 0) return r; - type = streq_ptr(sd_bus_message_get_member(msg), "PullTar") ? TRANSFER_TAR : TRANSFER_RAW; + type = streq_ptr(sd_bus_message_get_member(msg), "PullTar") ? TRANSFER_PULL_TAR : TRANSFER_PULL_RAW; if (manager_find(m, type, NULL, remote)) return sd_bus_error_setf(error, BUS_ERROR_TRANSFER_IN_PROGRESS, "Transfer for %s already in progress.", remote); @@ -710,9 +817,11 @@ static int method_pull_tar_or_raw(sd_bus *bus, sd_bus_message *msg, void *userda if (!t->remote) return -ENOMEM; - t->local = strdup(local); - if (!t->local) - return -ENOMEM; + if (local) { + t->local = strdup(local); + if (!t->local) + return -ENOMEM; + } r = transfer_start(t); if (r < 0) @@ -788,14 +897,14 @@ static int method_pull_dkr(sd_bus *bus, sd_bus_message *msg, void *userdata, sd_ if (r < 0) return r; - if (manager_find(m, TRANSFER_DKR, index_url, remote)) + if (manager_find(m, TRANSFER_PULL_DKR, index_url, remote)) return sd_bus_error_setf(error, BUS_ERROR_TRANSFER_IN_PROGRESS, "Transfer for %s already in progress.", remote); r = transfer_new(m, &t); if (r < 0) return r; - t->type = TRANSFER_DKR; + t->type = TRANSFER_PULL_DKR; t->verify = v; t->force_local = force; @@ -807,9 +916,11 @@ static int method_pull_dkr(sd_bus *bus, sd_bus_message *msg, void *userdata, sd_ if (!t->remote) return -ENOMEM; - t->local = strdup(local); - if (!t->local) - return -ENOMEM; + if (local) { + t->local = strdup(local); + if (!t->local) + return -ENOMEM; + } r = transfer_start(t); if (r < 0) @@ -967,6 +1078,8 @@ static const sd_bus_vtable transfer_vtable[] = { static const sd_bus_vtable manager_vtable[] = { SD_BUS_VTABLE_START(0), + SD_BUS_METHOD("ImportTar", "hsbb", "uo", method_import_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD("ImportRaw", "hsbb", "uo", method_import_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD("PullTar", "sssb", "uo", method_pull_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD("PullRaw", "sssb", "uo", method_pull_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD("PullDkr", "sssssb", "uo", method_pull_dkr, SD_BUS_VTABLE_UNPRIVILEGED), diff --git a/src/import/org.freedesktop.import1.policy.in b/src/import/org.freedesktop.import1.policy.in index 1003f4644..95a79d2ba 100644 --- a/src/import/org.freedesktop.import1.policy.in +++ b/src/import/org.freedesktop.import1.policy.in @@ -16,6 +16,16 @@ The systemd Project http://www.freedesktop.org/wiki/Software/systemd + + <_description>Import a VM or container image + <_message>Authentication is required to import a VM or container image + + auth_admin + auth_admin + auth_admin_keep + + + <_description>Download a VM or container image <_message>Authentication is required to download a VM or container image diff --git a/src/import/pull-common.c b/src/import/pull-common.c index 38380fc01..94dd54bd5 100644 --- a/src/import/pull-common.c +++ b/src/import/pull-common.c @@ -141,47 +141,6 @@ int pull_make_local_copy(const char *final, const char *image_root, const char * return 0; } -int pull_make_read_only_fd(int fd) { - int r; - - assert(fd >= 0); - - /* First, let's make this a read-only subvolume if it refers - * to a subvolume */ - r = btrfs_subvol_set_read_only_fd(fd, true); - if (r == -ENOTTY || r == -ENOTDIR || r == -EINVAL) { - struct stat st; - - /* This doesn't refer to a subvolume, or the file - * system isn't even btrfs. In that, case fall back to - * chmod()ing */ - - r = fstat(fd, &st); - if (r < 0) - return log_error_errno(errno, "Failed to stat temporary image: %m"); - - /* Drop "w" flag */ - if (fchmod(fd, st.st_mode & 07555) < 0) - return log_error_errno(errno, "Failed to chmod() final image: %m"); - - return 0; - - } else if (r < 0) - return log_error_errno(r, "Failed to make subvolume read-only: %m"); - - return 0; -} - -int pull_make_read_only(const char *path) { - _cleanup_close_ int fd = 1; - - fd = open(path, O_RDONLY|O_NOCTTY|O_CLOEXEC); - if (fd < 0) - return log_error_errno(errno, "Failed to open %s: %m", path); - - return pull_make_read_only_fd(fd); -} - int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret) { _cleanup_free_ char *escaped_url = NULL; char *path; @@ -463,83 +422,3 @@ finish: return r; } - -int pull_fork_tar(const char *path, pid_t *ret) { - _cleanup_close_pair_ int pipefd[2] = { -1, -1 }; - pid_t pid; - int r; - - assert(path); - assert(ret); - - if (pipe2(pipefd, O_CLOEXEC) < 0) - return log_error_errno(errno, "Failed to create pipe for tar: %m"); - - pid = fork(); - if (pid < 0) - return log_error_errno(errno, "Failed to fork off tar: %m"); - - if (pid == 0) { - int null_fd; - uint64_t retain = - (1ULL << CAP_CHOWN) | - (1ULL << CAP_FOWNER) | - (1ULL << CAP_FSETID) | - (1ULL << CAP_MKNOD) | - (1ULL << CAP_SETFCAP) | - (1ULL << CAP_DAC_OVERRIDE); - - /* Child */ - - reset_all_signal_handlers(); - reset_signal_mask(); - assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0); - - pipefd[1] = safe_close(pipefd[1]); - - if (dup2(pipefd[0], STDIN_FILENO) != STDIN_FILENO) { - log_error_errno(errno, "Failed to dup2() fd: %m"); - _exit(EXIT_FAILURE); - } - - if (pipefd[0] != STDIN_FILENO) - pipefd[0] = safe_close(pipefd[0]); - - null_fd = open("/dev/null", O_WRONLY|O_NOCTTY); - if (null_fd < 0) { - log_error_errno(errno, "Failed to open /dev/null: %m"); - _exit(EXIT_FAILURE); - } - - if (dup2(null_fd, STDOUT_FILENO) != STDOUT_FILENO) { - log_error_errno(errno, "Failed to dup2() fd: %m"); - _exit(EXIT_FAILURE); - } - - if (null_fd != STDOUT_FILENO) - null_fd = safe_close(null_fd); - - fd_cloexec(STDIN_FILENO, false); - fd_cloexec(STDOUT_FILENO, false); - fd_cloexec(STDERR_FILENO, false); - - if (unshare(CLONE_NEWNET) < 0) - log_error_errno(errno, "Failed to lock tar into network namespace, ignoring: %m"); - - r = capability_bounding_set_drop(~retain, true); - if (r < 0) - log_error_errno(r, "Failed to drop capabilities, ignoring: %m"); - - execlp("tar", "tar", "--numeric-owner", "-C", path, "-px", NULL); - log_error_errno(errno, "Failed to execute tar: %m"); - _exit(EXIT_FAILURE); - } - - pipefd[0] = safe_close(pipefd[0]); - r = pipefd[1]; - pipefd[1] = -1; - - *ret = pid; - - return r; -} diff --git a/src/import/pull-common.h b/src/import/pull-common.h index 4ac016bde..bb9cf3efc 100644 --- a/src/import/pull-common.h +++ b/src/import/pull-common.h @@ -30,12 +30,7 @@ int pull_make_local_copy(const char *final, const char *root, const char *local, int pull_find_old_etags(const char *url, const char *root, int dt, const char *prefix, const char *suffix, char ***etags); -int pull_make_read_only_fd(int fd); -int pull_make_read_only(const char *path); - int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret); int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata); int pull_verify(PullJob *main_job, PullJob *checksum_job, PullJob *signature_job); - -int pull_fork_tar(const char *path, pid_t *ret); diff --git a/src/import/pull-dkr.c b/src/import/pull-dkr.c index ecbf8063c..3a9beadf6 100644 --- a/src/import/pull-dkr.c +++ b/src/import/pull-dkr.c @@ -34,6 +34,7 @@ #include "aufs-util.h" #include "pull-job.h" #include "pull-common.h" +#include "import-common.h" #include "pull-dkr.h" typedef enum DkrProgress { @@ -491,7 +492,7 @@ static int dkr_pull_job_on_open_disk(PullJob *j) { if (r < 0) return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path); - j->disk_fd = pull_fork_tar(i->temp_path, &i->tar_pid); + j->disk_fd = import_fork_tar(i->temp_path, &i->tar_pid); if (j->disk_fd < 0) return j->disk_fd; diff --git a/src/import/pull-job.c b/src/import/pull-job.c index b5237449c..ed9af2351 100644 --- a/src/import/pull-job.c +++ b/src/import/pull-job.c @@ -25,9 +25,6 @@ #include "machine-pool.h" #include "pull-job.h" -/* Grow the /var/lib/machines directory after each 10MiB written */ -#define PULL_GROW_INTERVAL_BYTES (UINT64_C(10) * UINT64_C(1024) * UINT64_C(1024)) - PullJob* pull_job_unref(PullJob *j) { if (!j) return NULL; @@ -148,8 +145,7 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) { * sparse and we just seeked for the last part */ if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) { - log_error_errno(errno, "Failed to truncate file: %m"); - r = -errno; + r = log_error_errno(errno, "Failed to truncate file: %m"); goto finish; } @@ -197,7 +193,7 @@ static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) if (j->disk_fd >= 0) { - if (j->grow_machine_directory && j->written_since_last_grow >= PULL_GROW_INTERVAL_BYTES) { + if (j->grow_machine_directory && j->written_since_last_grow >= GROW_INTERVAL_BYTES) { j->written_since_last_grow = 0; grow_machine_directory(); } @@ -206,10 +202,8 @@ static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) n = sparse_write(j->disk_fd, p, sz, 64); else n = write(j->disk_fd, p, sz); - if (n < 0) { - log_error_errno(errno, "Failed to write file: %m"); - return -errno; - } + if (n < 0) + return log_error_errno(errno, "Failed to write file: %m"); if ((size_t) n < sz) { log_error("Short write"); return -EIO; diff --git a/src/import/pull-raw.c b/src/import/pull-raw.c index 4029c7ed1..d1d77d598 100644 --- a/src/import/pull-raw.c +++ b/src/import/pull-raw.c @@ -33,6 +33,7 @@ #include "mkdir.h" #include "path-util.h" #include "import-util.h" +#include "import-common.h" #include "curl-util.h" #include "qcow2-util.h" #include "pull-job.h" @@ -210,7 +211,7 @@ static int raw_pull_maybe_convert_qcow2(RawPull *i) { if (r < 0) return log_oom(); - converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644); + converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); if (converted_fd < 0) return log_error_errno(errno, "Failed to create %s: %m", t); @@ -226,9 +227,8 @@ static int raw_pull_maybe_convert_qcow2(RawPull *i) { return log_error_errno(r, "Failed to convert qcow2 image: %m"); } - unlink(i->temp_path); + (void) unlink(i->temp_path); free(i->temp_path); - i->temp_path = t; t = NULL; @@ -380,11 +380,11 @@ static void raw_pull_job_on_finished(PullJob *j) { raw_pull_report_progress(i, RAW_FINALIZING); - r = pull_make_read_only_fd(i->raw_job->disk_fd); + r = import_make_read_only_fd(i->raw_job->disk_fd); if (r < 0) goto finish; - r = rename(i->temp_path, i->final_path); + r = renameat2(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path, RENAME_NOREPLACE); if (r < 0) { r = log_error_errno(errno, "Failed to move RAW file into place: %m"); goto finish; @@ -426,12 +426,12 @@ static int raw_pull_job_on_open_disk(PullJob *j) { return log_oom(); r = tempfn_random(i->final_path, &i->temp_path); - if (r <0) + if (r < 0) return log_oom(); - mkdir_parents_label(i->temp_path, 0700); + (void) mkdir_parents_label(i->temp_path, 0700); - j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644); + j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); if (j->disk_fd < 0) return log_error_errno(errno, "Failed to create %s: %m", i->temp_path); diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c index de653a88f..504642fa2 100644 --- a/src/import/pull-tar.c +++ b/src/import/pull-tar.c @@ -32,6 +32,7 @@ #include "mkdir.h" #include "path-util.h" #include "import-util.h" +#include "import-common.h" #include "curl-util.h" #include "pull-job.h" #include "pull-common.h" @@ -276,11 +277,11 @@ static void tar_pull_job_on_finished(PullJob *j) { tar_pull_report_progress(i, TAR_FINALIZING); - r = pull_make_read_only(i->temp_path); + r = import_make_read_only(i->temp_path); if (r < 0) goto finish; - if (rename(i->temp_path, i->final_path) < 0) { + if (renameat2(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path, RENAME_NOREPLACE) < 0) { r = log_error_errno(errno, "Failed to rename to final image name: %m"); goto finish; } @@ -334,7 +335,7 @@ static int tar_pull_job_on_open_disk(PullJob *j) { } else if (r < 0) return log_error_errno(errno, "Failed to create subvolume %s: %m", i->temp_path); - j->disk_fd = pull_fork_tar(i->temp_path, &i->tar_pid); + j->disk_fd = import_fork_tar(i->temp_path, &i->tar_pid); if (j->disk_fd < 0) return j->disk_fd; diff --git a/src/import/pull.c b/src/import/pull.c index b3c48b320..03a17a5fe 100644 --- a/src/import/pull.c +++ b/src/import/pull.c @@ -433,6 +433,8 @@ int main(int argc, char *argv[]) { if (r <= 0) goto finish; + ignore_signals(SIGPIPE, -1); + r = pull_main(argc, argv); finish: diff --git a/src/machine/machinectl.c b/src/machine/machinectl.c index 49e28ee52..2f2b77114 100644 --- a/src/machine/machinectl.c +++ b/src/machine/machinectl.c @@ -1598,7 +1598,7 @@ static int pull_image_common(sd_bus *bus, sd_bus_message *m) { r = sd_bus_call(bus, m, 0, &error, &reply); if (r < 0) { - log_error("Failed pull image: %s", bus_error_message(&error, -r)); + log_error("Failed acquire image: %s", bus_error_message(&error, -r)); return r; } @@ -1621,6 +1621,140 @@ static int pull_image_common(sd_bus *bus, sd_bus_message *m) { return -r; } +static int import_tar(int argc, char *argv[], void *userdata) { + _cleanup_bus_message_unref_ sd_bus_message *m = NULL; + _cleanup_free_ char *ll = NULL; + _cleanup_close_ int fd = -1; + const char *local = NULL, *path = NULL; + sd_bus *bus = userdata; + int r; + + assert(bus); + + if (argc >= 2) + path = argv[1]; + if (isempty(path) || streq(path, "-")) + path = NULL; + + if (argc >= 3) + local = argv[2]; + else if (path) + local = basename(path); + if (isempty(local) || streq(local, "-")) + local = NULL; + + if (!local) { + log_error("Need either path or local name."); + return -EINVAL; + } + + r = tar_strip_suffixes(local, &ll); + if (r < 0) + return log_oom(); + + local = ll; + + if (!machine_name_is_valid(local)) { + log_error("Local name %s is not a suitable machine name.", local); + return -EINVAL; + } + + if (path) { + fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (fd < 0) + return log_error_errno(errno, "Failed to open %s: %m", path); + } + + r = sd_bus_message_new_method_call( + bus, + &m, + "org.freedesktop.import1", + "/org/freedesktop/import1", + "org.freedesktop.import1.Manager", + "ImportTar"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "hsbb", + fd >= 0 ? fd : STDIN_FILENO, + local, + arg_force, + arg_read_only); + if (r < 0) + return bus_log_create_error(r); + + return pull_image_common(bus, m); +} + +static int import_raw(int argc, char *argv[], void *userdata) { + _cleanup_bus_message_unref_ sd_bus_message *m = NULL; + _cleanup_free_ char *ll = NULL; + _cleanup_close_ int fd = -1; + const char *local = NULL, *path = NULL; + sd_bus *bus = userdata; + int r; + + assert(bus); + + if (argc >= 2) + path = argv[1]; + if (isempty(path) || streq(path, "-")) + path = NULL; + + if (argc >= 3) + local = argv[2]; + else if (path) + local = basename(path); + if (isempty(local) || streq(local, "-")) + local = NULL; + + if (!local) { + log_error("Need either path or local name."); + return -EINVAL; + } + + r = raw_strip_suffixes(local, &ll); + if (r < 0) + return log_oom(); + + local = ll; + + if (!machine_name_is_valid(local)) { + log_error("Local name %s is not a suitable machine name.", local); + return -EINVAL; + } + + if (path) { + fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (fd < 0) + return log_error_errno(errno, "Failed to open %s: %m", path); + } + + r = sd_bus_message_new_method_call( + bus, + &m, + "org.freedesktop.import1", + "/org/freedesktop/import1", + "org.freedesktop.import1.Manager", + "ImportRaw"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "hsbb", + fd >= 0 ? fd : STDIN_FILENO, + local, + arg_force, + arg_read_only); + if (r < 0) + return bus_log_create_error(r); + + return pull_image_common(bus, m); +} + static int pull_tar(int argc, char *argv[], void *userdata) { _cleanup_bus_message_unref_ sd_bus_message *m = NULL; _cleanup_free_ char *l = NULL, *ll = NULL; @@ -1652,7 +1786,7 @@ static int pull_tar(int argc, char *argv[], void *userdata) { if (local) { r = tar_strip_suffixes(local, &ll); if (r < 0) - return log_error_errno(r, "Failed to strip tar suffixes: %m"); + return log_oom(); local = ll; @@ -1716,7 +1850,7 @@ static int pull_raw(int argc, char *argv[], void *userdata) { if (local) { r = raw_strip_suffixes(local, &ll); if (r < 0) - return log_error_errno(r, "Failed to strip tar suffixes: %m"); + return log_oom(); local = ll; @@ -2065,6 +2199,8 @@ static int help(int argc, char *argv[], void *userdata) { " remove NAME... Remove an image\n" " set-limit [NAME] BYTES Set image or pool size limit (quota)\n\n" "Image Transfer Commands:\n" + " import-tar FILE [NAME] Import a local TAR container image\n" + " import-raw FILE [NAME] Import a local RAW container image\n" " pull-tar URL [NAME] Download a TAR container image\n" " pull-raw URL [NAME] Download a RAW container or VM image\n" " pull-dkr REMOTE [NAME] Download a DKR container image\n" @@ -2267,6 +2403,8 @@ static int machinectl_main(int argc, char *argv[], sd_bus *bus) { { "start", 2, VERB_ANY, 0, start_machine }, { "enable", 2, VERB_ANY, 0, enable_machine }, { "disable", 2, VERB_ANY, 0, enable_machine }, + { "import-tar", 2, 3, 0, import_tar }, + { "import-raw", 2, 3, 0, import_raw }, { "pull-tar", 2, 3, 0, pull_tar }, { "pull-raw", 2, 3, 0, pull_raw }, { "pull-dkr", 2, 3, 0, pull_dkr }, diff --git a/src/shared/machine-pool.h b/src/shared/machine-pool.h index 06c5d4058..fe01d3d47 100644 --- a/src/shared/machine-pool.h +++ b/src/shared/machine-pool.h @@ -23,5 +23,8 @@ #include "sd-bus.h" +/* Grow the /var/lib/machines directory after each 10MiB written */ +#define GROW_INTERVAL_BYTES (UINT64_C(10) * UINT64_C(1024) * UINT64_C(1024)) + int setup_machine_directory(uint64_t size, sd_bus_error *error); int grow_machine_directory(void); -- 2.30.2