From 587fec427c80b6c34dcf1d7570f891fcb652a7c5 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Mon, 9 Mar 2015 17:55:07 +0100 Subject: [PATCH] importd: add API for exporting container/VM images Also, expose it in machinectl. --- .gitignore | 1 + Makefile.am | 30 +- src/import/export-raw.c | 345 +++++++++++++++++++ src/import/export-raw.h | 37 ++ src/import/export-tar.c | 328 ++++++++++++++++++ src/import/export-tar.h | 37 ++ src/import/export.c | 319 +++++++++++++++++ src/import/import-common.c | 76 +++- src/import/import-common.h | 3 +- src/import/import-compress.c | 284 ++++++++++++++- src/import/import-compress.h | 7 +- src/import/import-raw.c | 9 +- src/import/import-tar.c | 15 +- src/import/import.c | 6 +- src/import/importd.c | 116 ++++++- src/import/org.freedesktop.import1.policy.in | 10 + src/import/pull-dkr.c | 2 +- src/import/pull-tar.c | 2 +- src/import/pull.c | 2 +- src/machine/machinectl.c | 151 +++++++- src/shared/btrfs-util.c | 53 +-- src/shared/btrfs-util.h | 2 + 22 files changed, 1770 insertions(+), 65 deletions(-) create mode 100644 src/import/export-raw.c create mode 100644 src/import/export-raw.h create mode 100644 src/import/export-tar.c create mode 100644 src/import/export-tar.h create mode 100644 src/import/export.c diff --git a/.gitignore b/.gitignore index 1308147f2..875ada5eb 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,7 @@ /systemd-efi-boot-generator /systemd-escape /systemd-evcat +/systemd-export /systemd-firstboot /systemd-fsck /systemd-fsckd diff --git a/Makefile.am b/Makefile.am index 42183f45d..3539e03c5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5371,7 +5371,8 @@ if HAVE_GCRYPT rootlibexec_PROGRAMS += \ systemd-importd \ systemd-pull \ - systemd-import + systemd-import \ + systemd-export systemd_importd_SOURCES = \ src/import/importd.c @@ -5379,7 +5380,8 @@ systemd_importd_SOURCES = \ systemd_importd_CFLAGS = \ $(AM_CFLAGS) \ -D SYSTEMD_PULL_PATH=\"$(rootlibexecdir)/systemd-pull\" \ - -D SYSTEMD_IMPORT_PATH=\"$(rootlibexecdir)/systemd-import\" + -D SYSTEMD_IMPORT_PATH=\"$(rootlibexecdir)/systemd-import\" \ + -D SYSTEMD_EXPORT_PATH=\"$(rootlibexecdir)/systemd-export\" systemd_importd_LDADD = \ libsystemd-internal.la \ @@ -5454,6 +5456,30 @@ systemd_import_LDADD = \ $(ZLIB_LIBS) \ -lbz2 +systemd_export_SOURCES = \ + src/import/export.c \ + src/import/export-tar.c \ + src/import/export-tar.h \ + src/import/export-raw.c \ + src/import/export-raw.h \ + src/import/import-common.c \ + src/import/import-common.h \ + src/import/import-compress.c \ + src/import/import-compress.h + +systemd_export_CFLAGS = \ + $(AM_CFLAGS) \ + $(XZ_CFLAGS) \ + $(ZLIB_CFLAGS) + +systemd_export_LDADD = \ + libsystemd-internal.la \ + libsystemd-label.la \ + libsystemd-shared.la \ + $(XZ_LIBS) \ + $(ZLIB_LIBS) \ + -lbz2 + dist_rootlibexec_DATA = \ src/import/import-pubring.gpg diff --git a/src/import/export-raw.c b/src/import/export-raw.c new file mode 100644 index 000000000..4b6d8dac3 --- /dev/null +++ b/src/import/export-raw.c @@ -0,0 +1,345 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include +#include +#undef basename + +#include "sd-daemon.h" +#include "util.h" +#include "ratelimit.h" +#include "btrfs-util.h" +#include "copy.h" +#include "import-common.h" +#include "export-raw.h" + +#define COPY_BUFFER_SIZE (16*1024) + +struct RawExport { + sd_event *event; + + RawExportFinished on_finished; + void *userdata; + + char *path; + + int input_fd; + int output_fd; + + ImportCompress compress; + + sd_event_source *output_event_source; + + void *buffer; + size_t buffer_size; + size_t buffer_allocated; + + uint64_t written_compressed; + uint64_t written_uncompressed; + + unsigned last_percent; + RateLimit progress_rate_limit; + + struct stat st; + + bool eof; + bool tried_reflink; + bool tried_sendfile; +}; + +RawExport *raw_export_unref(RawExport *e) { + if (!e) + return NULL; + + sd_event_source_unref(e->output_event_source); + + import_compress_free(&e->compress); + + sd_event_unref(e->event); + + safe_close(e->input_fd); + + free(e->buffer); + free(e->path); + free(e); + + return NULL; +} + +int raw_export_new( + RawExport **ret, + sd_event *event, + RawExportFinished on_finished, + void *userdata) { + + _cleanup_(raw_export_unrefp) RawExport *e = NULL; + int r; + + assert(ret); + + e = new0(RawExport, 1); + if (!e) + return -ENOMEM; + + e->output_fd = e->input_fd = -1; + e->on_finished = on_finished; + e->userdata = userdata; + + RATELIMIT_INIT(e->progress_rate_limit, 100 * USEC_PER_MSEC, 1); + e->last_percent = (unsigned) -1; + + if (event) + e->event = sd_event_ref(event); + else { + r = sd_event_default(&e->event); + if (r < 0) + return r; + } + + *ret = e; + e = NULL; + + return 0; +} + +static void raw_export_report_progress(RawExport *e) { + unsigned percent; + assert(e); + + if (e->written_uncompressed >= (uint64_t) e->st.st_size) + percent = 100; + else + percent = (unsigned) ((e->written_uncompressed * UINT64_C(100)) / (uint64_t) e->st.st_size); + + if (percent == e->last_percent) + return; + + if (!ratelimit_test(&e->progress_rate_limit)) + return; + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_info("Exported %u%%.", percent); + + e->last_percent = percent; +} + +static int raw_export_process(RawExport *e) { + ssize_t l; + int r; + + assert(e); + + if (!e->tried_reflink && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) { + + /* If we shall take an uncompressed snapshot we can + * reflink source to destination directly. Let's see + * if this works. */ + + r = btrfs_reflink(e->input_fd, e->output_fd); + if (r >= 0) { + r = 0; + goto finish; + } + + e->tried_reflink = true; + } + + if (!e->tried_sendfile && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) { + + l = sendfile(e->output_fd, e->input_fd, NULL, COPY_BUFFER_SIZE); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + e->tried_sendfile = true; + } else if (l == 0) { + r = 0; + goto finish; + } else { + e->written_uncompressed += l; + e->written_compressed += l; + + raw_export_report_progress(e); + + return 0; + } + } + + while (e->buffer_size <= 0) { + uint8_t input[COPY_BUFFER_SIZE]; + + if (e->eof) { + r = 0; + goto finish; + } + + l = read(e->input_fd, input, sizeof(input)); + if (l < 0) { + r = log_error_errno(errno, "Failed to read raw file: %m"); + goto finish; + } + + if (l == 0) { + e->eof = true; + r = import_compress_finish(&e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated); + } else { + e->written_uncompressed += l; + r = import_compress(&e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated); + } + if (r < 0) { + r = log_error_errno(r, "Failed to encode: %m"); + goto finish; + } + } + + l = write(e->output_fd, e->buffer, e->buffer_size); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + r = log_error_errno(errno, "Failed to write output file: %m"); + goto finish; + } + + assert((size_t) l <= e->buffer_size); + memmove(e->buffer, (uint8_t*) e->buffer + l, e->buffer_size - l); + e->buffer_size -= l; + e->written_compressed += l; + + raw_export_report_progress(e); + + return 0; + +finish: + if (r >= 0) { + (void) copy_times(e->input_fd, e->output_fd); + (void) copy_xattr(e->input_fd, e->output_fd); + } + + if (e->on_finished) + e->on_finished(e, r, e->userdata); + else + sd_event_exit(e->event, r); + + return 0; +} + +static int raw_export_on_output(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + RawExport *i = userdata; + + return raw_export_process(i); +} + +static int raw_export_on_defer(sd_event_source *s, void *userdata) { + RawExport *i = userdata; + + return raw_export_process(i); +} + +static int reflink_snapshot(int fd, const char *path) { + char *p, *d; + int new_fd, r; + + p = strdupa(path); + d = dirname(p); + + new_fd = open(d, O_TMPFILE|O_CLOEXEC|O_NOCTTY|O_RDWR, 0600); + if (new_fd < 0) { + _cleanup_free_ char *t = NULL; + + r = tempfn_random(path, &t); + if (r < 0) + return r; + + new_fd = open(t, O_CLOEXEC|O_CREAT|O_NOCTTY|O_RDWR, 0600); + if (new_fd < 0) + return -errno; + + (void) unlink(t); + } + + r = btrfs_reflink(fd, new_fd); + if (r < 0) { + safe_close(new_fd); + return r; + } + + return new_fd; +} + +int raw_export_start(RawExport *e, const char *path, int fd, ImportCompressType compress) { + _cleanup_close_ int sfd = -1, tfd = -1; + int r; + + assert(e); + assert(path); + assert(fd >= 0); + assert(compress < _IMPORT_COMPRESS_TYPE_MAX); + assert(compress != IMPORT_COMPRESS_UNKNOWN); + + if (e->output_fd >= 0) + return -EBUSY; + + r = fd_nonblock(fd, true); + if (r < 0) + return r; + + r = free_and_strdup(&e->path, path); + if (r < 0) + return r; + + sfd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (sfd < 0) + return -errno; + + if (fstat(sfd, &e->st) < 0) + return -errno; + if (!S_ISREG(e->st.st_mode)) + return -ENOTTY; + + /* Try to take a reflink snapshot of the file, if we can t make the export atomic */ + tfd = reflink_snapshot(sfd, path); + if (tfd >= 0) { + e->input_fd = tfd; + tfd = -1; + } else { + e->input_fd = sfd; + sfd = -1; + } + + r = import_compress_init(&e->compress, compress); + if (r < 0) + return r; + + r = sd_event_add_io(e->event, &e->output_event_source, fd, EPOLLOUT, raw_export_on_output, e); + if (r == -EPERM) { + r = sd_event_add_defer(e->event, &e->output_event_source, raw_export_on_defer, e); + if (r < 0) + return r; + + r = sd_event_source_set_enabled(e->output_event_source, SD_EVENT_ON); + } + if (r < 0) + return r; + + e->output_fd = fd; + return r; +} diff --git a/src/import/export-raw.h b/src/import/export-raw.h new file mode 100644 index 000000000..b71de6cb8 --- /dev/null +++ b/src/import/export-raw.h @@ -0,0 +1,37 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +#pragma once + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include "sd-event.h" +#include "macro.h" +#include "import-compress.h" + +typedef struct RawExport RawExport; + +typedef void (*RawExportFinished)(RawExport *export, int error, void *userdata); + +int raw_export_new(RawExport **export, sd_event *event, RawExportFinished on_finished, void *userdata); +RawExport* raw_export_unref(RawExport *export); + +DEFINE_TRIVIAL_CLEANUP_FUNC(RawExport*, raw_export_unref); + +int raw_export_start(RawExport *export, const char *path, int fd, ImportCompressType compress); diff --git a/src/import/export-tar.c b/src/import/export-tar.c new file mode 100644 index 000000000..80de83896 --- /dev/null +++ b/src/import/export-tar.c @@ -0,0 +1,328 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include + +#include "sd-daemon.h" +#include "util.h" +#include "ratelimit.h" +#include "btrfs-util.h" +#include "import-common.h" +#include "export-tar.h" + +#define COPY_BUFFER_SIZE (16*1024) + +struct TarExport { + sd_event *event; + + TarExportFinished on_finished; + void *userdata; + + char *path; + char *temp_path; + + int output_fd; + int tar_fd; + + ImportCompress compress; + + sd_event_source *output_event_source; + + void *buffer; + size_t buffer_size; + size_t buffer_allocated; + + uint64_t written_compressed; + uint64_t written_uncompressed; + + pid_t tar_pid; + + struct stat st; + uint64_t quota_referenced; + + unsigned last_percent; + RateLimit progress_rate_limit; + + bool eof; + bool tried_splice; +}; + +TarExport *tar_export_unref(TarExport *e) { + if (!e) + return NULL; + + sd_event_source_unref(e->output_event_source); + + if (e->tar_pid > 1) { + (void) kill_and_sigcont(e->tar_pid, SIGKILL); + (void) wait_for_terminate(e->tar_pid, NULL); + } + + if (e->temp_path) { + (void) btrfs_subvol_remove(e->temp_path); + free(e->temp_path); + } + + import_compress_free(&e->compress); + + sd_event_unref(e->event); + + safe_close(e->tar_fd); + + free(e->buffer); + free(e->path); + free(e); + + return NULL; +} + +int tar_export_new( + TarExport **ret, + sd_event *event, + TarExportFinished on_finished, + void *userdata) { + + _cleanup_(tar_export_unrefp) TarExport *e = NULL; + int r; + + assert(ret); + + e = new0(TarExport, 1); + if (!e) + return -ENOMEM; + + e->output_fd = e->tar_fd = -1; + e->on_finished = on_finished; + e->userdata = userdata; + e->quota_referenced = (uint64_t) -1; + + RATELIMIT_INIT(e->progress_rate_limit, 100 * USEC_PER_MSEC, 1); + e->last_percent = (unsigned) -1; + + if (event) + e->event = sd_event_ref(event); + else { + r = sd_event_default(&e->event); + if (r < 0) + return r; + } + + *ret = e; + e = NULL; + + return 0; +} + +static void tar_export_report_progress(TarExport *e) { + unsigned percent; + assert(e); + + /* Do we have any quota info? I fnot, we don't know anything about the progress */ + if (e->quota_referenced == (uint64_t) -1) + return; + + if (e->written_uncompressed >= e->quota_referenced) + percent = 100; + else + percent = (unsigned) ((e->written_uncompressed * UINT64_C(100)) / e->quota_referenced); + + if (percent == e->last_percent) + return; + + if (!ratelimit_test(&e->progress_rate_limit)) + return; + + sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); + log_info("Exported %u%%.", percent); + + e->last_percent = percent; +} + +static int tar_export_process(TarExport *e) { + ssize_t l; + int r; + + assert(e); + + if (!e->tried_splice && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) { + + l = splice(e->tar_fd, NULL, e->output_fd, NULL, COPY_BUFFER_SIZE, 0); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + e->tried_splice = true; + } else if (l == 0) { + r = 0; + goto finish; + } else { + e->written_uncompressed += l; + e->written_compressed += l; + + tar_export_report_progress(e); + + return 0; + } + } + + while (e->buffer_size <= 0) { + uint8_t input[COPY_BUFFER_SIZE]; + + if (e->eof) { + r = 0; + goto finish; + } + + l = read(e->tar_fd, input, sizeof(input)); + if (l < 0) { + r = log_error_errno(errno, "Failed to read tar file: %m"); + goto finish; + } + + if (l == 0) { + e->eof = true; + r = import_compress_finish(&e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated); + } else { + e->written_uncompressed += l; + r = import_compress(&e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated); + } + if (r < 0) { + r = log_error_errno(r, "Failed to encode: %m"); + goto finish; + } + } + + l = write(e->output_fd, e->buffer, e->buffer_size); + if (l < 0) { + if (errno == EAGAIN) + return 0; + + r = log_error_errno(errno, "Failed to write output file: %m"); + goto finish; + } + + assert((size_t) l <= e->buffer_size); + memmove(e->buffer, (uint8_t*) e->buffer + l, e->buffer_size - l); + e->buffer_size -= l; + e->written_compressed += l; + + tar_export_report_progress(e); + + return 0; + +finish: + if (e->on_finished) + e->on_finished(e, r, e->userdata); + else + sd_event_exit(e->event, r); + + return 0; +} + +static int tar_export_on_output(sd_event_source *s, int fd, uint32_t revents, void *userdata) { + TarExport *i = userdata; + + return tar_export_process(i); +} + +static int tar_export_on_defer(sd_event_source *s, void *userdata) { + TarExport *i = userdata; + + return tar_export_process(i); +} + +int tar_export_start(TarExport *e, const char *path, int fd, ImportCompressType compress) { + _cleanup_close_ int sfd = -1; + int r; + + assert(e); + assert(path); + assert(fd >= 0); + assert(compress < _IMPORT_COMPRESS_TYPE_MAX); + assert(compress != IMPORT_COMPRESS_UNKNOWN); + + if (e->output_fd >= 0) + return -EBUSY; + + sfd = open(path, O_DIRECTORY|O_RDONLY|O_NOCTTY|O_CLOEXEC); + if (sfd < 0) + return -errno; + + if (fstat(sfd, &e->st) < 0) + return -errno; + + r = fd_nonblock(fd, true); + if (r < 0) + return r; + + r = free_and_strdup(&e->path, path); + if (r < 0) + return r; + + e->quota_referenced = (uint64_t) -1; + + if (e->st.st_ino == 256) { /* might be a btrfs subvolume? */ + BtrfsQuotaInfo q; + + r = btrfs_subvol_get_quota_fd(sfd, &q); + if (r >= 0) + e->quota_referenced = q.referred; + + free(e->temp_path); + e->temp_path = NULL; + + r = tempfn_random(path, &e->temp_path); + if (r < 0) + return r; + + /* Let's try to make a snapshot, if we can, so that the export is atomic */ + r = btrfs_subvol_snapshot_fd(sfd, e->temp_path, true, false); + if (r < 0) { + log_debug_errno(r, "Couldn't create snapshot %s of %s, not exporting atomically: %m", e->temp_path, path); + free(e->temp_path); + e->temp_path = NULL; + } + } + + r = import_compress_init(&e->compress, compress); + if (r < 0) + return r; + + r = sd_event_add_io(e->event, &e->output_event_source, fd, EPOLLOUT, tar_export_on_output, e); + if (r == -EPERM) { + r = sd_event_add_defer(e->event, &e->output_event_source, tar_export_on_defer, e); + if (r < 0) + return r; + + r = sd_event_source_set_enabled(e->output_event_source, SD_EVENT_ON); + } + if (r < 0) + return r; + + e->tar_fd = import_fork_tar_c(e->temp_path ?: e->path, &e->tar_pid); + if (e->tar_fd < 0) { + e->output_event_source = sd_event_source_unref(e->output_event_source); + return e->tar_fd; + } + + e->output_fd = fd; + return r; +} diff --git a/src/import/export-tar.h b/src/import/export-tar.h new file mode 100644 index 000000000..ce27a9fc1 --- /dev/null +++ b/src/import/export-tar.h @@ -0,0 +1,37 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +#pragma once + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include "sd-event.h" +#include "macro.h" +#include "import-compress.h" + +typedef struct TarExport TarExport; + +typedef void (*TarExportFinished)(TarExport *export, int error, void *userdata); + +int tar_export_new(TarExport **export, sd_event *event, TarExportFinished on_finished, void *userdata); +TarExport* tar_export_unref(TarExport *export); + +DEFINE_TRIVIAL_CLEANUP_FUNC(TarExport*, tar_export_unref); + +int tar_export_start(TarExport *export, const char *path, int fd, ImportCompressType compress); diff --git a/src/import/export.c b/src/import/export.c new file mode 100644 index 000000000..201c5ab35 --- /dev/null +++ b/src/import/export.c @@ -0,0 +1,319 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include + +#include "sd-event.h" +#include "event-util.h" +#include "verbs.h" +#include "build.h" +#include "machine-image.h" +#include "import-util.h" +#include "export-tar.h" +#include "export-raw.h" + +static ImportCompressType arg_compress = IMPORT_COMPRESS_UNKNOWN; + +static void determine_compression_from_filename(const char *p) { + + if (arg_compress != IMPORT_COMPRESS_UNKNOWN) + return; + + if (!p) { + arg_compress = IMPORT_COMPRESS_UNCOMPRESSED; + return; + } + + if (endswith(p, ".xz")) + arg_compress = IMPORT_COMPRESS_XZ; + else if (endswith(p, ".gz")) + arg_compress = IMPORT_COMPRESS_GZIP; + else if (endswith(p, ".bz2")) + arg_compress = IMPORT_COMPRESS_BZIP2; + else + arg_compress = IMPORT_COMPRESS_UNCOMPRESSED; +} + +static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { + log_notice("Transfer aborted."); + sd_event_exit(sd_event_source_get_event(s), EINTR); + return 0; +} + +static void on_tar_finished(TarExport *export, int error, void *userdata) { + sd_event *event = userdata; + assert(export); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int export_tar(int argc, char *argv[], void *userdata) { + _cleanup_(tar_export_unrefp) TarExport *export = NULL; + _cleanup_event_unref_ sd_event *event = NULL; + _cleanup_(image_unrefp) Image *image = NULL; + const char *path = NULL, *local = NULL; + _cleanup_close_ int open_fd = -1; + int r, fd; + + if (machine_name_is_valid(argv[1])) { + r = image_find(argv[1], &image); + if (r < 0) + return log_error_errno(r, "Failed to look for machine %s: %m", argv[1]); + if (r == 0) { + log_error("Machine image %s not found.", argv[1]); + return -ENOENT; + } + + local = image->path; + } else + local = argv[1]; + + if (argc >= 3) + path = argv[2]; + if (isempty(path) || streq(path, "-")) + path = NULL; + + determine_compression_from_filename(path); + + if (path) { + open_fd = open(path, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOCTTY, 0666); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open tar image for export: %m"); + + fd = open_fd; + + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, import_compress_type_to_string(arg_compress)); + } else { + _cleanup_free_ char *pretty = NULL; + + fd = STDOUT_FILENO; + + (void) readlink_malloc("/proc/self/fd/1", &pretty); + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), import_compress_type_to_string(arg_compress)); + } + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, SIGTERM, SIGINT, -1) == 0); + sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + r = tar_export_new(&export, event, on_tar_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate exporter: %m"); + + r = tar_export_start(export, local, fd, arg_compress); + if (r < 0) + return log_error_errno(r, "Failed to export image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static void on_raw_finished(RawExport *export, int error, void *userdata) { + sd_event *event = userdata; + assert(export); + + if (error == 0) + log_info("Operation completed successfully."); + + sd_event_exit(event, abs(error)); +} + +static int export_raw(int argc, char *argv[], void *userdata) { + _cleanup_(raw_export_unrefp) RawExport *export = NULL; + _cleanup_event_unref_ sd_event *event = NULL; + _cleanup_(image_unrefp) Image *image = NULL; + const char *path = NULL, *local = NULL; + _cleanup_close_ int open_fd = -1; + int r, fd; + + if (machine_name_is_valid(argv[1])) { + r = image_find(argv[1], &image); + if (r < 0) + return log_error_errno(r, "Failed to look for machine %s: %m", argv[1]); + if (r == 0) { + log_error("Machine image %s not found.", argv[1]); + return -ENOENT; + } + + local = image->path; + } else + local = argv[1]; + + if (argc >= 3) + path = argv[2]; + if (isempty(path) || streq(path, "-")) + path = NULL; + + determine_compression_from_filename(path); + + if (path) { + open_fd = open(path, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOCTTY, 0666); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open raw image for export: %m"); + + fd = open_fd; + + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, import_compress_type_to_string(arg_compress)); + } else { + _cleanup_free_ char *pretty = NULL; + + fd = STDOUT_FILENO; + + (void) readlink_malloc("/proc/self/fd/1", &pretty); + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), import_compress_type_to_string(arg_compress)); + } + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, SIGTERM, SIGINT, -1) == 0); + sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + r = raw_export_new(&export, event, on_raw_finished, event); + if (r < 0) + return log_error_errno(r, "Failed to allocate exporter: %m"); + + r = raw_export_start(export, local, fd, arg_compress); + if (r < 0) + return log_error_errno(r, "Failed to export image: %m"); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + log_info("Exiting."); + return -r; +} + +static int help(int argc, char *argv[], void *userdata) { + + printf("%s [OPTIONS...] {COMMAND} ...\n\n" + "Export container or virtual machine images.\n\n" + " -h --help Show this help\n" + " --version Show package version\n" + " --format=FORMAT Select format\n\n" + "Commands:\n" + " tar NAME [FILE] Export a TAR image\n" + " raw NAME [FILE] Export a RAW image\n", + program_invocation_short_name); + + return 0; +} + +static int parse_argv(int argc, char *argv[]) { + + enum { + ARG_VERSION = 0x100, + ARG_FORMAT, + }; + + static const struct option options[] = { + { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, ARG_VERSION }, + { "format", required_argument, NULL, ARG_FORMAT }, + {} + }; + + int c; + + assert(argc >= 0); + assert(argv); + + while ((c = getopt_long(argc, argv, "h", options, NULL)) >= 0) + + switch (c) { + + case 'h': + return help(0, NULL, NULL); + + case ARG_VERSION: + puts(PACKAGE_STRING); + puts(SYSTEMD_FEATURES); + return 0; + + case ARG_FORMAT: + if (streq(optarg, "uncompressed")) + arg_compress = IMPORT_COMPRESS_UNCOMPRESSED; + else if (streq(optarg, "xz")) + arg_compress = IMPORT_COMPRESS_XZ; + else if (streq(optarg, "gzip")) + arg_compress = IMPORT_COMPRESS_GZIP; + else if (streq(optarg, "bzip2")) + arg_compress = IMPORT_COMPRESS_BZIP2; + else { + log_error("Unknown format: %s", optarg); + return -EINVAL; + } + break; + + case '?': + return -EINVAL; + + default: + assert_not_reached("Unhandled option"); + } + + return 1; +} + +static int export_main(int argc, char *argv[]) { + + static const Verb verbs[] = { + { "help", VERB_ANY, VERB_ANY, 0, help }, + { "tar", 2, 3, 0, export_tar }, + { "raw", 2, 3, 0, export_raw }, + {} + }; + + return dispatch_verb(argc, argv, verbs, NULL); +} + +int main(int argc, char *argv[]) { + int r; + + setlocale(LC_ALL, ""); + log_parse_environment(); + log_open(); + + r = parse_argv(argc, argv); + if (r <= 0) + goto finish; + + ignore_signals(SIGPIPE, -1); + + r = export_main(argc, argv); + +finish: + return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/src/import/import-common.c b/src/import/import-common.c index 6c3f347e7..aede2f9b3 100644 --- a/src/import/import-common.c +++ b/src/import/import-common.c @@ -69,7 +69,7 @@ int import_make_read_only(const char *path) { return import_make_read_only_fd(fd); } -int import_fork_tar(const char *path, pid_t *ret) { +int import_fork_tar_x(const char *path, pid_t *ret) { _cleanup_close_pair_ int pipefd[2] = { -1, -1 }; pid_t pid; int r; @@ -148,3 +148,77 @@ int import_fork_tar(const char *path, pid_t *ret) { return r; } + +int import_fork_tar_c(const char *path, pid_t *ret) { + _cleanup_close_pair_ int pipefd[2] = { -1, -1 }; + pid_t pid; + int r; + + assert(path); + assert(ret); + + if (pipe2(pipefd, O_CLOEXEC) < 0) + return log_error_errno(errno, "Failed to create pipe for tar: %m"); + + pid = fork(); + if (pid < 0) + return log_error_errno(errno, "Failed to fork off tar: %m"); + + if (pid == 0) { + int null_fd; + uint64_t retain = (1ULL << CAP_DAC_OVERRIDE); + + /* Child */ + + reset_all_signal_handlers(); + reset_signal_mask(); + assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0); + + pipefd[0] = safe_close(pipefd[0]); + + if (dup2(pipefd[1], STDOUT_FILENO) != STDOUT_FILENO) { + log_error_errno(errno, "Failed to dup2() fd: %m"); + _exit(EXIT_FAILURE); + } + + if (pipefd[1] != STDOUT_FILENO) + pipefd[1] = safe_close(pipefd[1]); + + null_fd = open("/dev/null", O_RDONLY|O_NOCTTY); + if (null_fd < 0) { + log_error_errno(errno, "Failed to open /dev/null: %m"); + _exit(EXIT_FAILURE); + } + + if (dup2(null_fd, STDIN_FILENO) != STDIN_FILENO) { + log_error_errno(errno, "Failed to dup2() fd: %m"); + _exit(EXIT_FAILURE); + } + + if (null_fd != STDIN_FILENO) + null_fd = safe_close(null_fd); + + fd_cloexec(STDIN_FILENO, false); + fd_cloexec(STDOUT_FILENO, false); + fd_cloexec(STDERR_FILENO, false); + + if (unshare(CLONE_NEWNET) < 0) + log_error_errno(errno, "Failed to lock tar into network namespace, ignoring: %m"); + + r = capability_bounding_set_drop(~retain, true); + if (r < 0) + log_error_errno(r, "Failed to drop capabilities, ignoring: %m"); + + execlp("tar", "tar", "--sparse", "-C", path, "-c", ".", NULL); + log_error_errno(errno, "Failed to execute tar: %m"); + _exit(EXIT_FAILURE); + } + + pipefd[1] = safe_close(pipefd[1]); + r = pipefd[0]; + pipefd[0] = -1; + + *ret = pid; + + return r; +} diff --git a/src/import/import-common.h b/src/import/import-common.h index 639ea1799..7b60de80c 100644 --- a/src/import/import-common.h +++ b/src/import/import-common.h @@ -24,4 +24,5 @@ int import_make_read_only_fd(int fd); int import_make_read_only(const char *path); -int import_fork_tar(const char *path, pid_t *ret); +int import_fork_tar_c(const char *path, pid_t *ret); +int import_fork_tar_x(const char *path, pid_t *ret); diff --git a/src/import/import-compress.c b/src/import/import-compress.c index 629605a44..fa3f162bf 100644 --- a/src/import/import-compress.c +++ b/src/import/import-compress.c @@ -27,10 +27,17 @@ void import_compress_free(ImportCompress *c) { if (c->type == IMPORT_COMPRESS_XZ) lzma_end(&c->xz); - else if (c->type == IMPORT_COMPRESS_GZIP) - inflateEnd(&c->gzip); - else if (c->type == IMPORT_COMPRESS_BZIP2) - BZ2_bzDecompressEnd(&c->bzip2); + else if (c->type == IMPORT_COMPRESS_GZIP) { + if (c->encoding) + deflateEnd(&c->gzip); + else + inflateEnd(&c->gzip); + } else if (c->type == IMPORT_COMPRESS_BZIP2) { + if (c->encoding) + BZ2_bzCompressEnd(&c->bzip2); + else + BZ2_bzDecompressEnd(&c->bzip2); + } c->type = IMPORT_COMPRESS_UNKNOWN; } @@ -85,6 +92,8 @@ int import_uncompress_detect(ImportCompress *c, const void *data, size_t size) { } else c->type = IMPORT_COMPRESS_UNCOMPRESSED; + c->encoding = false; + return 1; } @@ -98,6 +107,9 @@ int import_uncompress(ImportCompress *c, const void *data, size_t size, ImportCo if (r <= 0) return r; + if (c->encoding) + return -EINVAL; + if (size <= 0) return 1; @@ -183,6 +195,270 @@ int import_uncompress(ImportCompress *c, const void *data, size_t size, ImportCo return 1; } +int import_compress_init(ImportCompress *c, ImportCompressType t) { + int r; + + assert(c); + + switch (t) { + + case IMPORT_COMPRESS_XZ: { + lzma_ret xzr; + + xzr = lzma_easy_encoder(&c->xz, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64); + if (xzr != LZMA_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_XZ; + break; + } + + case IMPORT_COMPRESS_GZIP: + r = deflateInit2(&c->gzip, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY); + if (r != Z_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_GZIP; + break; + + case IMPORT_COMPRESS_BZIP2: + r = BZ2_bzCompressInit(&c->bzip2, 9, 0, 0); + if (r != BZ_OK) + return -EIO; + + c->type = IMPORT_COMPRESS_BZIP2; + break; + + case IMPORT_COMPRESS_UNCOMPRESSED: + c->type = IMPORT_COMPRESS_UNCOMPRESSED; + break; + + default: + return -ENOTSUP; + } + + c->encoding = true; + return 0; +} + +static int enlarge_buffer(void **buffer, size_t *buffer_size, size_t *buffer_allocated) { + size_t l; + void *p; + + if (*buffer_allocated > *buffer_size) + return 0; + + l = MAX(16*1024U, (*buffer_size * 2)); + p = realloc(*buffer, l); + if (!p) + return -ENOMEM; + + *buffer = p; + *buffer_allocated = l; + + return 1; +} + +int import_compress(ImportCompress *c, const void *data, size_t size, void **buffer, size_t *buffer_size, size_t *buffer_allocated) { + int r; + + assert(c); + assert(buffer); + assert(buffer_size); + assert(buffer_allocated); + + if (!c->encoding) + return -EINVAL; + + if (size <= 0) + return 0; + + assert(data); + + *buffer_size = 0; + + switch (c->type) { + + case IMPORT_COMPRESS_XZ: + + c->xz.next_in = data; + c->xz.avail_in = size; + + while (c->xz.avail_in > 0) { + lzma_ret lzr; + + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->xz.next_out = (uint8_t*) *buffer + *buffer_size; + c->xz.avail_out = *buffer_allocated - *buffer_size; + + lzr = lzma_code(&c->xz, LZMA_RUN); + if (lzr != LZMA_OK) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out; + } + + break; + + case IMPORT_COMPRESS_GZIP: + + c->gzip.next_in = (void*) data; + c->gzip.avail_in = size; + + while (c->gzip.avail_in > 0) { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->gzip.next_out = (uint8_t*) *buffer + *buffer_size; + c->gzip.avail_out = *buffer_allocated - *buffer_size; + + r = deflate(&c->gzip, Z_NO_FLUSH); + if (r != Z_OK) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->gzip.avail_out; + } + + break; + + case IMPORT_COMPRESS_BZIP2: + + c->bzip2.next_in = (void*) data; + c->bzip2.avail_in = size; + + while (c->bzip2.avail_in > 0) { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size); + c->bzip2.avail_out = *buffer_allocated - *buffer_size; + + r = BZ2_bzCompress(&c->bzip2, BZ_RUN); + if (r != BZ_RUN_OK) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->bzip2.avail_out; + } + + break; + + case IMPORT_COMPRESS_UNCOMPRESSED: + + if (*buffer_allocated < size) { + void *p; + + p = realloc(*buffer, size); + if (!p) + return -ENOMEM; + + *buffer = p; + *buffer_allocated = size; + } + + memcpy(*buffer, data, size); + *buffer_size = size; + break; + + default: + return -ENOTSUP; + } + + return 0; +} + +int import_compress_finish(ImportCompress *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated) { + int r; + + assert(c); + assert(buffer); + assert(buffer_size); + assert(buffer_allocated); + + if (!c->encoding) + return -EINVAL; + + *buffer_size = 0; + + switch (c->type) { + + case IMPORT_COMPRESS_XZ: { + lzma_ret lzr; + + c->xz.avail_in = 0; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->xz.next_out = (uint8_t*) *buffer + *buffer_size; + c->xz.avail_out = *buffer_allocated - *buffer_size; + + lzr = lzma_code(&c->xz, LZMA_FINISH); + if (lzr != LZMA_OK && lzr != LZMA_STREAM_END) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out; + } while (lzr != LZMA_STREAM_END); + + break; + } + + case IMPORT_COMPRESS_GZIP: + c->gzip.avail_in = 0; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->gzip.next_out = (uint8_t*) *buffer + *buffer_size; + c->gzip.avail_out = *buffer_allocated - *buffer_size; + + r = deflate(&c->gzip, Z_FINISH); + if (r != Z_OK && r != Z_STREAM_END) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->gzip.avail_out; + } while (r != Z_STREAM_END); + + break; + + case IMPORT_COMPRESS_BZIP2: + c->bzip2.avail_in = 0; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size); + c->bzip2.avail_out = *buffer_allocated - *buffer_size; + + r = BZ2_bzCompress(&c->bzip2, BZ_FINISH); + if (r != BZ_FINISH_OK && r != BZ_STREAM_END) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->bzip2.avail_out; + } while (r != BZ_STREAM_END); + + break; + + case IMPORT_COMPRESS_UNCOMPRESSED: + break; + + default: + return -ENOTSUP; + } + + return 0; +} + static const char* const import_compress_type_table[_IMPORT_COMPRESS_TYPE_MAX] = { [IMPORT_COMPRESS_UNKNOWN] = "unknown", [IMPORT_COMPRESS_UNCOMPRESSED] = "uncompressed", diff --git a/src/import/import-compress.h b/src/import/import-compress.h index 4f97f20b4..50d91f732 100644 --- a/src/import/import-compress.h +++ b/src/import/import-compress.h @@ -41,7 +41,7 @@ typedef enum ImportCompressType { typedef struct ImportCompress { ImportCompressType type; - + bool encoding; union { lzma_stream xz; z_stream gzip; @@ -54,8 +54,11 @@ typedef int (*ImportCompressCallback)(const void *data, size_t size, void *userd void import_compress_free(ImportCompress *c); int import_uncompress_detect(ImportCompress *c, const void *data, size_t size); - int import_uncompress(ImportCompress *c, const void *data, size_t size, ImportCompressCallback callback, void *userdata); +int import_compress_init(ImportCompress *c, ImportCompressType t); +int import_compress(ImportCompress *c, const void *data, size_t size, void **buffer, size_t *buffer_size, size_t *buffer_allocated); +int import_compress_finish(ImportCompress *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated); + const char* import_compress_type_to_string(ImportCompressType t) _const_; ImportCompressType import_compress_type_from_string(const char *s) _pure_; diff --git a/src/import/import-raw.c b/src/import/import-raw.c index 15e5eb2ca..25b52f7cb 100644 --- a/src/import/import-raw.c +++ b/src/import/import-raw.c @@ -117,7 +117,7 @@ int raw_import_new( i->on_finished = on_finished; i->userdata = userdata; - RATELIMIT_INIT(i->progress_rate_limit, 500 * USEC_PER_MSEC, 1); + RATELIMIT_INIT(i->progress_rate_limit, 100 * USEC_PER_MSEC, 1); i->last_percent = (unsigned) -1; i->image_root = strdup(image_root ?: "/var/lib/machines"); @@ -343,6 +343,9 @@ static int raw_import_process(RawImport *i) { l = read(i->input_fd, i->buffer + i->buffer_size, sizeof(i->buffer) - i->buffer_size); if (l < 0) { + if (errno == EAGAIN) + return 0; + r = log_error_errno(errno, "Failed to read input file: %m"); goto finish; } @@ -428,6 +431,10 @@ int raw_import_start(RawImport *i, int fd, const char *local, bool force_local, if (i->input_fd >= 0) return -EBUSY; + r = fd_nonblock(fd, true); + if (r < 0) + return r; + r = free_and_strdup(&i->local, local); if (r < 0) return r; diff --git a/src/import/import-tar.c b/src/import/import-tar.c index d5b6daddd..dd9557566 100644 --- a/src/import/import-tar.c +++ b/src/import/import-tar.c @@ -78,7 +78,7 @@ TarImport* tar_import_unref(TarImport *i) { if (!i) return NULL; - sd_event_unref(i->event); + sd_event_source_unref(i->input_event_source); if (i->tar_pid > 1) { (void) kill_and_sigcont(i->tar_pid, SIGKILL); @@ -93,7 +93,7 @@ TarImport* tar_import_unref(TarImport *i) { import_compress_free(&i->compress); - sd_event_source_unref(i->input_event_source); + sd_event_unref(i->event); safe_close(i->tar_fd); @@ -125,7 +125,7 @@ int tar_import_new( i->on_finished = on_finished; i->userdata = userdata; - RATELIMIT_INIT(i->progress_rate_limit, 500 * USEC_PER_MSEC, 1); + RATELIMIT_INIT(i->progress_rate_limit, 100 * USEC_PER_MSEC, 1); i->last_percent = (unsigned) -1; i->image_root = strdup(image_root ?: "/var/lib/machines"); @@ -236,7 +236,7 @@ static int tar_import_fork_tar(TarImport *i) { } else if (r < 0) return log_error_errno(errno, "Failed to create subvolume %s: %m", i->temp_path); - i->tar_fd = import_fork_tar(i->temp_path, &i->tar_pid); + i->tar_fd = import_fork_tar_x(i->temp_path, &i->tar_pid); if (i->tar_fd < 0) return i->tar_fd; @@ -271,6 +271,9 @@ static int tar_import_process(TarImport *i) { l = read(i->input_fd, i->buffer + i->buffer_size, sizeof(i->buffer) - i->buffer_size); if (l < 0) { + if (errno == EAGAIN) + return 0; + r = log_error_errno(errno, "Failed to read input file: %m"); goto finish; } @@ -348,6 +351,10 @@ int tar_import_start(TarImport *i, int fd, const char *local, bool force_local, if (i->input_fd >= 0) return -EBUSY; + r = fd_nonblock(fd, true); + if (r < 0) + return r; + r = free_and_strdup(&i->local, local); if (r < 0) return r; diff --git a/src/import/import.c b/src/import/import.c index 762c425e6..f3072b377 100644 --- a/src/import/import.c +++ b/src/import/import.c @@ -233,15 +233,15 @@ static int import_raw(int argc, char *argv[], void *userdata) { static int help(int argc, char *argv[], void *userdata) { printf("%s [OPTIONS...] {COMMAND} ...\n\n" - "Import container or virtual machine image.\n\n" + "Import container or virtual machine images.\n\n" " -h --help Show this help\n" " --version Show package version\n" " --force Force creation of image\n" " --image-root=PATH Image root directory\n" " --read-only Create a read-only image\n\n" "Commands:\n" - " tar FILE [NAME] Download a TAR image\n" - " raw FILE [NAME] Download a RAW image\n", + " tar FILE [NAME] Import a TAR image\n" + " raw FILE [NAME] Import a RAW image\n", program_invocation_short_name); return 0; diff --git a/src/import/importd.c b/src/import/importd.c index e2ff9f716..3e70fe733 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -41,6 +41,8 @@ typedef struct Manager Manager; typedef enum TransferType { TRANSFER_IMPORT_TAR, TRANSFER_IMPORT_RAW, + TRANSFER_EXPORT_TAR, + TRANSFER_EXPORT_RAW, TRANSFER_PULL_TAR, TRANSFER_PULL_RAW, TRANSFER_PULL_DKR, @@ -63,6 +65,7 @@ struct Transfer { bool read_only; char *dkr_index_url; + char *format; pid_t pid; @@ -78,6 +81,7 @@ struct Transfer { unsigned progress_percent; int stdin_fd; + int stdout_fd; }; struct Manager { @@ -99,6 +103,8 @@ struct Manager { static const char* const transfer_type_table[_TRANSFER_TYPE_MAX] = { [TRANSFER_IMPORT_TAR] = "import-tar", [TRANSFER_IMPORT_RAW] = "import-raw", + [TRANSFER_EXPORT_TAR] = "export-tar", + [TRANSFER_EXPORT_RAW] = "export-raw", [TRANSFER_PULL_TAR] = "pull-tar", [TRANSFER_PULL_RAW] = "pull-raw", [TRANSFER_PULL_DKR] = "pull-dkr", @@ -119,6 +125,7 @@ static Transfer *transfer_unref(Transfer *t) { free(t->remote); free(t->local); free(t->dkr_index_url); + free(t->format); free(t->object_path); if (t->pid > 0) { @@ -128,6 +135,7 @@ static Transfer *transfer_unref(Transfer *t) { safe_close(t->log_fd); safe_close(t->stdin_fd); + safe_close(t->stdout_fd); free(t); return NULL; @@ -363,14 +371,16 @@ static int transfer_start(Transfer *t) { return -errno; if (t->pid == 0) { const char *cmd[] = { - NULL, /* systemd-import or systemd-pull */ + NULL, /* systemd-import, systemd-export or systemd-pull */ NULL, /* tar, raw, dkr */ NULL, /* --verify= */ NULL, /* verify argument */ NULL, /* maybe --force */ NULL, /* maybe --read-only */ NULL, /* maybe --dkr-index-url */ - NULL, /* the actual URL */ + NULL, /* if so: the actual URL */ + NULL, /* maybe --format= */ + NULL, /* if so: the actual format */ NULL, /* remote */ NULL, /* local */ NULL @@ -385,14 +395,24 @@ static int transfer_start(Transfer *t) { pipefd[0] = safe_close(pipefd[0]); - if (dup2(pipefd[1], STDOUT_FILENO) != STDOUT_FILENO) { + if (dup2(pipefd[1], STDERR_FILENO) != STDERR_FILENO) { log_error_errno(errno, "Failed to dup2() fd: %m"); _exit(EXIT_FAILURE); } - if (dup2(pipefd[1], STDERR_FILENO) != STDERR_FILENO) { - log_error_errno(errno, "Failed to dup2() fd: %m"); - _exit(EXIT_FAILURE); + if (t->stdout_fd >= 0) { + if (dup2(t->stdout_fd, STDOUT_FILENO) != STDOUT_FILENO) { + log_error_errno(errno, "Failed to dup2() fd: %m"); + _exit(EXIT_FAILURE); + } + + if (t->stdout_fd != STDOUT_FILENO) + safe_close(t->stdout_fd); + } else { + if (dup2(pipefd[1], STDOUT_FILENO) != STDOUT_FILENO) { + log_error_errno(errno, "Failed to dup2() fd: %m"); + _exit(EXIT_FAILURE); + } } if (pipefd[1] != STDOUT_FILENO && pipefd[1] != STDERR_FILENO) @@ -433,12 +453,14 @@ static int transfer_start(Transfer *t) { if (IN_SET(t->type, TRANSFER_IMPORT_TAR, TRANSFER_IMPORT_RAW)) cmd[k++] = SYSTEMD_IMPORT_PATH; + else if (IN_SET(t->type, TRANSFER_EXPORT_TAR, TRANSFER_EXPORT_RAW)) + cmd[k++] = SYSTEMD_EXPORT_PATH; else cmd[k++] = SYSTEMD_PULL_PATH; - if (IN_SET(t->type, TRANSFER_IMPORT_TAR, TRANSFER_PULL_TAR)) + if (IN_SET(t->type, TRANSFER_IMPORT_TAR, TRANSFER_EXPORT_TAR, TRANSFER_PULL_TAR)) cmd[k++] = "tar"; - else if (IN_SET(t->type, TRANSFER_IMPORT_RAW, TRANSFER_PULL_RAW)) + else if (IN_SET(t->type, TRANSFER_IMPORT_RAW, TRANSFER_EXPORT_RAW, TRANSFER_PULL_RAW)) cmd[k++] = "raw"; else cmd[k++] = "dkr"; @@ -458,10 +480,17 @@ static int transfer_start(Transfer *t) { cmd[k++] = t->dkr_index_url; } - if (t->remote) - cmd[k++] = t->remote; - else - cmd[k++] = "-"; + if (t->format) { + cmd[k++] = "--format"; + cmd[k++] = t->format; + } + + if (!IN_SET(t->type, TRANSFER_EXPORT_TAR, TRANSFER_EXPORT_RAW)) { + if (t->remote) + cmd[k++] = t->remote; + else + cmd[k++] = "-"; + } if (t->local) cmd[k++] = t->local; @@ -751,6 +780,67 @@ static int method_import_tar_or_raw(sd_bus *bus, sd_bus_message *msg, void *user return sd_bus_reply_method_return(msg, "uo", id, object); } +static int method_export_tar_or_raw(sd_bus *bus, sd_bus_message *msg, void *userdata, sd_bus_error *error) { + _cleanup_(transfer_unrefp) Transfer *t = NULL; + int fd, r; + const char *local, *object, *format; + Manager *m = userdata; + TransferType type; + uint32_t id; + + r = bus_verify_polkit_async( + msg, + CAP_SYS_ADMIN, + "org.freedesktop.import1.export", + false, + UID_INVALID, + &m->polkit_registry, + error); + if (r < 0) + return r; + if (r == 0) + return 1; /* Will call us back */ + + r = sd_bus_message_read(msg, "shs", &local, &fd, &format); + if (r < 0) + return r; + + if (!machine_name_is_valid(local)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Local name %s is invalid", local); + + type = streq_ptr(sd_bus_message_get_member(msg), "ExportTar") ? TRANSFER_EXPORT_TAR : TRANSFER_EXPORT_RAW; + + r = transfer_new(m, &t); + if (r < 0) + return r; + + t->type = type; + + if (!isempty(format)) { + t->format = strdup(format); + if (!t->format) + return -ENOMEM; + } + + t->local = strdup(local); + if (!t->local) + return -ENOMEM; + + t->stdout_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); + if (t->stdout_fd < 0) + return -errno; + + r = transfer_start(t); + if (r < 0) + return r; + + object = t->object_path; + id = t->id; + t = NULL; + + return sd_bus_reply_method_return(msg, "uo", id, object); +} + static int method_pull_tar_or_raw(sd_bus *bus, sd_bus_message *msg, void *userdata, sd_bus_error *error) { _cleanup_(transfer_unrefp) Transfer *t = NULL; const char *remote, *local, *verify, *object; @@ -1080,6 +1170,8 @@ static const sd_bus_vtable manager_vtable[] = { SD_BUS_VTABLE_START(0), SD_BUS_METHOD("ImportTar", "hsbb", "uo", method_import_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD("ImportRaw", "hsbb", "uo", method_import_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD("ExportTar", "shs", "uo", method_export_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD("ExportRaw", "shs", "uo", method_export_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD("PullTar", "sssb", "uo", method_pull_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD("PullRaw", "sssb", "uo", method_pull_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD("PullDkr", "sssssb", "uo", method_pull_dkr, SD_BUS_VTABLE_UNPRIVILEGED), diff --git a/src/import/org.freedesktop.import1.policy.in b/src/import/org.freedesktop.import1.policy.in index 95a79d2ba..85924ed74 100644 --- a/src/import/org.freedesktop.import1.policy.in +++ b/src/import/org.freedesktop.import1.policy.in @@ -26,6 +26,16 @@ + + <_description>Export a VM or container image + <_message>Authentication is required to export a VM or container image + + auth_admin + auth_admin + auth_admin_keep + + + <_description>Download a VM or container image <_message>Authentication is required to download a VM or container image diff --git a/src/import/pull-dkr.c b/src/import/pull-dkr.c index 3a9beadf6..1a7dc310c 100644 --- a/src/import/pull-dkr.c +++ b/src/import/pull-dkr.c @@ -492,7 +492,7 @@ static int dkr_pull_job_on_open_disk(PullJob *j) { if (r < 0) return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path); - j->disk_fd = import_fork_tar(i->temp_path, &i->tar_pid); + j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid); if (j->disk_fd < 0) return j->disk_fd; diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c index 504642fa2..16994e1c2 100644 --- a/src/import/pull-tar.c +++ b/src/import/pull-tar.c @@ -335,7 +335,7 @@ static int tar_pull_job_on_open_disk(PullJob *j) { } else if (r < 0) return log_error_errno(errno, "Failed to create subvolume %s: %m", i->temp_path); - j->disk_fd = import_fork_tar(i->temp_path, &i->tar_pid); + j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid); if (j->disk_fd < 0) return j->disk_fd; diff --git a/src/import/pull.c b/src/import/pull.c index 03a17a5fe..ef7b0359a 100644 --- a/src/import/pull.c +++ b/src/import/pull.c @@ -319,7 +319,7 @@ static int pull_dkr(int argc, char *argv[], void *userdata) { static int help(int argc, char *argv[], void *userdata) { printf("%s [OPTIONS...] {COMMAND} ...\n\n" - "Download container or virtual machine image.\n\n" + "Download container or virtual machine images.\n\n" " -h --help Show this help\n" " --version Show package version\n" " --force Force creation of image\n" diff --git a/src/machine/machinectl.c b/src/machine/machinectl.c index afc4fd640..885604020 100644 --- a/src/machine/machinectl.c +++ b/src/machine/machinectl.c @@ -71,6 +71,7 @@ static OutputMode arg_output = OUTPUT_SHORT; static bool arg_force = false; static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE; static const char* arg_dkr_index_url = NULL; +static const char* arg_format = NULL; static void pager_open_if_enabled(void) { @@ -1551,7 +1552,7 @@ static int transfer_signal_handler(sd_event_source *s, const struct signalfd_sig return 0; } -static int pull_image_common(sd_bus *bus, sd_bus_message *m) { +static int transfer_image_common(sd_bus *bus, sd_bus_message *m) { _cleanup_bus_slot_unref_ sd_bus_slot *slot_job_removed = NULL, *slot_log_message = NULL; _cleanup_bus_error_free_ sd_bus_error error = SD_BUS_ERROR_NULL; _cleanup_bus_message_unref_ sd_bus_message *reply = NULL; @@ -1598,7 +1599,7 @@ static int pull_image_common(sd_bus *bus, sd_bus_message *m) { r = sd_bus_call(bus, m, 0, &error, &reply); if (r < 0) { - log_error("Failed acquire image: %s", bus_error_message(&error, -r)); + log_error("Failed transfer image: %s", bus_error_message(&error, -r)); return r; } @@ -1685,7 +1686,7 @@ static int import_tar(int argc, char *argv[], void *userdata) { if (r < 0) return bus_log_create_error(r); - return pull_image_common(bus, m); + return transfer_image_common(bus, m); } static int import_raw(int argc, char *argv[], void *userdata) { @@ -1752,7 +1753,124 @@ static int import_raw(int argc, char *argv[], void *userdata) { if (r < 0) return bus_log_create_error(r); - return pull_image_common(bus, m); + return transfer_image_common(bus, m); +} + +static void determine_compression_from_filename(const char *p) { + if (arg_format) + return; + + if (!p) + return; + + if (endswith(p, ".xz")) + arg_format = "xz"; + else if (endswith(p, ".gz")) + arg_format = "gzip"; + else if (endswith(p, ".bz2")) + arg_format = "bzip2"; +} + +static int export_tar(int argc, char *argv[], void *userdata) { + _cleanup_bus_message_unref_ sd_bus_message *m = NULL; + _cleanup_free_ char *ll = NULL; + _cleanup_close_ int fd = -1; + const char *local = NULL, *path = NULL; + sd_bus *bus = userdata; + int r; + + assert(bus); + + local = argv[1]; + if (!machine_name_is_valid(local)) { + log_error("Machine name %s is not valid.", local); + return -EINVAL; + } + + if (argc >= 3) + path = argv[2]; + if (isempty(path) || streq(path, "-")) + path = NULL; + + if (path) { + determine_compression_from_filename(path); + + fd = open(path, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOCTTY, 0666); + if (fd < 0) + return log_error_errno(errno, "Failed to open %s: %m", path); + } + + r = sd_bus_message_new_method_call( + bus, + &m, + "org.freedesktop.import1", + "/org/freedesktop/import1", + "org.freedesktop.import1.Manager", + "ExportTar"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "shs", + local, + fd >= 0 ? fd : STDOUT_FILENO, + arg_format); + if (r < 0) + return bus_log_create_error(r); + + return transfer_image_common(bus, m); +} + +static int export_raw(int argc, char *argv[], void *userdata) { + _cleanup_bus_message_unref_ sd_bus_message *m = NULL; + _cleanup_free_ char *ll = NULL; + _cleanup_close_ int fd = -1; + const char *local = NULL, *path = NULL; + sd_bus *bus = userdata; + int r; + + assert(bus); + + local = argv[1]; + if (!machine_name_is_valid(local)) { + log_error("Machine name %s is not valid.", local); + return -EINVAL; + } + + if (argc >= 3) + path = argv[2]; + if (isempty(path) || streq(path, "-")) + path = NULL; + + if (path) { + determine_compression_from_filename(path); + + fd = open(path, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOCTTY, 0666); + if (fd < 0) + return log_error_errno(errno, "Failed to open %s: %m", path); + } + + r = sd_bus_message_new_method_call( + bus, + &m, + "org.freedesktop.import1", + "/org/freedesktop/import1", + "org.freedesktop.import1.Manager", + "ExportRaw"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "shs", + local, + fd >= 0 ? fd : STDOUT_FILENO, + arg_format); + if (r < 0) + return bus_log_create_error(r); + + return transfer_image_common(bus, m); } static int pull_tar(int argc, char *argv[], void *userdata) { @@ -1816,7 +1934,7 @@ static int pull_tar(int argc, char *argv[], void *userdata) { if (r < 0) return bus_log_create_error(r); - return pull_image_common(bus, m); + return transfer_image_common(bus, m); } static int pull_raw(int argc, char *argv[], void *userdata) { @@ -1880,7 +1998,7 @@ static int pull_raw(int argc, char *argv[], void *userdata) { if (r < 0) return bus_log_create_error(r); - return pull_image_common(bus, m); + return transfer_image_common(bus, m); } static int pull_dkr(int argc, char *argv[], void *userdata) { @@ -1952,7 +2070,7 @@ static int pull_dkr(int argc, char *argv[], void *userdata) { if (r < 0) return bus_log_create_error(r); - return pull_image_common(bus, m); + return transfer_image_common(bus, m); } typedef struct TransferInfo { @@ -2199,11 +2317,13 @@ static int help(int argc, char *argv[], void *userdata) { " remove NAME... Remove an image\n" " set-limit [NAME] BYTES Set image or pool size limit (disk quota)\n\n" "Image Transfer Commands:\n" - " import-tar FILE [NAME] Import a local TAR container image\n" - " import-raw FILE [NAME] Import a local RAW container or VM image\n" " pull-tar URL [NAME] Download a TAR container image\n" " pull-raw URL [NAME] Download a RAW container or VM image\n" " pull-dkr REMOTE [NAME] Download a DKR container image\n" + " import-tar FILE [NAME] Import a local TAR container image\n" + " import-raw FILE [NAME] Import a local RAW container or VM image\n" + " export-tar FILE [NAME] Export a TAR container image locally\n" + " export-raw FILE [NAME] Export a RAW container or VM image locally\n" " list-transfers Show list of downloads in progress\n" " cancel-transfer Cancel a download\n" , program_invocation_short_name); @@ -2224,6 +2344,7 @@ static int parse_argv(int argc, char *argv[]) { ARG_VERIFY, ARG_FORCE, ARG_DKR_INDEX_URL, + ARG_FORMAT, }; static const struct option options[] = { @@ -2247,6 +2368,7 @@ static int parse_argv(int argc, char *argv[]) { { "verify", required_argument, NULL, ARG_VERIFY }, { "force", no_argument, NULL, ARG_FORCE }, { "dkr-index-url", required_argument, NULL, ARG_DKR_INDEX_URL }, + { "format", required_argument, NULL, ARG_FORMAT }, {} }; @@ -2368,6 +2490,15 @@ static int parse_argv(int argc, char *argv[]) { arg_dkr_index_url = optarg; break; + case ARG_FORMAT: + if (!STR_IN_SET(optarg, "uncompressed", "xz", "gzip", "bzip2")) { + log_error("Unknown format: %s", optarg); + return -EINVAL; + } + + arg_format = optarg; + break; + case '?': return -EINVAL; @@ -2405,6 +2536,8 @@ static int machinectl_main(int argc, char *argv[], sd_bus *bus) { { "disable", 2, VERB_ANY, 0, enable_machine }, { "import-tar", 2, 3, 0, import_tar }, { "import-raw", 2, 3, 0, import_raw }, + { "export-tar", 2, 3, 0, export_tar }, + { "export-raw", 2, 3, 0, export_raw }, { "pull-tar", 2, 3, 0, pull_tar }, { "pull-raw", 2, 3, 0, pull_raw }, { "pull-dkr", 2, 3, 0, pull_dkr }, diff --git a/src/shared/btrfs-util.c b/src/shared/btrfs-util.c index 256c5a699..23a22db66 100644 --- a/src/shared/btrfs-util.c +++ b/src/shared/btrfs-util.c @@ -101,48 +101,42 @@ int btrfs_is_snapshot(int fd) { return F_TYPE_EQUAL(sfs.f_type, BTRFS_SUPER_MAGIC); } -int btrfs_subvol_snapshot(const char *old_path, const char *new_path, bool read_only, bool fallback_copy) { +int btrfs_subvol_snapshot_fd(int old_fd, const char *new_path, bool read_only, bool fallback_copy) { struct btrfs_ioctl_vol_args_v2 args = { .flags = read_only ? BTRFS_SUBVOL_RDONLY : 0, }; - _cleanup_close_ int old_fd = -1, new_fd = -1; + _cleanup_close_ int new_fd = -1; const char *subvolume; int r; - assert(old_path); - - old_fd = open(old_path, O_RDONLY|O_NOCTTY|O_CLOEXEC|O_DIRECTORY); - if (old_fd < 0) - return -errno; + assert(new_path); r = btrfs_is_snapshot(old_fd); if (r < 0) return r; if (r == 0) { + if (!fallback_copy) + return -EISDIR; - if (fallback_copy) { - r = btrfs_subvol_make(new_path); - if (r < 0) - return r; + r = btrfs_subvol_make(new_path); + if (r < 0) + return r; + + r = copy_directory_fd(old_fd, new_path, true); + if (r < 0) { + btrfs_subvol_remove(new_path); + return r; + } - r = copy_directory_fd(old_fd, new_path, true); + if (read_only) { + r = btrfs_subvol_set_read_only(new_path, true); if (r < 0) { btrfs_subvol_remove(new_path); return r; } - - if (read_only) { - r = btrfs_subvol_set_read_only(new_path, true); - if (r < 0) { - btrfs_subvol_remove(new_path); - return r; - } - } - - return 0; } - return -EISDIR; + return 0; } r = extract_subvolume_name(new_path, &subvolume); @@ -162,6 +156,19 @@ int btrfs_subvol_snapshot(const char *old_path, const char *new_path, bool read_ return 0; } +int btrfs_subvol_snapshot(const char *old_path, const char *new_path, bool read_only, bool fallback_copy) { + _cleanup_close_ int old_fd = -1; + + assert(old_path); + assert(new_path); + + old_fd = open(old_path, O_RDONLY|O_NOCTTY|O_CLOEXEC|O_DIRECTORY); + if (old_fd < 0) + return -errno; + + return btrfs_subvol_snapshot_fd(old_fd, new_path, read_only, fallback_copy); +} + int btrfs_subvol_make(const char *path) { struct btrfs_ioctl_vol_args args = {}; _cleanup_close_ int fd = -1; diff --git a/src/shared/btrfs-util.h b/src/shared/btrfs-util.h index a2c246b8d..cb85b71d7 100644 --- a/src/shared/btrfs-util.h +++ b/src/shared/btrfs-util.h @@ -48,6 +48,8 @@ int btrfs_is_snapshot(int fd); int btrfs_subvol_make(const char *path); int btrfs_subvol_make_label(const char *path); int btrfs_subvol_remove(const char *path); + +int btrfs_subvol_snapshot_fd(int old_fd, const char *new_path, bool read_only, bool fallback_copy); int btrfs_subvol_snapshot(const char *old_path, const char *new_path, bool read_only, bool fallback_copy); int btrfs_subvol_set_read_only_fd(int fd, bool b); -- 2.30.2