From edce2aed3aa93b84f7b4c70412bdb665da2977b0 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 16 Jan 2015 20:07:25 +0100 Subject: [PATCH] import: support importing qcow2 images With this change the import tool will now unpack qcow2 images into normal raw disk images, suitable for usage with nspawn. This allows has the benefit of also allowing importing Ubuntu Cloud images for usage with nspawn. --- .gitignore | 1 + Makefile.am | 30 +++- configure.ac | 13 ++ src/import/import-raw.c | 48 ++++++ src/import/import.c | 52 +++++- src/import/qcow2-util.c | 347 ++++++++++++++++++++++++++++++++++++++++ src/import/qcow2-util.h | 25 +++ src/import/test-qcow2.c | 55 +++++++ 8 files changed, 560 insertions(+), 11 deletions(-) create mode 100644 src/import/qcow2-util.c create mode 100644 src/import/qcow2-util.h create mode 100644 src/import/test-qcow2.c diff --git a/.gitignore b/.gitignore index caef8310f..d4704feed 100644 --- a/.gitignore +++ b/.gitignore @@ -227,6 +227,7 @@ /test-pppoe /test-prioq /test-pty +/test-qcow2 /test-ratelimit /test-replace-var /test-resolve diff --git a/Makefile.am b/Makefile.am index ee73598cb..f91c78658 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5233,19 +5233,43 @@ systemd_import_SOURCES = \ src/import/curl-util.c \ src/import/curl-util.h \ src/import/aufs-util.c \ - src/import/aufs-util.h + src/import/aufs-util.h \ + src/import/qcow2-util.c \ + src/import/qcow2-util.h systemd_import_CFLAGS = \ $(AM_CFLAGS) \ $(LIBCURL_CFLAGS) \ - $(XZ_CFLAGS) + $(XZ_CFLAGS) \ + $(ZLIB_CFLAGS) systemd_import_LDADD = \ libsystemd-internal.la \ libsystemd-label.la \ libsystemd-shared.la \ $(LIBCURL_LIBS) \ - $(XZ_LIBS) + $(XZ_LIBS) \ + $(ZLIB_LIBS) + +endif + +if HAVE_ZLIB +manual_tests += \ + test-qcow2 + +test_qcow2_SOURCES = \ + src/import/test-qcow2.c \ + src/import/qcow2-util.c \ + src/import/qcow2-util.h + +test_qcow2_CFLAGS = \ + $(AM_CFLAGS) \ + $(ZLIB_CFLAGS) + +test_qcow2_LDADD = \ + libsystemd-internal.la \ + libsystemd-shared.la \ + $(ZLIB_LIBS) endif endif diff --git a/configure.ac b/configure.ac index 5057f8e99..6d510df33 100644 --- a/configure.ac +++ b/configure.ac @@ -563,6 +563,18 @@ if test "x$enable_xz" != "xno"; then fi AM_CONDITIONAL(HAVE_XZ, [test "$have_xz" = "yes"]) +# ------------------------------------------------------------------------------ +have_zlib=no +AC_ARG_ENABLE(zlib, AS_HELP_STRING([--disable-zlib], [Disable optional ZLIB support])) +if test "x$enable_zlib" != "xno"; then + PKG_CHECK_MODULES(ZLIB, [ zlib ], + [AC_DEFINE(HAVE_ZLIB, 1, [Define if ZLIB is available]) have_zlib=yes]) + if test "x$have_zlib" = xno -a "x$enable_zlib" = xyes; then + AC_MSG_ERROR([*** ZLIB support requested but libraries not found]) + fi +fi +AM_CONDITIONAL(HAVE_ZLIB, [test "$have_zlib" = "yes"]) + # ------------------------------------------------------------------------------ have_lz4=no AC_ARG_ENABLE(lz4, AS_HELP_STRING([--enable-lz4], [Enable optional LZ4 support])) @@ -1410,6 +1422,7 @@ AC_MSG_RESULT([ SELinux: ${have_selinux} SECCOMP: ${have_seccomp} SMACK: ${have_smack} + ZLIB: ${have_zlib} XZ: ${have_xz} LZ4: ${have_lz4} ACL: ${have_acl} diff --git a/src/import/import-raw.c b/src/import/import-raw.c index f1b36cbfc..c15765d51 100644 --- a/src/import/import-raw.c +++ b/src/import/import-raw.c @@ -27,6 +27,7 @@ #include "hashmap.h" #include "utf8.h" #include "curl-util.h" +#include "qcow2-util.h" #include "import-raw.h" #include "strv.h" #include "copy.h" @@ -235,6 +236,49 @@ finish: raw_import_finish(f->import, r); } +static int raw_import_maybe_convert_qcow2(RawImportFile *f) { + _cleanup_close_ int converted_fd = -1; + _cleanup_free_ char *t = NULL; + int r; + + assert(f); + assert(f->disk_fd); + assert(f->temp_path); + + r = qcow2_detect(f->disk_fd); + if (r < 0) + return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m"); + if (r == 0) + return 0; + + /* This is a QCOW2 image, let's convert it */ + r = tempfn_random(f->final_path, &t); + if (r < 0) + return log_oom(); + + converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644); + if (converted_fd < 0) + return log_error_errno(errno, "Failed to create %s: %m", t); + + r = qcow2_convert(f->disk_fd, converted_fd); + if (r < 0) { + unlink(t); + return log_error_errno(r, "Failed to convert qcow2 image: %m"); + } + + unlink(f->temp_path); + free(f->temp_path); + + f->temp_path = t; + t = NULL; + + safe_close(f->disk_fd); + f->disk_fd = converted_fd; + converted_fd = -1; + + return 1; +} + static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) { RawImportFile *f = NULL; struct stat st; @@ -288,6 +332,10 @@ static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result goto fail; } + r = raw_import_maybe_convert_qcow2(f); + if (r < 0) + goto fail; + if (f->etag) (void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0); if (f->url) diff --git a/src/import/import.c b/src/import/import.c index e457adf62..af8d0ec42 100644 --- a/src/import/import.c +++ b/src/import/import.c @@ -45,10 +45,48 @@ static void on_raw_finished(RawImport *import, int error, void *userdata) { sd_event_exit(event, error); } +static int strip_raw_suffixes(const char *p, char **ret) { + static const char suffixes[] = + ".xz\0" + ".raw\0" + ".qcow2\0" + ".img\0"; + + _cleanup_free_ char *q = NULL; + + q = strdup(p); + if (!q) + return -ENOMEM; + + for (;;) { + const char *sfx; + bool changed = false; + + NULSTR_FOREACH(sfx, suffixes) { + char *e; + + e = endswith(q, sfx); + if (e) { + *e = 0; + changed = true; + } + } + + if (!changed) + break; + } + + *ret = q; + q = NULL; + + return 0; +} + static int pull_raw(int argc, char *argv[], void *userdata) { _cleanup_(raw_import_unrefp) RawImport *import = NULL; _cleanup_event_unref_ sd_event *event = NULL; - const char *url, *local, *suffix; + const char *url, *local; + _cleanup_free_ char *l = NULL; int r; url = argv[1]; @@ -79,13 +117,11 @@ static int pull_raw(int argc, char *argv[], void *userdata) { if (local) { const char *p; - suffix = endswith(local, ".raw.xz"); - if (!suffix) - suffix = endswith(local, ".raw"); - if (!suffix) - suffix = endswith(local, ".xz"); - if (suffix) - local = strndupa(local, suffix - local); + r = strip_raw_suffixes(local, &l); + if (r < 0) + return log_oom(); + + local = l; if (!machine_name_is_valid(local)) { log_error("Local image name '%s' is not valid.", local); diff --git a/src/import/qcow2-util.c b/src/import/qcow2-util.c new file mode 100644 index 000000000..c84c6aa0d --- /dev/null +++ b/src/import/qcow2-util.c @@ -0,0 +1,347 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include + +#include "util.h" +#include "sparse-endian.h" +#include "qcow2-util.h" + +#define QCOW2_MAGIC 0x514649fb + +#define QCOW2_COPIED (1ULL << 63) +#define QCOW2_COMPRESSED (1ULL << 62) +#define QCOW2_ZERO (1ULL << 0) + +typedef struct _packed_ Header { + be32_t magic; + be32_t version; + + be64_t backing_file_offset; + be32_t backing_file_size; + + be32_t cluster_bits; + be64_t size; + be32_t crypt_method; + + be32_t l1_size; + be64_t l1_table_offset; + + be64_t refcount_table_offset; + be32_t refcount_table_clusters; + + be32_t nb_snapshots; + be64_t snapshots_offset; + + /* The remainder is only present on QCOW3 */ + be64_t incompatible_features; + be64_t compatible_features; + be64_t autoclear_features; + + be32_t refcount_order; + be32_t header_length; +} Header; + +#define HEADER_MAGIC(header) be32toh((header)->magic) +#define HEADER_VERSION(header) be32toh((header)->version) +#define HEADER_CLUSTER_BITS(header) be32toh((header)->cluster_bits) +#define HEADER_CLUSTER_SIZE(header) (1ULL << HEADER_CLUSTER_BITS(header)) +#define HEADER_L2_BITS(header) (HEADER_CLUSTER_BITS(header) - 3) +#define HEADER_SIZE(header) be64toh((header)->size) +#define HEADER_CRYPT_METHOD(header) be32toh((header)->crypt_method) +#define HEADER_L1_SIZE(header) be32toh((header)->l1_size) +#define HEADER_L2_SIZE(header) (HEADER_CLUSTER_SIZE(header)/sizeof(uint64_t)) +#define HEADER_L1_TABLE_OFFSET(header) be64toh((header)->l1_table_offset) + +static uint32_t HEADER_HEADER_LENGTH(const Header *h) { + if (HEADER_VERSION(h) < 3) + return offsetof(Header, incompatible_features); + + return be32toh(h->header_length); +} + +static int copy_cluster( + int sfd, uint64_t soffset, + int dfd, uint64_t doffset, + uint64_t cluster_size, + void *buffer) { + + ssize_t l; + + l = pread(sfd, buffer, cluster_size, soffset); + if (l < 0) + return -errno; + if ((uint64_t) l != cluster_size) + return -EIO; + + l = pwrite(dfd, buffer, cluster_size, doffset); + if (l < 0) + return -errno; + if ((uint64_t) l != cluster_size) + return -EIO; + + return 0; +} + +static int decompress_cluster( + int sfd, uint64_t soffset, + int dfd, uint64_t doffset, + uint64_t compressed_size, + uint64_t cluster_size, + void *buffer1, + void *buffer2) { + + _cleanup_free_ void *large_buffer = NULL; + z_stream s = {}; + uint64_t sz; + ssize_t l; + int r; + + if (compressed_size > cluster_size) { + /* The usual cluster buffer doesn't suffice, let's + * allocate a larger one, temporarily */ + + large_buffer = malloc(compressed_size); + if (!large_buffer) + return -ENOMEM; + + buffer1 = large_buffer; + } + + l = pread(sfd, buffer1, compressed_size, soffset); + if (l < 0) + return -errno; + if ((uint64_t) l != compressed_size) + return -EIO; + + s.next_in = buffer1; + s.avail_in = compressed_size; + s.next_out = buffer2; + s.avail_out = cluster_size; + + r = inflateInit2(&s, -12); + if (r != Z_OK) + return -EIO; + + r = inflate(&s, Z_FINISH); + sz = (uint8_t*) s.next_out - (uint8_t*) buffer2; + inflateEnd(&s); + if (r != Z_STREAM_END || sz != cluster_size) + return -EIO; + + l = pwrite(dfd, buffer2, cluster_size, doffset); + if (l < 0) + return -errno; + if ((uint64_t) l != cluster_size) + return -EIO; + + return 0; +} + +static int normalize_offset( + const Header *header, + uint64_t p, + uint64_t *ret, + bool *compressed, + uint64_t *compressed_size) { + + uint64_t q; + + q = be64toh(p); + + if (q & QCOW2_COMPRESSED) { + uint64_t sz, csize_shift, csize_mask; + + if (!compressed) + return -ENOTSUP; + + csize_shift = 64 - 2 - (HEADER_CLUSTER_BITS(header) - 8); + csize_mask = (1ULL << (HEADER_CLUSTER_BITS(header) - 8)) - 1; + sz = (((q >> csize_shift) & csize_mask) + 1) * 512 - (q & 511); + q &= ((1ULL << csize_shift) - 1); + + if (compressed_size) + *compressed_size = sz; + + *compressed = true; + + } else { + if (compressed) { + *compressed = false; + *compressed_size = 0; + } + + if (q & QCOW2_ZERO) { + /* We make no distinction between zero blocks and holes */ + *ret = 0; + return 0; + } + + q &= ~QCOW2_COPIED; + } + + *ret = q; + return q > 0; /* returns positive if not a hole */ +} + +static int verify_header(const Header *header) { + assert(header); + + if (HEADER_MAGIC(header) != QCOW2_MAGIC) + return -EBADMSG; + + if (HEADER_VERSION(header) != 2 && + HEADER_VERSION(header) != 3) + return -ENOTSUP; + + if (HEADER_CRYPT_METHOD(header) != 0) + return -ENOTSUP; + + if (HEADER_CLUSTER_BITS(header) < 9) /* 512K */ + return -EBADMSG; + + if (HEADER_CLUSTER_BITS(header) > 21) /* 2MB */ + return -EBADMSG; + + if (HEADER_SIZE(header) % HEADER_CLUSTER_SIZE(header) != 0) + return -EBADMSG; + + if (HEADER_L1_SIZE(header) > 32*1024*1024) /* 32MB */ + return -EBADMSG; + + if (HEADER_VERSION(header) == 3) { + + if (header->incompatible_features != 0) + return -ENOTSUP; + + if (HEADER_HEADER_LENGTH(header) < sizeof(Header)) + return -EBADMSG; + } + + return 0; +} + +int qcow2_convert(int qcow2_fd, int raw_fd) { + _cleanup_free_ void *buffer1 = NULL, *buffer2 = NULL; + _cleanup_free_ be64_t *l1_table = NULL, *l2_table = NULL; + uint64_t sz, i; + Header header; + ssize_t l; + int r; + + l = pread(qcow2_fd, &header, sizeof(header), 0); + if (l < 0) + return -errno; + if (l != sizeof(header)) + return -EIO; + + r = verify_header(&header); + if (r < 0) + return r; + + l1_table = new(be64_t, HEADER_L1_SIZE(&header)); + if (!l1_table) + return -ENOMEM; + + l2_table = malloc(HEADER_CLUSTER_SIZE(&header)); + if (!l2_table) + return -ENOMEM; + + buffer1 = malloc(HEADER_CLUSTER_SIZE(&header)); + if (!buffer1) + return -ENOMEM; + + buffer2 = malloc(HEADER_CLUSTER_SIZE(&header)); + if (!buffer2) + return -ENOMEM; + + /* Empty the file if it exists, we rely on zero bits */ + if (ftruncate(raw_fd, 0) < 0) + return -errno; + + if (ftruncate(raw_fd, HEADER_SIZE(&header)) < 0) + return -errno; + + sz = sizeof(uint64_t) * HEADER_L1_SIZE(&header); + l = pread(qcow2_fd, l1_table, sz, HEADER_L1_TABLE_OFFSET(&header)); + if (l < 0) + return -errno; + if ((uint64_t) l != sz) + return -EIO; + + for (i = 0; i < HEADER_L1_SIZE(&header); i ++) { + uint64_t l2_begin, j; + + r = normalize_offset(&header, l1_table[i], &l2_begin, NULL, NULL); + if (r < 0) + return r; + if (r == 0) + continue; + + l = pread(qcow2_fd, l2_table, HEADER_CLUSTER_SIZE(&header), l2_begin); + if (l < 0) + return -errno; + if ((uint64_t) l != HEADER_CLUSTER_SIZE(&header)) + return -EIO; + + for (j = 0; j < HEADER_L2_SIZE(&header); j++) { + uint64_t data_begin, p, compressed_size; + bool compressed; + + p = ((i << HEADER_L2_BITS(&header)) + j) << HEADER_CLUSTER_BITS(&header); + + r = normalize_offset(&header, l2_table[j], &data_begin, &compressed, &compressed_size); + if (r < 0) + return r; + if (r == 0) + continue; + + if (compressed) + r = decompress_cluster( + qcow2_fd, data_begin, + raw_fd, p, + compressed_size, HEADER_CLUSTER_SIZE(&header), + buffer1, buffer2); + else + r = copy_cluster( + qcow2_fd, data_begin, + raw_fd, p, + HEADER_CLUSTER_SIZE(&header), buffer1); + if (r < 0) + return r; + } + } + + return 0; +} + +int qcow2_detect(int fd) { + be32_t id; + ssize_t l; + + l = pread(fd, &id, sizeof(id), 0); + if (l < 0) + return -errno; + if (l != sizeof(id)) + return -EIO; + + return htobe32(QCOW2_MAGIC) == id; +} diff --git a/src/import/qcow2-util.h b/src/import/qcow2-util.h new file mode 100644 index 000000000..be7fd1d0c --- /dev/null +++ b/src/import/qcow2-util.h @@ -0,0 +1,25 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +#pragma once + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +int qcow2_detect(int fd); +int qcow2_convert(int qcow2_fd, int raw_fd); diff --git a/src/import/test-qcow2.c b/src/import/test-qcow2.c new file mode 100644 index 000000000..9a6c3e8b3 --- /dev/null +++ b/src/import/test-qcow2.c @@ -0,0 +1,55 @@ +/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ + +/*** + This file is part of systemd. + + Copyright 2015 Lennart Poettering + + systemd is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + systemd is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with systemd; If not, see . +***/ + +#include "log.h" +#include "util.h" + +#include "qcow2-util.h" + +int main(int argc, char *argv[]) { + _cleanup_close_ int sfd = -1, dfd = -1; + int r; + + if (argc != 3) { + log_error("Needs two arguments."); + return EXIT_FAILURE; + } + + sfd = open(argv[1], O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (sfd < 0) { + log_error_errno(errno, "Can't open source file: %m"); + return EXIT_FAILURE; + } + + dfd = open(argv[2], O_WRONLY|O_CREAT|O_CLOEXEC|O_NOCTTY, 0666); + if (dfd < 0) { + log_error_errno(errno, "Can't open destination file: %m"); + return EXIT_FAILURE; + } + + r = qcow2_convert(sfd, dfd); + if (r < 0) { + log_error_errno(r, "Failed to unpack: %m"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} -- 2.30.2