From 8af3cf74df03f7528f9e2605ec7896a5daf0f711 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 21 Jan 2015 13:43:55 +0100 Subject: [PATCH 1/1] import: support downloading bzip2-encoded images This way, we can import CoreOS images unmodified. --- Makefile.am | 3 +++ configure.ac | 11 +++++++++ src/import/import-job.c | 54 +++++++++++++++++++++++++++++++++++++---- src/import/import-job.h | 3 +++ src/import/import.c | 5 +++- 5 files changed, 70 insertions(+), 6 deletions(-) diff --git a/Makefile.am b/Makefile.am index 406910a3c..4f536ee41 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5250,6 +5250,7 @@ lib_LTLIBRARIES += \ if HAVE_LIBCURL if HAVE_XZ if HAVE_ZLIB +if HAVE_BZIP2 if HAVE_GCRYPT bin_PROGRAMS += \ @@ -5290,6 +5291,7 @@ systemd_import_LDADD = \ $(LIBCURL_LIBS) \ $(XZ_LIBS) \ $(ZLIB_LIBS) \ + -lbz2 \ $(GCRYPT_LIBS) manual_tests += \ @@ -5316,6 +5318,7 @@ endif endif endif endif +endif endif diff --git a/configure.ac b/configure.ac index d82cfe890..e0320057f 100644 --- a/configure.ac +++ b/configure.ac @@ -576,6 +576,16 @@ if test "x$enable_zlib" != "xno"; then fi AM_CONDITIONAL(HAVE_ZLIB, [test "$have_zlib" = "yes"]) +# ------------------------------------------------------------------------------ +have_bzip2=no +AC_ARG_ENABLE(bzip2, AS_HELP_STRING([--enable-bzip2], [Enable optional BZIP2 support])) +AS_IF([test "x$enable_bzip2" != "xno"], [ + AC_CHECK_HEADERS(bzlib.h, + [AC_DEFINE(HAVE_BZIP2, 1, [Define in BZIP2 is available]) have_bzip2=yes], + [AC_MSG_ERROR([*** BZIP2 support requested but headers not found])]) +]) +AM_CONDITIONAL(HAVE_BZIP2, [test "$have_bzip2" = "yes"]) + # ------------------------------------------------------------------------------ have_lz4=no AC_ARG_ENABLE(lz4, AS_HELP_STRING([--enable-lz4], [Enable optional LZ4 support])) @@ -1426,6 +1436,7 @@ AC_MSG_RESULT([ ZLIB: ${have_zlib} XZ: ${have_xz} LZ4: ${have_lz4} + BZIP2: ${have_bzip2} ACL: ${have_acl} GCRYPT: ${have_gcrypt} QRENCODE: ${have_qrencode} diff --git a/src/import/import-job.c b/src/import/import-job.c index 322aa1a18..337a5c5a2 100644 --- a/src/import/import-job.c +++ b/src/import/import-job.c @@ -37,6 +37,8 @@ ImportJob* import_job_unref(ImportJob *j) { lzma_end(&j->xz); else if (j->compressed == IMPORT_JOB_GZIP) inflateEnd(&j->gzip); + else if (j->compressed == IMPORT_JOB_BZIP2) + BZ2_bzDecompressEnd(&j->bzip2); if (j->checksum_context) gcry_md_close(j->checksum_context); @@ -178,7 +180,9 @@ static int import_job_write_uncompressed(ImportJob *j, void *p, size_t sz) { assert(j); assert(p); - assert(sz > 0); + + if (sz <= 0) + return 0; if (j->written_uncompressed + sz < j->written_uncompressed) { log_error("File too large, overflow"); @@ -209,7 +213,7 @@ static int import_job_write_uncompressed(ImportJob *j, void *p, size_t sz) { if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz)) return log_oom(); - memcpy((uint8_t*) j->payload + j->payload_size, p, sz); + memcpy(j->payload + j->payload_size, p, sz); j->payload_size += sz; } @@ -223,7 +227,9 @@ static int import_job_write_compressed(ImportJob *j, void *p, size_t sz) { assert(j); assert(p); - assert(sz > 0); + + if (sz <= 0) + return 0; if (j->written_compressed + sz < j->written_compressed) { log_error("File too large, overflow"); @@ -300,6 +306,29 @@ static int import_job_write_compressed(ImportJob *j, void *p, size_t sz) { break; + case IMPORT_JOB_BZIP2: + j->bzip2.next_in = p; + j->bzip2.avail_in = sz; + + while (j->bzip2.avail_in > 0) { + uint8_t buffer[16 * 1024]; + + j->bzip2.next_out = (char*) buffer; + j->bzip2.avail_out = sizeof(buffer); + + r = BZ2_bzDecompress(&j->bzip2); + if (r != BZ_OK && r != BZ_STREAM_END) { + log_error("Decompression error."); + return -EIO; + } + + r = import_job_write_uncompressed(j, buffer, sizeof(buffer) - j->bzip2.avail_out); + if (r < 0) + return r; + } + + break; + default: assert_not_reached("Unknown compression"); } @@ -350,6 +379,9 @@ static int import_job_detect_compression(ImportJob *j) { static const uint8_t gzip_signature[] = { 0x1f, 0x8b }; + static const uint8_t bzip2_signature[] = { + 'B', 'Z', 'h' + }; _cleanup_free_ uint8_t *stub = NULL; size_t stub_size; @@ -358,18 +390,23 @@ static int import_job_detect_compression(ImportJob *j) { assert(j); - if (j->payload_size < MAX(sizeof(xz_signature), sizeof(gzip_signature))) + if (j->payload_size < MAX3(sizeof(xz_signature), + sizeof(gzip_signature), + sizeof(bzip2_signature))) return 0; if (memcmp(j->payload, xz_signature, sizeof(xz_signature)) == 0) j->compressed = IMPORT_JOB_XZ; else if (memcmp(j->payload, gzip_signature, sizeof(gzip_signature)) == 0) j->compressed = IMPORT_JOB_GZIP; + else if (memcmp(j->payload, bzip2_signature, sizeof(bzip2_signature)) == 0) + j->compressed = IMPORT_JOB_BZIP2; else j->compressed = IMPORT_JOB_UNCOMPRESSED; log_debug("Stream is XZ compressed: %s", yes_no(j->compressed == IMPORT_JOB_XZ)); log_debug("Stream is GZIP compressed: %s", yes_no(j->compressed == IMPORT_JOB_GZIP)); + log_debug("Stream is BZIP2 compressed: %s", yes_no(j->compressed == IMPORT_JOB_BZIP2)); if (j->compressed == IMPORT_JOB_XZ) { lzma_ret xzr; @@ -387,6 +424,13 @@ static int import_job_detect_compression(ImportJob *j) { return -EIO; } } + if (j->compressed == IMPORT_JOB_BZIP2) { + r = BZ2_bzDecompressInit(&j->bzip2, 0, 0); + if (r != BZ_OK) { + log_error("Failed to initialize bzip2 decoder."); + return -EIO; + } + } r = import_job_open_disk(j); if (r < 0) @@ -427,7 +471,7 @@ static size_t import_job_write_callback(void *contents, size_t size, size_t nmem goto fail; } - memcpy((uint8_t*) j->payload + j->payload_size, contents, sz); + memcpy(j->payload + j->payload_size, contents, sz); j->payload_size += sz; r = import_job_detect_compression(j); diff --git a/src/import/import-job.h b/src/import/import-job.h index a9b58e633..b10b0ec0d 100644 --- a/src/import/import-job.h +++ b/src/import/import-job.h @@ -23,6 +23,7 @@ #include #include +#include #include #include "macro.h" @@ -49,6 +50,7 @@ typedef enum ImportJobCompression { IMPORT_JOB_UNCOMPRESSED, IMPORT_JOB_XZ, IMPORT_JOB_GZIP, + IMPORT_JOB_BZIP2, _IMPORT_JOB_COMPRESSION_MAX, _IMPORT_JOB_COMPRESSION_INVALID = -1, } ImportJobCompression; @@ -89,6 +91,7 @@ struct ImportJob { ImportJobCompression compressed; lzma_stream xz; z_stream gzip; + bz_stream bzip2; unsigned progress_percent; usec_t start_usec; diff --git a/src/import/import.c b/src/import/import.c index 0c215c648..b3ebb3535 100644 --- a/src/import/import.c +++ b/src/import/import.c @@ -51,10 +51,12 @@ static int strip_tar_suffixes(const char *name, char **ret) { char *s; e = endswith(name, ".tar"); + if (!e) + e = endswith(name, ".tar.xz"); if (!e) e = endswith(name, ".tar.gz"); if (!e) - e = endswith(name, ".tar.xz"); + e = endswith(name, ".tar.bz2"); if (!e) e = endswith(name, ".tgz"); if (!e) @@ -162,6 +164,7 @@ static int strip_raw_suffixes(const char *p, char **ret) { static const char suffixes[] = ".xz\0" ".gz\0" + ".bz2\0" ".raw\0" ".qcow2\0" ".img\0" -- 2.30.2