#include <sys/xattr.h>
#include <linux/fs.h>
#include <curl/curl.h>
+#include <lzma.h>
#include "hashmap.h"
#include "utf8.h"
#include "curl-util.h"
+#include "qcow2-util.h"
#include "import-raw.h"
#include "strv.h"
#include "copy.h"
char **old_etags;
uint64_t content_length;
- uint64_t written;
+ uint64_t written_compressed;
+ uint64_t written_uncompressed;
+
+ void *payload;
+ size_t payload_size;
usec_t mtime;
bool done;
int disk_fd;
+
+ lzma_stream lzma;
+ bool compressed;
+
+ unsigned progress_percent;
+ usec_t start_usec;
+ usec_t last_status_usec;
};
struct RawImport {
#define FILENAME_ESCAPE "/.#\"\'"
+#define RAW_MAX_SIZE (1024LLU*1024LLU*1024LLU*8) /* 8 GB */
+
static RawImportFile *raw_import_file_unref(RawImportFile *f) {
if (!f)
return NULL;
free(f->temp_path);
}
+ lzma_end(&f->lzma);
free(f->url);
free(f->local);
free(f->etag);
strv_free(f->old_etags);
+ free(f->payload);
free(f);
return NULL;
return 0;
}
-static void raw_import_file_success(RawImportFile *f) {
+static int raw_import_file_make_local_copy(RawImportFile *f) {
+ _cleanup_free_ char *tp = NULL;
+ _cleanup_close_ int dfd = -1;
+ const char *p;
int r;
assert(f);
- f->done = true;
+ if (!f->local)
+ return 0;
- if (f->local) {
- _cleanup_free_ char *tp = NULL;
- _cleanup_close_ int dfd = -1;
- const char *p;
+ if (f->disk_fd >= 0) {
+ if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1)
+ return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
+ } else {
+ r = raw_import_file_make_final_path(f);
+ if (r < 0)
+ return log_oom();
- if (f->disk_fd >= 0) {
- if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1) {
- r = log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
- goto finish;
- }
- } else {
- r = raw_import_file_make_final_path(f);
- if (r < 0) {
- log_oom();
- goto finish;
- }
+ f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
+ if (f->disk_fd < 0)
+ return log_error_errno(errno, "Failed to open vendor image: %m");
+ }
- f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
- if (f->disk_fd < 0) {
- r = log_error_errno(errno, "Failed to open vendor image: %m");
- goto finish;
- }
- }
+ p = strappenda(f->import->image_root, "/", f->local, ".raw");
+ if (f->force_local)
+ (void) rm_rf_dangerous(p, false, true, false);
- p = strappenda(f->import->image_root, "/", f->local, ".raw");
- if (f->force_local)
- (void) rm_rf_dangerous(p, false, true, false);
+ r = tempfn_random(p, &tp);
+ if (r < 0)
+ return log_oom();
- r = tempfn_random(p, &tp);
- if (r < 0) {
- log_oom();
- goto finish;
- }
+ dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
+ if (dfd < 0)
+ return log_error_errno(errno, "Failed to create writable copy of image: %m");
- dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
- if (dfd < 0) {
- r = log_error_errno(errno, "Failed to create writable copy of image: %m");
- goto finish;
- }
+ /* Turn off COW writing. This should greatly improve
+ * performance on COW file systems like btrfs, since it
+ * reduces fragmentation caused by not allowing in-place
+ * writes. */
+ r = chattr_fd(dfd, true, FS_NOCOW_FL);
+ if (r < 0)
+ log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
- /* Turn off COW writing. This should greatly improve
- * performance on COW file systems like btrfs, since it
- * reduces fragmentation caused by not allowing in-place
- * writes. */
- r = chattr_fd(dfd, true, FS_NOCOW_FL);
- if (r < 0)
- log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
+ r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
+ if (r < 0) {
+ unlink(tp);
+ return log_error_errno(r, "Failed to make writable copy of image: %m");
+ }
- r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
- if (r < 0) {
- log_error_errno(r, "Failed to make writable copy of image: %m");
- unlink(tp);
- goto finish;
- }
+ (void) copy_times(f->disk_fd, dfd);
+ (void) copy_xattr(f->disk_fd, dfd);
- (void) copy_times(f->disk_fd, dfd);
- (void) copy_xattr(f->disk_fd, dfd);
+ dfd = safe_close(dfd);
- dfd = safe_close(dfd);
+ r = rename(tp, p);
+ if (r < 0) {
+ unlink(tp);
+ return log_error_errno(errno, "Failed to move writable image into place: %m");
+ }
- r = rename(tp, p);
- if (r < 0) {
- r = log_error_errno(errno, "Failed to move writable image into place: %m");
- unlink(tp);
- goto finish;
- }
+ log_info("Created new local image %s.", p);
+ return 0;
+}
- log_info("Created new local image %s.", p);
- }
+static void raw_import_file_success(RawImportFile *f) {
+ int r;
+
+ assert(f);
+
+ f->done = true;
+
+ r = raw_import_file_make_local_copy(f);
+ if (r < 0)
+ goto finish;
f->disk_fd = safe_close(f->disk_fd);
r = 0;
raw_import_finish(f->import, r);
}
+static int raw_import_maybe_convert_qcow2(RawImportFile *f) {
+ _cleanup_close_ int converted_fd = -1;
+ _cleanup_free_ char *t = NULL;
+ int r;
+
+ assert(f);
+ assert(f->disk_fd);
+ assert(f->temp_path);
+
+ r = qcow2_detect(f->disk_fd);
+ if (r < 0)
+ return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
+ if (r == 0)
+ return 0;
+
+ /* This is a QCOW2 image, let's convert it */
+ r = tempfn_random(f->final_path, &t);
+ if (r < 0)
+ return log_oom();
+
+ converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
+ if (converted_fd < 0)
+ return log_error_errno(errno, "Failed to create %s: %m", t);
+
+ r = qcow2_convert(f->disk_fd, converted_fd);
+ if (r < 0) {
+ unlink(t);
+ return log_error_errno(r, "Failed to convert qcow2 image: %m");
+ }
+
+ unlink(f->temp_path);
+ free(f->temp_path);
+
+ f->temp_path = t;
+ t = NULL;
+
+ safe_close(f->disk_fd);
+ f->disk_fd = converted_fd;
+ converted_fd = -1;
+
+ return 1;
+}
+
static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
RawImportFile *f = NULL;
struct stat st;
}
if (f->content_length != (uint64_t) -1 &&
- f->content_length != f->written) {
+ f->content_length != f->written_compressed) {
log_error("Download truncated.");
r = -EIO;
goto fail;
}
+ /* Make sure the file size is right, in case the file was
+ * sparse and we just seeked for the last part */
+ if (ftruncate(f->disk_fd, f->written_uncompressed) < 0) {
+ log_error_errno(errno, "Failed to truncate file: %m");
+ r = -errno;
+ goto fail;
+ }
+
+ r = raw_import_maybe_convert_qcow2(f);
+ if (r < 0)
+ goto fail;
+
if (f->etag)
(void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
if (f->url)
if (f->disk_fd < 0)
return log_error_errno(errno, "Failed to create %s: %m", f->temp_path);
+ r = chattr_fd(f->disk_fd, true, FS_NOCOW_FL);
+ if (r < 0)
+ log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
+
return 0;
}
-static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
- RawImportFile *f = userdata;
- size_t sz = size * nmemb;
+static int raw_import_file_write_uncompressed(RawImportFile *f, void *p, size_t sz) {
ssize_t n;
- int r;
- assert(contents);
assert(f);
+ assert(p);
+ assert(sz > 0);
+ assert(f->disk_fd >= 0);
- if (f->done) {
- r = -ESTALE;
- goto fail;
+ if (f->written_uncompressed + sz < f->written_uncompressed) {
+ log_error("File too large, overflow");
+ return -EOVERFLOW;
}
- r = raw_import_file_open_disk_for_write(f);
- if (r < 0)
- goto fail;
+ if (f->written_uncompressed + sz > RAW_MAX_SIZE) {
+ log_error("File overly large, refusing");
+ return -EFBIG;
+ }
- if (f->written + sz < f->written) {
+ n = sparse_write(f->disk_fd, p, sz, 64);
+ if (n < 0) {
+ log_error_errno(errno, "Failed to write file: %m");
+ return -errno;
+ }
+ if ((size_t) n < sz) {
+ log_error("Short write");
+ return -EIO;
+ }
+
+ f->written_uncompressed += sz;
+
+ return 0;
+}
+
+static int raw_import_file_write_compressed(RawImportFile *f, void *p, size_t sz) {
+ int r;
+
+ assert(f);
+ assert(p);
+ assert(sz > 0);
+ assert(f->disk_fd >= 0);
+
+ if (f->written_compressed + sz < f->written_compressed) {
log_error("File too large, overflow");
- r = -EOVERFLOW;
- goto fail;
+ return -EOVERFLOW;
}
if (f->content_length != (uint64_t) -1 &&
- f->written + sz > f->content_length) {
+ f->written_compressed + sz > f->content_length) {
log_error("Content length incorrect.");
- r = -EFBIG;
- goto fail;
+ return -EFBIG;
}
- n = write(f->disk_fd, contents, sz);
- if (n < 0) {
- log_error_errno(errno, "Failed to write file: %m");
- goto fail;
+ if (!f->compressed) {
+ r = raw_import_file_write_uncompressed(f, p, sz);
+ if (r < 0)
+ return r;
+ } else {
+ f->lzma.next_in = p;
+ f->lzma.avail_in = sz;
+
+ while (f->lzma.avail_in > 0) {
+ uint8_t buffer[16 * 1024];
+ lzma_ret lzr;
+
+ f->lzma.next_out = buffer;
+ f->lzma.avail_out = sizeof(buffer);
+
+ lzr = lzma_code(&f->lzma, LZMA_RUN);
+ if (lzr != LZMA_OK && lzr != LZMA_STREAM_END) {
+ log_error("Decompression error.");
+ return -EIO;
+ }
+
+ r = raw_import_file_write_uncompressed(f, buffer, sizeof(buffer) - f->lzma.avail_out);
+ if (r < 0)
+ return r;
+ }
}
- if ((size_t) n < sz) {
- log_error("Short write");
- r = -EIO;
+ f->written_compressed += sz;
+
+ return 0;
+}
+
+static int raw_import_file_detect_xz(RawImportFile *f) {
+ static const uint8_t xz_signature[] = {
+ '\xfd', '7', 'z', 'X', 'Z', '\x00'
+ };
+ lzma_ret lzr;
+ int r;
+
+ assert(f);
+
+ if (f->payload_size < sizeof(xz_signature))
+ return 0;
+
+ f->compressed = memcmp(f->payload, xz_signature, sizeof(xz_signature)) == 0;
+ log_debug("Stream is XZ compressed: %s", yes_no(f->compressed));
+
+ if (f->compressed) {
+ lzr = lzma_stream_decoder(&f->lzma, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK);
+ if (lzr != LZMA_OK) {
+ log_error("Failed to initialize LZMA decoder.");
+ return -EIO;
+ }
+ }
+
+ r = raw_import_file_open_disk_for_write(f);
+ if (r < 0)
+ return r;
+
+ r = raw_import_file_write_compressed(f, f->payload, f->payload_size);
+ if (r < 0)
+ return r;
+
+ free(f->payload);
+ f->payload = NULL;
+ f->payload_size = 0;
+
+ return 0;
+}
+
+static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
+ RawImportFile *f = userdata;
+ size_t sz = size * nmemb;
+ int r;
+
+ assert(contents);
+ assert(f);
+
+ if (f->done) {
+ r = -ESTALE;
goto fail;
}
- f->written += sz;
+ if (f->disk_fd < 0) {
+ uint8_t *p;
+
+ /* We haven't opened the file yet, let's first check what it actually is */
+
+ p = realloc(f->payload, f->payload_size + sz);
+ if (!p) {
+ r = log_oom();
+ goto fail;
+ }
+
+ memcpy(p + f->payload_size, contents, sz);
+ f->payload_size = sz;
+ f->payload = p;
+
+ r = raw_import_file_detect_xz(f);
+ if (r < 0)
+ goto fail;
+
+ return sz;
+ }
+
+ r = raw_import_file_write_compressed(f, contents, sz);
+ if (r < 0)
+ goto fail;
return sz;
}
if (r > 0) {
(void) safe_atou64(length, &f->content_length);
+
+ if (f->content_length != (uint64_t) -1) {
+ char bytes[FORMAT_BYTES_MAX];
+ log_info("Downloading %s.", format_bytes(bytes, sizeof(bytes), f->content_length));
+ }
+
return sz;
}
return 0;
}
+static int raw_import_file_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
+ RawImportFile *f = userdata;
+ unsigned percent;
+ usec_t n;
+
+ assert(f);
+
+ if (dltotal <= 0)
+ return 0;
+
+ percent = ((100 * dlnow) / dltotal);
+ n = now(CLOCK_MONOTONIC);
+
+ if (n > f->last_status_usec + USEC_PER_SEC &&
+ percent != f->progress_percent) {
+ char buf[FORMAT_TIMESPAN_MAX];
+
+ if (n - f->start_usec > USEC_PER_SEC && dlnow > 0) {
+ usec_t left, done;
+
+ done = n - f->start_usec;
+ left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done;
+
+ log_info("Got %u%%. %s left.", percent, format_timespan(buf, sizeof(buf), left, USEC_PER_SEC));
+ } else
+ log_info("Got %u%%.", percent);
+
+ f->progress_percent = percent;
+ f->last_status_usec = n;
+ }
+
+ return 0;
+}
+
static bool etag_is_valid(const char *etag) {
if (!endswith(etag, "\""))
if (curl_easy_setopt(f->curl, CURLOPT_HEADERDATA, f) != CURLE_OK)
return -EIO;
+ if (curl_easy_setopt(f->curl, CURLOPT_XFERINFOFUNCTION, raw_import_file_progress_callback) != CURLE_OK)
+ return -EIO;
+
+ if (curl_easy_setopt(f->curl, CURLOPT_XFERINFODATA, f) != CURLE_OK)
+ return -EIO;
+
+ if (curl_easy_setopt(f->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK)
+ return -EIO;
+
r = curl_glue_add(f->import->glue, f->curl);
if (r < 0)
return r;
f->import = import;
f->disk_fd = -1;
f->content_length = (uint64_t) -1;
+ f->start_usec = now(CLOCK_MONOTONIC);
f->url = strdup(url);
if (!f->url)
~ianmdlvl / elogind.git / blobdiff