1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <sys/xattr.h>
24 #include <curl/curl.h>
29 #include "curl-util.h"
30 #include "qcow2-util.h"
33 #include "import-util.h"
34 #include "import-raw.h"
36 typedef struct RawImportFile RawImportFile;
38 struct RawImportFile {
45 struct curl_slist *request_header;
52 uint64_t content_length;
53 uint64_t written_compressed;
54 uint64_t written_uncompressed;
69 unsigned progress_percent;
71 usec_t last_status_usec;
81 raw_import_on_finished on_finished;
87 #define FILENAME_ESCAPE "/.#\"\'"
89 #define RAW_MAX_SIZE (1024LLU*1024LLU*1024LLU*8) /* 8 GB */
91 static RawImportFile *raw_import_file_unref(RawImportFile *f) {
96 curl_glue_remove_and_free(f->import->glue, f->curl);
97 curl_slist_free_all(f->request_header);
99 safe_close(f->disk_fd);
104 unlink(f->temp_path);
112 strv_free(f->old_etags);
119 DEFINE_TRIVIAL_CLEANUP_FUNC(RawImportFile*, raw_import_file_unref);
121 static void raw_import_finish(RawImport *import, int error) {
124 if (import->finished)
127 import->finished = true;
129 if (import->on_finished)
130 import->on_finished(import, error, import->userdata);
132 sd_event_exit(import->event, error);
135 static int raw_import_file_make_final_path(RawImportFile *f) {
136 _cleanup_free_ char *escaped_url = NULL, *escaped_etag = NULL;
143 escaped_url = xescape(f->url, FILENAME_ESCAPE);
148 escaped_etag = xescape(f->etag, FILENAME_ESCAPE);
152 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".", escaped_etag, ".raw", NULL);
154 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".raw", NULL);
161 static int raw_import_file_make_local_copy(RawImportFile *f) {
162 _cleanup_free_ char *tp = NULL;
163 _cleanup_close_ int dfd = -1;
172 if (f->disk_fd >= 0) {
173 if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1)
174 return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
176 r = raw_import_file_make_final_path(f);
180 f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
182 return log_error_errno(errno, "Failed to open vendor image: %m");
185 p = strappenda(f->import->image_root, "/", f->local, ".raw");
187 (void) rm_rf_dangerous(p, false, true, false);
189 r = tempfn_random(p, &tp);
193 dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
195 return log_error_errno(errno, "Failed to create writable copy of image: %m");
197 /* Turn off COW writing. This should greatly improve
198 * performance on COW file systems like btrfs, since it
199 * reduces fragmentation caused by not allowing in-place
201 r = chattr_fd(dfd, true, FS_NOCOW_FL);
203 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
205 r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
208 return log_error_errno(r, "Failed to make writable copy of image: %m");
211 (void) copy_times(f->disk_fd, dfd);
212 (void) copy_xattr(f->disk_fd, dfd);
214 dfd = safe_close(dfd);
219 return log_error_errno(errno, "Failed to move writable image into place: %m");
222 log_info("Created new local image %s.", p);
226 static void raw_import_file_success(RawImportFile *f) {
233 r = raw_import_file_make_local_copy(f);
237 f->disk_fd = safe_close(f->disk_fd);
241 raw_import_finish(f->import, r);
244 static int raw_import_maybe_convert_qcow2(RawImportFile *f) {
245 _cleanup_close_ int converted_fd = -1;
246 _cleanup_free_ char *t = NULL;
251 assert(f->temp_path);
253 r = qcow2_detect(f->disk_fd);
255 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
259 /* This is a QCOW2 image, let's convert it */
260 r = tempfn_random(f->final_path, &t);
264 converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
265 if (converted_fd < 0)
266 return log_error_errno(errno, "Failed to create %s: %m", t);
268 log_info("Unpacking QCOW2 file.");
270 r = qcow2_convert(f->disk_fd, converted_fd);
273 return log_error_errno(r, "Failed to convert qcow2 image: %m");
276 unlink(f->temp_path);
282 safe_close(f->disk_fd);
283 f->disk_fd = converted_fd;
289 static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
290 RawImportFile *f = NULL;
296 if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, &f) != CURLE_OK)
304 if (result != CURLE_OK) {
305 log_error("Transfer failed: %s", curl_easy_strerror(result));
310 code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
311 if (code != CURLE_OK) {
312 log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
315 } else if (status == 304) {
316 log_info("Image already downloaded. Skipping download.");
317 raw_import_file_success(f);
319 } else if (status >= 300) {
320 log_error("HTTP request to %s failed with code %li.", f->url, status);
323 } else if (status < 200) {
324 log_error("HTTP request to %s finished with unexpected code %li.", f->url, status);
329 if (f->disk_fd < 0) {
330 log_error("No data received.");
335 if (f->content_length != (uint64_t) -1 &&
336 f->content_length != f->written_compressed) {
337 log_error("Download truncated.");
342 /* Make sure the file size is right, in case the file was
343 * sparse and we just seeked for the last part */
344 if (ftruncate(f->disk_fd, f->written_uncompressed) < 0) {
345 log_error_errno(errno, "Failed to truncate file: %m");
350 r = raw_import_maybe_convert_qcow2(f);
355 (void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
357 (void) fsetxattr(f->disk_fd, "user.source_url", f->url, strlen(f->url), 0);
360 struct timespec ut[2];
362 timespec_store(&ut[0], f->mtime);
364 (void) futimens(f->disk_fd, ut);
366 fd_setcrtime(f->disk_fd, f->mtime);
369 if (fstat(f->disk_fd, &st) < 0) {
370 r = log_error_errno(errno, "Failed to stat file: %m");
375 (void) fchmod(f->disk_fd, st.st_mode & 07444);
377 assert(f->temp_path);
378 assert(f->final_path);
380 r = rename(f->temp_path, f->final_path);
382 r = log_error_errno(errno, "Failed to move RAW file into place: %m");
389 log_info("Completed writing vendor image %s.", f->final_path);
391 raw_import_file_success(f);
395 raw_import_finish(f->import, r);
398 static int raw_import_file_open_disk_for_write(RawImportFile *f) {
406 r = raw_import_file_make_final_path(f);
411 r = tempfn_random(f->final_path, &f->temp_path);
416 f->disk_fd = open(f->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
418 return log_error_errno(errno, "Failed to create %s: %m", f->temp_path);
420 r = chattr_fd(f->disk_fd, true, FS_NOCOW_FL);
422 log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
427 static int raw_import_file_write_uncompressed(RawImportFile *f, void *p, size_t sz) {
433 assert(f->disk_fd >= 0);
435 if (f->written_uncompressed + sz < f->written_uncompressed) {
436 log_error("File too large, overflow");
440 if (f->written_uncompressed + sz > RAW_MAX_SIZE) {
441 log_error("File overly large, refusing");
445 n = sparse_write(f->disk_fd, p, sz, 64);
447 log_error_errno(errno, "Failed to write file: %m");
450 if ((size_t) n < sz) {
451 log_error("Short write");
455 f->written_uncompressed += sz;
460 static int raw_import_file_write_compressed(RawImportFile *f, void *p, size_t sz) {
466 assert(f->disk_fd >= 0);
468 if (f->written_compressed + sz < f->written_compressed) {
469 log_error("File too large, overflow");
473 if (f->content_length != (uint64_t) -1 &&
474 f->written_compressed + sz > f->content_length) {
475 log_error("Content length incorrect.");
479 if (!f->compressed) {
480 r = raw_import_file_write_uncompressed(f, p, sz);
485 f->lzma.avail_in = sz;
487 while (f->lzma.avail_in > 0) {
488 uint8_t buffer[16 * 1024];
491 f->lzma.next_out = buffer;
492 f->lzma.avail_out = sizeof(buffer);
494 lzr = lzma_code(&f->lzma, LZMA_RUN);
495 if (lzr != LZMA_OK && lzr != LZMA_STREAM_END) {
496 log_error("Decompression error.");
500 r = raw_import_file_write_uncompressed(f, buffer, sizeof(buffer) - f->lzma.avail_out);
506 f->written_compressed += sz;
511 static int raw_import_file_detect_xz(RawImportFile *f) {
512 static const uint8_t xz_signature[] = {
513 '\xfd', '7', 'z', 'X', 'Z', '\x00'
520 if (f->payload_size < sizeof(xz_signature))
523 f->compressed = memcmp(f->payload, xz_signature, sizeof(xz_signature)) == 0;
524 log_debug("Stream is XZ compressed: %s", yes_no(f->compressed));
527 lzr = lzma_stream_decoder(&f->lzma, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK);
528 if (lzr != LZMA_OK) {
529 log_error("Failed to initialize LZMA decoder.");
534 r = raw_import_file_open_disk_for_write(f);
538 r = raw_import_file_write_compressed(f, f->payload, f->payload_size);
549 static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
550 RawImportFile *f = userdata;
551 size_t sz = size * nmemb;
562 if (f->disk_fd < 0) {
565 /* We haven't opened the file yet, let's first check what it actually is */
567 p = realloc(f->payload, f->payload_size + sz);
573 memcpy(p + f->payload_size, contents, sz);
574 f->payload_size = sz;
577 r = raw_import_file_detect_xz(f);
584 r = raw_import_file_write_compressed(f, contents, sz);
591 raw_import_finish(f->import, r);
595 static size_t raw_import_file_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
596 RawImportFile *f = userdata;
597 size_t sz = size * nmemb;
598 _cleanup_free_ char *length = NULL, *last_modified = NULL;
610 r = curl_header_strdup(contents, sz, "ETag:", &etag);
619 if (strv_contains(f->old_etags, f->etag)) {
620 log_info("Image already downloaded. Skipping download.");
621 raw_import_file_success(f);
628 r = curl_header_strdup(contents, sz, "Content-Length:", &length);
634 (void) safe_atou64(length, &f->content_length);
636 if (f->content_length != (uint64_t) -1) {
637 char bytes[FORMAT_BYTES_MAX];
638 log_info("Downloading %s.", format_bytes(bytes, sizeof(bytes), f->content_length));
644 r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
650 (void) curl_parse_http_time(last_modified, &f->mtime);
657 raw_import_finish(f->import, r);
661 static int raw_import_file_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
662 RawImportFile *f = userdata;
671 percent = ((100 * dlnow) / dltotal);
672 n = now(CLOCK_MONOTONIC);
674 if (n > f->last_status_usec + USEC_PER_SEC &&
675 percent != f->progress_percent) {
676 char buf[FORMAT_TIMESPAN_MAX];
678 if (n - f->start_usec > USEC_PER_SEC && dlnow > 0) {
681 done = n - f->start_usec;
682 left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done;
684 log_info("Got %u%%. %s left.", percent, format_timespan(buf, sizeof(buf), left, USEC_PER_SEC));
686 log_info("Got %u%%.", percent);
688 f->progress_percent = percent;
689 f->last_status_usec = n;
695 static int raw_import_file_find_old_etags(RawImportFile *f) {
696 _cleanup_free_ char *escaped_url = NULL;
697 _cleanup_closedir_ DIR *d = NULL;
701 escaped_url = xescape(f->url, FILENAME_ESCAPE);
705 d = opendir(f->import->image_root);
713 FOREACH_DIRENT_ALL(de, d, return -errno) {
717 if (de->d_type != DT_UNKNOWN &&
718 de->d_type != DT_REG)
721 a = startswith(de->d_name, ".raw-");
725 a = startswith(a, escaped_url);
729 a = startswith(a, ".");
733 b = endswith(de->d_name, ".raw");
740 u = cunescape_length(a, b - a);
744 if (!http_etag_is_valid(u)) {
749 r = strv_consume(&f->old_etags, u);
757 static int raw_import_file_begin(RawImportFile *f) {
763 log_info("Getting %s.", f->url);
765 r = raw_import_file_find_old_etags(f);
769 r = curl_glue_make(&f->curl, f->url, f);
773 if (!strv_isempty(f->old_etags)) {
774 _cleanup_free_ char *cc = NULL, *hdr = NULL;
776 cc = strv_join(f->old_etags, ", ");
780 hdr = strappend("If-None-Match: ", cc);
784 f->request_header = curl_slist_new(hdr, NULL);
785 if (!f->request_header)
788 if (curl_easy_setopt(f->curl, CURLOPT_HTTPHEADER, f->request_header) != CURLE_OK)
792 if (curl_easy_setopt(f->curl, CURLOPT_WRITEFUNCTION, raw_import_file_write_callback) != CURLE_OK)
795 if (curl_easy_setopt(f->curl, CURLOPT_WRITEDATA, f) != CURLE_OK)
798 if (curl_easy_setopt(f->curl, CURLOPT_HEADERFUNCTION, raw_import_file_header_callback) != CURLE_OK)
801 if (curl_easy_setopt(f->curl, CURLOPT_HEADERDATA, f) != CURLE_OK)
804 if (curl_easy_setopt(f->curl, CURLOPT_XFERINFOFUNCTION, raw_import_file_progress_callback) != CURLE_OK)
807 if (curl_easy_setopt(f->curl, CURLOPT_XFERINFODATA, f) != CURLE_OK)
810 if (curl_easy_setopt(f->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK)
813 r = curl_glue_add(f->import->glue, f->curl);
820 int raw_import_new(RawImport **import, sd_event *event, const char *image_root, raw_import_on_finished on_finished, void *userdata) {
821 _cleanup_(raw_import_unrefp) RawImport *i = NULL;
827 i = new0(RawImport, 1);
831 i->on_finished = on_finished;
832 i->userdata = userdata;
834 i->image_root = strdup(image_root);
839 i->event = sd_event_ref(event);
841 r = sd_event_default(&i->event);
846 r = curl_glue_new(&i->glue, i->event);
850 i->glue->on_finished = raw_import_curl_on_finished;
851 i->glue->userdata = i;
859 RawImport* raw_import_unref(RawImport *import) {
865 while ((f = hashmap_steal_first(import->files)))
866 raw_import_file_unref(f);
867 hashmap_free(import->files);
869 curl_glue_unref(import->glue);
870 sd_event_unref(import->event);
872 free(import->image_root);
878 int raw_import_cancel(RawImport *import, const char *url) {
884 f = hashmap_remove(import->files, url);
888 raw_import_file_unref(f);
892 int raw_import_pull(RawImport *import, const char *url, const char *local, bool force_local) {
893 _cleanup_(raw_import_file_unrefp) RawImportFile *f = NULL;
897 assert(http_url_is_valid(url));
898 assert(!local || machine_name_is_valid(local));
900 if (hashmap_get(import->files, url))
903 r = hashmap_ensure_allocated(&import->files, &string_hash_ops);
907 f = new0(RawImportFile, 1);
913 f->content_length = (uint64_t) -1;
914 f->start_usec = now(CLOCK_MONOTONIC);
916 f->url = strdup(url);
921 f->local = strdup(local);
925 f->force_local = force_local;
928 r = hashmap_put(import->files, f->url, f);
932 r = raw_import_file_begin(f);
934 raw_import_cancel(import, f->url);