1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <sys/xattr.h>
24 #include <curl/curl.h>
29 #include "curl-util.h"
30 #include "qcow2-util.h"
31 #include "import-raw.h"
35 typedef struct RawImportFile RawImportFile;
37 struct RawImportFile {
44 struct curl_slist *request_header;
51 uint64_t content_length;
52 uint64_t written_compressed;
53 uint64_t written_uncompressed;
68 unsigned progress_percent;
70 usec_t last_status_usec;
80 raw_import_on_finished on_finished;
86 #define FILENAME_ESCAPE "/.#\"\'"
88 #define RAW_MAX_SIZE (1024LLU*1024LLU*1024LLU*8) /* 8 GB */
90 static RawImportFile *raw_import_file_unref(RawImportFile *f) {
95 curl_glue_remove_and_free(f->import->glue, f->curl);
96 curl_slist_free_all(f->request_header);
98 safe_close(f->disk_fd);
103 unlink(f->temp_path);
111 strv_free(f->old_etags);
118 DEFINE_TRIVIAL_CLEANUP_FUNC(RawImportFile*, raw_import_file_unref);
120 static void raw_import_finish(RawImport *import, int error) {
123 if (import->finished)
126 import->finished = true;
128 if (import->on_finished)
129 import->on_finished(import, error, import->userdata);
131 sd_event_exit(import->event, error);
134 static int raw_import_file_make_final_path(RawImportFile *f) {
135 _cleanup_free_ char *escaped_url = NULL, *escaped_etag = NULL;
142 escaped_url = xescape(f->url, FILENAME_ESCAPE);
147 escaped_etag = xescape(f->etag, FILENAME_ESCAPE);
151 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".", escaped_etag, ".raw", NULL);
153 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".raw", NULL);
160 static int raw_import_file_make_local_copy(RawImportFile *f) {
161 _cleanup_free_ char *tp = NULL;
162 _cleanup_close_ int dfd = -1;
171 if (f->disk_fd >= 0) {
172 if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1)
173 return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
175 r = raw_import_file_make_final_path(f);
179 f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
181 return log_error_errno(errno, "Failed to open vendor image: %m");
184 p = strappenda(f->import->image_root, "/", f->local, ".raw");
186 (void) rm_rf_dangerous(p, false, true, false);
188 r = tempfn_random(p, &tp);
192 dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
194 return log_error_errno(errno, "Failed to create writable copy of image: %m");
196 /* Turn off COW writing. This should greatly improve
197 * performance on COW file systems like btrfs, since it
198 * reduces fragmentation caused by not allowing in-place
200 r = chattr_fd(dfd, true, FS_NOCOW_FL);
202 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
204 r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
207 return log_error_errno(r, "Failed to make writable copy of image: %m");
210 (void) copy_times(f->disk_fd, dfd);
211 (void) copy_xattr(f->disk_fd, dfd);
213 dfd = safe_close(dfd);
218 return log_error_errno(errno, "Failed to move writable image into place: %m");
221 log_info("Created new local image %s.", p);
225 static void raw_import_file_success(RawImportFile *f) {
232 r = raw_import_file_make_local_copy(f);
236 f->disk_fd = safe_close(f->disk_fd);
240 raw_import_finish(f->import, r);
243 static int raw_import_maybe_convert_qcow2(RawImportFile *f) {
244 _cleanup_close_ int converted_fd = -1;
245 _cleanup_free_ char *t = NULL;
250 assert(f->temp_path);
252 r = qcow2_detect(f->disk_fd);
254 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
258 /* This is a QCOW2 image, let's convert it */
259 r = tempfn_random(f->final_path, &t);
263 converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
264 if (converted_fd < 0)
265 return log_error_errno(errno, "Failed to create %s: %m", t);
267 r = qcow2_convert(f->disk_fd, converted_fd);
270 return log_error_errno(r, "Failed to convert qcow2 image: %m");
273 unlink(f->temp_path);
279 safe_close(f->disk_fd);
280 f->disk_fd = converted_fd;
286 static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
287 RawImportFile *f = NULL;
293 if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, &f) != CURLE_OK)
301 if (result != CURLE_OK) {
302 log_error("Transfer failed: %s", curl_easy_strerror(result));
307 code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
308 if (code != CURLE_OK) {
309 log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
312 } else if (status == 304) {
313 log_info("Image already downloaded. Skipping download.");
314 raw_import_file_success(f);
316 } else if (status >= 300) {
317 log_error("HTTP request to %s failed with code %li.", f->url, status);
320 } else if (status < 200) {
321 log_error("HTTP request to %s finished with unexpected code %li.", f->url, status);
326 if (f->disk_fd < 0) {
327 log_error("No data received.");
332 if (f->content_length != (uint64_t) -1 &&
333 f->content_length != f->written_compressed) {
334 log_error("Download truncated.");
339 /* Make sure the file size is right, in case the file was
340 * sparse and we just seeked for the last part */
341 if (ftruncate(f->disk_fd, f->written_uncompressed) < 0) {
342 log_error_errno(errno, "Failed to truncate file: %m");
347 r = raw_import_maybe_convert_qcow2(f);
352 (void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
354 (void) fsetxattr(f->disk_fd, "user.source_url", f->url, strlen(f->url), 0);
357 struct timespec ut[2];
359 timespec_store(&ut[0], f->mtime);
361 (void) futimens(f->disk_fd, ut);
363 fd_setcrtime(f->disk_fd, f->mtime);
366 if (fstat(f->disk_fd, &st) < 0) {
367 r = log_error_errno(errno, "Failed to stat file: %m");
372 (void) fchmod(f->disk_fd, st.st_mode & 07444);
374 assert(f->temp_path);
375 assert(f->final_path);
377 r = rename(f->temp_path, f->final_path);
379 r = log_error_errno(errno, "Failed to move RAW file into place: %m");
386 log_info("Completed writing vendor image %s.", f->final_path);
388 raw_import_file_success(f);
392 raw_import_finish(f->import, r);
395 static int raw_import_file_open_disk_for_write(RawImportFile *f) {
403 r = raw_import_file_make_final_path(f);
408 r = tempfn_random(f->final_path, &f->temp_path);
413 f->disk_fd = open(f->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
415 return log_error_errno(errno, "Failed to create %s: %m", f->temp_path);
417 r = chattr_fd(f->disk_fd, true, FS_NOCOW_FL);
419 log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
424 static int raw_import_file_write_uncompressed(RawImportFile *f, void *p, size_t sz) {
430 assert(f->disk_fd >= 0);
432 if (f->written_uncompressed + sz < f->written_uncompressed) {
433 log_error("File too large, overflow");
437 if (f->written_uncompressed + sz > RAW_MAX_SIZE) {
438 log_error("File overly large, refusing");
442 n = sparse_write(f->disk_fd, p, sz, 64);
444 log_error_errno(errno, "Failed to write file: %m");
447 if ((size_t) n < sz) {
448 log_error("Short write");
452 f->written_uncompressed += sz;
457 static int raw_import_file_write_compressed(RawImportFile *f, void *p, size_t sz) {
463 assert(f->disk_fd >= 0);
465 if (f->written_compressed + sz < f->written_compressed) {
466 log_error("File too large, overflow");
470 if (f->content_length != (uint64_t) -1 &&
471 f->written_compressed + sz > f->content_length) {
472 log_error("Content length incorrect.");
476 if (!f->compressed) {
477 r = raw_import_file_write_uncompressed(f, p, sz);
482 f->lzma.avail_in = sz;
484 while (f->lzma.avail_in > 0) {
485 uint8_t buffer[16 * 1024];
488 f->lzma.next_out = buffer;
489 f->lzma.avail_out = sizeof(buffer);
491 lzr = lzma_code(&f->lzma, LZMA_RUN);
492 if (lzr != LZMA_OK && lzr != LZMA_STREAM_END) {
493 log_error("Decompression error.");
497 r = raw_import_file_write_uncompressed(f, buffer, sizeof(buffer) - f->lzma.avail_out);
503 f->written_compressed += sz;
508 static int raw_import_file_detect_xz(RawImportFile *f) {
509 static const uint8_t xz_signature[] = {
510 '\xfd', '7', 'z', 'X', 'Z', '\x00'
517 if (f->payload_size < sizeof(xz_signature))
520 f->compressed = memcmp(f->payload, xz_signature, sizeof(xz_signature)) == 0;
521 log_debug("Stream is XZ compressed: %s", yes_no(f->compressed));
524 lzr = lzma_stream_decoder(&f->lzma, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK);
525 if (lzr != LZMA_OK) {
526 log_error("Failed to initialize LZMA decoder.");
531 r = raw_import_file_open_disk_for_write(f);
535 r = raw_import_file_write_compressed(f, f->payload, f->payload_size);
546 static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
547 RawImportFile *f = userdata;
548 size_t sz = size * nmemb;
559 if (f->disk_fd < 0) {
562 /* We haven't opened the file yet, let's first check what it actually is */
564 p = realloc(f->payload, f->payload_size + sz);
570 memcpy(p + f->payload_size, contents, sz);
571 f->payload_size = sz;
574 r = raw_import_file_detect_xz(f);
581 r = raw_import_file_write_compressed(f, contents, sz);
588 raw_import_finish(f->import, r);
592 static size_t raw_import_file_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
593 RawImportFile *f = userdata;
594 size_t sz = size * nmemb;
595 _cleanup_free_ char *length = NULL, *last_modified = NULL;
607 r = curl_header_strdup(contents, sz, "ETag:", &etag);
616 if (strv_contains(f->old_etags, f->etag)) {
617 log_info("Image already downloaded. Skipping download.");
618 raw_import_file_success(f);
625 r = curl_header_strdup(contents, sz, "Content-Length:", &length);
631 (void) safe_atou64(length, &f->content_length);
633 if (f->content_length != (uint64_t) -1) {
634 char bytes[FORMAT_BYTES_MAX];
635 log_info("Downloading %s.", format_bytes(bytes, sizeof(bytes), f->content_length));
641 r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
647 (void) curl_parse_http_time(last_modified, &f->mtime);
654 raw_import_finish(f->import, r);
658 static int raw_import_file_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
659 RawImportFile *f = userdata;
668 percent = ((100 * dlnow) / dltotal);
669 n = now(CLOCK_MONOTONIC);
671 if (n > f->last_status_usec + USEC_PER_SEC &&
672 percent != f->progress_percent) {
673 char buf[FORMAT_TIMESPAN_MAX];
675 if (n - f->start_usec > USEC_PER_SEC && dlnow > 0) {
678 done = n - f->start_usec;
679 left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done;
681 log_info("Got %u%%. %s left.", percent, format_timespan(buf, sizeof(buf), left, USEC_PER_SEC));
683 log_info("Got %u%%.", percent);
685 f->progress_percent = percent;
686 f->last_status_usec = n;
692 static bool etag_is_valid(const char *etag) {
694 if (!endswith(etag, "\""))
697 if (!startswith(etag, "\"") && !startswith(etag, "W/\""))
703 static int raw_import_file_find_old_etags(RawImportFile *f) {
704 _cleanup_free_ char *escaped_url = NULL;
705 _cleanup_closedir_ DIR *d = NULL;
709 escaped_url = xescape(f->url, FILENAME_ESCAPE);
713 d = opendir(f->import->image_root);
721 FOREACH_DIRENT_ALL(de, d, return -errno) {
725 if (de->d_type != DT_UNKNOWN &&
726 de->d_type != DT_REG)
729 a = startswith(de->d_name, ".raw-");
733 a = startswith(a, escaped_url);
737 a = startswith(a, ".");
741 b = endswith(de->d_name, ".raw");
748 u = cunescape_length(a, b - a);
752 if (!etag_is_valid(u)) {
757 r = strv_consume(&f->old_etags, u);
765 static int raw_import_file_begin(RawImportFile *f) {
771 log_info("Getting %s.", f->url);
773 r = raw_import_file_find_old_etags(f);
777 r = curl_glue_make(&f->curl, f->url, f);
781 if (!strv_isempty(f->old_etags)) {
782 _cleanup_free_ char *cc = NULL, *hdr = NULL;
784 cc = strv_join(f->old_etags, ", ");
788 hdr = strappend("If-None-Match: ", cc);
792 f->request_header = curl_slist_new(hdr, NULL);
793 if (!f->request_header)
796 if (curl_easy_setopt(f->curl, CURLOPT_HTTPHEADER, f->request_header) != CURLE_OK)
800 if (curl_easy_setopt(f->curl, CURLOPT_WRITEFUNCTION, raw_import_file_write_callback) != CURLE_OK)
803 if (curl_easy_setopt(f->curl, CURLOPT_WRITEDATA, f) != CURLE_OK)
806 if (curl_easy_setopt(f->curl, CURLOPT_HEADERFUNCTION, raw_import_file_header_callback) != CURLE_OK)
809 if (curl_easy_setopt(f->curl, CURLOPT_HEADERDATA, f) != CURLE_OK)
812 if (curl_easy_setopt(f->curl, CURLOPT_XFERINFOFUNCTION, raw_import_file_progress_callback) != CURLE_OK)
815 if (curl_easy_setopt(f->curl, CURLOPT_XFERINFODATA, f) != CURLE_OK)
818 if (curl_easy_setopt(f->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK)
821 r = curl_glue_add(f->import->glue, f->curl);
828 int raw_import_new(RawImport **import, sd_event *event, const char *image_root, raw_import_on_finished on_finished, void *userdata) {
829 _cleanup_(raw_import_unrefp) RawImport *i = NULL;
835 i = new0(RawImport, 1);
839 i->on_finished = on_finished;
840 i->userdata = userdata;
842 i->image_root = strdup(image_root);
847 i->event = sd_event_ref(event);
849 r = sd_event_default(&i->event);
854 r = curl_glue_new(&i->glue, i->event);
858 i->glue->on_finished = raw_import_curl_on_finished;
859 i->glue->userdata = i;
867 RawImport* raw_import_unref(RawImport *import) {
873 while ((f = hashmap_steal_first(import->files)))
874 raw_import_file_unref(f);
875 hashmap_free(import->files);
877 curl_glue_unref(import->glue);
878 sd_event_unref(import->event);
880 free(import->image_root);
886 int raw_import_cancel(RawImport *import, const char *url) {
892 f = hashmap_remove(import->files, url);
896 raw_import_file_unref(f);
900 int raw_import_pull(RawImport *import, const char *url, const char *local, bool force_local) {
901 _cleanup_(raw_import_file_unrefp) RawImportFile *f = NULL;
905 assert(raw_url_is_valid(url));
906 assert(!local || machine_name_is_valid(local));
908 if (hashmap_get(import->files, url))
911 r = hashmap_ensure_allocated(&import->files, &string_hash_ops);
915 f = new0(RawImportFile, 1);
921 f->content_length = (uint64_t) -1;
922 f->start_usec = now(CLOCK_MONOTONIC);
924 f->url = strdup(url);
929 f->local = strdup(local);
933 f->force_local = force_local;
936 r = hashmap_put(import->files, f->url, f);
940 r = raw_import_file_begin(f);
942 raw_import_cancel(import, f->url);
951 bool raw_url_is_valid(const char *url) {
955 if (!startswith(url, "http://") &&
956 !startswith(url, "https://"))
959 return ascii_is_valid(url);