1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <sys/xattr.h>
24 #include <curl/curl.h>
29 #include "curl-util.h"
30 #include "qcow2-util.h"
31 #include "import-raw.h"
35 typedef struct RawImportFile RawImportFile;
37 struct RawImportFile {
44 struct curl_slist *request_header;
51 uint64_t content_length;
52 uint64_t written_compressed;
53 uint64_t written_uncompressed;
68 unsigned progress_percent;
70 usec_t last_status_usec;
80 raw_import_on_finished on_finished;
86 #define FILENAME_ESCAPE "/.#\"\'"
88 #define RAW_MAX_SIZE (1024LLU*1024LLU*1024LLU*8) /* 8 GB */
90 static RawImportFile *raw_import_file_unref(RawImportFile *f) {
95 curl_glue_remove_and_free(f->import->glue, f->curl);
96 curl_slist_free_all(f->request_header);
98 safe_close(f->disk_fd);
103 unlink(f->temp_path);
111 strv_free(f->old_etags);
118 DEFINE_TRIVIAL_CLEANUP_FUNC(RawImportFile*, raw_import_file_unref);
120 static void raw_import_finish(RawImport *import, int error) {
123 if (import->finished)
126 import->finished = true;
128 if (import->on_finished)
129 import->on_finished(import, error, import->userdata);
131 sd_event_exit(import->event, error);
134 static int raw_import_file_make_final_path(RawImportFile *f) {
135 _cleanup_free_ char *escaped_url = NULL, *escaped_etag = NULL;
142 escaped_url = xescape(f->url, FILENAME_ESCAPE);
147 escaped_etag = xescape(f->etag, FILENAME_ESCAPE);
151 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".", escaped_etag, ".raw", NULL);
153 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".raw", NULL);
160 static int raw_import_file_make_local_copy(RawImportFile *f) {
161 _cleanup_free_ char *tp = NULL;
162 _cleanup_close_ int dfd = -1;
171 if (f->disk_fd >= 0) {
172 if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1)
173 return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
175 r = raw_import_file_make_final_path(f);
179 f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
181 return log_error_errno(errno, "Failed to open vendor image: %m");
184 p = strappenda(f->import->image_root, "/", f->local, ".raw");
186 (void) rm_rf_dangerous(p, false, true, false);
188 r = tempfn_random(p, &tp);
192 dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
194 return log_error_errno(errno, "Failed to create writable copy of image: %m");
196 /* Turn off COW writing. This should greatly improve
197 * performance on COW file systems like btrfs, since it
198 * reduces fragmentation caused by not allowing in-place
200 r = chattr_fd(dfd, true, FS_NOCOW_FL);
202 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
204 r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
207 return log_error_errno(r, "Failed to make writable copy of image: %m");
210 (void) copy_times(f->disk_fd, dfd);
211 (void) copy_xattr(f->disk_fd, dfd);
213 dfd = safe_close(dfd);
218 return log_error_errno(errno, "Failed to move writable image into place: %m");
221 log_info("Created new local image %s.", p);
225 static void raw_import_file_success(RawImportFile *f) {
232 r = raw_import_file_make_local_copy(f);
236 f->disk_fd = safe_close(f->disk_fd);
240 raw_import_finish(f->import, r);
243 static int raw_import_maybe_convert_qcow2(RawImportFile *f) {
244 _cleanup_close_ int converted_fd = -1;
245 _cleanup_free_ char *t = NULL;
250 assert(f->temp_path);
252 r = qcow2_detect(f->disk_fd);
254 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
258 /* This is a QCOW2 image, let's convert it */
259 r = tempfn_random(f->final_path, &t);
263 converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
264 if (converted_fd < 0)
265 return log_error_errno(errno, "Failed to create %s: %m", t);
267 log_info("Unpacking QCOW2 file.");
269 r = qcow2_convert(f->disk_fd, converted_fd);
272 return log_error_errno(r, "Failed to convert qcow2 image: %m");
275 unlink(f->temp_path);
281 safe_close(f->disk_fd);
282 f->disk_fd = converted_fd;
288 static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
289 RawImportFile *f = NULL;
295 if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, &f) != CURLE_OK)
303 if (result != CURLE_OK) {
304 log_error("Transfer failed: %s", curl_easy_strerror(result));
309 code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
310 if (code != CURLE_OK) {
311 log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
314 } else if (status == 304) {
315 log_info("Image already downloaded. Skipping download.");
316 raw_import_file_success(f);
318 } else if (status >= 300) {
319 log_error("HTTP request to %s failed with code %li.", f->url, status);
322 } else if (status < 200) {
323 log_error("HTTP request to %s finished with unexpected code %li.", f->url, status);
328 if (f->disk_fd < 0) {
329 log_error("No data received.");
334 if (f->content_length != (uint64_t) -1 &&
335 f->content_length != f->written_compressed) {
336 log_error("Download truncated.");
341 /* Make sure the file size is right, in case the file was
342 * sparse and we just seeked for the last part */
343 if (ftruncate(f->disk_fd, f->written_uncompressed) < 0) {
344 log_error_errno(errno, "Failed to truncate file: %m");
349 r = raw_import_maybe_convert_qcow2(f);
354 (void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
356 (void) fsetxattr(f->disk_fd, "user.source_url", f->url, strlen(f->url), 0);
359 struct timespec ut[2];
361 timespec_store(&ut[0], f->mtime);
363 (void) futimens(f->disk_fd, ut);
365 fd_setcrtime(f->disk_fd, f->mtime);
368 if (fstat(f->disk_fd, &st) < 0) {
369 r = log_error_errno(errno, "Failed to stat file: %m");
374 (void) fchmod(f->disk_fd, st.st_mode & 07444);
376 assert(f->temp_path);
377 assert(f->final_path);
379 r = rename(f->temp_path, f->final_path);
381 r = log_error_errno(errno, "Failed to move RAW file into place: %m");
388 log_info("Completed writing vendor image %s.", f->final_path);
390 raw_import_file_success(f);
394 raw_import_finish(f->import, r);
397 static int raw_import_file_open_disk_for_write(RawImportFile *f) {
405 r = raw_import_file_make_final_path(f);
410 r = tempfn_random(f->final_path, &f->temp_path);
415 f->disk_fd = open(f->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
417 return log_error_errno(errno, "Failed to create %s: %m", f->temp_path);
419 r = chattr_fd(f->disk_fd, true, FS_NOCOW_FL);
421 log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
426 static int raw_import_file_write_uncompressed(RawImportFile *f, void *p, size_t sz) {
432 assert(f->disk_fd >= 0);
434 if (f->written_uncompressed + sz < f->written_uncompressed) {
435 log_error("File too large, overflow");
439 if (f->written_uncompressed + sz > RAW_MAX_SIZE) {
440 log_error("File overly large, refusing");
444 n = sparse_write(f->disk_fd, p, sz, 64);
446 log_error_errno(errno, "Failed to write file: %m");
449 if ((size_t) n < sz) {
450 log_error("Short write");
454 f->written_uncompressed += sz;
459 static int raw_import_file_write_compressed(RawImportFile *f, void *p, size_t sz) {
465 assert(f->disk_fd >= 0);
467 if (f->written_compressed + sz < f->written_compressed) {
468 log_error("File too large, overflow");
472 if (f->content_length != (uint64_t) -1 &&
473 f->written_compressed + sz > f->content_length) {
474 log_error("Content length incorrect.");
478 if (!f->compressed) {
479 r = raw_import_file_write_uncompressed(f, p, sz);
484 f->lzma.avail_in = sz;
486 while (f->lzma.avail_in > 0) {
487 uint8_t buffer[16 * 1024];
490 f->lzma.next_out = buffer;
491 f->lzma.avail_out = sizeof(buffer);
493 lzr = lzma_code(&f->lzma, LZMA_RUN);
494 if (lzr != LZMA_OK && lzr != LZMA_STREAM_END) {
495 log_error("Decompression error.");
499 r = raw_import_file_write_uncompressed(f, buffer, sizeof(buffer) - f->lzma.avail_out);
505 f->written_compressed += sz;
510 static int raw_import_file_detect_xz(RawImportFile *f) {
511 static const uint8_t xz_signature[] = {
512 '\xfd', '7', 'z', 'X', 'Z', '\x00'
519 if (f->payload_size < sizeof(xz_signature))
522 f->compressed = memcmp(f->payload, xz_signature, sizeof(xz_signature)) == 0;
523 log_debug("Stream is XZ compressed: %s", yes_no(f->compressed));
526 lzr = lzma_stream_decoder(&f->lzma, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK);
527 if (lzr != LZMA_OK) {
528 log_error("Failed to initialize LZMA decoder.");
533 r = raw_import_file_open_disk_for_write(f);
537 r = raw_import_file_write_compressed(f, f->payload, f->payload_size);
548 static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
549 RawImportFile *f = userdata;
550 size_t sz = size * nmemb;
561 if (f->disk_fd < 0) {
564 /* We haven't opened the file yet, let's first check what it actually is */
566 p = realloc(f->payload, f->payload_size + sz);
572 memcpy(p + f->payload_size, contents, sz);
573 f->payload_size = sz;
576 r = raw_import_file_detect_xz(f);
583 r = raw_import_file_write_compressed(f, contents, sz);
590 raw_import_finish(f->import, r);
594 static size_t raw_import_file_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
595 RawImportFile *f = userdata;
596 size_t sz = size * nmemb;
597 _cleanup_free_ char *length = NULL, *last_modified = NULL;
609 r = curl_header_strdup(contents, sz, "ETag:", &etag);
618 if (strv_contains(f->old_etags, f->etag)) {
619 log_info("Image already downloaded. Skipping download.");
620 raw_import_file_success(f);
627 r = curl_header_strdup(contents, sz, "Content-Length:", &length);
633 (void) safe_atou64(length, &f->content_length);
635 if (f->content_length != (uint64_t) -1) {
636 char bytes[FORMAT_BYTES_MAX];
637 log_info("Downloading %s.", format_bytes(bytes, sizeof(bytes), f->content_length));
643 r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
649 (void) curl_parse_http_time(last_modified, &f->mtime);
656 raw_import_finish(f->import, r);
660 static int raw_import_file_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
661 RawImportFile *f = userdata;
670 percent = ((100 * dlnow) / dltotal);
671 n = now(CLOCK_MONOTONIC);
673 if (n > f->last_status_usec + USEC_PER_SEC &&
674 percent != f->progress_percent) {
675 char buf[FORMAT_TIMESPAN_MAX];
677 if (n - f->start_usec > USEC_PER_SEC && dlnow > 0) {
680 done = n - f->start_usec;
681 left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done;
683 log_info("Got %u%%. %s left.", percent, format_timespan(buf, sizeof(buf), left, USEC_PER_SEC));
685 log_info("Got %u%%.", percent);
687 f->progress_percent = percent;
688 f->last_status_usec = n;
694 static bool etag_is_valid(const char *etag) {
696 if (!endswith(etag, "\""))
699 if (!startswith(etag, "\"") && !startswith(etag, "W/\""))
705 static int raw_import_file_find_old_etags(RawImportFile *f) {
706 _cleanup_free_ char *escaped_url = NULL;
707 _cleanup_closedir_ DIR *d = NULL;
711 escaped_url = xescape(f->url, FILENAME_ESCAPE);
715 d = opendir(f->import->image_root);
723 FOREACH_DIRENT_ALL(de, d, return -errno) {
727 if (de->d_type != DT_UNKNOWN &&
728 de->d_type != DT_REG)
731 a = startswith(de->d_name, ".raw-");
735 a = startswith(a, escaped_url);
739 a = startswith(a, ".");
743 b = endswith(de->d_name, ".raw");
750 u = cunescape_length(a, b - a);
754 if (!etag_is_valid(u)) {
759 r = strv_consume(&f->old_etags, u);
767 static int raw_import_file_begin(RawImportFile *f) {
773 log_info("Getting %s.", f->url);
775 r = raw_import_file_find_old_etags(f);
779 r = curl_glue_make(&f->curl, f->url, f);
783 if (!strv_isempty(f->old_etags)) {
784 _cleanup_free_ char *cc = NULL, *hdr = NULL;
786 cc = strv_join(f->old_etags, ", ");
790 hdr = strappend("If-None-Match: ", cc);
794 f->request_header = curl_slist_new(hdr, NULL);
795 if (!f->request_header)
798 if (curl_easy_setopt(f->curl, CURLOPT_HTTPHEADER, f->request_header) != CURLE_OK)
802 if (curl_easy_setopt(f->curl, CURLOPT_WRITEFUNCTION, raw_import_file_write_callback) != CURLE_OK)
805 if (curl_easy_setopt(f->curl, CURLOPT_WRITEDATA, f) != CURLE_OK)
808 if (curl_easy_setopt(f->curl, CURLOPT_HEADERFUNCTION, raw_import_file_header_callback) != CURLE_OK)
811 if (curl_easy_setopt(f->curl, CURLOPT_HEADERDATA, f) != CURLE_OK)
814 if (curl_easy_setopt(f->curl, CURLOPT_XFERINFOFUNCTION, raw_import_file_progress_callback) != CURLE_OK)
817 if (curl_easy_setopt(f->curl, CURLOPT_XFERINFODATA, f) != CURLE_OK)
820 if (curl_easy_setopt(f->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK)
823 r = curl_glue_add(f->import->glue, f->curl);
830 int raw_import_new(RawImport **import, sd_event *event, const char *image_root, raw_import_on_finished on_finished, void *userdata) {
831 _cleanup_(raw_import_unrefp) RawImport *i = NULL;
837 i = new0(RawImport, 1);
841 i->on_finished = on_finished;
842 i->userdata = userdata;
844 i->image_root = strdup(image_root);
849 i->event = sd_event_ref(event);
851 r = sd_event_default(&i->event);
856 r = curl_glue_new(&i->glue, i->event);
860 i->glue->on_finished = raw_import_curl_on_finished;
861 i->glue->userdata = i;
869 RawImport* raw_import_unref(RawImport *import) {
875 while ((f = hashmap_steal_first(import->files)))
876 raw_import_file_unref(f);
877 hashmap_free(import->files);
879 curl_glue_unref(import->glue);
880 sd_event_unref(import->event);
882 free(import->image_root);
888 int raw_import_cancel(RawImport *import, const char *url) {
894 f = hashmap_remove(import->files, url);
898 raw_import_file_unref(f);
902 int raw_import_pull(RawImport *import, const char *url, const char *local, bool force_local) {
903 _cleanup_(raw_import_file_unrefp) RawImportFile *f = NULL;
907 assert(raw_url_is_valid(url));
908 assert(!local || machine_name_is_valid(local));
910 if (hashmap_get(import->files, url))
913 r = hashmap_ensure_allocated(&import->files, &string_hash_ops);
917 f = new0(RawImportFile, 1);
923 f->content_length = (uint64_t) -1;
924 f->start_usec = now(CLOCK_MONOTONIC);
926 f->url = strdup(url);
931 f->local = strdup(local);
935 f->force_local = force_local;
938 r = hashmap_put(import->files, f->url, f);
942 r = raw_import_file_begin(f);
944 raw_import_cancel(import, f->url);
953 bool raw_url_is_valid(const char *url) {
957 if (!startswith(url, "http://") &&
958 !startswith(url, "https://"))
961 return ascii_is_valid(url);