1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <sys/xattr.h>
24 #include <curl/curl.h>
29 #include "curl-util.h"
30 #include "qcow2-util.h"
31 #include "import-raw.h"
35 typedef struct RawImportFile RawImportFile;
37 struct RawImportFile {
44 struct curl_slist *request_header;
51 uint64_t content_length;
52 uint64_t written_compressed;
53 uint64_t written_uncompressed;
76 raw_import_on_finished on_finished;
82 #define FILENAME_ESCAPE "/.#\"\'"
84 #define RAW_MAX_SIZE (1024LLU*1024LLU*1024LLU*8) /* 8 GB */
86 static RawImportFile *raw_import_file_unref(RawImportFile *f) {
91 curl_glue_remove_and_free(f->import->glue, f->curl);
92 curl_slist_free_all(f->request_header);
94 safe_close(f->disk_fd);
106 strv_free(f->old_etags);
113 DEFINE_TRIVIAL_CLEANUP_FUNC(RawImportFile*, raw_import_file_unref);
115 static void raw_import_finish(RawImport *import, int error) {
118 if (import->finished)
121 import->finished = true;
123 if (import->on_finished)
124 import->on_finished(import, error, import->userdata);
126 sd_event_exit(import->event, error);
129 static int raw_import_file_make_final_path(RawImportFile *f) {
130 _cleanup_free_ char *escaped_url = NULL, *escaped_etag = NULL;
137 escaped_url = xescape(f->url, FILENAME_ESCAPE);
142 escaped_etag = xescape(f->etag, FILENAME_ESCAPE);
146 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".", escaped_etag, ".raw", NULL);
148 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".raw", NULL);
155 static void raw_import_file_success(RawImportFile *f) {
163 _cleanup_free_ char *tp = NULL;
164 _cleanup_close_ int dfd = -1;
167 if (f->disk_fd >= 0) {
168 if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1) {
169 r = log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
173 r = raw_import_file_make_final_path(f);
179 f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
180 if (f->disk_fd < 0) {
181 r = log_error_errno(errno, "Failed to open vendor image: %m");
186 p = strappenda(f->import->image_root, "/", f->local, ".raw");
188 (void) rm_rf_dangerous(p, false, true, false);
190 r = tempfn_random(p, &tp);
196 dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
198 r = log_error_errno(errno, "Failed to create writable copy of image: %m");
202 /* Turn off COW writing. This should greatly improve
203 * performance on COW file systems like btrfs, since it
204 * reduces fragmentation caused by not allowing in-place
206 r = chattr_fd(dfd, true, FS_NOCOW_FL);
208 log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
210 r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
212 log_error_errno(r, "Failed to make writable copy of image: %m");
217 (void) copy_times(f->disk_fd, dfd);
218 (void) copy_xattr(f->disk_fd, dfd);
220 dfd = safe_close(dfd);
224 r = log_error_errno(errno, "Failed to move writable image into place: %m");
229 log_info("Created new local image %s.", p);
232 f->disk_fd = safe_close(f->disk_fd);
236 raw_import_finish(f->import, r);
239 static int raw_import_maybe_convert_qcow2(RawImportFile *f) {
240 _cleanup_close_ int converted_fd = -1;
241 _cleanup_free_ char *t = NULL;
246 assert(f->temp_path);
248 r = qcow2_detect(f->disk_fd);
250 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
254 /* This is a QCOW2 image, let's convert it */
255 r = tempfn_random(f->final_path, &t);
259 converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
260 if (converted_fd < 0)
261 return log_error_errno(errno, "Failed to create %s: %m", t);
263 r = qcow2_convert(f->disk_fd, converted_fd);
266 return log_error_errno(r, "Failed to convert qcow2 image: %m");
269 unlink(f->temp_path);
275 safe_close(f->disk_fd);
276 f->disk_fd = converted_fd;
282 static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
283 RawImportFile *f = NULL;
289 if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, &f) != CURLE_OK)
297 if (result != CURLE_OK) {
298 log_error("Transfer failed: %s", curl_easy_strerror(result));
303 code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
304 if (code != CURLE_OK) {
305 log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
308 } else if (status == 304) {
309 log_info("Image already downloaded. Skipping download.");
310 raw_import_file_success(f);
312 } else if (status >= 300) {
313 log_error("HTTP request to %s failed with code %li.", f->url, status);
316 } else if (status < 200) {
317 log_error("HTTP request to %s finished with unexpected code %li.", f->url, status);
322 if (f->disk_fd < 0) {
323 log_error("No data received.");
328 if (f->content_length != (uint64_t) -1 &&
329 f->content_length != f->written_compressed) {
330 log_error("Download truncated.");
335 r = raw_import_maybe_convert_qcow2(f);
340 (void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
342 (void) fsetxattr(f->disk_fd, "user.source_url", f->url, strlen(f->url), 0);
345 struct timespec ut[2];
347 timespec_store(&ut[0], f->mtime);
349 (void) futimens(f->disk_fd, ut);
351 fd_setcrtime(f->disk_fd, f->mtime);
354 if (fstat(f->disk_fd, &st) < 0) {
355 r = log_error_errno(errno, "Failed to stat file: %m");
360 (void) fchmod(f->disk_fd, st.st_mode & 07444);
362 assert(f->temp_path);
363 assert(f->final_path);
365 r = rename(f->temp_path, f->final_path);
367 r = log_error_errno(errno, "Failed to move RAW file into place: %m");
374 log_info("Completed writing vendor image %s.", f->final_path);
376 raw_import_file_success(f);
380 raw_import_finish(f->import, r);
383 static int raw_import_file_open_disk_for_write(RawImportFile *f) {
391 r = raw_import_file_make_final_path(f);
396 r = tempfn_random(f->final_path, &f->temp_path);
401 f->disk_fd = open(f->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
403 return log_error_errno(errno, "Failed to create %s: %m", f->temp_path);
408 static int raw_import_file_write_uncompressed(RawImportFile *f, void *p, size_t sz) {
414 assert(f->disk_fd >= 0);
416 if (f->written_uncompressed + sz < f->written_uncompressed) {
417 log_error("File too large, overflow");
421 if (f->written_uncompressed + sz > RAW_MAX_SIZE) {
422 log_error("File overly large, refusing");
426 n = write(f->disk_fd, p, sz);
428 log_error_errno(errno, "Failed to write file: %m");
431 if ((size_t) n < sz) {
432 log_error("Short write");
436 f->written_uncompressed += sz;
441 static int raw_import_file_write_compressed(RawImportFile *f, void *p, size_t sz) {
447 assert(f->disk_fd >= 0);
449 if (f->written_compressed + sz < f->written_compressed) {
450 log_error("File too large, overflow");
454 if (f->content_length != (uint64_t) -1 &&
455 f->written_compressed + sz > f->content_length) {
456 log_error("Content length incorrect.");
460 if (!f->compressed) {
461 r = raw_import_file_write_uncompressed(f, p, sz);
466 f->lzma.avail_in = sz;
468 while (f->lzma.avail_in > 0) {
469 uint8_t buffer[16 * 1024];
472 f->lzma.next_out = buffer;
473 f->lzma.avail_out = sizeof(buffer);
475 lzr = lzma_code(&f->lzma, LZMA_RUN);
476 if (lzr != LZMA_OK && lzr != LZMA_STREAM_END) {
477 log_error("Decompression error.");
481 r = raw_import_file_write_uncompressed(f, buffer, sizeof(buffer) - f->lzma.avail_out);
487 f->written_compressed += sz;
492 static int raw_import_file_detect_xz(RawImportFile *f) {
493 static const uint8_t xz_signature[] = {
494 '\xfd', '7', 'z', 'X', 'Z', '\x00'
501 if (f->payload_size < sizeof(xz_signature))
504 f->compressed = memcmp(f->payload, xz_signature, sizeof(xz_signature)) == 0;
505 log_debug("Stream is XZ compressed: %s", yes_no(f->compressed));
508 lzr = lzma_stream_decoder(&f->lzma, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK);
509 if (lzr != LZMA_OK) {
510 log_error("Failed to initialize LZMA decoder.");
515 r = raw_import_file_open_disk_for_write(f);
519 r = raw_import_file_write_compressed(f, f->payload, f->payload_size);
530 static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
531 RawImportFile *f = userdata;
532 size_t sz = size * nmemb;
543 if (f->disk_fd < 0) {
546 /* We haven't opened the file yet, let's first check what it actually is */
548 p = realloc(f->payload, f->payload_size + sz);
554 memcpy(p + f->payload_size, contents, sz);
555 f->payload_size = sz;
558 r = raw_import_file_detect_xz(f);
565 r = raw_import_file_write_compressed(f, contents, sz);
572 raw_import_finish(f->import, r);
576 static size_t raw_import_file_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
577 RawImportFile *f = userdata;
578 size_t sz = size * nmemb;
579 _cleanup_free_ char *length = NULL, *last_modified = NULL;
591 r = curl_header_strdup(contents, sz, "ETag:", &etag);
600 if (strv_contains(f->old_etags, f->etag)) {
601 log_info("Image already downloaded. Skipping download.");
602 raw_import_file_success(f);
609 r = curl_header_strdup(contents, sz, "Content-Length:", &length);
615 (void) safe_atou64(length, &f->content_length);
617 if (f->content_length != (uint64_t) -1) {
618 char bytes[FORMAT_BYTES_MAX];
619 log_info("Downloading %s.", format_bytes(bytes, sizeof(bytes), f->content_length));
625 r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
631 (void) curl_parse_http_time(last_modified, &f->mtime);
638 raw_import_finish(f->import, r);
642 static bool etag_is_valid(const char *etag) {
644 if (!endswith(etag, "\""))
647 if (!startswith(etag, "\"") && !startswith(etag, "W/\""))
653 static int raw_import_file_find_old_etags(RawImportFile *f) {
654 _cleanup_free_ char *escaped_url = NULL;
655 _cleanup_closedir_ DIR *d = NULL;
659 escaped_url = xescape(f->url, FILENAME_ESCAPE);
663 d = opendir(f->import->image_root);
671 FOREACH_DIRENT_ALL(de, d, return -errno) {
675 if (de->d_type != DT_UNKNOWN &&
676 de->d_type != DT_REG)
679 a = startswith(de->d_name, ".raw-");
683 a = startswith(a, escaped_url);
687 a = startswith(a, ".");
691 b = endswith(de->d_name, ".raw");
698 u = cunescape_length(a, b - a);
702 if (!etag_is_valid(u)) {
707 r = strv_consume(&f->old_etags, u);
715 static int raw_import_file_begin(RawImportFile *f) {
721 log_info("Getting %s.", f->url);
723 r = raw_import_file_find_old_etags(f);
727 r = curl_glue_make(&f->curl, f->url, f);
731 if (!strv_isempty(f->old_etags)) {
732 _cleanup_free_ char *cc = NULL, *hdr = NULL;
734 cc = strv_join(f->old_etags, ", ");
738 hdr = strappend("If-None-Match: ", cc);
742 f->request_header = curl_slist_new(hdr, NULL);
743 if (!f->request_header)
746 if (curl_easy_setopt(f->curl, CURLOPT_HTTPHEADER, f->request_header) != CURLE_OK)
750 if (curl_easy_setopt(f->curl, CURLOPT_WRITEFUNCTION, raw_import_file_write_callback) != CURLE_OK)
753 if (curl_easy_setopt(f->curl, CURLOPT_WRITEDATA, f) != CURLE_OK)
756 if (curl_easy_setopt(f->curl, CURLOPT_HEADERFUNCTION, raw_import_file_header_callback) != CURLE_OK)
759 if (curl_easy_setopt(f->curl, CURLOPT_HEADERDATA, f) != CURLE_OK)
762 r = curl_glue_add(f->import->glue, f->curl);
769 int raw_import_new(RawImport **import, sd_event *event, const char *image_root, raw_import_on_finished on_finished, void *userdata) {
770 _cleanup_(raw_import_unrefp) RawImport *i = NULL;
776 i = new0(RawImport, 1);
780 i->on_finished = on_finished;
781 i->userdata = userdata;
783 i->image_root = strdup(image_root);
788 i->event = sd_event_ref(event);
790 r = sd_event_default(&i->event);
795 r = curl_glue_new(&i->glue, i->event);
799 i->glue->on_finished = raw_import_curl_on_finished;
800 i->glue->userdata = i;
808 RawImport* raw_import_unref(RawImport *import) {
814 while ((f = hashmap_steal_first(import->files)))
815 raw_import_file_unref(f);
816 hashmap_free(import->files);
818 curl_glue_unref(import->glue);
819 sd_event_unref(import->event);
821 free(import->image_root);
827 int raw_import_cancel(RawImport *import, const char *url) {
833 f = hashmap_remove(import->files, url);
837 raw_import_file_unref(f);
841 int raw_import_pull(RawImport *import, const char *url, const char *local, bool force_local) {
842 _cleanup_(raw_import_file_unrefp) RawImportFile *f = NULL;
846 assert(raw_url_is_valid(url));
847 assert(!local || machine_name_is_valid(local));
849 if (hashmap_get(import->files, url))
852 r = hashmap_ensure_allocated(&import->files, &string_hash_ops);
856 f = new0(RawImportFile, 1);
862 f->content_length = (uint64_t) -1;
864 f->url = strdup(url);
869 f->local = strdup(local);
873 f->force_local = force_local;
876 r = hashmap_put(import->files, f->url, f);
880 r = raw_import_file_begin(f);
882 raw_import_cancel(import, f->url);
891 bool raw_url_is_valid(const char *url) {
895 if (!startswith(url, "http://") &&
896 !startswith(url, "https://"))
899 return ascii_is_valid(url);