1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <sys/xattr.h>
24 #include <curl/curl.h>
29 #include "curl-util.h"
30 #include "qcow2-util.h"
31 #include "import-raw.h"
35 typedef struct RawImportFile RawImportFile;
37 struct RawImportFile {
44 struct curl_slist *request_header;
51 uint64_t content_length;
52 uint64_t written_compressed;
53 uint64_t written_uncompressed;
76 raw_import_on_finished on_finished;
82 #define FILENAME_ESCAPE "/.#\"\'"
84 #define RAW_MAX_SIZE (1024LLU*1024LLU*1024LLU*8) /* 8 GB */
86 static RawImportFile *raw_import_file_unref(RawImportFile *f) {
91 curl_glue_remove_and_free(f->import->glue, f->curl);
92 curl_slist_free_all(f->request_header);
94 safe_close(f->disk_fd);
106 strv_free(f->old_etags);
113 DEFINE_TRIVIAL_CLEANUP_FUNC(RawImportFile*, raw_import_file_unref);
115 static void raw_import_finish(RawImport *import, int error) {
118 if (import->finished)
121 import->finished = true;
123 if (import->on_finished)
124 import->on_finished(import, error, import->userdata);
126 sd_event_exit(import->event, error);
129 static int raw_import_file_make_final_path(RawImportFile *f) {
130 _cleanup_free_ char *escaped_url = NULL, *escaped_etag = NULL;
137 escaped_url = xescape(f->url, FILENAME_ESCAPE);
142 escaped_etag = xescape(f->etag, FILENAME_ESCAPE);
146 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".", escaped_etag, ".raw", NULL);
148 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".raw", NULL);
155 static void raw_import_file_success(RawImportFile *f) {
163 _cleanup_free_ char *tp = NULL;
164 _cleanup_close_ int dfd = -1;
167 if (f->disk_fd >= 0) {
168 if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1) {
169 r = log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
173 r = raw_import_file_make_final_path(f);
179 f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
180 if (f->disk_fd < 0) {
181 r = log_error_errno(errno, "Failed to open vendor image: %m");
186 p = strappenda(f->import->image_root, "/", f->local, ".raw");
188 (void) rm_rf_dangerous(p, false, true, false);
190 r = tempfn_random(p, &tp);
196 dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
198 r = log_error_errno(errno, "Failed to create writable copy of image: %m");
202 /* Turn off COW writing. This should greatly improve
203 * performance on COW file systems like btrfs, since it
204 * reduces fragmentation caused by not allowing in-place
206 r = chattr_fd(dfd, true, FS_NOCOW_FL);
208 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
210 r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
212 log_error_errno(r, "Failed to make writable copy of image: %m");
217 (void) copy_times(f->disk_fd, dfd);
218 (void) copy_xattr(f->disk_fd, dfd);
220 dfd = safe_close(dfd);
224 r = log_error_errno(errno, "Failed to move writable image into place: %m");
229 log_info("Created new local image %s.", p);
232 f->disk_fd = safe_close(f->disk_fd);
236 raw_import_finish(f->import, r);
239 static int raw_import_maybe_convert_qcow2(RawImportFile *f) {
240 _cleanup_close_ int converted_fd = -1;
241 _cleanup_free_ char *t = NULL;
246 assert(f->temp_path);
248 r = qcow2_detect(f->disk_fd);
250 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
254 /* This is a QCOW2 image, let's convert it */
255 r = tempfn_random(f->final_path, &t);
259 converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
260 if (converted_fd < 0)
261 return log_error_errno(errno, "Failed to create %s: %m", t);
263 r = qcow2_convert(f->disk_fd, converted_fd);
266 return log_error_errno(r, "Failed to convert qcow2 image: %m");
269 unlink(f->temp_path);
275 safe_close(f->disk_fd);
276 f->disk_fd = converted_fd;
282 static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
283 RawImportFile *f = NULL;
289 if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, &f) != CURLE_OK)
297 if (result != CURLE_OK) {
298 log_error("Transfer failed: %s", curl_easy_strerror(result));
303 code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
304 if (code != CURLE_OK) {
305 log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
308 } else if (status == 304) {
309 log_info("Image already downloaded. Skipping download.");
310 raw_import_file_success(f);
312 } else if (status >= 300) {
313 log_error("HTTP request to %s failed with code %li.", f->url, status);
316 } else if (status < 200) {
317 log_error("HTTP request to %s finished with unexpected code %li.", f->url, status);
322 if (f->disk_fd < 0) {
323 log_error("No data received.");
328 if (f->content_length != (uint64_t) -1 &&
329 f->content_length != f->written_compressed) {
330 log_error("Download truncated.");
335 r = raw_import_maybe_convert_qcow2(f);
340 (void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
342 (void) fsetxattr(f->disk_fd, "user.source_url", f->url, strlen(f->url), 0);
345 struct timespec ut[2];
347 timespec_store(&ut[0], f->mtime);
349 (void) futimens(f->disk_fd, ut);
351 fd_setcrtime(f->disk_fd, f->mtime);
354 if (fstat(f->disk_fd, &st) < 0) {
355 r = log_error_errno(errno, "Failed to stat file: %m");
360 (void) fchmod(f->disk_fd, st.st_mode & 07444);
362 assert(f->temp_path);
363 assert(f->final_path);
365 r = rename(f->temp_path, f->final_path);
367 r = log_error_errno(errno, "Failed to move RAW file into place: %m");
374 log_info("Completed writing vendor image %s.", f->final_path);
376 raw_import_file_success(f);
380 raw_import_finish(f->import, r);
383 static int raw_import_file_open_disk_for_write(RawImportFile *f) {
391 r = raw_import_file_make_final_path(f);
396 r = tempfn_random(f->final_path, &f->temp_path);
401 f->disk_fd = open(f->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
403 return log_error_errno(errno, "Failed to create %s: %m", f->temp_path);
405 r = chattr_fd(f->disk_fd, true, FS_NOCOW_FL);
407 log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
412 static int raw_import_file_write_uncompressed(RawImportFile *f, void *p, size_t sz) {
418 assert(f->disk_fd >= 0);
420 if (f->written_uncompressed + sz < f->written_uncompressed) {
421 log_error("File too large, overflow");
425 if (f->written_uncompressed + sz > RAW_MAX_SIZE) {
426 log_error("File overly large, refusing");
430 n = write(f->disk_fd, p, sz);
432 log_error_errno(errno, "Failed to write file: %m");
435 if ((size_t) n < sz) {
436 log_error("Short write");
440 f->written_uncompressed += sz;
445 static int raw_import_file_write_compressed(RawImportFile *f, void *p, size_t sz) {
451 assert(f->disk_fd >= 0);
453 if (f->written_compressed + sz < f->written_compressed) {
454 log_error("File too large, overflow");
458 if (f->content_length != (uint64_t) -1 &&
459 f->written_compressed + sz > f->content_length) {
460 log_error("Content length incorrect.");
464 if (!f->compressed) {
465 r = raw_import_file_write_uncompressed(f, p, sz);
470 f->lzma.avail_in = sz;
472 while (f->lzma.avail_in > 0) {
473 uint8_t buffer[16 * 1024];
476 f->lzma.next_out = buffer;
477 f->lzma.avail_out = sizeof(buffer);
479 lzr = lzma_code(&f->lzma, LZMA_RUN);
480 if (lzr != LZMA_OK && lzr != LZMA_STREAM_END) {
481 log_error("Decompression error.");
485 r = raw_import_file_write_uncompressed(f, buffer, sizeof(buffer) - f->lzma.avail_out);
491 f->written_compressed += sz;
496 static int raw_import_file_detect_xz(RawImportFile *f) {
497 static const uint8_t xz_signature[] = {
498 '\xfd', '7', 'z', 'X', 'Z', '\x00'
505 if (f->payload_size < sizeof(xz_signature))
508 f->compressed = memcmp(f->payload, xz_signature, sizeof(xz_signature)) == 0;
509 log_debug("Stream is XZ compressed: %s", yes_no(f->compressed));
512 lzr = lzma_stream_decoder(&f->lzma, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK);
513 if (lzr != LZMA_OK) {
514 log_error("Failed to initialize LZMA decoder.");
519 r = raw_import_file_open_disk_for_write(f);
523 r = raw_import_file_write_compressed(f, f->payload, f->payload_size);
534 static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
535 RawImportFile *f = userdata;
536 size_t sz = size * nmemb;
547 if (f->disk_fd < 0) {
550 /* We haven't opened the file yet, let's first check what it actually is */
552 p = realloc(f->payload, f->payload_size + sz);
558 memcpy(p + f->payload_size, contents, sz);
559 f->payload_size = sz;
562 r = raw_import_file_detect_xz(f);
569 r = raw_import_file_write_compressed(f, contents, sz);
576 raw_import_finish(f->import, r);
580 static size_t raw_import_file_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
581 RawImportFile *f = userdata;
582 size_t sz = size * nmemb;
583 _cleanup_free_ char *length = NULL, *last_modified = NULL;
595 r = curl_header_strdup(contents, sz, "ETag:", &etag);
604 if (strv_contains(f->old_etags, f->etag)) {
605 log_info("Image already downloaded. Skipping download.");
606 raw_import_file_success(f);
613 r = curl_header_strdup(contents, sz, "Content-Length:", &length);
619 (void) safe_atou64(length, &f->content_length);
621 if (f->content_length != (uint64_t) -1) {
622 char bytes[FORMAT_BYTES_MAX];
623 log_info("Downloading %s.", format_bytes(bytes, sizeof(bytes), f->content_length));
629 r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
635 (void) curl_parse_http_time(last_modified, &f->mtime);
642 raw_import_finish(f->import, r);
646 static bool etag_is_valid(const char *etag) {
648 if (!endswith(etag, "\""))
651 if (!startswith(etag, "\"") && !startswith(etag, "W/\""))
657 static int raw_import_file_find_old_etags(RawImportFile *f) {
658 _cleanup_free_ char *escaped_url = NULL;
659 _cleanup_closedir_ DIR *d = NULL;
663 escaped_url = xescape(f->url, FILENAME_ESCAPE);
667 d = opendir(f->import->image_root);
675 FOREACH_DIRENT_ALL(de, d, return -errno) {
679 if (de->d_type != DT_UNKNOWN &&
680 de->d_type != DT_REG)
683 a = startswith(de->d_name, ".raw-");
687 a = startswith(a, escaped_url);
691 a = startswith(a, ".");
695 b = endswith(de->d_name, ".raw");
702 u = cunescape_length(a, b - a);
706 if (!etag_is_valid(u)) {
711 r = strv_consume(&f->old_etags, u);
719 static int raw_import_file_begin(RawImportFile *f) {
725 log_info("Getting %s.", f->url);
727 r = raw_import_file_find_old_etags(f);
731 r = curl_glue_make(&f->curl, f->url, f);
735 if (!strv_isempty(f->old_etags)) {
736 _cleanup_free_ char *cc = NULL, *hdr = NULL;
738 cc = strv_join(f->old_etags, ", ");
742 hdr = strappend("If-None-Match: ", cc);
746 f->request_header = curl_slist_new(hdr, NULL);
747 if (!f->request_header)
750 if (curl_easy_setopt(f->curl, CURLOPT_HTTPHEADER, f->request_header) != CURLE_OK)
754 if (curl_easy_setopt(f->curl, CURLOPT_WRITEFUNCTION, raw_import_file_write_callback) != CURLE_OK)
757 if (curl_easy_setopt(f->curl, CURLOPT_WRITEDATA, f) != CURLE_OK)
760 if (curl_easy_setopt(f->curl, CURLOPT_HEADERFUNCTION, raw_import_file_header_callback) != CURLE_OK)
763 if (curl_easy_setopt(f->curl, CURLOPT_HEADERDATA, f) != CURLE_OK)
766 r = curl_glue_add(f->import->glue, f->curl);
773 int raw_import_new(RawImport **import, sd_event *event, const char *image_root, raw_import_on_finished on_finished, void *userdata) {
774 _cleanup_(raw_import_unrefp) RawImport *i = NULL;
780 i = new0(RawImport, 1);
784 i->on_finished = on_finished;
785 i->userdata = userdata;
787 i->image_root = strdup(image_root);
792 i->event = sd_event_ref(event);
794 r = sd_event_default(&i->event);
799 r = curl_glue_new(&i->glue, i->event);
803 i->glue->on_finished = raw_import_curl_on_finished;
804 i->glue->userdata = i;
812 RawImport* raw_import_unref(RawImport *import) {
818 while ((f = hashmap_steal_first(import->files)))
819 raw_import_file_unref(f);
820 hashmap_free(import->files);
822 curl_glue_unref(import->glue);
823 sd_event_unref(import->event);
825 free(import->image_root);
831 int raw_import_cancel(RawImport *import, const char *url) {
837 f = hashmap_remove(import->files, url);
841 raw_import_file_unref(f);
845 int raw_import_pull(RawImport *import, const char *url, const char *local, bool force_local) {
846 _cleanup_(raw_import_file_unrefp) RawImportFile *f = NULL;
850 assert(raw_url_is_valid(url));
851 assert(!local || machine_name_is_valid(local));
853 if (hashmap_get(import->files, url))
856 r = hashmap_ensure_allocated(&import->files, &string_hash_ops);
860 f = new0(RawImportFile, 1);
866 f->content_length = (uint64_t) -1;
868 f->url = strdup(url);
873 f->local = strdup(local);
877 f->force_local = force_local;
880 r = hashmap_put(import->files, f->url, f);
884 r = raw_import_file_begin(f);
886 raw_import_cancel(import, f->url);
895 bool raw_url_is_valid(const char *url) {
899 if (!startswith(url, "http://") &&
900 !startswith(url, "https://"))
903 return ascii_is_valid(url);