1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <sys/xattr.h>
24 #include <curl/curl.h>
29 #include "curl-util.h"
30 #include "import-raw.h"
34 typedef struct RawImportFile RawImportFile;
36 struct RawImportFile {
43 struct curl_slist *request_header;
50 uint64_t content_length;
51 uint64_t written_compressed;
52 uint64_t written_uncompressed;
75 raw_import_on_finished on_finished;
81 #define FILENAME_ESCAPE "/.#\"\'"
83 #define RAW_MAX_SIZE (1024LLU*1024LLU*1024LLU*8) /* 8 GB */
85 static RawImportFile *raw_import_file_unref(RawImportFile *f) {
90 curl_glue_remove_and_free(f->import->glue, f->curl);
91 curl_slist_free_all(f->request_header);
93 safe_close(f->disk_fd);
105 strv_free(f->old_etags);
112 DEFINE_TRIVIAL_CLEANUP_FUNC(RawImportFile*, raw_import_file_unref);
114 static void raw_import_finish(RawImport *import, int error) {
117 if (import->finished)
120 import->finished = true;
122 if (import->on_finished)
123 import->on_finished(import, error, import->userdata);
125 sd_event_exit(import->event, error);
128 static int raw_import_file_make_final_path(RawImportFile *f) {
129 _cleanup_free_ char *escaped_url = NULL, *escaped_etag = NULL;
136 escaped_url = xescape(f->url, FILENAME_ESCAPE);
141 escaped_etag = xescape(f->etag, FILENAME_ESCAPE);
145 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".", escaped_etag, ".raw", NULL);
147 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".raw", NULL);
154 static void raw_import_file_success(RawImportFile *f) {
162 _cleanup_free_ char *tp = NULL;
163 _cleanup_close_ int dfd = -1;
166 if (f->disk_fd >= 0) {
167 if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1) {
168 r = log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
172 r = raw_import_file_make_final_path(f);
178 f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
179 if (f->disk_fd < 0) {
180 r = log_error_errno(errno, "Failed to open vendor image: %m");
185 p = strappenda(f->import->image_root, "/", f->local, ".raw");
187 (void) rm_rf_dangerous(p, false, true, false);
189 r = tempfn_random(p, &tp);
195 dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
197 r = log_error_errno(errno, "Failed to create writable copy of image: %m");
201 /* Turn off COW writing. This should greatly improve
202 * performance on COW file systems like btrfs, since it
203 * reduces fragmentation caused by not allowing in-place
205 r = chattr_fd(dfd, true, FS_NOCOW_FL);
207 log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
209 r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
211 log_error_errno(r, "Failed to make writable copy of image: %m");
216 (void) copy_times(f->disk_fd, dfd);
217 (void) copy_xattr(f->disk_fd, dfd);
219 dfd = safe_close(dfd);
223 r = log_error_errno(errno, "Failed to move writable image into place: %m");
228 log_info("Created new local image %s.", p);
231 f->disk_fd = safe_close(f->disk_fd);
235 raw_import_finish(f->import, r);
238 static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
239 RawImportFile *f = NULL;
245 if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, &f) != CURLE_OK)
253 if (result != CURLE_OK) {
254 log_error("Transfer failed: %s", curl_easy_strerror(result));
259 code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
260 if (code != CURLE_OK) {
261 log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
264 } else if (status == 304) {
265 log_info("Image already downloaded. Skipping download.");
266 raw_import_file_success(f);
268 } else if (status >= 300) {
269 log_error("HTTP request to %s failed with code %li.", f->url, status);
272 } else if (status < 200) {
273 log_error("HTTP request to %s finished with unexpected code %li.", f->url, status);
278 if (f->disk_fd < 0) {
279 log_error("No data received.");
284 if (f->content_length != (uint64_t) -1 &&
285 f->content_length != f->written_compressed) {
286 log_error("Download truncated.");
292 (void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
294 (void) fsetxattr(f->disk_fd, "user.source_url", f->url, strlen(f->url), 0);
297 struct timespec ut[2];
299 timespec_store(&ut[0], f->mtime);
301 (void) futimens(f->disk_fd, ut);
303 fd_setcrtime(f->disk_fd, f->mtime);
306 if (fstat(f->disk_fd, &st) < 0) {
307 r = log_error_errno(errno, "Failed to stat file: %m");
312 (void) fchmod(f->disk_fd, st.st_mode & 07444);
314 assert(f->temp_path);
315 assert(f->final_path);
317 r = rename(f->temp_path, f->final_path);
319 r = log_error_errno(errno, "Failed to move RAW file into place: %m");
326 log_info("Completed writing vendor image %s.", f->final_path);
328 raw_import_file_success(f);
332 raw_import_finish(f->import, r);
335 static int raw_import_file_open_disk_for_write(RawImportFile *f) {
343 r = raw_import_file_make_final_path(f);
348 r = tempfn_random(f->final_path, &f->temp_path);
353 f->disk_fd = open(f->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
355 return log_error_errno(errno, "Failed to create %s: %m", f->temp_path);
360 static int raw_import_file_write_uncompressed(RawImportFile *f, void *p, size_t sz) {
366 assert(f->disk_fd >= 0);
368 if (f->written_uncompressed + sz < f->written_uncompressed) {
369 log_error("File too large, overflow");
373 if (f->written_uncompressed + sz > RAW_MAX_SIZE) {
374 log_error("File overly large, refusing");
378 n = write(f->disk_fd, p, sz);
380 log_error_errno(errno, "Failed to write file: %m");
383 if ((size_t) n < sz) {
384 log_error("Short write");
388 f->written_uncompressed += sz;
393 static int raw_import_file_write_compressed(RawImportFile *f, void *p, size_t sz) {
399 assert(f->disk_fd >= 0);
401 if (f->written_compressed + sz < f->written_compressed) {
402 log_error("File too large, overflow");
406 if (f->content_length != (uint64_t) -1 &&
407 f->written_compressed + sz > f->content_length) {
408 log_error("Content length incorrect.");
412 if (!f->compressed) {
413 r = raw_import_file_write_uncompressed(f, p, sz);
418 f->lzma.avail_in = sz;
420 while (f->lzma.avail_in > 0) {
421 uint8_t buffer[16 * 1024];
424 f->lzma.next_out = buffer;
425 f->lzma.avail_out = sizeof(buffer);
427 lzr = lzma_code(&f->lzma, LZMA_RUN);
428 if (lzr != LZMA_OK && lzr != LZMA_STREAM_END) {
429 log_error("Decompression error.");
433 r = raw_import_file_write_uncompressed(f, buffer, sizeof(buffer) - f->lzma.avail_out);
439 f->written_compressed += sz;
444 static int raw_import_file_detect_xz(RawImportFile *f) {
445 static const uint8_t xz_signature[] = {
446 '\xfd', '7', 'z', 'X', 'Z', '\x00'
453 if (f->payload_size < sizeof(xz_signature))
456 f->compressed = memcmp(f->payload, xz_signature, sizeof(xz_signature)) == 0;
457 log_debug("Stream is XZ compressed: %s", yes_no(f->compressed));
460 lzr = lzma_stream_decoder(&f->lzma, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK);
461 if (lzr != LZMA_OK) {
462 log_error("Failed to initialize LZMA decoder.");
467 r = raw_import_file_open_disk_for_write(f);
471 r = raw_import_file_write_compressed(f, f->payload, f->payload_size);
482 static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
483 RawImportFile *f = userdata;
484 size_t sz = size * nmemb;
495 if (f->disk_fd < 0) {
498 /* We haven't opened the file yet, let's first check what it actually is */
500 p = realloc(f->payload, f->payload_size + sz);
506 memcpy(p + f->payload_size, contents, sz);
507 f->payload_size = sz;
510 r = raw_import_file_detect_xz(f);
517 r = raw_import_file_write_compressed(f, contents, sz);
524 raw_import_finish(f->import, r);
528 static size_t raw_import_file_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
529 RawImportFile *f = userdata;
530 size_t sz = size * nmemb;
531 _cleanup_free_ char *length = NULL, *last_modified = NULL;
543 r = curl_header_strdup(contents, sz, "ETag:", &etag);
552 if (strv_contains(f->old_etags, f->etag)) {
553 log_info("Image already downloaded. Skipping download.");
554 raw_import_file_success(f);
561 r = curl_header_strdup(contents, sz, "Content-Length:", &length);
567 (void) safe_atou64(length, &f->content_length);
569 if (f->content_length != (uint64_t) -1) {
570 char bytes[FORMAT_BYTES_MAX];
571 log_info("Downloading %s.", format_bytes(bytes, sizeof(bytes), f->content_length));
577 r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
583 (void) curl_parse_http_time(last_modified, &f->mtime);
590 raw_import_finish(f->import, r);
594 static bool etag_is_valid(const char *etag) {
596 if (!endswith(etag, "\""))
599 if (!startswith(etag, "\"") && !startswith(etag, "W/\""))
605 static int raw_import_file_find_old_etags(RawImportFile *f) {
606 _cleanup_free_ char *escaped_url = NULL;
607 _cleanup_closedir_ DIR *d = NULL;
611 escaped_url = xescape(f->url, FILENAME_ESCAPE);
615 d = opendir(f->import->image_root);
623 FOREACH_DIRENT_ALL(de, d, return -errno) {
627 if (de->d_type != DT_UNKNOWN &&
628 de->d_type != DT_REG)
631 a = startswith(de->d_name, ".raw-");
635 a = startswith(a, escaped_url);
639 a = startswith(a, ".");
643 b = endswith(de->d_name, ".raw");
650 u = cunescape_length(a, b - a);
654 if (!etag_is_valid(u)) {
659 r = strv_consume(&f->old_etags, u);
667 static int raw_import_file_begin(RawImportFile *f) {
673 log_info("Getting %s.", f->url);
675 r = raw_import_file_find_old_etags(f);
679 r = curl_glue_make(&f->curl, f->url, f);
683 if (!strv_isempty(f->old_etags)) {
684 _cleanup_free_ char *cc = NULL, *hdr = NULL;
686 cc = strv_join(f->old_etags, ", ");
690 hdr = strappend("If-None-Match: ", cc);
694 f->request_header = curl_slist_new(hdr, NULL);
695 if (!f->request_header)
698 if (curl_easy_setopt(f->curl, CURLOPT_HTTPHEADER, f->request_header) != CURLE_OK)
702 if (curl_easy_setopt(f->curl, CURLOPT_WRITEFUNCTION, raw_import_file_write_callback) != CURLE_OK)
705 if (curl_easy_setopt(f->curl, CURLOPT_WRITEDATA, f) != CURLE_OK)
708 if (curl_easy_setopt(f->curl, CURLOPT_HEADERFUNCTION, raw_import_file_header_callback) != CURLE_OK)
711 if (curl_easy_setopt(f->curl, CURLOPT_HEADERDATA, f) != CURLE_OK)
714 r = curl_glue_add(f->import->glue, f->curl);
721 int raw_import_new(RawImport **import, sd_event *event, const char *image_root, raw_import_on_finished on_finished, void *userdata) {
722 _cleanup_(raw_import_unrefp) RawImport *i = NULL;
728 i = new0(RawImport, 1);
732 i->on_finished = on_finished;
733 i->userdata = userdata;
735 i->image_root = strdup(image_root);
740 i->event = sd_event_ref(event);
742 r = sd_event_default(&i->event);
747 r = curl_glue_new(&i->glue, i->event);
751 i->glue->on_finished = raw_import_curl_on_finished;
752 i->glue->userdata = i;
760 RawImport* raw_import_unref(RawImport *import) {
766 while ((f = hashmap_steal_first(import->files)))
767 raw_import_file_unref(f);
768 hashmap_free(import->files);
770 curl_glue_unref(import->glue);
771 sd_event_unref(import->event);
773 free(import->image_root);
779 int raw_import_cancel(RawImport *import, const char *url) {
785 f = hashmap_remove(import->files, url);
789 raw_import_file_unref(f);
793 int raw_import_pull(RawImport *import, const char *url, const char *local, bool force_local) {
794 _cleanup_(raw_import_file_unrefp) RawImportFile *f = NULL;
798 assert(raw_url_is_valid(url));
799 assert(!local || machine_name_is_valid(local));
801 if (hashmap_get(import->files, url))
804 r = hashmap_ensure_allocated(&import->files, &string_hash_ops);
808 f = new0(RawImportFile, 1);
814 f->content_length = (uint64_t) -1;
816 f->url = strdup(url);
821 f->local = strdup(local);
825 f->force_local = force_local;
828 r = hashmap_put(import->files, f->url, f);
832 r = raw_import_file_begin(f);
834 raw_import_cancel(import, f->url);
843 bool raw_url_is_valid(const char *url) {
847 if (!startswith(url, "http://") &&
848 !startswith(url, "https://"))
851 return ascii_is_valid(url);