1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <curl/curl.h>
23 #include <sys/prctl.h>
25 #include "sd-daemon.h"
28 #include "btrfs-util.h"
31 #include "path-util.h"
32 #include "import-util.h"
33 #include "curl-util.h"
34 #include "aufs-util.h"
36 #include "pull-common.h"
39 typedef enum DkrProgress {
56 PullJob *ancestry_job;
65 char **response_registries;
69 unsigned current_ancestry;
71 DkrPullFinished on_finished;
76 bool grow_machine_directory;
84 #define PROTOCOL_PREFIX "https://"
86 #define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"
87 #define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"
89 #define LAYERS_MAX 2048
91 static void dkr_pull_job_on_finished(PullJob *j);
93 DkrPull* dkr_pull_unref(DkrPull *i) {
98 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
99 (void) wait_for_terminate(i->tar_pid, NULL);
102 pull_job_unref(i->images_job);
103 pull_job_unref(i->tags_job);
104 pull_job_unref(i->ancestry_job);
105 pull_job_unref(i->json_job);
106 pull_job_unref(i->layer_job);
108 curl_glue_unref(i->glue);
109 sd_event_unref(i->event);
112 (void) btrfs_subvol_remove(i->temp_path);
113 (void) rm_rf_dangerous(i->temp_path, false, true, false);
120 free(i->response_token);
121 free(i->response_registries);
122 strv_free(i->ancestry);
135 const char *index_url,
136 const char *image_root,
137 DkrPullFinished on_finished,
140 _cleanup_(dkr_pull_unrefp) DkrPull *i = NULL;
147 if (!http_url_is_valid(index_url))
150 i = new0(DkrPull, 1);
154 i->on_finished = on_finished;
155 i->userdata = userdata;
157 i->image_root = strdup(image_root ?: "/var/lib/machines");
161 i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
163 i->index_url = strdup(index_url);
167 e = endswith(i->index_url, "/");
172 i->event = sd_event_ref(event);
174 r = sd_event_default(&i->event);
179 r = curl_glue_new(&i->glue, i->event);
183 i->glue->on_finished = pull_job_curl_on_finished;
184 i->glue->userdata = i;
192 static void dkr_pull_report_progress(DkrPull *i, DkrProgress p) {
202 percent += i->images_job->progress_percent * 5 / 100;
208 percent += i->tags_job->progress_percent * 5 / 100;
214 percent += i->ancestry_job->progress_percent * 5 / 100;
216 percent += i->json_job->progress_percent * 5 / 100;
219 case DKR_DOWNLOADING:
221 percent += 75 * i->current_ancestry / MAX(1U, i->n_ancestry);
223 percent += i->layer_job->progress_percent * 75 / MAX(1U, i->n_ancestry) / 100;
232 assert_not_reached("Unknown progress state");
235 sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
236 log_debug("Combined progress %u%%", percent);
239 static int parse_id(const void *payload, size_t size, char **ret) {
240 _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;
241 union json_value v = {};
242 void *json_state = NULL;
252 if (memchr(payload, 0, size))
255 buf = strndup(payload, size);
260 t = json_tokenize(&p, &id, &v, &json_state, NULL);
263 if (t != JSON_STRING)
266 t = json_tokenize(&p, &other, &v, &json_state, NULL);
272 if (!dkr_id_is_valid(id))
281 static int parse_ancestry(const void *payload, size_t size, char ***ret) {
282 _cleanup_free_ char *buf = NULL;
283 void *json_state = NULL;
290 } state = STATE_BEGIN;
291 _cleanup_strv_free_ char **l = NULL;
292 size_t n = 0, allocated = 0;
297 if (memchr(payload, 0, size))
300 buf = strndup(payload, size);
306 _cleanup_free_ char *str;
307 union json_value v = {};
310 t = json_tokenize(&p, &str, &v, &json_state, NULL);
317 if (t == JSON_ARRAY_OPEN)
325 if (t == JSON_STRING) {
326 if (!dkr_id_is_valid(str))
329 if (n+1 > LAYERS_MAX)
332 if (!GREEDY_REALLOC(l, allocated, n + 2))
341 } else if (t == JSON_ARRAY_CLOSE)
351 else if (t == JSON_ARRAY_CLOSE)
363 if (!strv_is_uniq(l))
378 static const char *dkr_pull_current_layer(DkrPull *i) {
381 if (strv_isempty(i->ancestry))
384 return i->ancestry[i->current_ancestry];
387 static const char *dkr_pull_current_base_layer(DkrPull *i) {
390 if (strv_isempty(i->ancestry))
393 if (i->current_ancestry <= 0)
396 return i->ancestry[i->current_ancestry-1];
399 static int dkr_pull_add_token(DkrPull *i, PullJob *j) {
405 if (i->response_token)
406 t = strjoina("Authorization: Token ", i->response_token);
408 t = HEADER_TOKEN " true";
410 j->request_header = curl_slist_new("Accept: application/json", t, NULL);
411 if (!j->request_header)
417 static bool dkr_pull_is_done(DkrPull *i) {
419 assert(i->images_job);
421 if (i->images_job->state != PULL_JOB_DONE)
424 if (!i->tags_job || i->tags_job->state != PULL_JOB_DONE)
427 if (!i->ancestry_job || i->ancestry_job->state != PULL_JOB_DONE)
430 if (!i->json_job || i->json_job->state != PULL_JOB_DONE)
433 if (i->layer_job && i->layer_job->state != PULL_JOB_DONE)
436 if (dkr_pull_current_layer(i))
442 static int dkr_pull_make_local_copy(DkrPull *i) {
450 if (!i->final_path) {
451 i->final_path = strjoin(i->image_root, "/.dkr-", i->id, NULL);
456 r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
463 static int dkr_pull_job_on_open_disk(PullJob *j) {
472 assert(i->layer_job == j);
473 assert(i->final_path);
474 assert(!i->temp_path);
475 assert(i->tar_pid <= 0);
477 r = tempfn_random(i->final_path, &i->temp_path);
481 mkdir_parents_label(i->temp_path, 0700);
483 base = dkr_pull_current_base_layer(i);
485 const char *base_path;
487 base_path = strjoina(i->image_root, "/.dkr-", base);
488 r = btrfs_subvol_snapshot(base_path, i->temp_path, false, true);
490 r = btrfs_subvol_make(i->temp_path);
492 return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path);
494 j->disk_fd = pull_fork_tar(i->temp_path, &i->tar_pid);
501 static void dkr_pull_job_on_progress(PullJob *j) {
509 dkr_pull_report_progress(
511 j == i->images_job ? DKR_SEARCHING :
512 j == i->tags_job ? DKR_RESOLVING :
513 j == i->ancestry_job || j == i->json_job ? DKR_METADATA :
517 static int dkr_pull_pull_layer(DkrPull *i) {
518 _cleanup_free_ char *path = NULL;
519 const char *url, *layer = NULL;
523 assert(!i->layer_job);
524 assert(!i->temp_path);
525 assert(!i->final_path);
528 layer = dkr_pull_current_layer(i);
530 return 0; /* no more layers */
532 path = strjoin(i->image_root, "/.dkr-", layer, NULL);
536 if (laccess(path, F_OK) < 0) {
540 return log_error_errno(errno, "Failed to check for container: %m");
543 log_info("Layer %s already exists, skipping.", layer);
545 i->current_ancestry++;
551 log_info("Pulling layer %s...", layer);
553 i->final_path = path;
556 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", layer, "/layer");
557 r = pull_job_new(&i->layer_job, url, i->glue, i);
559 return log_error_errno(r, "Failed to allocate layer job: %m");
561 r = dkr_pull_add_token(i, i->layer_job);
565 i->layer_job->on_finished = dkr_pull_job_on_finished;
566 i->layer_job->on_open_disk = dkr_pull_job_on_open_disk;
567 i->layer_job->on_progress = dkr_pull_job_on_progress;
568 i->layer_job->grow_machine_directory = i->grow_machine_directory;
570 r = pull_job_begin(i->layer_job);
572 return log_error_errno(r, "Failed to start layer job: %m");
577 static void dkr_pull_job_on_finished(PullJob *j) {
586 if (j == i->images_job)
587 log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)");
588 else if (j == i->tags_job)
589 log_error_errno(j->error, "Failed to retrieve tags list.");
590 else if (j == i->ancestry_job)
591 log_error_errno(j->error, "Failed to retrieve ancestry list.");
592 else if (j == i->json_job)
593 log_error_errno(j->error, "Failed to retrieve json data.");
595 log_error_errno(j->error, "Failed to retrieve layer data.");
601 if (i->images_job == j) {
604 assert(!i->tags_job);
605 assert(!i->ancestry_job);
606 assert(!i->json_job);
607 assert(!i->layer_job);
609 if (strv_isempty(i->response_registries)) {
611 log_error("Didn't get registry information.");
615 log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]);
616 dkr_pull_report_progress(i, DKR_RESOLVING);
618 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->tag);
619 r = pull_job_new(&i->tags_job, url, i->glue, i);
621 log_error_errno(r, "Failed to allocate tags job: %m");
625 r = dkr_pull_add_token(i, i->tags_job);
631 i->tags_job->on_finished = dkr_pull_job_on_finished;
632 i->tags_job->on_progress = dkr_pull_job_on_progress;
634 r = pull_job_begin(i->tags_job);
636 log_error_errno(r, "Failed to start tags job: %m");
640 } else if (i->tags_job == j) {
644 assert(!i->ancestry_job);
645 assert(!i->json_job);
646 assert(!i->layer_job);
648 r = parse_id(j->payload, j->payload_size, &id);
650 log_error_errno(r, "Failed to parse JSON id.");
657 log_info("Tag lookup succeeded, resolved to layer %s.", i->id);
658 dkr_pull_report_progress(i, DKR_METADATA);
660 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/ancestry");
661 r = pull_job_new(&i->ancestry_job, url, i->glue, i);
663 log_error_errno(r, "Failed to allocate ancestry job: %m");
667 r = dkr_pull_add_token(i, i->ancestry_job);
673 i->ancestry_job->on_finished = dkr_pull_job_on_finished;
674 i->ancestry_job->on_progress = dkr_pull_job_on_progress;
676 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/json");
677 r = pull_job_new(&i->json_job, url, i->glue, i);
679 log_error_errno(r, "Failed to allocate json job: %m");
683 r = dkr_pull_add_token(i, i->json_job);
689 i->json_job->on_finished = dkr_pull_job_on_finished;
690 i->json_job->on_progress = dkr_pull_job_on_progress;
692 r = pull_job_begin(i->ancestry_job);
694 log_error_errno(r, "Failed to start ancestry job: %m");
698 r = pull_job_begin(i->json_job);
700 log_error_errno(r, "Failed to start json job: %m");
704 } else if (i->ancestry_job == j) {
705 char **ancestry = NULL, **k;
708 assert(!i->layer_job);
710 r = parse_ancestry(j->payload, j->payload_size, &ancestry);
712 log_error_errno(r, "Failed to parse JSON id.");
716 n = strv_length(ancestry);
717 if (n <= 0 || !streq(ancestry[n-1], i->id)) {
718 log_error("Ancestry doesn't end in main layer.");
724 log_info("Ancestor lookup succeeded, requires layers:\n");
725 STRV_FOREACH(k, ancestry)
726 log_info("\t%s", *k);
728 strv_free(i->ancestry);
729 i->ancestry = ancestry;
731 i->current_ancestry = 0;
733 dkr_pull_report_progress(i, DKR_DOWNLOADING);
735 r = dkr_pull_pull_layer(i);
739 } else if (i->layer_job == j) {
740 assert(i->temp_path);
741 assert(i->final_path);
743 j->disk_fd = safe_close(j->disk_fd);
745 if (i->tar_pid > 0) {
746 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
752 r = aufs_resolve(i->temp_path);
754 log_error_errno(r, "Failed to resolve aufs whiteouts: %m");
758 r = btrfs_subvol_set_read_only(i->temp_path, true);
760 log_error_errno(r, "Failed to mark snapshot read-only: %m");
764 if (rename(i->temp_path, i->final_path) < 0) {
765 log_error_errno(errno, "Failed to rename snaphsot: %m");
769 log_info("Completed writing to layer %s.", i->final_path);
771 i->layer_job = pull_job_unref(i->layer_job);
775 i->final_path = NULL;
777 i->current_ancestry ++;
778 r = dkr_pull_pull_layer(i);
782 } else if (i->json_job != j)
783 assert_not_reached("Got finished event for unknown curl object");
785 if (!dkr_pull_is_done(i))
788 dkr_pull_report_progress(i, DKR_COPYING);
790 r = dkr_pull_make_local_copy(i);
798 i->on_finished(i, r, i->userdata);
800 sd_event_exit(i->event, r);
803 static int dkr_pull_job_on_header(PullJob *j, const char *header, size_t sz) {
804 _cleanup_free_ char *registry = NULL;
814 r = curl_header_strdup(header, sz, HEADER_TOKEN, &token);
818 free(i->response_token);
819 i->response_token = token;
823 r = curl_header_strdup(header, sz, HEADER_REGISTRY, ®istry);
829 l = strv_split(registry, ",");
834 if (!hostname_is_valid(*k)) {
835 log_error("Registry hostname is not valid.");
841 strv_free(i->response_registries);
842 i->response_registries = l;
848 int dkr_pull_start(DkrPull *i, const char *name, const char *tag, const char *local, bool force_local) {
854 if (!dkr_name_is_valid(name))
857 if (tag && !dkr_tag_is_valid(tag))
860 if (local && !machine_name_is_valid(local))
869 r = free_and_strdup(&i->local, local);
872 i->force_local = force_local;
874 r = free_and_strdup(&i->name, name);
877 r = free_and_strdup(&i->tag, tag);
881 url = strjoina(i->index_url, "/v1/repositories/", name, "/images");
883 r = pull_job_new(&i->images_job, url, i->glue, i);
887 r = dkr_pull_add_token(i, i->images_job);
891 i->images_job->on_finished = dkr_pull_job_on_finished;
892 i->images_job->on_header = dkr_pull_job_on_header;
893 i->images_job->on_progress = dkr_pull_job_on_progress;
895 return pull_job_begin(i->images_job);