1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <curl/curl.h>
23 #include <sys/prctl.h>
25 #include "sd-daemon.h"
28 #include "btrfs-util.h"
31 #include "path-util.h"
32 #include "import-util.h"
33 #include "curl-util.h"
34 #include "aufs-util.h"
36 #include "pull-common.h"
37 #include "import-common.h"
40 typedef enum DkrProgress {
57 PullJob *ancestry_job;
66 char **response_registries;
70 unsigned current_ancestry;
72 DkrPullFinished on_finished;
77 bool grow_machine_directory;
85 #define PROTOCOL_PREFIX "https://"
87 #define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"
88 #define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"
90 #define LAYERS_MAX 2048
92 static void dkr_pull_job_on_finished(PullJob *j);
94 DkrPull* dkr_pull_unref(DkrPull *i) {
99 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
100 (void) wait_for_terminate(i->tar_pid, NULL);
103 pull_job_unref(i->images_job);
104 pull_job_unref(i->tags_job);
105 pull_job_unref(i->ancestry_job);
106 pull_job_unref(i->json_job);
107 pull_job_unref(i->layer_job);
109 curl_glue_unref(i->glue);
110 sd_event_unref(i->event);
113 (void) btrfs_subvol_remove(i->temp_path);
114 (void) rm_rf_dangerous(i->temp_path, false, true, false);
121 free(i->response_token);
122 free(i->response_registries);
123 strv_free(i->ancestry);
136 const char *index_url,
137 const char *image_root,
138 DkrPullFinished on_finished,
141 _cleanup_(dkr_pull_unrefp) DkrPull *i = NULL;
148 if (!http_url_is_valid(index_url))
151 i = new0(DkrPull, 1);
155 i->on_finished = on_finished;
156 i->userdata = userdata;
158 i->image_root = strdup(image_root ?: "/var/lib/machines");
162 i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
164 i->index_url = strdup(index_url);
168 e = endswith(i->index_url, "/");
173 i->event = sd_event_ref(event);
175 r = sd_event_default(&i->event);
180 r = curl_glue_new(&i->glue, i->event);
184 i->glue->on_finished = pull_job_curl_on_finished;
185 i->glue->userdata = i;
193 static void dkr_pull_report_progress(DkrPull *i, DkrProgress p) {
203 percent += i->images_job->progress_percent * 5 / 100;
209 percent += i->tags_job->progress_percent * 5 / 100;
215 percent += i->ancestry_job->progress_percent * 5 / 100;
217 percent += i->json_job->progress_percent * 5 / 100;
220 case DKR_DOWNLOADING:
222 percent += 75 * i->current_ancestry / MAX(1U, i->n_ancestry);
224 percent += i->layer_job->progress_percent * 75 / MAX(1U, i->n_ancestry) / 100;
233 assert_not_reached("Unknown progress state");
236 sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
237 log_debug("Combined progress %u%%", percent);
240 static int parse_id(const void *payload, size_t size, char **ret) {
241 _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;
242 union json_value v = {};
243 void *json_state = NULL;
253 if (memchr(payload, 0, size))
256 buf = strndup(payload, size);
261 t = json_tokenize(&p, &id, &v, &json_state, NULL);
264 if (t != JSON_STRING)
267 t = json_tokenize(&p, &other, &v, &json_state, NULL);
273 if (!dkr_id_is_valid(id))
282 static int parse_ancestry(const void *payload, size_t size, char ***ret) {
283 _cleanup_free_ char *buf = NULL;
284 void *json_state = NULL;
291 } state = STATE_BEGIN;
292 _cleanup_strv_free_ char **l = NULL;
293 size_t n = 0, allocated = 0;
298 if (memchr(payload, 0, size))
301 buf = strndup(payload, size);
307 _cleanup_free_ char *str;
308 union json_value v = {};
311 t = json_tokenize(&p, &str, &v, &json_state, NULL);
318 if (t == JSON_ARRAY_OPEN)
326 if (t == JSON_STRING) {
327 if (!dkr_id_is_valid(str))
330 if (n+1 > LAYERS_MAX)
333 if (!GREEDY_REALLOC(l, allocated, n + 2))
342 } else if (t == JSON_ARRAY_CLOSE)
352 else if (t == JSON_ARRAY_CLOSE)
364 if (!strv_is_uniq(l))
379 static const char *dkr_pull_current_layer(DkrPull *i) {
382 if (strv_isempty(i->ancestry))
385 return i->ancestry[i->current_ancestry];
388 static const char *dkr_pull_current_base_layer(DkrPull *i) {
391 if (strv_isempty(i->ancestry))
394 if (i->current_ancestry <= 0)
397 return i->ancestry[i->current_ancestry-1];
400 static int dkr_pull_add_token(DkrPull *i, PullJob *j) {
406 if (i->response_token)
407 t = strjoina("Authorization: Token ", i->response_token);
409 t = HEADER_TOKEN " true";
411 j->request_header = curl_slist_new("Accept: application/json", t, NULL);
412 if (!j->request_header)
418 static bool dkr_pull_is_done(DkrPull *i) {
420 assert(i->images_job);
422 if (i->images_job->state != PULL_JOB_DONE)
425 if (!i->tags_job || i->tags_job->state != PULL_JOB_DONE)
428 if (!i->ancestry_job || i->ancestry_job->state != PULL_JOB_DONE)
431 if (!i->json_job || i->json_job->state != PULL_JOB_DONE)
434 if (i->layer_job && i->layer_job->state != PULL_JOB_DONE)
437 if (dkr_pull_current_layer(i))
443 static int dkr_pull_make_local_copy(DkrPull *i) {
451 if (!i->final_path) {
452 i->final_path = strjoin(i->image_root, "/.dkr-", i->id, NULL);
457 r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
464 static int dkr_pull_job_on_open_disk(PullJob *j) {
473 assert(i->layer_job == j);
474 assert(i->final_path);
475 assert(!i->temp_path);
476 assert(i->tar_pid <= 0);
478 r = tempfn_random(i->final_path, &i->temp_path);
482 mkdir_parents_label(i->temp_path, 0700);
484 base = dkr_pull_current_base_layer(i);
486 const char *base_path;
488 base_path = strjoina(i->image_root, "/.dkr-", base);
489 r = btrfs_subvol_snapshot(base_path, i->temp_path, false, true);
491 r = btrfs_subvol_make(i->temp_path);
493 return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path);
495 j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
502 static void dkr_pull_job_on_progress(PullJob *j) {
510 dkr_pull_report_progress(
512 j == i->images_job ? DKR_SEARCHING :
513 j == i->tags_job ? DKR_RESOLVING :
514 j == i->ancestry_job || j == i->json_job ? DKR_METADATA :
518 static int dkr_pull_pull_layer(DkrPull *i) {
519 _cleanup_free_ char *path = NULL;
520 const char *url, *layer = NULL;
524 assert(!i->layer_job);
525 assert(!i->temp_path);
526 assert(!i->final_path);
529 layer = dkr_pull_current_layer(i);
531 return 0; /* no more layers */
533 path = strjoin(i->image_root, "/.dkr-", layer, NULL);
537 if (laccess(path, F_OK) < 0) {
541 return log_error_errno(errno, "Failed to check for container: %m");
544 log_info("Layer %s already exists, skipping.", layer);
546 i->current_ancestry++;
552 log_info("Pulling layer %s...", layer);
554 i->final_path = path;
557 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", layer, "/layer");
558 r = pull_job_new(&i->layer_job, url, i->glue, i);
560 return log_error_errno(r, "Failed to allocate layer job: %m");
562 r = dkr_pull_add_token(i, i->layer_job);
566 i->layer_job->on_finished = dkr_pull_job_on_finished;
567 i->layer_job->on_open_disk = dkr_pull_job_on_open_disk;
568 i->layer_job->on_progress = dkr_pull_job_on_progress;
569 i->layer_job->grow_machine_directory = i->grow_machine_directory;
571 r = pull_job_begin(i->layer_job);
573 return log_error_errno(r, "Failed to start layer job: %m");
578 static void dkr_pull_job_on_finished(PullJob *j) {
587 if (j == i->images_job)
588 log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)");
589 else if (j == i->tags_job)
590 log_error_errno(j->error, "Failed to retrieve tags list.");
591 else if (j == i->ancestry_job)
592 log_error_errno(j->error, "Failed to retrieve ancestry list.");
593 else if (j == i->json_job)
594 log_error_errno(j->error, "Failed to retrieve json data.");
596 log_error_errno(j->error, "Failed to retrieve layer data.");
602 if (i->images_job == j) {
605 assert(!i->tags_job);
606 assert(!i->ancestry_job);
607 assert(!i->json_job);
608 assert(!i->layer_job);
610 if (strv_isempty(i->response_registries)) {
612 log_error("Didn't get registry information.");
616 log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]);
617 dkr_pull_report_progress(i, DKR_RESOLVING);
619 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->tag);
620 r = pull_job_new(&i->tags_job, url, i->glue, i);
622 log_error_errno(r, "Failed to allocate tags job: %m");
626 r = dkr_pull_add_token(i, i->tags_job);
632 i->tags_job->on_finished = dkr_pull_job_on_finished;
633 i->tags_job->on_progress = dkr_pull_job_on_progress;
635 r = pull_job_begin(i->tags_job);
637 log_error_errno(r, "Failed to start tags job: %m");
641 } else if (i->tags_job == j) {
645 assert(!i->ancestry_job);
646 assert(!i->json_job);
647 assert(!i->layer_job);
649 r = parse_id(j->payload, j->payload_size, &id);
651 log_error_errno(r, "Failed to parse JSON id.");
658 log_info("Tag lookup succeeded, resolved to layer %s.", i->id);
659 dkr_pull_report_progress(i, DKR_METADATA);
661 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/ancestry");
662 r = pull_job_new(&i->ancestry_job, url, i->glue, i);
664 log_error_errno(r, "Failed to allocate ancestry job: %m");
668 r = dkr_pull_add_token(i, i->ancestry_job);
674 i->ancestry_job->on_finished = dkr_pull_job_on_finished;
675 i->ancestry_job->on_progress = dkr_pull_job_on_progress;
677 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/json");
678 r = pull_job_new(&i->json_job, url, i->glue, i);
680 log_error_errno(r, "Failed to allocate json job: %m");
684 r = dkr_pull_add_token(i, i->json_job);
690 i->json_job->on_finished = dkr_pull_job_on_finished;
691 i->json_job->on_progress = dkr_pull_job_on_progress;
693 r = pull_job_begin(i->ancestry_job);
695 log_error_errno(r, "Failed to start ancestry job: %m");
699 r = pull_job_begin(i->json_job);
701 log_error_errno(r, "Failed to start json job: %m");
705 } else if (i->ancestry_job == j) {
706 char **ancestry = NULL, **k;
709 assert(!i->layer_job);
711 r = parse_ancestry(j->payload, j->payload_size, &ancestry);
713 log_error_errno(r, "Failed to parse JSON id.");
717 n = strv_length(ancestry);
718 if (n <= 0 || !streq(ancestry[n-1], i->id)) {
719 log_error("Ancestry doesn't end in main layer.");
725 log_info("Ancestor lookup succeeded, requires layers:\n");
726 STRV_FOREACH(k, ancestry)
727 log_info("\t%s", *k);
729 strv_free(i->ancestry);
730 i->ancestry = ancestry;
732 i->current_ancestry = 0;
734 dkr_pull_report_progress(i, DKR_DOWNLOADING);
736 r = dkr_pull_pull_layer(i);
740 } else if (i->layer_job == j) {
741 assert(i->temp_path);
742 assert(i->final_path);
744 j->disk_fd = safe_close(j->disk_fd);
746 if (i->tar_pid > 0) {
747 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
753 r = aufs_resolve(i->temp_path);
755 log_error_errno(r, "Failed to resolve aufs whiteouts: %m");
759 r = btrfs_subvol_set_read_only(i->temp_path, true);
761 log_error_errno(r, "Failed to mark snapshot read-only: %m");
765 if (rename(i->temp_path, i->final_path) < 0) {
766 log_error_errno(errno, "Failed to rename snaphsot: %m");
770 log_info("Completed writing to layer %s.", i->final_path);
772 i->layer_job = pull_job_unref(i->layer_job);
776 i->final_path = NULL;
778 i->current_ancestry ++;
779 r = dkr_pull_pull_layer(i);
783 } else if (i->json_job != j)
784 assert_not_reached("Got finished event for unknown curl object");
786 if (!dkr_pull_is_done(i))
789 dkr_pull_report_progress(i, DKR_COPYING);
791 r = dkr_pull_make_local_copy(i);
799 i->on_finished(i, r, i->userdata);
801 sd_event_exit(i->event, r);
804 static int dkr_pull_job_on_header(PullJob *j, const char *header, size_t sz) {
805 _cleanup_free_ char *registry = NULL;
815 r = curl_header_strdup(header, sz, HEADER_TOKEN, &token);
819 free(i->response_token);
820 i->response_token = token;
824 r = curl_header_strdup(header, sz, HEADER_REGISTRY, ®istry);
830 l = strv_split(registry, ",");
835 if (!hostname_is_valid(*k)) {
836 log_error("Registry hostname is not valid.");
842 strv_free(i->response_registries);
843 i->response_registries = l;
849 int dkr_pull_start(DkrPull *i, const char *name, const char *tag, const char *local, bool force_local) {
855 if (!dkr_name_is_valid(name))
858 if (tag && !dkr_tag_is_valid(tag))
861 if (local && !machine_name_is_valid(local))
870 r = free_and_strdup(&i->local, local);
873 i->force_local = force_local;
875 r = free_and_strdup(&i->name, name);
878 r = free_and_strdup(&i->tag, tag);
882 url = strjoina(i->index_url, "/v1/repositories/", name, "/images");
884 r = pull_job_new(&i->images_job, url, i->glue, i);
888 r = dkr_pull_add_token(i, i->images_job);
892 i->images_job->on_finished = dkr_pull_job_on_finished;
893 i->images_job->on_header = dkr_pull_job_on_header;
894 i->images_job->on_progress = dkr_pull_job_on_progress;
896 return pull_job_begin(i->images_job);