1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <curl/curl.h>
23 #include <sys/prctl.h>
25 #include "sd-daemon.h"
28 #include "btrfs-util.h"
31 #include "import-util.h"
32 #include "curl-util.h"
33 #include "aufs-util.h"
34 #include "import-job.h"
35 #include "import-common.h"
36 #include "import-dkr.h"
38 typedef enum DkrProgress {
53 ImportJob *images_job;
55 ImportJob *ancestry_job;
64 char **response_registries;
68 unsigned current_ancestry;
70 DkrImportFinished on_finished;
82 #define PROTOCOL_PREFIX "https://"
84 #define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"
85 #define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"
87 #define LAYERS_MAX 2048
89 static void dkr_import_job_on_finished(ImportJob *j);
91 DkrImport* dkr_import_unref(DkrImport *i) {
96 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
97 (void) wait_for_terminate(i->tar_pid, NULL);
100 import_job_unref(i->images_job);
101 import_job_unref(i->tags_job);
102 import_job_unref(i->ancestry_job);
103 import_job_unref(i->json_job);
104 import_job_unref(i->layer_job);
106 curl_glue_unref(i->glue);
107 sd_event_unref(i->event);
110 (void) btrfs_subvol_remove(i->temp_path);
111 (void) rm_rf_dangerous(i->temp_path, false, true, false);
118 free(i->response_token);
119 free(i->response_registries);
120 strv_free(i->ancestry);
133 const char *index_url,
134 const char *image_root,
135 DkrImportFinished on_finished,
138 _cleanup_(dkr_import_unrefp) DkrImport *i = NULL;
145 if (!http_url_is_valid(index_url))
148 i = new0(DkrImport, 1);
152 i->on_finished = on_finished;
153 i->userdata = userdata;
155 i->image_root = strdup(image_root ?: "/var/lib/machines");
159 i->index_url = strdup(index_url);
163 e = endswith(i->index_url, "/");
168 i->event = sd_event_ref(event);
170 r = sd_event_default(&i->event);
175 r = curl_glue_new(&i->glue, i->event);
179 i->glue->on_finished = import_job_curl_on_finished;
180 i->glue->userdata = i;
188 static void dkr_import_report_progress(DkrImport *i, DkrProgress p) {
198 percent += i->images_job->progress_percent * 5 / 100;
204 percent += i->tags_job->progress_percent * 5 / 100;
210 percent += i->ancestry_job->progress_percent * 5 / 100;
212 percent += i->json_job->progress_percent * 5 / 100;
215 case DKR_DOWNLOADING:
217 percent += 75 * i->current_ancestry / MAX(1U, i->n_ancestry);
219 percent += i->layer_job->progress_percent * 75 / MAX(1U, i->n_ancestry) / 100;
228 assert_not_reached("Unknown progress state");
231 sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
232 log_debug("Combined progress %u%%", percent);
235 static int parse_id(const void *payload, size_t size, char **ret) {
236 _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;
237 union json_value v = {};
238 void *json_state = NULL;
248 if (memchr(payload, 0, size))
251 buf = strndup(payload, size);
256 t = json_tokenize(&p, &id, &v, &json_state, NULL);
259 if (t != JSON_STRING)
262 t = json_tokenize(&p, &other, &v, &json_state, NULL);
268 if (!dkr_id_is_valid(id))
277 static int parse_ancestry(const void *payload, size_t size, char ***ret) {
278 _cleanup_free_ char *buf = NULL;
279 void *json_state = NULL;
286 } state = STATE_BEGIN;
287 _cleanup_strv_free_ char **l = NULL;
288 size_t n = 0, allocated = 0;
293 if (memchr(payload, 0, size))
296 buf = strndup(payload, size);
302 _cleanup_free_ char *str;
303 union json_value v = {};
306 t = json_tokenize(&p, &str, &v, &json_state, NULL);
313 if (t == JSON_ARRAY_OPEN)
321 if (t == JSON_STRING) {
322 if (!dkr_id_is_valid(str))
325 if (n+1 > LAYERS_MAX)
328 if (!GREEDY_REALLOC(l, allocated, n + 2))
337 } else if (t == JSON_ARRAY_CLOSE)
347 else if (t == JSON_ARRAY_CLOSE)
359 if (!strv_is_uniq(l))
374 static const char *dkr_import_current_layer(DkrImport *i) {
377 if (strv_isempty(i->ancestry))
380 return i->ancestry[i->current_ancestry];
383 static const char *dkr_import_current_base_layer(DkrImport *i) {
386 if (strv_isempty(i->ancestry))
389 if (i->current_ancestry <= 0)
392 return i->ancestry[i->current_ancestry-1];
395 static int dkr_import_add_token(DkrImport *i, ImportJob *j) {
401 if (i->response_token)
402 t = strjoina("Authorization: Token ", i->response_token);
404 t = HEADER_TOKEN " true";
406 j->request_header = curl_slist_new("Accept: application/json", t, NULL);
407 if (!j->request_header)
413 static bool dkr_import_is_done(DkrImport *i) {
415 assert(i->images_job);
417 if (i->images_job->state != IMPORT_JOB_DONE)
420 if (!i->tags_job || i->tags_job->state != IMPORT_JOB_DONE)
423 if (!i->ancestry_job || i->ancestry_job->state != IMPORT_JOB_DONE)
426 if (!i->json_job || i->json_job->state != IMPORT_JOB_DONE)
429 if (i->layer_job && i->layer_job->state != IMPORT_JOB_DONE)
432 if (dkr_import_current_layer(i))
438 static int dkr_import_make_local_copy(DkrImport *i) {
446 if (!i->final_path) {
447 i->final_path = strjoin(i->image_root, "/.dkr-", i->id, NULL);
452 r = import_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
459 static int dkr_import_job_on_open_disk(ImportJob *j) {
468 assert(i->layer_job == j);
469 assert(i->final_path);
470 assert(!i->temp_path);
471 assert(i->tar_pid <= 0);
473 r = tempfn_random(i->final_path, &i->temp_path);
477 mkdir_parents_label(i->temp_path, 0700);
479 base = dkr_import_current_base_layer(i);
481 const char *base_path;
483 base_path = strjoina(i->image_root, "/.dkr-", base);
484 r = btrfs_subvol_snapshot(base_path, i->temp_path, false, true);
486 r = btrfs_subvol_make(i->temp_path);
488 return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path);
490 j->disk_fd = import_fork_tar(i->temp_path, &i->tar_pid);
497 static void dkr_import_job_on_progress(ImportJob *j) {
505 dkr_import_report_progress(
507 j == i->images_job ? DKR_SEARCHING :
508 j == i->tags_job ? DKR_RESOLVING :
509 j == i->ancestry_job || j == i->json_job ? DKR_METADATA :
513 static int dkr_import_pull_layer(DkrImport *i) {
514 _cleanup_free_ char *path = NULL;
515 const char *url, *layer = NULL;
519 assert(!i->layer_job);
520 assert(!i->temp_path);
521 assert(!i->final_path);
524 layer = dkr_import_current_layer(i);
526 return 0; /* no more layers */
528 path = strjoin(i->image_root, "/.dkr-", layer, NULL);
532 if (laccess(path, F_OK) < 0) {
536 return log_error_errno(errno, "Failed to check for container: %m");
539 log_info("Layer %s already exists, skipping.", layer);
541 i->current_ancestry++;
547 log_info("Pulling layer %s...", layer);
549 i->final_path = path;
552 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", layer, "/layer");
553 r = import_job_new(&i->layer_job, url, i->glue, i);
555 return log_error_errno(r, "Failed to allocate layer job: %m");
557 r = dkr_import_add_token(i, i->layer_job);
561 i->layer_job->on_finished = dkr_import_job_on_finished;
562 i->layer_job->on_open_disk = dkr_import_job_on_open_disk;
563 i->layer_job->on_progress = dkr_import_job_on_progress;
565 r = import_job_begin(i->layer_job);
567 return log_error_errno(r, "Failed to start layer job: %m");
572 static void dkr_import_job_on_finished(ImportJob *j) {
581 if (j == i->images_job)
582 log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)");
583 else if (j == i->tags_job)
584 log_error_errno(j->error, "Failed to retrieve tags list.");
585 else if (j == i->ancestry_job)
586 log_error_errno(j->error, "Failed to retrieve ancestry list.");
587 else if (j == i->json_job)
588 log_error_errno(j->error, "Failed to retrieve json data.");
590 log_error_errno(j->error, "Failed to retrieve layer data.");
596 if (i->images_job == j) {
599 assert(!i->tags_job);
600 assert(!i->ancestry_job);
601 assert(!i->json_job);
602 assert(!i->layer_job);
604 if (strv_isempty(i->response_registries)) {
606 log_error("Didn't get registry information.");
610 log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]);
611 dkr_import_report_progress(i, DKR_RESOLVING);
613 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->tag);
614 r = import_job_new(&i->tags_job, url, i->glue, i);
616 log_error_errno(r, "Failed to allocate tags job: %m");
620 r = dkr_import_add_token(i, i->tags_job);
626 i->tags_job->on_finished = dkr_import_job_on_finished;
627 i->tags_job->on_progress = dkr_import_job_on_progress;
629 r = import_job_begin(i->tags_job);
631 log_error_errno(r, "Failed to start tags job: %m");
635 } else if (i->tags_job == j) {
639 assert(!i->ancestry_job);
640 assert(!i->json_job);
641 assert(!i->layer_job);
643 r = parse_id(j->payload, j->payload_size, &id);
645 log_error_errno(r, "Failed to parse JSON id.");
652 log_info("Tag lookup succeeded, resolved to layer %s.", i->id);
653 dkr_import_report_progress(i, DKR_METADATA);
655 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/ancestry");
656 r = import_job_new(&i->ancestry_job, url, i->glue, i);
658 log_error_errno(r, "Failed to allocate ancestry job: %m");
662 r = dkr_import_add_token(i, i->ancestry_job);
668 i->ancestry_job->on_finished = dkr_import_job_on_finished;
669 i->ancestry_job->on_progress = dkr_import_job_on_progress;
671 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/json");
672 r = import_job_new(&i->json_job, url, i->glue, i);
674 log_error_errno(r, "Failed to allocate json job: %m");
678 r = dkr_import_add_token(i, i->json_job);
684 i->json_job->on_finished = dkr_import_job_on_finished;
685 i->json_job->on_progress = dkr_import_job_on_progress;
687 r = import_job_begin(i->ancestry_job);
689 log_error_errno(r, "Failed to start ancestry job: %m");
693 r = import_job_begin(i->json_job);
695 log_error_errno(r, "Failed to start json job: %m");
699 } else if (i->ancestry_job == j) {
700 char **ancestry = NULL, **k;
703 assert(!i->layer_job);
705 r = parse_ancestry(j->payload, j->payload_size, &ancestry);
707 log_error_errno(r, "Failed to parse JSON id.");
711 n = strv_length(ancestry);
712 if (n <= 0 || !streq(ancestry[n-1], i->id)) {
713 log_error("Ancestry doesn't end in main layer.");
719 log_info("Ancestor lookup succeeded, requires layers:\n");
720 STRV_FOREACH(k, ancestry)
721 log_info("\t%s", *k);
723 strv_free(i->ancestry);
724 i->ancestry = ancestry;
726 i->current_ancestry = 0;
728 dkr_import_report_progress(i, DKR_DOWNLOADING);
730 r = dkr_import_pull_layer(i);
734 } else if (i->layer_job == j) {
735 assert(i->temp_path);
736 assert(i->final_path);
738 j->disk_fd = safe_close(j->disk_fd);
740 if (i->tar_pid > 0) {
741 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
747 r = aufs_resolve(i->temp_path);
749 log_error_errno(r, "Failed to resolve aufs whiteouts: %m");
753 r = btrfs_subvol_set_read_only(i->temp_path, true);
755 log_error_errno(r, "Failed to mark snapshot read-only: %m");
759 if (rename(i->temp_path, i->final_path) < 0) {
760 log_error_errno(errno, "Failed to rename snaphsot: %m");
764 log_info("Completed writing to layer %s.", i->final_path);
766 i->layer_job = import_job_unref(i->layer_job);
770 i->final_path = NULL;
772 i->current_ancestry ++;
773 r = dkr_import_pull_layer(i);
777 } else if (i->json_job != j)
778 assert_not_reached("Got finished event for unknown curl object");
780 if (!dkr_import_is_done(i))
783 dkr_import_report_progress(i, DKR_COPYING);
785 r = dkr_import_make_local_copy(i);
793 i->on_finished(i, r, i->userdata);
795 sd_event_exit(i->event, r);
798 static int dkr_import_job_on_header(ImportJob *j, const char *header, size_t sz) {
799 _cleanup_free_ char *registry = NULL;
809 r = curl_header_strdup(header, sz, HEADER_TOKEN, &token);
813 free(i->response_token);
814 i->response_token = token;
818 r = curl_header_strdup(header, sz, HEADER_REGISTRY, ®istry);
824 l = strv_split(registry, ",");
829 if (!hostname_is_valid(*k)) {
830 log_error("Registry hostname is not valid.");
836 strv_free(i->response_registries);
837 i->response_registries = l;
843 int dkr_import_pull(DkrImport *i, const char *name, const char *tag, const char *local, bool force_local) {
849 if (!dkr_name_is_valid(name))
852 if (tag && !dkr_tag_is_valid(tag))
855 if (local && !machine_name_is_valid(local))
864 r = free_and_strdup(&i->local, local);
867 i->force_local = force_local;
869 r = free_and_strdup(&i->name, name);
872 r = free_and_strdup(&i->tag, tag);
876 url = strjoina(i->index_url, "/v1/repositories/", name, "/images");
878 r = import_job_new(&i->images_job, url, i->glue, i);
882 r = dkr_import_add_token(i, i->images_job);
886 i->images_job->on_finished = dkr_import_job_on_finished;
887 i->images_job->on_header = dkr_import_job_on_header;
888 i->images_job->on_progress = dkr_import_job_on_progress;
890 return import_job_begin(i->images_job);