1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <curl/curl.h>
23 #include <sys/prctl.h>
28 #include "btrfs-util.h"
31 #include "import-util.h"
32 #include "curl-util.h"
33 #include "aufs-util.h"
34 #include "import-job.h"
35 #include "import-common.h"
36 #include "import-dkr.h"
45 ImportJob *images_job;
47 ImportJob *ancestry_job;
56 char **response_registries;
59 unsigned current_ancestry;
61 DkrImportFinished on_finished;
73 #define PROTOCOL_PREFIX "https://"
75 #define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"
76 #define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"
78 #define LAYERS_MAX 2048
80 static void dkr_import_job_on_finished(ImportJob *j);
82 DkrImport* dkr_import_unref(DkrImport *i) {
87 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
88 (void) wait_for_terminate(i->tar_pid, NULL);
91 import_job_unref(i->images_job);
92 import_job_unref(i->tags_job);
93 import_job_unref(i->ancestry_job);
94 import_job_unref(i->json_job);
95 import_job_unref(i->layer_job);
97 curl_glue_unref(i->glue);
98 sd_event_unref(i->event);
101 (void) btrfs_subvol_remove(i->temp_path);
102 (void) rm_rf_dangerous(i->temp_path, false, true, false);
109 free(i->response_token);
110 free(i->response_registries);
111 strv_free(i->ancestry);
124 const char *index_url,
125 const char *image_root,
126 DkrImportFinished on_finished,
129 _cleanup_(dkr_import_unrefp) DkrImport *i = NULL;
136 if (!http_url_is_valid(index_url))
139 i = new0(DkrImport, 1);
143 i->on_finished = on_finished;
144 i->userdata = userdata;
146 i->image_root = strdup(image_root ?: "/var/lib/machines");
150 i->index_url = strdup(index_url);
154 e = endswith(i->index_url, "/");
159 i->event = sd_event_ref(event);
161 r = sd_event_default(&i->event);
166 r = curl_glue_new(&i->glue, i->event);
170 i->glue->on_finished = import_job_curl_on_finished;
171 i->glue->userdata = i;
179 static int parse_id(const void *payload, size_t size, char **ret) {
180 _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;
181 union json_value v = {};
182 void *json_state = NULL;
192 if (memchr(payload, 0, size))
195 buf = strndup(payload, size);
200 t = json_tokenize(&p, &id, &v, &json_state, NULL);
203 if (t != JSON_STRING)
206 t = json_tokenize(&p, &other, &v, &json_state, NULL);
212 if (!dkr_id_is_valid(id))
221 static int parse_ancestry(const void *payload, size_t size, char ***ret) {
222 _cleanup_free_ char *buf = NULL;
223 void *json_state = NULL;
230 } state = STATE_BEGIN;
231 _cleanup_strv_free_ char **l = NULL;
232 size_t n = 0, allocated = 0;
237 if (memchr(payload, 0, size))
240 buf = strndup(payload, size);
246 _cleanup_free_ char *str;
247 union json_value v = {};
250 t = json_tokenize(&p, &str, &v, &json_state, NULL);
257 if (t == JSON_ARRAY_OPEN)
265 if (t == JSON_STRING) {
266 if (!dkr_id_is_valid(str))
269 if (n+1 > LAYERS_MAX)
272 if (!GREEDY_REALLOC(l, allocated, n + 2))
281 } else if (t == JSON_ARRAY_CLOSE)
291 else if (t == JSON_ARRAY_CLOSE)
303 if (!strv_is_uniq(l))
318 static const char *dkr_import_current_layer(DkrImport *i) {
321 if (strv_isempty(i->ancestry))
324 return i->ancestry[i->current_ancestry];
327 static const char *dkr_import_current_base_layer(DkrImport *i) {
330 if (strv_isempty(i->ancestry))
333 if (i->current_ancestry <= 0)
336 return i->ancestry[i->current_ancestry-1];
339 static int dkr_import_add_token(DkrImport *i, ImportJob *j) {
345 if (i->response_token)
346 t = strappenda("Authorization: Token ", i->response_token);
348 t = HEADER_TOKEN " true";
350 j->request_header = curl_slist_new("Accept: application/json", t, NULL);
351 if (!j->request_header)
357 static bool dkr_import_is_done(DkrImport *i) {
359 assert(i->images_job);
361 if (i->images_job->state != IMPORT_JOB_DONE)
364 if (!i->tags_job || i->tags_job->state != IMPORT_JOB_DONE)
367 if (!i->ancestry_job || i->ancestry_job->state != IMPORT_JOB_DONE)
370 if (!i->json_job || i->json_job->state != IMPORT_JOB_DONE)
373 if (i->layer_job && i->layer_job->state != IMPORT_JOB_DONE)
376 if (dkr_import_current_layer(i))
382 static int dkr_import_make_local_copy(DkrImport *i) {
390 if (!i->final_path) {
391 i->final_path = strjoin(i->image_root, "/.dkr-", i->id, NULL);
396 r = import_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
403 static int dkr_import_job_on_open_disk(ImportJob *j) {
412 assert(i->layer_job == j);
413 assert(i->final_path);
414 assert(!i->temp_path);
415 assert(i->tar_pid <= 0);
417 r = tempfn_random(i->final_path, &i->temp_path);
421 mkdir_parents_label(i->temp_path, 0700);
423 base = dkr_import_current_base_layer(i);
425 const char *base_path;
427 base_path = strappenda(i->image_root, "/.dkr-", base);
428 r = btrfs_subvol_snapshot(base_path, i->temp_path, false, true);
430 r = btrfs_subvol_make(i->temp_path);
432 return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path);
434 j->disk_fd = import_fork_tar(i->temp_path, &i->tar_pid);
441 static int dkr_import_pull_layer(DkrImport *i) {
442 _cleanup_free_ char *path = NULL;
443 const char *url, *layer = NULL;
447 assert(!i->layer_job);
448 assert(!i->temp_path);
449 assert(!i->final_path);
452 layer = dkr_import_current_layer(i);
454 return 0; /* no more layers */
456 path = strjoin(i->image_root, "/.dkr-", layer, NULL);
460 if (laccess(path, F_OK) < 0) {
464 return log_error_errno(errno, "Failed to check for container: %m");
467 log_info("Layer %s already exists, skipping.", layer);
469 i->current_ancestry++;
475 log_info("Pulling layer %s...", layer);
477 i->final_path = path;
480 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", layer, "/layer");
481 r = import_job_new(&i->layer_job, url, i->glue, i);
483 return log_error_errno(r, "Failed to allocate layer job: %m");
485 r = dkr_import_add_token(i, i->layer_job);
489 i->layer_job->on_finished = dkr_import_job_on_finished;
490 i->layer_job->on_open_disk = dkr_import_job_on_open_disk;
492 r = import_job_begin(i->layer_job);
494 return log_error_errno(r, "Failed to start layer job: %m");
499 static void dkr_import_job_on_finished(ImportJob *j) {
508 if (j == i->images_job)
509 log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)");
510 else if (j == i->tags_job)
511 log_error_errno(j->error, "Failed to retrieve tags list.");
512 else if (j == i->ancestry_job)
513 log_error_errno(j->error, "Failed to retrieve ancestry list.");
514 else if (j == i->json_job)
515 log_error_errno(j->error, "Failed to retrieve json data.");
517 log_error_errno(j->error, "Failed to retrieve layer data.");
523 if (i->images_job == j) {
526 assert(!i->tags_job);
527 assert(!i->ancestry_job);
528 assert(!i->json_job);
529 assert(!i->layer_job);
531 if (strv_isempty(i->response_registries)) {
533 log_error("Didn't get registry information.");
537 log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]);
539 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->tag);
540 r = import_job_new(&i->tags_job, url, i->glue, i);
542 log_error_errno(r, "Failed to allocate tags job: %m");
546 r = dkr_import_add_token(i, i->tags_job);
552 i->tags_job->on_finished = dkr_import_job_on_finished;
554 r = import_job_begin(i->tags_job);
556 log_error_errno(r, "Failed to start tags job: %m");
560 } else if (i->tags_job == j) {
564 assert(!i->ancestry_job);
565 assert(!i->json_job);
566 assert(!i->layer_job);
568 r = parse_id(j->payload, j->payload_size, &id);
570 log_error_errno(r, "Failed to parse JSON id.");
577 log_info("Tag lookup succeeded, resolved to layer %s.", i->id);
579 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/ancestry");
580 r = import_job_new(&i->ancestry_job, url, i->glue, i);
582 log_error_errno(r, "Failed to allocate ancestry job: %m");
586 r = dkr_import_add_token(i, i->ancestry_job);
592 i->ancestry_job->on_finished = dkr_import_job_on_finished;
594 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/json");
595 r = import_job_new(&i->json_job, url, i->glue, i);
597 log_error_errno(r, "Failed to allocate json job: %m");
601 r = dkr_import_add_token(i, i->json_job);
607 i->json_job->on_finished = dkr_import_job_on_finished;
609 r = import_job_begin(i->ancestry_job);
611 log_error_errno(r, "Failed to start ancestry job: %m");
615 r = import_job_begin(i->json_job);
617 log_error_errno(r, "Failed to start json job: %m");
621 } else if (i->ancestry_job == j) {
622 char **ancestry = NULL, **k;
625 assert(!i->layer_job);
627 r = parse_ancestry(j->payload, j->payload_size, &ancestry);
629 log_error_errno(r, "Failed to parse JSON id.");
633 n = strv_length(ancestry);
634 if (n <= 0 || !streq(ancestry[n-1], i->id)) {
635 log_error("Ancestry doesn't end in main layer.");
641 log_info("Ancestor lookup succeeded, requires layers:\n");
642 STRV_FOREACH(k, ancestry)
643 log_info("\t%s", *k);
645 strv_free(i->ancestry);
646 i->ancestry = ancestry;
648 i->current_ancestry = 0;
649 r = dkr_import_pull_layer(i);
653 } else if (i->layer_job == j) {
654 assert(i->temp_path);
655 assert(i->final_path);
657 j->disk_fd = safe_close(j->disk_fd);
659 if (i->tar_pid > 0) {
660 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
666 r = aufs_resolve(i->temp_path);
668 log_error_errno(r, "Failed to resolve aufs whiteouts: %m");
672 r = btrfs_subvol_set_read_only(i->temp_path, true);
674 log_error_errno(r, "Failed to mark snapshort read-only: %m");
678 if (rename(i->temp_path, i->final_path) < 0) {
679 log_error_errno(errno, "Failed to rename snaphsot: %m");
683 log_info("Completed writing to layer %s.", i->final_path);
685 i->layer_job = import_job_unref(i->layer_job);
689 i->final_path = NULL;
691 i->current_ancestry ++;
692 r = dkr_import_pull_layer(i);
696 } else if (i->json_job != j)
697 assert_not_reached("Got finished event for unknown curl object");
699 if (!dkr_import_is_done(i))
702 r = dkr_import_make_local_copy(i);
710 i->on_finished(i, r, i->userdata);
712 sd_event_exit(i->event, r);
715 static int dkr_import_job_on_header(ImportJob *j, const char *header, size_t sz) {
716 _cleanup_free_ char *registry = NULL;
726 r = curl_header_strdup(header, sz, HEADER_TOKEN, &token);
730 free(i->response_token);
731 i->response_token = token;
735 r = curl_header_strdup(header, sz, HEADER_REGISTRY, ®istry);
741 l = strv_split(registry, ",");
746 if (!hostname_is_valid(*k)) {
747 log_error("Registry hostname is not valid.");
753 strv_free(i->response_registries);
754 i->response_registries = l;
760 int dkr_import_pull(DkrImport *i, const char *name, const char *tag, const char *local, bool force_local) {
766 if (!dkr_name_is_valid(name))
769 if (tag && !dkr_tag_is_valid(tag))
772 if (local && !machine_name_is_valid(local))
781 r = free_and_strdup(&i->local, local);
784 i->force_local = force_local;
786 r = free_and_strdup(&i->name, name);
789 r = free_and_strdup(&i->tag, tag);
793 url = strappenda(i->index_url, "/v1/repositories/", name, "/images");
795 r = import_job_new(&i->images_job, url, i->glue, i);
799 r = dkr_import_add_token(i, i->images_job);
803 i->images_job->on_finished = dkr_import_job_on_finished;
804 i->images_job->on_header = dkr_import_job_on_header;
806 return import_job_begin(i->images_job);