1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <curl/curl.h>
23 #include <sys/prctl.h>
28 #include "btrfs-util.h"
31 #include "import-util.h"
32 #include "curl-util.h"
33 #include "aufs-util.h"
34 #include "import-job.h"
35 #include "import-common.h"
36 #include "import-dkr.h"
45 ImportJob *images_job;
47 ImportJob *ancestry_job;
56 char **response_registries;
59 unsigned current_ancestry;
61 DkrImportFinished on_finished;
73 #define PROTOCOL_PREFIX "https://"
75 #define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"
76 #define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"
78 #define LAYERS_MAX 2048
80 static void dkr_import_job_on_finished(ImportJob *j);
82 DkrImport* dkr_import_unref(DkrImport *i) {
87 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
88 (void) wait_for_terminate(i->tar_pid, NULL);
91 import_job_unref(i->images_job);
92 import_job_unref(i->tags_job);
93 import_job_unref(i->ancestry_job);
94 import_job_unref(i->json_job);
95 import_job_unref(i->layer_job);
97 curl_glue_unref(i->glue);
98 sd_event_unref(i->event);
101 (void) btrfs_subvol_remove(i->temp_path);
102 (void) rm_rf_dangerous(i->temp_path, false, true, false);
109 free(i->response_token);
110 free(i->response_registries);
111 strv_free(i->ancestry);
124 const char *index_url,
125 const char *image_root,
126 DkrImportFinished on_finished,
129 _cleanup_(dkr_import_unrefp) DkrImport *i = NULL;
136 if (!http_url_is_valid(index_url))
139 i = new0(DkrImport, 1);
143 i->on_finished = on_finished;
144 i->userdata = userdata;
146 i->image_root = strdup(image_root ?: "/var/lib/machines");
150 i->index_url = strdup(index_url);
154 e = endswith(i->index_url, "/");
159 i->event = sd_event_ref(event);
161 r = sd_event_default(&i->event);
166 r = curl_glue_new(&i->glue, i->event);
170 i->glue->on_finished = import_job_curl_on_finished;
171 i->glue->userdata = i;
179 static int parse_id(const void *payload, size_t size, char **ret) {
180 _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;
181 union json_value v = {};
182 void *json_state = NULL;
192 if (memchr(payload, 0, size))
195 buf = strndup(payload, size);
200 t = json_tokenize(&p, &id, &v, &json_state, NULL);
203 if (t != JSON_STRING)
206 t = json_tokenize(&p, &other, &v, &json_state, NULL);
212 if (!dkr_id_is_valid(id))
221 static int parse_ancestry(const void *payload, size_t size, char ***ret) {
222 _cleanup_free_ char *buf = NULL;
223 void *json_state = NULL;
230 } state = STATE_BEGIN;
231 _cleanup_strv_free_ char **l = NULL;
232 size_t n = 0, allocated = 0;
237 if (memchr(payload, 0, size))
240 buf = strndup(payload, size);
246 _cleanup_free_ char *str;
247 union json_value v = {};
250 t = json_tokenize(&p, &str, &v, &json_state, NULL);
257 if (t == JSON_ARRAY_OPEN)
265 if (t == JSON_STRING) {
266 if (!dkr_id_is_valid(str))
269 if (n+1 > LAYERS_MAX)
272 if (!GREEDY_REALLOC(l, allocated, n + 2))
281 } else if (t == JSON_ARRAY_CLOSE)
291 else if (t == JSON_ARRAY_CLOSE)
303 if (!strv_is_uniq(l))
318 static const char *dkr_import_current_layer(DkrImport *i) {
321 if (strv_isempty(i->ancestry))
324 return i->ancestry[i->current_ancestry];
327 static const char *dkr_import_current_base_layer(DkrImport *i) {
330 if (strv_isempty(i->ancestry))
333 if (i->current_ancestry <= 0)
336 return i->ancestry[i->current_ancestry-1];
339 static int dkr_import_add_token(DkrImport *i, ImportJob *j) {
345 if (i->response_token)
346 t = strappenda("Authorization: Token ", i->response_token);
348 t = HEADER_TOKEN " true";
350 j->request_header = curl_slist_new("Accept: application/json", t, NULL);
351 if (!j->request_header)
357 static bool dkr_import_is_done(DkrImport *i) {
359 assert(i->images_job);
361 if (i->images_job->state != IMPORT_JOB_DONE)
364 if (!i->tags_job || i->tags_job->state != IMPORT_JOB_DONE)
367 if (!i->ancestry_job || i->ancestry_job->state != IMPORT_JOB_DONE)
370 if (!i->json_job || i->json_job->state != IMPORT_JOB_DONE)
373 if (i->layer_job && i->layer_job->state != IMPORT_JOB_DONE)
376 if (dkr_import_current_layer(i))
382 static int dkr_import_make_local_copy(DkrImport *i) {
390 if (!i->final_path) {
391 i->final_path = strjoin(i->image_root, "/.dkr-", i->id, NULL);
396 r = import_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
403 static int dkr_import_job_on_open_disk(ImportJob *j) {
404 _cleanup_close_pair_ int pipefd[2] = { -1, -1 };
413 assert(i->layer_job == j);
414 assert(i->final_path);
415 assert(!i->temp_path);
416 assert(i->tar_pid <= 0);
418 r = tempfn_random(i->final_path, &i->temp_path);
422 mkdir_parents_label(i->temp_path, 0700);
424 base = dkr_import_current_base_layer(i);
426 const char *base_path;
428 base_path = strappenda(i->image_root, "/.dkr-", base);
429 r = btrfs_subvol_snapshot(base_path, i->temp_path, false, true);
431 r = btrfs_subvol_make(i->temp_path);
433 return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path);
435 if (pipe2(pipefd, O_CLOEXEC) < 0)
436 return log_error_errno(errno, "Failed to create pipe for tar: %m");
440 return log_error_errno(errno, "Failed to fork off tar: %m");
441 if (i->tar_pid == 0) {
446 reset_all_signal_handlers();
448 assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0);
450 pipefd[1] = safe_close(pipefd[1]);
452 if (dup2(pipefd[0], STDIN_FILENO) != STDIN_FILENO) {
453 log_error_errno(errno, "Failed to dup2() fd: %m");
457 if (pipefd[0] != STDIN_FILENO)
458 pipefd[0] = safe_close(pipefd[0]);
460 null_fd = open("/dev/null", O_WRONLY|O_NOCTTY);
462 log_error_errno(errno, "Failed to open /dev/null: %m");
466 if (dup2(null_fd, STDOUT_FILENO) != STDOUT_FILENO) {
467 log_error_errno(errno, "Failed to dup2() fd: %m");
471 if (null_fd != STDOUT_FILENO)
472 null_fd = safe_close(null_fd);
474 fd_cloexec(STDIN_FILENO, false);
475 fd_cloexec(STDOUT_FILENO, false);
476 fd_cloexec(STDERR_FILENO, false);
478 execlp("tar", "tar", "--numeric-owner", "-C", i->temp_path, "-px", NULL);
479 log_error_errno(errno, "Failed to execute tar: %m");
483 pipefd[0] = safe_close(pipefd[0]);
485 j->disk_fd = pipefd[1];
491 static int dkr_import_pull_layer(DkrImport *i) {
492 _cleanup_free_ char *path = NULL;
493 const char *url, *layer = NULL;
497 assert(!i->layer_job);
498 assert(!i->temp_path);
499 assert(!i->final_path);
502 layer = dkr_import_current_layer(i);
504 return 0; /* no more layers */
506 path = strjoin(i->image_root, "/.dkr-", layer, NULL);
510 if (laccess(path, F_OK) < 0) {
514 return log_error_errno(errno, "Failed to check for container: %m");
517 log_info("Layer %s already exists, skipping.", layer);
519 i->current_ancestry++;
525 log_info("Pulling layer %s...", layer);
527 i->final_path = path;
530 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", layer, "/layer");
531 r = import_job_new(&i->layer_job, url, i->glue, i);
533 return log_error_errno(r, "Failed to allocate layer job: %m");
535 r = dkr_import_add_token(i, i->layer_job);
539 i->layer_job->on_finished = dkr_import_job_on_finished;
540 i->layer_job->on_open_disk = dkr_import_job_on_open_disk;
542 r = import_job_begin(i->layer_job);
544 return log_error_errno(r, "Failed to start layer job: %m");
549 static void dkr_import_job_on_finished(ImportJob *j) {
558 if (j == i->images_job)
559 log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)");
560 else if (j == i->tags_job)
561 log_error_errno(j->error, "Failed to retrieve tags list.");
562 else if (j == i->ancestry_job)
563 log_error_errno(j->error, "Failed to retrieve ancestry list.");
564 else if (j == i->json_job)
565 log_error_errno(j->error, "Failed to retrieve json data.");
567 log_error_errno(j->error, "Failed to retrieve layer data.");
573 if (i->images_job == j) {
576 assert(!i->tags_job);
577 assert(!i->ancestry_job);
578 assert(!i->json_job);
579 assert(!i->layer_job);
581 if (strv_isempty(i->response_registries)) {
583 log_error("Didn't get registry information.");
587 log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]);
589 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->tag);
590 r = import_job_new(&i->tags_job, url, i->glue, i);
592 log_error_errno(r, "Failed to allocate tags job: %m");
596 r = dkr_import_add_token(i, i->tags_job);
602 i->tags_job->on_finished = dkr_import_job_on_finished;
604 r = import_job_begin(i->tags_job);
606 log_error_errno(r, "Failed to start tags job: %m");
610 } else if (i->tags_job == j) {
614 assert(!i->ancestry_job);
615 assert(!i->json_job);
616 assert(!i->layer_job);
618 r = parse_id(j->payload, j->payload_size, &id);
620 log_error_errno(r, "Failed to parse JSON id.");
627 log_info("Tag lookup succeeded, resolved to layer %s.", i->id);
629 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/ancestry");
630 r = import_job_new(&i->ancestry_job, url, i->glue, i);
632 log_error_errno(r, "Failed to allocate ancestry job: %m");
636 r = dkr_import_add_token(i, i->ancestry_job);
642 i->ancestry_job->on_finished = dkr_import_job_on_finished;
644 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/json");
645 r = import_job_new(&i->json_job, url, i->glue, i);
647 log_error_errno(r, "Failed to allocate json job: %m");
651 r = dkr_import_add_token(i, i->json_job);
657 i->json_job->on_finished = dkr_import_job_on_finished;
659 r = import_job_begin(i->ancestry_job);
661 log_error_errno(r, "Failed to start ancestry job: %m");
665 r = import_job_begin(i->json_job);
667 log_error_errno(r, "Failed to start json job: %m");
671 } else if (i->ancestry_job == j) {
672 char **ancestry = NULL, **k;
675 assert(!i->layer_job);
677 r = parse_ancestry(j->payload, j->payload_size, &ancestry);
679 log_error_errno(r, "Failed to parse JSON id.");
683 n = strv_length(ancestry);
684 if (n <= 0 || !streq(ancestry[n-1], i->id)) {
685 log_error("Ancestry doesn't end in main layer.");
691 log_info("Ancestor lookup succeeded, requires layers:\n");
692 STRV_FOREACH(k, ancestry)
693 log_info("\t%s", *k);
695 strv_free(i->ancestry);
696 i->ancestry = ancestry;
698 i->current_ancestry = 0;
699 r = dkr_import_pull_layer(i);
703 } else if (i->layer_job == j) {
704 assert(i->temp_path);
705 assert(i->final_path);
707 j->disk_fd = safe_close(j->disk_fd);
709 if (i->tar_pid > 0) {
710 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
716 r = aufs_resolve(i->temp_path);
718 log_error_errno(r, "Failed to resolve aufs whiteouts: %m");
722 r = btrfs_subvol_set_read_only(i->temp_path, true);
724 log_error_errno(r, "Failed to mark snapshort read-only: %m");
728 if (rename(i->temp_path, i->final_path) < 0) {
729 log_error_errno(errno, "Failed to rename snaphsot: %m");
733 log_info("Completed writing to layer %s.", i->final_path);
735 i->layer_job = import_job_unref(i->layer_job);
739 i->final_path = NULL;
741 i->current_ancestry ++;
742 r = dkr_import_pull_layer(i);
746 } else if (i->json_job != j)
747 assert_not_reached("Got finished event for unknown curl object");
749 if (!dkr_import_is_done(i))
752 r = dkr_import_make_local_copy(i);
760 i->on_finished(i, r, i->userdata);
762 sd_event_exit(i->event, r);
765 static int dkr_import_job_on_header(ImportJob *j, const char *header, size_t sz) {
766 _cleanup_free_ char *registry = NULL;
776 r = curl_header_strdup(header, sz, HEADER_TOKEN, &token);
780 free(i->response_token);
781 i->response_token = token;
785 r = curl_header_strdup(header, sz, HEADER_REGISTRY, ®istry);
791 l = strv_split(registry, ",");
796 if (!hostname_is_valid(*k)) {
797 log_error("Registry hostname is not valid.");
803 strv_free(i->response_registries);
804 i->response_registries = l;
810 int dkr_import_pull(DkrImport *i, const char *name, const char *tag, const char *local, bool force_local) {
816 if (!dkr_name_is_valid(name))
819 if (tag && !dkr_tag_is_valid(tag))
822 if (local && !machine_name_is_valid(local))
831 r = free_and_strdup(&i->local, local);
834 i->force_local = force_local;
836 r = free_and_strdup(&i->name, name);
839 r = free_and_strdup(&i->tag, tag);
843 url = strappenda(i->index_url, "/v1/repositories/", name, "/images");
845 r = import_job_new(&i->images_job, url, i->glue, i);
849 r = dkr_import_add_token(i, i->images_job);
853 i->images_job->on_finished = dkr_import_job_on_finished;
854 i->images_job->on_header = dkr_import_job_on_header;
856 return import_job_begin(i->images_job);