1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <curl/curl.h>
23 #include <sys/prctl.h>
28 #include "btrfs-util.h"
31 #include "curl-util.h"
32 #include "aufs-util.h"
33 #include "import-util.h"
34 #include "import-job.h"
35 #include "import-dkr.h"
44 ImportJob *images_job;
46 ImportJob *ancestry_job;
55 char **response_registries;
58 unsigned current_ancestry;
60 DkrImportFinished on_finished;
72 #define PROTOCOL_PREFIX "https://"
74 #define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"
75 #define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"
77 #define LAYERS_MAX 2048
79 static void dkr_import_job_on_finished(ImportJob *j);
81 DkrImport* dkr_import_unref(DkrImport *i) {
86 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
87 (void) wait_for_terminate(i->tar_pid, NULL);
90 import_job_unref(i->images_job);
91 import_job_unref(i->tags_job);
92 import_job_unref(i->ancestry_job);
93 import_job_unref(i->json_job);
94 import_job_unref(i->layer_job);
96 curl_glue_unref(i->glue);
97 sd_event_unref(i->event);
100 (void) btrfs_subvol_remove(i->temp_path);
101 (void) rm_rf_dangerous(i->temp_path, false, true, false);
108 free(i->response_token);
109 free(i->response_registries);
110 strv_free(i->ancestry);
123 const char *index_url,
124 const char *image_root,
125 DkrImportFinished on_finished,
128 _cleanup_(dkr_import_unrefp) DkrImport *i = NULL;
135 if (!http_url_is_valid(index_url))
138 i = new0(DkrImport, 1);
142 i->on_finished = on_finished;
143 i->userdata = userdata;
145 i->image_root = strdup(image_root ?: "/var/lib/machines");
149 i->index_url = strdup(index_url);
153 e = endswith(i->index_url, "/");
158 i->event = sd_event_ref(event);
160 r = sd_event_default(&i->event);
165 r = curl_glue_new(&i->glue, i->event);
169 i->glue->on_finished = import_job_curl_on_finished;
170 i->glue->userdata = i;
178 static int parse_id(const void *payload, size_t size, char **ret) {
179 _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;
180 union json_value v = {};
181 void *json_state = NULL;
191 if (memchr(payload, 0, size))
194 buf = strndup(payload, size);
199 t = json_tokenize(&p, &id, &v, &json_state, NULL);
202 if (t != JSON_STRING)
205 t = json_tokenize(&p, &other, &v, &json_state, NULL);
211 if (!dkr_id_is_valid(id))
220 static int parse_ancestry(const void *payload, size_t size, char ***ret) {
221 _cleanup_free_ char *buf = NULL;
222 void *json_state = NULL;
229 } state = STATE_BEGIN;
230 _cleanup_strv_free_ char **l = NULL;
231 size_t n = 0, allocated = 0;
236 if (memchr(payload, 0, size))
239 buf = strndup(payload, size);
245 _cleanup_free_ char *str;
246 union json_value v = {};
249 t = json_tokenize(&p, &str, &v, &json_state, NULL);
256 if (t == JSON_ARRAY_OPEN)
264 if (t == JSON_STRING) {
265 if (!dkr_id_is_valid(str))
268 if (n+1 > LAYERS_MAX)
271 if (!GREEDY_REALLOC(l, allocated, n + 2))
280 } else if (t == JSON_ARRAY_CLOSE)
290 else if (t == JSON_ARRAY_CLOSE)
302 if (!strv_is_uniq(l))
317 static const char *dkr_import_current_layer(DkrImport *i) {
320 if (strv_isempty(i->ancestry))
323 return i->ancestry[i->current_ancestry];
326 static const char *dkr_import_current_base_layer(DkrImport *i) {
329 if (strv_isempty(i->ancestry))
332 if (i->current_ancestry <= 0)
335 return i->ancestry[i->current_ancestry-1];
338 static int dkr_import_add_token(DkrImport *i, ImportJob *j) {
344 if (i->response_token)
345 t = strappenda("Authorization: Token ", i->response_token);
347 t = HEADER_TOKEN " true";
349 j->request_header = curl_slist_new("Accept: application/json", t, NULL);
350 if (!j->request_header)
356 static bool dkr_import_is_done(DkrImport *i) {
358 assert(i->images_job);
360 if (i->images_job->state != IMPORT_JOB_DONE)
363 if (!i->tags_job || i->tags_job->state != IMPORT_JOB_DONE)
366 if (!i->ancestry_job || i->ancestry_job->state != IMPORT_JOB_DONE)
369 if (!i->json_job || i->json_job->state != IMPORT_JOB_DONE)
372 if (i->layer_job && i->layer_job->state != IMPORT_JOB_DONE)
375 if (dkr_import_current_layer(i))
381 static int dkr_import_make_local_copy(DkrImport *i) {
389 if (!i->final_path) {
390 i->final_path = strjoin(i->image_root, "/.dkr-", i->id, NULL);
395 r = import_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
402 static int dkr_import_job_on_open_disk(ImportJob *j) {
403 _cleanup_close_pair_ int pipefd[2] = { -1, -1 };
412 assert(i->layer_job == j);
413 assert(i->final_path);
414 assert(!i->temp_path);
415 assert(i->tar_pid <= 0);
417 r = tempfn_random(i->final_path, &i->temp_path);
421 mkdir_parents_label(i->temp_path, 0700);
423 base = dkr_import_current_base_layer(i);
425 const char *base_path;
427 base_path = strappenda(i->image_root, "/.dkr-", base);
428 r = btrfs_subvol_snapshot(base_path, i->temp_path, false, true);
430 r = btrfs_subvol_make(i->temp_path);
432 return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path);
434 if (pipe2(pipefd, O_CLOEXEC) < 0)
435 return log_error_errno(errno, "Failed to create pipe for tar: %m");
439 return log_error_errno(errno, "Failed to fork off tar: %m");
440 if (i->tar_pid == 0) {
445 reset_all_signal_handlers();
447 assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0);
449 pipefd[1] = safe_close(pipefd[1]);
451 if (dup2(pipefd[0], STDIN_FILENO) != STDIN_FILENO) {
452 log_error_errno(errno, "Failed to dup2() fd: %m");
456 if (pipefd[0] != STDIN_FILENO)
457 pipefd[0] = safe_close(pipefd[0]);
459 null_fd = open("/dev/null", O_WRONLY|O_NOCTTY);
461 log_error_errno(errno, "Failed to open /dev/null: %m");
465 if (dup2(null_fd, STDOUT_FILENO) != STDOUT_FILENO) {
466 log_error_errno(errno, "Failed to dup2() fd: %m");
470 if (null_fd != STDOUT_FILENO)
471 null_fd = safe_close(null_fd);
473 fd_cloexec(STDIN_FILENO, false);
474 fd_cloexec(STDOUT_FILENO, false);
475 fd_cloexec(STDERR_FILENO, false);
477 execlp("tar", "tar", "--numeric-owner", "-C", i->temp_path, "-px", NULL);
478 log_error_errno(errno, "Failed to execute tar: %m");
482 pipefd[0] = safe_close(pipefd[0]);
484 j->disk_fd = pipefd[1];
490 static int dkr_import_pull_layer(DkrImport *i) {
491 _cleanup_free_ char *path = NULL;
492 const char *url, *layer = NULL;
496 assert(!i->layer_job);
497 assert(!i->temp_path);
498 assert(!i->final_path);
501 layer = dkr_import_current_layer(i);
503 return 0; /* no more layers */
505 path = strjoin(i->image_root, "/.dkr-", layer, NULL);
509 if (laccess(path, F_OK) < 0) {
513 return log_error_errno(errno, "Failed to check for container: %m");
516 log_info("Layer %s already exists, skipping.", layer);
518 i->current_ancestry++;
524 log_info("Pulling layer %s...", layer);
526 i->final_path = path;
529 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", layer, "/layer");
530 r = import_job_new(&i->layer_job, url, i->glue, i);
532 return log_error_errno(r, "Failed to allocate layer job: %m");
534 r = dkr_import_add_token(i, i->layer_job);
538 i->layer_job->on_finished = dkr_import_job_on_finished;
539 i->layer_job->on_open_disk = dkr_import_job_on_open_disk;
541 r = import_job_begin(i->layer_job);
543 return log_error_errno(r, "Failed to start layer job: %m");
548 static void dkr_import_job_on_finished(ImportJob *j) {
557 if (j == i->images_job)
558 log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)");
559 else if (j == i->tags_job)
560 log_error_errno(j->error, "Failed to retrieve tags list.");
561 else if (j == i->ancestry_job)
562 log_error_errno(j->error, "Failed to retrieve ancestry list.");
563 else if (j == i->json_job)
564 log_error_errno(j->error, "Failed to retrieve json data.");
566 log_error_errno(j->error, "Failed to retrieve layer data.");
572 if (i->images_job == j) {
575 assert(!i->tags_job);
576 assert(!i->ancestry_job);
577 assert(!i->json_job);
578 assert(!i->layer_job);
580 if (strv_isempty(i->response_registries)) {
582 log_error("Didn't get registry information.");
586 log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]);
588 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->tag);
589 r = import_job_new(&i->tags_job, url, i->glue, i);
591 log_error_errno(r, "Failed to allocate tags job: %m");
595 r = dkr_import_add_token(i, i->tags_job);
601 i->tags_job->on_finished = dkr_import_job_on_finished;
603 r = import_job_begin(i->tags_job);
605 log_error_errno(r, "Failed to start tags job: %m");
609 } else if (i->tags_job == j) {
613 assert(!i->ancestry_job);
614 assert(!i->json_job);
615 assert(!i->layer_job);
617 r = parse_id(j->payload, j->payload_size, &id);
619 log_error_errno(r, "Failed to parse JSON id.");
626 log_info("Tag lookup succeeded, resolved to layer %s.", i->id);
628 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/ancestry");
629 r = import_job_new(&i->ancestry_job, url, i->glue, i);
631 log_error_errno(r, "Failed to allocate ancestry job: %m");
635 r = dkr_import_add_token(i, i->ancestry_job);
641 i->ancestry_job->on_finished = dkr_import_job_on_finished;
643 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/json");
644 r = import_job_new(&i->json_job, url, i->glue, i);
646 log_error_errno(r, "Failed to allocate json job: %m");
650 r = dkr_import_add_token(i, i->json_job);
656 i->json_job->on_finished = dkr_import_job_on_finished;
658 r = import_job_begin(i->ancestry_job);
660 log_error_errno(r, "Failed to start ancestry job: %m");
664 r = import_job_begin(i->json_job);
666 log_error_errno(r, "Failed to start json job: %m");
670 } else if (i->ancestry_job == j) {
671 char **ancestry = NULL, **k;
674 assert(!i->layer_job);
676 r = parse_ancestry(j->payload, j->payload_size, &ancestry);
678 log_error_errno(r, "Failed to parse JSON id.");
682 n = strv_length(ancestry);
683 if (n <= 0 || !streq(ancestry[n-1], i->id)) {
684 log_error("Ancestry doesn't end in main layer.");
690 log_info("Ancestor lookup succeeded, requires layers:\n");
691 STRV_FOREACH(k, ancestry)
692 log_info("\t%s", *k);
694 strv_free(i->ancestry);
695 i->ancestry = ancestry;
697 i->current_ancestry = 0;
698 r = dkr_import_pull_layer(i);
702 } else if (i->layer_job == j) {
703 assert(i->temp_path);
704 assert(i->final_path);
706 j->disk_fd = safe_close(j->disk_fd);
708 if (i->tar_pid > 0) {
709 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
715 r = aufs_resolve(i->temp_path);
717 log_error_errno(r, "Failed to resolve aufs whiteouts: %m");
721 r = btrfs_subvol_set_read_only(i->temp_path, true);
723 log_error_errno(r, "Failed to mark snapshort read-only: %m");
727 if (rename(i->temp_path, i->final_path) < 0) {
728 log_error_errno(errno, "Failed to rename snaphsot: %m");
732 log_info("Completed writing to layer %s.", i->final_path);
734 i->layer_job = import_job_unref(i->layer_job);
738 i->final_path = NULL;
740 i->current_ancestry ++;
741 r = dkr_import_pull_layer(i);
745 } else if (i->json_job != j)
746 assert_not_reached("Got finished event for unknown curl object");
748 if (!dkr_import_is_done(i))
751 r = dkr_import_make_local_copy(i);
759 i->on_finished(i, r, i->userdata);
761 sd_event_exit(i->event, r);
764 static int dkr_import_job_on_header(ImportJob *j, const char *header, size_t sz) {
765 _cleanup_free_ char *registry = NULL;
775 r = curl_header_strdup(header, sz, HEADER_TOKEN, &token);
779 free(i->response_token);
780 i->response_token = token;
784 r = curl_header_strdup(header, sz, HEADER_REGISTRY, ®istry);
790 l = strv_split(registry, ",");
795 if (!hostname_is_valid(*k)) {
796 log_error("Registry hostname is not valid.");
802 strv_free(i->response_registries);
803 i->response_registries = l;
809 int dkr_import_pull(DkrImport *i, const char *name, const char *tag, const char *local, bool force_local) {
815 if (!dkr_name_is_valid(name))
818 if (tag && !dkr_tag_is_valid(tag))
821 if (local && !machine_name_is_valid(local))
830 r = free_and_strdup(&i->local, local);
833 i->force_local = force_local;
835 r = free_and_strdup(&i->name, name);
838 r = free_and_strdup(&i->tag, tag);
842 url = strappenda(i->index_url, "/v1/repositories/", name, "/images");
844 r = import_job_new(&i->images_job, url, i->glue, i);
848 r = dkr_import_add_token(i, i->images_job);
852 i->images_job->on_finished = dkr_import_job_on_finished;
853 i->images_job->on_header = dkr_import_job_on_header;
855 return import_job_begin(i->images_job);
858 bool dkr_name_is_valid(const char *name) {
859 const char *slash, *p;
864 slash = strchr(name, '/');
868 if (!filename_is_valid(slash + 1))
871 p = strndupa(name, slash - name);
872 if (!filename_is_valid(p))
878 bool dkr_id_is_valid(const char *id) {
880 if (!filename_is_valid(id))
883 if (!in_charset(id, "0123456789abcdef"))