1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <curl/curl.h>
23 #include <sys/prctl.h>
28 #include "btrfs-util.h"
31 #include "curl-util.h"
32 #include "aufs-util.h"
33 #include "import-util.h"
34 #include "import-job.h"
35 #include "import-dkr.h"
44 ImportJob *images_job;
46 ImportJob *ancestry_job;
55 char **response_registries;
58 unsigned current_ancestry;
60 DkrImportFinished on_finished;
72 #define PROTOCOL_PREFIX "https://"
74 #define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"
75 #define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"
77 #define LAYERS_MAX 2048
79 static void dkr_import_job_on_finished(ImportJob *j);
81 DkrImport* dkr_import_unref(DkrImport *i) {
86 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
87 (void) wait_for_terminate(i->tar_pid, NULL);
90 import_job_unref(i->images_job);
91 import_job_unref(i->tags_job);
92 import_job_unref(i->ancestry_job);
93 import_job_unref(i->json_job);
94 import_job_unref(i->layer_job);
96 curl_glue_unref(i->glue);
97 sd_event_unref(i->event);
100 (void) btrfs_subvol_remove(i->temp_path);
101 (void) rm_rf_dangerous(i->temp_path, false, true, false);
108 free(i->response_token);
109 free(i->response_registries);
110 strv_free(i->ancestry);
123 const char *index_url,
124 const char *image_root,
125 DkrImportFinished on_finished,
128 _cleanup_(dkr_import_unrefp) DkrImport *i = NULL;
135 if (!http_url_is_valid(index_url))
138 i = new0(DkrImport, 1);
142 i->on_finished = on_finished;
143 i->userdata = userdata;
145 i->image_root = strdup(image_root ?: "/var/lib/machines");
149 i->index_url = strdup(index_url);
153 e = endswith(i->index_url, "/");
158 i->event = sd_event_ref(event);
160 r = sd_event_default(&i->event);
165 r = curl_glue_new(&i->glue, i->event);
169 i->glue->on_finished = import_job_curl_on_finished;
170 i->glue->userdata = i;
178 static int parse_id(const void *payload, size_t size, char **ret) {
179 _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;
180 union json_value v = {};
181 void *json_state = NULL;
191 if (memchr(payload, 0, size))
194 buf = strndup(payload, size);
199 t = json_tokenize(&p, &id, &v, &json_state, NULL);
202 if (t != JSON_STRING)
205 t = json_tokenize(&p, &other, &v, &json_state, NULL);
211 if (!dkr_id_is_valid(id))
220 static int parse_ancestry(const void *payload, size_t size, char ***ret) {
221 _cleanup_free_ char *buf = NULL;
222 void *json_state = NULL;
229 } state = STATE_BEGIN;
230 _cleanup_strv_free_ char **l = NULL;
231 size_t n = 0, allocated = 0;
236 if (memchr(payload, 0, size))
239 buf = strndup(payload, size);
245 _cleanup_free_ char *str;
246 union json_value v = {};
249 t = json_tokenize(&p, &str, &v, &json_state, NULL);
256 if (t == JSON_ARRAY_OPEN)
264 if (t == JSON_STRING) {
265 if (!dkr_id_is_valid(str))
268 if (n+1 > LAYERS_MAX)
271 if (!GREEDY_REALLOC(l, allocated, n + 2))
280 } else if (t == JSON_ARRAY_CLOSE)
290 else if (t == JSON_ARRAY_CLOSE)
302 if (!strv_is_uniq(l))
317 static const char *dkr_import_current_layer(DkrImport *i) {
320 if (strv_isempty(i->ancestry))
323 return i->ancestry[i->current_ancestry];
326 static const char *dkr_import_current_base_layer(DkrImport *i) {
329 if (strv_isempty(i->ancestry))
332 if (i->current_ancestry <= 0)
335 return i->ancestry[i->current_ancestry-1];
338 static int dkr_import_add_token(DkrImport *i, ImportJob *j) {
344 if (i->response_token)
345 t = strappenda("Authorization: Token ", i->response_token);
347 t = HEADER_TOKEN " true";
349 j->request_header = curl_slist_new("Accept: application/json", t, NULL);
350 if (!j->request_header)
356 static bool dkr_import_is_done(DkrImport *i) {
358 assert(i->images_job);
360 if (i->images_job->state != IMPORT_JOB_DONE)
363 if (!i->tags_job || i->tags_job->state != IMPORT_JOB_DONE)
366 if (!i->ancestry_job || i->ancestry_job->state != IMPORT_JOB_DONE)
369 if (!i->json_job || i->json_job->state != IMPORT_JOB_DONE)
372 if (i->layer_job && i->layer_job->state != IMPORT_JOB_DONE)
375 if (dkr_import_current_layer(i))
381 static int dkr_import_make_local_copy(DkrImport *i) {
389 if (!i->final_path) {
390 i->final_path = strjoin(i->image_root, "/.dkr-", i->id, NULL);
395 r = import_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
402 static int dkr_import_job_on_open_disk(ImportJob *j) {
403 _cleanup_close_pair_ int pipefd[2] = { -1, -1 };
412 assert(i->layer_job == j);
413 assert(i->final_path);
414 assert(!i->temp_path);
415 assert(i->tar_pid <= 0);
417 r = tempfn_random(i->final_path, &i->temp_path);
421 mkdir_parents_label(i->temp_path, 0700);
423 base = dkr_import_current_base_layer(i);
425 const char *base_path;
427 base_path = strappenda(i->image_root, "/.dkr-", base);
428 r = btrfs_subvol_snapshot(base_path, i->temp_path, false, true);
430 r = btrfs_subvol_make(i->temp_path);
432 return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path);
434 if (pipe2(pipefd, O_CLOEXEC) < 0)
435 return log_error_errno(errno, "Failed to create pipe for tar: %m");
439 return log_error_errno(errno, "Failed to fork off tar: %m");
440 if (i->tar_pid == 0) {
445 reset_all_signal_handlers();
447 assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0);
449 pipefd[1] = safe_close(pipefd[1]);
451 if (dup2(pipefd[0], STDIN_FILENO) != STDIN_FILENO) {
452 log_error_errno(errno, "Failed to dup2() fd: %m");
456 if (pipefd[0] != STDIN_FILENO)
457 safe_close(pipefd[0]);
459 null_fd = open("/dev/null", O_WRONLY|O_NOCTTY);
461 log_error_errno(errno, "Failed to open /dev/null: %m");
465 if (dup2(null_fd, STDOUT_FILENO) != STDOUT_FILENO) {
466 log_error_errno(errno, "Failed to dup2() fd: %m");
470 if (null_fd != STDOUT_FILENO)
473 execlp("tar", "tar", "--numeric-owner", "-C", i->temp_path, "-px", NULL);
474 log_error_errno(errno, "Failed to execute tar: %m");
478 pipefd[0] = safe_close(pipefd[0]);
480 j->disk_fd = pipefd[1];
486 static int dkr_import_pull_layer(DkrImport *i) {
487 _cleanup_free_ char *path = NULL;
488 const char *url, *layer = NULL;
492 assert(!i->layer_job);
493 assert(!i->temp_path);
494 assert(!i->final_path);
497 layer = dkr_import_current_layer(i);
499 return 0; /* no more layers */
501 path = strjoin(i->image_root, "/.dkr-", layer, NULL);
505 if (laccess(path, F_OK) < 0) {
509 return log_error_errno(errno, "Failed to check for container: %m");
512 log_info("Layer %s already exists, skipping.", layer);
514 i->current_ancestry++;
520 log_info("Pulling layer %s...", layer);
522 i->final_path = path;
525 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", layer, "/layer");
526 r = import_job_new(&i->layer_job, url, i->glue, i);
528 return log_error_errno(r, "Failed to allocate layer job: %m");
530 r = dkr_import_add_token(i, i->layer_job);
534 i->layer_job->on_finished = dkr_import_job_on_finished;
535 i->layer_job->on_open_disk = dkr_import_job_on_open_disk;
537 r = import_job_begin(i->layer_job);
539 return log_error_errno(r, "Failed to start layer job: %m");
544 static void dkr_import_job_on_finished(ImportJob *j) {
553 if (j == i->images_job)
554 log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)");
555 else if (j == i->tags_job)
556 log_error_errno(j->error, "Failed to retrieve tags list.");
557 else if (j == i->ancestry_job)
558 log_error_errno(j->error, "Failed to retrieve ancestry list.");
559 else if (j == i->json_job)
560 log_error_errno(j->error, "Failed to retrieve json data.");
562 log_error_errno(j->error, "Failed to retrieve layer data.");
568 if (i->images_job == j) {
571 assert(!i->tags_job);
572 assert(!i->ancestry_job);
573 assert(!i->json_job);
574 assert(!i->layer_job);
576 if (strv_isempty(i->response_registries)) {
578 log_error("Didn't get registry information.");
582 log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]);
584 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->tag);
585 r = import_job_new(&i->tags_job, url, i->glue, i);
587 log_error_errno(r, "Failed to allocate tags job: %m");
591 r = dkr_import_add_token(i, i->tags_job);
597 i->tags_job->on_finished = dkr_import_job_on_finished;
599 r = import_job_begin(i->tags_job);
601 log_error_errno(r, "Failed to start tags job: %m");
605 } else if (i->tags_job == j) {
609 assert(!i->ancestry_job);
610 assert(!i->json_job);
611 assert(!i->layer_job);
613 r = parse_id(j->payload, j->payload_size, &id);
615 log_error_errno(r, "Failed to parse JSON id.");
622 log_info("Tag lookup succeeded, resolved to layer %s.", i->id);
624 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/ancestry");
625 r = import_job_new(&i->ancestry_job, url, i->glue, i);
627 log_error_errno(r, "Failed to allocate ancestry job: %m");
631 r = dkr_import_add_token(i, i->ancestry_job);
637 i->ancestry_job->on_finished = dkr_import_job_on_finished;
639 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/json");
640 r = import_job_new(&i->json_job, url, i->glue, i);
642 log_error_errno(r, "Failed to allocate json job: %m");
646 r = dkr_import_add_token(i, i->json_job);
652 i->json_job->on_finished = dkr_import_job_on_finished;
654 r = import_job_begin(i->ancestry_job);
656 log_error_errno(r, "Failed to start ancestry job: %m");
660 r = import_job_begin(i->json_job);
662 log_error_errno(r, "Failed to start json job: %m");
666 } else if (i->ancestry_job == j) {
667 char **ancestry = NULL, **k;
670 assert(!i->layer_job);
672 r = parse_ancestry(j->payload, j->payload_size, &ancestry);
674 log_error_errno(r, "Failed to parse JSON id.");
678 n = strv_length(ancestry);
679 if (n <= 0 || !streq(ancestry[n-1], i->id)) {
680 log_error("Ancestry doesn't end in main layer.");
686 log_info("Ancestor lookup succeeded, requires layers:\n");
687 STRV_FOREACH(k, ancestry)
688 log_info("\t%s", *k);
690 strv_free(i->ancestry);
691 i->ancestry = ancestry;
693 i->current_ancestry = 0;
694 r = dkr_import_pull_layer(i);
698 } else if (i->layer_job == j) {
699 assert(i->temp_path);
700 assert(i->final_path);
702 j->disk_fd = safe_close(j->disk_fd);
704 if (i->tar_pid > 0) {
705 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
711 r = aufs_resolve(i->temp_path);
713 log_error_errno(r, "Failed to resolve aufs whiteouts: %m");
717 r = btrfs_subvol_set_read_only(i->temp_path, true);
719 log_error_errno(r, "Failed to mark snapshort read-only: %m");
723 if (rename(i->temp_path, i->final_path) < 0) {
724 log_error_errno(errno, "Failed to rename snaphsot: %m");
728 log_info("Completed writing to layer %s.", i->final_path);
730 i->layer_job = import_job_unref(i->layer_job);
734 i->final_path = NULL;
736 i->current_ancestry ++;
737 r = dkr_import_pull_layer(i);
741 } else if (i->json_job != j)
742 assert_not_reached("Got finished event for unknown curl object");
744 if (!dkr_import_is_done(i))
747 r = dkr_import_make_local_copy(i);
755 i->on_finished(i, r, i->userdata);
757 sd_event_exit(i->event, r);
760 static int dkr_import_job_on_header(ImportJob *j, const char *header, size_t sz) {
761 _cleanup_free_ char *registry = NULL;
771 r = curl_header_strdup(header, sz, HEADER_TOKEN, &token);
775 free(i->response_token);
776 i->response_token = token;
780 r = curl_header_strdup(header, sz, HEADER_REGISTRY, ®istry);
786 l = strv_split(registry, ",");
791 if (!hostname_is_valid(*k)) {
792 log_error("Registry hostname is not valid.");
798 strv_free(i->response_registries);
799 i->response_registries = l;
805 int dkr_import_pull(DkrImport *i, const char *name, const char *tag, const char *local, bool force_local) {
811 if (!dkr_name_is_valid(name))
814 if (tag && !dkr_tag_is_valid(tag))
817 if (local && !machine_name_is_valid(local))
826 r = free_and_strdup(&i->local, local);
829 i->force_local = force_local;
831 r = free_and_strdup(&i->name, name);
834 r = free_and_strdup(&i->tag, tag);
838 url = strappenda(i->index_url, "/v1/repositories/", name, "/images");
840 r = import_job_new(&i->images_job, url, i->glue, i);
844 r = dkr_import_add_token(i, i->images_job);
848 i->images_job->on_finished = dkr_import_job_on_finished;
849 i->images_job->on_header = dkr_import_job_on_header;
851 return import_job_begin(i->images_job);
854 bool dkr_name_is_valid(const char *name) {
855 const char *slash, *p;
860 slash = strchr(name, '/');
864 if (!filename_is_valid(slash + 1))
867 p = strndupa(name, slash - name);
868 if (!filename_is_valid(p))
874 bool dkr_id_is_valid(const char *id) {
876 if (!filename_is_valid(id))
879 if (!in_charset(id, "0123456789abcdef"))