1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2014 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 #include <curl/curl.h>
23 #include <sys/prctl.h>
29 #include "curl-util.h"
30 #include "import-dkr.h"
31 #include "btrfs-util.h"
32 #include "aufs-util.h"
38 - fall back to btrfs loop pool device
41 typedef struct DkrImportJob DkrImportJob;
42 typedef struct DkrImportName DkrImportName;
44 typedef enum DkrImportJobType {
45 DKR_IMPORT_JOB_IMAGES,
47 DKR_IMPORT_JOB_ANCESTRY,
54 DkrImportJobType type;
59 Set *needed_by; /* DkrImport Name objects */
62 struct curl_slist *request_header;
67 char **response_registries;
76 struct DkrImportName {
84 DkrImportJob *job_images, *job_tags, *job_ancestry, *job_json, *job_layer;
87 unsigned current_ancestry;
102 dkr_import_on_finished on_finished;
108 #define PROTOCOL_PREFIX "https://"
110 #define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"
111 #define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"
113 #define PAYLOAD_MAX (16*1024*1024)
114 #define LAYERS_MAX 2048
116 static int dkr_import_name_add_job(DkrImportName *name, DkrImportJobType type, const char *url, DkrImportJob **ret);
118 static DkrImportJob *dkr_import_job_unref(DkrImportJob *job) {
123 curl_glue_remove_and_free(job->import->glue, job->curl);
124 curl_slist_free_all(job->request_header);
127 fclose(job->tar_stream);
129 free(job->final_path);
131 if (job->temp_path) {
132 btrfs_subvol_remove(job->temp_path);
133 free(job->temp_path);
136 set_free(job->needed_by);
138 if (job->tar_pid > 0)
139 kill(job->tar_pid, SIGTERM);
143 free(job->response_token);
144 strv_free(job->response_registries);
151 static DkrImportName *dkr_import_name_unref(DkrImportName *name) {
155 if (name->job_images)
156 set_remove(name->job_images->needed_by, name);
159 set_remove(name->job_tags->needed_by, name);
161 if (name->job_ancestry)
162 set_remove(name->job_ancestry->needed_by, name);
165 set_remove(name->job_json->needed_by, name);
168 set_remove(name->job_layer->needed_by, name);
175 strv_free(name->ancestry);
181 DEFINE_TRIVIAL_CLEANUP_FUNC(DkrImportJob*, dkr_import_job_unref);
182 DEFINE_TRIVIAL_CLEANUP_FUNC(DkrImportName*, dkr_import_name_unref);
184 static void dkr_import_finish(DkrImport *import, int error) {
187 if (import->finished)
190 import->finished = true;
192 if (import->on_finished)
193 import->on_finished(import, error, import->userdata);
195 sd_event_exit(import->event, error);
198 static int parse_id(const void *payload, size_t size, char **ret) {
199 _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;
200 union json_value v = {};
201 void *json_state = NULL;
211 if (memchr(payload, 0, size))
214 buf = strndup(payload, size);
219 t = json_tokenize(&p, &id, &v, &json_state, NULL);
222 if (t != JSON_STRING)
225 t = json_tokenize(&p, &other, &v, &json_state, NULL);
231 if (!dkr_id_is_valid(id))
240 static int parse_ancestry(const void *payload, size_t size, char ***ret) {
241 _cleanup_free_ char *buf = NULL;
242 void *json_state = NULL;
249 } state = STATE_BEGIN;
250 _cleanup_strv_free_ char **l = NULL;
251 size_t n = 0, allocated = 0;
256 if (memchr(payload, 0, size))
259 buf = strndup(payload, size);
265 _cleanup_free_ char *str;
266 union json_value v = {};
269 t = json_tokenize(&p, &str, &v, &json_state, NULL);
276 if (t == JSON_ARRAY_OPEN)
284 if (t == JSON_STRING) {
285 if (!dkr_id_is_valid(str))
288 if (n+1 > LAYERS_MAX)
291 if (!GREEDY_REALLOC(l, allocated, n + 2))
300 } else if (t == JSON_ARRAY_CLOSE)
310 else if (t == JSON_ARRAY_CLOSE)
322 if (!strv_is_uniq(l))
337 static const char *dkr_import_name_current_layer(DkrImportName *name) {
340 if (strv_isempty(name->ancestry))
343 return name->ancestry[name->current_ancestry];
346 static const char *dkr_import_name_current_base_layer(DkrImportName *name) {
349 if (strv_isempty(name->ancestry))
352 if (name->current_ancestry <= 0)
355 return name->ancestry[name->current_ancestry-1];
358 static char** dkr_import_name_get_registries(DkrImportName *name) {
361 if (!name->job_images)
364 if (!name->job_images->done)
367 if (strv_isempty(name->job_images->response_registries))
370 return name->job_images->response_registries;
373 static const char*dkr_import_name_get_token(DkrImportName *name) {
376 if (!name->job_images)
379 if (!name->job_images->done)
382 return name->job_images->response_token;
385 static void dkr_import_name_maybe_finish(DkrImportName *name) {
390 if (!name->job_images || !name->job_images->done)
393 if (!name->job_ancestry || !name->job_ancestry->done)
396 if (!name->job_json || !name->job_json->done)
399 if (name->job_layer && !name->job_json->done)
402 if (dkr_import_name_current_layer(name))
410 p = strappenda(name->import->image_root, "/", name->local);
411 q = strappenda(name->import->image_root, "/.dkr-", name->id);
413 if (name->force_local) {
414 (void) btrfs_subvol_remove(p);
415 (void) rm_rf_dangerous(p, false, true, false);
418 r = btrfs_subvol_snapshot(q, p, false, false);
420 log_error_errno(r, "Failed to snapshot local image: %m");
421 dkr_import_finish(name->import, r);
425 log_info("Created new local image %s.", p);
428 dkr_import_finish(name->import, 0);
431 static int dkr_import_job_run_tar(DkrImportJob *job) {
432 _cleanup_close_pair_ int pipefd[2] = { -1, -1 };
437 /* A stream to run tar on? */
444 /* Maybe fork off tar, if we have enough to figure out that
445 * something is gzip compressed or not */
447 if (job->payload_size < 2)
450 /* Detect gzip signature */
451 gzip = ((uint8_t*) job->payload)[0] == 0x1f &&
452 ((uint8_t*) job->payload)[1] == 0x8b;
454 assert(!job->tar_stream);
455 assert(job->tar_pid <= 0);
457 if (pipe2(pipefd, O_CLOEXEC) < 0)
458 return log_error_errno(errno, "Failed to create pipe for tar: %m");
460 job->tar_pid = fork();
461 if (job->tar_pid < 0)
462 return log_error_errno(errno, "Failed to fork off tar: %m");
463 if (job->tar_pid == 0) {
466 reset_all_signal_handlers();
468 assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0);
470 pipefd[1] = safe_close(pipefd[1]);
472 if (dup2(pipefd[0], STDIN_FILENO) != STDIN_FILENO) {
473 log_error_errno(errno, "Failed to dup2() fd: %m");
477 if (pipefd[0] != STDIN_FILENO)
478 safe_close(pipefd[0]);
479 if (pipefd[1] != STDIN_FILENO)
480 safe_close(pipefd[1]);
482 null_fd = open("/dev/null", O_WRONLY|O_NOCTTY);
484 log_error_errno(errno, "Failed to open /dev/null: %m");
488 if (dup2(null_fd, STDOUT_FILENO) != STDOUT_FILENO) {
489 log_error_errno(errno, "Failed to dup2() fd: %m");
493 if (null_fd != STDOUT_FILENO)
496 execlp("tar", "tar", "-C", job->temp_path, gzip ? "-xz" : "-x", NULL);
500 pipefd[0] = safe_close(pipefd[0]);
502 job->tar_stream = fdopen(pipefd[1], "w");
503 if (!job->tar_stream)
504 return log_error_errno(errno, "Failed to allocate tar stream: %m");
508 if (fwrite(job->payload, 1, job->payload_size, job->tar_stream) != job->payload_size)
509 return log_error_errno(errno, "Couldn't write payload: %m");
513 job->payload_size = 0;
518 static int dkr_import_name_pull_layer(DkrImportName *name) {
519 _cleanup_free_ char *path = NULL, *temp = NULL;
520 const char *url, *layer = NULL, *base = NULL;
526 if (name->job_layer) {
527 set_remove(name->job_layer->needed_by, name);
528 name->job_layer = NULL;
532 layer = dkr_import_name_current_layer(name);
534 dkr_import_name_maybe_finish(name);
538 path = strjoin(name->import->image_root, "/.dkr-", layer, NULL);
542 if (laccess(path, F_OK) < 0) {
546 return log_error_errno(errno, "Failed to check for container: %m");
549 log_info("Layer %s already exists, skipping.", layer);
551 name->current_ancestry++;
557 rg = dkr_import_name_get_registries(name);
560 url = strappenda(PROTOCOL_PREFIX, rg[0], "/v1/images/", layer, "/layer");
561 r = dkr_import_name_add_job(name, DKR_IMPORT_JOB_LAYER, url, &name->job_layer);
563 log_error_errno(r, "Failed to issue HTTP request: %m");
566 if (r == 0) /* Already downloading this one? */
569 log_info("Pulling layer %s...", layer);
571 r = tempfn_random(path, &temp);
575 base = dkr_import_name_current_base_layer(name);
577 const char *base_path;
579 base_path = strappenda(name->import->image_root, "/.dkr-", base);
580 r = btrfs_subvol_snapshot(base_path, temp, false, true);
582 r = btrfs_subvol_make(temp);
585 return log_error_errno(r, "Failed to make btrfs subvolume %s", temp);
587 name->job_layer->final_path = path;
588 name->job_layer->temp_path = temp;
594 static void dkr_import_name_job_finished(DkrImportName *name, DkrImportJob *job) {
600 if (name->job_images == job) {
604 assert(!name->job_tags);
605 assert(!name->job_ancestry);
606 assert(!name->job_json);
607 assert(!name->job_layer);
609 rg = dkr_import_name_get_registries(name);
610 if (strv_isempty(rg)) {
611 log_error("Didn't get registry information.");
616 log_info("Index lookup succeeded, directed to registry %s.", rg[0]);
618 url = strappenda(PROTOCOL_PREFIX, rg[0], "/v1/repositories/", name->name, "/tags/", name->tag);
620 r = dkr_import_name_add_job(name, DKR_IMPORT_JOB_TAGS, url, &name->job_tags);
622 log_error_errno(r, "Failed to issue HTTP request: %m");
626 } else if (name->job_tags == job) {
628 char *id = NULL, **rg;
630 assert(!name->job_ancestry);
631 assert(!name->job_json);
632 assert(!name->job_layer);
634 r = parse_id(job->payload, job->payload_size, &id);
636 log_error_errno(r, "Failed to parse JSON id.");
643 rg = dkr_import_name_get_registries(name);
646 log_info("Tag lookup succeeded, resolved to layer %s.", name->id);
648 url = strappenda(PROTOCOL_PREFIX, rg[0], "/v1/images/", name->id, "/ancestry");
649 r = dkr_import_name_add_job(name, DKR_IMPORT_JOB_ANCESTRY, url, &name->job_ancestry);
651 log_error_errno(r, "Failed to issue HTTP request: %m");
655 url = strappenda(PROTOCOL_PREFIX, rg[0], "/v1/images/", name->id, "/json");
656 r = dkr_import_name_add_job(name, DKR_IMPORT_JOB_JSON, url, &name->job_json);
658 log_error_errno(r, "Failed to issue HTTP request: %m");
662 } else if (name->job_ancestry == job) {
663 char **ancestry = NULL, **i;
666 r = parse_ancestry(job->payload, job->payload_size, &ancestry);
668 log_error_errno(r, "Failed to parse JSON id.");
672 n = strv_length(ancestry);
673 if (n <= 0 || !streq(ancestry[n-1], name->id)) {
674 log_error("Ancestry doesn't end in main layer.");
679 log_info("Ancestor lookup succeeded, requires layers:\n");
680 STRV_FOREACH(i, ancestry)
681 log_info("\t%s", *i);
683 strv_free(name->ancestry);
684 name->ancestry = ancestry;
686 name->current_ancestry = 0;
687 r = dkr_import_name_pull_layer(name);
691 } else if (name->job_json == job) {
693 dkr_import_name_maybe_finish(name);
695 } else if (name->job_layer == job) {
697 name->current_ancestry ++;
698 r = dkr_import_name_pull_layer(name);
703 assert_not_reached("Got finished event for unknown curl object");
708 dkr_import_finish(name->import, r);
711 static void dkr_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
712 DkrImportJob *job = NULL;
719 if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, &job) != CURLE_OK)
722 if (!job || job->done)
727 if (result != CURLE_OK) {
728 log_error("Transfer failed: %s", curl_easy_strerror(result));
733 code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
734 if (code != CURLE_OK) {
735 log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
738 } else if (status >= 300) {
739 log_error("HTTP request to %s failed with code %li.", job->url, status);
742 } else if (status < 200) {
743 log_error("HTTP request to %s finished with unexpected code %li.", job->url, status);
750 case DKR_IMPORT_JOB_LAYER: {
753 if (!job->tar_stream) {
754 log_error("Downloaded layer too short.");
759 fclose(job->tar_stream);
760 job->tar_stream = NULL;
762 assert(job->tar_pid > 0);
764 r = wait_for_terminate(job->tar_pid, &si);
766 log_error_errno(r, "Failed to wait for tar process: %m");
772 if (si.si_code != CLD_EXITED || si.si_status != EXIT_SUCCESS) {
773 log_error_errno(r, "tar failed abnormally.");
778 r = aufs_resolve(job->temp_path);
780 log_error_errno(r, "Couldn't resolve aufs whiteouts: %m");
784 r = btrfs_subvol_set_read_only(job->temp_path, true);
786 log_error_errno(r, "Failed to mark snapshot read-only: %m");
790 if (rename(job->temp_path, job->final_path) < 0) {
791 log_error_errno(r, "Failed to rename snapshot: %m");
795 log_info("Completed writing to layer %s", job->final_path);
803 SET_FOREACH(n, job->needed_by, i)
804 dkr_import_name_job_finished(n, job);
809 dkr_import_finish(job->import, r);
812 static size_t dkr_import_job_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
813 DkrImportJob *j = userdata;
814 size_t sz = size * nmemb;
829 l = fwrite(contents, size, nmemb, j->tar_stream);
831 r = log_error_errno(errno, "Failed to write to tar: %m");
838 if (j->payload_size + sz > PAYLOAD_MAX) {
839 log_error("Payload too large.");
844 p = realloc(j->payload, j->payload_size + sz);
850 memcpy(p + j->payload_size, contents, sz);
851 j->payload_size += sz;
854 r = dkr_import_job_run_tar(j);
861 dkr_import_finish(j->import, r);
865 static size_t dkr_import_job_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
866 _cleanup_free_ char *registry = NULL;
867 size_t sz = size * nmemb;
868 DkrImportJob *j = userdata;
880 r = curl_header_strdup(contents, sz, HEADER_TOKEN, &token);
886 free(j->response_token);
887 j->response_token = token;
890 r = curl_header_strdup(contents, sz, HEADER_REGISTRY, ®istry);
898 l = strv_split(registry, ",");
905 if (!hostname_is_valid(*i)) {
906 log_error("Registry hostname is not valid.");
913 strv_free(j->response_registries);
914 j->response_registries = l;
920 dkr_import_finish(j->import, r);
924 static int dkr_import_name_add_job(DkrImportName *name, DkrImportJobType type, const char *url, DkrImportJob **ret) {
925 _cleanup_(dkr_import_job_unrefp) DkrImportJob *j = NULL;
926 DkrImportJob *f = NULL;
927 const char *t, *token;
934 log_info("Getting %s.", url);
935 f = hashmap_get(name->import->jobs, url);
940 r = set_put(f->needed_by, name);
947 r = hashmap_ensure_allocated(&name->import->jobs, &string_hash_ops);
951 j = new0(DkrImportJob, 1);
955 j->import = name->import;
957 j->url = strdup(url);
961 r = set_ensure_allocated(&j->needed_by, &trivial_hash_ops);
965 r = curl_glue_make(&j->curl, j->url, j);
969 token = dkr_import_name_get_token(name);
971 t = strappenda("Authorization: Token ", token);
973 t = HEADER_TOKEN " true";
975 j->request_header = curl_slist_new("Accept: application/json", t, NULL);
976 if (!j->request_header)
979 if (curl_easy_setopt(j->curl, CURLOPT_HTTPHEADER, j->request_header) != CURLE_OK)
982 if (curl_easy_setopt(j->curl, CURLOPT_WRITEFUNCTION, dkr_import_job_write_callback) != CURLE_OK)
985 if (curl_easy_setopt(j->curl, CURLOPT_WRITEDATA, j) != CURLE_OK)
988 if (curl_easy_setopt(j->curl, CURLOPT_HEADERFUNCTION, dkr_import_job_header_callback) != CURLE_OK)
991 if (curl_easy_setopt(j->curl, CURLOPT_HEADERDATA, j) != CURLE_OK)
994 r = curl_glue_add(name->import->glue, j->curl);
998 r = hashmap_put(name->import->jobs, j->url, j);
1002 r = set_put(j->needed_by, name);
1004 hashmap_remove(name->import->jobs, url);
1014 static int dkr_import_name_begin(DkrImportName *name) {
1018 assert(!name->job_images);
1020 url = strappenda(name->import->index_url, "/v1/repositories/", name->name, "/images");
1022 return dkr_import_name_add_job(name, DKR_IMPORT_JOB_IMAGES, url, &name->job_images);
1028 const char *index_url,
1029 const char *image_root,
1030 dkr_import_on_finished on_finished,
1033 _cleanup_(dkr_import_unrefp) DkrImport *i = NULL;
1038 assert(dkr_url_is_valid(index_url));
1041 i = new0(DkrImport, 1);
1045 i->on_finished = on_finished;
1046 i->userdata = userdata;
1048 i->index_url = strdup(index_url);
1052 i->image_root = strdup(image_root);
1056 e = endswith(i->index_url, "/");
1061 i->event = sd_event_ref(event);
1063 r = sd_event_default(&i->event);
1068 r = curl_glue_new(&i->glue, i->event);
1072 i->glue->on_finished = dkr_import_curl_on_finished;
1073 i->glue->userdata = i;
1081 DkrImport* dkr_import_unref(DkrImport *import) {
1088 while ((n = hashmap_steal_first(import->names)))
1089 dkr_import_name_unref(n);
1090 hashmap_free(import->names);
1092 while ((j = hashmap_steal_first(import->jobs)))
1093 dkr_import_job_unref(j);
1094 hashmap_free(import->jobs);
1096 curl_glue_unref(import->glue);
1097 sd_event_unref(import->event);
1099 free(import->index_url);
1100 free(import->image_root);
1106 int dkr_import_cancel(DkrImport *import, const char *name) {
1112 n = hashmap_remove(import->names, name);
1116 dkr_import_name_unref(n);
1120 int dkr_import_pull(DkrImport *import, const char *name, const char *tag, const char *local, bool force_local) {
1121 _cleanup_(dkr_import_name_unrefp) DkrImportName *n = NULL;
1125 assert(dkr_name_is_valid(name));
1126 assert(dkr_tag_is_valid(tag));
1127 assert(!local || machine_name_is_valid(local));
1129 if (hashmap_get(import->names, name))
1132 r = hashmap_ensure_allocated(&import->names, &string_hash_ops);
1136 n = new0(DkrImportName, 1);
1142 n->name = strdup(name);
1146 n->tag = strdup(tag);
1151 n->local = strdup(local);
1154 n->force_local = force_local;
1157 r = hashmap_put(import->names, n->name, n);
1161 r = dkr_import_name_begin(n);
1163 dkr_import_cancel(import, n->name);
1172 bool dkr_name_is_valid(const char *name) {
1173 const char *slash, *p;
1178 slash = strchr(name, '/');
1182 if (!filename_is_valid(slash + 1))
1185 p = strndupa(name, slash - name);
1186 if (!filename_is_valid(p))
1192 bool dkr_id_is_valid(const char *id) {
1194 if (!filename_is_valid(id))
1197 if (!in_charset(id, "0123456789abcdef"))
1203 bool dkr_url_is_valid(const char *url) {
1207 if (!startswith(url, "http://") &&
1208 !startswith(url, "https://"))
1211 return ascii_is_valid(url);