chiark / gitweb /
import: drop all capabilities when invoking tar
[elogind.git] / src / import / import-dkr.c
1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
2
3 /***
4   This file is part of systemd.
5
6   Copyright 2014 Lennart Poettering
7
8   systemd is free software; you can redistribute it and/or modify it
9   under the terms of the GNU Lesser General Public License as published by
10   the Free Software Foundation; either version 2.1 of the License, or
11   (at your option) any later version.
12
13   systemd is distributed in the hope that it will be useful, but
14   WITHOUT ANY WARRANTY; without even the implied warranty of
15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16   Lesser General Public License for more details.
17
18   You should have received a copy of the GNU Lesser General Public License
19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
20 ***/
21
22 #include <curl/curl.h>
23 #include <sys/prctl.h>
24
25 #include "set.h"
26 #include "json.h"
27 #include "strv.h"
28 #include "btrfs-util.h"
29 #include "utf8.h"
30 #include "mkdir.h"
31 #include "import-util.h"
32 #include "curl-util.h"
33 #include "aufs-util.h"
34 #include "import-job.h"
35 #include "import-common.h"
36 #include "import-dkr.h"
37
38 struct DkrImport {
39         sd_event *event;
40         CurlGlue *glue;
41
42         char *index_url;
43         char *image_root;
44
45         ImportJob *images_job;
46         ImportJob *tags_job;
47         ImportJob *ancestry_job;
48         ImportJob *json_job;
49         ImportJob *layer_job;
50
51         char *name;
52         char *tag;
53         char *id;
54
55         char *response_token;
56         char **response_registries;
57
58         char **ancestry;
59         unsigned current_ancestry;
60
61         DkrImportFinished on_finished;
62         void *userdata;
63
64         char *local;
65         bool force_local;
66
67         char *temp_path;
68         char *final_path;
69
70         pid_t tar_pid;
71 };
72
73 #define PROTOCOL_PREFIX "https://"
74
75 #define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"
76 #define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"
77
78 #define LAYERS_MAX 2048
79
80 static void dkr_import_job_on_finished(ImportJob *j);
81
82 DkrImport* dkr_import_unref(DkrImport *i) {
83         if (!i)
84                 return NULL;
85
86         if (i->tar_pid > 1) {
87                 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
88                 (void) wait_for_terminate(i->tar_pid, NULL);
89         }
90
91         import_job_unref(i->images_job);
92         import_job_unref(i->tags_job);
93         import_job_unref(i->ancestry_job);
94         import_job_unref(i->json_job);
95         import_job_unref(i->layer_job);
96
97         curl_glue_unref(i->glue);
98         sd_event_unref(i->event);
99
100         if (i->temp_path) {
101                 (void) btrfs_subvol_remove(i->temp_path);
102                 (void) rm_rf_dangerous(i->temp_path, false, true, false);
103                 free(i->temp_path);
104         }
105
106         free(i->name);
107         free(i->tag);
108         free(i->id);
109         free(i->response_token);
110         free(i->response_registries);
111         strv_free(i->ancestry);
112         free(i->final_path);
113         free(i->index_url);
114         free(i->image_root);
115         free(i->local);
116         free(i);
117
118         return NULL;
119 }
120
121 int dkr_import_new(
122                 DkrImport **ret,
123                 sd_event *event,
124                 const char *index_url,
125                 const char *image_root,
126                 DkrImportFinished on_finished,
127                 void *userdata) {
128
129         _cleanup_(dkr_import_unrefp) DkrImport *i = NULL;
130         char *e;
131         int r;
132
133         assert(ret);
134         assert(index_url);
135
136         if (!http_url_is_valid(index_url))
137                 return -EINVAL;
138
139         i = new0(DkrImport, 1);
140         if (!i)
141                 return -ENOMEM;
142
143         i->on_finished = on_finished;
144         i->userdata = userdata;
145
146         i->image_root = strdup(image_root ?: "/var/lib/machines");
147         if (!i->image_root)
148                 return -ENOMEM;
149
150         i->index_url = strdup(index_url);
151         if (!i->index_url)
152                 return -ENOMEM;
153
154         e = endswith(i->index_url, "/");
155         if (e)
156                 *e = 0;
157
158         if (event)
159                 i->event = sd_event_ref(event);
160         else {
161                 r = sd_event_default(&i->event);
162                 if (r < 0)
163                         return r;
164         }
165
166         r = curl_glue_new(&i->glue, i->event);
167         if (r < 0)
168                 return r;
169
170         i->glue->on_finished = import_job_curl_on_finished;
171         i->glue->userdata = i;
172
173         *ret = i;
174         i = NULL;
175
176         return 0;
177 }
178
179 static int parse_id(const void *payload, size_t size, char **ret) {
180         _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;
181         union json_value v = {};
182         void *json_state = NULL;
183         const char *p;
184         int t;
185
186         assert(payload);
187         assert(ret);
188
189         if (size <= 0)
190                 return -EBADMSG;
191
192         if (memchr(payload, 0, size))
193                 return -EBADMSG;
194
195         buf = strndup(payload, size);
196         if (!buf)
197                 return -ENOMEM;
198
199         p = buf;
200         t = json_tokenize(&p, &id, &v, &json_state, NULL);
201         if (t < 0)
202                 return t;
203         if (t != JSON_STRING)
204                 return -EBADMSG;
205
206         t = json_tokenize(&p, &other, &v, &json_state, NULL);
207         if (t < 0)
208                 return t;
209         if (t != JSON_END)
210                 return -EBADMSG;
211
212         if (!dkr_id_is_valid(id))
213                 return -EBADMSG;
214
215         *ret = id;
216         id = NULL;
217
218         return 0;
219 }
220
221 static int parse_ancestry(const void *payload, size_t size, char ***ret) {
222         _cleanup_free_ char *buf = NULL;
223         void *json_state = NULL;
224         const char *p;
225         enum {
226                 STATE_BEGIN,
227                 STATE_ITEM,
228                 STATE_COMMA,
229                 STATE_END,
230         } state = STATE_BEGIN;
231         _cleanup_strv_free_ char **l = NULL;
232         size_t n = 0, allocated = 0;
233
234         if (size <= 0)
235                 return -EBADMSG;
236
237         if (memchr(payload, 0, size))
238                 return -EBADMSG;
239
240         buf = strndup(payload, size);
241         if (!buf)
242                 return -ENOMEM;
243
244         p = buf;
245         for (;;) {
246                 _cleanup_free_ char *str;
247                 union json_value v = {};
248                 int t;
249
250                 t = json_tokenize(&p, &str, &v, &json_state, NULL);
251                 if (t < 0)
252                         return t;
253
254                 switch (state) {
255
256                 case STATE_BEGIN:
257                         if (t == JSON_ARRAY_OPEN)
258                                 state = STATE_ITEM;
259                         else
260                                 return -EBADMSG;
261
262                         break;
263
264                 case STATE_ITEM:
265                         if (t == JSON_STRING) {
266                                 if (!dkr_id_is_valid(str))
267                                         return -EBADMSG;
268
269                                 if (n+1 > LAYERS_MAX)
270                                         return -EFBIG;
271
272                                 if (!GREEDY_REALLOC(l, allocated, n + 2))
273                                         return -ENOMEM;
274
275                                 l[n++] = str;
276                                 str = NULL;
277                                 l[n] = NULL;
278
279                                 state = STATE_COMMA;
280
281                         } else if (t == JSON_ARRAY_CLOSE)
282                                 state = STATE_END;
283                         else
284                                 return -EBADMSG;
285
286                         break;
287
288                 case STATE_COMMA:
289                         if (t == JSON_COMMA)
290                                 state = STATE_ITEM;
291                         else if (t == JSON_ARRAY_CLOSE)
292                                 state = STATE_END;
293                         else
294                                 return -EBADMSG;
295                         break;
296
297                 case STATE_END:
298                         if (t == JSON_END) {
299
300                                 if (strv_isempty(l))
301                                         return -EBADMSG;
302
303                                 if (!strv_is_uniq(l))
304                                         return -EBADMSG;
305
306                                 l = strv_reverse(l);
307
308                                 *ret = l;
309                                 l = NULL;
310                                 return 0;
311                         } else
312                                 return -EBADMSG;
313                 }
314
315         }
316 }
317
318 static const char *dkr_import_current_layer(DkrImport *i) {
319         assert(i);
320
321         if (strv_isempty(i->ancestry))
322                 return NULL;
323
324         return i->ancestry[i->current_ancestry];
325 }
326
327 static const char *dkr_import_current_base_layer(DkrImport *i) {
328         assert(i);
329
330         if (strv_isempty(i->ancestry))
331                 return NULL;
332
333         if (i->current_ancestry <= 0)
334                 return NULL;
335
336         return i->ancestry[i->current_ancestry-1];
337 }
338
339 static int dkr_import_add_token(DkrImport *i, ImportJob *j) {
340         const char *t;
341
342         assert(i);
343         assert(j);
344
345         if (i->response_token)
346                 t = strappenda("Authorization: Token ", i->response_token);
347         else
348                 t = HEADER_TOKEN " true";
349
350         j->request_header = curl_slist_new("Accept: application/json", t, NULL);
351         if (!j->request_header)
352                 return -ENOMEM;
353
354         return 0;
355 }
356
357 static bool dkr_import_is_done(DkrImport *i) {
358         assert(i);
359         assert(i->images_job);
360
361         if (i->images_job->state != IMPORT_JOB_DONE)
362                 return false;
363
364         if (!i->tags_job || i->tags_job->state != IMPORT_JOB_DONE)
365                 return false;
366
367         if (!i->ancestry_job || i->ancestry_job->state != IMPORT_JOB_DONE)
368                 return false;
369
370         if (!i->json_job || i->json_job->state != IMPORT_JOB_DONE)
371                 return false;
372
373         if (i->layer_job && i->layer_job->state != IMPORT_JOB_DONE)
374                 return false;
375
376         if (dkr_import_current_layer(i))
377                 return false;
378
379         return true;
380 }
381
382 static int dkr_import_make_local_copy(DkrImport *i) {
383         int r;
384
385         assert(i);
386
387         if (!i->local)
388                 return 0;
389
390         if (!i->final_path) {
391                 i->final_path = strjoin(i->image_root, "/.dkr-", i->id, NULL);
392                 if (!i->final_path)
393                         return log_oom();
394         }
395
396         r = import_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
397         if (r < 0)
398                 return r;
399
400         return 0;
401 }
402
403 static int dkr_import_job_on_open_disk(ImportJob *j) {
404         const char *base;
405         DkrImport *i;
406         int r;
407
408         assert(j);
409         assert(j->userdata);
410
411         i = j->userdata;
412         assert(i->layer_job == j);
413         assert(i->final_path);
414         assert(!i->temp_path);
415         assert(i->tar_pid <= 0);
416
417         r = tempfn_random(i->final_path, &i->temp_path);
418         if (r < 0)
419                 return log_oom();
420
421         mkdir_parents_label(i->temp_path, 0700);
422
423         base = dkr_import_current_base_layer(i);
424         if (base) {
425                 const char *base_path;
426
427                 base_path = strappenda(i->image_root, "/.dkr-", base);
428                 r = btrfs_subvol_snapshot(base_path, i->temp_path, false, true);
429         } else
430                 r = btrfs_subvol_make(i->temp_path);
431         if (r < 0)
432                 return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path);
433
434         j->disk_fd = import_fork_tar(i->temp_path, &i->tar_pid);
435         if (j->disk_fd < 0)
436                 return j->disk_fd;
437
438         return 0;
439 }
440
441 static int dkr_import_pull_layer(DkrImport *i) {
442         _cleanup_free_ char *path = NULL;
443         const char *url, *layer = NULL;
444         int r;
445
446         assert(i);
447         assert(!i->layer_job);
448         assert(!i->temp_path);
449         assert(!i->final_path);
450
451         for (;;) {
452                 layer = dkr_import_current_layer(i);
453                 if (!layer)
454                         return 0; /* no more layers */
455
456                 path = strjoin(i->image_root, "/.dkr-", layer, NULL);
457                 if (!path)
458                         return log_oom();
459
460                 if (laccess(path, F_OK) < 0) {
461                         if (errno == ENOENT)
462                                 break;
463
464                         return log_error_errno(errno, "Failed to check for container: %m");
465                 }
466
467                 log_info("Layer %s already exists, skipping.", layer);
468
469                 i->current_ancestry++;
470
471                 free(path);
472                 path = NULL;
473         }
474
475         log_info("Pulling layer %s...", layer);
476
477         i->final_path = path;
478         path = NULL;
479
480         url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", layer, "/layer");
481         r = import_job_new(&i->layer_job, url, i->glue, i);
482         if (r < 0)
483                 return log_error_errno(r, "Failed to allocate layer job: %m");
484
485         r = dkr_import_add_token(i, i->layer_job);
486         if (r < 0)
487                 return log_oom();
488
489         i->layer_job->on_finished = dkr_import_job_on_finished;
490         i->layer_job->on_open_disk = dkr_import_job_on_open_disk;
491
492         r = import_job_begin(i->layer_job);
493         if (r < 0)
494                 return log_error_errno(r, "Failed to start layer job: %m");
495
496         return 0;
497 }
498
499 static void dkr_import_job_on_finished(ImportJob *j) {
500         DkrImport *i;
501         int r;
502
503         assert(j);
504         assert(j->userdata);
505
506         i = j->userdata;
507         if (j->error != 0) {
508                 if (j == i->images_job)
509                         log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)");
510                 else if (j == i->tags_job)
511                         log_error_errno(j->error, "Failed to retrieve tags list.");
512                 else if (j == i->ancestry_job)
513                         log_error_errno(j->error, "Failed to retrieve ancestry list.");
514                 else if (j == i->json_job)
515                         log_error_errno(j->error, "Failed to retrieve json data.");
516                 else
517                         log_error_errno(j->error, "Failed to retrieve layer data.");
518
519                 r = j->error;
520                 goto finish;
521         }
522
523         if (i->images_job == j) {
524                 const char *url;
525
526                 assert(!i->tags_job);
527                 assert(!i->ancestry_job);
528                 assert(!i->json_job);
529                 assert(!i->layer_job);
530
531                 if (strv_isempty(i->response_registries)) {
532                         r = -EBADMSG;
533                         log_error("Didn't get registry information.");
534                         goto finish;
535                 }
536
537                 log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]);
538
539                 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->tag);
540                 r = import_job_new(&i->tags_job, url, i->glue, i);
541                 if (r < 0) {
542                         log_error_errno(r, "Failed to allocate tags job: %m");
543                         goto finish;
544                 }
545
546                 r = dkr_import_add_token(i, i->tags_job);
547                 if (r < 0) {
548                         log_oom();
549                         goto finish;
550                 }
551
552                 i->tags_job->on_finished = dkr_import_job_on_finished;
553
554                 r = import_job_begin(i->tags_job);
555                 if (r < 0) {
556                         log_error_errno(r, "Failed to start tags job: %m");
557                         goto finish;
558                 }
559
560         } else if (i->tags_job == j) {
561                 const char *url;
562                 char *id = NULL;
563
564                 assert(!i->ancestry_job);
565                 assert(!i->json_job);
566                 assert(!i->layer_job);
567
568                 r = parse_id(j->payload, j->payload_size, &id);
569                 if (r < 0) {
570                         log_error_errno(r, "Failed to parse JSON id.");
571                         goto finish;
572                 }
573
574                 free(i->id);
575                 i->id = id;
576
577                 log_info("Tag lookup succeeded, resolved to layer %s.", i->id);
578
579                 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/ancestry");
580                 r = import_job_new(&i->ancestry_job, url, i->glue, i);
581                 if (r < 0) {
582                         log_error_errno(r, "Failed to allocate ancestry job: %m");
583                         goto finish;
584                 }
585
586                 r = dkr_import_add_token(i, i->ancestry_job);
587                 if (r < 0) {
588                         log_oom();
589                         goto finish;
590                 }
591
592                 i->ancestry_job->on_finished = dkr_import_job_on_finished;
593
594                 url = strappenda(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/json");
595                 r = import_job_new(&i->json_job, url, i->glue, i);
596                 if (r < 0) {
597                         log_error_errno(r, "Failed to allocate json job: %m");
598                         goto finish;
599                 }
600
601                 r = dkr_import_add_token(i, i->json_job);
602                 if (r < 0) {
603                         log_oom();
604                         goto finish;
605                 }
606
607                 i->json_job->on_finished = dkr_import_job_on_finished;
608
609                 r = import_job_begin(i->ancestry_job);
610                 if (r < 0) {
611                         log_error_errno(r, "Failed to start ancestry job: %m");
612                         goto finish;
613                 }
614
615                 r = import_job_begin(i->json_job);
616                 if (r < 0) {
617                         log_error_errno(r, "Failed to start json job: %m");
618                         goto finish;
619                 }
620
621         } else if (i->ancestry_job == j) {
622                 char **ancestry = NULL, **k;
623                 unsigned n;
624
625                 assert(!i->layer_job);
626
627                 r = parse_ancestry(j->payload, j->payload_size, &ancestry);
628                 if (r < 0) {
629                         log_error_errno(r, "Failed to parse JSON id.");
630                         goto finish;
631                 }
632
633                 n = strv_length(ancestry);
634                 if (n <= 0 || !streq(ancestry[n-1], i->id)) {
635                         log_error("Ancestry doesn't end in main layer.");
636                         strv_free(ancestry);
637                         r = -EBADMSG;
638                         goto finish;
639                 }
640
641                 log_info("Ancestor lookup succeeded, requires layers:\n");
642                 STRV_FOREACH(k, ancestry)
643                         log_info("\t%s", *k);
644
645                 strv_free(i->ancestry);
646                 i->ancestry = ancestry;
647
648                 i->current_ancestry = 0;
649                 r = dkr_import_pull_layer(i);
650                 if (r < 0)
651                         goto finish;
652
653         } else if (i->layer_job == j) {
654                 assert(i->temp_path);
655                 assert(i->final_path);
656
657                 j->disk_fd = safe_close(j->disk_fd);
658
659                 if (i->tar_pid > 0) {
660                         r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
661                         i->tar_pid = 0;
662                         if (r < 0)
663                                 goto finish;
664                 }
665
666                 r = aufs_resolve(i->temp_path);
667                 if (r < 0) {
668                         log_error_errno(r, "Failed to resolve aufs whiteouts: %m");
669                         goto finish;
670                 }
671
672                 r = btrfs_subvol_set_read_only(i->temp_path, true);
673                 if (r < 0) {
674                         log_error_errno(r, "Failed to mark snapshort read-only: %m");
675                         goto finish;
676                 }
677
678                 if (rename(i->temp_path, i->final_path) < 0) {
679                         log_error_errno(errno, "Failed to rename snaphsot: %m");
680                         goto finish;
681                 }
682
683                 log_info("Completed writing to layer %s.", i->final_path);
684
685                 i->layer_job = import_job_unref(i->layer_job);
686                 free(i->temp_path);
687                 i->temp_path = NULL;
688                 free(i->final_path);
689                 i->final_path = NULL;
690
691                 i->current_ancestry ++;
692                 r = dkr_import_pull_layer(i);
693                 if (r < 0)
694                         goto finish;
695
696         } else if (i->json_job != j)
697                 assert_not_reached("Got finished event for unknown curl object");
698
699         if (!dkr_import_is_done(i))
700                 return;
701
702         r = dkr_import_make_local_copy(i);
703         if (r < 0)
704                 goto finish;
705
706         r = 0;
707
708 finish:
709         if (i->on_finished)
710                 i->on_finished(i, r, i->userdata);
711         else
712                 sd_event_exit(i->event, r);
713 }
714
715 static int dkr_import_job_on_header(ImportJob *j, const char *header, size_t sz)  {
716         _cleanup_free_ char *registry = NULL;
717         char *token;
718         DkrImport *i;
719         int r;
720
721         assert(j);
722         assert(j->userdata);
723
724         i = j->userdata;
725
726         r = curl_header_strdup(header, sz, HEADER_TOKEN, &token);
727         if (r < 0)
728                 return log_oom();
729         if (r > 0) {
730                 free(i->response_token);
731                 i->response_token = token;
732                 return 0;
733         }
734
735         r = curl_header_strdup(header, sz, HEADER_REGISTRY, &registry);
736         if (r < 0)
737                 return log_oom();
738         if (r > 0) {
739                 char **l, **k;
740
741                 l = strv_split(registry, ",");
742                 if (!l)
743                         return log_oom();
744
745                 STRV_FOREACH(k, l) {
746                         if (!hostname_is_valid(*k)) {
747                                 log_error("Registry hostname is not valid.");
748                                 strv_free(l);
749                                 return -EBADMSG;
750                         }
751                 }
752
753                 strv_free(i->response_registries);
754                 i->response_registries = l;
755         }
756
757         return 0;
758 }
759
760 int dkr_import_pull(DkrImport *i, const char *name, const char *tag, const char *local, bool force_local) {
761         const char *url;
762         int r;
763
764         assert(i);
765
766         if (!dkr_name_is_valid(name))
767                 return -EINVAL;
768
769         if (tag && !dkr_tag_is_valid(tag))
770                 return -EINVAL;
771
772         if (local && !machine_name_is_valid(local))
773                 return -EINVAL;
774
775         if (i->images_job)
776                 return -EBUSY;
777
778         if (!tag)
779                 tag = "latest";
780
781         r = free_and_strdup(&i->local, local);
782         if (r < 0)
783                 return r;
784         i->force_local = force_local;
785
786         r = free_and_strdup(&i->name, name);
787         if (r < 0)
788                 return r;
789         r = free_and_strdup(&i->tag, tag);
790         if (r < 0)
791                 return r;
792
793         url = strappenda(i->index_url, "/v1/repositories/", name, "/images");
794
795         r = import_job_new(&i->images_job, url, i->glue, i);
796         if (r < 0)
797                 return r;
798
799         r = dkr_import_add_token(i, i->images_job);
800         if (r < 0)
801                 return r;
802
803         i->images_job->on_finished = dkr_import_job_on_finished;
804         i->images_job->on_header = dkr_import_job_on_header;
805
806         return import_job_begin(i->images_job);
807 }