src/import/import-dkr.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <curl/curl.h>
  23 #include <sys/prctl.h>
  24
  25 #include "hashmap.h"
  26 #include "set.h"
  27 #include "json.h"
  28 #include "strv.h"
  29 #include "curl-util.h"
  30 #include "import-dkr.h"
  31 #include "btrfs-util.h"
  32 #include "aufs-util.h"
  33 #include "utf8.h"
  34
  35 /* TODO:
  36   - convert json bits
  37   - man page
  38   - fall back to btrfs loop pool device
  39 */
  40
  41 typedef struct DkrImportJob DkrImportJob;
  42 typedef struct DkrImportName DkrImportName;
  43
  44 typedef enum DkrImportJobType {
  45         DKR_IMPORT_JOB_IMAGES,
  46         DKR_IMPORT_JOB_TAGS,
  47         DKR_IMPORT_JOB_ANCESTRY,
  48         DKR_IMPORT_JOB_JSON,
  49         DKR_IMPORT_JOB_LAYER,
  50 } DkrImportJobType;
  51
  52 struct DkrImportJob {
  53         DkrImport *import;
  54         DkrImportJobType type;
  55         bool done;
  56
  57         char *url;
  58
  59         Set *needed_by; /* DkrImport Name objects */
  60
  61         CURL *curl;
  62         struct curl_slist *request_header;
  63         void *payload;
  64         size_t payload_size;
  65
  66         char *response_token;
  67         char **response_registries;
  68
  69         char *temp_path;
  70         char *final_path;
  71
  72         pid_t tar_pid;
  73         FILE *tar_stream;
  74 };
  75
  76 struct DkrImportName {
  77         DkrImport *import;
  78
  79         char *name;
  80         char *tag;
  81         char *id;
  82         char *local;
  83
  84         DkrImportJob *job_images, *job_tags, *job_ancestry, *job_json, *job_layer;
  85
  86         char **ancestry;
  87         unsigned current_ancestry;
  88
  89         bool force_local;
  90 };
  91
  92 struct DkrImport {
  93         sd_event *event;
  94         CurlGlue *glue;
  95
  96         char *index_url;
  97
  98         Hashmap *names;
  99         Hashmap *jobs;
 100
 101         dkr_import_on_finished on_finished;
 102         void *userdata;
 103 };
 104
 105 #define PROTOCOL_PREFIX "https://"
 106
 107 #define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"
 108 #define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"
 109
 110 #define PAYLOAD_MAX (16*1024*1024)
 111 #define LAYERS_MAX 2048
 112
 113 static int dkr_import_name_add_job(DkrImportName *name, DkrImportJobType type, const char *url, DkrImportJob **ret);
 114
 115 static DkrImportJob *dkr_import_job_unref(DkrImportJob *job) {
 116         if (!job)
 117                 return NULL;
 118
 119         if (job->import)
 120                 curl_glue_remove_and_free(job->import->glue, job->curl);
 121         curl_slist_free_all(job->request_header);
 122
 123         if (job->tar_stream)
 124                 fclose(job->tar_stream);
 125
 126         free(job->final_path);
 127
 128         if (job->temp_path) {
 129                 btrfs_subvol_remove(job->temp_path);
 130                 free(job->temp_path);
 131         }
 132
 133         set_free(job->needed_by);
 134
 135         if (job->tar_pid > 0)
 136                 kill(job->tar_pid, SIGTERM);
 137
 138         free(job->url);
 139         free(job->payload);
 140         free(job->response_token);
 141         strv_free(job->response_registries);
 142
 143         free(job);
 144
 145         return NULL;
 146 }
 147
 148 static DkrImportName *dkr_import_name_unref(DkrImportName *name) {
 149         if (!name)
 150                 return NULL;
 151
 152         if (name->job_images)
 153                 set_remove(name->job_images->needed_by, name);
 154
 155         if (name->job_tags)
 156                 set_remove(name->job_tags->needed_by, name);
 157
 158         if (name->job_ancestry)
 159                 set_remove(name->job_ancestry->needed_by, name);
 160
 161         if (name->job_json)
 162                 set_remove(name->job_json->needed_by, name);
 163
 164         if (name->job_layer)
 165                 set_remove(name->job_layer->needed_by, name);
 166
 167         free(name->name);
 168         free(name->id);
 169         free(name->tag);
 170         free(name->local);
 171
 172         strv_free(name->ancestry);
 173         free(name);
 174
 175         return NULL;
 176 }
 177
 178 DEFINE_TRIVIAL_CLEANUP_FUNC(DkrImportJob*, dkr_import_job_unref);
 179 DEFINE_TRIVIAL_CLEANUP_FUNC(DkrImportName*, dkr_import_name_unref);
 180
 181 static void dkr_import_finish(DkrImport *import, int error) {
 182         assert(import);
 183
 184         if (import->on_finished)
 185                 import->on_finished(import, error, import->userdata);
 186         else
 187                 sd_event_exit(import->event, error);
 188 }
 189
 190 static int parse_id(const void *payload, size_t size, char **ret) {
 191         _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;
 192         union json_value v = {};
 193         void *json_state = NULL;
 194         const char *p;
 195         int t;
 196
 197         assert(payload);
 198         assert(ret);
 199
 200         if (size <= 0)
 201                 return -EBADMSG;
 202
 203         if (memchr(payload, 0, size))
 204                 return -EBADMSG;
 205
 206         buf = strndup(payload, size);
 207         if (!buf)
 208                 return -ENOMEM;
 209
 210         p = buf;
 211         t = json_tokenize(&p, &id, &v, &json_state, NULL);
 212         if (t < 0)
 213                 return t;
 214         if (t != JSON_STRING)
 215                 return -EBADMSG;
 216
 217         t = json_tokenize(&p, &other, &v, &json_state, NULL);
 218         if (t < 0)
 219                 return t;
 220         if (t != JSON_END)
 221                 return -EBADMSG;
 222
 223         if (!dkr_id_is_valid(id))
 224                 return -EBADMSG;
 225
 226         *ret = id;
 227         id = NULL;
 228
 229         return 0;
 230 }
 231
 232 static int parse_ancestry(const void *payload, size_t size, char ***ret) {
 233         _cleanup_free_ char *buf = NULL;
 234         void *json_state = NULL;
 235         const char *p;
 236         enum {
 237                 STATE_BEGIN,
 238                 STATE_ITEM,
 239                 STATE_COMMA,
 240                 STATE_END,
 241         } state = STATE_BEGIN;
 242         _cleanup_strv_free_ char **l = NULL;
 243         size_t n = 0, allocated = 0;
 244
 245         if (size <= 0)
 246                 return -EBADMSG;
 247
 248         if (memchr(payload, 0, size))
 249                 return -EBADMSG;
 250
 251         buf = strndup(payload, size);
 252         if (!buf)
 253                 return -ENOMEM;
 254
 255         p = buf;
 256         for (;;) {
 257                 _cleanup_free_ char *str;
 258                 union json_value v = {};
 259                 int t;
 260
 261                 t = json_tokenize(&p, &str, &v, &json_state, NULL);
 262                 if (t < 0)
 263                         return t;
 264
 265                 switch (state) {
 266
 267                 case STATE_BEGIN:
 268                         if (t == JSON_ARRAY_OPEN)
 269                                 state = STATE_ITEM;
 270                         else
 271                                 return -EBADMSG;
 272
 273                         break;
 274
 275                 case STATE_ITEM:
 276                         if (t == JSON_STRING) {
 277                                 if (!dkr_id_is_valid(str))
 278                                         return -EBADMSG;
 279
 280                                 if (n+1 > LAYERS_MAX)
 281                                         return -EFBIG;
 282
 283                                 if (!GREEDY_REALLOC(l, allocated, n + 2))
 284                                         return -ENOMEM;
 285
 286                                 l[n++] = str;
 287                                 str = NULL;
 288                                 l[n] = NULL;
 289
 290                                 state = STATE_COMMA;
 291
 292                         } else if (t == JSON_ARRAY_CLOSE)
 293                                 state = STATE_END;
 294                         else
 295                                 return -EBADMSG;
 296
 297                         break;
 298
 299                 case STATE_COMMA:
 300                         if (t == JSON_COMMA)
 301                                 state = STATE_ITEM;
 302                         else if (t == JSON_ARRAY_CLOSE)
 303                                 state = STATE_END;
 304                         else
 305                                 return -EBADMSG;
 306                         break;
 307
 308                 case STATE_END:
 309                         if (t == JSON_END) {
 310
 311                                 if (strv_isempty(l))
 312                                         return -EBADMSG;
 313
 314                                 if (!strv_is_uniq(l))
 315                                         return -EBADMSG;
 316
 317                                 l = strv_reverse(l);
 318
 319                                 *ret = l;
 320                                 l = NULL;
 321                                 return 0;
 322                         } else
 323                                 return -EBADMSG;
 324                 }
 325
 326         }
 327 }
 328
 329 static const char *dkr_import_name_current_layer(DkrImportName *name) {
 330         assert(name);
 331
 332         if (strv_isempty(name->ancestry))
 333                 return NULL;
 334
 335         return name->ancestry[name->current_ancestry];
 336 }
 337
 338 static const char *dkr_import_name_current_base_layer(DkrImportName *name) {
 339         assert(name);
 340
 341         if (strv_isempty(name->ancestry))
 342                 return NULL;
 343
 344         if (name->current_ancestry <= 0)
 345                 return NULL;
 346
 347         return name->ancestry[name->current_ancestry-1];
 348 }
 349
 350 static char** dkr_import_name_get_registries(DkrImportName *name) {
 351         assert(name);
 352
 353         if (!name->job_images)
 354                 return NULL;
 355
 356         if (!name->job_images->done)
 357                 return NULL;
 358
 359         if (strv_isempty(name->job_images->response_registries))
 360                 return NULL;
 361
 362         return name->job_images->response_registries;
 363 }
 364
 365 static const char*dkr_import_name_get_token(DkrImportName *name) {
 366         assert(name);
 367
 368         if (!name->job_images)
 369                 return NULL;
 370
 371         if (!name->job_images->done)
 372                 return NULL;
 373
 374         return name->job_images->response_token;
 375 }
 376
 377 static void dkr_import_name_maybe_finish(DkrImportName *name) {
 378         int r;
 379
 380         assert(name);
 381
 382         if (!name->job_images || !name->job_images->done)
 383                 return;
 384
 385         if (!name->job_ancestry || !name->job_ancestry->done)
 386                 return;
 387
 388         if (!name->job_json || !name->job_json->done)
 389                 return;
 390
 391         if (name->job_layer && !name->job_json->done)
 392                 return;
 393
 394         if (dkr_import_name_current_layer(name))
 395                 return;
 396
 397         if (name->local) {
 398                 const char *p, *q;
 399
 400                 assert(name->id);
 401
 402                 p = strappenda("/var/lib/container/", name->local);
 403                 q = strappenda("/var/lib/container/.dkr-", name->id);
 404
 405                 if (name->force_local) {
 406                         (void) btrfs_subvol_remove(p);
 407                         (void) rm_rf(p, false, true, false);
 408                 }
 409
 410                 r = btrfs_subvol_snapshot(q, p, false, false);
 411                 if (r < 0) {
 412                         log_error_errno(r, "Failed to snapshot final image: %m");
 413                         dkr_import_finish(name->import, r);
 414                         return;
 415                 }
 416
 417                 log_info("Created new image %s.", p);
 418         }
 419
 420         dkr_import_finish(name->import, 0);
 421 }
 422
 423 static int dkr_import_job_run_tar(DkrImportJob *job) {
 424         _cleanup_close_pair_ int pipefd[2] = { -1, -1 };
 425         bool gzip;
 426
 427         assert(job);
 428
 429         /* A stream to run tar on? */
 430         if (!job->temp_path)
 431                 return 0;
 432
 433         if (job->tar_stream)
 434                 return 0;
 435
 436         /* Maybe fork off tar, if we have enough to figure out that
 437          * something is gzip compressed or not */
 438
 439         if (job->payload_size < 2)
 440                 return 0;
 441
 442         /* Detect gzip signature */
 443         gzip = ((uint8_t*) job->payload)[0] == 0x1f &&
 444                ((uint8_t*) job->payload)[1] == 0x8b;
 445
 446         assert(!job->tar_stream);
 447         assert(job->tar_pid <= 0);
 448
 449         if (pipe2(pipefd, O_CLOEXEC) < 0)
 450                 return log_error_errno(errno, "Failed to create pipe for tar: %m");
 451
 452         job->tar_pid = fork();
 453         if (job->tar_pid < 0)
 454                 return log_error_errno(errno, "Failed to fork off tar: %m");
 455         if (job->tar_pid == 0) {
 456                 int null_fd;
 457
 458                 reset_all_signal_handlers();
 459                 reset_signal_mask();
 460                 assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0);
 461
 462                 pipefd[1] = safe_close(pipefd[1]);
 463
 464                 if (dup2(pipefd[0], STDIN_FILENO) != STDIN_FILENO) {
 465                         log_error_errno(errno, "Failed to dup2() fd: %m");
 466                         _exit(EXIT_FAILURE);
 467                 }
 468
 469                 if (pipefd[0] != STDIN_FILENO)
 470                         safe_close(pipefd[0]);
 471                 if (pipefd[1] != STDIN_FILENO)
 472                         safe_close(pipefd[1]);
 473
 474                 null_fd = open("/dev/null", O_WRONLY|O_NOCTTY);
 475                 if (null_fd < 0) {
 476                         log_error_errno(errno, "Failed to open /dev/null: %m");
 477                         _exit(EXIT_FAILURE);
 478                 }
 479
 480                 if (dup2(null_fd, STDOUT_FILENO) != STDOUT_FILENO) {
 481                         log_error_errno(errno, "Failed to dup2() fd: %m");
 482                         _exit(EXIT_FAILURE);
 483                 }
 484
 485                 if (null_fd != STDOUT_FILENO)
 486                         safe_close(null_fd);
 487
 488                 execlp("tar", "tar", "-C", job->temp_path, gzip ? "-xz" : "-x", NULL);
 489                 _exit(EXIT_FAILURE);
 490         }
 491
 492         pipefd[0] = safe_close(pipefd[0]);
 493
 494         job->tar_stream = fdopen(pipefd[1], "w");
 495         if (!job->tar_stream)
 496                 return log_error_errno(errno, "Failed to allocate tar stream: %m");
 497
 498         pipefd[1] = -1;
 499
 500         if (fwrite(job->payload, 1, job->payload_size, job->tar_stream) != job->payload_size)
 501                 return log_error_errno(errno, "Couldn't write payload: %m");
 502
 503         free(job->payload);
 504         job->payload = NULL;
 505         job->payload_size = 0;
 506
 507         return 0;
 508 }
 509
 510 static int dkr_import_name_pull_layer(DkrImportName *name) {
 511         _cleanup_free_ char *path = NULL, *temp = NULL;
 512         const char *url, *layer = NULL, *base = NULL;
 513         char **rg;
 514         int r;
 515
 516         assert(name);
 517
 518         if (name->job_layer) {
 519                 set_remove(name->job_layer->needed_by, name);
 520                 name->job_layer = NULL;
 521         }
 522
 523         for (;;) {
 524                 layer = dkr_import_name_current_layer(name);
 525                 if (!layer) {
 526                         dkr_import_name_maybe_finish(name);
 527                         return 0;
 528                 }
 529
 530                 path = strjoin("/var/lib/container/.dkr-", layer, NULL);
 531                 if (!path)
 532                         return log_oom();
 533
 534                 if (laccess(path, F_OK) < 0) {
 535                         if (errno == ENOENT)
 536                                 break;
 537
 538                         return log_error_errno(errno, "Failed to check for container: %m");
 539                 }
 540
 541                 log_info("Layer %s already exists, skipping.", layer);
 542
 543                 name->current_ancestry++;
 544
 545                 free(path);
 546                 path = NULL;
 547         }
 548
 549         rg = dkr_import_name_get_registries(name);
 550         assert(rg && rg[0]);
 551
 552         url = strappenda(PROTOCOL_PREFIX, rg[0], "/v1/images/", layer, "/layer");
 553         r = dkr_import_name_add_job(name, DKR_IMPORT_JOB_LAYER, url, &name->job_layer);
 554         if (r < 0) {
 555                 log_error_errno(r, "Failed to issue HTTP request: %m");
 556                 return r;
 557         }
 558         if (r == 0) /* Already downloading this one? */
 559                 return 0;
 560
 561         log_info("Pulling layer %s...", layer);
 562
 563         r = tempfn_random(path, &temp);
 564         if (r < 0)
 565                 return log_oom();
 566
 567         base = dkr_import_name_current_base_layer(name);
 568         if (base) {
 569                 const char *base_path;
 570
 571                 base_path = strappend("/var/lib/container/.dkr-", base);
 572                 r = btrfs_subvol_snapshot(base_path, temp, false, true);
 573         } else
 574                 r = btrfs_subvol_make(temp);
 575
 576         if (r < 0)
 577                 return log_error_errno(r, "Failed to make btrfs subvolume %s", temp);
 578
 579         name->job_layer->final_path = path;
 580         name->job_layer->temp_path = temp;
 581         path = temp = NULL;
 582
 583         return 0;
 584 }
 585
 586 static void dkr_import_name_job_finished(DkrImportName *name, DkrImportJob *job) {
 587         int r;
 588
 589         assert(name);
 590         assert(job);
 591
 592         if (name->job_images == job) {
 593                 const char *url;
 594                 char **rg;
 595
 596                 assert(!name->job_tags);
 597                 assert(!name->job_ancestry);
 598                 assert(!name->job_json);
 599                 assert(!name->job_layer);
 600
 601                 rg = dkr_import_name_get_registries(name);
 602                 if (strv_isempty(rg)) {
 603                         log_error("Didn't get registry information.");
 604                         r = -EBADMSG;
 605                         goto fail;
 606                 }
 607
 608                 log_info("Index lookup succeeded, directed to registry %s.", rg[0]);
 609
 610                 url = strappenda(PROTOCOL_PREFIX, rg[0], "/v1/repositories/", name->name, "/tags/", name->tag);
 611
 612                 r = dkr_import_name_add_job(name, DKR_IMPORT_JOB_TAGS, url, &name->job_tags);
 613                 if (r < 0) {
 614                         log_error_errno(r, "Failed to issue HTTP request: %m");
 615                         goto fail;
 616                 }
 617
 618         } else if (name->job_tags == job) {
 619                 const char *url;
 620                 char *id = NULL, **rg;
 621
 622                 assert(!name->job_ancestry);
 623                 assert(!name->job_json);
 624                 assert(!name->job_layer);
 625
 626                 r = parse_id(job->payload, job->payload_size, &id);
 627                 if (r < 0) {
 628                         log_error_errno(r, "Failed to parse JSON id.");
 629                         goto fail;
 630                 }
 631
 632                 free(name->id);
 633                 name->id = id;
 634
 635                 rg = dkr_import_name_get_registries(name);
 636                 assert(rg && rg[0]);
 637
 638                 log_info("Tag lookup succeeded, resolved to layer %s.", name->id);
 639
 640                 url = strappenda(PROTOCOL_PREFIX, rg[0], "/v1/images/", name->id, "/ancestry");
 641                 r = dkr_import_name_add_job(name, DKR_IMPORT_JOB_ANCESTRY, url, &name->job_ancestry);
 642                 if (r < 0) {
 643                         log_error_errno(r, "Failed to issue HTTP request: %m");
 644                         goto fail;
 645                 }
 646
 647                 url = strappenda(PROTOCOL_PREFIX, rg[0], "/v1/images/", name->id, "/json");
 648                 r = dkr_import_name_add_job(name, DKR_IMPORT_JOB_JSON, url, &name->job_json);
 649                 if (r < 0) {
 650                         log_error_errno(r, "Failed to issue HTTP request: %m");
 651                         goto fail;
 652                 }
 653
 654         } else if (name->job_ancestry == job) {
 655                 char **ancestry = NULL, **i;
 656                 unsigned n;
 657
 658                 r = parse_ancestry(job->payload, job->payload_size, &ancestry);
 659                 if (r < 0) {
 660                         log_error_errno(r, "Failed to parse JSON id.");
 661                         goto fail;
 662                 }
 663
 664                 n = strv_length(ancestry);
 665                 if (n <= 0 || !streq(ancestry[n-1], name->id)) {
 666                         log_error("Ancestry doesn't end in main layer.");
 667                         r = -EBADMSG;
 668                         goto fail;
 669                 }
 670
 671                 log_info("Ancestor lookup succeeded, requires layers:\n");
 672                 STRV_FOREACH(i, ancestry)
 673                         log_info("\t%s", *i);
 674
 675                 strv_free(name->ancestry);
 676                 name->ancestry = ancestry;
 677
 678                 name->current_ancestry = 0;
 679                 r = dkr_import_name_pull_layer(name);
 680                 if (r < 0)
 681                         goto fail;
 682
 683         } else if (name->job_json == job) {
 684
 685                 dkr_import_name_maybe_finish(name);
 686
 687         } else if (name->job_layer == job) {
 688
 689                 name->current_ancestry ++;
 690                 r = dkr_import_name_pull_layer(name);
 691                 if (r < 0)
 692                         goto fail;
 693
 694         } else
 695                 assert_not_reached("Got finished event for unknown curl object");
 696
 697         return;
 698
 699 fail:
 700         dkr_import_finish(name->import, r);
 701 }
 702
 703 static void dkr_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
 704         DkrImportJob *job = NULL;
 705         CURLcode code;
 706         DkrImportName *n;
 707         long status;
 708         Iterator i;
 709         int r;
 710
 711         if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, &job) != CURLE_OK)
 712                 return;
 713
 714         if (!job)
 715                 return;
 716
 717         job->done = true;
 718
 719         if (result != CURLE_OK) {
 720                 log_error("Transfer failed: %s", curl_easy_strerror(result));
 721                 r = -EIO;
 722                 goto fail;
 723         }
 724
 725         code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
 726         if (code != CURLE_OK) {
 727                 log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
 728                 r = -EIO;
 729                 goto fail;
 730         } else if (status >= 300) {
 731                 log_error("HTTP request to %s failed with code %li.", job->url, status);
 732                 r = -EIO;
 733                 goto fail;
 734         } else if (status < 200) {
 735                 log_error("HTTP request to %s finished with unexpected code %li.", job->url, status);
 736                 r = -EIO;
 737                 goto fail;
 738         }
 739
 740         switch (job->type) {
 741
 742         case DKR_IMPORT_JOB_LAYER: {
 743                 siginfo_t si;
 744
 745                 if (!job->tar_stream) {
 746                         log_error("Downloaded layer too short.");
 747                         r = -EIO;
 748                         goto fail;
 749                 }
 750
 751                 fclose(job->tar_stream);
 752                 job->tar_stream = NULL;
 753
 754                 assert(job->tar_pid > 0);
 755
 756                 r = wait_for_terminate(job->tar_pid, &si);
 757                 if (r < 0) {
 758                         log_error_errno(r, "Failed to wait for tar process: %m");
 759                         goto fail;
 760                 }
 761
 762                 job->tar_pid = 0;
 763
 764                 if (si.si_code != CLD_EXITED || si.si_status != EXIT_SUCCESS) {
 765                         log_error_errno(r, "tar failed abnormally.");
 766                         r = -EIO;
 767                         goto fail;
 768                 }
 769
 770                 r = aufs_resolve(job->temp_path);
 771                 if (r < 0) {
 772                         log_error_errno(r, "Couldn't resolve aufs whiteouts: %m");
 773                         goto fail;
 774                 }
 775
 776                 r = btrfs_subvol_read_only(job->temp_path, true);
 777                 if (r < 0) {
 778                         log_error_errno(r, "Failed to mark snapshot read-only: %m");
 779                         goto fail;
 780                 }
 781
 782                 if (rename(job->temp_path, job->final_path) < 0) {
 783                         log_error_errno(r, "Failed to rename snapshot: %m");
 784                         goto fail;
 785                 }
 786
 787                 log_info("Completed writing to layer %s", job->final_path);
 788                 break;
 789         }
 790
 791         default:
 792                 ;
 793         }
 794
 795         SET_FOREACH(n, job->needed_by, i)
 796                 dkr_import_name_job_finished(n, job);
 797
 798         return;
 799
 800 fail:
 801         dkr_import_finish(job->import, r);
 802 }
 803
 804 static size_t dkr_import_job_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
 805         DkrImportJob *j = userdata;
 806         size_t sz = size * nmemb;
 807         char *p;
 808         int r;
 809
 810         assert(contents);
 811         assert(j);
 812
 813         if (j->tar_stream) {
 814                 size_t l;
 815
 816                 l = fwrite(contents, size, nmemb, j->tar_stream);
 817                 if (l != nmemb) {
 818                         r = -errno;
 819                         goto fail;
 820                 }
 821
 822                 return l;
 823         }
 824
 825         if (j->payload_size + sz > PAYLOAD_MAX) {
 826                 r = -EFBIG;
 827                 goto fail;
 828         }
 829
 830         p = realloc(j->payload, j->payload_size + sz);
 831         if (!p) {
 832                 r = -ENOMEM;
 833                 goto fail;
 834         }
 835
 836         memcpy(p + j->payload_size, contents, sz);
 837         j->payload_size += sz;
 838         j->payload = p;
 839
 840         r = dkr_import_job_run_tar(j);
 841         if (r < 0)
 842                 goto fail;
 843
 844         return sz;
 845
 846 fail:
 847         dkr_import_finish(j->import, r);
 848         return 0;
 849 }
 850
 851 static size_t dkr_import_job_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
 852         _cleanup_free_ char *registry = NULL;
 853         size_t sz = size * nmemb;
 854         DkrImportJob *j = userdata;
 855         char *token;
 856         int r;
 857
 858         assert(contents);
 859         assert(j);
 860
 861         r = curl_header_strdup(contents, sz, HEADER_TOKEN, &token);
 862         if (r < 0) {
 863                 log_oom();
 864                 goto fail;
 865         }
 866         if (r > 0) {
 867                 free(j->response_token);
 868                 j->response_token = token;
 869         }
 870
 871         r = curl_header_strdup(contents, sz, HEADER_REGISTRY, &registry);
 872         if (r < 0) {
 873                 log_oom();
 874                 goto fail;
 875         }
 876         if (r > 0) {
 877                 char **l, **i;
 878
 879                 l = strv_split(registry, ",");
 880                 if (!l) {
 881                         r = log_oom();
 882                         goto fail;
 883                 }
 884
 885                 STRV_FOREACH(i, l) {
 886                         if (!hostname_is_valid(*i)) {
 887                                 log_error("Registry hostname is not valid.");
 888                                 strv_free(l);
 889                                 r = -EBADMSG;
 890                                 goto fail;
 891                         }
 892                 }
 893
 894                 strv_free(j->response_registries);
 895                 j->response_registries = l;
 896         }
 897
 898         return sz;
 899
 900 fail:
 901         dkr_import_finish(j->import, r);
 902         return 0;
 903 }
 904
 905 static int dkr_import_name_add_job(DkrImportName *name, DkrImportJobType type, const char *url, DkrImportJob **ret) {
 906         _cleanup_(dkr_import_job_unrefp) DkrImportJob *j = NULL;
 907         DkrImportJob *f = NULL;
 908         const char *t, *token;
 909         int r;
 910
 911         assert(name);
 912         assert(url);
 913         assert(ret);
 914
 915         log_info("Getting %s.", url);
 916         f = hashmap_get(name->import->jobs, url);
 917         if (f) {
 918                 if (f->type != type)
 919                         return -EINVAL;
 920
 921                 r = set_put(f->needed_by, name);
 922                 if (r < 0)
 923                         return r;
 924
 925                 return 0;
 926         }
 927
 928         r = hashmap_ensure_allocated(&name->import->jobs, &string_hash_ops);
 929         if (r < 0)
 930                 return r;
 931
 932         j = new0(DkrImportJob, 1);
 933         if (!j)
 934                 return -ENOMEM;
 935
 936         j->import = name->import;
 937         j->type = type;
 938         j->url = strdup(url);
 939         if (!j->url)
 940                 return -ENOMEM;
 941
 942         r = set_ensure_allocated(&j->needed_by, &trivial_hash_ops);
 943         if (r < 0)
 944                 return r;
 945
 946         r = curl_glue_make(&j->curl, j->url, j);
 947         if (r < 0)
 948                 return r;
 949
 950         token = dkr_import_name_get_token(name);
 951         if (token)
 952                 t = strappenda("Authorization: Token ", token);
 953         else
 954                 t = HEADER_TOKEN " true";
 955
 956         j->request_header = curl_slist_new("Accept: application/json", t, NULL);
 957         if (!j->request_header)
 958                 return -ENOMEM;
 959
 960         if (curl_easy_setopt(j->curl, CURLOPT_HTTPHEADER, j->request_header) != CURLE_OK)
 961                 return -EIO;
 962
 963         if (curl_easy_setopt(j->curl, CURLOPT_WRITEFUNCTION, dkr_import_job_write_callback) != CURLE_OK)
 964                 return -EIO;
 965
 966         if (curl_easy_setopt(j->curl, CURLOPT_WRITEDATA, j) != CURLE_OK)
 967                 return -EIO;
 968
 969         if (curl_easy_setopt(j->curl, CURLOPT_HEADERFUNCTION, dkr_import_job_header_callback) != CURLE_OK)
 970                 return -EIO;
 971
 972         if (curl_easy_setopt(j->curl, CURLOPT_HEADERDATA, j) != CURLE_OK)
 973                 return -EIO;
 974
 975         r = curl_glue_add(name->import->glue, j->curl);
 976         if (r < 0)
 977                 return r;
 978
 979         r = hashmap_put(name->import->jobs, j->url, j);
 980         if (r < 0)
 981                 return r;
 982
 983         r = set_put(j->needed_by, name);
 984         if (r < 0) {
 985                 hashmap_remove(name->import->jobs, url);
 986                 return r;
 987         }
 988
 989         *ret = j;
 990         j = NULL;
 991
 992         return 1;
 993 }
 994
 995 static int dkr_import_name_begin(DkrImportName *name) {
 996         const char *url;
 997
 998         assert(name);
 999         assert(!name->job_images);
1000
1001         url = strappenda(name->import->index_url, "/v1/repositories/", name->name, "/images");
1002
1003         return dkr_import_name_add_job(name, DKR_IMPORT_JOB_IMAGES, url, &name->job_images);
1004 }
1005
1006 int dkr_import_new(DkrImport **import, sd_event *event, const char *index_url, dkr_import_on_finished on_finished, void *userdata) {
1007         _cleanup_(dkr_import_unrefp) DkrImport *i = NULL;
1008         char *e;
1009         int r;
1010
1011         assert(import);
1012         assert(dkr_url_is_valid(index_url));
1013
1014         i = new0(DkrImport, 1);
1015         if (!i)
1016                 return -ENOMEM;
1017
1018         i->on_finished = on_finished;
1019         i->userdata = userdata;
1020
1021         i->index_url = strdup(index_url);
1022         if (!i->index_url)
1023                 return -ENOMEM;
1024
1025         e = endswith(i->index_url, "/");
1026         if (e)
1027                 *e = 0;
1028
1029         if (event)
1030                 i->event = sd_event_ref(event);
1031         else {
1032                 r = sd_event_default(&i->event);
1033                 if (r < 0)
1034                         return r;
1035         }
1036
1037         r = curl_glue_new(&i->glue, i->event);
1038         if (r < 0)
1039                 return r;
1040
1041         i->glue->on_finished = dkr_import_curl_on_finished;
1042         i->glue->userdata = i;
1043
1044         *import = i;
1045         i = NULL;
1046
1047         return 0;
1048 }
1049
1050 DkrImport* dkr_import_unref(DkrImport *import) {
1051         DkrImportName *n;
1052         DkrImportJob *j;
1053
1054         if (!import)
1055                 return NULL;
1056
1057         while ((n = hashmap_steal_first(import->names)))
1058                dkr_import_name_unref(n);
1059         hashmap_free(import->names);
1060
1061         while ((j = hashmap_steal_first(import->jobs)))
1062                 dkr_import_job_unref(j);
1063         hashmap_free(import->jobs);
1064
1065         curl_glue_unref(import->glue);
1066         sd_event_unref(import->event);
1067
1068         free(import->index_url);
1069
1070         free(import);
1071
1072         return NULL;
1073 }
1074
1075 int dkr_import_cancel(DkrImport *import, const char *name) {
1076         DkrImportName *n;
1077
1078         assert(import);
1079         assert(name);
1080
1081         n = hashmap_remove(import->names, name);
1082         if (!n)
1083                 return 0;
1084
1085         dkr_import_name_unref(n);
1086         return 1;
1087 }
1088
1089 int dkr_import_pull(DkrImport *import, const char *name, const char *tag, const char *local, bool force_local) {
1090         _cleanup_(dkr_import_name_unrefp) DkrImportName *n = NULL;
1091         int r;
1092
1093         assert(import);
1094         assert(dkr_name_is_valid(name));
1095         assert(dkr_tag_is_valid(tag));
1096         assert(!local || machine_name_is_valid(local));
1097
1098         if (hashmap_get(import->names, name))
1099                 return -EEXIST;
1100
1101         r = hashmap_ensure_allocated(&import->names, &string_hash_ops);
1102         if (r < 0)
1103                 return r;
1104
1105         n = new0(DkrImportName, 1);
1106         if (!n)
1107                 return -ENOMEM;
1108
1109         n->import = import;
1110
1111         n->name = strdup(name);
1112         if (!n->name)
1113                 return -ENOMEM;
1114
1115         n->tag = strdup(tag);
1116         if (!n->tag)
1117                 return -ENOMEM;
1118
1119         if (local) {
1120                 n->local = strdup(local);
1121                 if (!n->local)
1122                         return -ENOMEM;
1123                 n->force_local = force_local;
1124         }
1125
1126         r = hashmap_put(import->names, name, n);
1127         if (r < 0)
1128                 return r;
1129
1130         r = dkr_import_name_begin(n);
1131         if (r < 0) {
1132                 dkr_import_cancel(import, n->name);
1133                 n = NULL;
1134                 return r;
1135         }
1136
1137         n = NULL;
1138         return 0;
1139 }
1140
1141 bool dkr_name_is_valid(const char *name) {
1142         const char *slash, *p;
1143
1144         if (isempty(name))
1145                 return false;
1146
1147         slash = strchr(name, '/');
1148         if (!slash)
1149                 return false;
1150
1151         if (!filename_is_valid(slash + 1))
1152                 return false;
1153
1154         p = strndupa(name, slash - name);
1155         if (!filename_is_valid(p))
1156                 return false;
1157
1158         return true;
1159 }
1160
1161 bool dkr_id_is_valid(const char *id) {
1162
1163         if (!filename_is_valid(id))
1164                 return false;
1165
1166         if (!in_charset(id, "0123456789abcdef"))
1167                 return false;
1168
1169         return true;
1170 }
1171
1172 bool dkr_url_is_valid(const char *url) {
1173         if (isempty(url))
1174                 return false;
1175
1176         if (!startswith(url, "http://") &&
1177             !startswith(url, "https://"))
1178                 return false;
1179
1180         return ascii_is_valid(url);
1181 }