src/import/import-dkr.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <curl/curl.h>
  23 #include <sys/prctl.h>
  24
  25 #include "hashmap.h"
  26 #include "set.h"
  27 #include "json.h"
  28 #include "strv.h"
  29 #include "curl-util.h"
  30 #include "import-dkr.h"
  31 #include "btrfs-util.h"
  32 #include "aufs-util.h"
  33 #include "utf8.h"
  34
  35 /* TODO:
  36   - convert json bits
  37   - man page
  38   - fall back to btrfs loop pool device
  39 */
  40
  41 typedef struct DkrImportJob DkrImportJob;
  42 typedef struct DkrImportName DkrImportName;
  43
  44 typedef enum DkrImportJobType {
  45         DKR_IMPORT_JOB_IMAGES,
  46         DKR_IMPORT_JOB_TAGS,
  47         DKR_IMPORT_JOB_ANCESTRY,
  48         DKR_IMPORT_JOB_JSON,
  49         DKR_IMPORT_JOB_LAYER,
  50 } DkrImportJobType;
  51
  52 struct DkrImportJob {
  53         DkrImport *import;
  54         DkrImportJobType type;
  55         bool done;
  56
  57         char *url;
  58
  59         Set *needed_by; /* DkrImport Name objects */
  60
  61         CURL *curl;
  62         struct curl_slist *request_header;
  63         void *payload;
  64         size_t payload_size;
  65
  66         char *response_token;
  67         char **response_registries;
  68
  69         char *temp_path;
  70         char *final_path;
  71
  72         pid_t tar_pid;
  73         FILE *tar_stream;
  74 };
  75
  76 struct DkrImportName {
  77         DkrImport *import;
  78
  79         char *name;
  80         char *tag;
  81         char *id;
  82         char *local;
  83
  84         DkrImportJob *job_images, *job_tags, *job_ancestry, *job_json, *job_layer;
  85
  86         char **ancestry;
  87         unsigned current_ancestry;
  88
  89         bool force_local;
  90 };
  91
  92 struct DkrImport {
  93         sd_event *event;
  94         CurlGlue *glue;
  95
  96         char *index_url;
  97         char *image_root;
  98
  99         Hashmap *names;
 100         Hashmap *jobs;
 101
 102         dkr_import_on_finished on_finished;
 103         void *userdata;
 104
 105         bool finished;
 106 };
 107
 108 #define PROTOCOL_PREFIX "https://"
 109
 110 #define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"
 111 #define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"
 112
 113 #define PAYLOAD_MAX (16*1024*1024)
 114 #define LAYERS_MAX 2048
 115
 116 static int dkr_import_name_add_job(DkrImportName *name, DkrImportJobType type, const char *url, DkrImportJob **ret);
 117
 118 static DkrImportJob *dkr_import_job_unref(DkrImportJob *job) {
 119         if (!job)
 120                 return NULL;
 121
 122         if (job->import)
 123                 curl_glue_remove_and_free(job->import->glue, job->curl);
 124         curl_slist_free_all(job->request_header);
 125
 126         if (job->tar_stream)
 127                 fclose(job->tar_stream);
 128
 129         free(job->final_path);
 130
 131         if (job->temp_path) {
 132                 btrfs_subvol_remove(job->temp_path);
 133                 free(job->temp_path);
 134         }
 135
 136         set_free(job->needed_by);
 137
 138         if (job->tar_pid > 0)
 139                 kill(job->tar_pid, SIGTERM);
 140
 141         free(job->url);
 142         free(job->payload);
 143         free(job->response_token);
 144         strv_free(job->response_registries);
 145
 146         free(job);
 147
 148         return NULL;
 149 }
 150
 151 static DkrImportName *dkr_import_name_unref(DkrImportName *name) {
 152         if (!name)
 153                 return NULL;
 154
 155         if (name->job_images)
 156                 set_remove(name->job_images->needed_by, name);
 157
 158         if (name->job_tags)
 159                 set_remove(name->job_tags->needed_by, name);
 160
 161         if (name->job_ancestry)
 162                 set_remove(name->job_ancestry->needed_by, name);
 163
 164         if (name->job_json)
 165                 set_remove(name->job_json->needed_by, name);
 166
 167         if (name->job_layer)
 168                 set_remove(name->job_layer->needed_by, name);
 169
 170         free(name->name);
 171         free(name->id);
 172         free(name->tag);
 173         free(name->local);
 174
 175         strv_free(name->ancestry);
 176         free(name);
 177
 178         return NULL;
 179 }
 180
 181 DEFINE_TRIVIAL_CLEANUP_FUNC(DkrImportJob*, dkr_import_job_unref);
 182 DEFINE_TRIVIAL_CLEANUP_FUNC(DkrImportName*, dkr_import_name_unref);
 183
 184 static void dkr_import_finish(DkrImport *import, int error) {
 185         assert(import);
 186
 187         if (import->finished)
 188                 return;
 189
 190         import->finished = true;
 191
 192         if (import->on_finished)
 193                 import->on_finished(import, error, import->userdata);
 194         else
 195                 sd_event_exit(import->event, error);
 196 }
 197
 198 static int parse_id(const void *payload, size_t size, char **ret) {
 199         _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;
 200         union json_value v = {};
 201         void *json_state = NULL;
 202         const char *p;
 203         int t;
 204
 205         assert(payload);
 206         assert(ret);
 207
 208         if (size <= 0)
 209                 return -EBADMSG;
 210
 211         if (memchr(payload, 0, size))
 212                 return -EBADMSG;
 213
 214         buf = strndup(payload, size);
 215         if (!buf)
 216                 return -ENOMEM;
 217
 218         p = buf;
 219         t = json_tokenize(&p, &id, &v, &json_state, NULL);
 220         if (t < 0)
 221                 return t;
 222         if (t != JSON_STRING)
 223                 return -EBADMSG;
 224
 225         t = json_tokenize(&p, &other, &v, &json_state, NULL);
 226         if (t < 0)
 227                 return t;
 228         if (t != JSON_END)
 229                 return -EBADMSG;
 230
 231         if (!dkr_id_is_valid(id))
 232                 return -EBADMSG;
 233
 234         *ret = id;
 235         id = NULL;
 236
 237         return 0;
 238 }
 239
 240 static int parse_ancestry(const void *payload, size_t size, char ***ret) {
 241         _cleanup_free_ char *buf = NULL;
 242         void *json_state = NULL;
 243         const char *p;
 244         enum {
 245                 STATE_BEGIN,
 246                 STATE_ITEM,
 247                 STATE_COMMA,
 248                 STATE_END,
 249         } state = STATE_BEGIN;
 250         _cleanup_strv_free_ char **l = NULL;
 251         size_t n = 0, allocated = 0;
 252
 253         if (size <= 0)
 254                 return -EBADMSG;
 255
 256         if (memchr(payload, 0, size))
 257                 return -EBADMSG;
 258
 259         buf = strndup(payload, size);
 260         if (!buf)
 261                 return -ENOMEM;
 262
 263         p = buf;
 264         for (;;) {
 265                 _cleanup_free_ char *str;
 266                 union json_value v = {};
 267                 int t;
 268
 269                 t = json_tokenize(&p, &str, &v, &json_state, NULL);
 270                 if (t < 0)
 271                         return t;
 272
 273                 switch (state) {
 274
 275                 case STATE_BEGIN:
 276                         if (t == JSON_ARRAY_OPEN)
 277                                 state = STATE_ITEM;
 278                         else
 279                                 return -EBADMSG;
 280
 281                         break;
 282
 283                 case STATE_ITEM:
 284                         if (t == JSON_STRING) {
 285                                 if (!dkr_id_is_valid(str))
 286                                         return -EBADMSG;
 287
 288                                 if (n+1 > LAYERS_MAX)
 289                                         return -EFBIG;
 290
 291                                 if (!GREEDY_REALLOC(l, allocated, n + 2))
 292                                         return -ENOMEM;
 293
 294                                 l[n++] = str;
 295                                 str = NULL;
 296                                 l[n] = NULL;
 297
 298                                 state = STATE_COMMA;
 299
 300                         } else if (t == JSON_ARRAY_CLOSE)
 301                                 state = STATE_END;
 302                         else
 303                                 return -EBADMSG;
 304
 305                         break;
 306
 307                 case STATE_COMMA:
 308                         if (t == JSON_COMMA)
 309                                 state = STATE_ITEM;
 310                         else if (t == JSON_ARRAY_CLOSE)
 311                                 state = STATE_END;
 312                         else
 313                                 return -EBADMSG;
 314                         break;
 315
 316                 case STATE_END:
 317                         if (t == JSON_END) {
 318
 319                                 if (strv_isempty(l))
 320                                         return -EBADMSG;
 321
 322                                 if (!strv_is_uniq(l))
 323                                         return -EBADMSG;
 324
 325                                 l = strv_reverse(l);
 326
 327                                 *ret = l;
 328                                 l = NULL;
 329                                 return 0;
 330                         } else
 331                                 return -EBADMSG;
 332                 }
 333
 334         }
 335 }
 336
 337 static const char *dkr_import_name_current_layer(DkrImportName *name) {
 338         assert(name);
 339
 340         if (strv_isempty(name->ancestry))
 341                 return NULL;
 342
 343         return name->ancestry[name->current_ancestry];
 344 }
 345
 346 static const char *dkr_import_name_current_base_layer(DkrImportName *name) {
 347         assert(name);
 348
 349         if (strv_isempty(name->ancestry))
 350                 return NULL;
 351
 352         if (name->current_ancestry <= 0)
 353                 return NULL;
 354
 355         return name->ancestry[name->current_ancestry-1];
 356 }
 357
 358 static char** dkr_import_name_get_registries(DkrImportName *name) {
 359         assert(name);
 360
 361         if (!name->job_images)
 362                 return NULL;
 363
 364         if (!name->job_images->done)
 365                 return NULL;
 366
 367         if (strv_isempty(name->job_images->response_registries))
 368                 return NULL;
 369
 370         return name->job_images->response_registries;
 371 }
 372
 373 static const char*dkr_import_name_get_token(DkrImportName *name) {
 374         assert(name);
 375
 376         if (!name->job_images)
 377                 return NULL;
 378
 379         if (!name->job_images->done)
 380                 return NULL;
 381
 382         return name->job_images->response_token;
 383 }
 384
 385 static void dkr_import_name_maybe_finish(DkrImportName *name) {
 386         int r;
 387
 388         assert(name);
 389
 390         if (!name->job_images || !name->job_images->done)
 391                 return;
 392
 393         if (!name->job_ancestry || !name->job_ancestry->done)
 394                 return;
 395
 396         if (!name->job_json || !name->job_json->done)
 397                 return;
 398
 399         if (name->job_layer && !name->job_json->done)
 400                 return;
 401
 402         if (dkr_import_name_current_layer(name))
 403                 return;
 404
 405         if (name->local) {
 406                 const char *p, *q;
 407
 408                 assert(name->id);
 409
 410                 p = strappenda(name->import->image_root, "/", name->local);
 411                 q = strappenda(name->import->image_root, "/.dkr-", name->id);
 412
 413                 if (name->force_local) {
 414                         (void) btrfs_subvol_remove(p);
 415                         (void) rm_rf_dangerous(p, false, true, false);
 416                 }
 417
 418                 r = btrfs_subvol_snapshot(q, p, false, false);
 419                 if (r < 0) {
 420                         log_error_errno(r, "Failed to snapshot local image: %m");
 421                         dkr_import_finish(name->import, r);
 422                         return;
 423                 }
 424
 425                 log_info("Created new local image '%s'.", name->local);
 426         }
 427
 428         dkr_import_finish(name->import, 0);
 429 }
 430
 431 static int dkr_import_job_run_tar(DkrImportJob *job) {
 432         _cleanup_close_pair_ int pipefd[2] = { -1, -1 };
 433         bool gzip;
 434
 435         assert(job);
 436
 437         /* A stream to run tar on? */
 438         if (!job->temp_path)
 439                 return 0;
 440
 441         if (job->tar_stream)
 442                 return 0;
 443
 444         /* Maybe fork off tar, if we have enough to figure out that
 445          * something is gzip compressed or not */
 446
 447         if (job->payload_size < 2)
 448                 return 0;
 449
 450         /* Detect gzip signature */
 451         gzip = ((uint8_t*) job->payload)[0] == 0x1f &&
 452                ((uint8_t*) job->payload)[1] == 0x8b;
 453
 454         assert(!job->tar_stream);
 455         assert(job->tar_pid <= 0);
 456
 457         if (pipe2(pipefd, O_CLOEXEC) < 0)
 458                 return log_error_errno(errno, "Failed to create pipe for tar: %m");
 459
 460         job->tar_pid = fork();
 461         if (job->tar_pid < 0)
 462                 return log_error_errno(errno, "Failed to fork off tar: %m");
 463         if (job->tar_pid == 0) {
 464                 int null_fd;
 465
 466                 reset_all_signal_handlers();
 467                 reset_signal_mask();
 468                 assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0);
 469
 470                 pipefd[1] = safe_close(pipefd[1]);
 471
 472                 if (dup2(pipefd[0], STDIN_FILENO) != STDIN_FILENO) {
 473                         log_error_errno(errno, "Failed to dup2() fd: %m");
 474                         _exit(EXIT_FAILURE);
 475                 }
 476
 477                 if (pipefd[0] != STDIN_FILENO)
 478                         safe_close(pipefd[0]);
 479                 if (pipefd[1] != STDIN_FILENO)
 480                         safe_close(pipefd[1]);
 481
 482                 null_fd = open("/dev/null", O_WRONLY|O_NOCTTY);
 483                 if (null_fd < 0) {
 484                         log_error_errno(errno, "Failed to open /dev/null: %m");
 485                         _exit(EXIT_FAILURE);
 486                 }
 487
 488                 if (dup2(null_fd, STDOUT_FILENO) != STDOUT_FILENO) {
 489                         log_error_errno(errno, "Failed to dup2() fd: %m");
 490                         _exit(EXIT_FAILURE);
 491                 }
 492
 493                 if (null_fd != STDOUT_FILENO)
 494                         safe_close(null_fd);
 495
 496                 execlp("tar", "tar", "-C", job->temp_path, gzip ? "-xpz" : "-px", "--numeric-owner", NULL);
 497                 _exit(EXIT_FAILURE);
 498         }
 499
 500         pipefd[0] = safe_close(pipefd[0]);
 501
 502         job->tar_stream = fdopen(pipefd[1], "w");
 503         if (!job->tar_stream)
 504                 return log_error_errno(errno, "Failed to allocate tar stream: %m");
 505
 506         pipefd[1] = -1;
 507
 508         if (fwrite(job->payload, 1, job->payload_size, job->tar_stream) != job->payload_size)
 509                 return log_error_errno(errno, "Couldn't write payload: %m");
 510
 511         free(job->payload);
 512         job->payload = NULL;
 513         job->payload_size = 0;
 514
 515         return 0;
 516 }
 517
 518 static int dkr_import_name_pull_layer(DkrImportName *name) {
 519         _cleanup_free_ char *path = NULL, *temp = NULL;
 520         const char *url, *layer = NULL, *base = NULL;
 521         char **rg;
 522         int r;
 523
 524         assert(name);
 525
 526         if (name->job_layer) {
 527                 set_remove(name->job_layer->needed_by, name);
 528                 name->job_layer = NULL;
 529         }
 530
 531         for (;;) {
 532                 layer = dkr_import_name_current_layer(name);
 533                 if (!layer) {
 534                         dkr_import_name_maybe_finish(name);
 535                         return 0;
 536                 }
 537
 538                 path = strjoin(name->import->image_root, "/.dkr-", layer, NULL);
 539                 if (!path)
 540                         return log_oom();
 541
 542                 if (laccess(path, F_OK) < 0) {
 543                         if (errno == ENOENT)
 544                                 break;
 545
 546                         return log_error_errno(errno, "Failed to check for container: %m");
 547                 }
 548
 549                 log_info("Layer %s already exists, skipping.", layer);
 550
 551                 name->current_ancestry++;
 552
 553                 free(path);
 554                 path = NULL;
 555         }
 556
 557         rg = dkr_import_name_get_registries(name);
 558         assert(rg && rg[0]);
 559
 560         url = strappenda(PROTOCOL_PREFIX, rg[0], "/v1/images/", layer, "/layer");
 561         r = dkr_import_name_add_job(name, DKR_IMPORT_JOB_LAYER, url, &name->job_layer);
 562         if (r < 0) {
 563                 log_error_errno(r, "Failed to issue HTTP request: %m");
 564                 return r;
 565         }
 566         if (r == 0) /* Already downloading this one? */
 567                 return 0;
 568
 569         log_info("Pulling layer %s...", layer);
 570
 571         r = tempfn_random(path, &temp);
 572         if (r < 0)
 573                 return log_oom();
 574
 575         base = dkr_import_name_current_base_layer(name);
 576         if (base) {
 577                 const char *base_path;
 578
 579                 base_path = strappenda(name->import->image_root, "/.dkr-", base);
 580                 r = btrfs_subvol_snapshot(base_path, temp, false, true);
 581         } else
 582                 r = btrfs_subvol_make(temp);
 583
 584         if (r < 0)
 585                 return log_error_errno(r, "Failed to make btrfs subvolume %s", temp);
 586
 587         name->job_layer->final_path = path;
 588         name->job_layer->temp_path = temp;
 589         path = temp = NULL;
 590
 591         return 0;
 592 }
 593
 594 static void dkr_import_name_job_finished(DkrImportName *name, DkrImportJob *job) {
 595         int r;
 596
 597         assert(name);
 598         assert(job);
 599
 600         if (name->job_images == job) {
 601                 const char *url;
 602                 char **rg;
 603
 604                 assert(!name->job_tags);
 605                 assert(!name->job_ancestry);
 606                 assert(!name->job_json);
 607                 assert(!name->job_layer);
 608
 609                 rg = dkr_import_name_get_registries(name);
 610                 if (strv_isempty(rg)) {
 611                         log_error("Didn't get registry information.");
 612                         r = -EBADMSG;
 613                         goto fail;
 614                 }
 615
 616                 log_info("Index lookup succeeded, directed to registry %s.", rg[0]);
 617
 618                 url = strappenda(PROTOCOL_PREFIX, rg[0], "/v1/repositories/", name->name, "/tags/", name->tag);
 619
 620                 r = dkr_import_name_add_job(name, DKR_IMPORT_JOB_TAGS, url, &name->job_tags);
 621                 if (r < 0) {
 622                         log_error_errno(r, "Failed to issue HTTP request: %m");
 623                         goto fail;
 624                 }
 625
 626         } else if (name->job_tags == job) {
 627                 const char *url;
 628                 char *id = NULL, **rg;
 629
 630                 assert(!name->job_ancestry);
 631                 assert(!name->job_json);
 632                 assert(!name->job_layer);
 633
 634                 r = parse_id(job->payload, job->payload_size, &id);
 635                 if (r < 0) {
 636                         log_error_errno(r, "Failed to parse JSON id.");
 637                         goto fail;
 638                 }
 639
 640                 free(name->id);
 641                 name->id = id;
 642
 643                 rg = dkr_import_name_get_registries(name);
 644                 assert(rg && rg[0]);
 645
 646                 log_info("Tag lookup succeeded, resolved to layer %s.", name->id);
 647
 648                 url = strappenda(PROTOCOL_PREFIX, rg[0], "/v1/images/", name->id, "/ancestry");
 649                 r = dkr_import_name_add_job(name, DKR_IMPORT_JOB_ANCESTRY, url, &name->job_ancestry);
 650                 if (r < 0) {
 651                         log_error_errno(r, "Failed to issue HTTP request: %m");
 652                         goto fail;
 653                 }
 654
 655                 url = strappenda(PROTOCOL_PREFIX, rg[0], "/v1/images/", name->id, "/json");
 656                 r = dkr_import_name_add_job(name, DKR_IMPORT_JOB_JSON, url, &name->job_json);
 657                 if (r < 0) {
 658                         log_error_errno(r, "Failed to issue HTTP request: %m");
 659                         goto fail;
 660                 }
 661
 662         } else if (name->job_ancestry == job) {
 663                 char **ancestry = NULL, **i;
 664                 unsigned n;
 665
 666                 r = parse_ancestry(job->payload, job->payload_size, &ancestry);
 667                 if (r < 0) {
 668                         log_error_errno(r, "Failed to parse JSON id.");
 669                         goto fail;
 670                 }
 671
 672                 n = strv_length(ancestry);
 673                 if (n <= 0 || !streq(ancestry[n-1], name->id)) {
 674                         log_error("Ancestry doesn't end in main layer.");
 675                         r = -EBADMSG;
 676                         goto fail;
 677                 }
 678
 679                 log_info("Ancestor lookup succeeded, requires layers:\n");
 680                 STRV_FOREACH(i, ancestry)
 681                         log_info("\t%s", *i);
 682
 683                 strv_free(name->ancestry);
 684                 name->ancestry = ancestry;
 685
 686                 name->current_ancestry = 0;
 687                 r = dkr_import_name_pull_layer(name);
 688                 if (r < 0)
 689                         goto fail;
 690
 691         } else if (name->job_json == job) {
 692
 693                 dkr_import_name_maybe_finish(name);
 694
 695         } else if (name->job_layer == job) {
 696
 697                 name->current_ancestry ++;
 698                 r = dkr_import_name_pull_layer(name);
 699                 if (r < 0)
 700                         goto fail;
 701
 702         } else
 703                 assert_not_reached("Got finished event for unknown curl object");
 704
 705         return;
 706
 707 fail:
 708         dkr_import_finish(name->import, r);
 709 }
 710
 711 static void dkr_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
 712         DkrImportJob *job = NULL;
 713         CURLcode code;
 714         DkrImportName *n;
 715         long status;
 716         Iterator i;
 717         int r;
 718
 719         if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, &job) != CURLE_OK)
 720                 return;
 721
 722         if (!job || job->done)
 723                 return;
 724
 725         job->done = true;
 726
 727         if (result != CURLE_OK) {
 728                 log_error("Transfer failed: %s", curl_easy_strerror(result));
 729                 r = -EIO;
 730                 goto fail;
 731         }
 732
 733         code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
 734         if (code != CURLE_OK) {
 735                 log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
 736                 r = -EIO;
 737                 goto fail;
 738         } else if (status >= 300) {
 739                 log_error("HTTP request to %s failed with code %li.", job->url, status);
 740                 r = -EIO;
 741                 goto fail;
 742         } else if (status < 200) {
 743                 log_error("HTTP request to %s finished with unexpected code %li.", job->url, status);
 744                 r = -EIO;
 745                 goto fail;
 746         }
 747
 748         switch (job->type) {
 749
 750         case DKR_IMPORT_JOB_LAYER: {
 751                 siginfo_t si;
 752
 753                 if (!job->tar_stream) {
 754                         log_error("Downloaded layer too short.");
 755                         r = -EIO;
 756                         goto fail;
 757                 }
 758
 759                 fclose(job->tar_stream);
 760                 job->tar_stream = NULL;
 761
 762                 assert(job->tar_pid > 0);
 763
 764                 r = wait_for_terminate(job->tar_pid, &si);
 765                 if (r < 0) {
 766                         log_error_errno(r, "Failed to wait for tar process: %m");
 767                         goto fail;
 768                 }
 769
 770                 job->tar_pid = 0;
 771
 772                 if (si.si_code != CLD_EXITED || si.si_status != EXIT_SUCCESS) {
 773                         log_error_errno(r, "tar failed abnormally.");
 774                         r = -EIO;
 775                         goto fail;
 776                 }
 777
 778                 r = aufs_resolve(job->temp_path);
 779                 if (r < 0) {
 780                         log_error_errno(r, "Couldn't resolve aufs whiteouts: %m");
 781                         goto fail;
 782                 }
 783
 784                 r = btrfs_subvol_set_read_only(job->temp_path, true);
 785                 if (r < 0) {
 786                         log_error_errno(r, "Failed to mark snapshot read-only: %m");
 787                         goto fail;
 788                 }
 789
 790                 if (rename(job->temp_path, job->final_path) < 0) {
 791                         log_error_errno(r, "Failed to rename snapshot: %m");
 792                         goto fail;
 793                 }
 794
 795                 log_info("Completed writing to layer %s", job->final_path);
 796                 break;
 797         }
 798
 799         default:
 800                 ;
 801         }
 802
 803         SET_FOREACH(n, job->needed_by, i)
 804                 dkr_import_name_job_finished(n, job);
 805
 806         return;
 807
 808 fail:
 809         dkr_import_finish(job->import, r);
 810 }
 811
 812 static size_t dkr_import_job_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
 813         DkrImportJob *j = userdata;
 814         size_t sz = size * nmemb;
 815         char *p;
 816         int r;
 817
 818         assert(contents);
 819         assert(j);
 820
 821         if (j->done) {
 822                 r = -ESTALE;
 823                 goto fail;
 824         }
 825
 826         if (j->tar_stream) {
 827                 size_t l;
 828
 829                 l = fwrite(contents, size, nmemb, j->tar_stream);
 830                 if (l != nmemb) {
 831                         r = log_error_errno(errno, "Failed to write to tar: %m");
 832                         goto fail;
 833                 }
 834
 835                 return l;
 836         }
 837
 838         if (j->payload_size + sz > PAYLOAD_MAX) {
 839                 log_error("Payload too large.");
 840                 r = -EFBIG;
 841                 goto fail;
 842         }
 843
 844         p = realloc(j->payload, j->payload_size + sz);
 845         if (!p) {
 846                 r = log_oom();
 847                 goto fail;
 848         }
 849
 850         memcpy(p + j->payload_size, contents, sz);
 851         j->payload_size += sz;
 852         j->payload = p;
 853
 854         r = dkr_import_job_run_tar(j);
 855         if (r < 0)
 856                 goto fail;
 857
 858         return sz;
 859
 860 fail:
 861         dkr_import_finish(j->import, r);
 862         return 0;
 863 }
 864
 865 static size_t dkr_import_job_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
 866         _cleanup_free_ char *registry = NULL;
 867         size_t sz = size * nmemb;
 868         DkrImportJob *j = userdata;
 869         char *token;
 870         int r;
 871
 872         assert(contents);
 873         assert(j);
 874
 875         if (j->done) {
 876                 r = -ESTALE;
 877                 goto fail;
 878         }
 879
 880         r = curl_header_strdup(contents, sz, HEADER_TOKEN, &token);
 881         if (r < 0) {
 882                 log_oom();
 883                 goto fail;
 884         }
 885         if (r > 0) {
 886                 free(j->response_token);
 887                 j->response_token = token;
 888         }
 889
 890         r = curl_header_strdup(contents, sz, HEADER_REGISTRY, &registry);
 891         if (r < 0) {
 892                 log_oom();
 893                 goto fail;
 894         }
 895         if (r > 0) {
 896                 char **l, **i;
 897
 898                 l = strv_split(registry, ",");
 899                 if (!l) {
 900                         r = log_oom();
 901                         goto fail;
 902                 }
 903
 904                 STRV_FOREACH(i, l) {
 905                         if (!hostname_is_valid(*i)) {
 906                                 log_error("Registry hostname is not valid.");
 907                                 strv_free(l);
 908                                 r = -EBADMSG;
 909                                 goto fail;
 910                         }
 911                 }
 912
 913                 strv_free(j->response_registries);
 914                 j->response_registries = l;
 915         }
 916
 917         return sz;
 918
 919 fail:
 920         dkr_import_finish(j->import, r);
 921         return 0;
 922 }
 923
 924 static int dkr_import_name_add_job(DkrImportName *name, DkrImportJobType type, const char *url, DkrImportJob **ret) {
 925         _cleanup_(dkr_import_job_unrefp) DkrImportJob *j = NULL;
 926         DkrImportJob *f = NULL;
 927         const char *t, *token;
 928         int r;
 929
 930         assert(name);
 931         assert(url);
 932         assert(ret);
 933
 934         log_info("Getting %s.", url);
 935         f = hashmap_get(name->import->jobs, url);
 936         if (f) {
 937                 if (f->type != type)
 938                         return -EINVAL;
 939
 940                 r = set_put(f->needed_by, name);
 941                 if (r < 0)
 942                         return r;
 943
 944                 return 0;
 945         }
 946
 947         r = hashmap_ensure_allocated(&name->import->jobs, &string_hash_ops);
 948         if (r < 0)
 949                 return r;
 950
 951         j = new0(DkrImportJob, 1);
 952         if (!j)
 953                 return -ENOMEM;
 954
 955         j->import = name->import;
 956         j->type = type;
 957         j->url = strdup(url);
 958         if (!j->url)
 959                 return -ENOMEM;
 960
 961         r = set_ensure_allocated(&j->needed_by, &trivial_hash_ops);
 962         if (r < 0)
 963                 return r;
 964
 965         r = curl_glue_make(&j->curl, j->url, j);
 966         if (r < 0)
 967                 return r;
 968
 969         token = dkr_import_name_get_token(name);
 970         if (token)
 971                 t = strappenda("Authorization: Token ", token);
 972         else
 973                 t = HEADER_TOKEN " true";
 974
 975         j->request_header = curl_slist_new("Accept: application/json", t, NULL);
 976         if (!j->request_header)
 977                 return -ENOMEM;
 978
 979         if (curl_easy_setopt(j->curl, CURLOPT_HTTPHEADER, j->request_header) != CURLE_OK)
 980                 return -EIO;
 981
 982         if (curl_easy_setopt(j->curl, CURLOPT_WRITEFUNCTION, dkr_import_job_write_callback) != CURLE_OK)
 983                 return -EIO;
 984
 985         if (curl_easy_setopt(j->curl, CURLOPT_WRITEDATA, j) != CURLE_OK)
 986                 return -EIO;
 987
 988         if (curl_easy_setopt(j->curl, CURLOPT_HEADERFUNCTION, dkr_import_job_header_callback) != CURLE_OK)
 989                 return -EIO;
 990
 991         if (curl_easy_setopt(j->curl, CURLOPT_HEADERDATA, j) != CURLE_OK)
 992                 return -EIO;
 993
 994         r = curl_glue_add(name->import->glue, j->curl);
 995         if (r < 0)
 996                 return r;
 997
 998         r = hashmap_put(name->import->jobs, j->url, j);
 999         if (r < 0)
1000                 return r;
1001
1002         r = set_put(j->needed_by, name);
1003         if (r < 0) {
1004                 hashmap_remove(name->import->jobs, url);
1005                 return r;
1006         }
1007
1008         *ret = j;
1009         j = NULL;
1010
1011         return 1;
1012 }
1013
1014 static int dkr_import_name_begin(DkrImportName *name) {
1015         const char *url;
1016
1017         assert(name);
1018         assert(!name->job_images);
1019
1020         url = strappenda(name->import->index_url, "/v1/repositories/", name->name, "/images");
1021
1022         return dkr_import_name_add_job(name, DKR_IMPORT_JOB_IMAGES, url, &name->job_images);
1023 }
1024
1025 int dkr_import_new(
1026                 DkrImport **import,
1027                 sd_event *event,
1028                 const char *index_url,
1029                 const char *image_root,
1030                 dkr_import_on_finished on_finished,
1031                 void *userdata) {
1032
1033         _cleanup_(dkr_import_unrefp) DkrImport *i = NULL;
1034         char *e;
1035         int r;
1036
1037         assert(import);
1038         assert(dkr_url_is_valid(index_url));
1039         assert(image_root);
1040
1041         i = new0(DkrImport, 1);
1042         if (!i)
1043                 return -ENOMEM;
1044
1045         i->on_finished = on_finished;
1046         i->userdata = userdata;
1047
1048         i->index_url = strdup(index_url);
1049         if (!i->index_url)
1050                 return -ENOMEM;
1051
1052         i->image_root = strdup(image_root);
1053         if (!i->image_root)
1054                 return -ENOMEM;
1055
1056         e = endswith(i->index_url, "/");
1057         if (e)
1058                 *e = 0;
1059
1060         if (event)
1061                 i->event = sd_event_ref(event);
1062         else {
1063                 r = sd_event_default(&i->event);
1064                 if (r < 0)
1065                         return r;
1066         }
1067
1068         r = curl_glue_new(&i->glue, i->event);
1069         if (r < 0)
1070                 return r;
1071
1072         i->glue->on_finished = dkr_import_curl_on_finished;
1073         i->glue->userdata = i;
1074
1075         *import = i;
1076         i = NULL;
1077
1078         return 0;
1079 }
1080
1081 DkrImport* dkr_import_unref(DkrImport *import) {
1082         DkrImportName *n;
1083         DkrImportJob *j;
1084
1085         if (!import)
1086                 return NULL;
1087
1088         while ((n = hashmap_steal_first(import->names)))
1089                dkr_import_name_unref(n);
1090         hashmap_free(import->names);
1091
1092         while ((j = hashmap_steal_first(import->jobs)))
1093                 dkr_import_job_unref(j);
1094         hashmap_free(import->jobs);
1095
1096         curl_glue_unref(import->glue);
1097         sd_event_unref(import->event);
1098
1099         free(import->index_url);
1100         free(import->image_root);
1101         free(import);
1102
1103         return NULL;
1104 }
1105
1106 int dkr_import_cancel(DkrImport *import, const char *name) {
1107         DkrImportName *n;
1108
1109         assert(import);
1110         assert(name);
1111
1112         n = hashmap_remove(import->names, name);
1113         if (!n)
1114                 return 0;
1115
1116         dkr_import_name_unref(n);
1117         return 1;
1118 }
1119
1120 int dkr_import_pull(DkrImport *import, const char *name, const char *tag, const char *local, bool force_local) {
1121         _cleanup_(dkr_import_name_unrefp) DkrImportName *n = NULL;
1122         int r;
1123
1124         assert(import);
1125         assert(dkr_name_is_valid(name));
1126         assert(dkr_tag_is_valid(tag));
1127         assert(!local || machine_name_is_valid(local));
1128
1129         if (hashmap_get(import->names, name))
1130                 return -EEXIST;
1131
1132         r = hashmap_ensure_allocated(&import->names, &string_hash_ops);
1133         if (r < 0)
1134                 return r;
1135
1136         n = new0(DkrImportName, 1);
1137         if (!n)
1138                 return -ENOMEM;
1139
1140         n->import = import;
1141
1142         n->name = strdup(name);
1143         if (!n->name)
1144                 return -ENOMEM;
1145
1146         n->tag = strdup(tag);
1147         if (!n->tag)
1148                 return -ENOMEM;
1149
1150         if (local) {
1151                 n->local = strdup(local);
1152                 if (!n->local)
1153                         return -ENOMEM;
1154                 n->force_local = force_local;
1155         }
1156
1157         r = hashmap_put(import->names, n->name, n);
1158         if (r < 0)
1159                 return r;
1160
1161         r = dkr_import_name_begin(n);
1162         if (r < 0) {
1163                 dkr_import_cancel(import, n->name);
1164                 n = NULL;
1165                 return r;
1166         }
1167
1168         n = NULL;
1169         return 0;
1170 }
1171
1172 bool dkr_name_is_valid(const char *name) {
1173         const char *slash, *p;
1174
1175         if (isempty(name))
1176                 return false;
1177
1178         slash = strchr(name, '/');
1179         if (!slash)
1180                 return false;
1181
1182         if (!filename_is_valid(slash + 1))
1183                 return false;
1184
1185         p = strndupa(name, slash - name);
1186         if (!filename_is_valid(p))
1187                 return false;
1188
1189         return true;
1190 }
1191
1192 bool dkr_id_is_valid(const char *id) {
1193
1194         if (!filename_is_valid(id))
1195                 return false;
1196
1197         if (!in_charset(id, "0123456789abcdef"))
1198                 return false;
1199
1200         return true;
1201 }
1202
1203 bool dkr_url_is_valid(const char *url) {
1204         if (isempty(url))
1205                 return false;
1206
1207         if (!startswith(url, "http://") &&
1208             !startswith(url, "https://"))
1209                 return false;
1210
1211         return ascii_is_valid(url);
1212 }