src/import/import-raw.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <sys/xattr.h>
  23 #include <linux/fs.h>
  24 #include <sys/prctl.h>
  25 #include <curl/curl.h>
  26 #include <gcrypt.h>
  27
  28 #include "utf8.h"
  29 #include "strv.h"
  30 #include "copy.h"
  31 #include "btrfs-util.h"
  32 #include "util.h"
  33 #include "macro.h"
  34 #include "mkdir.h"
  35 #include "curl-util.h"
  36 #include "qcow2-util.h"
  37 #include "import-job.h"
  38 #include "import-util.h"
  39 #include "import-raw.h"
  40
  41 typedef struct RawImportFile RawImportFile;
  42
  43 struct RawImport {
  44         sd_event *event;
  45         CurlGlue *glue;
  46
  47         char *image_root;
  48
  49         ImportJob *raw_job;
  50         ImportJob *sha256sums_job;
  51         ImportJob *signature_job;
  52
  53         RawImportFinished on_finished;
  54         void *userdata;
  55
  56         char *local;
  57         bool force_local;
  58
  59         char *temp_path;
  60         char *final_path;
  61
  62         ImportVerify verify;
  63 };
  64
  65 RawImport* raw_import_unref(RawImport *i) {
  66         if (!i)
  67                 return NULL;
  68
  69         import_job_unref(i->raw_job);
  70         import_job_unref(i->sha256sums_job);
  71         import_job_unref(i->signature_job);
  72
  73         curl_glue_unref(i->glue);
  74         sd_event_unref(i->event);
  75
  76         if (i->temp_path) {
  77                 (void) unlink(i->temp_path);
  78                 free(i->temp_path);
  79         }
  80
  81         free(i->final_path);
  82         free(i->image_root);
  83         free(i->local);
  84         free(i);
  85
  86         return NULL;
  87 }
  88
  89 int raw_import_new(RawImport **ret, sd_event *event, const char *image_root, RawImportFinished on_finished, void *userdata) {
  90         _cleanup_(raw_import_unrefp) RawImport *i = NULL;
  91         int r;
  92
  93         assert(ret);
  94
  95         i = new0(RawImport, 1);
  96         if (!i)
  97                 return -ENOMEM;
  98
  99         i->on_finished = on_finished;
 100         i->userdata = userdata;
 101
 102         i->image_root = strdup(image_root ?: "/var/lib/machines");
 103         if (!i->image_root)
 104                 return -ENOMEM;
 105
 106         if (event)
 107                 i->event = sd_event_ref(event);
 108         else {
 109                 r = sd_event_default(&i->event);
 110                 if (r < 0)
 111                         return r;
 112         }
 113
 114         r = curl_glue_new(&i->glue, i->event);
 115         if (r < 0)
 116                 return r;
 117
 118         i->glue->on_finished = import_job_curl_on_finished;
 119         i->glue->userdata = i;
 120
 121         *ret = i;
 122         i = NULL;
 123
 124         return 0;
 125 }
 126
 127 static int raw_import_maybe_convert_qcow2(RawImport *i) {
 128         _cleanup_close_ int converted_fd = -1;
 129         _cleanup_free_ char *t = NULL;
 130         int r;
 131
 132         assert(i);
 133         assert(i->raw_job);
 134
 135         r = qcow2_detect(i->raw_job->disk_fd);
 136         if (r < 0)
 137                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
 138         if (r == 0)
 139                 return 0;
 140
 141         /* This is a QCOW2 image, let's convert it */
 142         r = tempfn_random(i->final_path, &t);
 143         if (r < 0)
 144                 return log_oom();
 145
 146         converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
 147         if (converted_fd < 0)
 148                 return log_error_errno(errno, "Failed to create %s: %m", t);
 149
 150         r = chattr_fd(converted_fd, true, FS_NOCOW_FL);
 151         if (r < 0)
 152                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", t);
 153
 154         log_info("Unpacking QCOW2 file.");
 155
 156         r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
 157         if (r < 0) {
 158                 unlink(t);
 159                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
 160         }
 161
 162         unlink(i->temp_path);
 163         free(i->temp_path);
 164
 165         i->temp_path = t;
 166         t = NULL;
 167
 168         safe_close(i->raw_job->disk_fd);
 169         i->raw_job->disk_fd = converted_fd;
 170         converted_fd = -1;
 171
 172         return 1;
 173 }
 174
 175 static int raw_import_make_local_copy(RawImport *i) {
 176         _cleanup_free_ char *tp = NULL;
 177         _cleanup_close_ int dfd = -1;
 178         const char *p;
 179         int r;
 180
 181         assert(i);
 182         assert(i->raw_job);
 183
 184         if (!i->local)
 185                 return 0;
 186
 187         if (i->raw_job->etag_exists) {
 188                 /* We have downloaded this one previously, reopen it */
 189
 190                 assert(i->raw_job->disk_fd < 0);
 191
 192                 if (!i->final_path) {
 193                         r = import_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 194                         if (r < 0)
 195                                 return log_oom();
 196                 }
 197
 198                 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 199                 if (i->raw_job->disk_fd < 0)
 200                         return log_error_errno(errno, "Failed to open vendor image: %m");
 201         } else {
 202                 /* We freshly downloaded the image, use it */
 203
 204                 assert(i->raw_job->disk_fd >= 0);
 205
 206                 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
 207                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 208         }
 209
 210         p = strappenda(i->image_root, "/", i->local, ".raw");
 211
 212         if (i->force_local) {
 213                 (void) btrfs_subvol_remove(p);
 214                 (void) rm_rf_dangerous(p, false, true, false);
 215         }
 216
 217         r = tempfn_random(p, &tp);
 218         if (r < 0)
 219                 return log_oom();
 220
 221         dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 222         if (dfd < 0)
 223                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 224
 225         /* Turn off COW writing. This should greatly improve
 226          * performance on COW file systems like btrfs, since it
 227          * reduces fragmentation caused by not allowing in-place
 228          * writes. */
 229         r = chattr_fd(dfd, true, FS_NOCOW_FL);
 230         if (r < 0)
 231                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
 232
 233         r = copy_bytes(i->raw_job->disk_fd, dfd, (off_t) -1, true);
 234         if (r < 0) {
 235                 unlink(tp);
 236                 return log_error_errno(r, "Failed to make writable copy of image: %m");
 237         }
 238
 239         (void) copy_times(i->raw_job->disk_fd, dfd);
 240         (void) copy_xattr(i->raw_job->disk_fd, dfd);
 241
 242         dfd = safe_close(dfd);
 243
 244         r = rename(tp, p);
 245         if (r < 0)  {
 246                 unlink(tp);
 247                 return log_error_errno(errno, "Failed to move writable image into place: %m");
 248         }
 249
 250         log_info("Created new local image '%s'.", i->local);
 251         return 0;
 252 }
 253
 254 static int raw_import_verify_sha256sum(RawImport *i) {
 255         _cleanup_close_pair_ int gpg_pipe[2] = { -1, -1 };
 256         _cleanup_free_ char *fn = NULL;
 257         _cleanup_close_ int sig_file = -1;
 258         const char *p, *line;
 259         char sig_file_path[] = "/tmp/sigXXXXXX";
 260         _cleanup_sigkill_wait_ pid_t pid = 0;
 261         int r;
 262
 263         assert(i);
 264
 265         assert(i->raw_job);
 266
 267         if (!i->sha256sums_job)
 268                 return 0;
 269
 270         assert(i->raw_job->state == IMPORT_JOB_DONE);
 271         assert(i->raw_job->sha256);
 272
 273         assert(i->sha256sums_job->state == IMPORT_JOB_DONE);
 274         assert(i->sha256sums_job->payload);
 275         assert(i->sha256sums_job->payload_size > 0);
 276
 277         r = import_url_last_component(i->raw_job->url, &fn);
 278         if (r < 0)
 279                 return log_oom();
 280
 281         if (!filename_is_valid(fn)) {
 282                 log_error("Cannot verify checksum, could not determine valid server-side file name.");
 283                 return -EBADMSG;
 284         }
 285
 286         line = strappenda(i->raw_job->sha256, " *", fn, "\n");
 287
 288         p = memmem(i->sha256sums_job->payload,
 289                    i->sha256sums_job->payload_size,
 290                    line,
 291                    strlen(line));
 292
 293         if (!p || (p != (char*) i->sha256sums_job->payload && p[-1] != '\n')) {
 294                 log_error("Checksum did not check out, payload has been tempered with.");
 295                 return -EBADMSG;
 296         }
 297
 298         log_info("SHA256 checksum of %s is valid.", i->raw_job->url);
 299
 300         if (!i->signature_job)
 301                 return 0;
 302
 303         assert(i->signature_job->state == IMPORT_JOB_DONE);
 304         assert(i->signature_job->payload);
 305         assert(i->signature_job->payload_size > 0);
 306
 307         r = pipe2(gpg_pipe, O_CLOEXEC);
 308         if (r < 0)
 309                 return log_error_errno(errno, "Failed to create pipe: %m");
 310
 311         sig_file = mkostemp(sig_file_path, O_RDWR);
 312         if (sig_file < 0)
 313                 return log_error_errno(errno, "Failed to create temporary file: %m");
 314
 315         r = loop_write(sig_file, i->signature_job->payload, i->signature_job->payload_size, false);
 316         if (r < 0) {
 317                 log_error_errno(r, "Failed to write to temporary file: %m");
 318                 goto finish;
 319         }
 320
 321         pid = fork();
 322         if (pid < 0)
 323                 return log_error_errno(errno, "Failed to fork off gpg: %m");
 324         if (pid == 0) {
 325                 const char *cmd[] = {
 326                         "gpg",
 327                         "--no-options",
 328                         "--no-default-keyring",
 329                         "--no-auto-key-locate",
 330                         "--no-auto-check-trustdb",
 331                         "--batch",
 332                         "--trust-model=always",
 333                         "--keyring=" VENDOR_KEYRING_PATH,
 334                         NULL, /* maybe user keyring */
 335                         NULL, /* --verify */
 336                         NULL, /* signature file */
 337                         NULL, /* dash */
 338                         NULL  /* trailing NULL */
 339                 };
 340                 unsigned k = ELEMENTSOF(cmd) - 5;
 341                 int null_fd;
 342
 343                 /* Child */
 344
 345                 reset_all_signal_handlers();
 346                 reset_signal_mask();
 347                 assert_se(prctl(PR_SET_PDEATHSIG, SIGTERM) == 0);
 348
 349                 gpg_pipe[1] = safe_close(gpg_pipe[1]);
 350
 351                 if (dup2(gpg_pipe[0], STDIN_FILENO) != STDIN_FILENO) {
 352                         log_error_errno(errno, "Failed to dup2() fd: %m");
 353                         _exit(EXIT_FAILURE);
 354                 }
 355
 356                 if (gpg_pipe[0] != STDIN_FILENO)
 357                         gpg_pipe[0] = safe_close(gpg_pipe[0]);
 358
 359                 null_fd = open("/dev/null", O_WRONLY|O_NOCTTY);
 360                 if (null_fd < 0) {
 361                         log_error_errno(errno, "Failed to open /dev/null: %m");
 362                         _exit(EXIT_FAILURE);
 363                 }
 364
 365                 if (dup2(null_fd, STDOUT_FILENO) != STDOUT_FILENO) {
 366                         log_error_errno(errno, "Failed to dup2() fd: %m");
 367                         _exit(EXIT_FAILURE);
 368                 }
 369
 370                 if (null_fd != STDOUT_FILENO)
 371                         null_fd = safe_close(null_fd);
 372
 373                 /* We add the user keyring only to the command line
 374                  * arguments, if it's around since gpg fails
 375                  * otherwise. */
 376                 if (access(USER_KEYRING_PATH, F_OK) >= 0)
 377                         cmd[k++] = "--keyring=" USER_KEYRING_PATH;
 378
 379                 cmd[k++] = "--verify";
 380                 cmd[k++] = sig_file_path;
 381                 cmd[k++] = "-";
 382                 cmd[k++] = NULL;
 383
 384                 execvp("gpg", (char * const *) cmd);
 385                 log_error_errno(errno, "Failed to execute gpg: %m");
 386                 _exit(EXIT_FAILURE);
 387         }
 388
 389         gpg_pipe[0] = safe_close(gpg_pipe[0]);
 390
 391         r = loop_write(gpg_pipe[1], i->sha256sums_job->payload, i->sha256sums_job->payload_size, false);
 392         if (r < 0) {
 393                 log_error_errno(r, "Failed to write to pipe: %m");
 394                 goto finish;
 395         }
 396
 397         gpg_pipe[1] = safe_close(gpg_pipe[1]);
 398
 399         r = wait_for_terminate_and_warn("gpg", pid, true);
 400         pid = 0;
 401         if (r < 0)
 402                 goto finish;
 403         if (r > 0) {
 404                 log_error("Signature verification failed.");
 405                 r = -EBADMSG;
 406         } else {
 407                 log_info("Signature verification succeeded.");
 408                 r = 0;
 409         }
 410
 411 finish:
 412         if (sig_file >= 0)
 413                 unlink(sig_file_path);
 414
 415         return r;
 416 }
 417
 418 static void raw_import_job_on_finished(ImportJob *j) {
 419         RawImport *i;
 420         int r;
 421
 422         assert(j);
 423         assert(j->userdata);
 424
 425         i = j->userdata;
 426         if (j->error != 0) {
 427                 if (j == i->sha256sums_job)
 428                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 429                 else if (j == i->signature_job)
 430                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 431                 else
 432                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 433
 434                 r = j->error;
 435                 goto finish;
 436         }
 437
 438         /* This is invoked if either the download completed
 439          * successfully, or the download was skipped because we
 440          * already have the etag. In this case ->etag_exists is
 441          * true.
 442          *
 443          * We only do something when we got all three files */
 444
 445         if (!IMPORT_JOB_STATE_IS_COMPLETE(i->raw_job))
 446                 return;
 447         if (i->sha256sums_job && !IMPORT_JOB_STATE_IS_COMPLETE(i->sha256sums_job))
 448                 return;
 449         if (i->signature_job && !IMPORT_JOB_STATE_IS_COMPLETE(i->signature_job))
 450                 return;
 451
 452         if (!i->raw_job->etag_exists) {
 453                 assert(i->raw_job->disk_fd >= 0);
 454
 455                 r = raw_import_verify_sha256sum(i);
 456                 if (r < 0)
 457                         goto finish;
 458
 459                 r = raw_import_maybe_convert_qcow2(i);
 460                 if (r < 0)
 461                         goto finish;
 462
 463                 r = import_make_read_only_fd(i->raw_job->disk_fd);
 464                 if (r < 0)
 465                         goto finish;
 466
 467                 r = rename(i->temp_path, i->final_path);
 468                 if (r < 0) {
 469                         r = log_error_errno(errno, "Failed to move RAW file into place: %m");
 470                         goto finish;
 471                 }
 472
 473                 free(i->temp_path);
 474                 i->temp_path = NULL;
 475         }
 476
 477         r = raw_import_make_local_copy(i);
 478         if (r < 0)
 479                 goto finish;
 480
 481         r = 0;
 482
 483 finish:
 484         if (i->on_finished)
 485                 i->on_finished(i, r, i->userdata);
 486         else
 487                 sd_event_exit(i->event, r);
 488 }
 489
 490 static int raw_import_job_on_open_disk(ImportJob *j) {
 491         RawImport *i;
 492         int r;
 493
 494         assert(j);
 495         assert(j->userdata);
 496
 497         i = j->userdata;
 498
 499         r = import_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 500         if (r < 0)
 501                 return log_oom();
 502
 503         r = tempfn_random(i->final_path, &i->temp_path);
 504         if (r <0)
 505                 return log_oom();
 506
 507         mkdir_parents_label(i->temp_path, 0700);
 508
 509         j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
 510         if (j->disk_fd < 0)
 511                 return log_error_errno(errno, "Failed to create %s: %m", i->temp_path);
 512
 513         r = chattr_fd(j->disk_fd, true, FS_NOCOW_FL);
 514         if (r < 0)
 515                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", i->temp_path);
 516
 517         return 0;
 518 }
 519
 520 int raw_import_pull(RawImport *i, const char *url, const char *local, bool force_local, ImportVerify verify) {
 521         int r;
 522
 523         assert(i);
 524         assert(verify < _IMPORT_VERIFY_MAX);
 525         assert(verify >= 0);
 526
 527         if (i->raw_job)
 528                 return -EBUSY;
 529
 530         if (!http_url_is_valid(url))
 531                 return -EINVAL;
 532
 533         if (local && !machine_name_is_valid(local))
 534                 return -EINVAL;
 535
 536         r = free_and_strdup(&i->local, local);
 537         if (r < 0)
 538                 return r;
 539         i->force_local = force_local;
 540         i->verify = verify;
 541
 542         /* Queue job for the image itself */
 543         r = import_job_new(&i->raw_job, url, i->glue, i);
 544         if (r < 0)
 545                 return r;
 546
 547         i->raw_job->on_finished = raw_import_job_on_finished;
 548         i->raw_job->on_open_disk = raw_import_job_on_open_disk;
 549         i->raw_job->calc_hash = true;
 550
 551         r = import_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
 552         if (r < 0)
 553                 return r;
 554
 555         if (verify != IMPORT_VERIFY_NO) {
 556                 _cleanup_free_ char *sha256sums_url = NULL;
 557
 558                 /* Queue job for the SHA256SUMS file for the image */
 559                 r = import_url_change_last_component(url, "SHA256SUMS", &sha256sums_url);
 560                 if (r < 0)
 561                         return r;
 562
 563                 r = import_job_new(&i->sha256sums_job, sha256sums_url, i->glue, i);
 564                 if (r < 0)
 565                         return r;
 566
 567                 i->sha256sums_job->on_finished = raw_import_job_on_finished;
 568                 i->sha256sums_job->uncompressed_max = i->sha256sums_job->compressed_max = 1ULL * 1024ULL * 1024ULL;
 569         }
 570
 571         if (verify == IMPORT_VERIFY_SIGNATURE) {
 572                 _cleanup_free_ char *sha256sums_sig_url = NULL;
 573
 574                 /* Queue job for the SHA256SUMS.gpg file for the image. */
 575                 r = import_url_change_last_component(url, "SHA256SUMS.gpg", &sha256sums_sig_url);
 576                 if (r < 0)
 577                         return r;
 578
 579                 r = import_job_new(&i->signature_job, sha256sums_sig_url, i->glue, i);
 580                 if (r < 0)
 581                         return r;
 582
 583                 i->signature_job->on_finished = raw_import_job_on_finished;
 584                 i->signature_job->uncompressed_max = i->signature_job->compressed_max = 1ULL * 1024ULL * 1024ULL;
 585         }
 586
 587         r = import_job_begin(i->raw_job);
 588         if (r < 0)
 589                 return r;
 590
 591         if (i->sha256sums_job) {
 592                 r = import_job_begin(i->sha256sums_job);
 593                 if (r < 0)
 594                         return r;
 595         }
 596
 597         if (i->signature_job) {
 598                 r = import_job_begin(i->signature_job);
 599                 if (r < 0)
 600                         return r;
 601         }
 602
 603         return 0;
 604 }