src/import/pull-raw.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <sys/xattr.h>
  23 #include <linux/fs.h>
  24 #include <curl/curl.h>
  25
  26 #include "sd-daemon.h"
  27 #include "utf8.h"
  28 #include "strv.h"
  29 #include "copy.h"
  30 #include "btrfs-util.h"
  31 #include "util.h"
  32 #include "macro.h"
  33 #include "mkdir.h"
  34 #include "path-util.h"
  35 #include "import-util.h"
  36 #include "import-common.h"
  37 #include "curl-util.h"
  38 #include "qcow2-util.h"
  39 #include "pull-job.h"
  40 #include "pull-common.h"
  41 #include "pull-raw.h"
  42
  43 typedef enum RawProgress {
  44         RAW_DOWNLOADING,
  45         RAW_VERIFYING,
  46         RAW_UNPACKING,
  47         RAW_FINALIZING,
  48         RAW_COPYING,
  49 } RawProgress;
  50
  51 struct RawPull {
  52         sd_event *event;
  53         CurlGlue *glue;
  54
  55         char *image_root;
  56
  57         PullJob *raw_job;
  58         PullJob *checksum_job;
  59         PullJob *signature_job;
  60
  61         RawPullFinished on_finished;
  62         void *userdata;
  63
  64         char *local;
  65         bool force_local;
  66         bool grow_machine_directory;
  67
  68         char *temp_path;
  69         char *final_path;
  70
  71         ImportVerify verify;
  72 };
  73
  74 RawPull* raw_pull_unref(RawPull *i) {
  75         if (!i)
  76                 return NULL;
  77
  78         pull_job_unref(i->raw_job);
  79         pull_job_unref(i->checksum_job);
  80         pull_job_unref(i->signature_job);
  81
  82         curl_glue_unref(i->glue);
  83         sd_event_unref(i->event);
  84
  85         if (i->temp_path) {
  86                 (void) unlink(i->temp_path);
  87                 free(i->temp_path);
  88         }
  89
  90         free(i->final_path);
  91         free(i->image_root);
  92         free(i->local);
  93         free(i);
  94
  95         return NULL;
  96 }
  97
  98 int raw_pull_new(
  99                 RawPull **ret,
 100                 sd_event *event,
 101                 const char *image_root,
 102                 RawPullFinished on_finished,
 103                 void *userdata) {
 104
 105         _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
 106         int r;
 107
 108         assert(ret);
 109
 110         i = new0(RawPull, 1);
 111         if (!i)
 112                 return -ENOMEM;
 113
 114         i->on_finished = on_finished;
 115         i->userdata = userdata;
 116
 117         i->image_root = strdup(image_root ?: "/var/lib/machines");
 118         if (!i->image_root)
 119                 return -ENOMEM;
 120
 121         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 122
 123         if (event)
 124                 i->event = sd_event_ref(event);
 125         else {
 126                 r = sd_event_default(&i->event);
 127                 if (r < 0)
 128                         return r;
 129         }
 130
 131         r = curl_glue_new(&i->glue, i->event);
 132         if (r < 0)
 133                 return r;
 134
 135         i->glue->on_finished = pull_job_curl_on_finished;
 136         i->glue->userdata = i;
 137
 138         *ret = i;
 139         i = NULL;
 140
 141         return 0;
 142 }
 143
 144 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
 145         unsigned percent;
 146
 147         assert(i);
 148
 149         switch (p) {
 150
 151         case RAW_DOWNLOADING: {
 152                 unsigned remain = 80;
 153
 154                 percent = 0;
 155
 156                 if (i->checksum_job) {
 157                         percent += i->checksum_job->progress_percent * 5 / 100;
 158                         remain -= 5;
 159                 }
 160
 161                 if (i->signature_job) {
 162                         percent += i->signature_job->progress_percent * 5 / 100;
 163                         remain -= 5;
 164                 }
 165
 166                 if (i->raw_job)
 167                         percent += i->raw_job->progress_percent * remain / 100;
 168                 break;
 169         }
 170
 171         case RAW_VERIFYING:
 172                 percent = 80;
 173                 break;
 174
 175         case RAW_UNPACKING:
 176                 percent = 85;
 177                 break;
 178
 179         case RAW_FINALIZING:
 180                 percent = 90;
 181                 break;
 182
 183         case RAW_COPYING:
 184                 percent = 95;
 185                 break;
 186
 187         default:
 188                 assert_not_reached("Unknown progress state");
 189         }
 190
 191         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 192         log_debug("Combined progress %u%%", percent);
 193 }
 194
 195 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
 196         _cleanup_close_ int converted_fd = -1;
 197         _cleanup_free_ char *t = NULL;
 198         int r;
 199
 200         assert(i);
 201         assert(i->raw_job);
 202
 203         r = qcow2_detect(i->raw_job->disk_fd);
 204         if (r < 0)
 205                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
 206         if (r == 0)
 207                 return 0;
 208
 209         /* This is a QCOW2 image, let's convert it */
 210         r = tempfn_random(i->final_path, &t);
 211         if (r < 0)
 212                 return log_oom();
 213
 214         converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 215         if (converted_fd < 0)
 216                 return log_error_errno(errno, "Failed to create %s: %m", t);
 217
 218         r = chattr_fd(converted_fd, true, FS_NOCOW_FL);
 219         if (r < 0)
 220                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", t);
 221
 222         log_info("Unpacking QCOW2 file.");
 223
 224         r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
 225         if (r < 0) {
 226                 unlink(t);
 227                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
 228         }
 229
 230         (void) unlink(i->temp_path);
 231         free(i->temp_path);
 232         i->temp_path = t;
 233         t = NULL;
 234
 235         safe_close(i->raw_job->disk_fd);
 236         i->raw_job->disk_fd = converted_fd;
 237         converted_fd = -1;
 238
 239         return 1;
 240 }
 241
 242 static int raw_pull_make_local_copy(RawPull *i) {
 243         _cleanup_free_ char *tp = NULL;
 244         _cleanup_close_ int dfd = -1;
 245         const char *p;
 246         int r;
 247
 248         assert(i);
 249         assert(i->raw_job);
 250
 251         if (!i->local)
 252                 return 0;
 253
 254         if (i->raw_job->etag_exists) {
 255                 /* We have downloaded this one previously, reopen it */
 256
 257                 assert(i->raw_job->disk_fd < 0);
 258
 259                 if (!i->final_path) {
 260                         r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 261                         if (r < 0)
 262                                 return log_oom();
 263                 }
 264
 265                 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 266                 if (i->raw_job->disk_fd < 0)
 267                         return log_error_errno(errno, "Failed to open vendor image: %m");
 268         } else {
 269                 /* We freshly downloaded the image, use it */
 270
 271                 assert(i->raw_job->disk_fd >= 0);
 272
 273                 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
 274                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 275         }
 276
 277         p = strjoina(i->image_root, "/", i->local, ".raw");
 278
 279         if (i->force_local) {
 280                 (void) btrfs_subvol_remove(p);
 281                 (void) rm_rf_dangerous(p, false, true, false);
 282         }
 283
 284         r = tempfn_random(p, &tp);
 285         if (r < 0)
 286                 return log_oom();
 287
 288         dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 289         if (dfd < 0)
 290                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 291
 292         /* Turn off COW writing. This should greatly improve
 293          * performance on COW file systems like btrfs, since it
 294          * reduces fragmentation caused by not allowing in-place
 295          * writes. */
 296         r = chattr_fd(dfd, true, FS_NOCOW_FL);
 297         if (r < 0)
 298                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
 299
 300         r = copy_bytes(i->raw_job->disk_fd, dfd, (off_t) -1, true);
 301         if (r < 0) {
 302                 unlink(tp);
 303                 return log_error_errno(r, "Failed to make writable copy of image: %m");
 304         }
 305
 306         (void) copy_times(i->raw_job->disk_fd, dfd);
 307         (void) copy_xattr(i->raw_job->disk_fd, dfd);
 308
 309         dfd = safe_close(dfd);
 310
 311         r = rename(tp, p);
 312         if (r < 0)  {
 313                 unlink(tp);
 314                 return log_error_errno(errno, "Failed to move writable image into place: %m");
 315         }
 316
 317         log_info("Created new local image '%s'.", i->local);
 318         return 0;
 319 }
 320
 321 static bool raw_pull_is_done(RawPull *i) {
 322         assert(i);
 323         assert(i->raw_job);
 324
 325         if (i->raw_job->state != PULL_JOB_DONE)
 326                 return false;
 327         if (i->checksum_job && i->checksum_job->state != PULL_JOB_DONE)
 328                 return false;
 329         if (i->signature_job && i->signature_job->state != PULL_JOB_DONE)
 330                 return false;
 331
 332         return true;
 333 }
 334
 335 static void raw_pull_job_on_finished(PullJob *j) {
 336         RawPull *i;
 337         int r;
 338
 339         assert(j);
 340         assert(j->userdata);
 341
 342         i = j->userdata;
 343         if (j->error != 0) {
 344                 if (j == i->checksum_job)
 345                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 346                 else if (j == i->signature_job)
 347                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 348                 else
 349                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 350
 351                 r = j->error;
 352                 goto finish;
 353         }
 354
 355         /* This is invoked if either the download completed
 356          * successfully, or the download was skipped because we
 357          * already have the etag. In this case ->etag_exists is
 358          * true.
 359          *
 360          * We only do something when we got all three files */
 361
 362         if (!raw_pull_is_done(i))
 363                 return;
 364
 365         if (!i->raw_job->etag_exists) {
 366                 /* This is a new download, verify it, and move it into place */
 367                 assert(i->raw_job->disk_fd >= 0);
 368
 369                 raw_pull_report_progress(i, RAW_VERIFYING);
 370
 371                 r = pull_verify(i->raw_job, i->checksum_job, i->signature_job);
 372                 if (r < 0)
 373                         goto finish;
 374
 375                 raw_pull_report_progress(i, RAW_UNPACKING);
 376
 377                 r = raw_pull_maybe_convert_qcow2(i);
 378                 if (r < 0)
 379                         goto finish;
 380
 381                 raw_pull_report_progress(i, RAW_FINALIZING);
 382
 383                 r = import_make_read_only_fd(i->raw_job->disk_fd);
 384                 if (r < 0)
 385                         goto finish;
 386
 387                 r = renameat2(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path, RENAME_NOREPLACE);
 388                 if (r < 0) {
 389                         r = log_error_errno(errno, "Failed to move RAW file into place: %m");
 390                         goto finish;
 391                 }
 392
 393                 free(i->temp_path);
 394                 i->temp_path = NULL;
 395         }
 396
 397         raw_pull_report_progress(i, RAW_COPYING);
 398
 399         r = raw_pull_make_local_copy(i);
 400         if (r < 0)
 401                 goto finish;
 402
 403         r = 0;
 404
 405 finish:
 406         if (i->on_finished)
 407                 i->on_finished(i, r, i->userdata);
 408         else
 409                 sd_event_exit(i->event, r);
 410 }
 411
 412 static int raw_pull_job_on_open_disk(PullJob *j) {
 413         RawPull *i;
 414         int r;
 415
 416         assert(j);
 417         assert(j->userdata);
 418
 419         i = j->userdata;
 420         assert(i->raw_job == j);
 421         assert(!i->final_path);
 422         assert(!i->temp_path);
 423
 424         r = pull_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 425         if (r < 0)
 426                 return log_oom();
 427
 428         r = tempfn_random(i->final_path, &i->temp_path);
 429         if (r < 0)
 430                 return log_oom();
 431
 432         (void) mkdir_parents_label(i->temp_path, 0700);
 433
 434         j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 435         if (j->disk_fd < 0)
 436                 return log_error_errno(errno, "Failed to create %s: %m", i->temp_path);
 437
 438         r = chattr_fd(j->disk_fd, true, FS_NOCOW_FL);
 439         if (r < 0)
 440                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", i->temp_path);
 441
 442         return 0;
 443 }
 444
 445 static void raw_pull_job_on_progress(PullJob *j) {
 446         RawPull *i;
 447
 448         assert(j);
 449         assert(j->userdata);
 450
 451         i = j->userdata;
 452
 453         raw_pull_report_progress(i, RAW_DOWNLOADING);
 454 }
 455
 456 int raw_pull_start(RawPull *i, const char *url, const char *local, bool force_local, ImportVerify verify) {
 457         int r;
 458
 459         assert(i);
 460         assert(verify < _IMPORT_VERIFY_MAX);
 461         assert(verify >= 0);
 462
 463         if (!http_url_is_valid(url))
 464                 return -EINVAL;
 465
 466         if (local && !machine_name_is_valid(local))
 467                 return -EINVAL;
 468
 469         if (i->raw_job)
 470                 return -EBUSY;
 471
 472         r = free_and_strdup(&i->local, local);
 473         if (r < 0)
 474                 return r;
 475         i->force_local = force_local;
 476         i->verify = verify;
 477
 478         /* Queue job for the image itself */
 479         r = pull_job_new(&i->raw_job, url, i->glue, i);
 480         if (r < 0)
 481                 return r;
 482
 483         i->raw_job->on_finished = raw_pull_job_on_finished;
 484         i->raw_job->on_open_disk = raw_pull_job_on_open_disk;
 485         i->raw_job->on_progress = raw_pull_job_on_progress;
 486         i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 487         i->raw_job->grow_machine_directory = i->grow_machine_directory;
 488
 489         r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
 490         if (r < 0)
 491                 return r;
 492
 493         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
 494         if (r < 0)
 495                 return r;
 496
 497         r = pull_job_begin(i->raw_job);
 498         if (r < 0)
 499                 return r;
 500
 501         if (i->checksum_job) {
 502                 i->checksum_job->on_progress = raw_pull_job_on_progress;
 503
 504                 r = pull_job_begin(i->checksum_job);
 505                 if (r < 0)
 506                         return r;
 507         }
 508
 509         if (i->signature_job) {
 510                 i->signature_job->on_progress = raw_pull_job_on_progress;
 511
 512                 r = pull_job_begin(i->signature_job);
 513                 if (r < 0)
 514                         return r;
 515         }
 516
 517         return 0;
 518 }