src/import/import-raw.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <sys/xattr.h>
  23 #include <linux/fs.h>
  24 #include <curl/curl.h>
  25
  26 #include "hashmap.h"
  27 #include "utf8.h"
  28 #include "curl-util.h"
  29 #include "import-raw.h"
  30 #include "strv.h"
  31 #include "copy.h"
  32
  33 typedef struct RawImportFile RawImportFile;
  34
  35 struct RawImportFile {
  36         RawImport *import;
  37
  38         char *url;
  39         char *local;
  40
  41         CURL *curl;
  42         struct curl_slist *request_header;
  43
  44         char *temp_path;
  45         char *final_path;
  46         char *etag;
  47         char **old_etags;
  48
  49         uint64_t content_length;
  50         uint64_t written;
  51
  52         usec_t mtime;
  53
  54         bool force_local;
  55         bool done;
  56
  57         int disk_fd;
  58 };
  59
  60 struct RawImport {
  61         sd_event *event;
  62         CurlGlue *glue;
  63
  64         char *image_root;
  65         Hashmap *files;
  66
  67         raw_import_on_finished on_finished;
  68         void *userdata;
  69
  70         bool finished;
  71 };
  72
  73 #define FILENAME_ESCAPE "/.#\"\'"
  74
  75 static RawImportFile *raw_import_file_unref(RawImportFile *f) {
  76         if (!f)
  77                 return NULL;
  78
  79         if (f->import)
  80                 curl_glue_remove_and_free(f->import->glue, f->curl);
  81         curl_slist_free_all(f->request_header);
  82
  83         safe_close(f->disk_fd);
  84
  85         free(f->final_path);
  86
  87         if (f->temp_path) {
  88                 unlink(f->temp_path);
  89                 free(f->temp_path);
  90         }
  91
  92         free(f->url);
  93         free(f->local);
  94         free(f->etag);
  95         strv_free(f->old_etags);
  96         free(f);
  97
  98         return NULL;
  99 }
 100
 101 DEFINE_TRIVIAL_CLEANUP_FUNC(RawImportFile*, raw_import_file_unref);
 102
 103 static void raw_import_finish(RawImport *import, int error) {
 104         assert(import);
 105
 106         if (import->finished)
 107                 return;
 108
 109         import->finished = true;
 110
 111         if (import->on_finished)
 112                 import->on_finished(import, error, import->userdata);
 113         else
 114                 sd_event_exit(import->event, error);
 115 }
 116
 117 static int raw_import_file_make_final_path(RawImportFile *f) {
 118         _cleanup_free_ char *escaped_url = NULL, *escaped_etag = NULL;
 119
 120         assert(f);
 121
 122         if (f->final_path)
 123                 return 0;
 124
 125         escaped_url = xescape(f->url, FILENAME_ESCAPE);
 126         if (!escaped_url)
 127                 return -ENOMEM;
 128
 129         if (f->etag) {
 130                 escaped_etag = xescape(f->etag, FILENAME_ESCAPE);
 131                 if (!escaped_etag)
 132                         return -ENOMEM;
 133
 134                 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".", escaped_etag, ".raw", NULL);
 135         } else
 136                 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".raw", NULL);
 137         if (!f->final_path)
 138                 return -ENOMEM;
 139
 140         return 0;
 141 }
 142
 143 static void raw_import_file_success(RawImportFile *f) {
 144         int r;
 145
 146         assert(f);
 147
 148         f->done = true;
 149
 150         if (f->local) {
 151                 _cleanup_free_ char *tp = NULL;
 152                 _cleanup_close_ int dfd = -1;
 153                 const char *p;
 154
 155                 if (f->disk_fd >= 0) {
 156                         if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1) {
 157                                 r = log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 158                                 goto finish;
 159                         }
 160                 } else {
 161                         r = raw_import_file_make_final_path(f);
 162                         if (r < 0) {
 163                                 log_oom();
 164                                 goto finish;
 165                         }
 166
 167                         f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 168                         if (f->disk_fd < 0) {
 169                                 r = log_error_errno(errno, "Failed to open vendor image: %m");
 170                                 goto finish;
 171                         }
 172                 }
 173
 174                 p = strappenda(f->import->image_root, "/", f->local, ".raw");
 175                 if (f->force_local)
 176                         (void) rm_rf_dangerous(p, false, true, false);
 177
 178                 r = tempfn_random(p, &tp);
 179                 if (r < 0) {
 180                         log_oom();
 181                         goto finish;
 182                 }
 183
 184                 dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 185                 if (dfd < 0) {
 186                         r = log_error_errno(errno, "Failed to create writable copy of image: %m");
 187                         goto finish;
 188                 }
 189
 190                 /* Turn off COW writing. This should greatly improve
 191                  * performance on COW file systems like btrfs, since it
 192                  * reduces fragmentation caused by not allowing in-place
 193                  * writes. */
 194                 r = chattr_fd(dfd, true, FS_NOCOW_FL);
 195                 if (r < 0)
 196                         log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
 197
 198                 r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
 199                 if (r < 0) {
 200                         log_error_errno(r, "Failed to make writable copy of image: %m");
 201                         unlink(tp);
 202                         goto finish;
 203                 }
 204
 205                 (void) copy_times(f->disk_fd, dfd);
 206                 (void) copy_xattr(f->disk_fd, dfd);
 207
 208                 dfd = safe_close(dfd);
 209
 210                 r = rename(tp, p);
 211                 if (r < 0) {
 212                         r = log_error_errno(errno, "Failed to move writable image into place: %m");
 213                         unlink(tp);
 214                         goto finish;
 215                 }
 216
 217                 log_info("Created new local image %s.", p);
 218         }
 219
 220         f->disk_fd = safe_close(f->disk_fd);
 221         r = 0;
 222
 223 finish:
 224         raw_import_finish(f->import, r);
 225 }
 226
 227 static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
 228         RawImportFile *f = NULL;
 229         struct stat st;
 230         CURLcode code;
 231         long status;
 232         int r;
 233
 234         if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, &f) != CURLE_OK)
 235                 return;
 236
 237         if (!f || f->done)
 238                 return;
 239
 240         f->done = true;
 241
 242         if (result != CURLE_OK) {
 243                 log_error("Transfer failed: %s", curl_easy_strerror(result));
 244                 r = -EIO;
 245                 goto fail;
 246         }
 247
 248         code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
 249         if (code != CURLE_OK) {
 250                 log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
 251                 r = -EIO;
 252                 goto fail;
 253         } else if (status == 304) {
 254                 log_info("Image already downloaded. Skipping download.");
 255                 raw_import_file_success(f);
 256                 return;
 257         } else if (status >= 300) {
 258                 log_error("HTTP request to %s failed with code %li.", f->url, status);
 259                 r = -EIO;
 260                 goto fail;
 261         } else if (status < 200) {
 262                 log_error("HTTP request to %s finished with unexpected code %li.", f->url, status);
 263                 r = -EIO;
 264                 goto fail;
 265         }
 266
 267         if (f->disk_fd < 0) {
 268                 log_error("No data received.");
 269                 r = -EIO;
 270                 goto fail;
 271         }
 272
 273         if (f->content_length != (uint64_t) -1 &&
 274             f->content_length != f->written) {
 275                 log_error("Download truncated.");
 276                 r = -EIO;
 277                 goto fail;
 278         }
 279
 280         if (f->etag)
 281                 (void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
 282         if (f->url)
 283                 (void) fsetxattr(f->disk_fd, "user.source_url", f->url, strlen(f->url), 0);
 284
 285         if (f->mtime != 0) {
 286                 struct timespec ut[2];
 287
 288                 timespec_store(&ut[0], f->mtime);
 289                 ut[1] = ut[0];
 290                 (void) futimens(f->disk_fd, ut);
 291
 292                 fd_setcrtime(f->disk_fd, f->mtime);
 293         }
 294
 295         if (fstat(f->disk_fd, &st) < 0) {
 296                 r = log_error_errno(errno, "Failed to stat file: %m");
 297                 goto fail;
 298         }
 299
 300         /* Mark read-only */
 301         (void) fchmod(f->disk_fd, st.st_mode & 07444);
 302
 303         assert(f->temp_path);
 304         assert(f->final_path);
 305
 306         r = rename(f->temp_path, f->final_path);
 307         if (r < 0) {
 308                 r = log_error_errno(errno, "Failed to move RAW file into place: %m");
 309                 goto fail;
 310         }
 311
 312         free(f->temp_path);
 313         f->temp_path = NULL;
 314
 315         log_info("Completed writing vendor image %s.", f->final_path);
 316
 317         raw_import_file_success(f);
 318         return;
 319
 320 fail:
 321         raw_import_finish(f->import, r);
 322 }
 323
 324 static int raw_import_file_open_disk_for_write(RawImportFile *f) {
 325         int r;
 326
 327         assert(f);
 328
 329         if (f->disk_fd >= 0)
 330                 return 0;
 331
 332         r = raw_import_file_make_final_path(f);
 333         if (r < 0)
 334                 return log_oom();
 335
 336         if (!f->temp_path) {
 337                 r = tempfn_random(f->final_path, &f->temp_path);
 338                 if (r < 0)
 339                         return log_oom();
 340         }
 341
 342         f->disk_fd = open(f->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
 343         if (f->disk_fd < 0)
 344                 return log_error_errno(errno, "Failed to create %s: %m", f->temp_path);
 345
 346         return 0;
 347 }
 348
 349 static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
 350         RawImportFile *f = userdata;
 351         size_t sz = size * nmemb;
 352         ssize_t n;
 353         int r;
 354
 355         assert(contents);
 356         assert(f);
 357
 358         if (f->done) {
 359                 r = -ESTALE;
 360                 goto fail;
 361         }
 362
 363         r = raw_import_file_open_disk_for_write(f);
 364         if (r < 0)
 365                 goto fail;
 366
 367         if (f->written + sz < f->written) {
 368                 log_error("File too large, overflow");
 369                 r = -EOVERFLOW;
 370                 goto fail;
 371         }
 372
 373         if (f->content_length != (uint64_t) -1 &&
 374             f->written + sz > f->content_length) {
 375                 log_error("Content length incorrect.");
 376                 r = -EFBIG;
 377                 goto fail;
 378         }
 379
 380         n = write(f->disk_fd, contents, sz);
 381         if (n < 0) {
 382                 log_error_errno(errno, "Failed to write file: %m");
 383                 goto fail;
 384         }
 385
 386         if ((size_t) n < sz) {
 387                 log_error("Short write");
 388                 r = -EIO;
 389                 goto fail;
 390         }
 391
 392         f->written += sz;
 393
 394         return sz;
 395
 396 fail:
 397         raw_import_finish(f->import, r);
 398         return 0;
 399 }
 400
 401 static size_t raw_import_file_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
 402         RawImportFile *f = userdata;
 403         size_t sz = size * nmemb;
 404         _cleanup_free_ char *length = NULL, *last_modified = NULL;
 405         char *etag;
 406         int r;
 407
 408         assert(contents);
 409         assert(f);
 410
 411         if (f->done) {
 412                 r = -ESTALE;
 413                 goto fail;
 414         }
 415
 416         r = curl_header_strdup(contents, sz, "ETag:", &etag);
 417         if (r < 0) {
 418                 log_oom();
 419                 goto fail;
 420         }
 421         if (r > 0) {
 422                 free(f->etag);
 423                 f->etag = etag;
 424
 425                 if (strv_contains(f->old_etags, f->etag)) {
 426                         log_info("Image already downloaded. Skipping download.");
 427                         raw_import_file_success(f);
 428                         return sz;
 429                 }
 430
 431                 return sz;
 432         }
 433
 434         r = curl_header_strdup(contents, sz, "Content-Length:", &length);
 435         if (r < 0) {
 436                 log_oom();
 437                 goto fail;
 438         }
 439         if (r > 0) {
 440                 (void) safe_atou64(length, &f->content_length);
 441                 return sz;
 442         }
 443
 444         r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
 445         if (r < 0) {
 446                 log_oom();
 447                 goto fail;
 448         }
 449         if (r > 0) {
 450                 (void) curl_parse_http_time(last_modified, &f->mtime);
 451                 return sz;
 452         }
 453
 454         return sz;
 455
 456 fail:
 457         raw_import_finish(f->import, r);
 458         return 0;
 459 }
 460
 461 static bool etag_is_valid(const char *etag) {
 462
 463         if (!endswith(etag, "\""))
 464                 return false;
 465
 466         if (!startswith(etag, "\"") && !startswith(etag, "W/\""))
 467                 return false;
 468
 469         return true;
 470 }
 471
 472 static int raw_import_file_find_old_etags(RawImportFile *f) {
 473         _cleanup_free_ char *escaped_url = NULL;
 474         _cleanup_closedir_ DIR *d = NULL;
 475         struct dirent *de;
 476         int r;
 477
 478         escaped_url = xescape(f->url, FILENAME_ESCAPE);
 479         if (!escaped_url)
 480                 return -ENOMEM;
 481
 482         d = opendir(f->import->image_root);
 483         if (!d) {
 484                 if (errno == ENOENT)
 485                         return 0;
 486
 487                 return -errno;
 488         }
 489
 490         FOREACH_DIRENT_ALL(de, d, return -errno) {
 491                 const char *a, *b;
 492                 char *u;
 493
 494                 if (de->d_type != DT_UNKNOWN &&
 495                     de->d_type != DT_REG)
 496                         continue;
 497
 498                 a = startswith(de->d_name, ".raw-");
 499                 if (!a)
 500                         continue;
 501
 502                 a = startswith(a, escaped_url);
 503                 if (!a)
 504                         continue;
 505
 506                 a = startswith(a, ".");
 507                 if (!a)
 508                         continue;
 509
 510                 b = endswith(de->d_name, ".raw");
 511                 if (!b)
 512                         continue;
 513
 514                 if (a >= b)
 515                         continue;
 516
 517                 u = cunescape_length(a, b - a);
 518                 if (!u)
 519                         return -ENOMEM;
 520
 521                 if (!etag_is_valid(u)) {
 522                         free(u);
 523                         continue;
 524                 }
 525
 526                 r = strv_consume(&f->old_etags, u);
 527                 if (r < 0)
 528                         return r;
 529         }
 530
 531         return 0;
 532 }
 533
 534 static int raw_import_file_begin(RawImportFile *f) {
 535         int r;
 536
 537         assert(f);
 538         assert(!f->curl);
 539
 540         log_info("Getting %s.", f->url);
 541
 542         r = raw_import_file_find_old_etags(f);
 543         if (r < 0)
 544                 return r;
 545
 546         r = curl_glue_make(&f->curl, f->url, f);
 547         if (r < 0)
 548                 return r;
 549
 550         if (!strv_isempty(f->old_etags)) {
 551                 _cleanup_free_ char *cc = NULL, *hdr = NULL;
 552
 553                 cc = strv_join(f->old_etags, ", ");
 554                 if (!cc)
 555                         return -ENOMEM;
 556
 557                 hdr = strappend("If-None-Match: ", cc);
 558                 if (!hdr)
 559                         return -ENOMEM;
 560
 561                 f->request_header = curl_slist_new(hdr, NULL);
 562                 if (!f->request_header)
 563                         return -ENOMEM;
 564
 565                 if (curl_easy_setopt(f->curl, CURLOPT_HTTPHEADER, f->request_header) != CURLE_OK)
 566                         return -EIO;
 567         }
 568
 569         if (curl_easy_setopt(f->curl, CURLOPT_WRITEFUNCTION, raw_import_file_write_callback) != CURLE_OK)
 570                 return -EIO;
 571
 572         if (curl_easy_setopt(f->curl, CURLOPT_WRITEDATA, f) != CURLE_OK)
 573                 return -EIO;
 574
 575         if (curl_easy_setopt(f->curl, CURLOPT_HEADERFUNCTION, raw_import_file_header_callback) != CURLE_OK)
 576                 return -EIO;
 577
 578         if (curl_easy_setopt(f->curl, CURLOPT_HEADERDATA, f) != CURLE_OK)
 579                 return -EIO;
 580
 581         r = curl_glue_add(f->import->glue, f->curl);
 582         if (r < 0)
 583                 return r;
 584
 585         return 0;
 586 }
 587
 588 int raw_import_new(RawImport **import, sd_event *event, const char *image_root, raw_import_on_finished on_finished, void *userdata) {
 589         _cleanup_(raw_import_unrefp) RawImport *i = NULL;
 590         int r;
 591
 592         assert(import);
 593         assert(image_root);
 594
 595         i = new0(RawImport, 1);
 596         if (!i)
 597                 return -ENOMEM;
 598
 599         i->on_finished = on_finished;
 600         i->userdata = userdata;
 601
 602         i->image_root = strdup(image_root);
 603         if (!i->image_root)
 604                 return -ENOMEM;
 605
 606         if (event)
 607                 i->event = sd_event_ref(event);
 608         else {
 609                 r = sd_event_default(&i->event);
 610                 if (r < 0)
 611                         return r;
 612         }
 613
 614         r = curl_glue_new(&i->glue, i->event);
 615         if (r < 0)
 616                 return r;
 617
 618         i->glue->on_finished = raw_import_curl_on_finished;
 619         i->glue->userdata = i;
 620
 621         *import = i;
 622         i = NULL;
 623
 624         return 0;
 625 }
 626
 627 RawImport* raw_import_unref(RawImport *import) {
 628         RawImportFile *f;
 629
 630         if (!import)
 631                 return NULL;
 632
 633         while ((f = hashmap_steal_first(import->files)))
 634                 raw_import_file_unref(f);
 635         hashmap_free(import->files);
 636
 637         curl_glue_unref(import->glue);
 638         sd_event_unref(import->event);
 639
 640         free(import->image_root);
 641         free(import);
 642
 643         return NULL;
 644 }
 645
 646 int raw_import_cancel(RawImport *import, const char *url) {
 647         RawImportFile *f;
 648
 649         assert(import);
 650         assert(url);
 651
 652         f = hashmap_remove(import->files, url);
 653         if (!f)
 654                 return 0;
 655
 656         raw_import_file_unref(f);
 657         return 1;
 658 }
 659
 660 int raw_import_pull(RawImport *import, const char *url, const char *local, bool force_local) {
 661         _cleanup_(raw_import_file_unrefp) RawImportFile *f = NULL;
 662         int r;
 663
 664         assert(import);
 665         assert(raw_url_is_valid(url));
 666         assert(!local || machine_name_is_valid(local));
 667
 668         if (hashmap_get(import->files, url))
 669                 return -EEXIST;
 670
 671         r = hashmap_ensure_allocated(&import->files, &string_hash_ops);
 672         if (r < 0)
 673                 return r;
 674
 675         f = new0(RawImportFile, 1);
 676         if (!f)
 677                 return -ENOMEM;
 678
 679         f->import = import;
 680         f->disk_fd = -1;
 681         f->content_length = (uint64_t) -1;
 682
 683         f->url = strdup(url);
 684         if (!f->url)
 685                 return -ENOMEM;
 686
 687         if (local) {
 688                 f->local = strdup(local);
 689                 if (!f->local)
 690                         return -ENOMEM;
 691
 692                 f->force_local = force_local;
 693         }
 694
 695         r = hashmap_put(import->files, f->url, f);
 696         if (r < 0)
 697                 return r;
 698
 699         r = raw_import_file_begin(f);
 700         if (r < 0) {
 701                 raw_import_cancel(import, f->url);
 702                 f = NULL;
 703                 return r;
 704         }
 705
 706         f = NULL;
 707         return 0;
 708 }
 709
 710 bool raw_url_is_valid(const char *url) {
 711         if (isempty(url))
 712                 return false;
 713
 714         if (!startswith(url, "http://") &&
 715             !startswith(url, "https://"))
 716                 return false;
 717
 718         return ascii_is_valid(url);
 719 }