src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "strv.h"
  37 #include "path-util.h"
  38 #include "lookup3.h"
  39 #include "compress.h"
  40 #include "journal-internal.h"
  41 #include "missing.h"
  42 #include "catalog.h"
  43 #include "replace-var.h"
  44
  45 #define JOURNAL_FILES_MAX 1024
  46
  47 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  48
  49 #define REPLACE_VAR_MAX 256
  50
  51 #define DEFAULT_DATA_THRESHOLD (64*1024)
  52
  53 /* We return an error here only if we didn't manage to
  54    memorize the real error. */
  55 static int set_put_error(sd_journal *j, int r) {
  56         int k;
  57
  58         if (r >= 0)
  59                 return r;
  60
  61         k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
  62         if (k < 0)
  63                 return k;
  64
  65         return set_put(j->errors, INT_TO_PTR(r));
  66 }
  67
  68 static void detach_location(sd_journal *j) {
  69         Iterator i;
  70         JournalFile *f;
  71
  72         assert(j);
  73
  74         j->current_file = NULL;
  75         j->current_field = 0;
  76
  77         HASHMAP_FOREACH(f, j->files, i)
  78                 f->current_offset = 0;
  79 }
  80
  81 static void reset_location(sd_journal *j) {
  82         assert(j);
  83
  84         detach_location(j);
  85         zero(j->current_location);
  86 }
  87
  88 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  89         assert(l);
  90         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
  91         assert(f);
  92         assert(o->object.type == OBJECT_ENTRY);
  93
  94         l->type = type;
  95         l->seqnum = le64toh(o->entry.seqnum);
  96         l->seqnum_id = f->header->seqnum_id;
  97         l->realtime = le64toh(o->entry.realtime);
  98         l->monotonic = le64toh(o->entry.monotonic);
  99         l->boot_id = o->entry.boot_id;
 100         l->xor_hash = le64toh(o->entry.xor_hash);
 101
 102         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 103 }
 104
 105 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o, uint64_t offset) {
 106         assert(j);
 107         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 108         assert(f);
 109         assert(o);
 110
 111         init_location(&j->current_location, type, f, o);
 112
 113         if (j->current_file)
 114                 j->current_file->current_offset = 0;
 115
 116         j->current_file = f;
 117         j->current_field = 0;
 118
 119         f->current_offset = offset;
 120 }
 121
 122 static int match_is_valid(const void *data, size_t size) {
 123         const char *b, *p;
 124
 125         assert(data);
 126
 127         if (size < 2)
 128                 return false;
 129
 130         if (startswith(data, "__"))
 131                 return false;
 132
 133         b = data;
 134         for (p = b; p < b + size; p++) {
 135
 136                 if (*p == '=')
 137                         return p > b;
 138
 139                 if (*p == '_')
 140                         continue;
 141
 142                 if (*p >= 'A' && *p <= 'Z')
 143                         continue;
 144
 145                 if (*p >= '0' && *p <= '9')
 146                         continue;
 147
 148                 return false;
 149         }
 150
 151         return false;
 152 }
 153
 154 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 155         const uint8_t *a = _a, *b = _b;
 156         size_t j;
 157
 158         for (j = 0; j < s && j < t; j++) {
 159
 160                 if (a[j] != b[j])
 161                         return false;
 162
 163                 if (a[j] == '=')
 164                         return true;
 165         }
 166
 167         return true;
 168 }
 169
 170 static Match *match_new(Match *p, MatchType t) {
 171         Match *m;
 172
 173         m = new0(Match, 1);
 174         if (!m)
 175                 return NULL;
 176
 177         m->type = t;
 178
 179         if (p) {
 180                 m->parent = p;
 181                 LIST_PREPEND(Match, matches, p->matches, m);
 182         }
 183
 184         return m;
 185 }
 186
 187 static void match_free(Match *m) {
 188         assert(m);
 189
 190         while (m->matches)
 191                 match_free(m->matches);
 192
 193         if (m->parent)
 194                 LIST_REMOVE(Match, matches, m->parent->matches, m);
 195
 196         free(m->data);
 197         free(m);
 198 }
 199
 200 static void match_free_if_empty(Match *m) {
 201         if (!m || m->matches)
 202                 return;
 203
 204         match_free(m);
 205 }
 206
 207 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 208         Match *l3, *l4, *add_here = NULL, *m;
 209         le64_t le_hash;
 210
 211         if (!j)
 212                 return -EINVAL;
 213
 214         if (!data)
 215                 return -EINVAL;
 216
 217         if (size == 0)
 218                 size = strlen(data);
 219
 220         if (!match_is_valid(data, size))
 221                 return -EINVAL;
 222
 223         /* level 0: AND term
 224          * level 1: OR terms
 225          * level 2: AND terms
 226          * level 3: OR terms
 227          * level 4: concrete matches */
 228
 229         if (!j->level0) {
 230                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 231                 if (!j->level0)
 232                         return -ENOMEM;
 233         }
 234
 235         if (!j->level1) {
 236                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 237                 if (!j->level1)
 238                         return -ENOMEM;
 239         }
 240
 241         if (!j->level2) {
 242                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 243                 if (!j->level2)
 244                         return -ENOMEM;
 245         }
 246
 247         assert(j->level0->type == MATCH_AND_TERM);
 248         assert(j->level1->type == MATCH_OR_TERM);
 249         assert(j->level2->type == MATCH_AND_TERM);
 250
 251         le_hash = htole64(hash64(data, size));
 252
 253         LIST_FOREACH(matches, l3, j->level2->matches) {
 254                 assert(l3->type == MATCH_OR_TERM);
 255
 256                 LIST_FOREACH(matches, l4, l3->matches) {
 257                         assert(l4->type == MATCH_DISCRETE);
 258
 259                         /* Exactly the same match already? Then ignore
 260                          * this addition */
 261                         if (l4->le_hash == le_hash &&
 262                             l4->size == size &&
 263                             memcmp(l4->data, data, size) == 0)
 264                                 return 0;
 265
 266                         /* Same field? Then let's add this to this OR term */
 267                         if (same_field(data, size, l4->data, l4->size)) {
 268                                 add_here = l3;
 269                                 break;
 270                         }
 271                 }
 272
 273                 if (add_here)
 274                         break;
 275         }
 276
 277         if (!add_here) {
 278                 add_here = match_new(j->level2, MATCH_OR_TERM);
 279                 if (!add_here)
 280                         goto fail;
 281         }
 282
 283         m = match_new(add_here, MATCH_DISCRETE);
 284         if (!m)
 285                 goto fail;
 286
 287         m->le_hash = le_hash;
 288         m->size = size;
 289         m->data = memdup(data, size);
 290         if (!m->data)
 291                 goto fail;
 292
 293         detach_location(j);
 294
 295         return 0;
 296
 297 fail:
 298         match_free_if_empty(add_here);
 299         match_free_if_empty(j->level2);
 300         match_free_if_empty(j->level1);
 301         match_free_if_empty(j->level0);
 302
 303         return -ENOMEM;
 304 }
 305
 306 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 307         assert(j);
 308
 309         if (!j->level0)
 310                 return 0;
 311
 312         if (!j->level1)
 313                 return 0;
 314
 315         if (!j->level1->matches)
 316                 return 0;
 317
 318         j->level1 = NULL;
 319         j->level2 = NULL;
 320
 321         return 0;
 322 }
 323
 324 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 325         assert(j);
 326
 327         if (!j->level0)
 328                 return 0;
 329
 330         if (!j->level1)
 331                 return 0;
 332
 333         if (!j->level2)
 334                 return 0;
 335
 336         if (!j->level2->matches)
 337                 return 0;
 338
 339         j->level2 = NULL;
 340         return 0;
 341 }
 342
 343 static char *match_make_string(Match *m) {
 344         char *p, *r;
 345         Match *i;
 346         bool enclose = false;
 347
 348         if (!m)
 349                 return strdup("");
 350
 351         if (m->type == MATCH_DISCRETE)
 352                 return strndup(m->data, m->size);
 353
 354         p = NULL;
 355         LIST_FOREACH(matches, i, m->matches) {
 356                 char *t, *k;
 357
 358                 t = match_make_string(i);
 359                 if (!t) {
 360                         free(p);
 361                         return NULL;
 362                 }
 363
 364                 if (p) {
 365                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 366                         free(p);
 367                         free(t);
 368
 369                         if (!k)
 370                                 return NULL;
 371
 372                         p = k;
 373
 374                         enclose = true;
 375                 } else {
 376                         free(p);
 377                         p = t;
 378                 }
 379         }
 380
 381         if (enclose) {
 382                 r = strjoin("(", p, ")", NULL);
 383                 free(p);
 384                 return r;
 385         }
 386
 387         return p;
 388 }
 389
 390 char *journal_make_match_string(sd_journal *j) {
 391         assert(j);
 392
 393         return match_make_string(j->level0);
 394 }
 395
 396 _public_ void sd_journal_flush_matches(sd_journal *j) {
 397
 398         if (!j)
 399                 return;
 400
 401         if (j->level0)
 402                 match_free(j->level0);
 403
 404         j->level0 = j->level1 = j->level2 = NULL;
 405
 406         detach_location(j);
 407 }
 408
 409 static int compare_entry_order(JournalFile *af, Object *_ao,
 410                                JournalFile *bf, uint64_t bp) {
 411
 412         uint64_t a, b;
 413         Object *ao, *bo;
 414         int r;
 415
 416         assert(af);
 417         assert(bf);
 418         assert(_ao);
 419
 420         /* The mmap cache might invalidate the object from the first
 421          * file if we look at the one from the second file. Hence
 422          * temporarily copy the header of the first one, and look at
 423          * that only. */
 424         ao = alloca(offsetof(EntryObject, items));
 425         memcpy(ao, _ao, offsetof(EntryObject, items));
 426
 427         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 428         if (r < 0)
 429                 return strcmp(af->path, bf->path);
 430
 431         /* We operate on two different files here, hence we can access
 432          * two objects at the same time, which we normally can't.
 433          *
 434          * If contents and timestamps match, these entries are
 435          * identical, even if the seqnum does not match */
 436
 437         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 438             ao->entry.monotonic == bo->entry.monotonic &&
 439             ao->entry.realtime == bo->entry.realtime &&
 440             ao->entry.xor_hash == bo->entry.xor_hash)
 441                 return 0;
 442
 443         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 444
 445                 /* If this is from the same seqnum source, compare
 446                  * seqnums */
 447                 a = le64toh(ao->entry.seqnum);
 448                 b = le64toh(bo->entry.seqnum);
 449
 450                 if (a < b)
 451                         return -1;
 452                 if (a > b)
 453                         return 1;
 454
 455                 /* Wow! This is weird, different data but the same
 456                  * seqnums? Something is borked, but let's make the
 457                  * best of it and compare by time. */
 458         }
 459
 460         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 461
 462                 /* If the boot id matches compare monotonic time */
 463                 a = le64toh(ao->entry.monotonic);
 464                 b = le64toh(bo->entry.monotonic);
 465
 466                 if (a < b)
 467                         return -1;
 468                 if (a > b)
 469                         return 1;
 470         }
 471
 472         /* Otherwise compare UTC time */
 473         a = le64toh(ao->entry.realtime);
 474         b = le64toh(bo->entry.realtime);
 475
 476         if (a < b)
 477                 return -1;
 478         if (a > b)
 479                 return 1;
 480
 481         /* Finally, compare by contents */
 482         a = le64toh(ao->entry.xor_hash);
 483         b = le64toh(bo->entry.xor_hash);
 484
 485         if (a < b)
 486                 return -1;
 487         if (a > b)
 488                 return 1;
 489
 490         return 0;
 491 }
 492
 493 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 494         uint64_t a;
 495
 496         assert(af);
 497         assert(ao);
 498         assert(l);
 499         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 500
 501         if (l->monotonic_set &&
 502             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 503             l->realtime_set &&
 504             le64toh(ao->entry.realtime) == l->realtime &&
 505             l->xor_hash_set &&
 506             le64toh(ao->entry.xor_hash) == l->xor_hash)
 507                 return 0;
 508
 509         if (l->seqnum_set &&
 510             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 511
 512                 a = le64toh(ao->entry.seqnum);
 513
 514                 if (a < l->seqnum)
 515                         return -1;
 516                 if (a > l->seqnum)
 517                         return 1;
 518         }
 519
 520         if (l->monotonic_set &&
 521             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 522
 523                 a = le64toh(ao->entry.monotonic);
 524
 525                 if (a < l->monotonic)
 526                         return -1;
 527                 if (a > l->monotonic)
 528                         return 1;
 529         }
 530
 531         if (l->realtime_set) {
 532
 533                 a = le64toh(ao->entry.realtime);
 534
 535                 if (a < l->realtime)
 536                         return -1;
 537                 if (a > l->realtime)
 538                         return 1;
 539         }
 540
 541         if (l->xor_hash_set) {
 542                 a = le64toh(ao->entry.xor_hash);
 543
 544                 if (a < l->xor_hash)
 545                         return -1;
 546                 if (a > l->xor_hash)
 547                         return 1;
 548         }
 549
 550         return 0;
 551 }
 552
 553 static int next_for_match(
 554                 sd_journal *j,
 555                 Match *m,
 556                 JournalFile *f,
 557                 uint64_t after_offset,
 558                 direction_t direction,
 559                 Object **ret,
 560                 uint64_t *offset) {
 561
 562         int r;
 563         uint64_t np = 0;
 564         Object *n;
 565
 566         assert(j);
 567         assert(m);
 568         assert(f);
 569
 570         if (m->type == MATCH_DISCRETE) {
 571                 uint64_t dp;
 572
 573                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 574                 if (r <= 0)
 575                         return r;
 576
 577                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 578
 579         } else if (m->type == MATCH_OR_TERM) {
 580                 Match *i;
 581
 582                 /* Find the earliest match beyond after_offset */
 583
 584                 LIST_FOREACH(matches, i, m->matches) {
 585                         uint64_t cp;
 586
 587                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 588                         if (r < 0)
 589                                 return r;
 590                         else if (r > 0) {
 591                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 592                                         np = cp;
 593                         }
 594                 }
 595
 596         } else if (m->type == MATCH_AND_TERM) {
 597                 Match *i, *last_moved;
 598
 599                 /* Always jump to the next matching entry and repeat
 600                  * this until we find an offset that matches for all
 601                  * matches. */
 602
 603                 if (!m->matches)
 604                         return 0;
 605
 606                 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
 607                 if (r <= 0)
 608                         return r;
 609
 610                 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
 611                 last_moved = m->matches;
 612
 613                 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
 614                         uint64_t cp;
 615
 616                         r = next_for_match(j, i, f, np, direction, NULL, &cp);
 617                         if (r <= 0)
 618                                 return r;
 619
 620                         assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
 621                         if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
 622                                 np = cp;
 623                                 last_moved = i;
 624                         }
 625                 }
 626         }
 627
 628         if (np == 0)
 629                 return 0;
 630
 631         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 632         if (r < 0)
 633                 return r;
 634
 635         if (ret)
 636                 *ret = n;
 637         if (offset)
 638                 *offset = np;
 639
 640         return 1;
 641 }
 642
 643 static int find_location_for_match(
 644                 sd_journal *j,
 645                 Match *m,
 646                 JournalFile *f,
 647                 direction_t direction,
 648                 Object **ret,
 649                 uint64_t *offset) {
 650
 651         int r;
 652
 653         assert(j);
 654         assert(m);
 655         assert(f);
 656
 657         if (m->type == MATCH_DISCRETE) {
 658                 uint64_t dp;
 659
 660                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 661                 if (r <= 0)
 662                         return r;
 663
 664                 /* FIXME: missing: find by monotonic */
 665
 666                 if (j->current_location.type == LOCATION_HEAD)
 667                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 668                 if (j->current_location.type == LOCATION_TAIL)
 669                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 670                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 671                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 672                 if (j->current_location.monotonic_set) {
 673                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 674                         if (r != -ENOENT)
 675                                 return r;
 676                 }
 677                 if (j->current_location.realtime_set)
 678                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 679
 680                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 681
 682         } else if (m->type == MATCH_OR_TERM) {
 683                 uint64_t np = 0;
 684                 Object *n;
 685                 Match *i;
 686
 687                 /* Find the earliest match */
 688
 689                 LIST_FOREACH(matches, i, m->matches) {
 690                         uint64_t cp;
 691
 692                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 693                         if (r < 0)
 694                                 return r;
 695                         else if (r > 0) {
 696                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 697                                         np = cp;
 698                         }
 699                 }
 700
 701                 if (np == 0)
 702                         return 0;
 703
 704                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 705                 if (r < 0)
 706                         return r;
 707
 708                 if (ret)
 709                         *ret = n;
 710                 if (offset)
 711                         *offset = np;
 712
 713                 return 1;
 714
 715         } else {
 716                 Match *i;
 717                 uint64_t np = 0;
 718
 719                 assert(m->type == MATCH_AND_TERM);
 720
 721                 /* First jump to the last match, and then find the
 722                  * next one where all matches match */
 723
 724                 if (!m->matches)
 725                         return 0;
 726
 727                 LIST_FOREACH(matches, i, m->matches) {
 728                         uint64_t cp;
 729
 730                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 731                         if (r <= 0)
 732                                 return r;
 733
 734                         if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
 735                                 np = cp;
 736                 }
 737
 738                 return next_for_match(j, m, f, np, direction, ret, offset);
 739         }
 740 }
 741
 742 static int find_location_with_matches(
 743                 sd_journal *j,
 744                 JournalFile *f,
 745                 direction_t direction,
 746                 Object **ret,
 747                 uint64_t *offset) {
 748
 749         int r;
 750
 751         assert(j);
 752         assert(f);
 753         assert(ret);
 754         assert(offset);
 755
 756         if (!j->level0) {
 757                 /* No matches is simple */
 758
 759                 if (j->current_location.type == LOCATION_HEAD)
 760                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 761                 if (j->current_location.type == LOCATION_TAIL)
 762                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 763                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 764                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 765                 if (j->current_location.monotonic_set) {
 766                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 767                         if (r != -ENOENT)
 768                                 return r;
 769                 }
 770                 if (j->current_location.realtime_set)
 771                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 772
 773                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 774         } else
 775                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 776 }
 777
 778 static int next_with_matches(
 779                 sd_journal *j,
 780                 JournalFile *f,
 781                 direction_t direction,
 782                 Object **ret,
 783                 uint64_t *offset) {
 784
 785         Object *c;
 786         uint64_t cp;
 787
 788         assert(j);
 789         assert(f);
 790         assert(ret);
 791         assert(offset);
 792
 793         c = *ret;
 794         cp = *offset;
 795
 796         /* No matches is easy. We simple advance the file
 797          * pointer by one. */
 798         if (!j->level0)
 799                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 800
 801         /* If we have a match then we look for the next matching entry
 802          * with an offset at least one step larger */
 803         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 804 }
 805
 806 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 807         Object *c;
 808         uint64_t cp;
 809         int r;
 810
 811         assert(j);
 812         assert(f);
 813
 814         if (f->current_offset > 0) {
 815                 cp = f->current_offset;
 816
 817                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 818                 if (r < 0)
 819                         return r;
 820
 821                 r = next_with_matches(j, f, direction, &c, &cp);
 822                 if (r <= 0)
 823                         return r;
 824         } else {
 825                 r = find_location_with_matches(j, f, direction, &c, &cp);
 826                 if (r <= 0)
 827                         return r;
 828         }
 829
 830         /* OK, we found the spot, now let's advance until to an entry
 831          * that is actually different from what we were previously
 832          * looking at. This is necessary to handle entries which exist
 833          * in two (or more) journal files, and which shall all be
 834          * suppressed but one. */
 835
 836         for (;;) {
 837                 bool found;
 838
 839                 if (j->current_location.type == LOCATION_DISCRETE) {
 840                         int k;
 841
 842                         k = compare_with_location(f, c, &j->current_location);
 843                         if (direction == DIRECTION_DOWN)
 844                                 found = k > 0;
 845                         else
 846                                 found = k < 0;
 847                 } else
 848                         found = true;
 849
 850                 if (found) {
 851                         if (ret)
 852                                 *ret = c;
 853                         if (offset)
 854                                 *offset = cp;
 855                         return 1;
 856                 }
 857
 858                 r = next_with_matches(j, f, direction, &c, &cp);
 859                 if (r <= 0)
 860                         return r;
 861         }
 862 }
 863
 864 static int real_journal_next(sd_journal *j, direction_t direction) {
 865         JournalFile *f, *new_file = NULL;
 866         uint64_t new_offset = 0;
 867         Object *o;
 868         uint64_t p;
 869         Iterator i;
 870         int r;
 871
 872         if (!j)
 873                 return -EINVAL;
 874
 875         HASHMAP_FOREACH(f, j->files, i) {
 876                 bool found;
 877
 878                 r = next_beyond_location(j, f, direction, &o, &p);
 879                 if (r < 0) {
 880                         log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
 881                         continue;
 882                 } else if (r == 0)
 883                         continue;
 884
 885                 if (!new_file)
 886                         found = true;
 887                 else {
 888                         int k;
 889
 890                         k = compare_entry_order(f, o, new_file, new_offset);
 891
 892                         if (direction == DIRECTION_DOWN)
 893                                 found = k < 0;
 894                         else
 895                                 found = k > 0;
 896                 }
 897
 898                 if (found) {
 899                         new_file = f;
 900                         new_offset = p;
 901                 }
 902         }
 903
 904         if (!new_file)
 905                 return 0;
 906
 907         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 908         if (r < 0)
 909                 return r;
 910
 911         set_location(j, LOCATION_DISCRETE, new_file, o, new_offset);
 912
 913         return 1;
 914 }
 915
 916 _public_ int sd_journal_next(sd_journal *j) {
 917         return real_journal_next(j, DIRECTION_DOWN);
 918 }
 919
 920 _public_ int sd_journal_previous(sd_journal *j) {
 921         return real_journal_next(j, DIRECTION_UP);
 922 }
 923
 924 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 925         int c = 0, r;
 926
 927         if (!j)
 928                 return -EINVAL;
 929
 930         if (skip == 0) {
 931                 /* If this is not a discrete skip, then at least
 932                  * resolve the current location */
 933                 if (j->current_location.type != LOCATION_DISCRETE)
 934                         return real_journal_next(j, direction);
 935
 936                 return 0;
 937         }
 938
 939         do {
 940                 r = real_journal_next(j, direction);
 941                 if (r < 0)
 942                         return r;
 943
 944                 if (r == 0)
 945                         return c;
 946
 947                 skip--;
 948                 c++;
 949         } while (skip > 0);
 950
 951         return c;
 952 }
 953
 954 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 955         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 956 }
 957
 958 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 959         return real_journal_next_skip(j, DIRECTION_UP, skip);
 960 }
 961
 962 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 963         Object *o;
 964         int r;
 965         char bid[33], sid[33];
 966
 967         if (!j)
 968                 return -EINVAL;
 969         if (!cursor)
 970                 return -EINVAL;
 971
 972         if (!j->current_file || j->current_file->current_offset <= 0)
 973                 return -EADDRNOTAVAIL;
 974
 975         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 976         if (r < 0)
 977                 return r;
 978
 979         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 980         sd_id128_to_string(o->entry.boot_id, bid);
 981
 982         if (asprintf(cursor,
 983                      "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
 984                      sid, le64toh(o->entry.seqnum),
 985                      bid, le64toh(o->entry.monotonic),
 986                      le64toh(o->entry.realtime),
 987                      le64toh(o->entry.xor_hash)) < 0)
 988                 return -ENOMEM;
 989
 990         return 0;
 991 }
 992
 993 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
 994         char *w, *state;
 995         size_t l;
 996         unsigned long long seqnum, monotonic, realtime, xor_hash;
 997         bool
 998                 seqnum_id_set = false,
 999                 seqnum_set = false,
1000                 boot_id_set = false,
1001                 monotonic_set = false,
1002                 realtime_set = false,
1003                 xor_hash_set = false;
1004         sd_id128_t seqnum_id, boot_id;
1005
1006         if (!j)
1007                 return -EINVAL;
1008         if (isempty(cursor))
1009                 return -EINVAL;
1010
1011         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1012                 char *item;
1013                 int k = 0;
1014
1015                 if (l < 2 || w[1] != '=')
1016                         return -EINVAL;
1017
1018                 item = strndup(w, l);
1019                 if (!item)
1020                         return -ENOMEM;
1021
1022                 switch (w[0]) {
1023
1024                 case 's':
1025                         seqnum_id_set = true;
1026                         k = sd_id128_from_string(item+2, &seqnum_id);
1027                         break;
1028
1029                 case 'i':
1030                         seqnum_set = true;
1031                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1032                                 k = -EINVAL;
1033                         break;
1034
1035                 case 'b':
1036                         boot_id_set = true;
1037                         k = sd_id128_from_string(item+2, &boot_id);
1038                         break;
1039
1040                 case 'm':
1041                         monotonic_set = true;
1042                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1043                                 k = -EINVAL;
1044                         break;
1045
1046                 case 't':
1047                         realtime_set = true;
1048                         if (sscanf(item+2, "%llx", &realtime) != 1)
1049                                 k = -EINVAL;
1050                         break;
1051
1052                 case 'x':
1053                         xor_hash_set = true;
1054                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1055                                 k = -EINVAL;
1056                         break;
1057                 }
1058
1059                 free(item);
1060
1061                 if (k < 0)
1062                         return k;
1063         }
1064
1065         if ((!seqnum_set || !seqnum_id_set) &&
1066             (!monotonic_set || !boot_id_set) &&
1067             !realtime_set)
1068                 return -EINVAL;
1069
1070         reset_location(j);
1071
1072         j->current_location.type = LOCATION_SEEK;
1073
1074         if (realtime_set) {
1075                 j->current_location.realtime = (uint64_t) realtime;
1076                 j->current_location.realtime_set = true;
1077         }
1078
1079         if (seqnum_set && seqnum_id_set) {
1080                 j->current_location.seqnum = (uint64_t) seqnum;
1081                 j->current_location.seqnum_id = seqnum_id;
1082                 j->current_location.seqnum_set = true;
1083         }
1084
1085         if (monotonic_set && boot_id_set) {
1086                 j->current_location.monotonic = (uint64_t) monotonic;
1087                 j->current_location.boot_id = boot_id;
1088                 j->current_location.monotonic_set = true;
1089         }
1090
1091         if (xor_hash_set) {
1092                 j->current_location.xor_hash = (uint64_t) xor_hash;
1093                 j->current_location.xor_hash_set = true;
1094         }
1095
1096         return 0;
1097 }
1098
1099 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1100         int r;
1101         char *w, *state;
1102         size_t l;
1103         Object *o;
1104
1105         if (!j)
1106                 return -EINVAL;
1107         if (isempty(cursor))
1108                 return -EINVAL;
1109
1110         if (!j->current_file || j->current_file->current_offset <= 0)
1111                 return -EADDRNOTAVAIL;
1112
1113         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1114         if (r < 0)
1115                 return r;
1116
1117         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1118                 _cleanup_free_ char *item = NULL;
1119                 sd_id128_t id;
1120                 unsigned long long ll;
1121                 int k = 0;
1122
1123                 if (l < 2 || w[1] != '=')
1124                         return -EINVAL;
1125
1126                 item = strndup(w, l);
1127                 if (!item)
1128                         return -ENOMEM;
1129
1130                 switch (w[0]) {
1131
1132                 case 's':
1133                         k = sd_id128_from_string(item+2, &id);
1134                         if (k < 0)
1135                                 return k;
1136                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1137                                 return 0;
1138                         break;
1139
1140                 case 'i':
1141                         if (sscanf(item+2, "%llx", &ll) != 1)
1142                                 return -EINVAL;
1143                         if (ll != le64toh(o->entry.seqnum))
1144                                 return 0;
1145                         break;
1146
1147                 case 'b':
1148                         k = sd_id128_from_string(item+2, &id);
1149                         if (k < 0)
1150                                 return k;
1151                         if (!sd_id128_equal(id, o->entry.boot_id))
1152                                 return 0;
1153                         break;
1154
1155                 case 'm':
1156                         if (sscanf(item+2, "%llx", &ll) != 1)
1157                                 return -EINVAL;
1158                         if (ll != le64toh(o->entry.monotonic))
1159                                 return 0;
1160                         break;
1161
1162                 case 't':
1163                         if (sscanf(item+2, "%llx", &ll) != 1)
1164                                 return -EINVAL;
1165                         if (ll != le64toh(o->entry.realtime))
1166                                 return 0;
1167                         break;
1168
1169                 case 'x':
1170                         if (sscanf(item+2, "%llx", &ll) != 1)
1171                                 return -EINVAL;
1172                         if (ll != le64toh(o->entry.xor_hash))
1173                                 return 0;
1174                         break;
1175                 }
1176         }
1177
1178         return 1;
1179 }
1180
1181
1182 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1183         if (!j)
1184                 return -EINVAL;
1185
1186         reset_location(j);
1187         j->current_location.type = LOCATION_SEEK;
1188         j->current_location.boot_id = boot_id;
1189         j->current_location.monotonic = usec;
1190         j->current_location.monotonic_set = true;
1191
1192         return 0;
1193 }
1194
1195 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1196         if (!j)
1197                 return -EINVAL;
1198
1199         reset_location(j);
1200         j->current_location.type = LOCATION_SEEK;
1201         j->current_location.realtime = usec;
1202         j->current_location.realtime_set = true;
1203
1204         return 0;
1205 }
1206
1207 _public_ int sd_journal_seek_head(sd_journal *j) {
1208         if (!j)
1209                 return -EINVAL;
1210
1211         reset_location(j);
1212         j->current_location.type = LOCATION_HEAD;
1213
1214         return 0;
1215 }
1216
1217 _public_ int sd_journal_seek_tail(sd_journal *j) {
1218         if (!j)
1219                 return -EINVAL;
1220
1221         reset_location(j);
1222         j->current_location.type = LOCATION_TAIL;
1223
1224         return 0;
1225 }
1226
1227 static void check_network(sd_journal *j, int fd) {
1228         struct statfs sfs;
1229
1230         assert(j);
1231
1232         if (j->on_network)
1233                 return;
1234
1235         if (fstatfs(fd, &sfs) < 0)
1236                 return;
1237
1238         j->on_network =
1239                 F_TYPE_CMP(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1240                 F_TYPE_CMP(sfs.f_type, CODA_SUPER_MAGIC) ||
1241                 F_TYPE_CMP(sfs.f_type, NCP_SUPER_MAGIC) ||
1242                 F_TYPE_CMP(sfs.f_type, NFS_SUPER_MAGIC) ||
1243                 F_TYPE_CMP(sfs.f_type, SMB_SUPER_MAGIC);
1244 }
1245
1246 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1247         const char *full, *tilded, *atted;
1248
1249         full = strappend(prefix, ".journal");
1250         tilded = strappenda(full, "~");
1251         atted = strappenda(prefix, "@");
1252
1253         return streq(filename, full) ||
1254                streq(filename, tilded) ||
1255                startswith(filename, atted);
1256 }
1257
1258 static bool file_type_wanted(int flags, const char *filename) {
1259         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1260                 return false;
1261
1262         /* no flags set → every type is OK */
1263         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1264                 return true;
1265
1266         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1267                 return true;
1268
1269         if (flags & SD_JOURNAL_CURRENT_USER) {
1270                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1271
1272                 assert_se(snprintf(prefix, sizeof(prefix), "user-%lu", (unsigned long) getuid())
1273                           < (int) sizeof(prefix));
1274
1275                 if (file_has_type_prefix(prefix, filename))
1276                         return true;
1277         }
1278
1279         return false;
1280 }
1281
1282 static int add_any_file(sd_journal *j, const char *path) {
1283         JournalFile *f;
1284         int r;
1285
1286         assert(j);
1287         assert(path);
1288
1289         if (hashmap_get(j->files, path))
1290                 return 0;
1291
1292         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1293                 log_warning("Too many open journal files, not adding %s.", path);
1294                 return set_put_error(j, -ETOOMANYREFS);
1295         }
1296
1297         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1298         if (r < 0)
1299                 return r;
1300
1301         /* journal_file_dump(f); */
1302
1303         r = hashmap_put(j->files, f->path, f);
1304         if (r < 0) {
1305                 journal_file_close(f);
1306                 return r;
1307         }
1308
1309         log_debug("File %s added.", f->path);
1310
1311         check_network(j, f->fd);
1312
1313         j->current_invalidate_counter ++;
1314
1315         return 0;
1316 }
1317
1318 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1319         _cleanup_free_ char *path = NULL;
1320         int r;
1321
1322         assert(j);
1323         assert(prefix);
1324         assert(filename);
1325
1326         if (j->no_new_files ||
1327             !file_type_wanted(j->flags, filename))
1328                 return 0;
1329
1330         path = strjoin(prefix, "/", filename, NULL);
1331         if (!path)
1332                 return -ENOMEM;
1333
1334         r = add_any_file(j, path);
1335         if (r == -ENOENT)
1336                 return 0;
1337         return 0;
1338 }
1339
1340 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1341         char *path;
1342         JournalFile *f;
1343
1344         assert(j);
1345         assert(prefix);
1346         assert(filename);
1347
1348         path = strjoin(prefix, "/", filename, NULL);
1349         if (!path)
1350                 return -ENOMEM;
1351
1352         f = hashmap_get(j->files, path);
1353         free(path);
1354         if (!f)
1355                 return 0;
1356
1357         hashmap_remove(j->files, f->path);
1358
1359         log_debug("File %s removed.", f->path);
1360
1361         if (j->current_file == f) {
1362                 j->current_file = NULL;
1363                 j->current_field = 0;
1364         }
1365
1366         if (j->unique_file == f) {
1367                 j->unique_file = NULL;
1368                 j->unique_offset = 0;
1369         }
1370
1371         journal_file_close(f);
1372
1373         j->current_invalidate_counter ++;
1374
1375         return 0;
1376 }
1377
1378 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1379         _cleanup_free_ char *path = NULL;
1380         int r;
1381         _cleanup_closedir_ DIR *d = NULL;
1382         sd_id128_t id, mid;
1383         Directory *m;
1384
1385         assert(j);
1386         assert(prefix);
1387         assert(dirname);
1388
1389         log_debug("Considering %s/%s.", prefix, dirname);
1390
1391         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1392             (sd_id128_from_string(dirname, &id) < 0 ||
1393              sd_id128_get_machine(&mid) < 0 ||
1394              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1395             return 0;
1396
1397         path = strjoin(prefix, "/", dirname, NULL);
1398         if (!path)
1399                 return -ENOMEM;
1400
1401         d = opendir(path);
1402         if (!d) {
1403                 log_debug("Failed to open %s: %m", path);
1404                 if (errno == ENOENT)
1405                         return 0;
1406                 return -errno;
1407         }
1408
1409         m = hashmap_get(j->directories_by_path, path);
1410         if (!m) {
1411                 m = new0(Directory, 1);
1412                 if (!m)
1413                         return -ENOMEM;
1414
1415                 m->is_root = false;
1416                 m->path = path;
1417
1418                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1419                         free(m);
1420                         return -ENOMEM;
1421                 }
1422
1423                 path = NULL; /* avoid freeing in cleanup */
1424                 j->current_invalidate_counter ++;
1425
1426                 log_debug("Directory %s added.", m->path);
1427
1428         } else if (m->is_root)
1429                 return 0;
1430
1431         if (m->wd <= 0 && j->inotify_fd >= 0) {
1432
1433                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1434                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1435                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1436                                           IN_ONLYDIR);
1437
1438                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1439                         inotify_rm_watch(j->inotify_fd, m->wd);
1440         }
1441
1442         for (;;) {
1443                 struct dirent *de;
1444                 union dirent_storage buf;
1445
1446                 r = readdir_r(d, &buf.de, &de);
1447                 if (r != 0 || !de)
1448                         break;
1449
1450                 if (dirent_is_file_with_suffix(de, ".journal") ||
1451                     dirent_is_file_with_suffix(de, ".journal~")) {
1452                         r = add_file(j, m->path, de->d_name);
1453                         if (r < 0) {
1454                                 log_debug("Failed to add file %s/%s: %s",
1455                                           m->path, de->d_name, strerror(-r));
1456                                 r = set_put_error(j, r);
1457                                 if (r < 0)
1458                                         return r;
1459                         }
1460                 }
1461         }
1462
1463         check_network(j, dirfd(d));
1464
1465         return 0;
1466 }
1467
1468 static int add_root_directory(sd_journal *j, const char *p) {
1469         _cleanup_closedir_ DIR *d = NULL;
1470         Directory *m;
1471         int r;
1472
1473         assert(j);
1474         assert(p);
1475
1476         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1477             !path_startswith(p, "/run"))
1478                 return -EINVAL;
1479
1480         d = opendir(p);
1481         if (!d)
1482                 return -errno;
1483
1484         m = hashmap_get(j->directories_by_path, p);
1485         if (!m) {
1486                 m = new0(Directory, 1);
1487                 if (!m)
1488                         return -ENOMEM;
1489
1490                 m->is_root = true;
1491                 m->path = strdup(p);
1492                 if (!m->path) {
1493                         free(m);
1494                         return -ENOMEM;
1495                 }
1496
1497                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1498                         free(m->path);
1499                         free(m);
1500                         return -ENOMEM;
1501                 }
1502
1503                 j->current_invalidate_counter ++;
1504
1505                 log_debug("Root directory %s added.", m->path);
1506
1507         } else if (!m->is_root)
1508                 return 0;
1509
1510         if (m->wd <= 0 && j->inotify_fd >= 0) {
1511
1512                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1513                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1514                                           IN_ONLYDIR);
1515
1516                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1517                         inotify_rm_watch(j->inotify_fd, m->wd);
1518         }
1519
1520         if (j->no_new_files)
1521                 return 0;
1522
1523         for (;;) {
1524                 struct dirent *de;
1525                 union dirent_storage buf;
1526                 sd_id128_t id;
1527
1528                 r = readdir_r(d, &buf.de, &de);
1529                 if (r != 0 || !de)
1530                         break;
1531
1532                 if (dirent_is_file_with_suffix(de, ".journal") ||
1533                     dirent_is_file_with_suffix(de, ".journal~")) {
1534                         r = add_file(j, m->path, de->d_name);
1535                         if (r < 0) {
1536                                 log_debug("Failed to add file %s/%s: %s",
1537                                           m->path, de->d_name, strerror(-r));
1538                                 r = set_put_error(j, r);
1539                                 if (r < 0)
1540                                         return r;
1541                         }
1542                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1543                            sd_id128_from_string(de->d_name, &id) >= 0) {
1544
1545                         r = add_directory(j, m->path, de->d_name);
1546                         if (r < 0)
1547                                 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1548                 }
1549         }
1550
1551         check_network(j, dirfd(d));
1552
1553         return 0;
1554 }
1555
1556 static int remove_directory(sd_journal *j, Directory *d) {
1557         assert(j);
1558
1559         if (d->wd > 0) {
1560                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1561
1562                 if (j->inotify_fd >= 0)
1563                         inotify_rm_watch(j->inotify_fd, d->wd);
1564         }
1565
1566         hashmap_remove(j->directories_by_path, d->path);
1567
1568         if (d->is_root)
1569                 log_debug("Root directory %s removed.", d->path);
1570         else
1571                 log_debug("Directory %s removed.", d->path);
1572
1573         free(d->path);
1574         free(d);
1575
1576         return 0;
1577 }
1578
1579 static int add_search_paths(sd_journal *j) {
1580         int r;
1581         const char search_paths[] =
1582                 "/run/log/journal\0"
1583                 "/var/log/journal\0";
1584         const char *p;
1585
1586         assert(j);
1587
1588         /* We ignore most errors here, since the idea is to only open
1589          * what's actually accessible, and ignore the rest. */
1590
1591         NULSTR_FOREACH(p, search_paths) {
1592                 r = add_root_directory(j, p);
1593                 if (r < 0 && r != -ENOENT) {
1594                         r = set_put_error(j, r);
1595                         if (r < 0)
1596                                 return r;
1597                 }
1598         }
1599
1600         return 0;
1601 }
1602
1603 static int add_current_paths(sd_journal *j) {
1604         Iterator i;
1605         JournalFile *f;
1606
1607         assert(j);
1608         assert(j->no_new_files);
1609
1610         /* Simply adds all directories for files we have open as
1611          * "root" directories. We don't expect errors here, so we
1612          * treat them as fatal. */
1613
1614         HASHMAP_FOREACH(f, j->files, i) {
1615                 int r;
1616                 _cleanup_free_ char *dir;
1617
1618                 dir = dirname_malloc(f->path);
1619                 if (!dir)
1620                         return -ENOMEM;
1621
1622                 r = add_root_directory(j, dir);
1623                 if (r < 0) {
1624                         set_put_error(j, r);
1625                         return r;
1626                 }
1627         }
1628
1629         return 0;
1630 }
1631
1632
1633 static int allocate_inotify(sd_journal *j) {
1634         assert(j);
1635
1636         if (j->inotify_fd < 0) {
1637                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1638                 if (j->inotify_fd < 0)
1639                         return -errno;
1640         }
1641
1642         if (!j->directories_by_wd) {
1643                 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1644                 if (!j->directories_by_wd)
1645                         return -ENOMEM;
1646         }
1647
1648         return 0;
1649 }
1650
1651 static sd_journal *journal_new(int flags, const char *path) {
1652         sd_journal *j;
1653
1654         j = new0(sd_journal, 1);
1655         if (!j)
1656                 return NULL;
1657
1658         j->inotify_fd = -1;
1659         j->flags = flags;
1660         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1661
1662         if (path) {
1663                 j->path = strdup(path);
1664                 if (!j->path)
1665                         goto fail;
1666         }
1667
1668         j->files = hashmap_new(string_hash_func, string_compare_func);
1669         j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1670         j->mmap = mmap_cache_new();
1671         if (!j->files || !j->directories_by_path || !j->mmap)
1672                 goto fail;
1673
1674         return j;
1675
1676 fail:
1677         sd_journal_close(j);
1678         return NULL;
1679 }
1680
1681 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1682         sd_journal *j;
1683         int r;
1684
1685         if (!ret)
1686                 return -EINVAL;
1687
1688         if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1689                       SD_JOURNAL_RUNTIME_ONLY|
1690                       SD_JOURNAL_SYSTEM|
1691                       SD_JOURNAL_CURRENT_USER))
1692                 return -EINVAL;
1693
1694         j = journal_new(flags, NULL);
1695         if (!j)
1696                 return -ENOMEM;
1697
1698         r = add_search_paths(j);
1699         if (r < 0)
1700                 goto fail;
1701
1702         *ret = j;
1703         return 0;
1704
1705 fail:
1706         sd_journal_close(j);
1707
1708         return r;
1709 }
1710
1711 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1712         sd_journal *j;
1713         int r;
1714
1715         if (!ret)
1716                 return -EINVAL;
1717
1718         if (!path)
1719                 return -EINVAL;
1720
1721         if (flags != 0)
1722                 return -EINVAL;
1723
1724         j = journal_new(flags, path);
1725         if (!j)
1726                 return -ENOMEM;
1727
1728         r = add_root_directory(j, path);
1729         if (r < 0) {
1730                 set_put_error(j, r);
1731                 goto fail;
1732         }
1733
1734         *ret = j;
1735         return 0;
1736
1737 fail:
1738         sd_journal_close(j);
1739
1740         return r;
1741 }
1742
1743 _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int flags) {
1744         sd_journal *j;
1745         const char **path;
1746         int r;
1747
1748         if (!ret)
1749                 return -EINVAL;
1750
1751         if (flags != 0)
1752                 return -EINVAL;
1753
1754         j = journal_new(flags, NULL);
1755         if (!j)
1756                 return -ENOMEM;
1757
1758         STRV_FOREACH(path, paths) {
1759                 r = add_any_file(j, *path);
1760                 if (r < 0) {
1761                         log_error("Failed to open %s: %s", *path, strerror(-r));
1762                         goto fail;
1763                 }
1764         }
1765
1766         j->no_new_files = true;
1767
1768         *ret = j;
1769         return 0;
1770
1771 fail:
1772         sd_journal_close(j);
1773
1774         return r;
1775 }
1776
1777 _public_ void sd_journal_close(sd_journal *j) {
1778         Directory *d;
1779         JournalFile *f;
1780
1781         if (!j)
1782                 return;
1783
1784         sd_journal_flush_matches(j);
1785
1786         while ((f = hashmap_steal_first(j->files)))
1787                 journal_file_close(f);
1788
1789         hashmap_free(j->files);
1790
1791         while ((d = hashmap_first(j->directories_by_path)))
1792                 remove_directory(j, d);
1793
1794         while ((d = hashmap_first(j->directories_by_wd)))
1795                 remove_directory(j, d);
1796
1797         hashmap_free(j->directories_by_path);
1798         hashmap_free(j->directories_by_wd);
1799
1800         if (j->inotify_fd >= 0)
1801                 close_nointr_nofail(j->inotify_fd);
1802
1803         if (j->mmap)
1804                 mmap_cache_unref(j->mmap);
1805
1806         free(j->path);
1807         free(j->unique_field);
1808         set_free(j->errors);
1809         free(j);
1810 }
1811
1812 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1813         Object *o;
1814         JournalFile *f;
1815         int r;
1816
1817         if (!j)
1818                 return -EINVAL;
1819         if (!ret)
1820                 return -EINVAL;
1821
1822         f = j->current_file;
1823         if (!f)
1824                 return -EADDRNOTAVAIL;
1825
1826         if (f->current_offset <= 0)
1827                 return -EADDRNOTAVAIL;
1828
1829         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1830         if (r < 0)
1831                 return r;
1832
1833         *ret = le64toh(o->entry.realtime);
1834         return 0;
1835 }
1836
1837 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1838         Object *o;
1839         JournalFile *f;
1840         int r;
1841         sd_id128_t id;
1842
1843         if (!j)
1844                 return -EINVAL;
1845
1846         f = j->current_file;
1847         if (!f)
1848                 return -EADDRNOTAVAIL;
1849
1850         if (f->current_offset <= 0)
1851                 return -EADDRNOTAVAIL;
1852
1853         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1854         if (r < 0)
1855                 return r;
1856
1857         if (ret_boot_id)
1858                 *ret_boot_id = o->entry.boot_id;
1859         else {
1860                 r = sd_id128_get_boot(&id);
1861                 if (r < 0)
1862                         return r;
1863
1864                 if (!sd_id128_equal(id, o->entry.boot_id))
1865                         return -ESTALE;
1866         }
1867
1868         if (ret)
1869                 *ret = le64toh(o->entry.monotonic);
1870
1871         return 0;
1872 }
1873
1874 static bool field_is_valid(const char *field) {
1875         const char *p;
1876
1877         assert(field);
1878
1879         if (isempty(field))
1880                 return false;
1881
1882         if (startswith(field, "__"))
1883                 return false;
1884
1885         for (p = field; *p; p++) {
1886
1887                 if (*p == '_')
1888                         continue;
1889
1890                 if (*p >= 'A' && *p <= 'Z')
1891                         continue;
1892
1893                 if (*p >= '0' && *p <= '9')
1894                         continue;
1895
1896                 return false;
1897         }
1898
1899         return true;
1900 }
1901
1902 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1903         JournalFile *f;
1904         uint64_t i, n;
1905         size_t field_length;
1906         int r;
1907         Object *o;
1908
1909         if (!j)
1910                 return -EINVAL;
1911         if (!field)
1912                 return -EINVAL;
1913         if (!data)
1914                 return -EINVAL;
1915         if (!size)
1916                 return -EINVAL;
1917
1918         if (!field_is_valid(field))
1919                 return -EINVAL;
1920
1921         f = j->current_file;
1922         if (!f)
1923                 return -EADDRNOTAVAIL;
1924
1925         if (f->current_offset <= 0)
1926                 return -EADDRNOTAVAIL;
1927
1928         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1929         if (r < 0)
1930                 return r;
1931
1932         field_length = strlen(field);
1933
1934         n = journal_file_entry_n_items(o);
1935         for (i = 0; i < n; i++) {
1936                 uint64_t p, l;
1937                 le64_t le_hash;
1938                 size_t t;
1939
1940                 p = le64toh(o->entry.items[i].object_offset);
1941                 le_hash = o->entry.items[i].hash;
1942                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1943                 if (r < 0)
1944                         return r;
1945
1946                 if (le_hash != o->data.hash)
1947                         return -EBADMSG;
1948
1949                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1950
1951                 if (o->object.flags & OBJECT_COMPRESSED) {
1952
1953 #ifdef HAVE_XZ
1954                         if (uncompress_startswith(o->data.payload, l,
1955                                                   &f->compress_buffer, &f->compress_buffer_size,
1956                                                   field, field_length, '=')) {
1957
1958                                 uint64_t rsize;
1959
1960                                 if (!uncompress_blob(o->data.payload, l,
1961                                                      &f->compress_buffer, &f->compress_buffer_size, &rsize,
1962                                                      j->data_threshold))
1963                                         return -EBADMSG;
1964
1965                                 *data = f->compress_buffer;
1966                                 *size = (size_t) rsize;
1967
1968                                 return 0;
1969                         }
1970 #else
1971                         return -EPROTONOSUPPORT;
1972 #endif
1973
1974                 } else if (l >= field_length+1 &&
1975                            memcmp(o->data.payload, field, field_length) == 0 &&
1976                            o->data.payload[field_length] == '=') {
1977
1978                         t = (size_t) l;
1979
1980                         if ((uint64_t) t != l)
1981                                 return -E2BIG;
1982
1983                         *data = o->data.payload;
1984                         *size = t;
1985
1986                         return 0;
1987                 }
1988
1989                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1990                 if (r < 0)
1991                         return r;
1992         }
1993
1994         return -ENOENT;
1995 }
1996
1997 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1998         size_t t;
1999         uint64_t l;
2000
2001         l = le64toh(o->object.size) - offsetof(Object, data.payload);
2002         t = (size_t) l;
2003
2004         /* We can't read objects larger than 4G on a 32bit machine */
2005         if ((uint64_t) t != l)
2006                 return -E2BIG;
2007
2008         if (o->object.flags & OBJECT_COMPRESSED) {
2009 #ifdef HAVE_XZ
2010                 uint64_t rsize;
2011
2012                 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
2013                         return -EBADMSG;
2014
2015                 *data = f->compress_buffer;
2016                 *size = (size_t) rsize;
2017 #else
2018                 return -EPROTONOSUPPORT;
2019 #endif
2020         } else {
2021                 *data = o->data.payload;
2022                 *size = t;
2023         }
2024
2025         return 0;
2026 }
2027
2028 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
2029         JournalFile *f;
2030         uint64_t p, n;
2031         le64_t le_hash;
2032         int r;
2033         Object *o;
2034
2035         if (!j)
2036                 return -EINVAL;
2037         if (!data)
2038                 return -EINVAL;
2039         if (!size)
2040                 return -EINVAL;
2041
2042         f = j->current_file;
2043         if (!f)
2044                 return -EADDRNOTAVAIL;
2045
2046         if (f->current_offset <= 0)
2047                 return -EADDRNOTAVAIL;
2048
2049         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2050         if (r < 0)
2051                 return r;
2052
2053         n = journal_file_entry_n_items(o);
2054         if (j->current_field >= n)
2055                 return 0;
2056
2057         p = le64toh(o->entry.items[j->current_field].object_offset);
2058         le_hash = o->entry.items[j->current_field].hash;
2059         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2060         if (r < 0)
2061                 return r;
2062
2063         if (le_hash != o->data.hash)
2064                 return -EBADMSG;
2065
2066         r = return_data(j, f, o, data, size);
2067         if (r < 0)
2068                 return r;
2069
2070         j->current_field ++;
2071
2072         return 1;
2073 }
2074
2075 _public_ void sd_journal_restart_data(sd_journal *j) {
2076         if (!j)
2077                 return;
2078
2079         j->current_field = 0;
2080 }
2081
2082 _public_ int sd_journal_get_fd(sd_journal *j) {
2083         int r;
2084
2085         if (!j)
2086                 return -EINVAL;
2087
2088         if (j->inotify_fd >= 0)
2089                 return j->inotify_fd;
2090
2091         r = allocate_inotify(j);
2092         if (r < 0)
2093                 return r;
2094
2095         /* Iterate through all dirs again, to add them to the
2096          * inotify */
2097         if (j->no_new_files)
2098                 r = add_current_paths(j);
2099         else if (j->path)
2100                 r = add_root_directory(j, j->path);
2101         else
2102                 r = add_search_paths(j);
2103         if (r < 0)
2104                 return r;
2105
2106         return j->inotify_fd;
2107 }
2108
2109 _public_ int sd_journal_get_events(sd_journal *j) {
2110         int fd;
2111
2112         if (!j)
2113                 return -EINVAL;
2114
2115         fd = sd_journal_get_fd(j);
2116         if (fd < 0)
2117                 return fd;
2118
2119         return POLLIN;
2120 }
2121
2122 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2123         int fd;
2124
2125         if (!j)
2126                 return -EINVAL;
2127         if (!timeout_usec)
2128                 return -EINVAL;
2129
2130         fd = sd_journal_get_fd(j);
2131         if (fd < 0)
2132                 return fd;
2133
2134         if (!j->on_network) {
2135                 *timeout_usec = (uint64_t) -1;
2136                 return 0;
2137         }
2138
2139         /* If we are on the network we need to regularly check for
2140          * changes manually */
2141
2142         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2143         return 1;
2144 }
2145
2146 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2147         Directory *d;
2148         int r;
2149
2150         assert(j);
2151         assert(e);
2152
2153         /* Is this a subdirectory we watch? */
2154         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2155         if (d) {
2156                 sd_id128_t id;
2157
2158                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2159                     (endswith(e->name, ".journal") ||
2160                      endswith(e->name, ".journal~"))) {
2161
2162                         /* Event for a journal file */
2163
2164                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2165                                 r = add_file(j, d->path, e->name);
2166                                 if (r < 0) {
2167                                         log_debug("Failed to add file %s/%s: %s",
2168                                                   d->path, e->name, strerror(-r));
2169                                         set_put_error(j, r);
2170                                 }
2171
2172                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2173
2174                                 r = remove_file(j, d->path, e->name);
2175                                 if (r < 0)
2176                                         log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2177                         }
2178
2179                 } else if (!d->is_root && e->len == 0) {
2180
2181                         /* Event for a subdirectory */
2182
2183                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2184                                 r = remove_directory(j, d);
2185                                 if (r < 0)
2186                                         log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2187                         }
2188
2189
2190                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2191
2192                         /* Event for root directory */
2193
2194                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2195                                 r = add_directory(j, d->path, e->name);
2196                                 if (r < 0)
2197                                         log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2198                         }
2199                 }
2200
2201                 return;
2202         }
2203
2204         if (e->mask & IN_IGNORED)
2205                 return;
2206
2207         log_warning("Unknown inotify event.");
2208 }
2209
2210 static int determine_change(sd_journal *j) {
2211         bool b;
2212
2213         assert(j);
2214
2215         b = j->current_invalidate_counter != j->last_invalidate_counter;
2216         j->last_invalidate_counter = j->current_invalidate_counter;
2217
2218         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2219 }
2220
2221 _public_ int sd_journal_process(sd_journal *j) {
2222         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2223         bool got_something = false;
2224
2225         if (!j)
2226                 return -EINVAL;
2227
2228         j->last_process_usec = now(CLOCK_MONOTONIC);
2229
2230         for (;;) {
2231                 struct inotify_event *e;
2232                 ssize_t l;
2233
2234                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2235                 if (l < 0) {
2236                         if (errno == EAGAIN || errno == EINTR)
2237                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2238
2239                         return -errno;
2240                 }
2241
2242                 got_something = true;
2243
2244                 e = (struct inotify_event*) buffer;
2245                 while (l > 0) {
2246                         size_t step;
2247
2248                         process_inotify_event(j, e);
2249
2250                         step = sizeof(struct inotify_event) + e->len;
2251                         assert(step <= (size_t) l);
2252
2253                         e = (struct inotify_event*) ((uint8_t*) e + step);
2254                         l -= step;
2255                 }
2256         }
2257
2258         return determine_change(j);
2259 }
2260
2261 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2262         int r;
2263         uint64_t t;
2264
2265         assert(j);
2266
2267         if (j->inotify_fd < 0) {
2268
2269                 /* This is the first invocation, hence create the
2270                  * inotify watch */
2271                 r = sd_journal_get_fd(j);
2272                 if (r < 0)
2273                         return r;
2274
2275                 /* The journal might have changed since the context
2276                  * object was created and we weren't watching before,
2277                  * hence don't wait for anything, and return
2278                  * immediately. */
2279                 return determine_change(j);
2280         }
2281
2282         r = sd_journal_get_timeout(j, &t);
2283         if (r < 0)
2284                 return r;
2285
2286         if (t != (uint64_t) -1) {
2287                 usec_t n;
2288
2289                 n = now(CLOCK_MONOTONIC);
2290                 t = t > n ? t - n : 0;
2291
2292                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2293                         timeout_usec = t;
2294         }
2295
2296         do {
2297                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2298         } while (r == -EINTR);
2299
2300         if (r < 0)
2301                 return r;
2302
2303         return sd_journal_process(j);
2304 }
2305
2306 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2307         Iterator i;
2308         JournalFile *f;
2309         bool first = true;
2310         int r;
2311
2312         if (!j)
2313                 return -EINVAL;
2314         if (!from && !to)
2315                 return -EINVAL;
2316         if (from == to)
2317                 return -EINVAL;
2318
2319         HASHMAP_FOREACH(f, j->files, i) {
2320                 usec_t fr, t;
2321
2322                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2323                 if (r == -ENOENT)
2324                         continue;
2325                 if (r < 0)
2326                         return r;
2327                 if (r == 0)
2328                         continue;
2329
2330                 if (first) {
2331                         if (from)
2332                                 *from = fr;
2333                         if (to)
2334                                 *to = t;
2335                         first = false;
2336                 } else {
2337                         if (from)
2338                                 *from = MIN(fr, *from);
2339                         if (to)
2340                                 *to = MAX(t, *to);
2341                 }
2342         }
2343
2344         return first ? 0 : 1;
2345 }
2346
2347 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2348         Iterator i;
2349         JournalFile *f;
2350         bool first = true;
2351         int r;
2352
2353         if (!j)
2354                 return -EINVAL;
2355         if (!from && !to)
2356                 return -EINVAL;
2357         if (from == to)
2358                 return -EINVAL;
2359
2360         HASHMAP_FOREACH(f, j->files, i) {
2361                 usec_t fr, t;
2362
2363                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2364                 if (r == -ENOENT)
2365                         continue;
2366                 if (r < 0)
2367                         return r;
2368                 if (r == 0)
2369                         continue;
2370
2371                 if (first) {
2372                         if (from)
2373                                 *from = fr;
2374                         if (to)
2375                                 *to = t;
2376                         first = false;
2377                 } else {
2378                         if (from)
2379                                 *from = MIN(fr, *from);
2380                         if (to)
2381                                 *to = MAX(t, *to);
2382                 }
2383         }
2384
2385         return first ? 0 : 1;
2386 }
2387
2388 void journal_print_header(sd_journal *j) {
2389         Iterator i;
2390         JournalFile *f;
2391         bool newline = false;
2392
2393         assert(j);
2394
2395         HASHMAP_FOREACH(f, j->files, i) {
2396                 if (newline)
2397                         putchar('\n');
2398                 else
2399                         newline = true;
2400
2401                 journal_file_print_header(f);
2402         }
2403 }
2404
2405 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2406         Iterator i;
2407         JournalFile *f;
2408         uint64_t sum = 0;
2409
2410         if (!j)
2411                 return -EINVAL;
2412         if (!bytes)
2413                 return -EINVAL;
2414
2415         HASHMAP_FOREACH(f, j->files, i) {
2416                 struct stat st;
2417
2418                 if (fstat(f->fd, &st) < 0)
2419                         return -errno;
2420
2421                 sum += (uint64_t) st.st_blocks * 512ULL;
2422         }
2423
2424         *bytes = sum;
2425         return 0;
2426 }
2427
2428 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2429         char *f;
2430
2431         if (!j)
2432                 return -EINVAL;
2433         if (isempty(field))
2434                 return -EINVAL;
2435         if (!field_is_valid(field))
2436                 return -EINVAL;
2437
2438         f = strdup(field);
2439         if (!f)
2440                 return -ENOMEM;
2441
2442         free(j->unique_field);
2443         j->unique_field = f;
2444         j->unique_file = NULL;
2445         j->unique_offset = 0;
2446
2447         return 0;
2448 }
2449
2450 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2451         Object *o;
2452         size_t k;
2453         int r;
2454
2455         if (!j)
2456                 return -EINVAL;
2457         if (!data)
2458                 return -EINVAL;
2459         if (!l)
2460                 return -EINVAL;
2461         if (!j->unique_field)
2462                 return -EINVAL;
2463
2464         k = strlen(j->unique_field);
2465
2466         if (!j->unique_file) {
2467                 j->unique_file = hashmap_first(j->files);
2468                 if (!j->unique_file)
2469                         return 0;
2470                 j->unique_offset = 0;
2471         }
2472
2473         for (;;) {
2474                 JournalFile *of;
2475                 Iterator i;
2476                 const void *odata;
2477                 size_t ol;
2478                 bool found;
2479
2480                 /* Proceed to next data object in the field's linked list */
2481                 if (j->unique_offset == 0) {
2482                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2483                         if (r < 0)
2484                                 return r;
2485
2486                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2487                 } else {
2488                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2489                         if (r < 0)
2490                                 return r;
2491
2492                         j->unique_offset = le64toh(o->data.next_field_offset);
2493                 }
2494
2495                 /* We reached the end of the list? Then start again, with the next file */
2496                 if (j->unique_offset == 0) {
2497                         JournalFile *n;
2498
2499                         n = hashmap_next(j->files, j->unique_file->path);
2500                         if (!n)
2501                                 return 0;
2502
2503                         j->unique_file = n;
2504                         continue;
2505                 }
2506
2507                 /* We do not use the type context here, but 0 instead,
2508                  * so that we can look at this data object at the same
2509                  * time as one on another file */
2510                 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2511                 if (r < 0)
2512                         return r;
2513
2514                 /* Let's do the type check by hand, since we used 0 context above. */
2515                 if (o->object.type != OBJECT_DATA)
2516                         return -EBADMSG;
2517
2518                 r = return_data(j, j->unique_file, o, &odata, &ol);
2519                 if (r < 0)
2520                         return r;
2521
2522                 /* OK, now let's see if we already returned this data
2523                  * object by checking if it exists in the earlier
2524                  * traversed files. */
2525                 found = false;
2526                 HASHMAP_FOREACH(of, j->files, i) {
2527                         Object *oo;
2528                         uint64_t op;
2529
2530                         if (of == j->unique_file)
2531                                 break;
2532
2533                         /* Skip this file it didn't have any fields
2534                          * indexed */
2535                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2536                             le64toh(of->header->n_fields) <= 0)
2537                                 continue;
2538
2539                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2540                         if (r < 0)
2541                                 return r;
2542
2543                         if (r > 0)
2544                                 found = true;
2545                 }
2546
2547                 if (found)
2548                         continue;
2549
2550                 r = return_data(j, j->unique_file, o, data, l);
2551                 if (r < 0)
2552                         return r;
2553
2554                 return 1;
2555         }
2556 }
2557
2558 _public_ void sd_journal_restart_unique(sd_journal *j) {
2559         if (!j)
2560                 return;
2561
2562         j->unique_file = NULL;
2563         j->unique_offset = 0;
2564 }
2565
2566 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2567         if (!j)
2568                 return -EINVAL;
2569
2570         return !j->on_network;
2571 }
2572
2573 static char *lookup_field(const char *field, void *userdata) {
2574         sd_journal *j = userdata;
2575         const void *data;
2576         size_t size, d;
2577         int r;
2578
2579         assert(field);
2580         assert(j);
2581
2582         r = sd_journal_get_data(j, field, &data, &size);
2583         if (r < 0 ||
2584             size > REPLACE_VAR_MAX)
2585                 return strdup(field);
2586
2587         d = strlen(field) + 1;
2588
2589         return strndup((const char*) data + d, size - d);
2590 }
2591
2592 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2593         const void *data;
2594         size_t size;
2595         sd_id128_t id;
2596         _cleanup_free_ char *text = NULL, *cid = NULL;
2597         char *t;
2598         int r;
2599
2600         if (!j)
2601                 return -EINVAL;
2602         if (!ret)
2603                 return -EINVAL;
2604
2605         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2606         if (r < 0)
2607                 return r;
2608
2609         cid = strndup((const char*) data + 11, size - 11);
2610         if (!cid)
2611                 return -ENOMEM;
2612
2613         r = sd_id128_from_string(cid, &id);
2614         if (r < 0)
2615                 return r;
2616
2617         r = catalog_get(CATALOG_DATABASE, id, &text);
2618         if (r < 0)
2619                 return r;
2620
2621         t = replace_var(text, lookup_field, j);
2622         if (!t)
2623                 return -ENOMEM;
2624
2625         *ret = t;
2626         return 0;
2627 }
2628
2629 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2630         if (!ret)
2631                 return -EINVAL;
2632
2633         return catalog_get(CATALOG_DATABASE, id, ret);
2634 }
2635
2636 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2637         if (!j)
2638                 return -EINVAL;
2639
2640         j->data_threshold = sz;
2641         return 0;
2642 }
2643
2644 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2645         if (!j)
2646                 return -EINVAL;
2647         if (!sz)
2648                 return -EINVAL;
2649
2650         *sz = j->data_threshold;
2651         return 0;
2652 }