src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "path-util.h"
  37 #include "lookup3.h"
  38 #include "compress.h"
  39 #include "journal-internal.h"
  40 #include "missing.h"
  41 #include "catalog.h"
  42 #include "replace-var.h"
  43
  44 #define JOURNAL_FILES_MAX 1024
  45
  46 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  47
  48 #define REPLACE_VAR_MAX 256
  49
  50 #define DEFAULT_DATA_THRESHOLD (64*1024)
  51
  52 /* We return an error here only if we didn't manage to
  53    memorize the real error. */
  54 static int set_put_error(sd_journal *j, int r) {
  55         int k;
  56
  57         if (r >= 0)
  58                 return r;
  59
  60         k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
  61         if (k < 0)
  62                 return k;
  63
  64         return set_put(j->errors, INT_TO_PTR(r));
  65 }
  66
  67 static void detach_location(sd_journal *j) {
  68         Iterator i;
  69         JournalFile *f;
  70
  71         assert(j);
  72
  73         j->current_file = NULL;
  74         j->current_field = 0;
  75
  76         HASHMAP_FOREACH(f, j->files, i)
  77                 f->current_offset = 0;
  78 }
  79
  80 static void reset_location(sd_journal *j) {
  81         assert(j);
  82
  83         detach_location(j);
  84         zero(j->current_location);
  85 }
  86
  87 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  88         assert(l);
  89         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
  90         assert(f);
  91         assert(o->object.type == OBJECT_ENTRY);
  92
  93         l->type = type;
  94         l->seqnum = le64toh(o->entry.seqnum);
  95         l->seqnum_id = f->header->seqnum_id;
  96         l->realtime = le64toh(o->entry.realtime);
  97         l->monotonic = le64toh(o->entry.monotonic);
  98         l->boot_id = o->entry.boot_id;
  99         l->xor_hash = le64toh(o->entry.xor_hash);
 100
 101         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 102 }
 103
 104 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o, uint64_t offset) {
 105         assert(j);
 106         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 107         assert(f);
 108         assert(o);
 109
 110         init_location(&j->current_location, type, f, o);
 111
 112         if (j->current_file)
 113                 j->current_file->current_offset = 0;
 114
 115         j->current_file = f;
 116         j->current_field = 0;
 117
 118         f->current_offset = offset;
 119 }
 120
 121 static int match_is_valid(const void *data, size_t size) {
 122         const char *b, *p;
 123
 124         assert(data);
 125
 126         if (size < 2)
 127                 return false;
 128
 129         if (startswith(data, "__"))
 130                 return false;
 131
 132         b = data;
 133         for (p = b; p < b + size; p++) {
 134
 135                 if (*p == '=')
 136                         return p > b;
 137
 138                 if (*p == '_')
 139                         continue;
 140
 141                 if (*p >= 'A' && *p <= 'Z')
 142                         continue;
 143
 144                 if (*p >= '0' && *p <= '9')
 145                         continue;
 146
 147                 return false;
 148         }
 149
 150         return false;
 151 }
 152
 153 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 154         const uint8_t *a = _a, *b = _b;
 155         size_t j;
 156
 157         for (j = 0; j < s && j < t; j++) {
 158
 159                 if (a[j] != b[j])
 160                         return false;
 161
 162                 if (a[j] == '=')
 163                         return true;
 164         }
 165
 166         return true;
 167 }
 168
 169 static Match *match_new(Match *p, MatchType t) {
 170         Match *m;
 171
 172         m = new0(Match, 1);
 173         if (!m)
 174                 return NULL;
 175
 176         m->type = t;
 177
 178         if (p) {
 179                 m->parent = p;
 180                 LIST_PREPEND(Match, matches, p->matches, m);
 181         }
 182
 183         return m;
 184 }
 185
 186 static void match_free(Match *m) {
 187         assert(m);
 188
 189         while (m->matches)
 190                 match_free(m->matches);
 191
 192         if (m->parent)
 193                 LIST_REMOVE(Match, matches, m->parent->matches, m);
 194
 195         free(m->data);
 196         free(m);
 197 }
 198
 199 static void match_free_if_empty(Match *m) {
 200         if (!m || m->matches)
 201                 return;
 202
 203         match_free(m);
 204 }
 205
 206 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 207         Match *l3, *l4, *add_here = NULL, *m;
 208         le64_t le_hash;
 209
 210         if (!j)
 211                 return -EINVAL;
 212
 213         if (!data)
 214                 return -EINVAL;
 215
 216         if (size == 0)
 217                 size = strlen(data);
 218
 219         if (!match_is_valid(data, size))
 220                 return -EINVAL;
 221
 222         /* level 0: AND term
 223          * level 1: OR terms
 224          * level 2: AND terms
 225          * level 3: OR terms
 226          * level 4: concrete matches */
 227
 228         if (!j->level0) {
 229                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 230                 if (!j->level0)
 231                         return -ENOMEM;
 232         }
 233
 234         if (!j->level1) {
 235                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 236                 if (!j->level1)
 237                         return -ENOMEM;
 238         }
 239
 240         if (!j->level2) {
 241                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 242                 if (!j->level2)
 243                         return -ENOMEM;
 244         }
 245
 246         assert(j->level0->type == MATCH_AND_TERM);
 247         assert(j->level1->type == MATCH_OR_TERM);
 248         assert(j->level2->type == MATCH_AND_TERM);
 249
 250         le_hash = htole64(hash64(data, size));
 251
 252         LIST_FOREACH(matches, l3, j->level2->matches) {
 253                 assert(l3->type == MATCH_OR_TERM);
 254
 255                 LIST_FOREACH(matches, l4, l3->matches) {
 256                         assert(l4->type == MATCH_DISCRETE);
 257
 258                         /* Exactly the same match already? Then ignore
 259                          * this addition */
 260                         if (l4->le_hash == le_hash &&
 261                             l4->size == size &&
 262                             memcmp(l4->data, data, size) == 0)
 263                                 return 0;
 264
 265                         /* Same field? Then let's add this to this OR term */
 266                         if (same_field(data, size, l4->data, l4->size)) {
 267                                 add_here = l3;
 268                                 break;
 269                         }
 270                 }
 271
 272                 if (add_here)
 273                         break;
 274         }
 275
 276         if (!add_here) {
 277                 add_here = match_new(j->level2, MATCH_OR_TERM);
 278                 if (!add_here)
 279                         goto fail;
 280         }
 281
 282         m = match_new(add_here, MATCH_DISCRETE);
 283         if (!m)
 284                 goto fail;
 285
 286         m->le_hash = le_hash;
 287         m->size = size;
 288         m->data = memdup(data, size);
 289         if (!m->data)
 290                 goto fail;
 291
 292         detach_location(j);
 293
 294         return 0;
 295
 296 fail:
 297         match_free_if_empty(add_here);
 298         match_free_if_empty(j->level2);
 299         match_free_if_empty(j->level1);
 300         match_free_if_empty(j->level0);
 301
 302         return -ENOMEM;
 303 }
 304
 305 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 306         assert(j);
 307
 308         if (!j->level0)
 309                 return 0;
 310
 311         if (!j->level1)
 312                 return 0;
 313
 314         if (!j->level1->matches)
 315                 return 0;
 316
 317         j->level1 = NULL;
 318         j->level2 = NULL;
 319
 320         return 0;
 321 }
 322
 323 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 324         assert(j);
 325
 326         if (!j->level0)
 327                 return 0;
 328
 329         if (!j->level1)
 330                 return 0;
 331
 332         if (!j->level2)
 333                 return 0;
 334
 335         if (!j->level2->matches)
 336                 return 0;
 337
 338         j->level2 = NULL;
 339         return 0;
 340 }
 341
 342 static char *match_make_string(Match *m) {
 343         char *p, *r;
 344         Match *i;
 345         bool enclose = false;
 346
 347         if (!m)
 348                 return strdup("");
 349
 350         if (m->type == MATCH_DISCRETE)
 351                 return strndup(m->data, m->size);
 352
 353         p = NULL;
 354         LIST_FOREACH(matches, i, m->matches) {
 355                 char *t, *k;
 356
 357                 t = match_make_string(i);
 358                 if (!t) {
 359                         free(p);
 360                         return NULL;
 361                 }
 362
 363                 if (p) {
 364                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 365                         free(p);
 366                         free(t);
 367
 368                         if (!k)
 369                                 return NULL;
 370
 371                         p = k;
 372
 373                         enclose = true;
 374                 } else {
 375                         free(p);
 376                         p = t;
 377                 }
 378         }
 379
 380         if (enclose) {
 381                 r = strjoin("(", p, ")", NULL);
 382                 free(p);
 383                 return r;
 384         }
 385
 386         return p;
 387 }
 388
 389 char *journal_make_match_string(sd_journal *j) {
 390         assert(j);
 391
 392         return match_make_string(j->level0);
 393 }
 394
 395 _public_ void sd_journal_flush_matches(sd_journal *j) {
 396
 397         if (!j)
 398                 return;
 399
 400         if (j->level0)
 401                 match_free(j->level0);
 402
 403         j->level0 = j->level1 = j->level2 = NULL;
 404
 405         detach_location(j);
 406 }
 407
 408 static int compare_entry_order(JournalFile *af, Object *_ao,
 409                                JournalFile *bf, uint64_t bp) {
 410
 411         uint64_t a, b;
 412         Object *ao, *bo;
 413         int r;
 414
 415         assert(af);
 416         assert(bf);
 417         assert(_ao);
 418
 419         /* The mmap cache might invalidate the object from the first
 420          * file if we look at the one from the second file. Hence
 421          * temporarily copy the header of the first one, and look at
 422          * that only. */
 423         ao = alloca(offsetof(EntryObject, items));
 424         memcpy(ao, _ao, offsetof(EntryObject, items));
 425
 426         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 427         if (r < 0)
 428                 return strcmp(af->path, bf->path);
 429
 430         /* We operate on two different files here, hence we can access
 431          * two objects at the same time, which we normally can't.
 432          *
 433          * If contents and timestamps match, these entries are
 434          * identical, even if the seqnum does not match */
 435
 436         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 437             ao->entry.monotonic == bo->entry.monotonic &&
 438             ao->entry.realtime == bo->entry.realtime &&
 439             ao->entry.xor_hash == bo->entry.xor_hash)
 440                 return 0;
 441
 442         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 443
 444                 /* If this is from the same seqnum source, compare
 445                  * seqnums */
 446                 a = le64toh(ao->entry.seqnum);
 447                 b = le64toh(bo->entry.seqnum);
 448
 449                 if (a < b)
 450                         return -1;
 451                 if (a > b)
 452                         return 1;
 453
 454                 /* Wow! This is weird, different data but the same
 455                  * seqnums? Something is borked, but let's make the
 456                  * best of it and compare by time. */
 457         }
 458
 459         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 460
 461                 /* If the boot id matches compare monotonic time */
 462                 a = le64toh(ao->entry.monotonic);
 463                 b = le64toh(bo->entry.monotonic);
 464
 465                 if (a < b)
 466                         return -1;
 467                 if (a > b)
 468                         return 1;
 469         }
 470
 471         /* Otherwise compare UTC time */
 472         a = le64toh(ao->entry.realtime);
 473         b = le64toh(bo->entry.realtime);
 474
 475         if (a < b)
 476                 return -1;
 477         if (a > b)
 478                 return 1;
 479
 480         /* Finally, compare by contents */
 481         a = le64toh(ao->entry.xor_hash);
 482         b = le64toh(bo->entry.xor_hash);
 483
 484         if (a < b)
 485                 return -1;
 486         if (a > b)
 487                 return 1;
 488
 489         return 0;
 490 }
 491
 492 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 493         uint64_t a;
 494
 495         assert(af);
 496         assert(ao);
 497         assert(l);
 498         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 499
 500         if (l->monotonic_set &&
 501             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 502             l->realtime_set &&
 503             le64toh(ao->entry.realtime) == l->realtime &&
 504             l->xor_hash_set &&
 505             le64toh(ao->entry.xor_hash) == l->xor_hash)
 506                 return 0;
 507
 508         if (l->seqnum_set &&
 509             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 510
 511                 a = le64toh(ao->entry.seqnum);
 512
 513                 if (a < l->seqnum)
 514                         return -1;
 515                 if (a > l->seqnum)
 516                         return 1;
 517         }
 518
 519         if (l->monotonic_set &&
 520             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 521
 522                 a = le64toh(ao->entry.monotonic);
 523
 524                 if (a < l->monotonic)
 525                         return -1;
 526                 if (a > l->monotonic)
 527                         return 1;
 528         }
 529
 530         if (l->realtime_set) {
 531
 532                 a = le64toh(ao->entry.realtime);
 533
 534                 if (a < l->realtime)
 535                         return -1;
 536                 if (a > l->realtime)
 537                         return 1;
 538         }
 539
 540         if (l->xor_hash_set) {
 541                 a = le64toh(ao->entry.xor_hash);
 542
 543                 if (a < l->xor_hash)
 544                         return -1;
 545                 if (a > l->xor_hash)
 546                         return 1;
 547         }
 548
 549         return 0;
 550 }
 551
 552 static int next_for_match(
 553                 sd_journal *j,
 554                 Match *m,
 555                 JournalFile *f,
 556                 uint64_t after_offset,
 557                 direction_t direction,
 558                 Object **ret,
 559                 uint64_t *offset) {
 560
 561         int r;
 562         uint64_t np = 0;
 563         Object *n;
 564
 565         assert(j);
 566         assert(m);
 567         assert(f);
 568
 569         if (m->type == MATCH_DISCRETE) {
 570                 uint64_t dp;
 571
 572                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 573                 if (r <= 0)
 574                         return r;
 575
 576                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 577
 578         } else if (m->type == MATCH_OR_TERM) {
 579                 Match *i;
 580
 581                 /* Find the earliest match beyond after_offset */
 582
 583                 LIST_FOREACH(matches, i, m->matches) {
 584                         uint64_t cp;
 585
 586                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 587                         if (r < 0)
 588                                 return r;
 589                         else if (r > 0) {
 590                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 591                                         np = cp;
 592                         }
 593                 }
 594
 595         } else if (m->type == MATCH_AND_TERM) {
 596                 Match *i;
 597                 bool continue_looking;
 598
 599                 /* Always jump to the next matching entry and repeat
 600                  * this until we fine and offset that matches for all
 601                  * matches. */
 602
 603                 if (!m->matches)
 604                         return 0;
 605
 606                 np = 0;
 607                 do {
 608                         continue_looking = false;
 609
 610                         LIST_FOREACH(matches, i, m->matches) {
 611                                 uint64_t cp, limit;
 612
 613                                 if (np == 0)
 614                                         limit = after_offset;
 615                                 else if (direction == DIRECTION_DOWN)
 616                                         limit = MAX(np, after_offset);
 617                                 else
 618                                         limit = MIN(np, after_offset);
 619
 620                                 r = next_for_match(j, i, f, limit, direction, NULL, &cp);
 621                                 if (r <= 0)
 622                                         return r;
 623
 624                                 if ((direction == DIRECTION_DOWN ? cp >= after_offset : cp <= after_offset) &&
 625                                     (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))) {
 626                                         np = cp;
 627                                         continue_looking = true;
 628                                 }
 629                         }
 630
 631                 } while (continue_looking);
 632         }
 633
 634         if (np == 0)
 635                 return 0;
 636
 637         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 638         if (r < 0)
 639                 return r;
 640
 641         if (ret)
 642                 *ret = n;
 643         if (offset)
 644                 *offset = np;
 645
 646         return 1;
 647 }
 648
 649 static int find_location_for_match(
 650                 sd_journal *j,
 651                 Match *m,
 652                 JournalFile *f,
 653                 direction_t direction,
 654                 Object **ret,
 655                 uint64_t *offset) {
 656
 657         int r;
 658
 659         assert(j);
 660         assert(m);
 661         assert(f);
 662
 663         if (m->type == MATCH_DISCRETE) {
 664                 uint64_t dp;
 665
 666                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 667                 if (r <= 0)
 668                         return r;
 669
 670                 /* FIXME: missing: find by monotonic */
 671
 672                 if (j->current_location.type == LOCATION_HEAD)
 673                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 674                 if (j->current_location.type == LOCATION_TAIL)
 675                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 676                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 677                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 678                 if (j->current_location.monotonic_set) {
 679                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 680                         if (r != -ENOENT)
 681                                 return r;
 682                 }
 683                 if (j->current_location.realtime_set)
 684                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 685
 686                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 687
 688         } else if (m->type == MATCH_OR_TERM) {
 689                 uint64_t np = 0;
 690                 Object *n;
 691                 Match *i;
 692
 693                 /* Find the earliest match */
 694
 695                 LIST_FOREACH(matches, i, m->matches) {
 696                         uint64_t cp;
 697
 698                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 699                         if (r < 0)
 700                                 return r;
 701                         else if (r > 0) {
 702                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 703                                         np = cp;
 704                         }
 705                 }
 706
 707                 if (np == 0)
 708                         return 0;
 709
 710                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 711                 if (r < 0)
 712                         return r;
 713
 714                 if (ret)
 715                         *ret = n;
 716                 if (offset)
 717                         *offset = np;
 718
 719                 return 1;
 720
 721         } else {
 722                 Match *i;
 723                 uint64_t np = 0;
 724
 725                 assert(m->type == MATCH_AND_TERM);
 726
 727                 /* First jump to the last match, and then find the
 728                  * next one where all matches match */
 729
 730                 if (!m->matches)
 731                         return 0;
 732
 733                 LIST_FOREACH(matches, i, m->matches) {
 734                         uint64_t cp;
 735
 736                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 737                         if (r <= 0)
 738                                 return r;
 739
 740                         if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
 741                                 np = cp;
 742                 }
 743
 744                 return next_for_match(j, m, f, np, direction, ret, offset);
 745         }
 746 }
 747
 748 static int find_location_with_matches(
 749                 sd_journal *j,
 750                 JournalFile *f,
 751                 direction_t direction,
 752                 Object **ret,
 753                 uint64_t *offset) {
 754
 755         int r;
 756
 757         assert(j);
 758         assert(f);
 759         assert(ret);
 760         assert(offset);
 761
 762         if (!j->level0) {
 763                 /* No matches is simple */
 764
 765                 if (j->current_location.type == LOCATION_HEAD)
 766                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 767                 if (j->current_location.type == LOCATION_TAIL)
 768                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 769                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 770                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 771                 if (j->current_location.monotonic_set) {
 772                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 773                         if (r != -ENOENT)
 774                                 return r;
 775                 }
 776                 if (j->current_location.realtime_set)
 777                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 778
 779                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 780         } else
 781                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 782 }
 783
 784 static int next_with_matches(
 785                 sd_journal *j,
 786                 JournalFile *f,
 787                 direction_t direction,
 788                 Object **ret,
 789                 uint64_t *offset) {
 790
 791         Object *c;
 792         uint64_t cp;
 793
 794         assert(j);
 795         assert(f);
 796         assert(ret);
 797         assert(offset);
 798
 799         c = *ret;
 800         cp = *offset;
 801
 802         /* No matches is easy. We simple advance the file
 803          * pointer by one. */
 804         if (!j->level0)
 805                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 806
 807         /* If we have a match then we look for the next matching entry
 808          * with an offset at least one step larger */
 809         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 810 }
 811
 812 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 813         Object *c;
 814         uint64_t cp;
 815         int r;
 816
 817         assert(j);
 818         assert(f);
 819
 820         if (f->current_offset > 0) {
 821                 cp = f->current_offset;
 822
 823                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 824                 if (r < 0)
 825                         return r;
 826
 827                 r = next_with_matches(j, f, direction, &c, &cp);
 828                 if (r <= 0)
 829                         return r;
 830         } else {
 831                 r = find_location_with_matches(j, f, direction, &c, &cp);
 832                 if (r <= 0)
 833                         return r;
 834         }
 835
 836         /* OK, we found the spot, now let's advance until to an entry
 837          * that is actually different from what we were previously
 838          * looking at. This is necessary to handle entries which exist
 839          * in two (or more) journal files, and which shall all be
 840          * suppressed but one. */
 841
 842         for (;;) {
 843                 bool found;
 844
 845                 if (j->current_location.type == LOCATION_DISCRETE) {
 846                         int k;
 847
 848                         k = compare_with_location(f, c, &j->current_location);
 849                         if (direction == DIRECTION_DOWN)
 850                                 found = k > 0;
 851                         else
 852                                 found = k < 0;
 853                 } else
 854                         found = true;
 855
 856                 if (found) {
 857                         if (ret)
 858                                 *ret = c;
 859                         if (offset)
 860                                 *offset = cp;
 861                         return 1;
 862                 }
 863
 864                 r = next_with_matches(j, f, direction, &c, &cp);
 865                 if (r <= 0)
 866                         return r;
 867         }
 868 }
 869
 870 static int real_journal_next(sd_journal *j, direction_t direction) {
 871         JournalFile *f, *new_file = NULL;
 872         uint64_t new_offset = 0;
 873         Object *o;
 874         uint64_t p;
 875         Iterator i;
 876         int r;
 877
 878         if (!j)
 879                 return -EINVAL;
 880
 881         HASHMAP_FOREACH(f, j->files, i) {
 882                 bool found;
 883
 884                 r = next_beyond_location(j, f, direction, &o, &p);
 885                 if (r < 0) {
 886                         log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
 887                         continue;
 888                 } else if (r == 0)
 889                         continue;
 890
 891                 if (!new_file)
 892                         found = true;
 893                 else {
 894                         int k;
 895
 896                         k = compare_entry_order(f, o, new_file, new_offset);
 897
 898                         if (direction == DIRECTION_DOWN)
 899                                 found = k < 0;
 900                         else
 901                                 found = k > 0;
 902                 }
 903
 904                 if (found) {
 905                         new_file = f;
 906                         new_offset = p;
 907                 }
 908         }
 909
 910         if (!new_file)
 911                 return 0;
 912
 913         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 914         if (r < 0)
 915                 return r;
 916
 917         set_location(j, LOCATION_DISCRETE, new_file, o, new_offset);
 918
 919         return 1;
 920 }
 921
 922 _public_ int sd_journal_next(sd_journal *j) {
 923         return real_journal_next(j, DIRECTION_DOWN);
 924 }
 925
 926 _public_ int sd_journal_previous(sd_journal *j) {
 927         return real_journal_next(j, DIRECTION_UP);
 928 }
 929
 930 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 931         int c = 0, r;
 932
 933         if (!j)
 934                 return -EINVAL;
 935
 936         if (skip == 0) {
 937                 /* If this is not a discrete skip, then at least
 938                  * resolve the current location */
 939                 if (j->current_location.type != LOCATION_DISCRETE)
 940                         return real_journal_next(j, direction);
 941
 942                 return 0;
 943         }
 944
 945         do {
 946                 r = real_journal_next(j, direction);
 947                 if (r < 0)
 948                         return r;
 949
 950                 if (r == 0)
 951                         return c;
 952
 953                 skip--;
 954                 c++;
 955         } while (skip > 0);
 956
 957         return c;
 958 }
 959
 960 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 961         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 962 }
 963
 964 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 965         return real_journal_next_skip(j, DIRECTION_UP, skip);
 966 }
 967
 968 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 969         Object *o;
 970         int r;
 971         char bid[33], sid[33];
 972
 973         if (!j)
 974                 return -EINVAL;
 975         if (!cursor)
 976                 return -EINVAL;
 977
 978         if (!j->current_file || j->current_file->current_offset <= 0)
 979                 return -EADDRNOTAVAIL;
 980
 981         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 982         if (r < 0)
 983                 return r;
 984
 985         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 986         sd_id128_to_string(o->entry.boot_id, bid);
 987
 988         if (asprintf(cursor,
 989                      "s=%s;i=%llx;b=%s;m=%llx;t=%llx;x=%llx",
 990                      sid, (unsigned long long) le64toh(o->entry.seqnum),
 991                      bid, (unsigned long long) le64toh(o->entry.monotonic),
 992                      (unsigned long long) le64toh(o->entry.realtime),
 993                      (unsigned long long) le64toh(o->entry.xor_hash)) < 0)
 994                 return -ENOMEM;
 995
 996         return 0;
 997 }
 998
 999 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
1000         char *w, *state;
1001         size_t l;
1002         unsigned long long seqnum, monotonic, realtime, xor_hash;
1003         bool
1004                 seqnum_id_set = false,
1005                 seqnum_set = false,
1006                 boot_id_set = false,
1007                 monotonic_set = false,
1008                 realtime_set = false,
1009                 xor_hash_set = false;
1010         sd_id128_t seqnum_id, boot_id;
1011
1012         if (!j)
1013                 return -EINVAL;
1014         if (isempty(cursor))
1015                 return -EINVAL;
1016
1017         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1018                 char *item;
1019                 int k = 0;
1020
1021                 if (l < 2 || w[1] != '=')
1022                         return -EINVAL;
1023
1024                 item = strndup(w, l);
1025                 if (!item)
1026                         return -ENOMEM;
1027
1028                 switch (w[0]) {
1029
1030                 case 's':
1031                         seqnum_id_set = true;
1032                         k = sd_id128_from_string(item+2, &seqnum_id);
1033                         break;
1034
1035                 case 'i':
1036                         seqnum_set = true;
1037                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1038                                 k = -EINVAL;
1039                         break;
1040
1041                 case 'b':
1042                         boot_id_set = true;
1043                         k = sd_id128_from_string(item+2, &boot_id);
1044                         break;
1045
1046                 case 'm':
1047                         monotonic_set = true;
1048                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1049                                 k = -EINVAL;
1050                         break;
1051
1052                 case 't':
1053                         realtime_set = true;
1054                         if (sscanf(item+2, "%llx", &realtime) != 1)
1055                                 k = -EINVAL;
1056                         break;
1057
1058                 case 'x':
1059                         xor_hash_set = true;
1060                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1061                                 k = -EINVAL;
1062                         break;
1063                 }
1064
1065                 free(item);
1066
1067                 if (k < 0)
1068                         return k;
1069         }
1070
1071         if ((!seqnum_set || !seqnum_id_set) &&
1072             (!monotonic_set || !boot_id_set) &&
1073             !realtime_set)
1074                 return -EINVAL;
1075
1076         reset_location(j);
1077
1078         j->current_location.type = LOCATION_SEEK;
1079
1080         if (realtime_set) {
1081                 j->current_location.realtime = (uint64_t) realtime;
1082                 j->current_location.realtime_set = true;
1083         }
1084
1085         if (seqnum_set && seqnum_id_set) {
1086                 j->current_location.seqnum = (uint64_t) seqnum;
1087                 j->current_location.seqnum_id = seqnum_id;
1088                 j->current_location.seqnum_set = true;
1089         }
1090
1091         if (monotonic_set && boot_id_set) {
1092                 j->current_location.monotonic = (uint64_t) monotonic;
1093                 j->current_location.boot_id = boot_id;
1094                 j->current_location.monotonic_set = true;
1095         }
1096
1097         if (xor_hash_set) {
1098                 j->current_location.xor_hash = (uint64_t) xor_hash;
1099                 j->current_location.xor_hash_set = true;
1100         }
1101
1102         return 0;
1103 }
1104
1105 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1106         int r;
1107         char *w, *state;
1108         size_t l;
1109         Object *o;
1110
1111         if (!j)
1112                 return -EINVAL;
1113         if (isempty(cursor))
1114                 return -EINVAL;
1115
1116         if (!j->current_file || j->current_file->current_offset <= 0)
1117                 return -EADDRNOTAVAIL;
1118
1119         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1120         if (r < 0)
1121                 return r;
1122
1123         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1124                 _cleanup_free_ char *item = NULL;
1125                 sd_id128_t id;
1126                 unsigned long long ll;
1127                 int k = 0;
1128
1129                 if (l < 2 || w[1] != '=')
1130                         return -EINVAL;
1131
1132                 item = strndup(w, l);
1133                 if (!item)
1134                         return -ENOMEM;
1135
1136                 switch (w[0]) {
1137
1138                 case 's':
1139                         k = sd_id128_from_string(item+2, &id);
1140                         if (k < 0)
1141                                 return k;
1142                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1143                                 return 0;
1144                         break;
1145
1146                 case 'i':
1147                         if (sscanf(item+2, "%llx", &ll) != 1)
1148                                 return -EINVAL;
1149                         if (ll != le64toh(o->entry.seqnum))
1150                                 return 0;
1151                         break;
1152
1153                 case 'b':
1154                         k = sd_id128_from_string(item+2, &id);
1155                         if (k < 0)
1156                                 return k;
1157                         if (!sd_id128_equal(id, o->entry.boot_id))
1158                                 return 0;
1159                         break;
1160
1161                 case 'm':
1162                         if (sscanf(item+2, "%llx", &ll) != 1)
1163                                 return -EINVAL;
1164                         if (ll != le64toh(o->entry.monotonic))
1165                                 return 0;
1166                         break;
1167
1168                 case 't':
1169                         if (sscanf(item+2, "%llx", &ll) != 1)
1170                                 return -EINVAL;
1171                         if (ll != le64toh(o->entry.realtime))
1172                                 return 0;
1173                         break;
1174
1175                 case 'x':
1176                         if (sscanf(item+2, "%llx", &ll) != 1)
1177                                 return -EINVAL;
1178                         if (ll != le64toh(o->entry.xor_hash))
1179                                 return 0;
1180                         break;
1181                 }
1182         }
1183
1184         return 1;
1185 }
1186
1187
1188 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1189         if (!j)
1190                 return -EINVAL;
1191
1192         reset_location(j);
1193         j->current_location.type = LOCATION_SEEK;
1194         j->current_location.boot_id = boot_id;
1195         j->current_location.monotonic = usec;
1196         j->current_location.monotonic_set = true;
1197
1198         return 0;
1199 }
1200
1201 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1202         if (!j)
1203                 return -EINVAL;
1204
1205         reset_location(j);
1206         j->current_location.type = LOCATION_SEEK;
1207         j->current_location.realtime = usec;
1208         j->current_location.realtime_set = true;
1209
1210         return 0;
1211 }
1212
1213 _public_ int sd_journal_seek_head(sd_journal *j) {
1214         if (!j)
1215                 return -EINVAL;
1216
1217         reset_location(j);
1218         j->current_location.type = LOCATION_HEAD;
1219
1220         return 0;
1221 }
1222
1223 _public_ int sd_journal_seek_tail(sd_journal *j) {
1224         if (!j)
1225                 return -EINVAL;
1226
1227         reset_location(j);
1228         j->current_location.type = LOCATION_TAIL;
1229
1230         return 0;
1231 }
1232
1233 static void check_network(sd_journal *j, int fd) {
1234         struct statfs sfs;
1235
1236         assert(j);
1237
1238         if (j->on_network)
1239                 return;
1240
1241         if (fstatfs(fd, &sfs) < 0)
1242                 return;
1243
1244         j->on_network =
1245                 F_TYPE_CMP(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1246                 F_TYPE_CMP(sfs.f_type, CODA_SUPER_MAGIC) ||
1247                 F_TYPE_CMP(sfs.f_type, NCP_SUPER_MAGIC) ||
1248                 F_TYPE_CMP(sfs.f_type, NFS_SUPER_MAGIC) ||
1249                 F_TYPE_CMP(sfs.f_type, SMB_SUPER_MAGIC);
1250 }
1251
1252 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1253         _cleanup_free_ char *path = NULL;
1254         int r;
1255         JournalFile *f;
1256
1257         assert(j);
1258         assert(prefix);
1259         assert(filename);
1260
1261         if ((j->flags & SD_JOURNAL_SYSTEM_ONLY) &&
1262             !(streq(filename, "system.journal") ||
1263               streq(filename, "system.journal~") ||
1264               (startswith(filename, "system@") &&
1265                (endswith(filename, ".journal") || endswith(filename, ".journal~")))))
1266                 return 0;
1267
1268         path = strjoin(prefix, "/", filename, NULL);
1269         if (!path)
1270                 return -ENOMEM;
1271
1272         if (hashmap_get(j->files, path))
1273                 return 0;
1274
1275         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1276                 log_debug("Too many open journal files, not adding %s, ignoring.", path);
1277                 return set_put_error(j, -ETOOMANYREFS);
1278         }
1279
1280         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1281         if (r < 0) {
1282                 if (errno == ENOENT)
1283                         return 0;
1284
1285                 return r;
1286         }
1287
1288         /* journal_file_dump(f); */
1289
1290         r = hashmap_put(j->files, f->path, f);
1291         if (r < 0) {
1292                 journal_file_close(f);
1293                 return r;
1294         }
1295
1296         log_debug("File %s added.", f->path);
1297
1298         check_network(j, f->fd);
1299
1300         j->current_invalidate_counter ++;
1301
1302         return 0;
1303 }
1304
1305 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1306         char *path;
1307         JournalFile *f;
1308
1309         assert(j);
1310         assert(prefix);
1311         assert(filename);
1312
1313         path = strjoin(prefix, "/", filename, NULL);
1314         if (!path)
1315                 return -ENOMEM;
1316
1317         f = hashmap_get(j->files, path);
1318         free(path);
1319         if (!f)
1320                 return 0;
1321
1322         hashmap_remove(j->files, f->path);
1323
1324         log_debug("File %s removed.", f->path);
1325
1326         if (j->current_file == f) {
1327                 j->current_file = NULL;
1328                 j->current_field = 0;
1329         }
1330
1331         if (j->unique_file == f) {
1332                 j->unique_file = NULL;
1333                 j->unique_offset = 0;
1334         }
1335
1336         journal_file_close(f);
1337
1338         j->current_invalidate_counter ++;
1339
1340         return 0;
1341 }
1342
1343 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1344         _cleanup_free_ char *path = NULL;
1345         int r;
1346         _cleanup_closedir_ DIR *d = NULL;
1347         sd_id128_t id, mid;
1348         Directory *m;
1349
1350         assert(j);
1351         assert(prefix);
1352         assert(dirname);
1353
1354         log_debug("Considering %s/%s.", prefix, dirname);
1355
1356         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1357             (sd_id128_from_string(dirname, &id) < 0 ||
1358              sd_id128_get_machine(&mid) < 0 ||
1359              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1360             return 0;
1361
1362         path = strjoin(prefix, "/", dirname, NULL);
1363         if (!path)
1364                 return -ENOMEM;
1365
1366         d = opendir(path);
1367         if (!d) {
1368                 log_debug("Failed to open %s: %m", path);
1369                 if (errno == ENOENT)
1370                         return 0;
1371                 return -errno;
1372         }
1373
1374         m = hashmap_get(j->directories_by_path, path);
1375         if (!m) {
1376                 m = new0(Directory, 1);
1377                 if (!m)
1378                         return -ENOMEM;
1379
1380                 m->is_root = false;
1381                 m->path = path;
1382
1383                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1384                         free(m);
1385                         return -ENOMEM;
1386                 }
1387
1388                 path = NULL; /* avoid freeing in cleanup */
1389                 j->current_invalidate_counter ++;
1390
1391                 log_debug("Directory %s added.", m->path);
1392
1393         } else if (m->is_root)
1394                 return 0;
1395
1396         if (m->wd <= 0 && j->inotify_fd >= 0) {
1397
1398                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1399                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1400                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1401                                           IN_ONLYDIR);
1402
1403                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1404                         inotify_rm_watch(j->inotify_fd, m->wd);
1405         }
1406
1407         for (;;) {
1408                 struct dirent *de;
1409                 union dirent_storage buf;
1410
1411                 r = readdir_r(d, &buf.de, &de);
1412                 if (r != 0 || !de)
1413                         break;
1414
1415                 if (dirent_is_file_with_suffix(de, ".journal") ||
1416                     dirent_is_file_with_suffix(de, ".journal~")) {
1417                         r = add_file(j, m->path, de->d_name);
1418                         if (r < 0) {
1419                                 log_debug("Failed to add file %s/%s: %s",
1420                                           m->path, de->d_name, strerror(-r));
1421                                 r = set_put_error(j, r);
1422                                 if (r < 0)
1423                                         return r;
1424                         }
1425                 }
1426         }
1427
1428         check_network(j, dirfd(d));
1429
1430         return 0;
1431 }
1432
1433 static int add_root_directory(sd_journal *j, const char *p) {
1434         _cleanup_closedir_ DIR *d = NULL;
1435         Directory *m;
1436         int r;
1437
1438         assert(j);
1439         assert(p);
1440
1441         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1442             !path_startswith(p, "/run"))
1443                 return -EINVAL;
1444
1445         d = opendir(p);
1446         if (!d)
1447                 return -errno;
1448
1449         m = hashmap_get(j->directories_by_path, p);
1450         if (!m) {
1451                 m = new0(Directory, 1);
1452                 if (!m)
1453                         return -ENOMEM;
1454
1455                 m->is_root = true;
1456                 m->path = strdup(p);
1457                 if (!m->path) {
1458                         free(m);
1459                         return -ENOMEM;
1460                 }
1461
1462                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1463                         free(m->path);
1464                         free(m);
1465                         return -ENOMEM;
1466                 }
1467
1468                 j->current_invalidate_counter ++;
1469
1470                 log_debug("Root directory %s added.", m->path);
1471
1472         } else if (!m->is_root)
1473                 return 0;
1474
1475         if (m->wd <= 0 && j->inotify_fd >= 0) {
1476
1477                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1478                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1479                                           IN_ONLYDIR);
1480
1481                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1482                         inotify_rm_watch(j->inotify_fd, m->wd);
1483         }
1484
1485         for (;;) {
1486                 struct dirent *de;
1487                 union dirent_storage buf;
1488                 sd_id128_t id;
1489
1490                 r = readdir_r(d, &buf.de, &de);
1491                 if (r != 0 || !de)
1492                         break;
1493
1494                 if (dirent_is_file_with_suffix(de, ".journal") ||
1495                     dirent_is_file_with_suffix(de, ".journal~")) {
1496                         r = add_file(j, m->path, de->d_name);
1497                         if (r < 0) {
1498                                 log_debug("Failed to add file %s/%s: %s",
1499                                           m->path, de->d_name, strerror(-r));
1500                                 r = set_put_error(j, r);
1501                                 if (r < 0)
1502                                         return r;
1503                         }
1504                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1505                            sd_id128_from_string(de->d_name, &id) >= 0) {
1506
1507                         r = add_directory(j, m->path, de->d_name);
1508                         if (r < 0)
1509                                 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1510                 }
1511         }
1512
1513         check_network(j, dirfd(d));
1514
1515         return 0;
1516 }
1517
1518 static int remove_directory(sd_journal *j, Directory *d) {
1519         assert(j);
1520
1521         if (d->wd > 0) {
1522                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1523
1524                 if (j->inotify_fd >= 0)
1525                         inotify_rm_watch(j->inotify_fd, d->wd);
1526         }
1527
1528         hashmap_remove(j->directories_by_path, d->path);
1529
1530         if (d->is_root)
1531                 log_debug("Root directory %s removed.", d->path);
1532         else
1533                 log_debug("Directory %s removed.", d->path);
1534
1535         free(d->path);
1536         free(d);
1537
1538         return 0;
1539 }
1540
1541 static int add_search_paths(sd_journal *j) {
1542         int r;
1543         const char search_paths[] =
1544                 "/run/log/journal\0"
1545                 "/var/log/journal\0";
1546         const char *p;
1547
1548         assert(j);
1549
1550         /* We ignore most errors here, since the idea is to only open
1551          * what's actually accessible, and ignore the rest. */
1552
1553         NULSTR_FOREACH(p, search_paths) {
1554                 r = add_root_directory(j, p);
1555                 if (r < 0 && r != -ENOENT) {
1556                         r = set_put_error(j, r);
1557                         if (r < 0)
1558                                 return r;
1559                 }
1560         }
1561
1562         return 0;
1563 }
1564
1565 static int allocate_inotify(sd_journal *j) {
1566         assert(j);
1567
1568         if (j->inotify_fd < 0) {
1569                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1570                 if (j->inotify_fd < 0)
1571                         return -errno;
1572         }
1573
1574         if (!j->directories_by_wd) {
1575                 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1576                 if (!j->directories_by_wd)
1577                         return -ENOMEM;
1578         }
1579
1580         return 0;
1581 }
1582
1583 static sd_journal *journal_new(int flags, const char *path) {
1584         sd_journal *j;
1585
1586         j = new0(sd_journal, 1);
1587         if (!j)
1588                 return NULL;
1589
1590         j->inotify_fd = -1;
1591         j->flags = flags;
1592         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1593
1594         if (path) {
1595                 j->path = strdup(path);
1596                 if (!j->path)
1597                         goto fail;
1598         }
1599
1600         j->files = hashmap_new(string_hash_func, string_compare_func);
1601         j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1602         j->mmap = mmap_cache_new();
1603         if (!j->files || !j->directories_by_path || !j->mmap)
1604                 goto fail;
1605
1606         return j;
1607
1608 fail:
1609         sd_journal_close(j);
1610         return NULL;
1611 }
1612
1613 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1614         sd_journal *j;
1615         int r;
1616
1617         if (!ret)
1618                 return -EINVAL;
1619
1620         if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1621                       SD_JOURNAL_RUNTIME_ONLY|
1622                       SD_JOURNAL_SYSTEM_ONLY))
1623                 return -EINVAL;
1624
1625         j = journal_new(flags, NULL);
1626         if (!j)
1627                 return -ENOMEM;
1628
1629         r = add_search_paths(j);
1630         if (r < 0)
1631                 goto fail;
1632
1633         *ret = j;
1634         return 0;
1635
1636 fail:
1637         sd_journal_close(j);
1638
1639         return r;
1640 }
1641
1642 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1643         sd_journal *j;
1644         int r;
1645
1646         if (!ret)
1647                 return -EINVAL;
1648
1649         if (!path)
1650                 return -EINVAL;
1651
1652         if (flags != 0)
1653                 return -EINVAL;
1654
1655         j = journal_new(flags, path);
1656         if (!j)
1657                 return -ENOMEM;
1658
1659         r = add_root_directory(j, path);
1660         if (r < 0) {
1661                 set_put_error(j, r);
1662                 goto fail;
1663         }
1664
1665         *ret = j;
1666         return 0;
1667
1668 fail:
1669         sd_journal_close(j);
1670
1671         return r;
1672 }
1673
1674 _public_ void sd_journal_close(sd_journal *j) {
1675         Directory *d;
1676         JournalFile *f;
1677
1678         if (!j)
1679                 return;
1680
1681         sd_journal_flush_matches(j);
1682
1683         while ((f = hashmap_steal_first(j->files)))
1684                 journal_file_close(f);
1685
1686         hashmap_free(j->files);
1687
1688         while ((d = hashmap_first(j->directories_by_path)))
1689                 remove_directory(j, d);
1690
1691         while ((d = hashmap_first(j->directories_by_wd)))
1692                 remove_directory(j, d);
1693
1694         hashmap_free(j->directories_by_path);
1695         hashmap_free(j->directories_by_wd);
1696
1697         if (j->inotify_fd >= 0)
1698                 close_nointr_nofail(j->inotify_fd);
1699
1700         if (j->mmap)
1701                 mmap_cache_unref(j->mmap);
1702
1703         free(j->path);
1704         free(j->unique_field);
1705         set_free(j->errors);
1706         free(j);
1707 }
1708
1709 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1710         Object *o;
1711         JournalFile *f;
1712         int r;
1713
1714         if (!j)
1715                 return -EINVAL;
1716         if (!ret)
1717                 return -EINVAL;
1718
1719         f = j->current_file;
1720         if (!f)
1721                 return -EADDRNOTAVAIL;
1722
1723         if (f->current_offset <= 0)
1724                 return -EADDRNOTAVAIL;
1725
1726         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1727         if (r < 0)
1728                 return r;
1729
1730         *ret = le64toh(o->entry.realtime);
1731         return 0;
1732 }
1733
1734 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1735         Object *o;
1736         JournalFile *f;
1737         int r;
1738         sd_id128_t id;
1739
1740         if (!j)
1741                 return -EINVAL;
1742
1743         f = j->current_file;
1744         if (!f)
1745                 return -EADDRNOTAVAIL;
1746
1747         if (f->current_offset <= 0)
1748                 return -EADDRNOTAVAIL;
1749
1750         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1751         if (r < 0)
1752                 return r;
1753
1754         if (ret_boot_id)
1755                 *ret_boot_id = o->entry.boot_id;
1756         else {
1757                 r = sd_id128_get_boot(&id);
1758                 if (r < 0)
1759                         return r;
1760
1761                 if (!sd_id128_equal(id, o->entry.boot_id))
1762                         return -ESTALE;
1763         }
1764
1765         if (ret)
1766                 *ret = le64toh(o->entry.monotonic);
1767
1768         return 0;
1769 }
1770
1771 static bool field_is_valid(const char *field) {
1772         const char *p;
1773
1774         assert(field);
1775
1776         if (isempty(field))
1777                 return false;
1778
1779         if (startswith(field, "__"))
1780                 return false;
1781
1782         for (p = field; *p; p++) {
1783
1784                 if (*p == '_')
1785                         continue;
1786
1787                 if (*p >= 'A' && *p <= 'Z')
1788                         continue;
1789
1790                 if (*p >= '0' && *p <= '9')
1791                         continue;
1792
1793                 return false;
1794         }
1795
1796         return true;
1797 }
1798
1799 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1800         JournalFile *f;
1801         uint64_t i, n;
1802         size_t field_length;
1803         int r;
1804         Object *o;
1805
1806         if (!j)
1807                 return -EINVAL;
1808         if (!field)
1809                 return -EINVAL;
1810         if (!data)
1811                 return -EINVAL;
1812         if (!size)
1813                 return -EINVAL;
1814
1815         if (!field_is_valid(field))
1816                 return -EINVAL;
1817
1818         f = j->current_file;
1819         if (!f)
1820                 return -EADDRNOTAVAIL;
1821
1822         if (f->current_offset <= 0)
1823                 return -EADDRNOTAVAIL;
1824
1825         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1826         if (r < 0)
1827                 return r;
1828
1829         field_length = strlen(field);
1830
1831         n = journal_file_entry_n_items(o);
1832         for (i = 0; i < n; i++) {
1833                 uint64_t p, l;
1834                 le64_t le_hash;
1835                 size_t t;
1836
1837                 p = le64toh(o->entry.items[i].object_offset);
1838                 le_hash = o->entry.items[i].hash;
1839                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1840                 if (r < 0)
1841                         return r;
1842
1843                 if (le_hash != o->data.hash)
1844                         return -EBADMSG;
1845
1846                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1847
1848                 if (o->object.flags & OBJECT_COMPRESSED) {
1849
1850 #ifdef HAVE_XZ
1851                         if (uncompress_startswith(o->data.payload, l,
1852                                                   &f->compress_buffer, &f->compress_buffer_size,
1853                                                   field, field_length, '=')) {
1854
1855                                 uint64_t rsize;
1856
1857                                 if (!uncompress_blob(o->data.payload, l,
1858                                                      &f->compress_buffer, &f->compress_buffer_size, &rsize,
1859                                                      j->data_threshold))
1860                                         return -EBADMSG;
1861
1862                                 *data = f->compress_buffer;
1863                                 *size = (size_t) rsize;
1864
1865                                 return 0;
1866                         }
1867 #else
1868                         return -EPROTONOSUPPORT;
1869 #endif
1870
1871                 } else if (l >= field_length+1 &&
1872                            memcmp(o->data.payload, field, field_length) == 0 &&
1873                            o->data.payload[field_length] == '=') {
1874
1875                         t = (size_t) l;
1876
1877                         if ((uint64_t) t != l)
1878                                 return -E2BIG;
1879
1880                         *data = o->data.payload;
1881                         *size = t;
1882
1883                         return 0;
1884                 }
1885
1886                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1887                 if (r < 0)
1888                         return r;
1889         }
1890
1891         return -ENOENT;
1892 }
1893
1894 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1895         size_t t;
1896         uint64_t l;
1897
1898         l = le64toh(o->object.size) - offsetof(Object, data.payload);
1899         t = (size_t) l;
1900
1901         /* We can't read objects larger than 4G on a 32bit machine */
1902         if ((uint64_t) t != l)
1903                 return -E2BIG;
1904
1905         if (o->object.flags & OBJECT_COMPRESSED) {
1906 #ifdef HAVE_XZ
1907                 uint64_t rsize;
1908
1909                 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
1910                         return -EBADMSG;
1911
1912                 *data = f->compress_buffer;
1913                 *size = (size_t) rsize;
1914 #else
1915                 return -EPROTONOSUPPORT;
1916 #endif
1917         } else {
1918                 *data = o->data.payload;
1919                 *size = t;
1920         }
1921
1922         return 0;
1923 }
1924
1925 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1926         JournalFile *f;
1927         uint64_t p, n;
1928         le64_t le_hash;
1929         int r;
1930         Object *o;
1931
1932         if (!j)
1933                 return -EINVAL;
1934         if (!data)
1935                 return -EINVAL;
1936         if (!size)
1937                 return -EINVAL;
1938
1939         f = j->current_file;
1940         if (!f)
1941                 return -EADDRNOTAVAIL;
1942
1943         if (f->current_offset <= 0)
1944                 return -EADDRNOTAVAIL;
1945
1946         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1947         if (r < 0)
1948                 return r;
1949
1950         n = journal_file_entry_n_items(o);
1951         if (j->current_field >= n)
1952                 return 0;
1953
1954         p = le64toh(o->entry.items[j->current_field].object_offset);
1955         le_hash = o->entry.items[j->current_field].hash;
1956         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1957         if (r < 0)
1958                 return r;
1959
1960         if (le_hash != o->data.hash)
1961                 return -EBADMSG;
1962
1963         r = return_data(j, f, o, data, size);
1964         if (r < 0)
1965                 return r;
1966
1967         j->current_field ++;
1968
1969         return 1;
1970 }
1971
1972 _public_ void sd_journal_restart_data(sd_journal *j) {
1973         if (!j)
1974                 return;
1975
1976         j->current_field = 0;
1977 }
1978
1979 _public_ int sd_journal_get_fd(sd_journal *j) {
1980         int r;
1981
1982         if (!j)
1983                 return -EINVAL;
1984
1985         if (j->inotify_fd >= 0)
1986                 return j->inotify_fd;
1987
1988         r = allocate_inotify(j);
1989         if (r < 0)
1990                 return r;
1991
1992         /* Iterate through all dirs again, to add them to the
1993          * inotify */
1994         if (j->path)
1995                 r = add_root_directory(j, j->path);
1996         else
1997                 r = add_search_paths(j);
1998         if (r < 0)
1999                 return r;
2000
2001         return j->inotify_fd;
2002 }
2003
2004 _public_ int sd_journal_get_events(sd_journal *j) {
2005         int fd;
2006
2007         if (!j)
2008                 return -EINVAL;
2009
2010         fd = sd_journal_get_fd(j);
2011         if (fd < 0)
2012                 return fd;
2013
2014         return POLLIN;
2015 }
2016
2017 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2018         int fd;
2019
2020         if (!j)
2021                 return -EINVAL;
2022         if (!timeout_usec)
2023                 return -EINVAL;
2024
2025         fd = sd_journal_get_fd(j);
2026         if (fd < 0)
2027                 return fd;
2028
2029         if (!j->on_network) {
2030                 *timeout_usec = (uint64_t) -1;
2031                 return 0;
2032         }
2033
2034         /* If we are on the network we need to regularly check for
2035          * changes manually */
2036
2037         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2038         return 1;
2039 }
2040
2041 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2042         Directory *d;
2043         int r;
2044
2045         assert(j);
2046         assert(e);
2047
2048         /* Is this a subdirectory we watch? */
2049         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2050         if (d) {
2051                 sd_id128_t id;
2052
2053                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2054                     (endswith(e->name, ".journal") ||
2055                      endswith(e->name, ".journal~"))) {
2056
2057                         /* Event for a journal file */
2058
2059                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2060                                 r = add_file(j, d->path, e->name);
2061                                 if (r < 0) {
2062                                         log_debug("Failed to add file %s/%s: %s",
2063                                                   d->path, e->name, strerror(-r));
2064                                         set_put_error(j, r);
2065                                 }
2066
2067                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2068
2069                                 r = remove_file(j, d->path, e->name);
2070                                 if (r < 0)
2071                                         log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2072                         }
2073
2074                 } else if (!d->is_root && e->len == 0) {
2075
2076                         /* Event for a subdirectory */
2077
2078                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2079                                 r = remove_directory(j, d);
2080                                 if (r < 0)
2081                                         log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2082                         }
2083
2084
2085                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2086
2087                         /* Event for root directory */
2088
2089                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2090                                 r = add_directory(j, d->path, e->name);
2091                                 if (r < 0)
2092                                         log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2093                         }
2094                 }
2095
2096                 return;
2097         }
2098
2099         if (e->mask & IN_IGNORED)
2100                 return;
2101
2102         log_warning("Unknown inotify event.");
2103 }
2104
2105 static int determine_change(sd_journal *j) {
2106         bool b;
2107
2108         assert(j);
2109
2110         b = j->current_invalidate_counter != j->last_invalidate_counter;
2111         j->last_invalidate_counter = j->current_invalidate_counter;
2112
2113         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2114 }
2115
2116 _public_ int sd_journal_process(sd_journal *j) {
2117         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2118         bool got_something = false;
2119
2120         if (!j)
2121                 return -EINVAL;
2122
2123         j->last_process_usec = now(CLOCK_MONOTONIC);
2124
2125         for (;;) {
2126                 struct inotify_event *e;
2127                 ssize_t l;
2128
2129                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2130                 if (l < 0) {
2131                         if (errno == EAGAIN || errno == EINTR)
2132                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2133
2134                         return -errno;
2135                 }
2136
2137                 got_something = true;
2138
2139                 e = (struct inotify_event*) buffer;
2140                 while (l > 0) {
2141                         size_t step;
2142
2143                         process_inotify_event(j, e);
2144
2145                         step = sizeof(struct inotify_event) + e->len;
2146                         assert(step <= (size_t) l);
2147
2148                         e = (struct inotify_event*) ((uint8_t*) e + step);
2149                         l -= step;
2150                 }
2151         }
2152
2153         return determine_change(j);
2154 }
2155
2156 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2157         int r;
2158         uint64_t t;
2159
2160         assert(j);
2161
2162         if (j->inotify_fd < 0) {
2163
2164                 /* This is the first invocation, hence create the
2165                  * inotify watch */
2166                 r = sd_journal_get_fd(j);
2167                 if (r < 0)
2168                         return r;
2169
2170                 /* The journal might have changed since the context
2171                  * object was created and we weren't watching before,
2172                  * hence don't wait for anything, and return
2173                  * immediately. */
2174                 return determine_change(j);
2175         }
2176
2177         r = sd_journal_get_timeout(j, &t);
2178         if (r < 0)
2179                 return r;
2180
2181         if (t != (uint64_t) -1) {
2182                 usec_t n;
2183
2184                 n = now(CLOCK_MONOTONIC);
2185                 t = t > n ? t - n : 0;
2186
2187                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2188                         timeout_usec = t;
2189         }
2190
2191         do {
2192                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2193         } while (r == -EINTR);
2194
2195         if (r < 0)
2196                 return r;
2197
2198         return sd_journal_process(j);
2199 }
2200
2201 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2202         Iterator i;
2203         JournalFile *f;
2204         bool first = true;
2205         int r;
2206
2207         if (!j)
2208                 return -EINVAL;
2209         if (!from && !to)
2210                 return -EINVAL;
2211         if (from == to)
2212                 return -EINVAL;
2213
2214         HASHMAP_FOREACH(f, j->files, i) {
2215                 usec_t fr, t;
2216
2217                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2218                 if (r == -ENOENT)
2219                         continue;
2220                 if (r < 0)
2221                         return r;
2222                 if (r == 0)
2223                         continue;
2224
2225                 if (first) {
2226                         if (from)
2227                                 *from = fr;
2228                         if (to)
2229                                 *to = t;
2230                         first = false;
2231                 } else {
2232                         if (from)
2233                                 *from = MIN(fr, *from);
2234                         if (to)
2235                                 *to = MAX(t, *to);
2236                 }
2237         }
2238
2239         return first ? 0 : 1;
2240 }
2241
2242 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2243         Iterator i;
2244         JournalFile *f;
2245         bool first = true;
2246         int r;
2247
2248         if (!j)
2249                 return -EINVAL;
2250         if (!from && !to)
2251                 return -EINVAL;
2252         if (from == to)
2253                 return -EINVAL;
2254
2255         HASHMAP_FOREACH(f, j->files, i) {
2256                 usec_t fr, t;
2257
2258                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2259                 if (r == -ENOENT)
2260                         continue;
2261                 if (r < 0)
2262                         return r;
2263                 if (r == 0)
2264                         continue;
2265
2266                 if (first) {
2267                         if (from)
2268                                 *from = fr;
2269                         if (to)
2270                                 *to = t;
2271                         first = false;
2272                 } else {
2273                         if (from)
2274                                 *from = MIN(fr, *from);
2275                         if (to)
2276                                 *to = MAX(t, *to);
2277                 }
2278         }
2279
2280         return first ? 0 : 1;
2281 }
2282
2283 void journal_print_header(sd_journal *j) {
2284         Iterator i;
2285         JournalFile *f;
2286         bool newline = false;
2287
2288         assert(j);
2289
2290         HASHMAP_FOREACH(f, j->files, i) {
2291                 if (newline)
2292                         putchar('\n');
2293                 else
2294                         newline = true;
2295
2296                 journal_file_print_header(f);
2297         }
2298 }
2299
2300 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2301         Iterator i;
2302         JournalFile *f;
2303         uint64_t sum = 0;
2304
2305         if (!j)
2306                 return -EINVAL;
2307         if (!bytes)
2308                 return -EINVAL;
2309
2310         HASHMAP_FOREACH(f, j->files, i) {
2311                 struct stat st;
2312
2313                 if (fstat(f->fd, &st) < 0)
2314                         return -errno;
2315
2316                 sum += (uint64_t) st.st_blocks * 512ULL;
2317         }
2318
2319         *bytes = sum;
2320         return 0;
2321 }
2322
2323 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2324         char *f;
2325
2326         if (!j)
2327                 return -EINVAL;
2328         if (isempty(field))
2329                 return -EINVAL;
2330         if (!field_is_valid(field))
2331                 return -EINVAL;
2332
2333         f = strdup(field);
2334         if (!f)
2335                 return -ENOMEM;
2336
2337         free(j->unique_field);
2338         j->unique_field = f;
2339         j->unique_file = NULL;
2340         j->unique_offset = 0;
2341
2342         return 0;
2343 }
2344
2345 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2346         Object *o;
2347         size_t k;
2348         int r;
2349
2350         if (!j)
2351                 return -EINVAL;
2352         if (!data)
2353                 return -EINVAL;
2354         if (!l)
2355                 return -EINVAL;
2356         if (!j->unique_field)
2357                 return -EINVAL;
2358
2359         k = strlen(j->unique_field);
2360
2361         if (!j->unique_file) {
2362                 j->unique_file = hashmap_first(j->files);
2363                 if (!j->unique_file)
2364                         return 0;
2365                 j->unique_offset = 0;
2366         }
2367
2368         for (;;) {
2369                 JournalFile *of;
2370                 Iterator i;
2371                 const void *odata;
2372                 size_t ol;
2373                 bool found;
2374
2375                 /* Proceed to next data object in the field's linked list */
2376                 if (j->unique_offset == 0) {
2377                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2378                         if (r < 0)
2379                                 return r;
2380
2381                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2382                 } else {
2383                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2384                         if (r < 0)
2385                                 return r;
2386
2387                         j->unique_offset = le64toh(o->data.next_field_offset);
2388                 }
2389
2390                 /* We reached the end of the list? Then start again, with the next file */
2391                 if (j->unique_offset == 0) {
2392                         JournalFile *n;
2393
2394                         n = hashmap_next(j->files, j->unique_file->path);
2395                         if (!n)
2396                                 return 0;
2397
2398                         j->unique_file = n;
2399                         continue;
2400                 }
2401
2402                 /* We do not use the type context here, but 0 instead,
2403                  * so that we can look at this data object at the same
2404                  * time as one on another file */
2405                 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2406                 if (r < 0)
2407                         return r;
2408
2409                 /* Let's do the type check by hand, since we used 0 context above. */
2410                 if (o->object.type != OBJECT_DATA)
2411                         return -EBADMSG;
2412
2413                 r = return_data(j, j->unique_file, o, &odata, &ol);
2414                 if (r < 0)
2415                         return r;
2416
2417                 /* OK, now let's see if we already returned this data
2418                  * object by checking if it exists in the earlier
2419                  * traversed files. */
2420                 found = false;
2421                 HASHMAP_FOREACH(of, j->files, i) {
2422                         Object *oo;
2423                         uint64_t op;
2424
2425                         if (of == j->unique_file)
2426                                 break;
2427
2428                         /* Skip this file it didn't have any fields
2429                          * indexed */
2430                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2431                             le64toh(of->header->n_fields) <= 0)
2432                                 continue;
2433
2434                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2435                         if (r < 0)
2436                                 return r;
2437
2438                         if (r > 0)
2439                                 found = true;
2440                 }
2441
2442                 if (found)
2443                         continue;
2444
2445                 r = return_data(j, j->unique_file, o, data, l);
2446                 if (r < 0)
2447                         return r;
2448
2449                 return 1;
2450         }
2451 }
2452
2453 _public_ void sd_journal_restart_unique(sd_journal *j) {
2454         if (!j)
2455                 return;
2456
2457         j->unique_file = NULL;
2458         j->unique_offset = 0;
2459 }
2460
2461 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2462         if (!j)
2463                 return -EINVAL;
2464
2465         return !j->on_network;
2466 }
2467
2468 static char *lookup_field(const char *field, void *userdata) {
2469         sd_journal *j = userdata;
2470         const void *data;
2471         size_t size, d;
2472         int r;
2473
2474         assert(field);
2475         assert(j);
2476
2477         r = sd_journal_get_data(j, field, &data, &size);
2478         if (r < 0 ||
2479             size > REPLACE_VAR_MAX)
2480                 return strdup(field);
2481
2482         d = strlen(field) + 1;
2483
2484         return strndup((const char*) data + d, size - d);
2485 }
2486
2487 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2488         const void *data;
2489         size_t size;
2490         sd_id128_t id;
2491         _cleanup_free_ char *text = NULL, *cid = NULL;
2492         char *t;
2493         int r;
2494
2495         if (!j)
2496                 return -EINVAL;
2497         if (!ret)
2498                 return -EINVAL;
2499
2500         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2501         if (r < 0)
2502                 return r;
2503
2504         cid = strndup((const char*) data + 11, size - 11);
2505         if (!cid)
2506                 return -ENOMEM;
2507
2508         r = sd_id128_from_string(cid, &id);
2509         if (r < 0)
2510                 return r;
2511
2512         r = catalog_get(CATALOG_DATABASE, id, &text);
2513         if (r < 0)
2514                 return r;
2515
2516         t = replace_var(text, lookup_field, j);
2517         if (!t)
2518                 return -ENOMEM;
2519
2520         *ret = t;
2521         return 0;
2522 }
2523
2524 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2525         if (!ret)
2526                 return -EINVAL;
2527
2528         return catalog_get(CATALOG_DATABASE, id, ret);
2529 }
2530
2531 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2532         if (!j)
2533                 return -EINVAL;
2534
2535         j->data_threshold = sz;
2536         return 0;
2537 }
2538
2539 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2540         if (!j)
2541                 return -EINVAL;
2542         if (!sz)
2543                 return -EINVAL;
2544
2545         *sz = j->data_threshold;
2546         return 0;
2547 }