src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "path-util.h"
  37 #include "lookup3.h"
  38 #include "compress.h"
  39 #include "journal-internal.h"
  40 #include "missing.h"
  41 #include "catalog.h"
  42 #include "replace-var.h"
  43
  44 #define JOURNAL_FILES_MAX 1024
  45
  46 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  47
  48 #define REPLACE_VAR_MAX 256
  49
  50 #define DEFAULT_DATA_THRESHOLD (64*1024)
  51
  52 /* We return an error here only if we didn't manage to
  53    memorize the real error. */
  54 static int set_put_error(sd_journal *j, int r) {
  55         int k;
  56
  57         if (r >= 0)
  58                 return r;
  59
  60         k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
  61         if (k < 0)
  62                 return k;
  63
  64         return set_put(j->errors, INT_TO_PTR(r));
  65 }
  66
  67 static void detach_location(sd_journal *j) {
  68         Iterator i;
  69         JournalFile *f;
  70
  71         assert(j);
  72
  73         j->current_file = NULL;
  74         j->current_field = 0;
  75
  76         HASHMAP_FOREACH(f, j->files, i)
  77                 f->current_offset = 0;
  78 }
  79
  80 static void reset_location(sd_journal *j) {
  81         assert(j);
  82
  83         detach_location(j);
  84         zero(j->current_location);
  85 }
  86
  87 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  88         assert(l);
  89         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
  90         assert(f);
  91         assert(o->object.type == OBJECT_ENTRY);
  92
  93         l->type = type;
  94         l->seqnum = le64toh(o->entry.seqnum);
  95         l->seqnum_id = f->header->seqnum_id;
  96         l->realtime = le64toh(o->entry.realtime);
  97         l->monotonic = le64toh(o->entry.monotonic);
  98         l->boot_id = o->entry.boot_id;
  99         l->xor_hash = le64toh(o->entry.xor_hash);
 100
 101         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 102 }
 103
 104 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o, uint64_t offset) {
 105         assert(j);
 106         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 107         assert(f);
 108         assert(o);
 109
 110         init_location(&j->current_location, type, f, o);
 111
 112         if (j->current_file)
 113                 j->current_file->current_offset = 0;
 114
 115         j->current_file = f;
 116         j->current_field = 0;
 117
 118         f->current_offset = offset;
 119 }
 120
 121 static int match_is_valid(const void *data, size_t size) {
 122         const char *b, *p;
 123
 124         assert(data);
 125
 126         if (size < 2)
 127                 return false;
 128
 129         if (startswith(data, "__"))
 130                 return false;
 131
 132         b = data;
 133         for (p = b; p < b + size; p++) {
 134
 135                 if (*p == '=')
 136                         return p > b;
 137
 138                 if (*p == '_')
 139                         continue;
 140
 141                 if (*p >= 'A' && *p <= 'Z')
 142                         continue;
 143
 144                 if (*p >= '0' && *p <= '9')
 145                         continue;
 146
 147                 return false;
 148         }
 149
 150         return false;
 151 }
 152
 153 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 154         const uint8_t *a = _a, *b = _b;
 155         size_t j;
 156
 157         for (j = 0; j < s && j < t; j++) {
 158
 159                 if (a[j] != b[j])
 160                         return false;
 161
 162                 if (a[j] == '=')
 163                         return true;
 164         }
 165
 166         return true;
 167 }
 168
 169 static Match *match_new(Match *p, MatchType t) {
 170         Match *m;
 171
 172         m = new0(Match, 1);
 173         if (!m)
 174                 return NULL;
 175
 176         m->type = t;
 177
 178         if (p) {
 179                 m->parent = p;
 180                 LIST_PREPEND(Match, matches, p->matches, m);
 181         }
 182
 183         return m;
 184 }
 185
 186 static void match_free(Match *m) {
 187         assert(m);
 188
 189         while (m->matches)
 190                 match_free(m->matches);
 191
 192         if (m->parent)
 193                 LIST_REMOVE(Match, matches, m->parent->matches, m);
 194
 195         free(m->data);
 196         free(m);
 197 }
 198
 199 static void match_free_if_empty(Match *m) {
 200         if (!m || m->matches)
 201                 return;
 202
 203         match_free(m);
 204 }
 205
 206 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 207         Match *l3, *l4, *add_here = NULL, *m;
 208         le64_t le_hash;
 209
 210         if (!j)
 211                 return -EINVAL;
 212
 213         if (!data)
 214                 return -EINVAL;
 215
 216         if (size == 0)
 217                 size = strlen(data);
 218
 219         if (!match_is_valid(data, size))
 220                 return -EINVAL;
 221
 222         /* level 0: AND term
 223          * level 1: OR terms
 224          * level 2: AND terms
 225          * level 3: OR terms
 226          * level 4: concrete matches */
 227
 228         if (!j->level0) {
 229                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 230                 if (!j->level0)
 231                         return -ENOMEM;
 232         }
 233
 234         if (!j->level1) {
 235                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 236                 if (!j->level1)
 237                         return -ENOMEM;
 238         }
 239
 240         if (!j->level2) {
 241                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 242                 if (!j->level2)
 243                         return -ENOMEM;
 244         }
 245
 246         assert(j->level0->type == MATCH_AND_TERM);
 247         assert(j->level1->type == MATCH_OR_TERM);
 248         assert(j->level2->type == MATCH_AND_TERM);
 249
 250         le_hash = htole64(hash64(data, size));
 251
 252         LIST_FOREACH(matches, l3, j->level2->matches) {
 253                 assert(l3->type == MATCH_OR_TERM);
 254
 255                 LIST_FOREACH(matches, l4, l3->matches) {
 256                         assert(l4->type == MATCH_DISCRETE);
 257
 258                         /* Exactly the same match already? Then ignore
 259                          * this addition */
 260                         if (l4->le_hash == le_hash &&
 261                             l4->size == size &&
 262                             memcmp(l4->data, data, size) == 0)
 263                                 return 0;
 264
 265                         /* Same field? Then let's add this to this OR term */
 266                         if (same_field(data, size, l4->data, l4->size)) {
 267                                 add_here = l3;
 268                                 break;
 269                         }
 270                 }
 271
 272                 if (add_here)
 273                         break;
 274         }
 275
 276         if (!add_here) {
 277                 add_here = match_new(j->level2, MATCH_OR_TERM);
 278                 if (!add_here)
 279                         goto fail;
 280         }
 281
 282         m = match_new(add_here, MATCH_DISCRETE);
 283         if (!m)
 284                 goto fail;
 285
 286         m->le_hash = le_hash;
 287         m->size = size;
 288         m->data = memdup(data, size);
 289         if (!m->data)
 290                 goto fail;
 291
 292         detach_location(j);
 293
 294         return 0;
 295
 296 fail:
 297         match_free_if_empty(add_here);
 298         match_free_if_empty(j->level2);
 299         match_free_if_empty(j->level1);
 300         match_free_if_empty(j->level0);
 301
 302         return -ENOMEM;
 303 }
 304
 305 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 306         assert(j);
 307
 308         if (!j->level0)
 309                 return 0;
 310
 311         if (!j->level1)
 312                 return 0;
 313
 314         if (!j->level1->matches)
 315                 return 0;
 316
 317         j->level1 = NULL;
 318         j->level2 = NULL;
 319
 320         return 0;
 321 }
 322
 323 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 324         assert(j);
 325
 326         if (!j->level0)
 327                 return 0;
 328
 329         if (!j->level1)
 330                 return 0;
 331
 332         if (!j->level2)
 333                 return 0;
 334
 335         if (!j->level2->matches)
 336                 return 0;
 337
 338         j->level2 = NULL;
 339         return 0;
 340 }
 341
 342 static char *match_make_string(Match *m) {
 343         char *p, *r;
 344         Match *i;
 345         bool enclose = false;
 346
 347         if (!m)
 348                 return strdup("");
 349
 350         if (m->type == MATCH_DISCRETE)
 351                 return strndup(m->data, m->size);
 352
 353         p = NULL;
 354         LIST_FOREACH(matches, i, m->matches) {
 355                 char *t, *k;
 356
 357                 t = match_make_string(i);
 358                 if (!t) {
 359                         free(p);
 360                         return NULL;
 361                 }
 362
 363                 if (p) {
 364                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 365                         free(p);
 366                         free(t);
 367
 368                         if (!k)
 369                                 return NULL;
 370
 371                         p = k;
 372
 373                         enclose = true;
 374                 } else {
 375                         free(p);
 376                         p = t;
 377                 }
 378         }
 379
 380         if (enclose) {
 381                 r = strjoin("(", p, ")", NULL);
 382                 free(p);
 383                 return r;
 384         }
 385
 386         return p;
 387 }
 388
 389 char *journal_make_match_string(sd_journal *j) {
 390         assert(j);
 391
 392         return match_make_string(j->level0);
 393 }
 394
 395 _public_ void sd_journal_flush_matches(sd_journal *j) {
 396
 397         if (!j)
 398                 return;
 399
 400         if (j->level0)
 401                 match_free(j->level0);
 402
 403         j->level0 = j->level1 = j->level2 = NULL;
 404
 405         detach_location(j);
 406 }
 407
 408 static int compare_entry_order(JournalFile *af, Object *_ao,
 409                                JournalFile *bf, uint64_t bp) {
 410
 411         uint64_t a, b;
 412         Object *ao, *bo;
 413         int r;
 414
 415         assert(af);
 416         assert(bf);
 417         assert(_ao);
 418
 419         /* The mmap cache might invalidate the object from the first
 420          * file if we look at the one from the second file. Hence
 421          * temporarily copy the header of the first one, and look at
 422          * that only. */
 423         ao = alloca(offsetof(EntryObject, items));
 424         memcpy(ao, _ao, offsetof(EntryObject, items));
 425
 426         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 427         if (r < 0)
 428                 return strcmp(af->path, bf->path);
 429
 430         /* We operate on two different files here, hence we can access
 431          * two objects at the same time, which we normally can't.
 432          *
 433          * If contents and timestamps match, these entries are
 434          * identical, even if the seqnum does not match */
 435
 436         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 437             ao->entry.monotonic == bo->entry.monotonic &&
 438             ao->entry.realtime == bo->entry.realtime &&
 439             ao->entry.xor_hash == bo->entry.xor_hash)
 440                 return 0;
 441
 442         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 443
 444                 /* If this is from the same seqnum source, compare
 445                  * seqnums */
 446                 a = le64toh(ao->entry.seqnum);
 447                 b = le64toh(bo->entry.seqnum);
 448
 449                 if (a < b)
 450                         return -1;
 451                 if (a > b)
 452                         return 1;
 453
 454                 /* Wow! This is weird, different data but the same
 455                  * seqnums? Something is borked, but let's make the
 456                  * best of it and compare by time. */
 457         }
 458
 459         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 460
 461                 /* If the boot id matches compare monotonic time */
 462                 a = le64toh(ao->entry.monotonic);
 463                 b = le64toh(bo->entry.monotonic);
 464
 465                 if (a < b)
 466                         return -1;
 467                 if (a > b)
 468                         return 1;
 469         }
 470
 471         /* Otherwise compare UTC time */
 472         a = le64toh(ao->entry.realtime);
 473         b = le64toh(bo->entry.realtime);
 474
 475         if (a < b)
 476                 return -1;
 477         if (a > b)
 478                 return 1;
 479
 480         /* Finally, compare by contents */
 481         a = le64toh(ao->entry.xor_hash);
 482         b = le64toh(bo->entry.xor_hash);
 483
 484         if (a < b)
 485                 return -1;
 486         if (a > b)
 487                 return 1;
 488
 489         return 0;
 490 }
 491
 492 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 493         uint64_t a;
 494
 495         assert(af);
 496         assert(ao);
 497         assert(l);
 498         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 499
 500         if (l->monotonic_set &&
 501             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 502             l->realtime_set &&
 503             le64toh(ao->entry.realtime) == l->realtime &&
 504             l->xor_hash_set &&
 505             le64toh(ao->entry.xor_hash) == l->xor_hash)
 506                 return 0;
 507
 508         if (l->seqnum_set &&
 509             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 510
 511                 a = le64toh(ao->entry.seqnum);
 512
 513                 if (a < l->seqnum)
 514                         return -1;
 515                 if (a > l->seqnum)
 516                         return 1;
 517         }
 518
 519         if (l->monotonic_set &&
 520             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 521
 522                 a = le64toh(ao->entry.monotonic);
 523
 524                 if (a < l->monotonic)
 525                         return -1;
 526                 if (a > l->monotonic)
 527                         return 1;
 528         }
 529
 530         if (l->realtime_set) {
 531
 532                 a = le64toh(ao->entry.realtime);
 533
 534                 if (a < l->realtime)
 535                         return -1;
 536                 if (a > l->realtime)
 537                         return 1;
 538         }
 539
 540         if (l->xor_hash_set) {
 541                 a = le64toh(ao->entry.xor_hash);
 542
 543                 if (a < l->xor_hash)
 544                         return -1;
 545                 if (a > l->xor_hash)
 546                         return 1;
 547         }
 548
 549         return 0;
 550 }
 551
 552 static int next_for_match(
 553                 sd_journal *j,
 554                 Match *m,
 555                 JournalFile *f,
 556                 uint64_t after_offset,
 557                 direction_t direction,
 558                 Object **ret,
 559                 uint64_t *offset) {
 560
 561         int r;
 562         uint64_t np = 0;
 563         Object *n;
 564
 565         assert(j);
 566         assert(m);
 567         assert(f);
 568
 569         if (m->type == MATCH_DISCRETE) {
 570                 uint64_t dp;
 571
 572                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 573                 if (r <= 0)
 574                         return r;
 575
 576                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 577
 578         } else if (m->type == MATCH_OR_TERM) {
 579                 Match *i;
 580
 581                 /* Find the earliest match beyond after_offset */
 582
 583                 LIST_FOREACH(matches, i, m->matches) {
 584                         uint64_t cp;
 585
 586                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 587                         if (r < 0)
 588                                 return r;
 589                         else if (r > 0) {
 590                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 591                                         np = cp;
 592                         }
 593                 }
 594
 595         } else if (m->type == MATCH_AND_TERM) {
 596                 Match *i;
 597                 bool continue_looking;
 598
 599                 /* Always jump to the next matching entry and repeat
 600                  * this until we fine and offset that matches for all
 601                  * matches. */
 602
 603                 if (!m->matches)
 604                         return 0;
 605
 606                 np = 0;
 607                 do {
 608                         continue_looking = false;
 609
 610                         LIST_FOREACH(matches, i, m->matches) {
 611                                 uint64_t cp, limit;
 612
 613                                 if (np == 0)
 614                                         limit = after_offset;
 615                                 else if (direction == DIRECTION_DOWN)
 616                                         limit = MAX(np, after_offset);
 617                                 else
 618                                         limit = MIN(np, after_offset);
 619
 620                                 r = next_for_match(j, i, f, limit, direction, NULL, &cp);
 621                                 if (r <= 0)
 622                                         return r;
 623
 624                                 if ((direction == DIRECTION_DOWN ? cp >= after_offset : cp <= after_offset) &&
 625                                     (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))) {
 626                                         np = cp;
 627                                         continue_looking = true;
 628                                 }
 629                         }
 630
 631                 } while (continue_looking);
 632         }
 633
 634         if (np == 0)
 635                 return 0;
 636
 637         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 638         if (r < 0)
 639                 return r;
 640
 641         if (ret)
 642                 *ret = n;
 643         if (offset)
 644                 *offset = np;
 645
 646         return 1;
 647 }
 648
 649 static int find_location_for_match(
 650                 sd_journal *j,
 651                 Match *m,
 652                 JournalFile *f,
 653                 direction_t direction,
 654                 Object **ret,
 655                 uint64_t *offset) {
 656
 657         int r;
 658
 659         assert(j);
 660         assert(m);
 661         assert(f);
 662
 663         if (m->type == MATCH_DISCRETE) {
 664                 uint64_t dp;
 665
 666                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 667                 if (r <= 0)
 668                         return r;
 669
 670                 /* FIXME: missing: find by monotonic */
 671
 672                 if (j->current_location.type == LOCATION_HEAD)
 673                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 674                 if (j->current_location.type == LOCATION_TAIL)
 675                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 676                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 677                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 678                 if (j->current_location.monotonic_set) {
 679                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 680                         if (r != -ENOENT)
 681                                 return r;
 682                 }
 683                 if (j->current_location.realtime_set)
 684                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 685
 686                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 687
 688         } else if (m->type == MATCH_OR_TERM) {
 689                 uint64_t np = 0;
 690                 Object *n;
 691                 Match *i;
 692
 693                 /* Find the earliest match */
 694
 695                 LIST_FOREACH(matches, i, m->matches) {
 696                         uint64_t cp;
 697
 698                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 699                         if (r < 0)
 700                                 return r;
 701                         else if (r > 0) {
 702                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 703                                         np = cp;
 704                         }
 705                 }
 706
 707                 if (np == 0)
 708                         return 0;
 709
 710                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 711                 if (r < 0)
 712                         return r;
 713
 714                 if (ret)
 715                         *ret = n;
 716                 if (offset)
 717                         *offset = np;
 718
 719                 return 1;
 720
 721         } else {
 722                 Match *i;
 723                 uint64_t np = 0;
 724
 725                 assert(m->type == MATCH_AND_TERM);
 726
 727                 /* First jump to the last match, and then find the
 728                  * next one where all matches match */
 729
 730                 if (!m->matches)
 731                         return 0;
 732
 733                 LIST_FOREACH(matches, i, m->matches) {
 734                         uint64_t cp;
 735
 736                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 737                         if (r <= 0)
 738                                 return r;
 739
 740                         if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
 741                                 np = cp;
 742                 }
 743
 744                 return next_for_match(j, m, f, np, direction, ret, offset);
 745         }
 746 }
 747
 748 static int find_location_with_matches(
 749                 sd_journal *j,
 750                 JournalFile *f,
 751                 direction_t direction,
 752                 Object **ret,
 753                 uint64_t *offset) {
 754
 755         int r;
 756
 757         assert(j);
 758         assert(f);
 759         assert(ret);
 760         assert(offset);
 761
 762         if (!j->level0) {
 763                 /* No matches is simple */
 764
 765                 if (j->current_location.type == LOCATION_HEAD)
 766                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 767                 if (j->current_location.type == LOCATION_TAIL)
 768                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 769                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 770                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 771                 if (j->current_location.monotonic_set) {
 772                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 773                         if (r != -ENOENT)
 774                                 return r;
 775                 }
 776                 if (j->current_location.realtime_set)
 777                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 778
 779                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 780         } else
 781                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 782 }
 783
 784 static int next_with_matches(
 785                 sd_journal *j,
 786                 JournalFile *f,
 787                 direction_t direction,
 788                 Object **ret,
 789                 uint64_t *offset) {
 790
 791         Object *c;
 792         uint64_t cp;
 793
 794         assert(j);
 795         assert(f);
 796         assert(ret);
 797         assert(offset);
 798
 799         c = *ret;
 800         cp = *offset;
 801
 802         /* No matches is easy. We simple advance the file
 803          * pointer by one. */
 804         if (!j->level0)
 805                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 806
 807         /* If we have a match then we look for the next matching entry
 808          * with an offset at least one step larger */
 809         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 810 }
 811
 812 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 813         Object *c;
 814         uint64_t cp;
 815         int r;
 816
 817         assert(j);
 818         assert(f);
 819
 820         if (f->current_offset > 0) {
 821                 cp = f->current_offset;
 822
 823                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 824                 if (r < 0)
 825                         return r;
 826
 827                 r = next_with_matches(j, f, direction, &c, &cp);
 828                 if (r <= 0)
 829                         return r;
 830         } else {
 831                 r = find_location_with_matches(j, f, direction, &c, &cp);
 832                 if (r <= 0)
 833                         return r;
 834         }
 835
 836         /* OK, we found the spot, now let's advance until to an entry
 837          * that is actually different from what we were previously
 838          * looking at. This is necessary to handle entries which exist
 839          * in two (or more) journal files, and which shall all be
 840          * suppressed but one. */
 841
 842         for (;;) {
 843                 bool found;
 844
 845                 if (j->current_location.type == LOCATION_DISCRETE) {
 846                         int k;
 847
 848                         k = compare_with_location(f, c, &j->current_location);
 849                         if (direction == DIRECTION_DOWN)
 850                                 found = k > 0;
 851                         else
 852                                 found = k < 0;
 853                 } else
 854                         found = true;
 855
 856                 if (found) {
 857                         if (ret)
 858                                 *ret = c;
 859                         if (offset)
 860                                 *offset = cp;
 861                         return 1;
 862                 }
 863
 864                 r = next_with_matches(j, f, direction, &c, &cp);
 865                 if (r <= 0)
 866                         return r;
 867         }
 868 }
 869
 870 static int real_journal_next(sd_journal *j, direction_t direction) {
 871         JournalFile *f, *new_file = NULL;
 872         uint64_t new_offset = 0;
 873         Object *o;
 874         uint64_t p;
 875         Iterator i;
 876         int r;
 877
 878         if (!j)
 879                 return -EINVAL;
 880
 881         HASHMAP_FOREACH(f, j->files, i) {
 882                 bool found;
 883
 884                 r = next_beyond_location(j, f, direction, &o, &p);
 885                 if (r < 0) {
 886                         log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
 887                         continue;
 888                 } else if (r == 0)
 889                         continue;
 890
 891                 if (!new_file)
 892                         found = true;
 893                 else {
 894                         int k;
 895
 896                         k = compare_entry_order(f, o, new_file, new_offset);
 897
 898                         if (direction == DIRECTION_DOWN)
 899                                 found = k < 0;
 900                         else
 901                                 found = k > 0;
 902                 }
 903
 904                 if (found) {
 905                         new_file = f;
 906                         new_offset = p;
 907                 }
 908         }
 909
 910         if (!new_file)
 911                 return 0;
 912
 913         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 914         if (r < 0)
 915                 return r;
 916
 917         set_location(j, LOCATION_DISCRETE, new_file, o, new_offset);
 918
 919         return 1;
 920 }
 921
 922 _public_ int sd_journal_next(sd_journal *j) {
 923         return real_journal_next(j, DIRECTION_DOWN);
 924 }
 925
 926 _public_ int sd_journal_previous(sd_journal *j) {
 927         return real_journal_next(j, DIRECTION_UP);
 928 }
 929
 930 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 931         int c = 0, r;
 932
 933         if (!j)
 934                 return -EINVAL;
 935
 936         if (skip == 0) {
 937                 /* If this is not a discrete skip, then at least
 938                  * resolve the current location */
 939                 if (j->current_location.type != LOCATION_DISCRETE)
 940                         return real_journal_next(j, direction);
 941
 942                 return 0;
 943         }
 944
 945         do {
 946                 r = real_journal_next(j, direction);
 947                 if (r < 0)
 948                         return r;
 949
 950                 if (r == 0)
 951                         return c;
 952
 953                 skip--;
 954                 c++;
 955         } while (skip > 0);
 956
 957         return c;
 958 }
 959
 960 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 961         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 962 }
 963
 964 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 965         return real_journal_next_skip(j, DIRECTION_UP, skip);
 966 }
 967
 968 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 969         Object *o;
 970         int r;
 971         char bid[33], sid[33];
 972
 973         if (!j)
 974                 return -EINVAL;
 975         if (!cursor)
 976                 return -EINVAL;
 977
 978         if (!j->current_file || j->current_file->current_offset <= 0)
 979                 return -EADDRNOTAVAIL;
 980
 981         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 982         if (r < 0)
 983                 return r;
 984
 985         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 986         sd_id128_to_string(o->entry.boot_id, bid);
 987
 988         if (asprintf(cursor,
 989                      "s=%s;i=%llx;b=%s;m=%llx;t=%llx;x=%llx",
 990                      sid, (unsigned long long) le64toh(o->entry.seqnum),
 991                      bid, (unsigned long long) le64toh(o->entry.monotonic),
 992                      (unsigned long long) le64toh(o->entry.realtime),
 993                      (unsigned long long) le64toh(o->entry.xor_hash)) < 0)
 994                 return -ENOMEM;
 995
 996         return 0;
 997 }
 998
 999 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
1000         char *w, *state;
1001         size_t l;
1002         unsigned long long seqnum, monotonic, realtime, xor_hash;
1003         bool
1004                 seqnum_id_set = false,
1005                 seqnum_set = false,
1006                 boot_id_set = false,
1007                 monotonic_set = false,
1008                 realtime_set = false,
1009                 xor_hash_set = false;
1010         sd_id128_t seqnum_id, boot_id;
1011
1012         if (!j)
1013                 return -EINVAL;
1014         if (isempty(cursor))
1015                 return -EINVAL;
1016
1017         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1018                 char *item;
1019                 int k = 0;
1020
1021                 if (l < 2 || w[1] != '=')
1022                         return -EINVAL;
1023
1024                 item = strndup(w, l);
1025                 if (!item)
1026                         return -ENOMEM;
1027
1028                 switch (w[0]) {
1029
1030                 case 's':
1031                         seqnum_id_set = true;
1032                         k = sd_id128_from_string(item+2, &seqnum_id);
1033                         break;
1034
1035                 case 'i':
1036                         seqnum_set = true;
1037                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1038                                 k = -EINVAL;
1039                         break;
1040
1041                 case 'b':
1042                         boot_id_set = true;
1043                         k = sd_id128_from_string(item+2, &boot_id);
1044                         break;
1045
1046                 case 'm':
1047                         monotonic_set = true;
1048                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1049                                 k = -EINVAL;
1050                         break;
1051
1052                 case 't':
1053                         realtime_set = true;
1054                         if (sscanf(item+2, "%llx", &realtime) != 1)
1055                                 k = -EINVAL;
1056                         break;
1057
1058                 case 'x':
1059                         xor_hash_set = true;
1060                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1061                                 k = -EINVAL;
1062                         break;
1063                 }
1064
1065                 free(item);
1066
1067                 if (k < 0)
1068                         return k;
1069         }
1070
1071         if ((!seqnum_set || !seqnum_id_set) &&
1072             (!monotonic_set || !boot_id_set) &&
1073             !realtime_set)
1074                 return -EINVAL;
1075
1076         reset_location(j);
1077
1078         j->current_location.type = LOCATION_SEEK;
1079
1080         if (realtime_set) {
1081                 j->current_location.realtime = (uint64_t) realtime;
1082                 j->current_location.realtime_set = true;
1083         }
1084
1085         if (seqnum_set && seqnum_id_set) {
1086                 j->current_location.seqnum = (uint64_t) seqnum;
1087                 j->current_location.seqnum_id = seqnum_id;
1088                 j->current_location.seqnum_set = true;
1089         }
1090
1091         if (monotonic_set && boot_id_set) {
1092                 j->current_location.monotonic = (uint64_t) monotonic;
1093                 j->current_location.boot_id = boot_id;
1094                 j->current_location.monotonic_set = true;
1095         }
1096
1097         if (xor_hash_set) {
1098                 j->current_location.xor_hash = (uint64_t) xor_hash;
1099                 j->current_location.xor_hash_set = true;
1100         }
1101
1102         return 0;
1103 }
1104
1105 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1106         int r;
1107         char *w, *state;
1108         size_t l;
1109         Object *o;
1110
1111         if (!j)
1112                 return -EINVAL;
1113         if (isempty(cursor))
1114                 return -EINVAL;
1115
1116         if (!j->current_file || j->current_file->current_offset <= 0)
1117                 return -EADDRNOTAVAIL;
1118
1119         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1120         if (r < 0)
1121                 return r;
1122
1123         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1124                 _cleanup_free_ char *item = NULL;
1125                 sd_id128_t id;
1126                 unsigned long long ll;
1127                 int k = 0;
1128
1129                 if (l < 2 || w[1] != '=')
1130                         return -EINVAL;
1131
1132                 item = strndup(w, l);
1133                 if (!item)
1134                         return -ENOMEM;
1135
1136                 switch (w[0]) {
1137
1138                 case 's':
1139                         k = sd_id128_from_string(item+2, &id);
1140                         if (k < 0)
1141                                 return k;
1142                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1143                                 return 0;
1144                         break;
1145
1146                 case 'i':
1147                         if (sscanf(item+2, "%llx", &ll) != 1)
1148                                 return -EINVAL;
1149                         if (ll != le64toh(o->entry.seqnum))
1150                                 return 0;
1151                         break;
1152
1153                 case 'b':
1154                         k = sd_id128_from_string(item+2, &id);
1155                         if (k < 0)
1156                                 return k;
1157                         if (!sd_id128_equal(id, o->entry.boot_id))
1158                                 return 0;
1159                         break;
1160
1161                 case 'm':
1162                         if (sscanf(item+2, "%llx", &ll) != 1)
1163                                 return -EINVAL;
1164                         if (ll != le64toh(o->entry.monotonic))
1165                                 return 0;
1166                         break;
1167
1168                 case 't':
1169                         if (sscanf(item+2, "%llx", &ll) != 1)
1170                                 return -EINVAL;
1171                         if (ll != le64toh(o->entry.realtime))
1172                                 return 0;
1173                         break;
1174
1175                 case 'x':
1176                         if (sscanf(item+2, "%llx", &ll) != 1)
1177                                 return -EINVAL;
1178                         if (ll != le64toh(o->entry.xor_hash))
1179                                 return 0;
1180                         break;
1181                 }
1182         }
1183
1184         return 1;
1185 }
1186
1187
1188 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1189         if (!j)
1190                 return -EINVAL;
1191
1192         reset_location(j);
1193         j->current_location.type = LOCATION_SEEK;
1194         j->current_location.boot_id = boot_id;
1195         j->current_location.monotonic = usec;
1196         j->current_location.monotonic_set = true;
1197
1198         return 0;
1199 }
1200
1201 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1202         if (!j)
1203                 return -EINVAL;
1204
1205         reset_location(j);
1206         j->current_location.type = LOCATION_SEEK;
1207         j->current_location.realtime = usec;
1208         j->current_location.realtime_set = true;
1209
1210         return 0;
1211 }
1212
1213 _public_ int sd_journal_seek_head(sd_journal *j) {
1214         if (!j)
1215                 return -EINVAL;
1216
1217         reset_location(j);
1218         j->current_location.type = LOCATION_HEAD;
1219
1220         return 0;
1221 }
1222
1223 _public_ int sd_journal_seek_tail(sd_journal *j) {
1224         if (!j)
1225                 return -EINVAL;
1226
1227         reset_location(j);
1228         j->current_location.type = LOCATION_TAIL;
1229
1230         return 0;
1231 }
1232
1233 static void check_network(sd_journal *j, int fd) {
1234         struct statfs sfs;
1235
1236         assert(j);
1237
1238         if (j->on_network)
1239                 return;
1240
1241         if (fstatfs(fd, &sfs) < 0)
1242                 return;
1243
1244         j->on_network =
1245                 F_TYPE_CMP(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1246                 F_TYPE_CMP(sfs.f_type, CODA_SUPER_MAGIC) ||
1247                 F_TYPE_CMP(sfs.f_type, NCP_SUPER_MAGIC) ||
1248                 F_TYPE_CMP(sfs.f_type, NFS_SUPER_MAGIC) ||
1249                 F_TYPE_CMP(sfs.f_type, SMB_SUPER_MAGIC);
1250 }
1251
1252 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1253         const char *full, *tilded, *atted;
1254
1255         full = strappend(prefix, ".journal");
1256         tilded = strappenda(full, "~");
1257         atted = strappenda(prefix, "@");
1258
1259         return streq(filename, full) ||
1260                streq(filename, tilded) ||
1261                startswith(filename, atted);
1262 }
1263
1264 static bool file_type_wanted(int flags, const char *filename) {
1265         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1266                 return false;
1267
1268         /* no flags set → every type is OK */
1269         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1270                 return true;
1271
1272         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1273                 return true;
1274
1275         if (flags & SD_JOURNAL_CURRENT_USER) {
1276                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1277
1278                 assert_se(snprintf(prefix, sizeof(prefix), "user-%lu", (unsigned long) getuid())
1279                           < (int) sizeof(prefix));
1280
1281                 if (file_has_type_prefix(prefix, filename))
1282                         return true;
1283         }
1284
1285         return false;
1286 }
1287
1288 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1289         _cleanup_free_ char *path = NULL;
1290         int r;
1291         JournalFile *f;
1292
1293         assert(j);
1294         assert(prefix);
1295         assert(filename);
1296
1297         if (!file_type_wanted(j->flags, filename))
1298                 return 0;
1299
1300         path = strjoin(prefix, "/", filename, NULL);
1301         if (!path)
1302                 return -ENOMEM;
1303
1304         if (hashmap_get(j->files, path))
1305                 return 0;
1306
1307         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1308                 log_debug("Too many open journal files, not adding %s, ignoring.", path);
1309                 return set_put_error(j, -ETOOMANYREFS);
1310         }
1311
1312         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1313         if (r < 0) {
1314                 if (errno == ENOENT)
1315                         return 0;
1316
1317                 return r;
1318         }
1319
1320         /* journal_file_dump(f); */
1321
1322         r = hashmap_put(j->files, f->path, f);
1323         if (r < 0) {
1324                 journal_file_close(f);
1325                 return r;
1326         }
1327
1328         log_debug("File %s added.", f->path);
1329
1330         check_network(j, f->fd);
1331
1332         j->current_invalidate_counter ++;
1333
1334         return 0;
1335 }
1336
1337 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1338         char *path;
1339         JournalFile *f;
1340
1341         assert(j);
1342         assert(prefix);
1343         assert(filename);
1344
1345         path = strjoin(prefix, "/", filename, NULL);
1346         if (!path)
1347                 return -ENOMEM;
1348
1349         f = hashmap_get(j->files, path);
1350         free(path);
1351         if (!f)
1352                 return 0;
1353
1354         hashmap_remove(j->files, f->path);
1355
1356         log_debug("File %s removed.", f->path);
1357
1358         if (j->current_file == f) {
1359                 j->current_file = NULL;
1360                 j->current_field = 0;
1361         }
1362
1363         if (j->unique_file == f) {
1364                 j->unique_file = NULL;
1365                 j->unique_offset = 0;
1366         }
1367
1368         journal_file_close(f);
1369
1370         j->current_invalidate_counter ++;
1371
1372         return 0;
1373 }
1374
1375 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1376         _cleanup_free_ char *path = NULL;
1377         int r;
1378         _cleanup_closedir_ DIR *d = NULL;
1379         sd_id128_t id, mid;
1380         Directory *m;
1381
1382         assert(j);
1383         assert(prefix);
1384         assert(dirname);
1385
1386         log_debug("Considering %s/%s.", prefix, dirname);
1387
1388         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1389             (sd_id128_from_string(dirname, &id) < 0 ||
1390              sd_id128_get_machine(&mid) < 0 ||
1391              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1392             return 0;
1393
1394         path = strjoin(prefix, "/", dirname, NULL);
1395         if (!path)
1396                 return -ENOMEM;
1397
1398         d = opendir(path);
1399         if (!d) {
1400                 log_debug("Failed to open %s: %m", path);
1401                 if (errno == ENOENT)
1402                         return 0;
1403                 return -errno;
1404         }
1405
1406         m = hashmap_get(j->directories_by_path, path);
1407         if (!m) {
1408                 m = new0(Directory, 1);
1409                 if (!m)
1410                         return -ENOMEM;
1411
1412                 m->is_root = false;
1413                 m->path = path;
1414
1415                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1416                         free(m);
1417                         return -ENOMEM;
1418                 }
1419
1420                 path = NULL; /* avoid freeing in cleanup */
1421                 j->current_invalidate_counter ++;
1422
1423                 log_debug("Directory %s added.", m->path);
1424
1425         } else if (m->is_root)
1426                 return 0;
1427
1428         if (m->wd <= 0 && j->inotify_fd >= 0) {
1429
1430                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1431                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1432                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1433                                           IN_ONLYDIR);
1434
1435                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1436                         inotify_rm_watch(j->inotify_fd, m->wd);
1437         }
1438
1439         for (;;) {
1440                 struct dirent *de;
1441                 union dirent_storage buf;
1442
1443                 r = readdir_r(d, &buf.de, &de);
1444                 if (r != 0 || !de)
1445                         break;
1446
1447                 if (dirent_is_file_with_suffix(de, ".journal") ||
1448                     dirent_is_file_with_suffix(de, ".journal~")) {
1449                         r = add_file(j, m->path, de->d_name);
1450                         if (r < 0) {
1451                                 log_debug("Failed to add file %s/%s: %s",
1452                                           m->path, de->d_name, strerror(-r));
1453                                 r = set_put_error(j, r);
1454                                 if (r < 0)
1455                                         return r;
1456                         }
1457                 }
1458         }
1459
1460         check_network(j, dirfd(d));
1461
1462         return 0;
1463 }
1464
1465 static int add_root_directory(sd_journal *j, const char *p) {
1466         _cleanup_closedir_ DIR *d = NULL;
1467         Directory *m;
1468         int r;
1469
1470         assert(j);
1471         assert(p);
1472
1473         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1474             !path_startswith(p, "/run"))
1475                 return -EINVAL;
1476
1477         d = opendir(p);
1478         if (!d)
1479                 return -errno;
1480
1481         m = hashmap_get(j->directories_by_path, p);
1482         if (!m) {
1483                 m = new0(Directory, 1);
1484                 if (!m)
1485                         return -ENOMEM;
1486
1487                 m->is_root = true;
1488                 m->path = strdup(p);
1489                 if (!m->path) {
1490                         free(m);
1491                         return -ENOMEM;
1492                 }
1493
1494                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1495                         free(m->path);
1496                         free(m);
1497                         return -ENOMEM;
1498                 }
1499
1500                 j->current_invalidate_counter ++;
1501
1502                 log_debug("Root directory %s added.", m->path);
1503
1504         } else if (!m->is_root)
1505                 return 0;
1506
1507         if (m->wd <= 0 && j->inotify_fd >= 0) {
1508
1509                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1510                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1511                                           IN_ONLYDIR);
1512
1513                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1514                         inotify_rm_watch(j->inotify_fd, m->wd);
1515         }
1516
1517         for (;;) {
1518                 struct dirent *de;
1519                 union dirent_storage buf;
1520                 sd_id128_t id;
1521
1522                 r = readdir_r(d, &buf.de, &de);
1523                 if (r != 0 || !de)
1524                         break;
1525
1526                 if (dirent_is_file_with_suffix(de, ".journal") ||
1527                     dirent_is_file_with_suffix(de, ".journal~")) {
1528                         r = add_file(j, m->path, de->d_name);
1529                         if (r < 0) {
1530                                 log_debug("Failed to add file %s/%s: %s",
1531                                           m->path, de->d_name, strerror(-r));
1532                                 r = set_put_error(j, r);
1533                                 if (r < 0)
1534                                         return r;
1535                         }
1536                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1537                            sd_id128_from_string(de->d_name, &id) >= 0) {
1538
1539                         r = add_directory(j, m->path, de->d_name);
1540                         if (r < 0)
1541                                 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1542                 }
1543         }
1544
1545         check_network(j, dirfd(d));
1546
1547         return 0;
1548 }
1549
1550 static int remove_directory(sd_journal *j, Directory *d) {
1551         assert(j);
1552
1553         if (d->wd > 0) {
1554                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1555
1556                 if (j->inotify_fd >= 0)
1557                         inotify_rm_watch(j->inotify_fd, d->wd);
1558         }
1559
1560         hashmap_remove(j->directories_by_path, d->path);
1561
1562         if (d->is_root)
1563                 log_debug("Root directory %s removed.", d->path);
1564         else
1565                 log_debug("Directory %s removed.", d->path);
1566
1567         free(d->path);
1568         free(d);
1569
1570         return 0;
1571 }
1572
1573 static int add_search_paths(sd_journal *j) {
1574         int r;
1575         const char search_paths[] =
1576                 "/run/log/journal\0"
1577                 "/var/log/journal\0";
1578         const char *p;
1579
1580         assert(j);
1581
1582         /* We ignore most errors here, since the idea is to only open
1583          * what's actually accessible, and ignore the rest. */
1584
1585         NULSTR_FOREACH(p, search_paths) {
1586                 r = add_root_directory(j, p);
1587                 if (r < 0 && r != -ENOENT) {
1588                         r = set_put_error(j, r);
1589                         if (r < 0)
1590                                 return r;
1591                 }
1592         }
1593
1594         return 0;
1595 }
1596
1597 static int allocate_inotify(sd_journal *j) {
1598         assert(j);
1599
1600         if (j->inotify_fd < 0) {
1601                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1602                 if (j->inotify_fd < 0)
1603                         return -errno;
1604         }
1605
1606         if (!j->directories_by_wd) {
1607                 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1608                 if (!j->directories_by_wd)
1609                         return -ENOMEM;
1610         }
1611
1612         return 0;
1613 }
1614
1615 static sd_journal *journal_new(int flags, const char *path) {
1616         sd_journal *j;
1617
1618         j = new0(sd_journal, 1);
1619         if (!j)
1620                 return NULL;
1621
1622         j->inotify_fd = -1;
1623         j->flags = flags;
1624         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1625
1626         if (path) {
1627                 j->path = strdup(path);
1628                 if (!j->path)
1629                         goto fail;
1630         }
1631
1632         j->files = hashmap_new(string_hash_func, string_compare_func);
1633         j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1634         j->mmap = mmap_cache_new();
1635         if (!j->files || !j->directories_by_path || !j->mmap)
1636                 goto fail;
1637
1638         return j;
1639
1640 fail:
1641         sd_journal_close(j);
1642         return NULL;
1643 }
1644
1645 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1646         sd_journal *j;
1647         int r;
1648
1649         if (!ret)
1650                 return -EINVAL;
1651
1652         if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1653                       SD_JOURNAL_RUNTIME_ONLY|
1654                       SD_JOURNAL_SYSTEM|
1655                       SD_JOURNAL_CURRENT_USER))
1656                 return -EINVAL;
1657
1658         j = journal_new(flags, NULL);
1659         if (!j)
1660                 return -ENOMEM;
1661
1662         r = add_search_paths(j);
1663         if (r < 0)
1664                 goto fail;
1665
1666         *ret = j;
1667         return 0;
1668
1669 fail:
1670         sd_journal_close(j);
1671
1672         return r;
1673 }
1674
1675 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1676         sd_journal *j;
1677         int r;
1678
1679         if (!ret)
1680                 return -EINVAL;
1681
1682         if (!path)
1683                 return -EINVAL;
1684
1685         if (flags != 0)
1686                 return -EINVAL;
1687
1688         j = journal_new(flags, path);
1689         if (!j)
1690                 return -ENOMEM;
1691
1692         r = add_root_directory(j, path);
1693         if (r < 0) {
1694                 set_put_error(j, r);
1695                 goto fail;
1696         }
1697
1698         *ret = j;
1699         return 0;
1700
1701 fail:
1702         sd_journal_close(j);
1703
1704         return r;
1705 }
1706
1707 _public_ void sd_journal_close(sd_journal *j) {
1708         Directory *d;
1709         JournalFile *f;
1710
1711         if (!j)
1712                 return;
1713
1714         sd_journal_flush_matches(j);
1715
1716         while ((f = hashmap_steal_first(j->files)))
1717                 journal_file_close(f);
1718
1719         hashmap_free(j->files);
1720
1721         while ((d = hashmap_first(j->directories_by_path)))
1722                 remove_directory(j, d);
1723
1724         while ((d = hashmap_first(j->directories_by_wd)))
1725                 remove_directory(j, d);
1726
1727         hashmap_free(j->directories_by_path);
1728         hashmap_free(j->directories_by_wd);
1729
1730         if (j->inotify_fd >= 0)
1731                 close_nointr_nofail(j->inotify_fd);
1732
1733         if (j->mmap)
1734                 mmap_cache_unref(j->mmap);
1735
1736         free(j->path);
1737         free(j->unique_field);
1738         set_free(j->errors);
1739         free(j);
1740 }
1741
1742 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1743         Object *o;
1744         JournalFile *f;
1745         int r;
1746
1747         if (!j)
1748                 return -EINVAL;
1749         if (!ret)
1750                 return -EINVAL;
1751
1752         f = j->current_file;
1753         if (!f)
1754                 return -EADDRNOTAVAIL;
1755
1756         if (f->current_offset <= 0)
1757                 return -EADDRNOTAVAIL;
1758
1759         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1760         if (r < 0)
1761                 return r;
1762
1763         *ret = le64toh(o->entry.realtime);
1764         return 0;
1765 }
1766
1767 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1768         Object *o;
1769         JournalFile *f;
1770         int r;
1771         sd_id128_t id;
1772
1773         if (!j)
1774                 return -EINVAL;
1775
1776         f = j->current_file;
1777         if (!f)
1778                 return -EADDRNOTAVAIL;
1779
1780         if (f->current_offset <= 0)
1781                 return -EADDRNOTAVAIL;
1782
1783         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1784         if (r < 0)
1785                 return r;
1786
1787         if (ret_boot_id)
1788                 *ret_boot_id = o->entry.boot_id;
1789         else {
1790                 r = sd_id128_get_boot(&id);
1791                 if (r < 0)
1792                         return r;
1793
1794                 if (!sd_id128_equal(id, o->entry.boot_id))
1795                         return -ESTALE;
1796         }
1797
1798         if (ret)
1799                 *ret = le64toh(o->entry.monotonic);
1800
1801         return 0;
1802 }
1803
1804 static bool field_is_valid(const char *field) {
1805         const char *p;
1806
1807         assert(field);
1808
1809         if (isempty(field))
1810                 return false;
1811
1812         if (startswith(field, "__"))
1813                 return false;
1814
1815         for (p = field; *p; p++) {
1816
1817                 if (*p == '_')
1818                         continue;
1819
1820                 if (*p >= 'A' && *p <= 'Z')
1821                         continue;
1822
1823                 if (*p >= '0' && *p <= '9')
1824                         continue;
1825
1826                 return false;
1827         }
1828
1829         return true;
1830 }
1831
1832 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1833         JournalFile *f;
1834         uint64_t i, n;
1835         size_t field_length;
1836         int r;
1837         Object *o;
1838
1839         if (!j)
1840                 return -EINVAL;
1841         if (!field)
1842                 return -EINVAL;
1843         if (!data)
1844                 return -EINVAL;
1845         if (!size)
1846                 return -EINVAL;
1847
1848         if (!field_is_valid(field))
1849                 return -EINVAL;
1850
1851         f = j->current_file;
1852         if (!f)
1853                 return -EADDRNOTAVAIL;
1854
1855         if (f->current_offset <= 0)
1856                 return -EADDRNOTAVAIL;
1857
1858         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1859         if (r < 0)
1860                 return r;
1861
1862         field_length = strlen(field);
1863
1864         n = journal_file_entry_n_items(o);
1865         for (i = 0; i < n; i++) {
1866                 uint64_t p, l;
1867                 le64_t le_hash;
1868                 size_t t;
1869
1870                 p = le64toh(o->entry.items[i].object_offset);
1871                 le_hash = o->entry.items[i].hash;
1872                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1873                 if (r < 0)
1874                         return r;
1875
1876                 if (le_hash != o->data.hash)
1877                         return -EBADMSG;
1878
1879                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1880
1881                 if (o->object.flags & OBJECT_COMPRESSED) {
1882
1883 #ifdef HAVE_XZ
1884                         if (uncompress_startswith(o->data.payload, l,
1885                                                   &f->compress_buffer, &f->compress_buffer_size,
1886                                                   field, field_length, '=')) {
1887
1888                                 uint64_t rsize;
1889
1890                                 if (!uncompress_blob(o->data.payload, l,
1891                                                      &f->compress_buffer, &f->compress_buffer_size, &rsize,
1892                                                      j->data_threshold))
1893                                         return -EBADMSG;
1894
1895                                 *data = f->compress_buffer;
1896                                 *size = (size_t) rsize;
1897
1898                                 return 0;
1899                         }
1900 #else
1901                         return -EPROTONOSUPPORT;
1902 #endif
1903
1904                 } else if (l >= field_length+1 &&
1905                            memcmp(o->data.payload, field, field_length) == 0 &&
1906                            o->data.payload[field_length] == '=') {
1907
1908                         t = (size_t) l;
1909
1910                         if ((uint64_t) t != l)
1911                                 return -E2BIG;
1912
1913                         *data = o->data.payload;
1914                         *size = t;
1915
1916                         return 0;
1917                 }
1918
1919                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1920                 if (r < 0)
1921                         return r;
1922         }
1923
1924         return -ENOENT;
1925 }
1926
1927 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1928         size_t t;
1929         uint64_t l;
1930
1931         l = le64toh(o->object.size) - offsetof(Object, data.payload);
1932         t = (size_t) l;
1933
1934         /* We can't read objects larger than 4G on a 32bit machine */
1935         if ((uint64_t) t != l)
1936                 return -E2BIG;
1937
1938         if (o->object.flags & OBJECT_COMPRESSED) {
1939 #ifdef HAVE_XZ
1940                 uint64_t rsize;
1941
1942                 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
1943                         return -EBADMSG;
1944
1945                 *data = f->compress_buffer;
1946                 *size = (size_t) rsize;
1947 #else
1948                 return -EPROTONOSUPPORT;
1949 #endif
1950         } else {
1951                 *data = o->data.payload;
1952                 *size = t;
1953         }
1954
1955         return 0;
1956 }
1957
1958 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1959         JournalFile *f;
1960         uint64_t p, n;
1961         le64_t le_hash;
1962         int r;
1963         Object *o;
1964
1965         if (!j)
1966                 return -EINVAL;
1967         if (!data)
1968                 return -EINVAL;
1969         if (!size)
1970                 return -EINVAL;
1971
1972         f = j->current_file;
1973         if (!f)
1974                 return -EADDRNOTAVAIL;
1975
1976         if (f->current_offset <= 0)
1977                 return -EADDRNOTAVAIL;
1978
1979         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1980         if (r < 0)
1981                 return r;
1982
1983         n = journal_file_entry_n_items(o);
1984         if (j->current_field >= n)
1985                 return 0;
1986
1987         p = le64toh(o->entry.items[j->current_field].object_offset);
1988         le_hash = o->entry.items[j->current_field].hash;
1989         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1990         if (r < 0)
1991                 return r;
1992
1993         if (le_hash != o->data.hash)
1994                 return -EBADMSG;
1995
1996         r = return_data(j, f, o, data, size);
1997         if (r < 0)
1998                 return r;
1999
2000         j->current_field ++;
2001
2002         return 1;
2003 }
2004
2005 _public_ void sd_journal_restart_data(sd_journal *j) {
2006         if (!j)
2007                 return;
2008
2009         j->current_field = 0;
2010 }
2011
2012 _public_ int sd_journal_get_fd(sd_journal *j) {
2013         int r;
2014
2015         if (!j)
2016                 return -EINVAL;
2017
2018         if (j->inotify_fd >= 0)
2019                 return j->inotify_fd;
2020
2021         r = allocate_inotify(j);
2022         if (r < 0)
2023                 return r;
2024
2025         /* Iterate through all dirs again, to add them to the
2026          * inotify */
2027         if (j->path)
2028                 r = add_root_directory(j, j->path);
2029         else
2030                 r = add_search_paths(j);
2031         if (r < 0)
2032                 return r;
2033
2034         return j->inotify_fd;
2035 }
2036
2037 _public_ int sd_journal_get_events(sd_journal *j) {
2038         int fd;
2039
2040         if (!j)
2041                 return -EINVAL;
2042
2043         fd = sd_journal_get_fd(j);
2044         if (fd < 0)
2045                 return fd;
2046
2047         return POLLIN;
2048 }
2049
2050 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2051         int fd;
2052
2053         if (!j)
2054                 return -EINVAL;
2055         if (!timeout_usec)
2056                 return -EINVAL;
2057
2058         fd = sd_journal_get_fd(j);
2059         if (fd < 0)
2060                 return fd;
2061
2062         if (!j->on_network) {
2063                 *timeout_usec = (uint64_t) -1;
2064                 return 0;
2065         }
2066
2067         /* If we are on the network we need to regularly check for
2068          * changes manually */
2069
2070         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2071         return 1;
2072 }
2073
2074 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2075         Directory *d;
2076         int r;
2077
2078         assert(j);
2079         assert(e);
2080
2081         /* Is this a subdirectory we watch? */
2082         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2083         if (d) {
2084                 sd_id128_t id;
2085
2086                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2087                     (endswith(e->name, ".journal") ||
2088                      endswith(e->name, ".journal~"))) {
2089
2090                         /* Event for a journal file */
2091
2092                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2093                                 r = add_file(j, d->path, e->name);
2094                                 if (r < 0) {
2095                                         log_debug("Failed to add file %s/%s: %s",
2096                                                   d->path, e->name, strerror(-r));
2097                                         set_put_error(j, r);
2098                                 }
2099
2100                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2101
2102                                 r = remove_file(j, d->path, e->name);
2103                                 if (r < 0)
2104                                         log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2105                         }
2106
2107                 } else if (!d->is_root && e->len == 0) {
2108
2109                         /* Event for a subdirectory */
2110
2111                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2112                                 r = remove_directory(j, d);
2113                                 if (r < 0)
2114                                         log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2115                         }
2116
2117
2118                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2119
2120                         /* Event for root directory */
2121
2122                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2123                                 r = add_directory(j, d->path, e->name);
2124                                 if (r < 0)
2125                                         log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2126                         }
2127                 }
2128
2129                 return;
2130         }
2131
2132         if (e->mask & IN_IGNORED)
2133                 return;
2134
2135         log_warning("Unknown inotify event.");
2136 }
2137
2138 static int determine_change(sd_journal *j) {
2139         bool b;
2140
2141         assert(j);
2142
2143         b = j->current_invalidate_counter != j->last_invalidate_counter;
2144         j->last_invalidate_counter = j->current_invalidate_counter;
2145
2146         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2147 }
2148
2149 _public_ int sd_journal_process(sd_journal *j) {
2150         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2151         bool got_something = false;
2152
2153         if (!j)
2154                 return -EINVAL;
2155
2156         j->last_process_usec = now(CLOCK_MONOTONIC);
2157
2158         for (;;) {
2159                 struct inotify_event *e;
2160                 ssize_t l;
2161
2162                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2163                 if (l < 0) {
2164                         if (errno == EAGAIN || errno == EINTR)
2165                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2166
2167                         return -errno;
2168                 }
2169
2170                 got_something = true;
2171
2172                 e = (struct inotify_event*) buffer;
2173                 while (l > 0) {
2174                         size_t step;
2175
2176                         process_inotify_event(j, e);
2177
2178                         step = sizeof(struct inotify_event) + e->len;
2179                         assert(step <= (size_t) l);
2180
2181                         e = (struct inotify_event*) ((uint8_t*) e + step);
2182                         l -= step;
2183                 }
2184         }
2185
2186         return determine_change(j);
2187 }
2188
2189 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2190         int r;
2191         uint64_t t;
2192
2193         assert(j);
2194
2195         if (j->inotify_fd < 0) {
2196
2197                 /* This is the first invocation, hence create the
2198                  * inotify watch */
2199                 r = sd_journal_get_fd(j);
2200                 if (r < 0)
2201                         return r;
2202
2203                 /* The journal might have changed since the context
2204                  * object was created and we weren't watching before,
2205                  * hence don't wait for anything, and return
2206                  * immediately. */
2207                 return determine_change(j);
2208         }
2209
2210         r = sd_journal_get_timeout(j, &t);
2211         if (r < 0)
2212                 return r;
2213
2214         if (t != (uint64_t) -1) {
2215                 usec_t n;
2216
2217                 n = now(CLOCK_MONOTONIC);
2218                 t = t > n ? t - n : 0;
2219
2220                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2221                         timeout_usec = t;
2222         }
2223
2224         do {
2225                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2226         } while (r == -EINTR);
2227
2228         if (r < 0)
2229                 return r;
2230
2231         return sd_journal_process(j);
2232 }
2233
2234 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2235         Iterator i;
2236         JournalFile *f;
2237         bool first = true;
2238         int r;
2239
2240         if (!j)
2241                 return -EINVAL;
2242         if (!from && !to)
2243                 return -EINVAL;
2244         if (from == to)
2245                 return -EINVAL;
2246
2247         HASHMAP_FOREACH(f, j->files, i) {
2248                 usec_t fr, t;
2249
2250                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2251                 if (r == -ENOENT)
2252                         continue;
2253                 if (r < 0)
2254                         return r;
2255                 if (r == 0)
2256                         continue;
2257
2258                 if (first) {
2259                         if (from)
2260                                 *from = fr;
2261                         if (to)
2262                                 *to = t;
2263                         first = false;
2264                 } else {
2265                         if (from)
2266                                 *from = MIN(fr, *from);
2267                         if (to)
2268                                 *to = MAX(t, *to);
2269                 }
2270         }
2271
2272         return first ? 0 : 1;
2273 }
2274
2275 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2276         Iterator i;
2277         JournalFile *f;
2278         bool first = true;
2279         int r;
2280
2281         if (!j)
2282                 return -EINVAL;
2283         if (!from && !to)
2284                 return -EINVAL;
2285         if (from == to)
2286                 return -EINVAL;
2287
2288         HASHMAP_FOREACH(f, j->files, i) {
2289                 usec_t fr, t;
2290
2291                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2292                 if (r == -ENOENT)
2293                         continue;
2294                 if (r < 0)
2295                         return r;
2296                 if (r == 0)
2297                         continue;
2298
2299                 if (first) {
2300                         if (from)
2301                                 *from = fr;
2302                         if (to)
2303                                 *to = t;
2304                         first = false;
2305                 } else {
2306                         if (from)
2307                                 *from = MIN(fr, *from);
2308                         if (to)
2309                                 *to = MAX(t, *to);
2310                 }
2311         }
2312
2313         return first ? 0 : 1;
2314 }
2315
2316 void journal_print_header(sd_journal *j) {
2317         Iterator i;
2318         JournalFile *f;
2319         bool newline = false;
2320
2321         assert(j);
2322
2323         HASHMAP_FOREACH(f, j->files, i) {
2324                 if (newline)
2325                         putchar('\n');
2326                 else
2327                         newline = true;
2328
2329                 journal_file_print_header(f);
2330         }
2331 }
2332
2333 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2334         Iterator i;
2335         JournalFile *f;
2336         uint64_t sum = 0;
2337
2338         if (!j)
2339                 return -EINVAL;
2340         if (!bytes)
2341                 return -EINVAL;
2342
2343         HASHMAP_FOREACH(f, j->files, i) {
2344                 struct stat st;
2345
2346                 if (fstat(f->fd, &st) < 0)
2347                         return -errno;
2348
2349                 sum += (uint64_t) st.st_blocks * 512ULL;
2350         }
2351
2352         *bytes = sum;
2353         return 0;
2354 }
2355
2356 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2357         char *f;
2358
2359         if (!j)
2360                 return -EINVAL;
2361         if (isempty(field))
2362                 return -EINVAL;
2363         if (!field_is_valid(field))
2364                 return -EINVAL;
2365
2366         f = strdup(field);
2367         if (!f)
2368                 return -ENOMEM;
2369
2370         free(j->unique_field);
2371         j->unique_field = f;
2372         j->unique_file = NULL;
2373         j->unique_offset = 0;
2374
2375         return 0;
2376 }
2377
2378 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2379         Object *o;
2380         size_t k;
2381         int r;
2382
2383         if (!j)
2384                 return -EINVAL;
2385         if (!data)
2386                 return -EINVAL;
2387         if (!l)
2388                 return -EINVAL;
2389         if (!j->unique_field)
2390                 return -EINVAL;
2391
2392         k = strlen(j->unique_field);
2393
2394         if (!j->unique_file) {
2395                 j->unique_file = hashmap_first(j->files);
2396                 if (!j->unique_file)
2397                         return 0;
2398                 j->unique_offset = 0;
2399         }
2400
2401         for (;;) {
2402                 JournalFile *of;
2403                 Iterator i;
2404                 const void *odata;
2405                 size_t ol;
2406                 bool found;
2407
2408                 /* Proceed to next data object in the field's linked list */
2409                 if (j->unique_offset == 0) {
2410                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2411                         if (r < 0)
2412                                 return r;
2413
2414                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2415                 } else {
2416                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2417                         if (r < 0)
2418                                 return r;
2419
2420                         j->unique_offset = le64toh(o->data.next_field_offset);
2421                 }
2422
2423                 /* We reached the end of the list? Then start again, with the next file */
2424                 if (j->unique_offset == 0) {
2425                         JournalFile *n;
2426
2427                         n = hashmap_next(j->files, j->unique_file->path);
2428                         if (!n)
2429                                 return 0;
2430
2431                         j->unique_file = n;
2432                         continue;
2433                 }
2434
2435                 /* We do not use the type context here, but 0 instead,
2436                  * so that we can look at this data object at the same
2437                  * time as one on another file */
2438                 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2439                 if (r < 0)
2440                         return r;
2441
2442                 /* Let's do the type check by hand, since we used 0 context above. */
2443                 if (o->object.type != OBJECT_DATA)
2444                         return -EBADMSG;
2445
2446                 r = return_data(j, j->unique_file, o, &odata, &ol);
2447                 if (r < 0)
2448                         return r;
2449
2450                 /* OK, now let's see if we already returned this data
2451                  * object by checking if it exists in the earlier
2452                  * traversed files. */
2453                 found = false;
2454                 HASHMAP_FOREACH(of, j->files, i) {
2455                         Object *oo;
2456                         uint64_t op;
2457
2458                         if (of == j->unique_file)
2459                                 break;
2460
2461                         /* Skip this file it didn't have any fields
2462                          * indexed */
2463                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2464                             le64toh(of->header->n_fields) <= 0)
2465                                 continue;
2466
2467                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2468                         if (r < 0)
2469                                 return r;
2470
2471                         if (r > 0)
2472                                 found = true;
2473                 }
2474
2475                 if (found)
2476                         continue;
2477
2478                 r = return_data(j, j->unique_file, o, data, l);
2479                 if (r < 0)
2480                         return r;
2481
2482                 return 1;
2483         }
2484 }
2485
2486 _public_ void sd_journal_restart_unique(sd_journal *j) {
2487         if (!j)
2488                 return;
2489
2490         j->unique_file = NULL;
2491         j->unique_offset = 0;
2492 }
2493
2494 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2495         if (!j)
2496                 return -EINVAL;
2497
2498         return !j->on_network;
2499 }
2500
2501 static char *lookup_field(const char *field, void *userdata) {
2502         sd_journal *j = userdata;
2503         const void *data;
2504         size_t size, d;
2505         int r;
2506
2507         assert(field);
2508         assert(j);
2509
2510         r = sd_journal_get_data(j, field, &data, &size);
2511         if (r < 0 ||
2512             size > REPLACE_VAR_MAX)
2513                 return strdup(field);
2514
2515         d = strlen(field) + 1;
2516
2517         return strndup((const char*) data + d, size - d);
2518 }
2519
2520 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2521         const void *data;
2522         size_t size;
2523         sd_id128_t id;
2524         _cleanup_free_ char *text = NULL, *cid = NULL;
2525         char *t;
2526         int r;
2527
2528         if (!j)
2529                 return -EINVAL;
2530         if (!ret)
2531                 return -EINVAL;
2532
2533         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2534         if (r < 0)
2535                 return r;
2536
2537         cid = strndup((const char*) data + 11, size - 11);
2538         if (!cid)
2539                 return -ENOMEM;
2540
2541         r = sd_id128_from_string(cid, &id);
2542         if (r < 0)
2543                 return r;
2544
2545         r = catalog_get(CATALOG_DATABASE, id, &text);
2546         if (r < 0)
2547                 return r;
2548
2549         t = replace_var(text, lookup_field, j);
2550         if (!t)
2551                 return -ENOMEM;
2552
2553         *ret = t;
2554         return 0;
2555 }
2556
2557 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2558         if (!ret)
2559                 return -EINVAL;
2560
2561         return catalog_get(CATALOG_DATABASE, id, ret);
2562 }
2563
2564 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2565         if (!j)
2566                 return -EINVAL;
2567
2568         j->data_threshold = sz;
2569         return 0;
2570 }
2571
2572 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2573         if (!j)
2574                 return -EINVAL;
2575         if (!sz)
2576                 return -EINVAL;
2577
2578         *sz = j->data_threshold;
2579         return 0;
2580 }