src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "path-util.h"
  37 #include "lookup3.h"
  38 #include "compress.h"
  39 #include "journal-internal.h"
  40 #include "missing.h"
  41 #include "catalog.h"
  42 #include "replace-var.h"
  43
  44 #define JOURNAL_FILES_MAX 1024
  45
  46 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  47
  48 #define REPLACE_VAR_MAX 256
  49
  50 #define DEFAULT_DATA_THRESHOLD (64*1024)
  51
  52 /* We return an error here only if we didn't manage to
  53    memorize the real error. */
  54 static int set_put_error(sd_journal *j, int r) {
  55         int k;
  56
  57         if (r >= 0)
  58                 return r;
  59
  60         k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
  61         if (k < 0)
  62                 return k;
  63
  64         return set_put(j->errors, INT_TO_PTR(r));
  65 }
  66
  67 static void detach_location(sd_journal *j) {
  68         Iterator i;
  69         JournalFile *f;
  70
  71         assert(j);
  72
  73         j->current_file = NULL;
  74         j->current_field = 0;
  75
  76         HASHMAP_FOREACH(f, j->files, i)
  77                 f->current_offset = 0;
  78 }
  79
  80 static void reset_location(sd_journal *j) {
  81         assert(j);
  82
  83         detach_location(j);
  84         zero(j->current_location);
  85 }
  86
  87 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  88         assert(l);
  89         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
  90         assert(f);
  91         assert(o->object.type == OBJECT_ENTRY);
  92
  93         l->type = type;
  94         l->seqnum = le64toh(o->entry.seqnum);
  95         l->seqnum_id = f->header->seqnum_id;
  96         l->realtime = le64toh(o->entry.realtime);
  97         l->monotonic = le64toh(o->entry.monotonic);
  98         l->boot_id = o->entry.boot_id;
  99         l->xor_hash = le64toh(o->entry.xor_hash);
 100
 101         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 102 }
 103
 104 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o, uint64_t offset) {
 105         assert(j);
 106         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 107         assert(f);
 108         assert(o);
 109
 110         init_location(&j->current_location, type, f, o);
 111
 112         if (j->current_file)
 113                 j->current_file->current_offset = 0;
 114
 115         j->current_file = f;
 116         j->current_field = 0;
 117
 118         f->current_offset = offset;
 119 }
 120
 121 static int match_is_valid(const void *data, size_t size) {
 122         const char *b, *p;
 123
 124         assert(data);
 125
 126         if (size < 2)
 127                 return false;
 128
 129         if (startswith(data, "__"))
 130                 return false;
 131
 132         b = data;
 133         for (p = b; p < b + size; p++) {
 134
 135                 if (*p == '=')
 136                         return p > b;
 137
 138                 if (*p == '_')
 139                         continue;
 140
 141                 if (*p >= 'A' && *p <= 'Z')
 142                         continue;
 143
 144                 if (*p >= '0' && *p <= '9')
 145                         continue;
 146
 147                 return false;
 148         }
 149
 150         return false;
 151 }
 152
 153 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 154         const uint8_t *a = _a, *b = _b;
 155         size_t j;
 156
 157         for (j = 0; j < s && j < t; j++) {
 158
 159                 if (a[j] != b[j])
 160                         return false;
 161
 162                 if (a[j] == '=')
 163                         return true;
 164         }
 165
 166         return true;
 167 }
 168
 169 static Match *match_new(Match *p, MatchType t) {
 170         Match *m;
 171
 172         m = new0(Match, 1);
 173         if (!m)
 174                 return NULL;
 175
 176         m->type = t;
 177
 178         if (p) {
 179                 m->parent = p;
 180                 LIST_PREPEND(Match, matches, p->matches, m);
 181         }
 182
 183         return m;
 184 }
 185
 186 static void match_free(Match *m) {
 187         assert(m);
 188
 189         while (m->matches)
 190                 match_free(m->matches);
 191
 192         if (m->parent)
 193                 LIST_REMOVE(Match, matches, m->parent->matches, m);
 194
 195         free(m->data);
 196         free(m);
 197 }
 198
 199 static void match_free_if_empty(Match *m) {
 200         if (!m || m->matches)
 201                 return;
 202
 203         match_free(m);
 204 }
 205
 206 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 207         Match *l3, *l4, *add_here = NULL, *m;
 208         le64_t le_hash;
 209
 210         if (!j)
 211                 return -EINVAL;
 212
 213         if (!data)
 214                 return -EINVAL;
 215
 216         if (size == 0)
 217                 size = strlen(data);
 218
 219         if (!match_is_valid(data, size))
 220                 return -EINVAL;
 221
 222         /* level 0: AND term
 223          * level 1: OR terms
 224          * level 2: AND terms
 225          * level 3: OR terms
 226          * level 4: concrete matches */
 227
 228         if (!j->level0) {
 229                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 230                 if (!j->level0)
 231                         return -ENOMEM;
 232         }
 233
 234         if (!j->level1) {
 235                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 236                 if (!j->level1)
 237                         return -ENOMEM;
 238         }
 239
 240         if (!j->level2) {
 241                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 242                 if (!j->level2)
 243                         return -ENOMEM;
 244         }
 245
 246         assert(j->level0->type == MATCH_AND_TERM);
 247         assert(j->level1->type == MATCH_OR_TERM);
 248         assert(j->level2->type == MATCH_AND_TERM);
 249
 250         le_hash = htole64(hash64(data, size));
 251
 252         LIST_FOREACH(matches, l3, j->level2->matches) {
 253                 assert(l3->type == MATCH_OR_TERM);
 254
 255                 LIST_FOREACH(matches, l4, l3->matches) {
 256                         assert(l4->type == MATCH_DISCRETE);
 257
 258                         /* Exactly the same match already? Then ignore
 259                          * this addition */
 260                         if (l4->le_hash == le_hash &&
 261                             l4->size == size &&
 262                             memcmp(l4->data, data, size) == 0)
 263                                 return 0;
 264
 265                         /* Same field? Then let's add this to this OR term */
 266                         if (same_field(data, size, l4->data, l4->size)) {
 267                                 add_here = l3;
 268                                 break;
 269                         }
 270                 }
 271
 272                 if (add_here)
 273                         break;
 274         }
 275
 276         if (!add_here) {
 277                 add_here = match_new(j->level2, MATCH_OR_TERM);
 278                 if (!add_here)
 279                         goto fail;
 280         }
 281
 282         m = match_new(add_here, MATCH_DISCRETE);
 283         if (!m)
 284                 goto fail;
 285
 286         m->le_hash = le_hash;
 287         m->size = size;
 288         m->data = memdup(data, size);
 289         if (!m->data)
 290                 goto fail;
 291
 292         detach_location(j);
 293
 294         return 0;
 295
 296 fail:
 297         match_free_if_empty(add_here);
 298         match_free_if_empty(j->level2);
 299         match_free_if_empty(j->level1);
 300         match_free_if_empty(j->level0);
 301
 302         return -ENOMEM;
 303 }
 304
 305 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 306         assert(j);
 307
 308         if (!j->level0)
 309                 return 0;
 310
 311         if (!j->level1)
 312                 return 0;
 313
 314         if (!j->level1->matches)
 315                 return 0;
 316
 317         j->level1 = NULL;
 318         j->level2 = NULL;
 319
 320         return 0;
 321 }
 322
 323 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 324         assert(j);
 325
 326         if (!j->level0)
 327                 return 0;
 328
 329         if (!j->level1)
 330                 return 0;
 331
 332         if (!j->level2)
 333                 return 0;
 334
 335         if (!j->level2->matches)
 336                 return 0;
 337
 338         j->level2 = NULL;
 339         return 0;
 340 }
 341
 342 static char *match_make_string(Match *m) {
 343         char *p, *r;
 344         Match *i;
 345         bool enclose = false;
 346
 347         if (!m)
 348                 return strdup("");
 349
 350         if (m->type == MATCH_DISCRETE)
 351                 return strndup(m->data, m->size);
 352
 353         p = NULL;
 354         LIST_FOREACH(matches, i, m->matches) {
 355                 char *t, *k;
 356
 357                 t = match_make_string(i);
 358                 if (!t) {
 359                         free(p);
 360                         return NULL;
 361                 }
 362
 363                 if (p) {
 364                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 365                         free(p);
 366                         free(t);
 367
 368                         if (!k)
 369                                 return NULL;
 370
 371                         p = k;
 372
 373                         enclose = true;
 374                 } else {
 375                         free(p);
 376                         p = t;
 377                 }
 378         }
 379
 380         if (enclose) {
 381                 r = strjoin("(", p, ")", NULL);
 382                 free(p);
 383                 return r;
 384         }
 385
 386         return p;
 387 }
 388
 389 char *journal_make_match_string(sd_journal *j) {
 390         assert(j);
 391
 392         return match_make_string(j->level0);
 393 }
 394
 395 _public_ void sd_journal_flush_matches(sd_journal *j) {
 396
 397         if (!j)
 398                 return;
 399
 400         if (j->level0)
 401                 match_free(j->level0);
 402
 403         j->level0 = j->level1 = j->level2 = NULL;
 404
 405         detach_location(j);
 406 }
 407
 408 static int compare_entry_order(JournalFile *af, Object *_ao,
 409                                JournalFile *bf, uint64_t bp) {
 410
 411         uint64_t a, b;
 412         Object *ao, *bo;
 413         int r;
 414
 415         assert(af);
 416         assert(bf);
 417         assert(_ao);
 418
 419         /* The mmap cache might invalidate the object from the first
 420          * file if we look at the one from the second file. Hence
 421          * temporarily copy the header of the first one, and look at
 422          * that only. */
 423         ao = alloca(offsetof(EntryObject, items));
 424         memcpy(ao, _ao, offsetof(EntryObject, items));
 425
 426         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 427         if (r < 0)
 428                 return strcmp(af->path, bf->path);
 429
 430         /* We operate on two different files here, hence we can access
 431          * two objects at the same time, which we normally can't.
 432          *
 433          * If contents and timestamps match, these entries are
 434          * identical, even if the seqnum does not match */
 435
 436         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 437             ao->entry.monotonic == bo->entry.monotonic &&
 438             ao->entry.realtime == bo->entry.realtime &&
 439             ao->entry.xor_hash == bo->entry.xor_hash)
 440                 return 0;
 441
 442         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 443
 444                 /* If this is from the same seqnum source, compare
 445                  * seqnums */
 446                 a = le64toh(ao->entry.seqnum);
 447                 b = le64toh(bo->entry.seqnum);
 448
 449                 if (a < b)
 450                         return -1;
 451                 if (a > b)
 452                         return 1;
 453
 454                 /* Wow! This is weird, different data but the same
 455                  * seqnums? Something is borked, but let's make the
 456                  * best of it and compare by time. */
 457         }
 458
 459         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 460
 461                 /* If the boot id matches compare monotonic time */
 462                 a = le64toh(ao->entry.monotonic);
 463                 b = le64toh(bo->entry.monotonic);
 464
 465                 if (a < b)
 466                         return -1;
 467                 if (a > b)
 468                         return 1;
 469         }
 470
 471         /* Otherwise compare UTC time */
 472         a = le64toh(ao->entry.realtime);
 473         b = le64toh(bo->entry.realtime);
 474
 475         if (a < b)
 476                 return -1;
 477         if (a > b)
 478                 return 1;
 479
 480         /* Finally, compare by contents */
 481         a = le64toh(ao->entry.xor_hash);
 482         b = le64toh(bo->entry.xor_hash);
 483
 484         if (a < b)
 485                 return -1;
 486         if (a > b)
 487                 return 1;
 488
 489         return 0;
 490 }
 491
 492 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 493         uint64_t a;
 494
 495         assert(af);
 496         assert(ao);
 497         assert(l);
 498         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 499
 500         if (l->monotonic_set &&
 501             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 502             l->realtime_set &&
 503             le64toh(ao->entry.realtime) == l->realtime &&
 504             l->xor_hash_set &&
 505             le64toh(ao->entry.xor_hash) == l->xor_hash)
 506                 return 0;
 507
 508         if (l->seqnum_set &&
 509             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 510
 511                 a = le64toh(ao->entry.seqnum);
 512
 513                 if (a < l->seqnum)
 514                         return -1;
 515                 if (a > l->seqnum)
 516                         return 1;
 517         }
 518
 519         if (l->monotonic_set &&
 520             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 521
 522                 a = le64toh(ao->entry.monotonic);
 523
 524                 if (a < l->monotonic)
 525                         return -1;
 526                 if (a > l->monotonic)
 527                         return 1;
 528         }
 529
 530         if (l->realtime_set) {
 531
 532                 a = le64toh(ao->entry.realtime);
 533
 534                 if (a < l->realtime)
 535                         return -1;
 536                 if (a > l->realtime)
 537                         return 1;
 538         }
 539
 540         if (l->xor_hash_set) {
 541                 a = le64toh(ao->entry.xor_hash);
 542
 543                 if (a < l->xor_hash)
 544                         return -1;
 545                 if (a > l->xor_hash)
 546                         return 1;
 547         }
 548
 549         return 0;
 550 }
 551
 552 static int next_for_match(
 553                 sd_journal *j,
 554                 Match *m,
 555                 JournalFile *f,
 556                 uint64_t after_offset,
 557                 direction_t direction,
 558                 Object **ret,
 559                 uint64_t *offset) {
 560
 561         int r;
 562         uint64_t np = 0;
 563         Object *n;
 564
 565         assert(j);
 566         assert(m);
 567         assert(f);
 568
 569         if (m->type == MATCH_DISCRETE) {
 570                 uint64_t dp;
 571
 572                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 573                 if (r <= 0)
 574                         return r;
 575
 576                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 577
 578         } else if (m->type == MATCH_OR_TERM) {
 579                 Match *i;
 580
 581                 /* Find the earliest match beyond after_offset */
 582
 583                 LIST_FOREACH(matches, i, m->matches) {
 584                         uint64_t cp;
 585
 586                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 587                         if (r < 0)
 588                                 return r;
 589                         else if (r > 0) {
 590                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 591                                         np = cp;
 592                         }
 593                 }
 594
 595         } else if (m->type == MATCH_AND_TERM) {
 596                 Match *i, *last_moved;
 597
 598                 /* Always jump to the next matching entry and repeat
 599                  * this until we find an offset that matches for all
 600                  * matches. */
 601
 602                 if (!m->matches)
 603                         return 0;
 604
 605                 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
 606                 if (r <= 0)
 607                         return r;
 608
 609                 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
 610                 last_moved = m->matches;
 611
 612                 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
 613                         uint64_t cp;
 614
 615                         r = next_for_match(j, i, f, np, direction, NULL, &cp);
 616                         if (r <= 0)
 617                                 return r;
 618
 619                         assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
 620                         if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
 621                                 np = cp;
 622                                 last_moved = i;
 623                         }
 624                 }
 625         }
 626
 627         if (np == 0)
 628                 return 0;
 629
 630         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 631         if (r < 0)
 632                 return r;
 633
 634         if (ret)
 635                 *ret = n;
 636         if (offset)
 637                 *offset = np;
 638
 639         return 1;
 640 }
 641
 642 static int find_location_for_match(
 643                 sd_journal *j,
 644                 Match *m,
 645                 JournalFile *f,
 646                 direction_t direction,
 647                 Object **ret,
 648                 uint64_t *offset) {
 649
 650         int r;
 651
 652         assert(j);
 653         assert(m);
 654         assert(f);
 655
 656         if (m->type == MATCH_DISCRETE) {
 657                 uint64_t dp;
 658
 659                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 660                 if (r <= 0)
 661                         return r;
 662
 663                 /* FIXME: missing: find by monotonic */
 664
 665                 if (j->current_location.type == LOCATION_HEAD)
 666                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 667                 if (j->current_location.type == LOCATION_TAIL)
 668                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 669                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 670                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 671                 if (j->current_location.monotonic_set) {
 672                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 673                         if (r != -ENOENT)
 674                                 return r;
 675                 }
 676                 if (j->current_location.realtime_set)
 677                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 678
 679                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 680
 681         } else if (m->type == MATCH_OR_TERM) {
 682                 uint64_t np = 0;
 683                 Object *n;
 684                 Match *i;
 685
 686                 /* Find the earliest match */
 687
 688                 LIST_FOREACH(matches, i, m->matches) {
 689                         uint64_t cp;
 690
 691                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 692                         if (r < 0)
 693                                 return r;
 694                         else if (r > 0) {
 695                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 696                                         np = cp;
 697                         }
 698                 }
 699
 700                 if (np == 0)
 701                         return 0;
 702
 703                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 704                 if (r < 0)
 705                         return r;
 706
 707                 if (ret)
 708                         *ret = n;
 709                 if (offset)
 710                         *offset = np;
 711
 712                 return 1;
 713
 714         } else {
 715                 Match *i;
 716                 uint64_t np = 0;
 717
 718                 assert(m->type == MATCH_AND_TERM);
 719
 720                 /* First jump to the last match, and then find the
 721                  * next one where all matches match */
 722
 723                 if (!m->matches)
 724                         return 0;
 725
 726                 LIST_FOREACH(matches, i, m->matches) {
 727                         uint64_t cp;
 728
 729                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 730                         if (r <= 0)
 731                                 return r;
 732
 733                         if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
 734                                 np = cp;
 735                 }
 736
 737                 return next_for_match(j, m, f, np, direction, ret, offset);
 738         }
 739 }
 740
 741 static int find_location_with_matches(
 742                 sd_journal *j,
 743                 JournalFile *f,
 744                 direction_t direction,
 745                 Object **ret,
 746                 uint64_t *offset) {
 747
 748         int r;
 749
 750         assert(j);
 751         assert(f);
 752         assert(ret);
 753         assert(offset);
 754
 755         if (!j->level0) {
 756                 /* No matches is simple */
 757
 758                 if (j->current_location.type == LOCATION_HEAD)
 759                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 760                 if (j->current_location.type == LOCATION_TAIL)
 761                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 762                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 763                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 764                 if (j->current_location.monotonic_set) {
 765                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 766                         if (r != -ENOENT)
 767                                 return r;
 768                 }
 769                 if (j->current_location.realtime_set)
 770                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 771
 772                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 773         } else
 774                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 775 }
 776
 777 static int next_with_matches(
 778                 sd_journal *j,
 779                 JournalFile *f,
 780                 direction_t direction,
 781                 Object **ret,
 782                 uint64_t *offset) {
 783
 784         Object *c;
 785         uint64_t cp;
 786
 787         assert(j);
 788         assert(f);
 789         assert(ret);
 790         assert(offset);
 791
 792         c = *ret;
 793         cp = *offset;
 794
 795         /* No matches is easy. We simple advance the file
 796          * pointer by one. */
 797         if (!j->level0)
 798                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 799
 800         /* If we have a match then we look for the next matching entry
 801          * with an offset at least one step larger */
 802         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 803 }
 804
 805 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 806         Object *c;
 807         uint64_t cp;
 808         int r;
 809
 810         assert(j);
 811         assert(f);
 812
 813         if (f->current_offset > 0) {
 814                 cp = f->current_offset;
 815
 816                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 817                 if (r < 0)
 818                         return r;
 819
 820                 r = next_with_matches(j, f, direction, &c, &cp);
 821                 if (r <= 0)
 822                         return r;
 823         } else {
 824                 r = find_location_with_matches(j, f, direction, &c, &cp);
 825                 if (r <= 0)
 826                         return r;
 827         }
 828
 829         /* OK, we found the spot, now let's advance until to an entry
 830          * that is actually different from what we were previously
 831          * looking at. This is necessary to handle entries which exist
 832          * in two (or more) journal files, and which shall all be
 833          * suppressed but one. */
 834
 835         for (;;) {
 836                 bool found;
 837
 838                 if (j->current_location.type == LOCATION_DISCRETE) {
 839                         int k;
 840
 841                         k = compare_with_location(f, c, &j->current_location);
 842                         if (direction == DIRECTION_DOWN)
 843                                 found = k > 0;
 844                         else
 845                                 found = k < 0;
 846                 } else
 847                         found = true;
 848
 849                 if (found) {
 850                         if (ret)
 851                                 *ret = c;
 852                         if (offset)
 853                                 *offset = cp;
 854                         return 1;
 855                 }
 856
 857                 r = next_with_matches(j, f, direction, &c, &cp);
 858                 if (r <= 0)
 859                         return r;
 860         }
 861 }
 862
 863 static int real_journal_next(sd_journal *j, direction_t direction) {
 864         JournalFile *f, *new_file = NULL;
 865         uint64_t new_offset = 0;
 866         Object *o;
 867         uint64_t p;
 868         Iterator i;
 869         int r;
 870
 871         if (!j)
 872                 return -EINVAL;
 873
 874         HASHMAP_FOREACH(f, j->files, i) {
 875                 bool found;
 876
 877                 r = next_beyond_location(j, f, direction, &o, &p);
 878                 if (r < 0) {
 879                         log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
 880                         continue;
 881                 } else if (r == 0)
 882                         continue;
 883
 884                 if (!new_file)
 885                         found = true;
 886                 else {
 887                         int k;
 888
 889                         k = compare_entry_order(f, o, new_file, new_offset);
 890
 891                         if (direction == DIRECTION_DOWN)
 892                                 found = k < 0;
 893                         else
 894                                 found = k > 0;
 895                 }
 896
 897                 if (found) {
 898                         new_file = f;
 899                         new_offset = p;
 900                 }
 901         }
 902
 903         if (!new_file)
 904                 return 0;
 905
 906         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 907         if (r < 0)
 908                 return r;
 909
 910         set_location(j, LOCATION_DISCRETE, new_file, o, new_offset);
 911
 912         return 1;
 913 }
 914
 915 _public_ int sd_journal_next(sd_journal *j) {
 916         return real_journal_next(j, DIRECTION_DOWN);
 917 }
 918
 919 _public_ int sd_journal_previous(sd_journal *j) {
 920         return real_journal_next(j, DIRECTION_UP);
 921 }
 922
 923 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 924         int c = 0, r;
 925
 926         if (!j)
 927                 return -EINVAL;
 928
 929         if (skip == 0) {
 930                 /* If this is not a discrete skip, then at least
 931                  * resolve the current location */
 932                 if (j->current_location.type != LOCATION_DISCRETE)
 933                         return real_journal_next(j, direction);
 934
 935                 return 0;
 936         }
 937
 938         do {
 939                 r = real_journal_next(j, direction);
 940                 if (r < 0)
 941                         return r;
 942
 943                 if (r == 0)
 944                         return c;
 945
 946                 skip--;
 947                 c++;
 948         } while (skip > 0);
 949
 950         return c;
 951 }
 952
 953 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 954         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 955 }
 956
 957 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 958         return real_journal_next_skip(j, DIRECTION_UP, skip);
 959 }
 960
 961 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 962         Object *o;
 963         int r;
 964         char bid[33], sid[33];
 965
 966         if (!j)
 967                 return -EINVAL;
 968         if (!cursor)
 969                 return -EINVAL;
 970
 971         if (!j->current_file || j->current_file->current_offset <= 0)
 972                 return -EADDRNOTAVAIL;
 973
 974         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 975         if (r < 0)
 976                 return r;
 977
 978         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 979         sd_id128_to_string(o->entry.boot_id, bid);
 980
 981         if (asprintf(cursor,
 982                      "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
 983                      sid, le64toh(o->entry.seqnum),
 984                      bid, le64toh(o->entry.monotonic),
 985                      le64toh(o->entry.realtime),
 986                      le64toh(o->entry.xor_hash)) < 0)
 987                 return -ENOMEM;
 988
 989         return 0;
 990 }
 991
 992 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
 993         char *w, *state;
 994         size_t l;
 995         unsigned long long seqnum, monotonic, realtime, xor_hash;
 996         bool
 997                 seqnum_id_set = false,
 998                 seqnum_set = false,
 999                 boot_id_set = false,
1000                 monotonic_set = false,
1001                 realtime_set = false,
1002                 xor_hash_set = false;
1003         sd_id128_t seqnum_id, boot_id;
1004
1005         if (!j)
1006                 return -EINVAL;
1007         if (isempty(cursor))
1008                 return -EINVAL;
1009
1010         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1011                 char *item;
1012                 int k = 0;
1013
1014                 if (l < 2 || w[1] != '=')
1015                         return -EINVAL;
1016
1017                 item = strndup(w, l);
1018                 if (!item)
1019                         return -ENOMEM;
1020
1021                 switch (w[0]) {
1022
1023                 case 's':
1024                         seqnum_id_set = true;
1025                         k = sd_id128_from_string(item+2, &seqnum_id);
1026                         break;
1027
1028                 case 'i':
1029                         seqnum_set = true;
1030                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1031                                 k = -EINVAL;
1032                         break;
1033
1034                 case 'b':
1035                         boot_id_set = true;
1036                         k = sd_id128_from_string(item+2, &boot_id);
1037                         break;
1038
1039                 case 'm':
1040                         monotonic_set = true;
1041                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1042                                 k = -EINVAL;
1043                         break;
1044
1045                 case 't':
1046                         realtime_set = true;
1047                         if (sscanf(item+2, "%llx", &realtime) != 1)
1048                                 k = -EINVAL;
1049                         break;
1050
1051                 case 'x':
1052                         xor_hash_set = true;
1053                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1054                                 k = -EINVAL;
1055                         break;
1056                 }
1057
1058                 free(item);
1059
1060                 if (k < 0)
1061                         return k;
1062         }
1063
1064         if ((!seqnum_set || !seqnum_id_set) &&
1065             (!monotonic_set || !boot_id_set) &&
1066             !realtime_set)
1067                 return -EINVAL;
1068
1069         reset_location(j);
1070
1071         j->current_location.type = LOCATION_SEEK;
1072
1073         if (realtime_set) {
1074                 j->current_location.realtime = (uint64_t) realtime;
1075                 j->current_location.realtime_set = true;
1076         }
1077
1078         if (seqnum_set && seqnum_id_set) {
1079                 j->current_location.seqnum = (uint64_t) seqnum;
1080                 j->current_location.seqnum_id = seqnum_id;
1081                 j->current_location.seqnum_set = true;
1082         }
1083
1084         if (monotonic_set && boot_id_set) {
1085                 j->current_location.monotonic = (uint64_t) monotonic;
1086                 j->current_location.boot_id = boot_id;
1087                 j->current_location.monotonic_set = true;
1088         }
1089
1090         if (xor_hash_set) {
1091                 j->current_location.xor_hash = (uint64_t) xor_hash;
1092                 j->current_location.xor_hash_set = true;
1093         }
1094
1095         return 0;
1096 }
1097
1098 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1099         int r;
1100         char *w, *state;
1101         size_t l;
1102         Object *o;
1103
1104         if (!j)
1105                 return -EINVAL;
1106         if (isempty(cursor))
1107                 return -EINVAL;
1108
1109         if (!j->current_file || j->current_file->current_offset <= 0)
1110                 return -EADDRNOTAVAIL;
1111
1112         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1113         if (r < 0)
1114                 return r;
1115
1116         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1117                 _cleanup_free_ char *item = NULL;
1118                 sd_id128_t id;
1119                 unsigned long long ll;
1120                 int k = 0;
1121
1122                 if (l < 2 || w[1] != '=')
1123                         return -EINVAL;
1124
1125                 item = strndup(w, l);
1126                 if (!item)
1127                         return -ENOMEM;
1128
1129                 switch (w[0]) {
1130
1131                 case 's':
1132                         k = sd_id128_from_string(item+2, &id);
1133                         if (k < 0)
1134                                 return k;
1135                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1136                                 return 0;
1137                         break;
1138
1139                 case 'i':
1140                         if (sscanf(item+2, "%llx", &ll) != 1)
1141                                 return -EINVAL;
1142                         if (ll != le64toh(o->entry.seqnum))
1143                                 return 0;
1144                         break;
1145
1146                 case 'b':
1147                         k = sd_id128_from_string(item+2, &id);
1148                         if (k < 0)
1149                                 return k;
1150                         if (!sd_id128_equal(id, o->entry.boot_id))
1151                                 return 0;
1152                         break;
1153
1154                 case 'm':
1155                         if (sscanf(item+2, "%llx", &ll) != 1)
1156                                 return -EINVAL;
1157                         if (ll != le64toh(o->entry.monotonic))
1158                                 return 0;
1159                         break;
1160
1161                 case 't':
1162                         if (sscanf(item+2, "%llx", &ll) != 1)
1163                                 return -EINVAL;
1164                         if (ll != le64toh(o->entry.realtime))
1165                                 return 0;
1166                         break;
1167
1168                 case 'x':
1169                         if (sscanf(item+2, "%llx", &ll) != 1)
1170                                 return -EINVAL;
1171                         if (ll != le64toh(o->entry.xor_hash))
1172                                 return 0;
1173                         break;
1174                 }
1175         }
1176
1177         return 1;
1178 }
1179
1180
1181 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1182         if (!j)
1183                 return -EINVAL;
1184
1185         reset_location(j);
1186         j->current_location.type = LOCATION_SEEK;
1187         j->current_location.boot_id = boot_id;
1188         j->current_location.monotonic = usec;
1189         j->current_location.monotonic_set = true;
1190
1191         return 0;
1192 }
1193
1194 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1195         if (!j)
1196                 return -EINVAL;
1197
1198         reset_location(j);
1199         j->current_location.type = LOCATION_SEEK;
1200         j->current_location.realtime = usec;
1201         j->current_location.realtime_set = true;
1202
1203         return 0;
1204 }
1205
1206 _public_ int sd_journal_seek_head(sd_journal *j) {
1207         if (!j)
1208                 return -EINVAL;
1209
1210         reset_location(j);
1211         j->current_location.type = LOCATION_HEAD;
1212
1213         return 0;
1214 }
1215
1216 _public_ int sd_journal_seek_tail(sd_journal *j) {
1217         if (!j)
1218                 return -EINVAL;
1219
1220         reset_location(j);
1221         j->current_location.type = LOCATION_TAIL;
1222
1223         return 0;
1224 }
1225
1226 static void check_network(sd_journal *j, int fd) {
1227         struct statfs sfs;
1228
1229         assert(j);
1230
1231         if (j->on_network)
1232                 return;
1233
1234         if (fstatfs(fd, &sfs) < 0)
1235                 return;
1236
1237         j->on_network =
1238                 F_TYPE_CMP(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1239                 F_TYPE_CMP(sfs.f_type, CODA_SUPER_MAGIC) ||
1240                 F_TYPE_CMP(sfs.f_type, NCP_SUPER_MAGIC) ||
1241                 F_TYPE_CMP(sfs.f_type, NFS_SUPER_MAGIC) ||
1242                 F_TYPE_CMP(sfs.f_type, SMB_SUPER_MAGIC);
1243 }
1244
1245 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1246         const char *full, *tilded, *atted;
1247
1248         full = strappend(prefix, ".journal");
1249         tilded = strappenda(full, "~");
1250         atted = strappenda(prefix, "@");
1251
1252         return streq(filename, full) ||
1253                streq(filename, tilded) ||
1254                startswith(filename, atted);
1255 }
1256
1257 static bool file_type_wanted(int flags, const char *filename) {
1258         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1259                 return false;
1260
1261         /* no flags set → every type is OK */
1262         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1263                 return true;
1264
1265         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1266                 return true;
1267
1268         if (flags & SD_JOURNAL_CURRENT_USER) {
1269                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1270
1271                 assert_se(snprintf(prefix, sizeof(prefix), "user-%lu", (unsigned long) getuid())
1272                           < (int) sizeof(prefix));
1273
1274                 if (file_has_type_prefix(prefix, filename))
1275                         return true;
1276         }
1277
1278         return false;
1279 }
1280
1281 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1282         _cleanup_free_ char *path = NULL;
1283         int r;
1284         JournalFile *f;
1285
1286         assert(j);
1287         assert(prefix);
1288         assert(filename);
1289
1290         if (!file_type_wanted(j->flags, filename))
1291                 return 0;
1292
1293         path = strjoin(prefix, "/", filename, NULL);
1294         if (!path)
1295                 return -ENOMEM;
1296
1297         if (hashmap_get(j->files, path))
1298                 return 0;
1299
1300         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1301                 log_debug("Too many open journal files, not adding %s, ignoring.", path);
1302                 return set_put_error(j, -ETOOMANYREFS);
1303         }
1304
1305         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1306         if (r < 0) {
1307                 if (errno == ENOENT)
1308                         return 0;
1309
1310                 return r;
1311         }
1312
1313         /* journal_file_dump(f); */
1314
1315         r = hashmap_put(j->files, f->path, f);
1316         if (r < 0) {
1317                 journal_file_close(f);
1318                 return r;
1319         }
1320
1321         log_debug("File %s added.", f->path);
1322
1323         check_network(j, f->fd);
1324
1325         j->current_invalidate_counter ++;
1326
1327         return 0;
1328 }
1329
1330 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1331         char *path;
1332         JournalFile *f;
1333
1334         assert(j);
1335         assert(prefix);
1336         assert(filename);
1337
1338         path = strjoin(prefix, "/", filename, NULL);
1339         if (!path)
1340                 return -ENOMEM;
1341
1342         f = hashmap_get(j->files, path);
1343         free(path);
1344         if (!f)
1345                 return 0;
1346
1347         hashmap_remove(j->files, f->path);
1348
1349         log_debug("File %s removed.", f->path);
1350
1351         if (j->current_file == f) {
1352                 j->current_file = NULL;
1353                 j->current_field = 0;
1354         }
1355
1356         if (j->unique_file == f) {
1357                 j->unique_file = NULL;
1358                 j->unique_offset = 0;
1359         }
1360
1361         journal_file_close(f);
1362
1363         j->current_invalidate_counter ++;
1364
1365         return 0;
1366 }
1367
1368 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1369         _cleanup_free_ char *path = NULL;
1370         int r;
1371         _cleanup_closedir_ DIR *d = NULL;
1372         sd_id128_t id, mid;
1373         Directory *m;
1374
1375         assert(j);
1376         assert(prefix);
1377         assert(dirname);
1378
1379         log_debug("Considering %s/%s.", prefix, dirname);
1380
1381         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1382             (sd_id128_from_string(dirname, &id) < 0 ||
1383              sd_id128_get_machine(&mid) < 0 ||
1384              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1385             return 0;
1386
1387         path = strjoin(prefix, "/", dirname, NULL);
1388         if (!path)
1389                 return -ENOMEM;
1390
1391         d = opendir(path);
1392         if (!d) {
1393                 log_debug("Failed to open %s: %m", path);
1394                 if (errno == ENOENT)
1395                         return 0;
1396                 return -errno;
1397         }
1398
1399         m = hashmap_get(j->directories_by_path, path);
1400         if (!m) {
1401                 m = new0(Directory, 1);
1402                 if (!m)
1403                         return -ENOMEM;
1404
1405                 m->is_root = false;
1406                 m->path = path;
1407
1408                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1409                         free(m);
1410                         return -ENOMEM;
1411                 }
1412
1413                 path = NULL; /* avoid freeing in cleanup */
1414                 j->current_invalidate_counter ++;
1415
1416                 log_debug("Directory %s added.", m->path);
1417
1418         } else if (m->is_root)
1419                 return 0;
1420
1421         if (m->wd <= 0 && j->inotify_fd >= 0) {
1422
1423                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1424                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1425                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1426                                           IN_ONLYDIR);
1427
1428                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1429                         inotify_rm_watch(j->inotify_fd, m->wd);
1430         }
1431
1432         for (;;) {
1433                 struct dirent *de;
1434                 union dirent_storage buf;
1435
1436                 r = readdir_r(d, &buf.de, &de);
1437                 if (r != 0 || !de)
1438                         break;
1439
1440                 if (dirent_is_file_with_suffix(de, ".journal") ||
1441                     dirent_is_file_with_suffix(de, ".journal~")) {
1442                         r = add_file(j, m->path, de->d_name);
1443                         if (r < 0) {
1444                                 log_debug("Failed to add file %s/%s: %s",
1445                                           m->path, de->d_name, strerror(-r));
1446                                 r = set_put_error(j, r);
1447                                 if (r < 0)
1448                                         return r;
1449                         }
1450                 }
1451         }
1452
1453         check_network(j, dirfd(d));
1454
1455         return 0;
1456 }
1457
1458 static int add_root_directory(sd_journal *j, const char *p) {
1459         _cleanup_closedir_ DIR *d = NULL;
1460         Directory *m;
1461         int r;
1462
1463         assert(j);
1464         assert(p);
1465
1466         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1467             !path_startswith(p, "/run"))
1468                 return -EINVAL;
1469
1470         d = opendir(p);
1471         if (!d)
1472                 return -errno;
1473
1474         m = hashmap_get(j->directories_by_path, p);
1475         if (!m) {
1476                 m = new0(Directory, 1);
1477                 if (!m)
1478                         return -ENOMEM;
1479
1480                 m->is_root = true;
1481                 m->path = strdup(p);
1482                 if (!m->path) {
1483                         free(m);
1484                         return -ENOMEM;
1485                 }
1486
1487                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1488                         free(m->path);
1489                         free(m);
1490                         return -ENOMEM;
1491                 }
1492
1493                 j->current_invalidate_counter ++;
1494
1495                 log_debug("Root directory %s added.", m->path);
1496
1497         } else if (!m->is_root)
1498                 return 0;
1499
1500         if (m->wd <= 0 && j->inotify_fd >= 0) {
1501
1502                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1503                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1504                                           IN_ONLYDIR);
1505
1506                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1507                         inotify_rm_watch(j->inotify_fd, m->wd);
1508         }
1509
1510         for (;;) {
1511                 struct dirent *de;
1512                 union dirent_storage buf;
1513                 sd_id128_t id;
1514
1515                 r = readdir_r(d, &buf.de, &de);
1516                 if (r != 0 || !de)
1517                         break;
1518
1519                 if (dirent_is_file_with_suffix(de, ".journal") ||
1520                     dirent_is_file_with_suffix(de, ".journal~")) {
1521                         r = add_file(j, m->path, de->d_name);
1522                         if (r < 0) {
1523                                 log_debug("Failed to add file %s/%s: %s",
1524                                           m->path, de->d_name, strerror(-r));
1525                                 r = set_put_error(j, r);
1526                                 if (r < 0)
1527                                         return r;
1528                         }
1529                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1530                            sd_id128_from_string(de->d_name, &id) >= 0) {
1531
1532                         r = add_directory(j, m->path, de->d_name);
1533                         if (r < 0)
1534                                 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1535                 }
1536         }
1537
1538         check_network(j, dirfd(d));
1539
1540         return 0;
1541 }
1542
1543 static int remove_directory(sd_journal *j, Directory *d) {
1544         assert(j);
1545
1546         if (d->wd > 0) {
1547                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1548
1549                 if (j->inotify_fd >= 0)
1550                         inotify_rm_watch(j->inotify_fd, d->wd);
1551         }
1552
1553         hashmap_remove(j->directories_by_path, d->path);
1554
1555         if (d->is_root)
1556                 log_debug("Root directory %s removed.", d->path);
1557         else
1558                 log_debug("Directory %s removed.", d->path);
1559
1560         free(d->path);
1561         free(d);
1562
1563         return 0;
1564 }
1565
1566 static int add_search_paths(sd_journal *j) {
1567         int r;
1568         const char search_paths[] =
1569                 "/run/log/journal\0"
1570                 "/var/log/journal\0";
1571         const char *p;
1572
1573         assert(j);
1574
1575         /* We ignore most errors here, since the idea is to only open
1576          * what's actually accessible, and ignore the rest. */
1577
1578         NULSTR_FOREACH(p, search_paths) {
1579                 r = add_root_directory(j, p);
1580                 if (r < 0 && r != -ENOENT) {
1581                         r = set_put_error(j, r);
1582                         if (r < 0)
1583                                 return r;
1584                 }
1585         }
1586
1587         return 0;
1588 }
1589
1590 static int allocate_inotify(sd_journal *j) {
1591         assert(j);
1592
1593         if (j->inotify_fd < 0) {
1594                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1595                 if (j->inotify_fd < 0)
1596                         return -errno;
1597         }
1598
1599         if (!j->directories_by_wd) {
1600                 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1601                 if (!j->directories_by_wd)
1602                         return -ENOMEM;
1603         }
1604
1605         return 0;
1606 }
1607
1608 static sd_journal *journal_new(int flags, const char *path) {
1609         sd_journal *j;
1610
1611         j = new0(sd_journal, 1);
1612         if (!j)
1613                 return NULL;
1614
1615         j->inotify_fd = -1;
1616         j->flags = flags;
1617         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1618
1619         if (path) {
1620                 j->path = strdup(path);
1621                 if (!j->path)
1622                         goto fail;
1623         }
1624
1625         j->files = hashmap_new(string_hash_func, string_compare_func);
1626         j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1627         j->mmap = mmap_cache_new();
1628         if (!j->files || !j->directories_by_path || !j->mmap)
1629                 goto fail;
1630
1631         return j;
1632
1633 fail:
1634         sd_journal_close(j);
1635         return NULL;
1636 }
1637
1638 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1639         sd_journal *j;
1640         int r;
1641
1642         if (!ret)
1643                 return -EINVAL;
1644
1645         if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1646                       SD_JOURNAL_RUNTIME_ONLY|
1647                       SD_JOURNAL_SYSTEM|
1648                       SD_JOURNAL_CURRENT_USER))
1649                 return -EINVAL;
1650
1651         j = journal_new(flags, NULL);
1652         if (!j)
1653                 return -ENOMEM;
1654
1655         r = add_search_paths(j);
1656         if (r < 0)
1657                 goto fail;
1658
1659         *ret = j;
1660         return 0;
1661
1662 fail:
1663         sd_journal_close(j);
1664
1665         return r;
1666 }
1667
1668 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1669         sd_journal *j;
1670         int r;
1671
1672         if (!ret)
1673                 return -EINVAL;
1674
1675         if (!path)
1676                 return -EINVAL;
1677
1678         if (flags != 0)
1679                 return -EINVAL;
1680
1681         j = journal_new(flags, path);
1682         if (!j)
1683                 return -ENOMEM;
1684
1685         r = add_root_directory(j, path);
1686         if (r < 0) {
1687                 set_put_error(j, r);
1688                 goto fail;
1689         }
1690
1691         *ret = j;
1692         return 0;
1693
1694 fail:
1695         sd_journal_close(j);
1696
1697         return r;
1698 }
1699
1700 _public_ void sd_journal_close(sd_journal *j) {
1701         Directory *d;
1702         JournalFile *f;
1703
1704         if (!j)
1705                 return;
1706
1707         sd_journal_flush_matches(j);
1708
1709         while ((f = hashmap_steal_first(j->files)))
1710                 journal_file_close(f);
1711
1712         hashmap_free(j->files);
1713
1714         while ((d = hashmap_first(j->directories_by_path)))
1715                 remove_directory(j, d);
1716
1717         while ((d = hashmap_first(j->directories_by_wd)))
1718                 remove_directory(j, d);
1719
1720         hashmap_free(j->directories_by_path);
1721         hashmap_free(j->directories_by_wd);
1722
1723         if (j->inotify_fd >= 0)
1724                 close_nointr_nofail(j->inotify_fd);
1725
1726         if (j->mmap)
1727                 mmap_cache_unref(j->mmap);
1728
1729         free(j->path);
1730         free(j->unique_field);
1731         set_free(j->errors);
1732         free(j);
1733 }
1734
1735 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1736         Object *o;
1737         JournalFile *f;
1738         int r;
1739
1740         if (!j)
1741                 return -EINVAL;
1742         if (!ret)
1743                 return -EINVAL;
1744
1745         f = j->current_file;
1746         if (!f)
1747                 return -EADDRNOTAVAIL;
1748
1749         if (f->current_offset <= 0)
1750                 return -EADDRNOTAVAIL;
1751
1752         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1753         if (r < 0)
1754                 return r;
1755
1756         *ret = le64toh(o->entry.realtime);
1757         return 0;
1758 }
1759
1760 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1761         Object *o;
1762         JournalFile *f;
1763         int r;
1764         sd_id128_t id;
1765
1766         if (!j)
1767                 return -EINVAL;
1768
1769         f = j->current_file;
1770         if (!f)
1771                 return -EADDRNOTAVAIL;
1772
1773         if (f->current_offset <= 0)
1774                 return -EADDRNOTAVAIL;
1775
1776         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1777         if (r < 0)
1778                 return r;
1779
1780         if (ret_boot_id)
1781                 *ret_boot_id = o->entry.boot_id;
1782         else {
1783                 r = sd_id128_get_boot(&id);
1784                 if (r < 0)
1785                         return r;
1786
1787                 if (!sd_id128_equal(id, o->entry.boot_id))
1788                         return -ESTALE;
1789         }
1790
1791         if (ret)
1792                 *ret = le64toh(o->entry.monotonic);
1793
1794         return 0;
1795 }
1796
1797 static bool field_is_valid(const char *field) {
1798         const char *p;
1799
1800         assert(field);
1801
1802         if (isempty(field))
1803                 return false;
1804
1805         if (startswith(field, "__"))
1806                 return false;
1807
1808         for (p = field; *p; p++) {
1809
1810                 if (*p == '_')
1811                         continue;
1812
1813                 if (*p >= 'A' && *p <= 'Z')
1814                         continue;
1815
1816                 if (*p >= '0' && *p <= '9')
1817                         continue;
1818
1819                 return false;
1820         }
1821
1822         return true;
1823 }
1824
1825 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1826         JournalFile *f;
1827         uint64_t i, n;
1828         size_t field_length;
1829         int r;
1830         Object *o;
1831
1832         if (!j)
1833                 return -EINVAL;
1834         if (!field)
1835                 return -EINVAL;
1836         if (!data)
1837                 return -EINVAL;
1838         if (!size)
1839                 return -EINVAL;
1840
1841         if (!field_is_valid(field))
1842                 return -EINVAL;
1843
1844         f = j->current_file;
1845         if (!f)
1846                 return -EADDRNOTAVAIL;
1847
1848         if (f->current_offset <= 0)
1849                 return -EADDRNOTAVAIL;
1850
1851         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1852         if (r < 0)
1853                 return r;
1854
1855         field_length = strlen(field);
1856
1857         n = journal_file_entry_n_items(o);
1858         for (i = 0; i < n; i++) {
1859                 uint64_t p, l;
1860                 le64_t le_hash;
1861                 size_t t;
1862
1863                 p = le64toh(o->entry.items[i].object_offset);
1864                 le_hash = o->entry.items[i].hash;
1865                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1866                 if (r < 0)
1867                         return r;
1868
1869                 if (le_hash != o->data.hash)
1870                         return -EBADMSG;
1871
1872                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1873
1874                 if (o->object.flags & OBJECT_COMPRESSED) {
1875
1876 #ifdef HAVE_XZ
1877                         if (uncompress_startswith(o->data.payload, l,
1878                                                   &f->compress_buffer, &f->compress_buffer_size,
1879                                                   field, field_length, '=')) {
1880
1881                                 uint64_t rsize;
1882
1883                                 if (!uncompress_blob(o->data.payload, l,
1884                                                      &f->compress_buffer, &f->compress_buffer_size, &rsize,
1885                                                      j->data_threshold))
1886                                         return -EBADMSG;
1887
1888                                 *data = f->compress_buffer;
1889                                 *size = (size_t) rsize;
1890
1891                                 return 0;
1892                         }
1893 #else
1894                         return -EPROTONOSUPPORT;
1895 #endif
1896
1897                 } else if (l >= field_length+1 &&
1898                            memcmp(o->data.payload, field, field_length) == 0 &&
1899                            o->data.payload[field_length] == '=') {
1900
1901                         t = (size_t) l;
1902
1903                         if ((uint64_t) t != l)
1904                                 return -E2BIG;
1905
1906                         *data = o->data.payload;
1907                         *size = t;
1908
1909                         return 0;
1910                 }
1911
1912                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1913                 if (r < 0)
1914                         return r;
1915         }
1916
1917         return -ENOENT;
1918 }
1919
1920 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1921         size_t t;
1922         uint64_t l;
1923
1924         l = le64toh(o->object.size) - offsetof(Object, data.payload);
1925         t = (size_t) l;
1926
1927         /* We can't read objects larger than 4G on a 32bit machine */
1928         if ((uint64_t) t != l)
1929                 return -E2BIG;
1930
1931         if (o->object.flags & OBJECT_COMPRESSED) {
1932 #ifdef HAVE_XZ
1933                 uint64_t rsize;
1934
1935                 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
1936                         return -EBADMSG;
1937
1938                 *data = f->compress_buffer;
1939                 *size = (size_t) rsize;
1940 #else
1941                 return -EPROTONOSUPPORT;
1942 #endif
1943         } else {
1944                 *data = o->data.payload;
1945                 *size = t;
1946         }
1947
1948         return 0;
1949 }
1950
1951 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1952         JournalFile *f;
1953         uint64_t p, n;
1954         le64_t le_hash;
1955         int r;
1956         Object *o;
1957
1958         if (!j)
1959                 return -EINVAL;
1960         if (!data)
1961                 return -EINVAL;
1962         if (!size)
1963                 return -EINVAL;
1964
1965         f = j->current_file;
1966         if (!f)
1967                 return -EADDRNOTAVAIL;
1968
1969         if (f->current_offset <= 0)
1970                 return -EADDRNOTAVAIL;
1971
1972         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1973         if (r < 0)
1974                 return r;
1975
1976         n = journal_file_entry_n_items(o);
1977         if (j->current_field >= n)
1978                 return 0;
1979
1980         p = le64toh(o->entry.items[j->current_field].object_offset);
1981         le_hash = o->entry.items[j->current_field].hash;
1982         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1983         if (r < 0)
1984                 return r;
1985
1986         if (le_hash != o->data.hash)
1987                 return -EBADMSG;
1988
1989         r = return_data(j, f, o, data, size);
1990         if (r < 0)
1991                 return r;
1992
1993         j->current_field ++;
1994
1995         return 1;
1996 }
1997
1998 _public_ void sd_journal_restart_data(sd_journal *j) {
1999         if (!j)
2000                 return;
2001
2002         j->current_field = 0;
2003 }
2004
2005 _public_ int sd_journal_get_fd(sd_journal *j) {
2006         int r;
2007
2008         if (!j)
2009                 return -EINVAL;
2010
2011         if (j->inotify_fd >= 0)
2012                 return j->inotify_fd;
2013
2014         r = allocate_inotify(j);
2015         if (r < 0)
2016                 return r;
2017
2018         /* Iterate through all dirs again, to add them to the
2019          * inotify */
2020         if (j->path)
2021                 r = add_root_directory(j, j->path);
2022         else
2023                 r = add_search_paths(j);
2024         if (r < 0)
2025                 return r;
2026
2027         return j->inotify_fd;
2028 }
2029
2030 _public_ int sd_journal_get_events(sd_journal *j) {
2031         int fd;
2032
2033         if (!j)
2034                 return -EINVAL;
2035
2036         fd = sd_journal_get_fd(j);
2037         if (fd < 0)
2038                 return fd;
2039
2040         return POLLIN;
2041 }
2042
2043 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2044         int fd;
2045
2046         if (!j)
2047                 return -EINVAL;
2048         if (!timeout_usec)
2049                 return -EINVAL;
2050
2051         fd = sd_journal_get_fd(j);
2052         if (fd < 0)
2053                 return fd;
2054
2055         if (!j->on_network) {
2056                 *timeout_usec = (uint64_t) -1;
2057                 return 0;
2058         }
2059
2060         /* If we are on the network we need to regularly check for
2061          * changes manually */
2062
2063         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2064         return 1;
2065 }
2066
2067 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2068         Directory *d;
2069         int r;
2070
2071         assert(j);
2072         assert(e);
2073
2074         /* Is this a subdirectory we watch? */
2075         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2076         if (d) {
2077                 sd_id128_t id;
2078
2079                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2080                     (endswith(e->name, ".journal") ||
2081                      endswith(e->name, ".journal~"))) {
2082
2083                         /* Event for a journal file */
2084
2085                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2086                                 r = add_file(j, d->path, e->name);
2087                                 if (r < 0) {
2088                                         log_debug("Failed to add file %s/%s: %s",
2089                                                   d->path, e->name, strerror(-r));
2090                                         set_put_error(j, r);
2091                                 }
2092
2093                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2094
2095                                 r = remove_file(j, d->path, e->name);
2096                                 if (r < 0)
2097                                         log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2098                         }
2099
2100                 } else if (!d->is_root && e->len == 0) {
2101
2102                         /* Event for a subdirectory */
2103
2104                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2105                                 r = remove_directory(j, d);
2106                                 if (r < 0)
2107                                         log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2108                         }
2109
2110
2111                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2112
2113                         /* Event for root directory */
2114
2115                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2116                                 r = add_directory(j, d->path, e->name);
2117                                 if (r < 0)
2118                                         log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2119                         }
2120                 }
2121
2122                 return;
2123         }
2124
2125         if (e->mask & IN_IGNORED)
2126                 return;
2127
2128         log_warning("Unknown inotify event.");
2129 }
2130
2131 static int determine_change(sd_journal *j) {
2132         bool b;
2133
2134         assert(j);
2135
2136         b = j->current_invalidate_counter != j->last_invalidate_counter;
2137         j->last_invalidate_counter = j->current_invalidate_counter;
2138
2139         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2140 }
2141
2142 _public_ int sd_journal_process(sd_journal *j) {
2143         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2144         bool got_something = false;
2145
2146         if (!j)
2147                 return -EINVAL;
2148
2149         j->last_process_usec = now(CLOCK_MONOTONIC);
2150
2151         for (;;) {
2152                 struct inotify_event *e;
2153                 ssize_t l;
2154
2155                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2156                 if (l < 0) {
2157                         if (errno == EAGAIN || errno == EINTR)
2158                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2159
2160                         return -errno;
2161                 }
2162
2163                 got_something = true;
2164
2165                 e = (struct inotify_event*) buffer;
2166                 while (l > 0) {
2167                         size_t step;
2168
2169                         process_inotify_event(j, e);
2170
2171                         step = sizeof(struct inotify_event) + e->len;
2172                         assert(step <= (size_t) l);
2173
2174                         e = (struct inotify_event*) ((uint8_t*) e + step);
2175                         l -= step;
2176                 }
2177         }
2178
2179         return determine_change(j);
2180 }
2181
2182 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2183         int r;
2184         uint64_t t;
2185
2186         assert(j);
2187
2188         if (j->inotify_fd < 0) {
2189
2190                 /* This is the first invocation, hence create the
2191                  * inotify watch */
2192                 r = sd_journal_get_fd(j);
2193                 if (r < 0)
2194                         return r;
2195
2196                 /* The journal might have changed since the context
2197                  * object was created and we weren't watching before,
2198                  * hence don't wait for anything, and return
2199                  * immediately. */
2200                 return determine_change(j);
2201         }
2202
2203         r = sd_journal_get_timeout(j, &t);
2204         if (r < 0)
2205                 return r;
2206
2207         if (t != (uint64_t) -1) {
2208                 usec_t n;
2209
2210                 n = now(CLOCK_MONOTONIC);
2211                 t = t > n ? t - n : 0;
2212
2213                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2214                         timeout_usec = t;
2215         }
2216
2217         do {
2218                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2219         } while (r == -EINTR);
2220
2221         if (r < 0)
2222                 return r;
2223
2224         return sd_journal_process(j);
2225 }
2226
2227 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2228         Iterator i;
2229         JournalFile *f;
2230         bool first = true;
2231         int r;
2232
2233         if (!j)
2234                 return -EINVAL;
2235         if (!from && !to)
2236                 return -EINVAL;
2237         if (from == to)
2238                 return -EINVAL;
2239
2240         HASHMAP_FOREACH(f, j->files, i) {
2241                 usec_t fr, t;
2242
2243                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2244                 if (r == -ENOENT)
2245                         continue;
2246                 if (r < 0)
2247                         return r;
2248                 if (r == 0)
2249                         continue;
2250
2251                 if (first) {
2252                         if (from)
2253                                 *from = fr;
2254                         if (to)
2255                                 *to = t;
2256                         first = false;
2257                 } else {
2258                         if (from)
2259                                 *from = MIN(fr, *from);
2260                         if (to)
2261                                 *to = MAX(t, *to);
2262                 }
2263         }
2264
2265         return first ? 0 : 1;
2266 }
2267
2268 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2269         Iterator i;
2270         JournalFile *f;
2271         bool first = true;
2272         int r;
2273
2274         if (!j)
2275                 return -EINVAL;
2276         if (!from && !to)
2277                 return -EINVAL;
2278         if (from == to)
2279                 return -EINVAL;
2280
2281         HASHMAP_FOREACH(f, j->files, i) {
2282                 usec_t fr, t;
2283
2284                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2285                 if (r == -ENOENT)
2286                         continue;
2287                 if (r < 0)
2288                         return r;
2289                 if (r == 0)
2290                         continue;
2291
2292                 if (first) {
2293                         if (from)
2294                                 *from = fr;
2295                         if (to)
2296                                 *to = t;
2297                         first = false;
2298                 } else {
2299                         if (from)
2300                                 *from = MIN(fr, *from);
2301                         if (to)
2302                                 *to = MAX(t, *to);
2303                 }
2304         }
2305
2306         return first ? 0 : 1;
2307 }
2308
2309 void journal_print_header(sd_journal *j) {
2310         Iterator i;
2311         JournalFile *f;
2312         bool newline = false;
2313
2314         assert(j);
2315
2316         HASHMAP_FOREACH(f, j->files, i) {
2317                 if (newline)
2318                         putchar('\n');
2319                 else
2320                         newline = true;
2321
2322                 journal_file_print_header(f);
2323         }
2324 }
2325
2326 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2327         Iterator i;
2328         JournalFile *f;
2329         uint64_t sum = 0;
2330
2331         if (!j)
2332                 return -EINVAL;
2333         if (!bytes)
2334                 return -EINVAL;
2335
2336         HASHMAP_FOREACH(f, j->files, i) {
2337                 struct stat st;
2338
2339                 if (fstat(f->fd, &st) < 0)
2340                         return -errno;
2341
2342                 sum += (uint64_t) st.st_blocks * 512ULL;
2343         }
2344
2345         *bytes = sum;
2346         return 0;
2347 }
2348
2349 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2350         char *f;
2351
2352         if (!j)
2353                 return -EINVAL;
2354         if (isempty(field))
2355                 return -EINVAL;
2356         if (!field_is_valid(field))
2357                 return -EINVAL;
2358
2359         f = strdup(field);
2360         if (!f)
2361                 return -ENOMEM;
2362
2363         free(j->unique_field);
2364         j->unique_field = f;
2365         j->unique_file = NULL;
2366         j->unique_offset = 0;
2367
2368         return 0;
2369 }
2370
2371 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2372         Object *o;
2373         size_t k;
2374         int r;
2375
2376         if (!j)
2377                 return -EINVAL;
2378         if (!data)
2379                 return -EINVAL;
2380         if (!l)
2381                 return -EINVAL;
2382         if (!j->unique_field)
2383                 return -EINVAL;
2384
2385         k = strlen(j->unique_field);
2386
2387         if (!j->unique_file) {
2388                 j->unique_file = hashmap_first(j->files);
2389                 if (!j->unique_file)
2390                         return 0;
2391                 j->unique_offset = 0;
2392         }
2393
2394         for (;;) {
2395                 JournalFile *of;
2396                 Iterator i;
2397                 const void *odata;
2398                 size_t ol;
2399                 bool found;
2400
2401                 /* Proceed to next data object in the field's linked list */
2402                 if (j->unique_offset == 0) {
2403                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2404                         if (r < 0)
2405                                 return r;
2406
2407                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2408                 } else {
2409                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2410                         if (r < 0)
2411                                 return r;
2412
2413                         j->unique_offset = le64toh(o->data.next_field_offset);
2414                 }
2415
2416                 /* We reached the end of the list? Then start again, with the next file */
2417                 if (j->unique_offset == 0) {
2418                         JournalFile *n;
2419
2420                         n = hashmap_next(j->files, j->unique_file->path);
2421                         if (!n)
2422                                 return 0;
2423
2424                         j->unique_file = n;
2425                         continue;
2426                 }
2427
2428                 /* We do not use the type context here, but 0 instead,
2429                  * so that we can look at this data object at the same
2430                  * time as one on another file */
2431                 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2432                 if (r < 0)
2433                         return r;
2434
2435                 /* Let's do the type check by hand, since we used 0 context above. */
2436                 if (o->object.type != OBJECT_DATA)
2437                         return -EBADMSG;
2438
2439                 r = return_data(j, j->unique_file, o, &odata, &ol);
2440                 if (r < 0)
2441                         return r;
2442
2443                 /* OK, now let's see if we already returned this data
2444                  * object by checking if it exists in the earlier
2445                  * traversed files. */
2446                 found = false;
2447                 HASHMAP_FOREACH(of, j->files, i) {
2448                         Object *oo;
2449                         uint64_t op;
2450
2451                         if (of == j->unique_file)
2452                                 break;
2453
2454                         /* Skip this file it didn't have any fields
2455                          * indexed */
2456                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2457                             le64toh(of->header->n_fields) <= 0)
2458                                 continue;
2459
2460                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2461                         if (r < 0)
2462                                 return r;
2463
2464                         if (r > 0)
2465                                 found = true;
2466                 }
2467
2468                 if (found)
2469                         continue;
2470
2471                 r = return_data(j, j->unique_file, o, data, l);
2472                 if (r < 0)
2473                         return r;
2474
2475                 return 1;
2476         }
2477 }
2478
2479 _public_ void sd_journal_restart_unique(sd_journal *j) {
2480         if (!j)
2481                 return;
2482
2483         j->unique_file = NULL;
2484         j->unique_offset = 0;
2485 }
2486
2487 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2488         if (!j)
2489                 return -EINVAL;
2490
2491         return !j->on_network;
2492 }
2493
2494 static char *lookup_field(const char *field, void *userdata) {
2495         sd_journal *j = userdata;
2496         const void *data;
2497         size_t size, d;
2498         int r;
2499
2500         assert(field);
2501         assert(j);
2502
2503         r = sd_journal_get_data(j, field, &data, &size);
2504         if (r < 0 ||
2505             size > REPLACE_VAR_MAX)
2506                 return strdup(field);
2507
2508         d = strlen(field) + 1;
2509
2510         return strndup((const char*) data + d, size - d);
2511 }
2512
2513 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2514         const void *data;
2515         size_t size;
2516         sd_id128_t id;
2517         _cleanup_free_ char *text = NULL, *cid = NULL;
2518         char *t;
2519         int r;
2520
2521         if (!j)
2522                 return -EINVAL;
2523         if (!ret)
2524                 return -EINVAL;
2525
2526         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2527         if (r < 0)
2528                 return r;
2529
2530         cid = strndup((const char*) data + 11, size - 11);
2531         if (!cid)
2532                 return -ENOMEM;
2533
2534         r = sd_id128_from_string(cid, &id);
2535         if (r < 0)
2536                 return r;
2537
2538         r = catalog_get(CATALOG_DATABASE, id, &text);
2539         if (r < 0)
2540                 return r;
2541
2542         t = replace_var(text, lookup_field, j);
2543         if (!t)
2544                 return -ENOMEM;
2545
2546         *ret = t;
2547         return 0;
2548 }
2549
2550 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2551         if (!ret)
2552                 return -EINVAL;
2553
2554         return catalog_get(CATALOG_DATABASE, id, ret);
2555 }
2556
2557 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2558         if (!j)
2559                 return -EINVAL;
2560
2561         j->data_threshold = sz;
2562         return 0;
2563 }
2564
2565 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2566         if (!j)
2567                 return -EINVAL;
2568         if (!sz)
2569                 return -EINVAL;
2570
2571         *sz = j->data_threshold;
2572         return 0;
2573 }