src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "path-util.h"
  37 #include "lookup3.h"
  38 #include "compress.h"
  39 #include "journal-internal.h"
  40 #include "missing.h"
  41 #include "catalog.h"
  42 #include "replace-var.h"
  43
  44 #define JOURNAL_FILES_MAX 1024
  45
  46 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  47
  48 #define REPLACE_VAR_MAX 256
  49
  50 #define DEFAULT_DATA_THRESHOLD (64*1024)
  51
  52 /* We return an error here only if we didn't manage to
  53    memorize the real error. */
  54 static int set_put_error(sd_journal *j, int r) {
  55         int k;
  56
  57         if (r >= 0)
  58                 return r;
  59
  60         k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
  61         if (k < 0)
  62                 return k;
  63
  64         return set_put(j->errors, INT_TO_PTR(r));
  65 }
  66
  67 static void detach_location(sd_journal *j) {
  68         Iterator i;
  69         JournalFile *f;
  70
  71         assert(j);
  72
  73         j->current_file = NULL;
  74         j->current_field = 0;
  75
  76         HASHMAP_FOREACH(f, j->files, i)
  77                 f->current_offset = 0;
  78 }
  79
  80 static void reset_location(sd_journal *j) {
  81         assert(j);
  82
  83         detach_location(j);
  84         zero(j->current_location);
  85 }
  86
  87 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  88         assert(l);
  89         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
  90         assert(f);
  91         assert(o->object.type == OBJECT_ENTRY);
  92
  93         l->type = type;
  94         l->seqnum = le64toh(o->entry.seqnum);
  95         l->seqnum_id = f->header->seqnum_id;
  96         l->realtime = le64toh(o->entry.realtime);
  97         l->monotonic = le64toh(o->entry.monotonic);
  98         l->boot_id = o->entry.boot_id;
  99         l->xor_hash = le64toh(o->entry.xor_hash);
 100
 101         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 102 }
 103
 104 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o, uint64_t offset) {
 105         assert(j);
 106         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 107         assert(f);
 108         assert(o);
 109
 110         init_location(&j->current_location, type, f, o);
 111
 112         if (j->current_file)
 113                 j->current_file->current_offset = 0;
 114
 115         j->current_file = f;
 116         j->current_field = 0;
 117
 118         f->current_offset = offset;
 119 }
 120
 121 static int match_is_valid(const void *data, size_t size) {
 122         const char *b, *p;
 123
 124         assert(data);
 125
 126         if (size < 2)
 127                 return false;
 128
 129         if (startswith(data, "__"))
 130                 return false;
 131
 132         b = data;
 133         for (p = b; p < b + size; p++) {
 134
 135                 if (*p == '=')
 136                         return p > b;
 137
 138                 if (*p == '_')
 139                         continue;
 140
 141                 if (*p >= 'A' && *p <= 'Z')
 142                         continue;
 143
 144                 if (*p >= '0' && *p <= '9')
 145                         continue;
 146
 147                 return false;
 148         }
 149
 150         return false;
 151 }
 152
 153 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 154         const uint8_t *a = _a, *b = _b;
 155         size_t j;
 156
 157         for (j = 0; j < s && j < t; j++) {
 158
 159                 if (a[j] != b[j])
 160                         return false;
 161
 162                 if (a[j] == '=')
 163                         return true;
 164         }
 165
 166         return true;
 167 }
 168
 169 static Match *match_new(Match *p, MatchType t) {
 170         Match *m;
 171
 172         m = new0(Match, 1);
 173         if (!m)
 174                 return NULL;
 175
 176         m->type = t;
 177
 178         if (p) {
 179                 m->parent = p;
 180                 LIST_PREPEND(Match, matches, p->matches, m);
 181         }
 182
 183         return m;
 184 }
 185
 186 static void match_free(Match *m) {
 187         assert(m);
 188
 189         while (m->matches)
 190                 match_free(m->matches);
 191
 192         if (m->parent)
 193                 LIST_REMOVE(Match, matches, m->parent->matches, m);
 194
 195         free(m->data);
 196         free(m);
 197 }
 198
 199 static void match_free_if_empty(Match *m) {
 200         assert(m);
 201
 202         if (m->matches)
 203                 return;
 204
 205         match_free(m);
 206 }
 207
 208 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 209         Match *l3, *l4, *add_here = NULL, *m;
 210         le64_t le_hash;
 211
 212         if (!j)
 213                 return -EINVAL;
 214
 215         if (!data)
 216                 return -EINVAL;
 217
 218         if (size == 0)
 219                 size = strlen(data);
 220
 221         if (!match_is_valid(data, size))
 222                 return -EINVAL;
 223
 224         /* level 0: AND term
 225          * level 1: OR terms
 226          * level 2: AND terms
 227          * level 3: OR terms
 228          * level 4: concrete matches */
 229
 230         if (!j->level0) {
 231                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 232                 if (!j->level0)
 233                         return -ENOMEM;
 234         }
 235
 236         if (!j->level1) {
 237                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 238                 if (!j->level1)
 239                         return -ENOMEM;
 240         }
 241
 242         if (!j->level2) {
 243                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 244                 if (!j->level2)
 245                         return -ENOMEM;
 246         }
 247
 248         assert(j->level0->type == MATCH_AND_TERM);
 249         assert(j->level1->type == MATCH_OR_TERM);
 250         assert(j->level2->type == MATCH_AND_TERM);
 251
 252         le_hash = htole64(hash64(data, size));
 253
 254         LIST_FOREACH(matches, l3, j->level2->matches) {
 255                 assert(l3->type == MATCH_OR_TERM);
 256
 257                 LIST_FOREACH(matches, l4, l3->matches) {
 258                         assert(l4->type == MATCH_DISCRETE);
 259
 260                         /* Exactly the same match already? Then ignore
 261                          * this addition */
 262                         if (l4->le_hash == le_hash &&
 263                             l4->size == size &&
 264                             memcmp(l4->data, data, size) == 0)
 265                                 return 0;
 266
 267                         /* Same field? Then let's add this to this OR term */
 268                         if (same_field(data, size, l4->data, l4->size)) {
 269                                 add_here = l3;
 270                                 break;
 271                         }
 272                 }
 273
 274                 if (add_here)
 275                         break;
 276         }
 277
 278         if (!add_here) {
 279                 add_here = match_new(j->level2, MATCH_OR_TERM);
 280                 if (!add_here)
 281                         goto fail;
 282         }
 283
 284         m = match_new(add_here, MATCH_DISCRETE);
 285         if (!m)
 286                 goto fail;
 287
 288         m->le_hash = le_hash;
 289         m->size = size;
 290         m->data = memdup(data, size);
 291         if (!m->data)
 292                 goto fail;
 293
 294         detach_location(j);
 295
 296         return 0;
 297
 298 fail:
 299         if (add_here)
 300                 match_free_if_empty(add_here);
 301
 302         if (j->level2)
 303                 match_free_if_empty(j->level2);
 304
 305         if (j->level1)
 306                 match_free_if_empty(j->level1);
 307
 308         if (j->level0)
 309                 match_free_if_empty(j->level0);
 310
 311         return -ENOMEM;
 312 }
 313
 314 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 315         assert(j);
 316
 317         if (!j->level0)
 318                 return 0;
 319
 320         if (!j->level1)
 321                 return 0;
 322
 323         if (!j->level1->matches)
 324                 return 0;
 325
 326         j->level1 = NULL;
 327         j->level2 = NULL;
 328
 329         return 0;
 330 }
 331
 332 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 333         assert(j);
 334
 335         if (!j->level0)
 336                 return 0;
 337
 338         if (!j->level1)
 339                 return 0;
 340
 341         if (!j->level2)
 342                 return 0;
 343
 344         if (!j->level2->matches)
 345                 return 0;
 346
 347         j->level2 = NULL;
 348         return 0;
 349 }
 350
 351 static char *match_make_string(Match *m) {
 352         char *p, *r;
 353         Match *i;
 354         bool enclose = false;
 355
 356         if (!m)
 357                 return strdup("");
 358
 359         if (m->type == MATCH_DISCRETE)
 360                 return strndup(m->data, m->size);
 361
 362         p = NULL;
 363         LIST_FOREACH(matches, i, m->matches) {
 364                 char *t, *k;
 365
 366                 t = match_make_string(i);
 367                 if (!t) {
 368                         free(p);
 369                         return NULL;
 370                 }
 371
 372                 if (p) {
 373                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 374                         free(p);
 375                         free(t);
 376
 377                         if (!k)
 378                                 return NULL;
 379
 380                         p = k;
 381
 382                         enclose = true;
 383                 } else {
 384                         free(p);
 385                         p = t;
 386                 }
 387         }
 388
 389         if (enclose) {
 390                 r = strjoin("(", p, ")", NULL);
 391                 free(p);
 392                 return r;
 393         }
 394
 395         return p;
 396 }
 397
 398 char *journal_make_match_string(sd_journal *j) {
 399         assert(j);
 400
 401         return match_make_string(j->level0);
 402 }
 403
 404 _public_ void sd_journal_flush_matches(sd_journal *j) {
 405
 406         if (!j)
 407                 return;
 408
 409         if (j->level0)
 410                 match_free(j->level0);
 411
 412         j->level0 = j->level1 = j->level2 = NULL;
 413
 414         detach_location(j);
 415 }
 416
 417 static int compare_entry_order(JournalFile *af, Object *_ao,
 418                          JournalFile *bf, uint64_t bp) {
 419
 420         uint64_t a, b;
 421         Object *ao, *bo;
 422         int r;
 423
 424         assert(af);
 425         assert(bf);
 426         assert(_ao);
 427
 428         /* The mmap cache might invalidate the object from the first
 429          * file if we look at the one from the second file. Hence
 430          * temporarily copy the header of the first one, and look at
 431          * that only. */
 432         ao = alloca(offsetof(EntryObject, items));
 433         memcpy(ao, _ao, offsetof(EntryObject, items));
 434
 435         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 436         if (r < 0)
 437                 return strcmp(af->path, bf->path);
 438
 439         /* We operate on two different files here, hence we can access
 440          * two objects at the same time, which we normally can't.
 441          *
 442          * If contents and timestamps match, these entries are
 443          * identical, even if the seqnum does not match */
 444
 445         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 446             ao->entry.monotonic == bo->entry.monotonic &&
 447             ao->entry.realtime == bo->entry.realtime &&
 448             ao->entry.xor_hash == bo->entry.xor_hash)
 449                 return 0;
 450
 451         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 452
 453                 /* If this is from the same seqnum source, compare
 454                  * seqnums */
 455                 a = le64toh(ao->entry.seqnum);
 456                 b = le64toh(bo->entry.seqnum);
 457
 458                 if (a < b)
 459                         return -1;
 460                 if (a > b)
 461                         return 1;
 462
 463                 /* Wow! This is weird, different data but the same
 464                  * seqnums? Something is borked, but let's make the
 465                  * best of it and compare by time. */
 466         }
 467
 468         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 469
 470                 /* If the boot id matches compare monotonic time */
 471                 a = le64toh(ao->entry.monotonic);
 472                 b = le64toh(bo->entry.monotonic);
 473
 474                 if (a < b)
 475                         return -1;
 476                 if (a > b)
 477                         return 1;
 478         }
 479
 480         /* Otherwise compare UTC time */
 481         a = le64toh(ao->entry.realtime);
 482         b = le64toh(bo->entry.realtime);
 483
 484         if (a < b)
 485                 return -1;
 486         if (a > b)
 487                 return 1;
 488
 489         /* Finally, compare by contents */
 490         a = le64toh(ao->entry.xor_hash);
 491         b = le64toh(bo->entry.xor_hash);
 492
 493         if (a < b)
 494                 return -1;
 495         if (a > b)
 496                 return 1;
 497
 498         return 0;
 499 }
 500
 501 static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 502         uint64_t a;
 503
 504         assert(af);
 505         assert(ao);
 506         assert(l);
 507         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 508
 509         if (l->monotonic_set &&
 510             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 511             l->realtime_set &&
 512             le64toh(ao->entry.realtime) == l->realtime &&
 513             l->xor_hash_set &&
 514             le64toh(ao->entry.xor_hash) == l->xor_hash)
 515                 return 0;
 516
 517         if (l->seqnum_set &&
 518             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 519
 520                 a = le64toh(ao->entry.seqnum);
 521
 522                 if (a < l->seqnum)
 523                         return -1;
 524                 if (a > l->seqnum)
 525                         return 1;
 526         }
 527
 528         if (l->monotonic_set &&
 529             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 530
 531                 a = le64toh(ao->entry.monotonic);
 532
 533                 if (a < l->monotonic)
 534                         return -1;
 535                 if (a > l->monotonic)
 536                         return 1;
 537         }
 538
 539         if (l->realtime_set) {
 540
 541                 a = le64toh(ao->entry.realtime);
 542
 543                 if (a < l->realtime)
 544                         return -1;
 545                 if (a > l->realtime)
 546                         return 1;
 547         }
 548
 549         if (l->xor_hash_set) {
 550                 a = le64toh(ao->entry.xor_hash);
 551
 552                 if (a < l->xor_hash)
 553                         return -1;
 554                 if (a > l->xor_hash)
 555                         return 1;
 556         }
 557
 558         return 0;
 559 }
 560
 561 static int next_for_match(
 562                 sd_journal *j,
 563                 Match *m,
 564                 JournalFile *f,
 565                 uint64_t after_offset,
 566                 direction_t direction,
 567                 Object **ret,
 568                 uint64_t *offset) {
 569
 570         int r;
 571         uint64_t np = 0;
 572         Object *n;
 573
 574         assert(j);
 575         assert(m);
 576         assert(f);
 577
 578         if (m->type == MATCH_DISCRETE) {
 579                 uint64_t dp;
 580
 581                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 582                 if (r <= 0)
 583                         return r;
 584
 585                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 586
 587         } else if (m->type == MATCH_OR_TERM) {
 588                 Match *i;
 589
 590                 /* Find the earliest match beyond after_offset */
 591
 592                 LIST_FOREACH(matches, i, m->matches) {
 593                         uint64_t cp;
 594
 595                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 596                         if (r < 0)
 597                                 return r;
 598                         else if (r > 0) {
 599                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 600                                         np = cp;
 601                         }
 602                 }
 603
 604         } else if (m->type == MATCH_AND_TERM) {
 605                 Match *i;
 606                 bool continue_looking;
 607
 608                 /* Always jump to the next matching entry and repeat
 609                  * this until we fine and offset that matches for all
 610                  * matches. */
 611
 612                 if (!m->matches)
 613                         return 0;
 614
 615                 np = 0;
 616                 do {
 617                         continue_looking = false;
 618
 619                         LIST_FOREACH(matches, i, m->matches) {
 620                                 uint64_t cp, limit;
 621
 622                                 if (np == 0)
 623                                         limit = after_offset;
 624                                 else if (direction == DIRECTION_DOWN)
 625                                         limit = MAX(np, after_offset);
 626                                 else
 627                                         limit = MIN(np, after_offset);
 628
 629                                 r = next_for_match(j, i, f, limit, direction, NULL, &cp);
 630                                 if (r <= 0)
 631                                         return r;
 632
 633                                 if ((direction == DIRECTION_DOWN ? cp >= after_offset : cp <= after_offset) &&
 634                                     (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))) {
 635                                         np = cp;
 636                                         continue_looking = true;
 637                                 }
 638                         }
 639
 640                 } while (continue_looking);
 641         }
 642
 643         if (np == 0)
 644                 return 0;
 645
 646         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 647         if (r < 0)
 648                 return r;
 649
 650         if (ret)
 651                 *ret = n;
 652         if (offset)
 653                 *offset = np;
 654
 655         return 1;
 656 }
 657
 658 static int find_location_for_match(
 659                 sd_journal *j,
 660                 Match *m,
 661                 JournalFile *f,
 662                 direction_t direction,
 663                 Object **ret,
 664                 uint64_t *offset) {
 665
 666         int r;
 667
 668         assert(j);
 669         assert(m);
 670         assert(f);
 671
 672         if (m->type == MATCH_DISCRETE) {
 673                 uint64_t dp;
 674
 675                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 676                 if (r <= 0)
 677                         return r;
 678
 679                 /* FIXME: missing: find by monotonic */
 680
 681                 if (j->current_location.type == LOCATION_HEAD)
 682                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 683                 if (j->current_location.type == LOCATION_TAIL)
 684                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 685                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 686                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 687                 if (j->current_location.monotonic_set) {
 688                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 689                         if (r != -ENOENT)
 690                                 return r;
 691                 }
 692                 if (j->current_location.realtime_set)
 693                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 694
 695                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 696
 697         } else if (m->type == MATCH_OR_TERM) {
 698                 uint64_t np = 0;
 699                 Object *n;
 700                 Match *i;
 701
 702                 /* Find the earliest match */
 703
 704                 LIST_FOREACH(matches, i, m->matches) {
 705                         uint64_t cp;
 706
 707                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 708                         if (r < 0)
 709                                 return r;
 710                         else if (r > 0) {
 711                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 712                                         np = cp;
 713                         }
 714                 }
 715
 716                 if (np == 0)
 717                         return 0;
 718
 719                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 720                 if (r < 0)
 721                         return r;
 722
 723                 if (ret)
 724                         *ret = n;
 725                 if (offset)
 726                         *offset = np;
 727
 728                 return 1;
 729
 730         } else {
 731                 Match *i;
 732                 uint64_t np = 0;
 733
 734                 assert(m->type == MATCH_AND_TERM);
 735
 736                 /* First jump to the last match, and then find the
 737                  * next one where all matches match */
 738
 739                 if (!m->matches)
 740                         return 0;
 741
 742                 LIST_FOREACH(matches, i, m->matches) {
 743                         uint64_t cp;
 744
 745                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 746                         if (r <= 0)
 747                                 return r;
 748
 749                         if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
 750                                 np = cp;
 751                 }
 752
 753                 return next_for_match(j, m, f, np, direction, ret, offset);
 754         }
 755 }
 756
 757 static int find_location_with_matches(
 758                 sd_journal *j,
 759                 JournalFile *f,
 760                 direction_t direction,
 761                 Object **ret,
 762                 uint64_t *offset) {
 763
 764         int r;
 765
 766         assert(j);
 767         assert(f);
 768         assert(ret);
 769         assert(offset);
 770
 771         if (!j->level0) {
 772                 /* No matches is simple */
 773
 774                 if (j->current_location.type == LOCATION_HEAD)
 775                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 776                 if (j->current_location.type == LOCATION_TAIL)
 777                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 778                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 779                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 780                 if (j->current_location.monotonic_set) {
 781                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 782                         if (r != -ENOENT)
 783                                 return r;
 784                 }
 785                 if (j->current_location.realtime_set)
 786                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 787
 788                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 789         } else
 790                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 791 }
 792
 793 static int next_with_matches(
 794                 sd_journal *j,
 795                 JournalFile *f,
 796                 direction_t direction,
 797                 Object **ret,
 798                 uint64_t *offset) {
 799
 800         Object *c;
 801         uint64_t cp;
 802
 803         assert(j);
 804         assert(f);
 805         assert(ret);
 806         assert(offset);
 807
 808         c = *ret;
 809         cp = *offset;
 810
 811         /* No matches is easy. We simple advance the file
 812          * pointer by one. */
 813         if (!j->level0)
 814                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 815
 816         /* If we have a match then we look for the next matching entry
 817          * with an offset at least one step larger */
 818         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 819 }
 820
 821 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 822         Object *c;
 823         uint64_t cp;
 824         int r;
 825
 826         assert(j);
 827         assert(f);
 828
 829         if (f->current_offset > 0) {
 830                 cp = f->current_offset;
 831
 832                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 833                 if (r < 0)
 834                         return r;
 835
 836                 r = next_with_matches(j, f, direction, &c, &cp);
 837                 if (r <= 0)
 838                         return r;
 839         } else {
 840                 r = find_location_with_matches(j, f, direction, &c, &cp);
 841                 if (r <= 0)
 842                         return r;
 843         }
 844
 845         /* OK, we found the spot, now let's advance until to an entry
 846          * that is actually different from what we were previously
 847          * looking at. This is necessary to handle entries which exist
 848          * in two (or more) journal files, and which shall all be
 849          * suppressed but one. */
 850
 851         for (;;) {
 852                 bool found;
 853
 854                 if (j->current_location.type == LOCATION_DISCRETE) {
 855                         int k;
 856
 857                         k = compare_with_location(f, c, &j->current_location);
 858                         if (direction == DIRECTION_DOWN)
 859                                 found = k > 0;
 860                         else
 861                                 found = k < 0;
 862                 } else
 863                         found = true;
 864
 865                 if (found) {
 866                         if (ret)
 867                                 *ret = c;
 868                         if (offset)
 869                                 *offset = cp;
 870                         return 1;
 871                 }
 872
 873                 r = next_with_matches(j, f, direction, &c, &cp);
 874                 if (r <= 0)
 875                         return r;
 876         }
 877 }
 878
 879 static int real_journal_next(sd_journal *j, direction_t direction) {
 880         JournalFile *f, *new_file = NULL;
 881         uint64_t new_offset = 0;
 882         Object *o;
 883         uint64_t p;
 884         Iterator i;
 885         int r;
 886
 887         if (!j)
 888                 return -EINVAL;
 889
 890         HASHMAP_FOREACH(f, j->files, i) {
 891                 bool found;
 892
 893                 r = next_beyond_location(j, f, direction, &o, &p);
 894                 if (r < 0) {
 895                         log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
 896                         continue;
 897                 } else if (r == 0)
 898                         continue;
 899
 900                 if (!new_file)
 901                         found = true;
 902                 else {
 903                         int k;
 904
 905                         k = compare_entry_order(f, o, new_file, new_offset);
 906
 907                         if (direction == DIRECTION_DOWN)
 908                                 found = k < 0;
 909                         else
 910                                 found = k > 0;
 911                 }
 912
 913                 if (found) {
 914                         new_file = f;
 915                         new_offset = p;
 916                 }
 917         }
 918
 919         if (!new_file)
 920                 return 0;
 921
 922         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 923         if (r < 0)
 924                 return r;
 925
 926         set_location(j, LOCATION_DISCRETE, new_file, o, new_offset);
 927
 928         return 1;
 929 }
 930
 931 _public_ int sd_journal_next(sd_journal *j) {
 932         return real_journal_next(j, DIRECTION_DOWN);
 933 }
 934
 935 _public_ int sd_journal_previous(sd_journal *j) {
 936         return real_journal_next(j, DIRECTION_UP);
 937 }
 938
 939 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 940         int c = 0, r;
 941
 942         if (!j)
 943                 return -EINVAL;
 944
 945         if (skip == 0) {
 946                 /* If this is not a discrete skip, then at least
 947                  * resolve the current location */
 948                 if (j->current_location.type != LOCATION_DISCRETE)
 949                         return real_journal_next(j, direction);
 950
 951                 return 0;
 952         }
 953
 954         do {
 955                 r = real_journal_next(j, direction);
 956                 if (r < 0)
 957                         return r;
 958
 959                 if (r == 0)
 960                         return c;
 961
 962                 skip--;
 963                 c++;
 964         } while (skip > 0);
 965
 966         return c;
 967 }
 968
 969 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 970         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 971 }
 972
 973 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 974         return real_journal_next_skip(j, DIRECTION_UP, skip);
 975 }
 976
 977 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 978         Object *o;
 979         int r;
 980         char bid[33], sid[33];
 981
 982         if (!j)
 983                 return -EINVAL;
 984         if (!cursor)
 985                 return -EINVAL;
 986
 987         if (!j->current_file || j->current_file->current_offset <= 0)
 988                 return -EADDRNOTAVAIL;
 989
 990         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 991         if (r < 0)
 992                 return r;
 993
 994         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 995         sd_id128_to_string(o->entry.boot_id, bid);
 996
 997         if (asprintf(cursor,
 998                      "s=%s;i=%llx;b=%s;m=%llx;t=%llx;x=%llx",
 999                      sid, (unsigned long long) le64toh(o->entry.seqnum),
1000                      bid, (unsigned long long) le64toh(o->entry.monotonic),
1001                      (unsigned long long) le64toh(o->entry.realtime),
1002                      (unsigned long long) le64toh(o->entry.xor_hash)) < 0)
1003                 return -ENOMEM;
1004
1005         return 0;
1006 }
1007
1008 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
1009         char *w, *state;
1010         size_t l;
1011         unsigned long long seqnum, monotonic, realtime, xor_hash;
1012         bool
1013                 seqnum_id_set = false,
1014                 seqnum_set = false,
1015                 boot_id_set = false,
1016                 monotonic_set = false,
1017                 realtime_set = false,
1018                 xor_hash_set = false;
1019         sd_id128_t seqnum_id, boot_id;
1020
1021         if (!j)
1022                 return -EINVAL;
1023         if (isempty(cursor))
1024                 return -EINVAL;
1025
1026         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1027                 char *item;
1028                 int k = 0;
1029
1030                 if (l < 2 || w[1] != '=')
1031                         return -EINVAL;
1032
1033                 item = strndup(w, l);
1034                 if (!item)
1035                         return -ENOMEM;
1036
1037                 switch (w[0]) {
1038
1039                 case 's':
1040                         seqnum_id_set = true;
1041                         k = sd_id128_from_string(item+2, &seqnum_id);
1042                         break;
1043
1044                 case 'i':
1045                         seqnum_set = true;
1046                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1047                                 k = -EINVAL;
1048                         break;
1049
1050                 case 'b':
1051                         boot_id_set = true;
1052                         k = sd_id128_from_string(item+2, &boot_id);
1053                         break;
1054
1055                 case 'm':
1056                         monotonic_set = true;
1057                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1058                                 k = -EINVAL;
1059                         break;
1060
1061                 case 't':
1062                         realtime_set = true;
1063                         if (sscanf(item+2, "%llx", &realtime) != 1)
1064                                 k = -EINVAL;
1065                         break;
1066
1067                 case 'x':
1068                         xor_hash_set = true;
1069                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1070                                 k = -EINVAL;
1071                         break;
1072                 }
1073
1074                 free(item);
1075
1076                 if (k < 0)
1077                         return k;
1078         }
1079
1080         if ((!seqnum_set || !seqnum_id_set) &&
1081             (!monotonic_set || !boot_id_set) &&
1082             !realtime_set)
1083                 return -EINVAL;
1084
1085         reset_location(j);
1086
1087         j->current_location.type = LOCATION_SEEK;
1088
1089         if (realtime_set) {
1090                 j->current_location.realtime = (uint64_t) realtime;
1091                 j->current_location.realtime_set = true;
1092         }
1093
1094         if (seqnum_set && seqnum_id_set) {
1095                 j->current_location.seqnum = (uint64_t) seqnum;
1096                 j->current_location.seqnum_id = seqnum_id;
1097                 j->current_location.seqnum_set = true;
1098         }
1099
1100         if (monotonic_set && boot_id_set) {
1101                 j->current_location.monotonic = (uint64_t) monotonic;
1102                 j->current_location.boot_id = boot_id;
1103                 j->current_location.monotonic_set = true;
1104         }
1105
1106         if (xor_hash_set) {
1107                 j->current_location.xor_hash = (uint64_t) xor_hash;
1108                 j->current_location.xor_hash_set = true;
1109         }
1110
1111         return 0;
1112 }
1113
1114 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1115         int r;
1116         char *w, *state;
1117         size_t l;
1118         Object *o;
1119
1120         if (!j)
1121                 return -EINVAL;
1122         if (isempty(cursor))
1123                 return -EINVAL;
1124
1125         if (!j->current_file || j->current_file->current_offset <= 0)
1126                 return -EADDRNOTAVAIL;
1127
1128         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1129         if (r < 0)
1130                 return r;
1131
1132         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1133                 _cleanup_free_ char *item = NULL;
1134                 sd_id128_t id;
1135                 unsigned long long ll;
1136                 int k = 0;
1137
1138                 if (l < 2 || w[1] != '=')
1139                         return -EINVAL;
1140
1141                 item = strndup(w, l);
1142                 if (!item)
1143                         return -ENOMEM;
1144
1145                 switch (w[0]) {
1146
1147                 case 's':
1148                         k = sd_id128_from_string(item+2, &id);
1149                         if (k < 0)
1150                                 return k;
1151                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1152                                 return 0;
1153                         break;
1154
1155                 case 'i':
1156                         if (sscanf(item+2, "%llx", &ll) != 1)
1157                                 return -EINVAL;
1158                         if (ll != le64toh(o->entry.seqnum))
1159                                 return 0;
1160                         break;
1161
1162                 case 'b':
1163                         k = sd_id128_from_string(item+2, &id);
1164                         if (k < 0)
1165                                 return k;
1166                         if (!sd_id128_equal(id, o->entry.boot_id))
1167                                 return 0;
1168                         break;
1169
1170                 case 'm':
1171                         if (sscanf(item+2, "%llx", &ll) != 1)
1172                                 return -EINVAL;
1173                         if (ll != le64toh(o->entry.monotonic))
1174                                 return 0;
1175                         break;
1176
1177                 case 't':
1178                         if (sscanf(item+2, "%llx", &ll) != 1)
1179                                 return -EINVAL;
1180                         if (ll != le64toh(o->entry.realtime))
1181                                 return 0;
1182                         break;
1183
1184                 case 'x':
1185                         if (sscanf(item+2, "%llx", &ll) != 1)
1186                                 return -EINVAL;
1187                         if (ll != le64toh(o->entry.xor_hash))
1188                                 return 0;
1189                         break;
1190                 }
1191         }
1192
1193         return 1;
1194 }
1195
1196
1197 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1198         if (!j)
1199                 return -EINVAL;
1200
1201         reset_location(j);
1202         j->current_location.type = LOCATION_SEEK;
1203         j->current_location.boot_id = boot_id;
1204         j->current_location.monotonic = usec;
1205         j->current_location.monotonic_set = true;
1206
1207         return 0;
1208 }
1209
1210 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1211         if (!j)
1212                 return -EINVAL;
1213
1214         reset_location(j);
1215         j->current_location.type = LOCATION_SEEK;
1216         j->current_location.realtime = usec;
1217         j->current_location.realtime_set = true;
1218
1219         return 0;
1220 }
1221
1222 _public_ int sd_journal_seek_head(sd_journal *j) {
1223         if (!j)
1224                 return -EINVAL;
1225
1226         reset_location(j);
1227         j->current_location.type = LOCATION_HEAD;
1228
1229         return 0;
1230 }
1231
1232 _public_ int sd_journal_seek_tail(sd_journal *j) {
1233         if (!j)
1234                 return -EINVAL;
1235
1236         reset_location(j);
1237         j->current_location.type = LOCATION_TAIL;
1238
1239         return 0;
1240 }
1241
1242 static void check_network(sd_journal *j, int fd) {
1243         struct statfs sfs;
1244
1245         assert(j);
1246
1247         if (j->on_network)
1248                 return;
1249
1250         if (fstatfs(fd, &sfs) < 0)
1251                 return;
1252
1253         j->on_network =
1254                 F_TYPE_CMP(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1255                 F_TYPE_CMP(sfs.f_type, CODA_SUPER_MAGIC) ||
1256                 F_TYPE_CMP(sfs.f_type, NCP_SUPER_MAGIC) ||
1257                 F_TYPE_CMP(sfs.f_type, NFS_SUPER_MAGIC) ||
1258                 F_TYPE_CMP(sfs.f_type, SMB_SUPER_MAGIC);
1259 }
1260
1261 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1262         _cleanup_free_ char *path = NULL;
1263         int r;
1264         JournalFile *f;
1265
1266         assert(j);
1267         assert(prefix);
1268         assert(filename);
1269
1270         if ((j->flags & SD_JOURNAL_SYSTEM_ONLY) &&
1271             !(streq(filename, "system.journal") ||
1272               streq(filename, "system.journal~") ||
1273               (startswith(filename, "system@") &&
1274                (endswith(filename, ".journal") || endswith(filename, ".journal~")))))
1275                 return 0;
1276
1277         path = strjoin(prefix, "/", filename, NULL);
1278         if (!path)
1279                 return -ENOMEM;
1280
1281         if (hashmap_get(j->files, path))
1282                 return 0;
1283
1284         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1285                 log_debug("Too many open journal files, not adding %s, ignoring.", path);
1286                 return set_put_error(j, -ETOOMANYREFS);
1287         }
1288
1289         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1290         if (r < 0) {
1291                 if (errno == ENOENT)
1292                         return 0;
1293
1294                 return r;
1295         }
1296
1297         /* journal_file_dump(f); */
1298
1299         r = hashmap_put(j->files, f->path, f);
1300         if (r < 0) {
1301                 journal_file_close(f);
1302                 return r;
1303         }
1304
1305         log_debug("File %s got added.", f->path);
1306
1307         check_network(j, f->fd);
1308
1309         j->current_invalidate_counter ++;
1310
1311         return 0;
1312 }
1313
1314 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1315         char *path;
1316         JournalFile *f;
1317
1318         assert(j);
1319         assert(prefix);
1320         assert(filename);
1321
1322         path = strjoin(prefix, "/", filename, NULL);
1323         if (!path)
1324                 return -ENOMEM;
1325
1326         f = hashmap_get(j->files, path);
1327         free(path);
1328         if (!f)
1329                 return 0;
1330
1331         hashmap_remove(j->files, f->path);
1332
1333         log_debug("File %s got removed.", f->path);
1334
1335         if (j->current_file == f) {
1336                 j->current_file = NULL;
1337                 j->current_field = 0;
1338         }
1339
1340         if (j->unique_file == f) {
1341                 j->unique_file = NULL;
1342                 j->unique_offset = 0;
1343         }
1344
1345         journal_file_close(f);
1346
1347         j->current_invalidate_counter ++;
1348
1349         return 0;
1350 }
1351
1352 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1353         _cleanup_free_ char *path = NULL;
1354         int r;
1355         _cleanup_closedir_ DIR *d = NULL;
1356         sd_id128_t id, mid;
1357         Directory *m;
1358
1359         assert(j);
1360         assert(prefix);
1361         assert(dirname);
1362
1363         log_debug("Considering %s/%s.", prefix, dirname);
1364
1365         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1366             (sd_id128_from_string(dirname, &id) < 0 ||
1367              sd_id128_get_machine(&mid) < 0 ||
1368              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1369             return 0;
1370
1371         path = strjoin(prefix, "/", dirname, NULL);
1372         if (!path)
1373                 return -ENOMEM;
1374
1375         d = opendir(path);
1376         if (!d) {
1377                 log_debug("Failed to open %s: %m", path);
1378                 if (errno == ENOENT)
1379                         return 0;
1380                 return -errno;
1381         }
1382
1383         m = hashmap_get(j->directories_by_path, path);
1384         if (!m) {
1385                 m = new0(Directory, 1);
1386                 if (!m)
1387                         return -ENOMEM;
1388
1389                 m->is_root = false;
1390                 m->path = path;
1391
1392                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1393                         free(m);
1394                         return -ENOMEM;
1395                 }
1396
1397                 path = NULL; /* avoid freeing in cleanup */
1398                 j->current_invalidate_counter ++;
1399
1400                 log_debug("Directory %s got added.", m->path);
1401
1402         } else if (m->is_root)
1403                 return 0;
1404
1405         if (m->wd <= 0 && j->inotify_fd >= 0) {
1406
1407                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1408                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1409                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1410                                           IN_ONLYDIR);
1411
1412                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1413                         inotify_rm_watch(j->inotify_fd, m->wd);
1414         }
1415
1416         for (;;) {
1417                 struct dirent *de;
1418                 union dirent_storage buf;
1419
1420                 r = readdir_r(d, &buf.de, &de);
1421                 if (r != 0 || !de)
1422                         break;
1423
1424                 if (dirent_is_file_with_suffix(de, ".journal") ||
1425                     dirent_is_file_with_suffix(de, ".journal~")) {
1426                         r = add_file(j, m->path, de->d_name);
1427                         if (r < 0) {
1428                                 log_debug("Failed to add file %s/%s: %s",
1429                                           m->path, de->d_name, strerror(-r));
1430                                 r = set_put_error(j, r);
1431                                 if (r < 0)
1432                                         return r;
1433                         }
1434                 }
1435         }
1436
1437         check_network(j, dirfd(d));
1438
1439         return 0;
1440 }
1441
1442 static int add_root_directory(sd_journal *j, const char *p) {
1443         _cleanup_closedir_ DIR *d = NULL;
1444         Directory *m;
1445         int r;
1446
1447         assert(j);
1448         assert(p);
1449
1450         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1451             !path_startswith(p, "/run"))
1452                 return -EINVAL;
1453
1454         d = opendir(p);
1455         if (!d)
1456                 return -errno;
1457
1458         m = hashmap_get(j->directories_by_path, p);
1459         if (!m) {
1460                 m = new0(Directory, 1);
1461                 if (!m)
1462                         return -ENOMEM;
1463
1464                 m->is_root = true;
1465                 m->path = strdup(p);
1466                 if (!m->path) {
1467                         free(m);
1468                         return -ENOMEM;
1469                 }
1470
1471                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1472                         free(m->path);
1473                         free(m);
1474                         return -ENOMEM;
1475                 }
1476
1477                 j->current_invalidate_counter ++;
1478
1479                 log_debug("Root directory %s got added.", m->path);
1480
1481         } else if (!m->is_root)
1482                 return 0;
1483
1484         if (m->wd <= 0 && j->inotify_fd >= 0) {
1485
1486                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1487                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1488                                           IN_ONLYDIR);
1489
1490                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1491                         inotify_rm_watch(j->inotify_fd, m->wd);
1492         }
1493
1494         for (;;) {
1495                 struct dirent *de;
1496                 union dirent_storage buf;
1497                 sd_id128_t id;
1498
1499                 r = readdir_r(d, &buf.de, &de);
1500                 if (r != 0 || !de)
1501                         break;
1502
1503                 if (dirent_is_file_with_suffix(de, ".journal") ||
1504                     dirent_is_file_with_suffix(de, ".journal~")) {
1505                         r = add_file(j, m->path, de->d_name);
1506                         if (r < 0) {
1507                                 log_debug("Failed to add file %s/%s: %s",
1508                                           m->path, de->d_name, strerror(-r));
1509                                 r = set_put_error(j, r);
1510                                 if (r < 0)
1511                                         return r;
1512                         }
1513                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1514                            sd_id128_from_string(de->d_name, &id) >= 0) {
1515
1516                         r = add_directory(j, m->path, de->d_name);
1517                         if (r < 0)
1518                                 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1519                 }
1520         }
1521
1522         check_network(j, dirfd(d));
1523
1524         return 0;
1525 }
1526
1527 static int remove_directory(sd_journal *j, Directory *d) {
1528         assert(j);
1529
1530         if (d->wd > 0) {
1531                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1532
1533                 if (j->inotify_fd >= 0)
1534                         inotify_rm_watch(j->inotify_fd, d->wd);
1535         }
1536
1537         hashmap_remove(j->directories_by_path, d->path);
1538
1539         if (d->is_root)
1540                 log_debug("Root directory %s got removed.", d->path);
1541         else
1542                 log_debug("Directory %s got removed.", d->path);
1543
1544         free(d->path);
1545         free(d);
1546
1547         return 0;
1548 }
1549
1550 static int add_search_paths(sd_journal *j) {
1551         int r;
1552         const char search_paths[] =
1553                 "/run/log/journal\0"
1554                 "/var/log/journal\0";
1555         const char *p;
1556
1557         assert(j);
1558
1559         /* We ignore most errors here, since the idea is to only open
1560          * what's actually accessible, and ignore the rest. */
1561
1562         NULSTR_FOREACH(p, search_paths) {
1563                 r = add_root_directory(j, p);
1564                 if (r < 0 && r != -ENOENT) {
1565                         r = set_put_error(j, r);
1566                         if (r < 0)
1567                                 return r;
1568                 }
1569         }
1570
1571         return 0;
1572 }
1573
1574 static int allocate_inotify(sd_journal *j) {
1575         assert(j);
1576
1577         if (j->inotify_fd < 0) {
1578                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1579                 if (j->inotify_fd < 0)
1580                         return -errno;
1581         }
1582
1583         if (!j->directories_by_wd) {
1584                 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1585                 if (!j->directories_by_wd)
1586                         return -ENOMEM;
1587         }
1588
1589         return 0;
1590 }
1591
1592 static sd_journal *journal_new(int flags, const char *path) {
1593         sd_journal *j;
1594
1595         j = new0(sd_journal, 1);
1596         if (!j)
1597                 return NULL;
1598
1599         j->inotify_fd = -1;
1600         j->flags = flags;
1601         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1602
1603         if (path) {
1604                 j->path = strdup(path);
1605                 if (!j->path)
1606                         goto fail;
1607         }
1608
1609         j->files = hashmap_new(string_hash_func, string_compare_func);
1610         j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1611         j->mmap = mmap_cache_new();
1612         if (!j->files || !j->directories_by_path || !j->mmap)
1613                 goto fail;
1614
1615         return j;
1616
1617 fail:
1618         sd_journal_close(j);
1619         return NULL;
1620 }
1621
1622 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1623         sd_journal *j;
1624         int r;
1625
1626         if (!ret)
1627                 return -EINVAL;
1628
1629         if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1630                       SD_JOURNAL_RUNTIME_ONLY|
1631                       SD_JOURNAL_SYSTEM_ONLY))
1632                 return -EINVAL;
1633
1634         j = journal_new(flags, NULL);
1635         if (!j)
1636                 return -ENOMEM;
1637
1638         r = add_search_paths(j);
1639         if (r < 0)
1640                 goto fail;
1641
1642         *ret = j;
1643         return 0;
1644
1645 fail:
1646         sd_journal_close(j);
1647
1648         return r;
1649 }
1650
1651 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1652         sd_journal *j;
1653         int r;
1654
1655         if (!ret)
1656                 return -EINVAL;
1657
1658         if (!path)
1659                 return -EINVAL;
1660
1661         if (flags != 0)
1662                 return -EINVAL;
1663
1664         j = journal_new(flags, path);
1665         if (!j)
1666                 return -ENOMEM;
1667
1668         r = add_root_directory(j, path);
1669         if (r < 0) {
1670                 set_put_error(j, r);
1671                 goto fail;
1672         }
1673
1674         *ret = j;
1675         return 0;
1676
1677 fail:
1678         sd_journal_close(j);
1679
1680         return r;
1681 }
1682
1683 _public_ void sd_journal_close(sd_journal *j) {
1684         Directory *d;
1685         JournalFile *f;
1686
1687         if (!j)
1688                 return;
1689
1690         sd_journal_flush_matches(j);
1691
1692         while ((f = hashmap_steal_first(j->files)))
1693                 journal_file_close(f);
1694
1695         hashmap_free(j->files);
1696
1697         while ((d = hashmap_first(j->directories_by_path)))
1698                 remove_directory(j, d);
1699
1700         while ((d = hashmap_first(j->directories_by_wd)))
1701                 remove_directory(j, d);
1702
1703         hashmap_free(j->directories_by_path);
1704         hashmap_free(j->directories_by_wd);
1705
1706         if (j->inotify_fd >= 0)
1707                 close_nointr_nofail(j->inotify_fd);
1708
1709         if (j->mmap)
1710                 mmap_cache_unref(j->mmap);
1711
1712         free(j->path);
1713         free(j->unique_field);
1714         set_free(j->errors);
1715         free(j);
1716 }
1717
1718 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1719         Object *o;
1720         JournalFile *f;
1721         int r;
1722
1723         if (!j)
1724                 return -EINVAL;
1725         if (!ret)
1726                 return -EINVAL;
1727
1728         f = j->current_file;
1729         if (!f)
1730                 return -EADDRNOTAVAIL;
1731
1732         if (f->current_offset <= 0)
1733                 return -EADDRNOTAVAIL;
1734
1735         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1736         if (r < 0)
1737                 return r;
1738
1739         *ret = le64toh(o->entry.realtime);
1740         return 0;
1741 }
1742
1743 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1744         Object *o;
1745         JournalFile *f;
1746         int r;
1747         sd_id128_t id;
1748
1749         if (!j)
1750                 return -EINVAL;
1751
1752         f = j->current_file;
1753         if (!f)
1754                 return -EADDRNOTAVAIL;
1755
1756         if (f->current_offset <= 0)
1757                 return -EADDRNOTAVAIL;
1758
1759         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1760         if (r < 0)
1761                 return r;
1762
1763         if (ret_boot_id)
1764                 *ret_boot_id = o->entry.boot_id;
1765         else {
1766                 r = sd_id128_get_boot(&id);
1767                 if (r < 0)
1768                         return r;
1769
1770                 if (!sd_id128_equal(id, o->entry.boot_id))
1771                         return -ESTALE;
1772         }
1773
1774         if (ret)
1775                 *ret = le64toh(o->entry.monotonic);
1776
1777         return 0;
1778 }
1779
1780 static bool field_is_valid(const char *field) {
1781         const char *p;
1782
1783         assert(field);
1784
1785         if (isempty(field))
1786                 return false;
1787
1788         if (startswith(field, "__"))
1789                 return false;
1790
1791         for (p = field; *p; p++) {
1792
1793                 if (*p == '_')
1794                         continue;
1795
1796                 if (*p >= 'A' && *p <= 'Z')
1797                         continue;
1798
1799                 if (*p >= '0' && *p <= '9')
1800                         continue;
1801
1802                 return false;
1803         }
1804
1805         return true;
1806 }
1807
1808 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1809         JournalFile *f;
1810         uint64_t i, n;
1811         size_t field_length;
1812         int r;
1813         Object *o;
1814
1815         if (!j)
1816                 return -EINVAL;
1817         if (!field)
1818                 return -EINVAL;
1819         if (!data)
1820                 return -EINVAL;
1821         if (!size)
1822                 return -EINVAL;
1823
1824         if (!field_is_valid(field))
1825                 return -EINVAL;
1826
1827         f = j->current_file;
1828         if (!f)
1829                 return -EADDRNOTAVAIL;
1830
1831         if (f->current_offset <= 0)
1832                 return -EADDRNOTAVAIL;
1833
1834         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1835         if (r < 0)
1836                 return r;
1837
1838         field_length = strlen(field);
1839
1840         n = journal_file_entry_n_items(o);
1841         for (i = 0; i < n; i++) {
1842                 uint64_t p, l;
1843                 le64_t le_hash;
1844                 size_t t;
1845
1846                 p = le64toh(o->entry.items[i].object_offset);
1847                 le_hash = o->entry.items[i].hash;
1848                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1849                 if (r < 0)
1850                         return r;
1851
1852                 if (le_hash != o->data.hash)
1853                         return -EBADMSG;
1854
1855                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1856
1857                 if (o->object.flags & OBJECT_COMPRESSED) {
1858
1859 #ifdef HAVE_XZ
1860                         if (uncompress_startswith(o->data.payload, l,
1861                                                   &f->compress_buffer, &f->compress_buffer_size,
1862                                                   field, field_length, '=')) {
1863
1864                                 uint64_t rsize;
1865
1866                                 if (!uncompress_blob(o->data.payload, l,
1867                                                      &f->compress_buffer, &f->compress_buffer_size, &rsize,
1868                                                      j->data_threshold))
1869                                         return -EBADMSG;
1870
1871                                 *data = f->compress_buffer;
1872                                 *size = (size_t) rsize;
1873
1874                                 return 0;
1875                         }
1876 #else
1877                         return -EPROTONOSUPPORT;
1878 #endif
1879
1880                 } else if (l >= field_length+1 &&
1881                            memcmp(o->data.payload, field, field_length) == 0 &&
1882                            o->data.payload[field_length] == '=') {
1883
1884                         t = (size_t) l;
1885
1886                         if ((uint64_t) t != l)
1887                                 return -E2BIG;
1888
1889                         *data = o->data.payload;
1890                         *size = t;
1891
1892                         return 0;
1893                 }
1894
1895                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1896                 if (r < 0)
1897                         return r;
1898         }
1899
1900         return -ENOENT;
1901 }
1902
1903 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1904         size_t t;
1905         uint64_t l;
1906
1907         l = le64toh(o->object.size) - offsetof(Object, data.payload);
1908         t = (size_t) l;
1909
1910         /* We can't read objects larger than 4G on a 32bit machine */
1911         if ((uint64_t) t != l)
1912                 return -E2BIG;
1913
1914         if (o->object.flags & OBJECT_COMPRESSED) {
1915 #ifdef HAVE_XZ
1916                 uint64_t rsize;
1917
1918                 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
1919                         return -EBADMSG;
1920
1921                 *data = f->compress_buffer;
1922                 *size = (size_t) rsize;
1923 #else
1924                 return -EPROTONOSUPPORT;
1925 #endif
1926         } else {
1927                 *data = o->data.payload;
1928                 *size = t;
1929         }
1930
1931         return 0;
1932 }
1933
1934 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1935         JournalFile *f;
1936         uint64_t p, n;
1937         le64_t le_hash;
1938         int r;
1939         Object *o;
1940
1941         if (!j)
1942                 return -EINVAL;
1943         if (!data)
1944                 return -EINVAL;
1945         if (!size)
1946                 return -EINVAL;
1947
1948         f = j->current_file;
1949         if (!f)
1950                 return -EADDRNOTAVAIL;
1951
1952         if (f->current_offset <= 0)
1953                 return -EADDRNOTAVAIL;
1954
1955         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1956         if (r < 0)
1957                 return r;
1958
1959         n = journal_file_entry_n_items(o);
1960         if (j->current_field >= n)
1961                 return 0;
1962
1963         p = le64toh(o->entry.items[j->current_field].object_offset);
1964         le_hash = o->entry.items[j->current_field].hash;
1965         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1966         if (r < 0)
1967                 return r;
1968
1969         if (le_hash != o->data.hash)
1970                 return -EBADMSG;
1971
1972         r = return_data(j, f, o, data, size);
1973         if (r < 0)
1974                 return r;
1975
1976         j->current_field ++;
1977
1978         return 1;
1979 }
1980
1981 _public_ void sd_journal_restart_data(sd_journal *j) {
1982         if (!j)
1983                 return;
1984
1985         j->current_field = 0;
1986 }
1987
1988 _public_ int sd_journal_get_fd(sd_journal *j) {
1989         int r;
1990
1991         if (!j)
1992                 return -EINVAL;
1993
1994         if (j->inotify_fd >= 0)
1995                 return j->inotify_fd;
1996
1997         r = allocate_inotify(j);
1998         if (r < 0)
1999                 return r;
2000
2001         /* Iterate through all dirs again, to add them to the
2002          * inotify */
2003         if (j->path)
2004                 r = add_root_directory(j, j->path);
2005         else
2006                 r = add_search_paths(j);
2007         if (r < 0)
2008                 return r;
2009
2010         return j->inotify_fd;
2011 }
2012
2013 _public_ int sd_journal_get_events(sd_journal *j) {
2014         int fd;
2015
2016         if (!j)
2017                 return -EINVAL;
2018
2019         fd = sd_journal_get_fd(j);
2020         if (fd < 0)
2021                 return fd;
2022
2023         return POLLIN;
2024 }
2025
2026 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2027         int fd;
2028
2029         if (!j)
2030                 return -EINVAL;
2031         if (!timeout_usec)
2032                 return -EINVAL;
2033
2034         fd = sd_journal_get_fd(j);
2035         if (fd < 0)
2036                 return fd;
2037
2038         if (!j->on_network) {
2039                 *timeout_usec = (uint64_t) -1;
2040                 return 0;
2041         }
2042
2043         /* If we are on the network we need to regularly check for
2044          * changes manually */
2045
2046         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2047         return 1;
2048 }
2049
2050 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2051         Directory *d;
2052         int r;
2053
2054         assert(j);
2055         assert(e);
2056
2057         /* Is this a subdirectory we watch? */
2058         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2059         if (d) {
2060                 sd_id128_t id;
2061
2062                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2063                     (endswith(e->name, ".journal") ||
2064                      endswith(e->name, ".journal~"))) {
2065
2066                         /* Event for a journal file */
2067
2068                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2069                                 r = add_file(j, d->path, e->name);
2070                                 if (r < 0) {
2071                                         log_debug("Failed to add file %s/%s: %s",
2072                                                   d->path, e->name, strerror(-r));
2073                                         set_put_error(j, r);
2074                                 }
2075
2076                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2077
2078                                 r = remove_file(j, d->path, e->name);
2079                                 if (r < 0)
2080                                         log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2081                         }
2082
2083                 } else if (!d->is_root && e->len == 0) {
2084
2085                         /* Event for a subdirectory */
2086
2087                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2088                                 r = remove_directory(j, d);
2089                                 if (r < 0)
2090                                         log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2091                         }
2092
2093
2094                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2095
2096                         /* Event for root directory */
2097
2098                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2099                                 r = add_directory(j, d->path, e->name);
2100                                 if (r < 0)
2101                                         log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2102                         }
2103                 }
2104
2105                 return;
2106         }
2107
2108         if (e->mask & IN_IGNORED)
2109                 return;
2110
2111         log_warning("Unknown inotify event.");
2112 }
2113
2114 static int determine_change(sd_journal *j) {
2115         bool b;
2116
2117         assert(j);
2118
2119         b = j->current_invalidate_counter != j->last_invalidate_counter;
2120         j->last_invalidate_counter = j->current_invalidate_counter;
2121
2122         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2123 }
2124
2125 _public_ int sd_journal_process(sd_journal *j) {
2126         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2127         bool got_something = false;
2128
2129         if (!j)
2130                 return -EINVAL;
2131
2132         j->last_process_usec = now(CLOCK_MONOTONIC);
2133
2134         for (;;) {
2135                 struct inotify_event *e;
2136                 ssize_t l;
2137
2138                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2139                 if (l < 0) {
2140                         if (errno == EAGAIN || errno == EINTR)
2141                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2142
2143                         return -errno;
2144                 }
2145
2146                 got_something = true;
2147
2148                 e = (struct inotify_event*) buffer;
2149                 while (l > 0) {
2150                         size_t step;
2151
2152                         process_inotify_event(j, e);
2153
2154                         step = sizeof(struct inotify_event) + e->len;
2155                         assert(step <= (size_t) l);
2156
2157                         e = (struct inotify_event*) ((uint8_t*) e + step);
2158                         l -= step;
2159                 }
2160         }
2161
2162         return determine_change(j);
2163 }
2164
2165 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2166         int r;
2167         uint64_t t;
2168
2169         assert(j);
2170
2171         if (j->inotify_fd < 0) {
2172
2173                 /* This is the first invocation, hence create the
2174                  * inotify watch */
2175                 r = sd_journal_get_fd(j);
2176                 if (r < 0)
2177                         return r;
2178
2179                 /* The journal might have changed since the context
2180                  * object was created and we weren't watching before,
2181                  * hence don't wait for anything, and return
2182                  * immediately. */
2183                 return determine_change(j);
2184         }
2185
2186         r = sd_journal_get_timeout(j, &t);
2187         if (r < 0)
2188                 return r;
2189
2190         if (t != (uint64_t) -1) {
2191                 usec_t n;
2192
2193                 n = now(CLOCK_MONOTONIC);
2194                 t = t > n ? t - n : 0;
2195
2196                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2197                         timeout_usec = t;
2198         }
2199
2200         do {
2201                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2202         } while (r == -EINTR);
2203
2204         if (r < 0)
2205                 return r;
2206
2207         return sd_journal_process(j);
2208 }
2209
2210 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2211         Iterator i;
2212         JournalFile *f;
2213         bool first = true;
2214         int r;
2215
2216         if (!j)
2217                 return -EINVAL;
2218         if (!from && !to)
2219                 return -EINVAL;
2220
2221         HASHMAP_FOREACH(f, j->files, i) {
2222                 usec_t fr, t;
2223
2224                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2225                 if (r == -ENOENT)
2226                         continue;
2227                 if (r < 0)
2228                         return r;
2229                 if (r == 0)
2230                         continue;
2231
2232                 if (first) {
2233                         if (from)
2234                                 *from = fr;
2235                         if (to)
2236                                 *to = t;
2237                         first = false;
2238                 } else {
2239                         if (from)
2240                                 *from = MIN(fr, *from);
2241                         if (to)
2242                                 *to = MAX(t, *to);
2243                 }
2244         }
2245
2246         return first ? 0 : 1;
2247 }
2248
2249 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2250         Iterator i;
2251         JournalFile *f;
2252         bool first = true;
2253         int r;
2254
2255         if (!j)
2256                 return -EINVAL;
2257         if (!from && !to)
2258                 return -EINVAL;
2259
2260         HASHMAP_FOREACH(f, j->files, i) {
2261                 usec_t fr, t;
2262
2263                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2264                 if (r == -ENOENT)
2265                         continue;
2266                 if (r < 0)
2267                         return r;
2268                 if (r == 0)
2269                         continue;
2270
2271                 if (first) {
2272                         if (from)
2273                                 *from = fr;
2274                         if (to)
2275                                 *to = t;
2276                         first = false;
2277                 } else {
2278                         if (from)
2279                                 *from = MIN(fr, *from);
2280                         if (to)
2281                                 *to = MAX(t, *to);
2282                 }
2283         }
2284
2285         return first ? 0 : 1;
2286 }
2287
2288 void journal_print_header(sd_journal *j) {
2289         Iterator i;
2290         JournalFile *f;
2291         bool newline = false;
2292
2293         assert(j);
2294
2295         HASHMAP_FOREACH(f, j->files, i) {
2296                 if (newline)
2297                         putchar('\n');
2298                 else
2299                         newline = true;
2300
2301                 journal_file_print_header(f);
2302         }
2303 }
2304
2305 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2306         Iterator i;
2307         JournalFile *f;
2308         uint64_t sum = 0;
2309
2310         if (!j)
2311                 return -EINVAL;
2312         if (!bytes)
2313                 return -EINVAL;
2314
2315         HASHMAP_FOREACH(f, j->files, i) {
2316                 struct stat st;
2317
2318                 if (fstat(f->fd, &st) < 0)
2319                         return -errno;
2320
2321                 sum += (uint64_t) st.st_blocks * 512ULL;
2322         }
2323
2324         *bytes = sum;
2325         return 0;
2326 }
2327
2328 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2329         char *f;
2330
2331         if (!j)
2332                 return -EINVAL;
2333         if (isempty(field))
2334                 return -EINVAL;
2335         if (!field_is_valid(field))
2336                 return -EINVAL;
2337
2338         f = strdup(field);
2339         if (!f)
2340                 return -ENOMEM;
2341
2342         free(j->unique_field);
2343         j->unique_field = f;
2344         j->unique_file = NULL;
2345         j->unique_offset = 0;
2346
2347         return 0;
2348 }
2349
2350 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2351         Object *o;
2352         size_t k;
2353         int r;
2354
2355         if (!j)
2356                 return -EINVAL;
2357         if (!data)
2358                 return -EINVAL;
2359         if (!l)
2360                 return -EINVAL;
2361         if (!j->unique_field)
2362                 return -EINVAL;
2363
2364         k = strlen(j->unique_field);
2365
2366         if (!j->unique_file) {
2367                 j->unique_file = hashmap_first(j->files);
2368                 if (!j->unique_file)
2369                         return 0;
2370                 j->unique_offset = 0;
2371         }
2372
2373         for (;;) {
2374                 JournalFile *of;
2375                 Iterator i;
2376                 const void *odata;
2377                 size_t ol;
2378                 bool found;
2379
2380                 /* Proceed to next data object in the field's linked list */
2381                 if (j->unique_offset == 0) {
2382                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2383                         if (r < 0)
2384                                 return r;
2385
2386                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2387                 } else {
2388                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2389                         if (r < 0)
2390                                 return r;
2391
2392                         j->unique_offset = le64toh(o->data.next_field_offset);
2393                 }
2394
2395                 /* We reached the end of the list? Then start again, with the next file */
2396                 if (j->unique_offset == 0) {
2397                         JournalFile *n;
2398
2399                         n = hashmap_next(j->files, j->unique_file->path);
2400                         if (!n)
2401                                 return 0;
2402
2403                         j->unique_file = n;
2404                         continue;
2405                 }
2406
2407                 /* We do not use the type context here, but 0 instead,
2408                  * so that we can look at this data object at the same
2409                  * time as one on another file */
2410                 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2411                 if (r < 0)
2412                         return r;
2413
2414                 /* Let's do the type check by hand, since we used 0 context above. */
2415                 if (o->object.type != OBJECT_DATA)
2416                         return -EBADMSG;
2417
2418                 r = return_data(j, j->unique_file, o, &odata, &ol);
2419                 if (r < 0)
2420                         return r;
2421
2422                 /* OK, now let's see if we already returned this data
2423                  * object by checking if it exists in the earlier
2424                  * traversed files. */
2425                 found = false;
2426                 HASHMAP_FOREACH(of, j->files, i) {
2427                         Object *oo;
2428                         uint64_t op;
2429
2430                         if (of == j->unique_file)
2431                                 break;
2432
2433                         /* Skip this file it didn't have any fields
2434                          * indexed */
2435                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2436                             le64toh(of->header->n_fields) <= 0)
2437                                 continue;
2438
2439                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2440                         if (r < 0)
2441                                 return r;
2442
2443                         if (r > 0)
2444                                 found = true;
2445                 }
2446
2447                 if (found)
2448                         continue;
2449
2450                 r = return_data(j, j->unique_file, o, data, l);
2451                 if (r < 0)
2452                         return r;
2453
2454                 return 1;
2455         }
2456 }
2457
2458 _public_ void sd_journal_restart_unique(sd_journal *j) {
2459         if (!j)
2460                 return;
2461
2462         j->unique_file = NULL;
2463         j->unique_offset = 0;
2464 }
2465
2466 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2467         if (!j)
2468                 return -EINVAL;
2469
2470         return !j->on_network;
2471 }
2472
2473 static char *lookup_field(const char *field, void *userdata) {
2474         sd_journal *j = userdata;
2475         const void *data;
2476         size_t size, d;
2477         int r;
2478
2479         assert(field);
2480         assert(j);
2481
2482         r = sd_journal_get_data(j, field, &data, &size);
2483         if (r < 0 ||
2484             size > REPLACE_VAR_MAX)
2485                 return strdup(field);
2486
2487         d = strlen(field) + 1;
2488
2489         return strndup((const char*) data + d, size - d);
2490 }
2491
2492 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2493         const void *data;
2494         size_t size;
2495         sd_id128_t id;
2496         _cleanup_free_ char *text = NULL, *cid = NULL;
2497         char *t;
2498         int r;
2499
2500         if (!j)
2501                 return -EINVAL;
2502         if (!ret)
2503                 return -EINVAL;
2504
2505         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2506         if (r < 0)
2507                 return r;
2508
2509         cid = strndup((const char*) data + 11, size - 11);
2510         if (!cid)
2511                 return -ENOMEM;
2512
2513         r = sd_id128_from_string(cid, &id);
2514         if (r < 0)
2515                 return r;
2516
2517         r = catalog_get(CATALOG_DATABASE, id, &text);
2518         if (r < 0)
2519                 return r;
2520
2521         t = replace_var(text, lookup_field, j);
2522         if (!t)
2523                 return -ENOMEM;
2524
2525         *ret = t;
2526         return 0;
2527 }
2528
2529 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2530         if (!ret)
2531                 return -EINVAL;
2532
2533         return catalog_get(CATALOG_DATABASE, id, ret);
2534 }
2535
2536 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2537         if (!j)
2538                 return -EINVAL;
2539
2540         j->data_threshold = sz;
2541         return 0;
2542 }
2543
2544 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2545         if (!j)
2546                 return -EINVAL;
2547         if (!sz)
2548                 return -EINVAL;
2549
2550         *sz = j->data_threshold;
2551         return 0;
2552 }