src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "strv.h"
  37 #include "path-util.h"
  38 #include "lookup3.h"
  39 #include "compress.h"
  40 #include "journal-internal.h"
  41 #include "missing.h"
  42 #include "catalog.h"
  43 #include "replace-var.h"
  44
  45 #define JOURNAL_FILES_MAX 1024
  46
  47 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  48
  49 #define REPLACE_VAR_MAX 256
  50
  51 #define DEFAULT_DATA_THRESHOLD (64*1024)
  52
  53 static bool journal_pid_changed(sd_journal *j) {
  54         assert(j);
  55
  56         /* We don't support people creating a journal object and
  57          * keeping it around over a fork(). Let's complain. */
  58
  59         return j->original_pid != getpid();
  60 }
  61
  62 /* We return an error here only if we didn't manage to
  63    memorize the real error. */
  64 static int set_put_error(sd_journal *j, int r) {
  65         int k;
  66
  67         if (r >= 0)
  68                 return r;
  69
  70         k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
  71         if (k < 0)
  72                 return k;
  73
  74         return set_put(j->errors, INT_TO_PTR(r));
  75 }
  76
  77 static void detach_location(sd_journal *j) {
  78         Iterator i;
  79         JournalFile *f;
  80
  81         assert(j);
  82
  83         j->current_file = NULL;
  84         j->current_field = 0;
  85
  86         HASHMAP_FOREACH(f, j->files, i)
  87                 f->current_offset = 0;
  88 }
  89
  90 static void reset_location(sd_journal *j) {
  91         assert(j);
  92
  93         detach_location(j);
  94         zero(j->current_location);
  95 }
  96
  97 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  98         assert(l);
  99         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 100         assert(f);
 101         assert(o->object.type == OBJECT_ENTRY);
 102
 103         l->type = type;
 104         l->seqnum = le64toh(o->entry.seqnum);
 105         l->seqnum_id = f->header->seqnum_id;
 106         l->realtime = le64toh(o->entry.realtime);
 107         l->monotonic = le64toh(o->entry.monotonic);
 108         l->boot_id = o->entry.boot_id;
 109         l->xor_hash = le64toh(o->entry.xor_hash);
 110
 111         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 112 }
 113
 114 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o,
 115                          direction_t direction, uint64_t offset) {
 116         assert(j);
 117         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 118         assert(f);
 119         assert(o);
 120
 121         init_location(&j->current_location, type, f, o);
 122
 123         j->current_file = f;
 124         j->current_field = 0;
 125
 126         f->last_direction = direction;
 127         f->current_offset = offset;
 128 }
 129
 130 static int match_is_valid(const void *data, size_t size) {
 131         const char *b, *p;
 132
 133         assert(data);
 134
 135         if (size < 2)
 136                 return false;
 137
 138         if (startswith(data, "__"))
 139                 return false;
 140
 141         b = data;
 142         for (p = b; p < b + size; p++) {
 143
 144                 if (*p == '=')
 145                         return p > b;
 146
 147                 if (*p == '_')
 148                         continue;
 149
 150                 if (*p >= 'A' && *p <= 'Z')
 151                         continue;
 152
 153                 if (*p >= '0' && *p <= '9')
 154                         continue;
 155
 156                 return false;
 157         }
 158
 159         return false;
 160 }
 161
 162 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 163         const uint8_t *a = _a, *b = _b;
 164         size_t j;
 165
 166         for (j = 0; j < s && j < t; j++) {
 167
 168                 if (a[j] != b[j])
 169                         return false;
 170
 171                 if (a[j] == '=')
 172                         return true;
 173         }
 174
 175         assert_not_reached("\"=\" not found");
 176 }
 177
 178 static Match *match_new(Match *p, MatchType t) {
 179         Match *m;
 180
 181         m = new0(Match, 1);
 182         if (!m)
 183                 return NULL;
 184
 185         m->type = t;
 186
 187         if (p) {
 188                 m->parent = p;
 189                 LIST_PREPEND(matches, p->matches, m);
 190         }
 191
 192         return m;
 193 }
 194
 195 static void match_free(Match *m) {
 196         assert(m);
 197
 198         while (m->matches)
 199                 match_free(m->matches);
 200
 201         if (m->parent)
 202                 LIST_REMOVE(matches, m->parent->matches, m);
 203
 204         free(m->data);
 205         free(m);
 206 }
 207
 208 static void match_free_if_empty(Match *m) {
 209         if (!m || m->matches)
 210                 return;
 211
 212         match_free(m);
 213 }
 214
 215 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 216         Match *l3, *l4, *add_here = NULL, *m;
 217         le64_t le_hash;
 218
 219         if (!j)
 220                 return -EINVAL;
 221         if (journal_pid_changed(j))
 222                 return -ECHILD;
 223
 224         if (!data)
 225                 return -EINVAL;
 226
 227         if (size == 0)
 228                 size = strlen(data);
 229
 230         if (!match_is_valid(data, size))
 231                 return -EINVAL;
 232
 233         /* level 0: AND term
 234          * level 1: OR terms
 235          * level 2: AND terms
 236          * level 3: OR terms
 237          * level 4: concrete matches */
 238
 239         if (!j->level0) {
 240                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 241                 if (!j->level0)
 242                         return -ENOMEM;
 243         }
 244
 245         if (!j->level1) {
 246                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 247                 if (!j->level1)
 248                         return -ENOMEM;
 249         }
 250
 251         if (!j->level2) {
 252                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 253                 if (!j->level2)
 254                         return -ENOMEM;
 255         }
 256
 257         assert(j->level0->type == MATCH_AND_TERM);
 258         assert(j->level1->type == MATCH_OR_TERM);
 259         assert(j->level2->type == MATCH_AND_TERM);
 260
 261         le_hash = htole64(hash64(data, size));
 262
 263         LIST_FOREACH(matches, l3, j->level2->matches) {
 264                 assert(l3->type == MATCH_OR_TERM);
 265
 266                 LIST_FOREACH(matches, l4, l3->matches) {
 267                         assert(l4->type == MATCH_DISCRETE);
 268
 269                         /* Exactly the same match already? Then ignore
 270                          * this addition */
 271                         if (l4->le_hash == le_hash &&
 272                             l4->size == size &&
 273                             memcmp(l4->data, data, size) == 0)
 274                                 return 0;
 275
 276                         /* Same field? Then let's add this to this OR term */
 277                         if (same_field(data, size, l4->data, l4->size)) {
 278                                 add_here = l3;
 279                                 break;
 280                         }
 281                 }
 282
 283                 if (add_here)
 284                         break;
 285         }
 286
 287         if (!add_here) {
 288                 add_here = match_new(j->level2, MATCH_OR_TERM);
 289                 if (!add_here)
 290                         goto fail;
 291         }
 292
 293         m = match_new(add_here, MATCH_DISCRETE);
 294         if (!m)
 295                 goto fail;
 296
 297         m->le_hash = le_hash;
 298         m->size = size;
 299         m->data = memdup(data, size);
 300         if (!m->data)
 301                 goto fail;
 302
 303         detach_location(j);
 304
 305         return 0;
 306
 307 fail:
 308         match_free_if_empty(add_here);
 309         match_free_if_empty(j->level2);
 310         match_free_if_empty(j->level1);
 311         match_free_if_empty(j->level0);
 312
 313         return -ENOMEM;
 314 }
 315
 316 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 317         if (!j)
 318                 return -EINVAL;
 319         if (journal_pid_changed(j))
 320                 return -ECHILD;
 321
 322         if (!j->level0)
 323                 return 0;
 324
 325         if (!j->level1)
 326                 return 0;
 327
 328         if (!j->level1->matches)
 329                 return 0;
 330
 331         j->level1 = NULL;
 332         j->level2 = NULL;
 333
 334         return 0;
 335 }
 336
 337 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 338         if (!j)
 339                 return -EINVAL;
 340         if (journal_pid_changed(j))
 341                 return -ECHILD;
 342
 343         if (!j->level0)
 344                 return 0;
 345
 346         if (!j->level1)
 347                 return 0;
 348
 349         if (!j->level2)
 350                 return 0;
 351
 352         if (!j->level2->matches)
 353                 return 0;
 354
 355         j->level2 = NULL;
 356         return 0;
 357 }
 358
 359 static char *match_make_string(Match *m) {
 360         char *p, *r;
 361         Match *i;
 362         bool enclose = false;
 363
 364         if (!m)
 365                 return strdup("none");
 366
 367         if (m->type == MATCH_DISCRETE)
 368                 return strndup(m->data, m->size);
 369
 370         p = NULL;
 371         LIST_FOREACH(matches, i, m->matches) {
 372                 char *t, *k;
 373
 374                 t = match_make_string(i);
 375                 if (!t) {
 376                         free(p);
 377                         return NULL;
 378                 }
 379
 380                 if (p) {
 381                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 382                         free(p);
 383                         free(t);
 384
 385                         if (!k)
 386                                 return NULL;
 387
 388                         p = k;
 389
 390                         enclose = true;
 391                 } else
 392                         p = t;
 393         }
 394
 395         if (enclose) {
 396                 r = strjoin("(", p, ")", NULL);
 397                 free(p);
 398                 return r;
 399         }
 400
 401         return p;
 402 }
 403
 404 char *journal_make_match_string(sd_journal *j) {
 405         assert(j);
 406
 407         return match_make_string(j->level0);
 408 }
 409
 410 _public_ void sd_journal_flush_matches(sd_journal *j) {
 411         if (!j)
 412                 return;
 413
 414         if (j->level0)
 415                 match_free(j->level0);
 416
 417         j->level0 = j->level1 = j->level2 = NULL;
 418
 419         detach_location(j);
 420 }
 421
 422 static int compare_entry_order(JournalFile *af, Object *_ao,
 423                                JournalFile *bf, uint64_t bp) {
 424
 425         uint64_t a, b;
 426         Object *ao, *bo;
 427         int r;
 428
 429         assert(af);
 430         assert(bf);
 431         assert(_ao);
 432
 433         /* The mmap cache might invalidate the object from the first
 434          * file if we look at the one from the second file. Hence
 435          * temporarily copy the header of the first one, and look at
 436          * that only. */
 437         ao = alloca(offsetof(EntryObject, items));
 438         memcpy(ao, _ao, offsetof(EntryObject, items));
 439
 440         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 441         if (r < 0)
 442                 return strcmp(af->path, bf->path);
 443
 444         /* We operate on two different files here, hence we can access
 445          * two objects at the same time, which we normally can't.
 446          *
 447          * If contents and timestamps match, these entries are
 448          * identical, even if the seqnum does not match */
 449
 450         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 451             ao->entry.monotonic == bo->entry.monotonic &&
 452             ao->entry.realtime == bo->entry.realtime &&
 453             ao->entry.xor_hash == bo->entry.xor_hash)
 454                 return 0;
 455
 456         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 457
 458                 /* If this is from the same seqnum source, compare
 459                  * seqnums */
 460                 a = le64toh(ao->entry.seqnum);
 461                 b = le64toh(bo->entry.seqnum);
 462
 463                 if (a < b)
 464                         return -1;
 465                 if (a > b)
 466                         return 1;
 467
 468                 /* Wow! This is weird, different data but the same
 469                  * seqnums? Something is borked, but let's make the
 470                  * best of it and compare by time. */
 471         }
 472
 473         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 474
 475                 /* If the boot id matches compare monotonic time */
 476                 a = le64toh(ao->entry.monotonic);
 477                 b = le64toh(bo->entry.monotonic);
 478
 479                 if (a < b)
 480                         return -1;
 481                 if (a > b)
 482                         return 1;
 483         }
 484
 485         /* Otherwise compare UTC time */
 486         a = le64toh(ao->entry.realtime);
 487         b = le64toh(bo->entry.realtime);
 488
 489         if (a < b)
 490                 return -1;
 491         if (a > b)
 492                 return 1;
 493
 494         /* Finally, compare by contents */
 495         a = le64toh(ao->entry.xor_hash);
 496         b = le64toh(bo->entry.xor_hash);
 497
 498         if (a < b)
 499                 return -1;
 500         if (a > b)
 501                 return 1;
 502
 503         return 0;
 504 }
 505
 506 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 507         uint64_t a;
 508
 509         assert(af);
 510         assert(ao);
 511         assert(l);
 512         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 513
 514         if (l->monotonic_set &&
 515             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 516             l->realtime_set &&
 517             le64toh(ao->entry.realtime) == l->realtime &&
 518             l->xor_hash_set &&
 519             le64toh(ao->entry.xor_hash) == l->xor_hash)
 520                 return 0;
 521
 522         if (l->seqnum_set &&
 523             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 524
 525                 a = le64toh(ao->entry.seqnum);
 526
 527                 if (a < l->seqnum)
 528                         return -1;
 529                 if (a > l->seqnum)
 530                         return 1;
 531         }
 532
 533         if (l->monotonic_set &&
 534             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 535
 536                 a = le64toh(ao->entry.monotonic);
 537
 538                 if (a < l->monotonic)
 539                         return -1;
 540                 if (a > l->monotonic)
 541                         return 1;
 542         }
 543
 544         if (l->realtime_set) {
 545
 546                 a = le64toh(ao->entry.realtime);
 547
 548                 if (a < l->realtime)
 549                         return -1;
 550                 if (a > l->realtime)
 551                         return 1;
 552         }
 553
 554         if (l->xor_hash_set) {
 555                 a = le64toh(ao->entry.xor_hash);
 556
 557                 if (a < l->xor_hash)
 558                         return -1;
 559                 if (a > l->xor_hash)
 560                         return 1;
 561         }
 562
 563         return 0;
 564 }
 565
 566 static int next_for_match(
 567                 sd_journal *j,
 568                 Match *m,
 569                 JournalFile *f,
 570                 uint64_t after_offset,
 571                 direction_t direction,
 572                 Object **ret,
 573                 uint64_t *offset) {
 574
 575         int r;
 576         uint64_t np = 0;
 577         Object *n;
 578
 579         assert(j);
 580         assert(m);
 581         assert(f);
 582
 583         if (m->type == MATCH_DISCRETE) {
 584                 uint64_t dp;
 585
 586                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 587                 if (r <= 0)
 588                         return r;
 589
 590                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 591
 592         } else if (m->type == MATCH_OR_TERM) {
 593                 Match *i;
 594
 595                 /* Find the earliest match beyond after_offset */
 596
 597                 LIST_FOREACH(matches, i, m->matches) {
 598                         uint64_t cp;
 599
 600                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 601                         if (r < 0)
 602                                 return r;
 603                         else if (r > 0) {
 604                                 if (np == 0 || (direction == DIRECTION_DOWN ? cp < np : cp > np))
 605                                         np = cp;
 606                         }
 607                 }
 608
 609                 if (np == 0)
 610                         return 0;
 611
 612         } else if (m->type == MATCH_AND_TERM) {
 613                 Match *i, *last_moved;
 614
 615                 /* Always jump to the next matching entry and repeat
 616                  * this until we find an offset that matches for all
 617                  * matches. */
 618
 619                 if (!m->matches)
 620                         return 0;
 621
 622                 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
 623                 if (r <= 0)
 624                         return r;
 625
 626                 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
 627                 last_moved = m->matches;
 628
 629                 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
 630                         uint64_t cp;
 631
 632                         r = next_for_match(j, i, f, np, direction, NULL, &cp);
 633                         if (r <= 0)
 634                                 return r;
 635
 636                         assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
 637                         if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
 638                                 np = cp;
 639                                 last_moved = i;
 640                         }
 641                 }
 642         }
 643
 644         assert(np > 0);
 645
 646         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 647         if (r < 0)
 648                 return r;
 649
 650         if (ret)
 651                 *ret = n;
 652         if (offset)
 653                 *offset = np;
 654
 655         return 1;
 656 }
 657
 658 static int find_location_for_match(
 659                 sd_journal *j,
 660                 Match *m,
 661                 JournalFile *f,
 662                 direction_t direction,
 663                 Object **ret,
 664                 uint64_t *offset) {
 665
 666         int r;
 667
 668         assert(j);
 669         assert(m);
 670         assert(f);
 671
 672         if (m->type == MATCH_DISCRETE) {
 673                 uint64_t dp;
 674
 675                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 676                 if (r <= 0)
 677                         return r;
 678
 679                 /* FIXME: missing: find by monotonic */
 680
 681                 if (j->current_location.type == LOCATION_HEAD)
 682                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 683                 if (j->current_location.type == LOCATION_TAIL)
 684                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 685                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 686                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 687                 if (j->current_location.monotonic_set) {
 688                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 689                         if (r != -ENOENT)
 690                                 return r;
 691                 }
 692                 if (j->current_location.realtime_set)
 693                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 694
 695                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 696
 697         } else if (m->type == MATCH_OR_TERM) {
 698                 uint64_t np = 0;
 699                 Object *n;
 700                 Match *i;
 701
 702                 /* Find the earliest match */
 703
 704                 LIST_FOREACH(matches, i, m->matches) {
 705                         uint64_t cp;
 706
 707                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 708                         if (r < 0)
 709                                 return r;
 710                         else if (r > 0) {
 711                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 712                                         np = cp;
 713                         }
 714                 }
 715
 716                 if (np == 0)
 717                         return 0;
 718
 719                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 720                 if (r < 0)
 721                         return r;
 722
 723                 if (ret)
 724                         *ret = n;
 725                 if (offset)
 726                         *offset = np;
 727
 728                 return 1;
 729
 730         } else {
 731                 Match *i;
 732                 uint64_t np = 0;
 733
 734                 assert(m->type == MATCH_AND_TERM);
 735
 736                 /* First jump to the last match, and then find the
 737                  * next one where all matches match */
 738
 739                 if (!m->matches)
 740                         return 0;
 741
 742                 LIST_FOREACH(matches, i, m->matches) {
 743                         uint64_t cp;
 744
 745                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 746                         if (r <= 0)
 747                                 return r;
 748
 749                         if (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))
 750                                 np = cp;
 751                 }
 752
 753                 return next_for_match(j, m, f, np, direction, ret, offset);
 754         }
 755 }
 756
 757 static int find_location_with_matches(
 758                 sd_journal *j,
 759                 JournalFile *f,
 760                 direction_t direction,
 761                 Object **ret,
 762                 uint64_t *offset) {
 763
 764         int r;
 765
 766         assert(j);
 767         assert(f);
 768         assert(ret);
 769         assert(offset);
 770
 771         if (!j->level0) {
 772                 /* No matches is simple */
 773
 774                 if (j->current_location.type == LOCATION_HEAD)
 775                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 776                 if (j->current_location.type == LOCATION_TAIL)
 777                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 778                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 779                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 780                 if (j->current_location.monotonic_set) {
 781                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 782                         if (r != -ENOENT)
 783                                 return r;
 784                 }
 785                 if (j->current_location.realtime_set)
 786                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 787
 788                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 789         } else
 790                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 791 }
 792
 793 static int next_with_matches(
 794                 sd_journal *j,
 795                 JournalFile *f,
 796                 direction_t direction,
 797                 Object **ret,
 798                 uint64_t *offset) {
 799
 800         Object *c;
 801         uint64_t cp;
 802
 803         assert(j);
 804         assert(f);
 805         assert(ret);
 806         assert(offset);
 807
 808         c = *ret;
 809         cp = *offset;
 810
 811         /* No matches is easy. We simple advance the file
 812          * pointer by one. */
 813         if (!j->level0)
 814                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 815
 816         /* If we have a match then we look for the next matching entry
 817          * with an offset at least one step larger */
 818         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 819 }
 820
 821 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 822         Object *c;
 823         uint64_t cp;
 824         int r;
 825
 826         assert(j);
 827         assert(f);
 828
 829         if (f->last_direction == direction && f->current_offset > 0) {
 830                 cp = f->current_offset;
 831
 832                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 833                 if (r < 0)
 834                         return r;
 835
 836                 r = next_with_matches(j, f, direction, &c, &cp);
 837                 if (r <= 0)
 838                         return r;
 839         } else {
 840                 r = find_location_with_matches(j, f, direction, &c, &cp);
 841                 if (r <= 0)
 842                         return r;
 843         }
 844
 845         /* OK, we found the spot, now let's advance until an entry
 846          * that is actually different from what we were previously
 847          * looking at. This is necessary to handle entries which exist
 848          * in two (or more) journal files, and which shall all be
 849          * suppressed but one. */
 850
 851         for (;;) {
 852                 bool found;
 853
 854                 if (j->current_location.type == LOCATION_DISCRETE) {
 855                         int k;
 856
 857                         k = compare_with_location(f, c, &j->current_location);
 858                         if (direction == DIRECTION_DOWN)
 859                                 found = k > 0;
 860                         else
 861                                 found = k < 0;
 862                 } else
 863                         found = true;
 864
 865                 if (found) {
 866                         if (ret)
 867                                 *ret = c;
 868                         if (offset)
 869                                 *offset = cp;
 870                         return 1;
 871                 }
 872
 873                 r = next_with_matches(j, f, direction, &c, &cp);
 874                 if (r <= 0)
 875                         return r;
 876         }
 877 }
 878
 879 static int real_journal_next(sd_journal *j, direction_t direction) {
 880         JournalFile *f, *new_file = NULL;
 881         uint64_t new_offset = 0;
 882         Object *o;
 883         uint64_t p;
 884         Iterator i;
 885         int r;
 886
 887         if (!j)
 888                 return -EINVAL;
 889         if (journal_pid_changed(j))
 890                 return -ECHILD;
 891
 892         HASHMAP_FOREACH(f, j->files, i) {
 893                 bool found;
 894
 895                 r = next_beyond_location(j, f, direction, &o, &p);
 896                 if (r < 0) {
 897                         log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
 898                         continue;
 899                 } else if (r == 0)
 900                         continue;
 901
 902                 if (!new_file)
 903                         found = true;
 904                 else {
 905                         int k;
 906
 907                         k = compare_entry_order(f, o, new_file, new_offset);
 908
 909                         found = direction == DIRECTION_DOWN ? k < 0 : k > 0;
 910                 }
 911
 912                 if (found) {
 913                         new_file = f;
 914                         new_offset = p;
 915                 }
 916         }
 917
 918         if (!new_file)
 919                 return 0;
 920
 921         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 922         if (r < 0)
 923                 return r;
 924
 925         set_location(j, LOCATION_DISCRETE, new_file, o, direction, new_offset);
 926
 927         return 1;
 928 }
 929
 930 _public_ int sd_journal_next(sd_journal *j) {
 931         return real_journal_next(j, DIRECTION_DOWN);
 932 }
 933
 934 _public_ int sd_journal_previous(sd_journal *j) {
 935         return real_journal_next(j, DIRECTION_UP);
 936 }
 937
 938 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 939         int c = 0, r;
 940
 941         if (!j)
 942                 return -EINVAL;
 943         if (journal_pid_changed(j))
 944                 return -ECHILD;
 945
 946         if (skip == 0) {
 947                 /* If this is not a discrete skip, then at least
 948                  * resolve the current location */
 949                 if (j->current_location.type != LOCATION_DISCRETE)
 950                         return real_journal_next(j, direction);
 951
 952                 return 0;
 953         }
 954
 955         do {
 956                 r = real_journal_next(j, direction);
 957                 if (r < 0)
 958                         return r;
 959
 960                 if (r == 0)
 961                         return c;
 962
 963                 skip--;
 964                 c++;
 965         } while (skip > 0);
 966
 967         return c;
 968 }
 969
 970 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 971         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 972 }
 973
 974 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 975         return real_journal_next_skip(j, DIRECTION_UP, skip);
 976 }
 977
 978 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 979         Object *o;
 980         int r;
 981         char bid[33], sid[33];
 982
 983         if (!j)
 984                 return -EINVAL;
 985         if (journal_pid_changed(j))
 986                 return -ECHILD;
 987         if (!cursor)
 988                 return -EINVAL;
 989
 990         if (!j->current_file || j->current_file->current_offset <= 0)
 991                 return -EADDRNOTAVAIL;
 992
 993         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 994         if (r < 0)
 995                 return r;
 996
 997         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 998         sd_id128_to_string(o->entry.boot_id, bid);
 999
1000         if (asprintf(cursor,
1001                      "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
1002                      sid, le64toh(o->entry.seqnum),
1003                      bid, le64toh(o->entry.monotonic),
1004                      le64toh(o->entry.realtime),
1005                      le64toh(o->entry.xor_hash)) < 0)
1006                 return -ENOMEM;
1007
1008         return 0;
1009 }
1010
1011 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
1012         char *w, *state;
1013         size_t l;
1014         unsigned long long seqnum, monotonic, realtime, xor_hash;
1015         bool
1016                 seqnum_id_set = false,
1017                 seqnum_set = false,
1018                 boot_id_set = false,
1019                 monotonic_set = false,
1020                 realtime_set = false,
1021                 xor_hash_set = false;
1022         sd_id128_t seqnum_id, boot_id;
1023
1024         if (!j)
1025                 return -EINVAL;
1026         if (journal_pid_changed(j))
1027                 return -ECHILD;
1028         if (isempty(cursor))
1029                 return -EINVAL;
1030
1031         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1032                 char *item;
1033                 int k = 0;
1034
1035                 if (l < 2 || w[1] != '=')
1036                         return -EINVAL;
1037
1038                 item = strndup(w, l);
1039                 if (!item)
1040                         return -ENOMEM;
1041
1042                 switch (w[0]) {
1043
1044                 case 's':
1045                         seqnum_id_set = true;
1046                         k = sd_id128_from_string(item+2, &seqnum_id);
1047                         break;
1048
1049                 case 'i':
1050                         seqnum_set = true;
1051                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1052                                 k = -EINVAL;
1053                         break;
1054
1055                 case 'b':
1056                         boot_id_set = true;
1057                         k = sd_id128_from_string(item+2, &boot_id);
1058                         break;
1059
1060                 case 'm':
1061                         monotonic_set = true;
1062                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1063                                 k = -EINVAL;
1064                         break;
1065
1066                 case 't':
1067                         realtime_set = true;
1068                         if (sscanf(item+2, "%llx", &realtime) != 1)
1069                                 k = -EINVAL;
1070                         break;
1071
1072                 case 'x':
1073                         xor_hash_set = true;
1074                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1075                                 k = -EINVAL;
1076                         break;
1077                 }
1078
1079                 free(item);
1080
1081                 if (k < 0)
1082                         return k;
1083         }
1084
1085         if ((!seqnum_set || !seqnum_id_set) &&
1086             (!monotonic_set || !boot_id_set) &&
1087             !realtime_set)
1088                 return -EINVAL;
1089
1090         reset_location(j);
1091
1092         j->current_location.type = LOCATION_SEEK;
1093
1094         if (realtime_set) {
1095                 j->current_location.realtime = (uint64_t) realtime;
1096                 j->current_location.realtime_set = true;
1097         }
1098
1099         if (seqnum_set && seqnum_id_set) {
1100                 j->current_location.seqnum = (uint64_t) seqnum;
1101                 j->current_location.seqnum_id = seqnum_id;
1102                 j->current_location.seqnum_set = true;
1103         }
1104
1105         if (monotonic_set && boot_id_set) {
1106                 j->current_location.monotonic = (uint64_t) monotonic;
1107                 j->current_location.boot_id = boot_id;
1108                 j->current_location.monotonic_set = true;
1109         }
1110
1111         if (xor_hash_set) {
1112                 j->current_location.xor_hash = (uint64_t) xor_hash;
1113                 j->current_location.xor_hash_set = true;
1114         }
1115
1116         return 0;
1117 }
1118
1119 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1120         int r;
1121         char *w, *state;
1122         size_t l;
1123         Object *o;
1124
1125         if (!j)
1126                 return -EINVAL;
1127         if (journal_pid_changed(j))
1128                 return -ECHILD;
1129         if (isempty(cursor))
1130                 return -EINVAL;
1131
1132         if (!j->current_file || j->current_file->current_offset <= 0)
1133                 return -EADDRNOTAVAIL;
1134
1135         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1136         if (r < 0)
1137                 return r;
1138
1139         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1140                 _cleanup_free_ char *item = NULL;
1141                 sd_id128_t id;
1142                 unsigned long long ll;
1143                 int k = 0;
1144
1145                 if (l < 2 || w[1] != '=')
1146                         return -EINVAL;
1147
1148                 item = strndup(w, l);
1149                 if (!item)
1150                         return -ENOMEM;
1151
1152                 switch (w[0]) {
1153
1154                 case 's':
1155                         k = sd_id128_from_string(item+2, &id);
1156                         if (k < 0)
1157                                 return k;
1158                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1159                                 return 0;
1160                         break;
1161
1162                 case 'i':
1163                         if (sscanf(item+2, "%llx", &ll) != 1)
1164                                 return -EINVAL;
1165                         if (ll != le64toh(o->entry.seqnum))
1166                                 return 0;
1167                         break;
1168
1169                 case 'b':
1170                         k = sd_id128_from_string(item+2, &id);
1171                         if (k < 0)
1172                                 return k;
1173                         if (!sd_id128_equal(id, o->entry.boot_id))
1174                                 return 0;
1175                         break;
1176
1177                 case 'm':
1178                         if (sscanf(item+2, "%llx", &ll) != 1)
1179                                 return -EINVAL;
1180                         if (ll != le64toh(o->entry.monotonic))
1181                                 return 0;
1182                         break;
1183
1184                 case 't':
1185                         if (sscanf(item+2, "%llx", &ll) != 1)
1186                                 return -EINVAL;
1187                         if (ll != le64toh(o->entry.realtime))
1188                                 return 0;
1189                         break;
1190
1191                 case 'x':
1192                         if (sscanf(item+2, "%llx", &ll) != 1)
1193                                 return -EINVAL;
1194                         if (ll != le64toh(o->entry.xor_hash))
1195                                 return 0;
1196                         break;
1197                 }
1198         }
1199
1200         return 1;
1201 }
1202
1203
1204 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1205         if (!j)
1206                 return -EINVAL;
1207         if (journal_pid_changed(j))
1208                 return -ECHILD;
1209
1210         reset_location(j);
1211         j->current_location.type = LOCATION_SEEK;
1212         j->current_location.boot_id = boot_id;
1213         j->current_location.monotonic = usec;
1214         j->current_location.monotonic_set = true;
1215
1216         return 0;
1217 }
1218
1219 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1220         if (!j)
1221                 return -EINVAL;
1222         if (journal_pid_changed(j))
1223                 return -ECHILD;
1224
1225         reset_location(j);
1226         j->current_location.type = LOCATION_SEEK;
1227         j->current_location.realtime = usec;
1228         j->current_location.realtime_set = true;
1229
1230         return 0;
1231 }
1232
1233 _public_ int sd_journal_seek_head(sd_journal *j) {
1234         if (!j)
1235                 return -EINVAL;
1236         if (journal_pid_changed(j))
1237                 return -ECHILD;
1238
1239         reset_location(j);
1240         j->current_location.type = LOCATION_HEAD;
1241
1242         return 0;
1243 }
1244
1245 _public_ int sd_journal_seek_tail(sd_journal *j) {
1246         if (!j)
1247                 return -EINVAL;
1248         if (journal_pid_changed(j))
1249                 return -ECHILD;
1250
1251         reset_location(j);
1252         j->current_location.type = LOCATION_TAIL;
1253
1254         return 0;
1255 }
1256
1257 static void check_network(sd_journal *j, int fd) {
1258         struct statfs sfs;
1259
1260         assert(j);
1261
1262         if (j->on_network)
1263                 return;
1264
1265         if (fstatfs(fd, &sfs) < 0)
1266                 return;
1267
1268         j->on_network =
1269                 F_TYPE_EQUAL(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1270                 F_TYPE_EQUAL(sfs.f_type, CODA_SUPER_MAGIC) ||
1271                 F_TYPE_EQUAL(sfs.f_type, NCP_SUPER_MAGIC) ||
1272                 F_TYPE_EQUAL(sfs.f_type, NFS_SUPER_MAGIC) ||
1273                 F_TYPE_EQUAL(sfs.f_type, SMB_SUPER_MAGIC);
1274 }
1275
1276 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1277         const char *full, *tilded, *atted;
1278
1279         full = strappend(prefix, ".journal");
1280         tilded = strappenda(full, "~");
1281         atted = strappenda(prefix, "@");
1282
1283         return streq(filename, full) ||
1284                streq(filename, tilded) ||
1285                startswith(filename, atted);
1286 }
1287
1288 static bool file_type_wanted(int flags, const char *filename) {
1289         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1290                 return false;
1291
1292         /* no flags set → every type is OK */
1293         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1294                 return true;
1295
1296         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1297                 return true;
1298
1299         if (flags & SD_JOURNAL_CURRENT_USER) {
1300                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1301
1302                 assert_se(snprintf(prefix, sizeof(prefix), "user-%lu", (unsigned long) getuid())
1303                           < (int) sizeof(prefix));
1304
1305                 if (file_has_type_prefix(prefix, filename))
1306                         return true;
1307         }
1308
1309         return false;
1310 }
1311
1312 static int add_any_file(sd_journal *j, const char *path) {
1313         JournalFile *f;
1314         int r;
1315
1316         assert(j);
1317         assert(path);
1318
1319         if (hashmap_get(j->files, path))
1320                 return 0;
1321
1322         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1323                 log_warning("Too many open journal files, not adding %s.", path);
1324                 return set_put_error(j, -ETOOMANYREFS);
1325         }
1326
1327         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1328         if (r < 0)
1329                 return r;
1330
1331         /* journal_file_dump(f); */
1332
1333         r = hashmap_put(j->files, f->path, f);
1334         if (r < 0) {
1335                 journal_file_close(f);
1336                 return r;
1337         }
1338
1339         log_debug("File %s added.", f->path);
1340
1341         check_network(j, f->fd);
1342
1343         j->current_invalidate_counter ++;
1344
1345         return 0;
1346 }
1347
1348 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1349         _cleanup_free_ char *path = NULL;
1350         int r;
1351
1352         assert(j);
1353         assert(prefix);
1354         assert(filename);
1355
1356         if (j->no_new_files ||
1357             !file_type_wanted(j->flags, filename))
1358                 return 0;
1359
1360         path = strjoin(prefix, "/", filename, NULL);
1361         if (!path)
1362                 return -ENOMEM;
1363
1364         r = add_any_file(j, path);
1365         if (r == -ENOENT)
1366                 return 0;
1367         return 0;
1368 }
1369
1370 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1371         char *path;
1372         JournalFile *f;
1373
1374         assert(j);
1375         assert(prefix);
1376         assert(filename);
1377
1378         path = strjoin(prefix, "/", filename, NULL);
1379         if (!path)
1380                 return -ENOMEM;
1381
1382         f = hashmap_get(j->files, path);
1383         free(path);
1384         if (!f)
1385                 return 0;
1386
1387         hashmap_remove(j->files, f->path);
1388
1389         log_debug("File %s removed.", f->path);
1390
1391         if (j->current_file == f) {
1392                 j->current_file = NULL;
1393                 j->current_field = 0;
1394         }
1395
1396         if (j->unique_file == f) {
1397                 j->unique_file = NULL;
1398                 j->unique_offset = 0;
1399         }
1400
1401         journal_file_close(f);
1402
1403         j->current_invalidate_counter ++;
1404
1405         return 0;
1406 }
1407
1408 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1409         _cleanup_free_ char *path = NULL;
1410         int r;
1411         _cleanup_closedir_ DIR *d = NULL;
1412         sd_id128_t id, mid;
1413         Directory *m;
1414
1415         assert(j);
1416         assert(prefix);
1417         assert(dirname);
1418
1419         log_debug("Considering %s/%s.", prefix, dirname);
1420
1421         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1422             (sd_id128_from_string(dirname, &id) < 0 ||
1423              sd_id128_get_machine(&mid) < 0 ||
1424              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1425             return 0;
1426
1427         path = strjoin(prefix, "/", dirname, NULL);
1428         if (!path)
1429                 return -ENOMEM;
1430
1431         d = opendir(path);
1432         if (!d) {
1433                 log_debug("Failed to open %s: %m", path);
1434                 if (errno == ENOENT)
1435                         return 0;
1436                 return -errno;
1437         }
1438
1439         m = hashmap_get(j->directories_by_path, path);
1440         if (!m) {
1441                 m = new0(Directory, 1);
1442                 if (!m)
1443                         return -ENOMEM;
1444
1445                 m->is_root = false;
1446                 m->path = path;
1447
1448                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1449                         free(m);
1450                         return -ENOMEM;
1451                 }
1452
1453                 path = NULL; /* avoid freeing in cleanup */
1454                 j->current_invalidate_counter ++;
1455
1456                 log_debug("Directory %s added.", m->path);
1457
1458         } else if (m->is_root)
1459                 return 0;
1460
1461         if (m->wd <= 0 && j->inotify_fd >= 0) {
1462
1463                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1464                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1465                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1466                                           IN_ONLYDIR);
1467
1468                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1469                         inotify_rm_watch(j->inotify_fd, m->wd);
1470         }
1471
1472         for (;;) {
1473                 struct dirent *de;
1474                 union dirent_storage buf;
1475
1476                 r = readdir_r(d, &buf.de, &de);
1477                 if (r != 0 || !de)
1478                         break;
1479
1480                 if (dirent_is_file_with_suffix(de, ".journal") ||
1481                     dirent_is_file_with_suffix(de, ".journal~")) {
1482                         r = add_file(j, m->path, de->d_name);
1483                         if (r < 0) {
1484                                 log_debug("Failed to add file %s/%s: %s",
1485                                           m->path, de->d_name, strerror(-r));
1486                                 r = set_put_error(j, r);
1487                                 if (r < 0)
1488                                         return r;
1489                         }
1490                 }
1491         }
1492
1493         check_network(j, dirfd(d));
1494
1495         return 0;
1496 }
1497
1498 static int add_root_directory(sd_journal *j, const char *p) {
1499         _cleanup_closedir_ DIR *d = NULL;
1500         Directory *m;
1501         int r;
1502
1503         assert(j);
1504         assert(p);
1505
1506         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1507             !path_startswith(p, "/run"))
1508                 return -EINVAL;
1509
1510         d = opendir(p);
1511         if (!d)
1512                 return -errno;
1513
1514         m = hashmap_get(j->directories_by_path, p);
1515         if (!m) {
1516                 m = new0(Directory, 1);
1517                 if (!m)
1518                         return -ENOMEM;
1519
1520                 m->is_root = true;
1521                 m->path = strdup(p);
1522                 if (!m->path) {
1523                         free(m);
1524                         return -ENOMEM;
1525                 }
1526
1527                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1528                         free(m->path);
1529                         free(m);
1530                         return -ENOMEM;
1531                 }
1532
1533                 j->current_invalidate_counter ++;
1534
1535                 log_debug("Root directory %s added.", m->path);
1536
1537         } else if (!m->is_root)
1538                 return 0;
1539
1540         if (m->wd <= 0 && j->inotify_fd >= 0) {
1541
1542                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1543                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1544                                           IN_ONLYDIR);
1545
1546                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1547                         inotify_rm_watch(j->inotify_fd, m->wd);
1548         }
1549
1550         if (j->no_new_files)
1551                 return 0;
1552
1553         for (;;) {
1554                 struct dirent *de;
1555                 union dirent_storage buf;
1556                 sd_id128_t id;
1557
1558                 r = readdir_r(d, &buf.de, &de);
1559                 if (r != 0 || !de)
1560                         break;
1561
1562                 if (dirent_is_file_with_suffix(de, ".journal") ||
1563                     dirent_is_file_with_suffix(de, ".journal~")) {
1564                         r = add_file(j, m->path, de->d_name);
1565                         if (r < 0) {
1566                                 log_debug("Failed to add file %s/%s: %s",
1567                                           m->path, de->d_name, strerror(-r));
1568                                 r = set_put_error(j, r);
1569                                 if (r < 0)
1570                                         return r;
1571                         }
1572                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1573                            sd_id128_from_string(de->d_name, &id) >= 0) {
1574
1575                         r = add_directory(j, m->path, de->d_name);
1576                         if (r < 0)
1577                                 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1578                 }
1579         }
1580
1581         check_network(j, dirfd(d));
1582
1583         return 0;
1584 }
1585
1586 static int remove_directory(sd_journal *j, Directory *d) {
1587         assert(j);
1588
1589         if (d->wd > 0) {
1590                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1591
1592                 if (j->inotify_fd >= 0)
1593                         inotify_rm_watch(j->inotify_fd, d->wd);
1594         }
1595
1596         hashmap_remove(j->directories_by_path, d->path);
1597
1598         if (d->is_root)
1599                 log_debug("Root directory %s removed.", d->path);
1600         else
1601                 log_debug("Directory %s removed.", d->path);
1602
1603         free(d->path);
1604         free(d);
1605
1606         return 0;
1607 }
1608
1609 static int add_search_paths(sd_journal *j) {
1610         int r;
1611         const char search_paths[] =
1612                 "/run/log/journal\0"
1613                 "/var/log/journal\0";
1614         const char *p;
1615
1616         assert(j);
1617
1618         /* We ignore most errors here, since the idea is to only open
1619          * what's actually accessible, and ignore the rest. */
1620
1621         NULSTR_FOREACH(p, search_paths) {
1622                 r = add_root_directory(j, p);
1623                 if (r < 0 && r != -ENOENT) {
1624                         r = set_put_error(j, r);
1625                         if (r < 0)
1626                                 return r;
1627                 }
1628         }
1629
1630         return 0;
1631 }
1632
1633 static int add_current_paths(sd_journal *j) {
1634         Iterator i;
1635         JournalFile *f;
1636
1637         assert(j);
1638         assert(j->no_new_files);
1639
1640         /* Simply adds all directories for files we have open as
1641          * "root" directories. We don't expect errors here, so we
1642          * treat them as fatal. */
1643
1644         HASHMAP_FOREACH(f, j->files, i) {
1645                 int r;
1646                 _cleanup_free_ char *dir;
1647
1648                 dir = dirname_malloc(f->path);
1649                 if (!dir)
1650                         return -ENOMEM;
1651
1652                 r = add_root_directory(j, dir);
1653                 if (r < 0) {
1654                         set_put_error(j, r);
1655                         return r;
1656                 }
1657         }
1658
1659         return 0;
1660 }
1661
1662
1663 static int allocate_inotify(sd_journal *j) {
1664         assert(j);
1665
1666         if (j->inotify_fd < 0) {
1667                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1668                 if (j->inotify_fd < 0)
1669                         return -errno;
1670         }
1671
1672         if (!j->directories_by_wd) {
1673                 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1674                 if (!j->directories_by_wd)
1675                         return -ENOMEM;
1676         }
1677
1678         return 0;
1679 }
1680
1681 static sd_journal *journal_new(int flags, const char *path) {
1682         sd_journal *j;
1683
1684         j = new0(sd_journal, 1);
1685         if (!j)
1686                 return NULL;
1687
1688         j->original_pid = getpid();
1689         j->inotify_fd = -1;
1690         j->flags = flags;
1691         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1692
1693         if (path) {
1694                 j->path = strdup(path);
1695                 if (!j->path)
1696                         goto fail;
1697         }
1698
1699         j->files = hashmap_new(string_hash_func, string_compare_func);
1700         j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1701         j->mmap = mmap_cache_new();
1702         if (!j->files || !j->directories_by_path || !j->mmap)
1703                 goto fail;
1704
1705         return j;
1706
1707 fail:
1708         sd_journal_close(j);
1709         return NULL;
1710 }
1711
1712 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1713         sd_journal *j;
1714         int r;
1715
1716         if (!ret)
1717                 return -EINVAL;
1718
1719         if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1720                       SD_JOURNAL_RUNTIME_ONLY|
1721                       SD_JOURNAL_SYSTEM|
1722                       SD_JOURNAL_CURRENT_USER))
1723                 return -EINVAL;
1724
1725         j = journal_new(flags, NULL);
1726         if (!j)
1727                 return -ENOMEM;
1728
1729         r = add_search_paths(j);
1730         if (r < 0)
1731                 goto fail;
1732
1733         *ret = j;
1734         return 0;
1735
1736 fail:
1737         sd_journal_close(j);
1738
1739         return r;
1740 }
1741
1742 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1743         sd_journal *j;
1744         int r;
1745
1746         if (!ret)
1747                 return -EINVAL;
1748
1749         if (!path)
1750                 return -EINVAL;
1751
1752         if (flags != 0)
1753                 return -EINVAL;
1754
1755         j = journal_new(flags, path);
1756         if (!j)
1757                 return -ENOMEM;
1758
1759         r = add_root_directory(j, path);
1760         if (r < 0) {
1761                 set_put_error(j, r);
1762                 goto fail;
1763         }
1764
1765         *ret = j;
1766         return 0;
1767
1768 fail:
1769         sd_journal_close(j);
1770
1771         return r;
1772 }
1773
1774 _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int flags) {
1775         sd_journal *j;
1776         const char **path;
1777         int r;
1778
1779         if (!ret)
1780                 return -EINVAL;
1781
1782         if (flags != 0)
1783                 return -EINVAL;
1784
1785         j = journal_new(flags, NULL);
1786         if (!j)
1787                 return -ENOMEM;
1788
1789         STRV_FOREACH(path, paths) {
1790                 r = add_any_file(j, *path);
1791                 if (r < 0) {
1792                         log_error("Failed to open %s: %s", *path, strerror(-r));
1793                         goto fail;
1794                 }
1795         }
1796
1797         j->no_new_files = true;
1798
1799         *ret = j;
1800         return 0;
1801
1802 fail:
1803         sd_journal_close(j);
1804
1805         return r;
1806 }
1807
1808 _public_ void sd_journal_close(sd_journal *j) {
1809         Directory *d;
1810         JournalFile *f;
1811
1812         if (!j)
1813                 return;
1814
1815         sd_journal_flush_matches(j);
1816
1817         while ((f = hashmap_steal_first(j->files)))
1818                 journal_file_close(f);
1819
1820         hashmap_free(j->files);
1821
1822         while ((d = hashmap_first(j->directories_by_path)))
1823                 remove_directory(j, d);
1824
1825         while ((d = hashmap_first(j->directories_by_wd)))
1826                 remove_directory(j, d);
1827
1828         hashmap_free(j->directories_by_path);
1829         hashmap_free(j->directories_by_wd);
1830
1831         if (j->inotify_fd >= 0)
1832                 close_nointr_nofail(j->inotify_fd);
1833
1834         if (j->mmap)
1835                 mmap_cache_unref(j->mmap);
1836
1837         free(j->path);
1838         free(j->unique_field);
1839         set_free(j->errors);
1840         free(j);
1841 }
1842
1843 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1844         Object *o;
1845         JournalFile *f;
1846         int r;
1847
1848         if (!j)
1849                 return -EINVAL;
1850         if (journal_pid_changed(j))
1851                 return -ECHILD;
1852         if (!ret)
1853                 return -EINVAL;
1854
1855         f = j->current_file;
1856         if (!f)
1857                 return -EADDRNOTAVAIL;
1858
1859         if (f->current_offset <= 0)
1860                 return -EADDRNOTAVAIL;
1861
1862         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1863         if (r < 0)
1864                 return r;
1865
1866         *ret = le64toh(o->entry.realtime);
1867         return 0;
1868 }
1869
1870 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1871         Object *o;
1872         JournalFile *f;
1873         int r;
1874         sd_id128_t id;
1875
1876         if (!j)
1877                 return -EINVAL;
1878         if (journal_pid_changed(j))
1879                 return -ECHILD;
1880
1881         f = j->current_file;
1882         if (!f)
1883                 return -EADDRNOTAVAIL;
1884
1885         if (f->current_offset <= 0)
1886                 return -EADDRNOTAVAIL;
1887
1888         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1889         if (r < 0)
1890                 return r;
1891
1892         if (ret_boot_id)
1893                 *ret_boot_id = o->entry.boot_id;
1894         else {
1895                 r = sd_id128_get_boot(&id);
1896                 if (r < 0)
1897                         return r;
1898
1899                 if (!sd_id128_equal(id, o->entry.boot_id))
1900                         return -ESTALE;
1901         }
1902
1903         if (ret)
1904                 *ret = le64toh(o->entry.monotonic);
1905
1906         return 0;
1907 }
1908
1909 static bool field_is_valid(const char *field) {
1910         const char *p;
1911
1912         assert(field);
1913
1914         if (isempty(field))
1915                 return false;
1916
1917         if (startswith(field, "__"))
1918                 return false;
1919
1920         for (p = field; *p; p++) {
1921
1922                 if (*p == '_')
1923                         continue;
1924
1925                 if (*p >= 'A' && *p <= 'Z')
1926                         continue;
1927
1928                 if (*p >= '0' && *p <= '9')
1929                         continue;
1930
1931                 return false;
1932         }
1933
1934         return true;
1935 }
1936
1937 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1938         JournalFile *f;
1939         uint64_t i, n;
1940         size_t field_length;
1941         int r;
1942         Object *o;
1943
1944         if (!j)
1945                 return -EINVAL;
1946         if (journal_pid_changed(j))
1947                 return -ECHILD;
1948         if (!field)
1949                 return -EINVAL;
1950         if (!data)
1951                 return -EINVAL;
1952         if (!size)
1953                 return -EINVAL;
1954
1955         if (!field_is_valid(field))
1956                 return -EINVAL;
1957
1958         f = j->current_file;
1959         if (!f)
1960                 return -EADDRNOTAVAIL;
1961
1962         if (f->current_offset <= 0)
1963                 return -EADDRNOTAVAIL;
1964
1965         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1966         if (r < 0)
1967                 return r;
1968
1969         field_length = strlen(field);
1970
1971         n = journal_file_entry_n_items(o);
1972         for (i = 0; i < n; i++) {
1973                 uint64_t p, l;
1974                 le64_t le_hash;
1975                 size_t t;
1976
1977                 p = le64toh(o->entry.items[i].object_offset);
1978                 le_hash = o->entry.items[i].hash;
1979                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1980                 if (r < 0)
1981                         return r;
1982
1983                 if (le_hash != o->data.hash)
1984                         return -EBADMSG;
1985
1986                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1987
1988                 if (o->object.flags & OBJECT_COMPRESSED) {
1989
1990 #ifdef HAVE_XZ
1991                         if (uncompress_startswith(o->data.payload, l,
1992                                                   &f->compress_buffer, &f->compress_buffer_size,
1993                                                   field, field_length, '=')) {
1994
1995                                 uint64_t rsize;
1996
1997                                 if (!uncompress_blob(o->data.payload, l,
1998                                                      &f->compress_buffer, &f->compress_buffer_size, &rsize,
1999                                                      j->data_threshold))
2000                                         return -EBADMSG;
2001
2002                                 *data = f->compress_buffer;
2003                                 *size = (size_t) rsize;
2004
2005                                 return 0;
2006                         }
2007 #else
2008                         return -EPROTONOSUPPORT;
2009 #endif
2010
2011                 } else if (l >= field_length+1 &&
2012                            memcmp(o->data.payload, field, field_length) == 0 &&
2013                            o->data.payload[field_length] == '=') {
2014
2015                         t = (size_t) l;
2016
2017                         if ((uint64_t) t != l)
2018                                 return -E2BIG;
2019
2020                         *data = o->data.payload;
2021                         *size = t;
2022
2023                         return 0;
2024                 }
2025
2026                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2027                 if (r < 0)
2028                         return r;
2029         }
2030
2031         return -ENOENT;
2032 }
2033
2034 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
2035         size_t t;
2036         uint64_t l;
2037
2038         l = le64toh(o->object.size) - offsetof(Object, data.payload);
2039         t = (size_t) l;
2040
2041         /* We can't read objects larger than 4G on a 32bit machine */
2042         if ((uint64_t) t != l)
2043                 return -E2BIG;
2044
2045         if (o->object.flags & OBJECT_COMPRESSED) {
2046 #ifdef HAVE_XZ
2047                 uint64_t rsize;
2048
2049                 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
2050                         return -EBADMSG;
2051
2052                 *data = f->compress_buffer;
2053                 *size = (size_t) rsize;
2054 #else
2055                 return -EPROTONOSUPPORT;
2056 #endif
2057         } else {
2058                 *data = o->data.payload;
2059                 *size = t;
2060         }
2061
2062         return 0;
2063 }
2064
2065 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
2066         JournalFile *f;
2067         uint64_t p, n;
2068         le64_t le_hash;
2069         int r;
2070         Object *o;
2071
2072         if (!j)
2073                 return -EINVAL;
2074         if (journal_pid_changed(j))
2075                 return -ECHILD;
2076         if (!data)
2077                 return -EINVAL;
2078         if (!size)
2079                 return -EINVAL;
2080
2081         f = j->current_file;
2082         if (!f)
2083                 return -EADDRNOTAVAIL;
2084
2085         if (f->current_offset <= 0)
2086                 return -EADDRNOTAVAIL;
2087
2088         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2089         if (r < 0)
2090                 return r;
2091
2092         n = journal_file_entry_n_items(o);
2093         if (j->current_field >= n)
2094                 return 0;
2095
2096         p = le64toh(o->entry.items[j->current_field].object_offset);
2097         le_hash = o->entry.items[j->current_field].hash;
2098         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2099         if (r < 0)
2100                 return r;
2101
2102         if (le_hash != o->data.hash)
2103                 return -EBADMSG;
2104
2105         r = return_data(j, f, o, data, size);
2106         if (r < 0)
2107                 return r;
2108
2109         j->current_field ++;
2110
2111         return 1;
2112 }
2113
2114 _public_ void sd_journal_restart_data(sd_journal *j) {
2115         if (!j)
2116                 return;
2117
2118         j->current_field = 0;
2119 }
2120
2121 _public_ int sd_journal_get_fd(sd_journal *j) {
2122         int r;
2123
2124         if (!j)
2125                 return -EINVAL;
2126         if (journal_pid_changed(j))
2127                 return -ECHILD;
2128
2129         if (j->inotify_fd >= 0)
2130                 return j->inotify_fd;
2131
2132         r = allocate_inotify(j);
2133         if (r < 0)
2134                 return r;
2135
2136         /* Iterate through all dirs again, to add them to the
2137          * inotify */
2138         if (j->no_new_files)
2139                 r = add_current_paths(j);
2140         else if (j->path)
2141                 r = add_root_directory(j, j->path);
2142         else
2143                 r = add_search_paths(j);
2144         if (r < 0)
2145                 return r;
2146
2147         return j->inotify_fd;
2148 }
2149
2150 _public_ int sd_journal_get_events(sd_journal *j) {
2151         int fd;
2152
2153         if (!j)
2154                 return -EINVAL;
2155         if (journal_pid_changed(j))
2156                 return -ECHILD;
2157
2158         fd = sd_journal_get_fd(j);
2159         if (fd < 0)
2160                 return fd;
2161
2162         return POLLIN;
2163 }
2164
2165 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2166         int fd;
2167
2168         if (!j)
2169                 return -EINVAL;
2170         if (journal_pid_changed(j))
2171                 return -ECHILD;
2172         if (!timeout_usec)
2173                 return -EINVAL;
2174
2175         fd = sd_journal_get_fd(j);
2176         if (fd < 0)
2177                 return fd;
2178
2179         if (!j->on_network) {
2180                 *timeout_usec = (uint64_t) -1;
2181                 return 0;
2182         }
2183
2184         /* If we are on the network we need to regularly check for
2185          * changes manually */
2186
2187         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2188         return 1;
2189 }
2190
2191 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2192         Directory *d;
2193         int r;
2194
2195         assert(j);
2196         assert(e);
2197
2198         /* Is this a subdirectory we watch? */
2199         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2200         if (d) {
2201                 sd_id128_t id;
2202
2203                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2204                     (endswith(e->name, ".journal") ||
2205                      endswith(e->name, ".journal~"))) {
2206
2207                         /* Event for a journal file */
2208
2209                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2210                                 r = add_file(j, d->path, e->name);
2211                                 if (r < 0) {
2212                                         log_debug("Failed to add file %s/%s: %s",
2213                                                   d->path, e->name, strerror(-r));
2214                                         set_put_error(j, r);
2215                                 }
2216
2217                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2218
2219                                 r = remove_file(j, d->path, e->name);
2220                                 if (r < 0)
2221                                         log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2222                         }
2223
2224                 } else if (!d->is_root && e->len == 0) {
2225
2226                         /* Event for a subdirectory */
2227
2228                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2229                                 r = remove_directory(j, d);
2230                                 if (r < 0)
2231                                         log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2232                         }
2233
2234
2235                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2236
2237                         /* Event for root directory */
2238
2239                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2240                                 r = add_directory(j, d->path, e->name);
2241                                 if (r < 0)
2242                                         log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2243                         }
2244                 }
2245
2246                 return;
2247         }
2248
2249         if (e->mask & IN_IGNORED)
2250                 return;
2251
2252         log_warning("Unknown inotify event.");
2253 }
2254
2255 static int determine_change(sd_journal *j) {
2256         bool b;
2257
2258         assert(j);
2259
2260         b = j->current_invalidate_counter != j->last_invalidate_counter;
2261         j->last_invalidate_counter = j->current_invalidate_counter;
2262
2263         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2264 }
2265
2266 _public_ int sd_journal_process(sd_journal *j) {
2267         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2268         bool got_something = false;
2269
2270         if (!j)
2271                 return -EINVAL;
2272         if (journal_pid_changed(j))
2273                 return -ECHILD;
2274
2275         j->last_process_usec = now(CLOCK_MONOTONIC);
2276
2277         for (;;) {
2278                 struct inotify_event *e;
2279                 ssize_t l;
2280
2281                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2282                 if (l < 0) {
2283                         if (errno == EAGAIN || errno == EINTR)
2284                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2285
2286                         return -errno;
2287                 }
2288
2289                 got_something = true;
2290
2291                 e = (struct inotify_event*) buffer;
2292                 while (l > 0) {
2293                         size_t step;
2294
2295                         process_inotify_event(j, e);
2296
2297                         step = sizeof(struct inotify_event) + e->len;
2298                         assert(step <= (size_t) l);
2299
2300                         e = (struct inotify_event*) ((uint8_t*) e + step);
2301                         l -= step;
2302                 }
2303         }
2304
2305         return determine_change(j);
2306 }
2307
2308 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2309         int r;
2310         uint64_t t;
2311
2312         if (!j)
2313                 return -EINVAL;
2314         if (journal_pid_changed(j))
2315                 return -ECHILD;
2316
2317         if (j->inotify_fd < 0) {
2318
2319                 /* This is the first invocation, hence create the
2320                  * inotify watch */
2321                 r = sd_journal_get_fd(j);
2322                 if (r < 0)
2323                         return r;
2324
2325                 /* The journal might have changed since the context
2326                  * object was created and we weren't watching before,
2327                  * hence don't wait for anything, and return
2328                  * immediately. */
2329                 return determine_change(j);
2330         }
2331
2332         r = sd_journal_get_timeout(j, &t);
2333         if (r < 0)
2334                 return r;
2335
2336         if (t != (uint64_t) -1) {
2337                 usec_t n;
2338
2339                 n = now(CLOCK_MONOTONIC);
2340                 t = t > n ? t - n : 0;
2341
2342                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2343                         timeout_usec = t;
2344         }
2345
2346         do {
2347                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2348         } while (r == -EINTR);
2349
2350         if (r < 0)
2351                 return r;
2352
2353         return sd_journal_process(j);
2354 }
2355
2356 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2357         Iterator i;
2358         JournalFile *f;
2359         bool first = true;
2360         int r;
2361
2362         if (!j)
2363                 return -EINVAL;
2364         if (journal_pid_changed(j))
2365                 return -ECHILD;
2366         if (!from && !to)
2367                 return -EINVAL;
2368         if (from == to)
2369                 return -EINVAL;
2370
2371         HASHMAP_FOREACH(f, j->files, i) {
2372                 usec_t fr, t;
2373
2374                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2375                 if (r == -ENOENT)
2376                         continue;
2377                 if (r < 0)
2378                         return r;
2379                 if (r == 0)
2380                         continue;
2381
2382                 if (first) {
2383                         if (from)
2384                                 *from = fr;
2385                         if (to)
2386                                 *to = t;
2387                         first = false;
2388                 } else {
2389                         if (from)
2390                                 *from = MIN(fr, *from);
2391                         if (to)
2392                                 *to = MAX(t, *to);
2393                 }
2394         }
2395
2396         return first ? 0 : 1;
2397 }
2398
2399 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2400         Iterator i;
2401         JournalFile *f;
2402         bool first = true;
2403         int r;
2404
2405         if (!j)
2406                 return -EINVAL;
2407         if (journal_pid_changed(j))
2408                 return -ECHILD;
2409         if (!from && !to)
2410                 return -EINVAL;
2411         if (from == to)
2412                 return -EINVAL;
2413
2414         HASHMAP_FOREACH(f, j->files, i) {
2415                 usec_t fr, t;
2416
2417                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2418                 if (r == -ENOENT)
2419                         continue;
2420                 if (r < 0)
2421                         return r;
2422                 if (r == 0)
2423                         continue;
2424
2425                 if (first) {
2426                         if (from)
2427                                 *from = fr;
2428                         if (to)
2429                                 *to = t;
2430                         first = false;
2431                 } else {
2432                         if (from)
2433                                 *from = MIN(fr, *from);
2434                         if (to)
2435                                 *to = MAX(t, *to);
2436                 }
2437         }
2438
2439         return first ? 0 : 1;
2440 }
2441
2442 void journal_print_header(sd_journal *j) {
2443         Iterator i;
2444         JournalFile *f;
2445         bool newline = false;
2446
2447         assert(j);
2448
2449         HASHMAP_FOREACH(f, j->files, i) {
2450                 if (newline)
2451                         putchar('\n');
2452                 else
2453                         newline = true;
2454
2455                 journal_file_print_header(f);
2456         }
2457 }
2458
2459 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2460         Iterator i;
2461         JournalFile *f;
2462         uint64_t sum = 0;
2463
2464         if (!j)
2465                 return -EINVAL;
2466         if (journal_pid_changed(j))
2467                 return -ECHILD;
2468         if (!bytes)
2469                 return -EINVAL;
2470
2471         HASHMAP_FOREACH(f, j->files, i) {
2472                 struct stat st;
2473
2474                 if (fstat(f->fd, &st) < 0)
2475                         return -errno;
2476
2477                 sum += (uint64_t) st.st_blocks * 512ULL;
2478         }
2479
2480         *bytes = sum;
2481         return 0;
2482 }
2483
2484 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2485         char *f;
2486
2487         if (!j)
2488                 return -EINVAL;
2489         if (journal_pid_changed(j))
2490                 return -ECHILD;
2491         if (isempty(field))
2492                 return -EINVAL;
2493         if (!field_is_valid(field))
2494                 return -EINVAL;
2495
2496         f = strdup(field);
2497         if (!f)
2498                 return -ENOMEM;
2499
2500         free(j->unique_field);
2501         j->unique_field = f;
2502         j->unique_file = NULL;
2503         j->unique_offset = 0;
2504
2505         return 0;
2506 }
2507
2508 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2509         Object *o;
2510         size_t k;
2511         int r;
2512
2513         if (!j)
2514                 return -EINVAL;
2515         if (journal_pid_changed(j))
2516                 return -ECHILD;
2517         if (!data)
2518                 return -EINVAL;
2519         if (!l)
2520                 return -EINVAL;
2521         if (!j->unique_field)
2522                 return -EINVAL;
2523
2524         k = strlen(j->unique_field);
2525
2526         if (!j->unique_file) {
2527                 j->unique_file = hashmap_first(j->files);
2528                 if (!j->unique_file)
2529                         return 0;
2530                 j->unique_offset = 0;
2531         }
2532
2533         for (;;) {
2534                 JournalFile *of;
2535                 Iterator i;
2536                 const void *odata;
2537                 size_t ol;
2538                 bool found;
2539
2540                 /* Proceed to next data object in the field's linked list */
2541                 if (j->unique_offset == 0) {
2542                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2543                         if (r < 0)
2544                                 return r;
2545
2546                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2547                 } else {
2548                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2549                         if (r < 0)
2550                                 return r;
2551
2552                         j->unique_offset = le64toh(o->data.next_field_offset);
2553                 }
2554
2555                 /* We reached the end of the list? Then start again, with the next file */
2556                 if (j->unique_offset == 0) {
2557                         JournalFile *n;
2558
2559                         n = hashmap_next(j->files, j->unique_file->path);
2560                         if (!n)
2561                                 return 0;
2562
2563                         j->unique_file = n;
2564                         continue;
2565                 }
2566
2567                 /* We do not use the type context here, but 0 instead,
2568                  * so that we can look at this data object at the same
2569                  * time as one on another file */
2570                 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2571                 if (r < 0)
2572                         return r;
2573
2574                 /* Let's do the type check by hand, since we used 0 context above. */
2575                 if (o->object.type != OBJECT_DATA)
2576                         return -EBADMSG;
2577
2578                 r = return_data(j, j->unique_file, o, &odata, &ol);
2579                 if (r < 0)
2580                         return r;
2581
2582                 /* OK, now let's see if we already returned this data
2583                  * object by checking if it exists in the earlier
2584                  * traversed files. */
2585                 found = false;
2586                 HASHMAP_FOREACH(of, j->files, i) {
2587                         Object *oo;
2588                         uint64_t op;
2589
2590                         if (of == j->unique_file)
2591                                 break;
2592
2593                         /* Skip this file it didn't have any fields
2594                          * indexed */
2595                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2596                             le64toh(of->header->n_fields) <= 0)
2597                                 continue;
2598
2599                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2600                         if (r < 0)
2601                                 return r;
2602
2603                         if (r > 0)
2604                                 found = true;
2605                 }
2606
2607                 if (found)
2608                         continue;
2609
2610                 r = return_data(j, j->unique_file, o, data, l);
2611                 if (r < 0)
2612                         return r;
2613
2614                 return 1;
2615         }
2616 }
2617
2618 _public_ void sd_journal_restart_unique(sd_journal *j) {
2619         if (!j)
2620                 return;
2621
2622         j->unique_file = NULL;
2623         j->unique_offset = 0;
2624 }
2625
2626 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2627         if (!j)
2628                 return -EINVAL;
2629         if (journal_pid_changed(j))
2630                 return -ECHILD;
2631
2632         return !j->on_network;
2633 }
2634
2635 static char *lookup_field(const char *field, void *userdata) {
2636         sd_journal *j = userdata;
2637         const void *data;
2638         size_t size, d;
2639         int r;
2640
2641         assert(field);
2642         assert(j);
2643
2644         r = sd_journal_get_data(j, field, &data, &size);
2645         if (r < 0 ||
2646             size > REPLACE_VAR_MAX)
2647                 return strdup(field);
2648
2649         d = strlen(field) + 1;
2650
2651         return strndup((const char*) data + d, size - d);
2652 }
2653
2654 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2655         const void *data;
2656         size_t size;
2657         sd_id128_t id;
2658         _cleanup_free_ char *text = NULL, *cid = NULL;
2659         char *t;
2660         int r;
2661
2662         if (!j)
2663                 return -EINVAL;
2664         if (journal_pid_changed(j))
2665                 return -ECHILD;
2666         if (!ret)
2667                 return -EINVAL;
2668
2669         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2670         if (r < 0)
2671                 return r;
2672
2673         cid = strndup((const char*) data + 11, size - 11);
2674         if (!cid)
2675                 return -ENOMEM;
2676
2677         r = sd_id128_from_string(cid, &id);
2678         if (r < 0)
2679                 return r;
2680
2681         r = catalog_get(CATALOG_DATABASE, id, &text);
2682         if (r < 0)
2683                 return r;
2684
2685         t = replace_var(text, lookup_field, j);
2686         if (!t)
2687                 return -ENOMEM;
2688
2689         *ret = t;
2690         return 0;
2691 }
2692
2693 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2694         if (!ret)
2695                 return -EINVAL;
2696
2697         return catalog_get(CATALOG_DATABASE, id, ret);
2698 }
2699
2700 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2701         if (!j)
2702                 return -EINVAL;
2703         if (journal_pid_changed(j))
2704                 return -ECHILD;
2705
2706         j->data_threshold = sz;
2707         return 0;
2708 }
2709
2710 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2711         if (!j)
2712                 return -EINVAL;
2713         if (journal_pid_changed(j))
2714                 return -ECHILD;
2715         if (!sz)
2716                 return -EINVAL;
2717
2718         *sz = j->data_threshold;
2719         return 0;
2720 }