src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "strv.h"
  37 #include "path-util.h"
  38 #include "lookup3.h"
  39 #include "compress.h"
  40 #include "journal-internal.h"
  41 #include "missing.h"
  42 #include "catalog.h"
  43 #include "replace-var.h"
  44
  45 #define JOURNAL_FILES_MAX 1024
  46
  47 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  48
  49 #define REPLACE_VAR_MAX 256
  50
  51 #define DEFAULT_DATA_THRESHOLD (64*1024)
  52
  53 static bool journal_pid_changed(sd_journal *j) {
  54         assert(j);
  55
  56         /* We don't support people creating a journal object and
  57          * keeping it around over a fork(). Let's complain. */
  58
  59         return j->original_pid != getpid();
  60 }
  61
  62 /* We return an error here only if we didn't manage to
  63    memorize the real error. */
  64 static int set_put_error(sd_journal *j, int r) {
  65         int k;
  66
  67         if (r >= 0)
  68                 return r;
  69
  70         k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
  71         if (k < 0)
  72                 return k;
  73
  74         return set_put(j->errors, INT_TO_PTR(r));
  75 }
  76
  77 static void detach_location(sd_journal *j) {
  78         Iterator i;
  79         JournalFile *f;
  80
  81         assert(j);
  82
  83         j->current_file = NULL;
  84         j->current_field = 0;
  85
  86         HASHMAP_FOREACH(f, j->files, i)
  87                 f->current_offset = 0;
  88 }
  89
  90 static void reset_location(sd_journal *j) {
  91         assert(j);
  92
  93         detach_location(j);
  94         zero(j->current_location);
  95 }
  96
  97 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  98         assert(l);
  99         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 100         assert(f);
 101         assert(o->object.type == OBJECT_ENTRY);
 102
 103         l->type = type;
 104         l->seqnum = le64toh(o->entry.seqnum);
 105         l->seqnum_id = f->header->seqnum_id;
 106         l->realtime = le64toh(o->entry.realtime);
 107         l->monotonic = le64toh(o->entry.monotonic);
 108         l->boot_id = o->entry.boot_id;
 109         l->xor_hash = le64toh(o->entry.xor_hash);
 110
 111         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 112 }
 113
 114 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o,
 115                          direction_t direction, uint64_t offset) {
 116         assert(j);
 117         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 118         assert(f);
 119         assert(o);
 120
 121         init_location(&j->current_location, type, f, o);
 122
 123         j->current_file = f;
 124         j->current_field = 0;
 125
 126         f->last_direction = direction;
 127         f->current_offset = offset;
 128 }
 129
 130 static int match_is_valid(const void *data, size_t size) {
 131         const char *b, *p;
 132
 133         assert(data);
 134
 135         if (size < 2)
 136                 return false;
 137
 138         if (startswith(data, "__"))
 139                 return false;
 140
 141         b = data;
 142         for (p = b; p < b + size; p++) {
 143
 144                 if (*p == '=')
 145                         return p > b;
 146
 147                 if (*p == '_')
 148                         continue;
 149
 150                 if (*p >= 'A' && *p <= 'Z')
 151                         continue;
 152
 153                 if (*p >= '0' && *p <= '9')
 154                         continue;
 155
 156                 return false;
 157         }
 158
 159         return false;
 160 }
 161
 162 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 163         const uint8_t *a = _a, *b = _b;
 164         size_t j;
 165
 166         for (j = 0; j < s && j < t; j++) {
 167
 168                 if (a[j] != b[j])
 169                         return false;
 170
 171                 if (a[j] == '=')
 172                         return true;
 173         }
 174
 175         assert_not_reached("\"=\" not found");
 176 }
 177
 178 static Match *match_new(Match *p, MatchType t) {
 179         Match *m;
 180
 181         m = new0(Match, 1);
 182         if (!m)
 183                 return NULL;
 184
 185         m->type = t;
 186
 187         if (p) {
 188                 m->parent = p;
 189                 LIST_PREPEND(matches, p->matches, m);
 190         }
 191
 192         return m;
 193 }
 194
 195 static void match_free(Match *m) {
 196         assert(m);
 197
 198         while (m->matches)
 199                 match_free(m->matches);
 200
 201         if (m->parent)
 202                 LIST_REMOVE(matches, m->parent->matches, m);
 203
 204         free(m->data);
 205         free(m);
 206 }
 207
 208 static void match_free_if_empty(Match *m) {
 209         if (!m || m->matches)
 210                 return;
 211
 212         match_free(m);
 213 }
 214
 215 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 216         Match *l3, *l4, *add_here = NULL, *m;
 217         le64_t le_hash;
 218
 219         if (!j)
 220                 return -EINVAL;
 221         if (journal_pid_changed(j))
 222                 return -ECHILD;
 223
 224         if (!data)
 225                 return -EINVAL;
 226
 227         if (size == 0)
 228                 size = strlen(data);
 229
 230         if (!match_is_valid(data, size))
 231                 return -EINVAL;
 232
 233         /* level 0: AND term
 234          * level 1: OR terms
 235          * level 2: AND terms
 236          * level 3: OR terms
 237          * level 4: concrete matches */
 238
 239         if (!j->level0) {
 240                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 241                 if (!j->level0)
 242                         return -ENOMEM;
 243         }
 244
 245         if (!j->level1) {
 246                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 247                 if (!j->level1)
 248                         return -ENOMEM;
 249         }
 250
 251         if (!j->level2) {
 252                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 253                 if (!j->level2)
 254                         return -ENOMEM;
 255         }
 256
 257         assert(j->level0->type == MATCH_AND_TERM);
 258         assert(j->level1->type == MATCH_OR_TERM);
 259         assert(j->level2->type == MATCH_AND_TERM);
 260
 261         le_hash = htole64(hash64(data, size));
 262
 263         LIST_FOREACH(matches, l3, j->level2->matches) {
 264                 assert(l3->type == MATCH_OR_TERM);
 265
 266                 LIST_FOREACH(matches, l4, l3->matches) {
 267                         assert(l4->type == MATCH_DISCRETE);
 268
 269                         /* Exactly the same match already? Then ignore
 270                          * this addition */
 271                         if (l4->le_hash == le_hash &&
 272                             l4->size == size &&
 273                             memcmp(l4->data, data, size) == 0)
 274                                 return 0;
 275
 276                         /* Same field? Then let's add this to this OR term */
 277                         if (same_field(data, size, l4->data, l4->size)) {
 278                                 add_here = l3;
 279                                 break;
 280                         }
 281                 }
 282
 283                 if (add_here)
 284                         break;
 285         }
 286
 287         if (!add_here) {
 288                 add_here = match_new(j->level2, MATCH_OR_TERM);
 289                 if (!add_here)
 290                         goto fail;
 291         }
 292
 293         m = match_new(add_here, MATCH_DISCRETE);
 294         if (!m)
 295                 goto fail;
 296
 297         m->le_hash = le_hash;
 298         m->size = size;
 299         m->data = memdup(data, size);
 300         if (!m->data)
 301                 goto fail;
 302
 303         detach_location(j);
 304
 305         return 0;
 306
 307 fail:
 308         match_free_if_empty(add_here);
 309         match_free_if_empty(j->level2);
 310         match_free_if_empty(j->level1);
 311         match_free_if_empty(j->level0);
 312
 313         return -ENOMEM;
 314 }
 315
 316 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 317         if (!j)
 318                 return -EINVAL;
 319         if (journal_pid_changed(j))
 320                 return -ECHILD;
 321
 322         if (!j->level0)
 323                 return 0;
 324
 325         if (!j->level1)
 326                 return 0;
 327
 328         if (!j->level1->matches)
 329                 return 0;
 330
 331         j->level1 = NULL;
 332         j->level2 = NULL;
 333
 334         return 0;
 335 }
 336
 337 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 338         if (!j)
 339                 return -EINVAL;
 340         if (journal_pid_changed(j))
 341                 return -ECHILD;
 342
 343         if (!j->level0)
 344                 return 0;
 345
 346         if (!j->level1)
 347                 return 0;
 348
 349         if (!j->level2)
 350                 return 0;
 351
 352         if (!j->level2->matches)
 353                 return 0;
 354
 355         j->level2 = NULL;
 356         return 0;
 357 }
 358
 359 static char *match_make_string(Match *m) {
 360         char *p, *r;
 361         Match *i;
 362         bool enclose = false;
 363
 364         if (!m)
 365                 return strdup("none");
 366
 367         if (m->type == MATCH_DISCRETE)
 368                 return strndup(m->data, m->size);
 369
 370         p = NULL;
 371         LIST_FOREACH(matches, i, m->matches) {
 372                 char *t, *k;
 373
 374                 t = match_make_string(i);
 375                 if (!t) {
 376                         free(p);
 377                         return NULL;
 378                 }
 379
 380                 if (p) {
 381                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 382                         free(p);
 383                         free(t);
 384
 385                         if (!k)
 386                                 return NULL;
 387
 388                         p = k;
 389
 390                         enclose = true;
 391                 } else
 392                         p = t;
 393         }
 394
 395         if (enclose) {
 396                 r = strjoin("(", p, ")", NULL);
 397                 free(p);
 398                 return r;
 399         }
 400
 401         return p;
 402 }
 403
 404 char *journal_make_match_string(sd_journal *j) {
 405         assert(j);
 406
 407         return match_make_string(j->level0);
 408 }
 409
 410 _public_ void sd_journal_flush_matches(sd_journal *j) {
 411         if (!j)
 412                 return;
 413
 414         if (j->level0)
 415                 match_free(j->level0);
 416
 417         j->level0 = j->level1 = j->level2 = NULL;
 418
 419         detach_location(j);
 420 }
 421
 422 static int compare_entry_order(JournalFile *af, Object *_ao,
 423                                JournalFile *bf, uint64_t bp) {
 424
 425         uint64_t a, b;
 426         Object *ao, *bo;
 427         int r;
 428
 429         assert(af);
 430         assert(bf);
 431         assert(_ao);
 432
 433         /* The mmap cache might invalidate the object from the first
 434          * file if we look at the one from the second file. Hence
 435          * temporarily copy the header of the first one, and look at
 436          * that only. */
 437         ao = alloca(offsetof(EntryObject, items));
 438         memcpy(ao, _ao, offsetof(EntryObject, items));
 439
 440         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 441         if (r < 0)
 442                 return strcmp(af->path, bf->path);
 443
 444         /* We operate on two different files here, hence we can access
 445          * two objects at the same time, which we normally can't.
 446          *
 447          * If contents and timestamps match, these entries are
 448          * identical, even if the seqnum does not match */
 449
 450         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 451             ao->entry.monotonic == bo->entry.monotonic &&
 452             ao->entry.realtime == bo->entry.realtime &&
 453             ao->entry.xor_hash == bo->entry.xor_hash)
 454                 return 0;
 455
 456         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 457
 458                 /* If this is from the same seqnum source, compare
 459                  * seqnums */
 460                 a = le64toh(ao->entry.seqnum);
 461                 b = le64toh(bo->entry.seqnum);
 462
 463                 if (a < b)
 464                         return -1;
 465                 if (a > b)
 466                         return 1;
 467
 468                 /* Wow! This is weird, different data but the same
 469                  * seqnums? Something is borked, but let's make the
 470                  * best of it and compare by time. */
 471         }
 472
 473         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 474
 475                 /* If the boot id matches compare monotonic time */
 476                 a = le64toh(ao->entry.monotonic);
 477                 b = le64toh(bo->entry.monotonic);
 478
 479                 if (a < b)
 480                         return -1;
 481                 if (a > b)
 482                         return 1;
 483         }
 484
 485         /* Otherwise compare UTC time */
 486         a = le64toh(ao->entry.realtime);
 487         b = le64toh(bo->entry.realtime);
 488
 489         if (a < b)
 490                 return -1;
 491         if (a > b)
 492                 return 1;
 493
 494         /* Finally, compare by contents */
 495         a = le64toh(ao->entry.xor_hash);
 496         b = le64toh(bo->entry.xor_hash);
 497
 498         if (a < b)
 499                 return -1;
 500         if (a > b)
 501                 return 1;
 502
 503         return 0;
 504 }
 505
 506 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 507         uint64_t a;
 508
 509         assert(af);
 510         assert(ao);
 511         assert(l);
 512         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 513
 514         if (l->monotonic_set &&
 515             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 516             l->realtime_set &&
 517             le64toh(ao->entry.realtime) == l->realtime &&
 518             l->xor_hash_set &&
 519             le64toh(ao->entry.xor_hash) == l->xor_hash)
 520                 return 0;
 521
 522         if (l->seqnum_set &&
 523             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 524
 525                 a = le64toh(ao->entry.seqnum);
 526
 527                 if (a < l->seqnum)
 528                         return -1;
 529                 if (a > l->seqnum)
 530                         return 1;
 531         }
 532
 533         if (l->monotonic_set &&
 534             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 535
 536                 a = le64toh(ao->entry.monotonic);
 537
 538                 if (a < l->monotonic)
 539                         return -1;
 540                 if (a > l->monotonic)
 541                         return 1;
 542         }
 543
 544         if (l->realtime_set) {
 545
 546                 a = le64toh(ao->entry.realtime);
 547
 548                 if (a < l->realtime)
 549                         return -1;
 550                 if (a > l->realtime)
 551                         return 1;
 552         }
 553
 554         if (l->xor_hash_set) {
 555                 a = le64toh(ao->entry.xor_hash);
 556
 557                 if (a < l->xor_hash)
 558                         return -1;
 559                 if (a > l->xor_hash)
 560                         return 1;
 561         }
 562
 563         return 0;
 564 }
 565
 566 static int next_for_match(
 567                 sd_journal *j,
 568                 Match *m,
 569                 JournalFile *f,
 570                 uint64_t after_offset,
 571                 direction_t direction,
 572                 Object **ret,
 573                 uint64_t *offset) {
 574
 575         int r;
 576         uint64_t np = 0;
 577         Object *n;
 578
 579         assert(j);
 580         assert(m);
 581         assert(f);
 582
 583         if (m->type == MATCH_DISCRETE) {
 584                 uint64_t dp;
 585
 586                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 587                 if (r <= 0)
 588                         return r;
 589
 590                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 591
 592         } else if (m->type == MATCH_OR_TERM) {
 593                 Match *i;
 594
 595                 /* Find the earliest match beyond after_offset */
 596
 597                 LIST_FOREACH(matches, i, m->matches) {
 598                         uint64_t cp;
 599
 600                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 601                         if (r < 0)
 602                                 return r;
 603                         else if (r > 0) {
 604                                 if (np == 0 || (direction == DIRECTION_DOWN ? cp < np : cp > np))
 605                                         np = cp;
 606                         }
 607                 }
 608
 609                 if (np == 0)
 610                         return 0;
 611
 612         } else if (m->type == MATCH_AND_TERM) {
 613                 Match *i, *last_moved;
 614
 615                 /* Always jump to the next matching entry and repeat
 616                  * this until we find an offset that matches for all
 617                  * matches. */
 618
 619                 if (!m->matches)
 620                         return 0;
 621
 622                 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
 623                 if (r <= 0)
 624                         return r;
 625
 626                 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
 627                 last_moved = m->matches;
 628
 629                 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
 630                         uint64_t cp;
 631
 632                         r = next_for_match(j, i, f, np, direction, NULL, &cp);
 633                         if (r <= 0)
 634                                 return r;
 635
 636                         assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
 637                         if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
 638                                 np = cp;
 639                                 last_moved = i;
 640                         }
 641                 }
 642         }
 643
 644         assert(np > 0);
 645
 646         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 647         if (r < 0)
 648                 return r;
 649
 650         if (ret)
 651                 *ret = n;
 652         if (offset)
 653                 *offset = np;
 654
 655         return 1;
 656 }
 657
 658 static int find_location_for_match(
 659                 sd_journal *j,
 660                 Match *m,
 661                 JournalFile *f,
 662                 direction_t direction,
 663                 Object **ret,
 664                 uint64_t *offset) {
 665
 666         int r;
 667
 668         assert(j);
 669         assert(m);
 670         assert(f);
 671
 672         if (m->type == MATCH_DISCRETE) {
 673                 uint64_t dp;
 674
 675                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 676                 if (r <= 0)
 677                         return r;
 678
 679                 /* FIXME: missing: find by monotonic */
 680
 681                 if (j->current_location.type == LOCATION_HEAD)
 682                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 683                 if (j->current_location.type == LOCATION_TAIL)
 684                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 685                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 686                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 687                 if (j->current_location.monotonic_set) {
 688                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 689                         if (r != -ENOENT)
 690                                 return r;
 691                 }
 692                 if (j->current_location.realtime_set)
 693                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 694
 695                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 696
 697         } else if (m->type == MATCH_OR_TERM) {
 698                 uint64_t np = 0;
 699                 Object *n;
 700                 Match *i;
 701
 702                 /* Find the earliest match */
 703
 704                 LIST_FOREACH(matches, i, m->matches) {
 705                         uint64_t cp;
 706
 707                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 708                         if (r < 0)
 709                                 return r;
 710                         else if (r > 0) {
 711                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 712                                         np = cp;
 713                         }
 714                 }
 715
 716                 if (np == 0)
 717                         return 0;
 718
 719                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 720                 if (r < 0)
 721                         return r;
 722
 723                 if (ret)
 724                         *ret = n;
 725                 if (offset)
 726                         *offset = np;
 727
 728                 return 1;
 729
 730         } else {
 731                 Match *i;
 732                 uint64_t np = 0;
 733
 734                 assert(m->type == MATCH_AND_TERM);
 735
 736                 /* First jump to the last match, and then find the
 737                  * next one where all matches match */
 738
 739                 if (!m->matches)
 740                         return 0;
 741
 742                 LIST_FOREACH(matches, i, m->matches) {
 743                         uint64_t cp;
 744
 745                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 746                         if (r <= 0)
 747                                 return r;
 748
 749                         if (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))
 750                                 np = cp;
 751                 }
 752
 753                 return next_for_match(j, m, f, np, direction, ret, offset);
 754         }
 755 }
 756
 757 static int find_location_with_matches(
 758                 sd_journal *j,
 759                 JournalFile *f,
 760                 direction_t direction,
 761                 Object **ret,
 762                 uint64_t *offset) {
 763
 764         int r;
 765
 766         assert(j);
 767         assert(f);
 768         assert(ret);
 769         assert(offset);
 770
 771         if (!j->level0) {
 772                 /* No matches is simple */
 773
 774                 if (j->current_location.type == LOCATION_HEAD)
 775                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 776                 if (j->current_location.type == LOCATION_TAIL)
 777                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 778                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 779                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 780                 if (j->current_location.monotonic_set) {
 781                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 782                         if (r != -ENOENT)
 783                                 return r;
 784                 }
 785                 if (j->current_location.realtime_set)
 786                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 787
 788                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 789         } else
 790                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 791 }
 792
 793 static int next_with_matches(
 794                 sd_journal *j,
 795                 JournalFile *f,
 796                 direction_t direction,
 797                 Object **ret,
 798                 uint64_t *offset) {
 799
 800         Object *c;
 801         uint64_t cp;
 802
 803         assert(j);
 804         assert(f);
 805         assert(ret);
 806         assert(offset);
 807
 808         c = *ret;
 809         cp = *offset;
 810
 811         /* No matches is easy. We simple advance the file
 812          * pointer by one. */
 813         if (!j->level0)
 814                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 815
 816         /* If we have a match then we look for the next matching entry
 817          * with an offset at least one step larger */
 818         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 819 }
 820
 821 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 822         Object *c;
 823         uint64_t cp;
 824         int r;
 825
 826         assert(j);
 827         assert(f);
 828
 829         if (f->last_direction == direction && f->current_offset > 0) {
 830                 cp = f->current_offset;
 831
 832                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 833                 if (r < 0)
 834                         return r;
 835
 836                 r = next_with_matches(j, f, direction, &c, &cp);
 837                 if (r <= 0)
 838                         return r;
 839         } else {
 840                 r = find_location_with_matches(j, f, direction, &c, &cp);
 841                 if (r <= 0)
 842                         return r;
 843         }
 844
 845         /* OK, we found the spot, now let's advance until an entry
 846          * that is actually different from what we were previously
 847          * looking at. This is necessary to handle entries which exist
 848          * in two (or more) journal files, and which shall all be
 849          * suppressed but one. */
 850
 851         for (;;) {
 852                 bool found;
 853
 854                 if (j->current_location.type == LOCATION_DISCRETE) {
 855                         int k;
 856
 857                         k = compare_with_location(f, c, &j->current_location);
 858                         if (direction == DIRECTION_DOWN)
 859                                 found = k > 0;
 860                         else
 861                                 found = k < 0;
 862                 } else
 863                         found = true;
 864
 865                 if (found) {
 866                         if (ret)
 867                                 *ret = c;
 868                         if (offset)
 869                                 *offset = cp;
 870                         return 1;
 871                 }
 872
 873                 r = next_with_matches(j, f, direction, &c, &cp);
 874                 if (r <= 0)
 875                         return r;
 876         }
 877 }
 878
 879 static int real_journal_next(sd_journal *j, direction_t direction) {
 880         JournalFile *f, *new_file = NULL;
 881         uint64_t new_offset = 0;
 882         Object *o;
 883         uint64_t p;
 884         Iterator i;
 885         int r;
 886
 887         if (!j)
 888                 return -EINVAL;
 889         if (journal_pid_changed(j))
 890                 return -ECHILD;
 891
 892         HASHMAP_FOREACH(f, j->files, i) {
 893                 bool found;
 894
 895                 r = next_beyond_location(j, f, direction, &o, &p);
 896                 if (r < 0) {
 897                         log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
 898                         continue;
 899                 } else if (r == 0)
 900                         continue;
 901
 902                 if (!new_file)
 903                         found = true;
 904                 else {
 905                         int k;
 906
 907                         k = compare_entry_order(f, o, new_file, new_offset);
 908
 909                         found = direction == DIRECTION_DOWN ? k < 0 : k > 0;
 910                 }
 911
 912                 if (found) {
 913                         new_file = f;
 914                         new_offset = p;
 915                 }
 916         }
 917
 918         if (!new_file)
 919                 return 0;
 920
 921         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 922         if (r < 0)
 923                 return r;
 924
 925         set_location(j, LOCATION_DISCRETE, new_file, o, direction, new_offset);
 926
 927         return 1;
 928 }
 929
 930 _public_ int sd_journal_next(sd_journal *j) {
 931         return real_journal_next(j, DIRECTION_DOWN);
 932 }
 933
 934 _public_ int sd_journal_previous(sd_journal *j) {
 935         return real_journal_next(j, DIRECTION_UP);
 936 }
 937
 938 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 939         int c = 0, r;
 940
 941         if (!j)
 942                 return -EINVAL;
 943         if (journal_pid_changed(j))
 944                 return -ECHILD;
 945
 946         if (skip == 0) {
 947                 /* If this is not a discrete skip, then at least
 948                  * resolve the current location */
 949                 if (j->current_location.type != LOCATION_DISCRETE)
 950                         return real_journal_next(j, direction);
 951
 952                 return 0;
 953         }
 954
 955         do {
 956                 r = real_journal_next(j, direction);
 957                 if (r < 0)
 958                         return r;
 959
 960                 if (r == 0)
 961                         return c;
 962
 963                 skip--;
 964                 c++;
 965         } while (skip > 0);
 966
 967         return c;
 968 }
 969
 970 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 971         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 972 }
 973
 974 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 975         return real_journal_next_skip(j, DIRECTION_UP, skip);
 976 }
 977
 978 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 979         Object *o;
 980         int r;
 981         char bid[33], sid[33];
 982
 983         if (!j)
 984                 return -EINVAL;
 985         if (journal_pid_changed(j))
 986                 return -ECHILD;
 987         if (!cursor)
 988                 return -EINVAL;
 989
 990         if (!j->current_file || j->current_file->current_offset <= 0)
 991                 return -EADDRNOTAVAIL;
 992
 993         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 994         if (r < 0)
 995                 return r;
 996
 997         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 998         sd_id128_to_string(o->entry.boot_id, bid);
 999
1000         if (asprintf(cursor,
1001                      "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
1002                      sid, le64toh(o->entry.seqnum),
1003                      bid, le64toh(o->entry.monotonic),
1004                      le64toh(o->entry.realtime),
1005                      le64toh(o->entry.xor_hash)) < 0)
1006                 return -ENOMEM;
1007
1008         return 0;
1009 }
1010
1011 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
1012         char *w, *state;
1013         size_t l;
1014         unsigned long long seqnum, monotonic, realtime, xor_hash;
1015         bool
1016                 seqnum_id_set = false,
1017                 seqnum_set = false,
1018                 boot_id_set = false,
1019                 monotonic_set = false,
1020                 realtime_set = false,
1021                 xor_hash_set = false;
1022         sd_id128_t seqnum_id, boot_id;
1023
1024         if (!j)
1025                 return -EINVAL;
1026         if (journal_pid_changed(j))
1027                 return -ECHILD;
1028         if (isempty(cursor))
1029                 return -EINVAL;
1030
1031         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1032                 char *item;
1033                 int k = 0;
1034
1035                 if (l < 2 || w[1] != '=')
1036                         return -EINVAL;
1037
1038                 item = strndup(w, l);
1039                 if (!item)
1040                         return -ENOMEM;
1041
1042                 switch (w[0]) {
1043
1044                 case 's':
1045                         seqnum_id_set = true;
1046                         k = sd_id128_from_string(item+2, &seqnum_id);
1047                         break;
1048
1049                 case 'i':
1050                         seqnum_set = true;
1051                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1052                                 k = -EINVAL;
1053                         break;
1054
1055                 case 'b':
1056                         boot_id_set = true;
1057                         k = sd_id128_from_string(item+2, &boot_id);
1058                         break;
1059
1060                 case 'm':
1061                         monotonic_set = true;
1062                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1063                                 k = -EINVAL;
1064                         break;
1065
1066                 case 't':
1067                         realtime_set = true;
1068                         if (sscanf(item+2, "%llx", &realtime) != 1)
1069                                 k = -EINVAL;
1070                         break;
1071
1072                 case 'x':
1073                         xor_hash_set = true;
1074                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1075                                 k = -EINVAL;
1076                         break;
1077                 }
1078
1079                 free(item);
1080
1081                 if (k < 0)
1082                         return k;
1083         }
1084
1085         if ((!seqnum_set || !seqnum_id_set) &&
1086             (!monotonic_set || !boot_id_set) &&
1087             !realtime_set)
1088                 return -EINVAL;
1089
1090         reset_location(j);
1091
1092         j->current_location.type = LOCATION_SEEK;
1093
1094         if (realtime_set) {
1095                 j->current_location.realtime = (uint64_t) realtime;
1096                 j->current_location.realtime_set = true;
1097         }
1098
1099         if (seqnum_set && seqnum_id_set) {
1100                 j->current_location.seqnum = (uint64_t) seqnum;
1101                 j->current_location.seqnum_id = seqnum_id;
1102                 j->current_location.seqnum_set = true;
1103         }
1104
1105         if (monotonic_set && boot_id_set) {
1106                 j->current_location.monotonic = (uint64_t) monotonic;
1107                 j->current_location.boot_id = boot_id;
1108                 j->current_location.monotonic_set = true;
1109         }
1110
1111         if (xor_hash_set) {
1112                 j->current_location.xor_hash = (uint64_t) xor_hash;
1113                 j->current_location.xor_hash_set = true;
1114         }
1115
1116         return 0;
1117 }
1118
1119 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1120         int r;
1121         char *w, *state;
1122         size_t l;
1123         Object *o;
1124
1125         if (!j)
1126                 return -EINVAL;
1127         if (journal_pid_changed(j))
1128                 return -ECHILD;
1129         if (isempty(cursor))
1130                 return -EINVAL;
1131
1132         if (!j->current_file || j->current_file->current_offset <= 0)
1133                 return -EADDRNOTAVAIL;
1134
1135         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1136         if (r < 0)
1137                 return r;
1138
1139         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1140                 _cleanup_free_ char *item = NULL;
1141                 sd_id128_t id;
1142                 unsigned long long ll;
1143                 int k = 0;
1144
1145                 if (l < 2 || w[1] != '=')
1146                         return -EINVAL;
1147
1148                 item = strndup(w, l);
1149                 if (!item)
1150                         return -ENOMEM;
1151
1152                 switch (w[0]) {
1153
1154                 case 's':
1155                         k = sd_id128_from_string(item+2, &id);
1156                         if (k < 0)
1157                                 return k;
1158                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1159                                 return 0;
1160                         break;
1161
1162                 case 'i':
1163                         if (sscanf(item+2, "%llx", &ll) != 1)
1164                                 return -EINVAL;
1165                         if (ll != le64toh(o->entry.seqnum))
1166                                 return 0;
1167                         break;
1168
1169                 case 'b':
1170                         k = sd_id128_from_string(item+2, &id);
1171                         if (k < 0)
1172                                 return k;
1173                         if (!sd_id128_equal(id, o->entry.boot_id))
1174                                 return 0;
1175                         break;
1176
1177                 case 'm':
1178                         if (sscanf(item+2, "%llx", &ll) != 1)
1179                                 return -EINVAL;
1180                         if (ll != le64toh(o->entry.monotonic))
1181                                 return 0;
1182                         break;
1183
1184                 case 't':
1185                         if (sscanf(item+2, "%llx", &ll) != 1)
1186                                 return -EINVAL;
1187                         if (ll != le64toh(o->entry.realtime))
1188                                 return 0;
1189                         break;
1190
1191                 case 'x':
1192                         if (sscanf(item+2, "%llx", &ll) != 1)
1193                                 return -EINVAL;
1194                         if (ll != le64toh(o->entry.xor_hash))
1195                                 return 0;
1196                         break;
1197                 }
1198         }
1199
1200         return 1;
1201 }
1202
1203
1204 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1205         if (!j)
1206                 return -EINVAL;
1207         if (journal_pid_changed(j))
1208                 return -ECHILD;
1209
1210         reset_location(j);
1211         j->current_location.type = LOCATION_SEEK;
1212         j->current_location.boot_id = boot_id;
1213         j->current_location.monotonic = usec;
1214         j->current_location.monotonic_set = true;
1215
1216         return 0;
1217 }
1218
1219 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1220         if (!j)
1221                 return -EINVAL;
1222         if (journal_pid_changed(j))
1223                 return -ECHILD;
1224
1225         reset_location(j);
1226         j->current_location.type = LOCATION_SEEK;
1227         j->current_location.realtime = usec;
1228         j->current_location.realtime_set = true;
1229
1230         return 0;
1231 }
1232
1233 _public_ int sd_journal_seek_head(sd_journal *j) {
1234         if (!j)
1235                 return -EINVAL;
1236         if (journal_pid_changed(j))
1237                 return -ECHILD;
1238
1239         reset_location(j);
1240         j->current_location.type = LOCATION_HEAD;
1241
1242         return 0;
1243 }
1244
1245 _public_ int sd_journal_seek_tail(sd_journal *j) {
1246         if (!j)
1247                 return -EINVAL;
1248         if (journal_pid_changed(j))
1249                 return -ECHILD;
1250
1251         reset_location(j);
1252         j->current_location.type = LOCATION_TAIL;
1253
1254         return 0;
1255 }
1256
1257 static void check_network(sd_journal *j, int fd) {
1258         struct statfs sfs;
1259
1260         assert(j);
1261
1262         if (j->on_network)
1263                 return;
1264
1265         if (fstatfs(fd, &sfs) < 0)
1266                 return;
1267
1268         j->on_network =
1269                 F_TYPE_EQUAL(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1270                 F_TYPE_EQUAL(sfs.f_type, CODA_SUPER_MAGIC) ||
1271                 F_TYPE_EQUAL(sfs.f_type, NCP_SUPER_MAGIC) ||
1272                 F_TYPE_EQUAL(sfs.f_type, NFS_SUPER_MAGIC) ||
1273                 F_TYPE_EQUAL(sfs.f_type, SMB_SUPER_MAGIC);
1274 }
1275
1276 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1277         const char *full, *tilded, *atted;
1278
1279         full = strappenda(prefix, ".journal");
1280         tilded = strappenda(full, "~");
1281         atted = strappenda(prefix, "@");
1282
1283         return streq(filename, full) ||
1284                streq(filename, tilded) ||
1285                startswith(filename, atted);
1286 }
1287
1288 static bool file_type_wanted(int flags, const char *filename) {
1289         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1290                 return false;
1291
1292         /* no flags set → every type is OK */
1293         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1294                 return true;
1295
1296         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1297                 return true;
1298
1299         if (flags & SD_JOURNAL_CURRENT_USER) {
1300                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1301
1302                 assert_se(snprintf(prefix, sizeof(prefix), "user-%lu", (unsigned long) getuid())
1303                           < (int) sizeof(prefix));
1304
1305                 if (file_has_type_prefix(prefix, filename))
1306                         return true;
1307         }
1308
1309         return false;
1310 }
1311
1312 static int add_any_file(sd_journal *j, const char *path) {
1313         JournalFile *f;
1314         int r;
1315
1316         assert(j);
1317         assert(path);
1318
1319         if (hashmap_get(j->files, path))
1320                 return 0;
1321
1322         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1323                 log_warning("Too many open journal files, not adding %s.", path);
1324                 return set_put_error(j, -ETOOMANYREFS);
1325         }
1326
1327         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1328         if (r < 0)
1329                 return r;
1330
1331         /* journal_file_dump(f); */
1332
1333         r = hashmap_put(j->files, f->path, f);
1334         if (r < 0) {
1335                 journal_file_close(f);
1336                 return r;
1337         }
1338
1339         log_debug("File %s added.", f->path);
1340
1341         check_network(j, f->fd);
1342
1343         j->current_invalidate_counter ++;
1344
1345         return 0;
1346 }
1347
1348 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1349         _cleanup_free_ char *path = NULL;
1350         int r;
1351
1352         assert(j);
1353         assert(prefix);
1354         assert(filename);
1355
1356         if (j->no_new_files ||
1357             !file_type_wanted(j->flags, filename))
1358                 return 0;
1359
1360         path = strjoin(prefix, "/", filename, NULL);
1361         if (!path)
1362                 return -ENOMEM;
1363
1364         r = add_any_file(j, path);
1365         if (r == -ENOENT)
1366                 return 0;
1367         return 0;
1368 }
1369
1370 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1371         char *path;
1372         JournalFile *f;
1373
1374         assert(j);
1375         assert(prefix);
1376         assert(filename);
1377
1378         path = strjoin(prefix, "/", filename, NULL);
1379         if (!path)
1380                 return -ENOMEM;
1381
1382         f = hashmap_get(j->files, path);
1383         free(path);
1384         if (!f)
1385                 return 0;
1386
1387         hashmap_remove(j->files, f->path);
1388
1389         log_debug("File %s removed.", f->path);
1390
1391         if (j->current_file == f) {
1392                 j->current_file = NULL;
1393                 j->current_field = 0;
1394         }
1395
1396         if (j->unique_file == f) {
1397                 j->unique_file = NULL;
1398                 j->unique_offset = 0;
1399         }
1400
1401         journal_file_close(f);
1402
1403         j->current_invalidate_counter ++;
1404
1405         return 0;
1406 }
1407
1408 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1409         _cleanup_free_ char *path = NULL;
1410         int r;
1411         _cleanup_closedir_ DIR *d = NULL;
1412         sd_id128_t id, mid;
1413         Directory *m;
1414
1415         assert(j);
1416         assert(prefix);
1417         assert(dirname);
1418
1419         log_debug("Considering %s/%s.", prefix, dirname);
1420
1421         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1422             (sd_id128_from_string(dirname, &id) < 0 ||
1423              sd_id128_get_machine(&mid) < 0 ||
1424              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1425             return 0;
1426
1427         path = strjoin(prefix, "/", dirname, NULL);
1428         if (!path)
1429                 return -ENOMEM;
1430
1431         d = opendir(path);
1432         if (!d) {
1433                 log_debug("Failed to open %s: %m", path);
1434                 if (errno == ENOENT)
1435                         return 0;
1436                 return -errno;
1437         }
1438
1439         m = hashmap_get(j->directories_by_path, path);
1440         if (!m) {
1441                 m = new0(Directory, 1);
1442                 if (!m)
1443                         return -ENOMEM;
1444
1445                 m->is_root = false;
1446                 m->path = path;
1447
1448                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1449                         free(m);
1450                         return -ENOMEM;
1451                 }
1452
1453                 path = NULL; /* avoid freeing in cleanup */
1454                 j->current_invalidate_counter ++;
1455
1456                 log_debug("Directory %s added.", m->path);
1457
1458         } else if (m->is_root)
1459                 return 0;
1460
1461         if (m->wd <= 0 && j->inotify_fd >= 0) {
1462
1463                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1464                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1465                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1466                                           IN_ONLYDIR);
1467
1468                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1469                         inotify_rm_watch(j->inotify_fd, m->wd);
1470         }
1471
1472         for (;;) {
1473                 struct dirent *de;
1474                 union dirent_storage buf;
1475
1476                 r = readdir_r(d, &buf.de, &de);
1477                 if (r != 0 || !de)
1478                         break;
1479
1480                 if (dirent_is_file_with_suffix(de, ".journal") ||
1481                     dirent_is_file_with_suffix(de, ".journal~")) {
1482                         r = add_file(j, m->path, de->d_name);
1483                         if (r < 0) {
1484                                 log_debug("Failed to add file %s/%s: %s",
1485                                           m->path, de->d_name, strerror(-r));
1486                                 r = set_put_error(j, r);
1487                                 if (r < 0)
1488                                         return r;
1489                         }
1490                 }
1491         }
1492
1493         check_network(j, dirfd(d));
1494
1495         return 0;
1496 }
1497
1498 static int add_root_directory(sd_journal *j, const char *p) {
1499         _cleanup_closedir_ DIR *d = NULL;
1500         Directory *m;
1501         int r;
1502
1503         assert(j);
1504         assert(p);
1505
1506         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1507             !path_startswith(p, "/run"))
1508                 return -EINVAL;
1509
1510         d = opendir(p);
1511         if (!d)
1512                 return -errno;
1513
1514         m = hashmap_get(j->directories_by_path, p);
1515         if (!m) {
1516                 m = new0(Directory, 1);
1517                 if (!m)
1518                         return -ENOMEM;
1519
1520                 m->is_root = true;
1521                 m->path = strdup(p);
1522                 if (!m->path) {
1523                         free(m);
1524                         return -ENOMEM;
1525                 }
1526
1527                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1528                         free(m->path);
1529                         free(m);
1530                         return -ENOMEM;
1531                 }
1532
1533                 j->current_invalidate_counter ++;
1534
1535                 log_debug("Root directory %s added.", m->path);
1536
1537         } else if (!m->is_root)
1538                 return 0;
1539
1540         if (m->wd <= 0 && j->inotify_fd >= 0) {
1541
1542                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1543                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1544                                           IN_ONLYDIR);
1545
1546                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1547                         inotify_rm_watch(j->inotify_fd, m->wd);
1548         }
1549
1550         if (j->no_new_files)
1551                 return 0;
1552
1553         for (;;) {
1554                 struct dirent *de;
1555                 union dirent_storage buf;
1556                 sd_id128_t id;
1557
1558                 r = readdir_r(d, &buf.de, &de);
1559                 if (r != 0 || !de)
1560                         break;
1561
1562                 if (dirent_is_file_with_suffix(de, ".journal") ||
1563                     dirent_is_file_with_suffix(de, ".journal~")) {
1564                         r = add_file(j, m->path, de->d_name);
1565                         if (r < 0) {
1566                                 log_debug("Failed to add file %s/%s: %s",
1567                                           m->path, de->d_name, strerror(-r));
1568                                 r = set_put_error(j, r);
1569                                 if (r < 0)
1570                                         return r;
1571                         }
1572                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1573                            sd_id128_from_string(de->d_name, &id) >= 0) {
1574
1575                         r = add_directory(j, m->path, de->d_name);
1576                         if (r < 0)
1577                                 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1578                 }
1579         }
1580
1581         check_network(j, dirfd(d));
1582
1583         return 0;
1584 }
1585
1586 static int remove_directory(sd_journal *j, Directory *d) {
1587         assert(j);
1588
1589         if (d->wd > 0) {
1590                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1591
1592                 if (j->inotify_fd >= 0)
1593                         inotify_rm_watch(j->inotify_fd, d->wd);
1594         }
1595
1596         hashmap_remove(j->directories_by_path, d->path);
1597
1598         if (d->is_root)
1599                 log_debug("Root directory %s removed.", d->path);
1600         else
1601                 log_debug("Directory %s removed.", d->path);
1602
1603         free(d->path);
1604         free(d);
1605
1606         return 0;
1607 }
1608
1609 static int add_search_paths(sd_journal *j) {
1610         int r;
1611         const char search_paths[] =
1612                 "/run/log/journal\0"
1613                 "/var/log/journal\0";
1614         const char *p;
1615
1616         assert(j);
1617
1618         /* We ignore most errors here, since the idea is to only open
1619          * what's actually accessible, and ignore the rest. */
1620
1621         NULSTR_FOREACH(p, search_paths) {
1622                 r = add_root_directory(j, p);
1623                 if (r < 0 && r != -ENOENT) {
1624                         r = set_put_error(j, r);
1625                         if (r < 0)
1626                                 return r;
1627                 }
1628         }
1629
1630         return 0;
1631 }
1632
1633 static int add_current_paths(sd_journal *j) {
1634         Iterator i;
1635         JournalFile *f;
1636
1637         assert(j);
1638         assert(j->no_new_files);
1639
1640         /* Simply adds all directories for files we have open as
1641          * "root" directories. We don't expect errors here, so we
1642          * treat them as fatal. */
1643
1644         HASHMAP_FOREACH(f, j->files, i) {
1645                 int r;
1646                 _cleanup_free_ char *dir;
1647
1648                 dir = dirname_malloc(f->path);
1649                 if (!dir)
1650                         return -ENOMEM;
1651
1652                 r = add_root_directory(j, dir);
1653                 if (r < 0) {
1654                         set_put_error(j, r);
1655                         return r;
1656                 }
1657         }
1658
1659         return 0;
1660 }
1661
1662
1663 static int allocate_inotify(sd_journal *j) {
1664         assert(j);
1665
1666         if (j->inotify_fd < 0) {
1667                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1668                 if (j->inotify_fd < 0)
1669                         return -errno;
1670         }
1671
1672         if (!j->directories_by_wd) {
1673                 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1674                 if (!j->directories_by_wd)
1675                         return -ENOMEM;
1676         }
1677
1678         return 0;
1679 }
1680
1681 static sd_journal *journal_new(int flags, const char *path) {
1682         sd_journal *j;
1683
1684         j = new0(sd_journal, 1);
1685         if (!j)
1686                 return NULL;
1687
1688         j->original_pid = getpid();
1689         j->inotify_fd = -1;
1690         j->flags = flags;
1691         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1692
1693         if (path) {
1694                 j->path = strdup(path);
1695                 if (!j->path)
1696                         goto fail;
1697         }
1698
1699         j->files = hashmap_new(string_hash_func, string_compare_func);
1700         j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1701         j->mmap = mmap_cache_new();
1702         if (!j->files || !j->directories_by_path || !j->mmap)
1703                 goto fail;
1704
1705         return j;
1706
1707 fail:
1708         sd_journal_close(j);
1709         return NULL;
1710 }
1711
1712 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1713         sd_journal *j;
1714         int r;
1715
1716         if (!ret)
1717                 return -EINVAL;
1718
1719         if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1720                       SD_JOURNAL_RUNTIME_ONLY|
1721                       SD_JOURNAL_SYSTEM|
1722                       SD_JOURNAL_CURRENT_USER))
1723                 return -EINVAL;
1724
1725         j = journal_new(flags, NULL);
1726         if (!j)
1727                 return -ENOMEM;
1728
1729         r = add_search_paths(j);
1730         if (r < 0)
1731                 goto fail;
1732
1733         *ret = j;
1734         return 0;
1735
1736 fail:
1737         sd_journal_close(j);
1738
1739         return r;
1740 }
1741
1742 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1743         sd_journal *j;
1744         int r;
1745
1746         if (!ret)
1747                 return -EINVAL;
1748
1749         if (!path)
1750                 return -EINVAL;
1751
1752         if (flags != 0)
1753                 return -EINVAL;
1754
1755         j = journal_new(flags, path);
1756         if (!j)
1757                 return -ENOMEM;
1758
1759         r = add_root_directory(j, path);
1760         if (r < 0) {
1761                 set_put_error(j, r);
1762                 goto fail;
1763         }
1764
1765         *ret = j;
1766         return 0;
1767
1768 fail:
1769         sd_journal_close(j);
1770
1771         return r;
1772 }
1773
1774 _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int flags) {
1775         sd_journal *j;
1776         const char **path;
1777         int r;
1778
1779         if (!ret)
1780                 return -EINVAL;
1781
1782         if (flags != 0)
1783                 return -EINVAL;
1784
1785         j = journal_new(flags, NULL);
1786         if (!j)
1787                 return -ENOMEM;
1788
1789         STRV_FOREACH(path, paths) {
1790                 r = add_any_file(j, *path);
1791                 if (r < 0) {
1792                         log_error("Failed to open %s: %s", *path, strerror(-r));
1793                         goto fail;
1794                 }
1795         }
1796
1797         j->no_new_files = true;
1798
1799         *ret = j;
1800         return 0;
1801
1802 fail:
1803         sd_journal_close(j);
1804
1805         return r;
1806 }
1807
1808 _public_ void sd_journal_close(sd_journal *j) {
1809         Directory *d;
1810         JournalFile *f;
1811
1812         if (!j)
1813                 return;
1814
1815         sd_journal_flush_matches(j);
1816
1817         while ((f = hashmap_steal_first(j->files)))
1818                 journal_file_close(f);
1819
1820         hashmap_free(j->files);
1821
1822         while ((d = hashmap_first(j->directories_by_path)))
1823                 remove_directory(j, d);
1824
1825         while ((d = hashmap_first(j->directories_by_wd)))
1826                 remove_directory(j, d);
1827
1828         hashmap_free(j->directories_by_path);
1829         hashmap_free(j->directories_by_wd);
1830
1831         if (j->inotify_fd >= 0)
1832                 close_nointr_nofail(j->inotify_fd);
1833
1834         if (j->mmap) {
1835                 log_debug("mmap cache statistics: %u hit, %u miss", mmap_cache_get_hit(j->mmap), mmap_cache_get_missed(j->mmap));
1836                 mmap_cache_unref(j->mmap);
1837         }
1838
1839         free(j->path);
1840         free(j->unique_field);
1841         set_free(j->errors);
1842         free(j);
1843 }
1844
1845 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1846         Object *o;
1847         JournalFile *f;
1848         int r;
1849
1850         if (!j)
1851                 return -EINVAL;
1852         if (journal_pid_changed(j))
1853                 return -ECHILD;
1854         if (!ret)
1855                 return -EINVAL;
1856
1857         f = j->current_file;
1858         if (!f)
1859                 return -EADDRNOTAVAIL;
1860
1861         if (f->current_offset <= 0)
1862                 return -EADDRNOTAVAIL;
1863
1864         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1865         if (r < 0)
1866                 return r;
1867
1868         *ret = le64toh(o->entry.realtime);
1869         return 0;
1870 }
1871
1872 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1873         Object *o;
1874         JournalFile *f;
1875         int r;
1876         sd_id128_t id;
1877
1878         if (!j)
1879                 return -EINVAL;
1880         if (journal_pid_changed(j))
1881                 return -ECHILD;
1882
1883         f = j->current_file;
1884         if (!f)
1885                 return -EADDRNOTAVAIL;
1886
1887         if (f->current_offset <= 0)
1888                 return -EADDRNOTAVAIL;
1889
1890         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1891         if (r < 0)
1892                 return r;
1893
1894         if (ret_boot_id)
1895                 *ret_boot_id = o->entry.boot_id;
1896         else {
1897                 r = sd_id128_get_boot(&id);
1898                 if (r < 0)
1899                         return r;
1900
1901                 if (!sd_id128_equal(id, o->entry.boot_id))
1902                         return -ESTALE;
1903         }
1904
1905         if (ret)
1906                 *ret = le64toh(o->entry.monotonic);
1907
1908         return 0;
1909 }
1910
1911 static bool field_is_valid(const char *field) {
1912         const char *p;
1913
1914         assert(field);
1915
1916         if (isempty(field))
1917                 return false;
1918
1919         if (startswith(field, "__"))
1920                 return false;
1921
1922         for (p = field; *p; p++) {
1923
1924                 if (*p == '_')
1925                         continue;
1926
1927                 if (*p >= 'A' && *p <= 'Z')
1928                         continue;
1929
1930                 if (*p >= '0' && *p <= '9')
1931                         continue;
1932
1933                 return false;
1934         }
1935
1936         return true;
1937 }
1938
1939 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1940         JournalFile *f;
1941         uint64_t i, n;
1942         size_t field_length;
1943         int r;
1944         Object *o;
1945
1946         if (!j)
1947                 return -EINVAL;
1948         if (journal_pid_changed(j))
1949                 return -ECHILD;
1950         if (!field)
1951                 return -EINVAL;
1952         if (!data)
1953                 return -EINVAL;
1954         if (!size)
1955                 return -EINVAL;
1956
1957         if (!field_is_valid(field))
1958                 return -EINVAL;
1959
1960         f = j->current_file;
1961         if (!f)
1962                 return -EADDRNOTAVAIL;
1963
1964         if (f->current_offset <= 0)
1965                 return -EADDRNOTAVAIL;
1966
1967         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1968         if (r < 0)
1969                 return r;
1970
1971         field_length = strlen(field);
1972
1973         n = journal_file_entry_n_items(o);
1974         for (i = 0; i < n; i++) {
1975                 uint64_t p, l;
1976                 le64_t le_hash;
1977                 size_t t;
1978
1979                 p = le64toh(o->entry.items[i].object_offset);
1980                 le_hash = o->entry.items[i].hash;
1981                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1982                 if (r < 0)
1983                         return r;
1984
1985                 if (le_hash != o->data.hash)
1986                         return -EBADMSG;
1987
1988                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1989
1990                 if (o->object.flags & OBJECT_COMPRESSED) {
1991
1992 #ifdef HAVE_XZ
1993                         if (uncompress_startswith(o->data.payload, l,
1994                                                   &f->compress_buffer, &f->compress_buffer_size,
1995                                                   field, field_length, '=')) {
1996
1997                                 uint64_t rsize;
1998
1999                                 if (!uncompress_blob(o->data.payload, l,
2000                                                      &f->compress_buffer, &f->compress_buffer_size, &rsize,
2001                                                      j->data_threshold))
2002                                         return -EBADMSG;
2003
2004                                 *data = f->compress_buffer;
2005                                 *size = (size_t) rsize;
2006
2007                                 return 0;
2008                         }
2009 #else
2010                         return -EPROTONOSUPPORT;
2011 #endif
2012
2013                 } else if (l >= field_length+1 &&
2014                            memcmp(o->data.payload, field, field_length) == 0 &&
2015                            o->data.payload[field_length] == '=') {
2016
2017                         t = (size_t) l;
2018
2019                         if ((uint64_t) t != l)
2020                                 return -E2BIG;
2021
2022                         *data = o->data.payload;
2023                         *size = t;
2024
2025                         return 0;
2026                 }
2027
2028                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2029                 if (r < 0)
2030                         return r;
2031         }
2032
2033         return -ENOENT;
2034 }
2035
2036 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
2037         size_t t;
2038         uint64_t l;
2039
2040         l = le64toh(o->object.size) - offsetof(Object, data.payload);
2041         t = (size_t) l;
2042
2043         /* We can't read objects larger than 4G on a 32bit machine */
2044         if ((uint64_t) t != l)
2045                 return -E2BIG;
2046
2047         if (o->object.flags & OBJECT_COMPRESSED) {
2048 #ifdef HAVE_XZ
2049                 uint64_t rsize;
2050
2051                 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
2052                         return -EBADMSG;
2053
2054                 *data = f->compress_buffer;
2055                 *size = (size_t) rsize;
2056 #else
2057                 return -EPROTONOSUPPORT;
2058 #endif
2059         } else {
2060                 *data = o->data.payload;
2061                 *size = t;
2062         }
2063
2064         return 0;
2065 }
2066
2067 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
2068         JournalFile *f;
2069         uint64_t p, n;
2070         le64_t le_hash;
2071         int r;
2072         Object *o;
2073
2074         if (!j)
2075                 return -EINVAL;
2076         if (journal_pid_changed(j))
2077                 return -ECHILD;
2078         if (!data)
2079                 return -EINVAL;
2080         if (!size)
2081                 return -EINVAL;
2082
2083         f = j->current_file;
2084         if (!f)
2085                 return -EADDRNOTAVAIL;
2086
2087         if (f->current_offset <= 0)
2088                 return -EADDRNOTAVAIL;
2089
2090         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2091         if (r < 0)
2092                 return r;
2093
2094         n = journal_file_entry_n_items(o);
2095         if (j->current_field >= n)
2096                 return 0;
2097
2098         p = le64toh(o->entry.items[j->current_field].object_offset);
2099         le_hash = o->entry.items[j->current_field].hash;
2100         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2101         if (r < 0)
2102                 return r;
2103
2104         if (le_hash != o->data.hash)
2105                 return -EBADMSG;
2106
2107         r = return_data(j, f, o, data, size);
2108         if (r < 0)
2109                 return r;
2110
2111         j->current_field ++;
2112
2113         return 1;
2114 }
2115
2116 _public_ void sd_journal_restart_data(sd_journal *j) {
2117         if (!j)
2118                 return;
2119
2120         j->current_field = 0;
2121 }
2122
2123 _public_ int sd_journal_get_fd(sd_journal *j) {
2124         int r;
2125
2126         if (!j)
2127                 return -EINVAL;
2128         if (journal_pid_changed(j))
2129                 return -ECHILD;
2130
2131         if (j->inotify_fd >= 0)
2132                 return j->inotify_fd;
2133
2134         r = allocate_inotify(j);
2135         if (r < 0)
2136                 return r;
2137
2138         /* Iterate through all dirs again, to add them to the
2139          * inotify */
2140         if (j->no_new_files)
2141                 r = add_current_paths(j);
2142         else if (j->path)
2143                 r = add_root_directory(j, j->path);
2144         else
2145                 r = add_search_paths(j);
2146         if (r < 0)
2147                 return r;
2148
2149         return j->inotify_fd;
2150 }
2151
2152 _public_ int sd_journal_get_events(sd_journal *j) {
2153         int fd;
2154
2155         if (!j)
2156                 return -EINVAL;
2157         if (journal_pid_changed(j))
2158                 return -ECHILD;
2159
2160         fd = sd_journal_get_fd(j);
2161         if (fd < 0)
2162                 return fd;
2163
2164         return POLLIN;
2165 }
2166
2167 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2168         int fd;
2169
2170         if (!j)
2171                 return -EINVAL;
2172         if (journal_pid_changed(j))
2173                 return -ECHILD;
2174         if (!timeout_usec)
2175                 return -EINVAL;
2176
2177         fd = sd_journal_get_fd(j);
2178         if (fd < 0)
2179                 return fd;
2180
2181         if (!j->on_network) {
2182                 *timeout_usec = (uint64_t) -1;
2183                 return 0;
2184         }
2185
2186         /* If we are on the network we need to regularly check for
2187          * changes manually */
2188
2189         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2190         return 1;
2191 }
2192
2193 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2194         Directory *d;
2195         int r;
2196
2197         assert(j);
2198         assert(e);
2199
2200         /* Is this a subdirectory we watch? */
2201         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2202         if (d) {
2203                 sd_id128_t id;
2204
2205                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2206                     (endswith(e->name, ".journal") ||
2207                      endswith(e->name, ".journal~"))) {
2208
2209                         /* Event for a journal file */
2210
2211                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2212                                 r = add_file(j, d->path, e->name);
2213                                 if (r < 0) {
2214                                         log_debug("Failed to add file %s/%s: %s",
2215                                                   d->path, e->name, strerror(-r));
2216                                         set_put_error(j, r);
2217                                 }
2218
2219                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2220
2221                                 r = remove_file(j, d->path, e->name);
2222                                 if (r < 0)
2223                                         log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2224                         }
2225
2226                 } else if (!d->is_root && e->len == 0) {
2227
2228                         /* Event for a subdirectory */
2229
2230                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2231                                 r = remove_directory(j, d);
2232                                 if (r < 0)
2233                                         log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2234                         }
2235
2236
2237                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2238
2239                         /* Event for root directory */
2240
2241                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2242                                 r = add_directory(j, d->path, e->name);
2243                                 if (r < 0)
2244                                         log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2245                         }
2246                 }
2247
2248                 return;
2249         }
2250
2251         if (e->mask & IN_IGNORED)
2252                 return;
2253
2254         log_warning("Unknown inotify event.");
2255 }
2256
2257 static int determine_change(sd_journal *j) {
2258         bool b;
2259
2260         assert(j);
2261
2262         b = j->current_invalidate_counter != j->last_invalidate_counter;
2263         j->last_invalidate_counter = j->current_invalidate_counter;
2264
2265         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2266 }
2267
2268 _public_ int sd_journal_process(sd_journal *j) {
2269         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2270         bool got_something = false;
2271
2272         if (!j)
2273                 return -EINVAL;
2274         if (journal_pid_changed(j))
2275                 return -ECHILD;
2276
2277         j->last_process_usec = now(CLOCK_MONOTONIC);
2278
2279         for (;;) {
2280                 struct inotify_event *e;
2281                 ssize_t l;
2282
2283                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2284                 if (l < 0) {
2285                         if (errno == EAGAIN || errno == EINTR)
2286                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2287
2288                         return -errno;
2289                 }
2290
2291                 got_something = true;
2292
2293                 e = (struct inotify_event*) buffer;
2294                 while (l > 0) {
2295                         size_t step;
2296
2297                         process_inotify_event(j, e);
2298
2299                         step = sizeof(struct inotify_event) + e->len;
2300                         assert(step <= (size_t) l);
2301
2302                         e = (struct inotify_event*) ((uint8_t*) e + step);
2303                         l -= step;
2304                 }
2305         }
2306
2307         return determine_change(j);
2308 }
2309
2310 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2311         int r;
2312         uint64_t t;
2313
2314         if (!j)
2315                 return -EINVAL;
2316         if (journal_pid_changed(j))
2317                 return -ECHILD;
2318
2319         if (j->inotify_fd < 0) {
2320
2321                 /* This is the first invocation, hence create the
2322                  * inotify watch */
2323                 r = sd_journal_get_fd(j);
2324                 if (r < 0)
2325                         return r;
2326
2327                 /* The journal might have changed since the context
2328                  * object was created and we weren't watching before,
2329                  * hence don't wait for anything, and return
2330                  * immediately. */
2331                 return determine_change(j);
2332         }
2333
2334         r = sd_journal_get_timeout(j, &t);
2335         if (r < 0)
2336                 return r;
2337
2338         if (t != (uint64_t) -1) {
2339                 usec_t n;
2340
2341                 n = now(CLOCK_MONOTONIC);
2342                 t = t > n ? t - n : 0;
2343
2344                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2345                         timeout_usec = t;
2346         }
2347
2348         do {
2349                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2350         } while (r == -EINTR);
2351
2352         if (r < 0)
2353                 return r;
2354
2355         return sd_journal_process(j);
2356 }
2357
2358 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2359         Iterator i;
2360         JournalFile *f;
2361         bool first = true;
2362         int r;
2363
2364         if (!j)
2365                 return -EINVAL;
2366         if (journal_pid_changed(j))
2367                 return -ECHILD;
2368         if (!from && !to)
2369                 return -EINVAL;
2370         if (from == to)
2371                 return -EINVAL;
2372
2373         HASHMAP_FOREACH(f, j->files, i) {
2374                 usec_t fr, t;
2375
2376                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2377                 if (r == -ENOENT)
2378                         continue;
2379                 if (r < 0)
2380                         return r;
2381                 if (r == 0)
2382                         continue;
2383
2384                 if (first) {
2385                         if (from)
2386                                 *from = fr;
2387                         if (to)
2388                                 *to = t;
2389                         first = false;
2390                 } else {
2391                         if (from)
2392                                 *from = MIN(fr, *from);
2393                         if (to)
2394                                 *to = MAX(t, *to);
2395                 }
2396         }
2397
2398         return first ? 0 : 1;
2399 }
2400
2401 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2402         Iterator i;
2403         JournalFile *f;
2404         bool first = true;
2405         int r;
2406
2407         if (!j)
2408                 return -EINVAL;
2409         if (journal_pid_changed(j))
2410                 return -ECHILD;
2411         if (!from && !to)
2412                 return -EINVAL;
2413         if (from == to)
2414                 return -EINVAL;
2415
2416         HASHMAP_FOREACH(f, j->files, i) {
2417                 usec_t fr, t;
2418
2419                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2420                 if (r == -ENOENT)
2421                         continue;
2422                 if (r < 0)
2423                         return r;
2424                 if (r == 0)
2425                         continue;
2426
2427                 if (first) {
2428                         if (from)
2429                                 *from = fr;
2430                         if (to)
2431                                 *to = t;
2432                         first = false;
2433                 } else {
2434                         if (from)
2435                                 *from = MIN(fr, *from);
2436                         if (to)
2437                                 *to = MAX(t, *to);
2438                 }
2439         }
2440
2441         return first ? 0 : 1;
2442 }
2443
2444 void journal_print_header(sd_journal *j) {
2445         Iterator i;
2446         JournalFile *f;
2447         bool newline = false;
2448
2449         assert(j);
2450
2451         HASHMAP_FOREACH(f, j->files, i) {
2452                 if (newline)
2453                         putchar('\n');
2454                 else
2455                         newline = true;
2456
2457                 journal_file_print_header(f);
2458         }
2459 }
2460
2461 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2462         Iterator i;
2463         JournalFile *f;
2464         uint64_t sum = 0;
2465
2466         if (!j)
2467                 return -EINVAL;
2468         if (journal_pid_changed(j))
2469                 return -ECHILD;
2470         if (!bytes)
2471                 return -EINVAL;
2472
2473         HASHMAP_FOREACH(f, j->files, i) {
2474                 struct stat st;
2475
2476                 if (fstat(f->fd, &st) < 0)
2477                         return -errno;
2478
2479                 sum += (uint64_t) st.st_blocks * 512ULL;
2480         }
2481
2482         *bytes = sum;
2483         return 0;
2484 }
2485
2486 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2487         char *f;
2488
2489         if (!j)
2490                 return -EINVAL;
2491         if (journal_pid_changed(j))
2492                 return -ECHILD;
2493         if (isempty(field))
2494                 return -EINVAL;
2495         if (!field_is_valid(field))
2496                 return -EINVAL;
2497
2498         f = strdup(field);
2499         if (!f)
2500                 return -ENOMEM;
2501
2502         free(j->unique_field);
2503         j->unique_field = f;
2504         j->unique_file = NULL;
2505         j->unique_offset = 0;
2506
2507         return 0;
2508 }
2509
2510 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2511         Object *o;
2512         size_t k;
2513         int r;
2514
2515         if (!j)
2516                 return -EINVAL;
2517         if (journal_pid_changed(j))
2518                 return -ECHILD;
2519         if (!data)
2520                 return -EINVAL;
2521         if (!l)
2522                 return -EINVAL;
2523         if (!j->unique_field)
2524                 return -EINVAL;
2525
2526         k = strlen(j->unique_field);
2527
2528         if (!j->unique_file) {
2529                 j->unique_file = hashmap_first(j->files);
2530                 if (!j->unique_file)
2531                         return 0;
2532                 j->unique_offset = 0;
2533         }
2534
2535         for (;;) {
2536                 JournalFile *of;
2537                 Iterator i;
2538                 const void *odata;
2539                 size_t ol;
2540                 bool found;
2541
2542                 /* Proceed to next data object in the field's linked list */
2543                 if (j->unique_offset == 0) {
2544                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2545                         if (r < 0)
2546                                 return r;
2547
2548                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2549                 } else {
2550                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2551                         if (r < 0)
2552                                 return r;
2553
2554                         j->unique_offset = le64toh(o->data.next_field_offset);
2555                 }
2556
2557                 /* We reached the end of the list? Then start again, with the next file */
2558                 if (j->unique_offset == 0) {
2559                         JournalFile *n;
2560
2561                         n = hashmap_next(j->files, j->unique_file->path);
2562                         if (!n)
2563                                 return 0;
2564
2565                         j->unique_file = n;
2566                         continue;
2567                 }
2568
2569                 /* We do not use the type context here, but 0 instead,
2570                  * so that we can look at this data object at the same
2571                  * time as one on another file */
2572                 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2573                 if (r < 0)
2574                         return r;
2575
2576                 /* Let's do the type check by hand, since we used 0 context above. */
2577                 if (o->object.type != OBJECT_DATA)
2578                         return -EBADMSG;
2579
2580                 r = return_data(j, j->unique_file, o, &odata, &ol);
2581                 if (r < 0)
2582                         return r;
2583
2584                 /* OK, now let's see if we already returned this data
2585                  * object by checking if it exists in the earlier
2586                  * traversed files. */
2587                 found = false;
2588                 HASHMAP_FOREACH(of, j->files, i) {
2589                         Object *oo;
2590                         uint64_t op;
2591
2592                         if (of == j->unique_file)
2593                                 break;
2594
2595                         /* Skip this file it didn't have any fields
2596                          * indexed */
2597                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2598                             le64toh(of->header->n_fields) <= 0)
2599                                 continue;
2600
2601                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2602                         if (r < 0)
2603                                 return r;
2604
2605                         if (r > 0)
2606                                 found = true;
2607                 }
2608
2609                 if (found)
2610                         continue;
2611
2612                 r = return_data(j, j->unique_file, o, data, l);
2613                 if (r < 0)
2614                         return r;
2615
2616                 return 1;
2617         }
2618 }
2619
2620 _public_ void sd_journal_restart_unique(sd_journal *j) {
2621         if (!j)
2622                 return;
2623
2624         j->unique_file = NULL;
2625         j->unique_offset = 0;
2626 }
2627
2628 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2629         if (!j)
2630                 return -EINVAL;
2631         if (journal_pid_changed(j))
2632                 return -ECHILD;
2633
2634         return !j->on_network;
2635 }
2636
2637 static char *lookup_field(const char *field, void *userdata) {
2638         sd_journal *j = userdata;
2639         const void *data;
2640         size_t size, d;
2641         int r;
2642
2643         assert(field);
2644         assert(j);
2645
2646         r = sd_journal_get_data(j, field, &data, &size);
2647         if (r < 0 ||
2648             size > REPLACE_VAR_MAX)
2649                 return strdup(field);
2650
2651         d = strlen(field) + 1;
2652
2653         return strndup((const char*) data + d, size - d);
2654 }
2655
2656 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2657         const void *data;
2658         size_t size;
2659         sd_id128_t id;
2660         _cleanup_free_ char *text = NULL, *cid = NULL;
2661         char *t;
2662         int r;
2663
2664         if (!j)
2665                 return -EINVAL;
2666         if (journal_pid_changed(j))
2667                 return -ECHILD;
2668         if (!ret)
2669                 return -EINVAL;
2670
2671         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2672         if (r < 0)
2673                 return r;
2674
2675         cid = strndup((const char*) data + 11, size - 11);
2676         if (!cid)
2677                 return -ENOMEM;
2678
2679         r = sd_id128_from_string(cid, &id);
2680         if (r < 0)
2681                 return r;
2682
2683         r = catalog_get(CATALOG_DATABASE, id, &text);
2684         if (r < 0)
2685                 return r;
2686
2687         t = replace_var(text, lookup_field, j);
2688         if (!t)
2689                 return -ENOMEM;
2690
2691         *ret = t;
2692         return 0;
2693 }
2694
2695 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2696         if (!ret)
2697                 return -EINVAL;
2698
2699         return catalog_get(CATALOG_DATABASE, id, ret);
2700 }
2701
2702 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2703         if (!j)
2704                 return -EINVAL;
2705         if (journal_pid_changed(j))
2706                 return -ECHILD;
2707
2708         j->data_threshold = sz;
2709         return 0;
2710 }
2711
2712 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2713         if (!j)
2714                 return -EINVAL;
2715         if (journal_pid_changed(j))
2716                 return -ECHILD;
2717         if (!sz)
2718                 return -EINVAL;
2719
2720         *sz = j->data_threshold;
2721         return 0;
2722 }