src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "strv.h"
  37 #include "path-util.h"
  38 #include "lookup3.h"
  39 #include "compress.h"
  40 #include "journal-internal.h"
  41 #include "missing.h"
  42 #include "catalog.h"
  43 #include "replace-var.h"
  44
  45 #define JOURNAL_FILES_MAX 1024
  46
  47 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  48
  49 #define REPLACE_VAR_MAX 256
  50
  51 #define DEFAULT_DATA_THRESHOLD (64*1024)
  52
  53 static bool journal_pid_changed(sd_journal *j) {
  54         assert(j);
  55
  56         /* We don't support people creating a journal object and
  57          * keeping it around over a fork(). Let's complain. */
  58
  59         return j->original_pid != getpid();
  60 }
  61
  62 /* We return an error here only if we didn't manage to
  63    memorize the real error. */
  64 static int set_put_error(sd_journal *j, int r) {
  65         int k;
  66
  67         if (r >= 0)
  68                 return r;
  69
  70         k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
  71         if (k < 0)
  72                 return k;
  73
  74         return set_put(j->errors, INT_TO_PTR(r));
  75 }
  76
  77 static void detach_location(sd_journal *j) {
  78         Iterator i;
  79         JournalFile *f;
  80
  81         assert(j);
  82
  83         j->current_file = NULL;
  84         j->current_field = 0;
  85
  86         HASHMAP_FOREACH(f, j->files, i)
  87                 f->current_offset = 0;
  88 }
  89
  90 static void reset_location(sd_journal *j) {
  91         assert(j);
  92
  93         detach_location(j);
  94         zero(j->current_location);
  95 }
  96
  97 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  98         assert(l);
  99         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 100         assert(f);
 101         assert(o->object.type == OBJECT_ENTRY);
 102
 103         l->type = type;
 104         l->seqnum = le64toh(o->entry.seqnum);
 105         l->seqnum_id = f->header->seqnum_id;
 106         l->realtime = le64toh(o->entry.realtime);
 107         l->monotonic = le64toh(o->entry.monotonic);
 108         l->boot_id = o->entry.boot_id;
 109         l->xor_hash = le64toh(o->entry.xor_hash);
 110
 111         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 112 }
 113
 114 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o,
 115                          direction_t direction, uint64_t offset) {
 116         assert(j);
 117         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 118         assert(f);
 119         assert(o);
 120
 121         init_location(&j->current_location, type, f, o);
 122
 123         j->current_file = f;
 124         j->current_field = 0;
 125
 126         f->last_direction = direction;
 127         f->current_offset = offset;
 128 }
 129
 130 static int match_is_valid(const void *data, size_t size) {
 131         const char *b, *p;
 132
 133         assert(data);
 134
 135         if (size < 2)
 136                 return false;
 137
 138         if (startswith(data, "__"))
 139                 return false;
 140
 141         b = data;
 142         for (p = b; p < b + size; p++) {
 143
 144                 if (*p == '=')
 145                         return p > b;
 146
 147                 if (*p == '_')
 148                         continue;
 149
 150                 if (*p >= 'A' && *p <= 'Z')
 151                         continue;
 152
 153                 if (*p >= '0' && *p <= '9')
 154                         continue;
 155
 156                 return false;
 157         }
 158
 159         return false;
 160 }
 161
 162 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 163         const uint8_t *a = _a, *b = _b;
 164         size_t j;
 165
 166         for (j = 0; j < s && j < t; j++) {
 167
 168                 if (a[j] != b[j])
 169                         return false;
 170
 171                 if (a[j] == '=')
 172                         return true;
 173         }
 174
 175         assert_not_reached("\"=\" not found");
 176 }
 177
 178 static Match *match_new(Match *p, MatchType t) {
 179         Match *m;
 180
 181         m = new0(Match, 1);
 182         if (!m)
 183                 return NULL;
 184
 185         m->type = t;
 186
 187         if (p) {
 188                 m->parent = p;
 189                 LIST_PREPEND(matches, p->matches, m);
 190         }
 191
 192         return m;
 193 }
 194
 195 static void match_free(Match *m) {
 196         assert(m);
 197
 198         while (m->matches)
 199                 match_free(m->matches);
 200
 201         if (m->parent)
 202                 LIST_REMOVE(matches, m->parent->matches, m);
 203
 204         free(m->data);
 205         free(m);
 206 }
 207
 208 static void match_free_if_empty(Match *m) {
 209         if (!m || m->matches)
 210                 return;
 211
 212         match_free(m);
 213 }
 214
 215 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 216         Match *l3, *l4, *add_here = NULL, *m;
 217         le64_t le_hash;
 218
 219         assert_return(j, -EINVAL);
 220         assert_return(!journal_pid_changed(j), -ECHILD);
 221         assert_return(data, -EINVAL);
 222
 223         if (size == 0)
 224                 size = strlen(data);
 225
 226         assert_return(match_is_valid(data, size), -EINVAL);
 227
 228         /* level 0: AND term
 229          * level 1: OR terms
 230          * level 2: AND terms
 231          * level 3: OR terms
 232          * level 4: concrete matches */
 233
 234         if (!j->level0) {
 235                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 236                 if (!j->level0)
 237                         return -ENOMEM;
 238         }
 239
 240         if (!j->level1) {
 241                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 242                 if (!j->level1)
 243                         return -ENOMEM;
 244         }
 245
 246         if (!j->level2) {
 247                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 248                 if (!j->level2)
 249                         return -ENOMEM;
 250         }
 251
 252         assert(j->level0->type == MATCH_AND_TERM);
 253         assert(j->level1->type == MATCH_OR_TERM);
 254         assert(j->level2->type == MATCH_AND_TERM);
 255
 256         le_hash = htole64(hash64(data, size));
 257
 258         LIST_FOREACH(matches, l3, j->level2->matches) {
 259                 assert(l3->type == MATCH_OR_TERM);
 260
 261                 LIST_FOREACH(matches, l4, l3->matches) {
 262                         assert(l4->type == MATCH_DISCRETE);
 263
 264                         /* Exactly the same match already? Then ignore
 265                          * this addition */
 266                         if (l4->le_hash == le_hash &&
 267                             l4->size == size &&
 268                             memcmp(l4->data, data, size) == 0)
 269                                 return 0;
 270
 271                         /* Same field? Then let's add this to this OR term */
 272                         if (same_field(data, size, l4->data, l4->size)) {
 273                                 add_here = l3;
 274                                 break;
 275                         }
 276                 }
 277
 278                 if (add_here)
 279                         break;
 280         }
 281
 282         if (!add_here) {
 283                 add_here = match_new(j->level2, MATCH_OR_TERM);
 284                 if (!add_here)
 285                         goto fail;
 286         }
 287
 288         m = match_new(add_here, MATCH_DISCRETE);
 289         if (!m)
 290                 goto fail;
 291
 292         m->le_hash = le_hash;
 293         m->size = size;
 294         m->data = memdup(data, size);
 295         if (!m->data)
 296                 goto fail;
 297
 298         detach_location(j);
 299
 300         return 0;
 301
 302 fail:
 303         match_free_if_empty(add_here);
 304         match_free_if_empty(j->level2);
 305         match_free_if_empty(j->level1);
 306         match_free_if_empty(j->level0);
 307
 308         return -ENOMEM;
 309 }
 310
 311 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 312         assert_return(j, -EINVAL);
 313         assert_return(!journal_pid_changed(j), -ECHILD);
 314
 315         if (!j->level0)
 316                 return 0;
 317
 318         if (!j->level1)
 319                 return 0;
 320
 321         if (!j->level1->matches)
 322                 return 0;
 323
 324         j->level1 = NULL;
 325         j->level2 = NULL;
 326
 327         return 0;
 328 }
 329
 330 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 331         assert_return(j, -EINVAL);
 332         assert_return(!journal_pid_changed(j), -ECHILD);
 333
 334         if (!j->level0)
 335                 return 0;
 336
 337         if (!j->level1)
 338                 return 0;
 339
 340         if (!j->level2)
 341                 return 0;
 342
 343         if (!j->level2->matches)
 344                 return 0;
 345
 346         j->level2 = NULL;
 347         return 0;
 348 }
 349
 350 static char *match_make_string(Match *m) {
 351         char *p, *r;
 352         Match *i;
 353         bool enclose = false;
 354
 355         if (!m)
 356                 return strdup("none");
 357
 358         if (m->type == MATCH_DISCRETE)
 359                 return strndup(m->data, m->size);
 360
 361         p = NULL;
 362         LIST_FOREACH(matches, i, m->matches) {
 363                 char *t, *k;
 364
 365                 t = match_make_string(i);
 366                 if (!t) {
 367                         free(p);
 368                         return NULL;
 369                 }
 370
 371                 if (p) {
 372                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 373                         free(p);
 374                         free(t);
 375
 376                         if (!k)
 377                                 return NULL;
 378
 379                         p = k;
 380
 381                         enclose = true;
 382                 } else
 383                         p = t;
 384         }
 385
 386         if (enclose) {
 387                 r = strjoin("(", p, ")", NULL);
 388                 free(p);
 389                 return r;
 390         }
 391
 392         return p;
 393 }
 394
 395 char *journal_make_match_string(sd_journal *j) {
 396         assert(j);
 397
 398         return match_make_string(j->level0);
 399 }
 400
 401 _public_ void sd_journal_flush_matches(sd_journal *j) {
 402         if (!j)
 403                 return;
 404
 405         if (j->level0)
 406                 match_free(j->level0);
 407
 408         j->level0 = j->level1 = j->level2 = NULL;
 409
 410         detach_location(j);
 411 }
 412
 413 static int compare_entry_order(JournalFile *af, Object *_ao,
 414                                JournalFile *bf, uint64_t bp) {
 415
 416         uint64_t a, b;
 417         Object *ao, *bo;
 418         int r;
 419
 420         assert(af);
 421         assert(bf);
 422         assert(_ao);
 423
 424         /* The mmap cache might invalidate the object from the first
 425          * file if we look at the one from the second file. Hence
 426          * temporarily copy the header of the first one, and look at
 427          * that only. */
 428         ao = alloca(offsetof(EntryObject, items));
 429         memcpy(ao, _ao, offsetof(EntryObject, items));
 430
 431         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 432         if (r < 0)
 433                 return strcmp(af->path, bf->path);
 434
 435         /* We operate on two different files here, hence we can access
 436          * two objects at the same time, which we normally can't.
 437          *
 438          * If contents and timestamps match, these entries are
 439          * identical, even if the seqnum does not match */
 440
 441         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 442             ao->entry.monotonic == bo->entry.monotonic &&
 443             ao->entry.realtime == bo->entry.realtime &&
 444             ao->entry.xor_hash == bo->entry.xor_hash)
 445                 return 0;
 446
 447         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 448
 449                 /* If this is from the same seqnum source, compare
 450                  * seqnums */
 451                 a = le64toh(ao->entry.seqnum);
 452                 b = le64toh(bo->entry.seqnum);
 453
 454                 if (a < b)
 455                         return -1;
 456                 if (a > b)
 457                         return 1;
 458
 459                 /* Wow! This is weird, different data but the same
 460                  * seqnums? Something is borked, but let's make the
 461                  * best of it and compare by time. */
 462         }
 463
 464         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 465
 466                 /* If the boot id matches compare monotonic time */
 467                 a = le64toh(ao->entry.monotonic);
 468                 b = le64toh(bo->entry.monotonic);
 469
 470                 if (a < b)
 471                         return -1;
 472                 if (a > b)
 473                         return 1;
 474         }
 475
 476         /* Otherwise compare UTC time */
 477         a = le64toh(ao->entry.realtime);
 478         b = le64toh(bo->entry.realtime);
 479
 480         if (a < b)
 481                 return -1;
 482         if (a > b)
 483                 return 1;
 484
 485         /* Finally, compare by contents */
 486         a = le64toh(ao->entry.xor_hash);
 487         b = le64toh(bo->entry.xor_hash);
 488
 489         if (a < b)
 490                 return -1;
 491         if (a > b)
 492                 return 1;
 493
 494         return 0;
 495 }
 496
 497 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 498         uint64_t a;
 499
 500         assert(af);
 501         assert(ao);
 502         assert(l);
 503         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 504
 505         if (l->monotonic_set &&
 506             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 507             l->realtime_set &&
 508             le64toh(ao->entry.realtime) == l->realtime &&
 509             l->xor_hash_set &&
 510             le64toh(ao->entry.xor_hash) == l->xor_hash)
 511                 return 0;
 512
 513         if (l->seqnum_set &&
 514             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 515
 516                 a = le64toh(ao->entry.seqnum);
 517
 518                 if (a < l->seqnum)
 519                         return -1;
 520                 if (a > l->seqnum)
 521                         return 1;
 522         }
 523
 524         if (l->monotonic_set &&
 525             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 526
 527                 a = le64toh(ao->entry.monotonic);
 528
 529                 if (a < l->monotonic)
 530                         return -1;
 531                 if (a > l->monotonic)
 532                         return 1;
 533         }
 534
 535         if (l->realtime_set) {
 536
 537                 a = le64toh(ao->entry.realtime);
 538
 539                 if (a < l->realtime)
 540                         return -1;
 541                 if (a > l->realtime)
 542                         return 1;
 543         }
 544
 545         if (l->xor_hash_set) {
 546                 a = le64toh(ao->entry.xor_hash);
 547
 548                 if (a < l->xor_hash)
 549                         return -1;
 550                 if (a > l->xor_hash)
 551                         return 1;
 552         }
 553
 554         return 0;
 555 }
 556
 557 static int next_for_match(
 558                 sd_journal *j,
 559                 Match *m,
 560                 JournalFile *f,
 561                 uint64_t after_offset,
 562                 direction_t direction,
 563                 Object **ret,
 564                 uint64_t *offset) {
 565
 566         int r;
 567         uint64_t np = 0;
 568         Object *n;
 569
 570         assert(j);
 571         assert(m);
 572         assert(f);
 573
 574         if (m->type == MATCH_DISCRETE) {
 575                 uint64_t dp;
 576
 577                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 578                 if (r <= 0)
 579                         return r;
 580
 581                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 582
 583         } else if (m->type == MATCH_OR_TERM) {
 584                 Match *i;
 585
 586                 /* Find the earliest match beyond after_offset */
 587
 588                 LIST_FOREACH(matches, i, m->matches) {
 589                         uint64_t cp;
 590
 591                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 592                         if (r < 0)
 593                                 return r;
 594                         else if (r > 0) {
 595                                 if (np == 0 || (direction == DIRECTION_DOWN ? cp < np : cp > np))
 596                                         np = cp;
 597                         }
 598                 }
 599
 600                 if (np == 0)
 601                         return 0;
 602
 603         } else if (m->type == MATCH_AND_TERM) {
 604                 Match *i, *last_moved;
 605
 606                 /* Always jump to the next matching entry and repeat
 607                  * this until we find an offset that matches for all
 608                  * matches. */
 609
 610                 if (!m->matches)
 611                         return 0;
 612
 613                 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
 614                 if (r <= 0)
 615                         return r;
 616
 617                 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
 618                 last_moved = m->matches;
 619
 620                 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
 621                         uint64_t cp;
 622
 623                         r = next_for_match(j, i, f, np, direction, NULL, &cp);
 624                         if (r <= 0)
 625                                 return r;
 626
 627                         assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
 628                         if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
 629                                 np = cp;
 630                                 last_moved = i;
 631                         }
 632                 }
 633         }
 634
 635         assert(np > 0);
 636
 637         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 638         if (r < 0)
 639                 return r;
 640
 641         if (ret)
 642                 *ret = n;
 643         if (offset)
 644                 *offset = np;
 645
 646         return 1;
 647 }
 648
 649 static int find_location_for_match(
 650                 sd_journal *j,
 651                 Match *m,
 652                 JournalFile *f,
 653                 direction_t direction,
 654                 Object **ret,
 655                 uint64_t *offset) {
 656
 657         int r;
 658
 659         assert(j);
 660         assert(m);
 661         assert(f);
 662
 663         if (m->type == MATCH_DISCRETE) {
 664                 uint64_t dp;
 665
 666                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 667                 if (r <= 0)
 668                         return r;
 669
 670                 /* FIXME: missing: find by monotonic */
 671
 672                 if (j->current_location.type == LOCATION_HEAD)
 673                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 674                 if (j->current_location.type == LOCATION_TAIL)
 675                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 676                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 677                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 678                 if (j->current_location.monotonic_set) {
 679                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 680                         if (r != -ENOENT)
 681                                 return r;
 682                 }
 683                 if (j->current_location.realtime_set)
 684                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 685
 686                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 687
 688         } else if (m->type == MATCH_OR_TERM) {
 689                 uint64_t np = 0;
 690                 Object *n;
 691                 Match *i;
 692
 693                 /* Find the earliest match */
 694
 695                 LIST_FOREACH(matches, i, m->matches) {
 696                         uint64_t cp;
 697
 698                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 699                         if (r < 0)
 700                                 return r;
 701                         else if (r > 0) {
 702                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 703                                         np = cp;
 704                         }
 705                 }
 706
 707                 if (np == 0)
 708                         return 0;
 709
 710                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 711                 if (r < 0)
 712                         return r;
 713
 714                 if (ret)
 715                         *ret = n;
 716                 if (offset)
 717                         *offset = np;
 718
 719                 return 1;
 720
 721         } else {
 722                 Match *i;
 723                 uint64_t np = 0;
 724
 725                 assert(m->type == MATCH_AND_TERM);
 726
 727                 /* First jump to the last match, and then find the
 728                  * next one where all matches match */
 729
 730                 if (!m->matches)
 731                         return 0;
 732
 733                 LIST_FOREACH(matches, i, m->matches) {
 734                         uint64_t cp;
 735
 736                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 737                         if (r <= 0)
 738                                 return r;
 739
 740                         if (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))
 741                                 np = cp;
 742                 }
 743
 744                 return next_for_match(j, m, f, np, direction, ret, offset);
 745         }
 746 }
 747
 748 static int find_location_with_matches(
 749                 sd_journal *j,
 750                 JournalFile *f,
 751                 direction_t direction,
 752                 Object **ret,
 753                 uint64_t *offset) {
 754
 755         int r;
 756
 757         assert(j);
 758         assert(f);
 759         assert(ret);
 760         assert(offset);
 761
 762         if (!j->level0) {
 763                 /* No matches is simple */
 764
 765                 if (j->current_location.type == LOCATION_HEAD)
 766                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 767                 if (j->current_location.type == LOCATION_TAIL)
 768                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 769                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 770                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 771                 if (j->current_location.monotonic_set) {
 772                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 773                         if (r != -ENOENT)
 774                                 return r;
 775                 }
 776                 if (j->current_location.realtime_set)
 777                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 778
 779                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 780         } else
 781                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 782 }
 783
 784 static int next_with_matches(
 785                 sd_journal *j,
 786                 JournalFile *f,
 787                 direction_t direction,
 788                 Object **ret,
 789                 uint64_t *offset) {
 790
 791         Object *c;
 792         uint64_t cp;
 793
 794         assert(j);
 795         assert(f);
 796         assert(ret);
 797         assert(offset);
 798
 799         c = *ret;
 800         cp = *offset;
 801
 802         /* No matches is easy. We simple advance the file
 803          * pointer by one. */
 804         if (!j->level0)
 805                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 806
 807         /* If we have a match then we look for the next matching entry
 808          * with an offset at least one step larger */
 809         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 810 }
 811
 812 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 813         Object *c;
 814         uint64_t cp;
 815         int r;
 816
 817         assert(j);
 818         assert(f);
 819
 820         if (f->last_direction == direction && f->current_offset > 0) {
 821                 cp = f->current_offset;
 822
 823                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 824                 if (r < 0)
 825                         return r;
 826
 827                 r = next_with_matches(j, f, direction, &c, &cp);
 828                 if (r <= 0)
 829                         return r;
 830         } else {
 831                 r = find_location_with_matches(j, f, direction, &c, &cp);
 832                 if (r <= 0)
 833                         return r;
 834         }
 835
 836         /* OK, we found the spot, now let's advance until an entry
 837          * that is actually different from what we were previously
 838          * looking at. This is necessary to handle entries which exist
 839          * in two (or more) journal files, and which shall all be
 840          * suppressed but one. */
 841
 842         for (;;) {
 843                 bool found;
 844
 845                 if (j->current_location.type == LOCATION_DISCRETE) {
 846                         int k;
 847
 848                         k = compare_with_location(f, c, &j->current_location);
 849                         if (direction == DIRECTION_DOWN)
 850                                 found = k > 0;
 851                         else
 852                                 found = k < 0;
 853                 } else
 854                         found = true;
 855
 856                 if (found) {
 857                         if (ret)
 858                                 *ret = c;
 859                         if (offset)
 860                                 *offset = cp;
 861                         return 1;
 862                 }
 863
 864                 r = next_with_matches(j, f, direction, &c, &cp);
 865                 if (r <= 0)
 866                         return r;
 867         }
 868 }
 869
 870 static int real_journal_next(sd_journal *j, direction_t direction) {
 871         JournalFile *f, *new_file = NULL;
 872         uint64_t new_offset = 0;
 873         Object *o;
 874         uint64_t p;
 875         Iterator i;
 876         int r;
 877
 878         assert_return(j, -EINVAL);
 879         assert_return(!journal_pid_changed(j), -ECHILD);
 880
 881         HASHMAP_FOREACH(f, j->files, i) {
 882                 bool found;
 883
 884                 r = next_beyond_location(j, f, direction, &o, &p);
 885                 if (r < 0) {
 886                         log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
 887                         continue;
 888                 } else if (r == 0)
 889                         continue;
 890
 891                 if (!new_file)
 892                         found = true;
 893                 else {
 894                         int k;
 895
 896                         k = compare_entry_order(f, o, new_file, new_offset);
 897
 898                         found = direction == DIRECTION_DOWN ? k < 0 : k > 0;
 899                 }
 900
 901                 if (found) {
 902                         new_file = f;
 903                         new_offset = p;
 904                 }
 905         }
 906
 907         if (!new_file)
 908                 return 0;
 909
 910         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 911         if (r < 0)
 912                 return r;
 913
 914         set_location(j, LOCATION_DISCRETE, new_file, o, direction, new_offset);
 915
 916         return 1;
 917 }
 918
 919 _public_ int sd_journal_next(sd_journal *j) {
 920         return real_journal_next(j, DIRECTION_DOWN);
 921 }
 922
 923 _public_ int sd_journal_previous(sd_journal *j) {
 924         return real_journal_next(j, DIRECTION_UP);
 925 }
 926
 927 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 928         int c = 0, r;
 929
 930         assert_return(j, -EINVAL);
 931         assert_return(!journal_pid_changed(j), -ECHILD);
 932
 933         if (skip == 0) {
 934                 /* If this is not a discrete skip, then at least
 935                  * resolve the current location */
 936                 if (j->current_location.type != LOCATION_DISCRETE)
 937                         return real_journal_next(j, direction);
 938
 939                 return 0;
 940         }
 941
 942         do {
 943                 r = real_journal_next(j, direction);
 944                 if (r < 0)
 945                         return r;
 946
 947                 if (r == 0)
 948                         return c;
 949
 950                 skip--;
 951                 c++;
 952         } while (skip > 0);
 953
 954         return c;
 955 }
 956
 957 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 958         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 959 }
 960
 961 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 962         return real_journal_next_skip(j, DIRECTION_UP, skip);
 963 }
 964
 965 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 966         Object *o;
 967         int r;
 968         char bid[33], sid[33];
 969
 970         assert_return(j, -EINVAL);
 971         assert_return(!journal_pid_changed(j), -ECHILD);
 972         assert_return(cursor, -EINVAL);
 973
 974         if (!j->current_file || j->current_file->current_offset <= 0)
 975                 return -EADDRNOTAVAIL;
 976
 977         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 978         if (r < 0)
 979                 return r;
 980
 981         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 982         sd_id128_to_string(o->entry.boot_id, bid);
 983
 984         if (asprintf(cursor,
 985                      "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
 986                      sid, le64toh(o->entry.seqnum),
 987                      bid, le64toh(o->entry.monotonic),
 988                      le64toh(o->entry.realtime),
 989                      le64toh(o->entry.xor_hash)) < 0)
 990                 return -ENOMEM;
 991
 992         return 0;
 993 }
 994
 995 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
 996         char *w, *state;
 997         size_t l;
 998         unsigned long long seqnum, monotonic, realtime, xor_hash;
 999         bool
1000                 seqnum_id_set = false,
1001                 seqnum_set = false,
1002                 boot_id_set = false,
1003                 monotonic_set = false,
1004                 realtime_set = false,
1005                 xor_hash_set = false;
1006         sd_id128_t seqnum_id, boot_id;
1007
1008         assert_return(j, -EINVAL);
1009         assert_return(!journal_pid_changed(j), -ECHILD);
1010         assert_return(!isempty(cursor), -EINVAL);
1011
1012         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1013                 char *item;
1014                 int k = 0;
1015
1016                 if (l < 2 || w[1] != '=')
1017                         return -EINVAL;
1018
1019                 item = strndup(w, l);
1020                 if (!item)
1021                         return -ENOMEM;
1022
1023                 switch (w[0]) {
1024
1025                 case 's':
1026                         seqnum_id_set = true;
1027                         k = sd_id128_from_string(item+2, &seqnum_id);
1028                         break;
1029
1030                 case 'i':
1031                         seqnum_set = true;
1032                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1033                                 k = -EINVAL;
1034                         break;
1035
1036                 case 'b':
1037                         boot_id_set = true;
1038                         k = sd_id128_from_string(item+2, &boot_id);
1039                         break;
1040
1041                 case 'm':
1042                         monotonic_set = true;
1043                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1044                                 k = -EINVAL;
1045                         break;
1046
1047                 case 't':
1048                         realtime_set = true;
1049                         if (sscanf(item+2, "%llx", &realtime) != 1)
1050                                 k = -EINVAL;
1051                         break;
1052
1053                 case 'x':
1054                         xor_hash_set = true;
1055                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1056                                 k = -EINVAL;
1057                         break;
1058                 }
1059
1060                 free(item);
1061
1062                 if (k < 0)
1063                         return k;
1064         }
1065
1066         if ((!seqnum_set || !seqnum_id_set) &&
1067             (!monotonic_set || !boot_id_set) &&
1068             !realtime_set)
1069                 return -EINVAL;
1070
1071         reset_location(j);
1072
1073         j->current_location.type = LOCATION_SEEK;
1074
1075         if (realtime_set) {
1076                 j->current_location.realtime = (uint64_t) realtime;
1077                 j->current_location.realtime_set = true;
1078         }
1079
1080         if (seqnum_set && seqnum_id_set) {
1081                 j->current_location.seqnum = (uint64_t) seqnum;
1082                 j->current_location.seqnum_id = seqnum_id;
1083                 j->current_location.seqnum_set = true;
1084         }
1085
1086         if (monotonic_set && boot_id_set) {
1087                 j->current_location.monotonic = (uint64_t) monotonic;
1088                 j->current_location.boot_id = boot_id;
1089                 j->current_location.monotonic_set = true;
1090         }
1091
1092         if (xor_hash_set) {
1093                 j->current_location.xor_hash = (uint64_t) xor_hash;
1094                 j->current_location.xor_hash_set = true;
1095         }
1096
1097         return 0;
1098 }
1099
1100 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1101         int r;
1102         char *w, *state;
1103         size_t l;
1104         Object *o;
1105
1106         assert_return(j, -EINVAL);
1107         assert_return(!journal_pid_changed(j), -ECHILD);
1108         assert_return(!isempty(cursor), -EINVAL);
1109
1110         if (!j->current_file || j->current_file->current_offset <= 0)
1111                 return -EADDRNOTAVAIL;
1112
1113         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1114         if (r < 0)
1115                 return r;
1116
1117         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1118                 _cleanup_free_ char *item = NULL;
1119                 sd_id128_t id;
1120                 unsigned long long ll;
1121                 int k = 0;
1122
1123                 if (l < 2 || w[1] != '=')
1124                         return -EINVAL;
1125
1126                 item = strndup(w, l);
1127                 if (!item)
1128                         return -ENOMEM;
1129
1130                 switch (w[0]) {
1131
1132                 case 's':
1133                         k = sd_id128_from_string(item+2, &id);
1134                         if (k < 0)
1135                                 return k;
1136                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1137                                 return 0;
1138                         break;
1139
1140                 case 'i':
1141                         if (sscanf(item+2, "%llx", &ll) != 1)
1142                                 return -EINVAL;
1143                         if (ll != le64toh(o->entry.seqnum))
1144                                 return 0;
1145                         break;
1146
1147                 case 'b':
1148                         k = sd_id128_from_string(item+2, &id);
1149                         if (k < 0)
1150                                 return k;
1151                         if (!sd_id128_equal(id, o->entry.boot_id))
1152                                 return 0;
1153                         break;
1154
1155                 case 'm':
1156                         if (sscanf(item+2, "%llx", &ll) != 1)
1157                                 return -EINVAL;
1158                         if (ll != le64toh(o->entry.monotonic))
1159                                 return 0;
1160                         break;
1161
1162                 case 't':
1163                         if (sscanf(item+2, "%llx", &ll) != 1)
1164                                 return -EINVAL;
1165                         if (ll != le64toh(o->entry.realtime))
1166                                 return 0;
1167                         break;
1168
1169                 case 'x':
1170                         if (sscanf(item+2, "%llx", &ll) != 1)
1171                                 return -EINVAL;
1172                         if (ll != le64toh(o->entry.xor_hash))
1173                                 return 0;
1174                         break;
1175                 }
1176         }
1177
1178         return 1;
1179 }
1180
1181
1182 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1183         assert_return(j, -EINVAL);
1184         assert_return(!journal_pid_changed(j), -ECHILD);
1185
1186         reset_location(j);
1187         j->current_location.type = LOCATION_SEEK;
1188         j->current_location.boot_id = boot_id;
1189         j->current_location.monotonic = usec;
1190         j->current_location.monotonic_set = true;
1191
1192         return 0;
1193 }
1194
1195 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1196         assert_return(j, -EINVAL);
1197         assert_return(!journal_pid_changed(j), -ECHILD);
1198
1199         reset_location(j);
1200         j->current_location.type = LOCATION_SEEK;
1201         j->current_location.realtime = usec;
1202         j->current_location.realtime_set = true;
1203
1204         return 0;
1205 }
1206
1207 _public_ int sd_journal_seek_head(sd_journal *j) {
1208         assert_return(j, -EINVAL);
1209         assert_return(!journal_pid_changed(j), -ECHILD);
1210
1211         reset_location(j);
1212         j->current_location.type = LOCATION_HEAD;
1213
1214         return 0;
1215 }
1216
1217 _public_ int sd_journal_seek_tail(sd_journal *j) {
1218         assert_return(j, -EINVAL);
1219         assert_return(!journal_pid_changed(j), -ECHILD);
1220
1221         reset_location(j);
1222         j->current_location.type = LOCATION_TAIL;
1223
1224         return 0;
1225 }
1226
1227 static void check_network(sd_journal *j, int fd) {
1228         struct statfs sfs;
1229
1230         assert(j);
1231
1232         if (j->on_network)
1233                 return;
1234
1235         if (fstatfs(fd, &sfs) < 0)
1236                 return;
1237
1238         j->on_network =
1239                 F_TYPE_EQUAL(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1240                 F_TYPE_EQUAL(sfs.f_type, CODA_SUPER_MAGIC) ||
1241                 F_TYPE_EQUAL(sfs.f_type, NCP_SUPER_MAGIC) ||
1242                 F_TYPE_EQUAL(sfs.f_type, NFS_SUPER_MAGIC) ||
1243                 F_TYPE_EQUAL(sfs.f_type, SMB_SUPER_MAGIC);
1244 }
1245
1246 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1247         const char *full, *tilded, *atted;
1248
1249         full = strappenda(prefix, ".journal");
1250         tilded = strappenda(full, "~");
1251         atted = strappenda(prefix, "@");
1252
1253         return streq(filename, full) ||
1254                streq(filename, tilded) ||
1255                startswith(filename, atted);
1256 }
1257
1258 static bool file_type_wanted(int flags, const char *filename) {
1259         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1260                 return false;
1261
1262         /* no flags set → every type is OK */
1263         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1264                 return true;
1265
1266         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1267                 return true;
1268
1269         if (flags & SD_JOURNAL_CURRENT_USER) {
1270                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1271
1272                 assert_se(snprintf(prefix, sizeof(prefix), "user-%lu", (unsigned long) getuid())
1273                           < (int) sizeof(prefix));
1274
1275                 if (file_has_type_prefix(prefix, filename))
1276                         return true;
1277         }
1278
1279         return false;
1280 }
1281
1282 static int add_any_file(sd_journal *j, const char *path) {
1283         JournalFile *f;
1284         int r;
1285
1286         assert(j);
1287         assert(path);
1288
1289         if (hashmap_get(j->files, path))
1290                 return 0;
1291
1292         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1293                 log_warning("Too many open journal files, not adding %s.", path);
1294                 return set_put_error(j, -ETOOMANYREFS);
1295         }
1296
1297         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1298         if (r < 0)
1299                 return r;
1300
1301         /* journal_file_dump(f); */
1302
1303         r = hashmap_put(j->files, f->path, f);
1304         if (r < 0) {
1305                 journal_file_close(f);
1306                 return r;
1307         }
1308
1309         log_debug("File %s added.", f->path);
1310
1311         check_network(j, f->fd);
1312
1313         j->current_invalidate_counter ++;
1314
1315         return 0;
1316 }
1317
1318 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1319         _cleanup_free_ char *path = NULL;
1320         int r;
1321
1322         assert(j);
1323         assert(prefix);
1324         assert(filename);
1325
1326         if (j->no_new_files ||
1327             !file_type_wanted(j->flags, filename))
1328                 return 0;
1329
1330         path = strjoin(prefix, "/", filename, NULL);
1331         if (!path)
1332                 return -ENOMEM;
1333
1334         r = add_any_file(j, path);
1335         if (r == -ENOENT)
1336                 return 0;
1337         return 0;
1338 }
1339
1340 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1341         char *path;
1342         JournalFile *f;
1343
1344         assert(j);
1345         assert(prefix);
1346         assert(filename);
1347
1348         path = strjoin(prefix, "/", filename, NULL);
1349         if (!path)
1350                 return -ENOMEM;
1351
1352         f = hashmap_get(j->files, path);
1353         free(path);
1354         if (!f)
1355                 return 0;
1356
1357         hashmap_remove(j->files, f->path);
1358
1359         log_debug("File %s removed.", f->path);
1360
1361         if (j->current_file == f) {
1362                 j->current_file = NULL;
1363                 j->current_field = 0;
1364         }
1365
1366         if (j->unique_file == f) {
1367                 j->unique_file = NULL;
1368                 j->unique_offset = 0;
1369         }
1370
1371         journal_file_close(f);
1372
1373         j->current_invalidate_counter ++;
1374
1375         return 0;
1376 }
1377
1378 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1379         _cleanup_free_ char *path = NULL;
1380         int r;
1381         _cleanup_closedir_ DIR *d = NULL;
1382         sd_id128_t id, mid;
1383         Directory *m;
1384
1385         assert(j);
1386         assert(prefix);
1387         assert(dirname);
1388
1389         log_debug("Considering %s/%s.", prefix, dirname);
1390
1391         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1392             (sd_id128_from_string(dirname, &id) < 0 ||
1393              sd_id128_get_machine(&mid) < 0 ||
1394              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1395             return 0;
1396
1397         path = strjoin(prefix, "/", dirname, NULL);
1398         if (!path)
1399                 return -ENOMEM;
1400
1401         d = opendir(path);
1402         if (!d) {
1403                 log_debug("Failed to open %s: %m", path);
1404                 if (errno == ENOENT)
1405                         return 0;
1406                 return -errno;
1407         }
1408
1409         m = hashmap_get(j->directories_by_path, path);
1410         if (!m) {
1411                 m = new0(Directory, 1);
1412                 if (!m)
1413                         return -ENOMEM;
1414
1415                 m->is_root = false;
1416                 m->path = path;
1417
1418                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1419                         free(m);
1420                         return -ENOMEM;
1421                 }
1422
1423                 path = NULL; /* avoid freeing in cleanup */
1424                 j->current_invalidate_counter ++;
1425
1426                 log_debug("Directory %s added.", m->path);
1427
1428         } else if (m->is_root)
1429                 return 0;
1430
1431         if (m->wd <= 0 && j->inotify_fd >= 0) {
1432
1433                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1434                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1435                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1436                                           IN_ONLYDIR);
1437
1438                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1439                         inotify_rm_watch(j->inotify_fd, m->wd);
1440         }
1441
1442         for (;;) {
1443                 struct dirent *de;
1444                 union dirent_storage buf;
1445
1446                 r = readdir_r(d, &buf.de, &de);
1447                 if (r != 0 || !de)
1448                         break;
1449
1450                 if (dirent_is_file_with_suffix(de, ".journal") ||
1451                     dirent_is_file_with_suffix(de, ".journal~")) {
1452                         r = add_file(j, m->path, de->d_name);
1453                         if (r < 0) {
1454                                 log_debug("Failed to add file %s/%s: %s",
1455                                           m->path, de->d_name, strerror(-r));
1456                                 r = set_put_error(j, r);
1457                                 if (r < 0)
1458                                         return r;
1459                         }
1460                 }
1461         }
1462
1463         check_network(j, dirfd(d));
1464
1465         return 0;
1466 }
1467
1468 static int add_root_directory(sd_journal *j, const char *p) {
1469         _cleanup_closedir_ DIR *d = NULL;
1470         Directory *m;
1471         int r;
1472
1473         assert(j);
1474         assert(p);
1475
1476         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1477             !path_startswith(p, "/run"))
1478                 return -EINVAL;
1479
1480         d = opendir(p);
1481         if (!d)
1482                 return -errno;
1483
1484         m = hashmap_get(j->directories_by_path, p);
1485         if (!m) {
1486                 m = new0(Directory, 1);
1487                 if (!m)
1488                         return -ENOMEM;
1489
1490                 m->is_root = true;
1491                 m->path = strdup(p);
1492                 if (!m->path) {
1493                         free(m);
1494                         return -ENOMEM;
1495                 }
1496
1497                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1498                         free(m->path);
1499                         free(m);
1500                         return -ENOMEM;
1501                 }
1502
1503                 j->current_invalidate_counter ++;
1504
1505                 log_debug("Root directory %s added.", m->path);
1506
1507         } else if (!m->is_root)
1508                 return 0;
1509
1510         if (m->wd <= 0 && j->inotify_fd >= 0) {
1511
1512                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1513                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1514                                           IN_ONLYDIR);
1515
1516                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1517                         inotify_rm_watch(j->inotify_fd, m->wd);
1518         }
1519
1520         if (j->no_new_files)
1521                 return 0;
1522
1523         for (;;) {
1524                 struct dirent *de;
1525                 union dirent_storage buf;
1526                 sd_id128_t id;
1527
1528                 r = readdir_r(d, &buf.de, &de);
1529                 if (r != 0 || !de)
1530                         break;
1531
1532                 if (dirent_is_file_with_suffix(de, ".journal") ||
1533                     dirent_is_file_with_suffix(de, ".journal~")) {
1534                         r = add_file(j, m->path, de->d_name);
1535                         if (r < 0) {
1536                                 log_debug("Failed to add file %s/%s: %s",
1537                                           m->path, de->d_name, strerror(-r));
1538                                 r = set_put_error(j, r);
1539                                 if (r < 0)
1540                                         return r;
1541                         }
1542                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1543                            sd_id128_from_string(de->d_name, &id) >= 0) {
1544
1545                         r = add_directory(j, m->path, de->d_name);
1546                         if (r < 0)
1547                                 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1548                 }
1549         }
1550
1551         check_network(j, dirfd(d));
1552
1553         return 0;
1554 }
1555
1556 static int remove_directory(sd_journal *j, Directory *d) {
1557         assert(j);
1558
1559         if (d->wd > 0) {
1560                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1561
1562                 if (j->inotify_fd >= 0)
1563                         inotify_rm_watch(j->inotify_fd, d->wd);
1564         }
1565
1566         hashmap_remove(j->directories_by_path, d->path);
1567
1568         if (d->is_root)
1569                 log_debug("Root directory %s removed.", d->path);
1570         else
1571                 log_debug("Directory %s removed.", d->path);
1572
1573         free(d->path);
1574         free(d);
1575
1576         return 0;
1577 }
1578
1579 static int add_search_paths(sd_journal *j) {
1580         int r;
1581         const char search_paths[] =
1582                 "/run/log/journal\0"
1583                 "/var/log/journal\0";
1584         const char *p;
1585
1586         assert(j);
1587
1588         /* We ignore most errors here, since the idea is to only open
1589          * what's actually accessible, and ignore the rest. */
1590
1591         NULSTR_FOREACH(p, search_paths) {
1592                 r = add_root_directory(j, p);
1593                 if (r < 0 && r != -ENOENT) {
1594                         r = set_put_error(j, r);
1595                         if (r < 0)
1596                                 return r;
1597                 }
1598         }
1599
1600         return 0;
1601 }
1602
1603 static int add_current_paths(sd_journal *j) {
1604         Iterator i;
1605         JournalFile *f;
1606
1607         assert(j);
1608         assert(j->no_new_files);
1609
1610         /* Simply adds all directories for files we have open as
1611          * "root" directories. We don't expect errors here, so we
1612          * treat them as fatal. */
1613
1614         HASHMAP_FOREACH(f, j->files, i) {
1615                 int r;
1616                 _cleanup_free_ char *dir;
1617
1618                 dir = dirname_malloc(f->path);
1619                 if (!dir)
1620                         return -ENOMEM;
1621
1622                 r = add_root_directory(j, dir);
1623                 if (r < 0) {
1624                         set_put_error(j, r);
1625                         return r;
1626                 }
1627         }
1628
1629         return 0;
1630 }
1631
1632
1633 static int allocate_inotify(sd_journal *j) {
1634         assert(j);
1635
1636         if (j->inotify_fd < 0) {
1637                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1638                 if (j->inotify_fd < 0)
1639                         return -errno;
1640         }
1641
1642         if (!j->directories_by_wd) {
1643                 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1644                 if (!j->directories_by_wd)
1645                         return -ENOMEM;
1646         }
1647
1648         return 0;
1649 }
1650
1651 static sd_journal *journal_new(int flags, const char *path) {
1652         sd_journal *j;
1653
1654         j = new0(sd_journal, 1);
1655         if (!j)
1656                 return NULL;
1657
1658         j->original_pid = getpid();
1659         j->inotify_fd = -1;
1660         j->flags = flags;
1661         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1662
1663         if (path) {
1664                 j->path = strdup(path);
1665                 if (!j->path)
1666                         goto fail;
1667         }
1668
1669         j->files = hashmap_new(string_hash_func, string_compare_func);
1670         j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1671         j->mmap = mmap_cache_new();
1672         if (!j->files || !j->directories_by_path || !j->mmap)
1673                 goto fail;
1674
1675         return j;
1676
1677 fail:
1678         sd_journal_close(j);
1679         return NULL;
1680 }
1681
1682 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1683         sd_journal *j;
1684         int r;
1685
1686         assert_return(ret, -EINVAL);
1687
1688         if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1689                       SD_JOURNAL_RUNTIME_ONLY|
1690                       SD_JOURNAL_SYSTEM|
1691                       SD_JOURNAL_CURRENT_USER))
1692                 return -EINVAL;
1693
1694         j = journal_new(flags, NULL);
1695         if (!j)
1696                 return -ENOMEM;
1697
1698         r = add_search_paths(j);
1699         if (r < 0)
1700                 goto fail;
1701
1702         *ret = j;
1703         return 0;
1704
1705 fail:
1706         sd_journal_close(j);
1707
1708         return r;
1709 }
1710
1711 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1712         sd_journal *j;
1713         int r;
1714
1715         assert_return(ret, -EINVAL);
1716         assert_return(path, -EINVAL);
1717         assert_return(flags == 0, -EINVAL);
1718
1719         j = journal_new(flags, path);
1720         if (!j)
1721                 return -ENOMEM;
1722
1723         r = add_root_directory(j, path);
1724         if (r < 0) {
1725                 set_put_error(j, r);
1726                 goto fail;
1727         }
1728
1729         *ret = j;
1730         return 0;
1731
1732 fail:
1733         sd_journal_close(j);
1734
1735         return r;
1736 }
1737
1738 _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int flags) {
1739         sd_journal *j;
1740         const char **path;
1741         int r;
1742
1743         assert_return(ret, -EINVAL);
1744         assert_return(flags == 0, -EINVAL);
1745
1746         j = journal_new(flags, NULL);
1747         if (!j)
1748                 return -ENOMEM;
1749
1750         STRV_FOREACH(path, paths) {
1751                 r = add_any_file(j, *path);
1752                 if (r < 0) {
1753                         log_error("Failed to open %s: %s", *path, strerror(-r));
1754                         goto fail;
1755                 }
1756         }
1757
1758         j->no_new_files = true;
1759
1760         *ret = j;
1761         return 0;
1762
1763 fail:
1764         sd_journal_close(j);
1765
1766         return r;
1767 }
1768
1769 _public_ void sd_journal_close(sd_journal *j) {
1770         Directory *d;
1771         JournalFile *f;
1772
1773         if (!j)
1774                 return;
1775
1776         sd_journal_flush_matches(j);
1777
1778         while ((f = hashmap_steal_first(j->files)))
1779                 journal_file_close(f);
1780
1781         hashmap_free(j->files);
1782
1783         while ((d = hashmap_first(j->directories_by_path)))
1784                 remove_directory(j, d);
1785
1786         while ((d = hashmap_first(j->directories_by_wd)))
1787                 remove_directory(j, d);
1788
1789         hashmap_free(j->directories_by_path);
1790         hashmap_free(j->directories_by_wd);
1791
1792         if (j->inotify_fd >= 0)
1793                 close_nointr_nofail(j->inotify_fd);
1794
1795         if (j->mmap) {
1796                 log_debug("mmap cache statistics: %u hit, %u miss", mmap_cache_get_hit(j->mmap), mmap_cache_get_missed(j->mmap));
1797                 mmap_cache_unref(j->mmap);
1798         }
1799
1800         free(j->path);
1801         free(j->unique_field);
1802         set_free(j->errors);
1803         free(j);
1804 }
1805
1806 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1807         Object *o;
1808         JournalFile *f;
1809         int r;
1810
1811         assert_return(j, -EINVAL);
1812         assert_return(!journal_pid_changed(j), -ECHILD);
1813         assert_return(ret, -EINVAL);
1814
1815         f = j->current_file;
1816         if (!f)
1817                 return -EADDRNOTAVAIL;
1818
1819         if (f->current_offset <= 0)
1820                 return -EADDRNOTAVAIL;
1821
1822         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1823         if (r < 0)
1824                 return r;
1825
1826         *ret = le64toh(o->entry.realtime);
1827         return 0;
1828 }
1829
1830 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1831         Object *o;
1832         JournalFile *f;
1833         int r;
1834         sd_id128_t id;
1835
1836         assert_return(j, -EINVAL);
1837         assert_return(!journal_pid_changed(j), -ECHILD);
1838
1839         f = j->current_file;
1840         if (!f)
1841                 return -EADDRNOTAVAIL;
1842
1843         if (f->current_offset <= 0)
1844                 return -EADDRNOTAVAIL;
1845
1846         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1847         if (r < 0)
1848                 return r;
1849
1850         if (ret_boot_id)
1851                 *ret_boot_id = o->entry.boot_id;
1852         else {
1853                 r = sd_id128_get_boot(&id);
1854                 if (r < 0)
1855                         return r;
1856
1857                 if (!sd_id128_equal(id, o->entry.boot_id))
1858                         return -ESTALE;
1859         }
1860
1861         if (ret)
1862                 *ret = le64toh(o->entry.monotonic);
1863
1864         return 0;
1865 }
1866
1867 static bool field_is_valid(const char *field) {
1868         const char *p;
1869
1870         assert(field);
1871
1872         if (isempty(field))
1873                 return false;
1874
1875         if (startswith(field, "__"))
1876                 return false;
1877
1878         for (p = field; *p; p++) {
1879
1880                 if (*p == '_')
1881                         continue;
1882
1883                 if (*p >= 'A' && *p <= 'Z')
1884                         continue;
1885
1886                 if (*p >= '0' && *p <= '9')
1887                         continue;
1888
1889                 return false;
1890         }
1891
1892         return true;
1893 }
1894
1895 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1896         JournalFile *f;
1897         uint64_t i, n;
1898         size_t field_length;
1899         int r;
1900         Object *o;
1901
1902         assert_return(j, -EINVAL);
1903         assert_return(!journal_pid_changed(j), -ECHILD);
1904         assert_return(field, -EINVAL);
1905         assert_return(data, -EINVAL);
1906         assert_return(size, -EINVAL);
1907         assert_return(field_is_valid(field), -EINVAL);
1908
1909         f = j->current_file;
1910         if (!f)
1911                 return -EADDRNOTAVAIL;
1912
1913         if (f->current_offset <= 0)
1914                 return -EADDRNOTAVAIL;
1915
1916         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1917         if (r < 0)
1918                 return r;
1919
1920         field_length = strlen(field);
1921
1922         n = journal_file_entry_n_items(o);
1923         for (i = 0; i < n; i++) {
1924                 uint64_t p, l;
1925                 le64_t le_hash;
1926                 size_t t;
1927
1928                 p = le64toh(o->entry.items[i].object_offset);
1929                 le_hash = o->entry.items[i].hash;
1930                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1931                 if (r < 0)
1932                         return r;
1933
1934                 if (le_hash != o->data.hash)
1935                         return -EBADMSG;
1936
1937                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1938
1939                 if (o->object.flags & OBJECT_COMPRESSED) {
1940
1941 #ifdef HAVE_XZ
1942                         if (uncompress_startswith(o->data.payload, l,
1943                                                   &f->compress_buffer, &f->compress_buffer_size,
1944                                                   field, field_length, '=')) {
1945
1946                                 uint64_t rsize;
1947
1948                                 if (!uncompress_blob(o->data.payload, l,
1949                                                      &f->compress_buffer, &f->compress_buffer_size, &rsize,
1950                                                      j->data_threshold))
1951                                         return -EBADMSG;
1952
1953                                 *data = f->compress_buffer;
1954                                 *size = (size_t) rsize;
1955
1956                                 return 0;
1957                         }
1958 #else
1959                         return -EPROTONOSUPPORT;
1960 #endif
1961
1962                 } else if (l >= field_length+1 &&
1963                            memcmp(o->data.payload, field, field_length) == 0 &&
1964                            o->data.payload[field_length] == '=') {
1965
1966                         t = (size_t) l;
1967
1968                         if ((uint64_t) t != l)
1969                                 return -E2BIG;
1970
1971                         *data = o->data.payload;
1972                         *size = t;
1973
1974                         return 0;
1975                 }
1976
1977                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1978                 if (r < 0)
1979                         return r;
1980         }
1981
1982         return -ENOENT;
1983 }
1984
1985 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1986         size_t t;
1987         uint64_t l;
1988
1989         l = le64toh(o->object.size) - offsetof(Object, data.payload);
1990         t = (size_t) l;
1991
1992         /* We can't read objects larger than 4G on a 32bit machine */
1993         if ((uint64_t) t != l)
1994                 return -E2BIG;
1995
1996         if (o->object.flags & OBJECT_COMPRESSED) {
1997 #ifdef HAVE_XZ
1998                 uint64_t rsize;
1999
2000                 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
2001                         return -EBADMSG;
2002
2003                 *data = f->compress_buffer;
2004                 *size = (size_t) rsize;
2005 #else
2006                 return -EPROTONOSUPPORT;
2007 #endif
2008         } else {
2009                 *data = o->data.payload;
2010                 *size = t;
2011         }
2012
2013         return 0;
2014 }
2015
2016 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
2017         JournalFile *f;
2018         uint64_t p, n;
2019         le64_t le_hash;
2020         int r;
2021         Object *o;
2022
2023         assert_return(j, -EINVAL);
2024         assert_return(!journal_pid_changed(j), -ECHILD);
2025         assert_return(data, -EINVAL);
2026         assert_return(size, -EINVAL);
2027
2028         f = j->current_file;
2029         if (!f)
2030                 return -EADDRNOTAVAIL;
2031
2032         if (f->current_offset <= 0)
2033                 return -EADDRNOTAVAIL;
2034
2035         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2036         if (r < 0)
2037                 return r;
2038
2039         n = journal_file_entry_n_items(o);
2040         if (j->current_field >= n)
2041                 return 0;
2042
2043         p = le64toh(o->entry.items[j->current_field].object_offset);
2044         le_hash = o->entry.items[j->current_field].hash;
2045         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2046         if (r < 0)
2047                 return r;
2048
2049         if (le_hash != o->data.hash)
2050                 return -EBADMSG;
2051
2052         r = return_data(j, f, o, data, size);
2053         if (r < 0)
2054                 return r;
2055
2056         j->current_field ++;
2057
2058         return 1;
2059 }
2060
2061 _public_ void sd_journal_restart_data(sd_journal *j) {
2062         if (!j)
2063                 return;
2064
2065         j->current_field = 0;
2066 }
2067
2068 _public_ int sd_journal_get_fd(sd_journal *j) {
2069         int r;
2070
2071         assert_return(j, -EINVAL);
2072         assert_return(!journal_pid_changed(j), -ECHILD);
2073
2074         if (j->inotify_fd >= 0)
2075                 return j->inotify_fd;
2076
2077         r = allocate_inotify(j);
2078         if (r < 0)
2079                 return r;
2080
2081         /* Iterate through all dirs again, to add them to the
2082          * inotify */
2083         if (j->no_new_files)
2084                 r = add_current_paths(j);
2085         else if (j->path)
2086                 r = add_root_directory(j, j->path);
2087         else
2088                 r = add_search_paths(j);
2089         if (r < 0)
2090                 return r;
2091
2092         return j->inotify_fd;
2093 }
2094
2095 _public_ int sd_journal_get_events(sd_journal *j) {
2096         int fd;
2097
2098         assert_return(j, -EINVAL);
2099         assert_return(!journal_pid_changed(j), -ECHILD);
2100
2101         fd = sd_journal_get_fd(j);
2102         if (fd < 0)
2103                 return fd;
2104
2105         return POLLIN;
2106 }
2107
2108 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2109         int fd;
2110
2111         assert_return(j, -EINVAL);
2112         assert_return(!journal_pid_changed(j), -ECHILD);
2113         assert_return(timeout_usec, -EINVAL);
2114
2115         fd = sd_journal_get_fd(j);
2116         if (fd < 0)
2117                 return fd;
2118
2119         if (!j->on_network) {
2120                 *timeout_usec = (uint64_t) -1;
2121                 return 0;
2122         }
2123
2124         /* If we are on the network we need to regularly check for
2125          * changes manually */
2126
2127         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2128         return 1;
2129 }
2130
2131 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2132         Directory *d;
2133         int r;
2134
2135         assert(j);
2136         assert(e);
2137
2138         /* Is this a subdirectory we watch? */
2139         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2140         if (d) {
2141                 sd_id128_t id;
2142
2143                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2144                     (endswith(e->name, ".journal") ||
2145                      endswith(e->name, ".journal~"))) {
2146
2147                         /* Event for a journal file */
2148
2149                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2150                                 r = add_file(j, d->path, e->name);
2151                                 if (r < 0) {
2152                                         log_debug("Failed to add file %s/%s: %s",
2153                                                   d->path, e->name, strerror(-r));
2154                                         set_put_error(j, r);
2155                                 }
2156
2157                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2158
2159                                 r = remove_file(j, d->path, e->name);
2160                                 if (r < 0)
2161                                         log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2162                         }
2163
2164                 } else if (!d->is_root && e->len == 0) {
2165
2166                         /* Event for a subdirectory */
2167
2168                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2169                                 r = remove_directory(j, d);
2170                                 if (r < 0)
2171                                         log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2172                         }
2173
2174
2175                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2176
2177                         /* Event for root directory */
2178
2179                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2180                                 r = add_directory(j, d->path, e->name);
2181                                 if (r < 0)
2182                                         log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2183                         }
2184                 }
2185
2186                 return;
2187         }
2188
2189         if (e->mask & IN_IGNORED)
2190                 return;
2191
2192         log_warning("Unknown inotify event.");
2193 }
2194
2195 static int determine_change(sd_journal *j) {
2196         bool b;
2197
2198         assert(j);
2199
2200         b = j->current_invalidate_counter != j->last_invalidate_counter;
2201         j->last_invalidate_counter = j->current_invalidate_counter;
2202
2203         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2204 }
2205
2206 _public_ int sd_journal_process(sd_journal *j) {
2207         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2208         bool got_something = false;
2209
2210         assert_return(j, -EINVAL);
2211         assert_return(!journal_pid_changed(j), -ECHILD);
2212
2213         j->last_process_usec = now(CLOCK_MONOTONIC);
2214
2215         for (;;) {
2216                 struct inotify_event *e;
2217                 ssize_t l;
2218
2219                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2220                 if (l < 0) {
2221                         if (errno == EAGAIN || errno == EINTR)
2222                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2223
2224                         return -errno;
2225                 }
2226
2227                 got_something = true;
2228
2229                 e = (struct inotify_event*) buffer;
2230                 while (l > 0) {
2231                         size_t step;
2232
2233                         process_inotify_event(j, e);
2234
2235                         step = sizeof(struct inotify_event) + e->len;
2236                         assert(step <= (size_t) l);
2237
2238                         e = (struct inotify_event*) ((uint8_t*) e + step);
2239                         l -= step;
2240                 }
2241         }
2242
2243         return determine_change(j);
2244 }
2245
2246 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2247         int r;
2248         uint64_t t;
2249
2250         assert_return(j, -EINVAL);
2251         assert_return(!journal_pid_changed(j), -ECHILD);
2252
2253         if (j->inotify_fd < 0) {
2254
2255                 /* This is the first invocation, hence create the
2256                  * inotify watch */
2257                 r = sd_journal_get_fd(j);
2258                 if (r < 0)
2259                         return r;
2260
2261                 /* The journal might have changed since the context
2262                  * object was created and we weren't watching before,
2263                  * hence don't wait for anything, and return
2264                  * immediately. */
2265                 return determine_change(j);
2266         }
2267
2268         r = sd_journal_get_timeout(j, &t);
2269         if (r < 0)
2270                 return r;
2271
2272         if (t != (uint64_t) -1) {
2273                 usec_t n;
2274
2275                 n = now(CLOCK_MONOTONIC);
2276                 t = t > n ? t - n : 0;
2277
2278                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2279                         timeout_usec = t;
2280         }
2281
2282         do {
2283                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2284         } while (r == -EINTR);
2285
2286         if (r < 0)
2287                 return r;
2288
2289         return sd_journal_process(j);
2290 }
2291
2292 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2293         Iterator i;
2294         JournalFile *f;
2295         bool first = true;
2296         int r;
2297
2298         assert_return(j, -EINVAL);
2299         assert_return(!journal_pid_changed(j), -ECHILD);
2300         assert_return(from || to, -EINVAL);
2301         assert_return(from != to, -EINVAL);
2302
2303         HASHMAP_FOREACH(f, j->files, i) {
2304                 usec_t fr, t;
2305
2306                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2307                 if (r == -ENOENT)
2308                         continue;
2309                 if (r < 0)
2310                         return r;
2311                 if (r == 0)
2312                         continue;
2313
2314                 if (first) {
2315                         if (from)
2316                                 *from = fr;
2317                         if (to)
2318                                 *to = t;
2319                         first = false;
2320                 } else {
2321                         if (from)
2322                                 *from = MIN(fr, *from);
2323                         if (to)
2324                                 *to = MAX(t, *to);
2325                 }
2326         }
2327
2328         return first ? 0 : 1;
2329 }
2330
2331 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2332         Iterator i;
2333         JournalFile *f;
2334         bool first = true;
2335         int r;
2336
2337         assert_return(j, -EINVAL);
2338         assert_return(!journal_pid_changed(j), -ECHILD);
2339         assert_return(from || to, -EINVAL);
2340         assert_return(from != to, -EINVAL);
2341
2342         HASHMAP_FOREACH(f, j->files, i) {
2343                 usec_t fr, t;
2344
2345                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2346                 if (r == -ENOENT)
2347                         continue;
2348                 if (r < 0)
2349                         return r;
2350                 if (r == 0)
2351                         continue;
2352
2353                 if (first) {
2354                         if (from)
2355                                 *from = fr;
2356                         if (to)
2357                                 *to = t;
2358                         first = false;
2359                 } else {
2360                         if (from)
2361                                 *from = MIN(fr, *from);
2362                         if (to)
2363                                 *to = MAX(t, *to);
2364                 }
2365         }
2366
2367         return first ? 0 : 1;
2368 }
2369
2370 void journal_print_header(sd_journal *j) {
2371         Iterator i;
2372         JournalFile *f;
2373         bool newline = false;
2374
2375         assert(j);
2376
2377         HASHMAP_FOREACH(f, j->files, i) {
2378                 if (newline)
2379                         putchar('\n');
2380                 else
2381                         newline = true;
2382
2383                 journal_file_print_header(f);
2384         }
2385 }
2386
2387 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2388         Iterator i;
2389         JournalFile *f;
2390         uint64_t sum = 0;
2391
2392         assert_return(j, -EINVAL);
2393         assert_return(!journal_pid_changed(j), -ECHILD);
2394         assert_return(bytes, -EINVAL);
2395
2396         HASHMAP_FOREACH(f, j->files, i) {
2397                 struct stat st;
2398
2399                 if (fstat(f->fd, &st) < 0)
2400                         return -errno;
2401
2402                 sum += (uint64_t) st.st_blocks * 512ULL;
2403         }
2404
2405         *bytes = sum;
2406         return 0;
2407 }
2408
2409 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2410         char *f;
2411
2412         assert_return(j, -EINVAL);
2413         assert_return(!journal_pid_changed(j), -ECHILD);
2414         assert_return(!isempty(field), -EINVAL);
2415         assert_return(field_is_valid(field), -EINVAL);
2416
2417         f = strdup(field);
2418         if (!f)
2419                 return -ENOMEM;
2420
2421         free(j->unique_field);
2422         j->unique_field = f;
2423         j->unique_file = NULL;
2424         j->unique_offset = 0;
2425
2426         return 0;
2427 }
2428
2429 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2430         Object *o;
2431         size_t k;
2432         int r;
2433
2434         assert_return(j, -EINVAL);
2435         assert_return(!journal_pid_changed(j), -ECHILD);
2436         assert_return(data, -EINVAL);
2437         assert_return(l, -EINVAL);
2438         assert_return(j->unique_field, -EINVAL);
2439
2440         k = strlen(j->unique_field);
2441
2442         if (!j->unique_file) {
2443                 j->unique_file = hashmap_first(j->files);
2444                 if (!j->unique_file)
2445                         return 0;
2446                 j->unique_offset = 0;
2447         }
2448
2449         for (;;) {
2450                 JournalFile *of;
2451                 Iterator i;
2452                 const void *odata;
2453                 size_t ol;
2454                 bool found;
2455
2456                 /* Proceed to next data object in the field's linked list */
2457                 if (j->unique_offset == 0) {
2458                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2459                         if (r < 0)
2460                                 return r;
2461
2462                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2463                 } else {
2464                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2465                         if (r < 0)
2466                                 return r;
2467
2468                         j->unique_offset = le64toh(o->data.next_field_offset);
2469                 }
2470
2471                 /* We reached the end of the list? Then start again, with the next file */
2472                 if (j->unique_offset == 0) {
2473                         JournalFile *n;
2474
2475                         n = hashmap_next(j->files, j->unique_file->path);
2476                         if (!n)
2477                                 return 0;
2478
2479                         j->unique_file = n;
2480                         continue;
2481                 }
2482
2483                 /* We do not use the type context here, but 0 instead,
2484                  * so that we can look at this data object at the same
2485                  * time as one on another file */
2486                 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2487                 if (r < 0)
2488                         return r;
2489
2490                 /* Let's do the type check by hand, since we used 0 context above. */
2491                 if (o->object.type != OBJECT_DATA)
2492                         return -EBADMSG;
2493
2494                 r = return_data(j, j->unique_file, o, &odata, &ol);
2495                 if (r < 0)
2496                         return r;
2497
2498                 /* OK, now let's see if we already returned this data
2499                  * object by checking if it exists in the earlier
2500                  * traversed files. */
2501                 found = false;
2502                 HASHMAP_FOREACH(of, j->files, i) {
2503                         Object *oo;
2504                         uint64_t op;
2505
2506                         if (of == j->unique_file)
2507                                 break;
2508
2509                         /* Skip this file it didn't have any fields
2510                          * indexed */
2511                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2512                             le64toh(of->header->n_fields) <= 0)
2513                                 continue;
2514
2515                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2516                         if (r < 0)
2517                                 return r;
2518
2519                         if (r > 0)
2520                                 found = true;
2521                 }
2522
2523                 if (found)
2524                         continue;
2525
2526                 r = return_data(j, j->unique_file, o, data, l);
2527                 if (r < 0)
2528                         return r;
2529
2530                 return 1;
2531         }
2532 }
2533
2534 _public_ void sd_journal_restart_unique(sd_journal *j) {
2535         if (!j)
2536                 return;
2537
2538         j->unique_file = NULL;
2539         j->unique_offset = 0;
2540 }
2541
2542 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2543         assert_return(j, -EINVAL);
2544         assert_return(!journal_pid_changed(j), -ECHILD);
2545
2546         return !j->on_network;
2547 }
2548
2549 static char *lookup_field(const char *field, void *userdata) {
2550         sd_journal *j = userdata;
2551         const void *data;
2552         size_t size, d;
2553         int r;
2554
2555         assert(field);
2556         assert(j);
2557
2558         r = sd_journal_get_data(j, field, &data, &size);
2559         if (r < 0 ||
2560             size > REPLACE_VAR_MAX)
2561                 return strdup(field);
2562
2563         d = strlen(field) + 1;
2564
2565         return strndup((const char*) data + d, size - d);
2566 }
2567
2568 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2569         const void *data;
2570         size_t size;
2571         sd_id128_t id;
2572         _cleanup_free_ char *text = NULL, *cid = NULL;
2573         char *t;
2574         int r;
2575
2576         assert_return(j, -EINVAL);
2577         assert_return(!journal_pid_changed(j), -ECHILD);
2578         assert_return(ret, -EINVAL);
2579
2580         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2581         if (r < 0)
2582                 return r;
2583
2584         cid = strndup((const char*) data + 11, size - 11);
2585         if (!cid)
2586                 return -ENOMEM;
2587
2588         r = sd_id128_from_string(cid, &id);
2589         if (r < 0)
2590                 return r;
2591
2592         r = catalog_get(CATALOG_DATABASE, id, &text);
2593         if (r < 0)
2594                 return r;
2595
2596         t = replace_var(text, lookup_field, j);
2597         if (!t)
2598                 return -ENOMEM;
2599
2600         *ret = t;
2601         return 0;
2602 }
2603
2604 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2605         assert_return(ret, -EINVAL);
2606
2607         return catalog_get(CATALOG_DATABASE, id, ret);
2608 }
2609
2610 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2611         assert_return(j, -EINVAL);
2612         assert_return(!journal_pid_changed(j), -ECHILD);
2613
2614         j->data_threshold = sz;
2615         return 0;
2616 }
2617
2618 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2619         assert_return(j, -EINVAL);
2620         assert_return(!journal_pid_changed(j), -ECHILD);
2621         assert_return(sz, -EINVAL);
2622
2623         *sz = j->data_threshold;
2624         return 0;
2625 }