src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "strv.h"
  37 #include "path-util.h"
  38 #include "lookup3.h"
  39 #include "compress.h"
  40 #include "journal-internal.h"
  41 #include "missing.h"
  42 #include "catalog.h"
  43 #include "replace-var.h"
  44 #include "fileio.h"
  45
  46 #define JOURNAL_FILES_MAX 1024
  47
  48 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  49
  50 #define REPLACE_VAR_MAX 256
  51
  52 #define DEFAULT_DATA_THRESHOLD (64*1024)
  53
  54 static bool journal_pid_changed(sd_journal *j) {
  55         assert(j);
  56
  57         /* We don't support people creating a journal object and
  58          * keeping it around over a fork(). Let's complain. */
  59
  60         return j->original_pid != getpid();
  61 }
  62
  63 /* We return an error here only if we didn't manage to
  64    memorize the real error. */
  65 static int set_put_error(sd_journal *j, int r) {
  66         int k;
  67
  68         if (r >= 0)
  69                 return r;
  70
  71         k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
  72         if (k < 0)
  73                 return k;
  74
  75         return set_put(j->errors, INT_TO_PTR(r));
  76 }
  77
  78 static void detach_location(sd_journal *j) {
  79         Iterator i;
  80         JournalFile *f;
  81
  82         assert(j);
  83
  84         j->current_file = NULL;
  85         j->current_field = 0;
  86
  87         HASHMAP_FOREACH(f, j->files, i)
  88                 f->current_offset = 0;
  89 }
  90
  91 static void reset_location(sd_journal *j) {
  92         assert(j);
  93
  94         detach_location(j);
  95         zero(j->current_location);
  96 }
  97
  98 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  99         assert(l);
 100         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 101         assert(f);
 102         assert(o->object.type == OBJECT_ENTRY);
 103
 104         l->type = type;
 105         l->seqnum = le64toh(o->entry.seqnum);
 106         l->seqnum_id = f->header->seqnum_id;
 107         l->realtime = le64toh(o->entry.realtime);
 108         l->monotonic = le64toh(o->entry.monotonic);
 109         l->boot_id = o->entry.boot_id;
 110         l->xor_hash = le64toh(o->entry.xor_hash);
 111
 112         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 113 }
 114
 115 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o,
 116                          direction_t direction, uint64_t offset) {
 117         assert(j);
 118         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 119         assert(f);
 120         assert(o);
 121
 122         init_location(&j->current_location, type, f, o);
 123
 124         j->current_file = f;
 125         j->current_field = 0;
 126
 127         f->last_direction = direction;
 128         f->current_offset = offset;
 129 }
 130
 131 static int match_is_valid(const void *data, size_t size) {
 132         const char *b, *p;
 133
 134         assert(data);
 135
 136         if (size < 2)
 137                 return false;
 138
 139         if (startswith(data, "__"))
 140                 return false;
 141
 142         b = data;
 143         for (p = b; p < b + size; p++) {
 144
 145                 if (*p == '=')
 146                         return p > b;
 147
 148                 if (*p == '_')
 149                         continue;
 150
 151                 if (*p >= 'A' && *p <= 'Z')
 152                         continue;
 153
 154                 if (*p >= '0' && *p <= '9')
 155                         continue;
 156
 157                 return false;
 158         }
 159
 160         return false;
 161 }
 162
 163 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 164         const uint8_t *a = _a, *b = _b;
 165         size_t j;
 166
 167         for (j = 0; j < s && j < t; j++) {
 168
 169                 if (a[j] != b[j])
 170                         return false;
 171
 172                 if (a[j] == '=')
 173                         return true;
 174         }
 175
 176         assert_not_reached("\"=\" not found");
 177 }
 178
 179 static Match *match_new(Match *p, MatchType t) {
 180         Match *m;
 181
 182         m = new0(Match, 1);
 183         if (!m)
 184                 return NULL;
 185
 186         m->type = t;
 187
 188         if (p) {
 189                 m->parent = p;
 190                 LIST_PREPEND(matches, p->matches, m);
 191         }
 192
 193         return m;
 194 }
 195
 196 static void match_free(Match *m) {
 197         assert(m);
 198
 199         while (m->matches)
 200                 match_free(m->matches);
 201
 202         if (m->parent)
 203                 LIST_REMOVE(matches, m->parent->matches, m);
 204
 205         free(m->data);
 206         free(m);
 207 }
 208
 209 static void match_free_if_empty(Match *m) {
 210         if (!m || m->matches)
 211                 return;
 212
 213         match_free(m);
 214 }
 215
 216 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 217         Match *l3, *l4, *add_here = NULL, *m;
 218         le64_t le_hash;
 219
 220         assert_return(j, -EINVAL);
 221         assert_return(!journal_pid_changed(j), -ECHILD);
 222         assert_return(data, -EINVAL);
 223
 224         if (size == 0)
 225                 size = strlen(data);
 226
 227         assert_return(match_is_valid(data, size), -EINVAL);
 228
 229         /* level 0: AND term
 230          * level 1: OR terms
 231          * level 2: AND terms
 232          * level 3: OR terms
 233          * level 4: concrete matches */
 234
 235         if (!j->level0) {
 236                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 237                 if (!j->level0)
 238                         return -ENOMEM;
 239         }
 240
 241         if (!j->level1) {
 242                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 243                 if (!j->level1)
 244                         return -ENOMEM;
 245         }
 246
 247         if (!j->level2) {
 248                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 249                 if (!j->level2)
 250                         return -ENOMEM;
 251         }
 252
 253         assert(j->level0->type == MATCH_AND_TERM);
 254         assert(j->level1->type == MATCH_OR_TERM);
 255         assert(j->level2->type == MATCH_AND_TERM);
 256
 257         le_hash = htole64(hash64(data, size));
 258
 259         LIST_FOREACH(matches, l3, j->level2->matches) {
 260                 assert(l3->type == MATCH_OR_TERM);
 261
 262                 LIST_FOREACH(matches, l4, l3->matches) {
 263                         assert(l4->type == MATCH_DISCRETE);
 264
 265                         /* Exactly the same match already? Then ignore
 266                          * this addition */
 267                         if (l4->le_hash == le_hash &&
 268                             l4->size == size &&
 269                             memcmp(l4->data, data, size) == 0)
 270                                 return 0;
 271
 272                         /* Same field? Then let's add this to this OR term */
 273                         if (same_field(data, size, l4->data, l4->size)) {
 274                                 add_here = l3;
 275                                 break;
 276                         }
 277                 }
 278
 279                 if (add_here)
 280                         break;
 281         }
 282
 283         if (!add_here) {
 284                 add_here = match_new(j->level2, MATCH_OR_TERM);
 285                 if (!add_here)
 286                         goto fail;
 287         }
 288
 289         m = match_new(add_here, MATCH_DISCRETE);
 290         if (!m)
 291                 goto fail;
 292
 293         m->le_hash = le_hash;
 294         m->size = size;
 295         m->data = memdup(data, size);
 296         if (!m->data)
 297                 goto fail;
 298
 299         detach_location(j);
 300
 301         return 0;
 302
 303 fail:
 304         match_free_if_empty(add_here);
 305         match_free_if_empty(j->level2);
 306         match_free_if_empty(j->level1);
 307         match_free_if_empty(j->level0);
 308
 309         return -ENOMEM;
 310 }
 311
 312 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 313         assert_return(j, -EINVAL);
 314         assert_return(!journal_pid_changed(j), -ECHILD);
 315
 316         if (!j->level0)
 317                 return 0;
 318
 319         if (!j->level1)
 320                 return 0;
 321
 322         if (!j->level1->matches)
 323                 return 0;
 324
 325         j->level1 = NULL;
 326         j->level2 = NULL;
 327
 328         return 0;
 329 }
 330
 331 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 332         assert_return(j, -EINVAL);
 333         assert_return(!journal_pid_changed(j), -ECHILD);
 334
 335         if (!j->level0)
 336                 return 0;
 337
 338         if (!j->level1)
 339                 return 0;
 340
 341         if (!j->level2)
 342                 return 0;
 343
 344         if (!j->level2->matches)
 345                 return 0;
 346
 347         j->level2 = NULL;
 348         return 0;
 349 }
 350
 351 static char *match_make_string(Match *m) {
 352         char *p, *r;
 353         Match *i;
 354         bool enclose = false;
 355
 356         if (!m)
 357                 return strdup("none");
 358
 359         if (m->type == MATCH_DISCRETE)
 360                 return strndup(m->data, m->size);
 361
 362         p = NULL;
 363         LIST_FOREACH(matches, i, m->matches) {
 364                 char *t, *k;
 365
 366                 t = match_make_string(i);
 367                 if (!t) {
 368                         free(p);
 369                         return NULL;
 370                 }
 371
 372                 if (p) {
 373                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 374                         free(p);
 375                         free(t);
 376
 377                         if (!k)
 378                                 return NULL;
 379
 380                         p = k;
 381
 382                         enclose = true;
 383                 } else
 384                         p = t;
 385         }
 386
 387         if (enclose) {
 388                 r = strjoin("(", p, ")", NULL);
 389                 free(p);
 390                 return r;
 391         }
 392
 393         return p;
 394 }
 395
 396 char *journal_make_match_string(sd_journal *j) {
 397         assert(j);
 398
 399         return match_make_string(j->level0);
 400 }
 401
 402 _public_ void sd_journal_flush_matches(sd_journal *j) {
 403         if (!j)
 404                 return;
 405
 406         if (j->level0)
 407                 match_free(j->level0);
 408
 409         j->level0 = j->level1 = j->level2 = NULL;
 410
 411         detach_location(j);
 412 }
 413
 414 static int compare_entry_order(JournalFile *af, Object *_ao,
 415                                JournalFile *bf, uint64_t bp) {
 416
 417         uint64_t a, b;
 418         Object *ao, *bo;
 419         int r;
 420
 421         assert(af);
 422         assert(bf);
 423         assert(_ao);
 424
 425         /* The mmap cache might invalidate the object from the first
 426          * file if we look at the one from the second file. Hence
 427          * temporarily copy the header of the first one, and look at
 428          * that only. */
 429         ao = alloca(offsetof(EntryObject, items));
 430         memcpy(ao, _ao, offsetof(EntryObject, items));
 431
 432         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 433         if (r < 0)
 434                 return strcmp(af->path, bf->path);
 435
 436         /* We operate on two different files here, hence we can access
 437          * two objects at the same time, which we normally can't.
 438          *
 439          * If contents and timestamps match, these entries are
 440          * identical, even if the seqnum does not match */
 441
 442         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 443             ao->entry.monotonic == bo->entry.monotonic &&
 444             ao->entry.realtime == bo->entry.realtime &&
 445             ao->entry.xor_hash == bo->entry.xor_hash)
 446                 return 0;
 447
 448         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 449
 450                 /* If this is from the same seqnum source, compare
 451                  * seqnums */
 452                 a = le64toh(ao->entry.seqnum);
 453                 b = le64toh(bo->entry.seqnum);
 454
 455                 if (a < b)
 456                         return -1;
 457                 if (a > b)
 458                         return 1;
 459
 460                 /* Wow! This is weird, different data but the same
 461                  * seqnums? Something is borked, but let's make the
 462                  * best of it and compare by time. */
 463         }
 464
 465         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 466
 467                 /* If the boot id matches compare monotonic time */
 468                 a = le64toh(ao->entry.monotonic);
 469                 b = le64toh(bo->entry.monotonic);
 470
 471                 if (a < b)
 472                         return -1;
 473                 if (a > b)
 474                         return 1;
 475         }
 476
 477         /* Otherwise compare UTC time */
 478         a = le64toh(ao->entry.realtime);
 479         b = le64toh(bo->entry.realtime);
 480
 481         if (a < b)
 482                 return -1;
 483         if (a > b)
 484                 return 1;
 485
 486         /* Finally, compare by contents */
 487         a = le64toh(ao->entry.xor_hash);
 488         b = le64toh(bo->entry.xor_hash);
 489
 490         if (a < b)
 491                 return -1;
 492         if (a > b)
 493                 return 1;
 494
 495         return 0;
 496 }
 497
 498 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 499         uint64_t a;
 500
 501         assert(af);
 502         assert(ao);
 503         assert(l);
 504         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 505
 506         if (l->monotonic_set &&
 507             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 508             l->realtime_set &&
 509             le64toh(ao->entry.realtime) == l->realtime &&
 510             l->xor_hash_set &&
 511             le64toh(ao->entry.xor_hash) == l->xor_hash)
 512                 return 0;
 513
 514         if (l->seqnum_set &&
 515             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 516
 517                 a = le64toh(ao->entry.seqnum);
 518
 519                 if (a < l->seqnum)
 520                         return -1;
 521                 if (a > l->seqnum)
 522                         return 1;
 523         }
 524
 525         if (l->monotonic_set &&
 526             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 527
 528                 a = le64toh(ao->entry.monotonic);
 529
 530                 if (a < l->monotonic)
 531                         return -1;
 532                 if (a > l->monotonic)
 533                         return 1;
 534         }
 535
 536         if (l->realtime_set) {
 537
 538                 a = le64toh(ao->entry.realtime);
 539
 540                 if (a < l->realtime)
 541                         return -1;
 542                 if (a > l->realtime)
 543                         return 1;
 544         }
 545
 546         if (l->xor_hash_set) {
 547                 a = le64toh(ao->entry.xor_hash);
 548
 549                 if (a < l->xor_hash)
 550                         return -1;
 551                 if (a > l->xor_hash)
 552                         return 1;
 553         }
 554
 555         return 0;
 556 }
 557
 558 static int next_for_match(
 559                 sd_journal *j,
 560                 Match *m,
 561                 JournalFile *f,
 562                 uint64_t after_offset,
 563                 direction_t direction,
 564                 Object **ret,
 565                 uint64_t *offset) {
 566
 567         int r;
 568         uint64_t np = 0;
 569         Object *n;
 570
 571         assert(j);
 572         assert(m);
 573         assert(f);
 574
 575         if (m->type == MATCH_DISCRETE) {
 576                 uint64_t dp;
 577
 578                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 579                 if (r <= 0)
 580                         return r;
 581
 582                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 583
 584         } else if (m->type == MATCH_OR_TERM) {
 585                 Match *i;
 586
 587                 /* Find the earliest match beyond after_offset */
 588
 589                 LIST_FOREACH(matches, i, m->matches) {
 590                         uint64_t cp;
 591
 592                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 593                         if (r < 0)
 594                                 return r;
 595                         else if (r > 0) {
 596                                 if (np == 0 || (direction == DIRECTION_DOWN ? cp < np : cp > np))
 597                                         np = cp;
 598                         }
 599                 }
 600
 601                 if (np == 0)
 602                         return 0;
 603
 604         } else if (m->type == MATCH_AND_TERM) {
 605                 Match *i, *last_moved;
 606
 607                 /* Always jump to the next matching entry and repeat
 608                  * this until we find an offset that matches for all
 609                  * matches. */
 610
 611                 if (!m->matches)
 612                         return 0;
 613
 614                 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
 615                 if (r <= 0)
 616                         return r;
 617
 618                 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
 619                 last_moved = m->matches;
 620
 621                 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
 622                         uint64_t cp;
 623
 624                         r = next_for_match(j, i, f, np, direction, NULL, &cp);
 625                         if (r <= 0)
 626                                 return r;
 627
 628                         assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
 629                         if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
 630                                 np = cp;
 631                                 last_moved = i;
 632                         }
 633                 }
 634         }
 635
 636         assert(np > 0);
 637
 638         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 639         if (r < 0)
 640                 return r;
 641
 642         if (ret)
 643                 *ret = n;
 644         if (offset)
 645                 *offset = np;
 646
 647         return 1;
 648 }
 649
 650 static int find_location_for_match(
 651                 sd_journal *j,
 652                 Match *m,
 653                 JournalFile *f,
 654                 direction_t direction,
 655                 Object **ret,
 656                 uint64_t *offset) {
 657
 658         int r;
 659
 660         assert(j);
 661         assert(m);
 662         assert(f);
 663
 664         if (m->type == MATCH_DISCRETE) {
 665                 uint64_t dp;
 666
 667                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 668                 if (r <= 0)
 669                         return r;
 670
 671                 /* FIXME: missing: find by monotonic */
 672
 673                 if (j->current_location.type == LOCATION_HEAD)
 674                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 675                 if (j->current_location.type == LOCATION_TAIL)
 676                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 677                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 678                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 679                 if (j->current_location.monotonic_set) {
 680                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 681                         if (r != -ENOENT)
 682                                 return r;
 683                 }
 684                 if (j->current_location.realtime_set)
 685                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 686
 687                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 688
 689         } else if (m->type == MATCH_OR_TERM) {
 690                 uint64_t np = 0;
 691                 Object *n;
 692                 Match *i;
 693
 694                 /* Find the earliest match */
 695
 696                 LIST_FOREACH(matches, i, m->matches) {
 697                         uint64_t cp;
 698
 699                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 700                         if (r < 0)
 701                                 return r;
 702                         else if (r > 0) {
 703                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 704                                         np = cp;
 705                         }
 706                 }
 707
 708                 if (np == 0)
 709                         return 0;
 710
 711                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 712                 if (r < 0)
 713                         return r;
 714
 715                 if (ret)
 716                         *ret = n;
 717                 if (offset)
 718                         *offset = np;
 719
 720                 return 1;
 721
 722         } else {
 723                 Match *i;
 724                 uint64_t np = 0;
 725
 726                 assert(m->type == MATCH_AND_TERM);
 727
 728                 /* First jump to the last match, and then find the
 729                  * next one where all matches match */
 730
 731                 if (!m->matches)
 732                         return 0;
 733
 734                 LIST_FOREACH(matches, i, m->matches) {
 735                         uint64_t cp;
 736
 737                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 738                         if (r <= 0)
 739                                 return r;
 740
 741                         if (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))
 742                                 np = cp;
 743                 }
 744
 745                 return next_for_match(j, m, f, np, direction, ret, offset);
 746         }
 747 }
 748
 749 static int find_location_with_matches(
 750                 sd_journal *j,
 751                 JournalFile *f,
 752                 direction_t direction,
 753                 Object **ret,
 754                 uint64_t *offset) {
 755
 756         int r;
 757
 758         assert(j);
 759         assert(f);
 760         assert(ret);
 761         assert(offset);
 762
 763         if (!j->level0) {
 764                 /* No matches is simple */
 765
 766                 if (j->current_location.type == LOCATION_HEAD)
 767                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 768                 if (j->current_location.type == LOCATION_TAIL)
 769                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 770                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 771                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 772                 if (j->current_location.monotonic_set) {
 773                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 774                         if (r != -ENOENT)
 775                                 return r;
 776                 }
 777                 if (j->current_location.realtime_set)
 778                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 779
 780                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 781         } else
 782                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 783 }
 784
 785 static int next_with_matches(
 786                 sd_journal *j,
 787                 JournalFile *f,
 788                 direction_t direction,
 789                 Object **ret,
 790                 uint64_t *offset) {
 791
 792         Object *c;
 793         uint64_t cp;
 794
 795         assert(j);
 796         assert(f);
 797         assert(ret);
 798         assert(offset);
 799
 800         c = *ret;
 801         cp = *offset;
 802
 803         /* No matches is easy. We simple advance the file
 804          * pointer by one. */
 805         if (!j->level0)
 806                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 807
 808         /* If we have a match then we look for the next matching entry
 809          * with an offset at least one step larger */
 810         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 811 }
 812
 813 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 814         Object *c;
 815         uint64_t cp;
 816         int r;
 817
 818         assert(j);
 819         assert(f);
 820
 821         if (f->last_direction == direction && f->current_offset > 0) {
 822                 cp = f->current_offset;
 823
 824                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 825                 if (r < 0)
 826                         return r;
 827
 828                 r = next_with_matches(j, f, direction, &c, &cp);
 829                 if (r <= 0)
 830                         return r;
 831         } else {
 832                 r = find_location_with_matches(j, f, direction, &c, &cp);
 833                 if (r <= 0)
 834                         return r;
 835         }
 836
 837         /* OK, we found the spot, now let's advance until an entry
 838          * that is actually different from what we were previously
 839          * looking at. This is necessary to handle entries which exist
 840          * in two (or more) journal files, and which shall all be
 841          * suppressed but one. */
 842
 843         for (;;) {
 844                 bool found;
 845
 846                 if (j->current_location.type == LOCATION_DISCRETE) {
 847                         int k;
 848
 849                         k = compare_with_location(f, c, &j->current_location);
 850                         if (direction == DIRECTION_DOWN)
 851                                 found = k > 0;
 852                         else
 853                                 found = k < 0;
 854                 } else
 855                         found = true;
 856
 857                 if (found) {
 858                         if (ret)
 859                                 *ret = c;
 860                         if (offset)
 861                                 *offset = cp;
 862                         return 1;
 863                 }
 864
 865                 r = next_with_matches(j, f, direction, &c, &cp);
 866                 if (r <= 0)
 867                         return r;
 868         }
 869 }
 870
 871 static int real_journal_next(sd_journal *j, direction_t direction) {
 872         JournalFile *f, *new_file = NULL;
 873         uint64_t new_offset = 0;
 874         Object *o;
 875         uint64_t p;
 876         Iterator i;
 877         int r;
 878
 879         assert_return(j, -EINVAL);
 880         assert_return(!journal_pid_changed(j), -ECHILD);
 881
 882         HASHMAP_FOREACH(f, j->files, i) {
 883                 bool found;
 884
 885                 r = next_beyond_location(j, f, direction, &o, &p);
 886                 if (r < 0) {
 887                         log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
 888                         continue;
 889                 } else if (r == 0)
 890                         continue;
 891
 892                 if (!new_file)
 893                         found = true;
 894                 else {
 895                         int k;
 896
 897                         k = compare_entry_order(f, o, new_file, new_offset);
 898
 899                         found = direction == DIRECTION_DOWN ? k < 0 : k > 0;
 900                 }
 901
 902                 if (found) {
 903                         new_file = f;
 904                         new_offset = p;
 905                 }
 906         }
 907
 908         if (!new_file)
 909                 return 0;
 910
 911         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 912         if (r < 0)
 913                 return r;
 914
 915         set_location(j, LOCATION_DISCRETE, new_file, o, direction, new_offset);
 916
 917         return 1;
 918 }
 919
 920 _public_ int sd_journal_next(sd_journal *j) {
 921         return real_journal_next(j, DIRECTION_DOWN);
 922 }
 923
 924 _public_ int sd_journal_previous(sd_journal *j) {
 925         return real_journal_next(j, DIRECTION_UP);
 926 }
 927
 928 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 929         int c = 0, r;
 930
 931         assert_return(j, -EINVAL);
 932         assert_return(!journal_pid_changed(j), -ECHILD);
 933
 934         if (skip == 0) {
 935                 /* If this is not a discrete skip, then at least
 936                  * resolve the current location */
 937                 if (j->current_location.type != LOCATION_DISCRETE)
 938                         return real_journal_next(j, direction);
 939
 940                 return 0;
 941         }
 942
 943         do {
 944                 r = real_journal_next(j, direction);
 945                 if (r < 0)
 946                         return r;
 947
 948                 if (r == 0)
 949                         return c;
 950
 951                 skip--;
 952                 c++;
 953         } while (skip > 0);
 954
 955         return c;
 956 }
 957
 958 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 959         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 960 }
 961
 962 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 963         return real_journal_next_skip(j, DIRECTION_UP, skip);
 964 }
 965
 966 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 967         Object *o;
 968         int r;
 969         char bid[33], sid[33];
 970
 971         assert_return(j, -EINVAL);
 972         assert_return(!journal_pid_changed(j), -ECHILD);
 973         assert_return(cursor, -EINVAL);
 974
 975         if (!j->current_file || j->current_file->current_offset <= 0)
 976                 return -EADDRNOTAVAIL;
 977
 978         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 979         if (r < 0)
 980                 return r;
 981
 982         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 983         sd_id128_to_string(o->entry.boot_id, bid);
 984
 985         if (asprintf(cursor,
 986                      "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
 987                      sid, le64toh(o->entry.seqnum),
 988                      bid, le64toh(o->entry.monotonic),
 989                      le64toh(o->entry.realtime),
 990                      le64toh(o->entry.xor_hash)) < 0)
 991                 return -ENOMEM;
 992
 993         return 0;
 994 }
 995
 996 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
 997         char *w, *state;
 998         size_t l;
 999         unsigned long long seqnum, monotonic, realtime, xor_hash;
1000         bool
1001                 seqnum_id_set = false,
1002                 seqnum_set = false,
1003                 boot_id_set = false,
1004                 monotonic_set = false,
1005                 realtime_set = false,
1006                 xor_hash_set = false;
1007         sd_id128_t seqnum_id, boot_id;
1008
1009         assert_return(j, -EINVAL);
1010         assert_return(!journal_pid_changed(j), -ECHILD);
1011         assert_return(!isempty(cursor), -EINVAL);
1012
1013         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1014                 char *item;
1015                 int k = 0;
1016
1017                 if (l < 2 || w[1] != '=')
1018                         return -EINVAL;
1019
1020                 item = strndup(w, l);
1021                 if (!item)
1022                         return -ENOMEM;
1023
1024                 switch (w[0]) {
1025
1026                 case 's':
1027                         seqnum_id_set = true;
1028                         k = sd_id128_from_string(item+2, &seqnum_id);
1029                         break;
1030
1031                 case 'i':
1032                         seqnum_set = true;
1033                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1034                                 k = -EINVAL;
1035                         break;
1036
1037                 case 'b':
1038                         boot_id_set = true;
1039                         k = sd_id128_from_string(item+2, &boot_id);
1040                         break;
1041
1042                 case 'm':
1043                         monotonic_set = true;
1044                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1045                                 k = -EINVAL;
1046                         break;
1047
1048                 case 't':
1049                         realtime_set = true;
1050                         if (sscanf(item+2, "%llx", &realtime) != 1)
1051                                 k = -EINVAL;
1052                         break;
1053
1054                 case 'x':
1055                         xor_hash_set = true;
1056                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1057                                 k = -EINVAL;
1058                         break;
1059                 }
1060
1061                 free(item);
1062
1063                 if (k < 0)
1064                         return k;
1065         }
1066
1067         if ((!seqnum_set || !seqnum_id_set) &&
1068             (!monotonic_set || !boot_id_set) &&
1069             !realtime_set)
1070                 return -EINVAL;
1071
1072         reset_location(j);
1073
1074         j->current_location.type = LOCATION_SEEK;
1075
1076         if (realtime_set) {
1077                 j->current_location.realtime = (uint64_t) realtime;
1078                 j->current_location.realtime_set = true;
1079         }
1080
1081         if (seqnum_set && seqnum_id_set) {
1082                 j->current_location.seqnum = (uint64_t) seqnum;
1083                 j->current_location.seqnum_id = seqnum_id;
1084                 j->current_location.seqnum_set = true;
1085         }
1086
1087         if (monotonic_set && boot_id_set) {
1088                 j->current_location.monotonic = (uint64_t) monotonic;
1089                 j->current_location.boot_id = boot_id;
1090                 j->current_location.monotonic_set = true;
1091         }
1092
1093         if (xor_hash_set) {
1094                 j->current_location.xor_hash = (uint64_t) xor_hash;
1095                 j->current_location.xor_hash_set = true;
1096         }
1097
1098         return 0;
1099 }
1100
1101 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1102         int r;
1103         char *w, *state;
1104         size_t l;
1105         Object *o;
1106
1107         assert_return(j, -EINVAL);
1108         assert_return(!journal_pid_changed(j), -ECHILD);
1109         assert_return(!isempty(cursor), -EINVAL);
1110
1111         if (!j->current_file || j->current_file->current_offset <= 0)
1112                 return -EADDRNOTAVAIL;
1113
1114         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1115         if (r < 0)
1116                 return r;
1117
1118         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1119                 _cleanup_free_ char *item = NULL;
1120                 sd_id128_t id;
1121                 unsigned long long ll;
1122                 int k = 0;
1123
1124                 if (l < 2 || w[1] != '=')
1125                         return -EINVAL;
1126
1127                 item = strndup(w, l);
1128                 if (!item)
1129                         return -ENOMEM;
1130
1131                 switch (w[0]) {
1132
1133                 case 's':
1134                         k = sd_id128_from_string(item+2, &id);
1135                         if (k < 0)
1136                                 return k;
1137                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1138                                 return 0;
1139                         break;
1140
1141                 case 'i':
1142                         if (sscanf(item+2, "%llx", &ll) != 1)
1143                                 return -EINVAL;
1144                         if (ll != le64toh(o->entry.seqnum))
1145                                 return 0;
1146                         break;
1147
1148                 case 'b':
1149                         k = sd_id128_from_string(item+2, &id);
1150                         if (k < 0)
1151                                 return k;
1152                         if (!sd_id128_equal(id, o->entry.boot_id))
1153                                 return 0;
1154                         break;
1155
1156                 case 'm':
1157                         if (sscanf(item+2, "%llx", &ll) != 1)
1158                                 return -EINVAL;
1159                         if (ll != le64toh(o->entry.monotonic))
1160                                 return 0;
1161                         break;
1162
1163                 case 't':
1164                         if (sscanf(item+2, "%llx", &ll) != 1)
1165                                 return -EINVAL;
1166                         if (ll != le64toh(o->entry.realtime))
1167                                 return 0;
1168                         break;
1169
1170                 case 'x':
1171                         if (sscanf(item+2, "%llx", &ll) != 1)
1172                                 return -EINVAL;
1173                         if (ll != le64toh(o->entry.xor_hash))
1174                                 return 0;
1175                         break;
1176                 }
1177         }
1178
1179         return 1;
1180 }
1181
1182
1183 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1184         assert_return(j, -EINVAL);
1185         assert_return(!journal_pid_changed(j), -ECHILD);
1186
1187         reset_location(j);
1188         j->current_location.type = LOCATION_SEEK;
1189         j->current_location.boot_id = boot_id;
1190         j->current_location.monotonic = usec;
1191         j->current_location.monotonic_set = true;
1192
1193         return 0;
1194 }
1195
1196 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1197         assert_return(j, -EINVAL);
1198         assert_return(!journal_pid_changed(j), -ECHILD);
1199
1200         reset_location(j);
1201         j->current_location.type = LOCATION_SEEK;
1202         j->current_location.realtime = usec;
1203         j->current_location.realtime_set = true;
1204
1205         return 0;
1206 }
1207
1208 _public_ int sd_journal_seek_head(sd_journal *j) {
1209         assert_return(j, -EINVAL);
1210         assert_return(!journal_pid_changed(j), -ECHILD);
1211
1212         reset_location(j);
1213         j->current_location.type = LOCATION_HEAD;
1214
1215         return 0;
1216 }
1217
1218 _public_ int sd_journal_seek_tail(sd_journal *j) {
1219         assert_return(j, -EINVAL);
1220         assert_return(!journal_pid_changed(j), -ECHILD);
1221
1222         reset_location(j);
1223         j->current_location.type = LOCATION_TAIL;
1224
1225         return 0;
1226 }
1227
1228 static void check_network(sd_journal *j, int fd) {
1229         struct statfs sfs;
1230
1231         assert(j);
1232
1233         if (j->on_network)
1234                 return;
1235
1236         if (fstatfs(fd, &sfs) < 0)
1237                 return;
1238
1239         j->on_network =
1240                 F_TYPE_EQUAL(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1241                 F_TYPE_EQUAL(sfs.f_type, CODA_SUPER_MAGIC) ||
1242                 F_TYPE_EQUAL(sfs.f_type, NCP_SUPER_MAGIC) ||
1243                 F_TYPE_EQUAL(sfs.f_type, NFS_SUPER_MAGIC) ||
1244                 F_TYPE_EQUAL(sfs.f_type, SMB_SUPER_MAGIC);
1245 }
1246
1247 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1248         const char *full, *tilded, *atted;
1249
1250         full = strappenda(prefix, ".journal");
1251         tilded = strappenda(full, "~");
1252         atted = strappenda(prefix, "@");
1253
1254         return streq(filename, full) ||
1255                streq(filename, tilded) ||
1256                startswith(filename, atted);
1257 }
1258
1259 static bool file_type_wanted(int flags, const char *filename) {
1260         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1261                 return false;
1262
1263         /* no flags set → every type is OK */
1264         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1265                 return true;
1266
1267         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1268                 return true;
1269
1270         if (flags & SD_JOURNAL_CURRENT_USER) {
1271                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1272
1273                 assert_se(snprintf(prefix, sizeof(prefix), "user-%lu", (unsigned long) getuid())
1274                           < (int) sizeof(prefix));
1275
1276                 if (file_has_type_prefix(prefix, filename))
1277                         return true;
1278         }
1279
1280         return false;
1281 }
1282
1283 static int add_any_file(sd_journal *j, const char *path) {
1284         JournalFile *f;
1285         int r;
1286
1287         assert(j);
1288         assert(path);
1289
1290         if (hashmap_get(j->files, path))
1291                 return 0;
1292
1293         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1294                 log_warning("Too many open journal files, not adding %s.", path);
1295                 return set_put_error(j, -ETOOMANYREFS);
1296         }
1297
1298         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1299         if (r < 0)
1300                 return r;
1301
1302         /* journal_file_dump(f); */
1303
1304         r = hashmap_put(j->files, f->path, f);
1305         if (r < 0) {
1306                 journal_file_close(f);
1307                 return r;
1308         }
1309
1310         log_debug("File %s added.", f->path);
1311
1312         check_network(j, f->fd);
1313
1314         j->current_invalidate_counter ++;
1315
1316         return 0;
1317 }
1318
1319 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1320         _cleanup_free_ char *path = NULL;
1321         int r;
1322
1323         assert(j);
1324         assert(prefix);
1325         assert(filename);
1326
1327         if (j->no_new_files ||
1328             !file_type_wanted(j->flags, filename))
1329                 return 0;
1330
1331         path = strjoin(prefix, "/", filename, NULL);
1332         if (!path)
1333                 return -ENOMEM;
1334
1335         r = add_any_file(j, path);
1336         if (r == -ENOENT)
1337                 return 0;
1338         return 0;
1339 }
1340
1341 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1342         char *path;
1343         JournalFile *f;
1344
1345         assert(j);
1346         assert(prefix);
1347         assert(filename);
1348
1349         path = strjoin(prefix, "/", filename, NULL);
1350         if (!path)
1351                 return -ENOMEM;
1352
1353         f = hashmap_get(j->files, path);
1354         free(path);
1355         if (!f)
1356                 return 0;
1357
1358         hashmap_remove(j->files, f->path);
1359
1360         log_debug("File %s removed.", f->path);
1361
1362         if (j->current_file == f) {
1363                 j->current_file = NULL;
1364                 j->current_field = 0;
1365         }
1366
1367         if (j->unique_file == f) {
1368                 j->unique_file = NULL;
1369                 j->unique_offset = 0;
1370         }
1371
1372         journal_file_close(f);
1373
1374         j->current_invalidate_counter ++;
1375
1376         return 0;
1377 }
1378
1379 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1380         _cleanup_free_ char *path = NULL;
1381         int r;
1382         _cleanup_closedir_ DIR *d = NULL;
1383         sd_id128_t id, mid;
1384         Directory *m;
1385
1386         assert(j);
1387         assert(prefix);
1388         assert(dirname);
1389
1390         log_debug("Considering %s/%s.", prefix, dirname);
1391
1392         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1393             (sd_id128_from_string(dirname, &id) < 0 ||
1394              sd_id128_get_machine(&mid) < 0 ||
1395              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1396             return 0;
1397
1398         path = strjoin(prefix, "/", dirname, NULL);
1399         if (!path)
1400                 return -ENOMEM;
1401
1402         d = opendir(path);
1403         if (!d) {
1404                 log_debug("Failed to open %s: %m", path);
1405                 if (errno == ENOENT)
1406                         return 0;
1407                 return -errno;
1408         }
1409
1410         m = hashmap_get(j->directories_by_path, path);
1411         if (!m) {
1412                 m = new0(Directory, 1);
1413                 if (!m)
1414                         return -ENOMEM;
1415
1416                 m->is_root = false;
1417                 m->path = path;
1418
1419                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1420                         free(m);
1421                         return -ENOMEM;
1422                 }
1423
1424                 path = NULL; /* avoid freeing in cleanup */
1425                 j->current_invalidate_counter ++;
1426
1427                 log_debug("Directory %s added.", m->path);
1428
1429         } else if (m->is_root)
1430                 return 0;
1431
1432         if (m->wd <= 0 && j->inotify_fd >= 0) {
1433
1434                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1435                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1436                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1437                                           IN_ONLYDIR);
1438
1439                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1440                         inotify_rm_watch(j->inotify_fd, m->wd);
1441         }
1442
1443         for (;;) {
1444                 struct dirent *de;
1445                 union dirent_storage buf;
1446
1447                 r = readdir_r(d, &buf.de, &de);
1448                 if (r != 0 || !de)
1449                         break;
1450
1451                 if (dirent_is_file_with_suffix(de, ".journal") ||
1452                     dirent_is_file_with_suffix(de, ".journal~")) {
1453                         r = add_file(j, m->path, de->d_name);
1454                         if (r < 0) {
1455                                 log_debug("Failed to add file %s/%s: %s",
1456                                           m->path, de->d_name, strerror(-r));
1457                                 r = set_put_error(j, r);
1458                                 if (r < 0)
1459                                         return r;
1460                         }
1461                 }
1462         }
1463
1464         check_network(j, dirfd(d));
1465
1466         return 0;
1467 }
1468
1469 static int add_root_directory(sd_journal *j, const char *p, const char *prefix) {
1470         _cleanup_closedir_ DIR *d = NULL;
1471         Directory *m;
1472         int r;
1473
1474         assert(j);
1475         assert(p);
1476
1477         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1478             !path_startswith(p, "/run"))
1479                 return -EINVAL;
1480
1481         if (prefix)
1482                 p = strappenda(prefix, p);
1483
1484         d = opendir(p);
1485         if (!d)
1486                 return -errno;
1487
1488         m = hashmap_get(j->directories_by_path, p);
1489         if (!m) {
1490                 m = new0(Directory, 1);
1491                 if (!m)
1492                         return -ENOMEM;
1493
1494                 m->is_root = true;
1495                 m->path = strdup(p);
1496                 if (!m->path) {
1497                         free(m);
1498                         return -ENOMEM;
1499                 }
1500
1501                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1502                         free(m->path);
1503                         free(m);
1504                         return -ENOMEM;
1505                 }
1506
1507                 j->current_invalidate_counter ++;
1508
1509                 log_debug("Root directory %s added.", m->path);
1510
1511         } else if (!m->is_root)
1512                 return 0;
1513
1514         if (m->wd <= 0 && j->inotify_fd >= 0) {
1515
1516                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1517                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1518                                           IN_ONLYDIR);
1519
1520                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1521                         inotify_rm_watch(j->inotify_fd, m->wd);
1522         }
1523
1524         if (j->no_new_files)
1525                 return 0;
1526
1527         for (;;) {
1528                 struct dirent *de;
1529                 union dirent_storage buf;
1530                 sd_id128_t id;
1531
1532                 r = readdir_r(d, &buf.de, &de);
1533                 if (r != 0 || !de)
1534                         break;
1535
1536                 if (dirent_is_file_with_suffix(de, ".journal") ||
1537                     dirent_is_file_with_suffix(de, ".journal~")) {
1538                         r = add_file(j, m->path, de->d_name);
1539                         if (r < 0) {
1540                                 log_debug("Failed to add file %s/%s: %s",
1541                                           m->path, de->d_name, strerror(-r));
1542                                 r = set_put_error(j, r);
1543                                 if (r < 0)
1544                                         return r;
1545                         }
1546                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1547                            sd_id128_from_string(de->d_name, &id) >= 0) {
1548
1549                         r = add_directory(j, m->path, de->d_name);
1550                         if (r < 0)
1551                                 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1552                 }
1553         }
1554
1555         check_network(j, dirfd(d));
1556
1557         return 0;
1558 }
1559
1560 static int remove_directory(sd_journal *j, Directory *d) {
1561         assert(j);
1562
1563         if (d->wd > 0) {
1564                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1565
1566                 if (j->inotify_fd >= 0)
1567                         inotify_rm_watch(j->inotify_fd, d->wd);
1568         }
1569
1570         hashmap_remove(j->directories_by_path, d->path);
1571
1572         if (d->is_root)
1573                 log_debug("Root directory %s removed.", d->path);
1574         else
1575                 log_debug("Directory %s removed.", d->path);
1576
1577         free(d->path);
1578         free(d);
1579
1580         return 0;
1581 }
1582
1583 static int add_search_paths(sd_journal *j, const char *prefix) {
1584         int r;
1585         const char search_paths[] =
1586                 "/run/log/journal\0"
1587                 "/var/log/journal\0";
1588         const char *p;
1589
1590         assert(j);
1591
1592         /* We ignore most errors here, since the idea is to only open
1593          * what's actually accessible, and ignore the rest. */
1594
1595         NULSTR_FOREACH(p, search_paths) {
1596                 r = add_root_directory(j, p, prefix);
1597                 if (r < 0 && r != -ENOENT) {
1598                         r = set_put_error(j, r);
1599                         if (r < 0)
1600                                 return r;
1601                 }
1602         }
1603
1604         return 0;
1605 }
1606
1607 static int add_current_paths(sd_journal *j) {
1608         Iterator i;
1609         JournalFile *f;
1610
1611         assert(j);
1612         assert(j->no_new_files);
1613
1614         /* Simply adds all directories for files we have open as
1615          * "root" directories. We don't expect errors here, so we
1616          * treat them as fatal. */
1617
1618         HASHMAP_FOREACH(f, j->files, i) {
1619                 int r;
1620                 _cleanup_free_ char *dir;
1621
1622                 dir = dirname_malloc(f->path);
1623                 if (!dir)
1624                         return -ENOMEM;
1625
1626                 r = add_root_directory(j, dir, NULL);
1627                 if (r < 0) {
1628                         set_put_error(j, r);
1629                         return r;
1630                 }
1631         }
1632
1633         return 0;
1634 }
1635
1636
1637 static int allocate_inotify(sd_journal *j) {
1638         assert(j);
1639
1640         if (j->inotify_fd < 0) {
1641                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1642                 if (j->inotify_fd < 0)
1643                         return -errno;
1644         }
1645
1646         if (!j->directories_by_wd) {
1647                 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1648                 if (!j->directories_by_wd)
1649                         return -ENOMEM;
1650         }
1651
1652         return 0;
1653 }
1654
1655 static sd_journal *journal_new(int flags, const char *path) {
1656         sd_journal *j;
1657
1658         j = new0(sd_journal, 1);
1659         if (!j)
1660                 return NULL;
1661
1662         j->original_pid = getpid();
1663         j->inotify_fd = -1;
1664         j->flags = flags;
1665         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1666
1667         if (path) {
1668                 j->path = strdup(path);
1669                 if (!j->path)
1670                         goto fail;
1671         }
1672
1673         j->files = hashmap_new(string_hash_func, string_compare_func);
1674         j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1675         j->mmap = mmap_cache_new();
1676         if (!j->files || !j->directories_by_path || !j->mmap)
1677                 goto fail;
1678
1679         return j;
1680
1681 fail:
1682         sd_journal_close(j);
1683         return NULL;
1684 }
1685
1686 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1687         sd_journal *j;
1688         int r;
1689
1690         assert_return(ret, -EINVAL);
1691         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_RUNTIME_ONLY|SD_JOURNAL_SYSTEM|SD_JOURNAL_CURRENT_USER)) == 0, -EINVAL);
1692
1693         j = journal_new(flags, NULL);
1694         if (!j)
1695                 return -ENOMEM;
1696
1697         r = add_search_paths(j, NULL);
1698         if (r < 0)
1699                 goto fail;
1700
1701         *ret = j;
1702         return 0;
1703
1704 fail:
1705         sd_journal_close(j);
1706
1707         return r;
1708 }
1709
1710 _public_ int sd_journal_open_container(sd_journal **ret, const char *machine, int flags) {
1711         _cleanup_free_ char *root = NULL, *class = NULL;
1712         sd_journal *j;
1713         char *p;
1714         int r;
1715
1716         assert_return(machine, -EINVAL);
1717         assert_return(ret, -EINVAL);
1718         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_SYSTEM)) == 0, -EINVAL);
1719         assert_return(filename_is_safe(machine), -EINVAL);
1720
1721         p = strappenda("/run/systemd/machines/", machine);
1722         r = parse_env_file(p, NEWLINE, "ROOT", &root, "CLASS", &class, NULL);
1723         if (r == -ENOENT)
1724                 return -EHOSTDOWN;
1725         if (r < 0)
1726                 return r;
1727         if (!root)
1728                 return -ENODATA;
1729
1730         if (!streq_ptr(class, "container"))
1731                 return -EIO;
1732
1733         j = journal_new(flags, NULL);
1734         if (!j)
1735                 return -ENOMEM;
1736
1737         r = add_search_paths(j, root);
1738         if (r < 0)
1739                 goto fail;
1740
1741         *ret = j;
1742         return 0;
1743
1744 fail:
1745         sd_journal_close(j);
1746         return r;
1747 }
1748
1749 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1750         sd_journal *j;
1751         int r;
1752
1753         assert_return(ret, -EINVAL);
1754         assert_return(path, -EINVAL);
1755         assert_return(flags == 0, -EINVAL);
1756
1757         j = journal_new(flags, path);
1758         if (!j)
1759                 return -ENOMEM;
1760
1761         r = add_root_directory(j, path, NULL);
1762         if (r < 0) {
1763                 set_put_error(j, r);
1764                 goto fail;
1765         }
1766
1767         *ret = j;
1768         return 0;
1769
1770 fail:
1771         sd_journal_close(j);
1772
1773         return r;
1774 }
1775
1776 _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int flags) {
1777         sd_journal *j;
1778         const char **path;
1779         int r;
1780
1781         assert_return(ret, -EINVAL);
1782         assert_return(flags == 0, -EINVAL);
1783
1784         j = journal_new(flags, NULL);
1785         if (!j)
1786                 return -ENOMEM;
1787
1788         STRV_FOREACH(path, paths) {
1789                 r = add_any_file(j, *path);
1790                 if (r < 0) {
1791                         log_error("Failed to open %s: %s", *path, strerror(-r));
1792                         goto fail;
1793                 }
1794         }
1795
1796         j->no_new_files = true;
1797
1798         *ret = j;
1799         return 0;
1800
1801 fail:
1802         sd_journal_close(j);
1803
1804         return r;
1805 }
1806
1807 _public_ void sd_journal_close(sd_journal *j) {
1808         Directory *d;
1809         JournalFile *f;
1810
1811         if (!j)
1812                 return;
1813
1814         sd_journal_flush_matches(j);
1815
1816         while ((f = hashmap_steal_first(j->files)))
1817                 journal_file_close(f);
1818
1819         hashmap_free(j->files);
1820
1821         while ((d = hashmap_first(j->directories_by_path)))
1822                 remove_directory(j, d);
1823
1824         while ((d = hashmap_first(j->directories_by_wd)))
1825                 remove_directory(j, d);
1826
1827         hashmap_free(j->directories_by_path);
1828         hashmap_free(j->directories_by_wd);
1829
1830         if (j->inotify_fd >= 0)
1831                 close_nointr_nofail(j->inotify_fd);
1832
1833         if (j->mmap) {
1834                 log_debug("mmap cache statistics: %u hit, %u miss", mmap_cache_get_hit(j->mmap), mmap_cache_get_missed(j->mmap));
1835                 mmap_cache_unref(j->mmap);
1836         }
1837
1838         free(j->path);
1839         free(j->unique_field);
1840         set_free(j->errors);
1841         free(j);
1842 }
1843
1844 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1845         Object *o;
1846         JournalFile *f;
1847         int r;
1848
1849         assert_return(j, -EINVAL);
1850         assert_return(!journal_pid_changed(j), -ECHILD);
1851         assert_return(ret, -EINVAL);
1852
1853         f = j->current_file;
1854         if (!f)
1855                 return -EADDRNOTAVAIL;
1856
1857         if (f->current_offset <= 0)
1858                 return -EADDRNOTAVAIL;
1859
1860         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1861         if (r < 0)
1862                 return r;
1863
1864         *ret = le64toh(o->entry.realtime);
1865         return 0;
1866 }
1867
1868 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1869         Object *o;
1870         JournalFile *f;
1871         int r;
1872         sd_id128_t id;
1873
1874         assert_return(j, -EINVAL);
1875         assert_return(!journal_pid_changed(j), -ECHILD);
1876
1877         f = j->current_file;
1878         if (!f)
1879                 return -EADDRNOTAVAIL;
1880
1881         if (f->current_offset <= 0)
1882                 return -EADDRNOTAVAIL;
1883
1884         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1885         if (r < 0)
1886                 return r;
1887
1888         if (ret_boot_id)
1889                 *ret_boot_id = o->entry.boot_id;
1890         else {
1891                 r = sd_id128_get_boot(&id);
1892                 if (r < 0)
1893                         return r;
1894
1895                 if (!sd_id128_equal(id, o->entry.boot_id))
1896                         return -ESTALE;
1897         }
1898
1899         if (ret)
1900                 *ret = le64toh(o->entry.monotonic);
1901
1902         return 0;
1903 }
1904
1905 static bool field_is_valid(const char *field) {
1906         const char *p;
1907
1908         assert(field);
1909
1910         if (isempty(field))
1911                 return false;
1912
1913         if (startswith(field, "__"))
1914                 return false;
1915
1916         for (p = field; *p; p++) {
1917
1918                 if (*p == '_')
1919                         continue;
1920
1921                 if (*p >= 'A' && *p <= 'Z')
1922                         continue;
1923
1924                 if (*p >= '0' && *p <= '9')
1925                         continue;
1926
1927                 return false;
1928         }
1929
1930         return true;
1931 }
1932
1933 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1934         JournalFile *f;
1935         uint64_t i, n;
1936         size_t field_length;
1937         int r;
1938         Object *o;
1939
1940         assert_return(j, -EINVAL);
1941         assert_return(!journal_pid_changed(j), -ECHILD);
1942         assert_return(field, -EINVAL);
1943         assert_return(data, -EINVAL);
1944         assert_return(size, -EINVAL);
1945         assert_return(field_is_valid(field), -EINVAL);
1946
1947         f = j->current_file;
1948         if (!f)
1949                 return -EADDRNOTAVAIL;
1950
1951         if (f->current_offset <= 0)
1952                 return -EADDRNOTAVAIL;
1953
1954         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1955         if (r < 0)
1956                 return r;
1957
1958         field_length = strlen(field);
1959
1960         n = journal_file_entry_n_items(o);
1961         for (i = 0; i < n; i++) {
1962                 uint64_t p, l;
1963                 le64_t le_hash;
1964                 size_t t;
1965
1966                 p = le64toh(o->entry.items[i].object_offset);
1967                 le_hash = o->entry.items[i].hash;
1968                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1969                 if (r < 0)
1970                         return r;
1971
1972                 if (le_hash != o->data.hash)
1973                         return -EBADMSG;
1974
1975                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1976
1977                 if (o->object.flags & OBJECT_COMPRESSED) {
1978
1979 #ifdef HAVE_XZ
1980                         if (uncompress_startswith(o->data.payload, l,
1981                                                   &f->compress_buffer, &f->compress_buffer_size,
1982                                                   field, field_length, '=')) {
1983
1984                                 uint64_t rsize;
1985
1986                                 if (!uncompress_blob(o->data.payload, l,
1987                                                      &f->compress_buffer, &f->compress_buffer_size, &rsize,
1988                                                      j->data_threshold))
1989                                         return -EBADMSG;
1990
1991                                 *data = f->compress_buffer;
1992                                 *size = (size_t) rsize;
1993
1994                                 return 0;
1995                         }
1996 #else
1997                         return -EPROTONOSUPPORT;
1998 #endif
1999
2000                 } else if (l >= field_length+1 &&
2001                            memcmp(o->data.payload, field, field_length) == 0 &&
2002                            o->data.payload[field_length] == '=') {
2003
2004                         t = (size_t) l;
2005
2006                         if ((uint64_t) t != l)
2007                                 return -E2BIG;
2008
2009                         *data = o->data.payload;
2010                         *size = t;
2011
2012                         return 0;
2013                 }
2014
2015                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2016                 if (r < 0)
2017                         return r;
2018         }
2019
2020         return -ENOENT;
2021 }
2022
2023 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
2024         size_t t;
2025         uint64_t l;
2026
2027         l = le64toh(o->object.size) - offsetof(Object, data.payload);
2028         t = (size_t) l;
2029
2030         /* We can't read objects larger than 4G on a 32bit machine */
2031         if ((uint64_t) t != l)
2032                 return -E2BIG;
2033
2034         if (o->object.flags & OBJECT_COMPRESSED) {
2035 #ifdef HAVE_XZ
2036                 uint64_t rsize;
2037
2038                 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
2039                         return -EBADMSG;
2040
2041                 *data = f->compress_buffer;
2042                 *size = (size_t) rsize;
2043 #else
2044                 return -EPROTONOSUPPORT;
2045 #endif
2046         } else {
2047                 *data = o->data.payload;
2048                 *size = t;
2049         }
2050
2051         return 0;
2052 }
2053
2054 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
2055         JournalFile *f;
2056         uint64_t p, n;
2057         le64_t le_hash;
2058         int r;
2059         Object *o;
2060
2061         assert_return(j, -EINVAL);
2062         assert_return(!journal_pid_changed(j), -ECHILD);
2063         assert_return(data, -EINVAL);
2064         assert_return(size, -EINVAL);
2065
2066         f = j->current_file;
2067         if (!f)
2068                 return -EADDRNOTAVAIL;
2069
2070         if (f->current_offset <= 0)
2071                 return -EADDRNOTAVAIL;
2072
2073         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2074         if (r < 0)
2075                 return r;
2076
2077         n = journal_file_entry_n_items(o);
2078         if (j->current_field >= n)
2079                 return 0;
2080
2081         p = le64toh(o->entry.items[j->current_field].object_offset);
2082         le_hash = o->entry.items[j->current_field].hash;
2083         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2084         if (r < 0)
2085                 return r;
2086
2087         if (le_hash != o->data.hash)
2088                 return -EBADMSG;
2089
2090         r = return_data(j, f, o, data, size);
2091         if (r < 0)
2092                 return r;
2093
2094         j->current_field ++;
2095
2096         return 1;
2097 }
2098
2099 _public_ void sd_journal_restart_data(sd_journal *j) {
2100         if (!j)
2101                 return;
2102
2103         j->current_field = 0;
2104 }
2105
2106 _public_ int sd_journal_get_fd(sd_journal *j) {
2107         int r;
2108
2109         assert_return(j, -EINVAL);
2110         assert_return(!journal_pid_changed(j), -ECHILD);
2111
2112         if (j->inotify_fd >= 0)
2113                 return j->inotify_fd;
2114
2115         r = allocate_inotify(j);
2116         if (r < 0)
2117                 return r;
2118
2119         /* Iterate through all dirs again, to add them to the
2120          * inotify */
2121         if (j->no_new_files)
2122                 r = add_current_paths(j);
2123         else if (j->path)
2124                 r = add_root_directory(j, j->path, NULL);
2125         else
2126                 r = add_search_paths(j, NULL);
2127         if (r < 0)
2128                 return r;
2129
2130         return j->inotify_fd;
2131 }
2132
2133 _public_ int sd_journal_get_events(sd_journal *j) {
2134         int fd;
2135
2136         assert_return(j, -EINVAL);
2137         assert_return(!journal_pid_changed(j), -ECHILD);
2138
2139         fd = sd_journal_get_fd(j);
2140         if (fd < 0)
2141                 return fd;
2142
2143         return POLLIN;
2144 }
2145
2146 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2147         int fd;
2148
2149         assert_return(j, -EINVAL);
2150         assert_return(!journal_pid_changed(j), -ECHILD);
2151         assert_return(timeout_usec, -EINVAL);
2152
2153         fd = sd_journal_get_fd(j);
2154         if (fd < 0)
2155                 return fd;
2156
2157         if (!j->on_network) {
2158                 *timeout_usec = (uint64_t) -1;
2159                 return 0;
2160         }
2161
2162         /* If we are on the network we need to regularly check for
2163          * changes manually */
2164
2165         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2166         return 1;
2167 }
2168
2169 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2170         Directory *d;
2171         int r;
2172
2173         assert(j);
2174         assert(e);
2175
2176         /* Is this a subdirectory we watch? */
2177         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2178         if (d) {
2179                 sd_id128_t id;
2180
2181                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2182                     (endswith(e->name, ".journal") ||
2183                      endswith(e->name, ".journal~"))) {
2184
2185                         /* Event for a journal file */
2186
2187                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2188                                 r = add_file(j, d->path, e->name);
2189                                 if (r < 0) {
2190                                         log_debug("Failed to add file %s/%s: %s",
2191                                                   d->path, e->name, strerror(-r));
2192                                         set_put_error(j, r);
2193                                 }
2194
2195                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2196
2197                                 r = remove_file(j, d->path, e->name);
2198                                 if (r < 0)
2199                                         log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2200                         }
2201
2202                 } else if (!d->is_root && e->len == 0) {
2203
2204                         /* Event for a subdirectory */
2205
2206                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2207                                 r = remove_directory(j, d);
2208                                 if (r < 0)
2209                                         log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2210                         }
2211
2212
2213                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2214
2215                         /* Event for root directory */
2216
2217                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2218                                 r = add_directory(j, d->path, e->name);
2219                                 if (r < 0)
2220                                         log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2221                         }
2222                 }
2223
2224                 return;
2225         }
2226
2227         if (e->mask & IN_IGNORED)
2228                 return;
2229
2230         log_warning("Unknown inotify event.");
2231 }
2232
2233 static int determine_change(sd_journal *j) {
2234         bool b;
2235
2236         assert(j);
2237
2238         b = j->current_invalidate_counter != j->last_invalidate_counter;
2239         j->last_invalidate_counter = j->current_invalidate_counter;
2240
2241         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2242 }
2243
2244 _public_ int sd_journal_process(sd_journal *j) {
2245         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2246         bool got_something = false;
2247
2248         assert_return(j, -EINVAL);
2249         assert_return(!journal_pid_changed(j), -ECHILD);
2250
2251         j->last_process_usec = now(CLOCK_MONOTONIC);
2252
2253         for (;;) {
2254                 struct inotify_event *e;
2255                 ssize_t l;
2256
2257                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2258                 if (l < 0) {
2259                         if (errno == EAGAIN || errno == EINTR)
2260                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2261
2262                         return -errno;
2263                 }
2264
2265                 got_something = true;
2266
2267                 e = (struct inotify_event*) buffer;
2268                 while (l > 0) {
2269                         size_t step;
2270
2271                         process_inotify_event(j, e);
2272
2273                         step = sizeof(struct inotify_event) + e->len;
2274                         assert(step <= (size_t) l);
2275
2276                         e = (struct inotify_event*) ((uint8_t*) e + step);
2277                         l -= step;
2278                 }
2279         }
2280
2281         return determine_change(j);
2282 }
2283
2284 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2285         int r;
2286         uint64_t t;
2287
2288         assert_return(j, -EINVAL);
2289         assert_return(!journal_pid_changed(j), -ECHILD);
2290
2291         if (j->inotify_fd < 0) {
2292
2293                 /* This is the first invocation, hence create the
2294                  * inotify watch */
2295                 r = sd_journal_get_fd(j);
2296                 if (r < 0)
2297                         return r;
2298
2299                 /* The journal might have changed since the context
2300                  * object was created and we weren't watching before,
2301                  * hence don't wait for anything, and return
2302                  * immediately. */
2303                 return determine_change(j);
2304         }
2305
2306         r = sd_journal_get_timeout(j, &t);
2307         if (r < 0)
2308                 return r;
2309
2310         if (t != (uint64_t) -1) {
2311                 usec_t n;
2312
2313                 n = now(CLOCK_MONOTONIC);
2314                 t = t > n ? t - n : 0;
2315
2316                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2317                         timeout_usec = t;
2318         }
2319
2320         do {
2321                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2322         } while (r == -EINTR);
2323
2324         if (r < 0)
2325                 return r;
2326
2327         return sd_journal_process(j);
2328 }
2329
2330 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2331         Iterator i;
2332         JournalFile *f;
2333         bool first = true;
2334         int r;
2335
2336         assert_return(j, -EINVAL);
2337         assert_return(!journal_pid_changed(j), -ECHILD);
2338         assert_return(from || to, -EINVAL);
2339         assert_return(from != to, -EINVAL);
2340
2341         HASHMAP_FOREACH(f, j->files, i) {
2342                 usec_t fr, t;
2343
2344                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2345                 if (r == -ENOENT)
2346                         continue;
2347                 if (r < 0)
2348                         return r;
2349                 if (r == 0)
2350                         continue;
2351
2352                 if (first) {
2353                         if (from)
2354                                 *from = fr;
2355                         if (to)
2356                                 *to = t;
2357                         first = false;
2358                 } else {
2359                         if (from)
2360                                 *from = MIN(fr, *from);
2361                         if (to)
2362                                 *to = MAX(t, *to);
2363                 }
2364         }
2365
2366         return first ? 0 : 1;
2367 }
2368
2369 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2370         Iterator i;
2371         JournalFile *f;
2372         bool first = true;
2373         int r;
2374
2375         assert_return(j, -EINVAL);
2376         assert_return(!journal_pid_changed(j), -ECHILD);
2377         assert_return(from || to, -EINVAL);
2378         assert_return(from != to, -EINVAL);
2379
2380         HASHMAP_FOREACH(f, j->files, i) {
2381                 usec_t fr, t;
2382
2383                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2384                 if (r == -ENOENT)
2385                         continue;
2386                 if (r < 0)
2387                         return r;
2388                 if (r == 0)
2389                         continue;
2390
2391                 if (first) {
2392                         if (from)
2393                                 *from = fr;
2394                         if (to)
2395                                 *to = t;
2396                         first = false;
2397                 } else {
2398                         if (from)
2399                                 *from = MIN(fr, *from);
2400                         if (to)
2401                                 *to = MAX(t, *to);
2402                 }
2403         }
2404
2405         return first ? 0 : 1;
2406 }
2407
2408 void journal_print_header(sd_journal *j) {
2409         Iterator i;
2410         JournalFile *f;
2411         bool newline = false;
2412
2413         assert(j);
2414
2415         HASHMAP_FOREACH(f, j->files, i) {
2416                 if (newline)
2417                         putchar('\n');
2418                 else
2419                         newline = true;
2420
2421                 journal_file_print_header(f);
2422         }
2423 }
2424
2425 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2426         Iterator i;
2427         JournalFile *f;
2428         uint64_t sum = 0;
2429
2430         assert_return(j, -EINVAL);
2431         assert_return(!journal_pid_changed(j), -ECHILD);
2432         assert_return(bytes, -EINVAL);
2433
2434         HASHMAP_FOREACH(f, j->files, i) {
2435                 struct stat st;
2436
2437                 if (fstat(f->fd, &st) < 0)
2438                         return -errno;
2439
2440                 sum += (uint64_t) st.st_blocks * 512ULL;
2441         }
2442
2443         *bytes = sum;
2444         return 0;
2445 }
2446
2447 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2448         char *f;
2449
2450         assert_return(j, -EINVAL);
2451         assert_return(!journal_pid_changed(j), -ECHILD);
2452         assert_return(!isempty(field), -EINVAL);
2453         assert_return(field_is_valid(field), -EINVAL);
2454
2455         f = strdup(field);
2456         if (!f)
2457                 return -ENOMEM;
2458
2459         free(j->unique_field);
2460         j->unique_field = f;
2461         j->unique_file = NULL;
2462         j->unique_offset = 0;
2463
2464         return 0;
2465 }
2466
2467 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2468         Object *o;
2469         size_t k;
2470         int r;
2471
2472         assert_return(j, -EINVAL);
2473         assert_return(!journal_pid_changed(j), -ECHILD);
2474         assert_return(data, -EINVAL);
2475         assert_return(l, -EINVAL);
2476         assert_return(j->unique_field, -EINVAL);
2477
2478         k = strlen(j->unique_field);
2479
2480         if (!j->unique_file) {
2481                 j->unique_file = hashmap_first(j->files);
2482                 if (!j->unique_file)
2483                         return 0;
2484                 j->unique_offset = 0;
2485         }
2486
2487         for (;;) {
2488                 JournalFile *of;
2489                 Iterator i;
2490                 const void *odata;
2491                 size_t ol;
2492                 bool found;
2493
2494                 /* Proceed to next data object in the field's linked list */
2495                 if (j->unique_offset == 0) {
2496                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2497                         if (r < 0)
2498                                 return r;
2499
2500                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2501                 } else {
2502                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2503                         if (r < 0)
2504                                 return r;
2505
2506                         j->unique_offset = le64toh(o->data.next_field_offset);
2507                 }
2508
2509                 /* We reached the end of the list? Then start again, with the next file */
2510                 if (j->unique_offset == 0) {
2511                         JournalFile *n;
2512
2513                         n = hashmap_next(j->files, j->unique_file->path);
2514                         if (!n)
2515                                 return 0;
2516
2517                         j->unique_file = n;
2518                         continue;
2519                 }
2520
2521                 /* We do not use the type context here, but 0 instead,
2522                  * so that we can look at this data object at the same
2523                  * time as one on another file */
2524                 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2525                 if (r < 0)
2526                         return r;
2527
2528                 /* Let's do the type check by hand, since we used 0 context above. */
2529                 if (o->object.type != OBJECT_DATA)
2530                         return -EBADMSG;
2531
2532                 r = return_data(j, j->unique_file, o, &odata, &ol);
2533                 if (r < 0)
2534                         return r;
2535
2536                 /* OK, now let's see if we already returned this data
2537                  * object by checking if it exists in the earlier
2538                  * traversed files. */
2539                 found = false;
2540                 HASHMAP_FOREACH(of, j->files, i) {
2541                         Object *oo;
2542                         uint64_t op;
2543
2544                         if (of == j->unique_file)
2545                                 break;
2546
2547                         /* Skip this file it didn't have any fields
2548                          * indexed */
2549                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2550                             le64toh(of->header->n_fields) <= 0)
2551                                 continue;
2552
2553                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2554                         if (r < 0)
2555                                 return r;
2556
2557                         if (r > 0)
2558                                 found = true;
2559                 }
2560
2561                 if (found)
2562                         continue;
2563
2564                 r = return_data(j, j->unique_file, o, data, l);
2565                 if (r < 0)
2566                         return r;
2567
2568                 return 1;
2569         }
2570 }
2571
2572 _public_ void sd_journal_restart_unique(sd_journal *j) {
2573         if (!j)
2574                 return;
2575
2576         j->unique_file = NULL;
2577         j->unique_offset = 0;
2578 }
2579
2580 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2581         assert_return(j, -EINVAL);
2582         assert_return(!journal_pid_changed(j), -ECHILD);
2583
2584         return !j->on_network;
2585 }
2586
2587 static char *lookup_field(const char *field, void *userdata) {
2588         sd_journal *j = userdata;
2589         const void *data;
2590         size_t size, d;
2591         int r;
2592
2593         assert(field);
2594         assert(j);
2595
2596         r = sd_journal_get_data(j, field, &data, &size);
2597         if (r < 0 ||
2598             size > REPLACE_VAR_MAX)
2599                 return strdup(field);
2600
2601         d = strlen(field) + 1;
2602
2603         return strndup((const char*) data + d, size - d);
2604 }
2605
2606 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2607         const void *data;
2608         size_t size;
2609         sd_id128_t id;
2610         _cleanup_free_ char *text = NULL, *cid = NULL;
2611         char *t;
2612         int r;
2613
2614         assert_return(j, -EINVAL);
2615         assert_return(!journal_pid_changed(j), -ECHILD);
2616         assert_return(ret, -EINVAL);
2617
2618         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2619         if (r < 0)
2620                 return r;
2621
2622         cid = strndup((const char*) data + 11, size - 11);
2623         if (!cid)
2624                 return -ENOMEM;
2625
2626         r = sd_id128_from_string(cid, &id);
2627         if (r < 0)
2628                 return r;
2629
2630         r = catalog_get(CATALOG_DATABASE, id, &text);
2631         if (r < 0)
2632                 return r;
2633
2634         t = replace_var(text, lookup_field, j);
2635         if (!t)
2636                 return -ENOMEM;
2637
2638         *ret = t;
2639         return 0;
2640 }
2641
2642 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2643         assert_return(ret, -EINVAL);
2644
2645         return catalog_get(CATALOG_DATABASE, id, ret);
2646 }
2647
2648 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2649         assert_return(j, -EINVAL);
2650         assert_return(!journal_pid_changed(j), -ECHILD);
2651
2652         j->data_threshold = sz;
2653         return 0;
2654 }
2655
2656 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2657         assert_return(j, -EINVAL);
2658         assert_return(!journal_pid_changed(j), -ECHILD);
2659         assert_return(sz, -EINVAL);
2660
2661         *sz = j->data_threshold;
2662         return 0;
2663 }