src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "strv.h"
  37 #include "path-util.h"
  38 #include "lookup3.h"
  39 #include "compress.h"
  40 #include "journal-internal.h"
  41 #include "missing.h"
  42 #include "catalog.h"
  43 #include "replace-var.h"
  44 #include "fileio.h"
  45
  46 #define JOURNAL_FILES_MAX 1024
  47
  48 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  49
  50 #define REPLACE_VAR_MAX 256
  51
  52 #define DEFAULT_DATA_THRESHOLD (64*1024)
  53
  54 static void remove_file_real(sd_journal *j, JournalFile *f);
  55
  56 static bool journal_pid_changed(sd_journal *j) {
  57         assert(j);
  58
  59         /* We don't support people creating a journal object and
  60          * keeping it around over a fork(). Let's complain. */
  61
  62         return j->original_pid != getpid();
  63 }
  64
  65 /* We return an error here only if we didn't manage to
  66    memorize the real error. */
  67 static int set_put_error(sd_journal *j, int r) {
  68         int k;
  69
  70         if (r >= 0)
  71                 return r;
  72
  73         k = set_ensure_allocated(&j->errors, NULL);
  74         if (k < 0)
  75                 return k;
  76
  77         return set_put(j->errors, INT_TO_PTR(r));
  78 }
  79
  80 static void detach_location(sd_journal *j) {
  81         Iterator i;
  82         JournalFile *f;
  83
  84         assert(j);
  85
  86         j->current_file = NULL;
  87         j->current_field = 0;
  88
  89         ORDERED_HASHMAP_FOREACH(f, j->files, i)
  90                 f->current_offset = 0;
  91 }
  92
  93 static void reset_location(sd_journal *j) {
  94         assert(j);
  95
  96         detach_location(j);
  97         zero(j->current_location);
  98 }
  99
 100 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
 101         assert(l);
 102         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 103         assert(f);
 104         assert(o->object.type == OBJECT_ENTRY);
 105
 106         l->type = type;
 107         l->seqnum = le64toh(o->entry.seqnum);
 108         l->seqnum_id = f->header->seqnum_id;
 109         l->realtime = le64toh(o->entry.realtime);
 110         l->monotonic = le64toh(o->entry.monotonic);
 111         l->boot_id = o->entry.boot_id;
 112         l->xor_hash = le64toh(o->entry.xor_hash);
 113
 114         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 115 }
 116
 117 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o,
 118                          direction_t direction, uint64_t offset) {
 119         assert(j);
 120         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 121         assert(f);
 122         assert(o);
 123
 124         init_location(&j->current_location, type, f, o);
 125
 126         j->current_file = f;
 127         j->current_field = 0;
 128
 129         f->last_direction = direction;
 130         f->current_offset = offset;
 131 }
 132
 133 static int match_is_valid(const void *data, size_t size) {
 134         const char *b, *p;
 135
 136         assert(data);
 137
 138         if (size < 2)
 139                 return false;
 140
 141         if (startswith(data, "__"))
 142                 return false;
 143
 144         b = data;
 145         for (p = b; p < b + size; p++) {
 146
 147                 if (*p == '=')
 148                         return p > b;
 149
 150                 if (*p == '_')
 151                         continue;
 152
 153                 if (*p >= 'A' && *p <= 'Z')
 154                         continue;
 155
 156                 if (*p >= '0' && *p <= '9')
 157                         continue;
 158
 159                 return false;
 160         }
 161
 162         return false;
 163 }
 164
 165 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 166         const uint8_t *a = _a, *b = _b;
 167         size_t j;
 168
 169         for (j = 0; j < s && j < t; j++) {
 170
 171                 if (a[j] != b[j])
 172                         return false;
 173
 174                 if (a[j] == '=')
 175                         return true;
 176         }
 177
 178         assert_not_reached("\"=\" not found");
 179 }
 180
 181 static Match *match_new(Match *p, MatchType t) {
 182         Match *m;
 183
 184         m = new0(Match, 1);
 185         if (!m)
 186                 return NULL;
 187
 188         m->type = t;
 189
 190         if (p) {
 191                 m->parent = p;
 192                 LIST_PREPEND(matches, p->matches, m);
 193         }
 194
 195         return m;
 196 }
 197
 198 static void match_free(Match *m) {
 199         assert(m);
 200
 201         while (m->matches)
 202                 match_free(m->matches);
 203
 204         if (m->parent)
 205                 LIST_REMOVE(matches, m->parent->matches, m);
 206
 207         free(m->data);
 208         free(m);
 209 }
 210
 211 static void match_free_if_empty(Match *m) {
 212         if (!m || m->matches)
 213                 return;
 214
 215         match_free(m);
 216 }
 217
 218 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 219         Match *l3, *l4, *add_here = NULL, *m;
 220         le64_t le_hash;
 221
 222         assert_return(j, -EINVAL);
 223         assert_return(!journal_pid_changed(j), -ECHILD);
 224         assert_return(data, -EINVAL);
 225
 226         if (size == 0)
 227                 size = strlen(data);
 228
 229         assert_return(match_is_valid(data, size), -EINVAL);
 230
 231         /* level 0: AND term
 232          * level 1: OR terms
 233          * level 2: AND terms
 234          * level 3: OR terms
 235          * level 4: concrete matches */
 236
 237         if (!j->level0) {
 238                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 239                 if (!j->level0)
 240                         return -ENOMEM;
 241         }
 242
 243         if (!j->level1) {
 244                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 245                 if (!j->level1)
 246                         return -ENOMEM;
 247         }
 248
 249         if (!j->level2) {
 250                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 251                 if (!j->level2)
 252                         return -ENOMEM;
 253         }
 254
 255         assert(j->level0->type == MATCH_AND_TERM);
 256         assert(j->level1->type == MATCH_OR_TERM);
 257         assert(j->level2->type == MATCH_AND_TERM);
 258
 259         le_hash = htole64(hash64(data, size));
 260
 261         LIST_FOREACH(matches, l3, j->level2->matches) {
 262                 assert(l3->type == MATCH_OR_TERM);
 263
 264                 LIST_FOREACH(matches, l4, l3->matches) {
 265                         assert(l4->type == MATCH_DISCRETE);
 266
 267                         /* Exactly the same match already? Then ignore
 268                          * this addition */
 269                         if (l4->le_hash == le_hash &&
 270                             l4->size == size &&
 271                             memcmp(l4->data, data, size) == 0)
 272                                 return 0;
 273
 274                         /* Same field? Then let's add this to this OR term */
 275                         if (same_field(data, size, l4->data, l4->size)) {
 276                                 add_here = l3;
 277                                 break;
 278                         }
 279                 }
 280
 281                 if (add_here)
 282                         break;
 283         }
 284
 285         if (!add_here) {
 286                 add_here = match_new(j->level2, MATCH_OR_TERM);
 287                 if (!add_here)
 288                         goto fail;
 289         }
 290
 291         m = match_new(add_here, MATCH_DISCRETE);
 292         if (!m)
 293                 goto fail;
 294
 295         m->le_hash = le_hash;
 296         m->size = size;
 297         m->data = memdup(data, size);
 298         if (!m->data)
 299                 goto fail;
 300
 301         detach_location(j);
 302
 303         return 0;
 304
 305 fail:
 306         match_free_if_empty(add_here);
 307         match_free_if_empty(j->level2);
 308         match_free_if_empty(j->level1);
 309         match_free_if_empty(j->level0);
 310
 311         return -ENOMEM;
 312 }
 313
 314 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 315         assert_return(j, -EINVAL);
 316         assert_return(!journal_pid_changed(j), -ECHILD);
 317
 318         if (!j->level0)
 319                 return 0;
 320
 321         if (!j->level1)
 322                 return 0;
 323
 324         if (!j->level1->matches)
 325                 return 0;
 326
 327         j->level1 = NULL;
 328         j->level2 = NULL;
 329
 330         return 0;
 331 }
 332
 333 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 334         assert_return(j, -EINVAL);
 335         assert_return(!journal_pid_changed(j), -ECHILD);
 336
 337         if (!j->level0)
 338                 return 0;
 339
 340         if (!j->level1)
 341                 return 0;
 342
 343         if (!j->level2)
 344                 return 0;
 345
 346         if (!j->level2->matches)
 347                 return 0;
 348
 349         j->level2 = NULL;
 350         return 0;
 351 }
 352
 353 static char *match_make_string(Match *m) {
 354         char *p, *r;
 355         Match *i;
 356         bool enclose = false;
 357
 358         if (!m)
 359                 return strdup("none");
 360
 361         if (m->type == MATCH_DISCRETE)
 362                 return strndup(m->data, m->size);
 363
 364         p = NULL;
 365         LIST_FOREACH(matches, i, m->matches) {
 366                 char *t, *k;
 367
 368                 t = match_make_string(i);
 369                 if (!t) {
 370                         free(p);
 371                         return NULL;
 372                 }
 373
 374                 if (p) {
 375                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 376                         free(p);
 377                         free(t);
 378
 379                         if (!k)
 380                                 return NULL;
 381
 382                         p = k;
 383
 384                         enclose = true;
 385                 } else
 386                         p = t;
 387         }
 388
 389         if (enclose) {
 390                 r = strjoin("(", p, ")", NULL);
 391                 free(p);
 392                 return r;
 393         }
 394
 395         return p;
 396 }
 397
 398 char *journal_make_match_string(sd_journal *j) {
 399         assert(j);
 400
 401         return match_make_string(j->level0);
 402 }
 403
 404 _public_ void sd_journal_flush_matches(sd_journal *j) {
 405         if (!j)
 406                 return;
 407
 408         if (j->level0)
 409                 match_free(j->level0);
 410
 411         j->level0 = j->level1 = j->level2 = NULL;
 412
 413         detach_location(j);
 414 }
 415
 416 static int compare_entry_order(JournalFile *af, Object *_ao,
 417                                JournalFile *bf, uint64_t bp) {
 418
 419         uint64_t a, b;
 420         Object *ao, *bo;
 421         int r;
 422
 423         assert(af);
 424         assert(bf);
 425         assert(_ao);
 426
 427         /* The mmap cache might invalidate the object from the first
 428          * file if we look at the one from the second file. Hence
 429          * temporarily copy the header of the first one, and look at
 430          * that only. */
 431         ao = alloca(offsetof(EntryObject, items));
 432         memcpy(ao, _ao, offsetof(EntryObject, items));
 433
 434         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 435         if (r < 0)
 436                 return strcmp(af->path, bf->path);
 437
 438         /* We operate on two different files here, hence we can access
 439          * two objects at the same time, which we normally can't.
 440          *
 441          * If contents and timestamps match, these entries are
 442          * identical, even if the seqnum does not match */
 443
 444         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 445             ao->entry.monotonic == bo->entry.monotonic &&
 446             ao->entry.realtime == bo->entry.realtime &&
 447             ao->entry.xor_hash == bo->entry.xor_hash)
 448                 return 0;
 449
 450         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 451
 452                 /* If this is from the same seqnum source, compare
 453                  * seqnums */
 454                 a = le64toh(ao->entry.seqnum);
 455                 b = le64toh(bo->entry.seqnum);
 456
 457                 if (a < b)
 458                         return -1;
 459                 if (a > b)
 460                         return 1;
 461
 462                 /* Wow! This is weird, different data but the same
 463                  * seqnums? Something is borked, but let's make the
 464                  * best of it and compare by time. */
 465         }
 466
 467         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 468
 469                 /* If the boot id matches, compare monotonic time */
 470                 a = le64toh(ao->entry.monotonic);
 471                 b = le64toh(bo->entry.monotonic);
 472
 473                 if (a < b)
 474                         return -1;
 475                 if (a > b)
 476                         return 1;
 477         }
 478
 479         /* Otherwise, compare UTC time */
 480         a = le64toh(ao->entry.realtime);
 481         b = le64toh(bo->entry.realtime);
 482
 483         if (a < b)
 484                 return -1;
 485         if (a > b)
 486                 return 1;
 487
 488         /* Finally, compare by contents */
 489         a = le64toh(ao->entry.xor_hash);
 490         b = le64toh(bo->entry.xor_hash);
 491
 492         if (a < b)
 493                 return -1;
 494         if (a > b)
 495                 return 1;
 496
 497         return 0;
 498 }
 499
 500 static bool whole_file_precedes_location(JournalFile *f, Location *l, direction_t direction) {
 501         assert(f);
 502         assert(l);
 503
 504         if (l->type != LOCATION_DISCRETE && l->type != LOCATION_SEEK)
 505                 return false;
 506
 507         if (l->seqnum_set && sd_id128_equal(l->seqnum_id, f->header->seqnum_id))
 508                 return direction == DIRECTION_DOWN ?
 509                         l->seqnum > le64toh(f->header->tail_entry_seqnum) :
 510                         l->seqnum < le64toh(f->header->head_entry_seqnum);
 511
 512         if (l->realtime_set)
 513                 return direction == DIRECTION_DOWN ?
 514                         l->realtime > le64toh(f->header->tail_entry_realtime) :
 515                         l->realtime < le64toh(f->header->head_entry_realtime);
 516
 517         return false;
 518 }
 519
 520 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 521         uint64_t a;
 522
 523         assert(af);
 524         assert(ao);
 525         assert(l);
 526         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 527
 528         if (l->monotonic_set &&
 529             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 530             l->realtime_set &&
 531             le64toh(ao->entry.realtime) == l->realtime &&
 532             l->xor_hash_set &&
 533             le64toh(ao->entry.xor_hash) == l->xor_hash)
 534                 return 0;
 535
 536         if (l->seqnum_set &&
 537             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 538
 539                 a = le64toh(ao->entry.seqnum);
 540
 541                 if (a < l->seqnum)
 542                         return -1;
 543                 if (a > l->seqnum)
 544                         return 1;
 545         }
 546
 547         if (l->monotonic_set &&
 548             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 549
 550                 a = le64toh(ao->entry.monotonic);
 551
 552                 if (a < l->monotonic)
 553                         return -1;
 554                 if (a > l->monotonic)
 555                         return 1;
 556         }
 557
 558         if (l->realtime_set) {
 559
 560                 a = le64toh(ao->entry.realtime);
 561
 562                 if (a < l->realtime)
 563                         return -1;
 564                 if (a > l->realtime)
 565                         return 1;
 566         }
 567
 568         if (l->xor_hash_set) {
 569                 a = le64toh(ao->entry.xor_hash);
 570
 571                 if (a < l->xor_hash)
 572                         return -1;
 573                 if (a > l->xor_hash)
 574                         return 1;
 575         }
 576
 577         return 0;
 578 }
 579
 580 static int next_for_match(
 581                 sd_journal *j,
 582                 Match *m,
 583                 JournalFile *f,
 584                 uint64_t after_offset,
 585                 direction_t direction,
 586                 Object **ret,
 587                 uint64_t *offset) {
 588
 589         int r;
 590         uint64_t np = 0;
 591         Object *n;
 592
 593         assert(j);
 594         assert(m);
 595         assert(f);
 596
 597         if (m->type == MATCH_DISCRETE) {
 598                 uint64_t dp;
 599
 600                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 601                 if (r <= 0)
 602                         return r;
 603
 604                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 605
 606         } else if (m->type == MATCH_OR_TERM) {
 607                 Match *i;
 608
 609                 /* Find the earliest match beyond after_offset */
 610
 611                 LIST_FOREACH(matches, i, m->matches) {
 612                         uint64_t cp;
 613
 614                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 615                         if (r < 0)
 616                                 return r;
 617                         else if (r > 0) {
 618                                 if (np == 0 || (direction == DIRECTION_DOWN ? cp < np : cp > np))
 619                                         np = cp;
 620                         }
 621                 }
 622
 623                 if (np == 0)
 624                         return 0;
 625
 626         } else if (m->type == MATCH_AND_TERM) {
 627                 Match *i, *last_moved;
 628
 629                 /* Always jump to the next matching entry and repeat
 630                  * this until we find an offset that matches for all
 631                  * matches. */
 632
 633                 if (!m->matches)
 634                         return 0;
 635
 636                 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
 637                 if (r <= 0)
 638                         return r;
 639
 640                 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
 641                 last_moved = m->matches;
 642
 643                 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
 644                         uint64_t cp;
 645
 646                         r = next_for_match(j, i, f, np, direction, NULL, &cp);
 647                         if (r <= 0)
 648                                 return r;
 649
 650                         assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
 651                         if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
 652                                 np = cp;
 653                                 last_moved = i;
 654                         }
 655                 }
 656         }
 657
 658         assert(np > 0);
 659
 660         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 661         if (r < 0)
 662                 return r;
 663
 664         if (ret)
 665                 *ret = n;
 666         if (offset)
 667                 *offset = np;
 668
 669         return 1;
 670 }
 671
 672 static int find_location_for_match(
 673                 sd_journal *j,
 674                 Match *m,
 675                 JournalFile *f,
 676                 direction_t direction,
 677                 Object **ret,
 678                 uint64_t *offset) {
 679
 680         int r;
 681
 682         assert(j);
 683         assert(m);
 684         assert(f);
 685
 686         if (m->type == MATCH_DISCRETE) {
 687                 uint64_t dp;
 688
 689                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 690                 if (r <= 0)
 691                         return r;
 692
 693                 /* FIXME: missing: find by monotonic */
 694
 695                 if (j->current_location.type == LOCATION_HEAD)
 696                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 697                 if (j->current_location.type == LOCATION_TAIL)
 698                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 699                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 700                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 701                 if (j->current_location.monotonic_set) {
 702                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 703                         if (r != -ENOENT)
 704                                 return r;
 705                 }
 706                 if (j->current_location.realtime_set)
 707                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 708
 709                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 710
 711         } else if (m->type == MATCH_OR_TERM) {
 712                 uint64_t np = 0;
 713                 Object *n;
 714                 Match *i;
 715
 716                 /* Find the earliest match */
 717
 718                 LIST_FOREACH(matches, i, m->matches) {
 719                         uint64_t cp;
 720
 721                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 722                         if (r < 0)
 723                                 return r;
 724                         else if (r > 0) {
 725                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 726                                         np = cp;
 727                         }
 728                 }
 729
 730                 if (np == 0)
 731                         return 0;
 732
 733                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 734                 if (r < 0)
 735                         return r;
 736
 737                 if (ret)
 738                         *ret = n;
 739                 if (offset)
 740                         *offset = np;
 741
 742                 return 1;
 743
 744         } else {
 745                 Match *i;
 746                 uint64_t np = 0;
 747
 748                 assert(m->type == MATCH_AND_TERM);
 749
 750                 /* First jump to the last match, and then find the
 751                  * next one where all matches match */
 752
 753                 if (!m->matches)
 754                         return 0;
 755
 756                 LIST_FOREACH(matches, i, m->matches) {
 757                         uint64_t cp;
 758
 759                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 760                         if (r <= 0)
 761                                 return r;
 762
 763                         if (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))
 764                                 np = cp;
 765                 }
 766
 767                 return next_for_match(j, m, f, np, direction, ret, offset);
 768         }
 769 }
 770
 771 static int find_location_with_matches(
 772                 sd_journal *j,
 773                 JournalFile *f,
 774                 direction_t direction,
 775                 Object **ret,
 776                 uint64_t *offset) {
 777
 778         int r;
 779
 780         assert(j);
 781         assert(f);
 782         assert(ret);
 783         assert(offset);
 784
 785         if (!j->level0) {
 786                 /* No matches is simple */
 787
 788                 if (j->current_location.type == LOCATION_HEAD)
 789                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 790                 if (j->current_location.type == LOCATION_TAIL)
 791                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 792                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 793                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 794                 if (j->current_location.monotonic_set) {
 795                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 796                         if (r != -ENOENT)
 797                                 return r;
 798                 }
 799                 if (j->current_location.realtime_set)
 800                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 801
 802                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 803         } else
 804                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 805 }
 806
 807 static int next_with_matches(
 808                 sd_journal *j,
 809                 JournalFile *f,
 810                 direction_t direction,
 811                 Object **ret,
 812                 uint64_t *offset) {
 813
 814         Object *c;
 815         uint64_t cp;
 816
 817         assert(j);
 818         assert(f);
 819         assert(ret);
 820         assert(offset);
 821
 822         c = *ret;
 823         cp = *offset;
 824
 825         /* No matches is easy. We simple advance the file
 826          * pointer by one. */
 827         if (!j->level0)
 828                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 829
 830         /* If we have a match then we look for the next matching entry
 831          * with an offset at least one step larger */
 832         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 833 }
 834
 835 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 836         Object *c;
 837         uint64_t cp;
 838         int r;
 839
 840         assert(j);
 841         assert(f);
 842
 843         if (f->last_direction == direction && f->current_offset > 0) {
 844                 cp = f->current_offset;
 845
 846                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 847                 if (r < 0)
 848                         return r;
 849
 850                 r = next_with_matches(j, f, direction, &c, &cp);
 851                 if (r <= 0)
 852                         return r;
 853         } else {
 854                 r = find_location_with_matches(j, f, direction, &c, &cp);
 855                 if (r <= 0)
 856                         return r;
 857         }
 858
 859         /* OK, we found the spot, now let's advance until an entry
 860          * that is actually different from what we were previously
 861          * looking at. This is necessary to handle entries which exist
 862          * in two (or more) journal files, and which shall all be
 863          * suppressed but one. */
 864
 865         for (;;) {
 866                 bool found;
 867
 868                 if (j->current_location.type == LOCATION_DISCRETE) {
 869                         int k;
 870
 871                         k = compare_with_location(f, c, &j->current_location);
 872
 873                         found = direction == DIRECTION_DOWN ? k > 0 : k < 0;
 874                 } else
 875                         found = true;
 876
 877                 if (found) {
 878                         if (ret)
 879                                 *ret = c;
 880                         if (offset)
 881                                 *offset = cp;
 882                         return 1;
 883                 }
 884
 885                 r = next_with_matches(j, f, direction, &c, &cp);
 886                 if (r <= 0)
 887                         return r;
 888         }
 889 }
 890
 891 static int real_journal_next(sd_journal *j, direction_t direction) {
 892         JournalFile *f, *new_file = NULL;
 893         uint64_t new_offset = 0;
 894         uint64_t p = 0;
 895         Iterator i;
 896         Object *o;
 897         int r;
 898
 899         assert_return(j, -EINVAL);
 900         assert_return(!journal_pid_changed(j), -ECHILD);
 901
 902         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
 903                 bool found;
 904
 905                 if (whole_file_precedes_location(f, &j->current_location, direction))
 906                         continue;
 907
 908                 r = next_beyond_location(j, f, direction, &o, &p);
 909                 if (r < 0) {
 910                         log_debug_errno(r, "Can't iterate through %s, ignoring: %m", f->path);
 911                         remove_file_real(j, f);
 912                         continue;
 913                 } else if (r == 0)
 914                         continue;
 915
 916                 if (!new_file)
 917                         found = true;
 918                 else {
 919                         int k;
 920
 921                         k = compare_entry_order(f, o, new_file, new_offset);
 922
 923                         found = direction == DIRECTION_DOWN ? k < 0 : k > 0;
 924                 }
 925
 926                 if (found) {
 927                         new_file = f;
 928                         new_offset = p;
 929                 }
 930         }
 931
 932         if (!new_file)
 933                 return 0;
 934
 935         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 936         if (r < 0)
 937                 return r;
 938
 939         set_location(j, LOCATION_DISCRETE, new_file, o, direction, new_offset);
 940
 941         return 1;
 942 }
 943
 944 _public_ int sd_journal_next(sd_journal *j) {
 945         return real_journal_next(j, DIRECTION_DOWN);
 946 }
 947
 948 _public_ int sd_journal_previous(sd_journal *j) {
 949         return real_journal_next(j, DIRECTION_UP);
 950 }
 951
 952 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 953         int c = 0, r;
 954
 955         assert_return(j, -EINVAL);
 956         assert_return(!journal_pid_changed(j), -ECHILD);
 957
 958         if (skip == 0) {
 959                 /* If this is not a discrete skip, then at least
 960                  * resolve the current location */
 961                 if (j->current_location.type != LOCATION_DISCRETE)
 962                         return real_journal_next(j, direction);
 963
 964                 return 0;
 965         }
 966
 967         do {
 968                 r = real_journal_next(j, direction);
 969                 if (r < 0)
 970                         return r;
 971
 972                 if (r == 0)
 973                         return c;
 974
 975                 skip--;
 976                 c++;
 977         } while (skip > 0);
 978
 979         return c;
 980 }
 981
 982 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 983         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 984 }
 985
 986 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 987         return real_journal_next_skip(j, DIRECTION_UP, skip);
 988 }
 989
 990 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 991         Object *o;
 992         int r;
 993         char bid[33], sid[33];
 994
 995         assert_return(j, -EINVAL);
 996         assert_return(!journal_pid_changed(j), -ECHILD);
 997         assert_return(cursor, -EINVAL);
 998
 999         if (!j->current_file || j->current_file->current_offset <= 0)
1000                 return -EADDRNOTAVAIL;
1001
1002         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1003         if (r < 0)
1004                 return r;
1005
1006         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
1007         sd_id128_to_string(o->entry.boot_id, bid);
1008
1009         if (asprintf(cursor,
1010                      "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
1011                      sid, le64toh(o->entry.seqnum),
1012                      bid, le64toh(o->entry.monotonic),
1013                      le64toh(o->entry.realtime),
1014                      le64toh(o->entry.xor_hash)) < 0)
1015                 return -ENOMEM;
1016
1017         return 0;
1018 }
1019
1020 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
1021         const char *word, *state;
1022         size_t l;
1023         unsigned long long seqnum, monotonic, realtime, xor_hash;
1024         bool
1025                 seqnum_id_set = false,
1026                 seqnum_set = false,
1027                 boot_id_set = false,
1028                 monotonic_set = false,
1029                 realtime_set = false,
1030                 xor_hash_set = false;
1031         sd_id128_t seqnum_id, boot_id;
1032
1033         assert_return(j, -EINVAL);
1034         assert_return(!journal_pid_changed(j), -ECHILD);
1035         assert_return(!isempty(cursor), -EINVAL);
1036
1037         FOREACH_WORD_SEPARATOR(word, l, cursor, ";", state) {
1038                 char *item;
1039                 int k = 0;
1040
1041                 if (l < 2 || word[1] != '=')
1042                         return -EINVAL;
1043
1044                 item = strndup(word, l);
1045                 if (!item)
1046                         return -ENOMEM;
1047
1048                 switch (word[0]) {
1049
1050                 case 's':
1051                         seqnum_id_set = true;
1052                         k = sd_id128_from_string(item+2, &seqnum_id);
1053                         break;
1054
1055                 case 'i':
1056                         seqnum_set = true;
1057                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1058                                 k = -EINVAL;
1059                         break;
1060
1061                 case 'b':
1062                         boot_id_set = true;
1063                         k = sd_id128_from_string(item+2, &boot_id);
1064                         break;
1065
1066                 case 'm':
1067                         monotonic_set = true;
1068                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1069                                 k = -EINVAL;
1070                         break;
1071
1072                 case 't':
1073                         realtime_set = true;
1074                         if (sscanf(item+2, "%llx", &realtime) != 1)
1075                                 k = -EINVAL;
1076                         break;
1077
1078                 case 'x':
1079                         xor_hash_set = true;
1080                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1081                                 k = -EINVAL;
1082                         break;
1083                 }
1084
1085                 free(item);
1086
1087                 if (k < 0)
1088                         return k;
1089         }
1090
1091         if ((!seqnum_set || !seqnum_id_set) &&
1092             (!monotonic_set || !boot_id_set) &&
1093             !realtime_set)
1094                 return -EINVAL;
1095
1096         reset_location(j);
1097
1098         j->current_location.type = LOCATION_SEEK;
1099
1100         if (realtime_set) {
1101                 j->current_location.realtime = (uint64_t) realtime;
1102                 j->current_location.realtime_set = true;
1103         }
1104
1105         if (seqnum_set && seqnum_id_set) {
1106                 j->current_location.seqnum = (uint64_t) seqnum;
1107                 j->current_location.seqnum_id = seqnum_id;
1108                 j->current_location.seqnum_set = true;
1109         }
1110
1111         if (monotonic_set && boot_id_set) {
1112                 j->current_location.monotonic = (uint64_t) monotonic;
1113                 j->current_location.boot_id = boot_id;
1114                 j->current_location.monotonic_set = true;
1115         }
1116
1117         if (xor_hash_set) {
1118                 j->current_location.xor_hash = (uint64_t) xor_hash;
1119                 j->current_location.xor_hash_set = true;
1120         }
1121
1122         return 0;
1123 }
1124
1125 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1126         int r;
1127         const char *word, *state;
1128         size_t l;
1129         Object *o;
1130
1131         assert_return(j, -EINVAL);
1132         assert_return(!journal_pid_changed(j), -ECHILD);
1133         assert_return(!isempty(cursor), -EINVAL);
1134
1135         if (!j->current_file || j->current_file->current_offset <= 0)
1136                 return -EADDRNOTAVAIL;
1137
1138         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1139         if (r < 0)
1140                 return r;
1141
1142         FOREACH_WORD_SEPARATOR(word, l, cursor, ";", state) {
1143                 _cleanup_free_ char *item = NULL;
1144                 sd_id128_t id;
1145                 unsigned long long ll;
1146                 int k = 0;
1147
1148                 if (l < 2 || word[1] != '=')
1149                         return -EINVAL;
1150
1151                 item = strndup(word, l);
1152                 if (!item)
1153                         return -ENOMEM;
1154
1155                 switch (word[0]) {
1156
1157                 case 's':
1158                         k = sd_id128_from_string(item+2, &id);
1159                         if (k < 0)
1160                                 return k;
1161                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1162                                 return 0;
1163                         break;
1164
1165                 case 'i':
1166                         if (sscanf(item+2, "%llx", &ll) != 1)
1167                                 return -EINVAL;
1168                         if (ll != le64toh(o->entry.seqnum))
1169                                 return 0;
1170                         break;
1171
1172                 case 'b':
1173                         k = sd_id128_from_string(item+2, &id);
1174                         if (k < 0)
1175                                 return k;
1176                         if (!sd_id128_equal(id, o->entry.boot_id))
1177                                 return 0;
1178                         break;
1179
1180                 case 'm':
1181                         if (sscanf(item+2, "%llx", &ll) != 1)
1182                                 return -EINVAL;
1183                         if (ll != le64toh(o->entry.monotonic))
1184                                 return 0;
1185                         break;
1186
1187                 case 't':
1188                         if (sscanf(item+2, "%llx", &ll) != 1)
1189                                 return -EINVAL;
1190                         if (ll != le64toh(o->entry.realtime))
1191                                 return 0;
1192                         break;
1193
1194                 case 'x':
1195                         if (sscanf(item+2, "%llx", &ll) != 1)
1196                                 return -EINVAL;
1197                         if (ll != le64toh(o->entry.xor_hash))
1198                                 return 0;
1199                         break;
1200                 }
1201         }
1202
1203         return 1;
1204 }
1205
1206
1207 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1208         assert_return(j, -EINVAL);
1209         assert_return(!journal_pid_changed(j), -ECHILD);
1210
1211         reset_location(j);
1212         j->current_location.type = LOCATION_SEEK;
1213         j->current_location.boot_id = boot_id;
1214         j->current_location.monotonic = usec;
1215         j->current_location.monotonic_set = true;
1216
1217         return 0;
1218 }
1219
1220 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1221         assert_return(j, -EINVAL);
1222         assert_return(!journal_pid_changed(j), -ECHILD);
1223
1224         reset_location(j);
1225         j->current_location.type = LOCATION_SEEK;
1226         j->current_location.realtime = usec;
1227         j->current_location.realtime_set = true;
1228
1229         return 0;
1230 }
1231
1232 _public_ int sd_journal_seek_head(sd_journal *j) {
1233         assert_return(j, -EINVAL);
1234         assert_return(!journal_pid_changed(j), -ECHILD);
1235
1236         reset_location(j);
1237         j->current_location.type = LOCATION_HEAD;
1238
1239         return 0;
1240 }
1241
1242 _public_ int sd_journal_seek_tail(sd_journal *j) {
1243         assert_return(j, -EINVAL);
1244         assert_return(!journal_pid_changed(j), -ECHILD);
1245
1246         reset_location(j);
1247         j->current_location.type = LOCATION_TAIL;
1248
1249         return 0;
1250 }
1251
1252 static void check_network(sd_journal *j, int fd) {
1253         struct statfs sfs;
1254
1255         assert(j);
1256
1257         if (j->on_network)
1258                 return;
1259
1260         if (fstatfs(fd, &sfs) < 0)
1261                 return;
1262
1263         j->on_network =
1264                 F_TYPE_EQUAL(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1265                 F_TYPE_EQUAL(sfs.f_type, CODA_SUPER_MAGIC) ||
1266                 F_TYPE_EQUAL(sfs.f_type, NCP_SUPER_MAGIC) ||
1267                 F_TYPE_EQUAL(sfs.f_type, NFS_SUPER_MAGIC) ||
1268                 F_TYPE_EQUAL(sfs.f_type, SMB_SUPER_MAGIC);
1269 }
1270
1271 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1272         const char *full, *tilded, *atted;
1273
1274         full = strappenda(prefix, ".journal");
1275         tilded = strappenda(full, "~");
1276         atted = strappenda(prefix, "@");
1277
1278         return streq(filename, full) ||
1279                streq(filename, tilded) ||
1280                startswith(filename, atted);
1281 }
1282
1283 static bool file_type_wanted(int flags, const char *filename) {
1284         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1285                 return false;
1286
1287         /* no flags set → every type is OK */
1288         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1289                 return true;
1290
1291         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1292                 return true;
1293
1294         if (flags & SD_JOURNAL_CURRENT_USER) {
1295                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1296
1297                 assert_se(snprintf(prefix, sizeof(prefix), "user-"UID_FMT, getuid())
1298                           < (int) sizeof(prefix));
1299
1300                 if (file_has_type_prefix(prefix, filename))
1301                         return true;
1302         }
1303
1304         return false;
1305 }
1306
1307 static int add_any_file(sd_journal *j, const char *path) {
1308         JournalFile *f = NULL;
1309         int r;
1310
1311         assert(j);
1312         assert(path);
1313
1314         if (ordered_hashmap_get(j->files, path))
1315                 return 0;
1316
1317         if (ordered_hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1318                 log_warning("Too many open journal files, not adding %s.", path);
1319                 return set_put_error(j, -ETOOMANYREFS);
1320         }
1321
1322         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1323         if (r < 0)
1324                 return r;
1325
1326         /* journal_file_dump(f); */
1327
1328         r = ordered_hashmap_put(j->files, f->path, f);
1329         if (r < 0) {
1330                 journal_file_close(f);
1331                 return r;
1332         }
1333
1334         log_debug("File %s added.", f->path);
1335
1336         check_network(j, f->fd);
1337
1338         j->current_invalidate_counter ++;
1339
1340         return 0;
1341 }
1342
1343 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1344         _cleanup_free_ char *path = NULL;
1345         int r;
1346
1347         assert(j);
1348         assert(prefix);
1349         assert(filename);
1350
1351         if (j->no_new_files ||
1352             !file_type_wanted(j->flags, filename))
1353                 return 0;
1354
1355         path = strjoin(prefix, "/", filename, NULL);
1356         if (!path)
1357                 return -ENOMEM;
1358
1359         r = add_any_file(j, path);
1360         if (r == -ENOENT)
1361                 return 0;
1362         return 0;
1363 }
1364
1365 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1366         _cleanup_free_ char *path;
1367         JournalFile *f;
1368
1369         assert(j);
1370         assert(prefix);
1371         assert(filename);
1372
1373         path = strjoin(prefix, "/", filename, NULL);
1374         if (!path)
1375                 return -ENOMEM;
1376
1377         f = ordered_hashmap_get(j->files, path);
1378         if (!f)
1379                 return 0;
1380
1381         remove_file_real(j, f);
1382         return 0;
1383 }
1384
1385 static void remove_file_real(sd_journal *j, JournalFile *f) {
1386         assert(j);
1387         assert(f);
1388
1389         ordered_hashmap_remove(j->files, f->path);
1390
1391         log_debug("File %s removed.", f->path);
1392
1393         if (j->current_file == f) {
1394                 j->current_file = NULL;
1395                 j->current_field = 0;
1396         }
1397
1398         if (j->unique_file == f) {
1399                 /* Jump to the next unique_file or NULL if that one was last */
1400                 j->unique_file = ordered_hashmap_next(j->files, j->unique_file->path);
1401                 j->unique_offset = 0;
1402                 if (!j->unique_file)
1403                         j->unique_file_lost = true;
1404         }
1405
1406         journal_file_close(f);
1407
1408         j->current_invalidate_counter ++;
1409 }
1410
1411 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1412         _cleanup_free_ char *path = NULL;
1413         int r;
1414         _cleanup_closedir_ DIR *d = NULL;
1415         sd_id128_t id, mid;
1416         Directory *m;
1417
1418         assert(j);
1419         assert(prefix);
1420         assert(dirname);
1421
1422         log_debug("Considering %s/%s.", prefix, dirname);
1423
1424         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1425             (sd_id128_from_string(dirname, &id) < 0 ||
1426              sd_id128_get_machine(&mid) < 0 ||
1427              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1428             return 0;
1429
1430         path = strjoin(prefix, "/", dirname, NULL);
1431         if (!path)
1432                 return -ENOMEM;
1433
1434         d = opendir(path);
1435         if (!d) {
1436                 log_debug_errno(errno, "Failed to open %s: %m", path);
1437                 if (errno == ENOENT)
1438                         return 0;
1439                 return -errno;
1440         }
1441
1442         m = hashmap_get(j->directories_by_path, path);
1443         if (!m) {
1444                 m = new0(Directory, 1);
1445                 if (!m)
1446                         return -ENOMEM;
1447
1448                 m->is_root = false;
1449                 m->path = path;
1450
1451                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1452                         free(m);
1453                         return -ENOMEM;
1454                 }
1455
1456                 path = NULL; /* avoid freeing in cleanup */
1457                 j->current_invalidate_counter ++;
1458
1459                 log_debug("Directory %s added.", m->path);
1460
1461         } else if (m->is_root)
1462                 return 0;
1463
1464         if (m->wd <= 0 && j->inotify_fd >= 0) {
1465
1466                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1467                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1468                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1469                                           IN_ONLYDIR);
1470
1471                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1472                         inotify_rm_watch(j->inotify_fd, m->wd);
1473         }
1474
1475         for (;;) {
1476                 struct dirent *de;
1477
1478                 errno = 0;
1479                 de = readdir(d);
1480                 if (!de && errno != 0) {
1481                         r = -errno;
1482                         log_debug_errno(errno, "Failed to read directory %s: %m", m->path);
1483                         return r;
1484                 }
1485                 if (!de)
1486                         break;
1487
1488                 if (dirent_is_file_with_suffix(de, ".journal") ||
1489                     dirent_is_file_with_suffix(de, ".journal~")) {
1490                         r = add_file(j, m->path, de->d_name);
1491                         if (r < 0) {
1492                                 log_debug_errno(r, "Failed to add file %s/%s: %m",
1493                                                 m->path, de->d_name);
1494                                 r = set_put_error(j, r);
1495                                 if (r < 0)
1496                                         return r;
1497                         }
1498                 }
1499         }
1500
1501         check_network(j, dirfd(d));
1502
1503         return 0;
1504 }
1505
1506 static int add_root_directory(sd_journal *j, const char *p) {
1507         _cleanup_closedir_ DIR *d = NULL;
1508         Directory *m;
1509         int r;
1510
1511         assert(j);
1512         assert(p);
1513
1514         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1515             !path_startswith(p, "/run"))
1516                 return -EINVAL;
1517
1518         if (j->prefix)
1519                 p = strappenda(j->prefix, p);
1520
1521         d = opendir(p);
1522         if (!d)
1523                 return -errno;
1524
1525         m = hashmap_get(j->directories_by_path, p);
1526         if (!m) {
1527                 m = new0(Directory, 1);
1528                 if (!m)
1529                         return -ENOMEM;
1530
1531                 m->is_root = true;
1532                 m->path = strdup(p);
1533                 if (!m->path) {
1534                         free(m);
1535                         return -ENOMEM;
1536                 }
1537
1538                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1539                         free(m->path);
1540                         free(m);
1541                         return -ENOMEM;
1542                 }
1543
1544                 j->current_invalidate_counter ++;
1545
1546                 log_debug("Root directory %s added.", m->path);
1547
1548         } else if (!m->is_root)
1549                 return 0;
1550
1551         if (m->wd <= 0 && j->inotify_fd >= 0) {
1552
1553                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1554                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1555                                           IN_ONLYDIR);
1556
1557                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1558                         inotify_rm_watch(j->inotify_fd, m->wd);
1559         }
1560
1561         if (j->no_new_files)
1562                 return 0;
1563
1564         for (;;) {
1565                 struct dirent *de;
1566                 sd_id128_t id;
1567
1568                 errno = 0;
1569                 de = readdir(d);
1570                 if (!de && errno != 0) {
1571                         r = -errno;
1572                         log_debug_errno(errno, "Failed to read directory %s: %m", m->path);
1573                         return r;
1574                 }
1575                 if (!de)
1576                         break;
1577
1578                 if (dirent_is_file_with_suffix(de, ".journal") ||
1579                     dirent_is_file_with_suffix(de, ".journal~")) {
1580                         r = add_file(j, m->path, de->d_name);
1581                         if (r < 0) {
1582                                 log_debug_errno(r, "Failed to add file %s/%s: %m",
1583                                                 m->path, de->d_name);
1584                                 r = set_put_error(j, r);
1585                                 if (r < 0)
1586                                         return r;
1587                         }
1588                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1589                            sd_id128_from_string(de->d_name, &id) >= 0) {
1590
1591                         r = add_directory(j, m->path, de->d_name);
1592                         if (r < 0)
1593                                 log_debug_errno(r, "Failed to add directory %s/%s: %m", m->path, de->d_name);
1594                 }
1595         }
1596
1597         check_network(j, dirfd(d));
1598
1599         return 0;
1600 }
1601
1602 static int remove_directory(sd_journal *j, Directory *d) {
1603         assert(j);
1604
1605         if (d->wd > 0) {
1606                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1607
1608                 if (j->inotify_fd >= 0)
1609                         inotify_rm_watch(j->inotify_fd, d->wd);
1610         }
1611
1612         hashmap_remove(j->directories_by_path, d->path);
1613
1614         if (d->is_root)
1615                 log_debug("Root directory %s removed.", d->path);
1616         else
1617                 log_debug("Directory %s removed.", d->path);
1618
1619         free(d->path);
1620         free(d);
1621
1622         return 0;
1623 }
1624
1625 static int add_search_paths(sd_journal *j) {
1626         int r;
1627         const char search_paths[] =
1628                 "/run/log/journal\0"
1629                 "/var/log/journal\0";
1630         const char *p;
1631
1632         assert(j);
1633
1634         /* We ignore most errors here, since the idea is to only open
1635          * what's actually accessible, and ignore the rest. */
1636
1637         NULSTR_FOREACH(p, search_paths) {
1638                 r = add_root_directory(j, p);
1639                 if (r < 0 && r != -ENOENT) {
1640                         r = set_put_error(j, r);
1641                         if (r < 0)
1642                                 return r;
1643                 }
1644         }
1645
1646         return 0;
1647 }
1648
1649 static int add_current_paths(sd_journal *j) {
1650         Iterator i;
1651         JournalFile *f;
1652
1653         assert(j);
1654         assert(j->no_new_files);
1655
1656         /* Simply adds all directories for files we have open as
1657          * "root" directories. We don't expect errors here, so we
1658          * treat them as fatal. */
1659
1660         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
1661                 _cleanup_free_ char *dir;
1662                 int r;
1663
1664                 dir = dirname_malloc(f->path);
1665                 if (!dir)
1666                         return -ENOMEM;
1667
1668                 r = add_root_directory(j, dir);
1669                 if (r < 0) {
1670                         set_put_error(j, r);
1671                         return r;
1672                 }
1673         }
1674
1675         return 0;
1676 }
1677
1678
1679 static int allocate_inotify(sd_journal *j) {
1680         assert(j);
1681
1682         if (j->inotify_fd < 0) {
1683                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1684                 if (j->inotify_fd < 0)
1685                         return -errno;
1686         }
1687
1688         if (!j->directories_by_wd) {
1689                 j->directories_by_wd = hashmap_new(NULL);
1690                 if (!j->directories_by_wd)
1691                         return -ENOMEM;
1692         }
1693
1694         return 0;
1695 }
1696
1697 static sd_journal *journal_new(int flags, const char *path) {
1698         sd_journal *j;
1699
1700         j = new0(sd_journal, 1);
1701         if (!j)
1702                 return NULL;
1703
1704         j->original_pid = getpid();
1705         j->inotify_fd = -1;
1706         j->flags = flags;
1707         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1708
1709         if (path) {
1710                 j->path = strdup(path);
1711                 if (!j->path)
1712                         goto fail;
1713         }
1714
1715         j->files = ordered_hashmap_new(&string_hash_ops);
1716         j->directories_by_path = hashmap_new(&string_hash_ops);
1717         j->mmap = mmap_cache_new();
1718         if (!j->files || !j->directories_by_path || !j->mmap)
1719                 goto fail;
1720
1721         return j;
1722
1723 fail:
1724         sd_journal_close(j);
1725         return NULL;
1726 }
1727
1728 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1729         sd_journal *j;
1730         int r;
1731
1732         assert_return(ret, -EINVAL);
1733         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_RUNTIME_ONLY|SD_JOURNAL_SYSTEM|SD_JOURNAL_CURRENT_USER)) == 0, -EINVAL);
1734
1735         j = journal_new(flags, NULL);
1736         if (!j)
1737                 return -ENOMEM;
1738
1739         r = add_search_paths(j);
1740         if (r < 0)
1741                 goto fail;
1742
1743         *ret = j;
1744         return 0;
1745
1746 fail:
1747         sd_journal_close(j);
1748
1749         return r;
1750 }
1751
1752 _public_ int sd_journal_open_container(sd_journal **ret, const char *machine, int flags) {
1753         _cleanup_free_ char *root = NULL, *class = NULL;
1754         sd_journal *j;
1755         char *p;
1756         int r;
1757
1758         assert_return(machine, -EINVAL);
1759         assert_return(ret, -EINVAL);
1760         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_SYSTEM)) == 0, -EINVAL);
1761         assert_return(machine_name_is_valid(machine), -EINVAL);
1762
1763         p = strappenda("/run/systemd/machines/", machine);
1764         r = parse_env_file(p, NEWLINE, "ROOT", &root, "CLASS", &class, NULL);
1765         if (r == -ENOENT)
1766                 return -EHOSTDOWN;
1767         if (r < 0)
1768                 return r;
1769         if (!root)
1770                 return -ENODATA;
1771
1772         if (!streq_ptr(class, "container"))
1773                 return -EIO;
1774
1775         j = journal_new(flags, NULL);
1776         if (!j)
1777                 return -ENOMEM;
1778
1779         j->prefix = root;
1780         root = NULL;
1781
1782         r = add_search_paths(j);
1783         if (r < 0)
1784                 goto fail;
1785
1786         *ret = j;
1787         return 0;
1788
1789 fail:
1790         sd_journal_close(j);
1791         return r;
1792 }
1793
1794 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1795         sd_journal *j;
1796         int r;
1797
1798         assert_return(ret, -EINVAL);
1799         assert_return(path, -EINVAL);
1800         assert_return(flags == 0, -EINVAL);
1801
1802         j = journal_new(flags, path);
1803         if (!j)
1804                 return -ENOMEM;
1805
1806         r = add_root_directory(j, path);
1807         if (r < 0) {
1808                 set_put_error(j, r);
1809                 goto fail;
1810         }
1811
1812         *ret = j;
1813         return 0;
1814
1815 fail:
1816         sd_journal_close(j);
1817
1818         return r;
1819 }
1820
1821 _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int flags) {
1822         sd_journal *j;
1823         const char **path;
1824         int r;
1825
1826         assert_return(ret, -EINVAL);
1827         assert_return(flags == 0, -EINVAL);
1828
1829         j = journal_new(flags, NULL);
1830         if (!j)
1831                 return -ENOMEM;
1832
1833         STRV_FOREACH(path, paths) {
1834                 r = add_any_file(j, *path);
1835                 if (r < 0) {
1836                         log_error_errno(r, "Failed to open %s: %m", *path);
1837                         goto fail;
1838                 }
1839         }
1840
1841         j->no_new_files = true;
1842
1843         *ret = j;
1844         return 0;
1845
1846 fail:
1847         sd_journal_close(j);
1848
1849         return r;
1850 }
1851
1852 _public_ void sd_journal_close(sd_journal *j) {
1853         Directory *d;
1854         JournalFile *f;
1855
1856         if (!j)
1857                 return;
1858
1859         sd_journal_flush_matches(j);
1860
1861         while ((f = ordered_hashmap_steal_first(j->files)))
1862                 journal_file_close(f);
1863
1864         ordered_hashmap_free(j->files);
1865
1866         while ((d = hashmap_first(j->directories_by_path)))
1867                 remove_directory(j, d);
1868
1869         while ((d = hashmap_first(j->directories_by_wd)))
1870                 remove_directory(j, d);
1871
1872         hashmap_free(j->directories_by_path);
1873         hashmap_free(j->directories_by_wd);
1874
1875         safe_close(j->inotify_fd);
1876
1877         if (j->mmap) {
1878                 log_debug("mmap cache statistics: %u hit, %u miss", mmap_cache_get_hit(j->mmap), mmap_cache_get_missed(j->mmap));
1879                 mmap_cache_unref(j->mmap);
1880         }
1881
1882         free(j->path);
1883         free(j->prefix);
1884         free(j->unique_field);
1885         set_free(j->errors);
1886         free(j);
1887 }
1888
1889 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1890         Object *o;
1891         JournalFile *f;
1892         int r;
1893
1894         assert_return(j, -EINVAL);
1895         assert_return(!journal_pid_changed(j), -ECHILD);
1896         assert_return(ret, -EINVAL);
1897
1898         f = j->current_file;
1899         if (!f)
1900                 return -EADDRNOTAVAIL;
1901
1902         if (f->current_offset <= 0)
1903                 return -EADDRNOTAVAIL;
1904
1905         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1906         if (r < 0)
1907                 return r;
1908
1909         *ret = le64toh(o->entry.realtime);
1910         return 0;
1911 }
1912
1913 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1914         Object *o;
1915         JournalFile *f;
1916         int r;
1917         sd_id128_t id;
1918
1919         assert_return(j, -EINVAL);
1920         assert_return(!journal_pid_changed(j), -ECHILD);
1921
1922         f = j->current_file;
1923         if (!f)
1924                 return -EADDRNOTAVAIL;
1925
1926         if (f->current_offset <= 0)
1927                 return -EADDRNOTAVAIL;
1928
1929         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1930         if (r < 0)
1931                 return r;
1932
1933         if (ret_boot_id)
1934                 *ret_boot_id = o->entry.boot_id;
1935         else {
1936                 r = sd_id128_get_boot(&id);
1937                 if (r < 0)
1938                         return r;
1939
1940                 if (!sd_id128_equal(id, o->entry.boot_id))
1941                         return -ESTALE;
1942         }
1943
1944         if (ret)
1945                 *ret = le64toh(o->entry.monotonic);
1946
1947         return 0;
1948 }
1949
1950 static bool field_is_valid(const char *field) {
1951         const char *p;
1952
1953         assert(field);
1954
1955         if (isempty(field))
1956                 return false;
1957
1958         if (startswith(field, "__"))
1959                 return false;
1960
1961         for (p = field; *p; p++) {
1962
1963                 if (*p == '_')
1964                         continue;
1965
1966                 if (*p >= 'A' && *p <= 'Z')
1967                         continue;
1968
1969                 if (*p >= '0' && *p <= '9')
1970                         continue;
1971
1972                 return false;
1973         }
1974
1975         return true;
1976 }
1977
1978 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1979         JournalFile *f;
1980         uint64_t i, n;
1981         size_t field_length;
1982         int r;
1983         Object *o;
1984
1985         assert_return(j, -EINVAL);
1986         assert_return(!journal_pid_changed(j), -ECHILD);
1987         assert_return(field, -EINVAL);
1988         assert_return(data, -EINVAL);
1989         assert_return(size, -EINVAL);
1990         assert_return(field_is_valid(field), -EINVAL);
1991
1992         f = j->current_file;
1993         if (!f)
1994                 return -EADDRNOTAVAIL;
1995
1996         if (f->current_offset <= 0)
1997                 return -EADDRNOTAVAIL;
1998
1999         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2000         if (r < 0)
2001                 return r;
2002
2003         field_length = strlen(field);
2004
2005         n = journal_file_entry_n_items(o);
2006         for (i = 0; i < n; i++) {
2007                 uint64_t p, l;
2008                 le64_t le_hash;
2009                 size_t t;
2010                 int compression;
2011
2012                 p = le64toh(o->entry.items[i].object_offset);
2013                 le_hash = o->entry.items[i].hash;
2014                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2015                 if (r < 0)
2016                         return r;
2017
2018                 if (le_hash != o->data.hash)
2019                         return -EBADMSG;
2020
2021                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
2022
2023                 compression = o->object.flags & OBJECT_COMPRESSION_MASK;
2024                 if (compression) {
2025 #if defined(HAVE_XZ) || defined(HAVE_LZ4)
2026                         if (decompress_startswith(compression,
2027                                                   o->data.payload, l,
2028                                                   &f->compress_buffer, &f->compress_buffer_size,
2029                                                   field, field_length, '=')) {
2030
2031                                 size_t rsize;
2032
2033                                 r = decompress_blob(compression,
2034                                                     o->data.payload, l,
2035                                                     &f->compress_buffer, &f->compress_buffer_size, &rsize,
2036                                                     j->data_threshold);
2037                                 if (r < 0)
2038                                         return r;
2039
2040                                 *data = f->compress_buffer;
2041                                 *size = (size_t) rsize;
2042
2043                                 return 0;
2044                         }
2045 #else
2046                         return -EPROTONOSUPPORT;
2047 #endif
2048                 } else if (l >= field_length+1 &&
2049                            memcmp(o->data.payload, field, field_length) == 0 &&
2050                            o->data.payload[field_length] == '=') {
2051
2052                         t = (size_t) l;
2053
2054                         if ((uint64_t) t != l)
2055                                 return -E2BIG;
2056
2057                         *data = o->data.payload;
2058                         *size = t;
2059
2060                         return 0;
2061                 }
2062
2063                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2064                 if (r < 0)
2065                         return r;
2066         }
2067
2068         return -ENOENT;
2069 }
2070
2071 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
2072         size_t t;
2073         uint64_t l;
2074         int compression;
2075
2076         l = le64toh(o->object.size) - offsetof(Object, data.payload);
2077         t = (size_t) l;
2078
2079         /* We can't read objects larger than 4G on a 32bit machine */
2080         if ((uint64_t) t != l)
2081                 return -E2BIG;
2082
2083         compression = o->object.flags & OBJECT_COMPRESSION_MASK;
2084         if (compression) {
2085 #if defined(HAVE_XZ) || defined(HAVE_LZ4)
2086                 size_t rsize;
2087                 int r;
2088
2089                 r = decompress_blob(compression,
2090                                     o->data.payload, l, &f->compress_buffer,
2091                                     &f->compress_buffer_size, &rsize, j->data_threshold);
2092                 if (r < 0)
2093                         return r;
2094
2095                 *data = f->compress_buffer;
2096                 *size = (size_t) rsize;
2097 #else
2098                 return -EPROTONOSUPPORT;
2099 #endif
2100         } else {
2101                 *data = o->data.payload;
2102                 *size = t;
2103         }
2104
2105         return 0;
2106 }
2107
2108 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
2109         JournalFile *f;
2110         uint64_t p, n;
2111         le64_t le_hash;
2112         int r;
2113         Object *o;
2114
2115         assert_return(j, -EINVAL);
2116         assert_return(!journal_pid_changed(j), -ECHILD);
2117         assert_return(data, -EINVAL);
2118         assert_return(size, -EINVAL);
2119
2120         f = j->current_file;
2121         if (!f)
2122                 return -EADDRNOTAVAIL;
2123
2124         if (f->current_offset <= 0)
2125                 return -EADDRNOTAVAIL;
2126
2127         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2128         if (r < 0)
2129                 return r;
2130
2131         n = journal_file_entry_n_items(o);
2132         if (j->current_field >= n)
2133                 return 0;
2134
2135         p = le64toh(o->entry.items[j->current_field].object_offset);
2136         le_hash = o->entry.items[j->current_field].hash;
2137         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2138         if (r < 0)
2139                 return r;
2140
2141         if (le_hash != o->data.hash)
2142                 return -EBADMSG;
2143
2144         r = return_data(j, f, o, data, size);
2145         if (r < 0)
2146                 return r;
2147
2148         j->current_field ++;
2149
2150         return 1;
2151 }
2152
2153 _public_ void sd_journal_restart_data(sd_journal *j) {
2154         if (!j)
2155                 return;
2156
2157         j->current_field = 0;
2158 }
2159
2160 _public_ int sd_journal_get_fd(sd_journal *j) {
2161         int r;
2162
2163         assert_return(j, -EINVAL);
2164         assert_return(!journal_pid_changed(j), -ECHILD);
2165
2166         if (j->inotify_fd >= 0)
2167                 return j->inotify_fd;
2168
2169         r = allocate_inotify(j);
2170         if (r < 0)
2171                 return r;
2172
2173         /* Iterate through all dirs again, to add them to the
2174          * inotify */
2175         if (j->no_new_files)
2176                 r = add_current_paths(j);
2177         else if (j->path)
2178                 r = add_root_directory(j, j->path);
2179         else
2180                 r = add_search_paths(j);
2181         if (r < 0)
2182                 return r;
2183
2184         return j->inotify_fd;
2185 }
2186
2187 _public_ int sd_journal_get_events(sd_journal *j) {
2188         int fd;
2189
2190         assert_return(j, -EINVAL);
2191         assert_return(!journal_pid_changed(j), -ECHILD);
2192
2193         fd = sd_journal_get_fd(j);
2194         if (fd < 0)
2195                 return fd;
2196
2197         return POLLIN;
2198 }
2199
2200 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2201         int fd;
2202
2203         assert_return(j, -EINVAL);
2204         assert_return(!journal_pid_changed(j), -ECHILD);
2205         assert_return(timeout_usec, -EINVAL);
2206
2207         fd = sd_journal_get_fd(j);
2208         if (fd < 0)
2209                 return fd;
2210
2211         if (!j->on_network) {
2212                 *timeout_usec = (uint64_t) -1;
2213                 return 0;
2214         }
2215
2216         /* If we are on the network we need to regularly check for
2217          * changes manually */
2218
2219         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2220         return 1;
2221 }
2222
2223 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2224         Directory *d;
2225         int r;
2226
2227         assert(j);
2228         assert(e);
2229
2230         /* Is this a subdirectory we watch? */
2231         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2232         if (d) {
2233                 sd_id128_t id;
2234
2235                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2236                     (endswith(e->name, ".journal") ||
2237                      endswith(e->name, ".journal~"))) {
2238
2239                         /* Event for a journal file */
2240
2241                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2242                                 r = add_file(j, d->path, e->name);
2243                                 if (r < 0) {
2244                                         log_debug_errno(r, "Failed to add file %s/%s: %m",
2245                                                         d->path, e->name);
2246                                         set_put_error(j, r);
2247                                 }
2248
2249                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2250
2251                                 r = remove_file(j, d->path, e->name);
2252                                 if (r < 0)
2253                                         log_debug_errno(r, "Failed to remove file %s/%s: %m", d->path, e->name);
2254                         }
2255
2256                 } else if (!d->is_root && e->len == 0) {
2257
2258                         /* Event for a subdirectory */
2259
2260                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2261                                 r = remove_directory(j, d);
2262                                 if (r < 0)
2263                                         log_debug_errno(r, "Failed to remove directory %s: %m", d->path);
2264                         }
2265
2266
2267                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2268
2269                         /* Event for root directory */
2270
2271                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2272                                 r = add_directory(j, d->path, e->name);
2273                                 if (r < 0)
2274                                         log_debug_errno(r, "Failed to add directory %s/%s: %m", d->path, e->name);
2275                         }
2276                 }
2277
2278                 return;
2279         }
2280
2281         if (e->mask & IN_IGNORED)
2282                 return;
2283
2284         log_warning("Unknown inotify event.");
2285 }
2286
2287 static int determine_change(sd_journal *j) {
2288         bool b;
2289
2290         assert(j);
2291
2292         b = j->current_invalidate_counter != j->last_invalidate_counter;
2293         j->last_invalidate_counter = j->current_invalidate_counter;
2294
2295         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2296 }
2297
2298 _public_ int sd_journal_process(sd_journal *j) {
2299         bool got_something = false;
2300
2301         assert_return(j, -EINVAL);
2302         assert_return(!journal_pid_changed(j), -ECHILD);
2303
2304         j->last_process_usec = now(CLOCK_MONOTONIC);
2305
2306         for (;;) {
2307                 uint8_t buffer[INOTIFY_EVENT_MAX] _alignas_(struct inotify_event);
2308                 struct inotify_event *e;
2309                 ssize_t l;
2310
2311                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2312                 if (l < 0) {
2313                         if (errno == EAGAIN || errno == EINTR)
2314                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2315
2316                         return -errno;
2317                 }
2318
2319                 got_something = true;
2320
2321                 FOREACH_INOTIFY_EVENT(e, buffer, l)
2322                         process_inotify_event(j, e);
2323         }
2324 }
2325
2326 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2327         int r;
2328         uint64_t t;
2329
2330         assert_return(j, -EINVAL);
2331         assert_return(!journal_pid_changed(j), -ECHILD);
2332
2333         if (j->inotify_fd < 0) {
2334
2335                 /* This is the first invocation, hence create the
2336                  * inotify watch */
2337                 r = sd_journal_get_fd(j);
2338                 if (r < 0)
2339                         return r;
2340
2341                 /* The journal might have changed since the context
2342                  * object was created and we weren't watching before,
2343                  * hence don't wait for anything, and return
2344                  * immediately. */
2345                 return determine_change(j);
2346         }
2347
2348         r = sd_journal_get_timeout(j, &t);
2349         if (r < 0)
2350                 return r;
2351
2352         if (t != (uint64_t) -1) {
2353                 usec_t n;
2354
2355                 n = now(CLOCK_MONOTONIC);
2356                 t = t > n ? t - n : 0;
2357
2358                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2359                         timeout_usec = t;
2360         }
2361
2362         do {
2363                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2364         } while (r == -EINTR);
2365
2366         if (r < 0)
2367                 return r;
2368
2369         return sd_journal_process(j);
2370 }
2371
2372 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2373         Iterator i;
2374         JournalFile *f;
2375         bool first = true;
2376         uint64_t fmin = 0, tmax = 0;
2377         int r;
2378
2379         assert_return(j, -EINVAL);
2380         assert_return(!journal_pid_changed(j), -ECHILD);
2381         assert_return(from || to, -EINVAL);
2382         assert_return(from != to, -EINVAL);
2383
2384         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2385                 usec_t fr, t;
2386
2387                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2388                 if (r == -ENOENT)
2389                         continue;
2390                 if (r < 0)
2391                         return r;
2392                 if (r == 0)
2393                         continue;
2394
2395                 if (first) {
2396                         fmin = fr;
2397                         tmax = t;
2398                         first = false;
2399                 } else {
2400                         fmin = MIN(fr, fmin);
2401                         tmax = MAX(t, tmax);
2402                 }
2403         }
2404
2405         if (from)
2406                 *from = fmin;
2407         if (to)
2408                 *to = tmax;
2409
2410         return first ? 0 : 1;
2411 }
2412
2413 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2414         Iterator i;
2415         JournalFile *f;
2416         bool found = false;
2417         int r;
2418
2419         assert_return(j, -EINVAL);
2420         assert_return(!journal_pid_changed(j), -ECHILD);
2421         assert_return(from || to, -EINVAL);
2422         assert_return(from != to, -EINVAL);
2423
2424         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2425                 usec_t fr, t;
2426
2427                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2428                 if (r == -ENOENT)
2429                         continue;
2430                 if (r < 0)
2431                         return r;
2432                 if (r == 0)
2433                         continue;
2434
2435                 if (found) {
2436                         if (from)
2437                                 *from = MIN(fr, *from);
2438                         if (to)
2439                                 *to = MAX(t, *to);
2440                 } else {
2441                         if (from)
2442                                 *from = fr;
2443                         if (to)
2444                                 *to = t;
2445                         found = true;
2446                 }
2447         }
2448
2449         return found;
2450 }
2451
2452 void journal_print_header(sd_journal *j) {
2453         Iterator i;
2454         JournalFile *f;
2455         bool newline = false;
2456
2457         assert(j);
2458
2459         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2460                 if (newline)
2461                         putchar('\n');
2462                 else
2463                         newline = true;
2464
2465                 journal_file_print_header(f);
2466         }
2467 }
2468
2469 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2470         Iterator i;
2471         JournalFile *f;
2472         uint64_t sum = 0;
2473
2474         assert_return(j, -EINVAL);
2475         assert_return(!journal_pid_changed(j), -ECHILD);
2476         assert_return(bytes, -EINVAL);
2477
2478         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2479                 struct stat st;
2480
2481                 if (fstat(f->fd, &st) < 0)
2482                         return -errno;
2483
2484                 sum += (uint64_t) st.st_blocks * 512ULL;
2485         }
2486
2487         *bytes = sum;
2488         return 0;
2489 }
2490
2491 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2492         char *f;
2493
2494         assert_return(j, -EINVAL);
2495         assert_return(!journal_pid_changed(j), -ECHILD);
2496         assert_return(!isempty(field), -EINVAL);
2497         assert_return(field_is_valid(field), -EINVAL);
2498
2499         f = strdup(field);
2500         if (!f)
2501                 return -ENOMEM;
2502
2503         free(j->unique_field);
2504         j->unique_field = f;
2505         j->unique_file = NULL;
2506         j->unique_offset = 0;
2507         j->unique_file_lost = false;
2508
2509         return 0;
2510 }
2511
2512 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2513         size_t k;
2514
2515         assert_return(j, -EINVAL);
2516         assert_return(!journal_pid_changed(j), -ECHILD);
2517         assert_return(data, -EINVAL);
2518         assert_return(l, -EINVAL);
2519         assert_return(j->unique_field, -EINVAL);
2520
2521         k = strlen(j->unique_field);
2522
2523         if (!j->unique_file) {
2524                 if (j->unique_file_lost)
2525                         return 0;
2526
2527                 j->unique_file = ordered_hashmap_first(j->files);
2528                 if (!j->unique_file)
2529                         return 0;
2530
2531                 j->unique_offset = 0;
2532         }
2533
2534         for (;;) {
2535                 JournalFile *of;
2536                 Iterator i;
2537                 Object *o;
2538                 const void *odata;
2539                 size_t ol;
2540                 bool found;
2541                 int r;
2542
2543                 /* Proceed to next data object in the field's linked list */
2544                 if (j->unique_offset == 0) {
2545                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2546                         if (r < 0)
2547                                 return r;
2548
2549                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2550                 } else {
2551                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2552                         if (r < 0)
2553                                 return r;
2554
2555                         j->unique_offset = le64toh(o->data.next_field_offset);
2556                 }
2557
2558                 /* We reached the end of the list? Then start again, with the next file */
2559                 if (j->unique_offset == 0) {
2560                         j->unique_file = ordered_hashmap_next(j->files, j->unique_file->path);
2561                         if (!j->unique_file)
2562                                 return 0;
2563
2564                         continue;
2565                 }
2566
2567                 /* We do not use OBJECT_DATA context here, but OBJECT_UNUSED
2568                  * instead, so that we can look at this data object at the same
2569                  * time as one on another file */
2570                 r = journal_file_move_to_object(j->unique_file, OBJECT_UNUSED, j->unique_offset, &o);
2571                 if (r < 0)
2572                         return r;
2573
2574                 /* Let's do the type check by hand, since we used 0 context above. */
2575                 if (o->object.type != OBJECT_DATA) {
2576                         log_debug("%s:offset " OFSfmt ": object has type %d, expected %d",
2577                                   j->unique_file->path, j->unique_offset,
2578                                   o->object.type, OBJECT_DATA);
2579                         return -EBADMSG;
2580                 }
2581
2582                 r = return_data(j, j->unique_file, o, &odata, &ol);
2583                 if (r < 0)
2584                         return r;
2585
2586                 /* Check if we have at least the field name and "=". */
2587                 if (ol <= k) {
2588                         log_debug("%s:offset " OFSfmt ": object has size %zu, expected at least %zu",
2589                                   j->unique_file->path, j->unique_offset,
2590                                   ol, k + 1);
2591                         return -EBADMSG;
2592                 }
2593
2594                 if (memcmp(odata, j->unique_field, k) || ((const char*) odata)[k] != '=') {
2595                         log_debug("%s:offset " OFSfmt ": object does not start with \"%s=\"",
2596                                   j->unique_file->path, j->unique_offset,
2597                                   j->unique_field);
2598                         return -EBADMSG;
2599                 }
2600
2601                 /* OK, now let's see if we already returned this data
2602                  * object by checking if it exists in the earlier
2603                  * traversed files. */
2604                 found = false;
2605                 ORDERED_HASHMAP_FOREACH(of, j->files, i) {
2606                         Object *oo;
2607                         uint64_t op;
2608
2609                         if (of == j->unique_file)
2610                                 break;
2611
2612                         /* Skip this file it didn't have any fields
2613                          * indexed */
2614                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2615                             le64toh(of->header->n_fields) <= 0)
2616                                 continue;
2617
2618                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2619                         if (r < 0)
2620                                 return r;
2621
2622                         if (r > 0)
2623                                 found = true;
2624                 }
2625
2626                 if (found)
2627                         continue;
2628
2629                 r = return_data(j, j->unique_file, o, data, l);
2630                 if (r < 0)
2631                         return r;
2632
2633                 return 1;
2634         }
2635 }
2636
2637 _public_ void sd_journal_restart_unique(sd_journal *j) {
2638         if (!j)
2639                 return;
2640
2641         j->unique_file = NULL;
2642         j->unique_offset = 0;
2643         j->unique_file_lost = false;
2644 }
2645
2646 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2647         assert_return(j, -EINVAL);
2648         assert_return(!journal_pid_changed(j), -ECHILD);
2649
2650         return !j->on_network;
2651 }
2652
2653 static char *lookup_field(const char *field, void *userdata) {
2654         sd_journal *j = userdata;
2655         const void *data;
2656         size_t size, d;
2657         int r;
2658
2659         assert(field);
2660         assert(j);
2661
2662         r = sd_journal_get_data(j, field, &data, &size);
2663         if (r < 0 ||
2664             size > REPLACE_VAR_MAX)
2665                 return strdup(field);
2666
2667         d = strlen(field) + 1;
2668
2669         return strndup((const char*) data + d, size - d);
2670 }
2671
2672 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2673         const void *data;
2674         size_t size;
2675         sd_id128_t id;
2676         _cleanup_free_ char *text = NULL, *cid = NULL;
2677         char *t;
2678         int r;
2679
2680         assert_return(j, -EINVAL);
2681         assert_return(!journal_pid_changed(j), -ECHILD);
2682         assert_return(ret, -EINVAL);
2683
2684         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2685         if (r < 0)
2686                 return r;
2687
2688         cid = strndup((const char*) data + 11, size - 11);
2689         if (!cid)
2690                 return -ENOMEM;
2691
2692         r = sd_id128_from_string(cid, &id);
2693         if (r < 0)
2694                 return r;
2695
2696         r = catalog_get(CATALOG_DATABASE, id, &text);
2697         if (r < 0)
2698                 return r;
2699
2700         t = replace_var(text, lookup_field, j);
2701         if (!t)
2702                 return -ENOMEM;
2703
2704         *ret = t;
2705         return 0;
2706 }
2707
2708 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2709         assert_return(ret, -EINVAL);
2710
2711         return catalog_get(CATALOG_DATABASE, id, ret);
2712 }
2713
2714 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2715         assert_return(j, -EINVAL);
2716         assert_return(!journal_pid_changed(j), -ECHILD);
2717
2718         j->data_threshold = sz;
2719         return 0;
2720 }
2721
2722 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2723         assert_return(j, -EINVAL);
2724         assert_return(!journal_pid_changed(j), -ECHILD);
2725         assert_return(sz, -EINVAL);
2726
2727         *sz = j->data_threshold;
2728         return 0;
2729 }