src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "strv.h"
  37 #include "path-util.h"
  38 #include "lookup3.h"
  39 #include "compress.h"
  40 #include "journal-internal.h"
  41 #include "missing.h"
  42 #include "catalog.h"
  43 #include "replace-var.h"
  44 #include "fileio.h"
  45
  46 #define JOURNAL_FILES_MAX 1024
  47
  48 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  49
  50 #define REPLACE_VAR_MAX 256
  51
  52 #define DEFAULT_DATA_THRESHOLD (64*1024)
  53
  54 static void remove_file_real(sd_journal *j, JournalFile *f);
  55
  56 static bool journal_pid_changed(sd_journal *j) {
  57         assert(j);
  58
  59         /* We don't support people creating a journal object and
  60          * keeping it around over a fork(). Let's complain. */
  61
  62         return j->original_pid != getpid();
  63 }
  64
  65 /* We return an error here only if we didn't manage to
  66    memorize the real error. */
  67 static int set_put_error(sd_journal *j, int r) {
  68         int k;
  69
  70         if (r >= 0)
  71                 return r;
  72
  73         k = set_ensure_allocated(&j->errors, NULL);
  74         if (k < 0)
  75                 return k;
  76
  77         return set_put(j->errors, INT_TO_PTR(r));
  78 }
  79
  80 static void detach_location(sd_journal *j) {
  81         Iterator i;
  82         JournalFile *f;
  83
  84         assert(j);
  85
  86         j->current_file = NULL;
  87         j->current_field = 0;
  88
  89         ORDERED_HASHMAP_FOREACH(f, j->files, i)
  90                 journal_file_reset_location(f);
  91 }
  92
  93 static void reset_location(sd_journal *j) {
  94         assert(j);
  95
  96         detach_location(j);
  97         zero(j->current_location);
  98 }
  99
 100 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
 101         assert(l);
 102         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 103         assert(f);
 104         assert(o->object.type == OBJECT_ENTRY);
 105
 106         l->type = type;
 107         l->seqnum = le64toh(o->entry.seqnum);
 108         l->seqnum_id = f->header->seqnum_id;
 109         l->realtime = le64toh(o->entry.realtime);
 110         l->monotonic = le64toh(o->entry.monotonic);
 111         l->boot_id = o->entry.boot_id;
 112         l->xor_hash = le64toh(o->entry.xor_hash);
 113
 114         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 115 }
 116
 117 static void set_location(sd_journal *j, JournalFile *f, Object *o) {
 118         assert(j);
 119         assert(f);
 120         assert(o);
 121
 122         init_location(&j->current_location, LOCATION_DISCRETE, f, o);
 123
 124         j->current_file = f;
 125         j->current_field = 0;
 126
 127         /* Let f know its candidate entry was picked. */
 128         assert(f->location_type == LOCATION_SEEK);
 129         f->location_type = LOCATION_DISCRETE;
 130 }
 131
 132 static int match_is_valid(const void *data, size_t size) {
 133         const char *b, *p;
 134
 135         assert(data);
 136
 137         if (size < 2)
 138                 return false;
 139
 140         if (startswith(data, "__"))
 141                 return false;
 142
 143         b = data;
 144         for (p = b; p < b + size; p++) {
 145
 146                 if (*p == '=')
 147                         return p > b;
 148
 149                 if (*p == '_')
 150                         continue;
 151
 152                 if (*p >= 'A' && *p <= 'Z')
 153                         continue;
 154
 155                 if (*p >= '0' && *p <= '9')
 156                         continue;
 157
 158                 return false;
 159         }
 160
 161         return false;
 162 }
 163
 164 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 165         const uint8_t *a = _a, *b = _b;
 166         size_t j;
 167
 168         for (j = 0; j < s && j < t; j++) {
 169
 170                 if (a[j] != b[j])
 171                         return false;
 172
 173                 if (a[j] == '=')
 174                         return true;
 175         }
 176
 177         assert_not_reached("\"=\" not found");
 178 }
 179
 180 static Match *match_new(Match *p, MatchType t) {
 181         Match *m;
 182
 183         m = new0(Match, 1);
 184         if (!m)
 185                 return NULL;
 186
 187         m->type = t;
 188
 189         if (p) {
 190                 m->parent = p;
 191                 LIST_PREPEND(matches, p->matches, m);
 192         }
 193
 194         return m;
 195 }
 196
 197 static void match_free(Match *m) {
 198         assert(m);
 199
 200         while (m->matches)
 201                 match_free(m->matches);
 202
 203         if (m->parent)
 204                 LIST_REMOVE(matches, m->parent->matches, m);
 205
 206         free(m->data);
 207         free(m);
 208 }
 209
 210 static void match_free_if_empty(Match *m) {
 211         if (!m || m->matches)
 212                 return;
 213
 214         match_free(m);
 215 }
 216
 217 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 218         Match *l3, *l4, *add_here = NULL, *m;
 219         le64_t le_hash;
 220
 221         assert_return(j, -EINVAL);
 222         assert_return(!journal_pid_changed(j), -ECHILD);
 223         assert_return(data, -EINVAL);
 224
 225         if (size == 0)
 226                 size = strlen(data);
 227
 228         assert_return(match_is_valid(data, size), -EINVAL);
 229
 230         /* level 0: AND term
 231          * level 1: OR terms
 232          * level 2: AND terms
 233          * level 3: OR terms
 234          * level 4: concrete matches */
 235
 236         if (!j->level0) {
 237                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 238                 if (!j->level0)
 239                         return -ENOMEM;
 240         }
 241
 242         if (!j->level1) {
 243                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 244                 if (!j->level1)
 245                         return -ENOMEM;
 246         }
 247
 248         if (!j->level2) {
 249                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 250                 if (!j->level2)
 251                         return -ENOMEM;
 252         }
 253
 254         assert(j->level0->type == MATCH_AND_TERM);
 255         assert(j->level1->type == MATCH_OR_TERM);
 256         assert(j->level2->type == MATCH_AND_TERM);
 257
 258         le_hash = htole64(hash64(data, size));
 259
 260         LIST_FOREACH(matches, l3, j->level2->matches) {
 261                 assert(l3->type == MATCH_OR_TERM);
 262
 263                 LIST_FOREACH(matches, l4, l3->matches) {
 264                         assert(l4->type == MATCH_DISCRETE);
 265
 266                         /* Exactly the same match already? Then ignore
 267                          * this addition */
 268                         if (l4->le_hash == le_hash &&
 269                             l4->size == size &&
 270                             memcmp(l4->data, data, size) == 0)
 271                                 return 0;
 272
 273                         /* Same field? Then let's add this to this OR term */
 274                         if (same_field(data, size, l4->data, l4->size)) {
 275                                 add_here = l3;
 276                                 break;
 277                         }
 278                 }
 279
 280                 if (add_here)
 281                         break;
 282         }
 283
 284         if (!add_here) {
 285                 add_here = match_new(j->level2, MATCH_OR_TERM);
 286                 if (!add_here)
 287                         goto fail;
 288         }
 289
 290         m = match_new(add_here, MATCH_DISCRETE);
 291         if (!m)
 292                 goto fail;
 293
 294         m->le_hash = le_hash;
 295         m->size = size;
 296         m->data = memdup(data, size);
 297         if (!m->data)
 298                 goto fail;
 299
 300         detach_location(j);
 301
 302         return 0;
 303
 304 fail:
 305         match_free_if_empty(add_here);
 306         match_free_if_empty(j->level2);
 307         match_free_if_empty(j->level1);
 308         match_free_if_empty(j->level0);
 309
 310         return -ENOMEM;
 311 }
 312
 313 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 314         assert_return(j, -EINVAL);
 315         assert_return(!journal_pid_changed(j), -ECHILD);
 316
 317         if (!j->level0)
 318                 return 0;
 319
 320         if (!j->level1)
 321                 return 0;
 322
 323         if (!j->level1->matches)
 324                 return 0;
 325
 326         j->level1 = NULL;
 327         j->level2 = NULL;
 328
 329         return 0;
 330 }
 331
 332 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 333         assert_return(j, -EINVAL);
 334         assert_return(!journal_pid_changed(j), -ECHILD);
 335
 336         if (!j->level0)
 337                 return 0;
 338
 339         if (!j->level1)
 340                 return 0;
 341
 342         if (!j->level2)
 343                 return 0;
 344
 345         if (!j->level2->matches)
 346                 return 0;
 347
 348         j->level2 = NULL;
 349         return 0;
 350 }
 351
 352 static char *match_make_string(Match *m) {
 353         char *p, *r;
 354         Match *i;
 355         bool enclose = false;
 356
 357         if (!m)
 358                 return strdup("none");
 359
 360         if (m->type == MATCH_DISCRETE)
 361                 return strndup(m->data, m->size);
 362
 363         p = NULL;
 364         LIST_FOREACH(matches, i, m->matches) {
 365                 char *t, *k;
 366
 367                 t = match_make_string(i);
 368                 if (!t) {
 369                         free(p);
 370                         return NULL;
 371                 }
 372
 373                 if (p) {
 374                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 375                         free(p);
 376                         free(t);
 377
 378                         if (!k)
 379                                 return NULL;
 380
 381                         p = k;
 382
 383                         enclose = true;
 384                 } else
 385                         p = t;
 386         }
 387
 388         if (enclose) {
 389                 r = strjoin("(", p, ")", NULL);
 390                 free(p);
 391                 return r;
 392         }
 393
 394         return p;
 395 }
 396
 397 char *journal_make_match_string(sd_journal *j) {
 398         assert(j);
 399
 400         return match_make_string(j->level0);
 401 }
 402
 403 _public_ void sd_journal_flush_matches(sd_journal *j) {
 404         if (!j)
 405                 return;
 406
 407         if (j->level0)
 408                 match_free(j->level0);
 409
 410         j->level0 = j->level1 = j->level2 = NULL;
 411
 412         detach_location(j);
 413 }
 414
 415 static int compare_entry_order(JournalFile *af, Object *_ao,
 416                                JournalFile *bf, uint64_t bp) {
 417
 418         uint64_t a, b;
 419         Object *ao, *bo;
 420         int r;
 421
 422         assert(af);
 423         assert(bf);
 424         assert(_ao);
 425
 426         /* The mmap cache might invalidate the object from the first
 427          * file if we look at the one from the second file. Hence
 428          * temporarily copy the header of the first one, and look at
 429          * that only. */
 430         ao = alloca(offsetof(EntryObject, items));
 431         memcpy(ao, _ao, offsetof(EntryObject, items));
 432
 433         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 434         if (r < 0)
 435                 return strcmp(af->path, bf->path);
 436
 437         /* We operate on two different files here, hence we can access
 438          * two objects at the same time, which we normally can't.
 439          *
 440          * If contents and timestamps match, these entries are
 441          * identical, even if the seqnum does not match */
 442
 443         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 444             ao->entry.monotonic == bo->entry.monotonic &&
 445             ao->entry.realtime == bo->entry.realtime &&
 446             ao->entry.xor_hash == bo->entry.xor_hash)
 447                 return 0;
 448
 449         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 450
 451                 /* If this is from the same seqnum source, compare
 452                  * seqnums */
 453                 a = le64toh(ao->entry.seqnum);
 454                 b = le64toh(bo->entry.seqnum);
 455
 456                 if (a < b)
 457                         return -1;
 458                 if (a > b)
 459                         return 1;
 460
 461                 /* Wow! This is weird, different data but the same
 462                  * seqnums? Something is borked, but let's make the
 463                  * best of it and compare by time. */
 464         }
 465
 466         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 467
 468                 /* If the boot id matches, compare monotonic time */
 469                 a = le64toh(ao->entry.monotonic);
 470                 b = le64toh(bo->entry.monotonic);
 471
 472                 if (a < b)
 473                         return -1;
 474                 if (a > b)
 475                         return 1;
 476         }
 477
 478         /* Otherwise, compare UTC time */
 479         a = le64toh(ao->entry.realtime);
 480         b = le64toh(bo->entry.realtime);
 481
 482         if (a < b)
 483                 return -1;
 484         if (a > b)
 485                 return 1;
 486
 487         /* Finally, compare by contents */
 488         a = le64toh(ao->entry.xor_hash);
 489         b = le64toh(bo->entry.xor_hash);
 490
 491         if (a < b)
 492                 return -1;
 493         if (a > b)
 494                 return 1;
 495
 496         return 0;
 497 }
 498
 499 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 500         uint64_t a;
 501
 502         assert(af);
 503         assert(ao);
 504         assert(l);
 505         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 506
 507         if (l->monotonic_set &&
 508             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 509             l->realtime_set &&
 510             le64toh(ao->entry.realtime) == l->realtime &&
 511             l->xor_hash_set &&
 512             le64toh(ao->entry.xor_hash) == l->xor_hash)
 513                 return 0;
 514
 515         if (l->seqnum_set &&
 516             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 517
 518                 a = le64toh(ao->entry.seqnum);
 519
 520                 if (a < l->seqnum)
 521                         return -1;
 522                 if (a > l->seqnum)
 523                         return 1;
 524         }
 525
 526         if (l->monotonic_set &&
 527             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 528
 529                 a = le64toh(ao->entry.monotonic);
 530
 531                 if (a < l->monotonic)
 532                         return -1;
 533                 if (a > l->monotonic)
 534                         return 1;
 535         }
 536
 537         if (l->realtime_set) {
 538
 539                 a = le64toh(ao->entry.realtime);
 540
 541                 if (a < l->realtime)
 542                         return -1;
 543                 if (a > l->realtime)
 544                         return 1;
 545         }
 546
 547         if (l->xor_hash_set) {
 548                 a = le64toh(ao->entry.xor_hash);
 549
 550                 if (a < l->xor_hash)
 551                         return -1;
 552                 if (a > l->xor_hash)
 553                         return 1;
 554         }
 555
 556         return 0;
 557 }
 558
 559 static int next_for_match(
 560                 sd_journal *j,
 561                 Match *m,
 562                 JournalFile *f,
 563                 uint64_t after_offset,
 564                 direction_t direction,
 565                 Object **ret,
 566                 uint64_t *offset) {
 567
 568         int r;
 569         uint64_t np = 0;
 570         Object *n;
 571
 572         assert(j);
 573         assert(m);
 574         assert(f);
 575
 576         if (m->type == MATCH_DISCRETE) {
 577                 uint64_t dp;
 578
 579                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 580                 if (r <= 0)
 581                         return r;
 582
 583                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 584
 585         } else if (m->type == MATCH_OR_TERM) {
 586                 Match *i;
 587
 588                 /* Find the earliest match beyond after_offset */
 589
 590                 LIST_FOREACH(matches, i, m->matches) {
 591                         uint64_t cp;
 592
 593                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 594                         if (r < 0)
 595                                 return r;
 596                         else if (r > 0) {
 597                                 if (np == 0 || (direction == DIRECTION_DOWN ? cp < np : cp > np))
 598                                         np = cp;
 599                         }
 600                 }
 601
 602                 if (np == 0)
 603                         return 0;
 604
 605         } else if (m->type == MATCH_AND_TERM) {
 606                 Match *i, *last_moved;
 607
 608                 /* Always jump to the next matching entry and repeat
 609                  * this until we find an offset that matches for all
 610                  * matches. */
 611
 612                 if (!m->matches)
 613                         return 0;
 614
 615                 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
 616                 if (r <= 0)
 617                         return r;
 618
 619                 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
 620                 last_moved = m->matches;
 621
 622                 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
 623                         uint64_t cp;
 624
 625                         r = next_for_match(j, i, f, np, direction, NULL, &cp);
 626                         if (r <= 0)
 627                                 return r;
 628
 629                         assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
 630                         if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
 631                                 np = cp;
 632                                 last_moved = i;
 633                         }
 634                 }
 635         }
 636
 637         assert(np > 0);
 638
 639         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 640         if (r < 0)
 641                 return r;
 642
 643         if (ret)
 644                 *ret = n;
 645         if (offset)
 646                 *offset = np;
 647
 648         return 1;
 649 }
 650
 651 static int find_location_for_match(
 652                 sd_journal *j,
 653                 Match *m,
 654                 JournalFile *f,
 655                 direction_t direction,
 656                 Object **ret,
 657                 uint64_t *offset) {
 658
 659         int r;
 660
 661         assert(j);
 662         assert(m);
 663         assert(f);
 664
 665         if (m->type == MATCH_DISCRETE) {
 666                 uint64_t dp;
 667
 668                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 669                 if (r <= 0)
 670                         return r;
 671
 672                 /* FIXME: missing: find by monotonic */
 673
 674                 if (j->current_location.type == LOCATION_HEAD)
 675                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 676                 if (j->current_location.type == LOCATION_TAIL)
 677                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 678                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 679                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 680                 if (j->current_location.monotonic_set) {
 681                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 682                         if (r != -ENOENT)
 683                                 return r;
 684                 }
 685                 if (j->current_location.realtime_set)
 686                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 687
 688                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 689
 690         } else if (m->type == MATCH_OR_TERM) {
 691                 uint64_t np = 0;
 692                 Object *n;
 693                 Match *i;
 694
 695                 /* Find the earliest match */
 696
 697                 LIST_FOREACH(matches, i, m->matches) {
 698                         uint64_t cp;
 699
 700                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 701                         if (r < 0)
 702                                 return r;
 703                         else if (r > 0) {
 704                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 705                                         np = cp;
 706                         }
 707                 }
 708
 709                 if (np == 0)
 710                         return 0;
 711
 712                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 713                 if (r < 0)
 714                         return r;
 715
 716                 if (ret)
 717                         *ret = n;
 718                 if (offset)
 719                         *offset = np;
 720
 721                 return 1;
 722
 723         } else {
 724                 Match *i;
 725                 uint64_t np = 0;
 726
 727                 assert(m->type == MATCH_AND_TERM);
 728
 729                 /* First jump to the last match, and then find the
 730                  * next one where all matches match */
 731
 732                 if (!m->matches)
 733                         return 0;
 734
 735                 LIST_FOREACH(matches, i, m->matches) {
 736                         uint64_t cp;
 737
 738                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 739                         if (r <= 0)
 740                                 return r;
 741
 742                         if (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))
 743                                 np = cp;
 744                 }
 745
 746                 return next_for_match(j, m, f, np, direction, ret, offset);
 747         }
 748 }
 749
 750 static int find_location_with_matches(
 751                 sd_journal *j,
 752                 JournalFile *f,
 753                 direction_t direction,
 754                 Object **ret,
 755                 uint64_t *offset) {
 756
 757         int r;
 758
 759         assert(j);
 760         assert(f);
 761         assert(ret);
 762         assert(offset);
 763
 764         if (!j->level0) {
 765                 /* No matches is simple */
 766
 767                 if (j->current_location.type == LOCATION_HEAD)
 768                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 769                 if (j->current_location.type == LOCATION_TAIL)
 770                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 771                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 772                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 773                 if (j->current_location.monotonic_set) {
 774                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 775                         if (r != -ENOENT)
 776                                 return r;
 777                 }
 778                 if (j->current_location.realtime_set)
 779                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 780
 781                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 782         } else
 783                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 784 }
 785
 786 static int next_with_matches(
 787                 sd_journal *j,
 788                 JournalFile *f,
 789                 direction_t direction,
 790                 Object **ret,
 791                 uint64_t *offset) {
 792
 793         Object *c;
 794         uint64_t cp;
 795
 796         assert(j);
 797         assert(f);
 798         assert(ret);
 799         assert(offset);
 800
 801         c = *ret;
 802         cp = *offset;
 803
 804         /* No matches is easy. We simple advance the file
 805          * pointer by one. */
 806         if (!j->level0)
 807                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 808
 809         /* If we have a match then we look for the next matching entry
 810          * with an offset at least one step larger */
 811         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 812 }
 813
 814 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 815         Object *c;
 816         uint64_t cp;
 817         int r;
 818
 819         assert(j);
 820         assert(f);
 821
 822         if (f->last_direction == direction && f->current_offset > 0) {
 823                 cp = f->current_offset;
 824
 825                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 826                 if (r < 0)
 827                         return r;
 828
 829                 r = next_with_matches(j, f, direction, &c, &cp);
 830                 if (r <= 0)
 831                         return r;
 832         } else {
 833                 r = find_location_with_matches(j, f, direction, &c, &cp);
 834                 if (r <= 0)
 835                         return r;
 836         }
 837
 838         /* OK, we found the spot, now let's advance until an entry
 839          * that is actually different from what we were previously
 840          * looking at. This is necessary to handle entries which exist
 841          * in two (or more) journal files, and which shall all be
 842          * suppressed but one. */
 843
 844         for (;;) {
 845                 bool found;
 846
 847                 if (j->current_location.type == LOCATION_DISCRETE) {
 848                         int k;
 849
 850                         k = compare_with_location(f, c, &j->current_location);
 851
 852                         found = direction == DIRECTION_DOWN ? k > 0 : k < 0;
 853                 } else
 854                         found = true;
 855
 856                 if (found) {
 857                         journal_file_save_location(f, direction, c, cp);
 858
 859                         if (ret)
 860                                 *ret = c;
 861                         if (offset)
 862                                 *offset = cp;
 863                         return 1;
 864                 }
 865
 866                 r = next_with_matches(j, f, direction, &c, &cp);
 867                 if (r <= 0)
 868                         return r;
 869         }
 870 }
 871
 872 static int real_journal_next(sd_journal *j, direction_t direction) {
 873         JournalFile *f, *new_file = NULL;
 874         uint64_t new_offset = 0;
 875         uint64_t p = 0;
 876         Iterator i;
 877         Object *o;
 878         int r;
 879
 880         assert_return(j, -EINVAL);
 881         assert_return(!journal_pid_changed(j), -ECHILD);
 882
 883         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
 884                 bool found;
 885
 886                 r = next_beyond_location(j, f, direction, &o, &p);
 887                 if (r < 0) {
 888                         log_debug_errno(r, "Can't iterate through %s, ignoring: %m", f->path);
 889                         remove_file_real(j, f);
 890                         continue;
 891                 } else if (r == 0) {
 892                         f->location_type = LOCATION_TAIL;
 893                         continue;
 894                 }
 895
 896                 if (!new_file)
 897                         found = true;
 898                 else {
 899                         int k;
 900
 901                         k = compare_entry_order(f, o, new_file, new_offset);
 902
 903                         found = direction == DIRECTION_DOWN ? k < 0 : k > 0;
 904                 }
 905
 906                 if (found) {
 907                         new_file = f;
 908                         new_offset = p;
 909                 }
 910         }
 911
 912         if (!new_file)
 913                 return 0;
 914
 915         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 916         if (r < 0)
 917                 return r;
 918
 919         set_location(j, new_file, o);
 920
 921         return 1;
 922 }
 923
 924 _public_ int sd_journal_next(sd_journal *j) {
 925         return real_journal_next(j, DIRECTION_DOWN);
 926 }
 927
 928 _public_ int sd_journal_previous(sd_journal *j) {
 929         return real_journal_next(j, DIRECTION_UP);
 930 }
 931
 932 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 933         int c = 0, r;
 934
 935         assert_return(j, -EINVAL);
 936         assert_return(!journal_pid_changed(j), -ECHILD);
 937
 938         if (skip == 0) {
 939                 /* If this is not a discrete skip, then at least
 940                  * resolve the current location */
 941                 if (j->current_location.type != LOCATION_DISCRETE)
 942                         return real_journal_next(j, direction);
 943
 944                 return 0;
 945         }
 946
 947         do {
 948                 r = real_journal_next(j, direction);
 949                 if (r < 0)
 950                         return r;
 951
 952                 if (r == 0)
 953                         return c;
 954
 955                 skip--;
 956                 c++;
 957         } while (skip > 0);
 958
 959         return c;
 960 }
 961
 962 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 963         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 964 }
 965
 966 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 967         return real_journal_next_skip(j, DIRECTION_UP, skip);
 968 }
 969
 970 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 971         Object *o;
 972         int r;
 973         char bid[33], sid[33];
 974
 975         assert_return(j, -EINVAL);
 976         assert_return(!journal_pid_changed(j), -ECHILD);
 977         assert_return(cursor, -EINVAL);
 978
 979         if (!j->current_file || j->current_file->current_offset <= 0)
 980                 return -EADDRNOTAVAIL;
 981
 982         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 983         if (r < 0)
 984                 return r;
 985
 986         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 987         sd_id128_to_string(o->entry.boot_id, bid);
 988
 989         if (asprintf(cursor,
 990                      "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
 991                      sid, le64toh(o->entry.seqnum),
 992                      bid, le64toh(o->entry.monotonic),
 993                      le64toh(o->entry.realtime),
 994                      le64toh(o->entry.xor_hash)) < 0)
 995                 return -ENOMEM;
 996
 997         return 0;
 998 }
 999
1000 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
1001         const char *word, *state;
1002         size_t l;
1003         unsigned long long seqnum, monotonic, realtime, xor_hash;
1004         bool
1005                 seqnum_id_set = false,
1006                 seqnum_set = false,
1007                 boot_id_set = false,
1008                 monotonic_set = false,
1009                 realtime_set = false,
1010                 xor_hash_set = false;
1011         sd_id128_t seqnum_id, boot_id;
1012
1013         assert_return(j, -EINVAL);
1014         assert_return(!journal_pid_changed(j), -ECHILD);
1015         assert_return(!isempty(cursor), -EINVAL);
1016
1017         FOREACH_WORD_SEPARATOR(word, l, cursor, ";", state) {
1018                 char *item;
1019                 int k = 0;
1020
1021                 if (l < 2 || word[1] != '=')
1022                         return -EINVAL;
1023
1024                 item = strndup(word, l);
1025                 if (!item)
1026                         return -ENOMEM;
1027
1028                 switch (word[0]) {
1029
1030                 case 's':
1031                         seqnum_id_set = true;
1032                         k = sd_id128_from_string(item+2, &seqnum_id);
1033                         break;
1034
1035                 case 'i':
1036                         seqnum_set = true;
1037                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1038                                 k = -EINVAL;
1039                         break;
1040
1041                 case 'b':
1042                         boot_id_set = true;
1043                         k = sd_id128_from_string(item+2, &boot_id);
1044                         break;
1045
1046                 case 'm':
1047                         monotonic_set = true;
1048                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1049                                 k = -EINVAL;
1050                         break;
1051
1052                 case 't':
1053                         realtime_set = true;
1054                         if (sscanf(item+2, "%llx", &realtime) != 1)
1055                                 k = -EINVAL;
1056                         break;
1057
1058                 case 'x':
1059                         xor_hash_set = true;
1060                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1061                                 k = -EINVAL;
1062                         break;
1063                 }
1064
1065                 free(item);
1066
1067                 if (k < 0)
1068                         return k;
1069         }
1070
1071         if ((!seqnum_set || !seqnum_id_set) &&
1072             (!monotonic_set || !boot_id_set) &&
1073             !realtime_set)
1074                 return -EINVAL;
1075
1076         reset_location(j);
1077
1078         j->current_location.type = LOCATION_SEEK;
1079
1080         if (realtime_set) {
1081                 j->current_location.realtime = (uint64_t) realtime;
1082                 j->current_location.realtime_set = true;
1083         }
1084
1085         if (seqnum_set && seqnum_id_set) {
1086                 j->current_location.seqnum = (uint64_t) seqnum;
1087                 j->current_location.seqnum_id = seqnum_id;
1088                 j->current_location.seqnum_set = true;
1089         }
1090
1091         if (monotonic_set && boot_id_set) {
1092                 j->current_location.monotonic = (uint64_t) monotonic;
1093                 j->current_location.boot_id = boot_id;
1094                 j->current_location.monotonic_set = true;
1095         }
1096
1097         if (xor_hash_set) {
1098                 j->current_location.xor_hash = (uint64_t) xor_hash;
1099                 j->current_location.xor_hash_set = true;
1100         }
1101
1102         return 0;
1103 }
1104
1105 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1106         int r;
1107         const char *word, *state;
1108         size_t l;
1109         Object *o;
1110
1111         assert_return(j, -EINVAL);
1112         assert_return(!journal_pid_changed(j), -ECHILD);
1113         assert_return(!isempty(cursor), -EINVAL);
1114
1115         if (!j->current_file || j->current_file->current_offset <= 0)
1116                 return -EADDRNOTAVAIL;
1117
1118         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1119         if (r < 0)
1120                 return r;
1121
1122         FOREACH_WORD_SEPARATOR(word, l, cursor, ";", state) {
1123                 _cleanup_free_ char *item = NULL;
1124                 sd_id128_t id;
1125                 unsigned long long ll;
1126                 int k = 0;
1127
1128                 if (l < 2 || word[1] != '=')
1129                         return -EINVAL;
1130
1131                 item = strndup(word, l);
1132                 if (!item)
1133                         return -ENOMEM;
1134
1135                 switch (word[0]) {
1136
1137                 case 's':
1138                         k = sd_id128_from_string(item+2, &id);
1139                         if (k < 0)
1140                                 return k;
1141                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1142                                 return 0;
1143                         break;
1144
1145                 case 'i':
1146                         if (sscanf(item+2, "%llx", &ll) != 1)
1147                                 return -EINVAL;
1148                         if (ll != le64toh(o->entry.seqnum))
1149                                 return 0;
1150                         break;
1151
1152                 case 'b':
1153                         k = sd_id128_from_string(item+2, &id);
1154                         if (k < 0)
1155                                 return k;
1156                         if (!sd_id128_equal(id, o->entry.boot_id))
1157                                 return 0;
1158                         break;
1159
1160                 case 'm':
1161                         if (sscanf(item+2, "%llx", &ll) != 1)
1162                                 return -EINVAL;
1163                         if (ll != le64toh(o->entry.monotonic))
1164                                 return 0;
1165                         break;
1166
1167                 case 't':
1168                         if (sscanf(item+2, "%llx", &ll) != 1)
1169                                 return -EINVAL;
1170                         if (ll != le64toh(o->entry.realtime))
1171                                 return 0;
1172                         break;
1173
1174                 case 'x':
1175                         if (sscanf(item+2, "%llx", &ll) != 1)
1176                                 return -EINVAL;
1177                         if (ll != le64toh(o->entry.xor_hash))
1178                                 return 0;
1179                         break;
1180                 }
1181         }
1182
1183         return 1;
1184 }
1185
1186
1187 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1188         assert_return(j, -EINVAL);
1189         assert_return(!journal_pid_changed(j), -ECHILD);
1190
1191         reset_location(j);
1192         j->current_location.type = LOCATION_SEEK;
1193         j->current_location.boot_id = boot_id;
1194         j->current_location.monotonic = usec;
1195         j->current_location.monotonic_set = true;
1196
1197         return 0;
1198 }
1199
1200 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1201         assert_return(j, -EINVAL);
1202         assert_return(!journal_pid_changed(j), -ECHILD);
1203
1204         reset_location(j);
1205         j->current_location.type = LOCATION_SEEK;
1206         j->current_location.realtime = usec;
1207         j->current_location.realtime_set = true;
1208
1209         return 0;
1210 }
1211
1212 _public_ int sd_journal_seek_head(sd_journal *j) {
1213         assert_return(j, -EINVAL);
1214         assert_return(!journal_pid_changed(j), -ECHILD);
1215
1216         reset_location(j);
1217         j->current_location.type = LOCATION_HEAD;
1218
1219         return 0;
1220 }
1221
1222 _public_ int sd_journal_seek_tail(sd_journal *j) {
1223         assert_return(j, -EINVAL);
1224         assert_return(!journal_pid_changed(j), -ECHILD);
1225
1226         reset_location(j);
1227         j->current_location.type = LOCATION_TAIL;
1228
1229         return 0;
1230 }
1231
1232 static void check_network(sd_journal *j, int fd) {
1233         struct statfs sfs;
1234
1235         assert(j);
1236
1237         if (j->on_network)
1238                 return;
1239
1240         if (fstatfs(fd, &sfs) < 0)
1241                 return;
1242
1243         j->on_network =
1244                 F_TYPE_EQUAL(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1245                 F_TYPE_EQUAL(sfs.f_type, CODA_SUPER_MAGIC) ||
1246                 F_TYPE_EQUAL(sfs.f_type, NCP_SUPER_MAGIC) ||
1247                 F_TYPE_EQUAL(sfs.f_type, NFS_SUPER_MAGIC) ||
1248                 F_TYPE_EQUAL(sfs.f_type, SMB_SUPER_MAGIC);
1249 }
1250
1251 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1252         const char *full, *tilded, *atted;
1253
1254         full = strappenda(prefix, ".journal");
1255         tilded = strappenda(full, "~");
1256         atted = strappenda(prefix, "@");
1257
1258         return streq(filename, full) ||
1259                streq(filename, tilded) ||
1260                startswith(filename, atted);
1261 }
1262
1263 static bool file_type_wanted(int flags, const char *filename) {
1264         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1265                 return false;
1266
1267         /* no flags set → every type is OK */
1268         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1269                 return true;
1270
1271         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1272                 return true;
1273
1274         if (flags & SD_JOURNAL_CURRENT_USER) {
1275                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1276
1277                 assert_se(snprintf(prefix, sizeof(prefix), "user-"UID_FMT, getuid())
1278                           < (int) sizeof(prefix));
1279
1280                 if (file_has_type_prefix(prefix, filename))
1281                         return true;
1282         }
1283
1284         return false;
1285 }
1286
1287 static int add_any_file(sd_journal *j, const char *path) {
1288         JournalFile *f = NULL;
1289         int r;
1290
1291         assert(j);
1292         assert(path);
1293
1294         if (ordered_hashmap_get(j->files, path))
1295                 return 0;
1296
1297         if (ordered_hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1298                 log_warning("Too many open journal files, not adding %s.", path);
1299                 return set_put_error(j, -ETOOMANYREFS);
1300         }
1301
1302         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1303         if (r < 0)
1304                 return r;
1305
1306         /* journal_file_dump(f); */
1307
1308         r = ordered_hashmap_put(j->files, f->path, f);
1309         if (r < 0) {
1310                 journal_file_close(f);
1311                 return r;
1312         }
1313
1314         log_debug("File %s added.", f->path);
1315
1316         check_network(j, f->fd);
1317
1318         j->current_invalidate_counter ++;
1319
1320         return 0;
1321 }
1322
1323 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1324         _cleanup_free_ char *path = NULL;
1325         int r;
1326
1327         assert(j);
1328         assert(prefix);
1329         assert(filename);
1330
1331         if (j->no_new_files ||
1332             !file_type_wanted(j->flags, filename))
1333                 return 0;
1334
1335         path = strjoin(prefix, "/", filename, NULL);
1336         if (!path)
1337                 return -ENOMEM;
1338
1339         r = add_any_file(j, path);
1340         if (r == -ENOENT)
1341                 return 0;
1342         return 0;
1343 }
1344
1345 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1346         _cleanup_free_ char *path;
1347         JournalFile *f;
1348
1349         assert(j);
1350         assert(prefix);
1351         assert(filename);
1352
1353         path = strjoin(prefix, "/", filename, NULL);
1354         if (!path)
1355                 return -ENOMEM;
1356
1357         f = ordered_hashmap_get(j->files, path);
1358         if (!f)
1359                 return 0;
1360
1361         remove_file_real(j, f);
1362         return 0;
1363 }
1364
1365 static void remove_file_real(sd_journal *j, JournalFile *f) {
1366         assert(j);
1367         assert(f);
1368
1369         ordered_hashmap_remove(j->files, f->path);
1370
1371         log_debug("File %s removed.", f->path);
1372
1373         if (j->current_file == f) {
1374                 j->current_file = NULL;
1375                 j->current_field = 0;
1376         }
1377
1378         if (j->unique_file == f) {
1379                 /* Jump to the next unique_file or NULL if that one was last */
1380                 j->unique_file = ordered_hashmap_next(j->files, j->unique_file->path);
1381                 j->unique_offset = 0;
1382                 if (!j->unique_file)
1383                         j->unique_file_lost = true;
1384         }
1385
1386         journal_file_close(f);
1387
1388         j->current_invalidate_counter ++;
1389 }
1390
1391 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1392         _cleanup_free_ char *path = NULL;
1393         int r;
1394         _cleanup_closedir_ DIR *d = NULL;
1395         sd_id128_t id, mid;
1396         Directory *m;
1397
1398         assert(j);
1399         assert(prefix);
1400         assert(dirname);
1401
1402         log_debug("Considering %s/%s.", prefix, dirname);
1403
1404         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1405             (sd_id128_from_string(dirname, &id) < 0 ||
1406              sd_id128_get_machine(&mid) < 0 ||
1407              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1408             return 0;
1409
1410         path = strjoin(prefix, "/", dirname, NULL);
1411         if (!path)
1412                 return -ENOMEM;
1413
1414         d = opendir(path);
1415         if (!d) {
1416                 log_debug_errno(errno, "Failed to open %s: %m", path);
1417                 if (errno == ENOENT)
1418                         return 0;
1419                 return -errno;
1420         }
1421
1422         m = hashmap_get(j->directories_by_path, path);
1423         if (!m) {
1424                 m = new0(Directory, 1);
1425                 if (!m)
1426                         return -ENOMEM;
1427
1428                 m->is_root = false;
1429                 m->path = path;
1430
1431                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1432                         free(m);
1433                         return -ENOMEM;
1434                 }
1435
1436                 path = NULL; /* avoid freeing in cleanup */
1437                 j->current_invalidate_counter ++;
1438
1439                 log_debug("Directory %s added.", m->path);
1440
1441         } else if (m->is_root)
1442                 return 0;
1443
1444         if (m->wd <= 0 && j->inotify_fd >= 0) {
1445
1446                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1447                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1448                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1449                                           IN_ONLYDIR);
1450
1451                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1452                         inotify_rm_watch(j->inotify_fd, m->wd);
1453         }
1454
1455         for (;;) {
1456                 struct dirent *de;
1457
1458                 errno = 0;
1459                 de = readdir(d);
1460                 if (!de && errno != 0) {
1461                         r = -errno;
1462                         log_debug_errno(errno, "Failed to read directory %s: %m", m->path);
1463                         return r;
1464                 }
1465                 if (!de)
1466                         break;
1467
1468                 if (dirent_is_file_with_suffix(de, ".journal") ||
1469                     dirent_is_file_with_suffix(de, ".journal~")) {
1470                         r = add_file(j, m->path, de->d_name);
1471                         if (r < 0) {
1472                                 log_debug_errno(r, "Failed to add file %s/%s: %m",
1473                                                 m->path, de->d_name);
1474                                 r = set_put_error(j, r);
1475                                 if (r < 0)
1476                                         return r;
1477                         }
1478                 }
1479         }
1480
1481         check_network(j, dirfd(d));
1482
1483         return 0;
1484 }
1485
1486 static int add_root_directory(sd_journal *j, const char *p) {
1487         _cleanup_closedir_ DIR *d = NULL;
1488         Directory *m;
1489         int r;
1490
1491         assert(j);
1492         assert(p);
1493
1494         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1495             !path_startswith(p, "/run"))
1496                 return -EINVAL;
1497
1498         if (j->prefix)
1499                 p = strappenda(j->prefix, p);
1500
1501         d = opendir(p);
1502         if (!d)
1503                 return -errno;
1504
1505         m = hashmap_get(j->directories_by_path, p);
1506         if (!m) {
1507                 m = new0(Directory, 1);
1508                 if (!m)
1509                         return -ENOMEM;
1510
1511                 m->is_root = true;
1512                 m->path = strdup(p);
1513                 if (!m->path) {
1514                         free(m);
1515                         return -ENOMEM;
1516                 }
1517
1518                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1519                         free(m->path);
1520                         free(m);
1521                         return -ENOMEM;
1522                 }
1523
1524                 j->current_invalidate_counter ++;
1525
1526                 log_debug("Root directory %s added.", m->path);
1527
1528         } else if (!m->is_root)
1529                 return 0;
1530
1531         if (m->wd <= 0 && j->inotify_fd >= 0) {
1532
1533                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1534                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1535                                           IN_ONLYDIR);
1536
1537                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1538                         inotify_rm_watch(j->inotify_fd, m->wd);
1539         }
1540
1541         if (j->no_new_files)
1542                 return 0;
1543
1544         for (;;) {
1545                 struct dirent *de;
1546                 sd_id128_t id;
1547
1548                 errno = 0;
1549                 de = readdir(d);
1550                 if (!de && errno != 0) {
1551                         r = -errno;
1552                         log_debug_errno(errno, "Failed to read directory %s: %m", m->path);
1553                         return r;
1554                 }
1555                 if (!de)
1556                         break;
1557
1558                 if (dirent_is_file_with_suffix(de, ".journal") ||
1559                     dirent_is_file_with_suffix(de, ".journal~")) {
1560                         r = add_file(j, m->path, de->d_name);
1561                         if (r < 0) {
1562                                 log_debug_errno(r, "Failed to add file %s/%s: %m",
1563                                                 m->path, de->d_name);
1564                                 r = set_put_error(j, r);
1565                                 if (r < 0)
1566                                         return r;
1567                         }
1568                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1569                            sd_id128_from_string(de->d_name, &id) >= 0) {
1570
1571                         r = add_directory(j, m->path, de->d_name);
1572                         if (r < 0)
1573                                 log_debug_errno(r, "Failed to add directory %s/%s: %m", m->path, de->d_name);
1574                 }
1575         }
1576
1577         check_network(j, dirfd(d));
1578
1579         return 0;
1580 }
1581
1582 static int remove_directory(sd_journal *j, Directory *d) {
1583         assert(j);
1584
1585         if (d->wd > 0) {
1586                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1587
1588                 if (j->inotify_fd >= 0)
1589                         inotify_rm_watch(j->inotify_fd, d->wd);
1590         }
1591
1592         hashmap_remove(j->directories_by_path, d->path);
1593
1594         if (d->is_root)
1595                 log_debug("Root directory %s removed.", d->path);
1596         else
1597                 log_debug("Directory %s removed.", d->path);
1598
1599         free(d->path);
1600         free(d);
1601
1602         return 0;
1603 }
1604
1605 static int add_search_paths(sd_journal *j) {
1606         int r;
1607         const char search_paths[] =
1608                 "/run/log/journal\0"
1609                 "/var/log/journal\0";
1610         const char *p;
1611
1612         assert(j);
1613
1614         /* We ignore most errors here, since the idea is to only open
1615          * what's actually accessible, and ignore the rest. */
1616
1617         NULSTR_FOREACH(p, search_paths) {
1618                 r = add_root_directory(j, p);
1619                 if (r < 0 && r != -ENOENT) {
1620                         r = set_put_error(j, r);
1621                         if (r < 0)
1622                                 return r;
1623                 }
1624         }
1625
1626         return 0;
1627 }
1628
1629 static int add_current_paths(sd_journal *j) {
1630         Iterator i;
1631         JournalFile *f;
1632
1633         assert(j);
1634         assert(j->no_new_files);
1635
1636         /* Simply adds all directories for files we have open as
1637          * "root" directories. We don't expect errors here, so we
1638          * treat them as fatal. */
1639
1640         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
1641                 _cleanup_free_ char *dir;
1642                 int r;
1643
1644                 dir = dirname_malloc(f->path);
1645                 if (!dir)
1646                         return -ENOMEM;
1647
1648                 r = add_root_directory(j, dir);
1649                 if (r < 0) {
1650                         set_put_error(j, r);
1651                         return r;
1652                 }
1653         }
1654
1655         return 0;
1656 }
1657
1658
1659 static int allocate_inotify(sd_journal *j) {
1660         assert(j);
1661
1662         if (j->inotify_fd < 0) {
1663                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1664                 if (j->inotify_fd < 0)
1665                         return -errno;
1666         }
1667
1668         if (!j->directories_by_wd) {
1669                 j->directories_by_wd = hashmap_new(NULL);
1670                 if (!j->directories_by_wd)
1671                         return -ENOMEM;
1672         }
1673
1674         return 0;
1675 }
1676
1677 static sd_journal *journal_new(int flags, const char *path) {
1678         sd_journal *j;
1679
1680         j = new0(sd_journal, 1);
1681         if (!j)
1682                 return NULL;
1683
1684         j->original_pid = getpid();
1685         j->inotify_fd = -1;
1686         j->flags = flags;
1687         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1688
1689         if (path) {
1690                 j->path = strdup(path);
1691                 if (!j->path)
1692                         goto fail;
1693         }
1694
1695         j->files = ordered_hashmap_new(&string_hash_ops);
1696         j->directories_by_path = hashmap_new(&string_hash_ops);
1697         j->mmap = mmap_cache_new();
1698         if (!j->files || !j->directories_by_path || !j->mmap)
1699                 goto fail;
1700
1701         return j;
1702
1703 fail:
1704         sd_journal_close(j);
1705         return NULL;
1706 }
1707
1708 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1709         sd_journal *j;
1710         int r;
1711
1712         assert_return(ret, -EINVAL);
1713         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_RUNTIME_ONLY|SD_JOURNAL_SYSTEM|SD_JOURNAL_CURRENT_USER)) == 0, -EINVAL);
1714
1715         j = journal_new(flags, NULL);
1716         if (!j)
1717                 return -ENOMEM;
1718
1719         r = add_search_paths(j);
1720         if (r < 0)
1721                 goto fail;
1722
1723         *ret = j;
1724         return 0;
1725
1726 fail:
1727         sd_journal_close(j);
1728
1729         return r;
1730 }
1731
1732 _public_ int sd_journal_open_container(sd_journal **ret, const char *machine, int flags) {
1733         _cleanup_free_ char *root = NULL, *class = NULL;
1734         sd_journal *j;
1735         char *p;
1736         int r;
1737
1738         assert_return(machine, -EINVAL);
1739         assert_return(ret, -EINVAL);
1740         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_SYSTEM)) == 0, -EINVAL);
1741         assert_return(machine_name_is_valid(machine), -EINVAL);
1742
1743         p = strappenda("/run/systemd/machines/", machine);
1744         r = parse_env_file(p, NEWLINE, "ROOT", &root, "CLASS", &class, NULL);
1745         if (r == -ENOENT)
1746                 return -EHOSTDOWN;
1747         if (r < 0)
1748                 return r;
1749         if (!root)
1750                 return -ENODATA;
1751
1752         if (!streq_ptr(class, "container"))
1753                 return -EIO;
1754
1755         j = journal_new(flags, NULL);
1756         if (!j)
1757                 return -ENOMEM;
1758
1759         j->prefix = root;
1760         root = NULL;
1761
1762         r = add_search_paths(j);
1763         if (r < 0)
1764                 goto fail;
1765
1766         *ret = j;
1767         return 0;
1768
1769 fail:
1770         sd_journal_close(j);
1771         return r;
1772 }
1773
1774 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1775         sd_journal *j;
1776         int r;
1777
1778         assert_return(ret, -EINVAL);
1779         assert_return(path, -EINVAL);
1780         assert_return(flags == 0, -EINVAL);
1781
1782         j = journal_new(flags, path);
1783         if (!j)
1784                 return -ENOMEM;
1785
1786         r = add_root_directory(j, path);
1787         if (r < 0) {
1788                 set_put_error(j, r);
1789                 goto fail;
1790         }
1791
1792         *ret = j;
1793         return 0;
1794
1795 fail:
1796         sd_journal_close(j);
1797
1798         return r;
1799 }
1800
1801 _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int flags) {
1802         sd_journal *j;
1803         const char **path;
1804         int r;
1805
1806         assert_return(ret, -EINVAL);
1807         assert_return(flags == 0, -EINVAL);
1808
1809         j = journal_new(flags, NULL);
1810         if (!j)
1811                 return -ENOMEM;
1812
1813         STRV_FOREACH(path, paths) {
1814                 r = add_any_file(j, *path);
1815                 if (r < 0) {
1816                         log_error_errno(r, "Failed to open %s: %m", *path);
1817                         goto fail;
1818                 }
1819         }
1820
1821         j->no_new_files = true;
1822
1823         *ret = j;
1824         return 0;
1825
1826 fail:
1827         sd_journal_close(j);
1828
1829         return r;
1830 }
1831
1832 _public_ void sd_journal_close(sd_journal *j) {
1833         Directory *d;
1834         JournalFile *f;
1835
1836         if (!j)
1837                 return;
1838
1839         sd_journal_flush_matches(j);
1840
1841         while ((f = ordered_hashmap_steal_first(j->files)))
1842                 journal_file_close(f);
1843
1844         ordered_hashmap_free(j->files);
1845
1846         while ((d = hashmap_first(j->directories_by_path)))
1847                 remove_directory(j, d);
1848
1849         while ((d = hashmap_first(j->directories_by_wd)))
1850                 remove_directory(j, d);
1851
1852         hashmap_free(j->directories_by_path);
1853         hashmap_free(j->directories_by_wd);
1854
1855         safe_close(j->inotify_fd);
1856
1857         if (j->mmap) {
1858                 log_debug("mmap cache statistics: %u hit, %u miss", mmap_cache_get_hit(j->mmap), mmap_cache_get_missed(j->mmap));
1859                 mmap_cache_unref(j->mmap);
1860         }
1861
1862         free(j->path);
1863         free(j->prefix);
1864         free(j->unique_field);
1865         set_free(j->errors);
1866         free(j);
1867 }
1868
1869 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1870         Object *o;
1871         JournalFile *f;
1872         int r;
1873
1874         assert_return(j, -EINVAL);
1875         assert_return(!journal_pid_changed(j), -ECHILD);
1876         assert_return(ret, -EINVAL);
1877
1878         f = j->current_file;
1879         if (!f)
1880                 return -EADDRNOTAVAIL;
1881
1882         if (f->current_offset <= 0)
1883                 return -EADDRNOTAVAIL;
1884
1885         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1886         if (r < 0)
1887                 return r;
1888
1889         *ret = le64toh(o->entry.realtime);
1890         return 0;
1891 }
1892
1893 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1894         Object *o;
1895         JournalFile *f;
1896         int r;
1897         sd_id128_t id;
1898
1899         assert_return(j, -EINVAL);
1900         assert_return(!journal_pid_changed(j), -ECHILD);
1901
1902         f = j->current_file;
1903         if (!f)
1904                 return -EADDRNOTAVAIL;
1905
1906         if (f->current_offset <= 0)
1907                 return -EADDRNOTAVAIL;
1908
1909         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1910         if (r < 0)
1911                 return r;
1912
1913         if (ret_boot_id)
1914                 *ret_boot_id = o->entry.boot_id;
1915         else {
1916                 r = sd_id128_get_boot(&id);
1917                 if (r < 0)
1918                         return r;
1919
1920                 if (!sd_id128_equal(id, o->entry.boot_id))
1921                         return -ESTALE;
1922         }
1923
1924         if (ret)
1925                 *ret = le64toh(o->entry.monotonic);
1926
1927         return 0;
1928 }
1929
1930 static bool field_is_valid(const char *field) {
1931         const char *p;
1932
1933         assert(field);
1934
1935         if (isempty(field))
1936                 return false;
1937
1938         if (startswith(field, "__"))
1939                 return false;
1940
1941         for (p = field; *p; p++) {
1942
1943                 if (*p == '_')
1944                         continue;
1945
1946                 if (*p >= 'A' && *p <= 'Z')
1947                         continue;
1948
1949                 if (*p >= '0' && *p <= '9')
1950                         continue;
1951
1952                 return false;
1953         }
1954
1955         return true;
1956 }
1957
1958 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1959         JournalFile *f;
1960         uint64_t i, n;
1961         size_t field_length;
1962         int r;
1963         Object *o;
1964
1965         assert_return(j, -EINVAL);
1966         assert_return(!journal_pid_changed(j), -ECHILD);
1967         assert_return(field, -EINVAL);
1968         assert_return(data, -EINVAL);
1969         assert_return(size, -EINVAL);
1970         assert_return(field_is_valid(field), -EINVAL);
1971
1972         f = j->current_file;
1973         if (!f)
1974                 return -EADDRNOTAVAIL;
1975
1976         if (f->current_offset <= 0)
1977                 return -EADDRNOTAVAIL;
1978
1979         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1980         if (r < 0)
1981                 return r;
1982
1983         field_length = strlen(field);
1984
1985         n = journal_file_entry_n_items(o);
1986         for (i = 0; i < n; i++) {
1987                 uint64_t p, l;
1988                 le64_t le_hash;
1989                 size_t t;
1990                 int compression;
1991
1992                 p = le64toh(o->entry.items[i].object_offset);
1993                 le_hash = o->entry.items[i].hash;
1994                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1995                 if (r < 0)
1996                         return r;
1997
1998                 if (le_hash != o->data.hash)
1999                         return -EBADMSG;
2000
2001                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
2002
2003                 compression = o->object.flags & OBJECT_COMPRESSION_MASK;
2004                 if (compression) {
2005 #if defined(HAVE_XZ) || defined(HAVE_LZ4)
2006                         if (decompress_startswith(compression,
2007                                                   o->data.payload, l,
2008                                                   &f->compress_buffer, &f->compress_buffer_size,
2009                                                   field, field_length, '=')) {
2010
2011                                 size_t rsize;
2012
2013                                 r = decompress_blob(compression,
2014                                                     o->data.payload, l,
2015                                                     &f->compress_buffer, &f->compress_buffer_size, &rsize,
2016                                                     j->data_threshold);
2017                                 if (r < 0)
2018                                         return r;
2019
2020                                 *data = f->compress_buffer;
2021                                 *size = (size_t) rsize;
2022
2023                                 return 0;
2024                         }
2025 #else
2026                         return -EPROTONOSUPPORT;
2027 #endif
2028                 } else if (l >= field_length+1 &&
2029                            memcmp(o->data.payload, field, field_length) == 0 &&
2030                            o->data.payload[field_length] == '=') {
2031
2032                         t = (size_t) l;
2033
2034                         if ((uint64_t) t != l)
2035                                 return -E2BIG;
2036
2037                         *data = o->data.payload;
2038                         *size = t;
2039
2040                         return 0;
2041                 }
2042
2043                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2044                 if (r < 0)
2045                         return r;
2046         }
2047
2048         return -ENOENT;
2049 }
2050
2051 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
2052         size_t t;
2053         uint64_t l;
2054         int compression;
2055
2056         l = le64toh(o->object.size) - offsetof(Object, data.payload);
2057         t = (size_t) l;
2058
2059         /* We can't read objects larger than 4G on a 32bit machine */
2060         if ((uint64_t) t != l)
2061                 return -E2BIG;
2062
2063         compression = o->object.flags & OBJECT_COMPRESSION_MASK;
2064         if (compression) {
2065 #if defined(HAVE_XZ) || defined(HAVE_LZ4)
2066                 size_t rsize;
2067                 int r;
2068
2069                 r = decompress_blob(compression,
2070                                     o->data.payload, l, &f->compress_buffer,
2071                                     &f->compress_buffer_size, &rsize, j->data_threshold);
2072                 if (r < 0)
2073                         return r;
2074
2075                 *data = f->compress_buffer;
2076                 *size = (size_t) rsize;
2077 #else
2078                 return -EPROTONOSUPPORT;
2079 #endif
2080         } else {
2081                 *data = o->data.payload;
2082                 *size = t;
2083         }
2084
2085         return 0;
2086 }
2087
2088 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
2089         JournalFile *f;
2090         uint64_t p, n;
2091         le64_t le_hash;
2092         int r;
2093         Object *o;
2094
2095         assert_return(j, -EINVAL);
2096         assert_return(!journal_pid_changed(j), -ECHILD);
2097         assert_return(data, -EINVAL);
2098         assert_return(size, -EINVAL);
2099
2100         f = j->current_file;
2101         if (!f)
2102                 return -EADDRNOTAVAIL;
2103
2104         if (f->current_offset <= 0)
2105                 return -EADDRNOTAVAIL;
2106
2107         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2108         if (r < 0)
2109                 return r;
2110
2111         n = journal_file_entry_n_items(o);
2112         if (j->current_field >= n)
2113                 return 0;
2114
2115         p = le64toh(o->entry.items[j->current_field].object_offset);
2116         le_hash = o->entry.items[j->current_field].hash;
2117         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2118         if (r < 0)
2119                 return r;
2120
2121         if (le_hash != o->data.hash)
2122                 return -EBADMSG;
2123
2124         r = return_data(j, f, o, data, size);
2125         if (r < 0)
2126                 return r;
2127
2128         j->current_field ++;
2129
2130         return 1;
2131 }
2132
2133 _public_ void sd_journal_restart_data(sd_journal *j) {
2134         if (!j)
2135                 return;
2136
2137         j->current_field = 0;
2138 }
2139
2140 _public_ int sd_journal_get_fd(sd_journal *j) {
2141         int r;
2142
2143         assert_return(j, -EINVAL);
2144         assert_return(!journal_pid_changed(j), -ECHILD);
2145
2146         if (j->inotify_fd >= 0)
2147                 return j->inotify_fd;
2148
2149         r = allocate_inotify(j);
2150         if (r < 0)
2151                 return r;
2152
2153         /* Iterate through all dirs again, to add them to the
2154          * inotify */
2155         if (j->no_new_files)
2156                 r = add_current_paths(j);
2157         else if (j->path)
2158                 r = add_root_directory(j, j->path);
2159         else
2160                 r = add_search_paths(j);
2161         if (r < 0)
2162                 return r;
2163
2164         return j->inotify_fd;
2165 }
2166
2167 _public_ int sd_journal_get_events(sd_journal *j) {
2168         int fd;
2169
2170         assert_return(j, -EINVAL);
2171         assert_return(!journal_pid_changed(j), -ECHILD);
2172
2173         fd = sd_journal_get_fd(j);
2174         if (fd < 0)
2175                 return fd;
2176
2177         return POLLIN;
2178 }
2179
2180 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2181         int fd;
2182
2183         assert_return(j, -EINVAL);
2184         assert_return(!journal_pid_changed(j), -ECHILD);
2185         assert_return(timeout_usec, -EINVAL);
2186
2187         fd = sd_journal_get_fd(j);
2188         if (fd < 0)
2189                 return fd;
2190
2191         if (!j->on_network) {
2192                 *timeout_usec = (uint64_t) -1;
2193                 return 0;
2194         }
2195
2196         /* If we are on the network we need to regularly check for
2197          * changes manually */
2198
2199         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2200         return 1;
2201 }
2202
2203 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2204         Directory *d;
2205         int r;
2206
2207         assert(j);
2208         assert(e);
2209
2210         /* Is this a subdirectory we watch? */
2211         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2212         if (d) {
2213                 sd_id128_t id;
2214
2215                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2216                     (endswith(e->name, ".journal") ||
2217                      endswith(e->name, ".journal~"))) {
2218
2219                         /* Event for a journal file */
2220
2221                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2222                                 r = add_file(j, d->path, e->name);
2223                                 if (r < 0) {
2224                                         log_debug_errno(r, "Failed to add file %s/%s: %m",
2225                                                         d->path, e->name);
2226                                         set_put_error(j, r);
2227                                 }
2228
2229                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2230
2231                                 r = remove_file(j, d->path, e->name);
2232                                 if (r < 0)
2233                                         log_debug_errno(r, "Failed to remove file %s/%s: %m", d->path, e->name);
2234                         }
2235
2236                 } else if (!d->is_root && e->len == 0) {
2237
2238                         /* Event for a subdirectory */
2239
2240                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2241                                 r = remove_directory(j, d);
2242                                 if (r < 0)
2243                                         log_debug_errno(r, "Failed to remove directory %s: %m", d->path);
2244                         }
2245
2246
2247                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2248
2249                         /* Event for root directory */
2250
2251                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2252                                 r = add_directory(j, d->path, e->name);
2253                                 if (r < 0)
2254                                         log_debug_errno(r, "Failed to add directory %s/%s: %m", d->path, e->name);
2255                         }
2256                 }
2257
2258                 return;
2259         }
2260
2261         if (e->mask & IN_IGNORED)
2262                 return;
2263
2264         log_warning("Unknown inotify event.");
2265 }
2266
2267 static int determine_change(sd_journal *j) {
2268         bool b;
2269
2270         assert(j);
2271
2272         b = j->current_invalidate_counter != j->last_invalidate_counter;
2273         j->last_invalidate_counter = j->current_invalidate_counter;
2274
2275         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2276 }
2277
2278 _public_ int sd_journal_process(sd_journal *j) {
2279         bool got_something = false;
2280
2281         assert_return(j, -EINVAL);
2282         assert_return(!journal_pid_changed(j), -ECHILD);
2283
2284         j->last_process_usec = now(CLOCK_MONOTONIC);
2285
2286         for (;;) {
2287                 uint8_t buffer[INOTIFY_EVENT_MAX] _alignas_(struct inotify_event);
2288                 struct inotify_event *e;
2289                 ssize_t l;
2290
2291                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2292                 if (l < 0) {
2293                         if (errno == EAGAIN || errno == EINTR)
2294                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2295
2296                         return -errno;
2297                 }
2298
2299                 got_something = true;
2300
2301                 FOREACH_INOTIFY_EVENT(e, buffer, l)
2302                         process_inotify_event(j, e);
2303         }
2304 }
2305
2306 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2307         int r;
2308         uint64_t t;
2309
2310         assert_return(j, -EINVAL);
2311         assert_return(!journal_pid_changed(j), -ECHILD);
2312
2313         if (j->inotify_fd < 0) {
2314
2315                 /* This is the first invocation, hence create the
2316                  * inotify watch */
2317                 r = sd_journal_get_fd(j);
2318                 if (r < 0)
2319                         return r;
2320
2321                 /* The journal might have changed since the context
2322                  * object was created and we weren't watching before,
2323                  * hence don't wait for anything, and return
2324                  * immediately. */
2325                 return determine_change(j);
2326         }
2327
2328         r = sd_journal_get_timeout(j, &t);
2329         if (r < 0)
2330                 return r;
2331
2332         if (t != (uint64_t) -1) {
2333                 usec_t n;
2334
2335                 n = now(CLOCK_MONOTONIC);
2336                 t = t > n ? t - n : 0;
2337
2338                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2339                         timeout_usec = t;
2340         }
2341
2342         do {
2343                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2344         } while (r == -EINTR);
2345
2346         if (r < 0)
2347                 return r;
2348
2349         return sd_journal_process(j);
2350 }
2351
2352 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2353         Iterator i;
2354         JournalFile *f;
2355         bool first = true;
2356         uint64_t fmin = 0, tmax = 0;
2357         int r;
2358
2359         assert_return(j, -EINVAL);
2360         assert_return(!journal_pid_changed(j), -ECHILD);
2361         assert_return(from || to, -EINVAL);
2362         assert_return(from != to, -EINVAL);
2363
2364         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2365                 usec_t fr, t;
2366
2367                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2368                 if (r == -ENOENT)
2369                         continue;
2370                 if (r < 0)
2371                         return r;
2372                 if (r == 0)
2373                         continue;
2374
2375                 if (first) {
2376                         fmin = fr;
2377                         tmax = t;
2378                         first = false;
2379                 } else {
2380                         fmin = MIN(fr, fmin);
2381                         tmax = MAX(t, tmax);
2382                 }
2383         }
2384
2385         if (from)
2386                 *from = fmin;
2387         if (to)
2388                 *to = tmax;
2389
2390         return first ? 0 : 1;
2391 }
2392
2393 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2394         Iterator i;
2395         JournalFile *f;
2396         bool found = false;
2397         int r;
2398
2399         assert_return(j, -EINVAL);
2400         assert_return(!journal_pid_changed(j), -ECHILD);
2401         assert_return(from || to, -EINVAL);
2402         assert_return(from != to, -EINVAL);
2403
2404         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2405                 usec_t fr, t;
2406
2407                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2408                 if (r == -ENOENT)
2409                         continue;
2410                 if (r < 0)
2411                         return r;
2412                 if (r == 0)
2413                         continue;
2414
2415                 if (found) {
2416                         if (from)
2417                                 *from = MIN(fr, *from);
2418                         if (to)
2419                                 *to = MAX(t, *to);
2420                 } else {
2421                         if (from)
2422                                 *from = fr;
2423                         if (to)
2424                                 *to = t;
2425                         found = true;
2426                 }
2427         }
2428
2429         return found;
2430 }
2431
2432 void journal_print_header(sd_journal *j) {
2433         Iterator i;
2434         JournalFile *f;
2435         bool newline = false;
2436
2437         assert(j);
2438
2439         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2440                 if (newline)
2441                         putchar('\n');
2442                 else
2443                         newline = true;
2444
2445                 journal_file_print_header(f);
2446         }
2447 }
2448
2449 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2450         Iterator i;
2451         JournalFile *f;
2452         uint64_t sum = 0;
2453
2454         assert_return(j, -EINVAL);
2455         assert_return(!journal_pid_changed(j), -ECHILD);
2456         assert_return(bytes, -EINVAL);
2457
2458         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2459                 struct stat st;
2460
2461                 if (fstat(f->fd, &st) < 0)
2462                         return -errno;
2463
2464                 sum += (uint64_t) st.st_blocks * 512ULL;
2465         }
2466
2467         *bytes = sum;
2468         return 0;
2469 }
2470
2471 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2472         char *f;
2473
2474         assert_return(j, -EINVAL);
2475         assert_return(!journal_pid_changed(j), -ECHILD);
2476         assert_return(!isempty(field), -EINVAL);
2477         assert_return(field_is_valid(field), -EINVAL);
2478
2479         f = strdup(field);
2480         if (!f)
2481                 return -ENOMEM;
2482
2483         free(j->unique_field);
2484         j->unique_field = f;
2485         j->unique_file = NULL;
2486         j->unique_offset = 0;
2487         j->unique_file_lost = false;
2488
2489         return 0;
2490 }
2491
2492 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2493         size_t k;
2494
2495         assert_return(j, -EINVAL);
2496         assert_return(!journal_pid_changed(j), -ECHILD);
2497         assert_return(data, -EINVAL);
2498         assert_return(l, -EINVAL);
2499         assert_return(j->unique_field, -EINVAL);
2500
2501         k = strlen(j->unique_field);
2502
2503         if (!j->unique_file) {
2504                 if (j->unique_file_lost)
2505                         return 0;
2506
2507                 j->unique_file = ordered_hashmap_first(j->files);
2508                 if (!j->unique_file)
2509                         return 0;
2510
2511                 j->unique_offset = 0;
2512         }
2513
2514         for (;;) {
2515                 JournalFile *of;
2516                 Iterator i;
2517                 Object *o;
2518                 const void *odata;
2519                 size_t ol;
2520                 bool found;
2521                 int r;
2522
2523                 /* Proceed to next data object in the field's linked list */
2524                 if (j->unique_offset == 0) {
2525                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2526                         if (r < 0)
2527                                 return r;
2528
2529                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2530                 } else {
2531                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2532                         if (r < 0)
2533                                 return r;
2534
2535                         j->unique_offset = le64toh(o->data.next_field_offset);
2536                 }
2537
2538                 /* We reached the end of the list? Then start again, with the next file */
2539                 if (j->unique_offset == 0) {
2540                         j->unique_file = ordered_hashmap_next(j->files, j->unique_file->path);
2541                         if (!j->unique_file)
2542                                 return 0;
2543
2544                         continue;
2545                 }
2546
2547                 /* We do not use OBJECT_DATA context here, but OBJECT_UNUSED
2548                  * instead, so that we can look at this data object at the same
2549                  * time as one on another file */
2550                 r = journal_file_move_to_object(j->unique_file, OBJECT_UNUSED, j->unique_offset, &o);
2551                 if (r < 0)
2552                         return r;
2553
2554                 /* Let's do the type check by hand, since we used 0 context above. */
2555                 if (o->object.type != OBJECT_DATA) {
2556                         log_debug("%s:offset " OFSfmt ": object has type %d, expected %d",
2557                                   j->unique_file->path, j->unique_offset,
2558                                   o->object.type, OBJECT_DATA);
2559                         return -EBADMSG;
2560                 }
2561
2562                 r = return_data(j, j->unique_file, o, &odata, &ol);
2563                 if (r < 0)
2564                         return r;
2565
2566                 /* Check if we have at least the field name and "=". */
2567                 if (ol <= k) {
2568                         log_debug("%s:offset " OFSfmt ": object has size %zu, expected at least %zu",
2569                                   j->unique_file->path, j->unique_offset,
2570                                   ol, k + 1);
2571                         return -EBADMSG;
2572                 }
2573
2574                 if (memcmp(odata, j->unique_field, k) || ((const char*) odata)[k] != '=') {
2575                         log_debug("%s:offset " OFSfmt ": object does not start with \"%s=\"",
2576                                   j->unique_file->path, j->unique_offset,
2577                                   j->unique_field);
2578                         return -EBADMSG;
2579                 }
2580
2581                 /* OK, now let's see if we already returned this data
2582                  * object by checking if it exists in the earlier
2583                  * traversed files. */
2584                 found = false;
2585                 ORDERED_HASHMAP_FOREACH(of, j->files, i) {
2586                         Object *oo;
2587                         uint64_t op;
2588
2589                         if (of == j->unique_file)
2590                                 break;
2591
2592                         /* Skip this file it didn't have any fields
2593                          * indexed */
2594                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2595                             le64toh(of->header->n_fields) <= 0)
2596                                 continue;
2597
2598                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2599                         if (r < 0)
2600                                 return r;
2601
2602                         if (r > 0)
2603                                 found = true;
2604                 }
2605
2606                 if (found)
2607                         continue;
2608
2609                 r = return_data(j, j->unique_file, o, data, l);
2610                 if (r < 0)
2611                         return r;
2612
2613                 return 1;
2614         }
2615 }
2616
2617 _public_ void sd_journal_restart_unique(sd_journal *j) {
2618         if (!j)
2619                 return;
2620
2621         j->unique_file = NULL;
2622         j->unique_offset = 0;
2623         j->unique_file_lost = false;
2624 }
2625
2626 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2627         assert_return(j, -EINVAL);
2628         assert_return(!journal_pid_changed(j), -ECHILD);
2629
2630         return !j->on_network;
2631 }
2632
2633 static char *lookup_field(const char *field, void *userdata) {
2634         sd_journal *j = userdata;
2635         const void *data;
2636         size_t size, d;
2637         int r;
2638
2639         assert(field);
2640         assert(j);
2641
2642         r = sd_journal_get_data(j, field, &data, &size);
2643         if (r < 0 ||
2644             size > REPLACE_VAR_MAX)
2645                 return strdup(field);
2646
2647         d = strlen(field) + 1;
2648
2649         return strndup((const char*) data + d, size - d);
2650 }
2651
2652 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2653         const void *data;
2654         size_t size;
2655         sd_id128_t id;
2656         _cleanup_free_ char *text = NULL, *cid = NULL;
2657         char *t;
2658         int r;
2659
2660         assert_return(j, -EINVAL);
2661         assert_return(!journal_pid_changed(j), -ECHILD);
2662         assert_return(ret, -EINVAL);
2663
2664         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2665         if (r < 0)
2666                 return r;
2667
2668         cid = strndup((const char*) data + 11, size - 11);
2669         if (!cid)
2670                 return -ENOMEM;
2671
2672         r = sd_id128_from_string(cid, &id);
2673         if (r < 0)
2674                 return r;
2675
2676         r = catalog_get(CATALOG_DATABASE, id, &text);
2677         if (r < 0)
2678                 return r;
2679
2680         t = replace_var(text, lookup_field, j);
2681         if (!t)
2682                 return -ENOMEM;
2683
2684         *ret = t;
2685         return 0;
2686 }
2687
2688 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2689         assert_return(ret, -EINVAL);
2690
2691         return catalog_get(CATALOG_DATABASE, id, ret);
2692 }
2693
2694 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2695         assert_return(j, -EINVAL);
2696         assert_return(!journal_pid_changed(j), -ECHILD);
2697
2698         j->data_threshold = sz;
2699         return 0;
2700 }
2701
2702 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2703         assert_return(j, -EINVAL);
2704         assert_return(!journal_pid_changed(j), -ECHILD);
2705         assert_return(sz, -EINVAL);
2706
2707         *sz = j->data_threshold;
2708         return 0;
2709 }