src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "strv.h"
  37 #include "path-util.h"
  38 #include "lookup3.h"
  39 #include "compress.h"
  40 #include "journal-internal.h"
  41 #include "missing.h"
  42 #include "catalog.h"
  43 #include "replace-var.h"
  44 #include "fileio.h"
  45
  46 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  47
  48 #define REPLACE_VAR_MAX 256
  49
  50 #define DEFAULT_DATA_THRESHOLD (64*1024)
  51
  52 static void remove_file_real(sd_journal *j, JournalFile *f);
  53
  54 static bool journal_pid_changed(sd_journal *j) {
  55         assert(j);
  56
  57         /* We don't support people creating a journal object and
  58          * keeping it around over a fork(). Let's complain. */
  59
  60         return j->original_pid != getpid();
  61 }
  62
  63 /* We return an error here only if we didn't manage to
  64    memorize the real error. */
  65 static int set_put_error(sd_journal *j, int r) {
  66         int k;
  67
  68         if (r >= 0)
  69                 return r;
  70
  71         k = set_ensure_allocated(&j->errors, NULL);
  72         if (k < 0)
  73                 return k;
  74
  75         return set_put(j->errors, INT_TO_PTR(r));
  76 }
  77
  78 static void detach_location(sd_journal *j) {
  79         Iterator i;
  80         JournalFile *f;
  81
  82         assert(j);
  83
  84         j->current_file = NULL;
  85         j->current_field = 0;
  86
  87         ORDERED_HASHMAP_FOREACH(f, j->files, i)
  88                 journal_file_reset_location(f);
  89 }
  90
  91 static void reset_location(sd_journal *j) {
  92         assert(j);
  93
  94         detach_location(j);
  95         zero(j->current_location);
  96 }
  97
  98 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  99         assert(l);
 100         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 101         assert(f);
 102         assert(o->object.type == OBJECT_ENTRY);
 103
 104         l->type = type;
 105         l->seqnum = le64toh(o->entry.seqnum);
 106         l->seqnum_id = f->header->seqnum_id;
 107         l->realtime = le64toh(o->entry.realtime);
 108         l->monotonic = le64toh(o->entry.monotonic);
 109         l->boot_id = o->entry.boot_id;
 110         l->xor_hash = le64toh(o->entry.xor_hash);
 111
 112         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 113 }
 114
 115 static void set_location(sd_journal *j, JournalFile *f, Object *o) {
 116         assert(j);
 117         assert(f);
 118         assert(o);
 119
 120         init_location(&j->current_location, LOCATION_DISCRETE, f, o);
 121
 122         j->current_file = f;
 123         j->current_field = 0;
 124
 125         /* Let f know its candidate entry was picked. */
 126         assert(f->location_type == LOCATION_SEEK);
 127         f->location_type = LOCATION_DISCRETE;
 128 }
 129
 130 static int match_is_valid(const void *data, size_t size) {
 131         const char *b, *p;
 132
 133         assert(data);
 134
 135         if (size < 2)
 136                 return false;
 137
 138         if (startswith(data, "__"))
 139                 return false;
 140
 141         b = data;
 142         for (p = b; p < b + size; p++) {
 143
 144                 if (*p == '=')
 145                         return p > b;
 146
 147                 if (*p == '_')
 148                         continue;
 149
 150                 if (*p >= 'A' && *p <= 'Z')
 151                         continue;
 152
 153                 if (*p >= '0' && *p <= '9')
 154                         continue;
 155
 156                 return false;
 157         }
 158
 159         return false;
 160 }
 161
 162 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 163         const uint8_t *a = _a, *b = _b;
 164         size_t j;
 165
 166         for (j = 0; j < s && j < t; j++) {
 167
 168                 if (a[j] != b[j])
 169                         return false;
 170
 171                 if (a[j] == '=')
 172                         return true;
 173         }
 174
 175         assert_not_reached("\"=\" not found");
 176 }
 177
 178 static Match *match_new(Match *p, MatchType t) {
 179         Match *m;
 180
 181         m = new0(Match, 1);
 182         if (!m)
 183                 return NULL;
 184
 185         m->type = t;
 186
 187         if (p) {
 188                 m->parent = p;
 189                 LIST_PREPEND(matches, p->matches, m);
 190         }
 191
 192         return m;
 193 }
 194
 195 static void match_free(Match *m) {
 196         assert(m);
 197
 198         while (m->matches)
 199                 match_free(m->matches);
 200
 201         if (m->parent)
 202                 LIST_REMOVE(matches, m->parent->matches, m);
 203
 204         free(m->data);
 205         free(m);
 206 }
 207
 208 static void match_free_if_empty(Match *m) {
 209         if (!m || m->matches)
 210                 return;
 211
 212         match_free(m);
 213 }
 214
 215 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 216         Match *l3, *l4, *add_here = NULL, *m;
 217         le64_t le_hash;
 218
 219         assert_return(j, -EINVAL);
 220         assert_return(!journal_pid_changed(j), -ECHILD);
 221         assert_return(data, -EINVAL);
 222
 223         if (size == 0)
 224                 size = strlen(data);
 225
 226         assert_return(match_is_valid(data, size), -EINVAL);
 227
 228         /* level 0: AND term
 229          * level 1: OR terms
 230          * level 2: AND terms
 231          * level 3: OR terms
 232          * level 4: concrete matches */
 233
 234         if (!j->level0) {
 235                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 236                 if (!j->level0)
 237                         return -ENOMEM;
 238         }
 239
 240         if (!j->level1) {
 241                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 242                 if (!j->level1)
 243                         return -ENOMEM;
 244         }
 245
 246         if (!j->level2) {
 247                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 248                 if (!j->level2)
 249                         return -ENOMEM;
 250         }
 251
 252         assert(j->level0->type == MATCH_AND_TERM);
 253         assert(j->level1->type == MATCH_OR_TERM);
 254         assert(j->level2->type == MATCH_AND_TERM);
 255
 256         le_hash = htole64(hash64(data, size));
 257
 258         LIST_FOREACH(matches, l3, j->level2->matches) {
 259                 assert(l3->type == MATCH_OR_TERM);
 260
 261                 LIST_FOREACH(matches, l4, l3->matches) {
 262                         assert(l4->type == MATCH_DISCRETE);
 263
 264                         /* Exactly the same match already? Then ignore
 265                          * this addition */
 266                         if (l4->le_hash == le_hash &&
 267                             l4->size == size &&
 268                             memcmp(l4->data, data, size) == 0)
 269                                 return 0;
 270
 271                         /* Same field? Then let's add this to this OR term */
 272                         if (same_field(data, size, l4->data, l4->size)) {
 273                                 add_here = l3;
 274                                 break;
 275                         }
 276                 }
 277
 278                 if (add_here)
 279                         break;
 280         }
 281
 282         if (!add_here) {
 283                 add_here = match_new(j->level2, MATCH_OR_TERM);
 284                 if (!add_here)
 285                         goto fail;
 286         }
 287
 288         m = match_new(add_here, MATCH_DISCRETE);
 289         if (!m)
 290                 goto fail;
 291
 292         m->le_hash = le_hash;
 293         m->size = size;
 294         m->data = memdup(data, size);
 295         if (!m->data)
 296                 goto fail;
 297
 298         detach_location(j);
 299
 300         return 0;
 301
 302 fail:
 303         match_free_if_empty(add_here);
 304         match_free_if_empty(j->level2);
 305         match_free_if_empty(j->level1);
 306         match_free_if_empty(j->level0);
 307
 308         return -ENOMEM;
 309 }
 310
 311 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 312         assert_return(j, -EINVAL);
 313         assert_return(!journal_pid_changed(j), -ECHILD);
 314
 315         if (!j->level0)
 316                 return 0;
 317
 318         if (!j->level1)
 319                 return 0;
 320
 321         if (!j->level1->matches)
 322                 return 0;
 323
 324         j->level1 = NULL;
 325         j->level2 = NULL;
 326
 327         return 0;
 328 }
 329
 330 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 331         assert_return(j, -EINVAL);
 332         assert_return(!journal_pid_changed(j), -ECHILD);
 333
 334         if (!j->level0)
 335                 return 0;
 336
 337         if (!j->level1)
 338                 return 0;
 339
 340         if (!j->level2)
 341                 return 0;
 342
 343         if (!j->level2->matches)
 344                 return 0;
 345
 346         j->level2 = NULL;
 347         return 0;
 348 }
 349
 350 static char *match_make_string(Match *m) {
 351         char *p, *r;
 352         Match *i;
 353         bool enclose = false;
 354
 355         if (!m)
 356                 return strdup("none");
 357
 358         if (m->type == MATCH_DISCRETE)
 359                 return strndup(m->data, m->size);
 360
 361         p = NULL;
 362         LIST_FOREACH(matches, i, m->matches) {
 363                 char *t, *k;
 364
 365                 t = match_make_string(i);
 366                 if (!t) {
 367                         free(p);
 368                         return NULL;
 369                 }
 370
 371                 if (p) {
 372                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 373                         free(p);
 374                         free(t);
 375
 376                         if (!k)
 377                                 return NULL;
 378
 379                         p = k;
 380
 381                         enclose = true;
 382                 } else
 383                         p = t;
 384         }
 385
 386         if (enclose) {
 387                 r = strjoin("(", p, ")", NULL);
 388                 free(p);
 389                 return r;
 390         }
 391
 392         return p;
 393 }
 394
 395 char *journal_make_match_string(sd_journal *j) {
 396         assert(j);
 397
 398         return match_make_string(j->level0);
 399 }
 400
 401 _public_ void sd_journal_flush_matches(sd_journal *j) {
 402         if (!j)
 403                 return;
 404
 405         if (j->level0)
 406                 match_free(j->level0);
 407
 408         j->level0 = j->level1 = j->level2 = NULL;
 409
 410         detach_location(j);
 411 }
 412
 413 _pure_ static int compare_with_location(JournalFile *f, Location *l) {
 414         assert(f);
 415         assert(l);
 416         assert(f->location_type == LOCATION_SEEK);
 417         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 418
 419         if (l->monotonic_set &&
 420             sd_id128_equal(f->current_boot_id, l->boot_id) &&
 421             l->realtime_set &&
 422             f->current_realtime == l->realtime &&
 423             l->xor_hash_set &&
 424             f->current_xor_hash == l->xor_hash)
 425                 return 0;
 426
 427         if (l->seqnum_set &&
 428             sd_id128_equal(f->header->seqnum_id, l->seqnum_id)) {
 429
 430                 if (f->current_seqnum < l->seqnum)
 431                         return -1;
 432                 if (f->current_seqnum > l->seqnum)
 433                         return 1;
 434         }
 435
 436         if (l->monotonic_set &&
 437             sd_id128_equal(f->current_boot_id, l->boot_id)) {
 438
 439                 if (f->current_monotonic < l->monotonic)
 440                         return -1;
 441                 if (f->current_monotonic > l->monotonic)
 442                         return 1;
 443         }
 444
 445         if (l->realtime_set) {
 446
 447                 if (f->current_realtime < l->realtime)
 448                         return -1;
 449                 if (f->current_realtime > l->realtime)
 450                         return 1;
 451         }
 452
 453         if (l->xor_hash_set) {
 454
 455                 if (f->current_xor_hash < l->xor_hash)
 456                         return -1;
 457                 if (f->current_xor_hash > l->xor_hash)
 458                         return 1;
 459         }
 460
 461         return 0;
 462 }
 463
 464 static int next_for_match(
 465                 sd_journal *j,
 466                 Match *m,
 467                 JournalFile *f,
 468                 uint64_t after_offset,
 469                 direction_t direction,
 470                 Object **ret,
 471                 uint64_t *offset) {
 472
 473         int r;
 474         uint64_t np = 0;
 475         Object *n;
 476
 477         assert(j);
 478         assert(m);
 479         assert(f);
 480
 481         if (m->type == MATCH_DISCRETE) {
 482                 uint64_t dp;
 483
 484                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 485                 if (r <= 0)
 486                         return r;
 487
 488                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 489
 490         } else if (m->type == MATCH_OR_TERM) {
 491                 Match *i;
 492
 493                 /* Find the earliest match beyond after_offset */
 494
 495                 LIST_FOREACH(matches, i, m->matches) {
 496                         uint64_t cp;
 497
 498                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 499                         if (r < 0)
 500                                 return r;
 501                         else if (r > 0) {
 502                                 if (np == 0 || (direction == DIRECTION_DOWN ? cp < np : cp > np))
 503                                         np = cp;
 504                         }
 505                 }
 506
 507                 if (np == 0)
 508                         return 0;
 509
 510         } else if (m->type == MATCH_AND_TERM) {
 511                 Match *i, *last_moved;
 512
 513                 /* Always jump to the next matching entry and repeat
 514                  * this until we find an offset that matches for all
 515                  * matches. */
 516
 517                 if (!m->matches)
 518                         return 0;
 519
 520                 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
 521                 if (r <= 0)
 522                         return r;
 523
 524                 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
 525                 last_moved = m->matches;
 526
 527                 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
 528                         uint64_t cp;
 529
 530                         r = next_for_match(j, i, f, np, direction, NULL, &cp);
 531                         if (r <= 0)
 532                                 return r;
 533
 534                         assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
 535                         if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
 536                                 np = cp;
 537                                 last_moved = i;
 538                         }
 539                 }
 540         }
 541
 542         assert(np > 0);
 543
 544         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 545         if (r < 0)
 546                 return r;
 547
 548         if (ret)
 549                 *ret = n;
 550         if (offset)
 551                 *offset = np;
 552
 553         return 1;
 554 }
 555
 556 static int find_location_for_match(
 557                 sd_journal *j,
 558                 Match *m,
 559                 JournalFile *f,
 560                 direction_t direction,
 561                 Object **ret,
 562                 uint64_t *offset) {
 563
 564         int r;
 565
 566         assert(j);
 567         assert(m);
 568         assert(f);
 569
 570         if (m->type == MATCH_DISCRETE) {
 571                 uint64_t dp;
 572
 573                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 574                 if (r <= 0)
 575                         return r;
 576
 577                 /* FIXME: missing: find by monotonic */
 578
 579                 if (j->current_location.type == LOCATION_HEAD)
 580                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 581                 if (j->current_location.type == LOCATION_TAIL)
 582                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 583                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 584                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 585                 if (j->current_location.monotonic_set) {
 586                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 587                         if (r != -ENOENT)
 588                                 return r;
 589                 }
 590                 if (j->current_location.realtime_set)
 591                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 592
 593                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 594
 595         } else if (m->type == MATCH_OR_TERM) {
 596                 uint64_t np = 0;
 597                 Object *n;
 598                 Match *i;
 599
 600                 /* Find the earliest match */
 601
 602                 LIST_FOREACH(matches, i, m->matches) {
 603                         uint64_t cp;
 604
 605                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 606                         if (r < 0)
 607                                 return r;
 608                         else if (r > 0) {
 609                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 610                                         np = cp;
 611                         }
 612                 }
 613
 614                 if (np == 0)
 615                         return 0;
 616
 617                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 618                 if (r < 0)
 619                         return r;
 620
 621                 if (ret)
 622                         *ret = n;
 623                 if (offset)
 624                         *offset = np;
 625
 626                 return 1;
 627
 628         } else {
 629                 Match *i;
 630                 uint64_t np = 0;
 631
 632                 assert(m->type == MATCH_AND_TERM);
 633
 634                 /* First jump to the last match, and then find the
 635                  * next one where all matches match */
 636
 637                 if (!m->matches)
 638                         return 0;
 639
 640                 LIST_FOREACH(matches, i, m->matches) {
 641                         uint64_t cp;
 642
 643                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 644                         if (r <= 0)
 645                                 return r;
 646
 647                         if (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))
 648                                 np = cp;
 649                 }
 650
 651                 return next_for_match(j, m, f, np, direction, ret, offset);
 652         }
 653 }
 654
 655 static int find_location_with_matches(
 656                 sd_journal *j,
 657                 JournalFile *f,
 658                 direction_t direction,
 659                 Object **ret,
 660                 uint64_t *offset) {
 661
 662         int r;
 663
 664         assert(j);
 665         assert(f);
 666         assert(ret);
 667         assert(offset);
 668
 669         if (!j->level0) {
 670                 /* No matches is simple */
 671
 672                 if (j->current_location.type == LOCATION_HEAD)
 673                         return journal_file_next_entry(f, 0, DIRECTION_DOWN, ret, offset);
 674                 if (j->current_location.type == LOCATION_TAIL)
 675                         return journal_file_next_entry(f, 0, DIRECTION_UP, ret, offset);
 676                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 677                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 678                 if (j->current_location.monotonic_set) {
 679                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 680                         if (r != -ENOENT)
 681                                 return r;
 682                 }
 683                 if (j->current_location.realtime_set)
 684                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 685
 686                 return journal_file_next_entry(f, 0, direction, ret, offset);
 687         } else
 688                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 689 }
 690
 691 static int next_with_matches(
 692                 sd_journal *j,
 693                 JournalFile *f,
 694                 direction_t direction,
 695                 Object **ret,
 696                 uint64_t *offset) {
 697
 698         assert(j);
 699         assert(f);
 700         assert(ret);
 701         assert(offset);
 702
 703         /* No matches is easy. We simple advance the file
 704          * pointer by one. */
 705         if (!j->level0)
 706                 return journal_file_next_entry(f, f->current_offset, direction, ret, offset);
 707
 708         /* If we have a match then we look for the next matching entry
 709          * with an offset at least one step larger */
 710         return next_for_match(j, j->level0, f,
 711                               direction == DIRECTION_DOWN ? f->current_offset + 1
 712                                                           : f->current_offset - 1,
 713                               direction, ret, offset);
 714 }
 715
 716 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction) {
 717         Object *c;
 718         uint64_t cp, n_entries;
 719         int r;
 720
 721         assert(j);
 722         assert(f);
 723
 724         if (f->last_direction == direction && f->current_offset > 0) {
 725                 /* If we hit EOF before, recheck if any new entries arrived. */
 726                 n_entries = le64toh(f->header->n_entries);
 727                 if (f->location_type == LOCATION_TAIL && n_entries == f->last_n_entries)
 728                         return 0;
 729                 f->last_n_entries = n_entries;
 730
 731                 /* LOCATION_SEEK here means we did the work in a previous
 732                  * iteration and the current location already points to a
 733                  * candidate entry. */
 734                 if (f->location_type != LOCATION_SEEK) {
 735                         r = next_with_matches(j, f, direction, &c, &cp);
 736                         if (r <= 0)
 737                                 return r;
 738
 739                         journal_file_save_location(f, direction, c, cp);
 740                 }
 741         } else {
 742                 r = find_location_with_matches(j, f, direction, &c, &cp);
 743                 if (r <= 0)
 744                         return r;
 745
 746                 journal_file_save_location(f, direction, c, cp);
 747         }
 748
 749         /* OK, we found the spot, now let's advance until an entry
 750          * that is actually different from what we were previously
 751          * looking at. This is necessary to handle entries which exist
 752          * in two (or more) journal files, and which shall all be
 753          * suppressed but one. */
 754
 755         for (;;) {
 756                 bool found;
 757
 758                 if (j->current_location.type == LOCATION_DISCRETE) {
 759                         int k;
 760
 761                         k = compare_with_location(f, &j->current_location);
 762
 763                         found = direction == DIRECTION_DOWN ? k > 0 : k < 0;
 764                 } else
 765                         found = true;
 766
 767                 if (found)
 768                         return 1;
 769
 770                 r = next_with_matches(j, f, direction, &c, &cp);
 771                 if (r <= 0)
 772                         return r;
 773
 774                 journal_file_save_location(f, direction, c, cp);
 775         }
 776 }
 777
 778 static int real_journal_next(sd_journal *j, direction_t direction) {
 779         JournalFile *f, *new_file = NULL;
 780         Iterator i;
 781         Object *o;
 782         int r;
 783
 784         assert_return(j, -EINVAL);
 785         assert_return(!journal_pid_changed(j), -ECHILD);
 786
 787         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
 788                 bool found;
 789
 790                 r = next_beyond_location(j, f, direction);
 791                 if (r < 0) {
 792                         log_debug_errno(r, "Can't iterate through %s, ignoring: %m", f->path);
 793                         remove_file_real(j, f);
 794                         continue;
 795                 } else if (r == 0) {
 796                         f->location_type = LOCATION_TAIL;
 797                         continue;
 798                 }
 799
 800                 if (!new_file)
 801                         found = true;
 802                 else {
 803                         int k;
 804
 805                         k = journal_file_compare_locations(f, new_file);
 806
 807                         found = direction == DIRECTION_DOWN ? k < 0 : k > 0;
 808                 }
 809
 810                 if (found)
 811                         new_file = f;
 812         }
 813
 814         if (!new_file)
 815                 return 0;
 816
 817         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_file->current_offset, &o);
 818         if (r < 0)
 819                 return r;
 820
 821         set_location(j, new_file, o);
 822
 823         return 1;
 824 }
 825
 826 _public_ int sd_journal_next(sd_journal *j) {
 827         return real_journal_next(j, DIRECTION_DOWN);
 828 }
 829
 830 _public_ int sd_journal_previous(sd_journal *j) {
 831         return real_journal_next(j, DIRECTION_UP);
 832 }
 833
 834 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 835         int c = 0, r;
 836
 837         assert_return(j, -EINVAL);
 838         assert_return(!journal_pid_changed(j), -ECHILD);
 839
 840         if (skip == 0) {
 841                 /* If this is not a discrete skip, then at least
 842                  * resolve the current location */
 843                 if (j->current_location.type != LOCATION_DISCRETE)
 844                         return real_journal_next(j, direction);
 845
 846                 return 0;
 847         }
 848
 849         do {
 850                 r = real_journal_next(j, direction);
 851                 if (r < 0)
 852                         return r;
 853
 854                 if (r == 0)
 855                         return c;
 856
 857                 skip--;
 858                 c++;
 859         } while (skip > 0);
 860
 861         return c;
 862 }
 863
 864 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 865         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 866 }
 867
 868 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 869         return real_journal_next_skip(j, DIRECTION_UP, skip);
 870 }
 871
 872 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 873         Object *o;
 874         int r;
 875         char bid[33], sid[33];
 876
 877         assert_return(j, -EINVAL);
 878         assert_return(!journal_pid_changed(j), -ECHILD);
 879         assert_return(cursor, -EINVAL);
 880
 881         if (!j->current_file || j->current_file->current_offset <= 0)
 882                 return -EADDRNOTAVAIL;
 883
 884         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 885         if (r < 0)
 886                 return r;
 887
 888         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 889         sd_id128_to_string(o->entry.boot_id, bid);
 890
 891         if (asprintf(cursor,
 892                      "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
 893                      sid, le64toh(o->entry.seqnum),
 894                      bid, le64toh(o->entry.monotonic),
 895                      le64toh(o->entry.realtime),
 896                      le64toh(o->entry.xor_hash)) < 0)
 897                 return -ENOMEM;
 898
 899         return 0;
 900 }
 901
 902 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
 903         const char *word, *state;
 904         size_t l;
 905         unsigned long long seqnum, monotonic, realtime, xor_hash;
 906         bool
 907                 seqnum_id_set = false,
 908                 seqnum_set = false,
 909                 boot_id_set = false,
 910                 monotonic_set = false,
 911                 realtime_set = false,
 912                 xor_hash_set = false;
 913         sd_id128_t seqnum_id, boot_id;
 914
 915         assert_return(j, -EINVAL);
 916         assert_return(!journal_pid_changed(j), -ECHILD);
 917         assert_return(!isempty(cursor), -EINVAL);
 918
 919         FOREACH_WORD_SEPARATOR(word, l, cursor, ";", state) {
 920                 char *item;
 921                 int k = 0;
 922
 923                 if (l < 2 || word[1] != '=')
 924                         return -EINVAL;
 925
 926                 item = strndup(word, l);
 927                 if (!item)
 928                         return -ENOMEM;
 929
 930                 switch (word[0]) {
 931
 932                 case 's':
 933                         seqnum_id_set = true;
 934                         k = sd_id128_from_string(item+2, &seqnum_id);
 935                         break;
 936
 937                 case 'i':
 938                         seqnum_set = true;
 939                         if (sscanf(item+2, "%llx", &seqnum) != 1)
 940                                 k = -EINVAL;
 941                         break;
 942
 943                 case 'b':
 944                         boot_id_set = true;
 945                         k = sd_id128_from_string(item+2, &boot_id);
 946                         break;
 947
 948                 case 'm':
 949                         monotonic_set = true;
 950                         if (sscanf(item+2, "%llx", &monotonic) != 1)
 951                                 k = -EINVAL;
 952                         break;
 953
 954                 case 't':
 955                         realtime_set = true;
 956                         if (sscanf(item+2, "%llx", &realtime) != 1)
 957                                 k = -EINVAL;
 958                         break;
 959
 960                 case 'x':
 961                         xor_hash_set = true;
 962                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
 963                                 k = -EINVAL;
 964                         break;
 965                 }
 966
 967                 free(item);
 968
 969                 if (k < 0)
 970                         return k;
 971         }
 972
 973         if ((!seqnum_set || !seqnum_id_set) &&
 974             (!monotonic_set || !boot_id_set) &&
 975             !realtime_set)
 976                 return -EINVAL;
 977
 978         reset_location(j);
 979
 980         j->current_location.type = LOCATION_SEEK;
 981
 982         if (realtime_set) {
 983                 j->current_location.realtime = (uint64_t) realtime;
 984                 j->current_location.realtime_set = true;
 985         }
 986
 987         if (seqnum_set && seqnum_id_set) {
 988                 j->current_location.seqnum = (uint64_t) seqnum;
 989                 j->current_location.seqnum_id = seqnum_id;
 990                 j->current_location.seqnum_set = true;
 991         }
 992
 993         if (monotonic_set && boot_id_set) {
 994                 j->current_location.monotonic = (uint64_t) monotonic;
 995                 j->current_location.boot_id = boot_id;
 996                 j->current_location.monotonic_set = true;
 997         }
 998
 999         if (xor_hash_set) {
1000                 j->current_location.xor_hash = (uint64_t) xor_hash;
1001                 j->current_location.xor_hash_set = true;
1002         }
1003
1004         return 0;
1005 }
1006
1007 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1008         int r;
1009         const char *word, *state;
1010         size_t l;
1011         Object *o;
1012
1013         assert_return(j, -EINVAL);
1014         assert_return(!journal_pid_changed(j), -ECHILD);
1015         assert_return(!isempty(cursor), -EINVAL);
1016
1017         if (!j->current_file || j->current_file->current_offset <= 0)
1018                 return -EADDRNOTAVAIL;
1019
1020         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1021         if (r < 0)
1022                 return r;
1023
1024         FOREACH_WORD_SEPARATOR(word, l, cursor, ";", state) {
1025                 _cleanup_free_ char *item = NULL;
1026                 sd_id128_t id;
1027                 unsigned long long ll;
1028                 int k = 0;
1029
1030                 if (l < 2 || word[1] != '=')
1031                         return -EINVAL;
1032
1033                 item = strndup(word, l);
1034                 if (!item)
1035                         return -ENOMEM;
1036
1037                 switch (word[0]) {
1038
1039                 case 's':
1040                         k = sd_id128_from_string(item+2, &id);
1041                         if (k < 0)
1042                                 return k;
1043                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1044                                 return 0;
1045                         break;
1046
1047                 case 'i':
1048                         if (sscanf(item+2, "%llx", &ll) != 1)
1049                                 return -EINVAL;
1050                         if (ll != le64toh(o->entry.seqnum))
1051                                 return 0;
1052                         break;
1053
1054                 case 'b':
1055                         k = sd_id128_from_string(item+2, &id);
1056                         if (k < 0)
1057                                 return k;
1058                         if (!sd_id128_equal(id, o->entry.boot_id))
1059                                 return 0;
1060                         break;
1061
1062                 case 'm':
1063                         if (sscanf(item+2, "%llx", &ll) != 1)
1064                                 return -EINVAL;
1065                         if (ll != le64toh(o->entry.monotonic))
1066                                 return 0;
1067                         break;
1068
1069                 case 't':
1070                         if (sscanf(item+2, "%llx", &ll) != 1)
1071                                 return -EINVAL;
1072                         if (ll != le64toh(o->entry.realtime))
1073                                 return 0;
1074                         break;
1075
1076                 case 'x':
1077                         if (sscanf(item+2, "%llx", &ll) != 1)
1078                                 return -EINVAL;
1079                         if (ll != le64toh(o->entry.xor_hash))
1080                                 return 0;
1081                         break;
1082                 }
1083         }
1084
1085         return 1;
1086 }
1087
1088
1089 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1090         assert_return(j, -EINVAL);
1091         assert_return(!journal_pid_changed(j), -ECHILD);
1092
1093         reset_location(j);
1094         j->current_location.type = LOCATION_SEEK;
1095         j->current_location.boot_id = boot_id;
1096         j->current_location.monotonic = usec;
1097         j->current_location.monotonic_set = true;
1098
1099         return 0;
1100 }
1101
1102 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1103         assert_return(j, -EINVAL);
1104         assert_return(!journal_pid_changed(j), -ECHILD);
1105
1106         reset_location(j);
1107         j->current_location.type = LOCATION_SEEK;
1108         j->current_location.realtime = usec;
1109         j->current_location.realtime_set = true;
1110
1111         return 0;
1112 }
1113
1114 _public_ int sd_journal_seek_head(sd_journal *j) {
1115         assert_return(j, -EINVAL);
1116         assert_return(!journal_pid_changed(j), -ECHILD);
1117
1118         reset_location(j);
1119         j->current_location.type = LOCATION_HEAD;
1120
1121         return 0;
1122 }
1123
1124 _public_ int sd_journal_seek_tail(sd_journal *j) {
1125         assert_return(j, -EINVAL);
1126         assert_return(!journal_pid_changed(j), -ECHILD);
1127
1128         reset_location(j);
1129         j->current_location.type = LOCATION_TAIL;
1130
1131         return 0;
1132 }
1133
1134 static void check_network(sd_journal *j, int fd) {
1135         struct statfs sfs;
1136
1137         assert(j);
1138
1139         if (j->on_network)
1140                 return;
1141
1142         if (fstatfs(fd, &sfs) < 0)
1143                 return;
1144
1145         j->on_network =
1146                 F_TYPE_EQUAL(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1147                 F_TYPE_EQUAL(sfs.f_type, CODA_SUPER_MAGIC) ||
1148                 F_TYPE_EQUAL(sfs.f_type, NCP_SUPER_MAGIC) ||
1149                 F_TYPE_EQUAL(sfs.f_type, NFS_SUPER_MAGIC) ||
1150                 F_TYPE_EQUAL(sfs.f_type, SMB_SUPER_MAGIC);
1151 }
1152
1153 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1154         const char *full, *tilded, *atted;
1155
1156         full = strappenda(prefix, ".journal");
1157         tilded = strappenda(full, "~");
1158         atted = strappenda(prefix, "@");
1159
1160         return streq(filename, full) ||
1161                streq(filename, tilded) ||
1162                startswith(filename, atted);
1163 }
1164
1165 static bool file_type_wanted(int flags, const char *filename) {
1166         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1167                 return false;
1168
1169         /* no flags set → every type is OK */
1170         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1171                 return true;
1172
1173         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1174                 return true;
1175
1176         if (flags & SD_JOURNAL_CURRENT_USER) {
1177                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1178
1179                 assert_se(snprintf(prefix, sizeof(prefix), "user-"UID_FMT, getuid())
1180                           < (int) sizeof(prefix));
1181
1182                 if (file_has_type_prefix(prefix, filename))
1183                         return true;
1184         }
1185
1186         return false;
1187 }
1188
1189 static int add_any_file(sd_journal *j, const char *path) {
1190         JournalFile *f = NULL;
1191         int r;
1192
1193         assert(j);
1194         assert(path);
1195
1196         if (ordered_hashmap_get(j->files, path))
1197                 return 0;
1198
1199         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1200         if (r < 0)
1201                 return r;
1202
1203         /* journal_file_dump(f); */
1204
1205         r = ordered_hashmap_put(j->files, f->path, f);
1206         if (r < 0) {
1207                 journal_file_close(f);
1208                 return r;
1209         }
1210
1211         log_debug("File %s added.", f->path);
1212
1213         check_network(j, f->fd);
1214
1215         j->current_invalidate_counter ++;
1216
1217         return 0;
1218 }
1219
1220 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1221         _cleanup_free_ char *path = NULL;
1222         int r;
1223
1224         assert(j);
1225         assert(prefix);
1226         assert(filename);
1227
1228         if (j->no_new_files ||
1229             !file_type_wanted(j->flags, filename))
1230                 return 0;
1231
1232         path = strjoin(prefix, "/", filename, NULL);
1233         if (!path)
1234                 return -ENOMEM;
1235
1236         r = add_any_file(j, path);
1237         if (r == -ENOENT)
1238                 return 0;
1239         return 0;
1240 }
1241
1242 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1243         _cleanup_free_ char *path;
1244         JournalFile *f;
1245
1246         assert(j);
1247         assert(prefix);
1248         assert(filename);
1249
1250         path = strjoin(prefix, "/", filename, NULL);
1251         if (!path)
1252                 return -ENOMEM;
1253
1254         f = ordered_hashmap_get(j->files, path);
1255         if (!f)
1256                 return 0;
1257
1258         remove_file_real(j, f);
1259         return 0;
1260 }
1261
1262 static void remove_file_real(sd_journal *j, JournalFile *f) {
1263         assert(j);
1264         assert(f);
1265
1266         ordered_hashmap_remove(j->files, f->path);
1267
1268         log_debug("File %s removed.", f->path);
1269
1270         if (j->current_file == f) {
1271                 j->current_file = NULL;
1272                 j->current_field = 0;
1273         }
1274
1275         if (j->unique_file == f) {
1276                 /* Jump to the next unique_file or NULL if that one was last */
1277                 j->unique_file = ordered_hashmap_next(j->files, j->unique_file->path);
1278                 j->unique_offset = 0;
1279                 if (!j->unique_file)
1280                         j->unique_file_lost = true;
1281         }
1282
1283         journal_file_close(f);
1284
1285         j->current_invalidate_counter ++;
1286 }
1287
1288 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1289         _cleanup_free_ char *path = NULL;
1290         int r;
1291         _cleanup_closedir_ DIR *d = NULL;
1292         sd_id128_t id, mid;
1293         Directory *m;
1294
1295         assert(j);
1296         assert(prefix);
1297         assert(dirname);
1298
1299         log_debug("Considering %s/%s.", prefix, dirname);
1300
1301         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1302             (sd_id128_from_string(dirname, &id) < 0 ||
1303              sd_id128_get_machine(&mid) < 0 ||
1304              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1305             return 0;
1306
1307         path = strjoin(prefix, "/", dirname, NULL);
1308         if (!path)
1309                 return -ENOMEM;
1310
1311         d = opendir(path);
1312         if (!d) {
1313                 log_debug_errno(errno, "Failed to open %s: %m", path);
1314                 if (errno == ENOENT)
1315                         return 0;
1316                 return -errno;
1317         }
1318
1319         m = hashmap_get(j->directories_by_path, path);
1320         if (!m) {
1321                 m = new0(Directory, 1);
1322                 if (!m)
1323                         return -ENOMEM;
1324
1325                 m->is_root = false;
1326                 m->path = path;
1327
1328                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1329                         free(m);
1330                         return -ENOMEM;
1331                 }
1332
1333                 path = NULL; /* avoid freeing in cleanup */
1334                 j->current_invalidate_counter ++;
1335
1336                 log_debug("Directory %s added.", m->path);
1337
1338         } else if (m->is_root)
1339                 return 0;
1340
1341         if (m->wd <= 0 && j->inotify_fd >= 0) {
1342
1343                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1344                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1345                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1346                                           IN_ONLYDIR);
1347
1348                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1349                         inotify_rm_watch(j->inotify_fd, m->wd);
1350         }
1351
1352         for (;;) {
1353                 struct dirent *de;
1354
1355                 errno = 0;
1356                 de = readdir(d);
1357                 if (!de && errno != 0) {
1358                         r = -errno;
1359                         log_debug_errno(errno, "Failed to read directory %s: %m", m->path);
1360                         return r;
1361                 }
1362                 if (!de)
1363                         break;
1364
1365                 if (dirent_is_file_with_suffix(de, ".journal") ||
1366                     dirent_is_file_with_suffix(de, ".journal~")) {
1367                         r = add_file(j, m->path, de->d_name);
1368                         if (r < 0) {
1369                                 log_debug_errno(r, "Failed to add file %s/%s: %m",
1370                                                 m->path, de->d_name);
1371                                 r = set_put_error(j, r);
1372                                 if (r < 0)
1373                                         return r;
1374                         }
1375                 }
1376         }
1377
1378         check_network(j, dirfd(d));
1379
1380         return 0;
1381 }
1382
1383 static int add_root_directory(sd_journal *j, const char *p) {
1384         _cleanup_closedir_ DIR *d = NULL;
1385         Directory *m;
1386         int r;
1387
1388         assert(j);
1389         assert(p);
1390
1391         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1392             !path_startswith(p, "/run"))
1393                 return -EINVAL;
1394
1395         if (j->prefix)
1396                 p = strappenda(j->prefix, p);
1397
1398         d = opendir(p);
1399         if (!d)
1400                 return -errno;
1401
1402         m = hashmap_get(j->directories_by_path, p);
1403         if (!m) {
1404                 m = new0(Directory, 1);
1405                 if (!m)
1406                         return -ENOMEM;
1407
1408                 m->is_root = true;
1409                 m->path = strdup(p);
1410                 if (!m->path) {
1411                         free(m);
1412                         return -ENOMEM;
1413                 }
1414
1415                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1416                         free(m->path);
1417                         free(m);
1418                         return -ENOMEM;
1419                 }
1420
1421                 j->current_invalidate_counter ++;
1422
1423                 log_debug("Root directory %s added.", m->path);
1424
1425         } else if (!m->is_root)
1426                 return 0;
1427
1428         if (m->wd <= 0 && j->inotify_fd >= 0) {
1429
1430                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1431                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1432                                           IN_ONLYDIR);
1433
1434                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1435                         inotify_rm_watch(j->inotify_fd, m->wd);
1436         }
1437
1438         if (j->no_new_files)
1439                 return 0;
1440
1441         for (;;) {
1442                 struct dirent *de;
1443                 sd_id128_t id;
1444
1445                 errno = 0;
1446                 de = readdir(d);
1447                 if (!de && errno != 0) {
1448                         r = -errno;
1449                         log_debug_errno(errno, "Failed to read directory %s: %m", m->path);
1450                         return r;
1451                 }
1452                 if (!de)
1453                         break;
1454
1455                 if (dirent_is_file_with_suffix(de, ".journal") ||
1456                     dirent_is_file_with_suffix(de, ".journal~")) {
1457                         r = add_file(j, m->path, de->d_name);
1458                         if (r < 0) {
1459                                 log_debug_errno(r, "Failed to add file %s/%s: %m",
1460                                                 m->path, de->d_name);
1461                                 r = set_put_error(j, r);
1462                                 if (r < 0)
1463                                         return r;
1464                         }
1465                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1466                            sd_id128_from_string(de->d_name, &id) >= 0) {
1467
1468                         r = add_directory(j, m->path, de->d_name);
1469                         if (r < 0)
1470                                 log_debug_errno(r, "Failed to add directory %s/%s: %m", m->path, de->d_name);
1471                 }
1472         }
1473
1474         check_network(j, dirfd(d));
1475
1476         return 0;
1477 }
1478
1479 static int remove_directory(sd_journal *j, Directory *d) {
1480         assert(j);
1481
1482         if (d->wd > 0) {
1483                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1484
1485                 if (j->inotify_fd >= 0)
1486                         inotify_rm_watch(j->inotify_fd, d->wd);
1487         }
1488
1489         hashmap_remove(j->directories_by_path, d->path);
1490
1491         if (d->is_root)
1492                 log_debug("Root directory %s removed.", d->path);
1493         else
1494                 log_debug("Directory %s removed.", d->path);
1495
1496         free(d->path);
1497         free(d);
1498
1499         return 0;
1500 }
1501
1502 static int add_search_paths(sd_journal *j) {
1503         int r;
1504         const char search_paths[] =
1505                 "/run/log/journal\0"
1506                 "/var/log/journal\0";
1507         const char *p;
1508
1509         assert(j);
1510
1511         /* We ignore most errors here, since the idea is to only open
1512          * what's actually accessible, and ignore the rest. */
1513
1514         NULSTR_FOREACH(p, search_paths) {
1515                 r = add_root_directory(j, p);
1516                 if (r < 0 && r != -ENOENT) {
1517                         r = set_put_error(j, r);
1518                         if (r < 0)
1519                                 return r;
1520                 }
1521         }
1522
1523         return 0;
1524 }
1525
1526 static int add_current_paths(sd_journal *j) {
1527         Iterator i;
1528         JournalFile *f;
1529
1530         assert(j);
1531         assert(j->no_new_files);
1532
1533         /* Simply adds all directories for files we have open as
1534          * "root" directories. We don't expect errors here, so we
1535          * treat them as fatal. */
1536
1537         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
1538                 _cleanup_free_ char *dir;
1539                 int r;
1540
1541                 dir = dirname_malloc(f->path);
1542                 if (!dir)
1543                         return -ENOMEM;
1544
1545                 r = add_root_directory(j, dir);
1546                 if (r < 0) {
1547                         set_put_error(j, r);
1548                         return r;
1549                 }
1550         }
1551
1552         return 0;
1553 }
1554
1555
1556 static int allocate_inotify(sd_journal *j) {
1557         assert(j);
1558
1559         if (j->inotify_fd < 0) {
1560                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1561                 if (j->inotify_fd < 0)
1562                         return -errno;
1563         }
1564
1565         if (!j->directories_by_wd) {
1566                 j->directories_by_wd = hashmap_new(NULL);
1567                 if (!j->directories_by_wd)
1568                         return -ENOMEM;
1569         }
1570
1571         return 0;
1572 }
1573
1574 static sd_journal *journal_new(int flags, const char *path) {
1575         sd_journal *j;
1576
1577         j = new0(sd_journal, 1);
1578         if (!j)
1579                 return NULL;
1580
1581         j->original_pid = getpid();
1582         j->inotify_fd = -1;
1583         j->flags = flags;
1584         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1585
1586         if (path) {
1587                 j->path = strdup(path);
1588                 if (!j->path)
1589                         goto fail;
1590         }
1591
1592         j->files = ordered_hashmap_new(&string_hash_ops);
1593         j->directories_by_path = hashmap_new(&string_hash_ops);
1594         j->mmap = mmap_cache_new();
1595         if (!j->files || !j->directories_by_path || !j->mmap)
1596                 goto fail;
1597
1598         return j;
1599
1600 fail:
1601         sd_journal_close(j);
1602         return NULL;
1603 }
1604
1605 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1606         sd_journal *j;
1607         int r;
1608
1609         assert_return(ret, -EINVAL);
1610         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_RUNTIME_ONLY|SD_JOURNAL_SYSTEM|SD_JOURNAL_CURRENT_USER)) == 0, -EINVAL);
1611
1612         j = journal_new(flags, NULL);
1613         if (!j)
1614                 return -ENOMEM;
1615
1616         r = add_search_paths(j);
1617         if (r < 0)
1618                 goto fail;
1619
1620         *ret = j;
1621         return 0;
1622
1623 fail:
1624         sd_journal_close(j);
1625
1626         return r;
1627 }
1628
1629 _public_ int sd_journal_open_container(sd_journal **ret, const char *machine, int flags) {
1630         _cleanup_free_ char *root = NULL, *class = NULL;
1631         sd_journal *j;
1632         char *p;
1633         int r;
1634
1635         assert_return(machine, -EINVAL);
1636         assert_return(ret, -EINVAL);
1637         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_SYSTEM)) == 0, -EINVAL);
1638         assert_return(machine_name_is_valid(machine), -EINVAL);
1639
1640         p = strappenda("/run/systemd/machines/", machine);
1641         r = parse_env_file(p, NEWLINE, "ROOT", &root, "CLASS", &class, NULL);
1642         if (r == -ENOENT)
1643                 return -EHOSTDOWN;
1644         if (r < 0)
1645                 return r;
1646         if (!root)
1647                 return -ENODATA;
1648
1649         if (!streq_ptr(class, "container"))
1650                 return -EIO;
1651
1652         j = journal_new(flags, NULL);
1653         if (!j)
1654                 return -ENOMEM;
1655
1656         j->prefix = root;
1657         root = NULL;
1658
1659         r = add_search_paths(j);
1660         if (r < 0)
1661                 goto fail;
1662
1663         *ret = j;
1664         return 0;
1665
1666 fail:
1667         sd_journal_close(j);
1668         return r;
1669 }
1670
1671 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1672         sd_journal *j;
1673         int r;
1674
1675         assert_return(ret, -EINVAL);
1676         assert_return(path, -EINVAL);
1677         assert_return(flags == 0, -EINVAL);
1678
1679         j = journal_new(flags, path);
1680         if (!j)
1681                 return -ENOMEM;
1682
1683         r = add_root_directory(j, path);
1684         if (r < 0) {
1685                 set_put_error(j, r);
1686                 goto fail;
1687         }
1688
1689         *ret = j;
1690         return 0;
1691
1692 fail:
1693         sd_journal_close(j);
1694
1695         return r;
1696 }
1697
1698 _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int flags) {
1699         sd_journal *j;
1700         const char **path;
1701         int r;
1702
1703         assert_return(ret, -EINVAL);
1704         assert_return(flags == 0, -EINVAL);
1705
1706         j = journal_new(flags, NULL);
1707         if (!j)
1708                 return -ENOMEM;
1709
1710         STRV_FOREACH(path, paths) {
1711                 r = add_any_file(j, *path);
1712                 if (r < 0) {
1713                         log_error_errno(r, "Failed to open %s: %m", *path);
1714                         goto fail;
1715                 }
1716         }
1717
1718         j->no_new_files = true;
1719
1720         *ret = j;
1721         return 0;
1722
1723 fail:
1724         sd_journal_close(j);
1725
1726         return r;
1727 }
1728
1729 _public_ void sd_journal_close(sd_journal *j) {
1730         Directory *d;
1731         JournalFile *f;
1732
1733         if (!j)
1734                 return;
1735
1736         sd_journal_flush_matches(j);
1737
1738         while ((f = ordered_hashmap_steal_first(j->files)))
1739                 journal_file_close(f);
1740
1741         ordered_hashmap_free(j->files);
1742
1743         while ((d = hashmap_first(j->directories_by_path)))
1744                 remove_directory(j, d);
1745
1746         while ((d = hashmap_first(j->directories_by_wd)))
1747                 remove_directory(j, d);
1748
1749         hashmap_free(j->directories_by_path);
1750         hashmap_free(j->directories_by_wd);
1751
1752         safe_close(j->inotify_fd);
1753
1754         if (j->mmap) {
1755                 log_debug("mmap cache statistics: %u hit, %u miss", mmap_cache_get_hit(j->mmap), mmap_cache_get_missed(j->mmap));
1756                 mmap_cache_unref(j->mmap);
1757         }
1758
1759         free(j->path);
1760         free(j->prefix);
1761         free(j->unique_field);
1762         set_free(j->errors);
1763         free(j);
1764 }
1765
1766 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1767         Object *o;
1768         JournalFile *f;
1769         int r;
1770
1771         assert_return(j, -EINVAL);
1772         assert_return(!journal_pid_changed(j), -ECHILD);
1773         assert_return(ret, -EINVAL);
1774
1775         f = j->current_file;
1776         if (!f)
1777                 return -EADDRNOTAVAIL;
1778
1779         if (f->current_offset <= 0)
1780                 return -EADDRNOTAVAIL;
1781
1782         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1783         if (r < 0)
1784                 return r;
1785
1786         *ret = le64toh(o->entry.realtime);
1787         return 0;
1788 }
1789
1790 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1791         Object *o;
1792         JournalFile *f;
1793         int r;
1794         sd_id128_t id;
1795
1796         assert_return(j, -EINVAL);
1797         assert_return(!journal_pid_changed(j), -ECHILD);
1798
1799         f = j->current_file;
1800         if (!f)
1801                 return -EADDRNOTAVAIL;
1802
1803         if (f->current_offset <= 0)
1804                 return -EADDRNOTAVAIL;
1805
1806         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1807         if (r < 0)
1808                 return r;
1809
1810         if (ret_boot_id)
1811                 *ret_boot_id = o->entry.boot_id;
1812         else {
1813                 r = sd_id128_get_boot(&id);
1814                 if (r < 0)
1815                         return r;
1816
1817                 if (!sd_id128_equal(id, o->entry.boot_id))
1818                         return -ESTALE;
1819         }
1820
1821         if (ret)
1822                 *ret = le64toh(o->entry.monotonic);
1823
1824         return 0;
1825 }
1826
1827 static bool field_is_valid(const char *field) {
1828         const char *p;
1829
1830         assert(field);
1831
1832         if (isempty(field))
1833                 return false;
1834
1835         if (startswith(field, "__"))
1836                 return false;
1837
1838         for (p = field; *p; p++) {
1839
1840                 if (*p == '_')
1841                         continue;
1842
1843                 if (*p >= 'A' && *p <= 'Z')
1844                         continue;
1845
1846                 if (*p >= '0' && *p <= '9')
1847                         continue;
1848
1849                 return false;
1850         }
1851
1852         return true;
1853 }
1854
1855 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1856         JournalFile *f;
1857         uint64_t i, n;
1858         size_t field_length;
1859         int r;
1860         Object *o;
1861
1862         assert_return(j, -EINVAL);
1863         assert_return(!journal_pid_changed(j), -ECHILD);
1864         assert_return(field, -EINVAL);
1865         assert_return(data, -EINVAL);
1866         assert_return(size, -EINVAL);
1867         assert_return(field_is_valid(field), -EINVAL);
1868
1869         f = j->current_file;
1870         if (!f)
1871                 return -EADDRNOTAVAIL;
1872
1873         if (f->current_offset <= 0)
1874                 return -EADDRNOTAVAIL;
1875
1876         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1877         if (r < 0)
1878                 return r;
1879
1880         field_length = strlen(field);
1881
1882         n = journal_file_entry_n_items(o);
1883         for (i = 0; i < n; i++) {
1884                 uint64_t p, l;
1885                 le64_t le_hash;
1886                 size_t t;
1887                 int compression;
1888
1889                 p = le64toh(o->entry.items[i].object_offset);
1890                 le_hash = o->entry.items[i].hash;
1891                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1892                 if (r < 0)
1893                         return r;
1894
1895                 if (le_hash != o->data.hash)
1896                         return -EBADMSG;
1897
1898                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1899
1900                 compression = o->object.flags & OBJECT_COMPRESSION_MASK;
1901                 if (compression) {
1902 #if defined(HAVE_XZ) || defined(HAVE_LZ4)
1903                         if (decompress_startswith(compression,
1904                                                   o->data.payload, l,
1905                                                   &f->compress_buffer, &f->compress_buffer_size,
1906                                                   field, field_length, '=')) {
1907
1908                                 size_t rsize;
1909
1910                                 r = decompress_blob(compression,
1911                                                     o->data.payload, l,
1912                                                     &f->compress_buffer, &f->compress_buffer_size, &rsize,
1913                                                     j->data_threshold);
1914                                 if (r < 0)
1915                                         return r;
1916
1917                                 *data = f->compress_buffer;
1918                                 *size = (size_t) rsize;
1919
1920                                 return 0;
1921                         }
1922 #else
1923                         return -EPROTONOSUPPORT;
1924 #endif
1925                 } else if (l >= field_length+1 &&
1926                            memcmp(o->data.payload, field, field_length) == 0 &&
1927                            o->data.payload[field_length] == '=') {
1928
1929                         t = (size_t) l;
1930
1931                         if ((uint64_t) t != l)
1932                                 return -E2BIG;
1933
1934                         *data = o->data.payload;
1935                         *size = t;
1936
1937                         return 0;
1938                 }
1939
1940                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1941                 if (r < 0)
1942                         return r;
1943         }
1944
1945         return -ENOENT;
1946 }
1947
1948 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1949         size_t t;
1950         uint64_t l;
1951         int compression;
1952
1953         l = le64toh(o->object.size) - offsetof(Object, data.payload);
1954         t = (size_t) l;
1955
1956         /* We can't read objects larger than 4G on a 32bit machine */
1957         if ((uint64_t) t != l)
1958                 return -E2BIG;
1959
1960         compression = o->object.flags & OBJECT_COMPRESSION_MASK;
1961         if (compression) {
1962 #if defined(HAVE_XZ) || defined(HAVE_LZ4)
1963                 size_t rsize;
1964                 int r;
1965
1966                 r = decompress_blob(compression,
1967                                     o->data.payload, l, &f->compress_buffer,
1968                                     &f->compress_buffer_size, &rsize, j->data_threshold);
1969                 if (r < 0)
1970                         return r;
1971
1972                 *data = f->compress_buffer;
1973                 *size = (size_t) rsize;
1974 #else
1975                 return -EPROTONOSUPPORT;
1976 #endif
1977         } else {
1978                 *data = o->data.payload;
1979                 *size = t;
1980         }
1981
1982         return 0;
1983 }
1984
1985 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1986         JournalFile *f;
1987         uint64_t p, n;
1988         le64_t le_hash;
1989         int r;
1990         Object *o;
1991
1992         assert_return(j, -EINVAL);
1993         assert_return(!journal_pid_changed(j), -ECHILD);
1994         assert_return(data, -EINVAL);
1995         assert_return(size, -EINVAL);
1996
1997         f = j->current_file;
1998         if (!f)
1999                 return -EADDRNOTAVAIL;
2000
2001         if (f->current_offset <= 0)
2002                 return -EADDRNOTAVAIL;
2003
2004         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2005         if (r < 0)
2006                 return r;
2007
2008         n = journal_file_entry_n_items(o);
2009         if (j->current_field >= n)
2010                 return 0;
2011
2012         p = le64toh(o->entry.items[j->current_field].object_offset);
2013         le_hash = o->entry.items[j->current_field].hash;
2014         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2015         if (r < 0)
2016                 return r;
2017
2018         if (le_hash != o->data.hash)
2019                 return -EBADMSG;
2020
2021         r = return_data(j, f, o, data, size);
2022         if (r < 0)
2023                 return r;
2024
2025         j->current_field ++;
2026
2027         return 1;
2028 }
2029
2030 _public_ void sd_journal_restart_data(sd_journal *j) {
2031         if (!j)
2032                 return;
2033
2034         j->current_field = 0;
2035 }
2036
2037 _public_ int sd_journal_get_fd(sd_journal *j) {
2038         int r;
2039
2040         assert_return(j, -EINVAL);
2041         assert_return(!journal_pid_changed(j), -ECHILD);
2042
2043         if (j->inotify_fd >= 0)
2044                 return j->inotify_fd;
2045
2046         r = allocate_inotify(j);
2047         if (r < 0)
2048                 return r;
2049
2050         /* Iterate through all dirs again, to add them to the
2051          * inotify */
2052         if (j->no_new_files)
2053                 r = add_current_paths(j);
2054         else if (j->path)
2055                 r = add_root_directory(j, j->path);
2056         else
2057                 r = add_search_paths(j);
2058         if (r < 0)
2059                 return r;
2060
2061         return j->inotify_fd;
2062 }
2063
2064 _public_ int sd_journal_get_events(sd_journal *j) {
2065         int fd;
2066
2067         assert_return(j, -EINVAL);
2068         assert_return(!journal_pid_changed(j), -ECHILD);
2069
2070         fd = sd_journal_get_fd(j);
2071         if (fd < 0)
2072                 return fd;
2073
2074         return POLLIN;
2075 }
2076
2077 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2078         int fd;
2079
2080         assert_return(j, -EINVAL);
2081         assert_return(!journal_pid_changed(j), -ECHILD);
2082         assert_return(timeout_usec, -EINVAL);
2083
2084         fd = sd_journal_get_fd(j);
2085         if (fd < 0)
2086                 return fd;
2087
2088         if (!j->on_network) {
2089                 *timeout_usec = (uint64_t) -1;
2090                 return 0;
2091         }
2092
2093         /* If we are on the network we need to regularly check for
2094          * changes manually */
2095
2096         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2097         return 1;
2098 }
2099
2100 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2101         Directory *d;
2102         int r;
2103
2104         assert(j);
2105         assert(e);
2106
2107         /* Is this a subdirectory we watch? */
2108         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2109         if (d) {
2110                 sd_id128_t id;
2111
2112                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2113                     (endswith(e->name, ".journal") ||
2114                      endswith(e->name, ".journal~"))) {
2115
2116                         /* Event for a journal file */
2117
2118                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2119                                 r = add_file(j, d->path, e->name);
2120                                 if (r < 0) {
2121                                         log_debug_errno(r, "Failed to add file %s/%s: %m",
2122                                                         d->path, e->name);
2123                                         set_put_error(j, r);
2124                                 }
2125
2126                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2127
2128                                 r = remove_file(j, d->path, e->name);
2129                                 if (r < 0)
2130                                         log_debug_errno(r, "Failed to remove file %s/%s: %m", d->path, e->name);
2131                         }
2132
2133                 } else if (!d->is_root && e->len == 0) {
2134
2135                         /* Event for a subdirectory */
2136
2137                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2138                                 r = remove_directory(j, d);
2139                                 if (r < 0)
2140                                         log_debug_errno(r, "Failed to remove directory %s: %m", d->path);
2141                         }
2142
2143
2144                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2145
2146                         /* Event for root directory */
2147
2148                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2149                                 r = add_directory(j, d->path, e->name);
2150                                 if (r < 0)
2151                                         log_debug_errno(r, "Failed to add directory %s/%s: %m", d->path, e->name);
2152                         }
2153                 }
2154
2155                 return;
2156         }
2157
2158         if (e->mask & IN_IGNORED)
2159                 return;
2160
2161         log_warning("Unknown inotify event.");
2162 }
2163
2164 static int determine_change(sd_journal *j) {
2165         bool b;
2166
2167         assert(j);
2168
2169         b = j->current_invalidate_counter != j->last_invalidate_counter;
2170         j->last_invalidate_counter = j->current_invalidate_counter;
2171
2172         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2173 }
2174
2175 _public_ int sd_journal_process(sd_journal *j) {
2176         bool got_something = false;
2177
2178         assert_return(j, -EINVAL);
2179         assert_return(!journal_pid_changed(j), -ECHILD);
2180
2181         j->last_process_usec = now(CLOCK_MONOTONIC);
2182
2183         for (;;) {
2184                 union inotify_event_buffer buffer;
2185                 struct inotify_event *e;
2186                 ssize_t l;
2187
2188                 l = read(j->inotify_fd, &buffer, sizeof(buffer));
2189                 if (l < 0) {
2190                         if (errno == EAGAIN || errno == EINTR)
2191                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2192
2193                         return -errno;
2194                 }
2195
2196                 got_something = true;
2197
2198                 FOREACH_INOTIFY_EVENT(e, buffer, l)
2199                         process_inotify_event(j, e);
2200         }
2201 }
2202
2203 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2204         int r;
2205         uint64_t t;
2206
2207         assert_return(j, -EINVAL);
2208         assert_return(!journal_pid_changed(j), -ECHILD);
2209
2210         if (j->inotify_fd < 0) {
2211
2212                 /* This is the first invocation, hence create the
2213                  * inotify watch */
2214                 r = sd_journal_get_fd(j);
2215                 if (r < 0)
2216                         return r;
2217
2218                 /* The journal might have changed since the context
2219                  * object was created and we weren't watching before,
2220                  * hence don't wait for anything, and return
2221                  * immediately. */
2222                 return determine_change(j);
2223         }
2224
2225         r = sd_journal_get_timeout(j, &t);
2226         if (r < 0)
2227                 return r;
2228
2229         if (t != (uint64_t) -1) {
2230                 usec_t n;
2231
2232                 n = now(CLOCK_MONOTONIC);
2233                 t = t > n ? t - n : 0;
2234
2235                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2236                         timeout_usec = t;
2237         }
2238
2239         do {
2240                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2241         } while (r == -EINTR);
2242
2243         if (r < 0)
2244                 return r;
2245
2246         return sd_journal_process(j);
2247 }
2248
2249 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2250         Iterator i;
2251         JournalFile *f;
2252         bool first = true;
2253         uint64_t fmin = 0, tmax = 0;
2254         int r;
2255
2256         assert_return(j, -EINVAL);
2257         assert_return(!journal_pid_changed(j), -ECHILD);
2258         assert_return(from || to, -EINVAL);
2259         assert_return(from != to, -EINVAL);
2260
2261         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2262                 usec_t fr, t;
2263
2264                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2265                 if (r == -ENOENT)
2266                         continue;
2267                 if (r < 0)
2268                         return r;
2269                 if (r == 0)
2270                         continue;
2271
2272                 if (first) {
2273                         fmin = fr;
2274                         tmax = t;
2275                         first = false;
2276                 } else {
2277                         fmin = MIN(fr, fmin);
2278                         tmax = MAX(t, tmax);
2279                 }
2280         }
2281
2282         if (from)
2283                 *from = fmin;
2284         if (to)
2285                 *to = tmax;
2286
2287         return first ? 0 : 1;
2288 }
2289
2290 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2291         Iterator i;
2292         JournalFile *f;
2293         bool found = false;
2294         int r;
2295
2296         assert_return(j, -EINVAL);
2297         assert_return(!journal_pid_changed(j), -ECHILD);
2298         assert_return(from || to, -EINVAL);
2299         assert_return(from != to, -EINVAL);
2300
2301         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2302                 usec_t fr, t;
2303
2304                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2305                 if (r == -ENOENT)
2306                         continue;
2307                 if (r < 0)
2308                         return r;
2309                 if (r == 0)
2310                         continue;
2311
2312                 if (found) {
2313                         if (from)
2314                                 *from = MIN(fr, *from);
2315                         if (to)
2316                                 *to = MAX(t, *to);
2317                 } else {
2318                         if (from)
2319                                 *from = fr;
2320                         if (to)
2321                                 *to = t;
2322                         found = true;
2323                 }
2324         }
2325
2326         return found;
2327 }
2328
2329 void journal_print_header(sd_journal *j) {
2330         Iterator i;
2331         JournalFile *f;
2332         bool newline = false;
2333
2334         assert(j);
2335
2336         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2337                 if (newline)
2338                         putchar('\n');
2339                 else
2340                         newline = true;
2341
2342                 journal_file_print_header(f);
2343         }
2344 }
2345
2346 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2347         Iterator i;
2348         JournalFile *f;
2349         uint64_t sum = 0;
2350
2351         assert_return(j, -EINVAL);
2352         assert_return(!journal_pid_changed(j), -ECHILD);
2353         assert_return(bytes, -EINVAL);
2354
2355         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2356                 struct stat st;
2357
2358                 if (fstat(f->fd, &st) < 0)
2359                         return -errno;
2360
2361                 sum += (uint64_t) st.st_blocks * 512ULL;
2362         }
2363
2364         *bytes = sum;
2365         return 0;
2366 }
2367
2368 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2369         char *f;
2370
2371         assert_return(j, -EINVAL);
2372         assert_return(!journal_pid_changed(j), -ECHILD);
2373         assert_return(!isempty(field), -EINVAL);
2374         assert_return(field_is_valid(field), -EINVAL);
2375
2376         f = strdup(field);
2377         if (!f)
2378                 return -ENOMEM;
2379
2380         free(j->unique_field);
2381         j->unique_field = f;
2382         j->unique_file = NULL;
2383         j->unique_offset = 0;
2384         j->unique_file_lost = false;
2385
2386         return 0;
2387 }
2388
2389 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2390         size_t k;
2391
2392         assert_return(j, -EINVAL);
2393         assert_return(!journal_pid_changed(j), -ECHILD);
2394         assert_return(data, -EINVAL);
2395         assert_return(l, -EINVAL);
2396         assert_return(j->unique_field, -EINVAL);
2397
2398         k = strlen(j->unique_field);
2399
2400         if (!j->unique_file) {
2401                 if (j->unique_file_lost)
2402                         return 0;
2403
2404                 j->unique_file = ordered_hashmap_first(j->files);
2405                 if (!j->unique_file)
2406                         return 0;
2407
2408                 j->unique_offset = 0;
2409         }
2410
2411         for (;;) {
2412                 JournalFile *of;
2413                 Iterator i;
2414                 Object *o;
2415                 const void *odata;
2416                 size_t ol;
2417                 bool found;
2418                 int r;
2419
2420                 /* Proceed to next data object in the field's linked list */
2421                 if (j->unique_offset == 0) {
2422                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2423                         if (r < 0)
2424                                 return r;
2425
2426                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2427                 } else {
2428                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2429                         if (r < 0)
2430                                 return r;
2431
2432                         j->unique_offset = le64toh(o->data.next_field_offset);
2433                 }
2434
2435                 /* We reached the end of the list? Then start again, with the next file */
2436                 if (j->unique_offset == 0) {
2437                         j->unique_file = ordered_hashmap_next(j->files, j->unique_file->path);
2438                         if (!j->unique_file)
2439                                 return 0;
2440
2441                         continue;
2442                 }
2443
2444                 /* We do not use OBJECT_DATA context here, but OBJECT_UNUSED
2445                  * instead, so that we can look at this data object at the same
2446                  * time as one on another file */
2447                 r = journal_file_move_to_object(j->unique_file, OBJECT_UNUSED, j->unique_offset, &o);
2448                 if (r < 0)
2449                         return r;
2450
2451                 /* Let's do the type check by hand, since we used 0 context above. */
2452                 if (o->object.type != OBJECT_DATA) {
2453                         log_debug("%s:offset " OFSfmt ": object has type %d, expected %d",
2454                                   j->unique_file->path, j->unique_offset,
2455                                   o->object.type, OBJECT_DATA);
2456                         return -EBADMSG;
2457                 }
2458
2459                 r = return_data(j, j->unique_file, o, &odata, &ol);
2460                 if (r < 0)
2461                         return r;
2462
2463                 /* Check if we have at least the field name and "=". */
2464                 if (ol <= k) {
2465                         log_debug("%s:offset " OFSfmt ": object has size %zu, expected at least %zu",
2466                                   j->unique_file->path, j->unique_offset,
2467                                   ol, k + 1);
2468                         return -EBADMSG;
2469                 }
2470
2471                 if (memcmp(odata, j->unique_field, k) || ((const char*) odata)[k] != '=') {
2472                         log_debug("%s:offset " OFSfmt ": object does not start with \"%s=\"",
2473                                   j->unique_file->path, j->unique_offset,
2474                                   j->unique_field);
2475                         return -EBADMSG;
2476                 }
2477
2478                 /* OK, now let's see if we already returned this data
2479                  * object by checking if it exists in the earlier
2480                  * traversed files. */
2481                 found = false;
2482                 ORDERED_HASHMAP_FOREACH(of, j->files, i) {
2483                         Object *oo;
2484                         uint64_t op;
2485
2486                         if (of == j->unique_file)
2487                                 break;
2488
2489                         /* Skip this file it didn't have any fields
2490                          * indexed */
2491                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2492                             le64toh(of->header->n_fields) <= 0)
2493                                 continue;
2494
2495                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2496                         if (r < 0)
2497                                 return r;
2498
2499                         if (r > 0)
2500                                 found = true;
2501                 }
2502
2503                 if (found)
2504                         continue;
2505
2506                 r = return_data(j, j->unique_file, o, data, l);
2507                 if (r < 0)
2508                         return r;
2509
2510                 return 1;
2511         }
2512 }
2513
2514 _public_ void sd_journal_restart_unique(sd_journal *j) {
2515         if (!j)
2516                 return;
2517
2518         j->unique_file = NULL;
2519         j->unique_offset = 0;
2520         j->unique_file_lost = false;
2521 }
2522
2523 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2524         assert_return(j, -EINVAL);
2525         assert_return(!journal_pid_changed(j), -ECHILD);
2526
2527         return !j->on_network;
2528 }
2529
2530 static char *lookup_field(const char *field, void *userdata) {
2531         sd_journal *j = userdata;
2532         const void *data;
2533         size_t size, d;
2534         int r;
2535
2536         assert(field);
2537         assert(j);
2538
2539         r = sd_journal_get_data(j, field, &data, &size);
2540         if (r < 0 ||
2541             size > REPLACE_VAR_MAX)
2542                 return strdup(field);
2543
2544         d = strlen(field) + 1;
2545
2546         return strndup((const char*) data + d, size - d);
2547 }
2548
2549 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2550         const void *data;
2551         size_t size;
2552         sd_id128_t id;
2553         _cleanup_free_ char *text = NULL, *cid = NULL;
2554         char *t;
2555         int r;
2556
2557         assert_return(j, -EINVAL);
2558         assert_return(!journal_pid_changed(j), -ECHILD);
2559         assert_return(ret, -EINVAL);
2560
2561         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2562         if (r < 0)
2563                 return r;
2564
2565         cid = strndup((const char*) data + 11, size - 11);
2566         if (!cid)
2567                 return -ENOMEM;
2568
2569         r = sd_id128_from_string(cid, &id);
2570         if (r < 0)
2571                 return r;
2572
2573         r = catalog_get(CATALOG_DATABASE, id, &text);
2574         if (r < 0)
2575                 return r;
2576
2577         t = replace_var(text, lookup_field, j);
2578         if (!t)
2579                 return -ENOMEM;
2580
2581         *ret = t;
2582         return 0;
2583 }
2584
2585 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2586         assert_return(ret, -EINVAL);
2587
2588         return catalog_get(CATALOG_DATABASE, id, ret);
2589 }
2590
2591 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2592         assert_return(j, -EINVAL);
2593         assert_return(!journal_pid_changed(j), -ECHILD);
2594
2595         j->data_threshold = sz;
2596         return 0;
2597 }
2598
2599 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2600         assert_return(j, -EINVAL);
2601         assert_return(!journal_pid_changed(j), -ECHILD);
2602         assert_return(sz, -EINVAL);
2603
2604         *sz = j->data_threshold;
2605         return 0;
2606 }