src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "strv.h"
  37 #include "path-util.h"
  38 #include "lookup3.h"
  39 #include "compress.h"
  40 #include "journal-internal.h"
  41 #include "missing.h"
  42 #include "catalog.h"
  43 #include "replace-var.h"
  44
  45 #define JOURNAL_FILES_MAX 1024
  46
  47 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  48
  49 #define REPLACE_VAR_MAX 256
  50
  51 #define DEFAULT_DATA_THRESHOLD (64*1024)
  52
  53 /* We return an error here only if we didn't manage to
  54    memorize the real error. */
  55 static int set_put_error(sd_journal *j, int r) {
  56         int k;
  57
  58         if (r >= 0)
  59                 return r;
  60
  61         k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
  62         if (k < 0)
  63                 return k;
  64
  65         return set_put(j->errors, INT_TO_PTR(r));
  66 }
  67
  68 static void detach_location(sd_journal *j) {
  69         Iterator i;
  70         JournalFile *f;
  71
  72         assert(j);
  73
  74         j->current_file = NULL;
  75         j->current_field = 0;
  76
  77         HASHMAP_FOREACH(f, j->files, i)
  78                 f->current_offset = 0;
  79 }
  80
  81 static void reset_location(sd_journal *j) {
  82         assert(j);
  83
  84         detach_location(j);
  85         zero(j->current_location);
  86 }
  87
  88 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  89         assert(l);
  90         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
  91         assert(f);
  92         assert(o->object.type == OBJECT_ENTRY);
  93
  94         l->type = type;
  95         l->seqnum = le64toh(o->entry.seqnum);
  96         l->seqnum_id = f->header->seqnum_id;
  97         l->realtime = le64toh(o->entry.realtime);
  98         l->monotonic = le64toh(o->entry.monotonic);
  99         l->boot_id = o->entry.boot_id;
 100         l->xor_hash = le64toh(o->entry.xor_hash);
 101
 102         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 103 }
 104
 105 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o,
 106                          direction_t direction, uint64_t offset) {
 107         assert(j);
 108         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 109         assert(f);
 110         assert(o);
 111
 112         init_location(&j->current_location, type, f, o);
 113
 114         j->current_file = f;
 115         j->current_field = 0;
 116
 117         f->last_direction = direction;
 118         f->current_offset = offset;
 119 }
 120
 121 static int match_is_valid(const void *data, size_t size) {
 122         const char *b, *p;
 123
 124         assert(data);
 125
 126         if (size < 2)
 127                 return false;
 128
 129         if (startswith(data, "__"))
 130                 return false;
 131
 132         b = data;
 133         for (p = b; p < b + size; p++) {
 134
 135                 if (*p == '=')
 136                         return p > b;
 137
 138                 if (*p == '_')
 139                         continue;
 140
 141                 if (*p >= 'A' && *p <= 'Z')
 142                         continue;
 143
 144                 if (*p >= '0' && *p <= '9')
 145                         continue;
 146
 147                 return false;
 148         }
 149
 150         return false;
 151 }
 152
 153 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 154         const uint8_t *a = _a, *b = _b;
 155         size_t j;
 156
 157         for (j = 0; j < s && j < t; j++) {
 158
 159                 if (a[j] != b[j])
 160                         return false;
 161
 162                 if (a[j] == '=')
 163                         return true;
 164         }
 165
 166         assert_not_reached("\"=\" not found");
 167 }
 168
 169 static Match *match_new(Match *p, MatchType t) {
 170         Match *m;
 171
 172         m = new0(Match, 1);
 173         if (!m)
 174                 return NULL;
 175
 176         m->type = t;
 177
 178         if (p) {
 179                 m->parent = p;
 180                 LIST_PREPEND(Match, matches, p->matches, m);
 181         }
 182
 183         return m;
 184 }
 185
 186 static void match_free(Match *m) {
 187         assert(m);
 188
 189         while (m->matches)
 190                 match_free(m->matches);
 191
 192         if (m->parent)
 193                 LIST_REMOVE(Match, matches, m->parent->matches, m);
 194
 195         free(m->data);
 196         free(m);
 197 }
 198
 199 static void match_free_if_empty(Match *m) {
 200         if (!m || m->matches)
 201                 return;
 202
 203         match_free(m);
 204 }
 205
 206 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 207         Match *l3, *l4, *add_here = NULL, *m;
 208         le64_t le_hash;
 209
 210         if (!j)
 211                 return -EINVAL;
 212
 213         if (!data)
 214                 return -EINVAL;
 215
 216         if (size == 0)
 217                 size = strlen(data);
 218
 219         if (!match_is_valid(data, size))
 220                 return -EINVAL;
 221
 222         /* level 0: AND term
 223          * level 1: OR terms
 224          * level 2: AND terms
 225          * level 3: OR terms
 226          * level 4: concrete matches */
 227
 228         if (!j->level0) {
 229                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 230                 if (!j->level0)
 231                         return -ENOMEM;
 232         }
 233
 234         if (!j->level1) {
 235                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 236                 if (!j->level1)
 237                         return -ENOMEM;
 238         }
 239
 240         if (!j->level2) {
 241                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 242                 if (!j->level2)
 243                         return -ENOMEM;
 244         }
 245
 246         assert(j->level0->type == MATCH_AND_TERM);
 247         assert(j->level1->type == MATCH_OR_TERM);
 248         assert(j->level2->type == MATCH_AND_TERM);
 249
 250         le_hash = htole64(hash64(data, size));
 251
 252         LIST_FOREACH(matches, l3, j->level2->matches) {
 253                 assert(l3->type == MATCH_OR_TERM);
 254
 255                 LIST_FOREACH(matches, l4, l3->matches) {
 256                         assert(l4->type == MATCH_DISCRETE);
 257
 258                         /* Exactly the same match already? Then ignore
 259                          * this addition */
 260                         if (l4->le_hash == le_hash &&
 261                             l4->size == size &&
 262                             memcmp(l4->data, data, size) == 0)
 263                                 return 0;
 264
 265                         /* Same field? Then let's add this to this OR term */
 266                         if (same_field(data, size, l4->data, l4->size)) {
 267                                 add_here = l3;
 268                                 break;
 269                         }
 270                 }
 271
 272                 if (add_here)
 273                         break;
 274         }
 275
 276         if (!add_here) {
 277                 add_here = match_new(j->level2, MATCH_OR_TERM);
 278                 if (!add_here)
 279                         goto fail;
 280         }
 281
 282         m = match_new(add_here, MATCH_DISCRETE);
 283         if (!m)
 284                 goto fail;
 285
 286         m->le_hash = le_hash;
 287         m->size = size;
 288         m->data = memdup(data, size);
 289         if (!m->data)
 290                 goto fail;
 291
 292         detach_location(j);
 293
 294         return 0;
 295
 296 fail:
 297         match_free_if_empty(add_here);
 298         match_free_if_empty(j->level2);
 299         match_free_if_empty(j->level1);
 300         match_free_if_empty(j->level0);
 301
 302         return -ENOMEM;
 303 }
 304
 305 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 306         assert(j);
 307
 308         if (!j->level0)
 309                 return 0;
 310
 311         if (!j->level1)
 312                 return 0;
 313
 314         if (!j->level1->matches)
 315                 return 0;
 316
 317         j->level1 = NULL;
 318         j->level2 = NULL;
 319
 320         return 0;
 321 }
 322
 323 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 324         assert(j);
 325
 326         if (!j->level0)
 327                 return 0;
 328
 329         if (!j->level1)
 330                 return 0;
 331
 332         if (!j->level2)
 333                 return 0;
 334
 335         if (!j->level2->matches)
 336                 return 0;
 337
 338         j->level2 = NULL;
 339         return 0;
 340 }
 341
 342 static char *match_make_string(Match *m) {
 343         char *p, *r;
 344         Match *i;
 345         bool enclose = false;
 346
 347         if (!m)
 348                 return strdup("");
 349
 350         if (m->type == MATCH_DISCRETE)
 351                 return strndup(m->data, m->size);
 352
 353         p = NULL;
 354         LIST_FOREACH(matches, i, m->matches) {
 355                 char *t, *k;
 356
 357                 t = match_make_string(i);
 358                 if (!t) {
 359                         free(p);
 360                         return NULL;
 361                 }
 362
 363                 if (p) {
 364                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 365                         free(p);
 366                         free(t);
 367
 368                         if (!k)
 369                                 return NULL;
 370
 371                         p = k;
 372
 373                         enclose = true;
 374                 } else
 375                         p = t;
 376         }
 377
 378         if (enclose) {
 379                 r = strjoin("(", p, ")", NULL);
 380                 free(p);
 381                 return r;
 382         }
 383
 384         return p;
 385 }
 386
 387 char *journal_make_match_string(sd_journal *j) {
 388         assert(j);
 389
 390         return match_make_string(j->level0);
 391 }
 392
 393 _public_ void sd_journal_flush_matches(sd_journal *j) {
 394
 395         if (!j)
 396                 return;
 397
 398         if (j->level0)
 399                 match_free(j->level0);
 400
 401         j->level0 = j->level1 = j->level2 = NULL;
 402
 403         detach_location(j);
 404 }
 405
 406 static int compare_entry_order(JournalFile *af, Object *_ao,
 407                                JournalFile *bf, uint64_t bp) {
 408
 409         uint64_t a, b;
 410         Object *ao, *bo;
 411         int r;
 412
 413         assert(af);
 414         assert(bf);
 415         assert(_ao);
 416
 417         /* The mmap cache might invalidate the object from the first
 418          * file if we look at the one from the second file. Hence
 419          * temporarily copy the header of the first one, and look at
 420          * that only. */
 421         ao = alloca(offsetof(EntryObject, items));
 422         memcpy(ao, _ao, offsetof(EntryObject, items));
 423
 424         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 425         if (r < 0)
 426                 return strcmp(af->path, bf->path);
 427
 428         /* We operate on two different files here, hence we can access
 429          * two objects at the same time, which we normally can't.
 430          *
 431          * If contents and timestamps match, these entries are
 432          * identical, even if the seqnum does not match */
 433
 434         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 435             ao->entry.monotonic == bo->entry.monotonic &&
 436             ao->entry.realtime == bo->entry.realtime &&
 437             ao->entry.xor_hash == bo->entry.xor_hash)
 438                 return 0;
 439
 440         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 441
 442                 /* If this is from the same seqnum source, compare
 443                  * seqnums */
 444                 a = le64toh(ao->entry.seqnum);
 445                 b = le64toh(bo->entry.seqnum);
 446
 447                 if (a < b)
 448                         return -1;
 449                 if (a > b)
 450                         return 1;
 451
 452                 /* Wow! This is weird, different data but the same
 453                  * seqnums? Something is borked, but let's make the
 454                  * best of it and compare by time. */
 455         }
 456
 457         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 458
 459                 /* If the boot id matches compare monotonic time */
 460                 a = le64toh(ao->entry.monotonic);
 461                 b = le64toh(bo->entry.monotonic);
 462
 463                 if (a < b)
 464                         return -1;
 465                 if (a > b)
 466                         return 1;
 467         }
 468
 469         /* Otherwise compare UTC time */
 470         a = le64toh(ao->entry.realtime);
 471         b = le64toh(bo->entry.realtime);
 472
 473         if (a < b)
 474                 return -1;
 475         if (a > b)
 476                 return 1;
 477
 478         /* Finally, compare by contents */
 479         a = le64toh(ao->entry.xor_hash);
 480         b = le64toh(bo->entry.xor_hash);
 481
 482         if (a < b)
 483                 return -1;
 484         if (a > b)
 485                 return 1;
 486
 487         return 0;
 488 }
 489
 490 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 491         uint64_t a;
 492
 493         assert(af);
 494         assert(ao);
 495         assert(l);
 496         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 497
 498         if (l->monotonic_set &&
 499             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 500             l->realtime_set &&
 501             le64toh(ao->entry.realtime) == l->realtime &&
 502             l->xor_hash_set &&
 503             le64toh(ao->entry.xor_hash) == l->xor_hash)
 504                 return 0;
 505
 506         if (l->seqnum_set &&
 507             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 508
 509                 a = le64toh(ao->entry.seqnum);
 510
 511                 if (a < l->seqnum)
 512                         return -1;
 513                 if (a > l->seqnum)
 514                         return 1;
 515         }
 516
 517         if (l->monotonic_set &&
 518             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 519
 520                 a = le64toh(ao->entry.monotonic);
 521
 522                 if (a < l->monotonic)
 523                         return -1;
 524                 if (a > l->monotonic)
 525                         return 1;
 526         }
 527
 528         if (l->realtime_set) {
 529
 530                 a = le64toh(ao->entry.realtime);
 531
 532                 if (a < l->realtime)
 533                         return -1;
 534                 if (a > l->realtime)
 535                         return 1;
 536         }
 537
 538         if (l->xor_hash_set) {
 539                 a = le64toh(ao->entry.xor_hash);
 540
 541                 if (a < l->xor_hash)
 542                         return -1;
 543                 if (a > l->xor_hash)
 544                         return 1;
 545         }
 546
 547         return 0;
 548 }
 549
 550 static int next_for_match(
 551                 sd_journal *j,
 552                 Match *m,
 553                 JournalFile *f,
 554                 uint64_t after_offset,
 555                 direction_t direction,
 556                 Object **ret,
 557                 uint64_t *offset) {
 558
 559         int r;
 560         uint64_t np = 0;
 561         Object *n;
 562
 563         assert(j);
 564         assert(m);
 565         assert(f);
 566
 567         if (m->type == MATCH_DISCRETE) {
 568                 uint64_t dp;
 569
 570                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 571                 if (r <= 0)
 572                         return r;
 573
 574                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 575
 576         } else if (m->type == MATCH_OR_TERM) {
 577                 Match *i;
 578
 579                 /* Find the earliest match beyond after_offset */
 580
 581                 LIST_FOREACH(matches, i, m->matches) {
 582                         uint64_t cp;
 583
 584                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 585                         if (r < 0)
 586                                 return r;
 587                         else if (r > 0) {
 588                                 if (np == 0 || (direction == DIRECTION_DOWN ? cp < np : cp > np))
 589                                         np = cp;
 590                         }
 591                 }
 592
 593                 if (np == 0)
 594                         return 0;
 595
 596         } else if (m->type == MATCH_AND_TERM) {
 597                 Match *i, *last_moved;
 598
 599                 /* Always jump to the next matching entry and repeat
 600                  * this until we find an offset that matches for all
 601                  * matches. */
 602
 603                 if (!m->matches)
 604                         return 0;
 605
 606                 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
 607                 if (r <= 0)
 608                         return r;
 609
 610                 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
 611                 last_moved = m->matches;
 612
 613                 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
 614                         uint64_t cp;
 615
 616                         r = next_for_match(j, i, f, np, direction, NULL, &cp);
 617                         if (r <= 0)
 618                                 return r;
 619
 620                         assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
 621                         if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
 622                                 np = cp;
 623                                 last_moved = i;
 624                         }
 625                 }
 626         }
 627
 628         assert(np > 0);
 629
 630         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 631         if (r < 0)
 632                 return r;
 633
 634         if (ret)
 635                 *ret = n;
 636         if (offset)
 637                 *offset = np;
 638
 639         return 1;
 640 }
 641
 642 static int find_location_for_match(
 643                 sd_journal *j,
 644                 Match *m,
 645                 JournalFile *f,
 646                 direction_t direction,
 647                 Object **ret,
 648                 uint64_t *offset) {
 649
 650         int r;
 651
 652         assert(j);
 653         assert(m);
 654         assert(f);
 655
 656         if (m->type == MATCH_DISCRETE) {
 657                 uint64_t dp;
 658
 659                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 660                 if (r <= 0)
 661                         return r;
 662
 663                 /* FIXME: missing: find by monotonic */
 664
 665                 if (j->current_location.type == LOCATION_HEAD)
 666                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 667                 if (j->current_location.type == LOCATION_TAIL)
 668                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 669                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 670                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 671                 if (j->current_location.monotonic_set) {
 672                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 673                         if (r != -ENOENT)
 674                                 return r;
 675                 }
 676                 if (j->current_location.realtime_set)
 677                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 678
 679                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 680
 681         } else if (m->type == MATCH_OR_TERM) {
 682                 uint64_t np = 0;
 683                 Object *n;
 684                 Match *i;
 685
 686                 /* Find the earliest match */
 687
 688                 LIST_FOREACH(matches, i, m->matches) {
 689                         uint64_t cp;
 690
 691                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 692                         if (r < 0)
 693                                 return r;
 694                         else if (r > 0) {
 695                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 696                                         np = cp;
 697                         }
 698                 }
 699
 700                 if (np == 0)
 701                         return 0;
 702
 703                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 704                 if (r < 0)
 705                         return r;
 706
 707                 if (ret)
 708                         *ret = n;
 709                 if (offset)
 710                         *offset = np;
 711
 712                 return 1;
 713
 714         } else {
 715                 Match *i;
 716                 uint64_t np = 0;
 717
 718                 assert(m->type == MATCH_AND_TERM);
 719
 720                 /* First jump to the last match, and then find the
 721                  * next one where all matches match */
 722
 723                 if (!m->matches)
 724                         return 0;
 725
 726                 LIST_FOREACH(matches, i, m->matches) {
 727                         uint64_t cp;
 728
 729                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 730                         if (r <= 0)
 731                                 return r;
 732
 733                         if (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))
 734                                 np = cp;
 735                 }
 736
 737                 return next_for_match(j, m, f, np, direction, ret, offset);
 738         }
 739 }
 740
 741 static int find_location_with_matches(
 742                 sd_journal *j,
 743                 JournalFile *f,
 744                 direction_t direction,
 745                 Object **ret,
 746                 uint64_t *offset) {
 747
 748         int r;
 749
 750         assert(j);
 751         assert(f);
 752         assert(ret);
 753         assert(offset);
 754
 755         if (!j->level0) {
 756                 /* No matches is simple */
 757
 758                 if (j->current_location.type == LOCATION_HEAD)
 759                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 760                 if (j->current_location.type == LOCATION_TAIL)
 761                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 762                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 763                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 764                 if (j->current_location.monotonic_set) {
 765                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 766                         if (r != -ENOENT)
 767                                 return r;
 768                 }
 769                 if (j->current_location.realtime_set)
 770                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 771
 772                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 773         } else
 774                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 775 }
 776
 777 static int next_with_matches(
 778                 sd_journal *j,
 779                 JournalFile *f,
 780                 direction_t direction,
 781                 Object **ret,
 782                 uint64_t *offset) {
 783
 784         Object *c;
 785         uint64_t cp;
 786
 787         assert(j);
 788         assert(f);
 789         assert(ret);
 790         assert(offset);
 791
 792         c = *ret;
 793         cp = *offset;
 794
 795         /* No matches is easy. We simple advance the file
 796          * pointer by one. */
 797         if (!j->level0)
 798                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 799
 800         /* If we have a match then we look for the next matching entry
 801          * with an offset at least one step larger */
 802         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 803 }
 804
 805 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 806         Object *c;
 807         uint64_t cp;
 808         int r;
 809
 810         assert(j);
 811         assert(f);
 812
 813         if (f->last_direction == direction && f->current_offset > 0) {
 814                 cp = f->current_offset;
 815
 816                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 817                 if (r < 0)
 818                         return r;
 819
 820                 r = next_with_matches(j, f, direction, &c, &cp);
 821                 if (r <= 0)
 822                         return r;
 823         } else {
 824                 r = find_location_with_matches(j, f, direction, &c, &cp);
 825                 if (r <= 0)
 826                         return r;
 827         }
 828
 829         /* OK, we found the spot, now let's advance until an entry
 830          * that is actually different from what we were previously
 831          * looking at. This is necessary to handle entries which exist
 832          * in two (or more) journal files, and which shall all be
 833          * suppressed but one. */
 834
 835         for (;;) {
 836                 bool found;
 837
 838                 if (j->current_location.type == LOCATION_DISCRETE) {
 839                         int k;
 840
 841                         k = compare_with_location(f, c, &j->current_location);
 842                         if (direction == DIRECTION_DOWN)
 843                                 found = k > 0;
 844                         else
 845                                 found = k < 0;
 846                 } else
 847                         found = true;
 848
 849                 if (found) {
 850                         if (ret)
 851                                 *ret = c;
 852                         if (offset)
 853                                 *offset = cp;
 854                         return 1;
 855                 }
 856
 857                 r = next_with_matches(j, f, direction, &c, &cp);
 858                 if (r <= 0)
 859                         return r;
 860         }
 861 }
 862
 863 static int real_journal_next(sd_journal *j, direction_t direction) {
 864         JournalFile *f, *new_file = NULL;
 865         uint64_t new_offset = 0;
 866         Object *o;
 867         uint64_t p;
 868         Iterator i;
 869         int r;
 870
 871         if (!j)
 872                 return -EINVAL;
 873
 874         HASHMAP_FOREACH(f, j->files, i) {
 875                 bool found;
 876
 877                 r = next_beyond_location(j, f, direction, &o, &p);
 878                 if (r < 0) {
 879                         log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
 880                         continue;
 881                 } else if (r == 0)
 882                         continue;
 883
 884                 if (!new_file)
 885                         found = true;
 886                 else {
 887                         int k;
 888
 889                         k = compare_entry_order(f, o, new_file, new_offset);
 890
 891                         found = direction == DIRECTION_DOWN ? k < 0 : k > 0;
 892                 }
 893
 894                 if (found) {
 895                         new_file = f;
 896                         new_offset = p;
 897                 }
 898         }
 899
 900         if (!new_file)
 901                 return 0;
 902
 903         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 904         if (r < 0)
 905                 return r;
 906
 907         set_location(j, LOCATION_DISCRETE, new_file, o, direction, new_offset);
 908
 909         return 1;
 910 }
 911
 912 _public_ int sd_journal_next(sd_journal *j) {
 913         return real_journal_next(j, DIRECTION_DOWN);
 914 }
 915
 916 _public_ int sd_journal_previous(sd_journal *j) {
 917         return real_journal_next(j, DIRECTION_UP);
 918 }
 919
 920 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 921         int c = 0, r;
 922
 923         if (!j)
 924                 return -EINVAL;
 925
 926         if (skip == 0) {
 927                 /* If this is not a discrete skip, then at least
 928                  * resolve the current location */
 929                 if (j->current_location.type != LOCATION_DISCRETE)
 930                         return real_journal_next(j, direction);
 931
 932                 return 0;
 933         }
 934
 935         do {
 936                 r = real_journal_next(j, direction);
 937                 if (r < 0)
 938                         return r;
 939
 940                 if (r == 0)
 941                         return c;
 942
 943                 skip--;
 944                 c++;
 945         } while (skip > 0);
 946
 947         return c;
 948 }
 949
 950 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 951         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 952 }
 953
 954 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 955         return real_journal_next_skip(j, DIRECTION_UP, skip);
 956 }
 957
 958 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 959         Object *o;
 960         int r;
 961         char bid[33], sid[33];
 962
 963         if (!j)
 964                 return -EINVAL;
 965         if (!cursor)
 966                 return -EINVAL;
 967
 968         if (!j->current_file || j->current_file->current_offset <= 0)
 969                 return -EADDRNOTAVAIL;
 970
 971         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 972         if (r < 0)
 973                 return r;
 974
 975         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 976         sd_id128_to_string(o->entry.boot_id, bid);
 977
 978         if (asprintf(cursor,
 979                      "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
 980                      sid, le64toh(o->entry.seqnum),
 981                      bid, le64toh(o->entry.monotonic),
 982                      le64toh(o->entry.realtime),
 983                      le64toh(o->entry.xor_hash)) < 0)
 984                 return -ENOMEM;
 985
 986         return 0;
 987 }
 988
 989 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
 990         char *w, *state;
 991         size_t l;
 992         unsigned long long seqnum, monotonic, realtime, xor_hash;
 993         bool
 994                 seqnum_id_set = false,
 995                 seqnum_set = false,
 996                 boot_id_set = false,
 997                 monotonic_set = false,
 998                 realtime_set = false,
 999                 xor_hash_set = false;
1000         sd_id128_t seqnum_id, boot_id;
1001
1002         if (!j)
1003                 return -EINVAL;
1004         if (isempty(cursor))
1005                 return -EINVAL;
1006
1007         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1008                 char *item;
1009                 int k = 0;
1010
1011                 if (l < 2 || w[1] != '=')
1012                         return -EINVAL;
1013
1014                 item = strndup(w, l);
1015                 if (!item)
1016                         return -ENOMEM;
1017
1018                 switch (w[0]) {
1019
1020                 case 's':
1021                         seqnum_id_set = true;
1022                         k = sd_id128_from_string(item+2, &seqnum_id);
1023                         break;
1024
1025                 case 'i':
1026                         seqnum_set = true;
1027                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1028                                 k = -EINVAL;
1029                         break;
1030
1031                 case 'b':
1032                         boot_id_set = true;
1033                         k = sd_id128_from_string(item+2, &boot_id);
1034                         break;
1035
1036                 case 'm':
1037                         monotonic_set = true;
1038                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1039                                 k = -EINVAL;
1040                         break;
1041
1042                 case 't':
1043                         realtime_set = true;
1044                         if (sscanf(item+2, "%llx", &realtime) != 1)
1045                                 k = -EINVAL;
1046                         break;
1047
1048                 case 'x':
1049                         xor_hash_set = true;
1050                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1051                                 k = -EINVAL;
1052                         break;
1053                 }
1054
1055                 free(item);
1056
1057                 if (k < 0)
1058                         return k;
1059         }
1060
1061         if ((!seqnum_set || !seqnum_id_set) &&
1062             (!monotonic_set || !boot_id_set) &&
1063             !realtime_set)
1064                 return -EINVAL;
1065
1066         reset_location(j);
1067
1068         j->current_location.type = LOCATION_SEEK;
1069
1070         if (realtime_set) {
1071                 j->current_location.realtime = (uint64_t) realtime;
1072                 j->current_location.realtime_set = true;
1073         }
1074
1075         if (seqnum_set && seqnum_id_set) {
1076                 j->current_location.seqnum = (uint64_t) seqnum;
1077                 j->current_location.seqnum_id = seqnum_id;
1078                 j->current_location.seqnum_set = true;
1079         }
1080
1081         if (monotonic_set && boot_id_set) {
1082                 j->current_location.monotonic = (uint64_t) monotonic;
1083                 j->current_location.boot_id = boot_id;
1084                 j->current_location.monotonic_set = true;
1085         }
1086
1087         if (xor_hash_set) {
1088                 j->current_location.xor_hash = (uint64_t) xor_hash;
1089                 j->current_location.xor_hash_set = true;
1090         }
1091
1092         return 0;
1093 }
1094
1095 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1096         int r;
1097         char *w, *state;
1098         size_t l;
1099         Object *o;
1100
1101         if (!j)
1102                 return -EINVAL;
1103         if (isempty(cursor))
1104                 return -EINVAL;
1105
1106         if (!j->current_file || j->current_file->current_offset <= 0)
1107                 return -EADDRNOTAVAIL;
1108
1109         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1110         if (r < 0)
1111                 return r;
1112
1113         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1114                 _cleanup_free_ char *item = NULL;
1115                 sd_id128_t id;
1116                 unsigned long long ll;
1117                 int k = 0;
1118
1119                 if (l < 2 || w[1] != '=')
1120                         return -EINVAL;
1121
1122                 item = strndup(w, l);
1123                 if (!item)
1124                         return -ENOMEM;
1125
1126                 switch (w[0]) {
1127
1128                 case 's':
1129                         k = sd_id128_from_string(item+2, &id);
1130                         if (k < 0)
1131                                 return k;
1132                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1133                                 return 0;
1134                         break;
1135
1136                 case 'i':
1137                         if (sscanf(item+2, "%llx", &ll) != 1)
1138                                 return -EINVAL;
1139                         if (ll != le64toh(o->entry.seqnum))
1140                                 return 0;
1141                         break;
1142
1143                 case 'b':
1144                         k = sd_id128_from_string(item+2, &id);
1145                         if (k < 0)
1146                                 return k;
1147                         if (!sd_id128_equal(id, o->entry.boot_id))
1148                                 return 0;
1149                         break;
1150
1151                 case 'm':
1152                         if (sscanf(item+2, "%llx", &ll) != 1)
1153                                 return -EINVAL;
1154                         if (ll != le64toh(o->entry.monotonic))
1155                                 return 0;
1156                         break;
1157
1158                 case 't':
1159                         if (sscanf(item+2, "%llx", &ll) != 1)
1160                                 return -EINVAL;
1161                         if (ll != le64toh(o->entry.realtime))
1162                                 return 0;
1163                         break;
1164
1165                 case 'x':
1166                         if (sscanf(item+2, "%llx", &ll) != 1)
1167                                 return -EINVAL;
1168                         if (ll != le64toh(o->entry.xor_hash))
1169                                 return 0;
1170                         break;
1171                 }
1172         }
1173
1174         return 1;
1175 }
1176
1177
1178 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1179         if (!j)
1180                 return -EINVAL;
1181
1182         reset_location(j);
1183         j->current_location.type = LOCATION_SEEK;
1184         j->current_location.boot_id = boot_id;
1185         j->current_location.monotonic = usec;
1186         j->current_location.monotonic_set = true;
1187
1188         return 0;
1189 }
1190
1191 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1192         if (!j)
1193                 return -EINVAL;
1194
1195         reset_location(j);
1196         j->current_location.type = LOCATION_SEEK;
1197         j->current_location.realtime = usec;
1198         j->current_location.realtime_set = true;
1199
1200         return 0;
1201 }
1202
1203 _public_ int sd_journal_seek_head(sd_journal *j) {
1204         if (!j)
1205                 return -EINVAL;
1206
1207         reset_location(j);
1208         j->current_location.type = LOCATION_HEAD;
1209
1210         return 0;
1211 }
1212
1213 _public_ int sd_journal_seek_tail(sd_journal *j) {
1214         if (!j)
1215                 return -EINVAL;
1216
1217         reset_location(j);
1218         j->current_location.type = LOCATION_TAIL;
1219
1220         return 0;
1221 }
1222
1223 static void check_network(sd_journal *j, int fd) {
1224         struct statfs sfs;
1225
1226         assert(j);
1227
1228         if (j->on_network)
1229                 return;
1230
1231         if (fstatfs(fd, &sfs) < 0)
1232                 return;
1233
1234         j->on_network =
1235                 F_TYPE_CMP(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1236                 F_TYPE_CMP(sfs.f_type, CODA_SUPER_MAGIC) ||
1237                 F_TYPE_CMP(sfs.f_type, NCP_SUPER_MAGIC) ||
1238                 F_TYPE_CMP(sfs.f_type, NFS_SUPER_MAGIC) ||
1239                 F_TYPE_CMP(sfs.f_type, SMB_SUPER_MAGIC);
1240 }
1241
1242 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1243         const char *full, *tilded, *atted;
1244
1245         full = strappend(prefix, ".journal");
1246         tilded = strappenda(full, "~");
1247         atted = strappenda(prefix, "@");
1248
1249         return streq(filename, full) ||
1250                streq(filename, tilded) ||
1251                startswith(filename, atted);
1252 }
1253
1254 static bool file_type_wanted(int flags, const char *filename) {
1255         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1256                 return false;
1257
1258         /* no flags set → every type is OK */
1259         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1260                 return true;
1261
1262         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1263                 return true;
1264
1265         if (flags & SD_JOURNAL_CURRENT_USER) {
1266                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1267
1268                 assert_se(snprintf(prefix, sizeof(prefix), "user-%lu", (unsigned long) getuid())
1269                           < (int) sizeof(prefix));
1270
1271                 if (file_has_type_prefix(prefix, filename))
1272                         return true;
1273         }
1274
1275         return false;
1276 }
1277
1278 static int add_any_file(sd_journal *j, const char *path) {
1279         JournalFile *f;
1280         int r;
1281
1282         assert(j);
1283         assert(path);
1284
1285         if (hashmap_get(j->files, path))
1286                 return 0;
1287
1288         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1289                 log_warning("Too many open journal files, not adding %s.", path);
1290                 return set_put_error(j, -ETOOMANYREFS);
1291         }
1292
1293         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1294         if (r < 0)
1295                 return r;
1296
1297         /* journal_file_dump(f); */
1298
1299         r = hashmap_put(j->files, f->path, f);
1300         if (r < 0) {
1301                 journal_file_close(f);
1302                 return r;
1303         }
1304
1305         log_debug("File %s added.", f->path);
1306
1307         check_network(j, f->fd);
1308
1309         j->current_invalidate_counter ++;
1310
1311         return 0;
1312 }
1313
1314 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1315         _cleanup_free_ char *path = NULL;
1316         int r;
1317
1318         assert(j);
1319         assert(prefix);
1320         assert(filename);
1321
1322         if (j->no_new_files ||
1323             !file_type_wanted(j->flags, filename))
1324                 return 0;
1325
1326         path = strjoin(prefix, "/", filename, NULL);
1327         if (!path)
1328                 return -ENOMEM;
1329
1330         r = add_any_file(j, path);
1331         if (r == -ENOENT)
1332                 return 0;
1333         return 0;
1334 }
1335
1336 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1337         char *path;
1338         JournalFile *f;
1339
1340         assert(j);
1341         assert(prefix);
1342         assert(filename);
1343
1344         path = strjoin(prefix, "/", filename, NULL);
1345         if (!path)
1346                 return -ENOMEM;
1347
1348         f = hashmap_get(j->files, path);
1349         free(path);
1350         if (!f)
1351                 return 0;
1352
1353         hashmap_remove(j->files, f->path);
1354
1355         log_debug("File %s removed.", f->path);
1356
1357         if (j->current_file == f) {
1358                 j->current_file = NULL;
1359                 j->current_field = 0;
1360         }
1361
1362         if (j->unique_file == f) {
1363                 j->unique_file = NULL;
1364                 j->unique_offset = 0;
1365         }
1366
1367         journal_file_close(f);
1368
1369         j->current_invalidate_counter ++;
1370
1371         return 0;
1372 }
1373
1374 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1375         _cleanup_free_ char *path = NULL;
1376         int r;
1377         _cleanup_closedir_ DIR *d = NULL;
1378         sd_id128_t id, mid;
1379         Directory *m;
1380
1381         assert(j);
1382         assert(prefix);
1383         assert(dirname);
1384
1385         log_debug("Considering %s/%s.", prefix, dirname);
1386
1387         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1388             (sd_id128_from_string(dirname, &id) < 0 ||
1389              sd_id128_get_machine(&mid) < 0 ||
1390              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1391             return 0;
1392
1393         path = strjoin(prefix, "/", dirname, NULL);
1394         if (!path)
1395                 return -ENOMEM;
1396
1397         d = opendir(path);
1398         if (!d) {
1399                 log_debug("Failed to open %s: %m", path);
1400                 if (errno == ENOENT)
1401                         return 0;
1402                 return -errno;
1403         }
1404
1405         m = hashmap_get(j->directories_by_path, path);
1406         if (!m) {
1407                 m = new0(Directory, 1);
1408                 if (!m)
1409                         return -ENOMEM;
1410
1411                 m->is_root = false;
1412                 m->path = path;
1413
1414                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1415                         free(m);
1416                         return -ENOMEM;
1417                 }
1418
1419                 path = NULL; /* avoid freeing in cleanup */
1420                 j->current_invalidate_counter ++;
1421
1422                 log_debug("Directory %s added.", m->path);
1423
1424         } else if (m->is_root)
1425                 return 0;
1426
1427         if (m->wd <= 0 && j->inotify_fd >= 0) {
1428
1429                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1430                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1431                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1432                                           IN_ONLYDIR);
1433
1434                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1435                         inotify_rm_watch(j->inotify_fd, m->wd);
1436         }
1437
1438         for (;;) {
1439                 struct dirent *de;
1440                 union dirent_storage buf;
1441
1442                 r = readdir_r(d, &buf.de, &de);
1443                 if (r != 0 || !de)
1444                         break;
1445
1446                 if (dirent_is_file_with_suffix(de, ".journal") ||
1447                     dirent_is_file_with_suffix(de, ".journal~")) {
1448                         r = add_file(j, m->path, de->d_name);
1449                         if (r < 0) {
1450                                 log_debug("Failed to add file %s/%s: %s",
1451                                           m->path, de->d_name, strerror(-r));
1452                                 r = set_put_error(j, r);
1453                                 if (r < 0)
1454                                         return r;
1455                         }
1456                 }
1457         }
1458
1459         check_network(j, dirfd(d));
1460
1461         return 0;
1462 }
1463
1464 static int add_root_directory(sd_journal *j, const char *p) {
1465         _cleanup_closedir_ DIR *d = NULL;
1466         Directory *m;
1467         int r;
1468
1469         assert(j);
1470         assert(p);
1471
1472         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1473             !path_startswith(p, "/run"))
1474                 return -EINVAL;
1475
1476         d = opendir(p);
1477         if (!d)
1478                 return -errno;
1479
1480         m = hashmap_get(j->directories_by_path, p);
1481         if (!m) {
1482                 m = new0(Directory, 1);
1483                 if (!m)
1484                         return -ENOMEM;
1485
1486                 m->is_root = true;
1487                 m->path = strdup(p);
1488                 if (!m->path) {
1489                         free(m);
1490                         return -ENOMEM;
1491                 }
1492
1493                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1494                         free(m->path);
1495                         free(m);
1496                         return -ENOMEM;
1497                 }
1498
1499                 j->current_invalidate_counter ++;
1500
1501                 log_debug("Root directory %s added.", m->path);
1502
1503         } else if (!m->is_root)
1504                 return 0;
1505
1506         if (m->wd <= 0 && j->inotify_fd >= 0) {
1507
1508                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1509                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1510                                           IN_ONLYDIR);
1511
1512                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1513                         inotify_rm_watch(j->inotify_fd, m->wd);
1514         }
1515
1516         if (j->no_new_files)
1517                 return 0;
1518
1519         for (;;) {
1520                 struct dirent *de;
1521                 union dirent_storage buf;
1522                 sd_id128_t id;
1523
1524                 r = readdir_r(d, &buf.de, &de);
1525                 if (r != 0 || !de)
1526                         break;
1527
1528                 if (dirent_is_file_with_suffix(de, ".journal") ||
1529                     dirent_is_file_with_suffix(de, ".journal~")) {
1530                         r = add_file(j, m->path, de->d_name);
1531                         if (r < 0) {
1532                                 log_debug("Failed to add file %s/%s: %s",
1533                                           m->path, de->d_name, strerror(-r));
1534                                 r = set_put_error(j, r);
1535                                 if (r < 0)
1536                                         return r;
1537                         }
1538                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1539                            sd_id128_from_string(de->d_name, &id) >= 0) {
1540
1541                         r = add_directory(j, m->path, de->d_name);
1542                         if (r < 0)
1543                                 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1544                 }
1545         }
1546
1547         check_network(j, dirfd(d));
1548
1549         return 0;
1550 }
1551
1552 static int remove_directory(sd_journal *j, Directory *d) {
1553         assert(j);
1554
1555         if (d->wd > 0) {
1556                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1557
1558                 if (j->inotify_fd >= 0)
1559                         inotify_rm_watch(j->inotify_fd, d->wd);
1560         }
1561
1562         hashmap_remove(j->directories_by_path, d->path);
1563
1564         if (d->is_root)
1565                 log_debug("Root directory %s removed.", d->path);
1566         else
1567                 log_debug("Directory %s removed.", d->path);
1568
1569         free(d->path);
1570         free(d);
1571
1572         return 0;
1573 }
1574
1575 static int add_search_paths(sd_journal *j) {
1576         int r;
1577         const char search_paths[] =
1578                 "/run/log/journal\0"
1579                 "/var/log/journal\0";
1580         const char *p;
1581
1582         assert(j);
1583
1584         /* We ignore most errors here, since the idea is to only open
1585          * what's actually accessible, and ignore the rest. */
1586
1587         NULSTR_FOREACH(p, search_paths) {
1588                 r = add_root_directory(j, p);
1589                 if (r < 0 && r != -ENOENT) {
1590                         r = set_put_error(j, r);
1591                         if (r < 0)
1592                                 return r;
1593                 }
1594         }
1595
1596         return 0;
1597 }
1598
1599 static int add_current_paths(sd_journal *j) {
1600         Iterator i;
1601         JournalFile *f;
1602
1603         assert(j);
1604         assert(j->no_new_files);
1605
1606         /* Simply adds all directories for files we have open as
1607          * "root" directories. We don't expect errors here, so we
1608          * treat them as fatal. */
1609
1610         HASHMAP_FOREACH(f, j->files, i) {
1611                 int r;
1612                 _cleanup_free_ char *dir;
1613
1614                 dir = dirname_malloc(f->path);
1615                 if (!dir)
1616                         return -ENOMEM;
1617
1618                 r = add_root_directory(j, dir);
1619                 if (r < 0) {
1620                         set_put_error(j, r);
1621                         return r;
1622                 }
1623         }
1624
1625         return 0;
1626 }
1627
1628
1629 static int allocate_inotify(sd_journal *j) {
1630         assert(j);
1631
1632         if (j->inotify_fd < 0) {
1633                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1634                 if (j->inotify_fd < 0)
1635                         return -errno;
1636         }
1637
1638         if (!j->directories_by_wd) {
1639                 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1640                 if (!j->directories_by_wd)
1641                         return -ENOMEM;
1642         }
1643
1644         return 0;
1645 }
1646
1647 static sd_journal *journal_new(int flags, const char *path) {
1648         sd_journal *j;
1649
1650         j = new0(sd_journal, 1);
1651         if (!j)
1652                 return NULL;
1653
1654         j->inotify_fd = -1;
1655         j->flags = flags;
1656         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1657
1658         if (path) {
1659                 j->path = strdup(path);
1660                 if (!j->path)
1661                         goto fail;
1662         }
1663
1664         j->files = hashmap_new(string_hash_func, string_compare_func);
1665         j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1666         j->mmap = mmap_cache_new();
1667         if (!j->files || !j->directories_by_path || !j->mmap)
1668                 goto fail;
1669
1670         return j;
1671
1672 fail:
1673         sd_journal_close(j);
1674         return NULL;
1675 }
1676
1677 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1678         sd_journal *j;
1679         int r;
1680
1681         if (!ret)
1682                 return -EINVAL;
1683
1684         if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1685                       SD_JOURNAL_RUNTIME_ONLY|
1686                       SD_JOURNAL_SYSTEM|
1687                       SD_JOURNAL_CURRENT_USER))
1688                 return -EINVAL;
1689
1690         j = journal_new(flags, NULL);
1691         if (!j)
1692                 return -ENOMEM;
1693
1694         r = add_search_paths(j);
1695         if (r < 0)
1696                 goto fail;
1697
1698         *ret = j;
1699         return 0;
1700
1701 fail:
1702         sd_journal_close(j);
1703
1704         return r;
1705 }
1706
1707 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1708         sd_journal *j;
1709         int r;
1710
1711         if (!ret)
1712                 return -EINVAL;
1713
1714         if (!path)
1715                 return -EINVAL;
1716
1717         if (flags != 0)
1718                 return -EINVAL;
1719
1720         j = journal_new(flags, path);
1721         if (!j)
1722                 return -ENOMEM;
1723
1724         r = add_root_directory(j, path);
1725         if (r < 0) {
1726                 set_put_error(j, r);
1727                 goto fail;
1728         }
1729
1730         *ret = j;
1731         return 0;
1732
1733 fail:
1734         sd_journal_close(j);
1735
1736         return r;
1737 }
1738
1739 _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int flags) {
1740         sd_journal *j;
1741         const char **path;
1742         int r;
1743
1744         if (!ret)
1745                 return -EINVAL;
1746
1747         if (flags != 0)
1748                 return -EINVAL;
1749
1750         j = journal_new(flags, NULL);
1751         if (!j)
1752                 return -ENOMEM;
1753
1754         STRV_FOREACH(path, paths) {
1755                 r = add_any_file(j, *path);
1756                 if (r < 0) {
1757                         log_error("Failed to open %s: %s", *path, strerror(-r));
1758                         goto fail;
1759                 }
1760         }
1761
1762         j->no_new_files = true;
1763
1764         *ret = j;
1765         return 0;
1766
1767 fail:
1768         sd_journal_close(j);
1769
1770         return r;
1771 }
1772
1773 _public_ void sd_journal_close(sd_journal *j) {
1774         Directory *d;
1775         JournalFile *f;
1776
1777         if (!j)
1778                 return;
1779
1780         sd_journal_flush_matches(j);
1781
1782         while ((f = hashmap_steal_first(j->files)))
1783                 journal_file_close(f);
1784
1785         hashmap_free(j->files);
1786
1787         while ((d = hashmap_first(j->directories_by_path)))
1788                 remove_directory(j, d);
1789
1790         while ((d = hashmap_first(j->directories_by_wd)))
1791                 remove_directory(j, d);
1792
1793         hashmap_free(j->directories_by_path);
1794         hashmap_free(j->directories_by_wd);
1795
1796         if (j->inotify_fd >= 0)
1797                 close_nointr_nofail(j->inotify_fd);
1798
1799         if (j->mmap)
1800                 mmap_cache_unref(j->mmap);
1801
1802         free(j->path);
1803         free(j->unique_field);
1804         set_free(j->errors);
1805         free(j);
1806 }
1807
1808 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1809         Object *o;
1810         JournalFile *f;
1811         int r;
1812
1813         if (!j)
1814                 return -EINVAL;
1815         if (!ret)
1816                 return -EINVAL;
1817
1818         f = j->current_file;
1819         if (!f)
1820                 return -EADDRNOTAVAIL;
1821
1822         if (f->current_offset <= 0)
1823                 return -EADDRNOTAVAIL;
1824
1825         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1826         if (r < 0)
1827                 return r;
1828
1829         *ret = le64toh(o->entry.realtime);
1830         return 0;
1831 }
1832
1833 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1834         Object *o;
1835         JournalFile *f;
1836         int r;
1837         sd_id128_t id;
1838
1839         if (!j)
1840                 return -EINVAL;
1841
1842         f = j->current_file;
1843         if (!f)
1844                 return -EADDRNOTAVAIL;
1845
1846         if (f->current_offset <= 0)
1847                 return -EADDRNOTAVAIL;
1848
1849         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1850         if (r < 0)
1851                 return r;
1852
1853         if (ret_boot_id)
1854                 *ret_boot_id = o->entry.boot_id;
1855         else {
1856                 r = sd_id128_get_boot(&id);
1857                 if (r < 0)
1858                         return r;
1859
1860                 if (!sd_id128_equal(id, o->entry.boot_id))
1861                         return -ESTALE;
1862         }
1863
1864         if (ret)
1865                 *ret = le64toh(o->entry.monotonic);
1866
1867         return 0;
1868 }
1869
1870 static bool field_is_valid(const char *field) {
1871         const char *p;
1872
1873         assert(field);
1874
1875         if (isempty(field))
1876                 return false;
1877
1878         if (startswith(field, "__"))
1879                 return false;
1880
1881         for (p = field; *p; p++) {
1882
1883                 if (*p == '_')
1884                         continue;
1885
1886                 if (*p >= 'A' && *p <= 'Z')
1887                         continue;
1888
1889                 if (*p >= '0' && *p <= '9')
1890                         continue;
1891
1892                 return false;
1893         }
1894
1895         return true;
1896 }
1897
1898 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1899         JournalFile *f;
1900         uint64_t i, n;
1901         size_t field_length;
1902         int r;
1903         Object *o;
1904
1905         if (!j)
1906                 return -EINVAL;
1907         if (!field)
1908                 return -EINVAL;
1909         if (!data)
1910                 return -EINVAL;
1911         if (!size)
1912                 return -EINVAL;
1913
1914         if (!field_is_valid(field))
1915                 return -EINVAL;
1916
1917         f = j->current_file;
1918         if (!f)
1919                 return -EADDRNOTAVAIL;
1920
1921         if (f->current_offset <= 0)
1922                 return -EADDRNOTAVAIL;
1923
1924         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1925         if (r < 0)
1926                 return r;
1927
1928         field_length = strlen(field);
1929
1930         n = journal_file_entry_n_items(o);
1931         for (i = 0; i < n; i++) {
1932                 uint64_t p, l;
1933                 le64_t le_hash;
1934                 size_t t;
1935
1936                 p = le64toh(o->entry.items[i].object_offset);
1937                 le_hash = o->entry.items[i].hash;
1938                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1939                 if (r < 0)
1940                         return r;
1941
1942                 if (le_hash != o->data.hash)
1943                         return -EBADMSG;
1944
1945                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1946
1947                 if (o->object.flags & OBJECT_COMPRESSED) {
1948
1949 #ifdef HAVE_XZ
1950                         if (uncompress_startswith(o->data.payload, l,
1951                                                   &f->compress_buffer, &f->compress_buffer_size,
1952                                                   field, field_length, '=')) {
1953
1954                                 uint64_t rsize;
1955
1956                                 if (!uncompress_blob(o->data.payload, l,
1957                                                      &f->compress_buffer, &f->compress_buffer_size, &rsize,
1958                                                      j->data_threshold))
1959                                         return -EBADMSG;
1960
1961                                 *data = f->compress_buffer;
1962                                 *size = (size_t) rsize;
1963
1964                                 return 0;
1965                         }
1966 #else
1967                         return -EPROTONOSUPPORT;
1968 #endif
1969
1970                 } else if (l >= field_length+1 &&
1971                            memcmp(o->data.payload, field, field_length) == 0 &&
1972                            o->data.payload[field_length] == '=') {
1973
1974                         t = (size_t) l;
1975
1976                         if ((uint64_t) t != l)
1977                                 return -E2BIG;
1978
1979                         *data = o->data.payload;
1980                         *size = t;
1981
1982                         return 0;
1983                 }
1984
1985                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1986                 if (r < 0)
1987                         return r;
1988         }
1989
1990         return -ENOENT;
1991 }
1992
1993 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1994         size_t t;
1995         uint64_t l;
1996
1997         l = le64toh(o->object.size) - offsetof(Object, data.payload);
1998         t = (size_t) l;
1999
2000         /* We can't read objects larger than 4G on a 32bit machine */
2001         if ((uint64_t) t != l)
2002                 return -E2BIG;
2003
2004         if (o->object.flags & OBJECT_COMPRESSED) {
2005 #ifdef HAVE_XZ
2006                 uint64_t rsize;
2007
2008                 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
2009                         return -EBADMSG;
2010
2011                 *data = f->compress_buffer;
2012                 *size = (size_t) rsize;
2013 #else
2014                 return -EPROTONOSUPPORT;
2015 #endif
2016         } else {
2017                 *data = o->data.payload;
2018                 *size = t;
2019         }
2020
2021         return 0;
2022 }
2023
2024 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
2025         JournalFile *f;
2026         uint64_t p, n;
2027         le64_t le_hash;
2028         int r;
2029         Object *o;
2030
2031         if (!j)
2032                 return -EINVAL;
2033         if (!data)
2034                 return -EINVAL;
2035         if (!size)
2036                 return -EINVAL;
2037
2038         f = j->current_file;
2039         if (!f)
2040                 return -EADDRNOTAVAIL;
2041
2042         if (f->current_offset <= 0)
2043                 return -EADDRNOTAVAIL;
2044
2045         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2046         if (r < 0)
2047                 return r;
2048
2049         n = journal_file_entry_n_items(o);
2050         if (j->current_field >= n)
2051                 return 0;
2052
2053         p = le64toh(o->entry.items[j->current_field].object_offset);
2054         le_hash = o->entry.items[j->current_field].hash;
2055         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2056         if (r < 0)
2057                 return r;
2058
2059         if (le_hash != o->data.hash)
2060                 return -EBADMSG;
2061
2062         r = return_data(j, f, o, data, size);
2063         if (r < 0)
2064                 return r;
2065
2066         j->current_field ++;
2067
2068         return 1;
2069 }
2070
2071 _public_ void sd_journal_restart_data(sd_journal *j) {
2072         if (!j)
2073                 return;
2074
2075         j->current_field = 0;
2076 }
2077
2078 _public_ int sd_journal_get_fd(sd_journal *j) {
2079         int r;
2080
2081         if (!j)
2082                 return -EINVAL;
2083
2084         if (j->inotify_fd >= 0)
2085                 return j->inotify_fd;
2086
2087         r = allocate_inotify(j);
2088         if (r < 0)
2089                 return r;
2090
2091         /* Iterate through all dirs again, to add them to the
2092          * inotify */
2093         if (j->no_new_files)
2094                 r = add_current_paths(j);
2095         else if (j->path)
2096                 r = add_root_directory(j, j->path);
2097         else
2098                 r = add_search_paths(j);
2099         if (r < 0)
2100                 return r;
2101
2102         return j->inotify_fd;
2103 }
2104
2105 _public_ int sd_journal_get_events(sd_journal *j) {
2106         int fd;
2107
2108         if (!j)
2109                 return -EINVAL;
2110
2111         fd = sd_journal_get_fd(j);
2112         if (fd < 0)
2113                 return fd;
2114
2115         return POLLIN;
2116 }
2117
2118 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2119         int fd;
2120
2121         if (!j)
2122                 return -EINVAL;
2123         if (!timeout_usec)
2124                 return -EINVAL;
2125
2126         fd = sd_journal_get_fd(j);
2127         if (fd < 0)
2128                 return fd;
2129
2130         if (!j->on_network) {
2131                 *timeout_usec = (uint64_t) -1;
2132                 return 0;
2133         }
2134
2135         /* If we are on the network we need to regularly check for
2136          * changes manually */
2137
2138         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2139         return 1;
2140 }
2141
2142 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2143         Directory *d;
2144         int r;
2145
2146         assert(j);
2147         assert(e);
2148
2149         /* Is this a subdirectory we watch? */
2150         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2151         if (d) {
2152                 sd_id128_t id;
2153
2154                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2155                     (endswith(e->name, ".journal") ||
2156                      endswith(e->name, ".journal~"))) {
2157
2158                         /* Event for a journal file */
2159
2160                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2161                                 r = add_file(j, d->path, e->name);
2162                                 if (r < 0) {
2163                                         log_debug("Failed to add file %s/%s: %s",
2164                                                   d->path, e->name, strerror(-r));
2165                                         set_put_error(j, r);
2166                                 }
2167
2168                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2169
2170                                 r = remove_file(j, d->path, e->name);
2171                                 if (r < 0)
2172                                         log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2173                         }
2174
2175                 } else if (!d->is_root && e->len == 0) {
2176
2177                         /* Event for a subdirectory */
2178
2179                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2180                                 r = remove_directory(j, d);
2181                                 if (r < 0)
2182                                         log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2183                         }
2184
2185
2186                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2187
2188                         /* Event for root directory */
2189
2190                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2191                                 r = add_directory(j, d->path, e->name);
2192                                 if (r < 0)
2193                                         log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2194                         }
2195                 }
2196
2197                 return;
2198         }
2199
2200         if (e->mask & IN_IGNORED)
2201                 return;
2202
2203         log_warning("Unknown inotify event.");
2204 }
2205
2206 static int determine_change(sd_journal *j) {
2207         bool b;
2208
2209         assert(j);
2210
2211         b = j->current_invalidate_counter != j->last_invalidate_counter;
2212         j->last_invalidate_counter = j->current_invalidate_counter;
2213
2214         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2215 }
2216
2217 _public_ int sd_journal_process(sd_journal *j) {
2218         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2219         bool got_something = false;
2220
2221         if (!j)
2222                 return -EINVAL;
2223
2224         j->last_process_usec = now(CLOCK_MONOTONIC);
2225
2226         for (;;) {
2227                 struct inotify_event *e;
2228                 ssize_t l;
2229
2230                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2231                 if (l < 0) {
2232                         if (errno == EAGAIN || errno == EINTR)
2233                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2234
2235                         return -errno;
2236                 }
2237
2238                 got_something = true;
2239
2240                 e = (struct inotify_event*) buffer;
2241                 while (l > 0) {
2242                         size_t step;
2243
2244                         process_inotify_event(j, e);
2245
2246                         step = sizeof(struct inotify_event) + e->len;
2247                         assert(step <= (size_t) l);
2248
2249                         e = (struct inotify_event*) ((uint8_t*) e + step);
2250                         l -= step;
2251                 }
2252         }
2253
2254         return determine_change(j);
2255 }
2256
2257 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2258         int r;
2259         uint64_t t;
2260
2261         assert(j);
2262
2263         if (j->inotify_fd < 0) {
2264
2265                 /* This is the first invocation, hence create the
2266                  * inotify watch */
2267                 r = sd_journal_get_fd(j);
2268                 if (r < 0)
2269                         return r;
2270
2271                 /* The journal might have changed since the context
2272                  * object was created and we weren't watching before,
2273                  * hence don't wait for anything, and return
2274                  * immediately. */
2275                 return determine_change(j);
2276         }
2277
2278         r = sd_journal_get_timeout(j, &t);
2279         if (r < 0)
2280                 return r;
2281
2282         if (t != (uint64_t) -1) {
2283                 usec_t n;
2284
2285                 n = now(CLOCK_MONOTONIC);
2286                 t = t > n ? t - n : 0;
2287
2288                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2289                         timeout_usec = t;
2290         }
2291
2292         do {
2293                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2294         } while (r == -EINTR);
2295
2296         if (r < 0)
2297                 return r;
2298
2299         return sd_journal_process(j);
2300 }
2301
2302 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2303         Iterator i;
2304         JournalFile *f;
2305         bool first = true;
2306         int r;
2307
2308         if (!j)
2309                 return -EINVAL;
2310         if (!from && !to)
2311                 return -EINVAL;
2312         if (from == to)
2313                 return -EINVAL;
2314
2315         HASHMAP_FOREACH(f, j->files, i) {
2316                 usec_t fr, t;
2317
2318                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2319                 if (r == -ENOENT)
2320                         continue;
2321                 if (r < 0)
2322                         return r;
2323                 if (r == 0)
2324                         continue;
2325
2326                 if (first) {
2327                         if (from)
2328                                 *from = fr;
2329                         if (to)
2330                                 *to = t;
2331                         first = false;
2332                 } else {
2333                         if (from)
2334                                 *from = MIN(fr, *from);
2335                         if (to)
2336                                 *to = MAX(t, *to);
2337                 }
2338         }
2339
2340         return first ? 0 : 1;
2341 }
2342
2343 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2344         Iterator i;
2345         JournalFile *f;
2346         bool first = true;
2347         int r;
2348
2349         if (!j)
2350                 return -EINVAL;
2351         if (!from && !to)
2352                 return -EINVAL;
2353         if (from == to)
2354                 return -EINVAL;
2355
2356         HASHMAP_FOREACH(f, j->files, i) {
2357                 usec_t fr, t;
2358
2359                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2360                 if (r == -ENOENT)
2361                         continue;
2362                 if (r < 0)
2363                         return r;
2364                 if (r == 0)
2365                         continue;
2366
2367                 if (first) {
2368                         if (from)
2369                                 *from = fr;
2370                         if (to)
2371                                 *to = t;
2372                         first = false;
2373                 } else {
2374                         if (from)
2375                                 *from = MIN(fr, *from);
2376                         if (to)
2377                                 *to = MAX(t, *to);
2378                 }
2379         }
2380
2381         return first ? 0 : 1;
2382 }
2383
2384 void journal_print_header(sd_journal *j) {
2385         Iterator i;
2386         JournalFile *f;
2387         bool newline = false;
2388
2389         assert(j);
2390
2391         HASHMAP_FOREACH(f, j->files, i) {
2392                 if (newline)
2393                         putchar('\n');
2394                 else
2395                         newline = true;
2396
2397                 journal_file_print_header(f);
2398         }
2399 }
2400
2401 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2402         Iterator i;
2403         JournalFile *f;
2404         uint64_t sum = 0;
2405
2406         if (!j)
2407                 return -EINVAL;
2408         if (!bytes)
2409                 return -EINVAL;
2410
2411         HASHMAP_FOREACH(f, j->files, i) {
2412                 struct stat st;
2413
2414                 if (fstat(f->fd, &st) < 0)
2415                         return -errno;
2416
2417                 sum += (uint64_t) st.st_blocks * 512ULL;
2418         }
2419
2420         *bytes = sum;
2421         return 0;
2422 }
2423
2424 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2425         char *f;
2426
2427         if (!j)
2428                 return -EINVAL;
2429         if (isempty(field))
2430                 return -EINVAL;
2431         if (!field_is_valid(field))
2432                 return -EINVAL;
2433
2434         f = strdup(field);
2435         if (!f)
2436                 return -ENOMEM;
2437
2438         free(j->unique_field);
2439         j->unique_field = f;
2440         j->unique_file = NULL;
2441         j->unique_offset = 0;
2442
2443         return 0;
2444 }
2445
2446 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2447         Object *o;
2448         size_t k;
2449         int r;
2450
2451         if (!j)
2452                 return -EINVAL;
2453         if (!data)
2454                 return -EINVAL;
2455         if (!l)
2456                 return -EINVAL;
2457         if (!j->unique_field)
2458                 return -EINVAL;
2459
2460         k = strlen(j->unique_field);
2461
2462         if (!j->unique_file) {
2463                 j->unique_file = hashmap_first(j->files);
2464                 if (!j->unique_file)
2465                         return 0;
2466                 j->unique_offset = 0;
2467         }
2468
2469         for (;;) {
2470                 JournalFile *of;
2471                 Iterator i;
2472                 const void *odata;
2473                 size_t ol;
2474                 bool found;
2475
2476                 /* Proceed to next data object in the field's linked list */
2477                 if (j->unique_offset == 0) {
2478                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2479                         if (r < 0)
2480                                 return r;
2481
2482                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2483                 } else {
2484                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2485                         if (r < 0)
2486                                 return r;
2487
2488                         j->unique_offset = le64toh(o->data.next_field_offset);
2489                 }
2490
2491                 /* We reached the end of the list? Then start again, with the next file */
2492                 if (j->unique_offset == 0) {
2493                         JournalFile *n;
2494
2495                         n = hashmap_next(j->files, j->unique_file->path);
2496                         if (!n)
2497                                 return 0;
2498
2499                         j->unique_file = n;
2500                         continue;
2501                 }
2502
2503                 /* We do not use the type context here, but 0 instead,
2504                  * so that we can look at this data object at the same
2505                  * time as one on another file */
2506                 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2507                 if (r < 0)
2508                         return r;
2509
2510                 /* Let's do the type check by hand, since we used 0 context above. */
2511                 if (o->object.type != OBJECT_DATA)
2512                         return -EBADMSG;
2513
2514                 r = return_data(j, j->unique_file, o, &odata, &ol);
2515                 if (r < 0)
2516                         return r;
2517
2518                 /* OK, now let's see if we already returned this data
2519                  * object by checking if it exists in the earlier
2520                  * traversed files. */
2521                 found = false;
2522                 HASHMAP_FOREACH(of, j->files, i) {
2523                         Object *oo;
2524                         uint64_t op;
2525
2526                         if (of == j->unique_file)
2527                                 break;
2528
2529                         /* Skip this file it didn't have any fields
2530                          * indexed */
2531                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2532                             le64toh(of->header->n_fields) <= 0)
2533                                 continue;
2534
2535                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2536                         if (r < 0)
2537                                 return r;
2538
2539                         if (r > 0)
2540                                 found = true;
2541                 }
2542
2543                 if (found)
2544                         continue;
2545
2546                 r = return_data(j, j->unique_file, o, data, l);
2547                 if (r < 0)
2548                         return r;
2549
2550                 return 1;
2551         }
2552 }
2553
2554 _public_ void sd_journal_restart_unique(sd_journal *j) {
2555         if (!j)
2556                 return;
2557
2558         j->unique_file = NULL;
2559         j->unique_offset = 0;
2560 }
2561
2562 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2563         if (!j)
2564                 return -EINVAL;
2565
2566         return !j->on_network;
2567 }
2568
2569 static char *lookup_field(const char *field, void *userdata) {
2570         sd_journal *j = userdata;
2571         const void *data;
2572         size_t size, d;
2573         int r;
2574
2575         assert(field);
2576         assert(j);
2577
2578         r = sd_journal_get_data(j, field, &data, &size);
2579         if (r < 0 ||
2580             size > REPLACE_VAR_MAX)
2581                 return strdup(field);
2582
2583         d = strlen(field) + 1;
2584
2585         return strndup((const char*) data + d, size - d);
2586 }
2587
2588 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2589         const void *data;
2590         size_t size;
2591         sd_id128_t id;
2592         _cleanup_free_ char *text = NULL, *cid = NULL;
2593         char *t;
2594         int r;
2595
2596         if (!j)
2597                 return -EINVAL;
2598         if (!ret)
2599                 return -EINVAL;
2600
2601         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2602         if (r < 0)
2603                 return r;
2604
2605         cid = strndup((const char*) data + 11, size - 11);
2606         if (!cid)
2607                 return -ENOMEM;
2608
2609         r = sd_id128_from_string(cid, &id);
2610         if (r < 0)
2611                 return r;
2612
2613         r = catalog_get(CATALOG_DATABASE, id, &text);
2614         if (r < 0)
2615                 return r;
2616
2617         t = replace_var(text, lookup_field, j);
2618         if (!t)
2619                 return -ENOMEM;
2620
2621         *ret = t;
2622         return 0;
2623 }
2624
2625 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2626         if (!ret)
2627                 return -EINVAL;
2628
2629         return catalog_get(CATALOG_DATABASE, id, ret);
2630 }
2631
2632 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2633         if (!j)
2634                 return -EINVAL;
2635
2636         j->data_threshold = sz;
2637         return 0;
2638 }
2639
2640 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2641         if (!j)
2642                 return -EINVAL;
2643         if (!sz)
2644                 return -EINVAL;
2645
2646         *sz = j->data_threshold;
2647         return 0;
2648 }