src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "strv.h"
  37 #include "path-util.h"
  38 #include "lookup3.h"
  39 #include "compress.h"
  40 #include "journal-internal.h"
  41 #include "missing.h"
  42 #include "catalog.h"
  43 #include "replace-var.h"
  44 #include "fileio.h"
  45
  46 #define JOURNAL_FILES_MAX 1024
  47
  48 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  49
  50 #define REPLACE_VAR_MAX 256
  51
  52 #define DEFAULT_DATA_THRESHOLD (64*1024)
  53
  54 static void remove_file_real(sd_journal *j, JournalFile *f);
  55
  56 static bool journal_pid_changed(sd_journal *j) {
  57         assert(j);
  58
  59         /* We don't support people creating a journal object and
  60          * keeping it around over a fork(). Let's complain. */
  61
  62         return j->original_pid != getpid();
  63 }
  64
  65 /* We return an error here only if we didn't manage to
  66    memorize the real error. */
  67 static int set_put_error(sd_journal *j, int r) {
  68         int k;
  69
  70         if (r >= 0)
  71                 return r;
  72
  73         k = set_ensure_allocated(&j->errors, NULL);
  74         if (k < 0)
  75                 return k;
  76
  77         return set_put(j->errors, INT_TO_PTR(r));
  78 }
  79
  80 static void detach_location(sd_journal *j) {
  81         Iterator i;
  82         JournalFile *f;
  83
  84         assert(j);
  85
  86         j->current_file = NULL;
  87         j->current_field = 0;
  88
  89         ORDERED_HASHMAP_FOREACH(f, j->files, i)
  90                 journal_file_reset_location(f);
  91 }
  92
  93 static void reset_location(sd_journal *j) {
  94         assert(j);
  95
  96         detach_location(j);
  97         zero(j->current_location);
  98 }
  99
 100 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
 101         assert(l);
 102         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 103         assert(f);
 104         assert(o->object.type == OBJECT_ENTRY);
 105
 106         l->type = type;
 107         l->seqnum = le64toh(o->entry.seqnum);
 108         l->seqnum_id = f->header->seqnum_id;
 109         l->realtime = le64toh(o->entry.realtime);
 110         l->monotonic = le64toh(o->entry.monotonic);
 111         l->boot_id = o->entry.boot_id;
 112         l->xor_hash = le64toh(o->entry.xor_hash);
 113
 114         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 115 }
 116
 117 static void set_location(sd_journal *j, JournalFile *f, Object *o) {
 118         assert(j);
 119         assert(f);
 120         assert(o);
 121
 122         init_location(&j->current_location, LOCATION_DISCRETE, f, o);
 123
 124         j->current_file = f;
 125         j->current_field = 0;
 126
 127         /* Let f know its candidate entry was picked. */
 128         assert(f->location_type == LOCATION_SEEK);
 129         f->location_type = LOCATION_DISCRETE;
 130 }
 131
 132 static int match_is_valid(const void *data, size_t size) {
 133         const char *b, *p;
 134
 135         assert(data);
 136
 137         if (size < 2)
 138                 return false;
 139
 140         if (startswith(data, "__"))
 141                 return false;
 142
 143         b = data;
 144         for (p = b; p < b + size; p++) {
 145
 146                 if (*p == '=')
 147                         return p > b;
 148
 149                 if (*p == '_')
 150                         continue;
 151
 152                 if (*p >= 'A' && *p <= 'Z')
 153                         continue;
 154
 155                 if (*p >= '0' && *p <= '9')
 156                         continue;
 157
 158                 return false;
 159         }
 160
 161         return false;
 162 }
 163
 164 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 165         const uint8_t *a = _a, *b = _b;
 166         size_t j;
 167
 168         for (j = 0; j < s && j < t; j++) {
 169
 170                 if (a[j] != b[j])
 171                         return false;
 172
 173                 if (a[j] == '=')
 174                         return true;
 175         }
 176
 177         assert_not_reached("\"=\" not found");
 178 }
 179
 180 static Match *match_new(Match *p, MatchType t) {
 181         Match *m;
 182
 183         m = new0(Match, 1);
 184         if (!m)
 185                 return NULL;
 186
 187         m->type = t;
 188
 189         if (p) {
 190                 m->parent = p;
 191                 LIST_PREPEND(matches, p->matches, m);
 192         }
 193
 194         return m;
 195 }
 196
 197 static void match_free(Match *m) {
 198         assert(m);
 199
 200         while (m->matches)
 201                 match_free(m->matches);
 202
 203         if (m->parent)
 204                 LIST_REMOVE(matches, m->parent->matches, m);
 205
 206         free(m->data);
 207         free(m);
 208 }
 209
 210 static void match_free_if_empty(Match *m) {
 211         if (!m || m->matches)
 212                 return;
 213
 214         match_free(m);
 215 }
 216
 217 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 218         Match *l3, *l4, *add_here = NULL, *m;
 219         le64_t le_hash;
 220
 221         assert_return(j, -EINVAL);
 222         assert_return(!journal_pid_changed(j), -ECHILD);
 223         assert_return(data, -EINVAL);
 224
 225         if (size == 0)
 226                 size = strlen(data);
 227
 228         assert_return(match_is_valid(data, size), -EINVAL);
 229
 230         /* level 0: AND term
 231          * level 1: OR terms
 232          * level 2: AND terms
 233          * level 3: OR terms
 234          * level 4: concrete matches */
 235
 236         if (!j->level0) {
 237                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 238                 if (!j->level0)
 239                         return -ENOMEM;
 240         }
 241
 242         if (!j->level1) {
 243                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 244                 if (!j->level1)
 245                         return -ENOMEM;
 246         }
 247
 248         if (!j->level2) {
 249                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 250                 if (!j->level2)
 251                         return -ENOMEM;
 252         }
 253
 254         assert(j->level0->type == MATCH_AND_TERM);
 255         assert(j->level1->type == MATCH_OR_TERM);
 256         assert(j->level2->type == MATCH_AND_TERM);
 257
 258         le_hash = htole64(hash64(data, size));
 259
 260         LIST_FOREACH(matches, l3, j->level2->matches) {
 261                 assert(l3->type == MATCH_OR_TERM);
 262
 263                 LIST_FOREACH(matches, l4, l3->matches) {
 264                         assert(l4->type == MATCH_DISCRETE);
 265
 266                         /* Exactly the same match already? Then ignore
 267                          * this addition */
 268                         if (l4->le_hash == le_hash &&
 269                             l4->size == size &&
 270                             memcmp(l4->data, data, size) == 0)
 271                                 return 0;
 272
 273                         /* Same field? Then let's add this to this OR term */
 274                         if (same_field(data, size, l4->data, l4->size)) {
 275                                 add_here = l3;
 276                                 break;
 277                         }
 278                 }
 279
 280                 if (add_here)
 281                         break;
 282         }
 283
 284         if (!add_here) {
 285                 add_here = match_new(j->level2, MATCH_OR_TERM);
 286                 if (!add_here)
 287                         goto fail;
 288         }
 289
 290         m = match_new(add_here, MATCH_DISCRETE);
 291         if (!m)
 292                 goto fail;
 293
 294         m->le_hash = le_hash;
 295         m->size = size;
 296         m->data = memdup(data, size);
 297         if (!m->data)
 298                 goto fail;
 299
 300         detach_location(j);
 301
 302         return 0;
 303
 304 fail:
 305         match_free_if_empty(add_here);
 306         match_free_if_empty(j->level2);
 307         match_free_if_empty(j->level1);
 308         match_free_if_empty(j->level0);
 309
 310         return -ENOMEM;
 311 }
 312
 313 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 314         assert_return(j, -EINVAL);
 315         assert_return(!journal_pid_changed(j), -ECHILD);
 316
 317         if (!j->level0)
 318                 return 0;
 319
 320         if (!j->level1)
 321                 return 0;
 322
 323         if (!j->level1->matches)
 324                 return 0;
 325
 326         j->level1 = NULL;
 327         j->level2 = NULL;
 328
 329         return 0;
 330 }
 331
 332 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 333         assert_return(j, -EINVAL);
 334         assert_return(!journal_pid_changed(j), -ECHILD);
 335
 336         if (!j->level0)
 337                 return 0;
 338
 339         if (!j->level1)
 340                 return 0;
 341
 342         if (!j->level2)
 343                 return 0;
 344
 345         if (!j->level2->matches)
 346                 return 0;
 347
 348         j->level2 = NULL;
 349         return 0;
 350 }
 351
 352 static char *match_make_string(Match *m) {
 353         char *p, *r;
 354         Match *i;
 355         bool enclose = false;
 356
 357         if (!m)
 358                 return strdup("none");
 359
 360         if (m->type == MATCH_DISCRETE)
 361                 return strndup(m->data, m->size);
 362
 363         p = NULL;
 364         LIST_FOREACH(matches, i, m->matches) {
 365                 char *t, *k;
 366
 367                 t = match_make_string(i);
 368                 if (!t) {
 369                         free(p);
 370                         return NULL;
 371                 }
 372
 373                 if (p) {
 374                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 375                         free(p);
 376                         free(t);
 377
 378                         if (!k)
 379                                 return NULL;
 380
 381                         p = k;
 382
 383                         enclose = true;
 384                 } else
 385                         p = t;
 386         }
 387
 388         if (enclose) {
 389                 r = strjoin("(", p, ")", NULL);
 390                 free(p);
 391                 return r;
 392         }
 393
 394         return p;
 395 }
 396
 397 char *journal_make_match_string(sd_journal *j) {
 398         assert(j);
 399
 400         return match_make_string(j->level0);
 401 }
 402
 403 _public_ void sd_journal_flush_matches(sd_journal *j) {
 404         if (!j)
 405                 return;
 406
 407         if (j->level0)
 408                 match_free(j->level0);
 409
 410         j->level0 = j->level1 = j->level2 = NULL;
 411
 412         detach_location(j);
 413 }
 414
 415 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 416         uint64_t a;
 417
 418         assert(af);
 419         assert(ao);
 420         assert(l);
 421         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 422
 423         if (l->monotonic_set &&
 424             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 425             l->realtime_set &&
 426             le64toh(ao->entry.realtime) == l->realtime &&
 427             l->xor_hash_set &&
 428             le64toh(ao->entry.xor_hash) == l->xor_hash)
 429                 return 0;
 430
 431         if (l->seqnum_set &&
 432             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 433
 434                 a = le64toh(ao->entry.seqnum);
 435
 436                 if (a < l->seqnum)
 437                         return -1;
 438                 if (a > l->seqnum)
 439                         return 1;
 440         }
 441
 442         if (l->monotonic_set &&
 443             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 444
 445                 a = le64toh(ao->entry.monotonic);
 446
 447                 if (a < l->monotonic)
 448                         return -1;
 449                 if (a > l->monotonic)
 450                         return 1;
 451         }
 452
 453         if (l->realtime_set) {
 454
 455                 a = le64toh(ao->entry.realtime);
 456
 457                 if (a < l->realtime)
 458                         return -1;
 459                 if (a > l->realtime)
 460                         return 1;
 461         }
 462
 463         if (l->xor_hash_set) {
 464                 a = le64toh(ao->entry.xor_hash);
 465
 466                 if (a < l->xor_hash)
 467                         return -1;
 468                 if (a > l->xor_hash)
 469                         return 1;
 470         }
 471
 472         return 0;
 473 }
 474
 475 static int next_for_match(
 476                 sd_journal *j,
 477                 Match *m,
 478                 JournalFile *f,
 479                 uint64_t after_offset,
 480                 direction_t direction,
 481                 Object **ret,
 482                 uint64_t *offset) {
 483
 484         int r;
 485         uint64_t np = 0;
 486         Object *n;
 487
 488         assert(j);
 489         assert(m);
 490         assert(f);
 491
 492         if (m->type == MATCH_DISCRETE) {
 493                 uint64_t dp;
 494
 495                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 496                 if (r <= 0)
 497                         return r;
 498
 499                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 500
 501         } else if (m->type == MATCH_OR_TERM) {
 502                 Match *i;
 503
 504                 /* Find the earliest match beyond after_offset */
 505
 506                 LIST_FOREACH(matches, i, m->matches) {
 507                         uint64_t cp;
 508
 509                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 510                         if (r < 0)
 511                                 return r;
 512                         else if (r > 0) {
 513                                 if (np == 0 || (direction == DIRECTION_DOWN ? cp < np : cp > np))
 514                                         np = cp;
 515                         }
 516                 }
 517
 518                 if (np == 0)
 519                         return 0;
 520
 521         } else if (m->type == MATCH_AND_TERM) {
 522                 Match *i, *last_moved;
 523
 524                 /* Always jump to the next matching entry and repeat
 525                  * this until we find an offset that matches for all
 526                  * matches. */
 527
 528                 if (!m->matches)
 529                         return 0;
 530
 531                 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
 532                 if (r <= 0)
 533                         return r;
 534
 535                 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
 536                 last_moved = m->matches;
 537
 538                 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
 539                         uint64_t cp;
 540
 541                         r = next_for_match(j, i, f, np, direction, NULL, &cp);
 542                         if (r <= 0)
 543                                 return r;
 544
 545                         assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
 546                         if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
 547                                 np = cp;
 548                                 last_moved = i;
 549                         }
 550                 }
 551         }
 552
 553         assert(np > 0);
 554
 555         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 556         if (r < 0)
 557                 return r;
 558
 559         if (ret)
 560                 *ret = n;
 561         if (offset)
 562                 *offset = np;
 563
 564         return 1;
 565 }
 566
 567 static int find_location_for_match(
 568                 sd_journal *j,
 569                 Match *m,
 570                 JournalFile *f,
 571                 direction_t direction,
 572                 Object **ret,
 573                 uint64_t *offset) {
 574
 575         int r;
 576
 577         assert(j);
 578         assert(m);
 579         assert(f);
 580
 581         if (m->type == MATCH_DISCRETE) {
 582                 uint64_t dp;
 583
 584                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 585                 if (r <= 0)
 586                         return r;
 587
 588                 /* FIXME: missing: find by monotonic */
 589
 590                 if (j->current_location.type == LOCATION_HEAD)
 591                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 592                 if (j->current_location.type == LOCATION_TAIL)
 593                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 594                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 595                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 596                 if (j->current_location.monotonic_set) {
 597                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 598                         if (r != -ENOENT)
 599                                 return r;
 600                 }
 601                 if (j->current_location.realtime_set)
 602                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 603
 604                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 605
 606         } else if (m->type == MATCH_OR_TERM) {
 607                 uint64_t np = 0;
 608                 Object *n;
 609                 Match *i;
 610
 611                 /* Find the earliest match */
 612
 613                 LIST_FOREACH(matches, i, m->matches) {
 614                         uint64_t cp;
 615
 616                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 617                         if (r < 0)
 618                                 return r;
 619                         else if (r > 0) {
 620                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 621                                         np = cp;
 622                         }
 623                 }
 624
 625                 if (np == 0)
 626                         return 0;
 627
 628                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 629                 if (r < 0)
 630                         return r;
 631
 632                 if (ret)
 633                         *ret = n;
 634                 if (offset)
 635                         *offset = np;
 636
 637                 return 1;
 638
 639         } else {
 640                 Match *i;
 641                 uint64_t np = 0;
 642
 643                 assert(m->type == MATCH_AND_TERM);
 644
 645                 /* First jump to the last match, and then find the
 646                  * next one where all matches match */
 647
 648                 if (!m->matches)
 649                         return 0;
 650
 651                 LIST_FOREACH(matches, i, m->matches) {
 652                         uint64_t cp;
 653
 654                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 655                         if (r <= 0)
 656                                 return r;
 657
 658                         if (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))
 659                                 np = cp;
 660                 }
 661
 662                 return next_for_match(j, m, f, np, direction, ret, offset);
 663         }
 664 }
 665
 666 static int find_location_with_matches(
 667                 sd_journal *j,
 668                 JournalFile *f,
 669                 direction_t direction,
 670                 Object **ret,
 671                 uint64_t *offset) {
 672
 673         int r;
 674
 675         assert(j);
 676         assert(f);
 677         assert(ret);
 678         assert(offset);
 679
 680         if (!j->level0) {
 681                 /* No matches is simple */
 682
 683                 if (j->current_location.type == LOCATION_HEAD)
 684                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 685                 if (j->current_location.type == LOCATION_TAIL)
 686                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 687                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 688                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 689                 if (j->current_location.monotonic_set) {
 690                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 691                         if (r != -ENOENT)
 692                                 return r;
 693                 }
 694                 if (j->current_location.realtime_set)
 695                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 696
 697                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 698         } else
 699                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 700 }
 701
 702 static int next_with_matches(
 703                 sd_journal *j,
 704                 JournalFile *f,
 705                 direction_t direction,
 706                 Object **ret,
 707                 uint64_t *offset) {
 708
 709         Object *c;
 710         uint64_t cp;
 711
 712         assert(j);
 713         assert(f);
 714         assert(ret);
 715         assert(offset);
 716
 717         c = *ret;
 718         cp = *offset;
 719
 720         /* No matches is easy. We simple advance the file
 721          * pointer by one. */
 722         if (!j->level0)
 723                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 724
 725         /* If we have a match then we look for the next matching entry
 726          * with an offset at least one step larger */
 727         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 728 }
 729
 730 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction) {
 731         Object *c;
 732         uint64_t cp;
 733         int r;
 734
 735         assert(j);
 736         assert(f);
 737
 738         if (f->last_direction == direction && f->current_offset > 0) {
 739                 cp = f->current_offset;
 740
 741                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 742                 if (r < 0)
 743                         return r;
 744
 745                 r = next_with_matches(j, f, direction, &c, &cp);
 746                 if (r <= 0)
 747                         return r;
 748         } else {
 749                 r = find_location_with_matches(j, f, direction, &c, &cp);
 750                 if (r <= 0)
 751                         return r;
 752         }
 753
 754         /* OK, we found the spot, now let's advance until an entry
 755          * that is actually different from what we were previously
 756          * looking at. This is necessary to handle entries which exist
 757          * in two (or more) journal files, and which shall all be
 758          * suppressed but one. */
 759
 760         for (;;) {
 761                 bool found;
 762
 763                 if (j->current_location.type == LOCATION_DISCRETE) {
 764                         int k;
 765
 766                         k = compare_with_location(f, c, &j->current_location);
 767
 768                         found = direction == DIRECTION_DOWN ? k > 0 : k < 0;
 769                 } else
 770                         found = true;
 771
 772                 if (found) {
 773                         journal_file_save_location(f, direction, c, cp);
 774                         return 1;
 775                 }
 776
 777                 r = next_with_matches(j, f, direction, &c, &cp);
 778                 if (r <= 0)
 779                         return r;
 780         }
 781 }
 782
 783 static int real_journal_next(sd_journal *j, direction_t direction) {
 784         JournalFile *f, *new_file = NULL;
 785         Iterator i;
 786         Object *o;
 787         int r;
 788
 789         assert_return(j, -EINVAL);
 790         assert_return(!journal_pid_changed(j), -ECHILD);
 791
 792         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
 793                 bool found;
 794
 795                 r = next_beyond_location(j, f, direction);
 796                 if (r < 0) {
 797                         log_debug_errno(r, "Can't iterate through %s, ignoring: %m", f->path);
 798                         remove_file_real(j, f);
 799                         continue;
 800                 } else if (r == 0) {
 801                         f->location_type = LOCATION_TAIL;
 802                         continue;
 803                 }
 804
 805                 if (!new_file)
 806                         found = true;
 807                 else {
 808                         int k;
 809
 810                         k = journal_file_compare_locations(f, new_file);
 811
 812                         found = direction == DIRECTION_DOWN ? k < 0 : k > 0;
 813                 }
 814
 815                 if (found)
 816                         new_file = f;
 817         }
 818
 819         if (!new_file)
 820                 return 0;
 821
 822         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_file->current_offset, &o);
 823         if (r < 0)
 824                 return r;
 825
 826         set_location(j, new_file, o);
 827
 828         return 1;
 829 }
 830
 831 _public_ int sd_journal_next(sd_journal *j) {
 832         return real_journal_next(j, DIRECTION_DOWN);
 833 }
 834
 835 _public_ int sd_journal_previous(sd_journal *j) {
 836         return real_journal_next(j, DIRECTION_UP);
 837 }
 838
 839 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 840         int c = 0, r;
 841
 842         assert_return(j, -EINVAL);
 843         assert_return(!journal_pid_changed(j), -ECHILD);
 844
 845         if (skip == 0) {
 846                 /* If this is not a discrete skip, then at least
 847                  * resolve the current location */
 848                 if (j->current_location.type != LOCATION_DISCRETE)
 849                         return real_journal_next(j, direction);
 850
 851                 return 0;
 852         }
 853
 854         do {
 855                 r = real_journal_next(j, direction);
 856                 if (r < 0)
 857                         return r;
 858
 859                 if (r == 0)
 860                         return c;
 861
 862                 skip--;
 863                 c++;
 864         } while (skip > 0);
 865
 866         return c;
 867 }
 868
 869 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 870         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 871 }
 872
 873 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 874         return real_journal_next_skip(j, DIRECTION_UP, skip);
 875 }
 876
 877 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 878         Object *o;
 879         int r;
 880         char bid[33], sid[33];
 881
 882         assert_return(j, -EINVAL);
 883         assert_return(!journal_pid_changed(j), -ECHILD);
 884         assert_return(cursor, -EINVAL);
 885
 886         if (!j->current_file || j->current_file->current_offset <= 0)
 887                 return -EADDRNOTAVAIL;
 888
 889         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 890         if (r < 0)
 891                 return r;
 892
 893         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 894         sd_id128_to_string(o->entry.boot_id, bid);
 895
 896         if (asprintf(cursor,
 897                      "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
 898                      sid, le64toh(o->entry.seqnum),
 899                      bid, le64toh(o->entry.monotonic),
 900                      le64toh(o->entry.realtime),
 901                      le64toh(o->entry.xor_hash)) < 0)
 902                 return -ENOMEM;
 903
 904         return 0;
 905 }
 906
 907 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
 908         const char *word, *state;
 909         size_t l;
 910         unsigned long long seqnum, monotonic, realtime, xor_hash;
 911         bool
 912                 seqnum_id_set = false,
 913                 seqnum_set = false,
 914                 boot_id_set = false,
 915                 monotonic_set = false,
 916                 realtime_set = false,
 917                 xor_hash_set = false;
 918         sd_id128_t seqnum_id, boot_id;
 919
 920         assert_return(j, -EINVAL);
 921         assert_return(!journal_pid_changed(j), -ECHILD);
 922         assert_return(!isempty(cursor), -EINVAL);
 923
 924         FOREACH_WORD_SEPARATOR(word, l, cursor, ";", state) {
 925                 char *item;
 926                 int k = 0;
 927
 928                 if (l < 2 || word[1] != '=')
 929                         return -EINVAL;
 930
 931                 item = strndup(word, l);
 932                 if (!item)
 933                         return -ENOMEM;
 934
 935                 switch (word[0]) {
 936
 937                 case 's':
 938                         seqnum_id_set = true;
 939                         k = sd_id128_from_string(item+2, &seqnum_id);
 940                         break;
 941
 942                 case 'i':
 943                         seqnum_set = true;
 944                         if (sscanf(item+2, "%llx", &seqnum) != 1)
 945                                 k = -EINVAL;
 946                         break;
 947
 948                 case 'b':
 949                         boot_id_set = true;
 950                         k = sd_id128_from_string(item+2, &boot_id);
 951                         break;
 952
 953                 case 'm':
 954                         monotonic_set = true;
 955                         if (sscanf(item+2, "%llx", &monotonic) != 1)
 956                                 k = -EINVAL;
 957                         break;
 958
 959                 case 't':
 960                         realtime_set = true;
 961                         if (sscanf(item+2, "%llx", &realtime) != 1)
 962                                 k = -EINVAL;
 963                         break;
 964
 965                 case 'x':
 966                         xor_hash_set = true;
 967                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
 968                                 k = -EINVAL;
 969                         break;
 970                 }
 971
 972                 free(item);
 973
 974                 if (k < 0)
 975                         return k;
 976         }
 977
 978         if ((!seqnum_set || !seqnum_id_set) &&
 979             (!monotonic_set || !boot_id_set) &&
 980             !realtime_set)
 981                 return -EINVAL;
 982
 983         reset_location(j);
 984
 985         j->current_location.type = LOCATION_SEEK;
 986
 987         if (realtime_set) {
 988                 j->current_location.realtime = (uint64_t) realtime;
 989                 j->current_location.realtime_set = true;
 990         }
 991
 992         if (seqnum_set && seqnum_id_set) {
 993                 j->current_location.seqnum = (uint64_t) seqnum;
 994                 j->current_location.seqnum_id = seqnum_id;
 995                 j->current_location.seqnum_set = true;
 996         }
 997
 998         if (monotonic_set && boot_id_set) {
 999                 j->current_location.monotonic = (uint64_t) monotonic;
1000                 j->current_location.boot_id = boot_id;
1001                 j->current_location.monotonic_set = true;
1002         }
1003
1004         if (xor_hash_set) {
1005                 j->current_location.xor_hash = (uint64_t) xor_hash;
1006                 j->current_location.xor_hash_set = true;
1007         }
1008
1009         return 0;
1010 }
1011
1012 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1013         int r;
1014         const char *word, *state;
1015         size_t l;
1016         Object *o;
1017
1018         assert_return(j, -EINVAL);
1019         assert_return(!journal_pid_changed(j), -ECHILD);
1020         assert_return(!isempty(cursor), -EINVAL);
1021
1022         if (!j->current_file || j->current_file->current_offset <= 0)
1023                 return -EADDRNOTAVAIL;
1024
1025         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1026         if (r < 0)
1027                 return r;
1028
1029         FOREACH_WORD_SEPARATOR(word, l, cursor, ";", state) {
1030                 _cleanup_free_ char *item = NULL;
1031                 sd_id128_t id;
1032                 unsigned long long ll;
1033                 int k = 0;
1034
1035                 if (l < 2 || word[1] != '=')
1036                         return -EINVAL;
1037
1038                 item = strndup(word, l);
1039                 if (!item)
1040                         return -ENOMEM;
1041
1042                 switch (word[0]) {
1043
1044                 case 's':
1045                         k = sd_id128_from_string(item+2, &id);
1046                         if (k < 0)
1047                                 return k;
1048                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1049                                 return 0;
1050                         break;
1051
1052                 case 'i':
1053                         if (sscanf(item+2, "%llx", &ll) != 1)
1054                                 return -EINVAL;
1055                         if (ll != le64toh(o->entry.seqnum))
1056                                 return 0;
1057                         break;
1058
1059                 case 'b':
1060                         k = sd_id128_from_string(item+2, &id);
1061                         if (k < 0)
1062                                 return k;
1063                         if (!sd_id128_equal(id, o->entry.boot_id))
1064                                 return 0;
1065                         break;
1066
1067                 case 'm':
1068                         if (sscanf(item+2, "%llx", &ll) != 1)
1069                                 return -EINVAL;
1070                         if (ll != le64toh(o->entry.monotonic))
1071                                 return 0;
1072                         break;
1073
1074                 case 't':
1075                         if (sscanf(item+2, "%llx", &ll) != 1)
1076                                 return -EINVAL;
1077                         if (ll != le64toh(o->entry.realtime))
1078                                 return 0;
1079                         break;
1080
1081                 case 'x':
1082                         if (sscanf(item+2, "%llx", &ll) != 1)
1083                                 return -EINVAL;
1084                         if (ll != le64toh(o->entry.xor_hash))
1085                                 return 0;
1086                         break;
1087                 }
1088         }
1089
1090         return 1;
1091 }
1092
1093
1094 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1095         assert_return(j, -EINVAL);
1096         assert_return(!journal_pid_changed(j), -ECHILD);
1097
1098         reset_location(j);
1099         j->current_location.type = LOCATION_SEEK;
1100         j->current_location.boot_id = boot_id;
1101         j->current_location.monotonic = usec;
1102         j->current_location.monotonic_set = true;
1103
1104         return 0;
1105 }
1106
1107 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1108         assert_return(j, -EINVAL);
1109         assert_return(!journal_pid_changed(j), -ECHILD);
1110
1111         reset_location(j);
1112         j->current_location.type = LOCATION_SEEK;
1113         j->current_location.realtime = usec;
1114         j->current_location.realtime_set = true;
1115
1116         return 0;
1117 }
1118
1119 _public_ int sd_journal_seek_head(sd_journal *j) {
1120         assert_return(j, -EINVAL);
1121         assert_return(!journal_pid_changed(j), -ECHILD);
1122
1123         reset_location(j);
1124         j->current_location.type = LOCATION_HEAD;
1125
1126         return 0;
1127 }
1128
1129 _public_ int sd_journal_seek_tail(sd_journal *j) {
1130         assert_return(j, -EINVAL);
1131         assert_return(!journal_pid_changed(j), -ECHILD);
1132
1133         reset_location(j);
1134         j->current_location.type = LOCATION_TAIL;
1135
1136         return 0;
1137 }
1138
1139 static void check_network(sd_journal *j, int fd) {
1140         struct statfs sfs;
1141
1142         assert(j);
1143
1144         if (j->on_network)
1145                 return;
1146
1147         if (fstatfs(fd, &sfs) < 0)
1148                 return;
1149
1150         j->on_network =
1151                 F_TYPE_EQUAL(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1152                 F_TYPE_EQUAL(sfs.f_type, CODA_SUPER_MAGIC) ||
1153                 F_TYPE_EQUAL(sfs.f_type, NCP_SUPER_MAGIC) ||
1154                 F_TYPE_EQUAL(sfs.f_type, NFS_SUPER_MAGIC) ||
1155                 F_TYPE_EQUAL(sfs.f_type, SMB_SUPER_MAGIC);
1156 }
1157
1158 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1159         const char *full, *tilded, *atted;
1160
1161         full = strappenda(prefix, ".journal");
1162         tilded = strappenda(full, "~");
1163         atted = strappenda(prefix, "@");
1164
1165         return streq(filename, full) ||
1166                streq(filename, tilded) ||
1167                startswith(filename, atted);
1168 }
1169
1170 static bool file_type_wanted(int flags, const char *filename) {
1171         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1172                 return false;
1173
1174         /* no flags set → every type is OK */
1175         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1176                 return true;
1177
1178         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1179                 return true;
1180
1181         if (flags & SD_JOURNAL_CURRENT_USER) {
1182                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1183
1184                 assert_se(snprintf(prefix, sizeof(prefix), "user-"UID_FMT, getuid())
1185                           < (int) sizeof(prefix));
1186
1187                 if (file_has_type_prefix(prefix, filename))
1188                         return true;
1189         }
1190
1191         return false;
1192 }
1193
1194 static int add_any_file(sd_journal *j, const char *path) {
1195         JournalFile *f = NULL;
1196         int r;
1197
1198         assert(j);
1199         assert(path);
1200
1201         if (ordered_hashmap_get(j->files, path))
1202                 return 0;
1203
1204         if (ordered_hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1205                 log_warning("Too many open journal files, not adding %s.", path);
1206                 return set_put_error(j, -ETOOMANYREFS);
1207         }
1208
1209         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1210         if (r < 0)
1211                 return r;
1212
1213         /* journal_file_dump(f); */
1214
1215         r = ordered_hashmap_put(j->files, f->path, f);
1216         if (r < 0) {
1217                 journal_file_close(f);
1218                 return r;
1219         }
1220
1221         log_debug("File %s added.", f->path);
1222
1223         check_network(j, f->fd);
1224
1225         j->current_invalidate_counter ++;
1226
1227         return 0;
1228 }
1229
1230 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1231         _cleanup_free_ char *path = NULL;
1232         int r;
1233
1234         assert(j);
1235         assert(prefix);
1236         assert(filename);
1237
1238         if (j->no_new_files ||
1239             !file_type_wanted(j->flags, filename))
1240                 return 0;
1241
1242         path = strjoin(prefix, "/", filename, NULL);
1243         if (!path)
1244                 return -ENOMEM;
1245
1246         r = add_any_file(j, path);
1247         if (r == -ENOENT)
1248                 return 0;
1249         return 0;
1250 }
1251
1252 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1253         _cleanup_free_ char *path;
1254         JournalFile *f;
1255
1256         assert(j);
1257         assert(prefix);
1258         assert(filename);
1259
1260         path = strjoin(prefix, "/", filename, NULL);
1261         if (!path)
1262                 return -ENOMEM;
1263
1264         f = ordered_hashmap_get(j->files, path);
1265         if (!f)
1266                 return 0;
1267
1268         remove_file_real(j, f);
1269         return 0;
1270 }
1271
1272 static void remove_file_real(sd_journal *j, JournalFile *f) {
1273         assert(j);
1274         assert(f);
1275
1276         ordered_hashmap_remove(j->files, f->path);
1277
1278         log_debug("File %s removed.", f->path);
1279
1280         if (j->current_file == f) {
1281                 j->current_file = NULL;
1282                 j->current_field = 0;
1283         }
1284
1285         if (j->unique_file == f) {
1286                 /* Jump to the next unique_file or NULL if that one was last */
1287                 j->unique_file = ordered_hashmap_next(j->files, j->unique_file->path);
1288                 j->unique_offset = 0;
1289                 if (!j->unique_file)
1290                         j->unique_file_lost = true;
1291         }
1292
1293         journal_file_close(f);
1294
1295         j->current_invalidate_counter ++;
1296 }
1297
1298 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1299         _cleanup_free_ char *path = NULL;
1300         int r;
1301         _cleanup_closedir_ DIR *d = NULL;
1302         sd_id128_t id, mid;
1303         Directory *m;
1304
1305         assert(j);
1306         assert(prefix);
1307         assert(dirname);
1308
1309         log_debug("Considering %s/%s.", prefix, dirname);
1310
1311         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1312             (sd_id128_from_string(dirname, &id) < 0 ||
1313              sd_id128_get_machine(&mid) < 0 ||
1314              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1315             return 0;
1316
1317         path = strjoin(prefix, "/", dirname, NULL);
1318         if (!path)
1319                 return -ENOMEM;
1320
1321         d = opendir(path);
1322         if (!d) {
1323                 log_debug_errno(errno, "Failed to open %s: %m", path);
1324                 if (errno == ENOENT)
1325                         return 0;
1326                 return -errno;
1327         }
1328
1329         m = hashmap_get(j->directories_by_path, path);
1330         if (!m) {
1331                 m = new0(Directory, 1);
1332                 if (!m)
1333                         return -ENOMEM;
1334
1335                 m->is_root = false;
1336                 m->path = path;
1337
1338                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1339                         free(m);
1340                         return -ENOMEM;
1341                 }
1342
1343                 path = NULL; /* avoid freeing in cleanup */
1344                 j->current_invalidate_counter ++;
1345
1346                 log_debug("Directory %s added.", m->path);
1347
1348         } else if (m->is_root)
1349                 return 0;
1350
1351         if (m->wd <= 0 && j->inotify_fd >= 0) {
1352
1353                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1354                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1355                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1356                                           IN_ONLYDIR);
1357
1358                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1359                         inotify_rm_watch(j->inotify_fd, m->wd);
1360         }
1361
1362         for (;;) {
1363                 struct dirent *de;
1364
1365                 errno = 0;
1366                 de = readdir(d);
1367                 if (!de && errno != 0) {
1368                         r = -errno;
1369                         log_debug_errno(errno, "Failed to read directory %s: %m", m->path);
1370                         return r;
1371                 }
1372                 if (!de)
1373                         break;
1374
1375                 if (dirent_is_file_with_suffix(de, ".journal") ||
1376                     dirent_is_file_with_suffix(de, ".journal~")) {
1377                         r = add_file(j, m->path, de->d_name);
1378                         if (r < 0) {
1379                                 log_debug_errno(r, "Failed to add file %s/%s: %m",
1380                                                 m->path, de->d_name);
1381                                 r = set_put_error(j, r);
1382                                 if (r < 0)
1383                                         return r;
1384                         }
1385                 }
1386         }
1387
1388         check_network(j, dirfd(d));
1389
1390         return 0;
1391 }
1392
1393 static int add_root_directory(sd_journal *j, const char *p) {
1394         _cleanup_closedir_ DIR *d = NULL;
1395         Directory *m;
1396         int r;
1397
1398         assert(j);
1399         assert(p);
1400
1401         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1402             !path_startswith(p, "/run"))
1403                 return -EINVAL;
1404
1405         if (j->prefix)
1406                 p = strappenda(j->prefix, p);
1407
1408         d = opendir(p);
1409         if (!d)
1410                 return -errno;
1411
1412         m = hashmap_get(j->directories_by_path, p);
1413         if (!m) {
1414                 m = new0(Directory, 1);
1415                 if (!m)
1416                         return -ENOMEM;
1417
1418                 m->is_root = true;
1419                 m->path = strdup(p);
1420                 if (!m->path) {
1421                         free(m);
1422                         return -ENOMEM;
1423                 }
1424
1425                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1426                         free(m->path);
1427                         free(m);
1428                         return -ENOMEM;
1429                 }
1430
1431                 j->current_invalidate_counter ++;
1432
1433                 log_debug("Root directory %s added.", m->path);
1434
1435         } else if (!m->is_root)
1436                 return 0;
1437
1438         if (m->wd <= 0 && j->inotify_fd >= 0) {
1439
1440                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1441                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1442                                           IN_ONLYDIR);
1443
1444                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1445                         inotify_rm_watch(j->inotify_fd, m->wd);
1446         }
1447
1448         if (j->no_new_files)
1449                 return 0;
1450
1451         for (;;) {
1452                 struct dirent *de;
1453                 sd_id128_t id;
1454
1455                 errno = 0;
1456                 de = readdir(d);
1457                 if (!de && errno != 0) {
1458                         r = -errno;
1459                         log_debug_errno(errno, "Failed to read directory %s: %m", m->path);
1460                         return r;
1461                 }
1462                 if (!de)
1463                         break;
1464
1465                 if (dirent_is_file_with_suffix(de, ".journal") ||
1466                     dirent_is_file_with_suffix(de, ".journal~")) {
1467                         r = add_file(j, m->path, de->d_name);
1468                         if (r < 0) {
1469                                 log_debug_errno(r, "Failed to add file %s/%s: %m",
1470                                                 m->path, de->d_name);
1471                                 r = set_put_error(j, r);
1472                                 if (r < 0)
1473                                         return r;
1474                         }
1475                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1476                            sd_id128_from_string(de->d_name, &id) >= 0) {
1477
1478                         r = add_directory(j, m->path, de->d_name);
1479                         if (r < 0)
1480                                 log_debug_errno(r, "Failed to add directory %s/%s: %m", m->path, de->d_name);
1481                 }
1482         }
1483
1484         check_network(j, dirfd(d));
1485
1486         return 0;
1487 }
1488
1489 static int remove_directory(sd_journal *j, Directory *d) {
1490         assert(j);
1491
1492         if (d->wd > 0) {
1493                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1494
1495                 if (j->inotify_fd >= 0)
1496                         inotify_rm_watch(j->inotify_fd, d->wd);
1497         }
1498
1499         hashmap_remove(j->directories_by_path, d->path);
1500
1501         if (d->is_root)
1502                 log_debug("Root directory %s removed.", d->path);
1503         else
1504                 log_debug("Directory %s removed.", d->path);
1505
1506         free(d->path);
1507         free(d);
1508
1509         return 0;
1510 }
1511
1512 static int add_search_paths(sd_journal *j) {
1513         int r;
1514         const char search_paths[] =
1515                 "/run/log/journal\0"
1516                 "/var/log/journal\0";
1517         const char *p;
1518
1519         assert(j);
1520
1521         /* We ignore most errors here, since the idea is to only open
1522          * what's actually accessible, and ignore the rest. */
1523
1524         NULSTR_FOREACH(p, search_paths) {
1525                 r = add_root_directory(j, p);
1526                 if (r < 0 && r != -ENOENT) {
1527                         r = set_put_error(j, r);
1528                         if (r < 0)
1529                                 return r;
1530                 }
1531         }
1532
1533         return 0;
1534 }
1535
1536 static int add_current_paths(sd_journal *j) {
1537         Iterator i;
1538         JournalFile *f;
1539
1540         assert(j);
1541         assert(j->no_new_files);
1542
1543         /* Simply adds all directories for files we have open as
1544          * "root" directories. We don't expect errors here, so we
1545          * treat them as fatal. */
1546
1547         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
1548                 _cleanup_free_ char *dir;
1549                 int r;
1550
1551                 dir = dirname_malloc(f->path);
1552                 if (!dir)
1553                         return -ENOMEM;
1554
1555                 r = add_root_directory(j, dir);
1556                 if (r < 0) {
1557                         set_put_error(j, r);
1558                         return r;
1559                 }
1560         }
1561
1562         return 0;
1563 }
1564
1565
1566 static int allocate_inotify(sd_journal *j) {
1567         assert(j);
1568
1569         if (j->inotify_fd < 0) {
1570                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1571                 if (j->inotify_fd < 0)
1572                         return -errno;
1573         }
1574
1575         if (!j->directories_by_wd) {
1576                 j->directories_by_wd = hashmap_new(NULL);
1577                 if (!j->directories_by_wd)
1578                         return -ENOMEM;
1579         }
1580
1581         return 0;
1582 }
1583
1584 static sd_journal *journal_new(int flags, const char *path) {
1585         sd_journal *j;
1586
1587         j = new0(sd_journal, 1);
1588         if (!j)
1589                 return NULL;
1590
1591         j->original_pid = getpid();
1592         j->inotify_fd = -1;
1593         j->flags = flags;
1594         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1595
1596         if (path) {
1597                 j->path = strdup(path);
1598                 if (!j->path)
1599                         goto fail;
1600         }
1601
1602         j->files = ordered_hashmap_new(&string_hash_ops);
1603         j->directories_by_path = hashmap_new(&string_hash_ops);
1604         j->mmap = mmap_cache_new();
1605         if (!j->files || !j->directories_by_path || !j->mmap)
1606                 goto fail;
1607
1608         return j;
1609
1610 fail:
1611         sd_journal_close(j);
1612         return NULL;
1613 }
1614
1615 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1616         sd_journal *j;
1617         int r;
1618
1619         assert_return(ret, -EINVAL);
1620         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_RUNTIME_ONLY|SD_JOURNAL_SYSTEM|SD_JOURNAL_CURRENT_USER)) == 0, -EINVAL);
1621
1622         j = journal_new(flags, NULL);
1623         if (!j)
1624                 return -ENOMEM;
1625
1626         r = add_search_paths(j);
1627         if (r < 0)
1628                 goto fail;
1629
1630         *ret = j;
1631         return 0;
1632
1633 fail:
1634         sd_journal_close(j);
1635
1636         return r;
1637 }
1638
1639 _public_ int sd_journal_open_container(sd_journal **ret, const char *machine, int flags) {
1640         _cleanup_free_ char *root = NULL, *class = NULL;
1641         sd_journal *j;
1642         char *p;
1643         int r;
1644
1645         assert_return(machine, -EINVAL);
1646         assert_return(ret, -EINVAL);
1647         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_SYSTEM)) == 0, -EINVAL);
1648         assert_return(machine_name_is_valid(machine), -EINVAL);
1649
1650         p = strappenda("/run/systemd/machines/", machine);
1651         r = parse_env_file(p, NEWLINE, "ROOT", &root, "CLASS", &class, NULL);
1652         if (r == -ENOENT)
1653                 return -EHOSTDOWN;
1654         if (r < 0)
1655                 return r;
1656         if (!root)
1657                 return -ENODATA;
1658
1659         if (!streq_ptr(class, "container"))
1660                 return -EIO;
1661
1662         j = journal_new(flags, NULL);
1663         if (!j)
1664                 return -ENOMEM;
1665
1666         j->prefix = root;
1667         root = NULL;
1668
1669         r = add_search_paths(j);
1670         if (r < 0)
1671                 goto fail;
1672
1673         *ret = j;
1674         return 0;
1675
1676 fail:
1677         sd_journal_close(j);
1678         return r;
1679 }
1680
1681 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1682         sd_journal *j;
1683         int r;
1684
1685         assert_return(ret, -EINVAL);
1686         assert_return(path, -EINVAL);
1687         assert_return(flags == 0, -EINVAL);
1688
1689         j = journal_new(flags, path);
1690         if (!j)
1691                 return -ENOMEM;
1692
1693         r = add_root_directory(j, path);
1694         if (r < 0) {
1695                 set_put_error(j, r);
1696                 goto fail;
1697         }
1698
1699         *ret = j;
1700         return 0;
1701
1702 fail:
1703         sd_journal_close(j);
1704
1705         return r;
1706 }
1707
1708 _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int flags) {
1709         sd_journal *j;
1710         const char **path;
1711         int r;
1712
1713         assert_return(ret, -EINVAL);
1714         assert_return(flags == 0, -EINVAL);
1715
1716         j = journal_new(flags, NULL);
1717         if (!j)
1718                 return -ENOMEM;
1719
1720         STRV_FOREACH(path, paths) {
1721                 r = add_any_file(j, *path);
1722                 if (r < 0) {
1723                         log_error_errno(r, "Failed to open %s: %m", *path);
1724                         goto fail;
1725                 }
1726         }
1727
1728         j->no_new_files = true;
1729
1730         *ret = j;
1731         return 0;
1732
1733 fail:
1734         sd_journal_close(j);
1735
1736         return r;
1737 }
1738
1739 _public_ void sd_journal_close(sd_journal *j) {
1740         Directory *d;
1741         JournalFile *f;
1742
1743         if (!j)
1744                 return;
1745
1746         sd_journal_flush_matches(j);
1747
1748         while ((f = ordered_hashmap_steal_first(j->files)))
1749                 journal_file_close(f);
1750
1751         ordered_hashmap_free(j->files);
1752
1753         while ((d = hashmap_first(j->directories_by_path)))
1754                 remove_directory(j, d);
1755
1756         while ((d = hashmap_first(j->directories_by_wd)))
1757                 remove_directory(j, d);
1758
1759         hashmap_free(j->directories_by_path);
1760         hashmap_free(j->directories_by_wd);
1761
1762         safe_close(j->inotify_fd);
1763
1764         if (j->mmap) {
1765                 log_debug("mmap cache statistics: %u hit, %u miss", mmap_cache_get_hit(j->mmap), mmap_cache_get_missed(j->mmap));
1766                 mmap_cache_unref(j->mmap);
1767         }
1768
1769         free(j->path);
1770         free(j->prefix);
1771         free(j->unique_field);
1772         set_free(j->errors);
1773         free(j);
1774 }
1775
1776 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1777         Object *o;
1778         JournalFile *f;
1779         int r;
1780
1781         assert_return(j, -EINVAL);
1782         assert_return(!journal_pid_changed(j), -ECHILD);
1783         assert_return(ret, -EINVAL);
1784
1785         f = j->current_file;
1786         if (!f)
1787                 return -EADDRNOTAVAIL;
1788
1789         if (f->current_offset <= 0)
1790                 return -EADDRNOTAVAIL;
1791
1792         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1793         if (r < 0)
1794                 return r;
1795
1796         *ret = le64toh(o->entry.realtime);
1797         return 0;
1798 }
1799
1800 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1801         Object *o;
1802         JournalFile *f;
1803         int r;
1804         sd_id128_t id;
1805
1806         assert_return(j, -EINVAL);
1807         assert_return(!journal_pid_changed(j), -ECHILD);
1808
1809         f = j->current_file;
1810         if (!f)
1811                 return -EADDRNOTAVAIL;
1812
1813         if (f->current_offset <= 0)
1814                 return -EADDRNOTAVAIL;
1815
1816         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1817         if (r < 0)
1818                 return r;
1819
1820         if (ret_boot_id)
1821                 *ret_boot_id = o->entry.boot_id;
1822         else {
1823                 r = sd_id128_get_boot(&id);
1824                 if (r < 0)
1825                         return r;
1826
1827                 if (!sd_id128_equal(id, o->entry.boot_id))
1828                         return -ESTALE;
1829         }
1830
1831         if (ret)
1832                 *ret = le64toh(o->entry.monotonic);
1833
1834         return 0;
1835 }
1836
1837 static bool field_is_valid(const char *field) {
1838         const char *p;
1839
1840         assert(field);
1841
1842         if (isempty(field))
1843                 return false;
1844
1845         if (startswith(field, "__"))
1846                 return false;
1847
1848         for (p = field; *p; p++) {
1849
1850                 if (*p == '_')
1851                         continue;
1852
1853                 if (*p >= 'A' && *p <= 'Z')
1854                         continue;
1855
1856                 if (*p >= '0' && *p <= '9')
1857                         continue;
1858
1859                 return false;
1860         }
1861
1862         return true;
1863 }
1864
1865 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1866         JournalFile *f;
1867         uint64_t i, n;
1868         size_t field_length;
1869         int r;
1870         Object *o;
1871
1872         assert_return(j, -EINVAL);
1873         assert_return(!journal_pid_changed(j), -ECHILD);
1874         assert_return(field, -EINVAL);
1875         assert_return(data, -EINVAL);
1876         assert_return(size, -EINVAL);
1877         assert_return(field_is_valid(field), -EINVAL);
1878
1879         f = j->current_file;
1880         if (!f)
1881                 return -EADDRNOTAVAIL;
1882
1883         if (f->current_offset <= 0)
1884                 return -EADDRNOTAVAIL;
1885
1886         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1887         if (r < 0)
1888                 return r;
1889
1890         field_length = strlen(field);
1891
1892         n = journal_file_entry_n_items(o);
1893         for (i = 0; i < n; i++) {
1894                 uint64_t p, l;
1895                 le64_t le_hash;
1896                 size_t t;
1897                 int compression;
1898
1899                 p = le64toh(o->entry.items[i].object_offset);
1900                 le_hash = o->entry.items[i].hash;
1901                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1902                 if (r < 0)
1903                         return r;
1904
1905                 if (le_hash != o->data.hash)
1906                         return -EBADMSG;
1907
1908                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1909
1910                 compression = o->object.flags & OBJECT_COMPRESSION_MASK;
1911                 if (compression) {
1912 #if defined(HAVE_XZ) || defined(HAVE_LZ4)
1913                         if (decompress_startswith(compression,
1914                                                   o->data.payload, l,
1915                                                   &f->compress_buffer, &f->compress_buffer_size,
1916                                                   field, field_length, '=')) {
1917
1918                                 size_t rsize;
1919
1920                                 r = decompress_blob(compression,
1921                                                     o->data.payload, l,
1922                                                     &f->compress_buffer, &f->compress_buffer_size, &rsize,
1923                                                     j->data_threshold);
1924                                 if (r < 0)
1925                                         return r;
1926
1927                                 *data = f->compress_buffer;
1928                                 *size = (size_t) rsize;
1929
1930                                 return 0;
1931                         }
1932 #else
1933                         return -EPROTONOSUPPORT;
1934 #endif
1935                 } else if (l >= field_length+1 &&
1936                            memcmp(o->data.payload, field, field_length) == 0 &&
1937                            o->data.payload[field_length] == '=') {
1938
1939                         t = (size_t) l;
1940
1941                         if ((uint64_t) t != l)
1942                                 return -E2BIG;
1943
1944                         *data = o->data.payload;
1945                         *size = t;
1946
1947                         return 0;
1948                 }
1949
1950                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1951                 if (r < 0)
1952                         return r;
1953         }
1954
1955         return -ENOENT;
1956 }
1957
1958 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1959         size_t t;
1960         uint64_t l;
1961         int compression;
1962
1963         l = le64toh(o->object.size) - offsetof(Object, data.payload);
1964         t = (size_t) l;
1965
1966         /* We can't read objects larger than 4G on a 32bit machine */
1967         if ((uint64_t) t != l)
1968                 return -E2BIG;
1969
1970         compression = o->object.flags & OBJECT_COMPRESSION_MASK;
1971         if (compression) {
1972 #if defined(HAVE_XZ) || defined(HAVE_LZ4)
1973                 size_t rsize;
1974                 int r;
1975
1976                 r = decompress_blob(compression,
1977                                     o->data.payload, l, &f->compress_buffer,
1978                                     &f->compress_buffer_size, &rsize, j->data_threshold);
1979                 if (r < 0)
1980                         return r;
1981
1982                 *data = f->compress_buffer;
1983                 *size = (size_t) rsize;
1984 #else
1985                 return -EPROTONOSUPPORT;
1986 #endif
1987         } else {
1988                 *data = o->data.payload;
1989                 *size = t;
1990         }
1991
1992         return 0;
1993 }
1994
1995 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1996         JournalFile *f;
1997         uint64_t p, n;
1998         le64_t le_hash;
1999         int r;
2000         Object *o;
2001
2002         assert_return(j, -EINVAL);
2003         assert_return(!journal_pid_changed(j), -ECHILD);
2004         assert_return(data, -EINVAL);
2005         assert_return(size, -EINVAL);
2006
2007         f = j->current_file;
2008         if (!f)
2009                 return -EADDRNOTAVAIL;
2010
2011         if (f->current_offset <= 0)
2012                 return -EADDRNOTAVAIL;
2013
2014         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2015         if (r < 0)
2016                 return r;
2017
2018         n = journal_file_entry_n_items(o);
2019         if (j->current_field >= n)
2020                 return 0;
2021
2022         p = le64toh(o->entry.items[j->current_field].object_offset);
2023         le_hash = o->entry.items[j->current_field].hash;
2024         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2025         if (r < 0)
2026                 return r;
2027
2028         if (le_hash != o->data.hash)
2029                 return -EBADMSG;
2030
2031         r = return_data(j, f, o, data, size);
2032         if (r < 0)
2033                 return r;
2034
2035         j->current_field ++;
2036
2037         return 1;
2038 }
2039
2040 _public_ void sd_journal_restart_data(sd_journal *j) {
2041         if (!j)
2042                 return;
2043
2044         j->current_field = 0;
2045 }
2046
2047 _public_ int sd_journal_get_fd(sd_journal *j) {
2048         int r;
2049
2050         assert_return(j, -EINVAL);
2051         assert_return(!journal_pid_changed(j), -ECHILD);
2052
2053         if (j->inotify_fd >= 0)
2054                 return j->inotify_fd;
2055
2056         r = allocate_inotify(j);
2057         if (r < 0)
2058                 return r;
2059
2060         /* Iterate through all dirs again, to add them to the
2061          * inotify */
2062         if (j->no_new_files)
2063                 r = add_current_paths(j);
2064         else if (j->path)
2065                 r = add_root_directory(j, j->path);
2066         else
2067                 r = add_search_paths(j);
2068         if (r < 0)
2069                 return r;
2070
2071         return j->inotify_fd;
2072 }
2073
2074 _public_ int sd_journal_get_events(sd_journal *j) {
2075         int fd;
2076
2077         assert_return(j, -EINVAL);
2078         assert_return(!journal_pid_changed(j), -ECHILD);
2079
2080         fd = sd_journal_get_fd(j);
2081         if (fd < 0)
2082                 return fd;
2083
2084         return POLLIN;
2085 }
2086
2087 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2088         int fd;
2089
2090         assert_return(j, -EINVAL);
2091         assert_return(!journal_pid_changed(j), -ECHILD);
2092         assert_return(timeout_usec, -EINVAL);
2093
2094         fd = sd_journal_get_fd(j);
2095         if (fd < 0)
2096                 return fd;
2097
2098         if (!j->on_network) {
2099                 *timeout_usec = (uint64_t) -1;
2100                 return 0;
2101         }
2102
2103         /* If we are on the network we need to regularly check for
2104          * changes manually */
2105
2106         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2107         return 1;
2108 }
2109
2110 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2111         Directory *d;
2112         int r;
2113
2114         assert(j);
2115         assert(e);
2116
2117         /* Is this a subdirectory we watch? */
2118         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2119         if (d) {
2120                 sd_id128_t id;
2121
2122                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2123                     (endswith(e->name, ".journal") ||
2124                      endswith(e->name, ".journal~"))) {
2125
2126                         /* Event for a journal file */
2127
2128                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2129                                 r = add_file(j, d->path, e->name);
2130                                 if (r < 0) {
2131                                         log_debug_errno(r, "Failed to add file %s/%s: %m",
2132                                                         d->path, e->name);
2133                                         set_put_error(j, r);
2134                                 }
2135
2136                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2137
2138                                 r = remove_file(j, d->path, e->name);
2139                                 if (r < 0)
2140                                         log_debug_errno(r, "Failed to remove file %s/%s: %m", d->path, e->name);
2141                         }
2142
2143                 } else if (!d->is_root && e->len == 0) {
2144
2145                         /* Event for a subdirectory */
2146
2147                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2148                                 r = remove_directory(j, d);
2149                                 if (r < 0)
2150                                         log_debug_errno(r, "Failed to remove directory %s: %m", d->path);
2151                         }
2152
2153
2154                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2155
2156                         /* Event for root directory */
2157
2158                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2159                                 r = add_directory(j, d->path, e->name);
2160                                 if (r < 0)
2161                                         log_debug_errno(r, "Failed to add directory %s/%s: %m", d->path, e->name);
2162                         }
2163                 }
2164
2165                 return;
2166         }
2167
2168         if (e->mask & IN_IGNORED)
2169                 return;
2170
2171         log_warning("Unknown inotify event.");
2172 }
2173
2174 static int determine_change(sd_journal *j) {
2175         bool b;
2176
2177         assert(j);
2178
2179         b = j->current_invalidate_counter != j->last_invalidate_counter;
2180         j->last_invalidate_counter = j->current_invalidate_counter;
2181
2182         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2183 }
2184
2185 _public_ int sd_journal_process(sd_journal *j) {
2186         bool got_something = false;
2187
2188         assert_return(j, -EINVAL);
2189         assert_return(!journal_pid_changed(j), -ECHILD);
2190
2191         j->last_process_usec = now(CLOCK_MONOTONIC);
2192
2193         for (;;) {
2194                 uint8_t buffer[INOTIFY_EVENT_MAX] _alignas_(struct inotify_event);
2195                 struct inotify_event *e;
2196                 ssize_t l;
2197
2198                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2199                 if (l < 0) {
2200                         if (errno == EAGAIN || errno == EINTR)
2201                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2202
2203                         return -errno;
2204                 }
2205
2206                 got_something = true;
2207
2208                 FOREACH_INOTIFY_EVENT(e, buffer, l)
2209                         process_inotify_event(j, e);
2210         }
2211 }
2212
2213 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2214         int r;
2215         uint64_t t;
2216
2217         assert_return(j, -EINVAL);
2218         assert_return(!journal_pid_changed(j), -ECHILD);
2219
2220         if (j->inotify_fd < 0) {
2221
2222                 /* This is the first invocation, hence create the
2223                  * inotify watch */
2224                 r = sd_journal_get_fd(j);
2225                 if (r < 0)
2226                         return r;
2227
2228                 /* The journal might have changed since the context
2229                  * object was created and we weren't watching before,
2230                  * hence don't wait for anything, and return
2231                  * immediately. */
2232                 return determine_change(j);
2233         }
2234
2235         r = sd_journal_get_timeout(j, &t);
2236         if (r < 0)
2237                 return r;
2238
2239         if (t != (uint64_t) -1) {
2240                 usec_t n;
2241
2242                 n = now(CLOCK_MONOTONIC);
2243                 t = t > n ? t - n : 0;
2244
2245                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2246                         timeout_usec = t;
2247         }
2248
2249         do {
2250                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2251         } while (r == -EINTR);
2252
2253         if (r < 0)
2254                 return r;
2255
2256         return sd_journal_process(j);
2257 }
2258
2259 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2260         Iterator i;
2261         JournalFile *f;
2262         bool first = true;
2263         uint64_t fmin = 0, tmax = 0;
2264         int r;
2265
2266         assert_return(j, -EINVAL);
2267         assert_return(!journal_pid_changed(j), -ECHILD);
2268         assert_return(from || to, -EINVAL);
2269         assert_return(from != to, -EINVAL);
2270
2271         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2272                 usec_t fr, t;
2273
2274                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2275                 if (r == -ENOENT)
2276                         continue;
2277                 if (r < 0)
2278                         return r;
2279                 if (r == 0)
2280                         continue;
2281
2282                 if (first) {
2283                         fmin = fr;
2284                         tmax = t;
2285                         first = false;
2286                 } else {
2287                         fmin = MIN(fr, fmin);
2288                         tmax = MAX(t, tmax);
2289                 }
2290         }
2291
2292         if (from)
2293                 *from = fmin;
2294         if (to)
2295                 *to = tmax;
2296
2297         return first ? 0 : 1;
2298 }
2299
2300 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2301         Iterator i;
2302         JournalFile *f;
2303         bool found = false;
2304         int r;
2305
2306         assert_return(j, -EINVAL);
2307         assert_return(!journal_pid_changed(j), -ECHILD);
2308         assert_return(from || to, -EINVAL);
2309         assert_return(from != to, -EINVAL);
2310
2311         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2312                 usec_t fr, t;
2313
2314                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2315                 if (r == -ENOENT)
2316                         continue;
2317                 if (r < 0)
2318                         return r;
2319                 if (r == 0)
2320                         continue;
2321
2322                 if (found) {
2323                         if (from)
2324                                 *from = MIN(fr, *from);
2325                         if (to)
2326                                 *to = MAX(t, *to);
2327                 } else {
2328                         if (from)
2329                                 *from = fr;
2330                         if (to)
2331                                 *to = t;
2332                         found = true;
2333                 }
2334         }
2335
2336         return found;
2337 }
2338
2339 void journal_print_header(sd_journal *j) {
2340         Iterator i;
2341         JournalFile *f;
2342         bool newline = false;
2343
2344         assert(j);
2345
2346         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2347                 if (newline)
2348                         putchar('\n');
2349                 else
2350                         newline = true;
2351
2352                 journal_file_print_header(f);
2353         }
2354 }
2355
2356 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2357         Iterator i;
2358         JournalFile *f;
2359         uint64_t sum = 0;
2360
2361         assert_return(j, -EINVAL);
2362         assert_return(!journal_pid_changed(j), -ECHILD);
2363         assert_return(bytes, -EINVAL);
2364
2365         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2366                 struct stat st;
2367
2368                 if (fstat(f->fd, &st) < 0)
2369                         return -errno;
2370
2371                 sum += (uint64_t) st.st_blocks * 512ULL;
2372         }
2373
2374         *bytes = sum;
2375         return 0;
2376 }
2377
2378 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2379         char *f;
2380
2381         assert_return(j, -EINVAL);
2382         assert_return(!journal_pid_changed(j), -ECHILD);
2383         assert_return(!isempty(field), -EINVAL);
2384         assert_return(field_is_valid(field), -EINVAL);
2385
2386         f = strdup(field);
2387         if (!f)
2388                 return -ENOMEM;
2389
2390         free(j->unique_field);
2391         j->unique_field = f;
2392         j->unique_file = NULL;
2393         j->unique_offset = 0;
2394         j->unique_file_lost = false;
2395
2396         return 0;
2397 }
2398
2399 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2400         size_t k;
2401
2402         assert_return(j, -EINVAL);
2403         assert_return(!journal_pid_changed(j), -ECHILD);
2404         assert_return(data, -EINVAL);
2405         assert_return(l, -EINVAL);
2406         assert_return(j->unique_field, -EINVAL);
2407
2408         k = strlen(j->unique_field);
2409
2410         if (!j->unique_file) {
2411                 if (j->unique_file_lost)
2412                         return 0;
2413
2414                 j->unique_file = ordered_hashmap_first(j->files);
2415                 if (!j->unique_file)
2416                         return 0;
2417
2418                 j->unique_offset = 0;
2419         }
2420
2421         for (;;) {
2422                 JournalFile *of;
2423                 Iterator i;
2424                 Object *o;
2425                 const void *odata;
2426                 size_t ol;
2427                 bool found;
2428                 int r;
2429
2430                 /* Proceed to next data object in the field's linked list */
2431                 if (j->unique_offset == 0) {
2432                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2433                         if (r < 0)
2434                                 return r;
2435
2436                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2437                 } else {
2438                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2439                         if (r < 0)
2440                                 return r;
2441
2442                         j->unique_offset = le64toh(o->data.next_field_offset);
2443                 }
2444
2445                 /* We reached the end of the list? Then start again, with the next file */
2446                 if (j->unique_offset == 0) {
2447                         j->unique_file = ordered_hashmap_next(j->files, j->unique_file->path);
2448                         if (!j->unique_file)
2449                                 return 0;
2450
2451                         continue;
2452                 }
2453
2454                 /* We do not use OBJECT_DATA context here, but OBJECT_UNUSED
2455                  * instead, so that we can look at this data object at the same
2456                  * time as one on another file */
2457                 r = journal_file_move_to_object(j->unique_file, OBJECT_UNUSED, j->unique_offset, &o);
2458                 if (r < 0)
2459                         return r;
2460
2461                 /* Let's do the type check by hand, since we used 0 context above. */
2462                 if (o->object.type != OBJECT_DATA) {
2463                         log_debug("%s:offset " OFSfmt ": object has type %d, expected %d",
2464                                   j->unique_file->path, j->unique_offset,
2465                                   o->object.type, OBJECT_DATA);
2466                         return -EBADMSG;
2467                 }
2468
2469                 r = return_data(j, j->unique_file, o, &odata, &ol);
2470                 if (r < 0)
2471                         return r;
2472
2473                 /* Check if we have at least the field name and "=". */
2474                 if (ol <= k) {
2475                         log_debug("%s:offset " OFSfmt ": object has size %zu, expected at least %zu",
2476                                   j->unique_file->path, j->unique_offset,
2477                                   ol, k + 1);
2478                         return -EBADMSG;
2479                 }
2480
2481                 if (memcmp(odata, j->unique_field, k) || ((const char*) odata)[k] != '=') {
2482                         log_debug("%s:offset " OFSfmt ": object does not start with \"%s=\"",
2483                                   j->unique_file->path, j->unique_offset,
2484                                   j->unique_field);
2485                         return -EBADMSG;
2486                 }
2487
2488                 /* OK, now let's see if we already returned this data
2489                  * object by checking if it exists in the earlier
2490                  * traversed files. */
2491                 found = false;
2492                 ORDERED_HASHMAP_FOREACH(of, j->files, i) {
2493                         Object *oo;
2494                         uint64_t op;
2495
2496                         if (of == j->unique_file)
2497                                 break;
2498
2499                         /* Skip this file it didn't have any fields
2500                          * indexed */
2501                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2502                             le64toh(of->header->n_fields) <= 0)
2503                                 continue;
2504
2505                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2506                         if (r < 0)
2507                                 return r;
2508
2509                         if (r > 0)
2510                                 found = true;
2511                 }
2512
2513                 if (found)
2514                         continue;
2515
2516                 r = return_data(j, j->unique_file, o, data, l);
2517                 if (r < 0)
2518                         return r;
2519
2520                 return 1;
2521         }
2522 }
2523
2524 _public_ void sd_journal_restart_unique(sd_journal *j) {
2525         if (!j)
2526                 return;
2527
2528         j->unique_file = NULL;
2529         j->unique_offset = 0;
2530         j->unique_file_lost = false;
2531 }
2532
2533 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2534         assert_return(j, -EINVAL);
2535         assert_return(!journal_pid_changed(j), -ECHILD);
2536
2537         return !j->on_network;
2538 }
2539
2540 static char *lookup_field(const char *field, void *userdata) {
2541         sd_journal *j = userdata;
2542         const void *data;
2543         size_t size, d;
2544         int r;
2545
2546         assert(field);
2547         assert(j);
2548
2549         r = sd_journal_get_data(j, field, &data, &size);
2550         if (r < 0 ||
2551             size > REPLACE_VAR_MAX)
2552                 return strdup(field);
2553
2554         d = strlen(field) + 1;
2555
2556         return strndup((const char*) data + d, size - d);
2557 }
2558
2559 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2560         const void *data;
2561         size_t size;
2562         sd_id128_t id;
2563         _cleanup_free_ char *text = NULL, *cid = NULL;
2564         char *t;
2565         int r;
2566
2567         assert_return(j, -EINVAL);
2568         assert_return(!journal_pid_changed(j), -ECHILD);
2569         assert_return(ret, -EINVAL);
2570
2571         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2572         if (r < 0)
2573                 return r;
2574
2575         cid = strndup((const char*) data + 11, size - 11);
2576         if (!cid)
2577                 return -ENOMEM;
2578
2579         r = sd_id128_from_string(cid, &id);
2580         if (r < 0)
2581                 return r;
2582
2583         r = catalog_get(CATALOG_DATABASE, id, &text);
2584         if (r < 0)
2585                 return r;
2586
2587         t = replace_var(text, lookup_field, j);
2588         if (!t)
2589                 return -ENOMEM;
2590
2591         *ret = t;
2592         return 0;
2593 }
2594
2595 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2596         assert_return(ret, -EINVAL);
2597
2598         return catalog_get(CATALOG_DATABASE, id, ret);
2599 }
2600
2601 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2602         assert_return(j, -EINVAL);
2603         assert_return(!journal_pid_changed(j), -ECHILD);
2604
2605         j->data_threshold = sz;
2606         return 0;
2607 }
2608
2609 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2610         assert_return(j, -EINVAL);
2611         assert_return(!journal_pid_changed(j), -ECHILD);
2612         assert_return(sz, -EINVAL);
2613
2614         *sz = j->data_threshold;
2615         return 0;
2616 }