src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "path-util.h"
  37 #include "lookup3.h"
  38 #include "compress.h"
  39 #include "journal-internal.h"
  40 #include "missing.h"
  41
  42 #define JOURNAL_FILES_MAX 1024
  43
  44 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  45
  46 static void detach_location(sd_journal *j) {
  47         Iterator i;
  48         JournalFile *f;
  49
  50         assert(j);
  51
  52         j->current_file = NULL;
  53         j->current_field = 0;
  54
  55         HASHMAP_FOREACH(f, j->files, i)
  56                 f->current_offset = 0;
  57 }
  58
  59 static void reset_location(sd_journal *j) {
  60         assert(j);
  61
  62         detach_location(j);
  63         zero(j->current_location);
  64 }
  65
  66 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  67         assert(l);
  68         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
  69         assert(f);
  70         assert(o->object.type == OBJECT_ENTRY);
  71
  72         l->type = type;
  73         l->seqnum = le64toh(o->entry.seqnum);
  74         l->seqnum_id = f->header->seqnum_id;
  75         l->realtime = le64toh(o->entry.realtime);
  76         l->monotonic = le64toh(o->entry.monotonic);
  77         l->boot_id = o->entry.boot_id;
  78         l->xor_hash = le64toh(o->entry.xor_hash);
  79
  80         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
  81 }
  82
  83 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o, uint64_t offset) {
  84         assert(j);
  85         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
  86         assert(f);
  87         assert(o);
  88
  89         init_location(&j->current_location, type, f, o);
  90
  91         j->current_file = f;
  92         j->current_field = 0;
  93
  94         f->current_offset = offset;
  95 }
  96
  97 static int match_is_valid(const void *data, size_t size) {
  98         const char *b, *p;
  99
 100         assert(data);
 101
 102         if (size < 2)
 103                 return false;
 104
 105         if (startswith(data, "__"))
 106                 return false;
 107
 108         b = data;
 109         for (p = b; p < b + size; p++) {
 110
 111                 if (*p == '=')
 112                         return p > b;
 113
 114                 if (*p == '_')
 115                         continue;
 116
 117                 if (*p >= 'A' && *p <= 'Z')
 118                         continue;
 119
 120                 if (*p >= '0' && *p <= '9')
 121                         continue;
 122
 123                 return false;
 124         }
 125
 126         return false;
 127 }
 128
 129 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 130         const uint8_t *a = _a, *b = _b;
 131         size_t j;
 132
 133         for (j = 0; j < s && j < t; j++) {
 134
 135                 if (a[j] != b[j])
 136                         return false;
 137
 138                 if (a[j] == '=')
 139                         return true;
 140         }
 141
 142         return true;
 143 }
 144
 145 static Match *match_new(Match *p, MatchType t) {
 146         Match *m;
 147
 148         m = new0(Match, 1);
 149         if (!m)
 150                 return NULL;
 151
 152         m->type = t;
 153
 154         if (p) {
 155                 m->parent = p;
 156                 LIST_PREPEND(Match, matches, p->matches, m);
 157         }
 158
 159         return m;
 160 }
 161
 162 static void match_free(Match *m) {
 163         assert(m);
 164
 165         while (m->matches)
 166                 match_free(m->matches);
 167
 168         if (m->parent)
 169                 LIST_REMOVE(Match, matches, m->parent->matches, m);
 170
 171         free(m->data);
 172         free(m);
 173 }
 174
 175 static void match_free_if_empty(Match *m) {
 176         assert(m);
 177
 178         if (m->matches)
 179                 return;
 180
 181         match_free(m);
 182 }
 183
 184 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 185         Match *l2, *l3, *add_here = NULL, *m;
 186         le64_t le_hash;
 187
 188         if (!j)
 189                 return -EINVAL;
 190
 191         if (!data)
 192                 return -EINVAL;
 193
 194         if (size == 0)
 195                 size = strlen(data);
 196
 197         if (!match_is_valid(data, size))
 198                 return -EINVAL;
 199
 200         /* level 0: OR term
 201          * level 1: AND terms
 202          * level 2: OR terms
 203          * level 3: concrete matches */
 204
 205         if (!j->level0) {
 206                 j->level0 = match_new(NULL, MATCH_OR_TERM);
 207                 if (!j->level0)
 208                         return -ENOMEM;
 209         }
 210
 211         if (!j->level1) {
 212                 j->level1 = match_new(j->level0, MATCH_AND_TERM);
 213                 if (!j->level1)
 214                         return -ENOMEM;
 215         }
 216
 217         assert(j->level0->type == MATCH_OR_TERM);
 218         assert(j->level1->type == MATCH_AND_TERM);
 219
 220         le_hash = htole64(hash64(data, size));
 221
 222         LIST_FOREACH(matches, l2, j->level1->matches) {
 223                 assert(l2->type == MATCH_OR_TERM);
 224
 225                 LIST_FOREACH(matches, l3, l2->matches) {
 226                         assert(l3->type == MATCH_DISCRETE);
 227
 228                         /* Exactly the same match already? Then ignore
 229                          * this addition */
 230                         if (l3->le_hash == le_hash &&
 231                             l3->size == size &&
 232                             memcmp(l3->data, data, size) == 0)
 233                                 return 0;
 234
 235                         /* Same field? Then let's add this to this OR term */
 236                         if (same_field(data, size, l3->data, l3->size)) {
 237                                 add_here = l2;
 238                                 break;
 239                         }
 240                 }
 241
 242                 if (add_here)
 243                         break;
 244         }
 245
 246         if (!add_here) {
 247                 add_here = match_new(j->level1, MATCH_OR_TERM);
 248                 if (!add_here)
 249                         goto fail;
 250         }
 251
 252         m = match_new(add_here, MATCH_DISCRETE);
 253         if (!m)
 254                 goto fail;
 255
 256         m->le_hash = le_hash;
 257         m->size = size;
 258         m->data = memdup(data, size);
 259         if (!m->data)
 260                 goto fail;
 261
 262         detach_location(j);
 263
 264         return 0;
 265
 266 fail:
 267         if (add_here)
 268                 match_free_if_empty(add_here);
 269
 270         if (j->level1)
 271                 match_free_if_empty(j->level1);
 272
 273         if (j->level0)
 274                 match_free_if_empty(j->level0);
 275
 276         return -ENOMEM;
 277 }
 278
 279 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 280         Match *m;
 281
 282         assert(j);
 283
 284         if (!j->level0)
 285                 return 0;
 286
 287         if (!j->level1)
 288                 return 0;
 289
 290         if (!j->level1->matches)
 291                 return 0;
 292
 293         m = match_new(j->level0, MATCH_AND_TERM);
 294         if (!m)
 295                 return -ENOMEM;
 296
 297         j->level1 = m;
 298         return 0;
 299 }
 300
 301 static char *match_make_string(Match *m) {
 302         char *p, *r;
 303         Match *i;
 304         bool enclose = false;
 305
 306         if (!m)
 307                 return strdup("");
 308
 309         if (m->type == MATCH_DISCRETE)
 310                 return strndup(m->data, m->size);
 311
 312         p = NULL;
 313         LIST_FOREACH(matches, i, m->matches) {
 314                 char *t, *k;
 315
 316                 t = match_make_string(i);
 317                 if (!t) {
 318                         free(p);
 319                         return NULL;
 320                 }
 321
 322                 if (p) {
 323                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 324                         free(p);
 325                         free(t);
 326
 327                         if (!k)
 328                                 return NULL;
 329
 330                         p = k;
 331
 332                         enclose = true;
 333                 } else {
 334                         free(p);
 335                         p = t;
 336                 }
 337         }
 338
 339         if (enclose) {
 340                 r = strjoin("(", p, ")", NULL);
 341                 free(p);
 342                 return r;
 343         }
 344
 345         return p;
 346 }
 347
 348 char *journal_make_match_string(sd_journal *j) {
 349         assert(j);
 350
 351         return match_make_string(j->level0);
 352 }
 353
 354 _public_ void sd_journal_flush_matches(sd_journal *j) {
 355
 356         if (!j)
 357                 return;
 358
 359         if (j->level0)
 360                 match_free(j->level0);
 361
 362         j->level0 = j->level1 = NULL;
 363
 364         detach_location(j);
 365 }
 366
 367 static int compare_entry_order(JournalFile *af, Object *_ao,
 368                          JournalFile *bf, uint64_t bp) {
 369
 370         uint64_t a, b;
 371         Object *ao, *bo;
 372         int r;
 373
 374         assert(af);
 375         assert(bf);
 376         assert(_ao);
 377
 378         /* The mmap cache might invalidate the object from the first
 379          * file if we look at the one from the second file. Hence
 380          * temporarily copy the header of the first one, and look at
 381          * that only. */
 382         ao = alloca(offsetof(EntryObject, items));
 383         memcpy(ao, _ao, offsetof(EntryObject, items));
 384
 385         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 386         if (r < 0)
 387                 return strcmp(af->path, bf->path);
 388
 389         /* We operate on two different files here, hence we can access
 390          * two objects at the same time, which we normally can't.
 391          *
 392          * If contents and timestamps match, these entries are
 393          * identical, even if the seqnum does not match */
 394
 395         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 396             ao->entry.monotonic == bo->entry.monotonic &&
 397             ao->entry.realtime == bo->entry.realtime &&
 398             ao->entry.xor_hash == bo->entry.xor_hash)
 399                 return 0;
 400
 401         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 402
 403                 /* If this is from the same seqnum source, compare
 404                  * seqnums */
 405                 a = le64toh(ao->entry.seqnum);
 406                 b = le64toh(bo->entry.seqnum);
 407
 408                 if (a < b)
 409                         return -1;
 410                 if (a > b)
 411                         return 1;
 412
 413                 /* Wow! This is weird, different data but the same
 414                  * seqnums? Something is borked, but let's make the
 415                  * best of it and compare by time. */
 416         }
 417
 418         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 419
 420                 /* If the boot id matches compare monotonic time */
 421                 a = le64toh(ao->entry.monotonic);
 422                 b = le64toh(bo->entry.monotonic);
 423
 424                 if (a < b)
 425                         return -1;
 426                 if (a > b)
 427                         return 1;
 428         }
 429
 430         /* Otherwise compare UTC time */
 431         a = le64toh(ao->entry.realtime);
 432         b = le64toh(bo->entry.realtime);
 433
 434         if (a < b)
 435                 return -1;
 436         if (a > b)
 437                 return 1;
 438
 439         /* Finally, compare by contents */
 440         a = le64toh(ao->entry.xor_hash);
 441         b = le64toh(bo->entry.xor_hash);
 442
 443         if (a < b)
 444                 return -1;
 445         if (a > b)
 446                 return 1;
 447
 448         return 0;
 449 }
 450
 451 static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 452         uint64_t a;
 453
 454         assert(af);
 455         assert(ao);
 456         assert(l);
 457         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 458
 459         if (l->monotonic_set &&
 460             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 461             l->realtime_set &&
 462             le64toh(ao->entry.realtime) == l->realtime &&
 463             l->xor_hash_set &&
 464             le64toh(ao->entry.xor_hash) == l->xor_hash)
 465                 return 0;
 466
 467         if (l->seqnum_set &&
 468             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 469
 470                 a = le64toh(ao->entry.seqnum);
 471
 472                 if (a < l->seqnum)
 473                         return -1;
 474                 if (a > l->seqnum)
 475                         return 1;
 476         }
 477
 478         if (l->monotonic_set &&
 479             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 480
 481                 a = le64toh(ao->entry.monotonic);
 482
 483                 if (a < l->monotonic)
 484                         return -1;
 485                 if (a > l->monotonic)
 486                         return 1;
 487         }
 488
 489         if (l->realtime_set) {
 490
 491                 a = le64toh(ao->entry.realtime);
 492
 493                 if (a < l->realtime)
 494                         return -1;
 495                 if (a > l->realtime)
 496                         return 1;
 497         }
 498
 499         if (l->xor_hash_set) {
 500                 a = le64toh(ao->entry.xor_hash);
 501
 502                 if (a < l->xor_hash)
 503                         return -1;
 504                 if (a > l->xor_hash)
 505                         return 1;
 506         }
 507
 508         return 0;
 509 }
 510
 511 static int next_for_match(
 512                 sd_journal *j,
 513                 Match *m,
 514                 JournalFile *f,
 515                 uint64_t after_offset,
 516                 direction_t direction,
 517                 Object **ret,
 518                 uint64_t *offset) {
 519
 520         int r;
 521         uint64_t np = 0;
 522         Object *n;
 523
 524         assert(j);
 525         assert(m);
 526         assert(f);
 527
 528         if (m->type == MATCH_DISCRETE) {
 529                 uint64_t dp;
 530
 531                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 532                 if (r <= 0)
 533                         return r;
 534
 535                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 536
 537         } else if (m->type == MATCH_OR_TERM) {
 538                 Match *i;
 539
 540                 /* Find the earliest match beyond after_offset */
 541
 542                 LIST_FOREACH(matches, i, m->matches) {
 543                         uint64_t cp;
 544
 545                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 546                         if (r < 0)
 547                                 return r;
 548                         else if (r > 0) {
 549                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 550                                         np = cp;
 551                         }
 552                 }
 553
 554         } else if (m->type == MATCH_AND_TERM) {
 555                 Match *i;
 556                 bool continue_looking;
 557
 558                 /* Always jump to the next matching entry and repeat
 559                  * this until we fine and offset that matches for all
 560                  * matches. */
 561
 562                 if (!m->matches)
 563                         return 0;
 564
 565                 np = 0;
 566                 do {
 567                         continue_looking = false;
 568
 569                         LIST_FOREACH(matches, i, m->matches) {
 570                                 uint64_t cp, limit;
 571
 572                                 if (np == 0)
 573                                         limit = after_offset;
 574                                 else if (direction == DIRECTION_DOWN)
 575                                         limit = MAX(np, after_offset);
 576                                 else
 577                                         limit = MIN(np, after_offset);
 578
 579                                 r = next_for_match(j, i, f, limit, direction, NULL, &cp);
 580                                 if (r <= 0)
 581                                         return r;
 582
 583                                 if ((direction == DIRECTION_DOWN ? cp >= after_offset : cp <= after_offset) &&
 584                                     (np == 0 || (direction == DIRECTION_DOWN ? cp > np : np < cp))) {
 585                                         np = cp;
 586                                         continue_looking = true;
 587                                 }
 588                         }
 589
 590                 } while (continue_looking);
 591         }
 592
 593         if (np == 0)
 594                 return 0;
 595
 596         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 597         if (r < 0)
 598                 return r;
 599
 600         if (ret)
 601                 *ret = n;
 602         if (offset)
 603                 *offset = np;
 604
 605         return 1;
 606 }
 607
 608 static int find_location_for_match(
 609                 sd_journal *j,
 610                 Match *m,
 611                 JournalFile *f,
 612                 direction_t direction,
 613                 Object **ret,
 614                 uint64_t *offset) {
 615
 616         int r;
 617
 618         assert(j);
 619         assert(m);
 620         assert(f);
 621
 622         if (m->type == MATCH_DISCRETE) {
 623                 uint64_t dp;
 624
 625                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 626                 if (r <= 0)
 627                         return r;
 628
 629                 /* FIXME: missing: find by monotonic */
 630
 631                 if (j->current_location.type == LOCATION_HEAD)
 632                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 633                 if (j->current_location.type == LOCATION_TAIL)
 634                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 635                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 636                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 637                 if (j->current_location.monotonic_set) {
 638                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 639                         if (r != -ENOENT)
 640                                 return r;
 641                 }
 642                 if (j->current_location.realtime_set)
 643                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 644
 645                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 646
 647         } else if (m->type == MATCH_OR_TERM) {
 648                 uint64_t np = 0;
 649                 Object *n;
 650                 Match *i;
 651
 652                 /* Find the earliest match */
 653
 654                 LIST_FOREACH(matches, i, m->matches) {
 655                         uint64_t cp;
 656
 657                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 658                         if (r < 0)
 659                                 return r;
 660                         else if (r > 0) {
 661                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 662                                         np = cp;
 663                         }
 664                 }
 665
 666                 if (np == 0)
 667                         return 0;
 668
 669                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 670                 if (r < 0)
 671                         return r;
 672
 673                 if (ret)
 674                         *ret = n;
 675                 if (offset)
 676                         *offset = np;
 677
 678                 return 1;
 679
 680         } else {
 681                 Match *i;
 682                 uint64_t np = 0;
 683
 684                 assert(m->type == MATCH_AND_TERM);
 685
 686                 /* First jump to the last match, and then find the
 687                  * next one where all matches match */
 688
 689                 if (!m->matches)
 690                         return 0;
 691
 692                 LIST_FOREACH(matches, i, m->matches) {
 693                         uint64_t cp;
 694
 695                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 696                         if (r <= 0)
 697                                 return r;
 698
 699                         if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
 700                                 np = cp;
 701                 }
 702
 703                 return next_for_match(j, m, f, np, direction, ret, offset);
 704         }
 705 }
 706
 707 static int find_location_with_matches(
 708                 sd_journal *j,
 709                 JournalFile *f,
 710                 direction_t direction,
 711                 Object **ret,
 712                 uint64_t *offset) {
 713
 714         int r;
 715
 716         assert(j);
 717         assert(f);
 718         assert(ret);
 719         assert(offset);
 720
 721         if (!j->level0) {
 722                 /* No matches is simple */
 723
 724                 if (j->current_location.type == LOCATION_HEAD)
 725                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 726                 if (j->current_location.type == LOCATION_TAIL)
 727                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 728                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 729                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 730                 if (j->current_location.monotonic_set) {
 731                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 732                         if (r != -ENOENT)
 733                                 return r;
 734                 }
 735                 if (j->current_location.realtime_set)
 736                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 737
 738                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 739         } else
 740                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 741 }
 742
 743 static int next_with_matches(
 744                 sd_journal *j,
 745                 JournalFile *f,
 746                 direction_t direction,
 747                 Object **ret,
 748                 uint64_t *offset) {
 749
 750         Object *c;
 751         uint64_t cp;
 752
 753         assert(j);
 754         assert(f);
 755         assert(ret);
 756         assert(offset);
 757
 758         c = *ret;
 759         cp = *offset;
 760
 761         /* No matches is easy. We simple advance the file
 762          * pointer by one. */
 763         if (!j->level0)
 764                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 765
 766         /* If we have a match then we look for the next matching entry
 767          * with an offset at least one step larger */
 768         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 769 }
 770
 771 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 772         Object *c;
 773         uint64_t cp;
 774         int r;
 775
 776         assert(j);
 777         assert(f);
 778
 779         if (f->current_offset > 0) {
 780                 cp = f->current_offset;
 781
 782                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 783                 if (r < 0)
 784                         return r;
 785
 786                 r = next_with_matches(j, f, direction, &c, &cp);
 787                 if (r <= 0)
 788                         return r;
 789         } else {
 790                 r = find_location_with_matches(j, f, direction, &c, &cp);
 791                 if (r <= 0)
 792                         return r;
 793         }
 794
 795         /* OK, we found the spot, now let's advance until to an entry
 796          * that is actually different from what we were previously
 797          * looking at. This is necessary to handle entries which exist
 798          * in two (or more) journal files, and which shall all be
 799          * suppressed but one. */
 800
 801         for (;;) {
 802                 bool found;
 803
 804                 if (j->current_location.type == LOCATION_DISCRETE) {
 805                         int k;
 806
 807                         k = compare_with_location(f, c, &j->current_location);
 808                         if (direction == DIRECTION_DOWN)
 809                                 found = k > 0;
 810                         else
 811                                 found = k < 0;
 812                 } else
 813                         found = true;
 814
 815                 if (found) {
 816                         if (ret)
 817                                 *ret = c;
 818                         if (offset)
 819                                 *offset = cp;
 820                         return 1;
 821                 }
 822
 823                 r = next_with_matches(j, f, direction, &c, &cp);
 824                 if (r <= 0)
 825                         return r;
 826         }
 827 }
 828
 829 static int real_journal_next(sd_journal *j, direction_t direction) {
 830         JournalFile *f, *new_file = NULL;
 831         uint64_t new_offset = 0;
 832         Object *o;
 833         uint64_t p;
 834         Iterator i;
 835         int r;
 836
 837         if (!j)
 838                 return -EINVAL;
 839
 840         HASHMAP_FOREACH(f, j->files, i) {
 841                 bool found;
 842
 843                 r = next_beyond_location(j, f, direction, &o, &p);
 844                 if (r < 0) {
 845                         log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
 846                         continue;
 847                 } else if (r == 0)
 848                         continue;
 849
 850                 if (!new_file)
 851                         found = true;
 852                 else {
 853                         int k;
 854
 855                         k = compare_entry_order(f, o, new_file, new_offset);
 856
 857                         if (direction == DIRECTION_DOWN)
 858                                 found = k < 0;
 859                         else
 860                                 found = k > 0;
 861                 }
 862
 863                 if (found) {
 864                         new_file = f;
 865                         new_offset = p;
 866                 }
 867         }
 868
 869         if (!new_file)
 870                 return 0;
 871
 872         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 873         if (r < 0)
 874                 return r;
 875
 876         set_location(j, LOCATION_DISCRETE, new_file, o, new_offset);
 877
 878         return 1;
 879 }
 880
 881 _public_ int sd_journal_next(sd_journal *j) {
 882         return real_journal_next(j, DIRECTION_DOWN);
 883 }
 884
 885 _public_ int sd_journal_previous(sd_journal *j) {
 886         return real_journal_next(j, DIRECTION_UP);
 887 }
 888
 889 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 890         int c = 0, r;
 891
 892         if (!j)
 893                 return -EINVAL;
 894
 895         if (skip == 0) {
 896                 /* If this is not a discrete skip, then at least
 897                  * resolve the current location */
 898                 if (j->current_location.type != LOCATION_DISCRETE)
 899                         return real_journal_next(j, direction);
 900
 901                 return 0;
 902         }
 903
 904         do {
 905                 r = real_journal_next(j, direction);
 906                 if (r < 0)
 907                         return r;
 908
 909                 if (r == 0)
 910                         return c;
 911
 912                 skip--;
 913                 c++;
 914         } while (skip > 0);
 915
 916         return c;
 917 }
 918
 919 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 920         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 921 }
 922
 923 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 924         return real_journal_next_skip(j, DIRECTION_UP, skip);
 925 }
 926
 927 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 928         Object *o;
 929         int r;
 930         char bid[33], sid[33];
 931
 932         if (!j)
 933                 return -EINVAL;
 934         if (!cursor)
 935                 return -EINVAL;
 936
 937         if (!j->current_file || j->current_file->current_offset <= 0)
 938                 return -EADDRNOTAVAIL;
 939
 940         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 941         if (r < 0)
 942                 return r;
 943
 944         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 945         sd_id128_to_string(o->entry.boot_id, bid);
 946
 947         if (asprintf(cursor,
 948                      "s=%s;i=%llx;b=%s;m=%llx;t=%llx;x=%llx",
 949                      sid, (unsigned long long) le64toh(o->entry.seqnum),
 950                      bid, (unsigned long long) le64toh(o->entry.monotonic),
 951                      (unsigned long long) le64toh(o->entry.realtime),
 952                      (unsigned long long) le64toh(o->entry.xor_hash)) < 0)
 953                 return -ENOMEM;
 954
 955         return 1;
 956 }
 957
 958 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
 959         char *w, *state;
 960         size_t l;
 961         unsigned long long seqnum, monotonic, realtime, xor_hash;
 962         bool
 963                 seqnum_id_set = false,
 964                 seqnum_set = false,
 965                 boot_id_set = false,
 966                 monotonic_set = false,
 967                 realtime_set = false,
 968                 xor_hash_set = false;
 969         sd_id128_t seqnum_id, boot_id;
 970
 971         if (!j)
 972                 return -EINVAL;
 973         if (isempty(cursor))
 974                 return -EINVAL;
 975
 976         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
 977                 char *item;
 978                 int k = 0;
 979
 980                 if (l < 2 || w[1] != '=')
 981                         return -EINVAL;
 982
 983                 item = strndup(w, l);
 984                 if (!item)
 985                         return -ENOMEM;
 986
 987                 switch (w[0]) {
 988
 989                 case 's':
 990                         seqnum_id_set = true;
 991                         k = sd_id128_from_string(item+2, &seqnum_id);
 992                         break;
 993
 994                 case 'i':
 995                         seqnum_set = true;
 996                         if (sscanf(item+2, "%llx", &seqnum) != 1)
 997                                 k = -EINVAL;
 998                         break;
 999
1000                 case 'b':
1001                         boot_id_set = true;
1002                         k = sd_id128_from_string(item+2, &boot_id);
1003                         break;
1004
1005                 case 'm':
1006                         monotonic_set = true;
1007                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1008                                 k = -EINVAL;
1009                         break;
1010
1011                 case 't':
1012                         realtime_set = true;
1013                         if (sscanf(item+2, "%llx", &realtime) != 1)
1014                                 k = -EINVAL;
1015                         break;
1016
1017                 case 'x':
1018                         xor_hash_set = true;
1019                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1020                                 k = -EINVAL;
1021                         break;
1022                 }
1023
1024                 free(item);
1025
1026                 if (k < 0)
1027                         return k;
1028         }
1029
1030         if ((!seqnum_set || !seqnum_id_set) &&
1031             (!monotonic_set || !boot_id_set) &&
1032             !realtime_set)
1033                 return -EINVAL;
1034
1035         reset_location(j);
1036
1037         j->current_location.type = LOCATION_SEEK;
1038
1039         if (realtime_set) {
1040                 j->current_location.realtime = (uint64_t) realtime;
1041                 j->current_location.realtime_set = true;
1042         }
1043
1044         if (seqnum_set && seqnum_id_set) {
1045                 j->current_location.seqnum = (uint64_t) seqnum;
1046                 j->current_location.seqnum_id = seqnum_id;
1047                 j->current_location.seqnum_set = true;
1048         }
1049
1050         if (monotonic_set && boot_id_set) {
1051                 j->current_location.monotonic = (uint64_t) monotonic;
1052                 j->current_location.boot_id = boot_id;
1053                 j->current_location.monotonic_set = true;
1054         }
1055
1056         if (xor_hash_set) {
1057                 j->current_location.xor_hash = (uint64_t) xor_hash;
1058                 j->current_location.xor_hash_set = true;
1059         }
1060
1061         return 0;
1062 }
1063
1064 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1065         int r;
1066         char *w, *state;
1067         size_t l;
1068         Object *o;
1069
1070         if (!j)
1071                 return -EINVAL;
1072         if (isempty(cursor))
1073                 return -EINVAL;
1074
1075         if (!j->current_file || j->current_file->current_offset <= 0)
1076                 return -EADDRNOTAVAIL;
1077
1078         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1079         if (r < 0)
1080                 return r;
1081
1082         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1083                 _cleanup_free_ char *item = NULL;
1084                 sd_id128_t id;
1085                 unsigned long long ll;
1086                 int k = 0;
1087
1088                 if (l < 2 || w[1] != '=')
1089                         return -EINVAL;
1090
1091                 item = strndup(w, l);
1092                 if (!item)
1093                         return -ENOMEM;
1094
1095                 switch (w[0]) {
1096
1097                 case 's':
1098                         k = sd_id128_from_string(item+2, &id);
1099                         if (k < 0)
1100                                 return k;
1101                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1102                                 return 0;
1103                         break;
1104
1105                 case 'i':
1106                         if (sscanf(item+2, "%llx", &ll) != 1)
1107                                 return -EINVAL;
1108                         if (ll != le64toh(o->entry.seqnum))
1109                                 return 0;
1110                         break;
1111
1112                 case 'b':
1113                         k = sd_id128_from_string(item+2, &id);
1114                         if (k < 0)
1115                                 return k;
1116                         if (!sd_id128_equal(id, o->entry.boot_id))
1117                                 return 0;
1118                         break;
1119
1120                 case 'm':
1121                         if (sscanf(item+2, "%llx", &ll) != 1)
1122                                 return -EINVAL;
1123                         if (ll != le64toh(o->entry.monotonic))
1124                                 return 0;
1125                         break;
1126
1127                 case 't':
1128                         if (sscanf(item+2, "%llx", &ll) != 1)
1129                                 return -EINVAL;
1130                         if (ll != le64toh(o->entry.realtime))
1131                                 return 0;
1132                         break;
1133
1134                 case 'x':
1135                         if (sscanf(item+2, "%llx", &ll) != 1)
1136                                 return -EINVAL;
1137                         if (ll != le64toh(o->entry.xor_hash))
1138                                 return 0;
1139                         break;
1140                 }
1141         }
1142
1143         return 1;
1144 }
1145
1146
1147 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1148         if (!j)
1149                 return -EINVAL;
1150
1151         reset_location(j);
1152         j->current_location.type = LOCATION_SEEK;
1153         j->current_location.boot_id = boot_id;
1154         j->current_location.monotonic = usec;
1155         j->current_location.monotonic_set = true;
1156
1157         return 0;
1158 }
1159
1160 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1161         if (!j)
1162                 return -EINVAL;
1163
1164         reset_location(j);
1165         j->current_location.type = LOCATION_SEEK;
1166         j->current_location.realtime = usec;
1167         j->current_location.realtime_set = true;
1168
1169         return 0;
1170 }
1171
1172 _public_ int sd_journal_seek_head(sd_journal *j) {
1173         if (!j)
1174                 return -EINVAL;
1175
1176         reset_location(j);
1177         j->current_location.type = LOCATION_HEAD;
1178
1179         return 0;
1180 }
1181
1182 _public_ int sd_journal_seek_tail(sd_journal *j) {
1183         if (!j)
1184                 return -EINVAL;
1185
1186         reset_location(j);
1187         j->current_location.type = LOCATION_TAIL;
1188
1189         return 0;
1190 }
1191
1192 static void check_network(sd_journal *j, int fd) {
1193         struct statfs sfs;
1194
1195         assert(j);
1196
1197         if (j->on_network)
1198                 return;
1199
1200         if (fstatfs(fd, &sfs) < 0)
1201                 return;
1202
1203         j->on_network =
1204                 sfs.f_type == CIFS_MAGIC_NUMBER ||
1205                 sfs.f_type == CODA_SUPER_MAGIC ||
1206                 sfs.f_type == NCP_SUPER_MAGIC ||
1207                 sfs.f_type == NFS_SUPER_MAGIC ||
1208                 sfs.f_type == SMB_SUPER_MAGIC;
1209 }
1210
1211 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1212         char *path;
1213         int r;
1214         JournalFile *f;
1215
1216         assert(j);
1217         assert(prefix);
1218         assert(filename);
1219
1220         if ((j->flags & SD_JOURNAL_SYSTEM_ONLY) &&
1221             !(streq(filename, "system.journal") ||
1222               streq(filename, "system.journal~") ||
1223               (startswith(filename, "system@") &&
1224                (endswith(filename, ".journal") || endswith(filename, ".journal~")))))
1225                 return 0;
1226
1227         path = strjoin(prefix, "/", filename, NULL);
1228         if (!path)
1229                 return -ENOMEM;
1230
1231         if (hashmap_get(j->files, path)) {
1232                 free(path);
1233                 return 0;
1234         }
1235
1236         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1237                 log_debug("Too many open journal files, not adding %s, ignoring.", path);
1238                 free(path);
1239                 return 0;
1240         }
1241
1242         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1243         free(path);
1244
1245         if (r < 0) {
1246                 if (errno == ENOENT)
1247                         return 0;
1248
1249                 return r;
1250         }
1251
1252         /* journal_file_dump(f); */
1253
1254         r = hashmap_put(j->files, f->path, f);
1255         if (r < 0) {
1256                 journal_file_close(f);
1257                 return r;
1258         }
1259
1260         check_network(j, f->fd);
1261
1262         j->current_invalidate_counter ++;
1263
1264         log_debug("File %s got added.", f->path);
1265
1266         return 0;
1267 }
1268
1269 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1270         char *path;
1271         JournalFile *f;
1272
1273         assert(j);
1274         assert(prefix);
1275         assert(filename);
1276
1277         path = strjoin(prefix, "/", filename, NULL);
1278         if (!path)
1279                 return -ENOMEM;
1280
1281         f = hashmap_get(j->files, path);
1282         free(path);
1283         if (!f)
1284                 return 0;
1285
1286         hashmap_remove(j->files, f->path);
1287
1288         log_debug("File %s got removed.", f->path);
1289
1290         if (j->current_file == f) {
1291                 j->current_file = NULL;
1292                 j->current_field = 0;
1293         }
1294
1295         if (j->unique_file == f) {
1296                 j->unique_file = NULL;
1297                 j->unique_offset = 0;
1298         }
1299
1300         journal_file_close(f);
1301
1302         j->current_invalidate_counter ++;
1303
1304         return 0;
1305 }
1306
1307 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1308         char *path;
1309         int r;
1310         DIR *d;
1311         sd_id128_t id, mid;
1312         Directory *m;
1313
1314         assert(j);
1315         assert(prefix);
1316         assert(dirname);
1317
1318         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1319             (sd_id128_from_string(dirname, &id) < 0 ||
1320              sd_id128_get_machine(&mid) < 0 ||
1321              !sd_id128_equal(id, mid)))
1322             return 0;
1323
1324         path = strjoin(prefix, "/", dirname, NULL);
1325         if (!path)
1326                 return -ENOMEM;
1327
1328         d = opendir(path);
1329         if (!d) {
1330                 log_debug("Failed to open %s: %m", path);
1331                 free(path);
1332
1333                 if (errno == ENOENT)
1334                         return 0;
1335                 return -errno;
1336         }
1337
1338         m = hashmap_get(j->directories_by_path, path);
1339         if (!m) {
1340                 m = new0(Directory, 1);
1341                 if (!m) {
1342                         closedir(d);
1343                         free(path);
1344                         return -ENOMEM;
1345                 }
1346
1347                 m->is_root = false;
1348                 m->path = path;
1349
1350                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1351                         closedir(d);
1352                         free(m->path);
1353                         free(m);
1354                         return -ENOMEM;
1355                 }
1356
1357                 j->current_invalidate_counter ++;
1358
1359                 log_debug("Directory %s got added.", m->path);
1360
1361         } else if (m->is_root) {
1362                 free (path);
1363                 closedir(d);
1364                 return 0;
1365         }  else
1366                 free(path);
1367
1368         if (m->wd <= 0 && j->inotify_fd >= 0) {
1369
1370                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1371                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1372                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1373                                           IN_ONLYDIR);
1374
1375                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1376                         inotify_rm_watch(j->inotify_fd, m->wd);
1377         }
1378
1379         for (;;) {
1380                 struct dirent *de;
1381                 union dirent_storage buf;
1382
1383                 r = readdir_r(d, &buf.de, &de);
1384                 if (r != 0 || !de)
1385                         break;
1386
1387                 if (dirent_is_file_with_suffix(de, ".journal") ||
1388                     dirent_is_file_with_suffix(de, ".journal~")) {
1389                         r = add_file(j, m->path, de->d_name);
1390                         if (r < 0)
1391                                 log_debug("Failed to add file %s/%s: %s", m->path, de->d_name, strerror(-r));
1392                 }
1393         }
1394
1395         check_network(j, dirfd(d));
1396
1397         closedir(d);
1398
1399         return 0;
1400 }
1401
1402 static int add_root_directory(sd_journal *j, const char *p) {
1403         DIR *d;
1404         Directory *m;
1405         int r;
1406
1407         assert(j);
1408         assert(p);
1409
1410         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1411             !path_startswith(p, "/run"))
1412                 return -EINVAL;
1413
1414         d = opendir(p);
1415         if (!d)
1416                 return -errno;
1417
1418         m = hashmap_get(j->directories_by_path, p);
1419         if (!m) {
1420                 m = new0(Directory, 1);
1421                 if (!m) {
1422                         closedir(d);
1423                         return -ENOMEM;
1424                 }
1425
1426                 m->is_root = true;
1427                 m->path = strdup(p);
1428                 if (!m->path) {
1429                         closedir(d);
1430                         free(m);
1431                         return -ENOMEM;
1432                 }
1433
1434                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1435                         closedir(d);
1436                         free(m->path);
1437                         free(m);
1438                         return -ENOMEM;
1439                 }
1440
1441                 j->current_invalidate_counter ++;
1442
1443                 log_debug("Root directory %s got added.", m->path);
1444
1445         } else if (!m->is_root) {
1446                 closedir(d);
1447                 return 0;
1448         }
1449
1450         if (m->wd <= 0 && j->inotify_fd >= 0) {
1451
1452                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1453                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1454                                           IN_ONLYDIR);
1455
1456                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1457                         inotify_rm_watch(j->inotify_fd, m->wd);
1458         }
1459
1460         for (;;) {
1461                 struct dirent *de;
1462                 union dirent_storage buf;
1463                 sd_id128_t id;
1464
1465                 r = readdir_r(d, &buf.de, &de);
1466                 if (r != 0 || !de)
1467                         break;
1468
1469                 if (dirent_is_file_with_suffix(de, ".journal") ||
1470                     dirent_is_file_with_suffix(de, ".journal~")) {
1471                         r = add_file(j, m->path, de->d_name);
1472                         if (r < 0)
1473                                 log_debug("Failed to add file %s/%s: %s", m->path, de->d_name, strerror(-r));
1474
1475                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1476                            sd_id128_from_string(de->d_name, &id) >= 0) {
1477
1478                         r = add_directory(j, m->path, de->d_name);
1479                         if (r < 0)
1480                                 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1481                 }
1482         }
1483
1484         check_network(j, dirfd(d));
1485
1486         closedir(d);
1487
1488         return 0;
1489 }
1490
1491 static int remove_directory(sd_journal *j, Directory *d) {
1492         assert(j);
1493
1494         if (d->wd > 0) {
1495                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1496
1497                 if (j->inotify_fd >= 0)
1498                         inotify_rm_watch(j->inotify_fd, d->wd);
1499         }
1500
1501         hashmap_remove(j->directories_by_path, d->path);
1502
1503         if (d->is_root)
1504                 log_debug("Root directory %s got removed.", d->path);
1505         else
1506                 log_debug("Directory %s got removed.", d->path);
1507
1508         free(d->path);
1509         free(d);
1510
1511         return 0;
1512 }
1513
1514 static int add_search_paths(sd_journal *j) {
1515
1516         const char search_paths[] =
1517                 "/run/log/journal\0"
1518                 "/var/log/journal\0";
1519         const char *p;
1520
1521         assert(j);
1522
1523         /* We ignore most errors here, since the idea is to only open
1524          * what's actually accessible, and ignore the rest. */
1525
1526         NULSTR_FOREACH(p, search_paths)
1527                 add_root_directory(j, p);
1528
1529         return 0;
1530 }
1531
1532 static int allocate_inotify(sd_journal *j) {
1533         assert(j);
1534
1535         if (j->inotify_fd < 0) {
1536                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1537                 if (j->inotify_fd < 0)
1538                         return -errno;
1539         }
1540
1541         if (!j->directories_by_wd) {
1542                 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1543                 if (!j->directories_by_wd)
1544                         return -ENOMEM;
1545         }
1546
1547         return 0;
1548 }
1549
1550 static sd_journal *journal_new(int flags, const char *path) {
1551         sd_journal *j;
1552
1553         j = new0(sd_journal, 1);
1554         if (!j)
1555                 return NULL;
1556
1557         j->inotify_fd = -1;
1558         j->flags = flags;
1559
1560         if (path) {
1561                 j->path = strdup(path);
1562                 if (!j->path) {
1563                         free(j);
1564                         return NULL;
1565                 }
1566         }
1567
1568         j->files = hashmap_new(string_hash_func, string_compare_func);
1569         if (!j->files) {
1570                 free(j->path);
1571                 free(j);
1572                 return NULL;
1573         }
1574
1575         j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1576         if (!j->directories_by_path) {
1577                 hashmap_free(j->files);
1578                 free(j->path);
1579                 free(j);
1580                 return NULL;
1581         }
1582
1583         j->mmap = mmap_cache_new();
1584         if (!j->mmap) {
1585                 hashmap_free(j->files);
1586                 hashmap_free(j->directories_by_path);
1587                 free(j->path);
1588                 free(j);
1589                 return NULL;
1590         }
1591
1592         return j;
1593 }
1594
1595 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1596         sd_journal *j;
1597         int r;
1598
1599         if (!ret)
1600                 return -EINVAL;
1601
1602         if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1603                       SD_JOURNAL_RUNTIME_ONLY|
1604                       SD_JOURNAL_SYSTEM_ONLY))
1605                 return -EINVAL;
1606
1607         j = journal_new(flags, NULL);
1608         if (!j)
1609                 return -ENOMEM;
1610
1611         r = add_search_paths(j);
1612         if (r < 0)
1613                 goto fail;
1614
1615         *ret = j;
1616         return 0;
1617
1618 fail:
1619         sd_journal_close(j);
1620
1621         return r;
1622 }
1623
1624 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1625         sd_journal *j;
1626         int r;
1627
1628         if (!ret)
1629                 return -EINVAL;
1630
1631         if (!path || !path_is_absolute(path))
1632                 return -EINVAL;
1633
1634         if (flags != 0)
1635                 return -EINVAL;
1636
1637         j = journal_new(flags, path);
1638         if (!j)
1639                 return -ENOMEM;
1640
1641         r = add_root_directory(j, path);
1642         if (r < 0)
1643                 goto fail;
1644
1645         *ret = j;
1646         return 0;
1647
1648 fail:
1649         sd_journal_close(j);
1650
1651         return r;
1652 }
1653
1654 _public_ void sd_journal_close(sd_journal *j) {
1655         Directory *d;
1656         JournalFile *f;
1657
1658         if (!j)
1659                 return;
1660
1661         while ((f = hashmap_steal_first(j->files)))
1662                 journal_file_close(f);
1663
1664         hashmap_free(j->files);
1665
1666         while ((d = hashmap_first(j->directories_by_path)))
1667                 remove_directory(j, d);
1668
1669         while ((d = hashmap_first(j->directories_by_wd)))
1670                 remove_directory(j, d);
1671
1672         hashmap_free(j->directories_by_path);
1673         hashmap_free(j->directories_by_wd);
1674
1675         if (j->inotify_fd >= 0)
1676                 close_nointr_nofail(j->inotify_fd);
1677
1678         sd_journal_flush_matches(j);
1679
1680         if (j->mmap)
1681                 mmap_cache_unref(j->mmap);
1682
1683         free(j->path);
1684         free(j->unique_field);
1685         free(j);
1686 }
1687
1688 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1689         Object *o;
1690         JournalFile *f;
1691         int r;
1692
1693         if (!j)
1694                 return -EINVAL;
1695         if (!ret)
1696                 return -EINVAL;
1697
1698         f = j->current_file;
1699         if (!f)
1700                 return -EADDRNOTAVAIL;
1701
1702         if (f->current_offset <= 0)
1703                 return -EADDRNOTAVAIL;
1704
1705         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1706         if (r < 0)
1707                 return r;
1708
1709         *ret = le64toh(o->entry.realtime);
1710         return 0;
1711 }
1712
1713 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1714         Object *o;
1715         JournalFile *f;
1716         int r;
1717         sd_id128_t id;
1718
1719         if (!j)
1720                 return -EINVAL;
1721
1722         f = j->current_file;
1723         if (!f)
1724                 return -EADDRNOTAVAIL;
1725
1726         if (f->current_offset <= 0)
1727                 return -EADDRNOTAVAIL;
1728
1729         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1730         if (r < 0)
1731                 return r;
1732
1733         if (ret_boot_id)
1734                 *ret_boot_id = o->entry.boot_id;
1735         else {
1736                 r = sd_id128_get_boot(&id);
1737                 if (r < 0)
1738                         return r;
1739
1740                 if (!sd_id128_equal(id, o->entry.boot_id))
1741                         return -ESTALE;
1742         }
1743
1744         if (ret)
1745                 *ret = le64toh(o->entry.monotonic);
1746
1747         return 0;
1748 }
1749
1750 static bool field_is_valid(const char *field) {
1751         const char *p;
1752
1753         assert(field);
1754
1755         if (isempty(field))
1756                 return false;
1757
1758         if (startswith(field, "__"))
1759                 return false;
1760
1761         for (p = field; *p; p++) {
1762
1763                 if (*p == '_')
1764                         continue;
1765
1766                 if (*p >= 'A' && *p <= 'Z')
1767                         continue;
1768
1769                 if (*p >= '0' && *p <= '9')
1770                         continue;
1771
1772                 return false;
1773         }
1774
1775         return true;
1776 }
1777
1778 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1779         JournalFile *f;
1780         uint64_t i, n;
1781         size_t field_length;
1782         int r;
1783         Object *o;
1784
1785         if (!j)
1786                 return -EINVAL;
1787         if (!field)
1788                 return -EINVAL;
1789         if (!data)
1790                 return -EINVAL;
1791         if (!size)
1792                 return -EINVAL;
1793
1794         if (!field_is_valid(field))
1795                 return -EINVAL;
1796
1797         f = j->current_file;
1798         if (!f)
1799                 return -EADDRNOTAVAIL;
1800
1801         if (f->current_offset <= 0)
1802                 return -EADDRNOTAVAIL;
1803
1804         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1805         if (r < 0)
1806                 return r;
1807
1808         field_length = strlen(field);
1809
1810         n = journal_file_entry_n_items(o);
1811         for (i = 0; i < n; i++) {
1812                 uint64_t p, l;
1813                 le64_t le_hash;
1814                 size_t t;
1815
1816                 p = le64toh(o->entry.items[i].object_offset);
1817                 le_hash = o->entry.items[i].hash;
1818                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1819                 if (r < 0)
1820                         return r;
1821
1822                 if (le_hash != o->data.hash)
1823                         return -EBADMSG;
1824
1825                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1826
1827                 if (o->object.flags & OBJECT_COMPRESSED) {
1828
1829 #ifdef HAVE_XZ
1830                         if (uncompress_startswith(o->data.payload, l,
1831                                                   &f->compress_buffer, &f->compress_buffer_size,
1832                                                   field, field_length, '=')) {
1833
1834                                 uint64_t rsize;
1835
1836                                 if (!uncompress_blob(o->data.payload, l,
1837                                                      &f->compress_buffer, &f->compress_buffer_size, &rsize))
1838                                         return -EBADMSG;
1839
1840                                 *data = f->compress_buffer;
1841                                 *size = (size_t) rsize;
1842
1843                                 return 0;
1844                         }
1845 #else
1846                         return -EPROTONOSUPPORT;
1847 #endif
1848
1849                 } else if (l >= field_length+1 &&
1850                            memcmp(o->data.payload, field, field_length) == 0 &&
1851                            o->data.payload[field_length] == '=') {
1852
1853                         t = (size_t) l;
1854
1855                         if ((uint64_t) t != l)
1856                                 return -E2BIG;
1857
1858                         *data = o->data.payload;
1859                         *size = t;
1860
1861                         return 0;
1862                 }
1863
1864                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1865                 if (r < 0)
1866                         return r;
1867         }
1868
1869         return -ENOENT;
1870 }
1871
1872 static int return_data(JournalFile *f, Object *o, const void **data, size_t *size) {
1873         size_t t;
1874         uint64_t l;
1875
1876         l = le64toh(o->object.size) - offsetof(Object, data.payload);
1877         t = (size_t) l;
1878
1879         /* We can't read objects larger than 4G on a 32bit machine */
1880         if ((uint64_t) t != l)
1881                 return -E2BIG;
1882
1883         if (o->object.flags & OBJECT_COMPRESSED) {
1884 #ifdef HAVE_XZ
1885                 uint64_t rsize;
1886
1887                 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize))
1888                         return -EBADMSG;
1889
1890                 *data = f->compress_buffer;
1891                 *size = (size_t) rsize;
1892 #else
1893                 return -EPROTONOSUPPORT;
1894 #endif
1895         } else {
1896                 *data = o->data.payload;
1897                 *size = t;
1898         }
1899
1900         return 0;
1901 }
1902
1903 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1904         JournalFile *f;
1905         uint64_t p, n;
1906         le64_t le_hash;
1907         int r;
1908         Object *o;
1909
1910         if (!j)
1911                 return -EINVAL;
1912         if (!data)
1913                 return -EINVAL;
1914         if (!size)
1915                 return -EINVAL;
1916
1917         f = j->current_file;
1918         if (!f)
1919                 return -EADDRNOTAVAIL;
1920
1921         if (f->current_offset <= 0)
1922                 return -EADDRNOTAVAIL;
1923
1924         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1925         if (r < 0)
1926                 return r;
1927
1928         n = journal_file_entry_n_items(o);
1929         if (j->current_field >= n)
1930                 return 0;
1931
1932         p = le64toh(o->entry.items[j->current_field].object_offset);
1933         le_hash = o->entry.items[j->current_field].hash;
1934         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1935         if (r < 0)
1936                 return r;
1937
1938         if (le_hash != o->data.hash)
1939                 return -EBADMSG;
1940
1941         r = return_data(f, o, data, size);
1942         if (r < 0)
1943                 return r;
1944
1945         j->current_field ++;
1946
1947         return 1;
1948 }
1949
1950 _public_ void sd_journal_restart_data(sd_journal *j) {
1951         if (!j)
1952                 return;
1953
1954         j->current_field = 0;
1955 }
1956
1957 _public_ int sd_journal_get_fd(sd_journal *j) {
1958         int r;
1959
1960         if (!j)
1961                 return -EINVAL;
1962
1963         if (j->inotify_fd >= 0)
1964                 return j->inotify_fd;
1965
1966         r = allocate_inotify(j);
1967         if (r < 0)
1968                 return r;
1969
1970         /* Iterate through all dirs again, to add them to the
1971          * inotify */
1972         if (j->path)
1973                 r = add_root_directory(j, j->path);
1974         else
1975                 r = add_search_paths(j);
1976         if (r < 0)
1977                 return r;
1978
1979         return j->inotify_fd;
1980 }
1981
1982 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
1983         Directory *d;
1984         int r;
1985
1986         assert(j);
1987         assert(e);
1988
1989         /* Is this a subdirectory we watch? */
1990         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
1991         if (d) {
1992                 sd_id128_t id;
1993
1994                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
1995                     (endswith(e->name, ".journal") ||
1996                      endswith(e->name, ".journal~"))) {
1997
1998                         /* Event for a journal file */
1999
2000                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2001                                 r = add_file(j, d->path, e->name);
2002                                 if (r < 0)
2003                                         log_debug("Failed to add file %s/%s: %s", d->path, e->name, strerror(-r));
2004
2005                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2006
2007                                 r = remove_file(j, d->path, e->name);
2008                                 if (r < 0)
2009                                         log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2010                         }
2011
2012                 } else if (!d->is_root && e->len == 0) {
2013
2014                         /* Event for a subdirectory */
2015
2016                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2017                                 r = remove_directory(j, d);
2018                                 if (r < 0)
2019                                         log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2020                         }
2021
2022
2023                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2024
2025                         /* Event for root directory */
2026
2027                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2028                                 r = add_directory(j, d->path, e->name);
2029                                 if (r < 0)
2030                                         log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2031                         }
2032                 }
2033
2034                 return;
2035         }
2036
2037         if (e->mask & IN_IGNORED)
2038                 return;
2039
2040         log_warning("Unknown inotify event.");
2041 }
2042
2043 static int determine_change(sd_journal *j) {
2044         bool b;
2045
2046         assert(j);
2047
2048         b = j->current_invalidate_counter != j->last_invalidate_counter;
2049         j->last_invalidate_counter = j->current_invalidate_counter;
2050
2051         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2052 }
2053
2054 _public_ int sd_journal_process(sd_journal *j) {
2055         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2056         bool got_something = false;
2057
2058         if (!j)
2059                 return -EINVAL;
2060
2061         for (;;) {
2062                 struct inotify_event *e;
2063                 ssize_t l;
2064
2065                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2066                 if (l < 0) {
2067                         if (errno == EAGAIN || errno == EINTR)
2068                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2069
2070                         return -errno;
2071                 }
2072
2073                 got_something = true;
2074
2075                 e = (struct inotify_event*) buffer;
2076                 while (l > 0) {
2077                         size_t step;
2078
2079                         process_inotify_event(j, e);
2080
2081                         step = sizeof(struct inotify_event) + e->len;
2082                         assert(step <= (size_t) l);
2083
2084                         e = (struct inotify_event*) ((uint8_t*) e + step);
2085                         l -= step;
2086                 }
2087         }
2088
2089         return determine_change(j);
2090 }
2091
2092 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2093         int r;
2094
2095         assert(j);
2096
2097         if (j->inotify_fd < 0) {
2098
2099                 /* This is the first invocation, hence create the
2100                  * inotify watch */
2101                 r = sd_journal_get_fd(j);
2102                 if (r < 0)
2103                         return r;
2104
2105                 /* The journal might have changed since the context
2106                  * object was created and we weren't watching before,
2107                  * hence don't wait for anything, and return
2108                  * immediately. */
2109                 return determine_change(j);
2110         }
2111
2112         if (j->on_network) {
2113                 /* If we are on the network we need to regularly check
2114                  * for changes manually */
2115
2116                 if (timeout_usec == (uint64_t) -1 || timeout_usec > JOURNAL_FILES_RECHECK_USEC)
2117                         timeout_usec = JOURNAL_FILES_RECHECK_USEC;
2118         }
2119
2120         do {
2121                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2122         } while (r == -EINTR);
2123
2124         if (r < 0)
2125                 return r;
2126
2127         return sd_journal_process(j);
2128 }
2129
2130 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2131         Iterator i;
2132         JournalFile *f;
2133         bool first = true;
2134         int r;
2135
2136         if (!j)
2137                 return -EINVAL;
2138         if (!from && !to)
2139                 return -EINVAL;
2140
2141         HASHMAP_FOREACH(f, j->files, i) {
2142                 usec_t fr, t;
2143
2144                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2145                 if (r == -ENOENT)
2146                         continue;
2147                 if (r < 0)
2148                         return r;
2149                 if (r == 0)
2150                         continue;
2151
2152                 if (first) {
2153                         if (from)
2154                                 *from = fr;
2155                         if (to)
2156                                 *to = t;
2157                         first = false;
2158                 } else {
2159                         if (from)
2160                                 *from = MIN(fr, *from);
2161                         if (to)
2162                                 *to = MAX(t, *to);
2163                 }
2164         }
2165
2166         return first ? 0 : 1;
2167 }
2168
2169 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2170         Iterator i;
2171         JournalFile *f;
2172         bool first = true;
2173         int r;
2174
2175         if (!j)
2176                 return -EINVAL;
2177         if (!from && !to)
2178                 return -EINVAL;
2179
2180         HASHMAP_FOREACH(f, j->files, i) {
2181                 usec_t fr, t;
2182
2183                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2184                 if (r == -ENOENT)
2185                         continue;
2186                 if (r < 0)
2187                         return r;
2188                 if (r == 0)
2189                         continue;
2190
2191                 if (first) {
2192                         if (from)
2193                                 *from = fr;
2194                         if (to)
2195                                 *to = t;
2196                         first = false;
2197                 } else {
2198                         if (from)
2199                                 *from = MIN(fr, *from);
2200                         if (to)
2201                                 *to = MAX(t, *to);
2202                 }
2203         }
2204
2205         return first ? 0 : 1;
2206 }
2207
2208 void journal_print_header(sd_journal *j) {
2209         Iterator i;
2210         JournalFile *f;
2211         bool newline = false;
2212
2213         assert(j);
2214
2215         HASHMAP_FOREACH(f, j->files, i) {
2216                 if (newline)
2217                         putchar('\n');
2218                 else
2219                         newline = true;
2220
2221                 journal_file_print_header(f);
2222         }
2223 }
2224
2225 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2226         Iterator i;
2227         JournalFile *f;
2228         uint64_t sum = 0;
2229
2230         if (!j)
2231                 return -EINVAL;
2232         if (!bytes)
2233                 return -EINVAL;
2234
2235         HASHMAP_FOREACH(f, j->files, i) {
2236                 struct stat st;
2237
2238                 if (fstat(f->fd, &st) < 0)
2239                         return -errno;
2240
2241                 sum += (uint64_t) st.st_blocks * 512ULL;
2242         }
2243
2244         *bytes = sum;
2245         return 0;
2246 }
2247
2248 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2249         char *f;
2250
2251         if (!j)
2252                 return -EINVAL;
2253         if (isempty(field))
2254                 return -EINVAL;
2255         if (!field_is_valid(field))
2256                 return -EINVAL;
2257
2258         f = strdup(field);
2259         if (!f)
2260                 return -ENOMEM;
2261
2262         free(j->unique_field);
2263         j->unique_field = f;
2264         j->unique_file = NULL;
2265         j->unique_offset = 0;
2266
2267         return 0;
2268 }
2269
2270 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2271         Object *o;
2272         size_t k;
2273         int r;
2274
2275         if (!j)
2276                 return -EINVAL;
2277         if (!data)
2278                 return -EINVAL;
2279         if (!l)
2280                 return -EINVAL;
2281         if (!j->unique_field)
2282                 return -EINVAL;
2283
2284         k = strlen(j->unique_field);
2285
2286         if (!j->unique_file) {
2287                 j->unique_file = hashmap_first(j->files);
2288                 if (!j->unique_file)
2289                         return 0;
2290                 j->unique_offset = 0;
2291         }
2292
2293         for (;;) {
2294                 JournalFile *of;
2295                 Iterator i;
2296                 const void *odata;
2297                 size_t ol;
2298                 bool found;
2299
2300                 /* Proceed to next data object in the field's linked list */
2301                 if (j->unique_offset == 0) {
2302                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2303                         if (r < 0)
2304                                 return r;
2305
2306                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2307                 } else {
2308                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2309                         if (r < 0)
2310                                 return r;
2311
2312                         j->unique_offset = le64toh(o->data.next_field_offset);
2313                 }
2314
2315                 /* We reached the end of the list? Then start again, with the next file */
2316                 if (j->unique_offset == 0) {
2317                         JournalFile *n;
2318
2319                         n = hashmap_next(j->files, j->unique_file->path);
2320                         if (!n)
2321                                 return 0;
2322
2323                         j->unique_file = n;
2324                         continue;
2325                 }
2326
2327                 /* We do not use the type context here, but 0 instead,
2328                  * so that we can look at this data object at the same
2329                  * time as one on another file */
2330                 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2331                 if (r < 0)
2332                         return r;
2333
2334                 /* Let's do the type check by hand, since we used 0 context above. */
2335                 if (o->object.type != OBJECT_DATA)
2336                         return -EBADMSG;
2337
2338                 r = return_data(j->unique_file, o, &odata, &ol);
2339                 if (r < 0)
2340                         return r;
2341
2342                 /* OK, now let's see if we already returned this data
2343                  * object by checking if it exists in the earlier
2344                  * traversed files. */
2345                 found = false;
2346                 HASHMAP_FOREACH(of, j->files, i) {
2347                         Object *oo;
2348                         uint64_t op;
2349
2350                         if (of == j->unique_file)
2351                                 break;
2352
2353                         /* Skip this file it didn't have any fields
2354                          * indexed */
2355                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2356                             le64toh(of->header->n_fields) <= 0)
2357                                 continue;
2358
2359                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2360                         if (r < 0)
2361                                 return r;
2362
2363                         if (r > 0)
2364                                 found = true;
2365                 }
2366
2367                 if (found)
2368                         continue;
2369
2370                 r = return_data(j->unique_file, o, data, l);
2371                 if (r < 0)
2372                         return r;
2373
2374                 return 1;
2375         }
2376 }
2377
2378 _public_ void sd_journal_restart_unique(sd_journal *j) {
2379         if (!j)
2380                 return;
2381
2382         j->unique_file = NULL;
2383         j->unique_offset = 0;
2384 }
2385
2386 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2387         if (!j)
2388                 return -EINVAL;
2389
2390         return !j->on_network;
2391 }