1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
26 #include <sys/inotify.h>
29 #include <linux/magic.h>
31 #include "sd-journal.h"
32 #include "journal-def.h"
33 #include "journal-file.h"
36 #include "path-util.h"
39 #include "journal-internal.h"
42 #include "replace-var.h"
44 #define JOURNAL_FILES_MAX 1024
46 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
48 #define REPLACE_VAR_MAX 256
50 static void detach_location(sd_journal *j) {
56 j->current_file = NULL;
59 HASHMAP_FOREACH(f, j->files, i)
60 f->current_offset = 0;
63 static void reset_location(sd_journal *j) {
67 zero(j->current_location);
70 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
72 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
74 assert(o->object.type == OBJECT_ENTRY);
77 l->seqnum = le64toh(o->entry.seqnum);
78 l->seqnum_id = f->header->seqnum_id;
79 l->realtime = le64toh(o->entry.realtime);
80 l->monotonic = le64toh(o->entry.monotonic);
81 l->boot_id = o->entry.boot_id;
82 l->xor_hash = le64toh(o->entry.xor_hash);
84 l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
87 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o, uint64_t offset) {
89 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
93 init_location(&j->current_location, type, f, o);
98 f->current_offset = offset;
101 static int match_is_valid(const void *data, size_t size) {
109 if (startswith(data, "__"))
113 for (p = b; p < b + size; p++) {
121 if (*p >= 'A' && *p <= 'Z')
124 if (*p >= '0' && *p <= '9')
133 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
134 const uint8_t *a = _a, *b = _b;
137 for (j = 0; j < s && j < t; j++) {
149 static Match *match_new(Match *p, MatchType t) {
160 LIST_PREPEND(Match, matches, p->matches, m);
166 static void match_free(Match *m) {
170 match_free(m->matches);
173 LIST_REMOVE(Match, matches, m->parent->matches, m);
179 static void match_free_if_empty(Match *m) {
188 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
189 Match *l2, *l3, *add_here = NULL, *m;
201 if (!match_is_valid(data, size))
207 * level 3: concrete matches */
210 j->level0 = match_new(NULL, MATCH_OR_TERM);
216 j->level1 = match_new(j->level0, MATCH_AND_TERM);
221 assert(j->level0->type == MATCH_OR_TERM);
222 assert(j->level1->type == MATCH_AND_TERM);
224 le_hash = htole64(hash64(data, size));
226 LIST_FOREACH(matches, l2, j->level1->matches) {
227 assert(l2->type == MATCH_OR_TERM);
229 LIST_FOREACH(matches, l3, l2->matches) {
230 assert(l3->type == MATCH_DISCRETE);
232 /* Exactly the same match already? Then ignore
234 if (l3->le_hash == le_hash &&
236 memcmp(l3->data, data, size) == 0)
239 /* Same field? Then let's add this to this OR term */
240 if (same_field(data, size, l3->data, l3->size)) {
251 add_here = match_new(j->level1, MATCH_OR_TERM);
256 m = match_new(add_here, MATCH_DISCRETE);
260 m->le_hash = le_hash;
262 m->data = memdup(data, size);
272 match_free_if_empty(add_here);
275 match_free_if_empty(j->level1);
278 match_free_if_empty(j->level0);
283 _public_ int sd_journal_add_disjunction(sd_journal *j) {
294 if (!j->level1->matches)
297 m = match_new(j->level0, MATCH_AND_TERM);
305 static char *match_make_string(Match *m) {
308 bool enclose = false;
313 if (m->type == MATCH_DISCRETE)
314 return strndup(m->data, m->size);
317 LIST_FOREACH(matches, i, m->matches) {
320 t = match_make_string(i);
327 k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
344 r = strjoin("(", p, ")", NULL);
352 char *journal_make_match_string(sd_journal *j) {
355 return match_make_string(j->level0);
358 _public_ void sd_journal_flush_matches(sd_journal *j) {
364 match_free(j->level0);
366 j->level0 = j->level1 = NULL;
371 static int compare_entry_order(JournalFile *af, Object *_ao,
372 JournalFile *bf, uint64_t bp) {
382 /* The mmap cache might invalidate the object from the first
383 * file if we look at the one from the second file. Hence
384 * temporarily copy the header of the first one, and look at
386 ao = alloca(offsetof(EntryObject, items));
387 memcpy(ao, _ao, offsetof(EntryObject, items));
389 r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
391 return strcmp(af->path, bf->path);
393 /* We operate on two different files here, hence we can access
394 * two objects at the same time, which we normally can't.
396 * If contents and timestamps match, these entries are
397 * identical, even if the seqnum does not match */
399 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
400 ao->entry.monotonic == bo->entry.monotonic &&
401 ao->entry.realtime == bo->entry.realtime &&
402 ao->entry.xor_hash == bo->entry.xor_hash)
405 if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
407 /* If this is from the same seqnum source, compare
409 a = le64toh(ao->entry.seqnum);
410 b = le64toh(bo->entry.seqnum);
417 /* Wow! This is weird, different data but the same
418 * seqnums? Something is borked, but let's make the
419 * best of it and compare by time. */
422 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
424 /* If the boot id matches compare monotonic time */
425 a = le64toh(ao->entry.monotonic);
426 b = le64toh(bo->entry.monotonic);
434 /* Otherwise compare UTC time */
435 a = le64toh(ao->entry.realtime);
436 b = le64toh(bo->entry.realtime);
443 /* Finally, compare by contents */
444 a = le64toh(ao->entry.xor_hash);
445 b = le64toh(bo->entry.xor_hash);
455 static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
461 assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
463 if (l->monotonic_set &&
464 sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
466 le64toh(ao->entry.realtime) == l->realtime &&
468 le64toh(ao->entry.xor_hash) == l->xor_hash)
472 sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
474 a = le64toh(ao->entry.seqnum);
482 if (l->monotonic_set &&
483 sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
485 a = le64toh(ao->entry.monotonic);
487 if (a < l->monotonic)
489 if (a > l->monotonic)
493 if (l->realtime_set) {
495 a = le64toh(ao->entry.realtime);
503 if (l->xor_hash_set) {
504 a = le64toh(ao->entry.xor_hash);
515 static int next_for_match(
519 uint64_t after_offset,
520 direction_t direction,
532 if (m->type == MATCH_DISCRETE) {
535 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
539 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
541 } else if (m->type == MATCH_OR_TERM) {
544 /* Find the earliest match beyond after_offset */
546 LIST_FOREACH(matches, i, m->matches) {
549 r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
553 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
558 } else if (m->type == MATCH_AND_TERM) {
560 bool continue_looking;
562 /* Always jump to the next matching entry and repeat
563 * this until we fine and offset that matches for all
571 continue_looking = false;
573 LIST_FOREACH(matches, i, m->matches) {
577 limit = after_offset;
578 else if (direction == DIRECTION_DOWN)
579 limit = MAX(np, after_offset);
581 limit = MIN(np, after_offset);
583 r = next_for_match(j, i, f, limit, direction, NULL, &cp);
587 if ((direction == DIRECTION_DOWN ? cp >= after_offset : cp <= after_offset) &&
588 (np == 0 || (direction == DIRECTION_DOWN ? cp > np : np < cp))) {
590 continue_looking = true;
594 } while (continue_looking);
600 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
612 static int find_location_for_match(
616 direction_t direction,
626 if (m->type == MATCH_DISCRETE) {
629 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
633 /* FIXME: missing: find by monotonic */
635 if (j->current_location.type == LOCATION_HEAD)
636 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
637 if (j->current_location.type == LOCATION_TAIL)
638 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
639 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
640 return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
641 if (j->current_location.monotonic_set) {
642 r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
646 if (j->current_location.realtime_set)
647 return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
649 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
651 } else if (m->type == MATCH_OR_TERM) {
656 /* Find the earliest match */
658 LIST_FOREACH(matches, i, m->matches) {
661 r = find_location_for_match(j, i, f, direction, NULL, &cp);
665 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
673 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
688 assert(m->type == MATCH_AND_TERM);
690 /* First jump to the last match, and then find the
691 * next one where all matches match */
696 LIST_FOREACH(matches, i, m->matches) {
699 r = find_location_for_match(j, i, f, direction, NULL, &cp);
703 if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
707 return next_for_match(j, m, f, np, direction, ret, offset);
711 static int find_location_with_matches(
714 direction_t direction,
726 /* No matches is simple */
728 if (j->current_location.type == LOCATION_HEAD)
729 return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
730 if (j->current_location.type == LOCATION_TAIL)
731 return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
732 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
733 return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
734 if (j->current_location.monotonic_set) {
735 r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
739 if (j->current_location.realtime_set)
740 return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
742 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
744 return find_location_for_match(j, j->level0, f, direction, ret, offset);
747 static int next_with_matches(
750 direction_t direction,
765 /* No matches is easy. We simple advance the file
768 return journal_file_next_entry(f, c, cp, direction, ret, offset);
770 /* If we have a match then we look for the next matching entry
771 * with an offset at least one step larger */
772 return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
775 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
783 if (f->current_offset > 0) {
784 cp = f->current_offset;
786 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
790 r = next_with_matches(j, f, direction, &c, &cp);
794 r = find_location_with_matches(j, f, direction, &c, &cp);
799 /* OK, we found the spot, now let's advance until to an entry
800 * that is actually different from what we were previously
801 * looking at. This is necessary to handle entries which exist
802 * in two (or more) journal files, and which shall all be
803 * suppressed but one. */
808 if (j->current_location.type == LOCATION_DISCRETE) {
811 k = compare_with_location(f, c, &j->current_location);
812 if (direction == DIRECTION_DOWN)
827 r = next_with_matches(j, f, direction, &c, &cp);
833 static int real_journal_next(sd_journal *j, direction_t direction) {
834 JournalFile *f, *new_file = NULL;
835 uint64_t new_offset = 0;
844 HASHMAP_FOREACH(f, j->files, i) {
847 r = next_beyond_location(j, f, direction, &o, &p);
849 log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
859 k = compare_entry_order(f, o, new_file, new_offset);
861 if (direction == DIRECTION_DOWN)
876 r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
880 set_location(j, LOCATION_DISCRETE, new_file, o, new_offset);
885 _public_ int sd_journal_next(sd_journal *j) {
886 return real_journal_next(j, DIRECTION_DOWN);
889 _public_ int sd_journal_previous(sd_journal *j) {
890 return real_journal_next(j, DIRECTION_UP);
893 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
900 /* If this is not a discrete skip, then at least
901 * resolve the current location */
902 if (j->current_location.type != LOCATION_DISCRETE)
903 return real_journal_next(j, direction);
909 r = real_journal_next(j, direction);
923 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
924 return real_journal_next_skip(j, DIRECTION_DOWN, skip);
927 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
928 return real_journal_next_skip(j, DIRECTION_UP, skip);
931 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
934 char bid[33], sid[33];
941 if (!j->current_file || j->current_file->current_offset <= 0)
942 return -EADDRNOTAVAIL;
944 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
948 sd_id128_to_string(j->current_file->header->seqnum_id, sid);
949 sd_id128_to_string(o->entry.boot_id, bid);
952 "s=%s;i=%llx;b=%s;m=%llx;t=%llx;x=%llx",
953 sid, (unsigned long long) le64toh(o->entry.seqnum),
954 bid, (unsigned long long) le64toh(o->entry.monotonic),
955 (unsigned long long) le64toh(o->entry.realtime),
956 (unsigned long long) le64toh(o->entry.xor_hash)) < 0)
962 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
965 unsigned long long seqnum, monotonic, realtime, xor_hash;
967 seqnum_id_set = false,
970 monotonic_set = false,
971 realtime_set = false,
972 xor_hash_set = false;
973 sd_id128_t seqnum_id, boot_id;
980 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
984 if (l < 2 || w[1] != '=')
987 item = strndup(w, l);
994 seqnum_id_set = true;
995 k = sd_id128_from_string(item+2, &seqnum_id);
1000 if (sscanf(item+2, "%llx", &seqnum) != 1)
1006 k = sd_id128_from_string(item+2, &boot_id);
1010 monotonic_set = true;
1011 if (sscanf(item+2, "%llx", &monotonic) != 1)
1016 realtime_set = true;
1017 if (sscanf(item+2, "%llx", &realtime) != 1)
1022 xor_hash_set = true;
1023 if (sscanf(item+2, "%llx", &xor_hash) != 1)
1034 if ((!seqnum_set || !seqnum_id_set) &&
1035 (!monotonic_set || !boot_id_set) &&
1041 j->current_location.type = LOCATION_SEEK;
1044 j->current_location.realtime = (uint64_t) realtime;
1045 j->current_location.realtime_set = true;
1048 if (seqnum_set && seqnum_id_set) {
1049 j->current_location.seqnum = (uint64_t) seqnum;
1050 j->current_location.seqnum_id = seqnum_id;
1051 j->current_location.seqnum_set = true;
1054 if (monotonic_set && boot_id_set) {
1055 j->current_location.monotonic = (uint64_t) monotonic;
1056 j->current_location.boot_id = boot_id;
1057 j->current_location.monotonic_set = true;
1061 j->current_location.xor_hash = (uint64_t) xor_hash;
1062 j->current_location.xor_hash_set = true;
1068 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1076 if (isempty(cursor))
1079 if (!j->current_file || j->current_file->current_offset <= 0)
1080 return -EADDRNOTAVAIL;
1082 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1086 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1087 _cleanup_free_ char *item = NULL;
1089 unsigned long long ll;
1092 if (l < 2 || w[1] != '=')
1095 item = strndup(w, l);
1102 k = sd_id128_from_string(item+2, &id);
1105 if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1110 if (sscanf(item+2, "%llx", &ll) != 1)
1112 if (ll != le64toh(o->entry.seqnum))
1117 k = sd_id128_from_string(item+2, &id);
1120 if (!sd_id128_equal(id, o->entry.boot_id))
1125 if (sscanf(item+2, "%llx", &ll) != 1)
1127 if (ll != le64toh(o->entry.monotonic))
1132 if (sscanf(item+2, "%llx", &ll) != 1)
1134 if (ll != le64toh(o->entry.realtime))
1139 if (sscanf(item+2, "%llx", &ll) != 1)
1141 if (ll != le64toh(o->entry.xor_hash))
1151 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1156 j->current_location.type = LOCATION_SEEK;
1157 j->current_location.boot_id = boot_id;
1158 j->current_location.monotonic = usec;
1159 j->current_location.monotonic_set = true;
1164 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1169 j->current_location.type = LOCATION_SEEK;
1170 j->current_location.realtime = usec;
1171 j->current_location.realtime_set = true;
1176 _public_ int sd_journal_seek_head(sd_journal *j) {
1181 j->current_location.type = LOCATION_HEAD;
1186 _public_ int sd_journal_seek_tail(sd_journal *j) {
1191 j->current_location.type = LOCATION_TAIL;
1196 static void check_network(sd_journal *j, int fd) {
1204 if (fstatfs(fd, &sfs) < 0)
1208 (long)sfs.f_type == (long)CIFS_MAGIC_NUMBER ||
1209 sfs.f_type == CODA_SUPER_MAGIC ||
1210 sfs.f_type == NCP_SUPER_MAGIC ||
1211 sfs.f_type == NFS_SUPER_MAGIC ||
1212 sfs.f_type == SMB_SUPER_MAGIC;
1215 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1224 if ((j->flags & SD_JOURNAL_SYSTEM_ONLY) &&
1225 !(streq(filename, "system.journal") ||
1226 streq(filename, "system.journal~") ||
1227 (startswith(filename, "system@") &&
1228 (endswith(filename, ".journal") || endswith(filename, ".journal~")))))
1231 path = strjoin(prefix, "/", filename, NULL);
1235 if (hashmap_get(j->files, path)) {
1240 if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1241 log_debug("Too many open journal files, not adding %s, ignoring.", path);
1246 r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1250 if (errno == ENOENT)
1256 /* journal_file_dump(f); */
1258 r = hashmap_put(j->files, f->path, f);
1260 journal_file_close(f);
1264 check_network(j, f->fd);
1266 j->current_invalidate_counter ++;
1268 log_debug("File %s got added.", f->path);
1273 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1281 path = strjoin(prefix, "/", filename, NULL);
1285 f = hashmap_get(j->files, path);
1290 hashmap_remove(j->files, f->path);
1292 log_debug("File %s got removed.", f->path);
1294 if (j->current_file == f) {
1295 j->current_file = NULL;
1296 j->current_field = 0;
1299 if (j->unique_file == f) {
1300 j->unique_file = NULL;
1301 j->unique_offset = 0;
1304 journal_file_close(f);
1306 j->current_invalidate_counter ++;
1311 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1322 if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1323 (sd_id128_from_string(dirname, &id) < 0 ||
1324 sd_id128_get_machine(&mid) < 0 ||
1325 !sd_id128_equal(id, mid)))
1328 path = strjoin(prefix, "/", dirname, NULL);
1334 log_debug("Failed to open %s: %m", path);
1337 if (errno == ENOENT)
1342 m = hashmap_get(j->directories_by_path, path);
1344 m = new0(Directory, 1);
1354 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1361 j->current_invalidate_counter ++;
1363 log_debug("Directory %s got added.", m->path);
1365 } else if (m->is_root) {
1372 if (m->wd <= 0 && j->inotify_fd >= 0) {
1374 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1375 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1376 IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1379 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1380 inotify_rm_watch(j->inotify_fd, m->wd);
1385 union dirent_storage buf;
1387 r = readdir_r(d, &buf.de, &de);
1391 if (dirent_is_file_with_suffix(de, ".journal") ||
1392 dirent_is_file_with_suffix(de, ".journal~")) {
1393 r = add_file(j, m->path, de->d_name);
1395 log_debug("Failed to add file %s/%s: %s", m->path, de->d_name, strerror(-r));
1399 check_network(j, dirfd(d));
1406 static int add_root_directory(sd_journal *j, const char *p) {
1414 if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1415 !path_startswith(p, "/run"))
1422 m = hashmap_get(j->directories_by_path, p);
1424 m = new0(Directory, 1);
1431 m->path = strdup(p);
1438 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1445 j->current_invalidate_counter ++;
1447 log_debug("Root directory %s got added.", m->path);
1449 } else if (!m->is_root) {
1454 if (m->wd <= 0 && j->inotify_fd >= 0) {
1456 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1457 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1460 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1461 inotify_rm_watch(j->inotify_fd, m->wd);
1466 union dirent_storage buf;
1469 r = readdir_r(d, &buf.de, &de);
1473 if (dirent_is_file_with_suffix(de, ".journal") ||
1474 dirent_is_file_with_suffix(de, ".journal~")) {
1475 r = add_file(j, m->path, de->d_name);
1477 log_debug("Failed to add file %s/%s: %s", m->path, de->d_name, strerror(-r));
1479 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1480 sd_id128_from_string(de->d_name, &id) >= 0) {
1482 r = add_directory(j, m->path, de->d_name);
1484 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1488 check_network(j, dirfd(d));
1495 static int remove_directory(sd_journal *j, Directory *d) {
1499 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1501 if (j->inotify_fd >= 0)
1502 inotify_rm_watch(j->inotify_fd, d->wd);
1505 hashmap_remove(j->directories_by_path, d->path);
1508 log_debug("Root directory %s got removed.", d->path);
1510 log_debug("Directory %s got removed.", d->path);
1518 static int add_search_paths(sd_journal *j) {
1520 const char search_paths[] =
1521 "/run/log/journal\0"
1522 "/var/log/journal\0";
1527 /* We ignore most errors here, since the idea is to only open
1528 * what's actually accessible, and ignore the rest. */
1530 NULSTR_FOREACH(p, search_paths)
1531 add_root_directory(j, p);
1536 static int allocate_inotify(sd_journal *j) {
1539 if (j->inotify_fd < 0) {
1540 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1541 if (j->inotify_fd < 0)
1545 if (!j->directories_by_wd) {
1546 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1547 if (!j->directories_by_wd)
1554 static sd_journal *journal_new(int flags, const char *path) {
1557 j = new0(sd_journal, 1);
1565 j->path = strdup(path);
1572 j->files = hashmap_new(string_hash_func, string_compare_func);
1579 j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1580 if (!j->directories_by_path) {
1581 hashmap_free(j->files);
1587 j->mmap = mmap_cache_new();
1589 hashmap_free(j->files);
1590 hashmap_free(j->directories_by_path);
1599 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1606 if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1607 SD_JOURNAL_RUNTIME_ONLY|
1608 SD_JOURNAL_SYSTEM_ONLY))
1611 j = journal_new(flags, NULL);
1615 r = add_search_paths(j);
1623 sd_journal_close(j);
1628 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1635 if (!path || !path_is_absolute(path))
1641 j = journal_new(flags, path);
1645 r = add_root_directory(j, path);
1653 sd_journal_close(j);
1658 _public_ void sd_journal_close(sd_journal *j) {
1665 while ((f = hashmap_steal_first(j->files)))
1666 journal_file_close(f);
1668 hashmap_free(j->files);
1670 while ((d = hashmap_first(j->directories_by_path)))
1671 remove_directory(j, d);
1673 while ((d = hashmap_first(j->directories_by_wd)))
1674 remove_directory(j, d);
1676 hashmap_free(j->directories_by_path);
1677 hashmap_free(j->directories_by_wd);
1679 if (j->inotify_fd >= 0)
1680 close_nointr_nofail(j->inotify_fd);
1682 sd_journal_flush_matches(j);
1685 mmap_cache_unref(j->mmap);
1688 free(j->unique_field);
1692 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1702 f = j->current_file;
1704 return -EADDRNOTAVAIL;
1706 if (f->current_offset <= 0)
1707 return -EADDRNOTAVAIL;
1709 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1713 *ret = le64toh(o->entry.realtime);
1717 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1726 f = j->current_file;
1728 return -EADDRNOTAVAIL;
1730 if (f->current_offset <= 0)
1731 return -EADDRNOTAVAIL;
1733 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1738 *ret_boot_id = o->entry.boot_id;
1740 r = sd_id128_get_boot(&id);
1744 if (!sd_id128_equal(id, o->entry.boot_id))
1749 *ret = le64toh(o->entry.monotonic);
1754 static bool field_is_valid(const char *field) {
1762 if (startswith(field, "__"))
1765 for (p = field; *p; p++) {
1770 if (*p >= 'A' && *p <= 'Z')
1773 if (*p >= '0' && *p <= '9')
1782 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1785 size_t field_length;
1798 if (!field_is_valid(field))
1801 f = j->current_file;
1803 return -EADDRNOTAVAIL;
1805 if (f->current_offset <= 0)
1806 return -EADDRNOTAVAIL;
1808 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1812 field_length = strlen(field);
1814 n = journal_file_entry_n_items(o);
1815 for (i = 0; i < n; i++) {
1820 p = le64toh(o->entry.items[i].object_offset);
1821 le_hash = o->entry.items[i].hash;
1822 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1826 if (le_hash != o->data.hash)
1829 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1831 if (o->object.flags & OBJECT_COMPRESSED) {
1834 if (uncompress_startswith(o->data.payload, l,
1835 &f->compress_buffer, &f->compress_buffer_size,
1836 field, field_length, '=')) {
1840 if (!uncompress_blob(o->data.payload, l,
1841 &f->compress_buffer, &f->compress_buffer_size, &rsize))
1844 *data = f->compress_buffer;
1845 *size = (size_t) rsize;
1850 return -EPROTONOSUPPORT;
1853 } else if (l >= field_length+1 &&
1854 memcmp(o->data.payload, field, field_length) == 0 &&
1855 o->data.payload[field_length] == '=') {
1859 if ((uint64_t) t != l)
1862 *data = o->data.payload;
1868 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1876 static int return_data(JournalFile *f, Object *o, const void **data, size_t *size) {
1880 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1883 /* We can't read objects larger than 4G on a 32bit machine */
1884 if ((uint64_t) t != l)
1887 if (o->object.flags & OBJECT_COMPRESSED) {
1891 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize))
1894 *data = f->compress_buffer;
1895 *size = (size_t) rsize;
1897 return -EPROTONOSUPPORT;
1900 *data = o->data.payload;
1907 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1921 f = j->current_file;
1923 return -EADDRNOTAVAIL;
1925 if (f->current_offset <= 0)
1926 return -EADDRNOTAVAIL;
1928 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1932 n = journal_file_entry_n_items(o);
1933 if (j->current_field >= n)
1936 p = le64toh(o->entry.items[j->current_field].object_offset);
1937 le_hash = o->entry.items[j->current_field].hash;
1938 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1942 if (le_hash != o->data.hash)
1945 r = return_data(f, o, data, size);
1949 j->current_field ++;
1954 _public_ void sd_journal_restart_data(sd_journal *j) {
1958 j->current_field = 0;
1961 _public_ int sd_journal_get_fd(sd_journal *j) {
1967 if (j->inotify_fd >= 0)
1968 return j->inotify_fd;
1970 r = allocate_inotify(j);
1974 /* Iterate through all dirs again, to add them to the
1977 r = add_root_directory(j, j->path);
1979 r = add_search_paths(j);
1983 return j->inotify_fd;
1986 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
1993 /* Is this a subdirectory we watch? */
1994 d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
1998 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
1999 (endswith(e->name, ".journal") ||
2000 endswith(e->name, ".journal~"))) {
2002 /* Event for a journal file */
2004 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2005 r = add_file(j, d->path, e->name);
2007 log_debug("Failed to add file %s/%s: %s", d->path, e->name, strerror(-r));
2009 } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2011 r = remove_file(j, d->path, e->name);
2013 log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2016 } else if (!d->is_root && e->len == 0) {
2018 /* Event for a subdirectory */
2020 if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2021 r = remove_directory(j, d);
2023 log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2027 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2029 /* Event for root directory */
2031 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2032 r = add_directory(j, d->path, e->name);
2034 log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2041 if (e->mask & IN_IGNORED)
2044 log_warning("Unknown inotify event.");
2047 static int determine_change(sd_journal *j) {
2052 b = j->current_invalidate_counter != j->last_invalidate_counter;
2053 j->last_invalidate_counter = j->current_invalidate_counter;
2055 return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2058 _public_ int sd_journal_process(sd_journal *j) {
2059 uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2060 bool got_something = false;
2066 struct inotify_event *e;
2069 l = read(j->inotify_fd, buffer, sizeof(buffer));
2071 if (errno == EAGAIN || errno == EINTR)
2072 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2077 got_something = true;
2079 e = (struct inotify_event*) buffer;
2083 process_inotify_event(j, e);
2085 step = sizeof(struct inotify_event) + e->len;
2086 assert(step <= (size_t) l);
2088 e = (struct inotify_event*) ((uint8_t*) e + step);
2093 return determine_change(j);
2096 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2101 if (j->inotify_fd < 0) {
2103 /* This is the first invocation, hence create the
2105 r = sd_journal_get_fd(j);
2109 /* The journal might have changed since the context
2110 * object was created and we weren't watching before,
2111 * hence don't wait for anything, and return
2113 return determine_change(j);
2116 if (j->on_network) {
2117 /* If we are on the network we need to regularly check
2118 * for changes manually */
2120 if (timeout_usec == (uint64_t) -1 || timeout_usec > JOURNAL_FILES_RECHECK_USEC)
2121 timeout_usec = JOURNAL_FILES_RECHECK_USEC;
2125 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2126 } while (r == -EINTR);
2131 return sd_journal_process(j);
2134 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2145 HASHMAP_FOREACH(f, j->files, i) {
2148 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2164 *from = MIN(fr, *from);
2170 return first ? 0 : 1;
2173 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2184 HASHMAP_FOREACH(f, j->files, i) {
2187 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2203 *from = MIN(fr, *from);
2209 return first ? 0 : 1;
2212 void journal_print_header(sd_journal *j) {
2215 bool newline = false;
2219 HASHMAP_FOREACH(f, j->files, i) {
2225 journal_file_print_header(f);
2229 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2239 HASHMAP_FOREACH(f, j->files, i) {
2242 if (fstat(f->fd, &st) < 0)
2245 sum += (uint64_t) st.st_blocks * 512ULL;
2252 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2259 if (!field_is_valid(field))
2266 free(j->unique_field);
2267 j->unique_field = f;
2268 j->unique_file = NULL;
2269 j->unique_offset = 0;
2274 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2285 if (!j->unique_field)
2288 k = strlen(j->unique_field);
2290 if (!j->unique_file) {
2291 j->unique_file = hashmap_first(j->files);
2292 if (!j->unique_file)
2294 j->unique_offset = 0;
2304 /* Proceed to next data object in the field's linked list */
2305 if (j->unique_offset == 0) {
2306 r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2310 j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2312 r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2316 j->unique_offset = le64toh(o->data.next_field_offset);
2319 /* We reached the end of the list? Then start again, with the next file */
2320 if (j->unique_offset == 0) {
2323 n = hashmap_next(j->files, j->unique_file->path);
2331 /* We do not use the type context here, but 0 instead,
2332 * so that we can look at this data object at the same
2333 * time as one on another file */
2334 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2338 /* Let's do the type check by hand, since we used 0 context above. */
2339 if (o->object.type != OBJECT_DATA)
2342 r = return_data(j->unique_file, o, &odata, &ol);
2346 /* OK, now let's see if we already returned this data
2347 * object by checking if it exists in the earlier
2348 * traversed files. */
2350 HASHMAP_FOREACH(of, j->files, i) {
2354 if (of == j->unique_file)
2357 /* Skip this file it didn't have any fields
2359 if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2360 le64toh(of->header->n_fields) <= 0)
2363 r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2374 r = return_data(j->unique_file, o, data, l);
2382 _public_ void sd_journal_restart_unique(sd_journal *j) {
2386 j->unique_file = NULL;
2387 j->unique_offset = 0;
2390 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2394 return !j->on_network;
2397 static char *lookup_field(const char *field, void *userdata) {
2398 sd_journal *j = userdata;
2406 r = sd_journal_get_data(j, field, &data, &size);
2408 size > REPLACE_VAR_MAX)
2409 return strdup(field);
2411 d = strlen(field) + 1;
2413 return strndup((const char*) data + d, size - d);
2416 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2420 _cleanup_free_ char *text = NULL, *cid = NULL;
2429 r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2433 cid = strndup((const char*) data + 11, size - 11);
2437 r = sd_id128_from_string(cid, &id);
2441 r = catalog_get(id, &text);
2445 t = replace_var(text, lookup_field, j);
2453 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2457 return catalog_get(id, ret);