1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
26 #include <sys/inotify.h>
29 #include <linux/magic.h>
31 #include "sd-journal.h"
32 #include "journal-def.h"
33 #include "journal-file.h"
36 #include "path-util.h"
39 #include "journal-internal.h"
42 #include "replace-var.h"
44 #define JOURNAL_FILES_MAX 1024
46 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
48 #define REPLACE_VAR_MAX 256
50 #define DEFAULT_DATA_THRESHOLD (64*1024)
52 /* We return an error here only if we didn't manage to
53 memorize the real error. */
54 static int set_put_error(Set* errors, int r) {
58 return set_put(errors, INT_TO_PTR(r));
61 static void detach_location(sd_journal *j) {
67 j->current_file = NULL;
70 HASHMAP_FOREACH(f, j->files, i)
71 f->current_offset = 0;
74 static void reset_location(sd_journal *j) {
78 zero(j->current_location);
81 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
83 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
85 assert(o->object.type == OBJECT_ENTRY);
88 l->seqnum = le64toh(o->entry.seqnum);
89 l->seqnum_id = f->header->seqnum_id;
90 l->realtime = le64toh(o->entry.realtime);
91 l->monotonic = le64toh(o->entry.monotonic);
92 l->boot_id = o->entry.boot_id;
93 l->xor_hash = le64toh(o->entry.xor_hash);
95 l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
98 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o, uint64_t offset) {
100 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
104 init_location(&j->current_location, type, f, o);
107 j->current_field = 0;
109 f->current_offset = offset;
112 static int match_is_valid(const void *data, size_t size) {
120 if (startswith(data, "__"))
124 for (p = b; p < b + size; p++) {
132 if (*p >= 'A' && *p <= 'Z')
135 if (*p >= '0' && *p <= '9')
144 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
145 const uint8_t *a = _a, *b = _b;
148 for (j = 0; j < s && j < t; j++) {
160 static Match *match_new(Match *p, MatchType t) {
171 LIST_PREPEND(Match, matches, p->matches, m);
177 static void match_free(Match *m) {
181 match_free(m->matches);
184 LIST_REMOVE(Match, matches, m->parent->matches, m);
190 static void match_free_if_empty(Match *m) {
199 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
200 Match *l2, *l3, *add_here = NULL, *m;
212 if (!match_is_valid(data, size))
218 * level 3: concrete matches */
221 j->level0 = match_new(NULL, MATCH_OR_TERM);
227 j->level1 = match_new(j->level0, MATCH_AND_TERM);
232 assert(j->level0->type == MATCH_OR_TERM);
233 assert(j->level1->type == MATCH_AND_TERM);
235 le_hash = htole64(hash64(data, size));
237 LIST_FOREACH(matches, l2, j->level1->matches) {
238 assert(l2->type == MATCH_OR_TERM);
240 LIST_FOREACH(matches, l3, l2->matches) {
241 assert(l3->type == MATCH_DISCRETE);
243 /* Exactly the same match already? Then ignore
245 if (l3->le_hash == le_hash &&
247 memcmp(l3->data, data, size) == 0)
250 /* Same field? Then let's add this to this OR term */
251 if (same_field(data, size, l3->data, l3->size)) {
262 add_here = match_new(j->level1, MATCH_OR_TERM);
267 m = match_new(add_here, MATCH_DISCRETE);
271 m->le_hash = le_hash;
273 m->data = memdup(data, size);
283 match_free_if_empty(add_here);
286 match_free_if_empty(j->level1);
289 match_free_if_empty(j->level0);
294 _public_ int sd_journal_add_disjunction(sd_journal *j) {
305 if (!j->level1->matches)
308 m = match_new(j->level0, MATCH_AND_TERM);
316 static char *match_make_string(Match *m) {
319 bool enclose = false;
324 if (m->type == MATCH_DISCRETE)
325 return strndup(m->data, m->size);
328 LIST_FOREACH(matches, i, m->matches) {
331 t = match_make_string(i);
338 k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
355 r = strjoin("(", p, ")", NULL);
363 char *journal_make_match_string(sd_journal *j) {
366 return match_make_string(j->level0);
369 _public_ void sd_journal_flush_matches(sd_journal *j) {
375 match_free(j->level0);
377 j->level0 = j->level1 = NULL;
382 static int compare_entry_order(JournalFile *af, Object *_ao,
383 JournalFile *bf, uint64_t bp) {
393 /* The mmap cache might invalidate the object from the first
394 * file if we look at the one from the second file. Hence
395 * temporarily copy the header of the first one, and look at
397 ao = alloca(offsetof(EntryObject, items));
398 memcpy(ao, _ao, offsetof(EntryObject, items));
400 r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
402 return strcmp(af->path, bf->path);
404 /* We operate on two different files here, hence we can access
405 * two objects at the same time, which we normally can't.
407 * If contents and timestamps match, these entries are
408 * identical, even if the seqnum does not match */
410 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
411 ao->entry.monotonic == bo->entry.monotonic &&
412 ao->entry.realtime == bo->entry.realtime &&
413 ao->entry.xor_hash == bo->entry.xor_hash)
416 if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
418 /* If this is from the same seqnum source, compare
420 a = le64toh(ao->entry.seqnum);
421 b = le64toh(bo->entry.seqnum);
428 /* Wow! This is weird, different data but the same
429 * seqnums? Something is borked, but let's make the
430 * best of it and compare by time. */
433 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
435 /* If the boot id matches compare monotonic time */
436 a = le64toh(ao->entry.monotonic);
437 b = le64toh(bo->entry.monotonic);
445 /* Otherwise compare UTC time */
446 a = le64toh(ao->entry.realtime);
447 b = le64toh(bo->entry.realtime);
454 /* Finally, compare by contents */
455 a = le64toh(ao->entry.xor_hash);
456 b = le64toh(bo->entry.xor_hash);
466 static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
472 assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
474 if (l->monotonic_set &&
475 sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
477 le64toh(ao->entry.realtime) == l->realtime &&
479 le64toh(ao->entry.xor_hash) == l->xor_hash)
483 sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
485 a = le64toh(ao->entry.seqnum);
493 if (l->monotonic_set &&
494 sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
496 a = le64toh(ao->entry.monotonic);
498 if (a < l->monotonic)
500 if (a > l->monotonic)
504 if (l->realtime_set) {
506 a = le64toh(ao->entry.realtime);
514 if (l->xor_hash_set) {
515 a = le64toh(ao->entry.xor_hash);
526 static int next_for_match(
530 uint64_t after_offset,
531 direction_t direction,
543 if (m->type == MATCH_DISCRETE) {
546 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
550 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
552 } else if (m->type == MATCH_OR_TERM) {
555 /* Find the earliest match beyond after_offset */
557 LIST_FOREACH(matches, i, m->matches) {
560 r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
564 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
569 } else if (m->type == MATCH_AND_TERM) {
571 bool continue_looking;
573 /* Always jump to the next matching entry and repeat
574 * this until we fine and offset that matches for all
582 continue_looking = false;
584 LIST_FOREACH(matches, i, m->matches) {
588 limit = after_offset;
589 else if (direction == DIRECTION_DOWN)
590 limit = MAX(np, after_offset);
592 limit = MIN(np, after_offset);
594 r = next_for_match(j, i, f, limit, direction, NULL, &cp);
598 if ((direction == DIRECTION_DOWN ? cp >= after_offset : cp <= after_offset) &&
599 (np == 0 || (direction == DIRECTION_DOWN ? cp > np : np < cp))) {
601 continue_looking = true;
605 } while (continue_looking);
611 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
623 static int find_location_for_match(
627 direction_t direction,
637 if (m->type == MATCH_DISCRETE) {
640 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
644 /* FIXME: missing: find by monotonic */
646 if (j->current_location.type == LOCATION_HEAD)
647 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
648 if (j->current_location.type == LOCATION_TAIL)
649 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
650 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
651 return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
652 if (j->current_location.monotonic_set) {
653 r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
657 if (j->current_location.realtime_set)
658 return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
660 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
662 } else if (m->type == MATCH_OR_TERM) {
667 /* Find the earliest match */
669 LIST_FOREACH(matches, i, m->matches) {
672 r = find_location_for_match(j, i, f, direction, NULL, &cp);
676 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
684 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
699 assert(m->type == MATCH_AND_TERM);
701 /* First jump to the last match, and then find the
702 * next one where all matches match */
707 LIST_FOREACH(matches, i, m->matches) {
710 r = find_location_for_match(j, i, f, direction, NULL, &cp);
714 if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
718 return next_for_match(j, m, f, np, direction, ret, offset);
722 static int find_location_with_matches(
725 direction_t direction,
737 /* No matches is simple */
739 if (j->current_location.type == LOCATION_HEAD)
740 return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
741 if (j->current_location.type == LOCATION_TAIL)
742 return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
743 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
744 return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
745 if (j->current_location.monotonic_set) {
746 r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
750 if (j->current_location.realtime_set)
751 return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
753 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
755 return find_location_for_match(j, j->level0, f, direction, ret, offset);
758 static int next_with_matches(
761 direction_t direction,
776 /* No matches is easy. We simple advance the file
779 return journal_file_next_entry(f, c, cp, direction, ret, offset);
781 /* If we have a match then we look for the next matching entry
782 * with an offset at least one step larger */
783 return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
786 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
794 if (f->current_offset > 0) {
795 cp = f->current_offset;
797 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
801 r = next_with_matches(j, f, direction, &c, &cp);
805 r = find_location_with_matches(j, f, direction, &c, &cp);
810 /* OK, we found the spot, now let's advance until to an entry
811 * that is actually different from what we were previously
812 * looking at. This is necessary to handle entries which exist
813 * in two (or more) journal files, and which shall all be
814 * suppressed but one. */
819 if (j->current_location.type == LOCATION_DISCRETE) {
822 k = compare_with_location(f, c, &j->current_location);
823 if (direction == DIRECTION_DOWN)
838 r = next_with_matches(j, f, direction, &c, &cp);
844 static int real_journal_next(sd_journal *j, direction_t direction) {
845 JournalFile *f, *new_file = NULL;
846 uint64_t new_offset = 0;
855 HASHMAP_FOREACH(f, j->files, i) {
858 r = next_beyond_location(j, f, direction, &o, &p);
860 log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
870 k = compare_entry_order(f, o, new_file, new_offset);
872 if (direction == DIRECTION_DOWN)
887 r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
891 set_location(j, LOCATION_DISCRETE, new_file, o, new_offset);
896 _public_ int sd_journal_next(sd_journal *j) {
897 return real_journal_next(j, DIRECTION_DOWN);
900 _public_ int sd_journal_previous(sd_journal *j) {
901 return real_journal_next(j, DIRECTION_UP);
904 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
911 /* If this is not a discrete skip, then at least
912 * resolve the current location */
913 if (j->current_location.type != LOCATION_DISCRETE)
914 return real_journal_next(j, direction);
920 r = real_journal_next(j, direction);
934 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
935 return real_journal_next_skip(j, DIRECTION_DOWN, skip);
938 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
939 return real_journal_next_skip(j, DIRECTION_UP, skip);
942 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
945 char bid[33], sid[33];
952 if (!j->current_file || j->current_file->current_offset <= 0)
953 return -EADDRNOTAVAIL;
955 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
959 sd_id128_to_string(j->current_file->header->seqnum_id, sid);
960 sd_id128_to_string(o->entry.boot_id, bid);
963 "s=%s;i=%llx;b=%s;m=%llx;t=%llx;x=%llx",
964 sid, (unsigned long long) le64toh(o->entry.seqnum),
965 bid, (unsigned long long) le64toh(o->entry.monotonic),
966 (unsigned long long) le64toh(o->entry.realtime),
967 (unsigned long long) le64toh(o->entry.xor_hash)) < 0)
973 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
976 unsigned long long seqnum, monotonic, realtime, xor_hash;
978 seqnum_id_set = false,
981 monotonic_set = false,
982 realtime_set = false,
983 xor_hash_set = false;
984 sd_id128_t seqnum_id, boot_id;
991 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
995 if (l < 2 || w[1] != '=')
998 item = strndup(w, l);
1005 seqnum_id_set = true;
1006 k = sd_id128_from_string(item+2, &seqnum_id);
1011 if (sscanf(item+2, "%llx", &seqnum) != 1)
1017 k = sd_id128_from_string(item+2, &boot_id);
1021 monotonic_set = true;
1022 if (sscanf(item+2, "%llx", &monotonic) != 1)
1027 realtime_set = true;
1028 if (sscanf(item+2, "%llx", &realtime) != 1)
1033 xor_hash_set = true;
1034 if (sscanf(item+2, "%llx", &xor_hash) != 1)
1045 if ((!seqnum_set || !seqnum_id_set) &&
1046 (!monotonic_set || !boot_id_set) &&
1052 j->current_location.type = LOCATION_SEEK;
1055 j->current_location.realtime = (uint64_t) realtime;
1056 j->current_location.realtime_set = true;
1059 if (seqnum_set && seqnum_id_set) {
1060 j->current_location.seqnum = (uint64_t) seqnum;
1061 j->current_location.seqnum_id = seqnum_id;
1062 j->current_location.seqnum_set = true;
1065 if (monotonic_set && boot_id_set) {
1066 j->current_location.monotonic = (uint64_t) monotonic;
1067 j->current_location.boot_id = boot_id;
1068 j->current_location.monotonic_set = true;
1072 j->current_location.xor_hash = (uint64_t) xor_hash;
1073 j->current_location.xor_hash_set = true;
1079 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1087 if (isempty(cursor))
1090 if (!j->current_file || j->current_file->current_offset <= 0)
1091 return -EADDRNOTAVAIL;
1093 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1097 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1098 _cleanup_free_ char *item = NULL;
1100 unsigned long long ll;
1103 if (l < 2 || w[1] != '=')
1106 item = strndup(w, l);
1113 k = sd_id128_from_string(item+2, &id);
1116 if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1121 if (sscanf(item+2, "%llx", &ll) != 1)
1123 if (ll != le64toh(o->entry.seqnum))
1128 k = sd_id128_from_string(item+2, &id);
1131 if (!sd_id128_equal(id, o->entry.boot_id))
1136 if (sscanf(item+2, "%llx", &ll) != 1)
1138 if (ll != le64toh(o->entry.monotonic))
1143 if (sscanf(item+2, "%llx", &ll) != 1)
1145 if (ll != le64toh(o->entry.realtime))
1150 if (sscanf(item+2, "%llx", &ll) != 1)
1152 if (ll != le64toh(o->entry.xor_hash))
1162 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1167 j->current_location.type = LOCATION_SEEK;
1168 j->current_location.boot_id = boot_id;
1169 j->current_location.monotonic = usec;
1170 j->current_location.monotonic_set = true;
1175 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1180 j->current_location.type = LOCATION_SEEK;
1181 j->current_location.realtime = usec;
1182 j->current_location.realtime_set = true;
1187 _public_ int sd_journal_seek_head(sd_journal *j) {
1192 j->current_location.type = LOCATION_HEAD;
1197 _public_ int sd_journal_seek_tail(sd_journal *j) {
1202 j->current_location.type = LOCATION_TAIL;
1207 static void check_network(sd_journal *j, int fd) {
1215 if (fstatfs(fd, &sfs) < 0)
1219 (long)sfs.f_type == (long)CIFS_MAGIC_NUMBER ||
1220 sfs.f_type == CODA_SUPER_MAGIC ||
1221 sfs.f_type == NCP_SUPER_MAGIC ||
1222 sfs.f_type == NFS_SUPER_MAGIC ||
1223 sfs.f_type == SMB_SUPER_MAGIC;
1226 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1227 char _cleanup_free_ *path = NULL;
1235 if ((j->flags & SD_JOURNAL_SYSTEM_ONLY) &&
1236 !(streq(filename, "system.journal") ||
1237 streq(filename, "system.journal~") ||
1238 (startswith(filename, "system@") &&
1239 (endswith(filename, ".journal") || endswith(filename, ".journal~")))))
1242 path = strjoin(prefix, "/", filename, NULL);
1246 if (hashmap_get(j->files, path))
1249 if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1250 log_debug("Too many open journal files, not adding %s, ignoring.", path);
1251 return set_put_error(j->errors, -ETOOMANYREFS);
1254 r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1256 if (errno == ENOENT)
1262 /* journal_file_dump(f); */
1264 r = hashmap_put(j->files, f->path, f);
1266 journal_file_close(f);
1270 log_debug("File %s got added.", f->path);
1272 check_network(j, f->fd);
1274 j->current_invalidate_counter ++;
1279 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1287 path = strjoin(prefix, "/", filename, NULL);
1291 f = hashmap_get(j->files, path);
1296 hashmap_remove(j->files, f->path);
1298 log_debug("File %s got removed.", f->path);
1300 if (j->current_file == f) {
1301 j->current_file = NULL;
1302 j->current_field = 0;
1305 if (j->unique_file == f) {
1306 j->unique_file = NULL;
1307 j->unique_offset = 0;
1310 journal_file_close(f);
1312 j->current_invalidate_counter ++;
1317 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1318 char _cleanup_free_ *path = NULL;
1320 DIR _cleanup_closedir_ *d = NULL;
1328 log_debug("Considering %s/%s.", prefix, dirname);
1330 if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1331 (sd_id128_from_string(dirname, &id) < 0 ||
1332 sd_id128_get_machine(&mid) < 0 ||
1333 !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1336 path = strjoin(prefix, "/", dirname, NULL);
1342 log_debug("Failed to open %s: %m", path);
1343 if (errno == ENOENT)
1348 m = hashmap_get(j->directories_by_path, path);
1350 m = new0(Directory, 1);
1357 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1362 path = NULL; /* avoid freeing in cleanup */
1363 j->current_invalidate_counter ++;
1365 log_debug("Directory %s got added.", m->path);
1367 } else if (m->is_root)
1370 if (m->wd <= 0 && j->inotify_fd >= 0) {
1372 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1373 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1374 IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1377 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1378 inotify_rm_watch(j->inotify_fd, m->wd);
1383 union dirent_storage buf;
1385 r = readdir_r(d, &buf.de, &de);
1389 if (dirent_is_file_with_suffix(de, ".journal") ||
1390 dirent_is_file_with_suffix(de, ".journal~")) {
1391 r = add_file(j, m->path, de->d_name);
1393 log_debug("Failed to add file %s/%s: %s",
1394 m->path, de->d_name, strerror(-r));
1395 r = set_put_error(j->errors, r);
1402 check_network(j, dirfd(d));
1407 static int add_root_directory(sd_journal *j, const char *p) {
1408 DIR _cleanup_closedir_ *d = NULL;
1415 if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1416 !path_startswith(p, "/run"))
1423 m = hashmap_get(j->directories_by_path, p);
1425 m = new0(Directory, 1);
1430 m->path = strdup(p);
1436 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1442 j->current_invalidate_counter ++;
1444 log_debug("Root directory %s got added.", m->path);
1446 } else if (!m->is_root)
1449 if (m->wd <= 0 && j->inotify_fd >= 0) {
1451 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1452 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1455 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1456 inotify_rm_watch(j->inotify_fd, m->wd);
1461 union dirent_storage buf;
1464 r = readdir_r(d, &buf.de, &de);
1468 if (dirent_is_file_with_suffix(de, ".journal") ||
1469 dirent_is_file_with_suffix(de, ".journal~")) {
1470 r = add_file(j, m->path, de->d_name);
1472 log_debug("Failed to add file %s/%s: %s",
1473 m->path, de->d_name, strerror(-r));
1474 r = set_put_error(j->errors, r);
1478 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1479 sd_id128_from_string(de->d_name, &id) >= 0) {
1481 r = add_directory(j, m->path, de->d_name);
1483 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1487 check_network(j, dirfd(d));
1492 static int remove_directory(sd_journal *j, Directory *d) {
1496 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1498 if (j->inotify_fd >= 0)
1499 inotify_rm_watch(j->inotify_fd, d->wd);
1502 hashmap_remove(j->directories_by_path, d->path);
1505 log_debug("Root directory %s got removed.", d->path);
1507 log_debug("Directory %s got removed.", d->path);
1515 static int add_search_paths(sd_journal *j) {
1517 const char search_paths[] =
1518 "/run/log/journal\0"
1519 "/var/log/journal\0";
1524 /* We ignore most errors here, since the idea is to only open
1525 * what's actually accessible, and ignore the rest. */
1527 NULSTR_FOREACH(p, search_paths) {
1528 r = add_root_directory(j, p);
1530 return set_put_error(j->errors, r);
1536 static int allocate_inotify(sd_journal *j) {
1539 if (j->inotify_fd < 0) {
1540 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1541 if (j->inotify_fd < 0)
1545 if (!j->directories_by_wd) {
1546 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1547 if (!j->directories_by_wd)
1554 static sd_journal *journal_new(int flags, const char *path) {
1557 j = new0(sd_journal, 1);
1563 j->data_threshold = DEFAULT_DATA_THRESHOLD;
1566 j->path = strdup(path);
1571 j->files = hashmap_new(string_hash_func, string_compare_func);
1572 j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1573 j->mmap = mmap_cache_new();
1574 j->errors = set_new(trivial_hash_func, trivial_compare_func);
1575 if (!j->files || !j->directories_by_path || !j->mmap || !j->errors)
1581 sd_journal_close(j);
1585 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1592 if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1593 SD_JOURNAL_RUNTIME_ONLY|
1594 SD_JOURNAL_SYSTEM_ONLY))
1597 j = journal_new(flags, NULL);
1601 r = add_search_paths(j);
1609 sd_journal_close(j);
1614 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1627 j = journal_new(flags, path);
1631 r = add_root_directory(j, path);
1633 set_put_error(j->errors, r);
1641 sd_journal_close(j);
1646 _public_ void sd_journal_close(sd_journal *j) {
1653 while ((f = hashmap_steal_first(j->files)))
1654 journal_file_close(f);
1656 hashmap_free(j->files);
1658 while ((d = hashmap_first(j->directories_by_path)))
1659 remove_directory(j, d);
1661 while ((d = hashmap_first(j->directories_by_wd)))
1662 remove_directory(j, d);
1664 hashmap_free(j->directories_by_path);
1665 hashmap_free(j->directories_by_wd);
1667 if (j->inotify_fd >= 0)
1668 close_nointr_nofail(j->inotify_fd);
1670 sd_journal_flush_matches(j);
1673 mmap_cache_unref(j->mmap);
1676 free(j->unique_field);
1677 set_free(j->errors);
1681 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1691 f = j->current_file;
1693 return -EADDRNOTAVAIL;
1695 if (f->current_offset <= 0)
1696 return -EADDRNOTAVAIL;
1698 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1702 *ret = le64toh(o->entry.realtime);
1706 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1715 f = j->current_file;
1717 return -EADDRNOTAVAIL;
1719 if (f->current_offset <= 0)
1720 return -EADDRNOTAVAIL;
1722 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1727 *ret_boot_id = o->entry.boot_id;
1729 r = sd_id128_get_boot(&id);
1733 if (!sd_id128_equal(id, o->entry.boot_id))
1738 *ret = le64toh(o->entry.monotonic);
1743 static bool field_is_valid(const char *field) {
1751 if (startswith(field, "__"))
1754 for (p = field; *p; p++) {
1759 if (*p >= 'A' && *p <= 'Z')
1762 if (*p >= '0' && *p <= '9')
1771 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1774 size_t field_length;
1787 if (!field_is_valid(field))
1790 f = j->current_file;
1792 return -EADDRNOTAVAIL;
1794 if (f->current_offset <= 0)
1795 return -EADDRNOTAVAIL;
1797 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1801 field_length = strlen(field);
1803 n = journal_file_entry_n_items(o);
1804 for (i = 0; i < n; i++) {
1809 p = le64toh(o->entry.items[i].object_offset);
1810 le_hash = o->entry.items[i].hash;
1811 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1815 if (le_hash != o->data.hash)
1818 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1820 if (o->object.flags & OBJECT_COMPRESSED) {
1823 if (uncompress_startswith(o->data.payload, l,
1824 &f->compress_buffer, &f->compress_buffer_size,
1825 field, field_length, '=')) {
1829 if (!uncompress_blob(o->data.payload, l,
1830 &f->compress_buffer, &f->compress_buffer_size, &rsize,
1834 *data = f->compress_buffer;
1835 *size = (size_t) rsize;
1840 return -EPROTONOSUPPORT;
1843 } else if (l >= field_length+1 &&
1844 memcmp(o->data.payload, field, field_length) == 0 &&
1845 o->data.payload[field_length] == '=') {
1849 if ((uint64_t) t != l)
1852 *data = o->data.payload;
1858 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1866 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1870 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1873 /* We can't read objects larger than 4G on a 32bit machine */
1874 if ((uint64_t) t != l)
1877 if (o->object.flags & OBJECT_COMPRESSED) {
1881 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
1884 *data = f->compress_buffer;
1885 *size = (size_t) rsize;
1887 return -EPROTONOSUPPORT;
1890 *data = o->data.payload;
1897 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1911 f = j->current_file;
1913 return -EADDRNOTAVAIL;
1915 if (f->current_offset <= 0)
1916 return -EADDRNOTAVAIL;
1918 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1922 n = journal_file_entry_n_items(o);
1923 if (j->current_field >= n)
1926 p = le64toh(o->entry.items[j->current_field].object_offset);
1927 le_hash = o->entry.items[j->current_field].hash;
1928 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1932 if (le_hash != o->data.hash)
1935 r = return_data(j, f, o, data, size);
1939 j->current_field ++;
1944 _public_ void sd_journal_restart_data(sd_journal *j) {
1948 j->current_field = 0;
1951 _public_ int sd_journal_get_fd(sd_journal *j) {
1957 if (j->inotify_fd >= 0)
1958 return j->inotify_fd;
1960 r = allocate_inotify(j);
1964 /* Iterate through all dirs again, to add them to the
1967 r = add_root_directory(j, j->path);
1969 r = add_search_paths(j);
1973 return j->inotify_fd;
1976 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
1983 /* Is this a subdirectory we watch? */
1984 d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
1988 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
1989 (endswith(e->name, ".journal") ||
1990 endswith(e->name, ".journal~"))) {
1992 /* Event for a journal file */
1994 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
1995 r = add_file(j, d->path, e->name);
1997 log_debug("Failed to add file %s/%s: %s",
1998 d->path, e->name, strerror(-r));
1999 set_put_error(j->errors, r);
2002 } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2004 r = remove_file(j, d->path, e->name);
2006 log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2009 } else if (!d->is_root && e->len == 0) {
2011 /* Event for a subdirectory */
2013 if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2014 r = remove_directory(j, d);
2016 log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2020 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2022 /* Event for root directory */
2024 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2025 r = add_directory(j, d->path, e->name);
2027 log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2034 if (e->mask & IN_IGNORED)
2037 log_warning("Unknown inotify event.");
2040 static int determine_change(sd_journal *j) {
2045 b = j->current_invalidate_counter != j->last_invalidate_counter;
2046 j->last_invalidate_counter = j->current_invalidate_counter;
2048 return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2051 _public_ int sd_journal_process(sd_journal *j) {
2052 uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2053 bool got_something = false;
2059 struct inotify_event *e;
2062 l = read(j->inotify_fd, buffer, sizeof(buffer));
2064 if (errno == EAGAIN || errno == EINTR)
2065 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2070 got_something = true;
2072 e = (struct inotify_event*) buffer;
2076 process_inotify_event(j, e);
2078 step = sizeof(struct inotify_event) + e->len;
2079 assert(step <= (size_t) l);
2081 e = (struct inotify_event*) ((uint8_t*) e + step);
2086 return determine_change(j);
2089 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2094 if (j->inotify_fd < 0) {
2096 /* This is the first invocation, hence create the
2098 r = sd_journal_get_fd(j);
2102 /* The journal might have changed since the context
2103 * object was created and we weren't watching before,
2104 * hence don't wait for anything, and return
2106 return determine_change(j);
2109 if (j->on_network) {
2110 /* If we are on the network we need to regularly check
2111 * for changes manually */
2113 if (timeout_usec == (uint64_t) -1 || timeout_usec > JOURNAL_FILES_RECHECK_USEC)
2114 timeout_usec = JOURNAL_FILES_RECHECK_USEC;
2118 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2119 } while (r == -EINTR);
2124 return sd_journal_process(j);
2127 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2138 HASHMAP_FOREACH(f, j->files, i) {
2141 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2157 *from = MIN(fr, *from);
2163 return first ? 0 : 1;
2166 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2177 HASHMAP_FOREACH(f, j->files, i) {
2180 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2196 *from = MIN(fr, *from);
2202 return first ? 0 : 1;
2205 void journal_print_header(sd_journal *j) {
2208 bool newline = false;
2212 HASHMAP_FOREACH(f, j->files, i) {
2218 journal_file_print_header(f);
2222 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2232 HASHMAP_FOREACH(f, j->files, i) {
2235 if (fstat(f->fd, &st) < 0)
2238 sum += (uint64_t) st.st_blocks * 512ULL;
2245 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2252 if (!field_is_valid(field))
2259 free(j->unique_field);
2260 j->unique_field = f;
2261 j->unique_file = NULL;
2262 j->unique_offset = 0;
2267 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2278 if (!j->unique_field)
2281 k = strlen(j->unique_field);
2283 if (!j->unique_file) {
2284 j->unique_file = hashmap_first(j->files);
2285 if (!j->unique_file)
2287 j->unique_offset = 0;
2297 /* Proceed to next data object in the field's linked list */
2298 if (j->unique_offset == 0) {
2299 r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2303 j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2305 r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2309 j->unique_offset = le64toh(o->data.next_field_offset);
2312 /* We reached the end of the list? Then start again, with the next file */
2313 if (j->unique_offset == 0) {
2316 n = hashmap_next(j->files, j->unique_file->path);
2324 /* We do not use the type context here, but 0 instead,
2325 * so that we can look at this data object at the same
2326 * time as one on another file */
2327 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2331 /* Let's do the type check by hand, since we used 0 context above. */
2332 if (o->object.type != OBJECT_DATA)
2335 r = return_data(j, j->unique_file, o, &odata, &ol);
2339 /* OK, now let's see if we already returned this data
2340 * object by checking if it exists in the earlier
2341 * traversed files. */
2343 HASHMAP_FOREACH(of, j->files, i) {
2347 if (of == j->unique_file)
2350 /* Skip this file it didn't have any fields
2352 if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2353 le64toh(of->header->n_fields) <= 0)
2356 r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2367 r = return_data(j, j->unique_file, o, data, l);
2375 _public_ void sd_journal_restart_unique(sd_journal *j) {
2379 j->unique_file = NULL;
2380 j->unique_offset = 0;
2383 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2387 return !j->on_network;
2390 static char *lookup_field(const char *field, void *userdata) {
2391 sd_journal *j = userdata;
2399 r = sd_journal_get_data(j, field, &data, &size);
2401 size > REPLACE_VAR_MAX)
2402 return strdup(field);
2404 d = strlen(field) + 1;
2406 return strndup((const char*) data + d, size - d);
2409 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2413 _cleanup_free_ char *text = NULL, *cid = NULL;
2422 r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2426 cid = strndup((const char*) data + 11, size - 11);
2430 r = sd_id128_from_string(cid, &id);
2434 r = catalog_get(id, &text);
2438 t = replace_var(text, lookup_field, j);
2446 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2450 return catalog_get(id, ret);
2453 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2457 j->data_threshold = sz;
2461 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2467 *sz = j->data_threshold;