1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
26 #include <sys/inotify.h>
29 #include <linux/magic.h>
31 #include "sd-journal.h"
32 #include "journal-def.h"
33 #include "journal-file.h"
37 #include "path-util.h"
40 #include "journal-internal.h"
43 #include "replace-var.h"
45 #define JOURNAL_FILES_MAX 1024
47 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
49 #define REPLACE_VAR_MAX 256
51 #define DEFAULT_DATA_THRESHOLD (64*1024)
53 /* We return an error here only if we didn't manage to
54 memorize the real error. */
55 static int set_put_error(sd_journal *j, int r) {
61 k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
65 return set_put(j->errors, INT_TO_PTR(r));
68 static void detach_location(sd_journal *j) {
74 j->current_file = NULL;
77 HASHMAP_FOREACH(f, j->files, i)
78 f->current_offset = 0;
81 static void reset_location(sd_journal *j) {
85 zero(j->current_location);
88 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
90 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
92 assert(o->object.type == OBJECT_ENTRY);
95 l->seqnum = le64toh(o->entry.seqnum);
96 l->seqnum_id = f->header->seqnum_id;
97 l->realtime = le64toh(o->entry.realtime);
98 l->monotonic = le64toh(o->entry.monotonic);
99 l->boot_id = o->entry.boot_id;
100 l->xor_hash = le64toh(o->entry.xor_hash);
102 l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
105 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o,
106 direction_t direction, uint64_t offset) {
108 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
112 init_location(&j->current_location, type, f, o);
115 j->current_field = 0;
117 f->last_direction = direction;
118 f->current_offset = offset;
121 static int match_is_valid(const void *data, size_t size) {
129 if (startswith(data, "__"))
133 for (p = b; p < b + size; p++) {
141 if (*p >= 'A' && *p <= 'Z')
144 if (*p >= '0' && *p <= '9')
153 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
154 const uint8_t *a = _a, *b = _b;
157 for (j = 0; j < s && j < t; j++) {
169 static Match *match_new(Match *p, MatchType t) {
180 LIST_PREPEND(Match, matches, p->matches, m);
186 static void match_free(Match *m) {
190 match_free(m->matches);
193 LIST_REMOVE(Match, matches, m->parent->matches, m);
199 static void match_free_if_empty(Match *m) {
200 if (!m || m->matches)
206 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
207 Match *l3, *l4, *add_here = NULL, *m;
219 if (!match_is_valid(data, size))
226 * level 4: concrete matches */
229 j->level0 = match_new(NULL, MATCH_AND_TERM);
235 j->level1 = match_new(j->level0, MATCH_OR_TERM);
241 j->level2 = match_new(j->level1, MATCH_AND_TERM);
246 assert(j->level0->type == MATCH_AND_TERM);
247 assert(j->level1->type == MATCH_OR_TERM);
248 assert(j->level2->type == MATCH_AND_TERM);
250 le_hash = htole64(hash64(data, size));
252 LIST_FOREACH(matches, l3, j->level2->matches) {
253 assert(l3->type == MATCH_OR_TERM);
255 LIST_FOREACH(matches, l4, l3->matches) {
256 assert(l4->type == MATCH_DISCRETE);
258 /* Exactly the same match already? Then ignore
260 if (l4->le_hash == le_hash &&
262 memcmp(l4->data, data, size) == 0)
265 /* Same field? Then let's add this to this OR term */
266 if (same_field(data, size, l4->data, l4->size)) {
277 add_here = match_new(j->level2, MATCH_OR_TERM);
282 m = match_new(add_here, MATCH_DISCRETE);
286 m->le_hash = le_hash;
288 m->data = memdup(data, size);
297 match_free_if_empty(add_here);
298 match_free_if_empty(j->level2);
299 match_free_if_empty(j->level1);
300 match_free_if_empty(j->level0);
305 _public_ int sd_journal_add_conjunction(sd_journal *j) {
314 if (!j->level1->matches)
323 _public_ int sd_journal_add_disjunction(sd_journal *j) {
335 if (!j->level2->matches)
342 static char *match_make_string(Match *m) {
345 bool enclose = false;
350 if (m->type == MATCH_DISCRETE)
351 return strndup(m->data, m->size);
354 LIST_FOREACH(matches, i, m->matches) {
357 t = match_make_string(i);
364 k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
381 r = strjoin("(", p, ")", NULL);
389 char *journal_make_match_string(sd_journal *j) {
392 return match_make_string(j->level0);
395 _public_ void sd_journal_flush_matches(sd_journal *j) {
401 match_free(j->level0);
403 j->level0 = j->level1 = j->level2 = NULL;
408 static int compare_entry_order(JournalFile *af, Object *_ao,
409 JournalFile *bf, uint64_t bp) {
419 /* The mmap cache might invalidate the object from the first
420 * file if we look at the one from the second file. Hence
421 * temporarily copy the header of the first one, and look at
423 ao = alloca(offsetof(EntryObject, items));
424 memcpy(ao, _ao, offsetof(EntryObject, items));
426 r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
428 return strcmp(af->path, bf->path);
430 /* We operate on two different files here, hence we can access
431 * two objects at the same time, which we normally can't.
433 * If contents and timestamps match, these entries are
434 * identical, even if the seqnum does not match */
436 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
437 ao->entry.monotonic == bo->entry.monotonic &&
438 ao->entry.realtime == bo->entry.realtime &&
439 ao->entry.xor_hash == bo->entry.xor_hash)
442 if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
444 /* If this is from the same seqnum source, compare
446 a = le64toh(ao->entry.seqnum);
447 b = le64toh(bo->entry.seqnum);
454 /* Wow! This is weird, different data but the same
455 * seqnums? Something is borked, but let's make the
456 * best of it and compare by time. */
459 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
461 /* If the boot id matches compare monotonic time */
462 a = le64toh(ao->entry.monotonic);
463 b = le64toh(bo->entry.monotonic);
471 /* Otherwise compare UTC time */
472 a = le64toh(ao->entry.realtime);
473 b = le64toh(bo->entry.realtime);
480 /* Finally, compare by contents */
481 a = le64toh(ao->entry.xor_hash);
482 b = le64toh(bo->entry.xor_hash);
492 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
498 assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
500 if (l->monotonic_set &&
501 sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
503 le64toh(ao->entry.realtime) == l->realtime &&
505 le64toh(ao->entry.xor_hash) == l->xor_hash)
509 sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
511 a = le64toh(ao->entry.seqnum);
519 if (l->monotonic_set &&
520 sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
522 a = le64toh(ao->entry.monotonic);
524 if (a < l->monotonic)
526 if (a > l->monotonic)
530 if (l->realtime_set) {
532 a = le64toh(ao->entry.realtime);
540 if (l->xor_hash_set) {
541 a = le64toh(ao->entry.xor_hash);
552 static int next_for_match(
556 uint64_t after_offset,
557 direction_t direction,
569 if (m->type == MATCH_DISCRETE) {
572 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
576 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
578 } else if (m->type == MATCH_OR_TERM) {
581 /* Find the earliest match beyond after_offset */
583 LIST_FOREACH(matches, i, m->matches) {
586 r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
590 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
595 } else if (m->type == MATCH_AND_TERM) {
596 Match *i, *last_moved;
598 /* Always jump to the next matching entry and repeat
599 * this until we find an offset that matches for all
605 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
609 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
610 last_moved = m->matches;
612 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
615 r = next_for_match(j, i, f, np, direction, NULL, &cp);
619 assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
620 if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
630 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
642 static int find_location_for_match(
646 direction_t direction,
656 if (m->type == MATCH_DISCRETE) {
659 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
663 /* FIXME: missing: find by monotonic */
665 if (j->current_location.type == LOCATION_HEAD)
666 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
667 if (j->current_location.type == LOCATION_TAIL)
668 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
669 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
670 return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
671 if (j->current_location.monotonic_set) {
672 r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
676 if (j->current_location.realtime_set)
677 return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
679 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
681 } else if (m->type == MATCH_OR_TERM) {
686 /* Find the earliest match */
688 LIST_FOREACH(matches, i, m->matches) {
691 r = find_location_for_match(j, i, f, direction, NULL, &cp);
695 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
703 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
718 assert(m->type == MATCH_AND_TERM);
720 /* First jump to the last match, and then find the
721 * next one where all matches match */
726 LIST_FOREACH(matches, i, m->matches) {
729 r = find_location_for_match(j, i, f, direction, NULL, &cp);
733 if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
737 return next_for_match(j, m, f, np, direction, ret, offset);
741 static int find_location_with_matches(
744 direction_t direction,
756 /* No matches is simple */
758 if (j->current_location.type == LOCATION_HEAD)
759 return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
760 if (j->current_location.type == LOCATION_TAIL)
761 return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
762 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
763 return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
764 if (j->current_location.monotonic_set) {
765 r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
769 if (j->current_location.realtime_set)
770 return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
772 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
774 return find_location_for_match(j, j->level0, f, direction, ret, offset);
777 static int next_with_matches(
780 direction_t direction,
795 /* No matches is easy. We simple advance the file
798 return journal_file_next_entry(f, c, cp, direction, ret, offset);
800 /* If we have a match then we look for the next matching entry
801 * with an offset at least one step larger */
802 return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
805 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
813 if (f->last_direction == direction && f->current_offset > 0) {
814 cp = f->current_offset;
816 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
820 r = next_with_matches(j, f, direction, &c, &cp);
824 r = find_location_with_matches(j, f, direction, &c, &cp);
829 /* OK, we found the spot, now let's advance until to an entry
830 * that is actually different from what we were previously
831 * looking at. This is necessary to handle entries which exist
832 * in two (or more) journal files, and which shall all be
833 * suppressed but one. */
838 if (j->current_location.type == LOCATION_DISCRETE) {
841 k = compare_with_location(f, c, &j->current_location);
842 if (direction == DIRECTION_DOWN)
857 r = next_with_matches(j, f, direction, &c, &cp);
863 static int real_journal_next(sd_journal *j, direction_t direction) {
864 JournalFile *f, *new_file = NULL;
865 uint64_t new_offset = 0;
874 HASHMAP_FOREACH(f, j->files, i) {
877 r = next_beyond_location(j, f, direction, &o, &p);
879 log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
889 k = compare_entry_order(f, o, new_file, new_offset);
891 if (direction == DIRECTION_DOWN)
906 r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
910 set_location(j, LOCATION_DISCRETE, new_file, o, direction, new_offset);
915 _public_ int sd_journal_next(sd_journal *j) {
916 return real_journal_next(j, DIRECTION_DOWN);
919 _public_ int sd_journal_previous(sd_journal *j) {
920 return real_journal_next(j, DIRECTION_UP);
923 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
930 /* If this is not a discrete skip, then at least
931 * resolve the current location */
932 if (j->current_location.type != LOCATION_DISCRETE)
933 return real_journal_next(j, direction);
939 r = real_journal_next(j, direction);
953 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
954 return real_journal_next_skip(j, DIRECTION_DOWN, skip);
957 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
958 return real_journal_next_skip(j, DIRECTION_UP, skip);
961 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
964 char bid[33], sid[33];
971 if (!j->current_file || j->current_file->current_offset <= 0)
972 return -EADDRNOTAVAIL;
974 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
978 sd_id128_to_string(j->current_file->header->seqnum_id, sid);
979 sd_id128_to_string(o->entry.boot_id, bid);
982 "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
983 sid, le64toh(o->entry.seqnum),
984 bid, le64toh(o->entry.monotonic),
985 le64toh(o->entry.realtime),
986 le64toh(o->entry.xor_hash)) < 0)
992 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
995 unsigned long long seqnum, monotonic, realtime, xor_hash;
997 seqnum_id_set = false,
1000 monotonic_set = false,
1001 realtime_set = false,
1002 xor_hash_set = false;
1003 sd_id128_t seqnum_id, boot_id;
1007 if (isempty(cursor))
1010 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1014 if (l < 2 || w[1] != '=')
1017 item = strndup(w, l);
1024 seqnum_id_set = true;
1025 k = sd_id128_from_string(item+2, &seqnum_id);
1030 if (sscanf(item+2, "%llx", &seqnum) != 1)
1036 k = sd_id128_from_string(item+2, &boot_id);
1040 monotonic_set = true;
1041 if (sscanf(item+2, "%llx", &monotonic) != 1)
1046 realtime_set = true;
1047 if (sscanf(item+2, "%llx", &realtime) != 1)
1052 xor_hash_set = true;
1053 if (sscanf(item+2, "%llx", &xor_hash) != 1)
1064 if ((!seqnum_set || !seqnum_id_set) &&
1065 (!monotonic_set || !boot_id_set) &&
1071 j->current_location.type = LOCATION_SEEK;
1074 j->current_location.realtime = (uint64_t) realtime;
1075 j->current_location.realtime_set = true;
1078 if (seqnum_set && seqnum_id_set) {
1079 j->current_location.seqnum = (uint64_t) seqnum;
1080 j->current_location.seqnum_id = seqnum_id;
1081 j->current_location.seqnum_set = true;
1084 if (monotonic_set && boot_id_set) {
1085 j->current_location.monotonic = (uint64_t) monotonic;
1086 j->current_location.boot_id = boot_id;
1087 j->current_location.monotonic_set = true;
1091 j->current_location.xor_hash = (uint64_t) xor_hash;
1092 j->current_location.xor_hash_set = true;
1098 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1106 if (isempty(cursor))
1109 if (!j->current_file || j->current_file->current_offset <= 0)
1110 return -EADDRNOTAVAIL;
1112 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1116 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1117 _cleanup_free_ char *item = NULL;
1119 unsigned long long ll;
1122 if (l < 2 || w[1] != '=')
1125 item = strndup(w, l);
1132 k = sd_id128_from_string(item+2, &id);
1135 if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1140 if (sscanf(item+2, "%llx", &ll) != 1)
1142 if (ll != le64toh(o->entry.seqnum))
1147 k = sd_id128_from_string(item+2, &id);
1150 if (!sd_id128_equal(id, o->entry.boot_id))
1155 if (sscanf(item+2, "%llx", &ll) != 1)
1157 if (ll != le64toh(o->entry.monotonic))
1162 if (sscanf(item+2, "%llx", &ll) != 1)
1164 if (ll != le64toh(o->entry.realtime))
1169 if (sscanf(item+2, "%llx", &ll) != 1)
1171 if (ll != le64toh(o->entry.xor_hash))
1181 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1186 j->current_location.type = LOCATION_SEEK;
1187 j->current_location.boot_id = boot_id;
1188 j->current_location.monotonic = usec;
1189 j->current_location.monotonic_set = true;
1194 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1199 j->current_location.type = LOCATION_SEEK;
1200 j->current_location.realtime = usec;
1201 j->current_location.realtime_set = true;
1206 _public_ int sd_journal_seek_head(sd_journal *j) {
1211 j->current_location.type = LOCATION_HEAD;
1216 _public_ int sd_journal_seek_tail(sd_journal *j) {
1221 j->current_location.type = LOCATION_TAIL;
1226 static void check_network(sd_journal *j, int fd) {
1234 if (fstatfs(fd, &sfs) < 0)
1238 F_TYPE_CMP(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1239 F_TYPE_CMP(sfs.f_type, CODA_SUPER_MAGIC) ||
1240 F_TYPE_CMP(sfs.f_type, NCP_SUPER_MAGIC) ||
1241 F_TYPE_CMP(sfs.f_type, NFS_SUPER_MAGIC) ||
1242 F_TYPE_CMP(sfs.f_type, SMB_SUPER_MAGIC);
1245 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1246 const char *full, *tilded, *atted;
1248 full = strappend(prefix, ".journal");
1249 tilded = strappenda(full, "~");
1250 atted = strappenda(prefix, "@");
1252 return streq(filename, full) ||
1253 streq(filename, tilded) ||
1254 startswith(filename, atted);
1257 static bool file_type_wanted(int flags, const char *filename) {
1258 if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1261 /* no flags set → every type is OK */
1262 if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1265 if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1268 if (flags & SD_JOURNAL_CURRENT_USER) {
1269 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1271 assert_se(snprintf(prefix, sizeof(prefix), "user-%lu", (unsigned long) getuid())
1272 < (int) sizeof(prefix));
1274 if (file_has_type_prefix(prefix, filename))
1281 static int add_any_file(sd_journal *j, const char *path) {
1288 if (hashmap_get(j->files, path))
1291 if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1292 log_warning("Too many open journal files, not adding %s.", path);
1293 return set_put_error(j, -ETOOMANYREFS);
1296 r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1300 /* journal_file_dump(f); */
1302 r = hashmap_put(j->files, f->path, f);
1304 journal_file_close(f);
1308 log_debug("File %s added.", f->path);
1310 check_network(j, f->fd);
1312 j->current_invalidate_counter ++;
1317 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1318 _cleanup_free_ char *path = NULL;
1325 if (j->no_new_files ||
1326 !file_type_wanted(j->flags, filename))
1329 path = strjoin(prefix, "/", filename, NULL);
1333 r = add_any_file(j, path);
1339 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1347 path = strjoin(prefix, "/", filename, NULL);
1351 f = hashmap_get(j->files, path);
1356 hashmap_remove(j->files, f->path);
1358 log_debug("File %s removed.", f->path);
1360 if (j->current_file == f) {
1361 j->current_file = NULL;
1362 j->current_field = 0;
1365 if (j->unique_file == f) {
1366 j->unique_file = NULL;
1367 j->unique_offset = 0;
1370 journal_file_close(f);
1372 j->current_invalidate_counter ++;
1377 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1378 _cleanup_free_ char *path = NULL;
1380 _cleanup_closedir_ DIR *d = NULL;
1388 log_debug("Considering %s/%s.", prefix, dirname);
1390 if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1391 (sd_id128_from_string(dirname, &id) < 0 ||
1392 sd_id128_get_machine(&mid) < 0 ||
1393 !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1396 path = strjoin(prefix, "/", dirname, NULL);
1402 log_debug("Failed to open %s: %m", path);
1403 if (errno == ENOENT)
1408 m = hashmap_get(j->directories_by_path, path);
1410 m = new0(Directory, 1);
1417 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1422 path = NULL; /* avoid freeing in cleanup */
1423 j->current_invalidate_counter ++;
1425 log_debug("Directory %s added.", m->path);
1427 } else if (m->is_root)
1430 if (m->wd <= 0 && j->inotify_fd >= 0) {
1432 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1433 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1434 IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1437 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1438 inotify_rm_watch(j->inotify_fd, m->wd);
1443 union dirent_storage buf;
1445 r = readdir_r(d, &buf.de, &de);
1449 if (dirent_is_file_with_suffix(de, ".journal") ||
1450 dirent_is_file_with_suffix(de, ".journal~")) {
1451 r = add_file(j, m->path, de->d_name);
1453 log_debug("Failed to add file %s/%s: %s",
1454 m->path, de->d_name, strerror(-r));
1455 r = set_put_error(j, r);
1462 check_network(j, dirfd(d));
1467 static int add_root_directory(sd_journal *j, const char *p) {
1468 _cleanup_closedir_ DIR *d = NULL;
1475 if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1476 !path_startswith(p, "/run"))
1483 m = hashmap_get(j->directories_by_path, p);
1485 m = new0(Directory, 1);
1490 m->path = strdup(p);
1496 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1502 j->current_invalidate_counter ++;
1504 log_debug("Root directory %s added.", m->path);
1506 } else if (!m->is_root)
1509 if (m->wd <= 0 && j->inotify_fd >= 0) {
1511 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1512 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1515 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1516 inotify_rm_watch(j->inotify_fd, m->wd);
1519 if (j->no_new_files)
1524 union dirent_storage buf;
1527 r = readdir_r(d, &buf.de, &de);
1531 if (dirent_is_file_with_suffix(de, ".journal") ||
1532 dirent_is_file_with_suffix(de, ".journal~")) {
1533 r = add_file(j, m->path, de->d_name);
1535 log_debug("Failed to add file %s/%s: %s",
1536 m->path, de->d_name, strerror(-r));
1537 r = set_put_error(j, r);
1541 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1542 sd_id128_from_string(de->d_name, &id) >= 0) {
1544 r = add_directory(j, m->path, de->d_name);
1546 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1550 check_network(j, dirfd(d));
1555 static int remove_directory(sd_journal *j, Directory *d) {
1559 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1561 if (j->inotify_fd >= 0)
1562 inotify_rm_watch(j->inotify_fd, d->wd);
1565 hashmap_remove(j->directories_by_path, d->path);
1568 log_debug("Root directory %s removed.", d->path);
1570 log_debug("Directory %s removed.", d->path);
1578 static int add_search_paths(sd_journal *j) {
1580 const char search_paths[] =
1581 "/run/log/journal\0"
1582 "/var/log/journal\0";
1587 /* We ignore most errors here, since the idea is to only open
1588 * what's actually accessible, and ignore the rest. */
1590 NULSTR_FOREACH(p, search_paths) {
1591 r = add_root_directory(j, p);
1592 if (r < 0 && r != -ENOENT) {
1593 r = set_put_error(j, r);
1602 static int add_current_paths(sd_journal *j) {
1607 assert(j->no_new_files);
1609 /* Simply adds all directories for files we have open as
1610 * "root" directories. We don't expect errors here, so we
1611 * treat them as fatal. */
1613 HASHMAP_FOREACH(f, j->files, i) {
1615 _cleanup_free_ char *dir;
1617 dir = dirname_malloc(f->path);
1621 r = add_root_directory(j, dir);
1623 set_put_error(j, r);
1632 static int allocate_inotify(sd_journal *j) {
1635 if (j->inotify_fd < 0) {
1636 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1637 if (j->inotify_fd < 0)
1641 if (!j->directories_by_wd) {
1642 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1643 if (!j->directories_by_wd)
1650 static sd_journal *journal_new(int flags, const char *path) {
1653 j = new0(sd_journal, 1);
1659 j->data_threshold = DEFAULT_DATA_THRESHOLD;
1662 j->path = strdup(path);
1667 j->files = hashmap_new(string_hash_func, string_compare_func);
1668 j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1669 j->mmap = mmap_cache_new();
1670 if (!j->files || !j->directories_by_path || !j->mmap)
1676 sd_journal_close(j);
1680 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1687 if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1688 SD_JOURNAL_RUNTIME_ONLY|
1690 SD_JOURNAL_CURRENT_USER))
1693 j = journal_new(flags, NULL);
1697 r = add_search_paths(j);
1705 sd_journal_close(j);
1710 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1723 j = journal_new(flags, path);
1727 r = add_root_directory(j, path);
1729 set_put_error(j, r);
1737 sd_journal_close(j);
1742 _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int flags) {
1753 j = journal_new(flags, NULL);
1757 STRV_FOREACH(path, paths) {
1758 r = add_any_file(j, *path);
1760 log_error("Failed to open %s: %s", *path, strerror(-r));
1765 j->no_new_files = true;
1771 sd_journal_close(j);
1776 _public_ void sd_journal_close(sd_journal *j) {
1783 sd_journal_flush_matches(j);
1785 while ((f = hashmap_steal_first(j->files)))
1786 journal_file_close(f);
1788 hashmap_free(j->files);
1790 while ((d = hashmap_first(j->directories_by_path)))
1791 remove_directory(j, d);
1793 while ((d = hashmap_first(j->directories_by_wd)))
1794 remove_directory(j, d);
1796 hashmap_free(j->directories_by_path);
1797 hashmap_free(j->directories_by_wd);
1799 if (j->inotify_fd >= 0)
1800 close_nointr_nofail(j->inotify_fd);
1803 mmap_cache_unref(j->mmap);
1806 free(j->unique_field);
1807 set_free(j->errors);
1811 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1821 f = j->current_file;
1823 return -EADDRNOTAVAIL;
1825 if (f->current_offset <= 0)
1826 return -EADDRNOTAVAIL;
1828 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1832 *ret = le64toh(o->entry.realtime);
1836 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1845 f = j->current_file;
1847 return -EADDRNOTAVAIL;
1849 if (f->current_offset <= 0)
1850 return -EADDRNOTAVAIL;
1852 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1857 *ret_boot_id = o->entry.boot_id;
1859 r = sd_id128_get_boot(&id);
1863 if (!sd_id128_equal(id, o->entry.boot_id))
1868 *ret = le64toh(o->entry.monotonic);
1873 static bool field_is_valid(const char *field) {
1881 if (startswith(field, "__"))
1884 for (p = field; *p; p++) {
1889 if (*p >= 'A' && *p <= 'Z')
1892 if (*p >= '0' && *p <= '9')
1901 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1904 size_t field_length;
1917 if (!field_is_valid(field))
1920 f = j->current_file;
1922 return -EADDRNOTAVAIL;
1924 if (f->current_offset <= 0)
1925 return -EADDRNOTAVAIL;
1927 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1931 field_length = strlen(field);
1933 n = journal_file_entry_n_items(o);
1934 for (i = 0; i < n; i++) {
1939 p = le64toh(o->entry.items[i].object_offset);
1940 le_hash = o->entry.items[i].hash;
1941 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1945 if (le_hash != o->data.hash)
1948 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1950 if (o->object.flags & OBJECT_COMPRESSED) {
1953 if (uncompress_startswith(o->data.payload, l,
1954 &f->compress_buffer, &f->compress_buffer_size,
1955 field, field_length, '=')) {
1959 if (!uncompress_blob(o->data.payload, l,
1960 &f->compress_buffer, &f->compress_buffer_size, &rsize,
1964 *data = f->compress_buffer;
1965 *size = (size_t) rsize;
1970 return -EPROTONOSUPPORT;
1973 } else if (l >= field_length+1 &&
1974 memcmp(o->data.payload, field, field_length) == 0 &&
1975 o->data.payload[field_length] == '=') {
1979 if ((uint64_t) t != l)
1982 *data = o->data.payload;
1988 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1996 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
2000 l = le64toh(o->object.size) - offsetof(Object, data.payload);
2003 /* We can't read objects larger than 4G on a 32bit machine */
2004 if ((uint64_t) t != l)
2007 if (o->object.flags & OBJECT_COMPRESSED) {
2011 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
2014 *data = f->compress_buffer;
2015 *size = (size_t) rsize;
2017 return -EPROTONOSUPPORT;
2020 *data = o->data.payload;
2027 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
2041 f = j->current_file;
2043 return -EADDRNOTAVAIL;
2045 if (f->current_offset <= 0)
2046 return -EADDRNOTAVAIL;
2048 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2052 n = journal_file_entry_n_items(o);
2053 if (j->current_field >= n)
2056 p = le64toh(o->entry.items[j->current_field].object_offset);
2057 le_hash = o->entry.items[j->current_field].hash;
2058 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2062 if (le_hash != o->data.hash)
2065 r = return_data(j, f, o, data, size);
2069 j->current_field ++;
2074 _public_ void sd_journal_restart_data(sd_journal *j) {
2078 j->current_field = 0;
2081 _public_ int sd_journal_get_fd(sd_journal *j) {
2087 if (j->inotify_fd >= 0)
2088 return j->inotify_fd;
2090 r = allocate_inotify(j);
2094 /* Iterate through all dirs again, to add them to the
2096 if (j->no_new_files)
2097 r = add_current_paths(j);
2099 r = add_root_directory(j, j->path);
2101 r = add_search_paths(j);
2105 return j->inotify_fd;
2108 _public_ int sd_journal_get_events(sd_journal *j) {
2114 fd = sd_journal_get_fd(j);
2121 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2129 fd = sd_journal_get_fd(j);
2133 if (!j->on_network) {
2134 *timeout_usec = (uint64_t) -1;
2138 /* If we are on the network we need to regularly check for
2139 * changes manually */
2141 *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2145 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2152 /* Is this a subdirectory we watch? */
2153 d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2157 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2158 (endswith(e->name, ".journal") ||
2159 endswith(e->name, ".journal~"))) {
2161 /* Event for a journal file */
2163 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2164 r = add_file(j, d->path, e->name);
2166 log_debug("Failed to add file %s/%s: %s",
2167 d->path, e->name, strerror(-r));
2168 set_put_error(j, r);
2171 } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2173 r = remove_file(j, d->path, e->name);
2175 log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2178 } else if (!d->is_root && e->len == 0) {
2180 /* Event for a subdirectory */
2182 if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2183 r = remove_directory(j, d);
2185 log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2189 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2191 /* Event for root directory */
2193 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2194 r = add_directory(j, d->path, e->name);
2196 log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2203 if (e->mask & IN_IGNORED)
2206 log_warning("Unknown inotify event.");
2209 static int determine_change(sd_journal *j) {
2214 b = j->current_invalidate_counter != j->last_invalidate_counter;
2215 j->last_invalidate_counter = j->current_invalidate_counter;
2217 return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2220 _public_ int sd_journal_process(sd_journal *j) {
2221 uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2222 bool got_something = false;
2227 j->last_process_usec = now(CLOCK_MONOTONIC);
2230 struct inotify_event *e;
2233 l = read(j->inotify_fd, buffer, sizeof(buffer));
2235 if (errno == EAGAIN || errno == EINTR)
2236 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2241 got_something = true;
2243 e = (struct inotify_event*) buffer;
2247 process_inotify_event(j, e);
2249 step = sizeof(struct inotify_event) + e->len;
2250 assert(step <= (size_t) l);
2252 e = (struct inotify_event*) ((uint8_t*) e + step);
2257 return determine_change(j);
2260 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2266 if (j->inotify_fd < 0) {
2268 /* This is the first invocation, hence create the
2270 r = sd_journal_get_fd(j);
2274 /* The journal might have changed since the context
2275 * object was created and we weren't watching before,
2276 * hence don't wait for anything, and return
2278 return determine_change(j);
2281 r = sd_journal_get_timeout(j, &t);
2285 if (t != (uint64_t) -1) {
2288 n = now(CLOCK_MONOTONIC);
2289 t = t > n ? t - n : 0;
2291 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2296 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2297 } while (r == -EINTR);
2302 return sd_journal_process(j);
2305 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2318 HASHMAP_FOREACH(f, j->files, i) {
2321 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2337 *from = MIN(fr, *from);
2343 return first ? 0 : 1;
2346 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2359 HASHMAP_FOREACH(f, j->files, i) {
2362 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2378 *from = MIN(fr, *from);
2384 return first ? 0 : 1;
2387 void journal_print_header(sd_journal *j) {
2390 bool newline = false;
2394 HASHMAP_FOREACH(f, j->files, i) {
2400 journal_file_print_header(f);
2404 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2414 HASHMAP_FOREACH(f, j->files, i) {
2417 if (fstat(f->fd, &st) < 0)
2420 sum += (uint64_t) st.st_blocks * 512ULL;
2427 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2434 if (!field_is_valid(field))
2441 free(j->unique_field);
2442 j->unique_field = f;
2443 j->unique_file = NULL;
2444 j->unique_offset = 0;
2449 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2460 if (!j->unique_field)
2463 k = strlen(j->unique_field);
2465 if (!j->unique_file) {
2466 j->unique_file = hashmap_first(j->files);
2467 if (!j->unique_file)
2469 j->unique_offset = 0;
2479 /* Proceed to next data object in the field's linked list */
2480 if (j->unique_offset == 0) {
2481 r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2485 j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2487 r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2491 j->unique_offset = le64toh(o->data.next_field_offset);
2494 /* We reached the end of the list? Then start again, with the next file */
2495 if (j->unique_offset == 0) {
2498 n = hashmap_next(j->files, j->unique_file->path);
2506 /* We do not use the type context here, but 0 instead,
2507 * so that we can look at this data object at the same
2508 * time as one on another file */
2509 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2513 /* Let's do the type check by hand, since we used 0 context above. */
2514 if (o->object.type != OBJECT_DATA)
2517 r = return_data(j, j->unique_file, o, &odata, &ol);
2521 /* OK, now let's see if we already returned this data
2522 * object by checking if it exists in the earlier
2523 * traversed files. */
2525 HASHMAP_FOREACH(of, j->files, i) {
2529 if (of == j->unique_file)
2532 /* Skip this file it didn't have any fields
2534 if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2535 le64toh(of->header->n_fields) <= 0)
2538 r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2549 r = return_data(j, j->unique_file, o, data, l);
2557 _public_ void sd_journal_restart_unique(sd_journal *j) {
2561 j->unique_file = NULL;
2562 j->unique_offset = 0;
2565 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2569 return !j->on_network;
2572 static char *lookup_field(const char *field, void *userdata) {
2573 sd_journal *j = userdata;
2581 r = sd_journal_get_data(j, field, &data, &size);
2583 size > REPLACE_VAR_MAX)
2584 return strdup(field);
2586 d = strlen(field) + 1;
2588 return strndup((const char*) data + d, size - d);
2591 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2595 _cleanup_free_ char *text = NULL, *cid = NULL;
2604 r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2608 cid = strndup((const char*) data + 11, size - 11);
2612 r = sd_id128_from_string(cid, &id);
2616 r = catalog_get(CATALOG_DATABASE, id, &text);
2620 t = replace_var(text, lookup_field, j);
2628 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2632 return catalog_get(CATALOG_DATABASE, id, ret);
2635 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2639 j->data_threshold = sz;
2643 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2649 *sz = j->data_threshold;