1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
26 #include <sys/inotify.h>
29 #include <linux/magic.h>
31 #include "sd-journal.h"
32 #include "journal-def.h"
33 #include "journal-file.h"
36 #include "path-util.h"
39 #include "journal-internal.h"
42 #include "replace-var.h"
44 #define JOURNAL_FILES_MAX 1024
46 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
48 #define REPLACE_VAR_MAX 256
50 #define DEFAULT_DATA_THRESHOLD (64*1024)
52 /* We return an error here only if we didn't manage to
53 memorize the real error. */
54 static int set_put_error(sd_journal *j, int r) {
60 k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
64 return set_put(j->errors, INT_TO_PTR(r));
67 static void detach_location(sd_journal *j) {
73 j->current_file = NULL;
76 HASHMAP_FOREACH(f, j->files, i)
77 f->current_offset = 0;
80 static void reset_location(sd_journal *j) {
84 zero(j->current_location);
87 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
89 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
91 assert(o->object.type == OBJECT_ENTRY);
94 l->seqnum = le64toh(o->entry.seqnum);
95 l->seqnum_id = f->header->seqnum_id;
96 l->realtime = le64toh(o->entry.realtime);
97 l->monotonic = le64toh(o->entry.monotonic);
98 l->boot_id = o->entry.boot_id;
99 l->xor_hash = le64toh(o->entry.xor_hash);
101 l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
104 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o, uint64_t offset) {
106 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
110 init_location(&j->current_location, type, f, o);
113 j->current_field = 0;
115 f->current_offset = offset;
118 static int match_is_valid(const void *data, size_t size) {
126 if (startswith(data, "__"))
130 for (p = b; p < b + size; p++) {
138 if (*p >= 'A' && *p <= 'Z')
141 if (*p >= '0' && *p <= '9')
150 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
151 const uint8_t *a = _a, *b = _b;
154 for (j = 0; j < s && j < t; j++) {
166 static Match *match_new(Match *p, MatchType t) {
177 LIST_PREPEND(Match, matches, p->matches, m);
183 static void match_free(Match *m) {
187 match_free(m->matches);
190 LIST_REMOVE(Match, matches, m->parent->matches, m);
196 static void match_free_if_empty(Match *m) {
205 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
206 Match *l3, *l4, *add_here = NULL, *m;
218 if (!match_is_valid(data, size))
225 * level 4: concrete matches */
228 j->level0 = match_new(NULL, MATCH_AND_TERM);
234 j->level1 = match_new(j->level0, MATCH_OR_TERM);
240 j->level2 = match_new(j->level1, MATCH_AND_TERM);
245 assert(j->level0->type == MATCH_AND_TERM);
246 assert(j->level1->type == MATCH_OR_TERM);
247 assert(j->level2->type == MATCH_AND_TERM);
249 le_hash = htole64(hash64(data, size));
251 LIST_FOREACH(matches, l3, j->level2->matches) {
252 assert(l3->type == MATCH_OR_TERM);
254 LIST_FOREACH(matches, l4, l3->matches) {
255 assert(l4->type == MATCH_DISCRETE);
257 /* Exactly the same match already? Then ignore
259 if (l4->le_hash == le_hash &&
261 memcmp(l4->data, data, size) == 0)
264 /* Same field? Then let's add this to this OR term */
265 if (same_field(data, size, l4->data, l4->size)) {
276 add_here = match_new(j->level2, MATCH_OR_TERM);
281 m = match_new(add_here, MATCH_DISCRETE);
285 m->le_hash = le_hash;
287 m->data = memdup(data, size);
297 match_free_if_empty(add_here);
300 match_free_if_empty(j->level2);
303 match_free_if_empty(j->level1);
306 match_free_if_empty(j->level0);
311 _public_ int sd_journal_add_conjunction(sd_journal *j) {
320 if (!j->level1->matches)
329 _public_ int sd_journal_add_disjunction(sd_journal *j) {
341 if (!j->level2->matches)
348 static char *match_make_string(Match *m) {
351 bool enclose = false;
356 if (m->type == MATCH_DISCRETE)
357 return strndup(m->data, m->size);
360 LIST_FOREACH(matches, i, m->matches) {
363 t = match_make_string(i);
370 k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
387 r = strjoin("(", p, ")", NULL);
395 char *journal_make_match_string(sd_journal *j) {
398 return match_make_string(j->level0);
401 _public_ void sd_journal_flush_matches(sd_journal *j) {
407 match_free(j->level0);
409 j->level0 = j->level1 = j->level2 = NULL;
414 static int compare_entry_order(JournalFile *af, Object *_ao,
415 JournalFile *bf, uint64_t bp) {
425 /* The mmap cache might invalidate the object from the first
426 * file if we look at the one from the second file. Hence
427 * temporarily copy the header of the first one, and look at
429 ao = alloca(offsetof(EntryObject, items));
430 memcpy(ao, _ao, offsetof(EntryObject, items));
432 r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
434 return strcmp(af->path, bf->path);
436 /* We operate on two different files here, hence we can access
437 * two objects at the same time, which we normally can't.
439 * If contents and timestamps match, these entries are
440 * identical, even if the seqnum does not match */
442 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
443 ao->entry.monotonic == bo->entry.monotonic &&
444 ao->entry.realtime == bo->entry.realtime &&
445 ao->entry.xor_hash == bo->entry.xor_hash)
448 if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
450 /* If this is from the same seqnum source, compare
452 a = le64toh(ao->entry.seqnum);
453 b = le64toh(bo->entry.seqnum);
460 /* Wow! This is weird, different data but the same
461 * seqnums? Something is borked, but let's make the
462 * best of it and compare by time. */
465 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
467 /* If the boot id matches compare monotonic time */
468 a = le64toh(ao->entry.monotonic);
469 b = le64toh(bo->entry.monotonic);
477 /* Otherwise compare UTC time */
478 a = le64toh(ao->entry.realtime);
479 b = le64toh(bo->entry.realtime);
486 /* Finally, compare by contents */
487 a = le64toh(ao->entry.xor_hash);
488 b = le64toh(bo->entry.xor_hash);
498 static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
504 assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
506 if (l->monotonic_set &&
507 sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
509 le64toh(ao->entry.realtime) == l->realtime &&
511 le64toh(ao->entry.xor_hash) == l->xor_hash)
515 sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
517 a = le64toh(ao->entry.seqnum);
525 if (l->monotonic_set &&
526 sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
528 a = le64toh(ao->entry.monotonic);
530 if (a < l->monotonic)
532 if (a > l->monotonic)
536 if (l->realtime_set) {
538 a = le64toh(ao->entry.realtime);
546 if (l->xor_hash_set) {
547 a = le64toh(ao->entry.xor_hash);
558 static int next_for_match(
562 uint64_t after_offset,
563 direction_t direction,
575 if (m->type == MATCH_DISCRETE) {
578 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
582 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
584 } else if (m->type == MATCH_OR_TERM) {
587 /* Find the earliest match beyond after_offset */
589 LIST_FOREACH(matches, i, m->matches) {
592 r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
596 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
601 } else if (m->type == MATCH_AND_TERM) {
603 bool continue_looking;
605 /* Always jump to the next matching entry and repeat
606 * this until we fine and offset that matches for all
614 continue_looking = false;
616 LIST_FOREACH(matches, i, m->matches) {
620 limit = after_offset;
621 else if (direction == DIRECTION_DOWN)
622 limit = MAX(np, after_offset);
624 limit = MIN(np, after_offset);
626 r = next_for_match(j, i, f, limit, direction, NULL, &cp);
630 if ((direction == DIRECTION_DOWN ? cp >= after_offset : cp <= after_offset) &&
631 (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))) {
633 continue_looking = true;
637 } while (continue_looking);
643 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
655 static int find_location_for_match(
659 direction_t direction,
669 if (m->type == MATCH_DISCRETE) {
672 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
676 /* FIXME: missing: find by monotonic */
678 if (j->current_location.type == LOCATION_HEAD)
679 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
680 if (j->current_location.type == LOCATION_TAIL)
681 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
682 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
683 return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
684 if (j->current_location.monotonic_set) {
685 r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
689 if (j->current_location.realtime_set)
690 return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
692 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
694 } else if (m->type == MATCH_OR_TERM) {
699 /* Find the earliest match */
701 LIST_FOREACH(matches, i, m->matches) {
704 r = find_location_for_match(j, i, f, direction, NULL, &cp);
708 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
716 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
731 assert(m->type == MATCH_AND_TERM);
733 /* First jump to the last match, and then find the
734 * next one where all matches match */
739 LIST_FOREACH(matches, i, m->matches) {
742 r = find_location_for_match(j, i, f, direction, NULL, &cp);
746 if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
750 return next_for_match(j, m, f, np, direction, ret, offset);
754 static int find_location_with_matches(
757 direction_t direction,
769 /* No matches is simple */
771 if (j->current_location.type == LOCATION_HEAD)
772 return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
773 if (j->current_location.type == LOCATION_TAIL)
774 return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
775 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
776 return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
777 if (j->current_location.monotonic_set) {
778 r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
782 if (j->current_location.realtime_set)
783 return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
785 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
787 return find_location_for_match(j, j->level0, f, direction, ret, offset);
790 static int next_with_matches(
793 direction_t direction,
808 /* No matches is easy. We simple advance the file
811 return journal_file_next_entry(f, c, cp, direction, ret, offset);
813 /* If we have a match then we look for the next matching entry
814 * with an offset at least one step larger */
815 return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
818 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
826 if (f->current_offset > 0) {
827 cp = f->current_offset;
829 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
833 r = next_with_matches(j, f, direction, &c, &cp);
837 r = find_location_with_matches(j, f, direction, &c, &cp);
842 /* OK, we found the spot, now let's advance until to an entry
843 * that is actually different from what we were previously
844 * looking at. This is necessary to handle entries which exist
845 * in two (or more) journal files, and which shall all be
846 * suppressed but one. */
851 if (j->current_location.type == LOCATION_DISCRETE) {
854 k = compare_with_location(f, c, &j->current_location);
855 if (direction == DIRECTION_DOWN)
870 r = next_with_matches(j, f, direction, &c, &cp);
876 static int real_journal_next(sd_journal *j, direction_t direction) {
877 JournalFile *f, *new_file = NULL;
878 uint64_t new_offset = 0;
887 HASHMAP_FOREACH(f, j->files, i) {
890 r = next_beyond_location(j, f, direction, &o, &p);
892 log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
902 k = compare_entry_order(f, o, new_file, new_offset);
904 if (direction == DIRECTION_DOWN)
919 r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
923 set_location(j, LOCATION_DISCRETE, new_file, o, new_offset);
928 _public_ int sd_journal_next(sd_journal *j) {
929 return real_journal_next(j, DIRECTION_DOWN);
932 _public_ int sd_journal_previous(sd_journal *j) {
933 return real_journal_next(j, DIRECTION_UP);
936 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
943 /* If this is not a discrete skip, then at least
944 * resolve the current location */
945 if (j->current_location.type != LOCATION_DISCRETE)
946 return real_journal_next(j, direction);
952 r = real_journal_next(j, direction);
966 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
967 return real_journal_next_skip(j, DIRECTION_DOWN, skip);
970 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
971 return real_journal_next_skip(j, DIRECTION_UP, skip);
974 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
977 char bid[33], sid[33];
984 if (!j->current_file || j->current_file->current_offset <= 0)
985 return -EADDRNOTAVAIL;
987 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
991 sd_id128_to_string(j->current_file->header->seqnum_id, sid);
992 sd_id128_to_string(o->entry.boot_id, bid);
995 "s=%s;i=%llx;b=%s;m=%llx;t=%llx;x=%llx",
996 sid, (unsigned long long) le64toh(o->entry.seqnum),
997 bid, (unsigned long long) le64toh(o->entry.monotonic),
998 (unsigned long long) le64toh(o->entry.realtime),
999 (unsigned long long) le64toh(o->entry.xor_hash)) < 0)
1005 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
1008 unsigned long long seqnum, monotonic, realtime, xor_hash;
1010 seqnum_id_set = false,
1012 boot_id_set = false,
1013 monotonic_set = false,
1014 realtime_set = false,
1015 xor_hash_set = false;
1016 sd_id128_t seqnum_id, boot_id;
1020 if (isempty(cursor))
1023 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1027 if (l < 2 || w[1] != '=')
1030 item = strndup(w, l);
1037 seqnum_id_set = true;
1038 k = sd_id128_from_string(item+2, &seqnum_id);
1043 if (sscanf(item+2, "%llx", &seqnum) != 1)
1049 k = sd_id128_from_string(item+2, &boot_id);
1053 monotonic_set = true;
1054 if (sscanf(item+2, "%llx", &monotonic) != 1)
1059 realtime_set = true;
1060 if (sscanf(item+2, "%llx", &realtime) != 1)
1065 xor_hash_set = true;
1066 if (sscanf(item+2, "%llx", &xor_hash) != 1)
1077 if ((!seqnum_set || !seqnum_id_set) &&
1078 (!monotonic_set || !boot_id_set) &&
1084 j->current_location.type = LOCATION_SEEK;
1087 j->current_location.realtime = (uint64_t) realtime;
1088 j->current_location.realtime_set = true;
1091 if (seqnum_set && seqnum_id_set) {
1092 j->current_location.seqnum = (uint64_t) seqnum;
1093 j->current_location.seqnum_id = seqnum_id;
1094 j->current_location.seqnum_set = true;
1097 if (monotonic_set && boot_id_set) {
1098 j->current_location.monotonic = (uint64_t) monotonic;
1099 j->current_location.boot_id = boot_id;
1100 j->current_location.monotonic_set = true;
1104 j->current_location.xor_hash = (uint64_t) xor_hash;
1105 j->current_location.xor_hash_set = true;
1111 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1119 if (isempty(cursor))
1122 if (!j->current_file || j->current_file->current_offset <= 0)
1123 return -EADDRNOTAVAIL;
1125 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1129 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1130 _cleanup_free_ char *item = NULL;
1132 unsigned long long ll;
1135 if (l < 2 || w[1] != '=')
1138 item = strndup(w, l);
1145 k = sd_id128_from_string(item+2, &id);
1148 if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1153 if (sscanf(item+2, "%llx", &ll) != 1)
1155 if (ll != le64toh(o->entry.seqnum))
1160 k = sd_id128_from_string(item+2, &id);
1163 if (!sd_id128_equal(id, o->entry.boot_id))
1168 if (sscanf(item+2, "%llx", &ll) != 1)
1170 if (ll != le64toh(o->entry.monotonic))
1175 if (sscanf(item+2, "%llx", &ll) != 1)
1177 if (ll != le64toh(o->entry.realtime))
1182 if (sscanf(item+2, "%llx", &ll) != 1)
1184 if (ll != le64toh(o->entry.xor_hash))
1194 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1199 j->current_location.type = LOCATION_SEEK;
1200 j->current_location.boot_id = boot_id;
1201 j->current_location.monotonic = usec;
1202 j->current_location.monotonic_set = true;
1207 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1212 j->current_location.type = LOCATION_SEEK;
1213 j->current_location.realtime = usec;
1214 j->current_location.realtime_set = true;
1219 _public_ int sd_journal_seek_head(sd_journal *j) {
1224 j->current_location.type = LOCATION_HEAD;
1229 _public_ int sd_journal_seek_tail(sd_journal *j) {
1234 j->current_location.type = LOCATION_TAIL;
1239 static void check_network(sd_journal *j, int fd) {
1247 if (fstatfs(fd, &sfs) < 0)
1251 (unsigned long) sfs.f_type == CIFS_MAGIC_NUMBER ||
1252 (unsigned long) sfs.f_type == CODA_SUPER_MAGIC ||
1253 (unsigned long) sfs.f_type == NCP_SUPER_MAGIC ||
1254 (unsigned long) sfs.f_type == NFS_SUPER_MAGIC ||
1255 (unsigned long) sfs.f_type == SMB_SUPER_MAGIC;
1258 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1259 char _cleanup_free_ *path = NULL;
1267 if ((j->flags & SD_JOURNAL_SYSTEM_ONLY) &&
1268 !(streq(filename, "system.journal") ||
1269 streq(filename, "system.journal~") ||
1270 (startswith(filename, "system@") &&
1271 (endswith(filename, ".journal") || endswith(filename, ".journal~")))))
1274 path = strjoin(prefix, "/", filename, NULL);
1278 if (hashmap_get(j->files, path))
1281 if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1282 log_debug("Too many open journal files, not adding %s, ignoring.", path);
1283 return set_put_error(j, -ETOOMANYREFS);
1286 r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1288 if (errno == ENOENT)
1294 /* journal_file_dump(f); */
1296 r = hashmap_put(j->files, f->path, f);
1298 journal_file_close(f);
1302 log_debug("File %s got added.", f->path);
1304 check_network(j, f->fd);
1306 j->current_invalidate_counter ++;
1311 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1319 path = strjoin(prefix, "/", filename, NULL);
1323 f = hashmap_get(j->files, path);
1328 hashmap_remove(j->files, f->path);
1330 log_debug("File %s got removed.", f->path);
1332 if (j->current_file == f) {
1333 j->current_file = NULL;
1334 j->current_field = 0;
1337 if (j->unique_file == f) {
1338 j->unique_file = NULL;
1339 j->unique_offset = 0;
1342 journal_file_close(f);
1344 j->current_invalidate_counter ++;
1349 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1350 char _cleanup_free_ *path = NULL;
1352 DIR _cleanup_closedir_ *d = NULL;
1360 log_debug("Considering %s/%s.", prefix, dirname);
1362 if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1363 (sd_id128_from_string(dirname, &id) < 0 ||
1364 sd_id128_get_machine(&mid) < 0 ||
1365 !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1368 path = strjoin(prefix, "/", dirname, NULL);
1374 log_debug("Failed to open %s: %m", path);
1375 if (errno == ENOENT)
1380 m = hashmap_get(j->directories_by_path, path);
1382 m = new0(Directory, 1);
1389 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1394 path = NULL; /* avoid freeing in cleanup */
1395 j->current_invalidate_counter ++;
1397 log_debug("Directory %s got added.", m->path);
1399 } else if (m->is_root)
1402 if (m->wd <= 0 && j->inotify_fd >= 0) {
1404 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1405 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1406 IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1409 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1410 inotify_rm_watch(j->inotify_fd, m->wd);
1415 union dirent_storage buf;
1417 r = readdir_r(d, &buf.de, &de);
1421 if (dirent_is_file_with_suffix(de, ".journal") ||
1422 dirent_is_file_with_suffix(de, ".journal~")) {
1423 r = add_file(j, m->path, de->d_name);
1425 log_debug("Failed to add file %s/%s: %s",
1426 m->path, de->d_name, strerror(-r));
1427 r = set_put_error(j, r);
1434 check_network(j, dirfd(d));
1439 static int add_root_directory(sd_journal *j, const char *p) {
1440 DIR _cleanup_closedir_ *d = NULL;
1447 if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1448 !path_startswith(p, "/run"))
1455 m = hashmap_get(j->directories_by_path, p);
1457 m = new0(Directory, 1);
1462 m->path = strdup(p);
1468 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1474 j->current_invalidate_counter ++;
1476 log_debug("Root directory %s got added.", m->path);
1478 } else if (!m->is_root)
1481 if (m->wd <= 0 && j->inotify_fd >= 0) {
1483 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1484 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1487 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1488 inotify_rm_watch(j->inotify_fd, m->wd);
1493 union dirent_storage buf;
1496 r = readdir_r(d, &buf.de, &de);
1500 if (dirent_is_file_with_suffix(de, ".journal") ||
1501 dirent_is_file_with_suffix(de, ".journal~")) {
1502 r = add_file(j, m->path, de->d_name);
1504 log_debug("Failed to add file %s/%s: %s",
1505 m->path, de->d_name, strerror(-r));
1506 r = set_put_error(j, r);
1510 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1511 sd_id128_from_string(de->d_name, &id) >= 0) {
1513 r = add_directory(j, m->path, de->d_name);
1515 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1519 check_network(j, dirfd(d));
1524 static int remove_directory(sd_journal *j, Directory *d) {
1528 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1530 if (j->inotify_fd >= 0)
1531 inotify_rm_watch(j->inotify_fd, d->wd);
1534 hashmap_remove(j->directories_by_path, d->path);
1537 log_debug("Root directory %s got removed.", d->path);
1539 log_debug("Directory %s got removed.", d->path);
1547 static int add_search_paths(sd_journal *j) {
1549 const char search_paths[] =
1550 "/run/log/journal\0"
1551 "/var/log/journal\0";
1556 /* We ignore most errors here, since the idea is to only open
1557 * what's actually accessible, and ignore the rest. */
1559 NULSTR_FOREACH(p, search_paths) {
1560 r = add_root_directory(j, p);
1561 if (r < 0 && r != -ENOENT) {
1562 r = set_put_error(j, r);
1571 static int allocate_inotify(sd_journal *j) {
1574 if (j->inotify_fd < 0) {
1575 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1576 if (j->inotify_fd < 0)
1580 if (!j->directories_by_wd) {
1581 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1582 if (!j->directories_by_wd)
1589 static sd_journal *journal_new(int flags, const char *path) {
1592 j = new0(sd_journal, 1);
1598 j->data_threshold = DEFAULT_DATA_THRESHOLD;
1601 j->path = strdup(path);
1606 j->files = hashmap_new(string_hash_func, string_compare_func);
1607 j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1608 j->mmap = mmap_cache_new();
1609 if (!j->files || !j->directories_by_path || !j->mmap)
1615 sd_journal_close(j);
1619 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1626 if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1627 SD_JOURNAL_RUNTIME_ONLY|
1628 SD_JOURNAL_SYSTEM_ONLY))
1631 j = journal_new(flags, NULL);
1635 r = add_search_paths(j);
1643 sd_journal_close(j);
1648 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1661 j = journal_new(flags, path);
1665 r = add_root_directory(j, path);
1667 set_put_error(j, r);
1675 sd_journal_close(j);
1680 _public_ void sd_journal_close(sd_journal *j) {
1687 sd_journal_flush_matches(j);
1689 while ((f = hashmap_steal_first(j->files)))
1690 journal_file_close(f);
1692 hashmap_free(j->files);
1694 while ((d = hashmap_first(j->directories_by_path)))
1695 remove_directory(j, d);
1697 while ((d = hashmap_first(j->directories_by_wd)))
1698 remove_directory(j, d);
1700 hashmap_free(j->directories_by_path);
1701 hashmap_free(j->directories_by_wd);
1703 if (j->inotify_fd >= 0)
1704 close_nointr_nofail(j->inotify_fd);
1707 mmap_cache_unref(j->mmap);
1710 free(j->unique_field);
1711 set_free(j->errors);
1715 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1725 f = j->current_file;
1727 return -EADDRNOTAVAIL;
1729 if (f->current_offset <= 0)
1730 return -EADDRNOTAVAIL;
1732 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1736 *ret = le64toh(o->entry.realtime);
1740 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1749 f = j->current_file;
1751 return -EADDRNOTAVAIL;
1753 if (f->current_offset <= 0)
1754 return -EADDRNOTAVAIL;
1756 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1761 *ret_boot_id = o->entry.boot_id;
1763 r = sd_id128_get_boot(&id);
1767 if (!sd_id128_equal(id, o->entry.boot_id))
1772 *ret = le64toh(o->entry.monotonic);
1777 static bool field_is_valid(const char *field) {
1785 if (startswith(field, "__"))
1788 for (p = field; *p; p++) {
1793 if (*p >= 'A' && *p <= 'Z')
1796 if (*p >= '0' && *p <= '9')
1805 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1808 size_t field_length;
1821 if (!field_is_valid(field))
1824 f = j->current_file;
1826 return -EADDRNOTAVAIL;
1828 if (f->current_offset <= 0)
1829 return -EADDRNOTAVAIL;
1831 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1835 field_length = strlen(field);
1837 n = journal_file_entry_n_items(o);
1838 for (i = 0; i < n; i++) {
1843 p = le64toh(o->entry.items[i].object_offset);
1844 le_hash = o->entry.items[i].hash;
1845 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1849 if (le_hash != o->data.hash)
1852 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1854 if (o->object.flags & OBJECT_COMPRESSED) {
1857 if (uncompress_startswith(o->data.payload, l,
1858 &f->compress_buffer, &f->compress_buffer_size,
1859 field, field_length, '=')) {
1863 if (!uncompress_blob(o->data.payload, l,
1864 &f->compress_buffer, &f->compress_buffer_size, &rsize,
1868 *data = f->compress_buffer;
1869 *size = (size_t) rsize;
1874 return -EPROTONOSUPPORT;
1877 } else if (l >= field_length+1 &&
1878 memcmp(o->data.payload, field, field_length) == 0 &&
1879 o->data.payload[field_length] == '=') {
1883 if ((uint64_t) t != l)
1886 *data = o->data.payload;
1892 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1900 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1904 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1907 /* We can't read objects larger than 4G on a 32bit machine */
1908 if ((uint64_t) t != l)
1911 if (o->object.flags & OBJECT_COMPRESSED) {
1915 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
1918 *data = f->compress_buffer;
1919 *size = (size_t) rsize;
1921 return -EPROTONOSUPPORT;
1924 *data = o->data.payload;
1931 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1945 f = j->current_file;
1947 return -EADDRNOTAVAIL;
1949 if (f->current_offset <= 0)
1950 return -EADDRNOTAVAIL;
1952 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1956 n = journal_file_entry_n_items(o);
1957 if (j->current_field >= n)
1960 p = le64toh(o->entry.items[j->current_field].object_offset);
1961 le_hash = o->entry.items[j->current_field].hash;
1962 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1966 if (le_hash != o->data.hash)
1969 r = return_data(j, f, o, data, size);
1973 j->current_field ++;
1978 _public_ void sd_journal_restart_data(sd_journal *j) {
1982 j->current_field = 0;
1985 _public_ int sd_journal_get_fd(sd_journal *j) {
1991 if (j->inotify_fd >= 0)
1992 return j->inotify_fd;
1994 r = allocate_inotify(j);
1998 /* Iterate through all dirs again, to add them to the
2001 r = add_root_directory(j, j->path);
2003 r = add_search_paths(j);
2007 return j->inotify_fd;
2010 _public_ int sd_journal_get_events(sd_journal *j) {
2016 fd = sd_journal_get_fd(j);
2023 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2031 fd = sd_journal_get_fd(j);
2035 if (!j->on_network) {
2036 *timeout_usec = (uint64_t) -1;
2040 /* If we are on the network we need to regularly check for
2041 * changes manually */
2043 *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2047 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2054 /* Is this a subdirectory we watch? */
2055 d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2059 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2060 (endswith(e->name, ".journal") ||
2061 endswith(e->name, ".journal~"))) {
2063 /* Event for a journal file */
2065 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2066 r = add_file(j, d->path, e->name);
2068 log_debug("Failed to add file %s/%s: %s",
2069 d->path, e->name, strerror(-r));
2070 set_put_error(j, r);
2073 } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2075 r = remove_file(j, d->path, e->name);
2077 log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2080 } else if (!d->is_root && e->len == 0) {
2082 /* Event for a subdirectory */
2084 if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2085 r = remove_directory(j, d);
2087 log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2091 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2093 /* Event for root directory */
2095 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2096 r = add_directory(j, d->path, e->name);
2098 log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2105 if (e->mask & IN_IGNORED)
2108 log_warning("Unknown inotify event.");
2111 static int determine_change(sd_journal *j) {
2116 b = j->current_invalidate_counter != j->last_invalidate_counter;
2117 j->last_invalidate_counter = j->current_invalidate_counter;
2119 return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2122 _public_ int sd_journal_process(sd_journal *j) {
2123 uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2124 bool got_something = false;
2129 j->last_process_usec = now(CLOCK_MONOTONIC);
2132 struct inotify_event *e;
2135 l = read(j->inotify_fd, buffer, sizeof(buffer));
2137 if (errno == EAGAIN || errno == EINTR)
2138 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2143 got_something = true;
2145 e = (struct inotify_event*) buffer;
2149 process_inotify_event(j, e);
2151 step = sizeof(struct inotify_event) + e->len;
2152 assert(step <= (size_t) l);
2154 e = (struct inotify_event*) ((uint8_t*) e + step);
2159 return determine_change(j);
2162 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2168 if (j->inotify_fd < 0) {
2170 /* This is the first invocation, hence create the
2172 r = sd_journal_get_fd(j);
2176 /* The journal might have changed since the context
2177 * object was created and we weren't watching before,
2178 * hence don't wait for anything, and return
2180 return determine_change(j);
2183 r = sd_journal_get_timeout(j, &t);
2187 if (t != (uint64_t) -1) {
2190 n = now(CLOCK_MONOTONIC);
2191 t = t > n ? t - n : 0;
2193 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2198 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2199 } while (r == -EINTR);
2204 return sd_journal_process(j);
2207 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2218 HASHMAP_FOREACH(f, j->files, i) {
2221 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2237 *from = MIN(fr, *from);
2243 return first ? 0 : 1;
2246 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2257 HASHMAP_FOREACH(f, j->files, i) {
2260 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2276 *from = MIN(fr, *from);
2282 return first ? 0 : 1;
2285 void journal_print_header(sd_journal *j) {
2288 bool newline = false;
2292 HASHMAP_FOREACH(f, j->files, i) {
2298 journal_file_print_header(f);
2302 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2312 HASHMAP_FOREACH(f, j->files, i) {
2315 if (fstat(f->fd, &st) < 0)
2318 sum += (uint64_t) st.st_blocks * 512ULL;
2325 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2332 if (!field_is_valid(field))
2339 free(j->unique_field);
2340 j->unique_field = f;
2341 j->unique_file = NULL;
2342 j->unique_offset = 0;
2347 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2358 if (!j->unique_field)
2361 k = strlen(j->unique_field);
2363 if (!j->unique_file) {
2364 j->unique_file = hashmap_first(j->files);
2365 if (!j->unique_file)
2367 j->unique_offset = 0;
2377 /* Proceed to next data object in the field's linked list */
2378 if (j->unique_offset == 0) {
2379 r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2383 j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2385 r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2389 j->unique_offset = le64toh(o->data.next_field_offset);
2392 /* We reached the end of the list? Then start again, with the next file */
2393 if (j->unique_offset == 0) {
2396 n = hashmap_next(j->files, j->unique_file->path);
2404 /* We do not use the type context here, but 0 instead,
2405 * so that we can look at this data object at the same
2406 * time as one on another file */
2407 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2411 /* Let's do the type check by hand, since we used 0 context above. */
2412 if (o->object.type != OBJECT_DATA)
2415 r = return_data(j, j->unique_file, o, &odata, &ol);
2419 /* OK, now let's see if we already returned this data
2420 * object by checking if it exists in the earlier
2421 * traversed files. */
2423 HASHMAP_FOREACH(of, j->files, i) {
2427 if (of == j->unique_file)
2430 /* Skip this file it didn't have any fields
2432 if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2433 le64toh(of->header->n_fields) <= 0)
2436 r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2447 r = return_data(j, j->unique_file, o, data, l);
2455 _public_ void sd_journal_restart_unique(sd_journal *j) {
2459 j->unique_file = NULL;
2460 j->unique_offset = 0;
2463 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2467 return !j->on_network;
2470 static char *lookup_field(const char *field, void *userdata) {
2471 sd_journal *j = userdata;
2479 r = sd_journal_get_data(j, field, &data, &size);
2481 size > REPLACE_VAR_MAX)
2482 return strdup(field);
2484 d = strlen(field) + 1;
2486 return strndup((const char*) data + d, size - d);
2489 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2493 _cleanup_free_ char *text = NULL, *cid = NULL;
2502 r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2506 cid = strndup((const char*) data + 11, size - 11);
2510 r = sd_id128_from_string(cid, &id);
2514 r = catalog_get(CATALOG_DATABASE, id, &text);
2518 t = replace_var(text, lookup_field, j);
2526 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2530 return catalog_get(CATALOG_DATABASE, id, ret);
2533 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2537 j->data_threshold = sz;
2541 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2547 *sz = j->data_threshold;