1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
26 #include <sys/inotify.h>
29 #include <linux/magic.h>
31 #include "sd-journal.h"
32 #include "journal-def.h"
33 #include "journal-file.h"
36 #include "path-util.h"
39 #include "journal-internal.h"
42 #include "replace-var.h"
44 #define JOURNAL_FILES_MAX 1024
46 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
48 #define REPLACE_VAR_MAX 256
50 #define DEFAULT_DATA_THRESHOLD (64*1024)
52 /* We return an error here only if we didn't manage to
53 memorize the real error. */
54 static int set_put_error(sd_journal *j, int r) {
60 k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
64 return set_put(j->errors, INT_TO_PTR(r));
67 static void detach_location(sd_journal *j) {
73 j->current_file = NULL;
76 HASHMAP_FOREACH(f, j->files, i)
77 f->current_offset = 0;
80 static void reset_location(sd_journal *j) {
84 zero(j->current_location);
87 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
89 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
91 assert(o->object.type == OBJECT_ENTRY);
94 l->seqnum = le64toh(o->entry.seqnum);
95 l->seqnum_id = f->header->seqnum_id;
96 l->realtime = le64toh(o->entry.realtime);
97 l->monotonic = le64toh(o->entry.monotonic);
98 l->boot_id = o->entry.boot_id;
99 l->xor_hash = le64toh(o->entry.xor_hash);
101 l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
104 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o, uint64_t offset) {
106 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
110 init_location(&j->current_location, type, f, o);
113 j->current_file->current_offset = 0;
116 j->current_field = 0;
118 f->current_offset = offset;
121 static int match_is_valid(const void *data, size_t size) {
129 if (startswith(data, "__"))
133 for (p = b; p < b + size; p++) {
141 if (*p >= 'A' && *p <= 'Z')
144 if (*p >= '0' && *p <= '9')
153 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
154 const uint8_t *a = _a, *b = _b;
157 for (j = 0; j < s && j < t; j++) {
169 static Match *match_new(Match *p, MatchType t) {
180 LIST_PREPEND(Match, matches, p->matches, m);
186 static void match_free(Match *m) {
190 match_free(m->matches);
193 LIST_REMOVE(Match, matches, m->parent->matches, m);
199 static void match_free_if_empty(Match *m) {
200 if (!m || m->matches)
206 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
207 Match *l3, *l4, *add_here = NULL, *m;
219 if (!match_is_valid(data, size))
226 * level 4: concrete matches */
229 j->level0 = match_new(NULL, MATCH_AND_TERM);
235 j->level1 = match_new(j->level0, MATCH_OR_TERM);
241 j->level2 = match_new(j->level1, MATCH_AND_TERM);
246 assert(j->level0->type == MATCH_AND_TERM);
247 assert(j->level1->type == MATCH_OR_TERM);
248 assert(j->level2->type == MATCH_AND_TERM);
250 le_hash = htole64(hash64(data, size));
252 LIST_FOREACH(matches, l3, j->level2->matches) {
253 assert(l3->type == MATCH_OR_TERM);
255 LIST_FOREACH(matches, l4, l3->matches) {
256 assert(l4->type == MATCH_DISCRETE);
258 /* Exactly the same match already? Then ignore
260 if (l4->le_hash == le_hash &&
262 memcmp(l4->data, data, size) == 0)
265 /* Same field? Then let's add this to this OR term */
266 if (same_field(data, size, l4->data, l4->size)) {
277 add_here = match_new(j->level2, MATCH_OR_TERM);
282 m = match_new(add_here, MATCH_DISCRETE);
286 m->le_hash = le_hash;
288 m->data = memdup(data, size);
297 match_free_if_empty(add_here);
298 match_free_if_empty(j->level2);
299 match_free_if_empty(j->level1);
300 match_free_if_empty(j->level0);
305 _public_ int sd_journal_add_conjunction(sd_journal *j) {
314 if (!j->level1->matches)
323 _public_ int sd_journal_add_disjunction(sd_journal *j) {
335 if (!j->level2->matches)
342 static char *match_make_string(Match *m) {
345 bool enclose = false;
350 if (m->type == MATCH_DISCRETE)
351 return strndup(m->data, m->size);
354 LIST_FOREACH(matches, i, m->matches) {
357 t = match_make_string(i);
364 k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
381 r = strjoin("(", p, ")", NULL);
389 char *journal_make_match_string(sd_journal *j) {
392 return match_make_string(j->level0);
395 _public_ void sd_journal_flush_matches(sd_journal *j) {
401 match_free(j->level0);
403 j->level0 = j->level1 = j->level2 = NULL;
408 static int compare_entry_order(JournalFile *af, Object *_ao,
409 JournalFile *bf, uint64_t bp) {
419 /* The mmap cache might invalidate the object from the first
420 * file if we look at the one from the second file. Hence
421 * temporarily copy the header of the first one, and look at
423 ao = alloca(offsetof(EntryObject, items));
424 memcpy(ao, _ao, offsetof(EntryObject, items));
426 r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
428 return strcmp(af->path, bf->path);
430 /* We operate on two different files here, hence we can access
431 * two objects at the same time, which we normally can't.
433 * If contents and timestamps match, these entries are
434 * identical, even if the seqnum does not match */
436 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
437 ao->entry.monotonic == bo->entry.monotonic &&
438 ao->entry.realtime == bo->entry.realtime &&
439 ao->entry.xor_hash == bo->entry.xor_hash)
442 if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
444 /* If this is from the same seqnum source, compare
446 a = le64toh(ao->entry.seqnum);
447 b = le64toh(bo->entry.seqnum);
454 /* Wow! This is weird, different data but the same
455 * seqnums? Something is borked, but let's make the
456 * best of it and compare by time. */
459 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
461 /* If the boot id matches compare monotonic time */
462 a = le64toh(ao->entry.monotonic);
463 b = le64toh(bo->entry.monotonic);
471 /* Otherwise compare UTC time */
472 a = le64toh(ao->entry.realtime);
473 b = le64toh(bo->entry.realtime);
480 /* Finally, compare by contents */
481 a = le64toh(ao->entry.xor_hash);
482 b = le64toh(bo->entry.xor_hash);
492 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
498 assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
500 if (l->monotonic_set &&
501 sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
503 le64toh(ao->entry.realtime) == l->realtime &&
505 le64toh(ao->entry.xor_hash) == l->xor_hash)
509 sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
511 a = le64toh(ao->entry.seqnum);
519 if (l->monotonic_set &&
520 sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
522 a = le64toh(ao->entry.monotonic);
524 if (a < l->monotonic)
526 if (a > l->monotonic)
530 if (l->realtime_set) {
532 a = le64toh(ao->entry.realtime);
540 if (l->xor_hash_set) {
541 a = le64toh(ao->entry.xor_hash);
552 static int next_for_match(
556 uint64_t after_offset,
557 direction_t direction,
569 if (m->type == MATCH_DISCRETE) {
572 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
576 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
578 } else if (m->type == MATCH_OR_TERM) {
581 /* Find the earliest match beyond after_offset */
583 LIST_FOREACH(matches, i, m->matches) {
586 r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
590 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
595 } else if (m->type == MATCH_AND_TERM) {
597 bool continue_looking;
599 /* Always jump to the next matching entry and repeat
600 * this until we fine and offset that matches for all
608 continue_looking = false;
610 LIST_FOREACH(matches, i, m->matches) {
614 limit = after_offset;
615 else if (direction == DIRECTION_DOWN)
616 limit = MAX(np, after_offset);
618 limit = MIN(np, after_offset);
620 r = next_for_match(j, i, f, limit, direction, NULL, &cp);
624 if ((direction == DIRECTION_DOWN ? cp >= after_offset : cp <= after_offset) &&
625 (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))) {
627 continue_looking = true;
631 } while (continue_looking);
637 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
649 static int find_location_for_match(
653 direction_t direction,
663 if (m->type == MATCH_DISCRETE) {
666 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
670 /* FIXME: missing: find by monotonic */
672 if (j->current_location.type == LOCATION_HEAD)
673 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
674 if (j->current_location.type == LOCATION_TAIL)
675 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
676 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
677 return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
678 if (j->current_location.monotonic_set) {
679 r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
683 if (j->current_location.realtime_set)
684 return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
686 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
688 } else if (m->type == MATCH_OR_TERM) {
693 /* Find the earliest match */
695 LIST_FOREACH(matches, i, m->matches) {
698 r = find_location_for_match(j, i, f, direction, NULL, &cp);
702 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
710 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
725 assert(m->type == MATCH_AND_TERM);
727 /* First jump to the last match, and then find the
728 * next one where all matches match */
733 LIST_FOREACH(matches, i, m->matches) {
736 r = find_location_for_match(j, i, f, direction, NULL, &cp);
740 if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
744 return next_for_match(j, m, f, np, direction, ret, offset);
748 static int find_location_with_matches(
751 direction_t direction,
763 /* No matches is simple */
765 if (j->current_location.type == LOCATION_HEAD)
766 return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
767 if (j->current_location.type == LOCATION_TAIL)
768 return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
769 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
770 return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
771 if (j->current_location.monotonic_set) {
772 r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
776 if (j->current_location.realtime_set)
777 return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
779 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
781 return find_location_for_match(j, j->level0, f, direction, ret, offset);
784 static int next_with_matches(
787 direction_t direction,
802 /* No matches is easy. We simple advance the file
805 return journal_file_next_entry(f, c, cp, direction, ret, offset);
807 /* If we have a match then we look for the next matching entry
808 * with an offset at least one step larger */
809 return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
812 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
820 if (f->current_offset > 0) {
821 cp = f->current_offset;
823 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
827 r = next_with_matches(j, f, direction, &c, &cp);
831 r = find_location_with_matches(j, f, direction, &c, &cp);
836 /* OK, we found the spot, now let's advance until to an entry
837 * that is actually different from what we were previously
838 * looking at. This is necessary to handle entries which exist
839 * in two (or more) journal files, and which shall all be
840 * suppressed but one. */
845 if (j->current_location.type == LOCATION_DISCRETE) {
848 k = compare_with_location(f, c, &j->current_location);
849 if (direction == DIRECTION_DOWN)
864 r = next_with_matches(j, f, direction, &c, &cp);
870 static int real_journal_next(sd_journal *j, direction_t direction) {
871 JournalFile *f, *new_file = NULL;
872 uint64_t new_offset = 0;
881 HASHMAP_FOREACH(f, j->files, i) {
884 r = next_beyond_location(j, f, direction, &o, &p);
886 log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
896 k = compare_entry_order(f, o, new_file, new_offset);
898 if (direction == DIRECTION_DOWN)
913 r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
917 set_location(j, LOCATION_DISCRETE, new_file, o, new_offset);
922 _public_ int sd_journal_next(sd_journal *j) {
923 return real_journal_next(j, DIRECTION_DOWN);
926 _public_ int sd_journal_previous(sd_journal *j) {
927 return real_journal_next(j, DIRECTION_UP);
930 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
937 /* If this is not a discrete skip, then at least
938 * resolve the current location */
939 if (j->current_location.type != LOCATION_DISCRETE)
940 return real_journal_next(j, direction);
946 r = real_journal_next(j, direction);
960 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
961 return real_journal_next_skip(j, DIRECTION_DOWN, skip);
964 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
965 return real_journal_next_skip(j, DIRECTION_UP, skip);
968 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
971 char bid[33], sid[33];
978 if (!j->current_file || j->current_file->current_offset <= 0)
979 return -EADDRNOTAVAIL;
981 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
985 sd_id128_to_string(j->current_file->header->seqnum_id, sid);
986 sd_id128_to_string(o->entry.boot_id, bid);
989 "s=%s;i=%llx;b=%s;m=%llx;t=%llx;x=%llx",
990 sid, (unsigned long long) le64toh(o->entry.seqnum),
991 bid, (unsigned long long) le64toh(o->entry.monotonic),
992 (unsigned long long) le64toh(o->entry.realtime),
993 (unsigned long long) le64toh(o->entry.xor_hash)) < 0)
999 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
1002 unsigned long long seqnum, monotonic, realtime, xor_hash;
1004 seqnum_id_set = false,
1006 boot_id_set = false,
1007 monotonic_set = false,
1008 realtime_set = false,
1009 xor_hash_set = false;
1010 sd_id128_t seqnum_id, boot_id;
1014 if (isempty(cursor))
1017 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1021 if (l < 2 || w[1] != '=')
1024 item = strndup(w, l);
1031 seqnum_id_set = true;
1032 k = sd_id128_from_string(item+2, &seqnum_id);
1037 if (sscanf(item+2, "%llx", &seqnum) != 1)
1043 k = sd_id128_from_string(item+2, &boot_id);
1047 monotonic_set = true;
1048 if (sscanf(item+2, "%llx", &monotonic) != 1)
1053 realtime_set = true;
1054 if (sscanf(item+2, "%llx", &realtime) != 1)
1059 xor_hash_set = true;
1060 if (sscanf(item+2, "%llx", &xor_hash) != 1)
1071 if ((!seqnum_set || !seqnum_id_set) &&
1072 (!monotonic_set || !boot_id_set) &&
1078 j->current_location.type = LOCATION_SEEK;
1081 j->current_location.realtime = (uint64_t) realtime;
1082 j->current_location.realtime_set = true;
1085 if (seqnum_set && seqnum_id_set) {
1086 j->current_location.seqnum = (uint64_t) seqnum;
1087 j->current_location.seqnum_id = seqnum_id;
1088 j->current_location.seqnum_set = true;
1091 if (monotonic_set && boot_id_set) {
1092 j->current_location.monotonic = (uint64_t) monotonic;
1093 j->current_location.boot_id = boot_id;
1094 j->current_location.monotonic_set = true;
1098 j->current_location.xor_hash = (uint64_t) xor_hash;
1099 j->current_location.xor_hash_set = true;
1105 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1113 if (isempty(cursor))
1116 if (!j->current_file || j->current_file->current_offset <= 0)
1117 return -EADDRNOTAVAIL;
1119 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1123 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1124 _cleanup_free_ char *item = NULL;
1126 unsigned long long ll;
1129 if (l < 2 || w[1] != '=')
1132 item = strndup(w, l);
1139 k = sd_id128_from_string(item+2, &id);
1142 if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1147 if (sscanf(item+2, "%llx", &ll) != 1)
1149 if (ll != le64toh(o->entry.seqnum))
1154 k = sd_id128_from_string(item+2, &id);
1157 if (!sd_id128_equal(id, o->entry.boot_id))
1162 if (sscanf(item+2, "%llx", &ll) != 1)
1164 if (ll != le64toh(o->entry.monotonic))
1169 if (sscanf(item+2, "%llx", &ll) != 1)
1171 if (ll != le64toh(o->entry.realtime))
1176 if (sscanf(item+2, "%llx", &ll) != 1)
1178 if (ll != le64toh(o->entry.xor_hash))
1188 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1193 j->current_location.type = LOCATION_SEEK;
1194 j->current_location.boot_id = boot_id;
1195 j->current_location.monotonic = usec;
1196 j->current_location.monotonic_set = true;
1201 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1206 j->current_location.type = LOCATION_SEEK;
1207 j->current_location.realtime = usec;
1208 j->current_location.realtime_set = true;
1213 _public_ int sd_journal_seek_head(sd_journal *j) {
1218 j->current_location.type = LOCATION_HEAD;
1223 _public_ int sd_journal_seek_tail(sd_journal *j) {
1228 j->current_location.type = LOCATION_TAIL;
1233 static void check_network(sd_journal *j, int fd) {
1241 if (fstatfs(fd, &sfs) < 0)
1245 F_TYPE_CMP(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1246 F_TYPE_CMP(sfs.f_type, CODA_SUPER_MAGIC) ||
1247 F_TYPE_CMP(sfs.f_type, NCP_SUPER_MAGIC) ||
1248 F_TYPE_CMP(sfs.f_type, NFS_SUPER_MAGIC) ||
1249 F_TYPE_CMP(sfs.f_type, SMB_SUPER_MAGIC);
1252 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1253 const char *full, *tilded, *atted;
1255 full = strappend(prefix, ".journal");
1256 tilded = strappenda(full, "~");
1257 atted = strappenda(prefix, "@");
1259 return streq(filename, full) ||
1260 streq(filename, tilded) ||
1261 startswith(filename, atted);
1264 static bool file_type_wanted(int flags, const char *filename) {
1265 if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1268 /* no flags set → every type is OK */
1269 if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1272 if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1275 if (flags & SD_JOURNAL_CURRENT_USER) {
1276 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1278 assert_se(snprintf(prefix, sizeof(prefix), "user-%lu", (unsigned long) getuid())
1279 < (int) sizeof(prefix));
1281 if (file_has_type_prefix(prefix, filename))
1288 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1289 _cleanup_free_ char *path = NULL;
1297 if (!file_type_wanted(j->flags, filename))
1300 path = strjoin(prefix, "/", filename, NULL);
1304 if (hashmap_get(j->files, path))
1307 if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1308 log_debug("Too many open journal files, not adding %s, ignoring.", path);
1309 return set_put_error(j, -ETOOMANYREFS);
1312 r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1314 if (errno == ENOENT)
1320 /* journal_file_dump(f); */
1322 r = hashmap_put(j->files, f->path, f);
1324 journal_file_close(f);
1328 log_debug("File %s added.", f->path);
1330 check_network(j, f->fd);
1332 j->current_invalidate_counter ++;
1337 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1345 path = strjoin(prefix, "/", filename, NULL);
1349 f = hashmap_get(j->files, path);
1354 hashmap_remove(j->files, f->path);
1356 log_debug("File %s removed.", f->path);
1358 if (j->current_file == f) {
1359 j->current_file = NULL;
1360 j->current_field = 0;
1363 if (j->unique_file == f) {
1364 j->unique_file = NULL;
1365 j->unique_offset = 0;
1368 journal_file_close(f);
1370 j->current_invalidate_counter ++;
1375 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1376 _cleanup_free_ char *path = NULL;
1378 _cleanup_closedir_ DIR *d = NULL;
1386 log_debug("Considering %s/%s.", prefix, dirname);
1388 if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1389 (sd_id128_from_string(dirname, &id) < 0 ||
1390 sd_id128_get_machine(&mid) < 0 ||
1391 !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1394 path = strjoin(prefix, "/", dirname, NULL);
1400 log_debug("Failed to open %s: %m", path);
1401 if (errno == ENOENT)
1406 m = hashmap_get(j->directories_by_path, path);
1408 m = new0(Directory, 1);
1415 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1420 path = NULL; /* avoid freeing in cleanup */
1421 j->current_invalidate_counter ++;
1423 log_debug("Directory %s added.", m->path);
1425 } else if (m->is_root)
1428 if (m->wd <= 0 && j->inotify_fd >= 0) {
1430 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1431 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1432 IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1435 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1436 inotify_rm_watch(j->inotify_fd, m->wd);
1441 union dirent_storage buf;
1443 r = readdir_r(d, &buf.de, &de);
1447 if (dirent_is_file_with_suffix(de, ".journal") ||
1448 dirent_is_file_with_suffix(de, ".journal~")) {
1449 r = add_file(j, m->path, de->d_name);
1451 log_debug("Failed to add file %s/%s: %s",
1452 m->path, de->d_name, strerror(-r));
1453 r = set_put_error(j, r);
1460 check_network(j, dirfd(d));
1465 static int add_root_directory(sd_journal *j, const char *p) {
1466 _cleanup_closedir_ DIR *d = NULL;
1473 if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1474 !path_startswith(p, "/run"))
1481 m = hashmap_get(j->directories_by_path, p);
1483 m = new0(Directory, 1);
1488 m->path = strdup(p);
1494 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1500 j->current_invalidate_counter ++;
1502 log_debug("Root directory %s added.", m->path);
1504 } else if (!m->is_root)
1507 if (m->wd <= 0 && j->inotify_fd >= 0) {
1509 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1510 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1513 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1514 inotify_rm_watch(j->inotify_fd, m->wd);
1519 union dirent_storage buf;
1522 r = readdir_r(d, &buf.de, &de);
1526 if (dirent_is_file_with_suffix(de, ".journal") ||
1527 dirent_is_file_with_suffix(de, ".journal~")) {
1528 r = add_file(j, m->path, de->d_name);
1530 log_debug("Failed to add file %s/%s: %s",
1531 m->path, de->d_name, strerror(-r));
1532 r = set_put_error(j, r);
1536 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1537 sd_id128_from_string(de->d_name, &id) >= 0) {
1539 r = add_directory(j, m->path, de->d_name);
1541 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1545 check_network(j, dirfd(d));
1550 static int remove_directory(sd_journal *j, Directory *d) {
1554 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1556 if (j->inotify_fd >= 0)
1557 inotify_rm_watch(j->inotify_fd, d->wd);
1560 hashmap_remove(j->directories_by_path, d->path);
1563 log_debug("Root directory %s removed.", d->path);
1565 log_debug("Directory %s removed.", d->path);
1573 static int add_search_paths(sd_journal *j) {
1575 const char search_paths[] =
1576 "/run/log/journal\0"
1577 "/var/log/journal\0";
1582 /* We ignore most errors here, since the idea is to only open
1583 * what's actually accessible, and ignore the rest. */
1585 NULSTR_FOREACH(p, search_paths) {
1586 r = add_root_directory(j, p);
1587 if (r < 0 && r != -ENOENT) {
1588 r = set_put_error(j, r);
1597 static int allocate_inotify(sd_journal *j) {
1600 if (j->inotify_fd < 0) {
1601 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1602 if (j->inotify_fd < 0)
1606 if (!j->directories_by_wd) {
1607 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1608 if (!j->directories_by_wd)
1615 static sd_journal *journal_new(int flags, const char *path) {
1618 j = new0(sd_journal, 1);
1624 j->data_threshold = DEFAULT_DATA_THRESHOLD;
1627 j->path = strdup(path);
1632 j->files = hashmap_new(string_hash_func, string_compare_func);
1633 j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1634 j->mmap = mmap_cache_new();
1635 if (!j->files || !j->directories_by_path || !j->mmap)
1641 sd_journal_close(j);
1645 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1652 if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1653 SD_JOURNAL_RUNTIME_ONLY|
1655 SD_JOURNAL_CURRENT_USER))
1658 j = journal_new(flags, NULL);
1662 r = add_search_paths(j);
1670 sd_journal_close(j);
1675 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1688 j = journal_new(flags, path);
1692 r = add_root_directory(j, path);
1694 set_put_error(j, r);
1702 sd_journal_close(j);
1707 _public_ void sd_journal_close(sd_journal *j) {
1714 sd_journal_flush_matches(j);
1716 while ((f = hashmap_steal_first(j->files)))
1717 journal_file_close(f);
1719 hashmap_free(j->files);
1721 while ((d = hashmap_first(j->directories_by_path)))
1722 remove_directory(j, d);
1724 while ((d = hashmap_first(j->directories_by_wd)))
1725 remove_directory(j, d);
1727 hashmap_free(j->directories_by_path);
1728 hashmap_free(j->directories_by_wd);
1730 if (j->inotify_fd >= 0)
1731 close_nointr_nofail(j->inotify_fd);
1734 mmap_cache_unref(j->mmap);
1737 free(j->unique_field);
1738 set_free(j->errors);
1742 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1752 f = j->current_file;
1754 return -EADDRNOTAVAIL;
1756 if (f->current_offset <= 0)
1757 return -EADDRNOTAVAIL;
1759 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1763 *ret = le64toh(o->entry.realtime);
1767 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1776 f = j->current_file;
1778 return -EADDRNOTAVAIL;
1780 if (f->current_offset <= 0)
1781 return -EADDRNOTAVAIL;
1783 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1788 *ret_boot_id = o->entry.boot_id;
1790 r = sd_id128_get_boot(&id);
1794 if (!sd_id128_equal(id, o->entry.boot_id))
1799 *ret = le64toh(o->entry.monotonic);
1804 static bool field_is_valid(const char *field) {
1812 if (startswith(field, "__"))
1815 for (p = field; *p; p++) {
1820 if (*p >= 'A' && *p <= 'Z')
1823 if (*p >= '0' && *p <= '9')
1832 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1835 size_t field_length;
1848 if (!field_is_valid(field))
1851 f = j->current_file;
1853 return -EADDRNOTAVAIL;
1855 if (f->current_offset <= 0)
1856 return -EADDRNOTAVAIL;
1858 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1862 field_length = strlen(field);
1864 n = journal_file_entry_n_items(o);
1865 for (i = 0; i < n; i++) {
1870 p = le64toh(o->entry.items[i].object_offset);
1871 le_hash = o->entry.items[i].hash;
1872 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1876 if (le_hash != o->data.hash)
1879 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1881 if (o->object.flags & OBJECT_COMPRESSED) {
1884 if (uncompress_startswith(o->data.payload, l,
1885 &f->compress_buffer, &f->compress_buffer_size,
1886 field, field_length, '=')) {
1890 if (!uncompress_blob(o->data.payload, l,
1891 &f->compress_buffer, &f->compress_buffer_size, &rsize,
1895 *data = f->compress_buffer;
1896 *size = (size_t) rsize;
1901 return -EPROTONOSUPPORT;
1904 } else if (l >= field_length+1 &&
1905 memcmp(o->data.payload, field, field_length) == 0 &&
1906 o->data.payload[field_length] == '=') {
1910 if ((uint64_t) t != l)
1913 *data = o->data.payload;
1919 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1927 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1931 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1934 /* We can't read objects larger than 4G on a 32bit machine */
1935 if ((uint64_t) t != l)
1938 if (o->object.flags & OBJECT_COMPRESSED) {
1942 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
1945 *data = f->compress_buffer;
1946 *size = (size_t) rsize;
1948 return -EPROTONOSUPPORT;
1951 *data = o->data.payload;
1958 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1972 f = j->current_file;
1974 return -EADDRNOTAVAIL;
1976 if (f->current_offset <= 0)
1977 return -EADDRNOTAVAIL;
1979 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1983 n = journal_file_entry_n_items(o);
1984 if (j->current_field >= n)
1987 p = le64toh(o->entry.items[j->current_field].object_offset);
1988 le_hash = o->entry.items[j->current_field].hash;
1989 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1993 if (le_hash != o->data.hash)
1996 r = return_data(j, f, o, data, size);
2000 j->current_field ++;
2005 _public_ void sd_journal_restart_data(sd_journal *j) {
2009 j->current_field = 0;
2012 _public_ int sd_journal_get_fd(sd_journal *j) {
2018 if (j->inotify_fd >= 0)
2019 return j->inotify_fd;
2021 r = allocate_inotify(j);
2025 /* Iterate through all dirs again, to add them to the
2028 r = add_root_directory(j, j->path);
2030 r = add_search_paths(j);
2034 return j->inotify_fd;
2037 _public_ int sd_journal_get_events(sd_journal *j) {
2043 fd = sd_journal_get_fd(j);
2050 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2058 fd = sd_journal_get_fd(j);
2062 if (!j->on_network) {
2063 *timeout_usec = (uint64_t) -1;
2067 /* If we are on the network we need to regularly check for
2068 * changes manually */
2070 *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2074 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2081 /* Is this a subdirectory we watch? */
2082 d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2086 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2087 (endswith(e->name, ".journal") ||
2088 endswith(e->name, ".journal~"))) {
2090 /* Event for a journal file */
2092 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2093 r = add_file(j, d->path, e->name);
2095 log_debug("Failed to add file %s/%s: %s",
2096 d->path, e->name, strerror(-r));
2097 set_put_error(j, r);
2100 } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2102 r = remove_file(j, d->path, e->name);
2104 log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2107 } else if (!d->is_root && e->len == 0) {
2109 /* Event for a subdirectory */
2111 if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2112 r = remove_directory(j, d);
2114 log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2118 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2120 /* Event for root directory */
2122 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2123 r = add_directory(j, d->path, e->name);
2125 log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2132 if (e->mask & IN_IGNORED)
2135 log_warning("Unknown inotify event.");
2138 static int determine_change(sd_journal *j) {
2143 b = j->current_invalidate_counter != j->last_invalidate_counter;
2144 j->last_invalidate_counter = j->current_invalidate_counter;
2146 return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2149 _public_ int sd_journal_process(sd_journal *j) {
2150 uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2151 bool got_something = false;
2156 j->last_process_usec = now(CLOCK_MONOTONIC);
2159 struct inotify_event *e;
2162 l = read(j->inotify_fd, buffer, sizeof(buffer));
2164 if (errno == EAGAIN || errno == EINTR)
2165 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2170 got_something = true;
2172 e = (struct inotify_event*) buffer;
2176 process_inotify_event(j, e);
2178 step = sizeof(struct inotify_event) + e->len;
2179 assert(step <= (size_t) l);
2181 e = (struct inotify_event*) ((uint8_t*) e + step);
2186 return determine_change(j);
2189 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2195 if (j->inotify_fd < 0) {
2197 /* This is the first invocation, hence create the
2199 r = sd_journal_get_fd(j);
2203 /* The journal might have changed since the context
2204 * object was created and we weren't watching before,
2205 * hence don't wait for anything, and return
2207 return determine_change(j);
2210 r = sd_journal_get_timeout(j, &t);
2214 if (t != (uint64_t) -1) {
2217 n = now(CLOCK_MONOTONIC);
2218 t = t > n ? t - n : 0;
2220 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2225 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2226 } while (r == -EINTR);
2231 return sd_journal_process(j);
2234 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2247 HASHMAP_FOREACH(f, j->files, i) {
2250 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2266 *from = MIN(fr, *from);
2272 return first ? 0 : 1;
2275 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2288 HASHMAP_FOREACH(f, j->files, i) {
2291 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2307 *from = MIN(fr, *from);
2313 return first ? 0 : 1;
2316 void journal_print_header(sd_journal *j) {
2319 bool newline = false;
2323 HASHMAP_FOREACH(f, j->files, i) {
2329 journal_file_print_header(f);
2333 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2343 HASHMAP_FOREACH(f, j->files, i) {
2346 if (fstat(f->fd, &st) < 0)
2349 sum += (uint64_t) st.st_blocks * 512ULL;
2356 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2363 if (!field_is_valid(field))
2370 free(j->unique_field);
2371 j->unique_field = f;
2372 j->unique_file = NULL;
2373 j->unique_offset = 0;
2378 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2389 if (!j->unique_field)
2392 k = strlen(j->unique_field);
2394 if (!j->unique_file) {
2395 j->unique_file = hashmap_first(j->files);
2396 if (!j->unique_file)
2398 j->unique_offset = 0;
2408 /* Proceed to next data object in the field's linked list */
2409 if (j->unique_offset == 0) {
2410 r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2414 j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2416 r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2420 j->unique_offset = le64toh(o->data.next_field_offset);
2423 /* We reached the end of the list? Then start again, with the next file */
2424 if (j->unique_offset == 0) {
2427 n = hashmap_next(j->files, j->unique_file->path);
2435 /* We do not use the type context here, but 0 instead,
2436 * so that we can look at this data object at the same
2437 * time as one on another file */
2438 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2442 /* Let's do the type check by hand, since we used 0 context above. */
2443 if (o->object.type != OBJECT_DATA)
2446 r = return_data(j, j->unique_file, o, &odata, &ol);
2450 /* OK, now let's see if we already returned this data
2451 * object by checking if it exists in the earlier
2452 * traversed files. */
2454 HASHMAP_FOREACH(of, j->files, i) {
2458 if (of == j->unique_file)
2461 /* Skip this file it didn't have any fields
2463 if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2464 le64toh(of->header->n_fields) <= 0)
2467 r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2478 r = return_data(j, j->unique_file, o, data, l);
2486 _public_ void sd_journal_restart_unique(sd_journal *j) {
2490 j->unique_file = NULL;
2491 j->unique_offset = 0;
2494 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2498 return !j->on_network;
2501 static char *lookup_field(const char *field, void *userdata) {
2502 sd_journal *j = userdata;
2510 r = sd_journal_get_data(j, field, &data, &size);
2512 size > REPLACE_VAR_MAX)
2513 return strdup(field);
2515 d = strlen(field) + 1;
2517 return strndup((const char*) data + d, size - d);
2520 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2524 _cleanup_free_ char *text = NULL, *cid = NULL;
2533 r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2537 cid = strndup((const char*) data + 11, size - 11);
2541 r = sd_id128_from_string(cid, &id);
2545 r = catalog_get(CATALOG_DATABASE, id, &text);
2549 t = replace_var(text, lookup_field, j);
2557 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2561 return catalog_get(CATALOG_DATABASE, id, ret);
2564 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2568 j->data_threshold = sz;
2572 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2578 *sz = j->data_threshold;