1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
26 #include <sys/inotify.h>
29 #include <linux/magic.h>
31 #include "sd-journal.h"
32 #include "journal-def.h"
33 #include "journal-file.h"
36 #include "path-util.h"
39 #include "journal-internal.h"
42 #include "replace-var.h"
44 #define JOURNAL_FILES_MAX 1024
46 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
48 #define REPLACE_VAR_MAX 256
50 #define DEFAULT_DATA_THRESHOLD (64*1024)
52 /* We return an error here only if we didn't manage to
53 memorize the real error. */
54 static int set_put_error(sd_journal *j, int r) {
60 k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
64 return set_put(j->errors, INT_TO_PTR(r));
67 static void detach_location(sd_journal *j) {
73 j->current_file = NULL;
76 HASHMAP_FOREACH(f, j->files, i)
77 f->current_offset = 0;
80 static void reset_location(sd_journal *j) {
84 zero(j->current_location);
87 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
89 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
91 assert(o->object.type == OBJECT_ENTRY);
94 l->seqnum = le64toh(o->entry.seqnum);
95 l->seqnum_id = f->header->seqnum_id;
96 l->realtime = le64toh(o->entry.realtime);
97 l->monotonic = le64toh(o->entry.monotonic);
98 l->boot_id = o->entry.boot_id;
99 l->xor_hash = le64toh(o->entry.xor_hash);
101 l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
104 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o, uint64_t offset) {
106 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
110 init_location(&j->current_location, type, f, o);
113 j->current_field = 0;
115 f->current_offset = offset;
118 static int match_is_valid(const void *data, size_t size) {
126 if (startswith(data, "__"))
130 for (p = b; p < b + size; p++) {
138 if (*p >= 'A' && *p <= 'Z')
141 if (*p >= '0' && *p <= '9')
150 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
151 const uint8_t *a = _a, *b = _b;
154 for (j = 0; j < s && j < t; j++) {
166 static Match *match_new(Match *p, MatchType t) {
177 LIST_PREPEND(Match, matches, p->matches, m);
183 static void match_free(Match *m) {
187 match_free(m->matches);
190 LIST_REMOVE(Match, matches, m->parent->matches, m);
196 static void match_free_if_empty(Match *m) {
205 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
206 Match *l2, *l3, *add_here = NULL, *m;
218 if (!match_is_valid(data, size))
224 * level 3: concrete matches */
227 j->level0 = match_new(NULL, MATCH_OR_TERM);
233 j->level1 = match_new(j->level0, MATCH_AND_TERM);
238 assert(j->level0->type == MATCH_OR_TERM);
239 assert(j->level1->type == MATCH_AND_TERM);
241 le_hash = htole64(hash64(data, size));
243 LIST_FOREACH(matches, l2, j->level1->matches) {
244 assert(l2->type == MATCH_OR_TERM);
246 LIST_FOREACH(matches, l3, l2->matches) {
247 assert(l3->type == MATCH_DISCRETE);
249 /* Exactly the same match already? Then ignore
251 if (l3->le_hash == le_hash &&
253 memcmp(l3->data, data, size) == 0)
256 /* Same field? Then let's add this to this OR term */
257 if (same_field(data, size, l3->data, l3->size)) {
268 add_here = match_new(j->level1, MATCH_OR_TERM);
273 m = match_new(add_here, MATCH_DISCRETE);
277 m->le_hash = le_hash;
279 m->data = memdup(data, size);
289 match_free_if_empty(add_here);
292 match_free_if_empty(j->level1);
295 match_free_if_empty(j->level0);
300 _public_ int sd_journal_add_disjunction(sd_journal *j) {
311 if (!j->level1->matches)
314 m = match_new(j->level0, MATCH_AND_TERM);
322 static char *match_make_string(Match *m) {
325 bool enclose = false;
330 if (m->type == MATCH_DISCRETE)
331 return strndup(m->data, m->size);
334 LIST_FOREACH(matches, i, m->matches) {
337 t = match_make_string(i);
344 k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
361 r = strjoin("(", p, ")", NULL);
369 char *journal_make_match_string(sd_journal *j) {
372 return match_make_string(j->level0);
375 _public_ void sd_journal_flush_matches(sd_journal *j) {
381 match_free(j->level0);
383 j->level0 = j->level1 = NULL;
388 static int compare_entry_order(JournalFile *af, Object *_ao,
389 JournalFile *bf, uint64_t bp) {
399 /* The mmap cache might invalidate the object from the first
400 * file if we look at the one from the second file. Hence
401 * temporarily copy the header of the first one, and look at
403 ao = alloca(offsetof(EntryObject, items));
404 memcpy(ao, _ao, offsetof(EntryObject, items));
406 r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
408 return strcmp(af->path, bf->path);
410 /* We operate on two different files here, hence we can access
411 * two objects at the same time, which we normally can't.
413 * If contents and timestamps match, these entries are
414 * identical, even if the seqnum does not match */
416 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
417 ao->entry.monotonic == bo->entry.monotonic &&
418 ao->entry.realtime == bo->entry.realtime &&
419 ao->entry.xor_hash == bo->entry.xor_hash)
422 if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
424 /* If this is from the same seqnum source, compare
426 a = le64toh(ao->entry.seqnum);
427 b = le64toh(bo->entry.seqnum);
434 /* Wow! This is weird, different data but the same
435 * seqnums? Something is borked, but let's make the
436 * best of it and compare by time. */
439 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
441 /* If the boot id matches compare monotonic time */
442 a = le64toh(ao->entry.monotonic);
443 b = le64toh(bo->entry.monotonic);
451 /* Otherwise compare UTC time */
452 a = le64toh(ao->entry.realtime);
453 b = le64toh(bo->entry.realtime);
460 /* Finally, compare by contents */
461 a = le64toh(ao->entry.xor_hash);
462 b = le64toh(bo->entry.xor_hash);
472 static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
478 assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
480 if (l->monotonic_set &&
481 sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
483 le64toh(ao->entry.realtime) == l->realtime &&
485 le64toh(ao->entry.xor_hash) == l->xor_hash)
489 sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
491 a = le64toh(ao->entry.seqnum);
499 if (l->monotonic_set &&
500 sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
502 a = le64toh(ao->entry.monotonic);
504 if (a < l->monotonic)
506 if (a > l->monotonic)
510 if (l->realtime_set) {
512 a = le64toh(ao->entry.realtime);
520 if (l->xor_hash_set) {
521 a = le64toh(ao->entry.xor_hash);
532 static int next_for_match(
536 uint64_t after_offset,
537 direction_t direction,
549 if (m->type == MATCH_DISCRETE) {
552 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
556 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
558 } else if (m->type == MATCH_OR_TERM) {
561 /* Find the earliest match beyond after_offset */
563 LIST_FOREACH(matches, i, m->matches) {
566 r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
570 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
575 } else if (m->type == MATCH_AND_TERM) {
577 bool continue_looking;
579 /* Always jump to the next matching entry and repeat
580 * this until we fine and offset that matches for all
588 continue_looking = false;
590 LIST_FOREACH(matches, i, m->matches) {
594 limit = after_offset;
595 else if (direction == DIRECTION_DOWN)
596 limit = MAX(np, after_offset);
598 limit = MIN(np, after_offset);
600 r = next_for_match(j, i, f, limit, direction, NULL, &cp);
604 if ((direction == DIRECTION_DOWN ? cp >= after_offset : cp <= after_offset) &&
605 (np == 0 || (direction == DIRECTION_DOWN ? cp > np : np < cp))) {
607 continue_looking = true;
611 } while (continue_looking);
617 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
629 static int find_location_for_match(
633 direction_t direction,
643 if (m->type == MATCH_DISCRETE) {
646 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
650 /* FIXME: missing: find by monotonic */
652 if (j->current_location.type == LOCATION_HEAD)
653 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
654 if (j->current_location.type == LOCATION_TAIL)
655 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
656 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
657 return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
658 if (j->current_location.monotonic_set) {
659 r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
663 if (j->current_location.realtime_set)
664 return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
666 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
668 } else if (m->type == MATCH_OR_TERM) {
673 /* Find the earliest match */
675 LIST_FOREACH(matches, i, m->matches) {
678 r = find_location_for_match(j, i, f, direction, NULL, &cp);
682 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
690 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
705 assert(m->type == MATCH_AND_TERM);
707 /* First jump to the last match, and then find the
708 * next one where all matches match */
713 LIST_FOREACH(matches, i, m->matches) {
716 r = find_location_for_match(j, i, f, direction, NULL, &cp);
720 if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
724 return next_for_match(j, m, f, np, direction, ret, offset);
728 static int find_location_with_matches(
731 direction_t direction,
743 /* No matches is simple */
745 if (j->current_location.type == LOCATION_HEAD)
746 return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
747 if (j->current_location.type == LOCATION_TAIL)
748 return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
749 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
750 return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
751 if (j->current_location.monotonic_set) {
752 r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
756 if (j->current_location.realtime_set)
757 return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
759 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
761 return find_location_for_match(j, j->level0, f, direction, ret, offset);
764 static int next_with_matches(
767 direction_t direction,
782 /* No matches is easy. We simple advance the file
785 return journal_file_next_entry(f, c, cp, direction, ret, offset);
787 /* If we have a match then we look for the next matching entry
788 * with an offset at least one step larger */
789 return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
792 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
800 if (f->current_offset > 0) {
801 cp = f->current_offset;
803 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
807 r = next_with_matches(j, f, direction, &c, &cp);
811 r = find_location_with_matches(j, f, direction, &c, &cp);
816 /* OK, we found the spot, now let's advance until to an entry
817 * that is actually different from what we were previously
818 * looking at. This is necessary to handle entries which exist
819 * in two (or more) journal files, and which shall all be
820 * suppressed but one. */
825 if (j->current_location.type == LOCATION_DISCRETE) {
828 k = compare_with_location(f, c, &j->current_location);
829 if (direction == DIRECTION_DOWN)
844 r = next_with_matches(j, f, direction, &c, &cp);
850 static int real_journal_next(sd_journal *j, direction_t direction) {
851 JournalFile *f, *new_file = NULL;
852 uint64_t new_offset = 0;
861 HASHMAP_FOREACH(f, j->files, i) {
864 r = next_beyond_location(j, f, direction, &o, &p);
866 log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
876 k = compare_entry_order(f, o, new_file, new_offset);
878 if (direction == DIRECTION_DOWN)
893 r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
897 set_location(j, LOCATION_DISCRETE, new_file, o, new_offset);
902 _public_ int sd_journal_next(sd_journal *j) {
903 return real_journal_next(j, DIRECTION_DOWN);
906 _public_ int sd_journal_previous(sd_journal *j) {
907 return real_journal_next(j, DIRECTION_UP);
910 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
917 /* If this is not a discrete skip, then at least
918 * resolve the current location */
919 if (j->current_location.type != LOCATION_DISCRETE)
920 return real_journal_next(j, direction);
926 r = real_journal_next(j, direction);
940 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
941 return real_journal_next_skip(j, DIRECTION_DOWN, skip);
944 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
945 return real_journal_next_skip(j, DIRECTION_UP, skip);
948 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
951 char bid[33], sid[33];
958 if (!j->current_file || j->current_file->current_offset <= 0)
959 return -EADDRNOTAVAIL;
961 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
965 sd_id128_to_string(j->current_file->header->seqnum_id, sid);
966 sd_id128_to_string(o->entry.boot_id, bid);
969 "s=%s;i=%llx;b=%s;m=%llx;t=%llx;x=%llx",
970 sid, (unsigned long long) le64toh(o->entry.seqnum),
971 bid, (unsigned long long) le64toh(o->entry.monotonic),
972 (unsigned long long) le64toh(o->entry.realtime),
973 (unsigned long long) le64toh(o->entry.xor_hash)) < 0)
979 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
982 unsigned long long seqnum, monotonic, realtime, xor_hash;
984 seqnum_id_set = false,
987 monotonic_set = false,
988 realtime_set = false,
989 xor_hash_set = false;
990 sd_id128_t seqnum_id, boot_id;
997 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1001 if (l < 2 || w[1] != '=')
1004 item = strndup(w, l);
1011 seqnum_id_set = true;
1012 k = sd_id128_from_string(item+2, &seqnum_id);
1017 if (sscanf(item+2, "%llx", &seqnum) != 1)
1023 k = sd_id128_from_string(item+2, &boot_id);
1027 monotonic_set = true;
1028 if (sscanf(item+2, "%llx", &monotonic) != 1)
1033 realtime_set = true;
1034 if (sscanf(item+2, "%llx", &realtime) != 1)
1039 xor_hash_set = true;
1040 if (sscanf(item+2, "%llx", &xor_hash) != 1)
1051 if ((!seqnum_set || !seqnum_id_set) &&
1052 (!monotonic_set || !boot_id_set) &&
1058 j->current_location.type = LOCATION_SEEK;
1061 j->current_location.realtime = (uint64_t) realtime;
1062 j->current_location.realtime_set = true;
1065 if (seqnum_set && seqnum_id_set) {
1066 j->current_location.seqnum = (uint64_t) seqnum;
1067 j->current_location.seqnum_id = seqnum_id;
1068 j->current_location.seqnum_set = true;
1071 if (monotonic_set && boot_id_set) {
1072 j->current_location.monotonic = (uint64_t) monotonic;
1073 j->current_location.boot_id = boot_id;
1074 j->current_location.monotonic_set = true;
1078 j->current_location.xor_hash = (uint64_t) xor_hash;
1079 j->current_location.xor_hash_set = true;
1085 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1093 if (isempty(cursor))
1096 if (!j->current_file || j->current_file->current_offset <= 0)
1097 return -EADDRNOTAVAIL;
1099 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1103 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1104 _cleanup_free_ char *item = NULL;
1106 unsigned long long ll;
1109 if (l < 2 || w[1] != '=')
1112 item = strndup(w, l);
1119 k = sd_id128_from_string(item+2, &id);
1122 if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1127 if (sscanf(item+2, "%llx", &ll) != 1)
1129 if (ll != le64toh(o->entry.seqnum))
1134 k = sd_id128_from_string(item+2, &id);
1137 if (!sd_id128_equal(id, o->entry.boot_id))
1142 if (sscanf(item+2, "%llx", &ll) != 1)
1144 if (ll != le64toh(o->entry.monotonic))
1149 if (sscanf(item+2, "%llx", &ll) != 1)
1151 if (ll != le64toh(o->entry.realtime))
1156 if (sscanf(item+2, "%llx", &ll) != 1)
1158 if (ll != le64toh(o->entry.xor_hash))
1168 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1173 j->current_location.type = LOCATION_SEEK;
1174 j->current_location.boot_id = boot_id;
1175 j->current_location.monotonic = usec;
1176 j->current_location.monotonic_set = true;
1181 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1186 j->current_location.type = LOCATION_SEEK;
1187 j->current_location.realtime = usec;
1188 j->current_location.realtime_set = true;
1193 _public_ int sd_journal_seek_head(sd_journal *j) {
1198 j->current_location.type = LOCATION_HEAD;
1203 _public_ int sd_journal_seek_tail(sd_journal *j) {
1208 j->current_location.type = LOCATION_TAIL;
1213 static void check_network(sd_journal *j, int fd) {
1221 if (fstatfs(fd, &sfs) < 0)
1225 (long)sfs.f_type == (long)CIFS_MAGIC_NUMBER ||
1226 sfs.f_type == CODA_SUPER_MAGIC ||
1227 sfs.f_type == NCP_SUPER_MAGIC ||
1228 sfs.f_type == NFS_SUPER_MAGIC ||
1229 sfs.f_type == SMB_SUPER_MAGIC;
1232 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1233 char _cleanup_free_ *path = NULL;
1241 if ((j->flags & SD_JOURNAL_SYSTEM_ONLY) &&
1242 !(streq(filename, "system.journal") ||
1243 streq(filename, "system.journal~") ||
1244 (startswith(filename, "system@") &&
1245 (endswith(filename, ".journal") || endswith(filename, ".journal~")))))
1248 path = strjoin(prefix, "/", filename, NULL);
1252 if (hashmap_get(j->files, path))
1255 if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1256 log_debug("Too many open journal files, not adding %s, ignoring.", path);
1257 return set_put_error(j, -ETOOMANYREFS);
1260 r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1262 if (errno == ENOENT)
1268 /* journal_file_dump(f); */
1270 r = hashmap_put(j->files, f->path, f);
1272 journal_file_close(f);
1276 log_debug("File %s got added.", f->path);
1278 check_network(j, f->fd);
1280 j->current_invalidate_counter ++;
1285 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1293 path = strjoin(prefix, "/", filename, NULL);
1297 f = hashmap_get(j->files, path);
1302 hashmap_remove(j->files, f->path);
1304 log_debug("File %s got removed.", f->path);
1306 if (j->current_file == f) {
1307 j->current_file = NULL;
1308 j->current_field = 0;
1311 if (j->unique_file == f) {
1312 j->unique_file = NULL;
1313 j->unique_offset = 0;
1316 journal_file_close(f);
1318 j->current_invalidate_counter ++;
1323 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1324 char _cleanup_free_ *path = NULL;
1326 DIR _cleanup_closedir_ *d = NULL;
1334 log_debug("Considering %s/%s.", prefix, dirname);
1336 if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1337 (sd_id128_from_string(dirname, &id) < 0 ||
1338 sd_id128_get_machine(&mid) < 0 ||
1339 !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1342 path = strjoin(prefix, "/", dirname, NULL);
1348 log_debug("Failed to open %s: %m", path);
1349 if (errno == ENOENT)
1354 m = hashmap_get(j->directories_by_path, path);
1356 m = new0(Directory, 1);
1363 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1368 path = NULL; /* avoid freeing in cleanup */
1369 j->current_invalidate_counter ++;
1371 log_debug("Directory %s got added.", m->path);
1373 } else if (m->is_root)
1376 if (m->wd <= 0 && j->inotify_fd >= 0) {
1378 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1379 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1380 IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1383 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1384 inotify_rm_watch(j->inotify_fd, m->wd);
1389 union dirent_storage buf;
1391 r = readdir_r(d, &buf.de, &de);
1395 if (dirent_is_file_with_suffix(de, ".journal") ||
1396 dirent_is_file_with_suffix(de, ".journal~")) {
1397 r = add_file(j, m->path, de->d_name);
1399 log_debug("Failed to add file %s/%s: %s",
1400 m->path, de->d_name, strerror(-r));
1401 r = set_put_error(j, r);
1408 check_network(j, dirfd(d));
1413 static int add_root_directory(sd_journal *j, const char *p) {
1414 DIR _cleanup_closedir_ *d = NULL;
1421 if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1422 !path_startswith(p, "/run"))
1429 m = hashmap_get(j->directories_by_path, p);
1431 m = new0(Directory, 1);
1436 m->path = strdup(p);
1442 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1448 j->current_invalidate_counter ++;
1450 log_debug("Root directory %s got added.", m->path);
1452 } else if (!m->is_root)
1455 if (m->wd <= 0 && j->inotify_fd >= 0) {
1457 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1458 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1461 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1462 inotify_rm_watch(j->inotify_fd, m->wd);
1467 union dirent_storage buf;
1470 r = readdir_r(d, &buf.de, &de);
1474 if (dirent_is_file_with_suffix(de, ".journal") ||
1475 dirent_is_file_with_suffix(de, ".journal~")) {
1476 r = add_file(j, m->path, de->d_name);
1478 log_debug("Failed to add file %s/%s: %s",
1479 m->path, de->d_name, strerror(-r));
1480 r = set_put_error(j, r);
1484 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1485 sd_id128_from_string(de->d_name, &id) >= 0) {
1487 r = add_directory(j, m->path, de->d_name);
1489 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1493 check_network(j, dirfd(d));
1498 static int remove_directory(sd_journal *j, Directory *d) {
1502 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1504 if (j->inotify_fd >= 0)
1505 inotify_rm_watch(j->inotify_fd, d->wd);
1508 hashmap_remove(j->directories_by_path, d->path);
1511 log_debug("Root directory %s got removed.", d->path);
1513 log_debug("Directory %s got removed.", d->path);
1521 static int add_search_paths(sd_journal *j) {
1523 const char search_paths[] =
1524 "/run/log/journal\0"
1525 "/var/log/journal\0";
1530 /* We ignore most errors here, since the idea is to only open
1531 * what's actually accessible, and ignore the rest. */
1533 NULSTR_FOREACH(p, search_paths) {
1534 r = add_root_directory(j, p);
1535 if (r < 0 && r != -ENOENT) {
1536 r = set_put_error(j, r);
1545 static int allocate_inotify(sd_journal *j) {
1548 if (j->inotify_fd < 0) {
1549 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1550 if (j->inotify_fd < 0)
1554 if (!j->directories_by_wd) {
1555 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1556 if (!j->directories_by_wd)
1563 static sd_journal *journal_new(int flags, const char *path) {
1566 j = new0(sd_journal, 1);
1572 j->data_threshold = DEFAULT_DATA_THRESHOLD;
1575 j->path = strdup(path);
1580 j->files = hashmap_new(string_hash_func, string_compare_func);
1581 j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1582 j->mmap = mmap_cache_new();
1583 if (!j->files || !j->directories_by_path || !j->mmap)
1589 sd_journal_close(j);
1593 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1600 if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1601 SD_JOURNAL_RUNTIME_ONLY|
1602 SD_JOURNAL_SYSTEM_ONLY))
1605 j = journal_new(flags, NULL);
1609 r = add_search_paths(j);
1617 sd_journal_close(j);
1622 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1635 j = journal_new(flags, path);
1639 r = add_root_directory(j, path);
1641 set_put_error(j, r);
1649 sd_journal_close(j);
1654 _public_ void sd_journal_close(sd_journal *j) {
1661 sd_journal_flush_matches(j);
1663 while ((f = hashmap_steal_first(j->files)))
1664 journal_file_close(f);
1666 hashmap_free(j->files);
1668 while ((d = hashmap_first(j->directories_by_path)))
1669 remove_directory(j, d);
1671 while ((d = hashmap_first(j->directories_by_wd)))
1672 remove_directory(j, d);
1674 hashmap_free(j->directories_by_path);
1675 hashmap_free(j->directories_by_wd);
1677 if (j->inotify_fd >= 0)
1678 close_nointr_nofail(j->inotify_fd);
1681 mmap_cache_unref(j->mmap);
1684 free(j->unique_field);
1685 set_free(j->errors);
1689 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1699 f = j->current_file;
1701 return -EADDRNOTAVAIL;
1703 if (f->current_offset <= 0)
1704 return -EADDRNOTAVAIL;
1706 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1710 *ret = le64toh(o->entry.realtime);
1714 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1723 f = j->current_file;
1725 return -EADDRNOTAVAIL;
1727 if (f->current_offset <= 0)
1728 return -EADDRNOTAVAIL;
1730 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1735 *ret_boot_id = o->entry.boot_id;
1737 r = sd_id128_get_boot(&id);
1741 if (!sd_id128_equal(id, o->entry.boot_id))
1746 *ret = le64toh(o->entry.monotonic);
1751 static bool field_is_valid(const char *field) {
1759 if (startswith(field, "__"))
1762 for (p = field; *p; p++) {
1767 if (*p >= 'A' && *p <= 'Z')
1770 if (*p >= '0' && *p <= '9')
1779 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1782 size_t field_length;
1795 if (!field_is_valid(field))
1798 f = j->current_file;
1800 return -EADDRNOTAVAIL;
1802 if (f->current_offset <= 0)
1803 return -EADDRNOTAVAIL;
1805 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1809 field_length = strlen(field);
1811 n = journal_file_entry_n_items(o);
1812 for (i = 0; i < n; i++) {
1817 p = le64toh(o->entry.items[i].object_offset);
1818 le_hash = o->entry.items[i].hash;
1819 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1823 if (le_hash != o->data.hash)
1826 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1828 if (o->object.flags & OBJECT_COMPRESSED) {
1831 if (uncompress_startswith(o->data.payload, l,
1832 &f->compress_buffer, &f->compress_buffer_size,
1833 field, field_length, '=')) {
1837 if (!uncompress_blob(o->data.payload, l,
1838 &f->compress_buffer, &f->compress_buffer_size, &rsize,
1842 *data = f->compress_buffer;
1843 *size = (size_t) rsize;
1848 return -EPROTONOSUPPORT;
1851 } else if (l >= field_length+1 &&
1852 memcmp(o->data.payload, field, field_length) == 0 &&
1853 o->data.payload[field_length] == '=') {
1857 if ((uint64_t) t != l)
1860 *data = o->data.payload;
1866 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1874 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1878 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1881 /* We can't read objects larger than 4G on a 32bit machine */
1882 if ((uint64_t) t != l)
1885 if (o->object.flags & OBJECT_COMPRESSED) {
1889 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
1892 *data = f->compress_buffer;
1893 *size = (size_t) rsize;
1895 return -EPROTONOSUPPORT;
1898 *data = o->data.payload;
1905 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1919 f = j->current_file;
1921 return -EADDRNOTAVAIL;
1923 if (f->current_offset <= 0)
1924 return -EADDRNOTAVAIL;
1926 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1930 n = journal_file_entry_n_items(o);
1931 if (j->current_field >= n)
1934 p = le64toh(o->entry.items[j->current_field].object_offset);
1935 le_hash = o->entry.items[j->current_field].hash;
1936 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1940 if (le_hash != o->data.hash)
1943 r = return_data(j, f, o, data, size);
1947 j->current_field ++;
1952 _public_ void sd_journal_restart_data(sd_journal *j) {
1956 j->current_field = 0;
1959 _public_ int sd_journal_get_fd(sd_journal *j) {
1965 if (j->inotify_fd >= 0)
1966 return j->inotify_fd;
1968 r = allocate_inotify(j);
1972 /* Iterate through all dirs again, to add them to the
1975 r = add_root_directory(j, j->path);
1977 r = add_search_paths(j);
1981 return j->inotify_fd;
1984 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
1991 /* Is this a subdirectory we watch? */
1992 d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
1996 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
1997 (endswith(e->name, ".journal") ||
1998 endswith(e->name, ".journal~"))) {
2000 /* Event for a journal file */
2002 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2003 r = add_file(j, d->path, e->name);
2005 log_debug("Failed to add file %s/%s: %s",
2006 d->path, e->name, strerror(-r));
2007 set_put_error(j, r);
2010 } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2012 r = remove_file(j, d->path, e->name);
2014 log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2017 } else if (!d->is_root && e->len == 0) {
2019 /* Event for a subdirectory */
2021 if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2022 r = remove_directory(j, d);
2024 log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2028 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2030 /* Event for root directory */
2032 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2033 r = add_directory(j, d->path, e->name);
2035 log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2042 if (e->mask & IN_IGNORED)
2045 log_warning("Unknown inotify event.");
2048 static int determine_change(sd_journal *j) {
2053 b = j->current_invalidate_counter != j->last_invalidate_counter;
2054 j->last_invalidate_counter = j->current_invalidate_counter;
2056 return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2059 _public_ int sd_journal_process(sd_journal *j) {
2060 uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2061 bool got_something = false;
2067 struct inotify_event *e;
2070 l = read(j->inotify_fd, buffer, sizeof(buffer));
2072 if (errno == EAGAIN || errno == EINTR)
2073 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2078 got_something = true;
2080 e = (struct inotify_event*) buffer;
2084 process_inotify_event(j, e);
2086 step = sizeof(struct inotify_event) + e->len;
2087 assert(step <= (size_t) l);
2089 e = (struct inotify_event*) ((uint8_t*) e + step);
2094 return determine_change(j);
2097 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2102 if (j->inotify_fd < 0) {
2104 /* This is the first invocation, hence create the
2106 r = sd_journal_get_fd(j);
2110 /* The journal might have changed since the context
2111 * object was created and we weren't watching before,
2112 * hence don't wait for anything, and return
2114 return determine_change(j);
2117 if (j->on_network) {
2118 /* If we are on the network we need to regularly check
2119 * for changes manually */
2121 if (timeout_usec == (uint64_t) -1 || timeout_usec > JOURNAL_FILES_RECHECK_USEC)
2122 timeout_usec = JOURNAL_FILES_RECHECK_USEC;
2126 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2127 } while (r == -EINTR);
2132 return sd_journal_process(j);
2135 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2146 HASHMAP_FOREACH(f, j->files, i) {
2149 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2165 *from = MIN(fr, *from);
2171 return first ? 0 : 1;
2174 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2185 HASHMAP_FOREACH(f, j->files, i) {
2188 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2204 *from = MIN(fr, *from);
2210 return first ? 0 : 1;
2213 void journal_print_header(sd_journal *j) {
2216 bool newline = false;
2220 HASHMAP_FOREACH(f, j->files, i) {
2226 journal_file_print_header(f);
2230 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2240 HASHMAP_FOREACH(f, j->files, i) {
2243 if (fstat(f->fd, &st) < 0)
2246 sum += (uint64_t) st.st_blocks * 512ULL;
2253 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2260 if (!field_is_valid(field))
2267 free(j->unique_field);
2268 j->unique_field = f;
2269 j->unique_file = NULL;
2270 j->unique_offset = 0;
2275 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2286 if (!j->unique_field)
2289 k = strlen(j->unique_field);
2291 if (!j->unique_file) {
2292 j->unique_file = hashmap_first(j->files);
2293 if (!j->unique_file)
2295 j->unique_offset = 0;
2305 /* Proceed to next data object in the field's linked list */
2306 if (j->unique_offset == 0) {
2307 r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2311 j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2313 r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2317 j->unique_offset = le64toh(o->data.next_field_offset);
2320 /* We reached the end of the list? Then start again, with the next file */
2321 if (j->unique_offset == 0) {
2324 n = hashmap_next(j->files, j->unique_file->path);
2332 /* We do not use the type context here, but 0 instead,
2333 * so that we can look at this data object at the same
2334 * time as one on another file */
2335 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2339 /* Let's do the type check by hand, since we used 0 context above. */
2340 if (o->object.type != OBJECT_DATA)
2343 r = return_data(j, j->unique_file, o, &odata, &ol);
2347 /* OK, now let's see if we already returned this data
2348 * object by checking if it exists in the earlier
2349 * traversed files. */
2351 HASHMAP_FOREACH(of, j->files, i) {
2355 if (of == j->unique_file)
2358 /* Skip this file it didn't have any fields
2360 if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2361 le64toh(of->header->n_fields) <= 0)
2364 r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2375 r = return_data(j, j->unique_file, o, data, l);
2383 _public_ void sd_journal_restart_unique(sd_journal *j) {
2387 j->unique_file = NULL;
2388 j->unique_offset = 0;
2391 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2395 return !j->on_network;
2398 static char *lookup_field(const char *field, void *userdata) {
2399 sd_journal *j = userdata;
2407 r = sd_journal_get_data(j, field, &data, &size);
2409 size > REPLACE_VAR_MAX)
2410 return strdup(field);
2412 d = strlen(field) + 1;
2414 return strndup((const char*) data + d, size - d);
2417 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2421 _cleanup_free_ char *text = NULL, *cid = NULL;
2430 r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2434 cid = strndup((const char*) data + 11, size - 11);
2438 r = sd_id128_from_string(cid, &id);
2442 r = catalog_get(CATALOG_DATABASE, id, &text);
2446 t = replace_var(text, lookup_field, j);
2454 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2458 return catalog_get(CATALOG_DATABASE, id, ret);
2461 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2465 j->data_threshold = sz;
2469 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2475 *sz = j->data_threshold;