1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2011 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
26 #include <sys/inotify.h>
29 #include <linux/magic.h>
31 #include "sd-journal.h"
32 #include "journal-def.h"
33 #include "journal-file.h"
36 #include "path-util.h"
39 #include "journal-internal.h"
42 #include "replace-var.h"
44 #define JOURNAL_FILES_MAX 1024
46 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
48 #define REPLACE_VAR_MAX 256
50 #define DEFAULT_DATA_THRESHOLD (64*1024)
52 /* We return an error here only if we didn't manage to
53 memorize the real error. */
54 static int set_put_error(sd_journal *j, int r) {
60 k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
64 return set_put(j->errors, INT_TO_PTR(r));
67 static void detach_location(sd_journal *j) {
73 j->current_file = NULL;
76 HASHMAP_FOREACH(f, j->files, i)
77 f->current_offset = 0;
80 static void reset_location(sd_journal *j) {
84 zero(j->current_location);
87 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
89 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
91 assert(o->object.type == OBJECT_ENTRY);
94 l->seqnum = le64toh(o->entry.seqnum);
95 l->seqnum_id = f->header->seqnum_id;
96 l->realtime = le64toh(o->entry.realtime);
97 l->monotonic = le64toh(o->entry.monotonic);
98 l->boot_id = o->entry.boot_id;
99 l->xor_hash = le64toh(o->entry.xor_hash);
101 l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
104 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o, uint64_t offset) {
106 assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
110 init_location(&j->current_location, type, f, o);
113 j->current_file->current_offset = 0;
116 j->current_field = 0;
118 f->current_offset = offset;
121 static int match_is_valid(const void *data, size_t size) {
129 if (startswith(data, "__"))
133 for (p = b; p < b + size; p++) {
141 if (*p >= 'A' && *p <= 'Z')
144 if (*p >= '0' && *p <= '9')
153 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
154 const uint8_t *a = _a, *b = _b;
157 for (j = 0; j < s && j < t; j++) {
169 static Match *match_new(Match *p, MatchType t) {
180 LIST_PREPEND(Match, matches, p->matches, m);
186 static void match_free(Match *m) {
190 match_free(m->matches);
193 LIST_REMOVE(Match, matches, m->parent->matches, m);
199 static void match_free_if_empty(Match *m) {
208 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
209 Match *l3, *l4, *add_here = NULL, *m;
221 if (!match_is_valid(data, size))
228 * level 4: concrete matches */
231 j->level0 = match_new(NULL, MATCH_AND_TERM);
237 j->level1 = match_new(j->level0, MATCH_OR_TERM);
243 j->level2 = match_new(j->level1, MATCH_AND_TERM);
248 assert(j->level0->type == MATCH_AND_TERM);
249 assert(j->level1->type == MATCH_OR_TERM);
250 assert(j->level2->type == MATCH_AND_TERM);
252 le_hash = htole64(hash64(data, size));
254 LIST_FOREACH(matches, l3, j->level2->matches) {
255 assert(l3->type == MATCH_OR_TERM);
257 LIST_FOREACH(matches, l4, l3->matches) {
258 assert(l4->type == MATCH_DISCRETE);
260 /* Exactly the same match already? Then ignore
262 if (l4->le_hash == le_hash &&
264 memcmp(l4->data, data, size) == 0)
267 /* Same field? Then let's add this to this OR term */
268 if (same_field(data, size, l4->data, l4->size)) {
279 add_here = match_new(j->level2, MATCH_OR_TERM);
284 m = match_new(add_here, MATCH_DISCRETE);
288 m->le_hash = le_hash;
290 m->data = memdup(data, size);
300 match_free_if_empty(add_here);
303 match_free_if_empty(j->level2);
306 match_free_if_empty(j->level1);
309 match_free_if_empty(j->level0);
314 _public_ int sd_journal_add_conjunction(sd_journal *j) {
323 if (!j->level1->matches)
332 _public_ int sd_journal_add_disjunction(sd_journal *j) {
344 if (!j->level2->matches)
351 static char *match_make_string(Match *m) {
354 bool enclose = false;
359 if (m->type == MATCH_DISCRETE)
360 return strndup(m->data, m->size);
363 LIST_FOREACH(matches, i, m->matches) {
366 t = match_make_string(i);
373 k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
390 r = strjoin("(", p, ")", NULL);
398 char *journal_make_match_string(sd_journal *j) {
401 return match_make_string(j->level0);
404 _public_ void sd_journal_flush_matches(sd_journal *j) {
410 match_free(j->level0);
412 j->level0 = j->level1 = j->level2 = NULL;
417 static int compare_entry_order(JournalFile *af, Object *_ao,
418 JournalFile *bf, uint64_t bp) {
428 /* The mmap cache might invalidate the object from the first
429 * file if we look at the one from the second file. Hence
430 * temporarily copy the header of the first one, and look at
432 ao = alloca(offsetof(EntryObject, items));
433 memcpy(ao, _ao, offsetof(EntryObject, items));
435 r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
437 return strcmp(af->path, bf->path);
439 /* We operate on two different files here, hence we can access
440 * two objects at the same time, which we normally can't.
442 * If contents and timestamps match, these entries are
443 * identical, even if the seqnum does not match */
445 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
446 ao->entry.monotonic == bo->entry.monotonic &&
447 ao->entry.realtime == bo->entry.realtime &&
448 ao->entry.xor_hash == bo->entry.xor_hash)
451 if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
453 /* If this is from the same seqnum source, compare
455 a = le64toh(ao->entry.seqnum);
456 b = le64toh(bo->entry.seqnum);
463 /* Wow! This is weird, different data but the same
464 * seqnums? Something is borked, but let's make the
465 * best of it and compare by time. */
468 if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
470 /* If the boot id matches compare monotonic time */
471 a = le64toh(ao->entry.monotonic);
472 b = le64toh(bo->entry.monotonic);
480 /* Otherwise compare UTC time */
481 a = le64toh(ao->entry.realtime);
482 b = le64toh(bo->entry.realtime);
489 /* Finally, compare by contents */
490 a = le64toh(ao->entry.xor_hash);
491 b = le64toh(bo->entry.xor_hash);
501 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
507 assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
509 if (l->monotonic_set &&
510 sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
512 le64toh(ao->entry.realtime) == l->realtime &&
514 le64toh(ao->entry.xor_hash) == l->xor_hash)
518 sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
520 a = le64toh(ao->entry.seqnum);
528 if (l->monotonic_set &&
529 sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
531 a = le64toh(ao->entry.monotonic);
533 if (a < l->monotonic)
535 if (a > l->monotonic)
539 if (l->realtime_set) {
541 a = le64toh(ao->entry.realtime);
549 if (l->xor_hash_set) {
550 a = le64toh(ao->entry.xor_hash);
561 static int next_for_match(
565 uint64_t after_offset,
566 direction_t direction,
578 if (m->type == MATCH_DISCRETE) {
581 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
585 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
587 } else if (m->type == MATCH_OR_TERM) {
590 /* Find the earliest match beyond after_offset */
592 LIST_FOREACH(matches, i, m->matches) {
595 r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
599 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
604 } else if (m->type == MATCH_AND_TERM) {
606 bool continue_looking;
608 /* Always jump to the next matching entry and repeat
609 * this until we fine and offset that matches for all
617 continue_looking = false;
619 LIST_FOREACH(matches, i, m->matches) {
623 limit = after_offset;
624 else if (direction == DIRECTION_DOWN)
625 limit = MAX(np, after_offset);
627 limit = MIN(np, after_offset);
629 r = next_for_match(j, i, f, limit, direction, NULL, &cp);
633 if ((direction == DIRECTION_DOWN ? cp >= after_offset : cp <= after_offset) &&
634 (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))) {
636 continue_looking = true;
640 } while (continue_looking);
646 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
658 static int find_location_for_match(
662 direction_t direction,
672 if (m->type == MATCH_DISCRETE) {
675 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
679 /* FIXME: missing: find by monotonic */
681 if (j->current_location.type == LOCATION_HEAD)
682 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
683 if (j->current_location.type == LOCATION_TAIL)
684 return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
685 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
686 return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
687 if (j->current_location.monotonic_set) {
688 r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
692 if (j->current_location.realtime_set)
693 return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
695 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
697 } else if (m->type == MATCH_OR_TERM) {
702 /* Find the earliest match */
704 LIST_FOREACH(matches, i, m->matches) {
707 r = find_location_for_match(j, i, f, direction, NULL, &cp);
711 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
719 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
734 assert(m->type == MATCH_AND_TERM);
736 /* First jump to the last match, and then find the
737 * next one where all matches match */
742 LIST_FOREACH(matches, i, m->matches) {
745 r = find_location_for_match(j, i, f, direction, NULL, &cp);
749 if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
753 return next_for_match(j, m, f, np, direction, ret, offset);
757 static int find_location_with_matches(
760 direction_t direction,
772 /* No matches is simple */
774 if (j->current_location.type == LOCATION_HEAD)
775 return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
776 if (j->current_location.type == LOCATION_TAIL)
777 return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
778 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
779 return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
780 if (j->current_location.monotonic_set) {
781 r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
785 if (j->current_location.realtime_set)
786 return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
788 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
790 return find_location_for_match(j, j->level0, f, direction, ret, offset);
793 static int next_with_matches(
796 direction_t direction,
811 /* No matches is easy. We simple advance the file
814 return journal_file_next_entry(f, c, cp, direction, ret, offset);
816 /* If we have a match then we look for the next matching entry
817 * with an offset at least one step larger */
818 return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
821 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
829 if (f->current_offset > 0) {
830 cp = f->current_offset;
832 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
836 r = next_with_matches(j, f, direction, &c, &cp);
840 r = find_location_with_matches(j, f, direction, &c, &cp);
845 /* OK, we found the spot, now let's advance until to an entry
846 * that is actually different from what we were previously
847 * looking at. This is necessary to handle entries which exist
848 * in two (or more) journal files, and which shall all be
849 * suppressed but one. */
854 if (j->current_location.type == LOCATION_DISCRETE) {
857 k = compare_with_location(f, c, &j->current_location);
858 if (direction == DIRECTION_DOWN)
873 r = next_with_matches(j, f, direction, &c, &cp);
879 static int real_journal_next(sd_journal *j, direction_t direction) {
880 JournalFile *f, *new_file = NULL;
881 uint64_t new_offset = 0;
890 HASHMAP_FOREACH(f, j->files, i) {
893 r = next_beyond_location(j, f, direction, &o, &p);
895 log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
905 k = compare_entry_order(f, o, new_file, new_offset);
907 if (direction == DIRECTION_DOWN)
922 r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
926 set_location(j, LOCATION_DISCRETE, new_file, o, new_offset);
931 _public_ int sd_journal_next(sd_journal *j) {
932 return real_journal_next(j, DIRECTION_DOWN);
935 _public_ int sd_journal_previous(sd_journal *j) {
936 return real_journal_next(j, DIRECTION_UP);
939 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
946 /* If this is not a discrete skip, then at least
947 * resolve the current location */
948 if (j->current_location.type != LOCATION_DISCRETE)
949 return real_journal_next(j, direction);
955 r = real_journal_next(j, direction);
969 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
970 return real_journal_next_skip(j, DIRECTION_DOWN, skip);
973 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
974 return real_journal_next_skip(j, DIRECTION_UP, skip);
977 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
980 char bid[33], sid[33];
987 if (!j->current_file || j->current_file->current_offset <= 0)
988 return -EADDRNOTAVAIL;
990 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
994 sd_id128_to_string(j->current_file->header->seqnum_id, sid);
995 sd_id128_to_string(o->entry.boot_id, bid);
998 "s=%s;i=%llx;b=%s;m=%llx;t=%llx;x=%llx",
999 sid, (unsigned long long) le64toh(o->entry.seqnum),
1000 bid, (unsigned long long) le64toh(o->entry.monotonic),
1001 (unsigned long long) le64toh(o->entry.realtime),
1002 (unsigned long long) le64toh(o->entry.xor_hash)) < 0)
1008 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
1011 unsigned long long seqnum, monotonic, realtime, xor_hash;
1013 seqnum_id_set = false,
1015 boot_id_set = false,
1016 monotonic_set = false,
1017 realtime_set = false,
1018 xor_hash_set = false;
1019 sd_id128_t seqnum_id, boot_id;
1023 if (isempty(cursor))
1026 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1030 if (l < 2 || w[1] != '=')
1033 item = strndup(w, l);
1040 seqnum_id_set = true;
1041 k = sd_id128_from_string(item+2, &seqnum_id);
1046 if (sscanf(item+2, "%llx", &seqnum) != 1)
1052 k = sd_id128_from_string(item+2, &boot_id);
1056 monotonic_set = true;
1057 if (sscanf(item+2, "%llx", &monotonic) != 1)
1062 realtime_set = true;
1063 if (sscanf(item+2, "%llx", &realtime) != 1)
1068 xor_hash_set = true;
1069 if (sscanf(item+2, "%llx", &xor_hash) != 1)
1080 if ((!seqnum_set || !seqnum_id_set) &&
1081 (!monotonic_set || !boot_id_set) &&
1087 j->current_location.type = LOCATION_SEEK;
1090 j->current_location.realtime = (uint64_t) realtime;
1091 j->current_location.realtime_set = true;
1094 if (seqnum_set && seqnum_id_set) {
1095 j->current_location.seqnum = (uint64_t) seqnum;
1096 j->current_location.seqnum_id = seqnum_id;
1097 j->current_location.seqnum_set = true;
1100 if (monotonic_set && boot_id_set) {
1101 j->current_location.monotonic = (uint64_t) monotonic;
1102 j->current_location.boot_id = boot_id;
1103 j->current_location.monotonic_set = true;
1107 j->current_location.xor_hash = (uint64_t) xor_hash;
1108 j->current_location.xor_hash_set = true;
1114 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1122 if (isempty(cursor))
1125 if (!j->current_file || j->current_file->current_offset <= 0)
1126 return -EADDRNOTAVAIL;
1128 r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1132 FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1133 _cleanup_free_ char *item = NULL;
1135 unsigned long long ll;
1138 if (l < 2 || w[1] != '=')
1141 item = strndup(w, l);
1148 k = sd_id128_from_string(item+2, &id);
1151 if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1156 if (sscanf(item+2, "%llx", &ll) != 1)
1158 if (ll != le64toh(o->entry.seqnum))
1163 k = sd_id128_from_string(item+2, &id);
1166 if (!sd_id128_equal(id, o->entry.boot_id))
1171 if (sscanf(item+2, "%llx", &ll) != 1)
1173 if (ll != le64toh(o->entry.monotonic))
1178 if (sscanf(item+2, "%llx", &ll) != 1)
1180 if (ll != le64toh(o->entry.realtime))
1185 if (sscanf(item+2, "%llx", &ll) != 1)
1187 if (ll != le64toh(o->entry.xor_hash))
1197 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1202 j->current_location.type = LOCATION_SEEK;
1203 j->current_location.boot_id = boot_id;
1204 j->current_location.monotonic = usec;
1205 j->current_location.monotonic_set = true;
1210 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1215 j->current_location.type = LOCATION_SEEK;
1216 j->current_location.realtime = usec;
1217 j->current_location.realtime_set = true;
1222 _public_ int sd_journal_seek_head(sd_journal *j) {
1227 j->current_location.type = LOCATION_HEAD;
1232 _public_ int sd_journal_seek_tail(sd_journal *j) {
1237 j->current_location.type = LOCATION_TAIL;
1242 static void check_network(sd_journal *j, int fd) {
1250 if (fstatfs(fd, &sfs) < 0)
1254 F_TYPE_CMP(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1255 F_TYPE_CMP(sfs.f_type, CODA_SUPER_MAGIC) ||
1256 F_TYPE_CMP(sfs.f_type, NCP_SUPER_MAGIC) ||
1257 F_TYPE_CMP(sfs.f_type, NFS_SUPER_MAGIC) ||
1258 F_TYPE_CMP(sfs.f_type, SMB_SUPER_MAGIC);
1261 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1262 _cleanup_free_ char *path = NULL;
1270 if ((j->flags & SD_JOURNAL_SYSTEM_ONLY) &&
1271 !(streq(filename, "system.journal") ||
1272 streq(filename, "system.journal~") ||
1273 (startswith(filename, "system@") &&
1274 (endswith(filename, ".journal") || endswith(filename, ".journal~")))))
1277 path = strjoin(prefix, "/", filename, NULL);
1281 if (hashmap_get(j->files, path))
1284 if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1285 log_debug("Too many open journal files, not adding %s, ignoring.", path);
1286 return set_put_error(j, -ETOOMANYREFS);
1289 r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1291 if (errno == ENOENT)
1297 /* journal_file_dump(f); */
1299 r = hashmap_put(j->files, f->path, f);
1301 journal_file_close(f);
1305 log_debug("File %s got added.", f->path);
1307 check_network(j, f->fd);
1309 j->current_invalidate_counter ++;
1314 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1322 path = strjoin(prefix, "/", filename, NULL);
1326 f = hashmap_get(j->files, path);
1331 hashmap_remove(j->files, f->path);
1333 log_debug("File %s got removed.", f->path);
1335 if (j->current_file == f) {
1336 j->current_file = NULL;
1337 j->current_field = 0;
1340 if (j->unique_file == f) {
1341 j->unique_file = NULL;
1342 j->unique_offset = 0;
1345 journal_file_close(f);
1347 j->current_invalidate_counter ++;
1352 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1353 _cleanup_free_ char *path = NULL;
1355 _cleanup_closedir_ DIR *d = NULL;
1363 log_debug("Considering %s/%s.", prefix, dirname);
1365 if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1366 (sd_id128_from_string(dirname, &id) < 0 ||
1367 sd_id128_get_machine(&mid) < 0 ||
1368 !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1371 path = strjoin(prefix, "/", dirname, NULL);
1377 log_debug("Failed to open %s: %m", path);
1378 if (errno == ENOENT)
1383 m = hashmap_get(j->directories_by_path, path);
1385 m = new0(Directory, 1);
1392 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1397 path = NULL; /* avoid freeing in cleanup */
1398 j->current_invalidate_counter ++;
1400 log_debug("Directory %s got added.", m->path);
1402 } else if (m->is_root)
1405 if (m->wd <= 0 && j->inotify_fd >= 0) {
1407 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1408 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1409 IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1412 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1413 inotify_rm_watch(j->inotify_fd, m->wd);
1418 union dirent_storage buf;
1420 r = readdir_r(d, &buf.de, &de);
1424 if (dirent_is_file_with_suffix(de, ".journal") ||
1425 dirent_is_file_with_suffix(de, ".journal~")) {
1426 r = add_file(j, m->path, de->d_name);
1428 log_debug("Failed to add file %s/%s: %s",
1429 m->path, de->d_name, strerror(-r));
1430 r = set_put_error(j, r);
1437 check_network(j, dirfd(d));
1442 static int add_root_directory(sd_journal *j, const char *p) {
1443 _cleanup_closedir_ DIR *d = NULL;
1450 if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1451 !path_startswith(p, "/run"))
1458 m = hashmap_get(j->directories_by_path, p);
1460 m = new0(Directory, 1);
1465 m->path = strdup(p);
1471 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1477 j->current_invalidate_counter ++;
1479 log_debug("Root directory %s got added.", m->path);
1481 } else if (!m->is_root)
1484 if (m->wd <= 0 && j->inotify_fd >= 0) {
1486 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1487 IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1490 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1491 inotify_rm_watch(j->inotify_fd, m->wd);
1496 union dirent_storage buf;
1499 r = readdir_r(d, &buf.de, &de);
1503 if (dirent_is_file_with_suffix(de, ".journal") ||
1504 dirent_is_file_with_suffix(de, ".journal~")) {
1505 r = add_file(j, m->path, de->d_name);
1507 log_debug("Failed to add file %s/%s: %s",
1508 m->path, de->d_name, strerror(-r));
1509 r = set_put_error(j, r);
1513 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1514 sd_id128_from_string(de->d_name, &id) >= 0) {
1516 r = add_directory(j, m->path, de->d_name);
1518 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1522 check_network(j, dirfd(d));
1527 static int remove_directory(sd_journal *j, Directory *d) {
1531 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1533 if (j->inotify_fd >= 0)
1534 inotify_rm_watch(j->inotify_fd, d->wd);
1537 hashmap_remove(j->directories_by_path, d->path);
1540 log_debug("Root directory %s got removed.", d->path);
1542 log_debug("Directory %s got removed.", d->path);
1550 static int add_search_paths(sd_journal *j) {
1552 const char search_paths[] =
1553 "/run/log/journal\0"
1554 "/var/log/journal\0";
1559 /* We ignore most errors here, since the idea is to only open
1560 * what's actually accessible, and ignore the rest. */
1562 NULSTR_FOREACH(p, search_paths) {
1563 r = add_root_directory(j, p);
1564 if (r < 0 && r != -ENOENT) {
1565 r = set_put_error(j, r);
1574 static int allocate_inotify(sd_journal *j) {
1577 if (j->inotify_fd < 0) {
1578 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1579 if (j->inotify_fd < 0)
1583 if (!j->directories_by_wd) {
1584 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1585 if (!j->directories_by_wd)
1592 static sd_journal *journal_new(int flags, const char *path) {
1595 j = new0(sd_journal, 1);
1601 j->data_threshold = DEFAULT_DATA_THRESHOLD;
1604 j->path = strdup(path);
1609 j->files = hashmap_new(string_hash_func, string_compare_func);
1610 j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1611 j->mmap = mmap_cache_new();
1612 if (!j->files || !j->directories_by_path || !j->mmap)
1618 sd_journal_close(j);
1622 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1629 if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1630 SD_JOURNAL_RUNTIME_ONLY|
1631 SD_JOURNAL_SYSTEM_ONLY))
1634 j = journal_new(flags, NULL);
1638 r = add_search_paths(j);
1646 sd_journal_close(j);
1651 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1664 j = journal_new(flags, path);
1668 r = add_root_directory(j, path);
1670 set_put_error(j, r);
1678 sd_journal_close(j);
1683 _public_ void sd_journal_close(sd_journal *j) {
1690 sd_journal_flush_matches(j);
1692 while ((f = hashmap_steal_first(j->files)))
1693 journal_file_close(f);
1695 hashmap_free(j->files);
1697 while ((d = hashmap_first(j->directories_by_path)))
1698 remove_directory(j, d);
1700 while ((d = hashmap_first(j->directories_by_wd)))
1701 remove_directory(j, d);
1703 hashmap_free(j->directories_by_path);
1704 hashmap_free(j->directories_by_wd);
1706 if (j->inotify_fd >= 0)
1707 close_nointr_nofail(j->inotify_fd);
1710 mmap_cache_unref(j->mmap);
1713 free(j->unique_field);
1714 set_free(j->errors);
1718 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1728 f = j->current_file;
1730 return -EADDRNOTAVAIL;
1732 if (f->current_offset <= 0)
1733 return -EADDRNOTAVAIL;
1735 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1739 *ret = le64toh(o->entry.realtime);
1743 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1752 f = j->current_file;
1754 return -EADDRNOTAVAIL;
1756 if (f->current_offset <= 0)
1757 return -EADDRNOTAVAIL;
1759 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1764 *ret_boot_id = o->entry.boot_id;
1766 r = sd_id128_get_boot(&id);
1770 if (!sd_id128_equal(id, o->entry.boot_id))
1775 *ret = le64toh(o->entry.monotonic);
1780 static bool field_is_valid(const char *field) {
1788 if (startswith(field, "__"))
1791 for (p = field; *p; p++) {
1796 if (*p >= 'A' && *p <= 'Z')
1799 if (*p >= '0' && *p <= '9')
1808 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1811 size_t field_length;
1824 if (!field_is_valid(field))
1827 f = j->current_file;
1829 return -EADDRNOTAVAIL;
1831 if (f->current_offset <= 0)
1832 return -EADDRNOTAVAIL;
1834 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1838 field_length = strlen(field);
1840 n = journal_file_entry_n_items(o);
1841 for (i = 0; i < n; i++) {
1846 p = le64toh(o->entry.items[i].object_offset);
1847 le_hash = o->entry.items[i].hash;
1848 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1852 if (le_hash != o->data.hash)
1855 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1857 if (o->object.flags & OBJECT_COMPRESSED) {
1860 if (uncompress_startswith(o->data.payload, l,
1861 &f->compress_buffer, &f->compress_buffer_size,
1862 field, field_length, '=')) {
1866 if (!uncompress_blob(o->data.payload, l,
1867 &f->compress_buffer, &f->compress_buffer_size, &rsize,
1871 *data = f->compress_buffer;
1872 *size = (size_t) rsize;
1877 return -EPROTONOSUPPORT;
1880 } else if (l >= field_length+1 &&
1881 memcmp(o->data.payload, field, field_length) == 0 &&
1882 o->data.payload[field_length] == '=') {
1886 if ((uint64_t) t != l)
1889 *data = o->data.payload;
1895 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1903 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1907 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1910 /* We can't read objects larger than 4G on a 32bit machine */
1911 if ((uint64_t) t != l)
1914 if (o->object.flags & OBJECT_COMPRESSED) {
1918 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
1921 *data = f->compress_buffer;
1922 *size = (size_t) rsize;
1924 return -EPROTONOSUPPORT;
1927 *data = o->data.payload;
1934 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1948 f = j->current_file;
1950 return -EADDRNOTAVAIL;
1952 if (f->current_offset <= 0)
1953 return -EADDRNOTAVAIL;
1955 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1959 n = journal_file_entry_n_items(o);
1960 if (j->current_field >= n)
1963 p = le64toh(o->entry.items[j->current_field].object_offset);
1964 le_hash = o->entry.items[j->current_field].hash;
1965 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1969 if (le_hash != o->data.hash)
1972 r = return_data(j, f, o, data, size);
1976 j->current_field ++;
1981 _public_ void sd_journal_restart_data(sd_journal *j) {
1985 j->current_field = 0;
1988 _public_ int sd_journal_get_fd(sd_journal *j) {
1994 if (j->inotify_fd >= 0)
1995 return j->inotify_fd;
1997 r = allocate_inotify(j);
2001 /* Iterate through all dirs again, to add them to the
2004 r = add_root_directory(j, j->path);
2006 r = add_search_paths(j);
2010 return j->inotify_fd;
2013 _public_ int sd_journal_get_events(sd_journal *j) {
2019 fd = sd_journal_get_fd(j);
2026 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2034 fd = sd_journal_get_fd(j);
2038 if (!j->on_network) {
2039 *timeout_usec = (uint64_t) -1;
2043 /* If we are on the network we need to regularly check for
2044 * changes manually */
2046 *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2050 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2057 /* Is this a subdirectory we watch? */
2058 d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2062 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2063 (endswith(e->name, ".journal") ||
2064 endswith(e->name, ".journal~"))) {
2066 /* Event for a journal file */
2068 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2069 r = add_file(j, d->path, e->name);
2071 log_debug("Failed to add file %s/%s: %s",
2072 d->path, e->name, strerror(-r));
2073 set_put_error(j, r);
2076 } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2078 r = remove_file(j, d->path, e->name);
2080 log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2083 } else if (!d->is_root && e->len == 0) {
2085 /* Event for a subdirectory */
2087 if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2088 r = remove_directory(j, d);
2090 log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2094 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2096 /* Event for root directory */
2098 if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2099 r = add_directory(j, d->path, e->name);
2101 log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2108 if (e->mask & IN_IGNORED)
2111 log_warning("Unknown inotify event.");
2114 static int determine_change(sd_journal *j) {
2119 b = j->current_invalidate_counter != j->last_invalidate_counter;
2120 j->last_invalidate_counter = j->current_invalidate_counter;
2122 return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2125 _public_ int sd_journal_process(sd_journal *j) {
2126 uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2127 bool got_something = false;
2132 j->last_process_usec = now(CLOCK_MONOTONIC);
2135 struct inotify_event *e;
2138 l = read(j->inotify_fd, buffer, sizeof(buffer));
2140 if (errno == EAGAIN || errno == EINTR)
2141 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2146 got_something = true;
2148 e = (struct inotify_event*) buffer;
2152 process_inotify_event(j, e);
2154 step = sizeof(struct inotify_event) + e->len;
2155 assert(step <= (size_t) l);
2157 e = (struct inotify_event*) ((uint8_t*) e + step);
2162 return determine_change(j);
2165 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2171 if (j->inotify_fd < 0) {
2173 /* This is the first invocation, hence create the
2175 r = sd_journal_get_fd(j);
2179 /* The journal might have changed since the context
2180 * object was created and we weren't watching before,
2181 * hence don't wait for anything, and return
2183 return determine_change(j);
2186 r = sd_journal_get_timeout(j, &t);
2190 if (t != (uint64_t) -1) {
2193 n = now(CLOCK_MONOTONIC);
2194 t = t > n ? t - n : 0;
2196 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2201 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2202 } while (r == -EINTR);
2207 return sd_journal_process(j);
2210 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2223 HASHMAP_FOREACH(f, j->files, i) {
2226 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2242 *from = MIN(fr, *from);
2248 return first ? 0 : 1;
2251 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2264 HASHMAP_FOREACH(f, j->files, i) {
2267 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2283 *from = MIN(fr, *from);
2289 return first ? 0 : 1;
2292 void journal_print_header(sd_journal *j) {
2295 bool newline = false;
2299 HASHMAP_FOREACH(f, j->files, i) {
2305 journal_file_print_header(f);
2309 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2319 HASHMAP_FOREACH(f, j->files, i) {
2322 if (fstat(f->fd, &st) < 0)
2325 sum += (uint64_t) st.st_blocks * 512ULL;
2332 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2339 if (!field_is_valid(field))
2346 free(j->unique_field);
2347 j->unique_field = f;
2348 j->unique_file = NULL;
2349 j->unique_offset = 0;
2354 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2365 if (!j->unique_field)
2368 k = strlen(j->unique_field);
2370 if (!j->unique_file) {
2371 j->unique_file = hashmap_first(j->files);
2372 if (!j->unique_file)
2374 j->unique_offset = 0;
2384 /* Proceed to next data object in the field's linked list */
2385 if (j->unique_offset == 0) {
2386 r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2390 j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2392 r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2396 j->unique_offset = le64toh(o->data.next_field_offset);
2399 /* We reached the end of the list? Then start again, with the next file */
2400 if (j->unique_offset == 0) {
2403 n = hashmap_next(j->files, j->unique_file->path);
2411 /* We do not use the type context here, but 0 instead,
2412 * so that we can look at this data object at the same
2413 * time as one on another file */
2414 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2418 /* Let's do the type check by hand, since we used 0 context above. */
2419 if (o->object.type != OBJECT_DATA)
2422 r = return_data(j, j->unique_file, o, &odata, &ol);
2426 /* OK, now let's see if we already returned this data
2427 * object by checking if it exists in the earlier
2428 * traversed files. */
2430 HASHMAP_FOREACH(of, j->files, i) {
2434 if (of == j->unique_file)
2437 /* Skip this file it didn't have any fields
2439 if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2440 le64toh(of->header->n_fields) <= 0)
2443 r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2454 r = return_data(j, j->unique_file, o, data, l);
2462 _public_ void sd_journal_restart_unique(sd_journal *j) {
2466 j->unique_file = NULL;
2467 j->unique_offset = 0;
2470 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2474 return !j->on_network;
2477 static char *lookup_field(const char *field, void *userdata) {
2478 sd_journal *j = userdata;
2486 r = sd_journal_get_data(j, field, &data, &size);
2488 size > REPLACE_VAR_MAX)
2489 return strdup(field);
2491 d = strlen(field) + 1;
2493 return strndup((const char*) data + d, size - d);
2496 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2500 _cleanup_free_ char *text = NULL, *cid = NULL;
2509 r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2513 cid = strndup((const char*) data + 11, size - 11);
2517 r = sd_id128_from_string(cid, &id);
2521 r = catalog_get(CATALOG_DATABASE, id, &text);
2525 t = replace_var(text, lookup_field, j);
2533 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2537 return catalog_get(CATALOG_DATABASE, id, ret);
2540 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2544 j->data_threshold = sz;
2548 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2554 *sz = j->data_threshold;