X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=elogind.git;a=blobdiff_plain;f=src%2Fjournal%2Fjournal-file.c;h=9efa82cbe094eb0a136b7a089934ef458b18cba1;hp=419e15e46fa583c36c808720c2d3e1281e038e10;hb=361f9cbc7e6c7d3ac41892f5a5aebd1ef4049b95;hpb=b785c858c3baf855ec42e3b83f5d23dadfece069 diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c index 419e15e46..9efa82cbe 100644 --- a/src/journal/journal-file.c +++ b/src/journal/journal-file.c @@ -6,16 +6,16 @@ Copyright 2011 Lennart Poettering systemd is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. systemd is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. + Lesser General Public License for more details. - You should have received a copy of the GNU General Public License + You should have received a copy of the GNU Lesser General Public License along with systemd; If not, see . ***/ @@ -32,15 +32,15 @@ #include "lookup3.h" #include "compress.h" -#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*16ULL) -#define DEFAULT_FIELD_HASH_TABLE_SIZE (2047ULL*16ULL) +#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem)) +#define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem)) -#define DEFAULT_WINDOW_SIZE (128ULL*1024ULL*1024ULL) +#define DEFAULT_WINDOW_SIZE (8ULL*1024ULL*1024ULL) -#define COMPRESSION_SIZE_THRESHOLD (64ULL) +#define COMPRESSION_SIZE_THRESHOLD (512ULL) /* This is the minimum journal file size */ -#define JOURNAL_FILE_SIZE_MIN (64ULL*1024ULL) +#define JOURNAL_FILE_SIZE_MIN (64ULL*1024ULL) /* 64 KiB */ /* These are the lower and upper bounds if we deduce the max_use value * from the file system size */ @@ -48,7 +48,7 @@ #define DEFAULT_MAX_USE_UPPER (4ULL*1024ULL*1024ULL*1024ULL) /* 4 GiB */ /* This is the upper bound if we deduce max_size from max_use */ -#define DEFAULT_MAX_SIZE_UPPER (16ULL*1024ULL*1024ULL) /* 16 MiB */ +#define DEFAULT_MAX_SIZE_UPPER (128ULL*1024ULL*1024ULL) /* 128 MiB */ /* This is the upper bound if we deduce the keep_free value from the * file system size */ @@ -58,18 +58,28 @@ * size */ #define DEFAULT_KEEP_FREE (1024ULL*1024ULL) /* 1 MB */ -static const char signature[] = { 'L', 'P', 'K', 'S', 'H', 'H', 'R', 'H' }; +/* n_data was the first entry we added after the initial file format design */ +#define HEADER_SIZE_MIN ALIGN64(offsetof(Header, n_data)) #define ALIGN64(x) (((x) + 7ULL) & ~7ULL) +#define JOURNAL_HEADER_CONTAINS(h, field) \ + (le64toh((h)->header_size) >= offsetof(Header, field) + sizeof((h)->field)) + +static const char signature[] = { 'L', 'P', 'K', 'S', 'H', 'H', 'R', 'H' }; + void journal_file_close(JournalFile *f) { int t; assert(f); - if (f->header && f->writable) - f->header->state = STATE_OFFLINE; + if (f->header) { + /* Mark the file offline. Don't override the archived state if it already is set */ + if (f->writable && f->header->state == STATE_ONLINE) + f->header->state = STATE_OFFLINE; + munmap(f->header, PAGE_ALIGN(sizeof(Header))); + } for (t = 0; t < _WINDOW_MAX; t++) if (f->windows[t].ptr) @@ -96,7 +106,7 @@ static int journal_file_init_header(JournalFile *f, JournalFile *template) { zero(h); memcpy(h.signature, signature, 8); - h.arena_offset = htole64(ALIGN64(sizeof(h))); + h.header_size = htole64(ALIGN64(sizeof(h))); r = sd_id128_randomize(&h.file_id); if (r < 0) @@ -104,7 +114,7 @@ static int journal_file_init_header(JournalFile *f, JournalFile *template) { if (template) { h.seqnum_id = template->header->seqnum_id; - h.seqnum = template->header->seqnum; + h.tail_seqnum = template->header->tail_seqnum; } else h.seqnum_id = h.file_id; @@ -158,11 +168,15 @@ static int journal_file_verify_header(JournalFile *f) { return -EPROTONOSUPPORT; #endif - if ((uint64_t) f->last_stat.st_size < (le64toh(f->header->arena_offset) + le64toh(f->header->arena_size))) + /* The first addition was n_data, so check that we are at least this large */ + if (le64toh(f->header->header_size) < HEADER_SIZE_MIN) + return -EBADMSG; + + if ((uint64_t) f->last_stat.st_size < (le64toh(f->header->header_size) + le64toh(f->header->arena_size))) return -ENODATA; if (f->writable) { - uint32_t state; + uint8_t state; sd_id128_t machine_id; int r; @@ -175,12 +189,15 @@ static int journal_file_verify_header(JournalFile *f) { state = f->header->state; - if (state == STATE_ONLINE) - log_debug("Journal file %s is already online. Assuming unclean closing. Ignoring.", f->path); - else if (state == STATE_ARCHIVED) + if (state == STATE_ONLINE) { + log_debug("Journal file %s is already online. Assuming unclean closing.", f->path); + return -EBUSY; + } else if (state == STATE_ARCHIVED) return -ESHUTDOWN; - else if (state != STATE_OFFLINE) - log_debug("Journal file %s has unknown state %u. Ignoring.", f->path, state); + else if (state != STATE_OFFLINE) { + log_debug("Journal file %s has unknown state %u.", f->path, state); + return -EBUSY; + } } return 0; @@ -188,6 +205,7 @@ static int journal_file_verify_header(JournalFile *f) { static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size) { uint64_t old_size, new_size; + int r; assert(f); @@ -196,12 +214,12 @@ static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size) * ourselves */ old_size = - le64toh(f->header->arena_offset) + + le64toh(f->header->header_size) + le64toh(f->header->arena_size); new_size = PAGE_ALIGN(offset + size); - if (new_size < le64toh(f->header->arena_offset)) - new_size = le64toh(f->header->arena_offset); + if (new_size < le64toh(f->header->header_size)) + new_size = le64toh(f->header->header_size); if (new_size <= old_size) return 0; @@ -232,13 +250,14 @@ static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size) /* Note that the glibc fallocate() fallback is very inefficient, hence we try to minimize the allocation area as we can. */ - if (posix_fallocate(f->fd, old_size, new_size - old_size) < 0) - return -errno; + r = posix_fallocate(f->fd, old_size, new_size - old_size); + if (r != 0) + return -r; if (fstat(f->fd, &f->last_stat) < 0) return -errno; - f->header->arena_size = new_size - htole64(f->header->arena_offset); + f->header->arena_size = htole64(new_size - le64toh(f->header->header_size)); return 0; } @@ -418,7 +437,7 @@ static uint64_t journal_file_seqnum(JournalFile *f, uint64_t *seqnum) { assert(f); - r = le64toh(f->header->seqnum) + 1; + r = le64toh(f->header->tail_seqnum) + 1; if (seqnum) { /* If an external seqnum counter was passed, we update @@ -431,10 +450,10 @@ static uint64_t journal_file_seqnum(JournalFile *f, uint64_t *seqnum) { *seqnum = r; } - f->header->seqnum = htole64(r); + f->header->tail_seqnum = htole64(r); - if (f->header->first_seqnum == 0) - f->header->first_seqnum = htole64(r); + if (f->header->head_seqnum == 0) + f->header->head_seqnum = htole64(r); return r; } @@ -452,7 +471,7 @@ static int journal_file_append_object(JournalFile *f, int type, uint64_t size, O p = le64toh(f->header->tail_object_offset); if (p == 0) - p = le64toh(f->header->arena_offset); + p = le64toh(f->header->header_size); else { r = journal_file_move_to_object(f, -1, p, &tail); if (r < 0) @@ -491,7 +510,17 @@ static int journal_file_setup_data_hash_table(JournalFile *f) { assert(f); - s = DEFAULT_DATA_HASH_TABLE_SIZE; + /* We estimate that we need 1 hash table entry per 768 of + journal file and we want to make sure we never get beyond + 75% fill level. Calculate the hash table size for the + maximum file size based on these metrics. */ + + s = (f->metrics.max_size * 4 / 768 / 3) * sizeof(HashItem); + if (s < DEFAULT_DATA_HASH_TABLE_SIZE) + s = DEFAULT_DATA_HASH_TABLE_SIZE; + + log_info("Reserving %llu entries in hash table.", (unsigned long long) (s / sizeof(HashItem))); + r = journal_file_append_object(f, OBJECT_DATA_HASH_TABLE, offsetof(Object, hash_table.items) + s, @@ -581,32 +610,33 @@ static int journal_file_link_data(JournalFile *f, Object *o, uint64_t offset, ui assert(offset > 0); assert(o->object.type == OBJECT_DATA); + /* This might alter the window we are looking at */ + o->data.next_hash_offset = o->data.next_field_offset = 0; o->data.entry_offset = o->data.entry_array_offset = 0; o->data.n_entries = 0; h = hash % (le64toh(f->header->data_hash_table_size) / sizeof(HashItem)); - p = le64toh(f->data_hash_table[h].head_hash_offset); + p = le64toh(f->data_hash_table[h].tail_hash_offset); if (p == 0) { /* Only entry in the hash table is easy */ f->data_hash_table[h].head_hash_offset = htole64(offset); } else { - /* Temporarily move back to the previous data object, - * to patch in pointer */ + /* Move back to the previous data object, to patch in + * pointer */ r = journal_file_move_to_object(f, OBJECT_DATA, p, &o); if (r < 0) return r; o->data.next_hash_offset = htole64(offset); - - r = journal_file_move_to_object(f, OBJECT_DATA, offset, &o); - if (r < 0) - return r; } f->data_hash_table[h].tail_hash_offset = htole64(offset); + if (JOURNAL_HEADER_CONTAINS(f->header, n_data)) + f->header->n_data = htole64(le64toh(f->header->n_data) + 1); + return 0; } @@ -614,6 +644,7 @@ int journal_file_find_data_object_with_hash( JournalFile *f, const void *data, uint64_t size, uint64_t hash, Object **ret, uint64_t *offset) { + uint64_t p, osize, h; int r; @@ -702,7 +733,11 @@ int journal_file_find_data_object( ret, offset); } -static int journal_file_append_data(JournalFile *f, const void *data, uint64_t size, Object **ret, uint64_t *offset) { +static int journal_file_append_data( + JournalFile *f, + const void *data, uint64_t size, + Object **ret, uint64_t *offset) { + uint64_t hash, p; uint64_t osize; Object *o; @@ -760,6 +795,12 @@ static int journal_file_append_data(JournalFile *f, const void *data, uint64_t s if (r < 0) return r; + /* The linking might have altered the window, so let's + * refresh our pointer */ + r = journal_file_move_to_object(f, OBJECT_DATA, p, &o); + if (r < 0) + return r; + if (ret) *ret = o; @@ -771,21 +812,21 @@ static int journal_file_append_data(JournalFile *f, const void *data, uint64_t s uint64_t journal_file_entry_n_items(Object *o) { assert(o); - assert(o->object.type == htole64(OBJECT_ENTRY)); + assert(o->object.type == OBJECT_ENTRY); return (le64toh(o->object.size) - offsetof(Object, entry.items)) / sizeof(EntryItem); } static uint64_t journal_file_entry_array_n_items(Object *o) { assert(o); - assert(o->object.type == htole64(OBJECT_ENTRY_ARRAY)); + assert(o->object.type == OBJECT_ENTRY_ARRAY); return (le64toh(o->object.size) - offsetof(Object, entry_array.items)) / sizeof(uint64_t); } static int link_entry_into_array(JournalFile *f, - uint64_t *first, - uint64_t *idx, + le64_t *first, + le64_t *idx, uint64_t p) { int r; uint64_t n = 0, ap = 0, q, i, a, hidx; @@ -833,7 +874,7 @@ static int link_entry_into_array(JournalFile *f, o->entry_array.items[i] = htole64(p); if (ap == 0) - *first = q; + *first = htole64(q); else { r = journal_file_move_to_object(f, OBJECT_ENTRY_ARRAY, ap, &o); if (r < 0) @@ -848,9 +889,9 @@ static int link_entry_into_array(JournalFile *f, } static int link_entry_into_array_plus_one(JournalFile *f, - uint64_t *extra, - uint64_t *first, - uint64_t *idx, + le64_t *extra, + le64_t *first, + le64_t *idx, uint64_t p) { int r; @@ -864,9 +905,9 @@ static int link_entry_into_array_plus_one(JournalFile *f, if (*idx == 0) *extra = htole64(p); else { - uint64_t i; + le64_t i; - i = le64toh(*idx) - 1; + i = htole64(le64toh(*idx) - 1); r = link_entry_into_array(f, first, &i, p); if (r < 0) return r; @@ -1194,8 +1235,15 @@ static int generic_array_bisect(JournalFile *f, } } - if (k > n) + if (k > n) { + if (direction == DIRECTION_UP) { + i = n; + subtract_one = true; + goto found; + } + return 0; + } last_p = lp; @@ -1228,7 +1276,7 @@ found: *offset = p; if (idx) - *idx = t + i - (subtract_one ? 1 : 0); + *idx = t + i + (subtract_one ? -1 : 0); return 1; } @@ -1245,6 +1293,8 @@ static int generic_array_bisect_plus_one(JournalFile *f, uint64_t *idx) { int r; + bool step_back = false; + Object *o; assert(f); assert(test_object); @@ -1257,34 +1307,81 @@ static int generic_array_bisect_plus_one(JournalFile *f, r = test_object(f, extra, needle); if (r < 0) return r; - else if (r == TEST_FOUND) { - Object *o; - - r = journal_file_move_to_object(f, OBJECT_ENTRY, extra, &o); - if (r < 0) - return r; - if (ret) - *ret = o; - - if (offset) - *offset = extra; + if (r == TEST_FOUND) + r = direction == DIRECTION_DOWN ? TEST_RIGHT : TEST_LEFT; - if (idx) - *idx = 0; + /* if we are looking with DIRECTION_UP then we need to first + see if in the actual array there is a matching entry, and + return the last one of that. But if there isn't any we need + to return this one. Hence remember this, and return it + below. */ + if (r == TEST_LEFT) + step_back = direction == DIRECTION_UP; - return 1; - } else if (r == TEST_RIGHT) - return 0; + if (r == TEST_RIGHT) { + if (direction == DIRECTION_DOWN) + goto found; + else + return 0; + } r = generic_array_bisect(f, first, n-1, needle, test_object, direction, ret, offset, idx); - if (r > 0) + if (r == 0 && step_back) + goto found; + + if (r > 0 && idx) (*idx) ++; return r; + +found: + r = journal_file_move_to_object(f, OBJECT_ENTRY, extra, &o); + if (r < 0) + return r; + + if (ret) + *ret = o; + + if (offset) + *offset = extra; + + if (idx) + *idx = 0; + + return 1; +} + +static int test_object_offset(JournalFile *f, uint64_t p, uint64_t needle) { + assert(f); + assert(p > 0); + + if (p == needle) + return TEST_FOUND; + else if (p < needle) + return TEST_LEFT; + else + return TEST_RIGHT; } +int journal_file_move_to_entry_by_offset( + JournalFile *f, + uint64_t p, + direction_t direction, + Object **ret, + uint64_t *offset) { + + return generic_array_bisect(f, + le64toh(f->header->entry_array_offset), + le64toh(f->header->n_entries), + p, + test_object_offset, + direction, + ret, offset, NULL); +} + + static int test_object_seqnum(JournalFile *f, uint64_t p, uint64_t needle) { Object *o; int r; @@ -1382,16 +1479,17 @@ int journal_file_move_to_entry_by_monotonic( Object **ret, uint64_t *offset) { - char t[8+32+1] = "_BOOT_ID="; + char t[9+32+1] = "_BOOT_ID="; Object *o; int r; - sd_id128_to_string(boot_id, t + 8); + assert(f); + sd_id128_to_string(boot_id, t + 9); r = journal_file_find_data_object(f, t, strlen(t), &o, NULL); if (r < 0) return r; - else if (r == 0) + if (r == 0) return -ENOENT; return generic_array_bisect_plus_one(f, @@ -1404,18 +1502,6 @@ int journal_file_move_to_entry_by_monotonic( ret, offset, NULL); } -static int test_object_offset(JournalFile *f, uint64_t p, uint64_t needle) { - assert(f); - assert(p > 0); - - if (p == needle) - return TEST_FOUND; - else if (p < needle) - return TEST_LEFT; - else - return TEST_RIGHT; -} - int journal_file_next_entry( JournalFile *f, Object *o, uint64_t p, @@ -1580,6 +1666,119 @@ int journal_file_next_entry_for_data( ret, offset); } +int journal_file_move_to_entry_by_offset_for_data( + JournalFile *f, + uint64_t data_offset, + uint64_t p, + direction_t direction, + Object **ret, uint64_t *offset) { + + int r; + Object *d; + + assert(f); + + r = journal_file_move_to_object(f, OBJECT_DATA, data_offset, &d); + if (r < 0) + return r; + + return generic_array_bisect_plus_one(f, + le64toh(d->data.entry_offset), + le64toh(d->data.entry_array_offset), + le64toh(d->data.n_entries), + p, + test_object_offset, + direction, + ret, offset, NULL); +} + +int journal_file_move_to_entry_by_monotonic_for_data( + JournalFile *f, + uint64_t data_offset, + sd_id128_t boot_id, + uint64_t monotonic, + direction_t direction, + Object **ret, uint64_t *offset) { + + char t[9+32+1] = "_BOOT_ID="; + Object *o, *d; + int r; + uint64_t b, z; + + assert(f); + + /* First, seek by time */ + sd_id128_to_string(boot_id, t + 9); + r = journal_file_find_data_object(f, t, strlen(t), &o, &b); + if (r < 0) + return r; + if (r == 0) + return -ENOENT; + + r = generic_array_bisect_plus_one(f, + le64toh(o->data.entry_offset), + le64toh(o->data.entry_array_offset), + le64toh(o->data.n_entries), + monotonic, + test_object_monotonic, + direction, + NULL, &z, NULL); + if (r <= 0) + return r; + + /* And now, continue seeking until we find an entry that + * exists in both bisection arrays */ + + for (;;) { + Object *qo; + uint64_t p, q; + + r = journal_file_move_to_object(f, OBJECT_DATA, data_offset, &d); + if (r < 0) + return r; + + r = generic_array_bisect_plus_one(f, + le64toh(d->data.entry_offset), + le64toh(d->data.entry_array_offset), + le64toh(d->data.n_entries), + z, + test_object_offset, + direction, + NULL, &p, NULL); + if (r <= 0) + return r; + + r = journal_file_move_to_object(f, OBJECT_DATA, b, &o); + if (r < 0) + return r; + + r = generic_array_bisect_plus_one(f, + le64toh(o->data.entry_offset), + le64toh(o->data.entry_array_offset), + le64toh(o->data.n_entries), + p, + test_object_offset, + direction, + &qo, &q, NULL); + + if (r <= 0) + return r; + + if (p == q) { + if (ret) + *ret = qo; + if (offset) + *offset = q; + + return 1; + } + + z = q; + } + + return 0; +} + int journal_file_move_to_entry_by_seqnum_for_data( JournalFile *f, uint64_t data_offset, @@ -1590,8 +1789,10 @@ int journal_file_move_to_entry_by_seqnum_for_data( Object *d; int r; + assert(f); + r = journal_file_move_to_object(f, OBJECT_DATA, data_offset, &d); - if (r <= 0) + if (r < 0) return r; return generic_array_bisect_plus_one(f, @@ -1614,8 +1815,10 @@ int journal_file_move_to_entry_by_realtime_for_data( Object *d; int r; + assert(f); + r = journal_file_move_to_object(f, OBJECT_DATA, data_offset, &d); - if (r <= 0) + if (r < 0) return r; return generic_array_bisect_plus_one(f, @@ -1629,29 +1832,15 @@ int journal_file_move_to_entry_by_realtime_for_data( } void journal_file_dump(JournalFile *f) { - char a[33], b[33], c[33]; Object *o; int r; uint64_t p; assert(f); - printf("File Path: %s\n" - "File ID: %s\n" - "Machine ID: %s\n" - "Boot ID: %s\n" - "Arena size: %llu\n" - "Objects: %lu\n" - "Entries: %lu\n", - f->path, - sd_id128_to_string(f->header->file_id, a), - sd_id128_to_string(f->header->machine_id, b), - sd_id128_to_string(f->header->boot_id, c), - (unsigned long long) le64toh(f->header->arena_size), - (unsigned long) le64toh(f->header->n_objects), - (unsigned long) le64toh(f->header->n_entries)); + journal_file_print_header(f); - p = le64toh(f->header->arena_offset); + p = le64toh(f->header->header_size); while (p != 0) { r = journal_file_move_to_object(f, -1, p, &o); if (r < 0) @@ -1685,6 +1874,10 @@ void journal_file_dump(JournalFile *f) { case OBJECT_ENTRY_ARRAY: printf("Type: OBJECT_ENTRY_ARRAY\n"); break; + + case OBJECT_SIGNATURE: + printf("Type: OBJECT_SIGNATURE\n"); + break; } if (o->object.flags & OBJECT_COMPRESSED) @@ -1701,10 +1894,73 @@ fail: log_error("File corrupt"); } +void journal_file_print_header(JournalFile *f) { + char a[33], b[33], c[33]; + char x[FORMAT_TIMESTAMP_MAX], y[FORMAT_TIMESTAMP_MAX]; + + assert(f); + + printf("File Path: %s\n" + "File ID: %s\n" + "Machine ID: %s\n" + "Boot ID: %s\n" + "Sequential Number ID: %s\n" + "State: %s\n" + "Compatible Flags:%s%s\n" + "Incompatible Flags:%s%s\n" + "Header size: %llu\n" + "Arena size: %llu\n" + "Data Hash Table Size: %llu\n" + "Field Hash Table Size: %llu\n" + "Objects: %llu\n" + "Entry Objects: %llu\n" + "Rotate Suggested: %s\n" + "Head Sequential Number: %llu\n" + "Tail Sequential Number: %llu\n" + "Head Realtime Timestamp: %s\n" + "Tail Realtime Timestamp: %s\n", + f->path, + sd_id128_to_string(f->header->file_id, a), + sd_id128_to_string(f->header->machine_id, b), + sd_id128_to_string(f->header->boot_id, c), + sd_id128_to_string(f->header->seqnum_id, c), + f->header->state == STATE_OFFLINE ? "offline" : + f->header->state == STATE_ONLINE ? "online" : + f->header->state == STATE_ARCHIVED ? "archived" : "unknown", + (f->header->compatible_flags & HEADER_COMPATIBLE_SIGNED) ? " SIGNED" : "", + (f->header->compatible_flags & ~HEADER_COMPATIBLE_SIGNED) ? " ???" : "", + (f->header->incompatible_flags & HEADER_INCOMPATIBLE_COMPRESSED) ? " COMPRESSED" : "", + (f->header->incompatible_flags & ~HEADER_INCOMPATIBLE_COMPRESSED) ? " ???" : "", + (unsigned long long) le64toh(f->header->header_size), + (unsigned long long) le64toh(f->header->arena_size), + (unsigned long long) le64toh(f->header->data_hash_table_size) / sizeof(HashItem), + (unsigned long long) le64toh(f->header->field_hash_table_size) / sizeof(HashItem), + (unsigned long long) le64toh(f->header->n_objects), + (unsigned long long) le64toh(f->header->n_entries), + yes_no(journal_file_rotate_suggested(f)), + (unsigned long long) le64toh(f->header->head_seqnum), + (unsigned long long) le64toh(f->header->tail_seqnum), + format_timestamp(x, sizeof(x), le64toh(f->header->head_entry_realtime)), + format_timestamp(y, sizeof(y), le64toh(f->header->tail_entry_realtime))); + + if (JOURNAL_HEADER_CONTAINS(f->header, n_data)) + printf("Data Objects: %llu\n" + "Data Hash Table Fill: %.1f%%\n", + (unsigned long long) le64toh(f->header->n_data), + 100.0 * (double) le64toh(f->header->n_data) / ((double) (le64toh(f->header->data_hash_table_size) / sizeof(HashItem)))); + + if (JOURNAL_HEADER_CONTAINS(f->header, n_fields)) + printf("Field Objects: %llu\n" + "Field Hash Table Fill: %.1f%%\n", + (unsigned long long) le64toh(f->header->n_fields), + 100.0 * (double) le64toh(f->header->n_fields) / ((double) (le64toh(f->header->field_hash_table_size) / sizeof(HashItem)))); +} + int journal_file_open( const char *fname, int flags, mode_t mode, + JournalMetrics *metrics, JournalFile *template, JournalFile **ret) { @@ -1718,6 +1974,9 @@ int journal_file_open( (flags & O_ACCMODE) != O_RDWR) return -EINVAL; + if (!endswith(fname, ".journal")) + return -EINVAL; + f = new0(JournalFile, 1); if (!f) return -ENOMEM; @@ -1728,6 +1987,9 @@ int journal_file_open( f->writable = (flags & O_ACCMODE) != O_RDONLY; f->prot = prot_from_flags(flags); + if (template) + f->compress = template->compress; + f->path = strdup(fname); if (!f->path) { r = -ENOMEM; @@ -1758,7 +2020,7 @@ int journal_file_open( } } - if (f->last_stat.st_size < (off_t) sizeof(Header)) { + if (f->last_stat.st_size < (off_t) HEADER_SIZE_MIN) { r = -EIO; goto fail; } @@ -1777,6 +2039,12 @@ int journal_file_open( } if (f->writable) { + if (metrics) { + journal_default_metrics(metrics, f->fd); + f->metrics = *metrics; + } else if (template) + f->metrics = template->metrics; + r = journal_file_refresh_header(f); if (r < 0) goto fail; @@ -1831,7 +2099,7 @@ int journal_file_rotate(JournalFile **f) { l = strlen(old_file->path); - p = new(char, l + 1 + 16 + 1 + 32 + 1 + 16 + 1); + p = new(char, l + 1 + 32 + 1 + 16 + 1 + 16 + 1); if (!p) return -ENOMEM; @@ -1840,7 +2108,7 @@ int journal_file_rotate(JournalFile **f) { sd_id128_to_string(old_file->header->seqnum_id, p + l - 8 + 1); snprintf(p + l - 8 + 1 + 32, 1 + 16 + 1 + 16 + 8 + 1, "-%016llx-%016llx.journal", - (unsigned long long) le64toh((*f)->header->seqnum), + (unsigned long long) le64toh((*f)->header->tail_seqnum), (unsigned long long) le64toh((*f)->header->tail_entry_realtime)); r = rename(old_file->path, p); @@ -1849,15 +2117,59 @@ int journal_file_rotate(JournalFile **f) { if (r < 0) return -errno; - old_file->header->state = le32toh(STATE_ARCHIVED); + old_file->header->state = STATE_ARCHIVED; - r = journal_file_open(old_file->path, old_file->flags, old_file->mode, old_file, &new_file); + r = journal_file_open(old_file->path, old_file->flags, old_file->mode, NULL, old_file, &new_file); journal_file_close(old_file); *f = new_file; return r; } +int journal_file_open_reliably( + const char *fname, + int flags, + mode_t mode, + JournalMetrics *metrics, + JournalFile *template, + JournalFile **ret) { + + int r; + size_t l; + char *p; + + r = journal_file_open(fname, flags, mode, metrics, template, ret); + if (r != -EBADMSG && /* corrupted */ + r != -ENODATA && /* truncated */ + r != -EHOSTDOWN && /* other machine */ + r != -EPROTONOSUPPORT) /* incompatible feature */ + return r; + + if ((flags & O_ACCMODE) == O_RDONLY) + return r; + + if (!(flags & O_CREAT)) + return r; + + /* The file is corrupted. Rotate it away and try it again (but only once) */ + + l = strlen(fname); + if (asprintf(&p, "%.*s@%016llx-%016llx.journal~", + (int) (l-8), fname, + (unsigned long long) now(CLOCK_REALTIME), + random_ull()) < 0) + return -ENOMEM; + + r = rename(fname, p); + free(p); + if (r < 0) + return -errno; + + log_warning("File %s corrupted, renaming and replacing.", fname); + + return journal_file_open(fname, flags, mode, metrics, template, ret); +} + struct vacuum_info { off_t usage; char *filename; @@ -1865,6 +2177,8 @@ struct vacuum_info { uint64_t realtime; sd_id128_t seqnum_id; uint64_t seqnum; + + bool have_seqnum; }; static int vacuum_compare(const void *_a, const void *_b) { @@ -1873,7 +2187,8 @@ static int vacuum_compare(const void *_a, const void *_b) { a = _a; b = _b; - if (sd_id128_equal(a->seqnum_id, b->seqnum_id)) { + if (a->have_seqnum && b->have_seqnum && + sd_id128_equal(a->seqnum_id, b->seqnum_id)) { if (a->seqnum < b->seqnum) return -1; else if (a->seqnum > b->seqnum) @@ -1886,8 +2201,10 @@ static int vacuum_compare(const void *_a, const void *_b) { return -1; else if (a->realtime > b->realtime) return 1; - else + else if (a->have_seqnum && b->have_seqnum) return memcmp(&a->seqnum_id, &b->seqnum_id, 16); + else + return strcmp(a->filename, b->filename); } int journal_directory_vacuum(const char *directory, uint64_t max_use, uint64_t min_free) { @@ -1912,8 +2229,9 @@ int journal_directory_vacuum(const char *directory, uint64_t max_use, uint64_t m size_t q; struct stat st; char *p; - unsigned long long seqnum, realtime; + unsigned long long seqnum = 0, realtime; sd_id128_t seqnum_id; + bool have_seqnum; k = readdir_r(d, &buf, &de); if (k != 0) { @@ -1924,41 +2242,71 @@ int journal_directory_vacuum(const char *directory, uint64_t max_use, uint64_t m if (!de) break; - if (!dirent_is_file_with_suffix(de, ".journal")) + if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) + continue; + + if (!S_ISREG(st.st_mode)) continue; q = strlen(de->d_name); - if (q < 1 + 32 + 1 + 16 + 1 + 16 + 8) - continue; + if (endswith(de->d_name, ".journal")) { - if (de->d_name[q-8-16-1] != '-' || - de->d_name[q-8-16-1-16-1] != '-' || - de->d_name[q-8-16-1-16-1-32-1] != '@') - continue; + /* Vacuum archived files */ - if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) - continue; + if (q < 1 + 32 + 1 + 16 + 1 + 16 + 8) + continue; - if (!S_ISREG(st.st_mode)) - continue; + if (de->d_name[q-8-16-1] != '-' || + de->d_name[q-8-16-1-16-1] != '-' || + de->d_name[q-8-16-1-16-1-32-1] != '@') + continue; - p = strdup(de->d_name); - if (!p) { - r = -ENOMEM; - goto finish; - } + p = strdup(de->d_name); + if (!p) { + r = -ENOMEM; + goto finish; + } - de->d_name[q-8-16-1-16-1] = 0; - if (sd_id128_from_string(de->d_name + q-8-16-1-16-1-32, &seqnum_id) < 0) { - free(p); - continue; - } + de->d_name[q-8-16-1-16-1] = 0; + if (sd_id128_from_string(de->d_name + q-8-16-1-16-1-32, &seqnum_id) < 0) { + free(p); + continue; + } + + if (sscanf(de->d_name + q-8-16-1-16, "%16llx-%16llx.journal", &seqnum, &realtime) != 2) { + free(p); + continue; + } + + have_seqnum = true; + + } else if (endswith(de->d_name, ".journal~")) { + unsigned long long tmp; + + /* Vacuum corrupted files */ + + if (q < 1 + 16 + 1 + 16 + 8 + 1) + continue; + + if (de->d_name[q-1-8-16-1] != '-' || + de->d_name[q-1-8-16-1-16-1] != '@') + continue; - if (sscanf(de->d_name + q-8-16-1-16, "%16llx-%16llx.journal", &seqnum, &realtime) != 2) { - free(p); + p = strdup(de->d_name); + if (!p) { + r = -ENOMEM; + goto finish; + } + + if (sscanf(de->d_name + q-1-8-16-1-16, "%16llx-%16llx.journal~", &realtime, &tmp) != 2) { + free(p); + continue; + } + + have_seqnum = false; + } else continue; - } if (n_list >= n_allocated) { struct vacuum_info *j; @@ -1975,10 +2323,11 @@ int journal_directory_vacuum(const char *directory, uint64_t max_use, uint64_t m } list[n_list].filename = p; - list[n_list].usage = (uint64_t) st.st_blksize * (uint64_t) st.st_blocks; + list[n_list].usage = 512UL * (uint64_t) st.st_blocks; list[n_list].seqnum = seqnum; list[n_list].realtime = realtime; list[n_list].seqnum_id = seqnum_id; + list[n_list].have_seqnum = have_seqnum; sum += list[n_list].usage; @@ -2000,7 +2349,7 @@ int journal_directory_vacuum(const char *directory, uint64_t max_use, uint64_t m break; if (unlinkat(dirfd(d), list[i].filename, 0) >= 0) { - log_debug("Deleted archived journal %s/%s.", directory, list[i].filename); + log_info("Deleted archived journal %s/%s.", directory, list[i].filename); sum -= list[i].usage; } else if (errno != ENOENT) log_warning("Failed to delete %s/%s: %m", directory, list[i].filename); @@ -2040,14 +2389,12 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6 ts.monotonic < le64toh(to->header->tail_entry_monotonic)) return -EINVAL; - if (ts.realtime < le64toh(to->header->tail_entry_realtime)) - return -EINVAL; - n = journal_file_entry_n_items(o); items = alloca(sizeof(EntryItem) * n); for (i = 0; i < n; i++) { - uint64_t le_hash, l, h; + uint64_t l, h; + le64_t le_hash; size_t t; void *data; Object *u; @@ -2103,7 +2450,7 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6 void journal_default_metrics(JournalMetrics *m, int fd) { uint64_t fs_size = 0; struct statvfs ss; - char a[64], b[64], c[64], d[64]; + char a[FORMAT_BYTES_MAX], b[FORMAT_BYTES_MAX], c[FORMAT_BYTES_MAX], d[FORMAT_BYTES_MAX]; assert(m); assert(fd >= 0); @@ -2168,9 +2515,116 @@ void journal_default_metrics(JournalMetrics *m, int fd) { m->keep_free = DEFAULT_KEEP_FREE; } - log_debug("Fixed max_use=%s max_size=%s min_size=%s keep_free=%s", - format_bytes(a, sizeof(a), m->max_use), - format_bytes(b, sizeof(b), m->max_size), - format_bytes(c, sizeof(c), m->min_size), - format_bytes(d, sizeof(d), m->keep_free)); + log_info("Fixed max_use=%s max_size=%s min_size=%s keep_free=%s", + format_bytes(a, sizeof(a), m->max_use), + format_bytes(b, sizeof(b), m->max_size), + format_bytes(c, sizeof(c), m->min_size), + format_bytes(d, sizeof(d), m->keep_free)); +} + +int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *from, usec_t *to) { + assert(f); + assert(from || to); + + if (from) { + if (f->header->head_entry_realtime == 0) + return -ENOENT; + + *from = le64toh(f->header->head_entry_realtime); + } + + if (to) { + if (f->header->tail_entry_realtime == 0) + return -ENOENT; + + *to = le64toh(f->header->tail_entry_realtime); + } + + return 1; +} + +int journal_file_get_cutoff_monotonic_usec(JournalFile *f, sd_id128_t boot_id, usec_t *from, usec_t *to) { + char t[9+32+1] = "_BOOT_ID="; + Object *o; + uint64_t p; + int r; + + assert(f); + assert(from || to); + + sd_id128_to_string(boot_id, t + 9); + + r = journal_file_find_data_object(f, t, strlen(t), &o, &p); + if (r <= 0) + return r; + + if (le64toh(o->data.n_entries) <= 0) + return 0; + + if (from) { + r = journal_file_move_to_object(f, OBJECT_ENTRY, le64toh(o->data.entry_offset), &o); + if (r < 0) + return r; + + *from = le64toh(o->entry.monotonic); + } + + if (to) { + r = journal_file_move_to_object(f, OBJECT_DATA, p, &o); + if (r < 0) + return r; + + r = generic_array_get_plus_one(f, + le64toh(o->data.entry_offset), + le64toh(o->data.entry_array_offset), + le64toh(o->data.n_entries)-1, + &o, NULL); + if (r <= 0) + return r; + + *to = le64toh(o->entry.monotonic); + } + + return 1; +} + +bool journal_file_rotate_suggested(JournalFile *f) { + assert(f); + + /* If we gained new header fields we gained new features, + * hence suggest a rotation */ + if (le64toh(f->header->header_size) < sizeof(Header)) { + log_debug("%s uses an outdated header, suggesting rotation.", f->path); + return true; + } + + /* Let's check if the hash tables grew over a certain fill + * level (75%, borrowing this value from Java's hash table + * implementation), and if so suggest a rotation. To calculate + * the fill level we need the n_data field, which only exists + * in newer versions. */ + + if (JOURNAL_HEADER_CONTAINS(f->header, n_data)) + if (le64toh(f->header->n_data) * 4ULL > (le64toh(f->header->data_hash_table_size) / sizeof(HashItem)) * 3ULL) { + log_debug("Data hash table of %s has a fill level at %.1f (%llu of %llu items, %llu file size, %llu bytes per hash table item), suggesting rotation.", + f->path, + 100.0 * (double) le64toh(f->header->n_data) / ((double) (le64toh(f->header->data_hash_table_size) / sizeof(HashItem))), + (unsigned long long) le64toh(f->header->n_data), + (unsigned long long) (le64toh(f->header->data_hash_table_size) / sizeof(HashItem)), + (unsigned long long) (f->last_stat.st_size), + (unsigned long long) (f->last_stat.st_size / le64toh(f->header->n_data))); + return true; + } + + if (JOURNAL_HEADER_CONTAINS(f->header, n_fields)) + if (le64toh(f->header->n_fields) * 4ULL > (le64toh(f->header->field_hash_table_size) / sizeof(HashItem)) * 3ULL) { + log_debug("Field hash table of %s has a fill level at %.1f (%llu of %llu items), suggesting rotation.", + f->path, + 100.0 * (double) le64toh(f->header->n_fields) / ((double) (le64toh(f->header->field_hash_table_size) / sizeof(HashItem))), + (unsigned long long) le64toh(f->header->n_fields), + (unsigned long long) (le64toh(f->header->field_hash_table_size) / sizeof(HashItem))); + return true; + } + + return false; }