#include <fcntl.h>
#include <stddef.h>
+#ifdef HAVE_XATTR
+#include <attr/xattr.h>
+#endif
+
#include "journal-def.h"
#include "journal-file.h"
#include "journal-authenticate.h"
/* n_data was the first entry we added after the initial file format design */
#define HEADER_SIZE_MIN ALIGN64(offsetof(Header, n_data))
+/* How many entries to keep in the entry array chain cache at max */
+#define CHAIN_CACHE_MAX 20
+
void journal_file_close(JournalFile *f) {
assert(f);
if (f->mmap)
mmap_cache_unref(f->mmap);
+ hashmap_free_free(f->chain_cache);
+
#ifdef HAVE_XZ
free(f->compress_buffer);
#endif
f->compress = JOURNAL_HEADER_COMPRESSED(f->header);
- if (f->writable)
- f->seal = JOURNAL_HEADER_SEALED(f->header);
+ f->seal = JOURNAL_HEADER_SEALED(f->header);
return 0;
}
if (s < minimum_header_size(o))
return -EBADMSG;
- if (type >= 0 && o->object.type != type)
+ if (type > 0 && o->object.type != type)
return -EBADMSG;
if (s > sizeof(ObjectHeader)) {
assert(f);
+ /* We use a fixed size hash table for the fields as this
+ * number should grow very slowly only */
+
s = DEFAULT_FIELD_HASH_TABLE_SIZE;
r = journal_file_append_object(f,
OBJECT_FIELD_HASH_TABLE,
return 0;
}
-static int journal_file_link_data(JournalFile *f, Object *o, uint64_t offset, uint64_t hash) {
+static int journal_file_link_field(
+ JournalFile *f,
+ Object *o,
+ uint64_t offset,
+ uint64_t hash) {
+
+ uint64_t p, h;
+ int r;
+
+ assert(f);
+ assert(o);
+ assert(offset > 0);
+
+ if (o->object.type != OBJECT_FIELD)
+ return -EINVAL;
+
+ /* This might alter the window we are looking at */
+
+ o->field.next_hash_offset = o->field.head_data_offset = 0;
+
+ h = hash % (le64toh(f->header->field_hash_table_size) / sizeof(HashItem));
+ p = le64toh(f->field_hash_table[h].tail_hash_offset);
+ if (p == 0)
+ f->field_hash_table[h].head_hash_offset = htole64(offset);
+ else {
+ r = journal_file_move_to_object(f, OBJECT_FIELD, p, &o);
+ if (r < 0)
+ return r;
+
+ o->field.next_hash_offset = htole64(offset);
+ }
+
+ f->field_hash_table[h].tail_hash_offset = htole64(offset);
+
+ if (JOURNAL_HEADER_CONTAINS(f->header, n_fields))
+ f->header->n_fields = htole64(le64toh(f->header->n_fields) + 1);
+
+ return 0;
+}
+
+static int journal_file_link_data(
+ JournalFile *f,
+ Object *o,
+ uint64_t offset,
+ uint64_t hash) {
+
uint64_t p, h;
int r;
h = hash % (le64toh(f->header->data_hash_table_size) / sizeof(HashItem));
p = le64toh(f->data_hash_table[h].tail_hash_offset);
- if (p == 0) {
+ if (p == 0)
/* Only entry in the hash table is easy */
f->data_hash_table[h].head_hash_offset = htole64(offset);
- } else {
+ else {
/* Move back to the previous data object, to patch in
* pointer */
return 0;
}
+int journal_file_find_field_object_with_hash(
+ JournalFile *f,
+ const void *field, uint64_t size, uint64_t hash,
+ Object **ret, uint64_t *offset) {
+
+ uint64_t p, osize, h;
+ int r;
+
+ assert(f);
+ assert(field && size > 0);
+
+ osize = offsetof(Object, field.payload) + size;
+
+ if (f->header->field_hash_table_size == 0)
+ return -EBADMSG;
+
+ h = hash % (le64toh(f->header->field_hash_table_size) / sizeof(HashItem));
+ p = le64toh(f->field_hash_table[h].head_hash_offset);
+
+ while (p > 0) {
+ Object *o;
+
+ r = journal_file_move_to_object(f, OBJECT_FIELD, p, &o);
+ if (r < 0)
+ return r;
+
+ if (le64toh(o->field.hash) == hash &&
+ le64toh(o->object.size) == osize &&
+ memcmp(o->field.payload, field, size) == 0) {
+
+ if (ret)
+ *ret = o;
+ if (offset)
+ *offset = p;
+
+ return 1;
+ }
+
+ p = le64toh(o->field.next_hash_offset);
+ }
+
+ return 0;
+}
+
+int journal_file_find_field_object(
+ JournalFile *f,
+ const void *field, uint64_t size,
+ Object **ret, uint64_t *offset) {
+
+ uint64_t hash;
+
+ assert(f);
+ assert(field && size > 0);
+
+ hash = hash64(field, size);
+
+ return journal_file_find_field_object_with_hash(f,
+ field, size, hash,
+ ret, offset);
+}
+
int journal_file_find_data_object_with_hash(
JournalFile *f,
const void *data, uint64_t size, uint64_t hash,
ret, offset);
}
+static int journal_file_append_field(
+ JournalFile *f,
+ const void *field, uint64_t size,
+ Object **ret, uint64_t *offset) {
+
+ uint64_t hash, p;
+ uint64_t osize;
+ Object *o;
+ int r;
+
+ assert(f);
+ assert(field && size > 0);
+
+ hash = hash64(field, size);
+
+ r = journal_file_find_field_object_with_hash(f, field, size, hash, &o, &p);
+ if (r < 0)
+ return r;
+ else if (r > 0) {
+
+ if (ret)
+ *ret = o;
+
+ if (offset)
+ *offset = p;
+
+ return 0;
+ }
+
+ osize = offsetof(Object, field.payload) + size;
+ r = journal_file_append_object(f, OBJECT_FIELD, osize, &o, &p);
+
+ o->field.hash = htole64(hash);
+ memcpy(o->field.payload, field, size);
+
+ r = journal_file_link_field(f, o, p, hash);
+ if (r < 0)
+ return r;
+
+ /* The linking might have altered the window, so let's
+ * refresh our pointer */
+ r = journal_file_move_to_object(f, OBJECT_FIELD, p, &o);
+ if (r < 0)
+ return r;
+
+#ifdef HAVE_GCRYPT
+ r = journal_file_hmac_put_object(f, OBJECT_FIELD, o, p);
+ if (r < 0)
+ return r;
+#endif
+
+ if (ret)
+ *ret = o;
+
+ if (offset)
+ *offset = p;
+
+ return 0;
+}
+
static int journal_file_append_data(
JournalFile *f,
const void *data, uint64_t size,
Object *o;
int r;
bool compressed = false;
+ const void *eq;
assert(f);
assert(data || size == 0);
if (r < 0)
return r;
+ eq = memchr(data, '=', size);
+ if (eq && eq > data) {
+ uint64_t fp;
+ Object *fo;
+
+ /* Create field object ... */
+ r = journal_file_append_field(f, data, (uint8_t*) eq - (uint8_t*) data, &fo, &fp);
+ if (r < 0)
+ return r;
+
+ /* ... and link it in. */
+ o->data.next_field_offset = fo->field.head_data_offset;
+ fo->field.head_data_offset = le64toh(p);
+ }
+
#ifdef HAVE_GCRYPT
r = journal_file_hmac_put_object(f, OBJECT_DATA, o, p);
if (r < 0)
log_error("Failed to truncate file to its own size: %m");
}
+static int entry_item_cmp(const void *_a, const void *_b) {
+ const EntryItem *a = _a, *b = _b;
+
+ if (le64toh(a->object_offset) < le64toh(b->object_offset))
+ return -1;
+ if (le64toh(a->object_offset) > le64toh(b->object_offset))
+ return 1;
+ return 0;
+}
+
int journal_file_append_entry(JournalFile *f, const dual_timestamp *ts, const struct iovec iovec[], unsigned n_iovec, uint64_t *seqnum, Object **ret, uint64_t *offset) {
unsigned i;
EntryItem *items;
items[i].hash = o->data.hash;
}
+ /* Order by the position on disk, in order to improve seek
+ * times for rotating media. */
+ qsort(items, n_iovec, sizeof(EntryItem), entry_item_cmp);
+
r = journal_file_append_entry_internal(f, ts, xor_hash, items, n_iovec, seqnum, ret, offset);
journal_file_post_change(f);
return r;
}
+typedef struct ChainCacheItem {
+ uint64_t first; /* the array at the begin of the chain */
+ uint64_t array; /* the cached array */
+ uint64_t begin; /* the first item in the cached array */
+ uint64_t total; /* the total number of items in all arrays before this one in the chain */
+} ChainCacheItem;
+
+static void chain_cache_put(
+ Hashmap *h,
+ ChainCacheItem *ci,
+ uint64_t first,
+ uint64_t array,
+ uint64_t begin,
+ uint64_t total) {
+
+ if (!ci) {
+ /* If the chain item to cache for this chain is the
+ * first one it's not worth caching anything */
+ if (array == first)
+ return;
+
+ if (hashmap_size(h) >= CHAIN_CACHE_MAX)
+ ci = hashmap_steal_first(h);
+ else {
+ ci = new(ChainCacheItem, 1);
+ if (!ci)
+ return;
+ }
+
+ ci->first = first;
+
+ if (hashmap_put(h, &ci->first, ci) < 0) {
+ free(ci);
+ return;
+ }
+ } else
+ assert(ci->first == first);
+
+ ci->array = array;
+ ci->begin = begin;
+ ci->total = total;
+}
+
static int generic_array_get(JournalFile *f,
uint64_t first,
uint64_t i,
Object **ret, uint64_t *offset) {
Object *o;
- uint64_t p = 0, a;
+ uint64_t p = 0, a, t = 0;
int r;
+ ChainCacheItem *ci;
assert(f);
a = first;
+
+ /* Try the chain cache first */
+ ci = hashmap_get(f->chain_cache, &first);
+ if (ci && i > ci->total) {
+ a = ci->array;
+ i -= ci->total;
+ t = ci->total;
+ }
+
while (a > 0) {
- uint64_t n;
+ uint64_t k;
r = journal_file_move_to_object(f, OBJECT_ENTRY_ARRAY, a, &o);
if (r < 0)
return r;
- n = journal_file_entry_array_n_items(o);
- if (i < n) {
+ k = journal_file_entry_array_n_items(o);
+ if (i < k) {
p = le64toh(o->entry_array.items[i]);
- break;
+ goto found;
}
- i -= n;
+ i -= k;
+ t += k;
a = le64toh(o->entry_array.next_entry_array_offset);
}
- if (a <= 0 || p <= 0)
- return 0;
+ return 0;
+
+found:
+ /* Let's cache this item for the next invocation */
+ chain_cache_put(f->chain_cache, ci, first, a, o->entry_array.items[0], t);
r = journal_file_move_to_object(f, OBJECT_ENTRY, p, &o);
if (r < 0)
bool subtract_one = false;
Object *o, *array = NULL;
int r;
+ ChainCacheItem *ci;
assert(f);
assert(test_object);
+ /* Start with the first array in the chain */
a = first;
+
+ ci = hashmap_get(f->chain_cache, &first);
+ if (ci && n > ci->total) {
+ /* Ah, we have iterated this bisection array chain
+ * previously! Let's see if we can skip ahead in the
+ * chain, as far as the last time. But we can't jump
+ * backwards in the chain, so let's check that
+ * first. */
+
+ r = test_object(f, ci->begin, needle);
+ if (r < 0)
+ return r;
+
+ if (r == TEST_LEFT) {
+ /* OK, what we are looking for is right of th
+ * begin of this EntryArray, so let's jump
+ * straight to previously cached array in the
+ * chain */
+
+ a = ci->array;
+ n -= ci->total;
+ t = ci->total;
+ }
+ }
+
while (a > 0) {
uint64_t left, right, k, lp;
if (subtract_one && t == 0 && i == 0)
return 0;
+ /* Let's cache this item for the next invocation */
+ chain_cache_put(f->chain_cache, ci, first, a, array->entry_array.items[0], t);
+
if (subtract_one && i == 0)
p = last_p;
else if (subtract_one)
printf("Type: OBJECT_DATA\n");
break;
+ case OBJECT_FIELD:
+ printf("Type: OBJECT_FIELD\n");
+ break;
+
case OBJECT_ENTRY:
printf("Type: OBJECT_ENTRY seqnum=%llu monotonic=%llu realtime=%llu\n",
(unsigned long long) le64toh(o->entry.seqnum),
(unsigned long long) le64toh(o->tag.seqnum),
(unsigned long long) le64toh(o->tag.epoch));
break;
+
+ default:
+ printf("Type: unknown (%u)\n", o->object.type);
+ break;
}
if (o->object.flags & OBJECT_COMPRESSED)
(unsigned long long) le64toh(f->header->arena_size),
(unsigned long long) le64toh(f->header->data_hash_table_size) / sizeof(HashItem),
(unsigned long long) le64toh(f->header->field_hash_table_size) / sizeof(HashItem),
- yes_no(journal_file_rotate_suggested(f)),
+ yes_no(journal_file_rotate_suggested(f, 0)),
(unsigned long long) le64toh(f->header->head_entry_seqnum),
(unsigned long long) le64toh(f->header->tail_entry_seqnum),
format_timestamp(x, sizeof(x), le64toh(f->header->head_entry_realtime)),
goto fail;
}
+ f->chain_cache = hashmap_new(uint64_hash_func, uint64_compare_func);
+ if (!f->chain_cache) {
+ r = -ENOMEM;
+ goto fail;
+ }
+
f->fd = open(f->path, f->flags|O_CLOEXEC, f->mode);
if (f->fd < 0) {
r = -errno;
}
if (f->last_stat.st_size == 0 && f->writable) {
- newly_created = true;
+#ifdef HAVE_XATTR
+ uint64_t crtime;
+
+ /* Let's attach the creation time to the journal file,
+ * so that the vacuuming code knows the age of this
+ * file even if the file might end up corrupted one
+ * day... Ideally we'd just use the creation time many
+ * file systems maintain for each file, but there is
+ * currently no usable API to query this, hence let's
+ * emulate this via extended attributes. If extended
+ * attributes are not supported we'll just skip this,
+ * and rely solely on mtime/atime/ctime of the file.*/
+
+ crtime = htole64((uint64_t) now(CLOCK_REALTIME));
+ fsetxattr(f->fd, "user.crtime_usec", &crtime, sizeof(crtime), XATTR_CREATE);
+#endif
#ifdef HAVE_GCRYPT
/* Try to load the FSPRG state, and if we can't, then
r = -errno;
goto fail;
}
+
+ newly_created = true;
}
if (f->last_stat.st_size < (off_t) HEADER_SIZE_MIN) {
sd_id128_to_string(old_file->header->seqnum_id, p + l - 8 + 1);
snprintf(p + l - 8 + 1 + 32, 1 + 16 + 1 + 16 + 8 + 1,
"-%016llx-%016llx.journal",
- (unsigned long long) le64toh((*f)->header->tail_entry_seqnum),
- (unsigned long long) le64toh((*f)->header->tail_entry_realtime));
+ (unsigned long long) le64toh((*f)->header->head_entry_seqnum),
+ (unsigned long long) le64toh((*f)->header->head_entry_realtime));
r = rename(old_file->path, p);
free(p);
return 1;
}
-bool journal_file_rotate_suggested(JournalFile *f) {
+bool journal_file_rotate_suggested(JournalFile *f, usec_t max_file_usec) {
assert(f);
/* If we gained new header fields we gained new features,
return true;
}
+ /* Are the data objects properly indexed by field objects? */
+ if (JOURNAL_HEADER_CONTAINS(f->header, n_data) &&
+ JOURNAL_HEADER_CONTAINS(f->header, n_fields) &&
+ le64toh(f->header->n_data) > 0 &&
+ le64toh(f->header->n_fields) == 0)
+ return true;
+
+ if (max_file_usec > 0) {
+ usec_t t, h;
+
+ h = le64toh(f->header->head_entry_realtime);
+ t = now(CLOCK_REALTIME);
+
+ if (h > 0 && t > h + max_file_usec)
+ return true;
+ }
+
return false;
}