chiark / gitweb /
journal: include tag object header in hmac
[elogind.git] / src / journal / journal-file.c
index b9ab009b3f732ce3cf1f9e74b8920053590eb1d1..9235e5fea2df5711acbe92ce141567ef175c7665 100644 (file)
@@ -31,6 +31,7 @@
 #include "journal-file.h"
 #include "lookup3.h"
 #include "compress.h"
+#include "fsprg.h"
 
 #define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem))
 #define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem))
 #define JOURNAL_HEADER_CONTAINS(h, field) \
         (le64toh((h)->header_size) >= offsetof(Header, field) + sizeof((h)->field))
 
-static const char signature[] = { 'L', 'P', 'K', 'S', 'H', 'H', 'R', 'H' };
+static int journal_file_maybe_append_tag(JournalFile *f, uint64_t realtime);
+static int journal_file_hmac_put_object(JournalFile *f, int type, uint64_t p);
 
 void journal_file_close(JournalFile *f) {
         int t;
 
         assert(f);
 
+        /* Write the final tag */
+        if (f->authenticate)
+                journal_file_append_tag(f);
+
+        /* Sync everything to disk, before we mark the file offline */
+        for (t = 0; t < _WINDOW_MAX; t++)
+                if (f->windows[t].ptr)
+                        munmap(f->windows[t].ptr, f->windows[t].size);
+
+        if (f->writable && f->fd >= 0)
+                fdatasync(f->fd);
+
         if (f->header) {
                 /* Mark the file offline. Don't override the archived state if it already is set */
                 if (f->writable && f->header->state == STATE_ONLINE)
@@ -81,10 +95,6 @@ void journal_file_close(JournalFile *f) {
                 munmap(f->header, PAGE_ALIGN(sizeof(Header)));
         }
 
-        for (t = 0; t < _WINDOW_MAX; t++)
-                if (f->windows[t].ptr)
-                        munmap(f->windows[t].ptr, f->windows[t].size);
-
         if (f->fd >= 0)
                 close_nointr_nofail(f->fd);
 
@@ -94,6 +104,14 @@ void journal_file_close(JournalFile *f) {
         free(f->compress_buffer);
 #endif
 
+#ifdef HAVE_GCRYPT
+        if (f->fsprg_header)
+                munmap(f->fsprg_header, PAGE_ALIGN(f->fsprg_size));
+
+        if (f->hmac)
+                gcry_md_close(f->hmac);
+#endif
+
         free(f);
 }
 
@@ -105,9 +123,15 @@ static int journal_file_init_header(JournalFile *f, JournalFile *template) {
         assert(f);
 
         zero(h);
-        memcpy(h.signature, signature, 8);
+        memcpy(h.signature, HEADER_SIGNATURE, 8);
         h.header_size = htole64(ALIGN64(sizeof(h)));
 
+        h.incompatible_flags =
+                htole32(f->compress ? HEADER_INCOMPATIBLE_COMPRESSED : 0);
+
+        h.compatible_flags =
+                htole32(f->authenticate ? HEADER_COMPATIBLE_AUTHENTICATED : 0);
+
         r = sd_id128_randomize(&h.file_id);
         if (r < 0)
                 return r;
@@ -149,7 +173,9 @@ static int journal_file_refresh_header(JournalFile *f) {
 
         f->header->state = STATE_ONLINE;
 
-        __sync_synchronize();
+        /* Sync the online state to disk */
+        msync(f->header, PAGE_ALIGN(sizeof(Header)), MS_SYNC);
+        fdatasync(f->fd);
 
         return 0;
 }
@@ -157,17 +183,31 @@ static int journal_file_refresh_header(JournalFile *f) {
 static int journal_file_verify_header(JournalFile *f) {
         assert(f);
 
-        if (memcmp(f->header, signature, 8))
+        if (memcmp(f->header->signature, HEADER_SIGNATURE, 8))
                 return -EBADMSG;
 
+        /* In both read and write mode we refuse to open files with
+         * incompatible flags we don't know */
 #ifdef HAVE_XZ
-        if ((le64toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_COMPRESSED) != 0)
+        if ((le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_COMPRESSED) != 0)
                 return -EPROTONOSUPPORT;
 #else
         if (f->header->incompatible_flags != 0)
                 return -EPROTONOSUPPORT;
 #endif
 
+        /* When open for writing we refuse to open files with
+         * compatible flags, too */
+        if (f->writable) {
+#ifdef HAVE_GCRYPT
+                if ((le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_AUTHENTICATED) != 0)
+                        return -EPROTONOSUPPORT;
+#else
+                if (f->header->compatible_flags != 0)
+                        return -EPROTONOSUPPORT;
+#endif
+        }
+
         /* The first addition was n_data, so check that we are at least this large */
         if (le64toh(f->header->header_size) < HEADER_SIZE_MIN)
                 return -EBADMSG;
@@ -200,6 +240,9 @@ static int journal_file_verify_header(JournalFile *f) {
                 }
         }
 
+        f->compress = !!(le32toh(f->header->incompatible_flags) & HEADER_INCOMPATIBLE_COMPRESSED);
+        f->authenticate = !!(le32toh(f->header->compatible_flags) & HEADER_COMPATIBLE_AUTHENTICATED);
+
         return 0;
 }
 
@@ -432,7 +475,7 @@ int journal_file_move_to_object(JournalFile *f, int type, uint64_t offset, Objec
         return 0;
 }
 
-static uint64_t journal_file_seqnum(JournalFile *f, uint64_t *seqnum) {
+static uint64_t journal_file_entry_seqnum(JournalFile *f, uint64_t *seqnum) {
         uint64_t r;
 
         assert(f);
@@ -781,20 +824,22 @@ static int journal_file_append_data(
                         o->object.size = htole64(offsetof(Object, data.payload) + rsize);
                         o->object.flags |= OBJECT_COMPRESSED;
 
-                        f->header->incompatible_flags = htole32(le32toh(f->header->incompatible_flags) | HEADER_INCOMPATIBLE_COMPRESSED);
-
                         log_debug("Compressed data object %lu -> %lu", (unsigned long) size, (unsigned long) rsize);
                 }
         }
 #endif
 
-        if (!compressed)
+        if (!compressed && size > 0)
                 memcpy(o->data.payload, data, size);
 
         r = journal_file_link_data(f, o, p, hash);
         if (r < 0)
                 return r;
 
+        r = journal_file_hmac_put_object(f, OBJECT_DATA, p);
+        if (r < 0)
+                return r;
+
         /* The linking might have altered the window, so let's
          * refresh our pointer */
         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
@@ -871,6 +916,10 @@ static int link_entry_into_array(JournalFile *f,
         if (r < 0)
                 return r;
 
+        r = journal_file_hmac_put_object(f, OBJECT_ENTRY_ARRAY, q);
+        if (r < 0)
+                return r;
+
         o->entry_array.items[i] = htole64(p);
 
         if (ap == 0)
@@ -1001,13 +1050,17 @@ static int journal_file_append_entry_internal(
         if (r < 0)
                 return r;
 
-        o->entry.seqnum = htole64(journal_file_seqnum(f, seqnum));
+        o->entry.seqnum = htole64(journal_file_entry_seqnum(f, seqnum));
         memcpy(o->entry.items, items, n_items * sizeof(EntryItem));
         o->entry.realtime = htole64(ts->realtime);
         o->entry.monotonic = htole64(ts->monotonic);
         o->entry.xor_hash = htole64(xor_hash);
         o->entry.boot_id = f->header->boot_id;
 
+        r = journal_file_hmac_put_object(f, OBJECT_ENTRY, np);
+        if (r < 0)
+                return r;
+
         r = journal_file_link_entry(f, o, np);
         if (r < 0)
                 return r;
@@ -1057,7 +1110,12 @@ int journal_file_append_entry(JournalFile *f, const dual_timestamp *ts, const st
             ts->monotonic < le64toh(f->header->tail_entry_monotonic))
                 return -EINVAL;
 
-        items = alloca(sizeof(EntryItem) * n_iovec);
+        r = journal_file_maybe_append_tag(f, ts->realtime);
+        if (r < 0)
+                return r;
+
+        /* alloca() can't take 0, hence let's allocate at least one */
+        items = alloca(sizeof(EntryItem) * MAX(1, n_iovec));
 
         for (i = 0; i < n_iovec; i++) {
                 uint64_t p;
@@ -1831,6 +1889,412 @@ int journal_file_move_to_entry_by_realtime_for_data(
                                              ret, offset, NULL);
 }
 
+static void *fsprg_state(JournalFile *f) {
+        uint64_t a, b;
+        assert(f);
+
+        if (!f->authenticate)
+                return NULL;
+
+        a = le64toh(f->fsprg_header->header_size);
+        b = le64toh(f->fsprg_header->state_size);
+
+        if (a + b > f->fsprg_size)
+                return NULL;
+
+        return (uint8_t*) f->fsprg_header + a;
+}
+
+static uint64_t journal_file_tag_seqnum(JournalFile *f) {
+        uint64_t r;
+
+        assert(f);
+
+        r = le64toh(f->header->n_tags) + 1;
+        f->header->n_tags = htole64(r);
+
+        return r;
+}
+
+int journal_file_append_tag(JournalFile *f) {
+        Object *o;
+        uint64_t p;
+        int r;
+
+        assert(f);
+
+        if (!f->authenticate)
+                return 0;
+
+        if (!f->hmac_running)
+                return 0;
+
+        log_debug("Writing tag for epoch %llu\n", (unsigned long long) FSPRG_GetEpoch(fsprg_state(f)));
+
+        assert(f->hmac);
+
+        r = journal_file_append_object(f, OBJECT_TAG, sizeof(struct TagObject), &o, &p);
+        if (r < 0)
+                return r;
+
+        o->tag.seqnum = htole64(journal_file_tag_seqnum(f));
+
+        /* Add the tag object itself, so that we can protect its
+         * header. This will exclude the actual hash value in it */
+        r = journal_file_hmac_put_object(f, OBJECT_TAG, p);
+        if (r < 0)
+                return r;
+
+        /* Get the HMAC tag and store it in the object */
+        memcpy(o->tag.tag, gcry_md_read(f->hmac, 0), TAG_LENGTH);
+        f->hmac_running = false;
+
+        return 0;
+}
+
+static int journal_file_hmac_start(JournalFile *f) {
+        uint8_t key[256 / 8]; /* Let's pass 256 bit from FSPRG to HMAC */
+
+        assert(f);
+
+        if (!f->authenticate)
+                return 0;
+
+        if (f->hmac_running)
+                return 0;
+
+        /* Prepare HMAC for next cycle */
+        gcry_md_reset(f->hmac);
+        FSPRG_GetKey(fsprg_state(f), key, sizeof(key), 0);
+        gcry_md_setkey(f->hmac, key, sizeof(key));
+
+        f->hmac_running = true;
+
+        return 0;
+}
+
+static int journal_file_get_epoch(JournalFile *f, uint64_t realtime, uint64_t *epoch) {
+        uint64_t t;
+
+        assert(f);
+        assert(epoch);
+        assert(f->authenticate);
+
+        if (le64toh(f->fsprg_header->fsprg_start_usec) == 0 ||
+            le64toh(f->fsprg_header->fsprg_interval_usec) == 0)
+                return -ENOTSUP;
+
+        if (realtime < le64toh(f->fsprg_header->fsprg_start_usec))
+                return -ESTALE;
+
+        t = realtime - le64toh(f->fsprg_header->fsprg_start_usec);
+        t = t / le64toh(f->fsprg_header->fsprg_interval_usec);
+
+        *epoch = t;
+        return 0;
+}
+
+static int journal_file_need_evolve(JournalFile *f, uint64_t realtime) {
+        uint64_t goal, epoch;
+        int r;
+        assert(f);
+
+        if (!f->authenticate)
+                return 0;
+
+        r = journal_file_get_epoch(f, realtime, &goal);
+        if (r < 0)
+                return r;
+
+        epoch = FSPRG_GetEpoch(fsprg_state(f));
+        if (epoch > goal)
+                return -ESTALE;
+
+        return epoch != goal;
+}
+
+static int journal_file_evolve(JournalFile *f, uint64_t realtime) {
+        uint64_t goal, epoch;
+        int r;
+
+        assert(f);
+
+        if (!f->authenticate)
+                return 0;
+
+        r = journal_file_get_epoch(f, realtime, &goal);
+        if (r < 0)
+                return r;
+
+        epoch = FSPRG_GetEpoch(fsprg_state(f));
+        if (epoch < goal)
+                log_debug("Evolving FSPRG key from epoch %llu to %llu.", (unsigned long long) epoch, (unsigned long long) goal);
+
+        for (;;) {
+                if (epoch > goal)
+                        return -ESTALE;
+                if (epoch == goal)
+                        return 0;
+
+                FSPRG_Evolve(fsprg_state(f));
+                epoch = FSPRG_GetEpoch(fsprg_state(f));
+        }
+}
+
+static int journal_file_maybe_append_tag(JournalFile *f, uint64_t realtime) {
+        int r;
+
+        assert(f);
+
+        if (!f->authenticate)
+                return 0;
+
+        r = journal_file_need_evolve(f, realtime);
+        if (r <= 0)
+                return 0;
+
+        r = journal_file_append_tag(f);
+        if (r < 0)
+                return r;
+
+        r = journal_file_evolve(f, realtime);
+        if (r < 0)
+                return r;
+
+        r = journal_file_hmac_start(f);
+        if (r < 0)
+                return r;
+
+        return 0;
+}
+
+static int journal_file_hmac_put_object(JournalFile *f, int type, uint64_t p) {
+        int r;
+        Object *o;
+
+        assert(f);
+
+        if (!f->authenticate)
+                return 0;
+
+        r = journal_file_hmac_start(f);
+        if (r < 0)
+                return r;
+
+        r = journal_file_move_to_object(f, type, p, &o);
+        if (r < 0)
+                return r;
+
+        gcry_md_write(f->hmac, o, offsetof(ObjectHeader, payload));
+
+        switch (o->object.type) {
+
+        case OBJECT_DATA:
+                /* All but: hash and payload are mutable */
+                gcry_md_write(f->hmac, &o->data.hash, sizeof(o->data.hash));
+                gcry_md_write(f->hmac, o->data.payload, le64toh(o->object.size) - offsetof(DataObject, payload));
+                break;
+
+        case OBJECT_ENTRY:
+                /* All */
+                gcry_md_write(f->hmac, &o->entry.seqnum, le64toh(o->object.size) - offsetof(EntryObject, seqnum));
+                break;
+
+        case OBJECT_FIELD_HASH_TABLE:
+        case OBJECT_DATA_HASH_TABLE:
+        case OBJECT_ENTRY_ARRAY:
+                /* Nothing: everything is mutable */
+                break;
+
+        case OBJECT_TAG:
+                /* All but the tag itself */
+                gcry_md_write(f->hmac, &o->tag.seqnum, sizeof(o->tag.seqnum));
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+static int journal_file_hmac_put_header(JournalFile *f) {
+        int r;
+
+        assert(f);
+
+        if (!f->authenticate)
+                return 0;
+
+        r = journal_file_hmac_start(f);
+        if (r < 0)
+                return r;
+
+        /* All but state+reserved, boot_id, arena_size,
+         * tail_object_offset, n_objects, n_entries, tail_seqnum,
+         * head_entry_realtime, tail_entry_realtime,
+         * tail_entry_monotonic, n_data, n_fields, header_tag */
+
+        gcry_md_write(f->hmac, f->header->signature, offsetof(Header, state) - offsetof(Header, signature));
+        gcry_md_write(f->hmac, &f->header->file_id, offsetof(Header, boot_id) - offsetof(Header, file_id));
+        gcry_md_write(f->hmac, &f->header->seqnum_id, offsetof(Header, arena_size) - offsetof(Header, seqnum_id));
+        gcry_md_write(f->hmac, &f->header->data_hash_table_offset, offsetof(Header, tail_object_offset) - offsetof(Header, data_hash_table_offset));
+        gcry_md_write(f->hmac, &f->header->head_seqnum, offsetof(Header, head_entry_realtime) - offsetof(Header, head_seqnum));
+
+        return 0;
+}
+
+static int journal_file_load_fsprg(JournalFile *f) {
+        int r, fd = -1;
+        char *p = NULL;
+        struct stat st;
+        FSPRGHeader *m = NULL;
+        sd_id128_t machine;
+
+        assert(f);
+
+        if (!f->authenticate)
+                return 0;
+
+        r = sd_id128_get_machine(&machine);
+        if (r < 0)
+                return r;
+
+        if (asprintf(&p, "/var/log/journal/" SD_ID128_FORMAT_STR "/fsprg",
+                     SD_ID128_FORMAT_VAL(machine)) < 0)
+                return -ENOMEM;
+
+        fd = open(p, O_RDWR|O_CLOEXEC|O_NOCTTY, 0600);
+        if (fd < 0) {
+                log_error("Failed to open %s: %m", p);
+                r = -errno;
+                goto finish;
+        }
+
+        if (fstat(fd, &st) < 0) {
+                r = -errno;
+                goto finish;
+        }
+
+        if (st.st_size < (off_t) sizeof(FSPRGHeader)) {
+                r = -ENODATA;
+                goto finish;
+        }
+
+        m = mmap(NULL, PAGE_ALIGN(sizeof(FSPRGHeader)), PROT_READ, MAP_SHARED, fd, 0);
+        if (m == MAP_FAILED) {
+                m = NULL;
+                r = -errno;
+                goto finish;
+        }
+
+        if (memcmp(m->signature, FSPRG_HEADER_SIGNATURE, 8) != 0) {
+                r = -EBADMSG;
+                goto finish;
+        }
+
+        if (m->incompatible_flags != 0) {
+                r = -EPROTONOSUPPORT;
+                goto finish;
+        }
+
+        if (le64toh(m->header_size) < sizeof(FSPRGHeader)) {
+                r = -EBADMSG;
+                goto finish;
+        }
+
+        if (le64toh(m->state_size) != FSPRG_stateinbytes(m->secpar)) {
+                r = -EBADMSG;
+                goto finish;
+        }
+
+        f->fsprg_size = le64toh(m->header_size) + le64toh(m->state_size);
+        if ((uint64_t) st.st_size < f->fsprg_size) {
+                r = -ENODATA;
+                goto finish;
+        }
+
+        if (!sd_id128_equal(machine, m->machine_id)) {
+                r = -EHOSTDOWN;
+                goto finish;
+        }
+
+        if (le64toh(m->fsprg_start_usec) <= 0 ||
+            le64toh(m->fsprg_interval_usec) <= 0) {
+                r = -EBADMSG;
+                goto finish;
+        }
+
+        f->fsprg_header = mmap(NULL, PAGE_ALIGN(f->fsprg_size), PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+        if (f->fsprg_header == MAP_FAILED) {
+                f->fsprg_header = NULL;
+                r = -errno;
+                goto finish;
+        }
+
+        r = 0;
+
+finish:
+        if (m)
+                munmap(m, PAGE_ALIGN(sizeof(FSPRGHeader)));
+
+        if (fd >= 0)
+                close_nointr_nofail(fd);
+
+        free(p);
+        return r;
+}
+
+static int journal_file_setup_hmac(JournalFile *f) {
+        gcry_error_t e;
+
+        if (!f->authenticate)
+                return 0;
+
+        e = gcry_md_open(&f->hmac, GCRY_MD_SHA256, GCRY_MD_FLAG_HMAC);
+        if (e != 0)
+                return -ENOTSUP;
+
+        return 0;
+}
+
+static int journal_file_append_first_tag(JournalFile *f) {
+        int r;
+        uint64_t p;
+
+        if (!f->authenticate)
+                return 0;
+
+        log_debug("Calculating first tag...");
+
+        r = journal_file_hmac_put_header(f);
+        if (r < 0)
+                return r;
+
+        p = le64toh(f->header->field_hash_table_offset);
+        if (p < offsetof(Object, hash_table.items))
+                return -EINVAL;
+        p -= offsetof(Object, hash_table.items);
+
+        r = journal_file_hmac_put_object(f, OBJECT_FIELD_HASH_TABLE, p);
+        if (r < 0)
+                return r;
+
+        p = le64toh(f->header->data_hash_table_offset);
+        if (p < offsetof(Object, hash_table.items))
+                return -EINVAL;
+        p -= offsetof(Object, hash_table.items);
+
+        r = journal_file_hmac_put_object(f, OBJECT_DATA_HASH_TABLE, p);
+        if (r < 0)
+                return r;
+
+        r = journal_file_append_tag(f);
+        if (r < 0)
+                return r;
+
+        return 0;
+}
+
 void journal_file_dump(JournalFile *f) {
         Object *o;
         int r;
@@ -1875,8 +2339,9 @@ void journal_file_dump(JournalFile *f) {
                         printf("Type: OBJECT_ENTRY_ARRAY\n");
                         break;
 
-                case OBJECT_SIGNATURE:
-                        printf("Type: OBJECT_SIGNATURE\n");
+                case OBJECT_TAG:
+                        printf("Type: OBJECT_TAG %llu\n",
+                               (unsigned long long) le64toh(o->tag.seqnum));
                         break;
                 }
 
@@ -1927,8 +2392,8 @@ void journal_file_print_header(JournalFile *f) {
                f->header->state == STATE_OFFLINE ? "offline" :
                f->header->state == STATE_ONLINE ? "online" :
                f->header->state == STATE_ARCHIVED ? "archived" : "unknown",
-               (f->header->compatible_flags & HEADER_COMPATIBLE_SIGNED) ? " SIGNED" : "",
-               (f->header->compatible_flags & ~HEADER_COMPATIBLE_SIGNED) ? " ???" : "",
+               (f->header->compatible_flags & HEADER_COMPATIBLE_AUTHENTICATED) ? " AUTHENTICATED" : "",
+               (f->header->compatible_flags & ~HEADER_COMPATIBLE_AUTHENTICATED) ? " ???" : "",
                (f->header->incompatible_flags & HEADER_INCOMPATIBLE_COMPRESSED) ? " COMPRESSED" : "",
                (f->header->incompatible_flags & ~HEADER_INCOMPATIBLE_COMPRESSED) ? " ???" : "",
                (unsigned long long) le64toh(f->header->header_size),
@@ -1960,6 +2425,8 @@ int journal_file_open(
                 const char *fname,
                 int flags,
                 mode_t mode,
+                bool compress,
+                bool authenticate,
                 JournalMetrics *metrics,
                 JournalFile *template,
                 JournalFile **ret) {
@@ -1982,13 +2449,13 @@ int journal_file_open(
                 return -ENOMEM;
 
         f->fd = -1;
-        f->flags = flags;
         f->mode = mode;
-        f->writable = (flags & O_ACCMODE) != O_RDONLY;
-        f->prot = prot_from_flags(flags);
 
-        if (template)
-                f->compress = template->compress;
+        f->flags = flags;
+        f->prot = prot_from_flags(flags);
+        f->writable = (flags & O_ACCMODE) != O_RDONLY;
+        f->compress = compress;
+        f->authenticate = authenticate;
 
         f->path = strdup(fname);
         if (!f->path) {
@@ -2010,6 +2477,12 @@ int journal_file_open(
         if (f->last_stat.st_size == 0 && f->writable) {
                 newly_created = true;
 
+                /* Try to load the FSPRG state, and if we can't, then
+                 * just don't do authentication */
+                r = journal_file_load_fsprg(f);
+                if (r < 0)
+                        f->authenticate = false;
+
                 r = journal_file_init_header(f, template);
                 if (r < 0)
                         goto fail;
@@ -2038,6 +2511,12 @@ int journal_file_open(
                         goto fail;
         }
 
+        if (!newly_created && f->writable) {
+                r = journal_file_load_fsprg(f);
+                if (r < 0)
+                        goto fail;
+        }
+
         if (f->writable) {
                 if (metrics) {
                         journal_default_metrics(metrics, f->fd);
@@ -2048,10 +2527,13 @@ int journal_file_open(
                 r = journal_file_refresh_header(f);
                 if (r < 0)
                         goto fail;
+
+                r = journal_file_setup_hmac(f);
+                if (r < 0)
+                        goto fail;
         }
 
         if (newly_created) {
-
                 r = journal_file_setup_field_hash_table(f);
                 if (r < 0)
                         goto fail;
@@ -2059,6 +2541,10 @@ int journal_file_open(
                 r = journal_file_setup_data_hash_table(f);
                 if (r < 0)
                         goto fail;
+
+                r = journal_file_append_first_tag(f);
+                if (r < 0)
+                        goto fail;
         }
 
         r = journal_file_map_field_hash_table(f);
@@ -2080,7 +2566,7 @@ fail:
         return r;
 }
 
-int journal_file_rotate(JournalFile **f) {
+int journal_file_rotate(JournalFile **f, bool compress, bool authenticate) {
         char *p;
         size_t l;
         JournalFile *old_file, *new_file = NULL;
@@ -2119,7 +2605,7 @@ int journal_file_rotate(JournalFile **f) {
 
         old_file->header->state = STATE_ARCHIVED;
 
-        r = journal_file_open(old_file->path, old_file->flags, old_file->mode, NULL, old_file, &new_file);
+        r = journal_file_open(old_file->path, old_file->flags, old_file->mode, compress, authenticate, NULL, old_file, &new_file);
         journal_file_close(old_file);
 
         *f = new_file;
@@ -2130,6 +2616,8 @@ int journal_file_open_reliably(
                 const char *fname,
                 int flags,
                 mode_t mode,
+                bool compress,
+                bool authenticate,
                 JournalMetrics *metrics,
                 JournalFile *template,
                 JournalFile **ret) {
@@ -2138,11 +2626,13 @@ int journal_file_open_reliably(
         size_t l;
         char *p;
 
-        r = journal_file_open(fname, flags, mode, metrics, template, ret);
+        r = journal_file_open(fname, flags, mode, compress, authenticate, metrics, template, ret);
         if (r != -EBADMSG && /* corrupted */
             r != -ENODATA && /* truncated */
             r != -EHOSTDOWN && /* other machine */
-            r != -EPROTONOSUPPORT) /* incompatible feature */
+            r != -EPROTONOSUPPORT && /* incompatible feature */
+            r != -EBUSY && /* unclean shutdown */
+            r != -ESHUTDOWN /* already archived */)
                 return r;
 
         if ((flags & O_ACCMODE) == O_RDONLY)
@@ -2151,6 +2641,9 @@ int journal_file_open_reliably(
         if (!(flags & O_CREAT))
                 return r;
 
+        if (!endswith(fname, ".journal"))
+                return r;
+
         /* The file is corrupted. Rotate it away and try it again (but only once) */
 
         l = strlen(fname);
@@ -2165,9 +2658,9 @@ int journal_file_open_reliably(
         if (r < 0)
                 return -errno;
 
-        log_warning("File %s corrupted, renaming and replacing.", fname);
+        log_warning("File %s corrupted or uncleanly shut down, renaming and replacing.", fname);
 
-        return journal_file_open(fname, flags, mode, metrics, template, ret);
+        return journal_file_open(fname, flags, mode, compress, authenticate, metrics, template, ret);
 }
 
 struct vacuum_info {
@@ -2334,7 +2827,8 @@ int journal_directory_vacuum(const char *directory, uint64_t max_use, uint64_t m
                 n_list ++;
         }
 
-        qsort(list, n_list, sizeof(struct vacuum_info), vacuum_compare);
+        if (n_list > 0)
+                qsort(list, n_list, sizeof(struct vacuum_info), vacuum_compare);
 
         for(i = 0; i < n_list; i++) {
                 struct statvfs ss;
@@ -2593,8 +3087,10 @@ bool journal_file_rotate_suggested(JournalFile *f) {
 
         /* If we gained new header fields we gained new features,
          * hence suggest a rotation */
-        if (le64toh(f->header->header_size) < sizeof(Header))
+        if (le64toh(f->header->header_size) < sizeof(Header)) {
+                log_debug("%s uses an outdated header, suggesting rotation.", f->path);
                 return true;
+        }
 
         /* Let's check if the hash tables grew over a certain fill
          * level (75%, borrowing this value from Java's hash table
@@ -2603,12 +3099,26 @@ bool journal_file_rotate_suggested(JournalFile *f) {
          * in newer versions. */
 
         if (JOURNAL_HEADER_CONTAINS(f->header, n_data))
-                if (le64toh(f->header->n_data) * 4ULL > (le64toh(f->header->data_hash_table_size) / sizeof(HashItem)) * 3ULL)
+                if (le64toh(f->header->n_data) * 4ULL > (le64toh(f->header->data_hash_table_size) / sizeof(HashItem)) * 3ULL) {
+                        log_debug("Data hash table of %s has a fill level at %.1f (%llu of %llu items, %llu file size, %llu bytes per hash table item), suggesting rotation.",
+                                  f->path,
+                                  100.0 * (double) le64toh(f->header->n_data) / ((double) (le64toh(f->header->data_hash_table_size) / sizeof(HashItem))),
+                                  (unsigned long long) le64toh(f->header->n_data),
+                                  (unsigned long long) (le64toh(f->header->data_hash_table_size) / sizeof(HashItem)),
+                                  (unsigned long long) (f->last_stat.st_size),
+                                  (unsigned long long) (f->last_stat.st_size / le64toh(f->header->n_data)));
                         return true;
+                }
 
         if (JOURNAL_HEADER_CONTAINS(f->header, n_fields))
-                if (le64toh(f->header->n_fields) * 4ULL > (le64toh(f->header->field_hash_table_size) / sizeof(HashItem)) * 3ULL)
+                if (le64toh(f->header->n_fields) * 4ULL > (le64toh(f->header->field_hash_table_size) / sizeof(HashItem)) * 3ULL) {
+                        log_debug("Field hash table of %s has a fill level at %.1f (%llu of %llu items), suggesting rotation.",
+                                  f->path,
+                                  100.0 * (double) le64toh(f->header->n_fields) / ((double) (le64toh(f->header->field_hash_table_size) / sizeof(HashItem))),
+                                  (unsigned long long) le64toh(f->header->n_fields),
+                                  (unsigned long long) (le64toh(f->header->field_hash_table_size) / sizeof(HashItem)));
                         return true;
+                }
 
         return false;
 }