journal: size journal data hash table based on maximum file size metrics

The default of 2047 hash table entries turned out to result in way too
many collisions for bigger files, hence scale the hash table size by the
estimated maximum file size.

diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c
index 9d8c9e686d3ec186e29d859a5a435afb85725d16..04564d989e4f23621016d3697d2889e500d1b4af 100644 (file)
--- a/src/journal/journal-file.c
+++ b/src/journal/journal-file.c
@@ -32,8 +32,8 @@
 #include "lookup3.h"
 #include "compress.h"
 
-#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*16ULL)
-#define DEFAULT_FIELD_HASH_TABLE_SIZE (2047ULL*16ULL)
+#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem))
+#define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem))
 
 #define DEFAULT_WINDOW_SIZE (8ULL*1024ULL*1024ULL)
 
@@ -509,7 +509,17 @@ static int journal_file_setup_data_hash_table(JournalFile *f) {
 
         assert(f);
 
-        s = DEFAULT_DATA_HASH_TABLE_SIZE;
+        /* We estimate that we need 1 hash table entry per 2K of
+           journal file and we want to make sure we never get beyond
+           75% fill level. Calculate the hash table size for the
+           maximum file size based on these metrics. */
+
+        s = (f->metrics.max_size * 4 / 2048 / 3) * sizeof(HashItem);
+        if (s < DEFAULT_DATA_HASH_TABLE_SIZE)
+                s = DEFAULT_DATA_HASH_TABLE_SIZE;
+
+        log_info("Reserving %llu entries in hash table.", (unsigned long long) s);
+
         r = journal_file_append_object(f,
                                        OBJECT_DATA_HASH_TABLE,
                                        offsetof(Object, hash_table.items) + s,
@@ -1939,6 +1949,7 @@ int journal_file_open(
                 const char *fname,
                 int flags,
                 mode_t mode,
+                JournalMetrics *metrics,
                 JournalFile *template,
                 JournalFile **ret) {
 
@@ -1965,10 +1976,8 @@ int journal_file_open(
         f->writable = (flags & O_ACCMODE) != O_RDONLY;
         f->prot = prot_from_flags(flags);
 
-        if (template) {
-                f->metrics = template->metrics;
+        if (template)
                 f->compress = template->compress;
-        }
 
         f->path = strdup(fname);
         if (!f->path) {
@@ -2019,6 +2028,12 @@ int journal_file_open(
         }
 
         if (f->writable) {
+                if (metrics) {
+                        journal_default_metrics(metrics, f->fd);
+                        f->metrics = *metrics;
+                } else if (template)
+                        f->metrics = template->metrics;
+
                 r = journal_file_refresh_header(f);
                 if (r < 0)
                         goto fail;
@@ -2093,7 +2108,7 @@ int journal_file_rotate(JournalFile **f) {
 
         old_file->header->state = STATE_ARCHIVED;
 
-        r = journal_file_open(old_file->path, old_file->flags, old_file->mode, old_file, &new_file);
+        r = journal_file_open(old_file->path, old_file->flags, old_file->mode, NULL, old_file, &new_file);
         journal_file_close(old_file);
 
         *f = new_file;
@@ -2104,6 +2119,7 @@ int journal_file_open_reliably(
                 const char *fname,
                 int flags,
                 mode_t mode,
+                JournalMetrics *metrics,
                 JournalFile *template,
                 JournalFile **ret) {
 
@@ -2111,7 +2127,7 @@ int journal_file_open_reliably(
         size_t l;
         char *p;
 
-        r = journal_file_open(fname, flags, mode, template, ret);
+        r = journal_file_open(fname, flags, mode, metrics, template, ret);
         if (r != -EBADMSG && /* corrupted */
             r != -ENODATA && /* truncated */
             r != -EHOSTDOWN && /* other machine */
@@ -2140,7 +2156,7 @@ int journal_file_open_reliably(
 
         log_warning("File %s corrupted, renaming and replacing.", fname);
 
-        return journal_file_open(fname, flags, mode, template, ret);
+        return journal_file_open(fname, flags, mode, metrics, template, ret);
 }
 
 struct vacuum_info {

diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h
index 62197429ae01caa6408ea7162ddd26642566870e..1f7f73a78db9b7eacdc782713b35eaf721a93d2a 100644 (file)
--- a/src/journal/journal-file.h
+++ b/src/journal/journal-file.h
@@ -88,10 +88,23 @@ typedef enum direction {
         DIRECTION_DOWN
 } direction_t;
 
-int journal_file_open(const char *fname, int flags, mode_t mode, JournalFile *template, JournalFile **ret);
+int journal_file_open(
+                const char *fname,
+                int flags,
+                mode_t mode,
+                JournalMetrics *metrics,
+                JournalFile *template,
+                JournalFile **ret);
+
 void journal_file_close(JournalFile *j);
 
-int journal_file_open_reliably(const char *fname, int flags, mode_t mode, JournalFile *template, JournalFile **ret);
+int journal_file_open_reliably(
+                const char *fname,
+                int flags,
+                mode_t mode,
+                JournalMetrics *metrics,
+                JournalFile *template,
+                JournalFile **ret);
 
 int journal_file_move_to_object(JournalFile *f, int type, uint64_t offset, Object **ret);
 

diff --git a/src/journal/journald.c b/src/journal/journald.c
index 8f00aef5bd41a313994b65629677d57bd5fccd9a..d8cfa66b75dfb10e7bb51e2927a25386e2eda867 100644 (file)
--- a/src/journal/journald.c
+++ b/src/journal/journald.c
@@ -313,7 +313,7 @@ static JournalFile* find_journal(Server *s, uid_t uid) {
                 journal_file_close(f);
         }
 
-        r = journal_file_open_reliably(p, O_RDWR|O_CREAT, 0640, s->system_journal, &f);
+        r = journal_file_open_reliably(p, O_RDWR|O_CREAT, 0640, &s->system_metrics, s->system_journal, &f);
         free(p);
 
         if (r < 0)
@@ -2005,13 +2005,10 @@ static int system_journal_open(Server *s) {
                 if (!fn)
                         return -ENOMEM;
 
-                r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, NULL, &s->system_journal);
+                r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, &s->system_metrics, NULL, &s->system_journal);
                 free(fn);
 
                 if (r >= 0) {
-                        journal_default_metrics(&s->system_metrics, s->system_journal->fd);
-
-                        s->system_journal->metrics = s->system_metrics;
                         s->system_journal->compress = s->compress;
 
                         server_fix_perms(s, s->system_journal, 0);
@@ -2037,7 +2034,7 @@ static int system_journal_open(Server *s) {
                          * if it already exists, so that we can flush
                          * it into the system journal */
 
-                        r = journal_file_open(fn, O_RDWR, 0640, NULL, &s->runtime_journal);
+                        r = journal_file_open(fn, O_RDWR, 0640, &s->runtime_metrics, NULL, &s->runtime_journal);
                         free(fn);
 
                         if (r < 0) {
@@ -2053,7 +2050,7 @@ static int system_journal_open(Server *s) {
                          * it if necessary. */
 
                         (void) mkdir_parents(fn, 0755);
-                        r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, NULL, &s->runtime_journal);
+                        r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, &s->runtime_metrics, NULL, &s->runtime_journal);
                         free(fn);
 
                         if (r < 0) {
@@ -2063,9 +2060,6 @@ static int system_journal_open(Server *s) {
                 }
 
                 if (s->runtime_journal) {
-                        journal_default_metrics(&s->runtime_metrics, s->runtime_journal->fd);
-
-                        s->runtime_journal->metrics = s->runtime_metrics;
                         s->runtime_journal->compress = s->compress;
 
                         server_fix_perms(s, s->runtime_journal, 0);

diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index 77469e9ebd342319cc7118132d360c0471424b3c..497f79cb525a33ae0b195fd3da3e1ab20f440eed 100644 (file)
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -1116,7 +1116,7 @@ static int add_file(sd_journal *j, const char *prefix, const char *filename) {
                 return 0;
         }
 
-        r = journal_file_open(path, O_RDONLY, 0, NULL, &f);
+        r = journal_file_open(path, O_RDONLY, 0, NULL, NULL, &f);
         free(path);
 
         if (r < 0) {

diff --git a/src/journal/test-journal-stream.c b/src/journal/test-journal-stream.c
index 313606f9be63faa4f8cf3af749a98903b4cdae07..627faa7eb56d0e756ee3d29e7ac01b0d41ce3389 100644 (file)
--- a/src/journal/test-journal-stream.c
+++ b/src/journal/test-journal-stream.c
@@ -79,9 +79,9 @@ int main(int argc, char *argv[]) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open("one.journal", O_RDWR|O_CREAT, 0666, NULL, &one) == 0);
-        assert_se(journal_file_open("two.journal", O_RDWR|O_CREAT, 0666, NULL, &two) == 0);
-        assert_se(journal_file_open("three.journal", O_RDWR|O_CREAT, 0666, NULL, &three) == 0);
+        assert_se(journal_file_open("one.journal", O_RDWR|O_CREAT, 0666, NULL, NULL, &one) == 0);
+        assert_se(journal_file_open("two.journal", O_RDWR|O_CREAT, 0666, NULL, NULL, &two) == 0);
+        assert_se(journal_file_open("three.journal", O_RDWR|O_CREAT, 0666, NULL, NULL, &three) == 0);
 
         for (i = 0; i < N_ENTRIES; i++) {
                 char *p, *q;

diff --git a/src/journal/test-journal.c b/src/journal/test-journal.c
index 9e1a4f57baba4cf2e74da66ec8e8db6ff5a693da..7271a3facbd2ff2e44f7017d6286842394d84c69 100644 (file)
--- a/src/journal/test-journal.c
+++ b/src/journal/test-journal.c
@@ -41,7 +41,7 @@ int main(int argc, char *argv[]) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, NULL, &f) == 0);
+        assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, NULL, NULL, &f) == 0);
 
         dual_timestamp_get(&ts);