The default of 2047 hash table entries turned out to result in way too
many collisions for bigger files, hence scale the hash table size by the
estimated maximum file size.
#include "lookup3.h"
#include "compress.h"
#include "lookup3.h"
#include "compress.h"
-#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*16ULL)
-#define DEFAULT_FIELD_HASH_TABLE_SIZE (2047ULL*16ULL)
+#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem))
+#define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem))
#define DEFAULT_WINDOW_SIZE (8ULL*1024ULL*1024ULL)
#define DEFAULT_WINDOW_SIZE (8ULL*1024ULL*1024ULL)
- s = DEFAULT_DATA_HASH_TABLE_SIZE;
+ /* We estimate that we need 1 hash table entry per 2K of
+ journal file and we want to make sure we never get beyond
+ 75% fill level. Calculate the hash table size for the
+ maximum file size based on these metrics. */
+
+ s = (f->metrics.max_size * 4 / 2048 / 3) * sizeof(HashItem);
+ if (s < DEFAULT_DATA_HASH_TABLE_SIZE)
+ s = DEFAULT_DATA_HASH_TABLE_SIZE;
+
+ log_info("Reserving %llu entries in hash table.", (unsigned long long) s);
+
r = journal_file_append_object(f,
OBJECT_DATA_HASH_TABLE,
offsetof(Object, hash_table.items) + s,
r = journal_file_append_object(f,
OBJECT_DATA_HASH_TABLE,
offsetof(Object, hash_table.items) + s,
const char *fname,
int flags,
mode_t mode,
const char *fname,
int flags,
mode_t mode,
+ JournalMetrics *metrics,
JournalFile *template,
JournalFile **ret) {
JournalFile *template,
JournalFile **ret) {
f->writable = (flags & O_ACCMODE) != O_RDONLY;
f->prot = prot_from_flags(flags);
f->writable = (flags & O_ACCMODE) != O_RDONLY;
f->prot = prot_from_flags(flags);
- if (template) {
- f->metrics = template->metrics;
f->compress = template->compress;
f->compress = template->compress;
f->path = strdup(fname);
if (!f->path) {
f->path = strdup(fname);
if (!f->path) {
+ if (metrics) {
+ journal_default_metrics(metrics, f->fd);
+ f->metrics = *metrics;
+ } else if (template)
+ f->metrics = template->metrics;
+
r = journal_file_refresh_header(f);
if (r < 0)
goto fail;
r = journal_file_refresh_header(f);
if (r < 0)
goto fail;
old_file->header->state = STATE_ARCHIVED;
old_file->header->state = STATE_ARCHIVED;
- r = journal_file_open(old_file->path, old_file->flags, old_file->mode, old_file, &new_file);
+ r = journal_file_open(old_file->path, old_file->flags, old_file->mode, NULL, old_file, &new_file);
journal_file_close(old_file);
*f = new_file;
journal_file_close(old_file);
*f = new_file;
const char *fname,
int flags,
mode_t mode,
const char *fname,
int flags,
mode_t mode,
+ JournalMetrics *metrics,
JournalFile *template,
JournalFile **ret) {
JournalFile *template,
JournalFile **ret) {
- r = journal_file_open(fname, flags, mode, template, ret);
+ r = journal_file_open(fname, flags, mode, metrics, template, ret);
if (r != -EBADMSG && /* corrupted */
r != -ENODATA && /* truncated */
r != -EHOSTDOWN && /* other machine */
if (r != -EBADMSG && /* corrupted */
r != -ENODATA && /* truncated */
r != -EHOSTDOWN && /* other machine */
log_warning("File %s corrupted, renaming and replacing.", fname);
log_warning("File %s corrupted, renaming and replacing.", fname);
- return journal_file_open(fname, flags, mode, template, ret);
+ return journal_file_open(fname, flags, mode, metrics, template, ret);
DIRECTION_DOWN
} direction_t;
DIRECTION_DOWN
} direction_t;
-int journal_file_open(const char *fname, int flags, mode_t mode, JournalFile *template, JournalFile **ret);
+int journal_file_open(
+ const char *fname,
+ int flags,
+ mode_t mode,
+ JournalMetrics *metrics,
+ JournalFile *template,
+ JournalFile **ret);
+
void journal_file_close(JournalFile *j);
void journal_file_close(JournalFile *j);
-int journal_file_open_reliably(const char *fname, int flags, mode_t mode, JournalFile *template, JournalFile **ret);
+int journal_file_open_reliably(
+ const char *fname,
+ int flags,
+ mode_t mode,
+ JournalMetrics *metrics,
+ JournalFile *template,
+ JournalFile **ret);
int journal_file_move_to_object(JournalFile *f, int type, uint64_t offset, Object **ret);
int journal_file_move_to_object(JournalFile *f, int type, uint64_t offset, Object **ret);
- r = journal_file_open_reliably(p, O_RDWR|O_CREAT, 0640, s->system_journal, &f);
+ r = journal_file_open_reliably(p, O_RDWR|O_CREAT, 0640, &s->system_metrics, s->system_journal, &f);
- r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, NULL, &s->system_journal);
+ r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, &s->system_metrics, NULL, &s->system_journal);
- journal_default_metrics(&s->system_metrics, s->system_journal->fd);
-
- s->system_journal->metrics = s->system_metrics;
s->system_journal->compress = s->compress;
server_fix_perms(s, s->system_journal, 0);
s->system_journal->compress = s->compress;
server_fix_perms(s, s->system_journal, 0);
* if it already exists, so that we can flush
* it into the system journal */
* if it already exists, so that we can flush
* it into the system journal */
- r = journal_file_open(fn, O_RDWR, 0640, NULL, &s->runtime_journal);
+ r = journal_file_open(fn, O_RDWR, 0640, &s->runtime_metrics, NULL, &s->runtime_journal);
* it if necessary. */
(void) mkdir_parents(fn, 0755);
* it if necessary. */
(void) mkdir_parents(fn, 0755);
- r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, NULL, &s->runtime_journal);
+ r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, &s->runtime_metrics, NULL, &s->runtime_journal);
}
if (s->runtime_journal) {
}
if (s->runtime_journal) {
- journal_default_metrics(&s->runtime_metrics, s->runtime_journal->fd);
-
- s->runtime_journal->metrics = s->runtime_metrics;
s->runtime_journal->compress = s->compress;
server_fix_perms(s, s->runtime_journal, 0);
s->runtime_journal->compress = s->compress;
server_fix_perms(s, s->runtime_journal, 0);
- r = journal_file_open(path, O_RDONLY, 0, NULL, &f);
+ r = journal_file_open(path, O_RDONLY, 0, NULL, NULL, &f);
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
- assert_se(journal_file_open("one.journal", O_RDWR|O_CREAT, 0666, NULL, &one) == 0);
- assert_se(journal_file_open("two.journal", O_RDWR|O_CREAT, 0666, NULL, &two) == 0);
- assert_se(journal_file_open("three.journal", O_RDWR|O_CREAT, 0666, NULL, &three) == 0);
+ assert_se(journal_file_open("one.journal", O_RDWR|O_CREAT, 0666, NULL, NULL, &one) == 0);
+ assert_se(journal_file_open("two.journal", O_RDWR|O_CREAT, 0666, NULL, NULL, &two) == 0);
+ assert_se(journal_file_open("three.journal", O_RDWR|O_CREAT, 0666, NULL, NULL, &three) == 0);
for (i = 0; i < N_ENTRIES; i++) {
char *p, *q;
for (i = 0; i < N_ENTRIES; i++) {
char *p, *q;
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
- assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, NULL, &f) == 0);
+ assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, NULL, NULL, &f) == 0);