X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=blobdiff_plain;f=src%2Fjournal%2Fmmap-cache.c;h=ab21cdc28873e36ac88465336252caa44cef4a4d;hb=c7abe32be167ded048a5bb8a912faf2e745b20cb;hp=69efb20adea08f95e39098da596eb199b4e25fa8;hpb=84168d8068bb67dcd5468ab3b376535d81643aef;p=elogind.git
diff --git a/src/journal/mmap-cache.c b/src/journal/mmap-cache.c
index 69efb20ad..ab21cdc28 100644
--- a/src/journal/mmap-cache.c
+++ b/src/journal/mmap-cache.c
@@ -19,354 +19,464 @@
along with systemd; If not, see .
***/
-#include
-#include
#include
#include
+#include
#include
+#include "hashmap.h"
+#include "list.h"
+#include "log.h"
#include "util.h"
-
+#include "macro.h"
+#include "sigbus.h"
#include "mmap-cache.h"
-#define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
+typedef struct Window Window;
+typedef struct Context Context;
+typedef struct FileDescriptor FileDescriptor;
-#define DEFAULT_WINDOWS_MAX 64
-#define DEFAULT_FDS_MAX 32
-#define DEFAULT_CONTEXTS_MAX 32
+struct Window {
+ MMapCache *cache;
-typedef struct Window {
- int fd;
+ bool invalidated;
+ bool keep_always;
+ bool in_unused;
+
+ int prot;
void *ptr;
uint64_t offset;
- uint64_t size;
+ size_t size;
+
+ FileDescriptor *fd;
+
+ LIST_FIELDS(Window, by_fd);
+ LIST_FIELDS(Window, unused);
+
+ LIST_HEAD(Context, contexts);
+};
- unsigned n_ref;
- unsigned lru_prev;
- unsigned lru_next;
+struct Context {
+ MMapCache *cache;
+ unsigned id;
+ Window *window;
- unsigned by_fd_prev;
- unsigned by_fd_next;
-} Window;
+ LIST_FIELDS(Context, by_window);
+};
-typedef struct FileDescriptor {
+struct FileDescriptor {
+ MMapCache *cache;
int fd;
- unsigned windows;
-} FileDescriptor;
+ bool sigbus;
+ LIST_HEAD(Window, windows);
+};
struct MMapCache {
- unsigned n_ref;
+ int n_ref;
+ unsigned n_windows;
- unsigned contexts_max;
- unsigned windows_max;
- unsigned fds_max;
+ unsigned n_hit, n_missed;
- unsigned n_windows;
- unsigned n_fds;
- unsigned lru_first, lru_last;
+ Hashmap *fds;
+ Context *contexts[MMAP_CACHE_MAX_CONTEXTS];
- Window *windows;
- unsigned *by_context;
- FileDescriptor *by_fd;
+ LIST_HEAD(Window, unused);
+ Window *last_unused;
};
-static int mmap_cache_peek_fd_index(MMapCache *m, int fd, unsigned *fd_index);
+#define WINDOWS_MIN 64
-static void mmap_cache_window_unmap(MMapCache *m, unsigned w) {
- Window *v;
+#ifdef ENABLE_DEBUG_MMAP_CACHE
+/* Tiny windows increase mmap activity and the chance of exposing unsafe use. */
+# define WINDOW_SIZE (page_size())
+#else
+# define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
+#endif
- assert(m);
- assert(w < m->n_windows);
+MMapCache* mmap_cache_new(void) {
+ MMapCache *m;
- v = m->windows + w;
- if (!v->ptr)
- return;
+ m = new0(MMapCache, 1);
+ if (!m)
+ return NULL;
- munmap(v->ptr, v->size);
- v->ptr = NULL;
+ m->n_ref = 1;
+ return m;
}
-static void mmap_cache_window_add_lru(MMapCache *m, unsigned w) {
- Window *v;
-
+MMapCache* mmap_cache_ref(MMapCache *m) {
assert(m);
- assert(w < m->n_windows);
-
- v = m->windows + w;
- assert(v->n_ref == 0);
+ assert(m->n_ref > 0);
- if (m->lru_last != (unsigned) -1) {
- assert(m->windows[m->lru_last].lru_next == (unsigned) -1);
- m->windows[m->lru_last].lru_next = w;
- }
+ m->n_ref ++;
+ return m;
+}
- v->lru_prev = m->lru_last;
- v->lru_next = (unsigned) -1;
+static void window_unlink(Window *w) {
+ Context *c;
- m->lru_last = w;
- if (m->lru_first == (unsigned) -1)
- m->lru_first = w;
-}
+ assert(w);
-static void mmap_cache_window_remove_lru(MMapCache *m, unsigned w) {
- Window *v;
+ if (w->ptr)
+ munmap(w->ptr, w->size);
- assert(m);
- assert(w < m->n_windows);
+ if (w->fd)
+ LIST_REMOVE(by_fd, w->fd->windows, w);
- v = m->windows + w;
+ if (w->in_unused) {
+ if (w->cache->last_unused == w)
+ w->cache->last_unused = w->unused_prev;
- if (v->lru_prev == (unsigned) -1) {
- assert(m->lru_first == w);
- m->lru_first = v->lru_next;
- } else {
- assert(m->windows[v->lru_prev].lru_next == w);
- m->windows[v->lru_prev].lru_next = v->lru_next;
+ LIST_REMOVE(unused, w->cache->unused, w);
}
- if (v->lru_next == (unsigned) -1) {
- assert(m->lru_last == w);
- m->lru_last = v->lru_prev;
- } else {
- assert(m->windows[v->lru_next].lru_prev == w);
- m->windows[v->lru_next].lru_prev = v->lru_prev;
+ LIST_FOREACH(by_window, c, w->contexts) {
+ assert(c->window == w);
+ c->window = NULL;
}
}
-static void mmap_cache_fd_add(MMapCache *m, unsigned fd_index, unsigned w) {
- Window *v;
+static void window_invalidate(Window *w) {
+ assert(w);
- assert(m);
- assert(fd_index < m->n_fds);
+ if (w->invalidated)
+ return;
- v = m->windows + w;
- assert(m->by_fd[fd_index].fd == v->fd);
+ /* Replace the window with anonymous pages. This is useful
+ * when we hit a SIGBUS and want to make sure the file cannot
+ * trigger any further SIGBUS, possibly overrunning the sigbus
+ * queue. */
- if (m->by_fd[fd_index].windows != (unsigned) -1) {
- assert(m->windows[m->by_fd[fd_index].windows].by_fd_prev == (unsigned) -1);
- m->windows[m->by_fd[fd_index].windows].by_fd_prev = w;
- }
+ assert_se(mmap(w->ptr, w->size, w->prot, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0) == w->ptr);
+ w->invalidated = true;
+}
- v->by_fd_next = m->by_fd[fd_index].windows;
- v->by_fd_prev = (unsigned) -1;
+static void window_free(Window *w) {
+ assert(w);
- m->by_fd[fd_index].windows = w;
+ window_unlink(w);
+ w->cache->n_windows--;
+ free(w);
}
-static void mmap_cache_fd_remove(MMapCache *m, unsigned fd_index, unsigned w) {
- Window *v;
+_pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
+ assert(w);
+ assert(fd >= 0);
+ assert(size > 0);
+
+ return
+ w->fd &&
+ fd == w->fd->fd &&
+ prot == w->prot &&
+ offset >= w->offset &&
+ offset + size <= w->offset + w->size;
+}
+
+static Window *window_add(MMapCache *m) {
+ Window *w;
assert(m);
- assert(fd_index < m->n_fds);
- v = m->windows + w;
- assert(m->by_fd[fd_index].fd == v->fd);
- assert(v->by_fd_next == (unsigned) -1 || m->windows[v->by_fd_next].fd == v->fd);
- assert(v->by_fd_prev == (unsigned) -1 || m->windows[v->by_fd_prev].fd == v->fd);
+ if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
- if (v->by_fd_prev == (unsigned) -1) {
- assert(m->by_fd[fd_index].windows == w);
- m->by_fd[fd_index].windows = v->by_fd_next;
+ /* Allocate a new window */
+ w = new0(Window, 1);
+ if (!w)
+ return NULL;
+ m->n_windows++;
} else {
- assert(m->windows[v->by_fd_prev].by_fd_next == w);
- m->windows[v->by_fd_prev].by_fd_next = v->by_fd_next;
- }
- if (v->by_fd_next != (unsigned) -1) {
- assert(m->windows[v->by_fd_next].by_fd_prev == w);
- m->windows[v->by_fd_next].by_fd_prev = v->by_fd_prev;
+ /* Reuse an existing one */
+ w = m->last_unused;
+ window_unlink(w);
+ zero(*w);
}
+
+ w->cache = m;
+ return w;
}
-static void mmap_cache_context_unset(MMapCache *m, unsigned c) {
- Window *v;
- unsigned w;
+static void context_detach_window(Context *c) {
+ Window *w;
- assert(m);
- assert(c < m->contexts_max);
+ assert(c);
+
+ if (!c->window)
+ return;
- if (m->by_context[c] == (unsigned) -1)
+ w = c->window;
+ c->window = NULL;
+ LIST_REMOVE(by_window, w->contexts, c);
+
+ if (!w->contexts && !w->keep_always) {
+ /* Not used anymore? */
+#ifdef ENABLE_DEBUG_MMAP_CACHE
+ /* Unmap unused windows immediately to expose use-after-unmap
+ * by SIGSEGV. */
+ window_free(w);
+#else
+ LIST_PREPEND(unused, c->cache->unused, w);
+ if (!c->cache->last_unused)
+ c->cache->last_unused = w;
+
+ w->in_unused = true;
+#endif
+ }
+}
+
+static void context_attach_window(Context *c, Window *w) {
+ assert(c);
+ assert(w);
+
+ if (c->window == w)
return;
- w = m->by_context[c];
- m->by_context[c] = (unsigned) -1;
+ context_detach_window(c);
- v = m->windows + w;
- assert(v->n_ref > 0);
- v->n_ref --;
+ if (w->in_unused) {
+ /* Used again? */
+ LIST_REMOVE(unused, c->cache->unused, w);
+ if (c->cache->last_unused == w)
+ c->cache->last_unused = w->unused_prev;
- if (v->n_ref == 0)
- mmap_cache_window_add_lru(m, w);
+ w->in_unused = false;
+ }
+
+ c->window = w;
+ LIST_PREPEND(by_window, w->contexts, c);
}
-static void mmap_cache_context_set(MMapCache *m, unsigned c, unsigned w) {
- Window *v;
+static Context *context_add(MMapCache *m, unsigned id) {
+ Context *c;
assert(m);
- assert(c < m->contexts_max);
- assert(w < m->n_windows);
- if (m->by_context[c] == w)
- return;
+ c = m->contexts[id];
+ if (c)
+ return c;
- mmap_cache_context_unset(m, c);
+ c = new0(Context, 1);
+ if (!c)
+ return NULL;
- m->by_context[c] = w;
+ c->cache = m;
+ c->id = id;
- v = m->windows + w;
- v->n_ref ++;
+ assert(!m->contexts[id]);
+ m->contexts[id] = c;
- if (v->n_ref == 1)
- mmap_cache_window_remove_lru(m, w);
+ return c;
}
-static void mmap_cache_free(MMapCache *m) {
+static void context_free(Context *c) {
+ assert(c);
- assert(m);
+ context_detach_window(c);
+
+ if (c->cache) {
+ assert(c->cache->contexts[c->id] == c);
+ c->cache->contexts[c->id] = NULL;
+ }
- if (m->windows) {
- unsigned w;
+ free(c);
+}
- for (w = 0; w < m->n_windows; w++)
- mmap_cache_window_unmap(m, w);
+static void fd_free(FileDescriptor *f) {
+ assert(f);
- free(m->windows);
- }
+ while (f->windows)
+ window_free(f->windows);
- free(m->by_context);
- free(m->by_fd);
- free(m);
+ if (f->cache)
+ assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1)));
+
+ free(f);
}
-MMapCache* mmap_cache_new(void) {
- MMapCache *m;
+static FileDescriptor* fd_add(MMapCache *m, int fd) {
+ FileDescriptor *f;
+ int r;
- m = new0(MMapCache, 1);
- if (!m)
- return NULL;
+ assert(m);
+ assert(fd >= 0);
- m->contexts_max = DEFAULT_CONTEXTS_MAX;
- m->fds_max = DEFAULT_FDS_MAX;
- m->windows_max = DEFAULT_WINDOWS_MAX;
- m->n_ref = 1;
- m->lru_first = (unsigned) -1;
- m->lru_last = (unsigned) -1;
+ f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
+ if (f)
+ return f;
- m->windows = new(Window, m->windows_max);
- if (!m->windows) {
- mmap_cache_free(m);
+ r = hashmap_ensure_allocated(&m->fds, NULL);
+ if (r < 0)
return NULL;
- }
- m->by_context = new(unsigned, m->contexts_max);
- if (!m->by_context) {
- mmap_cache_free(m);
+ f = new0(FileDescriptor, 1);
+ if (!f)
return NULL;
- }
- memset(m->by_context, -1, m->contexts_max * sizeof(unsigned));
- m->by_fd = new(FileDescriptor, m->fds_max);
- if (!m->by_fd) {
- mmap_cache_free(m);
+ f->cache = m;
+ f->fd = fd;
+
+ r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f);
+ if (r < 0) {
+ free(f);
return NULL;
}
- return m;
+ return f;
}
-MMapCache* mmap_cache_ref(MMapCache *m) {
+static void mmap_cache_free(MMapCache *m) {
+ FileDescriptor *f;
+ int i;
+
assert(m);
- assert(m->n_ref > 0);
- m->n_ref++;
- return m;
+ for (i = 0; i < MMAP_CACHE_MAX_CONTEXTS; i++)
+ if (m->contexts[i])
+ context_free(m->contexts[i]);
+
+ while ((f = hashmap_first(m->fds)))
+ fd_free(f);
+
+ hashmap_free(m->fds);
+
+ while (m->unused)
+ window_free(m->unused);
+
+ free(m);
}
MMapCache* mmap_cache_unref(MMapCache *m) {
assert(m);
assert(m->n_ref > 0);
- if (m->n_ref == 1)
+ m->n_ref --;
+ if (m->n_ref == 0)
mmap_cache_free(m);
- else
- m->n_ref--;
return NULL;
}
-static int mmap_cache_allocate_window(MMapCache *m, unsigned *w) {
- Window *v;
- unsigned fd_index;
+static int make_room(MMapCache *m) {
+ assert(m);
+
+ if (!m->last_unused)
+ return 0;
+
+ window_free(m->last_unused);
+ return 1;
+}
+
+static int try_context(
+ MMapCache *m,
+ int fd,
+ int prot,
+ unsigned context,
+ bool keep_always,
+ uint64_t offset,
+ size_t size,
+ void **ret) {
+
+ Context *c;
assert(m);
- assert(w);
+ assert(m->n_ref > 0);
+ assert(fd >= 0);
+ assert(size > 0);
+ assert(ret);
- if (m->n_windows < m->windows_max) {
- *w = m->n_windows ++;
+ c = m->contexts[context];
+ if (!c)
return 0;
- }
- if (m->lru_first == (unsigned) -1)
- return -E2BIG;
+ assert(c->id == context);
- *w = m->lru_first;
- v = m->windows + *w;
- assert(v->n_ref == 0);
+ if (!c->window)
+ return 0;
- mmap_cache_window_unmap(m, *w);
+ if (!window_matches(c->window, fd, prot, offset, size)) {
- if (v->fd >= 0) {
- assert_se(mmap_cache_peek_fd_index(m, v->fd, &fd_index) > 0);
- mmap_cache_fd_remove(m, fd_index, *w);
+ /* Drop the reference to the window, since it's unnecessary now */
+ context_detach_window(c);
+ return 0;
}
- mmap_cache_window_remove_lru(m, *w);
+ if (c->window->fd->sigbus)
+ return -EIO;
+
+ c->window->keep_always |= keep_always;
- return 0;
+ *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
+ return 1;
}
-static int mmap_cache_make_room(MMapCache *m) {
- unsigned w;
+static int find_mmap(
+ MMapCache *m,
+ int fd,
+ int prot,
+ unsigned context,
+ bool keep_always,
+ uint64_t offset,
+ size_t size,
+ void **ret) {
+
+ FileDescriptor *f;
+ Window *w;
+ Context *c;
assert(m);
+ assert(m->n_ref > 0);
+ assert(fd >= 0);
+ assert(size > 0);
- w = m->lru_first;
- while (w != (unsigned) -1) {
- Window *v;
+ f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
+ if (!f)
+ return 0;
- v = m->windows + w;
+ assert(f->fd == fd);
- if (v->ptr) {
- mmap_cache_window_unmap(m, w);
- return 1;
- }
+ if (f->sigbus)
+ return -EIO;
- w = v->lru_next;
- }
+ LIST_FOREACH(by_fd, w, f->windows)
+ if (window_matches(w, fd, prot, offset, size))
+ break;
+
+ if (!w)
+ return 0;
- return 0;
+ c = context_add(m, context);
+ if (!c)
+ return -ENOMEM;
+
+ context_attach_window(c, w);
+ w->keep_always += keep_always;
+
+ *ret = (uint8_t*) w->ptr + (offset - w->offset);
+ return 1;
}
-static int mmap_cache_put(
+static int add_mmap(
MMapCache *m,
int fd,
- unsigned fd_index,
int prot,
unsigned context,
+ bool keep_always,
uint64_t offset,
- uint64_t size,
+ size_t size,
+ struct stat *st,
void **ret) {
- unsigned w;
- Window *v;
- void *d;
uint64_t woffset, wsize;
+ Context *c;
+ FileDescriptor *f;
+ Window *w;
+ void *d;
int r;
assert(m);
+ assert(m->n_ref > 0);
assert(fd >= 0);
- assert(context < m->contexts_max);
assert(size > 0);
assert(ret);
@@ -387,6 +497,18 @@ static int mmap_cache_put(
wsize = WINDOW_SIZE;
}
+ if (st) {
+ /* Memory maps that are larger then the files
+ underneath have undefined behavior. Hence, clamp
+ things to the file size if we know it */
+
+ if (woffset >= (uint64_t) st->st_size)
+ return -EADDRNOTAVAIL;
+
+ if (woffset + wsize > (uint64_t) st->st_size)
+ wsize = PAGE_ALIGN(st->st_size - woffset);
+ }
+
for (;;) {
d = mmap(NULL, wsize, prot, MAP_SHARED, fd, woffset);
if (d != MAP_FAILED)
@@ -394,336 +516,191 @@ static int mmap_cache_put(
if (errno != ENOMEM)
return -errno;
- r = mmap_cache_make_room(m);
+ r = make_room(m);
if (r < 0)
return r;
if (r == 0)
return -ENOMEM;
}
- r = mmap_cache_allocate_window(m, &w);
- if (r < 0) {
- munmap(d, wsize);
- return r;
- }
-
- v = m->windows + w;
- v->fd = fd;
- v->ptr = d;
- v->offset = woffset;
- v->size = wsize;
-
- v->n_ref = 0;
- mmap_cache_window_add_lru(m, w);
- mmap_cache_fd_add(m, fd_index, w);
- mmap_cache_context_set(m, context, w);
-
- *ret = (uint8_t*) d + (offset - woffset);
- return 1;
-}
-
-static int fd_cmp(const void *_a, const void *_b) {
- const FileDescriptor *a = _a, *b = _b;
+ c = context_add(m, context);
+ if (!c)
+ goto outofmem;
- if (a->fd < b->fd)
- return -1;
- if (a->fd > b->fd)
- return 1;
+ f = fd_add(m, fd);
+ if (!f)
+ goto outofmem;
- return 0;
-}
+ w = window_add(m);
+ if (!w)
+ goto outofmem;
-static int mmap_cache_peek_fd_index(MMapCache *m, int fd, unsigned *fd_index) {
- FileDescriptor *j;
- unsigned r;
+ w->keep_always = keep_always;
+ w->ptr = d;
+ w->offset = woffset;
+ w->prot = prot;
+ w->size = wsize;
+ w->fd = f;
- assert(m);
- assert(fd >= 0);
- assert(fd_index);
+ LIST_PREPEND(by_fd, f->windows, w);
- for (r = 0; r < m->n_fds; r++)
- assert(m->by_fd[r].windows == (unsigned) -1 ||
- m->windows[m->by_fd[r].windows].fd == m->by_fd[r].fd);
+ context_detach_window(c);
+ c->window = w;
+ LIST_PREPEND(by_window, w->contexts, c);
- j = bsearch(&fd, m->by_fd, m->n_fds, sizeof(FileDescriptor), fd_cmp);
- if (!j)
- return 0;
-
- *fd_index = (unsigned) (j - m->by_fd);
+ *ret = (uint8_t*) w->ptr + (offset - w->offset);
return 1;
-}
-
-static int mmap_cache_get_fd_index(MMapCache *m, int fd, unsigned *fd_index) {
- FileDescriptor *j;
- int r;
-
- assert(m);
- assert(fd >= 0);
- assert(fd_index);
-
- r = mmap_cache_peek_fd_index(m, fd, fd_index);
- if (r != 0)
- return r;
-
- if (m->n_fds >= m->fds_max) {
- unsigned k;
- FileDescriptor *n;
-
- k = m->n_fds * 2;
- n = realloc(m->by_fd, sizeof(FileDescriptor) * k);
- if (!n)
- return -ENOMEM;
-
- m->fds_max = k;
- m->by_fd = n;
- }
-
- j = m->by_fd + m->n_fds ++;
- j->fd = fd;
- j->windows = (unsigned) -1;
-
- qsort(m->by_fd, m->n_fds, sizeof(FileDescriptor), fd_cmp);
- return mmap_cache_peek_fd_index(m, fd, fd_index);
+outofmem:
+ munmap(d, wsize);
+ return -ENOMEM;
}
-static bool mmap_cache_test_window(
- MMapCache *m,
- unsigned w,
- uint64_t offset,
- uint64_t size) {
- Window *v;
-
- assert(m);
- assert(w < m->n_windows);
- assert(size > 0);
-
- v = m->windows + w;
-
- return offset >= v->offset &&
- offset + size <= v->offset + v->size;
-}
-
-static int mmap_cache_current(
+int mmap_cache_get(
MMapCache *m,
int fd,
+ int prot,
unsigned context,
+ bool keep_always,
uint64_t offset,
- uint64_t size,
+ size_t size,
+ struct stat *st,
void **ret) {
- Window *v;
- unsigned w;
+ int r;
assert(m);
+ assert(m->n_ref > 0);
assert(fd >= 0);
- assert(context < m->contexts_max);
assert(size > 0);
assert(ret);
+ assert(context < MMAP_CACHE_MAX_CONTEXTS);
- if (m->by_context[context] == (unsigned) -1)
- return 0;
-
- w = m->by_context[context];
- v = m->windows + w;
+ /* Check whether the current context is the right one already */
+ r = try_context(m, fd, prot, context, keep_always, offset, size, ret);
+ if (r != 0) {
+ m->n_hit ++;
+ return r;
+ }
- if (v->fd != fd)
- return 0;
+ /* Search for a matching mmap */
+ r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret);
+ if (r != 0) {
+ m->n_hit ++;
+ return r;
+ }
- if (!mmap_cache_test_window(m, w, offset, size))
- return 0;
+ m->n_missed++;
- *ret = (uint8_t*) v->ptr + (offset - v->offset);
- return 1;
+ /* Create a new mmap */
+ return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret);
}
-static int mmap_cache_find(
- MMapCache *m,
- int fd,
- unsigned fd_index,
- unsigned context,
- uint64_t offset,
- uint64_t size,
- void **ret) {
-
- Window *v = NULL;
- unsigned w;
-
+unsigned mmap_cache_get_hit(MMapCache *m) {
assert(m);
- assert(fd >= 0);
- assert(fd_index < m->n_fds);
- assert(context < m->contexts_max);
- assert(size > 0);
- assert(ret);
- w = m->by_fd[fd_index].windows;
- while (w != (unsigned) -1) {
- v = m->windows + w;
- assert(v->fd == fd);
-
- if (mmap_cache_test_window(m, w, offset, size))
- break;
-
- w = v->by_fd_next;
- }
-
- if (w == (unsigned) -1)
- return 0;
+ return m->n_hit;
+}
- mmap_cache_context_set(m, context, w);
+unsigned mmap_cache_get_missed(MMapCache *m) {
+ assert(m);
- *ret = (uint8_t*) v->ptr + (offset - v->offset);
- return 1;
+ return m->n_missed;
}
-int mmap_cache_get(
- MMapCache *m,
- int fd,
- int prot,
- unsigned context,
- uint64_t offset,
- uint64_t size,
- void **ret) {
-
- unsigned fd_index;
+static void mmap_cache_process_sigbus(MMapCache *m) {
+ bool found = false;
+ FileDescriptor *f;
+ Iterator i;
int r;
assert(m);
- assert(fd >= 0);
- assert(size > 0);
- assert(ret);
-
- if (context >= m->contexts_max) {
- unsigned k, *n;
- Window *w;
-
- /* Increase the number of contexts if necessary, and
- * make sure we have twice the number of windows */
-
- k = context * 2;
- n = realloc(m->by_context, sizeof(unsigned) * k);
- if (!n)
- return -ENOMEM;
- memset(n + m->contexts_max, -1, (k - m->contexts_max) * sizeof(unsigned));
- m->contexts_max = k;
- m->by_context = n;
-
- k = MAX(m->windows_max, m->contexts_max*2);
- w = realloc(m->windows, sizeof(Window) * k);
- if (!w)
- return -ENOMEM;
-
- m->windows_max = k;
- m->windows = w;
- }
-
- /* Maybe the current pointer for this context is already the
- * right one? */
- r = mmap_cache_current(m, fd, context, offset, size, ret);
- if (r != 0)
- return r;
-
- /* Hmm, drop the reference to the current one, since it wasn't
- * good enough */
- mmap_cache_context_unset(m, context);
-
- /* OK, let's find the chain for this FD */
- r = mmap_cache_get_fd_index(m, fd, &fd_index);
- if (r < 0)
- return r;
- /* And let's look through the available mmaps */
- r = mmap_cache_find(m, fd, fd_index, context, offset, size, ret);
- if (r != 0)
- return r;
-
- /* Not found? Then, let's add it */
- return mmap_cache_put(m, fd, fd_index, prot, context, offset, size, ret);
-}
+ /* Iterate through all triggered pages and mark their files as
+ * invalidated */
+ for (;;) {
+ bool ours;
+ void *addr;
-void mmap_cache_close_fd(MMapCache *m, int fd) {
- unsigned fd_index, c, w;
- int r;
+ r = sigbus_pop(&addr);
+ if (_likely_(r == 0))
+ break;
+ if (r < 0) {
+ log_error_errno(r, "SIGBUS handling failed: %m");
+ abort();
+ }
- assert(m);
- assert(fd > 0);
+ ours = false;
+ HASHMAP_FOREACH(f, m->fds, i) {
+ Window *w;
- r = mmap_cache_peek_fd_index(m, fd, &fd_index);
- if (r <= 0)
- return;
+ LIST_FOREACH(by_fd, w, f->windows) {
+ if ((uint8_t*) addr >= (uint8_t*) w->ptr &&
+ (uint8_t*) addr < (uint8_t*) w->ptr + w->size) {
+ found = ours = f->sigbus = true;
+ break;
+ }
+ }
- for (c = 0; c < m->contexts_max; c++) {
- w = m->by_context[c];
- if (w == (unsigned) -1)
- continue;
+ if (ours)
+ break;
+ }
- if (m->windows[w].fd == fd)
- mmap_cache_context_unset(m, c);
+ /* Didn't find a matching window, give up */
+ if (!ours) {
+ log_error("Unknown SIGBUS page, aborting.");
+ abort();
+ }
}
- w = m->by_fd[fd_index].windows;
- while (w != (unsigned) -1) {
- Window *v;
+ /* The list of triggered pages is now empty. Now, let's remap
+ * all windows of the triggered file to anonymous maps, so
+ * that no page of the file in question is triggered again, so
+ * that we can be sure not to hit the queue size limit. */
+ if (_likely_(!found))
+ return;
- v = m->windows + w;
- assert(v->fd == fd);
+ HASHMAP_FOREACH(f, m->fds, i) {
+ Window *w;
- mmap_cache_window_unmap(m, w);
- mmap_cache_fd_remove(m, fd_index, w);
- v->fd = -1;
+ if (!f->sigbus)
+ continue;
- w = m->by_fd[fd_index].windows;
+ LIST_FOREACH(by_fd, w, f->windows)
+ window_invalidate(w);
}
-
- memmove(m->by_fd + fd_index, m->by_fd + fd_index + 1, (m->n_fds - (fd_index + 1)) * sizeof(FileDescriptor));
- m->n_fds --;
}
-void mmap_cache_close_fd_range(MMapCache *m, int fd, uint64_t p) {
- unsigned fd_index, c, w;
- int r;
+bool mmap_cache_got_sigbus(MMapCache *m, int fd) {
+ FileDescriptor *f;
assert(m);
- assert(fd > 0);
-
- /* This drops all windows that include space right of the
- * specified offset. This is useful to ensure that after the
- * file size is extended we drop our mappings of the end and
- * create it anew, since otherwise it is undefined whether
- * mapping will continue to work as intended. */
+ assert(fd >= 0);
- r = mmap_cache_peek_fd_index(m, fd, &fd_index);
- if (r <= 0)
- return;
+ mmap_cache_process_sigbus(m);
- for (c = 0; c < m->contexts_max; c++) {
- w = m->by_context[c];
+ f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
+ if (!f)
+ return false;
- if (w != (unsigned) -1 && m->windows[w].fd == fd)
- mmap_cache_context_unset(m, c);
- }
+ return f->sigbus;
+}
- w = m->by_fd[fd_index].windows;
- while (w != (unsigned) -1) {
- Window *v;
+void mmap_cache_close_fd(MMapCache *m, int fd) {
+ FileDescriptor *f;
- v = m->windows + w;
- assert(v->fd == fd);
- assert(v->by_fd_next == (unsigned) -1 ||
- m->windows[v->by_fd_next].fd == fd);
+ assert(m);
+ assert(fd >= 0);
- if (v->offset + v->size > p) {
+ /* Make sure that any queued SIGBUS are first dispatched, so
+ * that we don't end up with a SIGBUS entry we cannot relate
+ * to any existing memory map */
- mmap_cache_window_unmap(m, w);
- mmap_cache_fd_remove(m, fd_index, w);
- v->fd = -1;
+ mmap_cache_process_sigbus(m);
- w = m->by_fd[fd_index].windows;
- } else
- w = v->by_fd_next;
- }
-}
+ f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
+ if (!f)
+ return;
-void mmap_cache_close_context(MMapCache *m, unsigned context) {
- mmap_cache_context_unset(m, context);
+ fd_free(f);
}