1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2012 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
33 #include "mmap-cache.h"
35 typedef struct Window Window;
36 typedef struct Context Context;
37 typedef struct FileDescriptor FileDescriptor;
53 LIST_FIELDS(Window, by_fd);
54 LIST_FIELDS(Window, unused);
56 LIST_HEAD(Context, contexts);
64 LIST_FIELDS(Context, by_window);
67 struct FileDescriptor {
71 LIST_HEAD(Window, windows);
78 unsigned n_hit, n_missed;
82 Context *contexts[MMAP_CACHE_MAX_CONTEXTS];
84 LIST_HEAD(Window, unused);
88 #define WINDOWS_MIN 64
90 #ifdef ENABLE_DEBUG_MMAP_CACHE
91 /* Tiny windows increase mmap activity and the chance of exposing unsafe use. */
92 # define WINDOW_SIZE (page_size())
94 # define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
97 MMapCache* mmap_cache_new(void) {
100 m = new0(MMapCache, 1);
108 MMapCache* mmap_cache_ref(MMapCache *m) {
110 assert(m->n_ref > 0);
116 static void window_unlink(Window *w) {
122 munmap(w->ptr, w->size);
125 LIST_REMOVE(by_fd, w->fd->windows, w);
128 if (w->cache->last_unused == w)
129 w->cache->last_unused = w->unused_prev;
131 LIST_REMOVE(unused, w->cache->unused, w);
134 LIST_FOREACH(by_window, c, w->contexts) {
135 assert(c->window == w);
140 static void window_invalidate(Window *w) {
146 /* Replace the window with anonymous pages. This is useful
147 * when we hit a SIGBUS and want to make sure the file cannot
148 * trigger any further SIGBUS, possibly overrunning the sigbus
151 assert_se(mmap(w->ptr, w->size, w->prot, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0) == w->ptr);
152 w->invalidated = true;
155 static void window_free(Window *w) {
159 w->cache->n_windows--;
163 _pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
172 offset >= w->offset &&
173 offset + size <= w->offset + w->size;
176 static Window *window_add(MMapCache *m) {
181 if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
183 /* Allocate a new window */
190 /* Reuse an existing one */
200 static void context_detach_window(Context *c) {
210 LIST_REMOVE(by_window, w->contexts, c);
212 if (!w->contexts && !w->keep_always) {
213 /* Not used anymore? */
214 #ifdef ENABLE_DEBUG_MMAP_CACHE
215 /* Unmap unused windows immediately to expose use-after-unmap
219 LIST_PREPEND(unused, c->cache->unused, w);
220 if (!c->cache->last_unused)
221 c->cache->last_unused = w;
228 static void context_attach_window(Context *c, Window *w) {
235 context_detach_window(c);
239 LIST_REMOVE(unused, c->cache->unused, w);
240 if (c->cache->last_unused == w)
241 c->cache->last_unused = w->unused_prev;
243 w->in_unused = false;
247 LIST_PREPEND(by_window, w->contexts, c);
250 static Context *context_add(MMapCache *m, unsigned id) {
259 c = new0(Context, 1);
266 assert(!m->contexts[id]);
272 static void context_free(Context *c) {
275 context_detach_window(c);
278 assert(c->cache->contexts[c->id] == c);
279 c->cache->contexts[c->id] = NULL;
285 static void fd_free(FileDescriptor *f) {
289 window_free(f->windows);
292 assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1)));
297 static FileDescriptor* fd_add(MMapCache *m, int fd) {
304 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
308 r = hashmap_ensure_allocated(&m->fds, NULL);
312 f = new0(FileDescriptor, 1);
319 r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f);
328 static void mmap_cache_free(MMapCache *m) {
334 for (i = 0; i < MMAP_CACHE_MAX_CONTEXTS; i++)
336 context_free(m->contexts[i]);
338 while ((f = hashmap_first(m->fds)))
341 hashmap_free(m->fds);
344 window_free(m->unused);
349 MMapCache* mmap_cache_unref(MMapCache *m) {
351 assert(m->n_ref > 0);
360 static int make_room(MMapCache *m) {
366 window_free(m->last_unused);
370 static int try_context(
383 assert(m->n_ref > 0);
388 c = m->contexts[context];
392 assert(c->id == context);
397 if (!window_matches(c->window, fd, prot, offset, size)) {
399 /* Drop the reference to the window, since it's unnecessary now */
400 context_detach_window(c);
404 if (c->window->fd->sigbus)
407 c->window->keep_always |= keep_always;
409 *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
413 static int find_mmap(
428 assert(m->n_ref > 0);
432 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
441 LIST_FOREACH(by_fd, w, f->windows)
442 if (window_matches(w, fd, prot, offset, size))
448 c = context_add(m, context);
452 context_attach_window(c, w);
453 w->keep_always += keep_always;
455 *ret = (uint8_t*) w->ptr + (offset - w->offset);
470 uint64_t woffset, wsize;
478 assert(m->n_ref > 0);
483 woffset = offset & ~((uint64_t) page_size() - 1ULL);
484 wsize = size + (offset - woffset);
485 wsize = PAGE_ALIGN(wsize);
487 if (wsize < WINDOW_SIZE) {
490 delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2);
501 /* Memory maps that are larger then the files
502 underneath have undefined behavior. Hence, clamp
503 things to the file size if we know it */
505 if (woffset >= (uint64_t) st->st_size)
506 return -EADDRNOTAVAIL;
508 if (woffset + wsize > (uint64_t) st->st_size)
509 wsize = PAGE_ALIGN(st->st_size - woffset);
513 d = mmap(NULL, wsize, prot, MAP_SHARED, fd, woffset);
526 c = context_add(m, context);
538 w->keep_always = keep_always;
545 LIST_PREPEND(by_fd, f->windows, w);
547 context_detach_window(c);
549 LIST_PREPEND(by_window, w->contexts, c);
551 *ret = (uint8_t*) w->ptr + (offset - w->offset);
573 assert(m->n_ref > 0);
577 assert(context < MMAP_CACHE_MAX_CONTEXTS);
579 /* Check whether the current context is the right one already */
580 r = try_context(m, fd, prot, context, keep_always, offset, size, ret);
586 /* Search for a matching mmap */
587 r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret);
595 /* Create a new mmap */
596 return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret);
599 unsigned mmap_cache_get_hit(MMapCache *m) {
605 unsigned mmap_cache_get_missed(MMapCache *m) {
611 static void mmap_cache_process_sigbus(MMapCache *m) {
619 /* Iterate through all triggered pages and mark their files as
625 r = sigbus_pop(&addr);
626 if (_likely_(r == 0))
629 log_error_errno(r, "SIGBUS handling failed: %m");
634 HASHMAP_FOREACH(f, m->fds, i) {
637 LIST_FOREACH(by_fd, w, f->windows) {
638 if ((uint8_t*) addr >= (uint8_t*) w->ptr &&
639 (uint8_t*) addr < (uint8_t*) w->ptr + w->size) {
640 found = ours = f->sigbus = true;
649 /* Didn't find a matching window, give up */
651 log_error("Unknown SIGBUS page, aborting.");
656 /* The list of triggered pages is now empty. Now, let's remap
657 * all windows of the triggered file to anonymous maps, so
658 * that no page of the file in question is triggered again, so
659 * that we can be sure not to hit the queue size limit. */
660 if (_likely_(!found))
663 HASHMAP_FOREACH(f, m->fds, i) {
669 LIST_FOREACH(by_fd, w, f->windows)
670 window_invalidate(w);
674 bool mmap_cache_got_sigbus(MMapCache *m, int fd) {
680 mmap_cache_process_sigbus(m);
682 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
689 void mmap_cache_close_fd(MMapCache *m, int fd) {
695 /* Make sure that any queued SIGBUS are first dispatched, so
696 * that we don't end up with a SIGBUS entry we cannot relate
697 * to any existing memory map */
699 mmap_cache_process_sigbus(m);
701 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));