1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2012 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
32 #include "mmap-cache.h"
34 typedef struct Window Window;
35 typedef struct Context Context;
36 typedef struct FileDescriptor FileDescriptor;
51 LIST_FIELDS(Window, by_fd);
52 LIST_FIELDS(Window, unused);
54 LIST_HEAD(Context, contexts);
62 LIST_FIELDS(Context, by_window);
65 struct FileDescriptor {
68 LIST_HEAD(Window, windows);
75 unsigned n_hit, n_missed;
79 Context *contexts[MMAP_CACHE_MAX_CONTEXTS];
81 LIST_HEAD(Window, unused);
85 #define WINDOWS_MIN 64
87 #ifdef ENABLE_DEBUG_MMAP_CACHE
88 /* Tiny windows increase mmap activity and the chance of exposing unsafe use. */
89 # define WINDOW_SIZE (page_size())
91 # define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
94 MMapCache* mmap_cache_new(void) {
97 m = new0(MMapCache, 1);
105 MMapCache* mmap_cache_ref(MMapCache *m) {
107 assert(m->n_ref > 0);
113 static void window_unlink(Window *w) {
119 munmap(w->ptr, w->size);
122 LIST_REMOVE(by_fd, w->fd->windows, w);
125 if (w->cache->last_unused == w)
126 w->cache->last_unused = w->unused_prev;
128 LIST_REMOVE(unused, w->cache->unused, w);
131 LIST_FOREACH(by_window, c, w->contexts) {
132 assert(c->window == w);
137 static void window_free(Window *w) {
141 w->cache->n_windows--;
145 _pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
154 offset >= w->offset &&
155 offset + size <= w->offset + w->size;
158 static Window *window_add(MMapCache *m) {
163 if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
165 /* Allocate a new window */
172 /* Reuse an existing one */
182 static void context_detach_window(Context *c) {
192 LIST_REMOVE(by_window, w->contexts, c);
194 if (!w->contexts && !w->keep_always) {
195 /* Not used anymore? */
196 #ifdef ENABLE_DEBUG_MMAP_CACHE
197 /* Unmap unused windows immediately to expose use-after-unmap
201 LIST_PREPEND(unused, c->cache->unused, w);
202 if (!c->cache->last_unused)
203 c->cache->last_unused = w;
210 static void context_attach_window(Context *c, Window *w) {
217 context_detach_window(c);
221 LIST_REMOVE(unused, c->cache->unused, w);
222 if (c->cache->last_unused == w)
223 c->cache->last_unused = w->unused_prev;
225 w->in_unused = false;
229 LIST_PREPEND(by_window, w->contexts, c);
232 static Context *context_add(MMapCache *m, unsigned id) {
241 c = new0(Context, 1);
248 assert(!m->contexts[id]);
254 static void context_free(Context *c) {
257 context_detach_window(c);
260 assert(c->cache->contexts[c->id] == c);
261 c->cache->contexts[c->id] = NULL;
267 static void fd_free(FileDescriptor *f) {
271 window_free(f->windows);
274 assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1)));
279 static FileDescriptor* fd_add(MMapCache *m, int fd) {
286 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
290 r = hashmap_ensure_allocated(&m->fds, NULL);
294 f = new0(FileDescriptor, 1);
301 r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f);
310 static void mmap_cache_free(MMapCache *m) {
316 for (i = 0; i < MMAP_CACHE_MAX_CONTEXTS; i++)
318 context_free(m->contexts[i]);
320 while ((f = hashmap_first(m->fds)))
323 hashmap_free(m->fds);
326 window_free(m->unused);
331 MMapCache* mmap_cache_unref(MMapCache *m) {
333 assert(m->n_ref > 0);
342 static int make_room(MMapCache *m) {
348 window_free(m->last_unused);
352 static int try_context(
365 assert(m->n_ref > 0);
370 c = m->contexts[context];
374 assert(c->id == context);
379 if (!window_matches(c->window, fd, prot, offset, size)) {
381 /* Drop the reference to the window, since it's unnecessary now */
382 context_detach_window(c);
386 c->window->keep_always |= keep_always;
388 *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
392 static int find_mmap(
407 assert(m->n_ref > 0);
411 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
417 LIST_FOREACH(by_fd, w, f->windows)
418 if (window_matches(w, fd, prot, offset, size))
424 c = context_add(m, context);
428 context_attach_window(c, w);
429 w->keep_always += keep_always;
431 *ret = (uint8_t*) w->ptr + (offset - w->offset);
446 uint64_t woffset, wsize;
454 assert(m->n_ref > 0);
459 woffset = offset & ~((uint64_t) page_size() - 1ULL);
460 wsize = size + (offset - woffset);
461 wsize = PAGE_ALIGN(wsize);
463 if (wsize < WINDOW_SIZE) {
466 delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2);
477 /* Memory maps that are larger then the files
478 underneath have undefined behavior. Hence, clamp
479 things to the file size if we know it */
481 if (woffset >= (uint64_t) st->st_size)
482 return -EADDRNOTAVAIL;
484 if (woffset + wsize > (uint64_t) st->st_size)
485 wsize = PAGE_ALIGN(st->st_size - woffset);
489 d = mmap(NULL, wsize, prot, MAP_SHARED, fd, woffset);
502 c = context_add(m, context);
514 w->keep_always = keep_always;
521 LIST_PREPEND(by_fd, f->windows, w);
523 context_detach_window(c);
525 LIST_PREPEND(by_window, w->contexts, c);
527 *ret = (uint8_t*) w->ptr + (offset - w->offset);
549 assert(m->n_ref > 0);
553 assert(context < MMAP_CACHE_MAX_CONTEXTS);
555 /* Check whether the current context is the right one already */
556 r = try_context(m, fd, prot, context, keep_always, offset, size, ret);
562 /* Search for a matching mmap */
563 r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret);
571 /* Create a new mmap */
572 return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret);
575 void mmap_cache_close_fd(MMapCache *m, int fd) {
581 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
588 unsigned mmap_cache_get_hit(MMapCache *m) {
594 unsigned mmap_cache_get_missed(MMapCache *m) {