1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2012 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
32 #include "mmap-cache.h"
34 typedef struct Window Window;
35 typedef struct Context Context;
36 typedef struct FileDescriptor FileDescriptor;
51 LIST_FIELDS(Window, by_fd);
52 LIST_FIELDS(Window, unused);
54 LIST_HEAD(Context, contexts);
62 LIST_FIELDS(Context, by_window);
65 struct FileDescriptor {
68 LIST_HEAD(Window, windows);
75 unsigned n_hit, n_missed;
81 LIST_HEAD(Window, unused);
85 #define WINDOWS_MIN 64
87 #ifdef ENABLE_DEBUG_MMAP_CACHE
88 /* Tiny windows increase mmap activity and the chance of exposing unsafe use. */
89 # define WINDOW_SIZE (page_size())
91 # define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
94 MMapCache* mmap_cache_new(void) {
97 m = new0(MMapCache, 1);
105 MMapCache* mmap_cache_ref(MMapCache *m) {
107 assert(m->n_ref > 0);
113 static void window_unlink(Window *w) {
119 munmap(w->ptr, w->size);
122 LIST_REMOVE(by_fd, w->fd->windows, w);
125 if (w->cache->last_unused == w)
126 w->cache->last_unused = w->unused_prev;
128 LIST_REMOVE(unused, w->cache->unused, w);
131 LIST_FOREACH(by_window, c, w->contexts) {
132 assert(c->window == w);
137 static void window_free(Window *w) {
141 w->cache->n_windows--;
145 _pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
154 offset >= w->offset &&
155 offset + size <= w->offset + w->size;
158 static Window *window_add(MMapCache *m) {
163 if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
165 /* Allocate a new window */
172 /* Reuse an existing one */
182 static void context_detach_window(Context *c) {
192 LIST_REMOVE(by_window, w->contexts, c);
194 if (!w->contexts && !w->keep_always) {
195 /* Not used anymore? */
196 #ifdef ENABLE_DEBUG_MMAP_CACHE
197 /* Unmap unused windows immediately to expose use-after-unmap
201 LIST_PREPEND(unused, c->cache->unused, w);
202 if (!c->cache->last_unused)
203 c->cache->last_unused = w;
210 static void context_attach_window(Context *c, Window *w) {
217 context_detach_window(c);
221 LIST_REMOVE(unused, c->cache->unused, w);
222 if (c->cache->last_unused == w)
223 c->cache->last_unused = w->unused_prev;
225 w->in_unused = false;
229 LIST_PREPEND(by_window, w->contexts, c);
232 static Context *context_add(MMapCache *m, unsigned id) {
238 c = hashmap_get(m->contexts, UINT_TO_PTR(id + 1));
242 r = hashmap_ensure_allocated(&m->contexts, NULL);
246 c = new0(Context, 1);
253 r = hashmap_put(m->contexts, UINT_TO_PTR(id + 1), c);
262 static void context_free(Context *c) {
265 context_detach_window(c);
268 assert_se(hashmap_remove(c->cache->contexts, UINT_TO_PTR(c->id + 1)));
273 static void fd_free(FileDescriptor *f) {
277 window_free(f->windows);
280 assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1)));
285 static FileDescriptor* fd_add(MMapCache *m, int fd) {
292 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
296 r = hashmap_ensure_allocated(&m->fds, NULL);
300 f = new0(FileDescriptor, 1);
307 r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f);
316 static void mmap_cache_free(MMapCache *m) {
322 while ((c = hashmap_first(m->contexts)))
325 hashmap_free(m->contexts);
327 while ((f = hashmap_first(m->fds)))
330 hashmap_free(m->fds);
333 window_free(m->unused);
338 MMapCache* mmap_cache_unref(MMapCache *m) {
340 assert(m->n_ref > 0);
349 static int make_room(MMapCache *m) {
355 window_free(m->last_unused);
359 static int try_context(
372 assert(m->n_ref > 0);
377 c = hashmap_get(m->contexts, UINT_TO_PTR(context+1));
381 assert(c->id == context);
386 if (!window_matches(c->window, fd, prot, offset, size)) {
388 /* Drop the reference to the window, since it's unnecessary now */
389 context_detach_window(c);
393 c->window->keep_always |= keep_always;
395 *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
399 static int find_mmap(
414 assert(m->n_ref > 0);
418 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
424 LIST_FOREACH(by_fd, w, f->windows)
425 if (window_matches(w, fd, prot, offset, size))
431 c = context_add(m, context);
435 context_attach_window(c, w);
436 w->keep_always += keep_always;
438 *ret = (uint8_t*) w->ptr + (offset - w->offset);
453 uint64_t woffset, wsize;
461 assert(m->n_ref > 0);
466 woffset = offset & ~((uint64_t) page_size() - 1ULL);
467 wsize = size + (offset - woffset);
468 wsize = PAGE_ALIGN(wsize);
470 if (wsize < WINDOW_SIZE) {
473 delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2);
484 /* Memory maps that are larger then the files
485 underneath have undefined behavior. Hence, clamp
486 things to the file size if we know it */
488 if (woffset >= (uint64_t) st->st_size)
489 return -EADDRNOTAVAIL;
491 if (woffset + wsize > (uint64_t) st->st_size)
492 wsize = PAGE_ALIGN(st->st_size - woffset);
496 d = mmap(NULL, wsize, prot, MAP_SHARED, fd, woffset);
509 c = context_add(m, context);
521 w->keep_always = keep_always;
528 LIST_PREPEND(by_fd, f->windows, w);
530 context_detach_window(c);
532 LIST_PREPEND(by_window, w->contexts, c);
534 *ret = (uint8_t*) w->ptr + (offset - w->offset);
556 assert(m->n_ref > 0);
561 /* Check whether the current context is the right one already */
562 r = try_context(m, fd, prot, context, keep_always, offset, size, ret);
568 /* Search for a matching mmap */
569 r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret);
577 /* Create a new mmap */
578 return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret);
581 void mmap_cache_close_fd(MMapCache *m, int fd) {
587 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
594 void mmap_cache_close_context(MMapCache *m, unsigned context) {
599 c = hashmap_get(m->contexts, UINT_TO_PTR(context + 1));
606 unsigned mmap_cache_get_hit(MMapCache *m) {
612 unsigned mmap_cache_get_missed(MMapCache *m) {