1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2012 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
32 #include "mmap-cache.h"
34 typedef struct Window Window;
35 typedef struct Context Context;
36 typedef struct FileDescriptor FileDescriptor;
51 LIST_FIELDS(Window, by_fd);
52 LIST_FIELDS(Window, unused);
54 LIST_HEAD(Context, contexts);
62 LIST_FIELDS(Context, by_window);
65 struct FileDescriptor {
68 LIST_HEAD(Window, windows);
75 unsigned n_hit, n_missed;
81 LIST_HEAD(Window, unused);
85 #define WINDOWS_MIN 64
87 #ifdef ENABLE_DEBUG_MMAP_CACHE
88 /* Tiny windows increase mmap activity and the chance of exposing unsafe use. */
89 # define WINDOW_SIZE (page_size())
91 # define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
94 MMapCache* mmap_cache_new(void) {
97 m = new0(MMapCache, 1);
105 MMapCache* mmap_cache_ref(MMapCache *m) {
107 assert(m->n_ref > 0);
113 static void window_unlink(Window *w) {
119 munmap(w->ptr, w->size);
122 LIST_REMOVE(by_fd, w->fd->windows, w);
125 if (w->cache->last_unused == w)
126 w->cache->last_unused = w->unused_prev;
128 LIST_REMOVE(unused, w->cache->unused, w);
131 LIST_FOREACH(by_window, c, w->contexts) {
132 assert(c->window == w);
137 static void window_free(Window *w) {
141 w->cache->n_windows--;
145 _pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
154 offset >= w->offset &&
155 offset + size <= w->offset + w->size;
158 static Window *window_add(MMapCache *m) {
163 if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
165 /* Allocate a new window */
172 /* Reuse an existing one */
182 static void context_detach_window(Context *c) {
192 LIST_REMOVE(by_window, w->contexts, c);
194 if (!w->contexts && w->keep_always == 0) {
195 /* Not used anymore? */
196 #ifdef ENABLE_DEBUG_MMAP_CACHE
197 /* Unmap unused windows immediately to expose use-after-unmap
201 LIST_PREPEND(unused, c->cache->unused, w);
202 if (!c->cache->last_unused)
203 c->cache->last_unused = w;
210 static void context_attach_window(Context *c, Window *w) {
217 context_detach_window(c);
221 LIST_REMOVE(unused, c->cache->unused, w);
222 if (c->cache->last_unused == w)
223 c->cache->last_unused = w->unused_prev;
225 w->in_unused = false;
229 LIST_PREPEND(by_window, w->contexts, c);
232 static Context *context_add(MMapCache *m, unsigned id) {
238 c = hashmap_get(m->contexts, UINT_TO_PTR(id + 1));
242 r = hashmap_ensure_allocated(&m->contexts, NULL);
246 c = new0(Context, 1);
253 r = hashmap_put(m->contexts, UINT_TO_PTR(id + 1), c);
262 static void context_free(Context *c) {
265 context_detach_window(c);
268 assert_se(hashmap_remove(c->cache->contexts, UINT_TO_PTR(c->id + 1)));
273 static void fd_free(FileDescriptor *f) {
277 window_free(f->windows);
280 assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1)));
285 static FileDescriptor* fd_add(MMapCache *m, int fd) {
292 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
296 r = hashmap_ensure_allocated(&m->fds, NULL);
300 f = new0(FileDescriptor, 1);
307 r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f);
316 static void mmap_cache_free(MMapCache *m) {
322 while ((c = hashmap_first(m->contexts)))
325 hashmap_free(m->contexts);
327 while ((f = hashmap_first(m->fds)))
330 hashmap_free(m->fds);
333 window_free(m->unused);
338 MMapCache* mmap_cache_unref(MMapCache *m) {
340 assert(m->n_ref > 0);
349 static int make_room(MMapCache *m) {
355 window_free(m->last_unused);
359 static int try_context(
368 void **release_cookie) {
373 assert(m->n_ref > 0);
377 c = hashmap_get(m->contexts, UINT_TO_PTR(context+1));
381 assert(c->id == context);
386 if (!window_matches(c->window, fd, prot, offset, size)) {
388 /* Drop the reference to the window, since it's unnecessary now */
389 context_detach_window(c);
393 c->window->keep_always += keep_always;
396 *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
397 if (keep_always && release_cookie)
398 *release_cookie = c->window;
402 static int find_mmap(
411 void **release_cookie) {
418 assert(m->n_ref > 0);
422 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
428 LIST_FOREACH(by_fd, w, f->windows)
429 if (window_matches(w, fd, prot, offset, size))
435 c = context_add(m, context);
439 context_attach_window(c, w);
440 w->keep_always += keep_always;
443 *ret = (uint8_t*) w->ptr + (offset - w->offset);
444 if (keep_always && release_cookie)
445 *release_cookie = c->window;
459 void **release_cookie) {
461 uint64_t woffset, wsize;
469 assert(m->n_ref > 0);
473 woffset = offset & ~((uint64_t) page_size() - 1ULL);
474 wsize = size + (offset - woffset);
475 wsize = PAGE_ALIGN(wsize);
477 if (wsize < WINDOW_SIZE) {
480 delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2);
491 /* Memory maps that are larger then the files
492 underneath have undefined behavior. Hence, clamp
493 things to the file size if we know it */
495 if (woffset >= (uint64_t) st->st_size)
496 return -EADDRNOTAVAIL;
498 if (woffset + wsize > (uint64_t) st->st_size)
499 wsize = PAGE_ALIGN(st->st_size - woffset);
503 d = mmap(NULL, wsize, prot, MAP_SHARED, fd, woffset);
516 c = context_add(m, context);
528 w->keep_always = keep_always;
535 LIST_PREPEND(by_fd, f->windows, w);
537 context_detach_window(c);
539 LIST_PREPEND(by_window, w->contexts, c);
542 *ret = (uint8_t*) w->ptr + (offset - w->offset);
543 if (keep_always && release_cookie)
544 *release_cookie = c->window;
562 void **release_cookie) {
567 assert(m->n_ref > 0);
571 /* Check whether the current context is the right one already */
572 r = try_context(m, fd, prot, context, keep_always, offset, size, ret, release_cookie);
578 /* Search for a matching mmap */
579 r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret, release_cookie);
587 /* Create a new mmap */
588 return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret, release_cookie);
591 int mmap_cache_release(
594 void *release_cookie) {
600 assert(m->n_ref > 0);
603 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
609 LIST_FOREACH(by_fd, w, f->windows)
610 if (w == release_cookie)
616 if (w->keep_always == 0)
623 void mmap_cache_close_fd(MMapCache *m, int fd) {
629 f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
636 void mmap_cache_close_context(MMapCache *m, unsigned context) {
641 c = hashmap_get(m->contexts, UINT_TO_PTR(context + 1));
648 unsigned mmap_cache_get_hit(MMapCache *m) {
654 unsigned mmap_cache_get_missed(MMapCache *m) {