Single-threaded version of sark-branch memory page management (c78247f9) · Commits · 戴 / Bird

lib/resource.c

+2 −0

Original line number	Diff line number	Diff line
		@@ -270,6 +270,8 @@ rlookup(unsigned long a)
		void
		resource_init(void)
		{
		resource_sys_init();

		root_pool.r.class = &pool_class;
		root_pool.name = "Root";
		init_list(&root_pool.inside);

lib/resource.h

+3 −2

Original line number	Diff line number	Diff line
		@@ -110,10 +110,11 @@ void sl_free(slab , void );
		void buffer_realloc(void *buf, unsigned size, unsigned need, unsigned item_size);

		/* Allocator of whole pages; for use in slabs and other high-level allocators. */
		u64 get_page_size(void);
		extern long page_size;
		void *alloc_page(void);
		void free_page(void *);
		extern uint pages_kept;

		void resource_sys_init(void);

		#ifdef HAVE_LIBDMALLOC
		/*

lib/slab.c

+6 −7

Original line number	Diff line number	Diff line
		@@ -180,7 +180,7 @@ struct sl_alignment { /* Magic structure for testing of alignment */
		int x[0];
		};

		#define SL_GET_HEAD(x) ((struct sl_head *) (((uintptr_t) (x)) & ~(get_page_size()-1)))
		#define SL_GET_HEAD(x) ((struct sl_head *) (((uintptr_t) (x)) & ~(page_size-1)))

		/**
		* sl_new - create a new Slab
		@@ -202,7 +202,6 @@ sl_new(pool *p, uint size)
		s->obj_size = size;

		s->head_size = sizeof(struct sl_head);
		u64 page_size = get_page_size();

		do {
		s->objs_per_slab = (page_size - s->head_size) / size;
		@@ -273,7 +272,7 @@ no_partial:
		}
		h = alloc_page();
		#ifdef POISON
		memset(h, 0xba, get_page_size());
		memset(h, 0xba, page_size);
		#endif
		ASSERT_DIE(SL_GET_HEAD(h) == h);
		memset(h, 0, s->head_size);
		@@ -332,7 +331,7 @@ sl_free(slab s, void oo)
		if (s->num_empty_heads >= MAX_EMPTY_HEADS)
		{
		#ifdef POISON
		memset(h, 0xde, get_page_size());
		memset(h, 0xde, page_size);
		#endif
		free_page(h);
		}
		@@ -399,7 +398,7 @@ slab_memsize(resource *r)

		return (struct resmem) {
		.effective = eff,
		.overhead = ALLOC_OVERHEAD + sizeof(struct slab) + heads * get_page_size() - eff,
		.overhead = ALLOC_OVERHEAD + sizeof(struct slab) + heads * page_size - eff,
		};
		}

		@@ -410,10 +409,10 @@ slab_lookup(resource *r, unsigned long a)
		struct sl_head *h;

		WALK_LIST(h, s->partial_heads)
		if ((unsigned long) h < a && (unsigned long) h + get_page_size() < a)
		if ((unsigned long) h < a && (unsigned long) h + page_size < a)
		return r;
		WALK_LIST(h, s->full_heads)
		if ((unsigned long) h < a && (unsigned long) h + get_page_size() < a)
		if ((unsigned long) h < a && (unsigned long) h + page_size < a)
		return r;
		return NULL;
		}

nest/cmds.c

+3 −2

Original line number	Diff line number	Diff line
		@@ -108,6 +108,7 @@ print_size(char *dsc, struct resmem vals)

		extern pool *rt_table_pool;
		extern pool *rta_pool;
		extern uint *pages_kept;

		void
		cmd_show_memory(void)
		@@ -119,8 +120,8 @@ cmd_show_memory(void)
		print_size("Protocols:", rmemsize(proto_pool));
		struct resmem total = rmemsize(&root_pool);
		#ifdef HAVE_MMAP
		print_size("Standby memory:", (struct resmem) { .overhead = get_page_size() * pages_kept });
		total.overhead += get_page_size() * pages_kept;
		print_size("Standby memory:", (struct resmem) { .overhead = page_size * *pages_kept });
		total.overhead += page_size * *pages_kept;
		#endif
		print_size("Total:", total);
		cli_msg(0, "");

sysdep/unix/alloc.c

+123 −68

Original line number	Diff line number	Diff line
		@@ -11,6 +11,7 @@
		#include "lib/lists.h"
		#include "lib/event.h"

		#include <errno.h>
		#include <stdlib.h>
		#include <unistd.h>

		@@ -18,113 +19,167 @@
		#include <sys/mman.h>
		#endif

		#ifdef HAVE_MMAP
		#define KEEP_PAGES 512
		long page_size = 0;

		static u64 page_size = 0;
		static _Bool use_fake = 0;
		#ifdef HAVE_MMAP
		#define KEEP_PAGES_MAIN_MAX 256
		#define KEEP_PAGES_MAIN_MIN 8
		#define CLEANUP_PAGES_BULK 256

		uint pages_kept = 0;
		static list pages_list;
		_Static_assert(KEEP_PAGES_MAIN_MIN * 4 < KEEP_PAGES_MAIN_MAX);

		static void cleanup_pages(void *data);
		static event page_cleanup_event = { .hook = cleanup_pages };
		static _Bool use_fake = 0;

		#if DEBUGGING
		struct free_page {
		node unused[42];
		node n;
		};
		#else
		static const u64 page_size = 4096; /* Fake page size */
		struct free_page {
		node n;
		};
		#endif

		u64 get_page_size(void)
		{
		if (page_size)
		return page_size;
		struct free_pages {
		list pages;
		u16 min, max; /* Minimal and maximal number of free pages kept */
		uint cnt; /* Number of empty pages */
		event cleanup;
		};

		#ifdef HAVE_MMAP
		if (page_size = sysconf(_SC_PAGESIZE))
		{
		if ((u64_popcount(page_size) > 1) \|\| (page_size > 16384))
		static void global_free_pages_cleanup_event(void *);

		static struct free_pages global_free_pages = {
		.min = KEEP_PAGES_MAIN_MIN,
		.max = KEEP_PAGES_MAIN_MAX,
		.cleanup = { .hook = global_free_pages_cleanup_event },
		};

		uint *pages_kept = &global_free_pages.cnt;

		static void *
		alloc_sys_page(void)
		{
		/* Too big or strange page, use the aligned allocator instead */
		page_size = 4096;
		use_fake = 1;
		}
		return page_size;
		void *ptr = mmap(NULL, page_size, PROT_WRITE \| PROT_READ, MAP_PRIVATE \| MAP_ANONYMOUS, -1, 0);

		if (ptr == MAP_FAILED)
		bug("mmap(%lu) failed: %m", page_size);

		return ptr;
		}

		bug("Page size must be non-zero");
		extern int shutting_down; /* Shutdown requested. */

		#else // ! HAVE_MMAP
		#define use_fake 1
		#endif
		}

		void *
		alloc_page(void)
		{
		#ifdef HAVE_MMAP
		if (pages_kept)
		{
		node *page = TAIL(pages_list);
		rem_node(page);
		pages_kept--;
		memset(page, 0, get_page_size());
		return page;
		}

		if (!use_fake)
		{
		void *ret = mmap(NULL, get_page_size(), PROT_WRITE \| PROT_READ, MAP_PRIVATE \| MAP_ANONYMOUS, -1, 0);
		if (ret == MAP_FAILED)
		bug("mmap(%lu) failed: %m", (long unsigned int) page_size);
		return ret;
		}
		else
		#endif
		if (use_fake)
		{
		void *ptr = NULL;
		int err = posix_memalign(&ptr, page_size, page_size);

		if (err \|\| !ptr)
		bug("posix_memalign(%lu) failed", (long unsigned int) page_size);

		return ptr;
		}

		#ifdef HAVE_MMAP
		struct free_pages *fps = &global_free_pages;

		if (fps->cnt)
		{
		struct free_page *fp = SKIP_BACK(struct free_page, n, HEAD(fps->pages));
		rem_node(&fp->n);
		if ((--fps->cnt < fps->min) && !shutting_down)
		ev_schedule(&fps->cleanup);

		bzero(fp, page_size);
		return fp;
		}

		return alloc_sys_page();
		#endif
		}

		void
		free_page(void *ptr)
		{
		#ifdef HAVE_MMAP
		if (!use_fake)
		if (use_fake)
		{
		if (!pages_kept)
		init_list(&pages_list);
		free(ptr);
		return;
		}

		memset(ptr, 0, sizeof(node));
		add_tail(&pages_list, ptr);
		#ifdef HAVE_MMAP
		struct free_pages *fps = &global_free_pages;
		struct free_page *fp = ptr;

		if (++pages_kept > KEEP_PAGES)
		ev_schedule(&page_cleanup_event);
		}
		else
		fp->n = (node) {};
		add_tail(&fps->pages, &fp->n);

		if ((++fps->cnt > fps->max) && !shutting_down)
		ev_schedule(&fps->cleanup);
		#endif
		free(ptr);
		}

		#ifdef HAVE_MMAP
		static void
		cleanup_pages(void *data UNUSED)
		global_free_pages_cleanup_event(void *data UNUSED)
		{
		if (shutting_down)
		return;

		struct free_pages *fps = &global_free_pages;

		while (fps->cnt / 2 < fps->min)
		{
		for (uint seen = 0; (pages_kept > KEEP_PAGES) && (seen < KEEP_PAGES); seen++)
		struct free_page *fp = alloc_sys_page();
		fp->n = (node) {};
		add_tail(&fps->pages, &fp->n);
		fps->cnt++;
		}

		for (uint seen = 0; (seen < CLEANUP_PAGES_BULK) && (fps->cnt > fps->max / 2); seen++)
		{
		void *ptr = HEAD(pages_list);
		rem_node(ptr);
		if (munmap(ptr, get_page_size()) == 0)
		pages_kept--;
		#ifdef ENOMEM
		struct free_page *fp = SKIP_BACK(struct free_page, n, TAIL(fps->pages));
		rem_node(&fp->n);

		if (munmap(fp, page_size) == 0)
		fps->cnt--;
		else if (errno == ENOMEM)
		add_tail(&pages_list, ptr);
		#endif
		add_head(&fps->pages, &fp->n);
		else
		bug("munmap(%p) failed: %m", ptr);
		bug("munmap(%p) failed: %m", fp);
		}
		}
		#endif

		void
		resource_sys_init(void)
		{
		#ifdef HAVE_MMAP
		if (!(page_size = sysconf(_SC_PAGESIZE)))
		die("System page size must be non-zero");

		if (pages_kept > KEEP_PAGES)
		ev_schedule(&page_cleanup_event);
		if (u64_popcount(page_size) == 1)
		{
		struct free_pages *fps = &global_free_pages;

		init_list(&fps->pages);
		global_free_pages_cleanup_event(NULL);
		return;
		}

		/* Too big or strange page, use the aligned allocator instead */
		log(L_WARN "Got strange memory page size (%lu), using the aligned allocator instead", page_size);
		use_fake = 1;
		#endif

		page_size = 4096;
		}

Admin message