x86,percpu: generalize lpage first chunk allocator (8c4bfc6e) · Commits · 戴 / test

arch/x86/include/asm/percpu.h

+0 −9

Original line number	Diff line number	Diff line
		@@ -156,15 +156,6 @@ do { \
		/* We can use this directly for local CPU (faster). */
		DECLARE_PER_CPU(unsigned long, this_cpu_off);

		#ifdef CONFIG_NEED_MULTIPLE_NODES
		void pcpu_lpage_remapped(void kaddr);
		#else
		static inline void pcpu_lpage_remapped(void kaddr)
		{
		return NULL;
		}
		#endif

		#endif /* !__ASSEMBLY__ */

		#ifdef CONFIG_SMP

arch/x86/kernel/setup_percpu.c

+11 −158

Original line number	Diff line number	Diff line
		@@ -137,44 +137,21 @@ static void __init pcpu_fc_free(void *ptr, size_t size)
		}

		/*
		* Large page remap allocator
		*
		* This allocator uses PMD page as unit. A PMD page is allocated for
		* each cpu and each is remapped into vmalloc area using PMD mapping.
		* As PMD page is quite large, only part of it is used for the first
		* chunk. Unused part is returned to the bootmem allocator.
		*
		* So, the PMD pages are mapped twice - once to the physical mapping
		* and to the vmalloc area for the first percpu chunk. The double
		* mapping does add one more PMD TLB entry pressure but still is much
		* better than only using 4k mappings while still being NUMA friendly.
		* Large page remapping allocator
		*/
		#ifdef CONFIG_NEED_MULTIPLE_NODES
		struct pcpul_ent {
		unsigned int cpu;
		void *ptr;
		};

		static size_t pcpul_size;
		static struct pcpul_ent *pcpul_map;
		static struct vm_struct pcpul_vm;

		static struct page * __init pcpul_get_page(unsigned int cpu, int pageno)
		static void __init pcpul_map(void ptr, size_t size, void addr)
		{
		size_t off = (size_t)pageno << PAGE_SHIFT;

		if (off >= pcpul_size)
		return NULL;
		pmd_t *pmd, pmd_v;

		return virt_to_page(pcpul_map[cpu].ptr + off);
		pmd = populate_extra_pmd((unsigned long)addr);
		pmd_v = pfn_pmd(page_to_pfn(virt_to_page(ptr)), PAGE_KERNEL_LARGE);
		set_pmd(pmd, pmd_v);
		}

		static ssize_t __init setup_pcpu_lpage(size_t static_size, bool chosen)
		{
		size_t map_size, dyn_size;
		unsigned int cpu;
		int i, j;
		ssize_t ret;
		size_t reserve = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE;

		if (!chosen) {
		size_t vm_size = VMALLOC_END - VMALLOC_START;
		@@ -198,134 +175,10 @@ static ssize_t __init setup_pcpu_lpage(size_t static_size, bool chosen)
		return -EINVAL;
		}

		/*
		* Currently supports only single page. Supporting multiple
		* pages won't be too difficult if it ever becomes necessary.
		*/
		pcpul_size = PFN_ALIGN(static_size + PERCPU_MODULE_RESERVE +
		PERCPU_DYNAMIC_RESERVE);
		if (pcpul_size > PMD_SIZE) {
		pr_warning("PERCPU: static data is larger than large page, "
		"can't use large page\n");
		return -EINVAL;
		}
		dyn_size = pcpul_size - static_size - PERCPU_FIRST_CHUNK_RESERVE;

		/* allocate pointer array and alloc large pages */
		map_size = PFN_ALIGN(num_possible_cpus() * sizeof(pcpul_map[0]));
		pcpul_map = alloc_bootmem(map_size);

		for_each_possible_cpu(cpu) {
		pcpul_map[cpu].cpu = cpu;
		pcpul_map[cpu].ptr = pcpu_alloc_bootmem(cpu, PMD_SIZE,
		PMD_SIZE);
		if (!pcpul_map[cpu].ptr) {
		pr_warning("PERCPU: failed to allocate large page "
		"for cpu%u\n", cpu);
		goto enomem;
		}

		/*
		* Only use pcpul_size bytes and give back the rest.
		*
		* Ingo: The 2MB up-rounding bootmem is needed to make
		* sure the partial 2MB page is still fully RAM - it's
		* not well-specified to have a PAT-incompatible area
		* (unmapped RAM, device memory, etc.) in that hole.
		*/
		free_bootmem(__pa(pcpul_map[cpu].ptr + pcpul_size),
		PMD_SIZE - pcpul_size);

		memcpy(pcpul_map[cpu].ptr, __per_cpu_load, static_size);
		}

		/* allocate address and map */
		pcpul_vm.flags = VM_ALLOC;
		pcpul_vm.size = num_possible_cpus() * PMD_SIZE;
		vm_area_register_early(&pcpul_vm, PMD_SIZE);

		for_each_possible_cpu(cpu) {
		pmd_t *pmd, pmd_v;

		pmd = populate_extra_pmd((unsigned long)pcpul_vm.addr +
		cpu * PMD_SIZE);
		pmd_v = pfn_pmd(page_to_pfn(virt_to_page(pcpul_map[cpu].ptr)),
		PAGE_KERNEL_LARGE);
		set_pmd(pmd, pmd_v);
		}

		/* we're ready, commit */
		pr_info("PERCPU: Remapped at %p with large pages, static data "
		"%zu bytes\n", pcpul_vm.addr, static_size);

		ret = pcpu_setup_first_chunk(pcpul_get_page, static_size,
		PERCPU_FIRST_CHUNK_RESERVE, dyn_size,
		PMD_SIZE, pcpul_vm.addr, NULL);

		/* sort pcpul_map array for pcpu_lpage_remapped() */
		for (i = 0; i < num_possible_cpus() - 1; i++)
		for (j = i + 1; j < num_possible_cpus(); j++)
		if (pcpul_map[i].ptr > pcpul_map[j].ptr) {
		struct pcpul_ent tmp = pcpul_map[i];
		pcpul_map[i] = pcpul_map[j];
		pcpul_map[j] = tmp;
		}

		return ret;

		enomem:
		for_each_possible_cpu(cpu)
		if (pcpul_map[cpu].ptr)
		free_bootmem(__pa(pcpul_map[cpu].ptr), pcpul_size);
		free_bootmem(__pa(pcpul_map), map_size);
		return -ENOMEM;
		}

		/**
		* pcpu_lpage_remapped - determine whether a kaddr is in pcpul recycled area
		* @kaddr: the kernel address in question
		*
		* Determine whether @kaddr falls in the pcpul recycled area. This is
		* used by pageattr to detect VM aliases and break up the pcpu PMD
		* mapping such that the same physical page is not mapped under
		* different attributes.
		*
		* The recycled area is always at the tail of a partially used PMD
		* page.
		*
		* RETURNS:
		* Address of corresponding remapped pcpu address if match is found;
		* otherwise, NULL.
		*/
		void pcpu_lpage_remapped(void kaddr)
		{
		void pmd_addr = (void )((unsigned long)kaddr & PMD_MASK);
		unsigned long offset = (unsigned long)kaddr & ~PMD_MASK;
		int left = 0, right = num_possible_cpus() - 1;
		int pos;

		/* pcpul in use at all? */
		if (!pcpul_map)
		return NULL;

		/* okay, perform binary search */
		while (left <= right) {
		pos = (left + right) / 2;

		if (pcpul_map[pos].ptr < pmd_addr)
		left = pos + 1;
		else if (pcpul_map[pos].ptr > pmd_addr)
		right = pos - 1;
		else {
		/* it shouldn't be in the area for the first chunk */
		WARN_ON(offset < pcpul_size);

		return pcpul_vm.addr +
		pcpul_map[pos].cpu * PMD_SIZE + offset;
		}
		}

		return NULL;
		return pcpu_lpage_first_chunk(static_size, PERCPU_FIRST_CHUNK_RESERVE,
		reserve - PERCPU_FIRST_CHUNK_RESERVE,
		PMD_SIZE,
		pcpu_fc_alloc, pcpu_fc_free, pcpul_map);
		}
		#else
		static ssize_t __init setup_pcpu_lpage(size_t static_size, bool chosen)

arch/x86/mm/pageattr.c

+1 −0

Original line number	Diff line number	Diff line
		@@ -12,6 +12,7 @@
		#include <linux/seq_file.h>
		#include <linux/debugfs.h>
		#include <linux/pfn.h>
		#include <linux/percpu.h>

		#include <asm/e820.h>
		#include <asm/processor.h>

include/linux/percpu.h

+27 −0

Original line number	Diff line number	Diff line
		@@ -62,6 +62,7 @@ typedef struct page * (*pcpu_get_page_fn_t)(unsigned int cpu, int pageno);
		typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size);
		typedef void (pcpu_fc_free_fn_t)(void ptr, size_t size);
		typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr);
		typedef void (pcpu_fc_map_fn_t)(void ptr, size_t size, void *addr);

		extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
		size_t static_size, size_t reserved_size,
		@@ -79,6 +80,32 @@ extern ssize_t __init pcpu_4k_first_chunk(
		pcpu_fc_free_fn_t free_fn,
		pcpu_fc_populate_pte_fn_t populate_pte_fn);

		#ifdef CONFIG_NEED_MULTIPLE_NODES
		extern ssize_t __init pcpu_lpage_first_chunk(
		size_t static_size, size_t reserved_size,
		ssize_t dyn_size, size_t lpage_size,
		pcpu_fc_alloc_fn_t alloc_fn,
		pcpu_fc_free_fn_t free_fn,
		pcpu_fc_map_fn_t map_fn);

		extern void pcpu_lpage_remapped(void kaddr);
		#else
		static inline ssize_t __init pcpu_lpage_first_chunk(
		size_t static_size, size_t reserved_size,
		ssize_t dyn_size, size_t lpage_size,
		pcpu_fc_alloc_fn_t alloc_fn,
		pcpu_fc_free_fn_t free_fn,
		pcpu_fc_map_fn_t map_fn)
		{
		return -EINVAL;
		}

		static inline void pcpu_lpage_remapped(void kaddr)
		{
		return NULL;
		}
		#endif

		/*
		* Use this to get to a cpu's version of the per-cpu object
		* dynamically allocated. Non-atomic access to the current CPU's

mm/percpu.c

+205 −4

Original line number	Diff line number	Diff line
		@@ -1190,6 +1190,19 @@ size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
		return pcpu_unit_size;
		}

		static size_t pcpu_calc_fc_sizes(size_t static_size, size_t reserved_size,
		ssize_t *dyn_sizep)
		{
		size_t size_sum;

		size_sum = PFN_ALIGN(static_size + reserved_size +
		(dyn_sizep >= 0 ? dyn_sizep : 0));
		if (*dyn_sizep != 0)
		*dyn_sizep = size_sum - static_size - reserved_size;

		return size_sum;
		}

		/*
		* Embedding first chunk setup helper.
		*/
		@@ -1241,10 +1254,7 @@ ssize_t __init pcpu_embed_first_chunk(size_t static_size, size_t reserved_size,
		unsigned int cpu;

		/* determine parameters and allocate */
		pcpue_size = PFN_ALIGN(static_size + reserved_size +
		(dyn_size >= 0 ? dyn_size : 0));
		if (dyn_size != 0)
		dyn_size = pcpue_size - static_size - reserved_size;
		pcpue_size = pcpu_calc_fc_sizes(static_size, reserved_size, &dyn_size);

		pcpue_unit_size = max_t(size_t, pcpue_size, PCPU_MIN_UNIT_SIZE);
		chunk_size = pcpue_unit_size * num_possible_cpus();
		@@ -1390,6 +1400,197 @@ out_free_ar:
		return ret;
		}

		/*
		* Large page remapping first chunk setup helper
		*/
		#ifdef CONFIG_NEED_MULTIPLE_NODES
		struct pcpul_ent {
		unsigned int cpu;
		void *ptr;
		};

		static size_t pcpul_size;
		static size_t pcpul_unit_size;
		static struct pcpul_ent *pcpul_map;
		static struct vm_struct pcpul_vm;

		static struct page * __init pcpul_get_page(unsigned int cpu, int pageno)
		{
		size_t off = (size_t)pageno << PAGE_SHIFT;

		if (off >= pcpul_size)
		return NULL;

		return virt_to_page(pcpul_map[cpu].ptr + off);
		}

		/**
		* pcpu_lpage_first_chunk - remap the first percpu chunk using large page
		* @static_size: the size of static percpu area in bytes
		* @reserved_size: the size of reserved percpu area in bytes
		* @dyn_size: free size for dynamic allocation in bytes, -1 for auto
		* @lpage_size: the size of a large page
		* @alloc_fn: function to allocate percpu lpage, always called with lpage_size
		* @free_fn: function to free percpu memory, @size <= lpage_size
		* @map_fn: function to map percpu lpage, always called with lpage_size
		*
		* This allocator uses large page as unit. A large page is allocated
		* for each cpu and each is remapped into vmalloc area using large
		* page mapping. As large page can be quite large, only part of it is
		* used for the first chunk. Unused part is returned to the bootmem
		* allocator.
		*
		* So, the large pages are mapped twice - once to the physical mapping
		* and to the vmalloc area for the first percpu chunk. The double
		* mapping does add one more large TLB entry pressure but still is
		* much better than only using 4k mappings while still being NUMA
		* friendly.
		*
		* RETURNS:
		* The determined pcpu_unit_size which can be used to initialize
		* percpu access on success, -errno on failure.
		*/
		ssize_t __init pcpu_lpage_first_chunk(size_t static_size, size_t reserved_size,
		ssize_t dyn_size, size_t lpage_size,
		pcpu_fc_alloc_fn_t alloc_fn,
		pcpu_fc_free_fn_t free_fn,
		pcpu_fc_map_fn_t map_fn)
		{
		size_t size_sum;
		size_t map_size;
		unsigned int cpu;
		int i, j;
		ssize_t ret;

		/*
		* Currently supports only single page. Supporting multiple
		* pages won't be too difficult if it ever becomes necessary.
		*/
		size_sum = pcpu_calc_fc_sizes(static_size, reserved_size, &dyn_size);

		pcpul_unit_size = lpage_size;
		pcpul_size = max_t(size_t, size_sum, PCPU_MIN_UNIT_SIZE);
		if (pcpul_size > pcpul_unit_size) {
		pr_warning("PERCPU: static data is larger than large page, "
		"can't use large page\n");
		return -EINVAL;
		}

		/* allocate pointer array and alloc large pages */
		map_size = PFN_ALIGN(num_possible_cpus() * sizeof(pcpul_map[0]));
		pcpul_map = alloc_bootmem(map_size);

		for_each_possible_cpu(cpu) {
		void *ptr;

		ptr = alloc_fn(cpu, lpage_size);
		if (!ptr) {
		pr_warning("PERCPU: failed to allocate large page "
		"for cpu%u\n", cpu);
		goto enomem;
		}

		/*
		* Only use pcpul_size bytes and give back the rest.
		*
		* Ingo: The lpage_size up-rounding bootmem is needed
		* to make sure the partial lpage is still fully RAM -
		* it's not well-specified to have a incompatible area
		* (unmapped RAM, device memory, etc.) in that hole.
		*/
		free_fn(ptr + pcpul_size, lpage_size - pcpul_size);

		pcpul_map[cpu].cpu = cpu;
		pcpul_map[cpu].ptr = ptr;

		memcpy(ptr, __per_cpu_load, static_size);
		}

		/* allocate address and map */
		pcpul_vm.flags = VM_ALLOC;
		pcpul_vm.size = num_possible_cpus() * pcpul_unit_size;
		vm_area_register_early(&pcpul_vm, pcpul_unit_size);

		for_each_possible_cpu(cpu)
		map_fn(pcpul_map[cpu].ptr, pcpul_unit_size,
		pcpul_vm.addr + cpu * pcpul_unit_size);

		/* we're ready, commit */
		pr_info("PERCPU: Remapped at %p with large pages, static data "
		"%zu bytes\n", pcpul_vm.addr, static_size);

		ret = pcpu_setup_first_chunk(pcpul_get_page, static_size,
		reserved_size, dyn_size, pcpul_unit_size,
		pcpul_vm.addr, NULL);

		/* sort pcpul_map array for pcpu_lpage_remapped() */
		for (i = 0; i < num_possible_cpus() - 1; i++)
		for (j = i + 1; j < num_possible_cpus(); j++)
		if (pcpul_map[i].ptr > pcpul_map[j].ptr) {
		struct pcpul_ent tmp = pcpul_map[i];
		pcpul_map[i] = pcpul_map[j];
		pcpul_map[j] = tmp;
		}

		return ret;

		enomem:
		for_each_possible_cpu(cpu)
		if (pcpul_map[cpu].ptr)
		free_fn(pcpul_map[cpu].ptr, pcpul_size);
		free_bootmem(__pa(pcpul_map), map_size);
		return -ENOMEM;
		}

		/**
		* pcpu_lpage_remapped - determine whether a kaddr is in pcpul recycled area
		* @kaddr: the kernel address in question
		*
		* Determine whether @kaddr falls in the pcpul recycled area. This is
		* used by pageattr to detect VM aliases and break up the pcpu large
		* page mapping such that the same physical page is not mapped under
		* different attributes.
		*
		* The recycled area is always at the tail of a partially used large
		* page.
		*
		* RETURNS:
		* Address of corresponding remapped pcpu address if match is found;
		* otherwise, NULL.
		*/
		void pcpu_lpage_remapped(void kaddr)
		{
		unsigned long unit_mask = pcpul_unit_size - 1;
		void lpage_addr = (void )((unsigned long)kaddr & ~unit_mask);
		unsigned long offset = (unsigned long)kaddr & unit_mask;
		int left = 0, right = num_possible_cpus() - 1;
		int pos;

		/* pcpul in use at all? */
		if (!pcpul_map)
		return NULL;

		/* okay, perform binary search */
		while (left <= right) {
		pos = (left + right) / 2;

		if (pcpul_map[pos].ptr < lpage_addr)
		left = pos + 1;
		else if (pcpul_map[pos].ptr > lpage_addr)
		right = pos - 1;
		else {
		/* it shouldn't be in the area for the first chunk */
		WARN_ON(offset < pcpul_size);

		return pcpul_vm.addr +
		pcpul_map[pos].cpu * pcpul_unit_size + offset;
		}
		}

		return NULL;
		}
		#endif

		/*
		* Generic percpu area setup.
		*

Admin message