Commit ce3141a2 authored by Tejun Heo's avatar Tejun Heo
Browse files

percpu: drop pcpu_chunk->page[] 



percpu core doesn't need to tack all the allocated pages.  It needs to
know whether certain pages are populated and a way to reverse map
address to page when freeing.  This patch drops pcpu_chunk->page[] and
use populated bitmap and vmalloc_to_page() lookup instead.  Using
vmalloc_to_page() exclusively is also possible but complicates first
chunk handling, inflates cache footprint and prevents non-standard
memory allocation for percpu memory.

pcpu_chunk->page[] was used to track each page's allocation and
allowed asymmetric population which happens during failure path;
however, with single bitmap for all units, this is no longer possible.
Bite the bullet and rewrite (de)populate functions so that things are
done in clearly separated steps such that asymmetric population
doesn't happen.  This makes the (de)population process much more
modular and will also ease implementing non-standard memory usage in
the future (e.g. large pages).

This makes @get_page_fn parameter to pcpu_setup_first_chunk()
unnecessary.  The parameter is dropped and all first chunk helpers are
updated accordingly.  Please note that despite the volume most changes
to first chunk helpers are symbol renames for variables which don't
need to be referenced outside of the helper anymore.

This change reduces memory usage and cache footprint of pcpu_chunk.
Now only #unit_pages bits are necessary per chunk.

[ Impact: reduced memory usage and cache footprint for bookkeeping ]

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: David Miller <davem@davemloft.net>
parent c8a51be4

arch/sparc/kernel/smp_64.c

+15 −27
Original line number Diff line number Diff line
@@ -1415,19 +1415,6 @@ static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size, 
#endif

}



static size_t pcpur_size __initdata;

static void **pcpur_ptrs __initdata;



static struct page * __init pcpur_get_page(unsigned int cpu, int pageno)

{

	size_t off = (size_t)pageno << PAGE_SHIFT;



	if (off >= pcpur_size)

		return NULL;



	return virt_to_page(pcpur_ptrs[cpu] + off);

}



#define PCPU_CHUNK_SIZE (4UL * 1024UL * 1024UL)



static void __init pcpu_map_range(unsigned long start, unsigned long end,
@@ -1491,25 +1478,26 @@ void __init setup_per_cpu_areas(void) 
	size_t dyn_size, static_size = __per_cpu_end - __per_cpu_start;

	static struct vm_struct vm;

	unsigned long delta, cpu;

	size_t pcpu_unit_size;

	size_t size_sum, pcpu_unit_size;

	size_t ptrs_size;

	void **ptrs;



	pcpur_size = PFN_ALIGN(static_size + PERCPU_MODULE_RESERVE +

	size_sum = PFN_ALIGN(static_size + PERCPU_MODULE_RESERVE +

			     PERCPU_DYNAMIC_RESERVE);

	dyn_size = pcpur_size - static_size - PERCPU_MODULE_RESERVE;

	dyn_size = size_sum - static_size - PERCPU_MODULE_RESERVE;





	ptrs_size = PFN_ALIGN(num_possible_cpus() * sizeof(pcpur_ptrs[0]));

	pcpur_ptrs = alloc_bootmem(ptrs_size);

	ptrs_size = PFN_ALIGN(num_possible_cpus() * sizeof(ptrs[0]));

	ptrs = alloc_bootmem(ptrs_size);



	for_each_possible_cpu(cpu) {

		pcpur_ptrs[cpu] = pcpu_alloc_bootmem(cpu, PCPU_CHUNK_SIZE,

		ptrs[cpu] = pcpu_alloc_bootmem(cpu, PCPU_CHUNK_SIZE,

					       PCPU_CHUNK_SIZE);



		free_bootmem(__pa(pcpur_ptrs[cpu] + pcpur_size),

			     PCPU_CHUNK_SIZE - pcpur_size);

		free_bootmem(__pa(ptrs[cpu] + size_sum),

			     PCPU_CHUNK_SIZE - size_sum);



		memcpy(pcpur_ptrs[cpu], __per_cpu_load, static_size);

		memcpy(ptrs[cpu], __per_cpu_load, static_size);

	}



	/* allocate address and map */
@@ -1523,14 +1511,14 @@ void __init setup_per_cpu_areas(void) 


		start += cpu * PCPU_CHUNK_SIZE;

		end = start + PCPU_CHUNK_SIZE;

		pcpu_map_range(start, end, virt_to_page(pcpur_ptrs[cpu]));

		pcpu_map_range(start, end, virt_to_page(ptrs[cpu]));

	}



	pcpu_unit_size = pcpu_setup_first_chunk(pcpur_get_page, static_size,

	pcpu_unit_size = pcpu_setup_first_chunk(static_size,

						PERCPU_MODULE_RESERVE, dyn_size,

						PCPU_CHUNK_SIZE, vm.addr);



	free_bootmem(__pa(pcpur_ptrs), ptrs_size);

	free_bootmem(__pa(ptrs), ptrs_size);



	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;

	for_each_possible_cpu(cpu) {

include/linux/percpu.h

+1 −2
Original line number Diff line number Diff line
@@ -58,13 +58,12 @@ 


extern void *pcpu_base_addr;



typedef struct page * (*pcpu_get_page_fn_t)(unsigned int cpu, int pageno);

typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size);

typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size);

typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr);

typedef void (*pcpu_fc_map_fn_t)(void *ptr, size_t size, void *addr);



extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,

extern size_t __init pcpu_setup_first_chunk(

				size_t static_size, size_t reserved_size,

				ssize_t dyn_size, size_t unit_size,

				void *base_addr);

mm/percpu.c

+384 −220

File changed.

Preview size limit exceeded, changes collapsed.

CRA Git | Maintained and supported by SUSTech CRA and CCSE