Commit 61ace7fa authored by Tejun Heo's avatar Tejun Heo
Browse files

percpu: improve first chunk initial area map handling 



Impact: no functional change

When the first chunk is created, its initial area map is not allocated
because kmalloc isn't online yet.  The map is allocated and
initialized on the first allocation request on the chunk.  This works
fine but the scattering of initialization logic between the init
function and allocation path is a bit confusing.

This patch makes the first chunk initialize and use minimal statically
allocated map from pcpu_setpu_first_chunk().  The map resizing path
still needs to handle this specially but it's more straight-forward
and gives more latitude to the init path.  This will ease future
changes.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent 2441d15c

mm/percpu.c

+27 −26
Original line number Original line Diff line number Diff line
@@ -93,9 +93,6 @@ static size_t pcpu_chunk_struct_size __read_mostly; 
void *pcpu_base_addr __read_mostly;

void *pcpu_base_addr __read_mostly;

EXPORT_SYMBOL_GPL(pcpu_base_addr);

EXPORT_SYMBOL_GPL(pcpu_base_addr);





/* the size of kernel static area */

static int pcpu_static_size __read_mostly;



/*

/*

 * One mutex to rule them all.

 * One mutex to rule them all.

 *

 *
@@ -316,12 +313,25 @@ static int pcpu_split_block(struct pcpu_chunk *chunk, int i, int head, int tail) 




	/* reallocation required? */

	/* reallocation required? */

	if (chunk->map_alloc < target) {

	if (chunk->map_alloc < target) {

		int new_alloc = chunk->map_alloc;

		int new_alloc;

		int *new;

		int *new;





		new_alloc = PCPU_DFL_MAP_ALLOC;

		while (new_alloc < target)

		while (new_alloc < target)

			new_alloc *= 2;

			new_alloc *= 2;





		if (chunk->map_alloc < PCPU_DFL_MAP_ALLOC) {

			/*

			 * map_alloc smaller than the default size

			 * indicates that the chunk is one of the

			 * first chunks and still using static map.

			 * Allocate a dynamic one and copy.

			 */

			new = pcpu_realloc(NULL, 0, new_alloc * sizeof(new[0]));

			if (new)

				memcpy(new, chunk->map,

				       chunk->map_alloc * sizeof(new[0]));

		} else

			new = pcpu_realloc(chunk->map,

			new = pcpu_realloc(chunk->map,

					   chunk->map_alloc * sizeof(new[0]),

					   chunk->map_alloc * sizeof(new[0]),

					   new_alloc * sizeof(new[0]));

					   new_alloc * sizeof(new[0]));
@@ -367,22 +377,6 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align) 
	int max_contig = 0;

	int max_contig = 0;

	int i, off;

	int i, off;





	/*

	 * The static chunk initially doesn't have map attached

	 * because kmalloc wasn't available during init.  Give it one.

	 */

	if (unlikely(!chunk->map)) {

		chunk->map = pcpu_realloc(NULL, 0,

				PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));

		if (!chunk->map)

			return -ENOMEM;



		chunk->map_alloc = PCPU_DFL_MAP_ALLOC;

		chunk->map[chunk->map_used++] = -pcpu_static_size;

		if (chunk->free_size)

			chunk->map[chunk->map_used++] = chunk->free_size;

	}



	for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) {

	for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) {

		bool is_last = i + 1 == chunk->map_used;

		bool is_last = i + 1 == chunk->map_used;

		int head, tail;

		int head, tail;
@@ -874,12 +868,14 @@ size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, 
				     pcpu_populate_pte_fn_t populate_pte_fn)

				     pcpu_populate_pte_fn_t populate_pte_fn)

{

{

	static struct vm_struct first_vm;

	static struct vm_struct first_vm;

	static int smap[2];

	struct pcpu_chunk *schunk;

	struct pcpu_chunk *schunk;

	unsigned int cpu;

	unsigned int cpu;

	int nr_pages;

	int nr_pages;

	int err, i;

	int err, i;





	/* santiy checks */

	/* santiy checks */

	BUILD_BUG_ON(ARRAY_SIZE(smap) >= PCPU_DFL_MAP_ALLOC);

	BUG_ON(!static_size);

	BUG_ON(!static_size);

	BUG_ON(!unit_size && dyn_size);

	BUG_ON(!unit_size && dyn_size);

	BUG_ON(unit_size && unit_size < static_size + dyn_size);

	BUG_ON(unit_size && unit_size < static_size + dyn_size);
@@ -893,7 +889,6 @@ size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, 
		pcpu_unit_pages = max_t(int, PCPU_MIN_UNIT_SIZE >> PAGE_SHIFT,

		pcpu_unit_pages = max_t(int, PCPU_MIN_UNIT_SIZE >> PAGE_SHIFT,

					PFN_UP(static_size));

					PFN_UP(static_size));





	pcpu_static_size = static_size;

	pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT;

	pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT;

	pcpu_chunk_size = num_possible_cpus() * pcpu_unit_size;

	pcpu_chunk_size = num_possible_cpus() * pcpu_unit_size;

	pcpu_chunk_struct_size = sizeof(struct pcpu_chunk)

	pcpu_chunk_struct_size = sizeof(struct pcpu_chunk)
@@ -912,14 +907,20 @@ size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, 
	schunk = alloc_bootmem(pcpu_chunk_struct_size);

	schunk = alloc_bootmem(pcpu_chunk_struct_size);

	INIT_LIST_HEAD(&schunk->list);

	INIT_LIST_HEAD(&schunk->list);

	schunk->vm = &first_vm;

	schunk->vm = &first_vm;

	schunk->map = smap;

	schunk->map_alloc = ARRAY_SIZE(smap);





	if (dyn_size)

	if (dyn_size)

		schunk->free_size = dyn_size;

		schunk->free_size = dyn_size;

	else

	else

		schunk->free_size = pcpu_unit_size - pcpu_static_size;

		schunk->free_size = pcpu_unit_size - static_size;





	schunk->contig_hint = schunk->free_size;

	schunk->contig_hint = schunk->free_size;





	schunk->map[schunk->map_used++] = -static_size;

	if (schunk->free_size)

		schunk->map[schunk->map_used++] = schunk->free_size;



	/* allocate vm address */

	/* allocate vm address */

	first_vm.flags = VM_ALLOC;

	first_vm.flags = VM_ALLOC;

	first_vm.size = pcpu_chunk_size;

	first_vm.size = pcpu_chunk_size;
@@ -948,7 +949,7 @@ size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, 
			*pcpu_chunk_pagep(schunk, cpu, i) = page;

			*pcpu_chunk_pagep(schunk, cpu, i) = page;

		}

		}





		BUG_ON(i < PFN_UP(pcpu_static_size));

		BUG_ON(i < PFN_UP(static_size));





		if (nr_pages < 0)

		if (nr_pages < 0)

			nr_pages = i;

			nr_pages = i;

CRA Git | Maintained and supported by SUSTech CRA and CCSE