Commit d168286d authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'map-charge-cleanup' 



Roman Gushchin says:

====================
During my work on memcg-based memory accounting for bpf maps
I've done some cleanups and refactorings of the existing
memlock rlimit-based code. It makes it more robust, unifies
size to pages conversion, size checks and corresponding error
codes. Also it adds coverage for cgroup local storage and
socket local storage maps.

It looks like some preliminary work on the mm side might be
required to start working on the memcg-based accounting,
so I'm sending these patches as a separate patchset.
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 576240cf c85d6913

include/linux/bpf.h

+11 −4
Original line number Diff line number Diff line
@@ -66,6 +66,11 @@ struct bpf_map_ops { 
				     u64 imm, u32 *off);

};



struct bpf_map_memory {

	u32 pages;

	struct user_struct *user;

};



struct bpf_map {

	/* The first two cachelines with read-mostly members of which some

	 * are also accessed in fast-path (e.g. ops, max_entries).
@@ -86,7 +91,7 @@ struct bpf_map { 
	u32 btf_key_type_id;

	u32 btf_value_type_id;

	struct btf *btf;

	u32 pages;

	struct bpf_map_memory memory;

	bool unpriv_array;

	bool frozen; /* write-once */

	/* 48 bytes hole */
@@ -94,8 +99,7 @@ struct bpf_map { 
	/* The 3rd and 4th cacheline with misc members to avoid false sharing

	 * particularly with refcounting.

	 */

	struct user_struct *user ____cacheline_aligned;

	atomic_t refcnt;

	atomic_t refcnt ____cacheline_aligned;

	atomic_t usercnt;

	struct work_struct work;

	char name[BPF_OBJ_NAME_LEN];
@@ -646,9 +650,12 @@ struct bpf_map *__bpf_map_get(struct fd f); 
struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref);

void bpf_map_put_with_uref(struct bpf_map *map);

void bpf_map_put(struct bpf_map *map);

int bpf_map_precharge_memlock(u32 pages);

int bpf_map_charge_memlock(struct bpf_map *map, u32 pages);

void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages);

int bpf_map_charge_init(struct bpf_map_memory *mem, size_t size);

void bpf_map_charge_finish(struct bpf_map_memory *mem);

void bpf_map_charge_move(struct bpf_map_memory *dst,

			 struct bpf_map_memory *src);

void *bpf_map_area_alloc(size_t size, int numa_node);

void bpf_map_area_free(void *base);

void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);

kernel/bpf/arraymap.c

+8 −10
Original line number Diff line number Diff line
@@ -83,6 +83,7 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) 
	u32 elem_size, index_mask, max_entries;

	bool unpriv = !capable(CAP_SYS_ADMIN);

	u64 cost, array_size, mask64;

	struct bpf_map_memory mem;

	struct bpf_array *array;



	elem_size = round_up(attr->value_size, 8);
@@ -116,32 +117,29 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) 


	/* make sure there is no u32 overflow later in round_up() */

	cost = array_size;

	if (cost >= U32_MAX - PAGE_SIZE)

		return ERR_PTR(-ENOMEM);

	if (percpu) {

	if (percpu)

		cost += (u64)attr->max_entries * elem_size * num_possible_cpus();

		if (cost >= U32_MAX - PAGE_SIZE)

			return ERR_PTR(-ENOMEM);

	}

	cost = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;



	ret = bpf_map_precharge_memlock(cost);

	ret = bpf_map_charge_init(&mem, cost);

	if (ret < 0)

		return ERR_PTR(ret);



	/* allocate all map elements and zero-initialize them */

	array = bpf_map_area_alloc(array_size, numa_node);

	if (!array)

	if (!array) {

		bpf_map_charge_finish(&mem);

		return ERR_PTR(-ENOMEM);

	}

	array->index_mask = index_mask;

	array->map.unpriv_array = unpriv;



	/* copy mandatory map attributes */

	bpf_map_init_from_attr(&array->map, attr);

	array->map.pages = cost;

	bpf_map_charge_move(&array->map.memory, &mem);

	array->elem_size = elem_size;



	if (percpu && bpf_array_alloc_percpu(array)) {

		bpf_map_charge_finish(&array->map.memory);

		bpf_map_area_free(array);

		return ERR_PTR(-ENOMEM);

	}

kernel/bpf/cpumap.c

+4 −5
Original line number Diff line number Diff line
@@ -106,12 +106,9 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) 
	/* make sure page count doesn't overflow */

	cost = (u64) cmap->map.max_entries * sizeof(struct bpf_cpu_map_entry *);

	cost += cpu_map_bitmap_size(attr) * num_possible_cpus();

	if (cost >= U32_MAX - PAGE_SIZE)

		goto free_cmap;

	cmap->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;



	/* Notice returns -EPERM on if map size is larger than memlock limit */

	ret = bpf_map_precharge_memlock(cmap->map.pages);

	ret = bpf_map_charge_init(&cmap->map.memory, cost);

	if (ret) {

		err = ret;

		goto free_cmap;
@@ -121,7 +118,7 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) 
	cmap->flush_needed = __alloc_percpu(cpu_map_bitmap_size(attr),

					    __alignof__(unsigned long));

	if (!cmap->flush_needed)

		goto free_cmap;

		goto free_charge;



	/* Alloc array for possible remote "destination" CPUs */

	cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
@@ -133,6 +130,8 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) 
	return &cmap->map;

free_percpu:

	free_percpu(cmap->flush_needed);

free_charge:

	bpf_map_charge_finish(&cmap->map.memory);

free_cmap:

	kfree(cmap);

	return ERR_PTR(err);

kernel/bpf/devmap.c

+6 −8
Original line number Diff line number Diff line
@@ -108,13 +108,9 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr) 
	/* make sure page count doesn't overflow */

	cost = (u64) dtab->map.max_entries * sizeof(struct bpf_dtab_netdev *);

	cost += dev_map_bitmap_size(attr) * num_possible_cpus();

	if (cost >= U32_MAX - PAGE_SIZE)

		goto free_dtab;



	dtab->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;



	/* if map size is larger than memlock limit, reject it early */

	err = bpf_map_precharge_memlock(dtab->map.pages);

	/* if map size is larger than memlock limit, reject it */

	err = bpf_map_charge_init(&dtab->map.memory, cost);

	if (err)

		goto free_dtab;
@@ -125,19 +121,21 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr) 
						__alignof__(unsigned long),

						GFP_KERNEL | __GFP_NOWARN);

	if (!dtab->flush_needed)

		goto free_dtab;

		goto free_charge;



	dtab->netdev_map = bpf_map_area_alloc(dtab->map.max_entries *

					      sizeof(struct bpf_dtab_netdev *),

					      dtab->map.numa_node);

	if (!dtab->netdev_map)

		goto free_dtab;

		goto free_charge;



	spin_lock(&dev_map_lock);

	list_add_tail_rcu(&dtab->list, &dev_map_list);

	spin_unlock(&dev_map_lock);



	return &dtab->map;

free_charge:

	bpf_map_charge_finish(&dtab->map.memory);

free_dtab:

	free_percpu(dtab->flush_needed);

	kfree(dtab);

kernel/bpf/hashtab.c

+5 −9
Original line number Diff line number Diff line
@@ -360,14 +360,8 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) 
	else

	       cost += (u64) htab->elem_size * num_possible_cpus();



	if (cost >= U32_MAX - PAGE_SIZE)

		/* make sure page count doesn't overflow */

		goto free_htab;



	htab->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;



	/* if map size is larger than memlock limit, reject it early */

	err = bpf_map_precharge_memlock(htab->map.pages);

	/* if map size is larger than memlock limit, reject it */

	err = bpf_map_charge_init(&htab->map.memory, cost);

	if (err)

		goto free_htab;
@@ -376,7 +370,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) 
					   sizeof(struct bucket),

					   htab->map.numa_node);

	if (!htab->buckets)

		goto free_htab;

		goto free_charge;



	if (htab->map.map_flags & BPF_F_ZERO_SEED)

		htab->hashrnd = 0;
@@ -409,6 +403,8 @@ free_prealloc: 
	prealloc_destroy(htab);

free_buckets:

	bpf_map_area_free(htab->buckets);

free_charge:

	bpf_map_charge_finish(&htab->map.memory);

free_htab:

	kfree(htab);

	return ERR_PTR(err);

CRA Git | Maintained and supported by SUSTech CRA and CCSE