Commit b936ca64 authored by Roman Gushchin's avatar Roman Gushchin Committed by Alexei Starovoitov
Browse files

bpf: rework memlock-based memory accounting for maps 



In order to unify the existing memlock charging code with the
memcg-based memory accounting, which will be added later, let's
rework the current scheme.

Currently the following design is used:
  1) .alloc() callback optionally checks if the allocation will likely
     succeed using bpf_map_precharge_memlock()
  2) .alloc() performs actual allocations
  3) .alloc() callback calculates map cost and sets map.memory.pages
  4) map_create() calls bpf_map_init_memlock() which sets map.memory.user
     and performs actual charging; in case of failure the map is
     destroyed
  <map is in use>
  1) bpf_map_free_deferred() calls bpf_map_release_memlock(), which
     performs uncharge and releases the user
  2) .map_free() callback releases the memory

The scheme can be simplified and made more robust:
  1) .alloc() calculates map cost and calls bpf_map_charge_init()
  2) bpf_map_charge_init() sets map.memory.user and performs actual
    charge
  3) .alloc() performs actual allocations
  <map is in use>
  1) .map_free() callback releases the memory
  2) bpf_map_charge_finish() performs uncharge and releases the user

The new scheme also allows to reuse bpf_map_charge_init()/finish()
functions for memcg-based accounting. Because charges are performed
before actual allocations and uncharges after freeing the memory,
no bogus memory pressure can be created.

In cases when the map structure is not available (e.g. it's not
created yet, or is already destroyed), on-stack bpf_map_memory
structure is used. The charge can be transferred with the
bpf_map_charge_move() function.

Signed-off-by: default avatarRoman Gushchin <guro@fb.com>
Acked-by: default avatarSong Liu <songliubraving@fb.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 3539b96e

include/linux/bpf.h

+4 −1
Original line number Diff line number Diff line
@@ -650,9 +650,12 @@ struct bpf_map *__bpf_map_get(struct fd f); 
struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref);

void bpf_map_put_with_uref(struct bpf_map *map);

void bpf_map_put(struct bpf_map *map);

int bpf_map_precharge_memlock(u32 pages);

int bpf_map_charge_memlock(struct bpf_map *map, u32 pages);

void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages);

int bpf_map_charge_init(struct bpf_map_memory *mem, u32 pages);

void bpf_map_charge_finish(struct bpf_map_memory *mem);

void bpf_map_charge_move(struct bpf_map_memory *dst,

			 struct bpf_map_memory *src);

void *bpf_map_area_alloc(size_t size, int numa_node);

void bpf_map_area_free(void *base);

void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);

kernel/bpf/arraymap.c

+7 −3
Original line number Diff line number Diff line
@@ -83,6 +83,7 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) 
	u32 elem_size, index_mask, max_entries;

	bool unpriv = !capable(CAP_SYS_ADMIN);

	u64 cost, array_size, mask64;

	struct bpf_map_memory mem;

	struct bpf_array *array;



	elem_size = round_up(attr->value_size, 8);
@@ -125,23 +126,26 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) 
	}

	cost = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;



	ret = bpf_map_precharge_memlock(cost);

	ret = bpf_map_charge_init(&mem, cost);

	if (ret < 0)

		return ERR_PTR(ret);



	/* allocate all map elements and zero-initialize them */

	array = bpf_map_area_alloc(array_size, numa_node);

	if (!array)

	if (!array) {

		bpf_map_charge_finish(&mem);

		return ERR_PTR(-ENOMEM);

	}

	array->index_mask = index_mask;

	array->map.unpriv_array = unpriv;



	/* copy mandatory map attributes */

	bpf_map_init_from_attr(&array->map, attr);

	array->map.memory.pages = cost;

	bpf_map_charge_move(&array->map.memory, &mem);

	array->elem_size = elem_size;



	if (percpu && bpf_array_alloc_percpu(array)) {

		bpf_map_charge_finish(&array->map.memory);

		bpf_map_area_free(array);

		return ERR_PTR(-ENOMEM);

	}

kernel/bpf/cpumap.c

+5 −3
Original line number Diff line number Diff line
@@ -108,10 +108,10 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) 
	cost += cpu_map_bitmap_size(attr) * num_possible_cpus();

	if (cost >= U32_MAX - PAGE_SIZE)

		goto free_cmap;

	cmap->map.memory.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;



	/* Notice returns -EPERM on if map size is larger than memlock limit */

	ret = bpf_map_precharge_memlock(cmap->map.memory.pages);

	ret = bpf_map_charge_init(&cmap->map.memory,

				  round_up(cost, PAGE_SIZE) >> PAGE_SHIFT);

	if (ret) {

		err = ret;

		goto free_cmap;
@@ -121,7 +121,7 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) 
	cmap->flush_needed = __alloc_percpu(cpu_map_bitmap_size(attr),

					    __alignof__(unsigned long));

	if (!cmap->flush_needed)

		goto free_cmap;

		goto free_charge;



	/* Alloc array for possible remote "destination" CPUs */

	cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
@@ -133,6 +133,8 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) 
	return &cmap->map;

free_percpu:

	free_percpu(cmap->flush_needed);

free_charge:

	bpf_map_charge_finish(&cmap->map.memory);

free_cmap:

	kfree(cmap);

	return ERR_PTR(err);

kernel/bpf/devmap.c

+7 −6
Original line number Diff line number Diff line
@@ -111,10 +111,9 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr) 
	if (cost >= U32_MAX - PAGE_SIZE)

		goto free_dtab;



	dtab->map.memory.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;



	/* if map size is larger than memlock limit, reject it early */

	err = bpf_map_precharge_memlock(dtab->map.memory.pages);

	/* if map size is larger than memlock limit, reject it */

	err = bpf_map_charge_init(&dtab->map.memory,

				  round_up(cost, PAGE_SIZE) >> PAGE_SHIFT);

	if (err)

		goto free_dtab;
@@ -125,19 +124,21 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr) 
						__alignof__(unsigned long),

						GFP_KERNEL | __GFP_NOWARN);

	if (!dtab->flush_needed)

		goto free_dtab;

		goto free_charge;



	dtab->netdev_map = bpf_map_area_alloc(dtab->map.max_entries *

					      sizeof(struct bpf_dtab_netdev *),

					      dtab->map.numa_node);

	if (!dtab->netdev_map)

		goto free_dtab;

		goto free_charge;



	spin_lock(&dev_map_lock);

	list_add_tail_rcu(&dtab->list, &dev_map_list);

	spin_unlock(&dev_map_lock);



	return &dtab->map;

free_charge:

	bpf_map_charge_finish(&dtab->map.memory);

free_dtab:

	free_percpu(dtab->flush_needed);

	kfree(dtab);

kernel/bpf/hashtab.c

+6 −5
Original line number Diff line number Diff line
@@ -364,10 +364,9 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) 
		/* make sure page count doesn't overflow */

		goto free_htab;



	htab->map.memory.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;



	/* if map size is larger than memlock limit, reject it early */

	err = bpf_map_precharge_memlock(htab->map.memory.pages);

	/* if map size is larger than memlock limit, reject it */

	err = bpf_map_charge_init(&htab->map.memory,

				  round_up(cost, PAGE_SIZE) >> PAGE_SHIFT);

	if (err)

		goto free_htab;
@@ -376,7 +375,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) 
					   sizeof(struct bucket),

					   htab->map.numa_node);

	if (!htab->buckets)

		goto free_htab;

		goto free_charge;



	if (htab->map.map_flags & BPF_F_ZERO_SEED)

		htab->hashrnd = 0;
@@ -409,6 +408,8 @@ free_prealloc: 
	prealloc_destroy(htab);

free_buckets:

	bpf_map_area_free(htab->buckets);

free_charge:

	bpf_map_charge_finish(&htab->map.memory);

free_htab:

	kfree(htab);

	return ERR_PTR(err);

CRA Git | Maintained and supported by SUSTech CRA and CCSE