Commit 15c14a3d authored by Brian Vazquez's avatar Brian Vazquez Committed by Alexei Starovoitov
Browse files

bpf: Add bpf_map_{value_size, update_value, map_copy_value} functions 



This commit moves reusable code from map_lookup_elem and map_update_elem
to avoid code duplication in kernel/bpf/syscall.c.

Signed-off-by: default avatarBrian Vazquez <brianvv@google.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarJohn Fastabend <john.fastabend@gmail.com>
Acked-by: default avatarYonghong Song <yhs@fb.com>
Link: https://lore.kernel.org/bpf/20200115184308.162644-2-brianvv@google.com
parent 83e4b88b

kernel/bpf/syscall.c

+152 −128
Original line number Diff line number Diff line
@@ -129,6 +129,154 @@ static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 
	return map;

}



static u32 bpf_map_value_size(struct bpf_map *map)

{

	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||

	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||

	    map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||

	    map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)

		return round_up(map->value_size, 8) * num_possible_cpus();

	else if (IS_FD_MAP(map))

		return sizeof(u32);

	else

		return  map->value_size;

}



static void maybe_wait_bpf_programs(struct bpf_map *map)

{

	/* Wait for any running BPF programs to complete so that

	 * userspace, when we return to it, knows that all programs

	 * that could be running use the new map value.

	 */

	if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||

	    map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)

		synchronize_rcu();

}



static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key,

				void *value, __u64 flags)

{

	int err;



	/* Need to create a kthread, thus must support schedule */

	if (bpf_map_is_dev_bound(map)) {

		return bpf_map_offload_update_elem(map, key, value, flags);

	} else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||

		   map->map_type == BPF_MAP_TYPE_SOCKHASH ||

		   map->map_type == BPF_MAP_TYPE_SOCKMAP ||

		   map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {

		return map->ops->map_update_elem(map, key, value, flags);

	} else if (IS_FD_PROG_ARRAY(map)) {

		return bpf_fd_array_map_update_elem(map, f.file, key, value,

						    flags);

	}



	/* must increment bpf_prog_active to avoid kprobe+bpf triggering from

	 * inside bpf map update or delete otherwise deadlocks are possible

	 */

	preempt_disable();

	__this_cpu_inc(bpf_prog_active);

	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||

	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {

		err = bpf_percpu_hash_update(map, key, value, flags);

	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {

		err = bpf_percpu_array_update(map, key, value, flags);

	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {

		err = bpf_percpu_cgroup_storage_update(map, key, value,

						       flags);

	} else if (IS_FD_ARRAY(map)) {

		rcu_read_lock();

		err = bpf_fd_array_map_update_elem(map, f.file, key, value,

						   flags);

		rcu_read_unlock();

	} else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {

		rcu_read_lock();

		err = bpf_fd_htab_map_update_elem(map, f.file, key, value,

						  flags);

		rcu_read_unlock();

	} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {

		/* rcu_read_lock() is not needed */

		err = bpf_fd_reuseport_array_update_elem(map, key, value,

							 flags);

	} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||

		   map->map_type == BPF_MAP_TYPE_STACK) {

		err = map->ops->map_push_elem(map, value, flags);

	} else {

		rcu_read_lock();

		err = map->ops->map_update_elem(map, key, value, flags);

		rcu_read_unlock();

	}

	__this_cpu_dec(bpf_prog_active);

	preempt_enable();

	maybe_wait_bpf_programs(map);



	return err;

}



static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,

			      __u64 flags)

{

	void *ptr;

	int err;



	if (bpf_map_is_dev_bound(map)) {

		err =  bpf_map_offload_lookup_elem(map, key, value);

		return err;

	}



	preempt_disable();

	this_cpu_inc(bpf_prog_active);

	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||

	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {

		err = bpf_percpu_hash_copy(map, key, value);

	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {

		err = bpf_percpu_array_copy(map, key, value);

	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {

		err = bpf_percpu_cgroup_storage_copy(map, key, value);

	} else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {

		err = bpf_stackmap_copy(map, key, value);

	} else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {

		err = bpf_fd_array_map_lookup_elem(map, key, value);

	} else if (IS_FD_HASH(map)) {

		err = bpf_fd_htab_map_lookup_elem(map, key, value);

	} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {

		err = bpf_fd_reuseport_array_lookup_elem(map, key, value);

	} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||

		   map->map_type == BPF_MAP_TYPE_STACK) {

		err = map->ops->map_peek_elem(map, value);

	} else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {

		/* struct_ops map requires directly updating "value" */

		err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);

	} else {

		rcu_read_lock();

		if (map->ops->map_lookup_elem_sys_only)

			ptr = map->ops->map_lookup_elem_sys_only(map, key);

		else

			ptr = map->ops->map_lookup_elem(map, key);

		if (IS_ERR(ptr)) {

			err = PTR_ERR(ptr);

		} else if (!ptr) {

			err = -ENOENT;

		} else {

			err = 0;

			if (flags & BPF_F_LOCK)

				/* lock 'ptr' and copy everything but lock */

				copy_map_value_locked(map, value, ptr, true);

			else

				copy_map_value(map, value, ptr);

			/* mask lock, since value wasn't zero inited */

			check_and_init_map_lock(map, value);

		}

		rcu_read_unlock();

	}



	this_cpu_dec(bpf_prog_active);

	preempt_enable();

	maybe_wait_bpf_programs(map);



	return err;

}



static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)

{

	/* We really just want to fail instead of triggering OOM killer
@@ -827,7 +975,7 @@ static int map_lookup_elem(union bpf_attr *attr) 
	void __user *uvalue = u64_to_user_ptr(attr->value);

	int ufd = attr->map_fd;

	struct bpf_map *map;

	void *key, *value, *ptr;

	void *key, *value;

	u32 value_size;

	struct fd f;

	int err;
@@ -859,75 +1007,14 @@ static int map_lookup_elem(union bpf_attr *attr) 
		goto err_put;

	}



	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||

	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||

	    map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||

	    map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)

		value_size = round_up(map->value_size, 8) * num_possible_cpus();

	else if (IS_FD_MAP(map))

		value_size = sizeof(u32);

	else

		value_size = map->value_size;

	value_size = bpf_map_value_size(map);



	err = -ENOMEM;

	value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);

	if (!value)

		goto free_key;



	if (bpf_map_is_dev_bound(map)) {

		err = bpf_map_offload_lookup_elem(map, key, value);

		goto done;

	}



	preempt_disable();

	this_cpu_inc(bpf_prog_active);

	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||

	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {

		err = bpf_percpu_hash_copy(map, key, value);

	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {

		err = bpf_percpu_array_copy(map, key, value);

	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {

		err = bpf_percpu_cgroup_storage_copy(map, key, value);

	} else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {

		err = bpf_stackmap_copy(map, key, value);

	} else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {

		err = bpf_fd_array_map_lookup_elem(map, key, value);

	} else if (IS_FD_HASH(map)) {

		err = bpf_fd_htab_map_lookup_elem(map, key, value);

	} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {

		err = bpf_fd_reuseport_array_lookup_elem(map, key, value);

	} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||

		   map->map_type == BPF_MAP_TYPE_STACK) {

		err = map->ops->map_peek_elem(map, value);

	} else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {

		/* struct_ops map requires directly updating "value" */

		err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);

	} else {

		rcu_read_lock();

		if (map->ops->map_lookup_elem_sys_only)

			ptr = map->ops->map_lookup_elem_sys_only(map, key);

		else

			ptr = map->ops->map_lookup_elem(map, key);

		if (IS_ERR(ptr)) {

			err = PTR_ERR(ptr);

		} else if (!ptr) {

			err = -ENOENT;

		} else {

			err = 0;

			if (attr->flags & BPF_F_LOCK)

				/* lock 'ptr' and copy everything but lock */

				copy_map_value_locked(map, value, ptr, true);

			else

				copy_map_value(map, value, ptr);

			/* mask lock, since value wasn't zero inited */

			check_and_init_map_lock(map, value);

		}

		rcu_read_unlock();

	}

	this_cpu_dec(bpf_prog_active);

	preempt_enable();



done:

	err = bpf_map_copy_value(map, key, value, attr->flags);

	if (err)

		goto free_value;
@@ -946,16 +1033,6 @@ err_put: 
	return err;

}



static void maybe_wait_bpf_programs(struct bpf_map *map)

{

	/* Wait for any running BPF programs to complete so that

	 * userspace, when we return to it, knows that all programs

	 * that could be running use the new map value.

	 */

	if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||

	    map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)

		synchronize_rcu();

}



#define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
@@ -1011,61 +1088,8 @@ static int map_update_elem(union bpf_attr *attr) 
	if (copy_from_user(value, uvalue, value_size) != 0)

		goto free_value;



	/* Need to create a kthread, thus must support schedule */

	if (bpf_map_is_dev_bound(map)) {

		err = bpf_map_offload_update_elem(map, key, value, attr->flags);

		goto out;

	} else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||

		   map->map_type == BPF_MAP_TYPE_SOCKHASH ||

		   map->map_type == BPF_MAP_TYPE_SOCKMAP ||

		   map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {

		err = map->ops->map_update_elem(map, key, value, attr->flags);

		goto out;

	} else if (IS_FD_PROG_ARRAY(map)) {

		err = bpf_fd_array_map_update_elem(map, f.file, key, value,

						   attr->flags);

		goto out;

	}

	err = bpf_map_update_value(map, f, key, value, attr->flags);



	/* must increment bpf_prog_active to avoid kprobe+bpf triggering from

	 * inside bpf map update or delete otherwise deadlocks are possible

	 */

	preempt_disable();

	__this_cpu_inc(bpf_prog_active);

	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||

	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {

		err = bpf_percpu_hash_update(map, key, value, attr->flags);

	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {

		err = bpf_percpu_array_update(map, key, value, attr->flags);

	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {

		err = bpf_percpu_cgroup_storage_update(map, key, value,

						       attr->flags);

	} else if (IS_FD_ARRAY(map)) {

		rcu_read_lock();

		err = bpf_fd_array_map_update_elem(map, f.file, key, value,

						   attr->flags);

		rcu_read_unlock();

	} else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {

		rcu_read_lock();

		err = bpf_fd_htab_map_update_elem(map, f.file, key, value,

						  attr->flags);

		rcu_read_unlock();

	} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {

		/* rcu_read_lock() is not needed */

		err = bpf_fd_reuseport_array_update_elem(map, key, value,

							 attr->flags);

	} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||

		   map->map_type == BPF_MAP_TYPE_STACK) {

		err = map->ops->map_push_elem(map, value, attr->flags);

	} else {

		rcu_read_lock();

		err = map->ops->map_update_elem(map, key, value, attr->flags);

		rcu_read_unlock();

	}

	__this_cpu_dec(bpf_prog_active);

	preempt_enable();

	maybe_wait_bpf_programs(map);

out:

free_value:

	kfree(value);

free_key:

CRA Git | Maintained and supported by SUSTech CRA and CCSE