Commit 96049f3a authored by Alexei Starovoitov's avatar Alexei Starovoitov Committed by Daniel Borkmann
Browse files

bpf: introduce BPF_F_LOCK flag 



Introduce BPF_F_LOCK flag for map_lookup and map_update syscall commands
and for map_update() helper function.
In all these cases take a lock of existing element (which was provided
in BTF description) before copying (in or out) the rest of map value.

Implementation details that are part of uapi:

Array:
The array map takes the element lock for lookup/update.

Hash:
hash map also takes the lock for lookup/update and tries to avoid the bucket lock.
If old element exists it takes the element lock and updates the element in place.
If element doesn't exist it allocates new one and inserts into hash table
while holding the bucket lock.
In rare case the hashmap has to take both the bucket lock and the element lock
to update old value in place.

Cgroup local storage:
It is similar to array. update in place and lookup are done with lock taken.

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parent ab963beb

include/linux/bpf.h

+2 −0
Original line number Diff line number Diff line
@@ -119,6 +119,8 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) 
		memcpy(dst, src, map->value_size);

	}

}

void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,

			   bool lock_src);



struct bpf_offload_dev;

struct bpf_offloaded_map;

include/uapi/linux/bpf.h

+1 −0
Original line number Diff line number Diff line
@@ -267,6 +267,7 @@ enum bpf_attach_type { 
#define BPF_ANY		0 /* create new element or update existing */

#define BPF_NOEXIST	1 /* create new element if it didn't exist */

#define BPF_EXIST	2 /* update existing element */

#define BPF_F_LOCK	4 /* spin_lock-ed map_lookup/map_update */



/* flags for BPF_MAP_CREATE command */

#define BPF_F_NO_PREALLOC	(1U << 0)

kernel/bpf/arraymap.c

+16 −8
Original line number Diff line number Diff line
@@ -253,8 +253,9 @@ static int array_map_update_elem(struct bpf_map *map, void *key, void *value, 
{

	struct bpf_array *array = container_of(map, struct bpf_array, map);

	u32 index = *(u32 *)key;

	char *val;



	if (unlikely(map_flags > BPF_EXIST))

	if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST))

		/* unknown flags */

		return -EINVAL;
@@ -262,18 +263,25 @@ static int array_map_update_elem(struct bpf_map *map, void *key, void *value, 
		/* all elements were pre-allocated, cannot insert a new one */

		return -E2BIG;



	if (unlikely(map_flags == BPF_NOEXIST))

	if (unlikely(map_flags & BPF_NOEXIST))

		/* all elements already exist */

		return -EEXIST;



	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)

	if (unlikely((map_flags & BPF_F_LOCK) &&

		     !map_value_has_spin_lock(map)))

		return -EINVAL;



	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {

		memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]),

		       value, map->value_size);

	} else {

		val = array->value +

			array->elem_size * (index & array->index_mask);

		if (map_flags & BPF_F_LOCK)

			copy_map_value_locked(map, val, value, false);

		else

		copy_map_value(map,

			       array->value +

			       array->elem_size * (index & array->index_mask),

			       value);

			copy_map_value(map, val, value);

	}

	return 0;

}

kernel/bpf/hashtab.c

+39 −3
Original line number Diff line number Diff line
@@ -804,11 +804,11 @@ dec_count: 
static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old,

		       u64 map_flags)

{

	if (l_old && map_flags == BPF_NOEXIST)

	if (l_old && (map_flags & ~BPF_F_LOCK) == BPF_NOEXIST)

		/* elem already exists */

		return -EEXIST;



	if (!l_old && map_flags == BPF_EXIST)

	if (!l_old && (map_flags & ~BPF_F_LOCK) == BPF_EXIST)

		/* elem doesn't exist, cannot update it */

		return -ENOENT;
@@ -827,7 +827,7 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value, 
	u32 key_size, hash;

	int ret;



	if (unlikely(map_flags > BPF_EXIST))

	if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST))

		/* unknown flags */

		return -EINVAL;
@@ -840,6 +840,28 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value, 
	b = __select_bucket(htab, hash);

	head = &b->head;



	if (unlikely(map_flags & BPF_F_LOCK)) {

		if (unlikely(!map_value_has_spin_lock(map)))

			return -EINVAL;

		/* find an element without taking the bucket lock */

		l_old = lookup_nulls_elem_raw(head, hash, key, key_size,

					      htab->n_buckets);

		ret = check_flags(htab, l_old, map_flags);

		if (ret)

			return ret;

		if (l_old) {

			/* grab the element lock and update value in place */

			copy_map_value_locked(map,

					      l_old->key + round_up(key_size, 8),

					      value, false);

			return 0;

		}

		/* fall through, grab the bucket lock and lookup again.

		 * 99.9% chance that the element won't be found,

		 * but second lookup under lock has to be done.

		 */

	}



	/* bpf_map_update_elem() can be called in_irq() */

	raw_spin_lock_irqsave(&b->lock, flags);
@@ -849,6 +871,20 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value, 
	if (ret)

		goto err;



	if (unlikely(l_old && (map_flags & BPF_F_LOCK))) {

		/* first lookup without the bucket lock didn't find the element,

		 * but second lookup with the bucket lock found it.

		 * This case is highly unlikely, but has to be dealt with:

		 * grab the element lock in addition to the bucket lock

		 * and update element in place

		 */

		copy_map_value_locked(map,

				      l_old->key + round_up(key_size, 8),

				      value, false);

		ret = 0;

		goto err;

	}



	l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false,

				l_old);

	if (IS_ERR(l_new)) {

kernel/bpf/helpers.c

+16 −0
Original line number Diff line number Diff line
@@ -301,6 +301,22 @@ const struct bpf_func_proto bpf_spin_unlock_proto = { 
	.arg1_type	= ARG_PTR_TO_SPIN_LOCK,

};



void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,

			   bool lock_src)

{

	struct bpf_spin_lock *lock;



	if (lock_src)

		lock = src + map->spin_lock_off;

	else

		lock = dst + map->spin_lock_off;

	preempt_disable();

	____bpf_spin_lock(lock);

	copy_map_value(map, dst, src);

	____bpf_spin_unlock(lock);

	preempt_enable();

}



#ifdef CONFIG_CGROUPS

BPF_CALL_0(bpf_get_current_cgroup_id)

{

CRA Git | Maintained and supported by SUSTech CRA and CCSE