Commit 43ed375f authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'queue_stack_maps' 



Mauricio Vasquez says:

====================
In some applications this is needed have a pool of free elements, for
example the list of free L4 ports in a SNAT.  None of the current maps allow
to do it as it is not possible to get any element without having they key
it is associated to, even if it were possible, the lack of locking mecanishms in
eBPF would do it almost impossible to be implemented without data races.

This patchset implements two new kind of eBPF maps: queue and stack.
Those maps provide to eBPF programs the peek, push and pop operations, and for
userspace applications a new bpf_map_lookup_and_delete_elem() is added.

Signed-off-by: default avatarMauricio Vasquez B <mauricio.vasquez@polito.it>

v2 -> v3:
 - Remove "almost dead code" in syscall.c
 - Remove unnecessary copy_from_user in bpf_map_lookup_and_delete_elem
 - Rebase

v1 -> v2:
 - Put ARG_PTR_TO_UNINIT_MAP_VALUE logic into a separated patch
 - Fix missing __this_cpu_dec & preempt_enable calls in kernel/bpf/syscall.c

RFC v4 -> v1:
 - Remove roundup to power of 2 in memory allocation
 - Remove count and use a free slot to check if queue/stack is empty
 - Use if + assigment for wrapping indexes
 - Fix some minor style issues
 - Squash two patches together

RFC v3 -> RFC v4:
 - Revert renaming of kernel/bpf/stackmap.c
 - Remove restriction on value size
 - Remove len arguments from peek/pop helpers
 - Add new ARG_PTR_TO_UNINIT_MAP_VALUE

RFC v2 -> RFC v3:
 - Return elements by value instead that by reference
 - Implement queue/stack base on array and head + tail indexes
 - Rename stack trace related files to avoid confusion and conflicts

RFC v1 -> RFC v2:
 - Create two separate maps instead of single one + flags
 - Implement bpf_map_lookup_and_delete syscall
 - Support peek operation
 - Define replacement policy through flags in the update() method
 - Add eBPF side tests
====================

Acked-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 3ddeac67 43b987d2

include/linux/bpf.h

+7 −0
Original line number Original line Diff line number Diff line
@@ -39,6 +39,9 @@ struct bpf_map_ops { 
	void *(*map_lookup_elem)(struct bpf_map *map, void *key);

	void *(*map_lookup_elem)(struct bpf_map *map, void *key);

	int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);

	int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);

	int (*map_delete_elem)(struct bpf_map *map, void *key);

	int (*map_delete_elem)(struct bpf_map *map, void *key);

	int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);

	int (*map_pop_elem)(struct bpf_map *map, void *value);

	int (*map_peek_elem)(struct bpf_map *map, void *value);





	/* funcs called by prog_array and perf_event_array map */

	/* funcs called by prog_array and perf_event_array map */

	void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,

	void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
@@ -138,6 +141,7 @@ enum bpf_arg_type { 
	ARG_CONST_MAP_PTR,	/* const argument used as pointer to bpf_map */

	ARG_CONST_MAP_PTR,	/* const argument used as pointer to bpf_map */

	ARG_PTR_TO_MAP_KEY,	/* pointer to stack used as map key */

	ARG_PTR_TO_MAP_KEY,	/* pointer to stack used as map key */

	ARG_PTR_TO_MAP_VALUE,	/* pointer to stack used as map value */

	ARG_PTR_TO_MAP_VALUE,	/* pointer to stack used as map value */

	ARG_PTR_TO_UNINIT_MAP_VALUE,	/* pointer to valid memory used to store a map value */





	/* the following constraints used to prototype bpf_memcmp() and other

	/* the following constraints used to prototype bpf_memcmp() and other

	 * functions that access data on eBPF program stack

	 * functions that access data on eBPF program stack
@@ -810,6 +814,9 @@ static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, 
extern const struct bpf_func_proto bpf_map_lookup_elem_proto;

extern const struct bpf_func_proto bpf_map_lookup_elem_proto;

extern const struct bpf_func_proto bpf_map_update_elem_proto;

extern const struct bpf_func_proto bpf_map_update_elem_proto;

extern const struct bpf_func_proto bpf_map_delete_elem_proto;

extern const struct bpf_func_proto bpf_map_delete_elem_proto;

extern const struct bpf_func_proto bpf_map_push_elem_proto;

extern const struct bpf_func_proto bpf_map_pop_elem_proto;

extern const struct bpf_func_proto bpf_map_peek_elem_proto;





extern const struct bpf_func_proto bpf_get_prandom_u32_proto;

extern const struct bpf_func_proto bpf_get_prandom_u32_proto;

extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;

extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;

include/linux/bpf_types.h

+3 −1
Original line number Original line Diff line number Diff line
@@ -51,7 +51,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_LRU_HASH, htab_lru_map_ops) 
BPF_MAP_TYPE(BPF_MAP_TYPE_LRU_PERCPU_HASH, htab_lru_percpu_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_LRU_PERCPU_HASH, htab_lru_percpu_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_LPM_TRIE, trie_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_LPM_TRIE, trie_map_ops)

#ifdef CONFIG_PERF_EVENTS

#ifdef CONFIG_PERF_EVENTS

BPF_MAP_TYPE(BPF_MAP_TYPE_STACK_TRACE, stack_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_STACK_TRACE, stack_trace_map_ops)

#endif

#endif

BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY_OF_MAPS, array_of_maps_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY_OF_MAPS, array_of_maps_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_HASH_OF_MAPS, htab_of_maps_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_HASH_OF_MAPS, htab_of_maps_map_ops)
@@ -69,3 +69,5 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_XSKMAP, xsk_map_ops) 
BPF_MAP_TYPE(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, reuseport_array_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, reuseport_array_ops)

#endif

#endif

#endif

#endif

BPF_MAP_TYPE(BPF_MAP_TYPE_QUEUE, queue_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_STACK, stack_map_ops)

include/uapi/linux/bpf.h

+29 −1
Original line number Original line Diff line number Diff line
@@ -103,6 +103,7 @@ enum bpf_cmd { 
	BPF_BTF_LOAD,

	BPF_BTF_LOAD,

	BPF_BTF_GET_FD_BY_ID,

	BPF_BTF_GET_FD_BY_ID,

	BPF_TASK_FD_QUERY,

	BPF_TASK_FD_QUERY,

	BPF_MAP_LOOKUP_AND_DELETE_ELEM,

};

};





enum bpf_map_type {

enum bpf_map_type {
@@ -128,6 +129,8 @@ enum bpf_map_type { 
	BPF_MAP_TYPE_CGROUP_STORAGE,

	BPF_MAP_TYPE_CGROUP_STORAGE,

	BPF_MAP_TYPE_REUSEPORT_SOCKARRAY,

	BPF_MAP_TYPE_REUSEPORT_SOCKARRAY,

	BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE,

	BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE,

	BPF_MAP_TYPE_QUEUE,

	BPF_MAP_TYPE_STACK,

};

};





enum bpf_prog_type {

enum bpf_prog_type {
@@ -462,6 +465,28 @@ union bpf_attr { 
 * 	Return

 * 	Return

 * 		0 on success, or a negative error in case of failure.

 * 		0 on success, or a negative error in case of failure.

 *

 *

 * int bpf_map_push_elem(struct bpf_map *map, const void *value, u64 flags)

 * 	Description

 * 		Push an element *value* in *map*. *flags* is one of:

 *

 * 		**BPF_EXIST**

 * 		If the queue/stack is full, the oldest element is removed to

 * 		make room for this.

 * 	Return

 * 		0 on success, or a negative error in case of failure.

 *

 * int bpf_map_pop_elem(struct bpf_map *map, void *value)

 * 	Description

 * 		Pop an element from *map*.

 * Return

 * 		0 on success, or a negative error in case of failure.

 *

 * int bpf_map_peek_elem(struct bpf_map *map, void *value)

 * 	Description

 * 		Get an element from *map* without removing it.

 * Return

 * 		0 on success, or a negative error in case of failure.

 *

 * int bpf_probe_read(void *dst, u32 size, const void *src)

 * int bpf_probe_read(void *dst, u32 size, const void *src)

 * 	Description

 * 	Description

 * 		For tracing programs, safely attempt to read *size* bytes from

 * 		For tracing programs, safely attempt to read *size* bytes from
@@ -2303,7 +2328,10 @@ union bpf_attr { 
	FN(skb_ancestor_cgroup_id),	\

	FN(skb_ancestor_cgroup_id),	\

	FN(sk_lookup_tcp),		\

	FN(sk_lookup_tcp),		\

	FN(sk_lookup_udp),		\

	FN(sk_lookup_udp),		\

	FN(sk_release),

	FN(sk_release),			\

	FN(map_push_elem),		\

	FN(map_pop_elem),		\

	FN(map_peek_elem),





/* integer value in 'imm' field of BPF_CALL instruction selects which helper

/* integer value in 'imm' field of BPF_CALL instruction selects which helper

 * function eBPF program intends to call

 * function eBPF program intends to call

kernel/bpf/Makefile

+1 −1
Original line number Original line Diff line number Diff line
@@ -3,7 +3,7 @@ obj-y := core.o 




obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o

obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o

obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o

obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o

obj-$(CONFIG_BPF_SYSCALL) += local_storage.o

obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o

obj-$(CONFIG_BPF_SYSCALL) += disasm.o

obj-$(CONFIG_BPF_SYSCALL) += disasm.o

obj-$(CONFIG_BPF_SYSCALL) += btf.o

obj-$(CONFIG_BPF_SYSCALL) += btf.o

ifeq ($(CONFIG_NET),y)

ifeq ($(CONFIG_NET),y)

kernel/bpf/core.c

+3 −0
Original line number Original line Diff line number Diff line
@@ -1783,6 +1783,9 @@ BPF_CALL_0(bpf_user_rnd_u32) 
const struct bpf_func_proto bpf_map_lookup_elem_proto __weak;

const struct bpf_func_proto bpf_map_lookup_elem_proto __weak;

const struct bpf_func_proto bpf_map_update_elem_proto __weak;

const struct bpf_func_proto bpf_map_update_elem_proto __weak;

const struct bpf_func_proto bpf_map_delete_elem_proto __weak;

const struct bpf_func_proto bpf_map_delete_elem_proto __weak;

const struct bpf_func_proto bpf_map_push_elem_proto __weak;

const struct bpf_func_proto bpf_map_pop_elem_proto __weak;

const struct bpf_func_proto bpf_map_peek_elem_proto __weak;





const struct bpf_func_proto bpf_get_prandom_u32_proto __weak;

const struct bpf_func_proto bpf_get_prandom_u32_proto __weak;

const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak;

const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak;

CRA Git | Maintained and supported by SUSTech CRA and CCSE