Commit 1e6c62a8 authored by Alexei Starovoitov's avatar Alexei Starovoitov Committed by Daniel Borkmann
Browse files

bpf: Introduce sleepable BPF programs 



Introduce sleepable BPF programs that can request such property for themselves
via BPF_F_SLEEPABLE flag at program load time. In such case they will be able
to use helpers like bpf_copy_from_user() that might sleep. At present only
fentry/fexit/fmod_ret and lsm programs can request to be sleepable and only
when they are attached to kernel functions that are known to allow sleeping.

The non-sleepable programs are relying on implicit rcu_read_lock() and
migrate_disable() to protect life time of programs, maps that they use and
per-cpu kernel structures used to pass info between bpf programs and the
kernel. The sleepable programs cannot be enclosed into rcu_read_lock().
migrate_disable() maps to preempt_disable() in non-RT kernels, so the progs
should not be enclosed in migrate_disable() as well. Therefore
rcu_read_lock_trace is used to protect the life time of sleepable progs.

There are many networking and tracing program types. In many cases the
'struct bpf_prog *' pointer itself is rcu protected within some other kernel
data structure and the kernel code is using rcu_dereference() to load that
program pointer and call BPF_PROG_RUN() on it. All these cases are not touched.
Instead sleepable bpf programs are allowed with bpf trampoline only. The
program pointers are hard-coded into generated assembly of bpf trampoline and
synchronize_rcu_tasks_trace() is used to protect the life time of the program.
The same trampoline can hold both sleepable and non-sleepable progs.

When rcu_read_lock_trace is held it means that some sleepable bpf program is
running from bpf trampoline. Those programs can use bpf arrays and preallocated
hash/lru maps. These map types are waiting on programs to complete via
synchronize_rcu_tasks_trace();

Updates to trampoline now has to do synchronize_rcu_tasks_trace() and
synchronize_rcu_tasks() to wait for sleepable progs to finish and for
trampoline assembly to finish.

This is the first step of introducing sleepable progs. Eventually dynamically
allocated hash maps can be allowed and networking program types can become
sleepable too.

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Reviewed-by: default avatarJosef Bacik <josef@toxicpanda.com>
Acked-by: default avatarAndrii Nakryiko <andriin@fb.com>
Acked-by: default avatarKP Singh <kpsingh@google.com>
Link: https://lore.kernel.org/bpf/20200827220114.69225-3-alexei.starovoitov@gmail.com
parent 76cd6173

arch/x86/net/bpf_jit_comp.c

+21 −11
Original line number Diff line number Diff line
@@ -1379,10 +1379,15 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, 
	u8 *prog = *pprog;

	int cnt = 0;



	if (p->aux->sleepable) {

		if (emit_call(&prog, __bpf_prog_enter_sleepable, prog))

			return -EINVAL;

	} else {

		if (emit_call(&prog, __bpf_prog_enter, prog))

			return -EINVAL;

		/* remember prog start time returned by __bpf_prog_enter */

		emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);

	}



	/* arg1: lea rdi, [rbp - stack_size] */

	EMIT4(0x48, 0x8D, 0x7D, -stack_size);
@@ -1402,6 +1407,10 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, 
	if (mod_ret)

		emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);



	if (p->aux->sleepable) {

		if (emit_call(&prog, __bpf_prog_exit_sleepable, prog))

			return -EINVAL;

	} else {

		/* arg1: mov rdi, progs[i] */

		emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32,

			       (u32) (long) p);
@@ -1409,6 +1418,7 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, 
		emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);

		if (emit_call(&prog, __bpf_prog_exit, prog))

			return -EINVAL;

	}



	*pprog = prog;

	return 0;

include/linux/bpf.h

+3 −0
Original line number Diff line number Diff line
@@ -539,6 +539,8 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, 
/* these two functions are called from generated trampoline */

u64 notrace __bpf_prog_enter(void);

void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start);

void notrace __bpf_prog_enter_sleepable(void);

void notrace __bpf_prog_exit_sleepable(void);



struct bpf_ksym {

	unsigned long		 start;
@@ -734,6 +736,7 @@ struct bpf_prog_aux { 
	bool offload_requested;

	bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */

	bool func_proto_unreliable;

	bool sleepable;

	enum bpf_tramp_prog_type trampoline_prog_type;

	struct bpf_trampoline *trampoline;

	struct hlist_node tramp_hlist;

include/uapi/linux/bpf.h

+8 −0
Original line number Diff line number Diff line
@@ -346,6 +346,14 @@ enum bpf_link_type { 
/* The verifier internal test flag. Behavior is undefined */

#define BPF_F_TEST_STATE_FREQ	(1U << 3)



/* If BPF_F_SLEEPABLE is used in BPF_PROG_LOAD command, the verifier will

 * restrict map and helper usage for such programs. Sleepable BPF programs can

 * only be attached to hooks where kernel execution context allows sleeping.

 * Such programs are allowed to use helpers that may sleep like

 * bpf_copy_from_user().

 */

#define BPF_F_SLEEPABLE		(1U << 4)



/* When BPF ldimm64's insn[0].src_reg != 0 then this can have

 * two extensions:

 *

init/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -1691,6 +1691,7 @@ config BPF_SYSCALL 
	bool "Enable bpf() system call"

	select BPF

	select IRQ_WORK

	select TASKS_TRACE_RCU

	default n

	help

	  Enable the bpf() system call that allows to manipulate eBPF

kernel/bpf/arraymap.c

+1 −0
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ 
#include <linux/filter.h>

#include <linux/perf_event.h>

#include <uapi/linux/btf.h>

#include <linux/rcupdate_trace.h>



#include "map_in_map.h"

CRA Git | Maintained and supported by SUSTech CRA and CCSE