Commit 23a1f682 authored by David S. Miller's avatar David S. Miller
Browse files

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next 



Daniel Borkmann says:

====================
pull-request: bpf-next 2020-10-01

The following pull-request contains BPF updates for your *net-next* tree.

We've added 90 non-merge commits during the last 8 day(s) which contain
a total of 103 files changed, 7662 insertions(+), 1894 deletions(-).

Note that once bpf(/net) tree gets merged into net-next, there will be a small
merge conflict in tools/lib/bpf/btf.c between commit 12450081 ("libbpf: Fix
native endian assumption when parsing BTF") from the bpf tree and the commit
3289959b ("libbpf: Support BTF loading and raw data output in both endianness")
from the bpf-next tree. Correct resolution would be to stick with bpf-next, it
should look like:

  [...]
        /* check BTF magic */
        if (fread(&magic, 1, sizeof(magic), f) < sizeof(magic)) {
                err = -EIO;
                goto err_out;
        }
        if (magic != BTF_MAGIC && magic != bswap_16(BTF_MAGIC)) {
                /* definitely not a raw BTF */
                err = -EPROTO;
                goto err_out;
        }

        /* get file size */
  [...]

The main changes are:

1) Add bpf_snprintf_btf() and bpf_seq_printf_btf() helpers to support displaying
   BTF-based kernel data structures out of BPF programs, from Alan Maguire.

2) Speed up RCU tasks trace grace periods by a factor of 50 & fix a few race
   conditions exposed by it. It was discussed to take these via BPF and
   networking tree to get better testing exposure, from Paul E. McKenney.

3) Support multi-attach for freplace programs, needed for incremental attachment
   of multiple XDP progs using libxdp dispatcher model, from Toke Høiland-Jørgensen.

4) libbpf support for appending new BTF types at the end of BTF object, allowing
   intrusive changes of prog's BTF (useful for future linking), from Andrii Nakryiko.

5) Several BPF helper improvements e.g. avoid atomic op in cookie generator and add
   a redirect helper into neighboring subsys, from Daniel Borkmann.

6) Allow map updates on sockmaps from bpf_iter context in order to migrate sockmaps
   from one to another, from Lorenz Bauer.

7) Fix 32 bit to 64 bit assignment from latest alu32 bounds tracking which caused
   a verifier issue due to type downgrade to scalar, from John Fastabend.

8) Follow-up on tail-call support in BPF subprogs which optimizes x64 JIT prologue
   and epilogue sections, from Maciej Fijalkowski.

9) Add an option to perf RB map to improve sharing of event entries by avoiding remove-
   on-close behavior. Also, add BPF_PROG_TEST_RUN for raw_tracepoint, from Song Liu.

10) Fix a crash in AF_XDP's socket_release when memory allocation for UMEMs fails,
    from Magnus Karlsson.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 7c89d9d9 6208689f

arch/x86/net/bpf_jit_comp.c

+23 −12
Original line number Diff line number Diff line
@@ -281,6 +281,7 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, 
	EMIT1(0x55);             /* push rbp */

	EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */

	/* sub rsp, rounded_stack_depth */

	if (stack_depth)

		EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8));

	if (tail_call_reachable)

		EMIT1(0x50);         /* push rax */
@@ -407,9 +408,9 @@ static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, 
	int tcc_off = -4 - round_up(stack_depth, 8);

	u8 *prog = *pprog;

	int pop_bytes = 0;

	int off1 = 49;

	int off2 = 38;

	int off3 = 16;

	int off1 = 42;

	int off2 = 31;

	int off3 = 9;

	int cnt = 0;



	/* count the additional bytes used for popping callee regs from stack
@@ -421,6 +422,12 @@ static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, 
	off2 += pop_bytes;

	off3 += pop_bytes;



	if (stack_depth) {

		off1 += 7;

		off2 += 7;

		off3 += 7;

	}



	/*

	 * rdi - pointer to ctx

	 * rsi - pointer to bpf_array
@@ -465,6 +472,7 @@ static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, 
	prog = *pprog;



	EMIT1(0x58);                              /* pop rax */

	if (stack_depth)

		EMIT3_off32(0x48, 0x81, 0xC4,     /* add rsp, sd */

			    round_up(stack_depth, 8));
@@ -491,7 +499,7 @@ static void emit_bpf_tail_call_direct(struct bpf_jit_poke_descriptor *poke, 
	int tcc_off = -4 - round_up(stack_depth, 8);

	u8 *prog = *pprog;

	int pop_bytes = 0;

	int off1 = 27;

	int off1 = 20;

	int poke_off;

	int cnt = 0;
@@ -506,10 +514,14 @@ static void emit_bpf_tail_call_direct(struct bpf_jit_poke_descriptor *poke, 
	 * total bytes for:

	 * - nop5/ jmpq $off

	 * - pop callee regs

	 * - sub rsp, $val

	 * - sub rsp, $val if depth > 0

	 * - pop rax

	 */

	poke_off = X86_PATCH_SIZE + pop_bytes + 7 + 1;

	poke_off = X86_PATCH_SIZE + pop_bytes + 1;

	if (stack_depth) {

		poke_off += 7;

		off1 += 7;

	}



	/*

	 * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
@@ -533,6 +545,7 @@ static void emit_bpf_tail_call_direct(struct bpf_jit_poke_descriptor *poke, 
	pop_callee_regs(pprog, callee_regs_used);

	prog = *pprog;

	EMIT1(0x58);                                  /* pop rax */

	if (stack_depth)

		EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8));



	memcpy(prog, ideal_nops[NOP_ATOMIC5], X86_PATCH_SIZE);
@@ -1441,8 +1454,6 @@ emit_jmp: 
			/* Update cleanup_addr */

			ctx->cleanup_addr = proglen;

			pop_callee_regs(&prog, callee_regs_used);

			if (tail_call_reachable)

				EMIT1(0x59); /* pop rcx, get rid of tail_call_cnt */

			EMIT1(0xC9);         /* leave */

			EMIT1(0xC3);         /* ret */

			break;

include/linux/bpf.h

+48 −13
Original line number Diff line number Diff line
@@ -292,6 +292,7 @@ enum bpf_arg_type { 
	ARG_PTR_TO_ALLOC_MEM,	/* pointer to dynamically allocated memory */

	ARG_PTR_TO_ALLOC_MEM_OR_NULL,	/* pointer to dynamically allocated memory or NULL */

	ARG_CONST_ALLOC_SIZE_OR_ZERO,	/* number of allocated bytes requested */

	ARG_PTR_TO_BTF_ID_SOCK_COMMON,	/* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */

	__BPF_ARG_TYPE_MAX,

};
@@ -382,8 +383,22 @@ enum bpf_reg_type { 
	PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */

	PTR_TO_TP_BUFFER,	 /* reg points to a writable raw tp's buffer */

	PTR_TO_XDP_SOCK,	 /* reg points to struct xdp_sock */

	PTR_TO_BTF_ID,		 /* reg points to kernel struct */

	PTR_TO_BTF_ID_OR_NULL,	 /* reg points to kernel struct or NULL */

	/* PTR_TO_BTF_ID points to a kernel struct that does not need

	 * to be null checked by the BPF program. This does not imply the

	 * pointer is _not_ null and in practice this can easily be a null

	 * pointer when reading pointer chains. The assumption is program

	 * context will handle null pointer dereference typically via fault

	 * handling. The verifier must keep this in mind and can make no

	 * assumptions about null or non-null when doing branch analysis.

	 * Further, when passed into helpers the helpers can not, without

	 * additional context, assume the value is non-null.

	 */

	PTR_TO_BTF_ID,

	/* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not

	 * been checked for null. Used primarily to inform the verifier

	 * an explicit null check is required for this struct.

	 */

	PTR_TO_BTF_ID_OR_NULL,

	PTR_TO_MEM,		 /* reg points to valid memory region */

	PTR_TO_MEM_OR_NULL,	 /* reg points to valid memory region or NULL */

	PTR_TO_RDONLY_BUF,	 /* reg points to a readonly buffer */
@@ -591,6 +606,13 @@ struct bpf_trampoline { 
	struct bpf_ksym ksym;

};



struct bpf_attach_target_info {

	struct btf_func_model fmodel;

	long tgt_addr;

	const char *tgt_name;

	const struct btf_type *tgt_type;

};



#define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */



struct bpf_dispatcher_prog {
@@ -618,9 +640,10 @@ static __always_inline unsigned int bpf_dispatcher_nop_func( 
	return bpf_func(ctx, insnsi);

}

#ifdef CONFIG_BPF_JIT

struct bpf_trampoline *bpf_trampoline_lookup(u64 key);

int bpf_trampoline_link_prog(struct bpf_prog *prog);

int bpf_trampoline_unlink_prog(struct bpf_prog *prog);

int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);

int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);

struct bpf_trampoline *bpf_trampoline_get(u64 key,

					  struct bpf_attach_target_info *tgt_info);

void bpf_trampoline_put(struct bpf_trampoline *tr);

#define BPF_DISPATCHER_INIT(_name) {				\

	.mutex = __MUTEX_INITIALIZER(_name.mutex),		\
@@ -665,17 +688,20 @@ void bpf_image_ksym_del(struct bpf_ksym *ksym); 
void bpf_ksym_add(struct bpf_ksym *ksym);

void bpf_ksym_del(struct bpf_ksym *ksym);

#else

static inline struct bpf_trampoline *bpf_trampoline_lookup(u64 key)

static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,

					   struct bpf_trampoline *tr)

{

	return NULL;

	return -ENOTSUPP;

}

static inline int bpf_trampoline_link_prog(struct bpf_prog *prog)

static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,

					     struct bpf_trampoline *tr)

{

	return -ENOTSUPP;

}

static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog)

static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,

							struct bpf_attach_target_info *tgt_info)

{

	return -ENOTSUPP;

	return ERR_PTR(-EOPNOTSUPP);

}

static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {}

#define DEFINE_BPF_DISPATCHER(name)
@@ -739,7 +765,11 @@ struct bpf_prog_aux { 
	u32 max_rdonly_access;

	u32 max_rdwr_access;

	const struct bpf_ctx_arg_aux *ctx_arg_info;

	struct bpf_prog *linked_prog;

	struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */

	struct bpf_prog *dst_prog;

	struct bpf_trampoline *dst_trampoline;

	enum bpf_prog_type saved_dst_prog_type;

	enum bpf_attach_type saved_dst_attach_type;

	bool verifier_zext; /* Zero extensions has been inserted by verifier. */

	bool offload_requested;

	bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
@@ -747,7 +777,6 @@ struct bpf_prog_aux { 
	bool sleepable;

	bool tail_call_reachable;

	enum bpf_tramp_prog_type trampoline_prog_type;

	struct bpf_trampoline *trampoline;

	struct hlist_node tramp_hlist;

	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */

	const struct btf_type *attach_func_proto;
@@ -1340,6 +1369,8 @@ int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, 
	      union bpf_attr __user *uattr);

void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);



struct btf *bpf_get_btf_vmlinux(void);



/* Map specifics */

struct xdp_buff;

struct sk_buff;
@@ -1381,6 +1412,9 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog, 
int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,

				     const union bpf_attr *kattr,

				     union bpf_attr __user *uattr);

int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,

			     const union bpf_attr *kattr,

			     union bpf_attr __user *uattr);

bool btf_ctx_access(int off, int size, enum bpf_access_type type,

		    const struct bpf_prog *prog,

		    struct bpf_insn_access_aux *info);
@@ -1402,7 +1436,7 @@ int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, 
			     struct bpf_reg_state *regs);

int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,

			  struct bpf_reg_state *reg);

int btf_check_type_match(struct bpf_verifier_env *env, struct bpf_prog *prog,

int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,

			 struct btf *btf, const struct btf_type *t);



struct bpf_prog *bpf_prog_by_id(u32 id);
@@ -1793,6 +1827,7 @@ extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto; 
extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto;

extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;

extern const struct bpf_func_proto bpf_copy_from_user_proto;

extern const struct bpf_func_proto bpf_snprintf_btf_proto;



const struct bpf_func_proto *bpf_tracing_func_proto(

	enum bpf_func_id func_id, const struct bpf_prog *prog);

include/linux/bpf_verifier.h

+16 −2
Original line number Diff line number Diff line
@@ -347,8 +347,9 @@ static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log) 


static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)

{

	return (log->level && log->ubuf && !bpf_verifier_log_full(log)) ||

		log->level == BPF_LOG_KERNEL;

	return log &&

		((log->level && log->ubuf && !bpf_verifier_log_full(log)) ||

		 log->level == BPF_LOG_KERNEL);

}



#define BPF_MAX_SUBPROGS 256
@@ -449,4 +450,17 @@ bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt); 
int check_ctx_reg(struct bpf_verifier_env *env,

		  const struct bpf_reg_state *reg, int regno);



/* this lives here instead of in bpf.h because it needs to dereference tgt_prog */

static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog,

					     u32 btf_id)

{

        return tgt_prog ? (((u64)tgt_prog->aux->id) << 32 | btf_id) : btf_id;

}



int bpf_check_attach_target(struct bpf_verifier_log *log,

			    const struct bpf_prog *prog,

			    const struct bpf_prog *tgt_prog,

			    u32 btf_id,

			    struct bpf_attach_target_info *tgt_info);



#endif /* _LINUX_BPF_VERIFIER_H */

include/linux/btf.h

+39 −0
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 


#include <linux/types.h>

#include <uapi/linux/btf.h>

#include <uapi/linux/bpf.h>



#define BTF_TYPE_EMIT(type) ((void)(type *)0)
@@ -13,6 +14,7 @@ struct btf; 
struct btf_member;

struct btf_type;

union bpf_attr;

struct btf_show;



extern const struct file_operations btf_fops;
@@ -46,8 +48,45 @@ int btf_get_info_by_fd(const struct btf *btf, 
const struct btf_type *btf_type_id_size(const struct btf *btf,

					u32 *type_id,

					u32 *ret_size);



/*

 * Options to control show behaviour.

 *	- BTF_SHOW_COMPACT: no formatting around type information

 *	- BTF_SHOW_NONAME: no struct/union member names/types

 *	- BTF_SHOW_PTR_RAW: show raw (unobfuscated) pointer values;

 *	  equivalent to %px.

 *	- BTF_SHOW_ZERO: show zero-valued struct/union members; they

 *	  are not displayed by default

 *	- BTF_SHOW_UNSAFE: skip use of bpf_probe_read() to safely read

 *	  data before displaying it.

 */

#define BTF_SHOW_COMPACT	BTF_F_COMPACT

#define BTF_SHOW_NONAME		BTF_F_NONAME

#define BTF_SHOW_PTR_RAW	BTF_F_PTR_RAW

#define BTF_SHOW_ZERO		BTF_F_ZERO

#define BTF_SHOW_UNSAFE		(1ULL << 4)



void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj,

		       struct seq_file *m);

int btf_type_seq_show_flags(const struct btf *btf, u32 type_id, void *obj,

			    struct seq_file *m, u64 flags);



/*

 * Copy len bytes of string representation of obj of BTF type_id into buf.

 *

 * @btf: struct btf object

 * @type_id: type id of type obj points to

 * @obj: pointer to typed data

 * @buf: buffer to write to

 * @len: maximum length to write to buf

 * @flags: show options (see above)

 *

 * Return: length that would have been/was copied as per snprintf, or

 *	   negative error.

 */

int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj,

			   char *buf, int len, u64 flags);



int btf_get_fd_by_id(u32 id);

u32 btf_id(const struct btf *btf);

bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s,

include/linux/cookie.h

0 → 100644
+51 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

#ifndef __LINUX_COOKIE_H

#define __LINUX_COOKIE_H



#include <linux/atomic.h>

#include <linux/percpu.h>

#include <asm/local.h>



struct pcpu_gen_cookie {

	local_t nesting;

	u64 last;

} __aligned(16);



struct gen_cookie {

	struct pcpu_gen_cookie __percpu *local;

	atomic64_t forward_last ____cacheline_aligned_in_smp;

	atomic64_t reverse_last;

};



#define COOKIE_LOCAL_BATCH	4096



#define DEFINE_COOKIE(name)						\

	static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name);	\

	static struct gen_cookie name = {				\

		.local		= &__##name,				\

		.forward_last	= ATOMIC64_INIT(0),			\

		.reverse_last	= ATOMIC64_INIT(0),			\

	}



static __always_inline u64 gen_cookie_next(struct gen_cookie *gc)

{

	struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local);

	u64 val;



	if (likely(local_inc_return(&local->nesting) == 1)) {

		val = local->last;

		if (__is_defined(CONFIG_SMP) &&

		    unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) {

			s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH,

						       &gc->forward_last);

			val = next - COOKIE_LOCAL_BATCH;

		}

		local->last = ++val;

	} else {

		val = atomic64_dec_return(&gc->reverse_last);

	}

	local_dec(&local->nesting);

	return val;

}



#endif /* __LINUX_COOKIE_H */

CRA Git | Maintained and supported by SUSTech CRA and CCSE