Commit 3aac1ead authored by Toke Høiland-Jørgensen's avatar Toke Høiland-Jørgensen Committed by Alexei Starovoitov
Browse files

bpf: Move prog->aux->linked_prog and trampoline into bpf_link on attach 



In preparation for allowing multiple attachments of freplace programs, move
the references to the target program and trampoline into the
bpf_tracing_link structure when that is created. To do this atomically,
introduce a new mutex in prog->aux to protect writing to the two pointers
to target prog and trampoline, and rename the members to make it clear that
they are related.

With this change, it is no longer possible to attach the same tracing
program multiple times (detaching in-between), since the reference from the
tracing program to the target disappears on the first attach. However,
since the next patch will let the caller supply an attach target, that will
also make it possible to attach to the same place multiple times.

Signed-off-by: default avatarToke Høiland-Jørgensen <toke@redhat.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarAndrii Nakryiko <andriin@fb.com>
Link: https://lore.kernel.org/bpf/160138355059.48470.2503076992210324984.stgit@toke.dk
parent 85e3f318

include/linux/bpf.h

+9 −6
Original line number Diff line number Diff line
@@ -640,8 +640,8 @@ static __always_inline unsigned int bpf_dispatcher_nop_func( 
	return bpf_func(ctx, insnsi);

}

#ifdef CONFIG_BPF_JIT

int bpf_trampoline_link_prog(struct bpf_prog *prog);

int bpf_trampoline_unlink_prog(struct bpf_prog *prog);

int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);

int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);

struct bpf_trampoline *bpf_trampoline_get(u64 key,

					  struct bpf_attach_target_info *tgt_info);

void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -688,11 +688,13 @@ void bpf_image_ksym_del(struct bpf_ksym *ksym); 
void bpf_ksym_add(struct bpf_ksym *ksym);

void bpf_ksym_del(struct bpf_ksym *ksym);

#else

static inline int bpf_trampoline_link_prog(struct bpf_prog *prog)

static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,

					   struct bpf_trampoline *tr)

{

	return -ENOTSUPP;

}

static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog)

static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,

					     struct bpf_trampoline *tr)

{

	return -ENOTSUPP;

}
@@ -763,7 +765,9 @@ struct bpf_prog_aux { 
	u32 max_rdonly_access;

	u32 max_rdwr_access;

	const struct bpf_ctx_arg_aux *ctx_arg_info;

	struct bpf_prog *linked_prog;

	struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */

	struct bpf_prog *dst_prog;

	struct bpf_trampoline *dst_trampoline;

	bool verifier_zext; /* Zero extensions has been inserted by verifier. */

	bool offload_requested;

	bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
@@ -771,7 +775,6 @@ struct bpf_prog_aux { 
	bool sleepable;

	bool tail_call_reachable;

	enum bpf_tramp_prog_type trampoline_prog_type;

	struct bpf_trampoline *trampoline;

	struct hlist_node tramp_hlist;

	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */

	const struct btf_type *attach_func_proto;

kernel/bpf/btf.c

+3 −3
Original line number Diff line number Diff line
@@ -4428,7 +4428,7 @@ errout: 


struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog)

{

	struct bpf_prog *tgt_prog = prog->aux->linked_prog;

	struct bpf_prog *tgt_prog = prog->aux->dst_prog;



	if (tgt_prog) {

		return tgt_prog->aux->btf;
@@ -4455,7 +4455,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, 
		    struct bpf_insn_access_aux *info)

{

	const struct btf_type *t = prog->aux->attach_func_proto;

	struct bpf_prog *tgt_prog = prog->aux->linked_prog;

	struct bpf_prog *tgt_prog = prog->aux->dst_prog;

	struct btf *btf = bpf_prog_get_target_btf(prog);

	const char *tname = prog->aux->attach_func_name;

	struct bpf_verifier_log *log = info->log;
@@ -5281,7 +5281,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, 
		return -EFAULT;

	}

	if (prog_type == BPF_PROG_TYPE_EXT)

		prog_type = prog->aux->linked_prog->type;

		prog_type = prog->aux->dst_prog->type;



	t = btf_type_by_id(btf, t->type);

	if (!t || !btf_type_is_func_proto(t)) {

kernel/bpf/core.c

+6 −3
Original line number Diff line number Diff line
@@ -99,6 +99,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag 


	INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode);

	mutex_init(&fp->aux->used_maps_mutex);

	mutex_init(&fp->aux->dst_mutex);



	return fp;

}
@@ -255,6 +256,7 @@ void __bpf_prog_free(struct bpf_prog *fp) 
{

	if (fp->aux) {

		mutex_destroy(&fp->aux->used_maps_mutex);

		mutex_destroy(&fp->aux->dst_mutex);

		free_percpu(fp->aux->stats);

		kfree(fp->aux->poke_tab);

		kfree(fp->aux);
@@ -2138,7 +2140,8 @@ static void bpf_prog_free_deferred(struct work_struct *work) 
	if (aux->prog->has_callchain_buf)

		put_callchain_buffers();

#endif

	bpf_trampoline_put(aux->trampoline);

	if (aux->dst_trampoline)

		bpf_trampoline_put(aux->dst_trampoline);

	for (i = 0; i < aux->func_cnt; i++)

		bpf_jit_free(aux->func[i]);

	if (aux->func_cnt) {
@@ -2154,8 +2157,8 @@ void bpf_prog_free(struct bpf_prog *fp) 
{

	struct bpf_prog_aux *aux = fp->aux;



	if (aux->linked_prog)

		bpf_prog_put(aux->linked_prog);

	if (aux->dst_prog)

		bpf_prog_put(aux->dst_prog);

	INIT_WORK(&aux->work, bpf_prog_free_deferred);

	schedule_work(&aux->work);

}

kernel/bpf/preload/iterators/iterators.bpf.c

+2 −2
Original line number Diff line number Diff line
@@ -42,7 +42,7 @@ struct bpf_prog_aux { 
	__u32 id;

	char name[16];

	const char *attach_func_name;

	struct bpf_prog *linked_prog;

	struct bpf_prog *dst_prog;

	struct bpf_func_info *func_info;

	struct btf *btf;

};
@@ -108,7 +108,7 @@ int dump_bpf_prog(struct bpf_iter__bpf_prog *ctx) 


	BPF_SEQ_PRINTF(seq, "%4u %-16s %s %s\n", aux->id,

		       get_name(aux->btf, aux->func_info[0].type_id, aux->name),

		       aux->attach_func_name, aux->linked_prog->aux->name);

		       aux->attach_func_name, aux->dst_prog->aux->name);

	return 0;

}

char LICENSE[] SEC("license") = "GPL";

kernel/bpf/preload/iterators/iterators.skel.h

+223 −221

File changed.

Preview size limit exceeded, changes collapsed.

CRA Git | Maintained and supported by SUSTech CRA and CCSE