Commit b89c2998 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'non-jit-btf-func_info' 



Yonghong Song says:

====================
Commit 838e9690 ("bpf: Introduce bpf_func_info")
added bpf func info support. The userspace is able
to get better ksym's for bpf programs with jit, and
is able to print out func prototypes.

For a program containing func-to-func calls, the existing
implementation returns user specified number of function
calls and BTF types if jit is enabled. If the jit is not
enabled, it only returns the type for the main function.

This is undesirable. Interpreter may still be used
and we should keep feature identical regardless of
whether jit is enabled or not.
This patch fixed this discrepancy.

The following example shows bpftool output for
the bpf program in selftests test_btf_haskv.o when jit
is disabled:
  $ bpftool prog dump xlated id 1490
  int _dummy_tracepoint(struct dummy_tracepoint_args * arg):
     0: (85) call pc+2#__bpf_prog_run_args32
     1: (b7) r0 = 0
     2: (95) exit
  int test_long_fname_1(struct dummy_tracepoint_args * arg):
     3: (85) call pc+1#__bpf_prog_run_args32
     4: (95) exit
  int test_long_fname_2(struct dummy_tracepoint_args * arg):
     5: (b7) r2 = 0
     6: (63) *(u32 *)(r10 -4) = r2
     7: (79) r1 = *(u64 *)(r1 +8)
     8: (15) if r1 == 0x0 goto pc+9
     9: (bf) r2 = r10
    10: (07) r2 += -4
    11: (18) r1 = map[id:1173]
    13: (85) call bpf_map_lookup_elem#77088
    14: (15) if r0 == 0x0 goto pc+3
    15: (61) r1 = *(u32 *)(r0 +4)
    16: (07) r1 += 1
    17: (63) *(u32 *)(r0 +4) = r1
    18: (95) exit
  $ bpftool prog dump jited id 1490
    no instructions returned
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents d8f3e978 812dd689

include/linux/bpf.h

+4 −2
Original line number Original line Diff line number Diff line
@@ -299,7 +299,8 @@ struct bpf_prog_aux { 
	u32 max_pkt_offset;

	u32 max_pkt_offset;

	u32 stack_depth;

	u32 stack_depth;

	u32 id;

	u32 id;

	u32 func_cnt;

	u32 func_cnt; /* used by non-func prog as the number of func progs */

	u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */

	bool offload_requested;

	bool offload_requested;

	struct bpf_prog **func;

	struct bpf_prog **func;

	void *jit_data; /* JIT specific data. arch dependent */

	void *jit_data; /* JIT specific data. arch dependent */
@@ -317,7 +318,8 @@ struct bpf_prog_aux { 
#endif

#endif

	struct bpf_prog_offload *offload;

	struct bpf_prog_offload *offload;

	struct btf *btf;

	struct btf *btf;

	u32 type_id; /* type id for this prog/func */

	struct bpf_func_info *func_info;

	u32 func_info_cnt;

	union {

	union {

		struct work_struct work;

		struct work_struct work;

		struct rcu_head	rcu;

		struct rcu_head	rcu;

include/linux/bpf_verifier.h

+0 −1
Original line number Original line Diff line number Diff line
@@ -204,7 +204,6 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log) 
struct bpf_subprog_info {

struct bpf_subprog_info {

	u32 start; /* insn idx of function entry point */

	u32 start; /* insn idx of function entry point */

	u16 stack_depth; /* max. stack depth used by this function */

	u16 stack_depth; /* max. stack depth used by this function */

	u32 type_id; /* btf type_id for this subprog */

};

};





/* single container for all structs

/* single container for all structs

kernel/bpf/core.c

+2 −1
Original line number Original line Diff line number Diff line
@@ -411,7 +411,8 @@ static void bpf_get_prog_name(const struct bpf_prog *prog, char *sym) 




	/* prog->aux->name will be ignored if full btf name is available */

	/* prog->aux->name will be ignored if full btf name is available */

	if (prog->aux->btf) {

	if (prog->aux->btf) {

		type = btf_type_by_id(prog->aux->btf, prog->aux->type_id);

		type = btf_type_by_id(prog->aux->btf,

				      prog->aux->func_info[prog->aux->func_idx].type_id);

		func_name = btf_name_by_offset(prog->aux->btf, type->name_off);

		func_name = btf_name_by_offset(prog->aux->btf, type->name_off);

		snprintf(sym, (size_t)(end - sym), "_%s", func_name);

		snprintf(sym, (size_t)(end - sym), "_%s", func_name);

		return;

		return;

kernel/bpf/syscall.c

+8 −25
Original line number Original line Diff line number Diff line
@@ -1214,6 +1214,7 @@ static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 
		bpf_prog_free_id(prog, do_idr_lock);

		bpf_prog_free_id(prog, do_idr_lock);

		bpf_prog_kallsyms_del_all(prog);

		bpf_prog_kallsyms_del_all(prog);

		btf_put(prog->aux->btf);

		btf_put(prog->aux->btf);

		kvfree(prog->aux->func_info);





		call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);

		call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);

	}

	}
@@ -2219,46 +2220,28 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 
	}

	}





	if (prog->aux->btf) {

	if (prog->aux->btf) {

		u32 krec_size = sizeof(struct bpf_func_info);

		u32 ucnt, urec_size;

		u32 ucnt, urec_size;





		info.btf_id = btf_id(prog->aux->btf);

		info.btf_id = btf_id(prog->aux->btf);





		ucnt = info.func_info_cnt;

		ucnt = info.func_info_cnt;

		info.func_info_cnt = prog->aux->func_cnt ? : 1;

		info.func_info_cnt = prog->aux->func_info_cnt;

		urec_size = info.func_info_rec_size;

		urec_size = info.func_info_rec_size;

		info.func_info_rec_size = sizeof(struct bpf_func_info);

		info.func_info_rec_size = krec_size;

		if (ucnt) {

		if (ucnt) {

			/* expect passed-in urec_size is what the kernel expects */

			/* expect passed-in urec_size is what the kernel expects */

			if (urec_size != info.func_info_rec_size)

			if (urec_size != info.func_info_rec_size)

				return -EINVAL;

				return -EINVAL;





			if (bpf_dump_raw_ok()) {

			if (bpf_dump_raw_ok()) {

				struct bpf_func_info kern_finfo;

				char __user *user_finfo;

				char __user *user_finfo;

				u32 i, insn_offset;





				user_finfo = u64_to_user_ptr(info.func_info);

				user_finfo = u64_to_user_ptr(info.func_info);

				if (prog->aux->func_cnt) {

				ucnt = min_t(u32, info.func_info_cnt, ucnt);

				ucnt = min_t(u32, info.func_info_cnt, ucnt);

					insn_offset = 0;

				if (copy_to_user(user_finfo, prog->aux->func_info,

					for (i = 0; i < ucnt; i++) {

						 krec_size * ucnt))

						kern_finfo.insn_offset = insn_offset;

						kern_finfo.type_id = prog->aux->func[i]->aux->type_id;

						if (copy_to_user(user_finfo, &kern_finfo,

								 sizeof(kern_finfo)))

							return -EFAULT;



						/* func[i]->len holds the prog len */

						insn_offset += prog->aux->func[i]->len;

						user_finfo += urec_size;

					}

				} else {

					kern_finfo.insn_offset = 0;

					kern_finfo.type_id = prog->aux->type_id;

					if (copy_to_user(user_finfo, &kern_finfo,

							 sizeof(kern_finfo)))

					return -EFAULT;

					return -EFAULT;

				}

			} else {

			} else {

				info.func_info_cnt = 0;

				info.func_info_cnt = 0;

			}

			}

kernel/bpf/verifier.c

+38 −17
Original line number Original line Diff line number Diff line
@@ -4650,7 +4650,7 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, 
{

{

	u32 i, nfuncs, urec_size, min_size, prev_offset;

	u32 i, nfuncs, urec_size, min_size, prev_offset;

	u32 krec_size = sizeof(struct bpf_func_info);

	u32 krec_size = sizeof(struct bpf_func_info);

	struct bpf_func_info krecord = {};

	struct bpf_func_info *krecord = NULL;

	const struct btf_type *type;

	const struct btf_type *type;

	void __user *urecord;

	void __user *urecord;

	struct btf *btf;

	struct btf *btf;
@@ -4682,6 +4682,12 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, 
	urecord = u64_to_user_ptr(attr->func_info);

	urecord = u64_to_user_ptr(attr->func_info);

	min_size = min_t(u32, krec_size, urec_size);

	min_size = min_t(u32, krec_size, urec_size);





	krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);

	if (!krecord) {

		ret = -ENOMEM;

		goto free_btf;

	}



	for (i = 0; i < nfuncs; i++) {

	for (i = 0; i < nfuncs; i++) {

		ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);

		ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);

		if (ret) {

		if (ret) {
@@ -4696,59 +4702,69 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, 
			goto free_btf;

			goto free_btf;

		}

		}





		if (copy_from_user(&krecord, urecord, min_size)) {

		if (copy_from_user(&krecord[i], urecord, min_size)) {

			ret = -EFAULT;

			ret = -EFAULT;

			goto free_btf;

			goto free_btf;

		}

		}





		/* check insn_offset */

		/* check insn_offset */

		if (i == 0) {

		if (i == 0) {

			if (krecord.insn_offset) {

			if (krecord[i].insn_offset) {

				verbose(env,

				verbose(env,

					"nonzero insn_offset %u for the first func info record",

					"nonzero insn_offset %u for the first func info record",

					krecord.insn_offset);

					krecord[i].insn_offset);

				ret = -EINVAL;

				ret = -EINVAL;

				goto free_btf;

				goto free_btf;

			}

			}

		} else if (krecord.insn_offset <= prev_offset) {

		} else if (krecord[i].insn_offset <= prev_offset) {

			verbose(env,

			verbose(env,

				"same or smaller insn offset (%u) than previous func info record (%u)",

				"same or smaller insn offset (%u) than previous func info record (%u)",

				krecord.insn_offset, prev_offset);

				krecord[i].insn_offset, prev_offset);

			ret = -EINVAL;

			ret = -EINVAL;

			goto free_btf;

			goto free_btf;

		}

		}





		if (env->subprog_info[i].start != krecord.insn_offset) {

		if (env->subprog_info[i].start != krecord[i].insn_offset) {

			verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");

			verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");

			ret = -EINVAL;

			ret = -EINVAL;

			goto free_btf;

			goto free_btf;

		}

		}





		/* check type_id */

		/* check type_id */

		type = btf_type_by_id(btf, krecord.type_id);

		type = btf_type_by_id(btf, krecord[i].type_id);

		if (!type || BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) {

		if (!type || BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) {

			verbose(env, "invalid type id %d in func info",

			verbose(env, "invalid type id %d in func info",

				krecord.type_id);

				krecord[i].type_id);

			ret = -EINVAL;

			ret = -EINVAL;

			goto free_btf;

			goto free_btf;

		}

		}





		if (i == 0)

		prev_offset = krecord[i].insn_offset;

			prog->aux->type_id = krecord.type_id;

		env->subprog_info[i].type_id = krecord.type_id;



		prev_offset = krecord.insn_offset;

		urecord += urec_size;

		urecord += urec_size;

	}

	}





	prog->aux->btf = btf;

	prog->aux->btf = btf;

	prog->aux->func_info = krecord;

	prog->aux->func_info_cnt = nfuncs;

	return 0;

	return 0;





free_btf:

free_btf:

	btf_put(btf);

	btf_put(btf);

	kvfree(krecord);

	return ret;

	return ret;

}

}





static void adjust_btf_func(struct bpf_verifier_env *env)

{

	int i;



	if (!env->prog->aux->func_info)

		return;



	for (i = 0; i < env->subprog_cnt; i++)

		env->prog->aux->func_info[i].insn_offset = env->subprog_info[i].start;

}



/* check %cur's range satisfies %old's */

/* check %cur's range satisfies %old's */

static bool range_within(struct bpf_reg_state *old,

static bool range_within(struct bpf_reg_state *old,

			 struct bpf_reg_state *cur)

			 struct bpf_reg_state *cur)
@@ -6043,15 +6059,17 @@ static int jit_subprogs(struct bpf_verifier_env *env) 
		if (bpf_prog_calc_tag(func[i]))

		if (bpf_prog_calc_tag(func[i]))

			goto out_free;

			goto out_free;

		func[i]->is_func = 1;

		func[i]->is_func = 1;

		func[i]->aux->func_idx = i;

		/* the btf and func_info will be freed only at prog->aux */

		func[i]->aux->btf = prog->aux->btf;

		func[i]->aux->func_info = prog->aux->func_info;



		/* Use bpf_prog_F_tag to indicate functions in stack traces.

		/* Use bpf_prog_F_tag to indicate functions in stack traces.

		 * Long term would need debug info to populate names

		 * Long term would need debug info to populate names

		 */

		 */

		func[i]->aux->name[0] = 'F';

		func[i]->aux->name[0] = 'F';

		func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;

		func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;

		func[i]->jit_requested = 1;

		func[i]->jit_requested = 1;

		/* the btf will be freed only at prog->aux */

		func[i]->aux->btf = prog->aux->btf;

		func[i]->aux->type_id = env->subprog_info[i].type_id;

		func[i] = bpf_int_jit_compile(func[i]);

		func[i] = bpf_int_jit_compile(func[i]);

		if (!func[i]->jited) {

		if (!func[i]->jited) {

			err = -ENOTSUPP;

			err = -ENOTSUPP;
@@ -6572,6 +6590,9 @@ skip_full_check: 
		convert_pseudo_ld_imm64(env);

		convert_pseudo_ld_imm64(env);

	}

	}





	if (ret == 0)

		adjust_btf_func(env);



err_release_maps:

err_release_maps:

	if (!env->prog->aux->used_maps)

	if (!env->prog->aux->used_maps)

		/* if we didn't copy map pointers into bpf_prog_info, release

		/* if we didn't copy map pointers into bpf_prog_info, release

CRA Git | Maintained and supported by SUSTech CRA and CCSE