Commit da4e023e authored by Daniel Borkmann's avatar Daniel Borkmann
Browse files

Merge branch 'bpf-prog-stats' 



Alexei Starovoitov says:

====================
Introduce per program stats to monitor the usage BPF.

v2->v3:
- rename to run_time_ns/run_cnt everywhere

v1->v2:
- fixed u64 stats on 32-bit archs. Thanks Eric
- use more verbose run_time_ns in json output as suggested by Andrii
- refactored prog_alloc and clarified behavior of stats in subprogs
====================

Acked-by: default avatarAndrii Nakryiko <andriin@fb.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parents 143bdc2e 88ad472b

include/linux/bpf.h

+9 −0
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 
#include <linux/rbtree_latch.h>

#include <linux/numa.h>

#include <linux/wait.h>

#include <linux/u64_stats_sync.h>



struct bpf_verifier_env;

struct perf_event;
@@ -340,6 +341,12 @@ enum bpf_cgroup_storage_type { 


#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX



struct bpf_prog_stats {

	u64 cnt;

	u64 nsecs;

	struct u64_stats_sync syncp;

};



struct bpf_prog_aux {

	atomic_t refcnt;

	u32 used_map_cnt;
@@ -389,6 +396,7 @@ struct bpf_prog_aux { 
	 * main prog always has linfo_idx == 0

	 */

	u32 linfo_idx;

	struct bpf_prog_stats __percpu *stats;

	union {

		struct work_struct work;

		struct rcu_head	rcu;
@@ -559,6 +567,7 @@ void bpf_map_area_free(void *base); 
void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);



extern int sysctl_unprivileged_bpf_disabled;

extern int sysctl_bpf_stats_enabled;



int bpf_map_new_fd(struct bpf_map *map, int flags);

int bpf_prog_new_fd(struct bpf_prog *prog);

include/linux/filter.h

+19 −1
Original line number Diff line number Diff line
@@ -533,7 +533,24 @@ struct sk_filter { 
	struct bpf_prog	*prog;

};



#define BPF_PROG_RUN(filter, ctx)  ({ cant_sleep(); (*(filter)->bpf_func)(ctx, (filter)->insnsi); })

DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key);



#define BPF_PROG_RUN(prog, ctx)	({				\

	u32 ret;						\

	cant_sleep();						\

	if (static_branch_unlikely(&bpf_stats_enabled_key)) {	\

		struct bpf_prog_stats *stats;			\

		u64 start = sched_clock();			\

		ret = (*(prog)->bpf_func)(ctx, (prog)->insnsi);	\

		stats = this_cpu_ptr(prog->aux->stats);		\

		u64_stats_update_begin(&stats->syncp);		\

		stats->cnt++;					\

		stats->nsecs += sched_clock() - start;		\

		u64_stats_update_end(&stats->syncp);		\

	} else {						\

		ret = (*(prog)->bpf_func)(ctx, (prog)->insnsi);	\

	}							\

	ret; })



#define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN
@@ -764,6 +781,7 @@ void bpf_prog_free_jited_linfo(struct bpf_prog *prog); 
void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog);



struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags);

struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags);

struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,

				  gfp_t gfp_extra_flags);

void __bpf_prog_free(struct bpf_prog *fp);

include/uapi/linux/bpf.h

+2 −0
Original line number Diff line number Diff line
@@ -2813,6 +2813,8 @@ struct bpf_prog_info { 
	__u32 jited_line_info_rec_size;

	__u32 nr_prog_tags;

	__aligned_u64 prog_tags;

	__u64 run_time_ns;

	__u64 run_cnt;

} __attribute__((aligned(8)));



struct bpf_map_info {

kernel/bpf/core.c

+29 −2
Original line number Diff line number Diff line
@@ -78,7 +78,7 @@ void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, uns 
	return NULL;

}



struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)

struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags)

{

	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;

	struct bpf_prog_aux *aux;
@@ -104,6 +104,26 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) 


	return fp;

}



struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)

{

	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;

	struct bpf_prog *prog;



	prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags);

	if (!prog)

		return NULL;



	prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);

	if (!prog->aux->stats) {

		kfree(prog->aux);

		vfree(prog);

		return NULL;

	}



	u64_stats_init(&prog->aux->stats->syncp);

	return prog;

}

EXPORT_SYMBOL_GPL(bpf_prog_alloc);



int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog)
@@ -231,7 +251,10 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, 


void __bpf_prog_free(struct bpf_prog *fp)

{

	if (fp->aux) {

		free_percpu(fp->aux->stats);

		kfree(fp->aux);

	}

	vfree(fp);

}
@@ -2069,6 +2092,10 @@ int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to, 
	return -EFAULT;

}



DEFINE_STATIC_KEY_FALSE(bpf_stats_enabled_key);

EXPORT_SYMBOL(bpf_stats_enabled_key);

int sysctl_bpf_stats_enabled __read_mostly;



/* All definitions of tracepoints related to BPF. */

#define CREATE_TRACE_POINTS

#include <linux/bpf_trace.h>

kernel/bpf/syscall.c

+37 −2
Original line number Diff line number Diff line
@@ -1283,24 +1283,54 @@ static int bpf_prog_release(struct inode *inode, struct file *filp) 
	return 0;

}



static void bpf_prog_get_stats(const struct bpf_prog *prog,

			       struct bpf_prog_stats *stats)

{

	u64 nsecs = 0, cnt = 0;

	int cpu;



	for_each_possible_cpu(cpu) {

		const struct bpf_prog_stats *st;

		unsigned int start;

		u64 tnsecs, tcnt;



		st = per_cpu_ptr(prog->aux->stats, cpu);

		do {

			start = u64_stats_fetch_begin_irq(&st->syncp);

			tnsecs = st->nsecs;

			tcnt = st->cnt;

		} while (u64_stats_fetch_retry_irq(&st->syncp, start));

		nsecs += tnsecs;

		cnt += tcnt;

	}

	stats->nsecs = nsecs;

	stats->cnt = cnt;

}



#ifdef CONFIG_PROC_FS

static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)

{

	const struct bpf_prog *prog = filp->private_data;

	char prog_tag[sizeof(prog->tag) * 2 + 1] = { };

	struct bpf_prog_stats stats;



	bpf_prog_get_stats(prog, &stats);

	bin2hex(prog_tag, prog->tag, sizeof(prog->tag));

	seq_printf(m,

		   "prog_type:\t%u\n"

		   "prog_jited:\t%u\n"

		   "prog_tag:\t%s\n"

		   "memlock:\t%llu\n"

		   "prog_id:\t%u\n",

		   "prog_id:\t%u\n"

		   "run_time_ns:\t%llu\n"

		   "run_cnt:\t%llu\n",

		   prog->type,

		   prog->jited,

		   prog_tag,

		   prog->pages * 1ULL << PAGE_SHIFT,

		   prog->aux->id);

		   prog->aux->id,

		   stats.nsecs,

		   stats.cnt);

}

#endif
@@ -2122,6 +2152,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 
	struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);

	struct bpf_prog_info info = {};

	u32 info_len = attr->info.info_len;

	struct bpf_prog_stats stats;

	char __user *uinsns;

	u32 ulen;

	int err;
@@ -2161,6 +2192,10 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 
	if (err)

		return err;



	bpf_prog_get_stats(prog, &stats);

	info.run_time_ns = stats.nsecs;

	info.run_cnt = stats.cnt;



	if (!capable(CAP_SYS_ADMIN)) {

		info.jited_prog_len = 0;

		info.xlated_prog_len = 0;

CRA Git | Maintained and supported by SUSTech CRA and CCSE