Commit 6219d055 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'bpf-fixes' 



Daniel Borkmann says:

====================
Couple of BPF fixes

This set contains three BPF fixes for net, one that addresses the
complaint from Geert wrt static allocations, and the other is a fix
wrt mem accounting that I found recently during testing and there's
still one more fix on the map value marking.

Thanks!

v1 -> v2:
  - Patch 1 as is.
  - Fixed kbuild bot issue by letting charging helpers stay in the
    syscall.c, since there locked_vm is valid and only export the
    ones needed by bpf_prog_realloc(). Add empty stubs in case the
    bpf syscall is not enabled.
  - Added patch 3 that addresses one more issue in map val marking.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 40e972ab 6760bf2d

include/linux/bpf.h

+12 −1
Original line number Diff line number Diff line
@@ -216,7 +216,7 @@ u64 bpf_tail_call(u64 ctx, u64 r2, u64 index, u64 r4, u64 r5); 
u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);



bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);

void bpf_prog_calc_digest(struct bpf_prog *fp);

int bpf_prog_calc_digest(struct bpf_prog *fp);



const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
@@ -238,6 +238,8 @@ struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, int i); 
void bpf_prog_sub(struct bpf_prog *prog, int i);

struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog);

void bpf_prog_put(struct bpf_prog *prog);

int __bpf_prog_charge(struct user_struct *user, u32 pages);

void __bpf_prog_uncharge(struct user_struct *user, u32 pages);



struct bpf_map *bpf_map_get_with_uref(u32 ufd);

struct bpf_map *__bpf_map_get(struct fd f);
@@ -318,6 +320,15 @@ static inline struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog) 
{

	return ERR_PTR(-EOPNOTSUPP);

}



static inline int __bpf_prog_charge(struct user_struct *user, u32 pages)

{

	return 0;

}



static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages)

{

}

#endif /* CONFIG_BPF_SYSCALL */



/* verifier prototypes for helper functions called from eBPF programs */

include/linux/filter.h

+11 −3
Original line number Diff line number Diff line
@@ -57,9 +57,6 @@ struct bpf_prog_aux; 
/* BPF program can access up to 512 bytes of stack space. */

#define MAX_BPF_STACK	512



/* Maximum BPF program size in bytes. */

#define MAX_BPF_SIZE	(BPF_MAXINSNS * sizeof(struct bpf_insn))



/* Helper macros for filter block array initializers. */



/* ALU ops on registers, bpf_add|sub|...: dst_reg += src_reg */
@@ -517,6 +514,17 @@ static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog, 
	return BPF_PROG_RUN(prog, xdp);

}



static inline u32 bpf_prog_insn_size(const struct bpf_prog *prog)

{

	return prog->len * sizeof(struct bpf_insn);

}



static inline u32 bpf_prog_digest_scratch_size(const struct bpf_prog *prog)

{

	return round_up(bpf_prog_insn_size(prog) +

			sizeof(__be64) + 1, SHA_MESSAGE_BYTES);

}



static inline unsigned int bpf_prog_size(unsigned int proglen)

{

	return max(sizeof(struct bpf_prog),

kernel/bpf/core.c

+29 −14
Original line number Diff line number Diff line
@@ -105,19 +105,29 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, 
	gfp_t gfp_flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO |

			  gfp_extra_flags;

	struct bpf_prog *fp;

	u32 pages, delta;

	int ret;



	BUG_ON(fp_old == NULL);



	size = round_up(size, PAGE_SIZE);

	if (size <= fp_old->pages * PAGE_SIZE)

	pages = size / PAGE_SIZE;

	if (pages <= fp_old->pages)

		return fp_old;



	delta = pages - fp_old->pages;

	ret = __bpf_prog_charge(fp_old->aux->user, delta);

	if (ret)

		return NULL;



	fp = __vmalloc(size, gfp_flags, PAGE_KERNEL);

	if (fp != NULL) {

	if (fp == NULL) {

		__bpf_prog_uncharge(fp_old->aux->user, delta);

	} else {

		kmemcheck_annotate_bitfield(fp, meta);



		memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE);

		fp->pages = size / PAGE_SIZE;

		fp->pages = pages;

		fp->aux->prog = fp;



		/* We keep fp->aux from fp_old around in the new
@@ -136,28 +146,29 @@ void __bpf_prog_free(struct bpf_prog *fp) 
	vfree(fp);

}



#define SHA_BPF_RAW_SIZE						\

	round_up(MAX_BPF_SIZE + sizeof(__be64) + 1, SHA_MESSAGE_BYTES)



/* Called under verifier mutex. */

void bpf_prog_calc_digest(struct bpf_prog *fp)

int bpf_prog_calc_digest(struct bpf_prog *fp)

{

	const u32 bits_offset = SHA_MESSAGE_BYTES - sizeof(__be64);

	static u32 ws[SHA_WORKSPACE_WORDS];

	static u8 raw[SHA_BPF_RAW_SIZE];

	struct bpf_insn *dst = (void *)raw;

	u32 raw_size = bpf_prog_digest_scratch_size(fp);

	u32 ws[SHA_WORKSPACE_WORDS];

	u32 i, bsize, psize, blocks;

	struct bpf_insn *dst;

	bool was_ld_map;

	u8 *todo = raw;

	u8 *raw, *todo;

	__be32 *result;

	__be64 *bits;



	raw = vmalloc(raw_size);

	if (!raw)

		return -ENOMEM;



	sha_init(fp->digest);

	memset(ws, 0, sizeof(ws));



	/* We need to take out the map fd for the digest calculation

	 * since they are unstable from user space side.

	 */

	dst = (void *)raw;

	for (i = 0, was_ld_map = false; i < fp->len; i++) {

		dst[i] = fp->insnsi[i];

		if (!was_ld_map &&
@@ -177,12 +188,13 @@ void bpf_prog_calc_digest(struct bpf_prog *fp) 
		}

	}



	psize = fp->len * sizeof(struct bpf_insn);

	memset(&raw[psize], 0, sizeof(raw) - psize);

	psize = bpf_prog_insn_size(fp);

	memset(&raw[psize], 0, raw_size - psize);

	raw[psize++] = 0x80;



	bsize  = round_up(psize, SHA_MESSAGE_BYTES);

	blocks = bsize / SHA_MESSAGE_BYTES;

	todo   = raw;

	if (bsize - psize >= sizeof(__be64)) {

		bits = (__be64 *)(todo + bsize - sizeof(__be64));

	} else {
@@ -199,6 +211,9 @@ void bpf_prog_calc_digest(struct bpf_prog *fp) 
	result = (__force __be32 *)fp->digest;

	for (i = 0; i < SHA_DIGEST_WORDS; i++)

		result[i] = cpu_to_be32(fp->digest[i]);



	vfree(raw);

	return 0;

}



static bool bpf_is_jmp_and_has_target(const struct bpf_insn *insn)

kernel/bpf/syscall.c

+29 −9
Original line number Diff line number Diff line
@@ -615,19 +615,39 @@ static void free_used_maps(struct bpf_prog_aux *aux) 
	kfree(aux->used_maps);

}



int __bpf_prog_charge(struct user_struct *user, u32 pages)

{

	unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

	unsigned long user_bufs;



	if (user) {

		user_bufs = atomic_long_add_return(pages, &user->locked_vm);

		if (user_bufs > memlock_limit) {

			atomic_long_sub(pages, &user->locked_vm);

			return -EPERM;

		}

	}



	return 0;

}



void __bpf_prog_uncharge(struct user_struct *user, u32 pages)

{

	if (user)

		atomic_long_sub(pages, &user->locked_vm);

}



static int bpf_prog_charge_memlock(struct bpf_prog *prog)

{

	struct user_struct *user = get_current_user();

	unsigned long memlock_limit;

	int ret;



	memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;



	atomic_long_add(prog->pages, &user->locked_vm);

	if (atomic_long_read(&user->locked_vm) > memlock_limit) {

		atomic_long_sub(prog->pages, &user->locked_vm);

	ret = __bpf_prog_charge(user, prog->pages);

	if (ret) {

		free_uid(user);

		return -EPERM;

		return ret;

	}



	prog->aux->user = user;

	return 0;

}
@@ -636,7 +656,7 @@ static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 
{

	struct user_struct *user = prog->aux->user;



	atomic_long_sub(prog->pages, &user->locked_vm);

	__bpf_prog_uncharge(user, prog->pages);

	free_uid(user);

}
@@ -811,7 +831,7 @@ static int bpf_prog_load(union bpf_attr *attr) 


	err = -EFAULT;

	if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),

			   prog->len * sizeof(struct bpf_insn)) != 0)

			   bpf_prog_insn_size(prog)) != 0)

		goto free_prog;



	prog->orig_prog = NULL;

kernel/bpf/verifier.c

+12 −5
Original line number Diff line number Diff line
@@ -462,14 +462,19 @@ static void init_reg_state(struct bpf_reg_state *regs) 
	regs[BPF_REG_1].type = PTR_TO_CTX;

}



static void mark_reg_unknown_value(struct bpf_reg_state *regs, u32 regno)

static void __mark_reg_unknown_value(struct bpf_reg_state *regs, u32 regno)

{

	BUG_ON(regno >= MAX_BPF_REG);

	regs[regno].type = UNKNOWN_VALUE;

	regs[regno].id = 0;

	regs[regno].imm = 0;

}



static void mark_reg_unknown_value(struct bpf_reg_state *regs, u32 regno)

{

	BUG_ON(regno >= MAX_BPF_REG);

	__mark_reg_unknown_value(regs, regno);

}



static void reset_reg_range_values(struct bpf_reg_state *regs, u32 regno)

{

	regs[regno].min_value = BPF_REGISTER_MIN_RANGE;
@@ -1976,7 +1981,7 @@ static void mark_map_reg(struct bpf_reg_state *regs, u32 regno, u32 id, 
		 */

		reg->id = 0;

		if (type == UNKNOWN_VALUE)

			mark_reg_unknown_value(regs, regno);

			__mark_reg_unknown_value(regs, regno);

	}

}
@@ -2931,6 +2936,10 @@ static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) 
	int insn_cnt = env->prog->len;

	int i, j, err;



	err = bpf_prog_calc_digest(env->prog);

	if (err)

		return err;



	for (i = 0; i < insn_cnt; i++, insn++) {

		if (BPF_CLASS(insn->code) == BPF_LDX &&

		    (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) {
@@ -3178,8 +3187,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr) 
		log_level = 0;

	}



	bpf_prog_calc_digest(env->prog);



	ret = replace_map_fd_with_map_ptr(env);

	if (ret < 0)

		goto skip_full_check;

CRA Git | Maintained and supported by SUSTech CRA and CCSE