Commit 923cefe3 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'dead-code-elimination' 



Jakub Kicinski says:

====================
This set adds support for complete removal of dead code.

Patch 3 contains all the code removal logic, patches 2 and 4
additionally optimize branches around and to dead code.

Patches 6 and 7 allow offload JITs to take advantage of the
optimization.  After a few small clean ups (8, 9, 10) nfp
support is added (11, 12).

Removing code directly in the verifier makes it easy to adjust
the relevant metadata (line info, subprogram info).  JITs for
code store constrained architectures would have hard time
performing such adjustments at JIT level.  Removing subprograms
or line info is very hard once BPF core finished the verification.
For user space to perform dead code removal it would have to perform
the execution simulation/analysis similar to what the verifier does.

v3:
 - fix uninitilized var warning in GCC 6 (buildbot).
v4:
 - simplify the linfo-keeping logic (Yonghong).  Instead of
   trying to figure out that we are removing first instruction
   of a subprogram, just always keep last dead line info, if
   first live instruction doesn't have one.
v5:
 - improve comments (Martin Lau).
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents bbebce8e 9a06927e

drivers/net/ethernet/netronome/nfp/bpf/jit.c

+22 −20
Original line number Diff line number Diff line
@@ -1266,7 +1266,7 @@ wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 
	u64 imm = insn->imm; /* sign extend */



	if (skip) {

		meta->skip = true;

		meta->flags |= FLAG_INSN_SKIP_NOOP;

		return 0;

	}
@@ -1296,7 +1296,7 @@ wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 
	const struct bpf_insn *insn = &meta->insn;



	if (skip) {

		meta->skip = true;

		meta->flags |= FLAG_INSN_SKIP_NOOP;

		return 0;

	}
@@ -3182,7 +3182,7 @@ bpf_to_bpf_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 
		wrp_immed_relo(nfp_prog, imm_b(nfp_prog), 0, RELO_IMMED_REL);

	} else {

		ret_tgt = nfp_prog_current_offset(nfp_prog) + 2;

		emit_br(nfp_prog, BR_UNC, meta->n + 1 + meta->insn.imm, 1);

		emit_br(nfp_prog, BR_UNC, meta->insn.imm, 1);

		offset_br = nfp_prog_current_offset(nfp_prog);

	}

	wrp_immed_relo(nfp_prog, ret_reg(nfp_prog), ret_tgt, RELO_IMMED_REL);
@@ -3395,7 +3395,7 @@ static int nfp_fixup_branches(struct nfp_prog *nfp_prog) 
	int err;



	list_for_each_entry(meta, &nfp_prog->insns, l) {

		if (meta->skip)

		if (meta->flags & FLAG_INSN_SKIP_MASK)

			continue;

		if (BPF_CLASS(meta->insn.code) != BPF_JMP)

			continue;
@@ -3439,7 +3439,7 @@ static int nfp_fixup_branches(struct nfp_prog *nfp_prog) 


		jmp_dst = meta->jmp_dst;



		if (jmp_dst->skip) {

		if (jmp_dst->flags & FLAG_INSN_SKIP_PREC_DEPENDENT) {

			pr_err("Branch landing on removed instruction!!\n");

			return -ELOOP;

		}
@@ -3689,7 +3689,7 @@ static int nfp_translate(struct nfp_prog *nfp_prog) 
				return nfp_prog->error;

		}



		if (meta->skip) {

		if (meta->flags & FLAG_INSN_SKIP_MASK) {

			nfp_prog->n_translated++;

			continue;

		}
@@ -3737,10 +3737,10 @@ static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog) 
		/* Programs start with R6 = R1 but we ignore the skb pointer */

		if (insn.code == (BPF_ALU64 | BPF_MOV | BPF_X) &&

		    insn.src_reg == 1 && insn.dst_reg == 6)

			meta->skip = true;

			meta->flags |= FLAG_INSN_SKIP_PREC_DEPENDENT;



		/* Return as soon as something doesn't match */

		if (!meta->skip)

		if (!(meta->flags & FLAG_INSN_SKIP_MASK))

			return;

	}

}
@@ -3755,7 +3755,7 @@ static void nfp_bpf_opt_neg_add_sub(struct nfp_prog *nfp_prog) 
	list_for_each_entry(meta, &nfp_prog->insns, l) {

		struct bpf_insn insn = meta->insn;



		if (meta->skip)

		if (meta->flags & FLAG_INSN_SKIP_MASK)

			continue;



		if (BPF_CLASS(insn.code) != BPF_ALU &&
@@ -3829,7 +3829,7 @@ static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog) 
		if (meta2->flags & FLAG_INSN_IS_JUMP_DST)

			continue;



		meta2->skip = true;

		meta2->flags |= FLAG_INSN_SKIP_PREC_DEPENDENT;

	}

}
@@ -3869,8 +3869,8 @@ static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog) 
		    meta3->flags & FLAG_INSN_IS_JUMP_DST)

			continue;



		meta2->skip = true;

		meta3->skip = true;

		meta2->flags |= FLAG_INSN_SKIP_PREC_DEPENDENT;

		meta3->flags |= FLAG_INSN_SKIP_PREC_DEPENDENT;

	}

}
@@ -4065,7 +4065,8 @@ static void nfp_bpf_opt_ldst_gather(struct nfp_prog *nfp_prog) 
				}



				head_ld_meta->paired_st = &head_st_meta->insn;

				head_st_meta->skip = true;

				head_st_meta->flags |=

					FLAG_INSN_SKIP_PREC_DEPENDENT;

			} else {

				head_ld_meta->ldst_gather_len = 0;

			}
@@ -4098,8 +4099,8 @@ static void nfp_bpf_opt_ldst_gather(struct nfp_prog *nfp_prog) 
			head_ld_meta = meta1;

			head_st_meta = meta2;

		} else {

			meta1->skip = true;

			meta2->skip = true;

			meta1->flags |= FLAG_INSN_SKIP_PREC_DEPENDENT;

			meta2->flags |= FLAG_INSN_SKIP_PREC_DEPENDENT;

		}



		head_ld_meta->ldst_gather_len += BPF_LDST_BYTES(ld);
@@ -4124,7 +4125,7 @@ static void nfp_bpf_opt_pkt_cache(struct nfp_prog *nfp_prog) 
		if (meta->flags & FLAG_INSN_IS_JUMP_DST)

			cache_avail = false;



		if (meta->skip)

		if (meta->flags & FLAG_INSN_SKIP_MASK)

			continue;



		insn = &meta->insn;
@@ -4210,7 +4211,7 @@ start_new: 
	}



	list_for_each_entry(meta, &nfp_prog->insns, l) {

		if (meta->skip)

		if (meta->flags & FLAG_INSN_SKIP_MASK)

			continue;



		if (is_mbpf_load_pkt(meta) && !meta->ldst_gather_len) {
@@ -4246,7 +4247,8 @@ static int nfp_bpf_replace_map_ptrs(struct nfp_prog *nfp_prog) 
	u32 id;



	nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {

		if (meta1->skip || meta2->skip)

		if (meta1->flags & FLAG_INSN_SKIP_MASK ||

		    meta2->flags & FLAG_INSN_SKIP_MASK)

			continue;



		if (meta1->insn.code != (BPF_LD | BPF_IMM | BPF_DW) ||
@@ -4325,7 +4327,7 @@ int nfp_bpf_jit(struct nfp_prog *nfp_prog) 
	return ret;

}



void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt)

void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog)

{

	struct nfp_insn_meta *meta;
@@ -4353,7 +4355,7 @@ void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt) 
		else

			dst_idx = meta->n + 1 + meta->insn.off;



		dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_idx, cnt);

		dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_idx);



		if (pseudo_call)

			dst_meta->flags |= FLAG_INSN_IS_SUBPROG_START;

drivers/net/ethernet/netronome/nfp/bpf/main.h

+29 −4
Original line number Diff line number Diff line
@@ -243,6 +243,16 @@ struct nfp_bpf_reg_state { 
#define FLAG_INSN_IS_JUMP_DST			BIT(0)

#define FLAG_INSN_IS_SUBPROG_START		BIT(1)

#define FLAG_INSN_PTR_CALLER_STACK_FRAME	BIT(2)

/* Instruction is pointless, noop even on its own */

#define FLAG_INSN_SKIP_NOOP			BIT(3)

/* Instruction is optimized out based on preceding instructions */

#define FLAG_INSN_SKIP_PREC_DEPENDENT		BIT(4)

/* Instruction is optimized by the verifier */

#define FLAG_INSN_SKIP_VERIFIER_OPT		BIT(5)



#define FLAG_INSN_SKIP_MASK		(FLAG_INSN_SKIP_NOOP | \

					 FLAG_INSN_SKIP_PREC_DEPENDENT | \

					 FLAG_INSN_SKIP_VERIFIER_OPT)



/**

 * struct nfp_insn_meta - BPF instruction wrapper
@@ -271,7 +281,6 @@ struct nfp_bpf_reg_state { 
 * @n: eBPF instruction number

 * @flags: eBPF instruction extra optimization flags

 * @subprog_idx: index of subprogram to which the instruction belongs

 * @skip: skip this instruction (optimized out)

 * @double_cb: callback for second part of the instruction

 * @l: link on nfp_prog->insns list

 */
@@ -319,7 +328,6 @@ struct nfp_insn_meta { 
	unsigned short n;

	unsigned short flags;

	unsigned short subprog_idx;

	bool skip;

	instr_cb_t double_cb;



	struct list_head l;
@@ -407,6 +415,17 @@ static inline bool is_mbpf_div(const struct nfp_insn_meta *meta) 
	return is_mbpf_alu(meta) && mbpf_op(meta) == BPF_DIV;

}



static inline bool is_mbpf_cond_jump(const struct nfp_insn_meta *meta)

{

	u8 op;



	if (BPF_CLASS(meta->insn.code) != BPF_JMP)

		return false;



	op = BPF_OP(meta->insn.code);

	return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;

}



static inline bool is_mbpf_helper_call(const struct nfp_insn_meta *meta)

{

	struct bpf_insn insn = meta->insn;
@@ -457,6 +476,7 @@ struct nfp_bpf_subprog_info { 
 * @subprog_cnt: number of sub-programs, including main function

 * @map_records: the map record pointers from bpf->maps_neutral

 * @subprog: pointer to an array of objects holding info about sub-programs

 * @n_insns: number of instructions on @insns list

 * @insns: list of BPF instruction wrappers (struct nfp_insn_meta)

 */

struct nfp_prog {
@@ -489,6 +509,7 @@ struct nfp_prog { 
	struct nfp_bpf_neutral_map **map_records;

	struct nfp_bpf_subprog_info *subprog;



	unsigned int n_insns;

	struct list_head insns;

};
@@ -505,7 +526,7 @@ struct nfp_bpf_vnic { 
};



bool nfp_is_subprog_start(struct nfp_insn_meta *meta);

void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt);

void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog);

int nfp_bpf_jit(struct nfp_prog *prog);

bool nfp_bpf_supported_opcode(u8 code);
@@ -513,6 +534,10 @@ int nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, 
		    int prev_insn_idx);

int nfp_bpf_finalize(struct bpf_verifier_env *env);



int nfp_bpf_opt_replace_insn(struct bpf_verifier_env *env, u32 off,

			     struct bpf_insn *insn);

int nfp_bpf_opt_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt);



extern const struct bpf_prog_offload_ops nfp_bpf_dev_ops;



struct netdev_bpf;
@@ -526,7 +551,7 @@ int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog, 


struct nfp_insn_meta *

nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,

		  unsigned int insn_idx, unsigned int n_insns);

		  unsigned int insn_idx);



void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv);

drivers/net/ethernet/netronome/nfp/bpf/offload.c

+8 −1
Original line number Diff line number Diff line
@@ -163,8 +163,9 @@ nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog, 


		list_add_tail(&meta->l, &nfp_prog->insns);

	}

	nfp_prog->n_insns = cnt;



	nfp_bpf_jit_prepare(nfp_prog, cnt);

	nfp_bpf_jit_prepare(nfp_prog);



	return 0;

}
@@ -219,6 +220,10 @@ static int nfp_bpf_translate(struct bpf_prog *prog) 
	unsigned int max_instr;

	int err;



	/* We depend on dead code elimination succeeding */

	if (prog->aux->offload->opt_failed)

		return -EINVAL;



	max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);

	nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);
@@ -591,6 +596,8 @@ int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog, 
const struct bpf_prog_offload_ops nfp_bpf_dev_ops = {

	.insn_hook	= nfp_verify_insn,

	.finalize	= nfp_bpf_finalize,

	.replace_insn	= nfp_bpf_opt_replace_insn,

	.remove_insns	= nfp_bpf_opt_remove_insns,

	.prepare	= nfp_bpf_verifier_prep,

	.translate	= nfp_bpf_translate,

	.destroy	= nfp_bpf_destroy,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

+65 −9
Original line number Diff line number Diff line
@@ -18,15 +18,15 @@ 


struct nfp_insn_meta *

nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,

		  unsigned int insn_idx, unsigned int n_insns)

		  unsigned int insn_idx)

{

	unsigned int forward, backward, i;



	backward = meta->n - insn_idx;

	forward = insn_idx - meta->n;



	if (min(forward, backward) > n_insns - insn_idx - 1) {

		backward = n_insns - insn_idx - 1;

	if (min(forward, backward) > nfp_prog->n_insns - insn_idx - 1) {

		backward = nfp_prog->n_insns - insn_idx - 1;

		meta = nfp_prog_last_meta(nfp_prog);

	}

	if (min(forward, backward) > insn_idx && backward > insn_idx) {
@@ -629,7 +629,7 @@ int nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, 
	struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;

	struct nfp_insn_meta *meta = nfp_prog->verifier_meta;



	meta = nfp_bpf_goto_meta(nfp_prog, meta, insn_idx, env->prog->len);

	meta = nfp_bpf_goto_meta(nfp_prog, meta, insn_idx);

	nfp_prog->verifier_meta = meta;



	if (!nfp_bpf_supported_opcode(meta->insn.code)) {
@@ -690,8 +690,7 @@ nfp_assign_subprog_idx_and_regs(struct bpf_verifier_env *env, 
	return 0;

}



static unsigned int

nfp_bpf_get_stack_usage(struct nfp_prog *nfp_prog, unsigned int cnt)

static unsigned int nfp_bpf_get_stack_usage(struct nfp_prog *nfp_prog)

{

	struct nfp_insn_meta *meta = nfp_prog_first_meta(nfp_prog);

	unsigned int max_depth = 0, depth = 0, frame = 0;
@@ -726,7 +725,7 @@ continue_subprog: 


		/* Find the callee and start processing it. */

		meta = nfp_bpf_goto_meta(nfp_prog, meta,

					 meta->n + 1 + meta->insn.imm, cnt);

					 meta->n + 1 + meta->insn.imm);

		idx = meta->subprog_idx;

		frame++;

		goto process_subprog;
@@ -778,8 +777,7 @@ int nfp_bpf_finalize(struct bpf_verifier_env *env) 


	nn = netdev_priv(env->prog->aux->offload->netdev);

	max_stack = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;

	nfp_prog->stack_size = nfp_bpf_get_stack_usage(nfp_prog,

						       env->prog->len);

	nfp_prog->stack_size = nfp_bpf_get_stack_usage(nfp_prog);

	if (nfp_prog->stack_size > max_stack) {

		pr_vlog(env, "stack too large: program %dB > FW stack %dB\n",

			nfp_prog->stack_size, max_stack);
@@ -788,3 +786,61 @@ int nfp_bpf_finalize(struct bpf_verifier_env *env) 


	return 0;

}



int nfp_bpf_opt_replace_insn(struct bpf_verifier_env *env, u32 off,

			     struct bpf_insn *insn)

{

	struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;

	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

	struct nfp_insn_meta *meta = nfp_prog->verifier_meta;



	meta = nfp_bpf_goto_meta(nfp_prog, meta, aux_data[off].orig_idx);

	nfp_prog->verifier_meta = meta;



	/* conditional jump to jump conversion */

	if (is_mbpf_cond_jump(meta) &&

	    insn->code == (BPF_JMP | BPF_JA | BPF_K)) {

		unsigned int tgt_off;



		tgt_off = off + insn->off + 1;



		if (!insn->off) {

			meta->jmp_dst = list_next_entry(meta, l);

			meta->jump_neg_op = false;

		} else if (meta->jmp_dst->n != aux_data[tgt_off].orig_idx) {

			pr_vlog(env, "branch hard wire at %d changes target %d -> %d\n",

				off, meta->jmp_dst->n,

				aux_data[tgt_off].orig_idx);

			return -EINVAL;

		}

		return 0;

	}



	pr_vlog(env, "unsupported instruction replacement %hhx -> %hhx\n",

		meta->insn.code, insn->code);

	return -EINVAL;

}



int nfp_bpf_opt_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)

{

	struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;

	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

	struct nfp_insn_meta *meta = nfp_prog->verifier_meta;

	unsigned int i;



	meta = nfp_bpf_goto_meta(nfp_prog, meta, aux_data[off].orig_idx);



	for (i = 0; i < cnt; i++) {

		if (WARN_ON_ONCE(&meta->l == &nfp_prog->insns))

			return -EINVAL;



		/* doesn't count if it already has the flag */

		if (meta->flags & FLAG_INSN_SKIP_VERIFIER_OPT)

			i--;



		meta->flags |= FLAG_INSN_SKIP_VERIFIER_OPT;

		meta = list_next_entry(meta, l);

	}



	return 0;

}

include/linux/bpf.h

+7 −0
Original line number Diff line number Diff line
@@ -268,9 +268,15 @@ struct bpf_verifier_ops { 
};



struct bpf_prog_offload_ops {

	/* verifier basic callbacks */

	int (*insn_hook)(struct bpf_verifier_env *env,

			 int insn_idx, int prev_insn_idx);

	int (*finalize)(struct bpf_verifier_env *env);

	/* verifier optimization callbacks (called after .finalize) */

	int (*replace_insn)(struct bpf_verifier_env *env, u32 off,

			    struct bpf_insn *insn);

	int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);

	/* program management callbacks */

	int (*prepare)(struct bpf_prog *prog);

	int (*translate)(struct bpf_prog *prog);

	void (*destroy)(struct bpf_prog *prog);
@@ -283,6 +289,7 @@ struct bpf_prog_offload { 
	void			*dev_priv;

	struct list_head	offloads;

	bool			dev_state;

	bool			opt_failed;

	void			*jited_image;

	u32			jited_len;

};

CRA Git | Maintained and supported by SUSTech CRA and CCSE