Commit 08ca90af authored by Jakub Kicinski's avatar Jakub Kicinski Committed by Alexei Starovoitov
Browse files

bpf: notify offload JITs about optimizations 



Let offload JITs know when instructions are replaced and optimized
out, so they can update their state appropriately.  The optimizations
are best effort, if JIT returns an error from any callback verifier
will stop notifying it as state may now be out of sync, but the
verifier continues making progress.

Signed-off-by: default avatarJakub Kicinski <jakub.kicinski@netronome.com>
Reviewed-by: default avatarQuentin Monnet <quentin.monnet@netronome.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 9e4c24e7

include/linux/bpf.h

+7 −0
Original line number Diff line number Diff line
@@ -268,9 +268,15 @@ struct bpf_verifier_ops { 
};



struct bpf_prog_offload_ops {

	/* verifier basic callbacks */

	int (*insn_hook)(struct bpf_verifier_env *env,

			 int insn_idx, int prev_insn_idx);

	int (*finalize)(struct bpf_verifier_env *env);

	/* verifier optimization callbacks (called after .finalize) */

	int (*replace_insn)(struct bpf_verifier_env *env, u32 off,

			    struct bpf_insn *insn);

	int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);

	/* program management callbacks */

	int (*prepare)(struct bpf_prog *prog);

	int (*translate)(struct bpf_prog *prog);

	void (*destroy)(struct bpf_prog *prog);
@@ -283,6 +289,7 @@ struct bpf_prog_offload { 
	void			*dev_priv;

	struct list_head	offloads;

	bool			dev_state;

	bool			opt_failed;

	void			*jited_image;

	u32			jited_len;

};

include/linux/bpf_verifier.h

+5 −0
Original line number Diff line number Diff line
@@ -266,5 +266,10 @@ int bpf_prog_offload_verifier_prep(struct bpf_prog *prog); 
int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env,

				 int insn_idx, int prev_insn_idx);

int bpf_prog_offload_finalize(struct bpf_verifier_env *env);

void

bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,

			      struct bpf_insn *insn);

void

bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt);



#endif /* _LINUX_BPF_VERIFIER_H */

kernel/bpf/offload.c

+35 −0
Original line number Diff line number Diff line
@@ -173,6 +173,41 @@ int bpf_prog_offload_finalize(struct bpf_verifier_env *env) 
	return ret;

}



void

bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,

			      struct bpf_insn *insn)

{

	const struct bpf_prog_offload_ops *ops;

	struct bpf_prog_offload *offload;

	int ret = -EOPNOTSUPP;



	down_read(&bpf_devs_lock);

	offload = env->prog->aux->offload;

	if (offload) {

		ops = offload->offdev->ops;

		if (!offload->opt_failed && ops->replace_insn)

			ret = ops->replace_insn(env, off, insn);

		offload->opt_failed |= ret;

	}

	up_read(&bpf_devs_lock);

}



void

bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)

{

	struct bpf_prog_offload *offload;

	int ret = -EOPNOTSUPP;



	down_read(&bpf_devs_lock);

	offload = env->prog->aux->offload;

	if (offload) {

		if (!offload->opt_failed && offload->offdev->ops->remove_insns)

			ret = offload->offdev->ops->remove_insns(env, off, cnt);

		offload->opt_failed |= ret;

	}

	up_read(&bpf_devs_lock);

}



static void __bpf_prog_offload_destroy(struct bpf_prog *prog)

{

	struct bpf_prog_offload *offload = prog->aux->offload;

kernel/bpf/verifier.c

+6 −0
Original line number Diff line number Diff line
@@ -6558,6 +6558,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 
	unsigned int orig_prog_len = env->prog->len;

	int err;



	if (bpf_prog_is_dev_bound(env->prog->aux))

		bpf_prog_offload_remove_insns(env, off, cnt);



	err = bpf_remove_insns(env->prog, off, cnt);

	if (err)

		return err;
@@ -6632,6 +6635,9 @@ static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 
		else

			continue;



		if (bpf_prog_is_dev_bound(env->prog->aux))

			bpf_prog_offload_replace_insn(env, i, &ja);



		memcpy(insn, &ja, sizeof(ja));

	}

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE