Commit dee72f8a authored by David S. Miller's avatar David S. Miller
Browse files

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next 



Alexei Starovoitov says:

====================
pull-request: bpf-next 2020-07-21

The following pull-request contains BPF updates for your *net-next* tree.

We've added 46 non-merge commits during the last 6 day(s) which contain
a total of 68 files changed, 4929 insertions(+), 526 deletions(-).

The main changes are:

1) Run BPF program on socket lookup, from Jakub.

2) Introduce cpumap, from Lorenzo.

3) s390 JIT fixes, from Ilya.

4) teach riscv JIT to emit compressed insns, from Luke.

5) use build time computed BTF ids in bpf iter, from Yonghong.
====================

Purely independent overlapping changes in both filter.h and xdp.h

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents fa56a987 9165e1d7

arch/riscv/net/bpf_jit.h

+480 −3
Original line number Diff line number Diff line
@@ -13,6 +13,11 @@ 
#include <linux/filter.h>

#include <asm/cacheflush.h>



static inline bool rvc_enabled(void)

{

	return IS_ENABLED(CONFIG_RISCV_ISA_C);

}



enum {

	RV_REG_ZERO =	0,	/* The constant value 0 */

	RV_REG_RA =	1,	/* Return address */
@@ -48,9 +53,21 @@ enum { 
	RV_REG_T6 =	31,

};



static inline bool is_creg(u8 reg)

{

	return (1 << reg) & (BIT(RV_REG_FP) |

			     BIT(RV_REG_S1) |

			     BIT(RV_REG_A0) |

			     BIT(RV_REG_A1) |

			     BIT(RV_REG_A2) |

			     BIT(RV_REG_A3) |

			     BIT(RV_REG_A4) |

			     BIT(RV_REG_A5));

}



struct rv_jit_context {

	struct bpf_prog *prog;

	u32 *insns;		/* RV insns */

	u16 *insns;		/* RV insns */

	int ninsns;

	int epilogue_offset;

	int *offset;		/* BPF to RV */
@@ -58,6 +75,12 @@ struct rv_jit_context { 
	int stack_size;

};



/* Convert from ninsns to bytes. */

static inline int ninsns_rvoff(int ninsns)

{

	return ninsns << 1;

}



struct rv_jit_data {

	struct bpf_binary_header *header;

	u8 *image;
@@ -74,8 +97,22 @@ static inline void bpf_flush_icache(void *start, void *end) 
	flush_icache_range((unsigned long)start, (unsigned long)end);

}



/* Emit a 4-byte riscv instruction. */

static inline void emit(const u32 insn, struct rv_jit_context *ctx)

{

	if (ctx->insns) {

		ctx->insns[ctx->ninsns] = insn;

		ctx->insns[ctx->ninsns + 1] = (insn >> 16);

	}



	ctx->ninsns += 2;

}



/* Emit a 2-byte riscv compressed instruction. */

static inline void emitc(const u16 insn, struct rv_jit_context *ctx)

{

	BUILD_BUG_ON(!rvc_enabled());



	if (ctx->insns)

		ctx->insns[ctx->ninsns] = insn;
@@ -86,7 +123,7 @@ static inline int epilogue_offset(struct rv_jit_context *ctx) 
{

	int to = ctx->epilogue_offset, from = ctx->ninsns;



	return (to - from) << 2;

	return ninsns_rvoff(to - from);

}



/* Return -1 or inverted cond. */
@@ -117,6 +154,36 @@ static inline int invert_bpf_cond(u8 cond) 
	return -1;

}



static inline bool is_6b_int(long val)

{

	return -(1L << 5) <= val && val < (1L << 5);

}



static inline bool is_7b_uint(unsigned long val)

{

	return val < (1UL << 7);

}



static inline bool is_8b_uint(unsigned long val)

{

	return val < (1UL << 8);

}



static inline bool is_9b_uint(unsigned long val)

{

	return val < (1UL << 9);

}



static inline bool is_10b_int(long val)

{

	return -(1L << 9) <= val && val < (1L << 9);

}



static inline bool is_10b_uint(unsigned long val)

{

	return val < (1UL << 10);

}



static inline bool is_12b_int(long val)

{

	return -(1L << 11) <= val && val < (1L << 11);
@@ -149,7 +216,7 @@ static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx) 
	off++; /* BPF branch is from PC+1, RV is from PC */

	from = (insn > 0) ? ctx->offset[insn - 1] : 0;

	to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0;

	return (to - from) << 2;

	return ninsns_rvoff(to - from);

}



/* Instruction formats. */
@@ -207,6 +274,59 @@ static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1, 
	return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode);

}



/* RISC-V compressed instruction formats. */



static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op)

{

	return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op;

}



static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op)

{

	u32 imm;



	imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);

	return (funct3 << 13) | (rd << 7) | op | imm;

}



static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op)

{

	return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op;

}



static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op)

{

	return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op;

}



static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd,

			     u8 op)

{

	return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |

		(imm_lo << 5) | ((rd & 0x7) << 2) | op;

}



static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2,

			     u8 op)

{

	return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |

		(imm_lo << 5) | ((rs2 & 0x7) << 2) | op;

}



static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op)

{

	return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) |

		((rs2 & 0x7) << 2) | op;

}



static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op)

{

	u32 imm;



	imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);

	return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm;

}



/* Instructions shared by both RV32 and RV64. */



static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0)
@@ -414,6 +534,135 @@ static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 
	return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);

}



/* RVC instrutions. */



static inline u16 rvc_addi4spn(u8 rd, u32 imm10)

{

	u32 imm;



	imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) |

		((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3);

	return rv_ciw_insn(0x0, imm, rd, 0x0);

}



static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1)

{

	u32 imm_hi, imm_lo;



	imm_hi = (imm7 & 0x38) >> 3;

	imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);

	return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0);

}



static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2)

{

	u32 imm_hi, imm_lo;



	imm_hi = (imm7 & 0x38) >> 3;

	imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);

	return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0);

}



static inline u16 rvc_addi(u8 rd, u32 imm6)

{

	return rv_ci_insn(0, imm6, rd, 0x1);

}



static inline u16 rvc_li(u8 rd, u32 imm6)

{

	return rv_ci_insn(0x2, imm6, rd, 0x1);

}



static inline u16 rvc_addi16sp(u32 imm10)

{

	u32 imm;



	imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) |

		((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5);

	return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1);

}



static inline u16 rvc_lui(u8 rd, u32 imm6)

{

	return rv_ci_insn(0x3, imm6, rd, 0x1);

}



static inline u16 rvc_srli(u8 rd, u32 imm6)

{

	return rv_cb_insn(0x4, imm6, 0, rd, 0x1);

}



static inline u16 rvc_srai(u8 rd, u32 imm6)

{

	return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1);

}



static inline u16 rvc_andi(u8 rd, u32 imm6)

{

	return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1);

}



static inline u16 rvc_sub(u8 rd, u8 rs)

{

	return rv_ca_insn(0x23, rd, 0, rs, 0x1);

}



static inline u16 rvc_xor(u8 rd, u8 rs)

{

	return rv_ca_insn(0x23, rd, 0x1, rs, 0x1);

}



static inline u16 rvc_or(u8 rd, u8 rs)

{

	return rv_ca_insn(0x23, rd, 0x2, rs, 0x1);

}



static inline u16 rvc_and(u8 rd, u8 rs)

{

	return rv_ca_insn(0x23, rd, 0x3, rs, 0x1);

}



static inline u16 rvc_slli(u8 rd, u32 imm6)

{

	return rv_ci_insn(0, imm6, rd, 0x2);

}



static inline u16 rvc_lwsp(u8 rd, u32 imm8)

{

	u32 imm;



	imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c);

	return rv_ci_insn(0x2, imm, rd, 0x2);

}



static inline u16 rvc_jr(u8 rs1)

{

	return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2);

}



static inline u16 rvc_mv(u8 rd, u8 rs)

{

	return rv_cr_insn(0x8, rd, rs, 0x2);

}



static inline u16 rvc_jalr(u8 rs1)

{

	return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2);

}



static inline u16 rvc_add(u8 rd, u8 rs)

{

	return rv_cr_insn(0x9, rd, rs, 0x2);

}



static inline u16 rvc_swsp(u32 imm8, u8 rs2)

{

	u32 imm;



	imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6);

	return rv_css_insn(0x6, imm, rs2, 0x2);

}



/*

 * RV64-only instructions.

 *
@@ -503,6 +752,234 @@ static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 
	return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);

}



/* RV64-only RVC instructions. */



static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1)

{

	u32 imm_hi, imm_lo;



	imm_hi = (imm8 & 0x38) >> 3;

	imm_lo = (imm8 & 0xc0) >> 6;

	return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0);

}



static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2)

{

	u32 imm_hi, imm_lo;



	imm_hi = (imm8 & 0x38) >> 3;

	imm_lo = (imm8 & 0xc0) >> 6;

	return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0);

}



static inline u16 rvc_subw(u8 rd, u8 rs)

{

	return rv_ca_insn(0x27, rd, 0, rs, 0x1);

}



static inline u16 rvc_addiw(u8 rd, u32 imm6)

{

	return rv_ci_insn(0x1, imm6, rd, 0x1);

}



static inline u16 rvc_ldsp(u8 rd, u32 imm9)

{

	u32 imm;



	imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38);

	return rv_ci_insn(0x3, imm, rd, 0x2);

}



static inline u16 rvc_sdsp(u32 imm9, u8 rs2)

{

	u32 imm;



	imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6);

	return rv_css_insn(0x7, imm, rs2, 0x2);

}



#endif /* __riscv_xlen == 64 */



/* Helper functions that emit RVC instructions when possible. */



static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && rd == RV_REG_RA && rs && !imm)

		emitc(rvc_jalr(rs), ctx);

	else if (rvc_enabled() && !rd && rs && !imm)

		emitc(rvc_jr(rs), ctx);

	else

		emit(rv_jalr(rd, rs, imm), ctx);

}



static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && rd && rs)

		emitc(rvc_mv(rd, rs), ctx);

	else

		emit(rv_addi(rd, rs, 0), ctx);

}



static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && rd && rd == rs1 && rs2)

		emitc(rvc_add(rd, rs2), ctx);

	else

		emit(rv_add(rd, rs1, rs2), ctx);

}



static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf))

		emitc(rvc_addi16sp(imm), ctx);

	else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) &&

		 !(imm & 0x3) && imm)

		emitc(rvc_addi4spn(rd, imm), ctx);

	else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm))

		emitc(rvc_addi(rd, imm), ctx);

	else

		emit(rv_addi(rd, rs, imm), ctx);

}



static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && rd && is_6b_int(imm))

		emitc(rvc_li(rd, imm), ctx);

	else

		emit(rv_addi(rd, RV_REG_ZERO, imm), ctx);

}



static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm)

		emitc(rvc_lui(rd, imm), ctx);

	else

		emit(rv_lui(rd, imm), ctx);

}



static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen)

		emitc(rvc_slli(rd, imm), ctx);

	else

		emit(rv_slli(rd, rs, imm), ctx);

}



static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm))

		emitc(rvc_andi(rd, imm), ctx);

	else

		emit(rv_andi(rd, rs, imm), ctx);

}



static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)

		emitc(rvc_srli(rd, imm), ctx);

	else

		emit(rv_srli(rd, rs, imm), ctx);

}



static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)

		emitc(rvc_srai(rd, imm), ctx);

	else

		emit(rv_srai(rd, rs, imm), ctx);

}



static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))

		emitc(rvc_sub(rd, rs2), ctx);

	else

		emit(rv_sub(rd, rs1, rs2), ctx);

}



static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))

		emitc(rvc_or(rd, rs2), ctx);

	else

		emit(rv_or(rd, rs1, rs2), ctx);

}



static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))

		emitc(rvc_and(rd, rs2), ctx);

	else

		emit(rv_and(rd, rs1, rs2), ctx);

}



static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))

		emitc(rvc_xor(rd, rs2), ctx);

	else

		emit(rv_xor(rd, rs1, rs2), ctx);

}



static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3))

		emitc(rvc_lwsp(rd, off), ctx);

	else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3))

		emitc(rvc_lw(rd, off, rs1), ctx);

	else

		emit(rv_lw(rd, off, rs1), ctx);

}



static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3))

		emitc(rvc_swsp(off, rs2), ctx);

	else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3))

		emitc(rvc_sw(rs1, off, rs2), ctx);

	else

		emit(rv_sw(rs1, off, rs2), ctx);

}



/* RV64-only helper functions. */

#if __riscv_xlen == 64



static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && rd && rd == rs && is_6b_int(imm))

		emitc(rvc_addiw(rd, imm), ctx);

	else

		emit(rv_addiw(rd, rs, imm), ctx);

}



static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7))

		emitc(rvc_ldsp(rd, off), ctx);

	else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7))

		emitc(rvc_ld(rd, off, rs1), ctx);

	else

		emit(rv_ld(rd, off, rs1), ctx);

}



static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7))

		emitc(rvc_sdsp(off, rs2), ctx);

	else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7))

		emitc(rvc_sd(rs1, off, rs2), ctx);

	else

		emit(rv_sd(rs1, off, rs2), ctx);

}



static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)

{

	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))

		emitc(rvc_subw(rd, rs2), ctx);

	else

		emit(rv_subw(rd, rs1, rs2), ctx);

}



#endif /* __riscv_xlen == 64 */



void bpf_jit_build_prologue(struct rv_jit_context *ctx);

arch/riscv/net/bpf_jit_comp32.c

+7 −7
Original line number Diff line number Diff line
@@ -644,7 +644,7 @@ static int emit_branch_r64(const s8 *src1, const s8 *src2, s32 rvoff, 


	e = ctx->ninsns;

	/* Adjust for extra insns. */

	rvoff -= (e - s) << 2;

	rvoff -= ninsns_rvoff(e - s);

	emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx);

	return 0;

}
@@ -713,7 +713,7 @@ static int emit_bcc(u8 op, u8 rd, u8 rs, int rvoff, struct rv_jit_context *ctx) 
	if (far) {

		e = ctx->ninsns;

		/* Adjust for extra insns. */

		rvoff -= (e - s) << 2;

		rvoff -= ninsns_rvoff(e - s);

		emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx);

	}

	return 0;
@@ -731,7 +731,7 @@ static int emit_branch_r32(const s8 *src1, const s8 *src2, s32 rvoff, 


	e = ctx->ninsns;

	/* Adjust for extra insns. */

	rvoff -= (e - s) << 2;

	rvoff -= ninsns_rvoff(e - s);



	if (emit_bcc(op, lo(rs1), lo(rs2), rvoff, ctx))

		return -1;
@@ -795,7 +795,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 
	 * if (index >= max_entries)

	 *   goto out;

	 */

	off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;

	off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));

	emit_bcc(BPF_JGE, lo(idx_reg), RV_REG_T1, off, ctx);



	/*
@@ -804,7 +804,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 
	 *   goto out;

	 */

	emit(rv_addi(RV_REG_T1, RV_REG_TCC, -1), ctx);

	off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;

	off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));

	emit_bcc(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx);



	/*
@@ -818,7 +818,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 
	if (is_12b_check(off, insn))

		return -1;

	emit(rv_lw(RV_REG_T0, off, RV_REG_T0), ctx);

	off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;

	off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));

	emit_bcc(BPF_JEQ, RV_REG_T0, RV_REG_ZERO, off, ctx);



	/*
@@ -1214,7 +1214,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
			emit_imm32(tmp2, imm, ctx);

			src = tmp2;

			e = ctx->ninsns;

			rvoff -= (e - s) << 2;

			rvoff -= ninsns_rvoff(e - s);

		}



		if (is64)

arch/riscv/net/bpf_jit_comp64.c

+153 −140

File changed.

Preview size limit exceeded, changes collapsed.

arch/riscv/net/bpf_jit_core.c

+3 −3
Original line number Diff line number Diff line
@@ -73,7 +73,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 


	if (ctx->offset) {

		extra_pass = true;

		image_size = sizeof(u32) * ctx->ninsns;

		image_size = sizeof(*ctx->insns) * ctx->ninsns;

		goto skip_init_ctx;

	}
@@ -103,7 +103,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 
			if (jit_data->header)

				break;



			image_size = sizeof(u32) * ctx->ninsns;

			image_size = sizeof(*ctx->insns) * ctx->ninsns;

			jit_data->header =

				bpf_jit_binary_alloc(image_size,

						     &jit_data->image,
@@ -114,7 +114,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 
				goto out_offset;

			}



			ctx->insns = (u32 *)jit_data->image;

			ctx->insns = (u16 *)jit_data->image;

			/*

			 * Now, when the image is allocated, the image can

			 * potentially shrink more (auipc/jalr -> jal).

arch/s390/net/bpf_jit_comp.c

+41 −22
Original line number Diff line number Diff line
@@ -489,6 +489,24 @@ static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth) 
	} while (re <= last);

}



static void bpf_skip(struct bpf_jit *jit, int size)

{

	if (size >= 6 && !is_valid_rel(size)) {

		/* brcl 0xf,size */

		EMIT6_PCREL_RIL(0xc0f4000000, size);

		size -= 6;

	} else if (size >= 4 && is_valid_rel(size)) {

		/* brc 0xf,size */

		EMIT4_PCREL(0xa7f40000, size);

		size -= 4;

	}

	while (size >= 2) {

		/* bcr 0,%0 */

		_EMIT2(0x0700);

		size -= 2;

	}

}



/*

 * Emit function prologue

 *
@@ -501,10 +519,11 @@ static void bpf_jit_prologue(struct bpf_jit *jit, u32 stack_depth) 
		/* xc STK_OFF_TCCNT(4,%r15),STK_OFF_TCCNT(%r15) */

		_EMIT6(0xd703f000 | STK_OFF_TCCNT, 0xf000 | STK_OFF_TCCNT);

	} else {

		/* j tail_call_start: NOP if no tail calls are used */

		EMIT4_PCREL(0xa7f40000, 6);

		/* bcr 0,%0 */

		EMIT2(0x0700, 0, REG_0);

		/*

		 * There are no tail calls. Insert nops in order to have

		 * tail_call_start at a predictable offset.

		 */

		bpf_skip(jit, 6);

	}

	/* Tail calls have to skip above initialization */

	jit->tail_call_start = jit->prg;
@@ -1268,8 +1287,12 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 
		last = (i == fp->len - 1) ? 1 : 0;

		if (last)

			break;

		/* j <exit> */

		EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg);

		if (!is_first_pass(jit) && can_use_rel(jit, jit->exit_ip))

			/* brc 0xf, <exit> */

			EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip);

		else

			/* brcl 0xf, <exit> */

			EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip);

		break;

	/*

	 * Branch relative (number of skipped instructions) to offset on
@@ -1417,21 +1440,10 @@ branch_ks: 
		}

		break;

branch_ku:

		is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;

		/* clfi or clgfi %dst,imm */

		EMIT6_IMM(is_jmp32 ? 0xc20f0000 : 0xc20e0000,

			  dst_reg, imm);

		if (!is_first_pass(jit) &&

		    can_use_rel(jit, addrs[i + off + 1])) {

			/* brc mask,off */

			EMIT4_PCREL_RIC(0xa7040000,

					mask >> 12, addrs[i + off + 1]);

		} else {

			/* brcl mask,off */

			EMIT6_PCREL_RILC(0xc0040000,

					 mask >> 12, addrs[i + off + 1]);

		}

		break;

		/* lgfi %w1,imm (load sign extend imm) */

		src_reg = REG_1;

		EMIT6_IMM(0xc0010000, src_reg, imm);

		goto branch_xu;

branch_xs:

		is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;

		if (!is_first_pass(jit) &&
@@ -1510,7 +1522,14 @@ static bool bpf_is_new_addr_sane(struct bpf_jit *jit, int i) 
 */

static int bpf_set_addr(struct bpf_jit *jit, int i)

{

	if (!bpf_is_new_addr_sane(jit, i))

	int delta;



	if (is_codegen_pass(jit)) {

		delta = jit->prg - jit->addrs[i];

		if (delta < 0)

			bpf_skip(jit, -delta);

	}

	if (WARN_ON_ONCE(!bpf_is_new_addr_sane(jit, i)))

		return -1;

	jit->addrs[i] = jit->prg;

	return 0;

CRA Git | Maintained and supported by SUSTech CRA and CCSE