Commit 5234ccf2 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'move-ld_abs-to-native-BPF' 



Daniel Borkmann says:

====================
This set simplifies BPF JITs significantly by moving ld_abs/ld_ind
to native BPF, for details see individual patches. Main rationale
is in patch 'implement ld_abs/ld_ind in native bpf'. Thanks!

v1 -> v2:
  - Added missing seen_lds_abs in LDX_MSH and use X = A
    initially due to being preserved on func call.
  - Added a large batch of cBPF tests into test_bpf.
  - Added x32 removal of LD_ABS/LD_IND, so all JITs are
    covered.
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 08dbc7a6 32b3652c

arch/arm/net/bpf_jit_32.c

+0 −77
Original line number Diff line number Diff line
@@ -1452,83 +1452,6 @@ exit: 
			emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);

		emit_ldx_r(dst, rn, dstk, off, ctx, BPF_SIZE(code));

		break;

	/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */

	case BPF_LD | BPF_ABS | BPF_W:

	case BPF_LD | BPF_ABS | BPF_H:

	case BPF_LD | BPF_ABS | BPF_B:

	/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */

	case BPF_LD | BPF_IND | BPF_W:

	case BPF_LD | BPF_IND | BPF_H:

	case BPF_LD | BPF_IND | BPF_B:

	{

		const u8 r4 = bpf2a32[BPF_REG_6][1]; /* r4 = ptr to sk_buff */

		const u8 r0 = bpf2a32[BPF_REG_0][1]; /*r0: struct sk_buff *skb*/

						     /* rtn value */

		const u8 r1 = bpf2a32[BPF_REG_0][0]; /* r1: int k */

		const u8 r2 = bpf2a32[BPF_REG_1][1]; /* r2: unsigned int size */

		const u8 r3 = bpf2a32[BPF_REG_1][0]; /* r3: void *buffer */

		const u8 r6 = bpf2a32[TMP_REG_1][1]; /* r6: void *(*func)(..) */

		int size;



		/* Setting up first argument */

		emit(ARM_MOV_R(r0, r4), ctx);



		/* Setting up second argument */

		emit_a32_mov_i(r1, imm, false, ctx);

		if (BPF_MODE(code) == BPF_IND)

			emit_a32_alu_r(r1, src_lo, false, sstk, ctx,

				       false, false, BPF_ADD);



		/* Setting up third argument */

		switch (BPF_SIZE(code)) {

		case BPF_W:

			size = 4;

			break;

		case BPF_H:

			size = 2;

			break;

		case BPF_B:

			size = 1;

			break;

		default:

			return -EINVAL;

		}

		emit_a32_mov_i(r2, size, false, ctx);



		/* Setting up fourth argument */

		emit(ARM_ADD_I(r3, ARM_SP, imm8m(SKB_BUFFER)), ctx);



		/* Setting up function pointer to call */

		emit_a32_mov_i(r6, (unsigned int)bpf_load_pointer, false, ctx);

		emit_blx_r(r6, ctx);



		emit(ARM_EOR_R(r1, r1, r1), ctx);

		/* Check if return address is NULL or not.

		 * if NULL then jump to epilogue

		 * else continue to load the value from retn address

		 */

		emit(ARM_CMP_I(r0, 0), ctx);

		jmp_offset = epilogue_offset(ctx);

		check_imm24(jmp_offset);

		_emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);



		/* Load value from the address */

		switch (BPF_SIZE(code)) {

		case BPF_W:

			emit(ARM_LDR_I(r0, r0, 0), ctx);

			emit_rev32(r0, r0, ctx);

			break;

		case BPF_H:

			emit(ARM_LDRH_I(r0, r0, 0), ctx);

			emit_rev16(r0, r0, ctx);

			break;

		case BPF_B:

			emit(ARM_LDRB_I(r0, r0, 0), ctx);

			/* No need to reverse */

			break;

		}

		break;

	}

	/* ST: *(size *)(dst + off) = imm */

	case BPF_ST | BPF_MEM | BPF_W:

	case BPF_ST | BPF_MEM | BPF_H:

arch/arm64/net/bpf_jit_comp.c

+0 −65
Original line number Diff line number Diff line
@@ -723,71 +723,6 @@ emit_cond_jmp: 
		emit(A64_CBNZ(0, tmp3, jmp_offset), ctx);

		break;



	/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */

	case BPF_LD | BPF_ABS | BPF_W:

	case BPF_LD | BPF_ABS | BPF_H:

	case BPF_LD | BPF_ABS | BPF_B:

	/* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */

	case BPF_LD | BPF_IND | BPF_W:

	case BPF_LD | BPF_IND | BPF_H:

	case BPF_LD | BPF_IND | BPF_B:

	{

		const u8 r0 = bpf2a64[BPF_REG_0]; /* r0 = return value */

		const u8 r6 = bpf2a64[BPF_REG_6]; /* r6 = pointer to sk_buff */

		const u8 fp = bpf2a64[BPF_REG_FP];

		const u8 r1 = bpf2a64[BPF_REG_1]; /* r1: struct sk_buff *skb */

		const u8 r2 = bpf2a64[BPF_REG_2]; /* r2: int k */

		const u8 r3 = bpf2a64[BPF_REG_3]; /* r3: unsigned int size */

		const u8 r4 = bpf2a64[BPF_REG_4]; /* r4: void *buffer */

		const u8 r5 = bpf2a64[BPF_REG_5]; /* r5: void *(*func)(...) */

		int size;



		emit(A64_MOV(1, r1, r6), ctx);

		emit_a64_mov_i(0, r2, imm, ctx);

		if (BPF_MODE(code) == BPF_IND)

			emit(A64_ADD(0, r2, r2, src), ctx);

		switch (BPF_SIZE(code)) {

		case BPF_W:

			size = 4;

			break;

		case BPF_H:

			size = 2;

			break;

		case BPF_B:

			size = 1;

			break;

		default:

			return -EINVAL;

		}

		emit_a64_mov_i64(r3, size, ctx);

		emit(A64_SUB_I(1, r4, fp, ctx->stack_size), ctx);

		emit_a64_mov_i64(r5, (unsigned long)bpf_load_pointer, ctx);

		emit(A64_BLR(r5), ctx);

		emit(A64_MOV(1, r0, A64_R(0)), ctx);



		jmp_offset = epilogue_offset(ctx);

		check_imm19(jmp_offset);

		emit(A64_CBZ(1, r0, jmp_offset), ctx);

		emit(A64_MOV(1, r5, r0), ctx);

		switch (BPF_SIZE(code)) {

		case BPF_W:

			emit(A64_LDR32(r0, r5, A64_ZR), ctx);

#ifndef CONFIG_CPU_BIG_ENDIAN

			emit(A64_REV32(0, r0, r0), ctx);

#endif

			break;

		case BPF_H:

			emit(A64_LDRH(r0, r5, A64_ZR), ctx);

#ifndef CONFIG_CPU_BIG_ENDIAN

			emit(A64_REV16(0, r0, r0), ctx);

#endif

			break;

		case BPF_B:

			emit(A64_LDRB(r0, r5, A64_ZR), ctx);

			break;

		}

		break;

	}

	default:

		pr_err_once("unknown opcode %02x\n", code);

		return -EINVAL;

arch/mips/net/ebpf_jit.c

+0 −104
Original line number Diff line number Diff line
@@ -1267,110 +1267,6 @@ jeq_common: 
			return -EINVAL;

		break;



	case BPF_LD | BPF_B | BPF_ABS:

	case BPF_LD | BPF_H | BPF_ABS:

	case BPF_LD | BPF_W | BPF_ABS:

	case BPF_LD | BPF_DW | BPF_ABS:

		ctx->flags |= EBPF_SAVE_RA;



		gen_imm_to_reg(insn, MIPS_R_A1, ctx);

		emit_instr(ctx, addiu, MIPS_R_A2, MIPS_R_ZERO, size_to_len(insn));



		if (insn->imm < 0) {

			emit_const_to_reg(ctx, MIPS_R_T9, (u64)bpf_internal_load_pointer_neg_helper);

		} else {

			emit_const_to_reg(ctx, MIPS_R_T9, (u64)ool_skb_header_pointer);

			emit_instr(ctx, daddiu, MIPS_R_A3, MIPS_R_SP, ctx->tmp_offset);

		}

		goto ld_skb_common;



	case BPF_LD | BPF_B | BPF_IND:

	case BPF_LD | BPF_H | BPF_IND:

	case BPF_LD | BPF_W | BPF_IND:

	case BPF_LD | BPF_DW | BPF_IND:

		ctx->flags |= EBPF_SAVE_RA;

		src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);

		if (src < 0)

			return src;

		ts = get_reg_val_type(ctx, this_idx, insn->src_reg);

		if (ts == REG_32BIT_ZERO_EX) {

			/* sign extend */

			emit_instr(ctx, sll, MIPS_R_A1, src, 0);

			src = MIPS_R_A1;

		}

		if (insn->imm >= S16_MIN && insn->imm <= S16_MAX) {

			emit_instr(ctx, daddiu, MIPS_R_A1, src, insn->imm);

		} else {

			gen_imm_to_reg(insn, MIPS_R_AT, ctx);

			emit_instr(ctx, daddu, MIPS_R_A1, MIPS_R_AT, src);

		}

		/* truncate to 32-bit int */

		emit_instr(ctx, sll, MIPS_R_A1, MIPS_R_A1, 0);

		emit_instr(ctx, daddiu, MIPS_R_A3, MIPS_R_SP, ctx->tmp_offset);

		emit_instr(ctx, slt, MIPS_R_AT, MIPS_R_A1, MIPS_R_ZERO);



		emit_const_to_reg(ctx, MIPS_R_T8, (u64)bpf_internal_load_pointer_neg_helper);

		emit_const_to_reg(ctx, MIPS_R_T9, (u64)ool_skb_header_pointer);

		emit_instr(ctx, addiu, MIPS_R_A2, MIPS_R_ZERO, size_to_len(insn));

		emit_instr(ctx, movn, MIPS_R_T9, MIPS_R_T8, MIPS_R_AT);



ld_skb_common:

		emit_instr(ctx, jalr, MIPS_R_RA, MIPS_R_T9);

		/* delay slot move */

		emit_instr(ctx, daddu, MIPS_R_A0, MIPS_R_S0, MIPS_R_ZERO);



		/* Check the error value */

		b_off = b_imm(exit_idx, ctx);

		if (is_bad_offset(b_off)) {

			target = j_target(ctx, exit_idx);

			if (target == (unsigned int)-1)

				return -E2BIG;



			if (!(ctx->offsets[this_idx] & OFFSETS_B_CONV)) {

				ctx->offsets[this_idx] |= OFFSETS_B_CONV;

				ctx->long_b_conversion = 1;

			}

			emit_instr(ctx, bne, MIPS_R_V0, MIPS_R_ZERO, 4 * 3);

			emit_instr(ctx, nop);

			emit_instr(ctx, j, target);

			emit_instr(ctx, nop);

		} else {

			emit_instr(ctx, beq, MIPS_R_V0, MIPS_R_ZERO, b_off);

			emit_instr(ctx, nop);

		}



#ifdef __BIG_ENDIAN

		need_swap = false;

#else

		need_swap = true;

#endif

		dst = MIPS_R_V0;

		switch (BPF_SIZE(insn->code)) {

		case BPF_B:

			emit_instr(ctx, lbu, dst, 0, MIPS_R_V0);

			break;

		case BPF_H:

			emit_instr(ctx, lhu, dst, 0, MIPS_R_V0);

			if (need_swap)

				emit_instr(ctx, wsbh, dst, dst);

			break;

		case BPF_W:

			emit_instr(ctx, lw, dst, 0, MIPS_R_V0);

			if (need_swap) {

				emit_instr(ctx, wsbh, dst, dst);

				emit_instr(ctx, rotr, dst, dst, 16);

			}

			break;

		case BPF_DW:

			emit_instr(ctx, ld, dst, 0, MIPS_R_V0);

			if (need_swap) {

				emit_instr(ctx, dsbh, dst, dst);

				emit_instr(ctx, dshd, dst, dst);

			}

			break;

		}



		break;

	case BPF_ALU | BPF_END | BPF_FROM_BE:

	case BPF_ALU | BPF_END | BPF_FROM_LE:

		dst = ebpf_to_mips_reg(ctx, insn, dst_reg);

arch/powerpc/net/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -3,7 +3,7 @@ 
# Arch-specific network modules

#

ifeq ($(CONFIG_PPC64),y)

obj-$(CONFIG_BPF_JIT) += bpf_jit_asm64.o bpf_jit_comp64.o

obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o

else

obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o

endif

arch/powerpc/net/bpf_jit64.h

+9 −28
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ 
 * with our redzone usage.

 *

 *		[	prev sp		] <-------------

 *		[   nv gpr save area	] 8*8		|

 *		[   nv gpr save area	] 6*8		|

 *		[    tail_call_cnt	] 8		|

 *		[    local_tmp_var	] 8		|

 * fp (r31) -->	[   ebpf stack space	] upto 512	|
@@ -28,8 +28,8 @@ 
 * sp (r1) --->	[    stack pointer	] --------------

 */



/* for gpr non volatile registers BPG_REG_6 to 10, plus skb cache registers */

#define BPF_PPC_STACK_SAVE	(8*8)

/* for gpr non volatile registers BPG_REG_6 to 10 */

#define BPF_PPC_STACK_SAVE	(6*8)

/* for bpf JIT code internal usage */

#define BPF_PPC_STACK_LOCALS	16

/* stack frame excluding BPF stack, ensure this is quadword aligned */
@@ -39,10 +39,8 @@ 
#ifndef __ASSEMBLY__



/* BPF register usage */

#define SKB_HLEN_REG	(MAX_BPF_JIT_REG + 0)

#define SKB_DATA_REG	(MAX_BPF_JIT_REG + 1)

#define TMP_REG_1	(MAX_BPF_JIT_REG + 2)

#define TMP_REG_2	(MAX_BPF_JIT_REG + 3)

#define TMP_REG_1	(MAX_BPF_JIT_REG + 0)

#define TMP_REG_2	(MAX_BPF_JIT_REG + 1)



/* BPF to ppc register mappings */

static const int b2p[] = {
@@ -63,40 +61,23 @@ static const int b2p[] = { 
	[BPF_REG_FP] = 31,

	/* eBPF jit internal registers */

	[BPF_REG_AX] = 2,

	[SKB_HLEN_REG] = 25,

	[SKB_DATA_REG] = 26,

	[TMP_REG_1] = 9,

	[TMP_REG_2] = 10

};



/* PPC NVR range -- update this if we ever use NVRs below r24 */

#define BPF_PPC_NVR_MIN		24



/* Assembly helpers */

#define DECLARE_LOAD_FUNC(func)	u64 func(u64 r3, u64 r4);			\

				u64 func##_negative_offset(u64 r3, u64 r4);	\

				u64 func##_positive_offset(u64 r3, u64 r4);



DECLARE_LOAD_FUNC(sk_load_word);

DECLARE_LOAD_FUNC(sk_load_half);

DECLARE_LOAD_FUNC(sk_load_byte);



#define CHOOSE_LOAD_FUNC(imm, func)						\

			(imm < 0 ?						\

			(imm >= SKF_LL_OFF ? func##_negative_offset : func) :	\

			func##_positive_offset)

/* PPC NVR range -- update this if we ever use NVRs below r27 */

#define BPF_PPC_NVR_MIN		27



#define SEEN_FUNC	0x1000 /* might call external helpers */

#define SEEN_STACK	0x2000 /* uses BPF stack */

#define SEEN_SKB	0x4000 /* uses sk_buff */

#define SEEN_TAILCALL	0x8000 /* uses tail calls */

#define SEEN_TAILCALL	0x4000 /* uses tail calls */



struct codegen_context {

	/*

	 * This is used to track register usage as well

	 * as calls to external helpers.

	 * - register usage is tracked with corresponding

	 *   bits (r3-r10 and r25-r31)

	 *   bits (r3-r10 and r27-r31)

	 * - rest of the bits can be used to track other

	 *   things -- for now, we use bits 16 to 23

	 *   encoded in SEEN_* macros above

CRA Git | Maintained and supported by SUSTech CRA and CCSE