Commit 836256bf authored by Jiong Wang's avatar Jiong Wang Committed by Alexei Starovoitov
Browse files

x32: bpf: eliminate zero extension code-gen 



Cc: Wang YanQing <udknight@gmail.com>
Tested-by: default avatarWang YanQing <udknight@gmail.com>
Signed-off-by: default avatarJiong Wang <jiong.wang@netronome.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 3e2a33cf

arch/x86/net/bpf_jit_comp32.c

+56 −27
Original line number Diff line number Diff line
@@ -253,13 +253,14 @@ static inline void emit_ia32_mov_r(const u8 dst, const u8 src, bool dstk, 
/* dst = src */

static inline void emit_ia32_mov_r64(const bool is64, const u8 dst[],

				     const u8 src[], bool dstk,

				     bool sstk, u8 **pprog)

				     bool sstk, u8 **pprog,

				     const struct bpf_prog_aux *aux)

{

	emit_ia32_mov_r(dst_lo, src_lo, dstk, sstk, pprog);

	if (is64)

		/* complete 8 byte move */

		emit_ia32_mov_r(dst_hi, src_hi, dstk, sstk, pprog);

	else

	else if (!aux->verifier_zext)

		/* zero out high 4 bytes */

		emit_ia32_mov_i(dst_hi, 0, dstk, pprog);

}
@@ -313,7 +314,8 @@ static inline void emit_ia32_mul_r(const u8 dst, const u8 src, bool dstk, 
}



static inline void emit_ia32_to_le_r64(const u8 dst[], s32 val,

					 bool dstk, u8 **pprog)

					 bool dstk, u8 **pprog,

					 const struct bpf_prog_aux *aux)

{

	u8 *prog = *pprog;

	int cnt = 0;
@@ -334,10 +336,12 @@ static inline void emit_ia32_to_le_r64(const u8 dst[], s32 val, 
		 */

		EMIT2(0x0F, 0xB7);

		EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo));

		if (!aux->verifier_zext)

			/* xor dreg_hi,dreg_hi */

			EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));

		break;

	case 32:

		if (!aux->verifier_zext)

			/* xor dreg_hi,dreg_hi */

			EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));

		break;
@@ -358,7 +362,8 @@ static inline void emit_ia32_to_le_r64(const u8 dst[], s32 val, 
}



static inline void emit_ia32_to_be_r64(const u8 dst[], s32 val,

				       bool dstk, u8 **pprog)

				       bool dstk, u8 **pprog,

				       const struct bpf_prog_aux *aux)

{

	u8 *prog = *pprog;

	int cnt = 0;
@@ -380,6 +385,7 @@ static inline void emit_ia32_to_be_r64(const u8 dst[], s32 val, 
		EMIT2(0x0F, 0xB7);

		EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo));



		if (!aux->verifier_zext)

			/* xor dreg_hi,dreg_hi */

			EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));

		break;
@@ -388,6 +394,7 @@ static inline void emit_ia32_to_be_r64(const u8 dst[], s32 val, 
		EMIT1(0x0F);

		EMIT1(add_1reg(0xC8, dreg_lo));



		if (!aux->verifier_zext)

			/* xor dreg_hi,dreg_hi */

			EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));

		break;
@@ -569,7 +576,7 @@ static inline void emit_ia32_alu_r(const bool is64, const bool hi, const u8 op, 
static inline void emit_ia32_alu_r64(const bool is64, const u8 op,

				     const u8 dst[], const u8 src[],

				     bool dstk,  bool sstk,

				     u8 **pprog)

				     u8 **pprog, const struct bpf_prog_aux *aux)

{

	u8 *prog = *pprog;
@@ -577,7 +584,7 @@ static inline void emit_ia32_alu_r64(const bool is64, const u8 op, 
	if (is64)

		emit_ia32_alu_r(is64, true, op, dst_hi, src_hi, dstk, sstk,

				&prog);

	else

	else if (!aux->verifier_zext)

		emit_ia32_mov_i(dst_hi, 0, dstk, &prog);

	*pprog = prog;

}
@@ -668,7 +675,8 @@ static inline void emit_ia32_alu_i(const bool is64, const bool hi, const u8 op, 
/* ALU operation (64 bit) */

static inline void emit_ia32_alu_i64(const bool is64, const u8 op,

				     const u8 dst[], const u32 val,

				     bool dstk, u8 **pprog)

				     bool dstk, u8 **pprog,

				     const struct bpf_prog_aux *aux)

{

	u8 *prog = *pprog;

	u32 hi = 0;
@@ -679,7 +687,7 @@ static inline void emit_ia32_alu_i64(const bool is64, const u8 op, 
	emit_ia32_alu_i(is64, false, op, dst_lo, val, dstk, &prog);

	if (is64)

		emit_ia32_alu_i(is64, true, op, dst_hi, hi, dstk, &prog);

	else

	else if (!aux->verifier_zext)

		emit_ia32_mov_i(dst_hi, 0, dstk, &prog);



	*pprog = prog;
@@ -1713,8 +1721,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, 
		case BPF_ALU64 | BPF_MOV | BPF_X:

			switch (BPF_SRC(code)) {

			case BPF_X:

				emit_ia32_mov_r64(is64, dst, src, dstk,

						  sstk, &prog);

				if (imm32 == 1) {

					/* Special mov32 for zext. */

					emit_ia32_mov_i(dst_hi, 0, dstk, &prog);

					break;

				}

				emit_ia32_mov_r64(is64, dst, src, dstk, sstk,

						  &prog, bpf_prog->aux);

				break;

			case BPF_K:

				/* Sign-extend immediate value to dst reg */
@@ -1754,11 +1767,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, 
			switch (BPF_SRC(code)) {

			case BPF_X:

				emit_ia32_alu_r64(is64, BPF_OP(code), dst,

						  src, dstk, sstk, &prog);

						  src, dstk, sstk, &prog,

						  bpf_prog->aux);

				break;

			case BPF_K:

				emit_ia32_alu_i64(is64, BPF_OP(code), dst,

						  imm32, dstk, &prog);

						  imm32, dstk, &prog,

						  bpf_prog->aux);

				break;

			}

			break;
@@ -1777,6 +1792,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, 
						false, &prog);

				break;

			}

			if (!bpf_prog->aux->verifier_zext)

				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);

			break;

		case BPF_ALU | BPF_LSH | BPF_X:
@@ -1797,6 +1813,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, 
						  &prog);

				break;

			}

			if (!bpf_prog->aux->verifier_zext)

				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);

			break;

		/* dst = dst / src(imm) */
@@ -1819,6 +1836,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, 
						    &prog);

				break;

			}

			if (!bpf_prog->aux->verifier_zext)

				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);

			break;

		case BPF_ALU64 | BPF_DIV | BPF_K:
@@ -1836,6 +1854,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, 
			EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32);

			emit_ia32_shift_r(BPF_OP(code), dst_lo, IA32_ECX, dstk,

					  false, &prog);

			if (!bpf_prog->aux->verifier_zext)

				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);

			break;

		/* dst = dst << imm */
@@ -1872,6 +1891,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, 
		case BPF_ALU | BPF_NEG:

			emit_ia32_alu_i(is64, false, BPF_OP(code),

					dst_lo, 0, dstk, &prog);

			if (!bpf_prog->aux->verifier_zext)

				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);

			break;

		/* dst = ~dst (64 bit) */
@@ -1892,11 +1912,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, 
			break;

		/* dst = htole(dst) */

		case BPF_ALU | BPF_END | BPF_FROM_LE:

			emit_ia32_to_le_r64(dst, imm32, dstk, &prog);

			emit_ia32_to_le_r64(dst, imm32, dstk, &prog,

					    bpf_prog->aux);

			break;

		/* dst = htobe(dst) */

		case BPF_ALU | BPF_END | BPF_FROM_BE:

			emit_ia32_to_be_r64(dst, imm32, dstk, &prog);

			emit_ia32_to_be_r64(dst, imm32, dstk, &prog,

					    bpf_prog->aux);

			break;

		/* dst = imm64 */

		case BPF_LD | BPF_IMM | BPF_DW: {
@@ -2051,6 +2073,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, 
			case BPF_B:

			case BPF_H:

			case BPF_W:

				if (!bpf_prog->aux->verifier_zext)

					break;

				if (dstk) {

					EMIT3(0xC7, add_1reg(0x40, IA32_EBP),

					      STACK_VAR(dst_hi));
@@ -2475,6 +2499,11 @@ notyet: 
	return proglen;

}



bool bpf_jit_needs_zext(void)

{

	return true;

}



struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)

{

	struct bpf_binary_header *header = NULL;

CRA Git | Maintained and supported by SUSTech CRA and CCSE