Commit 3b2744e6 authored by Alexei Starovoitov's avatar Alexei Starovoitov Committed by Daniel Borkmann
Browse files

bpf: Refactor x86 JIT into helpers 



Refactor x86 JITing of LDX, STX, CALL instructions into separate helper
functions.  No functional changes in LDX and STX helpers.  There is a minor
change in CALL helper. It will populate target address correctly on the first
pass of JIT instead of second pass. That won't reduce total number of JIT
passes though.

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Acked-by: default avatarSong Liu <songliubraving@fb.com>
Acked-by: default avatarAndrii Nakryiko <andriin@fb.com>
Link: https://lore.kernel.org/bpf/20191114185720.1641606-3-ast@kernel.org
parent c3d6324f

arch/x86/net/bpf_jit_comp.c

+98 −54
Original line number Diff line number Diff line
@@ -198,6 +198,8 @@ struct jit_context { 
/* Maximum number of bytes emitted while JITing one eBPF insn */

#define BPF_MAX_INSN_SIZE	128

#define BPF_INSN_SAFETY		64

/* number of bytes emit_call() needs to generate call instruction */

#define X86_CALL_SIZE		5



#define PROLOGUE_SIZE		20
@@ -390,6 +392,99 @@ static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg) 
	*pprog = prog;

}



/* LDX: dst_reg = *(u8*)(src_reg + off) */

static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)

{

	u8 *prog = *pprog;

	int cnt = 0;



	switch (size) {

	case BPF_B:

		/* Emit 'movzx rax, byte ptr [rax + off]' */

		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);

		break;

	case BPF_H:

		/* Emit 'movzx rax, word ptr [rax + off]' */

		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);

		break;

	case BPF_W:

		/* Emit 'mov eax, dword ptr [rax+0x14]' */

		if (is_ereg(dst_reg) || is_ereg(src_reg))

			EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);

		else

			EMIT1(0x8B);

		break;

	case BPF_DW:

		/* Emit 'mov rax, qword ptr [rax+0x14]' */

		EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);

		break;

	}

	/*

	 * If insn->off == 0 we can save one extra byte, but

	 * special case of x86 R13 which always needs an offset

	 * is not worth the hassle

	 */

	if (is_imm8(off))

		EMIT2(add_2reg(0x40, src_reg, dst_reg), off);

	else

		EMIT1_off32(add_2reg(0x80, src_reg, dst_reg), off);

	*pprog = prog;

}



/* STX: *(u8*)(dst_reg + off) = src_reg */

static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)

{

	u8 *prog = *pprog;

	int cnt = 0;



	switch (size) {

	case BPF_B:

		/* Emit 'mov byte ptr [rax + off], al' */

		if (is_ereg(dst_reg) || is_ereg(src_reg) ||

		    /* We have to add extra byte for x86 SIL, DIL regs */

		    src_reg == BPF_REG_1 || src_reg == BPF_REG_2)

			EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88);

		else

			EMIT1(0x88);

		break;

	case BPF_H:

		if (is_ereg(dst_reg) || is_ereg(src_reg))

			EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89);

		else

			EMIT2(0x66, 0x89);

		break;

	case BPF_W:

		if (is_ereg(dst_reg) || is_ereg(src_reg))

			EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89);

		else

			EMIT1(0x89);

		break;

	case BPF_DW:

		EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89);

		break;

	}

	if (is_imm8(off))

		EMIT2(add_2reg(0x40, dst_reg, src_reg), off);

	else

		EMIT1_off32(add_2reg(0x80, dst_reg, src_reg), off);

	*pprog = prog;

}



static int emit_call(u8 **pprog, void *func, void *ip)

{

	u8 *prog = *pprog;

	int cnt = 0;

	s64 offset;



	offset = func - (ip + X86_CALL_SIZE);

	if (!is_simm32(offset)) {

		pr_err("Target call %p is out of range\n", func);

		return -EINVAL;

	}

	EMIT1_off32(0xE8, offset);

	*pprog = prog;

	return 0;

}



static bool ex_handler_bpf(const struct exception_table_entry *x,

			   struct pt_regs *regs, int trapnr,
@@ -773,68 +868,22 @@ st: if (is_imm8(insn->off)) 


			/* STX: *(u8*)(dst_reg + off) = src_reg */

		case BPF_STX | BPF_MEM | BPF_B:

			/* Emit 'mov byte ptr [rax + off], al' */

			if (is_ereg(dst_reg) || is_ereg(src_reg) ||

			    /* We have to add extra byte for x86 SIL, DIL regs */

			    src_reg == BPF_REG_1 || src_reg == BPF_REG_2)

				EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88);

			else

				EMIT1(0x88);

			goto stx;

		case BPF_STX | BPF_MEM | BPF_H:

			if (is_ereg(dst_reg) || is_ereg(src_reg))

				EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89);

			else

				EMIT2(0x66, 0x89);

			goto stx;

		case BPF_STX | BPF_MEM | BPF_W:

			if (is_ereg(dst_reg) || is_ereg(src_reg))

				EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89);

			else

				EMIT1(0x89);

			goto stx;

		case BPF_STX | BPF_MEM | BPF_DW:

			EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89);

stx:			if (is_imm8(insn->off))

				EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off);

			else

				EMIT1_off32(add_2reg(0x80, dst_reg, src_reg),

					    insn->off);

			emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);

			break;



			/* LDX: dst_reg = *(u8*)(src_reg + off) */

		case BPF_LDX | BPF_MEM | BPF_B:

		case BPF_LDX | BPF_PROBE_MEM | BPF_B:

			/* Emit 'movzx rax, byte ptr [rax + off]' */

			EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);

			goto ldx;

		case BPF_LDX | BPF_MEM | BPF_H:

		case BPF_LDX | BPF_PROBE_MEM | BPF_H:

			/* Emit 'movzx rax, word ptr [rax + off]' */

			EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);

			goto ldx;

		case BPF_LDX | BPF_MEM | BPF_W:

		case BPF_LDX | BPF_PROBE_MEM | BPF_W:

			/* Emit 'mov eax, dword ptr [rax+0x14]' */

			if (is_ereg(dst_reg) || is_ereg(src_reg))

				EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);

			else

				EMIT1(0x8B);

			goto ldx;

		case BPF_LDX | BPF_MEM | BPF_DW:

		case BPF_LDX | BPF_PROBE_MEM | BPF_DW:

			/* Emit 'mov rax, qword ptr [rax+0x14]' */

			EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);

ldx:			/*

			 * If insn->off == 0 we can save one extra byte, but

			 * special case of x86 R13 which always needs an offset

			 * is not worth the hassle

			 */

			if (is_imm8(insn->off))

				EMIT2(add_2reg(0x40, src_reg, dst_reg), insn->off);

			else

				EMIT1_off32(add_2reg(0x80, src_reg, dst_reg),

					    insn->off);

			emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);

			if (BPF_MODE(insn->code) == BPF_PROBE_MEM) {

				struct exception_table_entry *ex;

				u8 *_insn = image + proglen;
@@ -899,13 +948,8 @@ xadd: if (is_imm8(insn->off)) 
			/* call */

		case BPF_JMP | BPF_CALL:

			func = (u8 *) __bpf_call_base + imm32;

			jmp_offset = func - (image + addrs[i]);

			if (!imm32 || !is_simm32(jmp_offset)) {

				pr_err("unsupported BPF func %d addr %p image %p\n",

				       imm32, func, image);

			if (!imm32 || emit_call(&prog, func, image + addrs[i - 1]))

				return -EINVAL;

			}

			EMIT1_off32(0xE8, jmp_offset);

			break;



		case BPF_JMP | BPF_TAIL_CALL:

CRA Git | Maintained and supported by SUSTech CRA and CCSE