Commit 116eb788 authored by Björn Töpel's avatar Björn Töpel Committed by Alexei Starovoitov
Browse files

bpf, x86: Align dispatcher branch targets to 16B 



>From Intel 64 and IA-32 Architectures Optimization Reference Manual,
3.4.1.4 Code Alignment, Assembly/Compiler Coding Rule 11: All branch
targets should be 16-byte aligned.

This commits aligns branch targets according to the Intel manual.

The nops used to align branch targets make the dispatcher larger, and
therefore the number of supported dispatch points/programs are
descreased from 64 to 48.

Signed-off-by: default avatarBjörn Töpel <bjorn.topel@intel.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20191213175112.30208-7-bjorn.topel@gmail.com
parent e754f5a6

arch/x86/net/bpf_jit_comp.c

+29 −1
Original line number Diff line number Diff line
@@ -1548,6 +1548,26 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) 
	return 0;

}



static void emit_nops(u8 **pprog, unsigned int len)

{

	unsigned int i, noplen;

	u8 *prog = *pprog;

	int cnt = 0;



	while (len > 0) {

		noplen = len;



		if (noplen > ASM_NOP_MAX)

			noplen = ASM_NOP_MAX;



		for (i = 0; i < noplen; i++)

			EMIT1(ideal_nops[noplen][i]);

		len -= noplen;

	}



	*pprog = prog;

}



static int emit_fallback_jump(u8 **pprog)

{

	u8 *prog = *pprog;
@@ -1570,8 +1590,8 @@ static int emit_fallback_jump(u8 **pprog) 


static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs)

{

	u8 *jg_reloc, *jg_target, *prog = *pprog;

	int pivot, err, jg_bytes = 1, cnt = 0;

	u8 *jg_reloc, *prog = *pprog;

	s64 jg_offset;



	if (a == b) {
@@ -1620,6 +1640,14 @@ static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs) 
	if (err)

		return err;



	/* From Intel 64 and IA-32 Architectures Optimization

	 * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler

	 * Coding Rule 11: All branch targets should be 16-byte

	 * aligned.

	 */

	jg_target = PTR_ALIGN(prog, 16);

	if (jg_target != prog)

		emit_nops(&prog, jg_target - prog);

	jg_offset = prog - jg_reloc;

	emit_code(jg_reloc - jg_bytes, jg_offset, jg_bytes);

include/linux/bpf.h

+1 −1
Original line number Diff line number Diff line
@@ -471,7 +471,7 @@ struct bpf_trampoline { 
	u64 selector;

};



#define BPF_DISPATCHER_MAX 64 /* Fits in 2048B */

#define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */



struct bpf_dispatcher_prog {

	struct bpf_prog *prog;

CRA Git | Maintained and supported by SUSTech CRA and CCSE