Commit de5cb6eb authored by Martin Schwidefsky's avatar Martin Schwidefsky
Browse files

s390: use expoline thunks in the BPF JIT 



The BPF JIT need safe guarding against spectre v2 in the sk_load_xxx
assembler stubs and the indirect branches generated by the JIT itself
need to be converted to expolines.

Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
parent 6deaa3bb

arch/s390/net/bpf_jit.S

+10 −6
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ 
 */



#include <linux/linkage.h>

#include <asm/nospec-insn.h>

#include "bpf_jit.h"



/*
@@ -54,7 +55,7 @@ ENTRY(sk_load_##NAME##_pos); \ 
	clg	%r3,STK_OFF_HLEN(%r15);	/* Offset + SIZE > hlen? */	\

	jh	sk_load_##NAME##_slow;					\

	LOAD	%r14,-SIZE(%r3,%r12);	/* Get data from skb */		\

	b	OFF_OK(%r6);		/* Return */			\

	B_EX	OFF_OK,%r6;		/* Return */			\

									\

sk_load_##NAME##_slow:;							\

	lgr	%r2,%r7;		/* Arg1 = skb pointer */	\
@@ -64,11 +65,14 @@ sk_load_##NAME##_slow:; \ 
	brasl	%r14,skb_copy_bits;	/* Get data from skb */		\

	LOAD	%r14,STK_OFF_TMP(%r15);	/* Load from temp bufffer */	\

	ltgr	%r2,%r2;		/* Set cc to (%r2 != 0) */	\

	br	%r6;			/* Return */

	BR_EX	%r6;			/* Return */



sk_load_common(word, 4, llgf)	/* r14 = *(u32 *) (skb->data+offset) */

sk_load_common(half, 2, llgh)	/* r14 = *(u16 *) (skb->data+offset) */



	GEN_BR_THUNK %r6

	GEN_B_THUNK OFF_OK,%r6



/*

 * Load 1 byte from SKB (optimized version)

 */
@@ -80,7 +84,7 @@ ENTRY(sk_load_byte_pos) 
	clg	%r3,STK_OFF_HLEN(%r15)	# Offset >= hlen?

	jnl	sk_load_byte_slow

	llgc	%r14,0(%r3,%r12)	# Get byte from skb

	b	OFF_OK(%r6)		# Return OK

	B_EX	OFF_OK,%r6		# Return OK



sk_load_byte_slow:

	lgr	%r2,%r7			# Arg1 = skb pointer
@@ -90,7 +94,7 @@ sk_load_byte_slow: 
	brasl	%r14,skb_copy_bits	# Get data from skb

	llgc	%r14,STK_OFF_TMP(%r15)	# Load result from temp buffer

	ltgr	%r2,%r2			# Set cc to (%r2 != 0)

	br	%r6			# Return cc

	BR_EX	%r6			# Return cc



#define sk_negative_common(NAME, SIZE, LOAD)				\

sk_load_##NAME##_slow_neg:;						\
@@ -104,7 +108,7 @@ sk_load_##NAME##_slow_neg:; \ 
	jz	bpf_error;						\

	LOAD	%r14,0(%r2);		/* Get data from pointer */	\

	xr	%r3,%r3;		/* Set cc to zero */		\

	br	%r6;			/* Return cc */

	BR_EX	%r6;			/* Return cc */



sk_negative_common(word, 4, llgf)

sk_negative_common(half, 2, llgh)
@@ -113,4 +117,4 @@ sk_negative_common(byte, 1, llgc) 
bpf_error:

# force a return 0 from jit handler

	ltgr	%r15,%r15	# Set condition code

	br	%r6
 
	BR_EX	%r6

arch/s390/net/bpf_jit_comp.c

+61 −2
Original line number Diff line number Diff line
@@ -25,6 +25,8 @@ 
#include <linux/bpf.h>

#include <asm/cacheflush.h>

#include <asm/dis.h>

#include <asm/facility.h>

#include <asm/nospec-branch.h>

#include <asm/set_memory.h>

#include "bpf_jit.h"
@@ -41,6 +43,8 @@ struct bpf_jit { 
	int base_ip;		/* Base address for literal pool */

	int ret0_ip;		/* Address of return 0 */

	int exit_ip;		/* Address of exit */

	int r1_thunk_ip;	/* Address of expoline thunk for 'br %r1' */

	int r14_thunk_ip;	/* Address of expoline thunk for 'br %r14' */

	int tail_call_start;	/* Tail call start offset */

	int labels[1];		/* Labels for local jumps */

};
@@ -250,6 +254,19 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) 
	REG_SET_SEEN(b2);					\

})



#define EMIT6_PCREL_RILB(op, b, target)				\

({								\

	int rel = (target - jit->prg) / 2;			\

	_EMIT6(op | reg_high(b) << 16 | rel >> 16, rel & 0xffff);	\

	REG_SET_SEEN(b);					\

})



#define EMIT6_PCREL_RIL(op, target)				\

({								\

	int rel = (target - jit->prg) / 2;			\

	_EMIT6(op | rel >> 16, rel & 0xffff);			\

})



#define _EMIT6_IMM(op, imm)					\

({								\

	unsigned int __imm = (imm);				\
@@ -469,8 +486,45 @@ static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth) 
	EMIT4(0xb9040000, REG_2, BPF_REG_0);

	/* Restore registers */

	save_restore_regs(jit, REGS_RESTORE, stack_depth);

	if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) {

		jit->r14_thunk_ip = jit->prg;

		/* Generate __s390_indirect_jump_r14 thunk */

		if (test_facility(35)) {

			/* exrl %r0,.+10 */

			EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);

		} else {

			/* larl %r1,.+14 */

			EMIT6_PCREL_RILB(0xc0000000, REG_1, jit->prg + 14);

			/* ex 0,0(%r1) */

			EMIT4_DISP(0x44000000, REG_0, REG_1, 0);

		}

		/* j . */

		EMIT4_PCREL(0xa7f40000, 0);

	}

	/* br %r14 */

	_EMIT2(0x07fe);



	if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable &&

	    (jit->seen & SEEN_FUNC)) {

		jit->r1_thunk_ip = jit->prg;

		/* Generate __s390_indirect_jump_r1 thunk */

		if (test_facility(35)) {

			/* exrl %r0,.+10 */

			EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);

			/* j . */

			EMIT4_PCREL(0xa7f40000, 0);

			/* br %r1 */

			_EMIT2(0x07f1);

		} else {

			/* larl %r1,.+14 */

			EMIT6_PCREL_RILB(0xc0000000, REG_1, jit->prg + 14);

			/* ex 0,S390_lowcore.br_r1_tampoline */

			EMIT4_DISP(0x44000000, REG_0, REG_0,

				   offsetof(struct lowcore, br_r1_trampoline));

			/* j . */

			EMIT4_PCREL(0xa7f40000, 0);

		}

	}

}



/*
@@ -966,8 +1020,13 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i 
		/* lg %w1,<d(imm)>(%l) */

		EMIT6_DISP_LH(0xe3000000, 0x0004, REG_W1, REG_0, REG_L,

			      EMIT_CONST_U64(func));

		if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) {

			/* brasl %r14,__s390_indirect_jump_r1 */

			EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);

		} else {

			/* basr %r14,%w1 */

			EMIT2(0x0d00, REG_14, REG_W1);

		}

		/* lgr %b0,%r2: load return value into %b0 */

		EMIT4(0xb9040000, BPF_REG_0, REG_2);

		if ((jit->seen & SEEN_SKB) &&

CRA Git | Maintained and supported by SUSTech CRA and CCSE