Commit 3f161e0a authored by Ilya Leoshkevich's avatar Ilya Leoshkevich Committed by Heiko Carstens
Browse files

s390/bpf: implement BPF_PROBE_MEM 



This is a s390 port of x86 commit 3dec541b ("bpf: Add support for BTF
pointers to x86 JIT").

Signed-off-by: default avatarIlya Leoshkevich <iii@linux.ibm.com>
Reviewed-by: default avatarHeiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: default avatarHeiko Carstens <hca@linux.ibm.com>
parent 05a68e89

arch/s390/net/bpf_jit_comp.c

+138 −1
Original line number Diff line number Diff line
@@ -49,6 +49,7 @@ struct bpf_jit { 
	int r1_thunk_ip;	/* Address of expoline thunk for 'br %r1' */

	int r14_thunk_ip;	/* Address of expoline thunk for 'br %r14' */

	int tail_call_start;	/* Tail call start offset */

	int excnt;		/* Number of exception table entries */

	int labels[1];		/* Labels for local jumps */

};
@@ -588,6 +589,84 @@ static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth) 
	}

}



static int get_probe_mem_regno(const u8 *insn)

{

	/*

	 * insn must point to llgc, llgh, llgf or lg, which have destination

	 * register at the same position.

	 */

	if (insn[0] != 0xe3) /* common llgc, llgh, llgf and lg prefix */

		return -1;

	if (insn[5] != 0x90 && /* llgc */

	    insn[5] != 0x91 && /* llgh */

	    insn[5] != 0x16 && /* llgf */

	    insn[5] != 0x04) /* lg */

		return -1;

	return insn[1] >> 4;

}



static bool ex_handler_bpf(const struct exception_table_entry *x,

			   struct pt_regs *regs)

{

	int regno;

	u8 *insn;



	regs->psw.addr = extable_fixup(x);

	insn = (u8 *)__rewind_psw(regs->psw, regs->int_code >> 16);

	regno = get_probe_mem_regno(insn);

	if (WARN_ON_ONCE(regno < 0))

		/* JIT bug - unexpected instruction. */

		return false;

	regs->gprs[regno] = 0;

	return true;

}



static int bpf_jit_probe_mem(struct bpf_jit *jit, struct bpf_prog *fp,

			     int probe_prg, int nop_prg)

{

	struct exception_table_entry *ex;

	s64 delta;

	u8 *insn;

	int prg;

	int i;



	if (!fp->aux->extable)

		/* Do nothing during early JIT passes. */

		return 0;

	insn = jit->prg_buf + probe_prg;

	if (WARN_ON_ONCE(get_probe_mem_regno(insn) < 0))

		/* JIT bug - unexpected probe instruction. */

		return -1;

	if (WARN_ON_ONCE(probe_prg + insn_length(*insn) != nop_prg))

		/* JIT bug - gap between probe and nop instructions. */

		return -1;

	for (i = 0; i < 2; i++) {

		if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries))

			/* Verifier bug - not enough entries. */

			return -1;

		ex = &fp->aux->extable[jit->excnt];

		/* Add extable entries for probe and nop instructions. */

		prg = i == 0 ? probe_prg : nop_prg;

		delta = jit->prg_buf + prg - (u8 *)&ex->insn;

		if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX))

			/* JIT bug - code and extable must be close. */

			return -1;

		ex->insn = delta;

		/*

		 * Always land on the nop. Note that extable infrastructure

		 * ignores fixup field, it is handled by ex_handler_bpf().

		 */

		delta = jit->prg_buf + nop_prg - (u8 *)&ex->fixup;

		if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX))

			/* JIT bug - landing pad and extable must be close. */

			return -1;

		ex->fixup = delta;

		ex->handler = (u8 *)ex_handler_bpf - (u8 *)&ex->handler;

		jit->excnt++;

	}

	return 0;

}



/*

 * Compile one eBPF instruction into s390x code

 *
@@ -604,7 +683,14 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 
	u32 *addrs = jit->addrs;

	s32 imm = insn->imm;

	s16 off = insn->off;

	int probe_prg = -1;

	unsigned int mask;

	int nop_prg;

	int err;



	if (BPF_CLASS(insn->code) == BPF_LDX &&

	    BPF_MODE(insn->code) == BPF_PROBE_MEM)

		probe_prg = jit->prg;



	switch (insn->code) {

	/*
@@ -1119,6 +1205,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 
	 * BPF_LDX

	 */

	case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */

	case BPF_LDX | BPF_PROBE_MEM | BPF_B:

		/* llgc %dst,0(off,%src) */

		EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, REG_0, off);

		jit->seen |= SEEN_MEM;
@@ -1126,6 +1213,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 
			insn_count = 2;

		break;

	case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */

	case BPF_LDX | BPF_PROBE_MEM | BPF_H:

		/* llgh %dst,0(off,%src) */

		EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, REG_0, off);

		jit->seen |= SEEN_MEM;
@@ -1133,6 +1221,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 
			insn_count = 2;

		break;

	case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */

	case BPF_LDX | BPF_PROBE_MEM | BPF_W:

		/* llgf %dst,off(%src) */

		jit->seen |= SEEN_MEM;

		EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, REG_0, off);
@@ -1140,6 +1229,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 
			insn_count = 2;

		break;

	case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */

	case BPF_LDX | BPF_PROBE_MEM | BPF_DW:

		/* lg %dst,0(off,%src) */

		jit->seen |= SEEN_MEM;

		EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, REG_0, off);
@@ -1485,6 +1575,23 @@ branch_oc: 
		pr_err("Unknown opcode %02x\n", insn->code);

		return -1;

	}



	if (probe_prg != -1) {

		/*

		 * Handlers of certain exceptions leave psw.addr pointing to

		 * the instruction directly after the failing one. Therefore,

		 * create two exception table entries and also add a nop in

		 * case two probing instructions come directly after each

		 * other.

		 */

		nop_prg = jit->prg;

		/* bcr 0,%0 */

		_EMIT2(0x0700);

		err = bpf_jit_probe_mem(jit, fp, probe_prg, nop_prg);

		if (err < 0)

			return err;

	}



	return insn_count;

}
@@ -1527,6 +1634,7 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp, 
	jit->lit32 = jit->lit32_start;

	jit->lit64 = jit->lit64_start;

	jit->prg = 0;

	jit->excnt = 0;



	bpf_jit_prologue(jit, stack_depth);

	if (bpf_set_addr(jit, 0) < 0)
@@ -1551,6 +1659,12 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp, 
		jit->lit64_start = ALIGN(jit->lit64_start, 8);

	jit->size = jit->lit64_start + lit64_size;

	jit->size_prg = jit->prg;



	if (WARN_ON_ONCE(fp->aux->extable &&

			 jit->excnt != fp->aux->num_exentries))

		/* Verifier bug - too many entries. */

		return -1;



	return 0;

}
@@ -1565,6 +1679,29 @@ struct s390_jit_data { 
	int pass;

};



static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,

					       struct bpf_prog *fp)

{

	struct bpf_binary_header *header;

	u32 extable_size;

	u32 code_size;



	/* We need two entries per insn. */

	fp->aux->num_exentries *= 2;



	code_size = roundup(jit->size,

			    __alignof__(struct exception_table_entry));

	extable_size = fp->aux->num_exentries *

		sizeof(struct exception_table_entry);

	header = bpf_jit_binary_alloc(code_size + extable_size, &jit->prg_buf,

				      8, jit_fill_hole);

	if (!header)

		return NULL;

	fp->aux->extable = (struct exception_table_entry *)

		(jit->prg_buf + code_size);

	return header;

}



/*

 * Compile eBPF program "fp"

 */
@@ -1631,7 +1768,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 
	/*

	 * Final pass: Allocate and generate program

	 */

	header = bpf_jit_binary_alloc(jit.size, &jit.prg_buf, 8, jit_fill_hole);

	header = bpf_jit_alloc(&jit, fp);

	if (!header) {

		fp = orig_fp;

		goto free_addrs;

CRA Git | Maintained and supported by SUSTech CRA and CCSE