Commit b25c57b6 authored by Ilya Leoshkevich's avatar Ilya Leoshkevich Committed by Alexei Starovoitov
Browse files

s390/bpf: Use lg(f)rl when long displacement cannot be used 



If literal pool grows past 524287 mark, it's no longer possible to use
long displacement to reference literal pool entries. In JIT setting
maintaining multiple literal pool registers is next to impossible, since
we operate on one instruction at a time.

Therefore, fall back to loading literal pool entry using PC-relative
addressing, and then using a register-register form of the following
machine instruction.

Signed-off-by: default avatarIlya Leoshkevich <iii@linux.ibm.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20191118180340.68373-6-iii@linux.ibm.com
parent 451e448f

arch/s390/net/bpf_jit_comp.c

+81 −15
Original line number Diff line number Diff line
@@ -371,6 +371,24 @@ static bool is_valid_ldisp(int disp) 
	return disp >= -524288 && disp <= 524287;

}



/*

 * Return whether the next 32-bit literal pool entry can be referenced using

 * Long-Displacement Facility

 */

static bool can_use_ldisp_for_lit32(struct bpf_jit *jit)

{

	return is_valid_ldisp(jit->lit32 - jit->base_ip);

}



/*

 * Return whether the next 64-bit literal pool entry can be referenced using

 * Long-Displacement Facility

 */

static bool can_use_ldisp_for_lit64(struct bpf_jit *jit)

{

	return is_valid_ldisp(jit->lit64 - jit->base_ip);

}



/*

 * Fill whole space with illegal instructions

 */
@@ -752,9 +770,18 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 
		EMIT4_IMM(0xa7080000, REG_W0, 0);

		/* lr %w1,%dst */

		EMIT2(0x1800, REG_W1, dst_reg);

		if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) {

			/* dl %w0,<d(imm)>(%l) */

			EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, REG_L,

				      EMIT_CONST_U32(imm));

		} else {

			/* lgfrl %dst,imm */

			EMIT6_PCREL_RILB(0xc40c0000, dst_reg,

					 _EMIT_CONST_U32(imm));

			jit->seen |= SEEN_LITERAL;

			/* dlr %w0,%dst */

			EMIT4(0xb9970000, REG_W0, dst_reg);

		}

		/* llgfr %dst,%rc */

		EMIT4(0xb9160000, dst_reg, rc_reg);

		if (insn_is_zext(&insn[1]))
@@ -776,9 +803,18 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 
		EMIT4_IMM(0xa7090000, REG_W0, 0);

		/* lgr %w1,%dst */

		EMIT4(0xb9040000, REG_W1, dst_reg);

		if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {

			/* dlg %w0,<d(imm)>(%l) */

			EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, REG_L,

				      EMIT_CONST_U64(imm));

		} else {

			/* lgrl %dst,imm */

			EMIT6_PCREL_RILB(0xc4080000, dst_reg,

					 _EMIT_CONST_U64(imm));

			jit->seen |= SEEN_LITERAL;

			/* dlgr %w0,%dst */

			EMIT4(0xb9870000, REG_W0, dst_reg);

		}

		/* lgr %dst,%rc */

		EMIT4(0xb9040000, dst_reg, rc_reg);

		break;
@@ -801,9 +837,19 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 
		EMIT_ZERO(dst_reg);

		break;

	case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */

		if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {

			/* ng %dst,<d(imm)>(%l) */

		EMIT6_DISP_LH(0xe3000000, 0x0080, dst_reg, REG_0, REG_L,

			EMIT6_DISP_LH(0xe3000000, 0x0080,

				      dst_reg, REG_0, REG_L,

				      EMIT_CONST_U64(imm));

		} else {

			/* lgrl %w0,imm */

			EMIT6_PCREL_RILB(0xc4080000, REG_W0,

					 _EMIT_CONST_U64(imm));

			jit->seen |= SEEN_LITERAL;

			/* ngr %dst,%w0 */

			EMIT4(0xb9800000, dst_reg, REG_W0);

		}

		break;

	/*

	 * BPF_OR
@@ -823,9 +869,19 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 
		EMIT_ZERO(dst_reg);

		break;

	case BPF_ALU64 | BPF_OR | BPF_K: /* dst = dst | imm */

		if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {

			/* og %dst,<d(imm)>(%l) */

		EMIT6_DISP_LH(0xe3000000, 0x0081, dst_reg, REG_0, REG_L,

			EMIT6_DISP_LH(0xe3000000, 0x0081,

				      dst_reg, REG_0, REG_L,

				      EMIT_CONST_U64(imm));

		} else {

			/* lgrl %w0,imm */

			EMIT6_PCREL_RILB(0xc4080000, REG_W0,

					 _EMIT_CONST_U64(imm));

			jit->seen |= SEEN_LITERAL;

			/* ogr %dst,%w0 */

			EMIT4(0xb9810000, dst_reg, REG_W0);

		}

		break;

	/*

	 * BPF_XOR
@@ -847,9 +903,19 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 
		EMIT_ZERO(dst_reg);

		break;

	case BPF_ALU64 | BPF_XOR | BPF_K: /* dst = dst ^ imm */

		if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {

			/* xg %dst,<d(imm)>(%l) */

		EMIT6_DISP_LH(0xe3000000, 0x0082, dst_reg, REG_0, REG_L,

			EMIT6_DISP_LH(0xe3000000, 0x0082,

				      dst_reg, REG_0, REG_L,

				      EMIT_CONST_U64(imm));

		} else {

			/* lgrl %w0,imm */

			EMIT6_PCREL_RILB(0xc4080000, REG_W0,

					 _EMIT_CONST_U64(imm));

			jit->seen |= SEEN_LITERAL;

			/* xgr %dst,%w0 */

			EMIT4(0xb9820000, dst_reg, REG_W0);

		}

		break;

	/*

	 * BPF_LSH

CRA Git | Maintained and supported by SUSTech CRA and CCSE