Commit a085a1ee authored by Daniel Borkmann's avatar Daniel Borkmann
Browse files

Merge branch 'bpf-rv64-jit' 



Luke Nelson says:

====================
This patch series introduces a set of optimizations to the BPF JIT
on RV64. The optimizations are related to the verifier zero-extension
optimization and BPF_JMP BPF_K.

We tested the optimizations on a QEMU riscv64 virt machine, using
lib/test_bpf and test_verifier, and formally verified their correctness
using Serval.
====================

Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parents 5447e8e0 073ca6a0

arch/riscv/net/bpf_jit_comp64.c

+44 −20
Original line number Diff line number Diff line
@@ -515,7 +515,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
	case BPF_ALU | BPF_LSH | BPF_X:

	case BPF_ALU64 | BPF_LSH | BPF_X:

		emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx);

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_RSH | BPF_X:
@@ -542,13 +542,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 


	/* dst = BSWAP##imm(dst) */

	case BPF_ALU | BPF_END | BPF_FROM_LE:

	{

		int shift = 64 - imm;



		emit(rv_slli(rd, rd, shift), ctx);

		emit(rv_srli(rd, rd, shift), ctx);

		switch (imm) {

		case 16:

			emit(rv_slli(rd, rd, 48), ctx);

			emit(rv_srli(rd, rd, 48), ctx);

			break;

		case 32:

			if (!aux->verifier_zext)

				emit_zext_32(rd, ctx);

			break;

		case 64:

			/* Do nothing */

			break;

		}

		break;



	case BPF_ALU | BPF_END | BPF_FROM_BE:

		emit(rv_addi(RV_REG_T2, RV_REG_ZERO, 0), ctx);
@@ -692,19 +700,19 @@ out_be: 
	case BPF_ALU | BPF_LSH | BPF_K:

	case BPF_ALU64 | BPF_LSH | BPF_K:

		emit(is64 ? rv_slli(rd, rd, imm) : rv_slliw(rd, rd, imm), ctx);

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_RSH | BPF_K:

	case BPF_ALU64 | BPF_RSH | BPF_K:

		emit(is64 ? rv_srli(rd, rd, imm) : rv_srliw(rd, rd, imm), ctx);

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_ARSH | BPF_K:

	case BPF_ALU64 | BPF_ARSH | BPF_K:

		emit(is64 ? rv_srai(rd, rd, imm) : rv_sraiw(rd, rd, imm), ctx);

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;
@@ -784,11 +792,15 @@ out_be: 
	case BPF_JMP32 | BPF_JSGE | BPF_K:

	case BPF_JMP | BPF_JSLE | BPF_K:

	case BPF_JMP32 | BPF_JSLE | BPF_K:

	case BPF_JMP | BPF_JSET | BPF_K:

	case BPF_JMP32 | BPF_JSET | BPF_K:

		rvoff = rv_offset(i, off, ctx);

		s = ctx->ninsns;

		if (imm) {

			emit_imm(RV_REG_T1, imm, ctx);

			rs = RV_REG_T1;

		} else {

			/* If imm is 0, simply use zero register. */

			rs = RV_REG_ZERO;

		}

		if (!is64) {

			if (is_signed_bpf_cond(BPF_OP(code)))

				emit_sext_32_rd(&rd, ctx);
@@ -799,16 +811,28 @@ out_be: 


		/* Adjust for extra insns */

		rvoff -= (e - s) << 2;

		emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);

		break;



		if (BPF_OP(code) == BPF_JSET) {

			/* Adjust for and */

			rvoff -= 4;

			emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx);

			emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff,

				    ctx);

	case BPF_JMP | BPF_JSET | BPF_K:

	case BPF_JMP32 | BPF_JSET | BPF_K:

		rvoff = rv_offset(i, off, ctx);

		s = ctx->ninsns;

		if (is_12b_int(imm)) {

			emit(rv_andi(RV_REG_T1, rd, imm), ctx);

		} else {

			emit_branch(BPF_OP(code), rd, RV_REG_T1, rvoff, ctx);

			emit_imm(RV_REG_T1, imm, ctx);

			emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx);

		}

		/* For jset32, we should clear the upper 32 bits of t1, but

		 * sign-extension is sufficient here and saves one instruction,

		 * as t1 is used only in comparison against zero.

		 */

		if (!is64 && imm < 0)

			emit(rv_addiw(RV_REG_T1, RV_REG_T1, 0), ctx);

		e = ctx->ninsns;

		rvoff -= (e - s) << 2;

		emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx);

		break;



	/* function call */

CRA Git | Maintained and supported by SUSTech CRA and CCSE