MIPS: eBPF: Initial eBPF support for MIPS32 architecture. (716850ab) · Commits · 戴 / test

arch/mips/Kconfig

+1 −2

Original line number	Diff line number	Diff line
		@@ -44,8 +44,7 @@ config MIPS
		select HAVE_ARCH_SECCOMP_FILTER
		select HAVE_ARCH_TRACEHOOK
		select HAVE_ARCH_TRANSPARENT_HUGEPAGE if CPU_SUPPORTS_HUGEPAGES && 64BIT
		select HAVE_CBPF_JIT if (!64BIT && !CPU_MICROMIPS)
		select HAVE_EBPF_JIT if (64BIT && !CPU_MICROMIPS)
		select HAVE_EBPF_JIT if (!CPU_MICROMIPS)
		select HAVE_CONTEXT_TRACKING
		select HAVE_COPY_THREAD_TLS
		select HAVE_C_RECORDMCOUNT

arch/mips/net/Makefile

+0 −1

Original line number	Diff line number	Diff line
		# MIPS networking code

		obj-$(CONFIG_MIPS_CBPF_JIT) += bpf_jit.o bpf_jit_asm.o
		obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o

arch/mips/net/bpf_jit.c

deleted100644 → 0

+0 −1270

File deleted.

Preview size limit exceeded, changes collapsed.

arch/mips/net/bpf_jit_asm.S

deleted100644 → 0

+0 −285

Original line number	Diff line number	Diff line
		/*
		* bpf_jib_asm.S: Packet/header access helper functions for MIPS/MIPS64 BPF
		* compiler.
		*
		* Copyright (C) 2015 Imagination Technologies Ltd.
		* Author: Markos Chandras <markos.chandras@imgtec.com>
		*
		* This program is free software; you can redistribute it and/or modify it
		* under the terms of the GNU General Public License as published by the
		* Free Software Foundation; version 2 of the License.
		*/

		#include <asm/asm.h>
		#include <asm/isa-rev.h>
		#include <asm/regdef.h>
		#include "bpf_jit.h"

		/* ABI
		*
		* r_skb_hl skb header length
		* r_skb_data skb data
		* r_off(a1) offset register
		* r_A BPF register A
		* r_X PF register X
		* r_skb(a0) *skb
		* r_M *scratch memory
		* r_skb_le skb length
		* r_s0 Scratch register 0
		* r_s1 Scratch register 1
		*
		* On entry:
		* a0: *skb
		* a1: offset (imm or imm + X)
		*
		* All non-BPF-ABI registers are free for use. On return, we only
		* care about r_ret. The BPF-ABI registers are assumed to remain
		* unmodified during the entire filter operation.
		*/

		#define skb a0
		#define offset a1
		#define SKF_LL_OFF (-0x200000) /* Can't include linux/filter.h in assembly */

		/* We know better :) so prevent assembler reordering etc */
		.set noreorder

		#define is_offset_negative(TYPE) \
		/* If offset is negative we have more work to do */ \
		slti t0, offset, 0; \
		bgtz t0, bpf_slow_path_##TYPE##_neg; \
		/* Be careful what follows in DS. */

		#define is_offset_in_header(SIZE, TYPE) \
		/* Reading from header? */ \
		addiu $r_s0, $r_skb_hl, -SIZE; \
		slt t0, $r_s0, offset; \
		bgtz t0, bpf_slow_path_##TYPE; \

		LEAF(sk_load_word)
		is_offset_negative(word)
		FEXPORT(sk_load_word_positive)
		is_offset_in_header(4, word)
		/* Offset within header boundaries */
		PTR_ADDU t1, $r_skb_data, offset
		.set reorder
		lw $r_A, 0(t1)
		.set noreorder
		#ifdef CONFIG_CPU_LITTLE_ENDIAN
		# if MIPS_ISA_REV >= 2
		wsbh t0, $r_A
		rotr $r_A, t0, 16
		# else
		sll t0, $r_A, 24
		srl t1, $r_A, 24
		srl t2, $r_A, 8
		or t0, t0, t1
		andi t2, t2, 0xff00
		andi t1, $r_A, 0xff00
		or t0, t0, t2
		sll t1, t1, 8
		or $r_A, t0, t1
		# endif
		#endif
		jr $r_ra
		move $r_ret, zero
		END(sk_load_word)

		LEAF(sk_load_half)
		is_offset_negative(half)
		FEXPORT(sk_load_half_positive)
		is_offset_in_header(2, half)
		/* Offset within header boundaries */
		PTR_ADDU t1, $r_skb_data, offset
		lhu $r_A, 0(t1)
		#ifdef CONFIG_CPU_LITTLE_ENDIAN
		# if MIPS_ISA_REV >= 2
		wsbh $r_A, $r_A
		# else
		sll t0, $r_A, 8
		srl t1, $r_A, 8
		andi t0, t0, 0xff00
		or $r_A, t0, t1
		# endif
		#endif
		jr $r_ra
		move $r_ret, zero
		END(sk_load_half)

		LEAF(sk_load_byte)
		is_offset_negative(byte)
		FEXPORT(sk_load_byte_positive)
		is_offset_in_header(1, byte)
		/* Offset within header boundaries */
		PTR_ADDU t1, $r_skb_data, offset
		lbu $r_A, 0(t1)
		jr $r_ra
		move $r_ret, zero
		END(sk_load_byte)

		/*
		* call skb_copy_bits:
		* (prototype in linux/skbuff.h)
		*
		* int skb_copy_bits(sk_buff skb, int offset, void to, int len)
		*
		* o32 mandates we leave 4 spaces for argument registers in case
		* the callee needs to use them. Even though we don't care about
		* the argument registers ourselves, we need to allocate that space
		* to remain ABI compliant since the callee may want to use that space.
		* We also allocate 2 more spaces for $r_ra and our return register (*to).
		*
		* n64 is a bit different. The caller will allocate the space to preserve
		* the arguments. So in 64-bit kernels, we allocate the 4-arg space for no
		* good reason but it does not matter that much really.
		*
		* (void *to) is returned in r_s0
		*
		*/
		#ifdef CONFIG_CPU_LITTLE_ENDIAN
		#define DS_OFFSET(SIZE) (4 * SZREG)
		#else
		#define DS_OFFSET(SIZE) ((4 * SZREG) + (4 - SIZE))
		#endif
		#define bpf_slow_path_common(SIZE) \
		/* Quick check. Are we within reasonable boundaries? */ \
		LONG_ADDIU $r_s1, $r_skb_len, -SIZE; \
		sltu $r_s0, offset, $r_s1; \
		beqz $r_s0, fault; \
		/* Load 4th argument in DS */ \
		LONG_ADDIU a3, zero, SIZE; \
		PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \
		PTR_LA t0, skb_copy_bits; \
		PTR_S $r_ra, (5 * SZREG)($r_sp); \
		/* Assign low slot to a2 */ \
		PTR_ADDIU a2, $r_sp, DS_OFFSET(SIZE); \
		jalr t0; \
		/* Reset our destination slot (DS but it's ok) */ \
		INT_S zero, (4 * SZREG)($r_sp); \
		/* \
		* skb_copy_bits returns 0 on success and -EFAULT \
		* on error. Our data live in a2. Do not bother with \
		* our data if an error has been returned. \
		*/ \
		/* Restore our frame */ \
		PTR_L $r_ra, (5 * SZREG)($r_sp); \
		INT_L $r_s0, (4 * SZREG)($r_sp); \
		bltz v0, fault; \
		PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \
		move $r_ret, zero; \

		NESTED(bpf_slow_path_word, (6 * SZREG), $r_sp)
		bpf_slow_path_common(4)
		#ifdef CONFIG_CPU_LITTLE_ENDIAN
		# if MIPS_ISA_REV >= 2
		wsbh t0, $r_s0
		jr $r_ra
		rotr $r_A, t0, 16
		# else
		sll t0, $r_s0, 24
		srl t1, $r_s0, 24
		srl t2, $r_s0, 8
		or t0, t0, t1
		andi t2, t2, 0xff00
		andi t1, $r_s0, 0xff00
		or t0, t0, t2
		sll t1, t1, 8
		jr $r_ra
		or $r_A, t0, t1
		# endif
		#else
		jr $r_ra
		move $r_A, $r_s0
		#endif

		END(bpf_slow_path_word)

		NESTED(bpf_slow_path_half, (6 * SZREG), $r_sp)
		bpf_slow_path_common(2)
		#ifdef CONFIG_CPU_LITTLE_ENDIAN
		# if MIPS_ISA_REV >= 2
		jr $r_ra
		wsbh $r_A, $r_s0
		# else
		sll t0, $r_s0, 8
		andi t1, $r_s0, 0xff00
		andi t0, t0, 0xff00
		srl t1, t1, 8
		jr $r_ra
		or $r_A, t0, t1
		# endif
		#else
		jr $r_ra
		move $r_A, $r_s0
		#endif

		END(bpf_slow_path_half)

		NESTED(bpf_slow_path_byte, (6 * SZREG), $r_sp)
		bpf_slow_path_common(1)
		jr $r_ra
		move $r_A, $r_s0

		END(bpf_slow_path_byte)

		/*
		* Negative entry points
		*/
		.macro bpf_is_end_of_data
		li t0, SKF_LL_OFF
		/* Reading link layer data? */
		slt t1, offset, t0
		bgtz t1, fault
		/* Be careful what follows in DS. */
		.endm
		/*
		* call skb_copy_bits:
		* (prototype in linux/filter.h)
		*
		* void bpf_internal_load_pointer_neg_helper(const struct sk_buff skb,
		* int k, unsigned int size)
		*
		* see above (bpf_slow_path_common) for ABI restrictions
		*/
		#define bpf_negative_common(SIZE) \
		PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \
		PTR_LA t0, bpf_internal_load_pointer_neg_helper; \
		PTR_S $r_ra, (5 * SZREG)($r_sp); \
		jalr t0; \
		li a2, SIZE; \
		PTR_L $r_ra, (5 * SZREG)($r_sp); \
		/* Check return pointer */ \
		beqz v0, fault; \
		PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \
		/* Preserve our pointer */ \
		move $r_s0, v0; \
		/* Set return value */ \
		move $r_ret, zero; \

		bpf_slow_path_word_neg:
		bpf_is_end_of_data
		NESTED(sk_load_word_negative, (6 * SZREG), $r_sp)
		bpf_negative_common(4)
		jr $r_ra
		lw $r_A, 0($r_s0)
		END(sk_load_word_negative)

		bpf_slow_path_half_neg:
		bpf_is_end_of_data
		NESTED(sk_load_half_negative, (6 * SZREG), $r_sp)
		bpf_negative_common(2)
		jr $r_ra
		lhu $r_A, 0($r_s0)
		END(sk_load_half_negative)

		bpf_slow_path_byte_neg:
		bpf_is_end_of_data
		NESTED(sk_load_byte_negative, (6 * SZREG), $r_sp)
		bpf_negative_common(1)
		jr $r_ra
		lbu $r_A, 0($r_s0)
		END(sk_load_byte_negative)

		fault:
		jr $r_ra
		addiu $r_ret, zero, 1

arch/mips/net/ebpf_jit.c

+69 −44

Original line number	Diff line number	Diff line
		@@ -126,15 +126,21 @@ static enum reg_val_type get_reg_val_type(const struct jit_ctx *ctx,
		}

		/* Simply emit the instruction if the JIT memory space has been allocated */
		#define emit_instr(ctx, func, ...) \
		#define emit_instr_long(ctx, func64, func32, ...) \
		do { \
		if ((ctx)->target != NULL) { \
		u32 *p = &(ctx)->target[ctx->idx]; \
		uasm_i_##func(&p, ##__VA_ARGS__); \
		if (IS_ENABLED(CONFIG_64BIT)) \
		uasm_i_##func64(&p, ##__VA_ARGS__); \
		else \
		uasm_i_##func32(&p, ##__VA_ARGS__); \
		} \
		(ctx)->idx++; \
		} while (0)

		#define emit_instr(ctx, func, ...) \
		emit_instr_long(ctx, func, func, ##__VA_ARGS__)

		static unsigned int j_target(struct jit_ctx *ctx, int target_idx)
		{
		unsigned long target_va, base_va;
		@@ -274,17 +280,17 @@ static int gen_int_prologue(struct jit_ctx *ctx)
		* If RA we are doing a function call and may need
		* extra 8-byte tmp area.
		*/
		stack_adjust += 16;
		stack_adjust += 2 * sizeof(long);
		if (ctx->flags & EBPF_SAVE_S0)
		stack_adjust += 8;
		stack_adjust += sizeof(long);
		if (ctx->flags & EBPF_SAVE_S1)
		stack_adjust += 8;
		stack_adjust += sizeof(long);
		if (ctx->flags & EBPF_SAVE_S2)
		stack_adjust += 8;
		stack_adjust += sizeof(long);
		if (ctx->flags & EBPF_SAVE_S3)
		stack_adjust += 8;
		stack_adjust += sizeof(long);
		if (ctx->flags & EBPF_SAVE_S4)
		stack_adjust += 8;
		stack_adjust += sizeof(long);

		BUILD_BUG_ON(MAX_BPF_STACK & 7);
		locals_size = (ctx->flags & EBPF_SEEN_FP) ? MAX_BPF_STACK : 0;
		@@ -298,41 +304,49 @@ static int gen_int_prologue(struct jit_ctx *ctx)
		* On tail call we skip this instruction, and the TCC is
		* passed in $v1 from the caller.
		*/
		emit_instr(ctx, daddiu, MIPS_R_V1, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
		emit_instr(ctx, addiu, MIPS_R_V1, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
		if (stack_adjust)
		emit_instr(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack_adjust);
		emit_instr_long(ctx, daddiu, addiu,
		MIPS_R_SP, MIPS_R_SP, -stack_adjust);
		else
		return 0;

		store_offset = stack_adjust - 8;
		store_offset = stack_adjust - sizeof(long);

		if (ctx->flags & EBPF_SAVE_RA) {
		emit_instr(ctx, sd, MIPS_R_RA, store_offset, MIPS_R_SP);
		store_offset -= 8;
		emit_instr_long(ctx, sd, sw,
		MIPS_R_RA, store_offset, MIPS_R_SP);
		store_offset -= sizeof(long);
		}
		if (ctx->flags & EBPF_SAVE_S0) {
		emit_instr(ctx, sd, MIPS_R_S0, store_offset, MIPS_R_SP);
		store_offset -= 8;
		emit_instr_long(ctx, sd, sw,
		MIPS_R_S0, store_offset, MIPS_R_SP);
		store_offset -= sizeof(long);
		}
		if (ctx->flags & EBPF_SAVE_S1) {
		emit_instr(ctx, sd, MIPS_R_S1, store_offset, MIPS_R_SP);
		store_offset -= 8;
		emit_instr_long(ctx, sd, sw,
		MIPS_R_S1, store_offset, MIPS_R_SP);
		store_offset -= sizeof(long);
		}
		if (ctx->flags & EBPF_SAVE_S2) {
		emit_instr(ctx, sd, MIPS_R_S2, store_offset, MIPS_R_SP);
		store_offset -= 8;
		emit_instr_long(ctx, sd, sw,
		MIPS_R_S2, store_offset, MIPS_R_SP);
		store_offset -= sizeof(long);
		}
		if (ctx->flags & EBPF_SAVE_S3) {
		emit_instr(ctx, sd, MIPS_R_S3, store_offset, MIPS_R_SP);
		store_offset -= 8;
		emit_instr_long(ctx, sd, sw,
		MIPS_R_S3, store_offset, MIPS_R_SP);
		store_offset -= sizeof(long);
		}
		if (ctx->flags & EBPF_SAVE_S4) {
		emit_instr(ctx, sd, MIPS_R_S4, store_offset, MIPS_R_SP);
		store_offset -= 8;
		emit_instr_long(ctx, sd, sw,
		MIPS_R_S4, store_offset, MIPS_R_SP);
		store_offset -= sizeof(long);
		}

		if ((ctx->flags & EBPF_SEEN_TC) && !(ctx->flags & EBPF_TCC_IN_V1))
		emit_instr(ctx, daddu, MIPS_R_S4, MIPS_R_V1, MIPS_R_ZERO);
		emit_instr_long(ctx, daddu, addu,
		MIPS_R_S4, MIPS_R_V1, MIPS_R_ZERO);

		return 0;
		}
		@@ -341,7 +355,7 @@ static int build_int_epilogue(struct jit_ctx *ctx, int dest_reg)
		{
		const struct bpf_prog *prog = ctx->skf;
		int stack_adjust = ctx->stack_size;
		int store_offset = stack_adjust - 8;
		int store_offset = stack_adjust - sizeof(long);
		enum reg_val_type td;
		int r0 = MIPS_R_V0;

		@@ -353,33 +367,40 @@ static int build_int_epilogue(struct jit_ctx *ctx, int dest_reg)
		}

		if (ctx->flags & EBPF_SAVE_RA) {
		emit_instr(ctx, ld, MIPS_R_RA, store_offset, MIPS_R_SP);
		store_offset -= 8;
		emit_instr_long(ctx, ld, lw,
		MIPS_R_RA, store_offset, MIPS_R_SP);
		store_offset -= sizeof(long);
		}
		if (ctx->flags & EBPF_SAVE_S0) {
		emit_instr(ctx, ld, MIPS_R_S0, store_offset, MIPS_R_SP);
		store_offset -= 8;
		emit_instr_long(ctx, ld, lw,
		MIPS_R_S0, store_offset, MIPS_R_SP);
		store_offset -= sizeof(long);
		}
		if (ctx->flags & EBPF_SAVE_S1) {
		emit_instr(ctx, ld, MIPS_R_S1, store_offset, MIPS_R_SP);
		store_offset -= 8;
		emit_instr_long(ctx, ld, lw,
		MIPS_R_S1, store_offset, MIPS_R_SP);
		store_offset -= sizeof(long);
		}
		if (ctx->flags & EBPF_SAVE_S2) {
		emit_instr(ctx, ld, MIPS_R_S2, store_offset, MIPS_R_SP);
		store_offset -= 8;
		emit_instr_long(ctx, ld, lw,
		MIPS_R_S2, store_offset, MIPS_R_SP);
		store_offset -= sizeof(long);
		}
		if (ctx->flags & EBPF_SAVE_S3) {
		emit_instr(ctx, ld, MIPS_R_S3, store_offset, MIPS_R_SP);
		store_offset -= 8;
		emit_instr_long(ctx, ld, lw,
		MIPS_R_S3, store_offset, MIPS_R_SP);
		store_offset -= sizeof(long);
		}
		if (ctx->flags & EBPF_SAVE_S4) {
		emit_instr(ctx, ld, MIPS_R_S4, store_offset, MIPS_R_SP);
		store_offset -= 8;
		emit_instr_long(ctx, ld, lw,
		MIPS_R_S4, store_offset, MIPS_R_SP);
		store_offset -= sizeof(long);
		}
		emit_instr(ctx, jr, dest_reg);

		if (stack_adjust)
		emit_instr(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, stack_adjust);
		emit_instr_long(ctx, daddiu, addiu,
		MIPS_R_SP, MIPS_R_SP, stack_adjust);
		else
		emit_instr(ctx, nop);

		@@ -646,6 +667,10 @@ static int build_one_insn(const struct bpf_insn insn, struct jit_ctx ctx,
		s64 t64s;
		int bpf_op = BPF_OP(insn->code);

		if (IS_ENABLED(CONFIG_32BIT) && ((BPF_CLASS(insn->code) == BPF_ALU64)
		\|\| (bpf_op == BPF_DW)))
		return -EINVAL;

		switch (insn->code) {
		case BPF_ALU64 \| BPF_ADD \| BPF_K: /* ALU64_IMM */
		case BPF_ALU64 \| BPF_SUB \| BPF_K: /* ALU64_IMM */
		@@ -1283,7 +1308,7 @@ jeq_common:

		case BPF_JMP \| BPF_CALL:
		ctx->flags \|= EBPF_SAVE_RA;
		t64s = (s64)insn->imm + (s64)__bpf_call_base;
		t64s = (s64)insn->imm + (long)__bpf_call_base;
		emit_const_to_reg(ctx, MIPS_R_T9, (u64)t64s);
		emit_instr(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
		/* delay slot */

Admin message