Commit 9db7f2b8 authored by Michael Holzheu's avatar Michael Holzheu Committed by David S. Miller
Browse files

s390/bpf: recache skb->data/hlen for skb_vlan_push/pop 



Allow eBPF programs attached to TC qdiscs call skb_vlan_push/pop
via helper functions. These functions may change skb->data/hlen.
This data is cached by s390 JIT to improve performance of ld_abs/ld_ind
instructions. Therefore after a change we have to reload the data.

In case of usage of skb_vlan_push/pop, in the prologue we store
the SKB pointer on the stack and restore it after BPF_JMP_CALL
to skb_vlan_push/pop.

Signed-off-by: default avatarMichael Holzheu <holzheu@linux.vnet.ibm.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent cde66c2d

arch/s390/net/bpf_jit.h

+4 −1
Original line number Diff line number Diff line
@@ -36,6 +36,8 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[]; 
 *	      |   BPF stack   |     |

 *	      |		      |     |

 *	      +---------------+     |

 *	      | 8 byte skbp   |     |

 * R15+170 -> +---------------+     |

 *	      | 8 byte hlen   |     |

 * R15+168 -> +---------------+     |

 *	      | 4 byte align  |     |
@@ -51,11 +53,12 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[]; 
 * We get 160 bytes stack space from calling function, but only use

 * 12 * 8 byte for old backchain, r15..r6, and tail_call_cnt.

 */

#define STK_SPACE	(MAX_BPF_STACK + 8 + 4 + 4 + 160)

#define STK_SPACE	(MAX_BPF_STACK + 8 + 8 + 4 + 4 + 160)

#define STK_160_UNUSED	(160 - 12 * 8)

#define STK_OFF		(STK_SPACE - STK_160_UNUSED)

#define STK_OFF_TMP	160	/* Offset of tmp buffer on stack */

#define STK_OFF_HLEN	168	/* Offset of SKB header length on stack */

#define STK_OFF_SKBP	170	/* Offset of SKB pointer on stack */



#define STK_OFF_R6	(160 - 11 * 8)	/* Offset of r6 on stack */

#define STK_OFF_TCCNT	(160 - 12 * 8)	/* Offset of tail_call_cnt on stack */

arch/s390/net/bpf_jit_comp.c

+33 −22
Original line number Diff line number Diff line
@@ -53,6 +53,7 @@ struct bpf_jit { 
#define SEEN_LITERAL	8	/* code uses literals */

#define SEEN_FUNC	16	/* calls C functions */

#define SEEN_TAIL_CALL	32	/* code uses tail calls */

#define SEEN_SKB_CHANGE	64	/* code changes skb data */

#define SEEN_STACK	(SEEN_FUNC | SEEN_MEM | SEEN_SKB)



/*
@@ -381,6 +382,26 @@ static void save_restore_regs(struct bpf_jit *jit, int op) 
	} while (re <= 15);

}



/*

 * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S"

 * we store the SKB header length on the stack and the SKB data

 * pointer in REG_SKB_DATA.

 */

static void emit_load_skb_data_hlen(struct bpf_jit *jit)

{

	/* Header length: llgf %w1,<len>(%b1) */

	EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1,

		      offsetof(struct sk_buff, len));

	/* s %w1,<data_len>(%b1) */

	EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1,

		   offsetof(struct sk_buff, data_len));

	/* stg %w1,ST_OFF_HLEN(%r0,%r15) */

	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, STK_OFF_HLEN);

	/* lg %skb_data,data_off(%b1) */

	EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,

		      BPF_REG_1, offsetof(struct sk_buff, data));

}



/*

 * Emit function prologue

 *
@@ -421,25 +442,12 @@ static void bpf_jit_prologue(struct bpf_jit *jit, bool is_classic) 
			EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,

				      REG_15, 152);

	}

	/*

	 * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S"

	 * we store the SKB header length on the stack and the SKB data

	 * pointer in REG_SKB_DATA.

	 */

	if (jit->seen & SEEN_SKB) {

		/* Header length: llgf %w1,<len>(%b1) */

		EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1,

			      offsetof(struct sk_buff, len));

		/* s %w1,<data_len>(%b1) */

		EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1,

			   offsetof(struct sk_buff, data_len));

		/* stg %w1,ST_OFF_HLEN(%r0,%r15) */

	if (jit->seen & SEEN_SKB)

		emit_load_skb_data_hlen(jit);

	if (jit->seen & SEEN_SKB_CHANGE)

		/* stg %b1,ST_OFF_SKBP(%r0,%r15) */

		EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15,

			      STK_OFF_HLEN);

		/* lg %skb_data,data_off(%b1) */

		EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,

			      BPF_REG_1, offsetof(struct sk_buff, data));

	}

			      STK_OFF_SKBP);

	/* Clear A (%b0) and X (%b7) registers for converted BPF programs */

	if (is_classic) {

		if (REG_SEEN(BPF_REG_A))
@@ -967,10 +975,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i 
		 */

		const u64 func = (u64)__bpf_call_base + imm;



		if (bpf_helper_changes_skb_data((void *)func))

			/* TODO reload skb->data, hlen */

			return -1;



		REG_SET_SEEN(BPF_REG_5);

		jit->seen |= SEEN_FUNC;

		/* lg %w1,<d(imm)>(%l) */
@@ -980,6 +984,13 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i 
		EMIT2(0x0d00, REG_14, REG_W1);

		/* lgr %b0,%r2: load return value into %b0 */

		EMIT4(0xb9040000, BPF_REG_0, REG_2);

		if (bpf_helper_changes_skb_data((void *)func)) {

			jit->seen |= SEEN_SKB_CHANGE;

			/* lg %b1,ST_OFF_SKBP(%r15) */

			EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_1, REG_0,

				      REG_15, STK_OFF_SKBP);

			emit_load_skb_data_hlen(jit);

		}

		break;

	}

	case BPF_JMP | BPF_CALL | BPF_X:

CRA Git | Maintained and supported by SUSTech CRA and CCSE