Commit c44768a3 authored by David Miller's avatar David Miller Committed by Alexei Starovoitov
Browse files

sparc: Fix JIT fused branch convergance. 



On T4 and later sparc64 cpus we can use the fused compare and branch
instruction.

However, it can only be used if the branch destination is in the range
of a signed 10-bit immediate offset.  This amounts to 1024
instructions forwards or backwards.

After the commit referenced in the Fixes: tag, the largest possible
size program seen by the JIT explodes by a significant factor.

As a result of this convergance takes many more passes since the
expanded "BPF_LDX | BPF_MSH | BPF_B" code sequence, for example,
contains several embedded branch on condition instructions.

On each pass, as suddenly new fused compare and branch instances
become valid, this makes thousands more in range for the next pass.
And so on and so forth.

This is most greatly exemplified by "BPF_MAXINSNS: exec all MSH" which
takes 35 passes to converge, and shrinks the image by about 64K.

To decrease the cost of this number of convergance passes, do the
convergance pass before we have the program image allocated, just like
other JITs (such as x86) do.

Fixes: e0cea7ce ("bpf: implement ld_abs/ld_ind in native bpf")
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent fdac315d

arch/sparc/net/bpf_jit_comp_64.c

+49 −28
Original line number Diff line number Diff line
@@ -1425,12 +1425,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 
	struct bpf_prog *tmp, *orig_prog = prog;

	struct sparc64_jit_data *jit_data;

	struct bpf_binary_header *header;

	u32 prev_image_size, image_size;

	bool tmp_blinded = false;

	bool extra_pass = false;

	struct jit_ctx ctx;

	u32 image_size;

	u8 *image_ptr;

	int pass;

	int pass, i;



	if (!prog->jit_requested)

		return orig_prog;
@@ -1461,28 +1461,53 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 
		header = jit_data->header;

		extra_pass = true;

		image_size = sizeof(u32) * ctx.idx;

		prev_image_size = image_size;

		pass = 1;

		goto skip_init_ctx;

	}



	memset(&ctx, 0, sizeof(ctx));

	ctx.prog = prog;



	ctx.offset = kcalloc(prog->len, sizeof(unsigned int), GFP_KERNEL);

	ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL);

	if (ctx.offset == NULL) {

		prog = orig_prog;

		goto out_off;

	}



	/* Fake pass to detect features used, and get an accurate assessment

	 * of what the final image size will be.

	/* Longest sequence emitted is for bswap32, 12 instructions.  Pre-cook

	 * the offset array so that we converge faster.

	 */

	for (i = 0; i < prog->len; i++)

		ctx.offset[i] = i * (12 * 4);



	prev_image_size = ~0U;

	for (pass = 1; pass < 40; pass++) {

		ctx.idx = 0;



		build_prologue(&ctx);

		if (build_body(&ctx)) {

			prog = orig_prog;

			goto out_off;

		}

	build_prologue(&ctx);

		build_epilogue(&ctx);



		if (bpf_jit_enable > 1)

			pr_info("Pass %d: size = %u, seen = [%c%c%c%c%c%c]\n", pass,

				ctx.idx * 4,

				ctx.tmp_1_used ? '1' : ' ',

				ctx.tmp_2_used ? '2' : ' ',

				ctx.tmp_3_used ? '3' : ' ',

				ctx.saw_frame_pointer ? 'F' : ' ',

				ctx.saw_call ? 'C' : ' ',

				ctx.saw_tail_call ? 'T' : ' ');



		if (ctx.idx * 4 == prev_image_size)

			break;

		prev_image_size = ctx.idx * 4;

		cond_resched();

	}



	/* Now we know the actual image size. */

	image_size = sizeof(u32) * ctx.idx;

	header = bpf_jit_binary_alloc(image_size, &image_ptr,
@@ -1494,7 +1519,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 


	ctx.image = (u32 *)image_ptr;

skip_init_ctx:

	for (pass = 1; pass < 3; pass++) {

	ctx.idx = 0;



	build_prologue(&ctx);
@@ -1507,15 +1531,12 @@ skip_init_ctx: 


	build_epilogue(&ctx);



		if (bpf_jit_enable > 1)

			pr_info("Pass %d: shrink = %d, seen = [%c%c%c%c%c%c]\n", pass,

				image_size - (ctx.idx * 4),

				ctx.tmp_1_used ? '1' : ' ',

				ctx.tmp_2_used ? '2' : ' ',

				ctx.tmp_3_used ? '3' : ' ',

				ctx.saw_frame_pointer ? 'F' : ' ',

				ctx.saw_call ? 'C' : ' ',

				ctx.saw_tail_call ? 'T' : ' ');

	if (ctx.idx * 4 != prev_image_size) {

		pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n",

		       prev_image_size, ctx.idx * 4);

		bpf_jit_binary_free(header);

		prog = orig_prog;

		goto out_off;

	}



	if (bpf_jit_enable > 1)

CRA Git | Maintained and supported by SUSTech CRA and CCSE