Commit 3a70418b authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'bpf-inline-lookups' 



Alexei Starovoitov says:

====================
bpf: inline bpf_map_lookup_elem()

bpf_map_lookup_elem() is one of the most frequently used helper functions.
Improve JITed program performance by inlining this helper.

bpf_map_type	before  after
hash		58M	74M
array		174M	280M

The values are number of lookups per second in ideal conditions
measured by micro-benchmark in patch 6.

The 'perf report' for HASH map type:
before:
    54.23%  map_perf_test  [kernel.kallsyms]  [k] __htab_map_lookup_elem
    14.24%  map_perf_test  [kernel.kallsyms]  [k] lookup_elem_raw
     8.84%  map_perf_test  [kernel.kallsyms]  [k] htab_map_lookup_elem
     5.93%  map_perf_test  [kernel.kallsyms]  [k] bpf_map_lookup_elem
     2.30%  map_perf_test  [kernel.kallsyms]  [k] bpf_prog_da4fc6a3f41761a2
     1.49%  map_perf_test  [kernel.kallsyms]  [k] kprobe_ftrace_handler

after:
    60.03%  map_perf_test  [kernel.kallsyms]  [k] __htab_map_lookup_elem
    18.07%  map_perf_test  [kernel.kallsyms]  [k] lookup_elem_raw
     2.91%  map_perf_test  [kernel.kallsyms]  [k] bpf_prog_da4fc6a3f41761a2
     1.94%  map_perf_test  [kernel.kallsyms]  [k] _einittext
     1.90%  map_perf_test  [kernel.kallsyms]  [k] __audit_syscall_exit
     1.72%  map_perf_test  [kernel.kallsyms]  [k] kprobe_ftrace_handler

so the cost of htab_map_lookup_elem() and bpf_map_lookup_elem()
is gone after inlining.

'per-cpu' and 'lru' map types can be optimized similarly in the future.

Note the sparse will complain that bpf is addictive ;)
kernel/bpf/hashtab.c:438:19: sparse: subtraction of functions? Share your drugs
kernel/bpf/verifier.c:3342:38: sparse: subtraction of functions? Share your drugs
it's not a new warning, just in new places.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 4396e461 95ff141e

include/linux/bpf.h

+1 −0
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ struct bpf_map_ops { 
	void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,

				int fd);

	void (*map_fd_put_ptr)(void *ptr);

	u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);

};



struct bpf_map {

include/linux/bpf_verifier.h

+4 −1
Original line number Diff line number Diff line
@@ -66,7 +66,10 @@ struct bpf_verifier_state_list { 
};



struct bpf_insn_aux_data {

	union {

		enum bpf_reg_type ptr_type;	/* pointer type for load/store insns */

		struct bpf_map *map_ptr;	/* pointer for call insn into lookup_elem */

	};

};



#define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */

include/linux/filter.h

+10 −0
Original line number Diff line number Diff line
@@ -693,6 +693,11 @@ static inline bool bpf_jit_is_ebpf(void) 
# endif

}



static inline bool ebpf_jit_enabled(void)

{

	return bpf_jit_enable && bpf_jit_is_ebpf();

}



static inline bool bpf_prog_ebpf_jited(const struct bpf_prog *fp)

{

	return fp->jited && bpf_jit_is_ebpf();
@@ -753,6 +758,11 @@ void bpf_prog_kallsyms_del(struct bpf_prog *fp); 


#else /* CONFIG_BPF_JIT */



static inline bool ebpf_jit_enabled(void)

{

	return false;

}



static inline bool bpf_prog_ebpf_jited(const struct bpf_prog *fp)

{

	return false;

kernel/bpf/arraymap.c

+29 −0
Original line number Diff line number Diff line 
/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com

 * Copyright (c) 2016,2017 Facebook

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of version 2 of the GNU General Public
@@ -113,6 +114,33 @@ static void *array_map_lookup_elem(struct bpf_map *map, void *key) 
	return array->value + array->elem_size * index;

}



/* emit BPF instructions equivalent to C code of array_map_lookup_elem() */

static u32 array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)

{

	struct bpf_array *array = container_of(map, struct bpf_array, map);

	struct bpf_insn *insn = insn_buf;

	u32 elem_size = array->elem_size;

	const int ret = BPF_REG_0;

	const int map_ptr = BPF_REG_1;

	const int index = BPF_REG_2;



	*insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));

	*insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);

	*insn++ = BPF_JMP_IMM(BPF_JGE, ret, array->map.max_entries,

			      elem_size == 1 ? 2 : 3);

	if (elem_size == 1) {

		/* nop */

	} else if (is_power_of_2(elem_size)) {

		*insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size));

	} else {

		*insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size);

	}

	*insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr);

	*insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);

	*insn++ = BPF_MOV64_IMM(ret, 0);

	return insn - insn_buf;

}



/* Called from eBPF program */

static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key)

{
@@ -267,6 +295,7 @@ static const struct bpf_map_ops array_ops = { 
	.map_lookup_elem = array_map_lookup_elem,

	.map_update_elem = array_map_update_elem,

	.map_delete_elem = array_map_delete_elem,

	.map_gen_lookup = array_map_gen_lookup,

};



static struct bpf_map_type_list array_type __ro_after_init = {

kernel/bpf/hashtab.c

+30 −1
Original line number Diff line number Diff line
@@ -419,7 +419,11 @@ again: 
	return NULL;

}



/* Called from syscall or from eBPF program */

/* Called from syscall or from eBPF program directly, so

 * arguments have to match bpf_map_lookup_elem() exactly.

 * The return value is adjusted by BPF instructions

 * in htab_map_gen_lookup().

 */

static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)

{

	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
@@ -451,6 +455,30 @@ static void *htab_map_lookup_elem(struct bpf_map *map, void *key) 
	return NULL;

}



/* inline bpf_map_lookup_elem() call.

 * Instead of:

 * bpf_prog

 *   bpf_map_lookup_elem

 *     map->ops->map_lookup_elem

 *       htab_map_lookup_elem

 *         __htab_map_lookup_elem

 * do:

 * bpf_prog

 *   __htab_map_lookup_elem

 */

static u32 htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)

{

	struct bpf_insn *insn = insn_buf;

	const int ret = BPF_REG_0;



	*insn++ = BPF_EMIT_CALL((u64 (*)(u64, u64, u64, u64, u64))__htab_map_lookup_elem);

	*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);

	*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,

				offsetof(struct htab_elem, key) +

				round_up(map->key_size, 8));

	return insn - insn_buf;

}



static void *htab_lru_map_lookup_elem(struct bpf_map *map, void *key)

{

	struct htab_elem *l = __htab_map_lookup_elem(map, key);
@@ -1062,6 +1090,7 @@ static const struct bpf_map_ops htab_ops = { 
	.map_lookup_elem = htab_map_lookup_elem,

	.map_update_elem = htab_map_update_elem,

	.map_delete_elem = htab_map_delete_elem,

	.map_gen_lookup = htab_map_gen_lookup,

};



static struct bpf_map_type_list htab_type __ro_after_init = {

CRA Git | Maintained and supported by SUSTech CRA and CCSE