Commit b741f163 authored by Roman Gushchin's avatar Roman Gushchin Committed by Daniel Borkmann
Browse files

bpf: introduce per-cpu cgroup local storage 



This commit introduced per-cpu cgroup local storage.

Per-cpu cgroup local storage is very similar to simple cgroup storage
(let's call it shared), except all the data is per-cpu.

The main goal of per-cpu variant is to implement super fast
counters (e.g. packet counters), which don't require neither
lookups, neither atomic operations.

>From userspace's point of view, accessing a per-cpu cgroup storage
is similar to other per-cpu map types (e.g. per-cpu hashmaps and
arrays).

Writing to a per-cpu cgroup storage is not atomic, but is performed
by copying longs, so some minimal atomicity is here, exactly
as with other per-cpu maps.

Signed-off-by: default avatarRoman Gushchin <guro@fb.com>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Cc: Alexei Starovoitov <ast@kernel.org>
Acked-by: default avatarSong Liu <songliubraving@fb.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parent f294b37e

include/linux/bpf-cgroup.h

+19 −1
Original line number Diff line number Diff line
@@ -37,7 +37,10 @@ struct bpf_storage_buffer { 
};



struct bpf_cgroup_storage {

	union {

		struct bpf_storage_buffer *buf;

		void __percpu *percpu_buf;

	};

	struct bpf_cgroup_storage_map *map;

	struct bpf_cgroup_storage_key key;

	struct list_head list;
@@ -109,6 +112,9 @@ int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor, 
static inline enum bpf_cgroup_storage_type cgroup_storage_type(

	struct bpf_map *map)

{

	if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)

		return BPF_CGROUP_STORAGE_PERCPU;



	return BPF_CGROUP_STORAGE_SHARED;

}
@@ -131,6 +137,10 @@ void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage); 
int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *map);

void bpf_cgroup_storage_release(struct bpf_prog *prog, struct bpf_map *map);



int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, void *value);

int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,

				     void *value, u64 flags);



/* Wrappers for __cgroup_bpf_run_filter_skb() guarded by cgroup_bpf_enabled. */

#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb)			      \

({									      \
@@ -285,6 +295,14 @@ static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc( 
	struct bpf_prog *prog, enum bpf_cgroup_storage_type stype) { return 0; }

static inline void bpf_cgroup_storage_free(

	struct bpf_cgroup_storage *storage) {}

static inline int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key,

						 void *value) {

	return 0;

}

static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,

					void *key, void *value, u64 flags) {

	return 0;

}



#define cgroup_bpf_enabled (0)

#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0)

include/linux/bpf.h

+1 −0
Original line number Diff line number Diff line
@@ -274,6 +274,7 @@ struct bpf_prog_offload { 


enum bpf_cgroup_storage_type {

	BPF_CGROUP_STORAGE_SHARED,

	BPF_CGROUP_STORAGE_PERCPU,

	__BPF_CGROUP_STORAGE_MAX

};

include/linux/bpf_types.h

+1 −0
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_CGROUP_ARRAY, cgroup_array_map_ops) 
#endif

#ifdef CONFIG_CGROUP_BPF

BPF_MAP_TYPE(BPF_MAP_TYPE_CGROUP_STORAGE, cgroup_storage_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, cgroup_storage_map_ops)

#endif

BPF_MAP_TYPE(BPF_MAP_TYPE_HASH, htab_map_ops)

BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_HASH, htab_percpu_map_ops)

include/uapi/linux/bpf.h

+1 −0
Original line number Diff line number Diff line
@@ -127,6 +127,7 @@ enum bpf_map_type { 
	BPF_MAP_TYPE_SOCKHASH,

	BPF_MAP_TYPE_CGROUP_STORAGE,

	BPF_MAP_TYPE_REUSEPORT_SOCKARRAY,

	BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE,

};



enum bpf_prog_type {

kernel/bpf/helpers.c

+7 −1
Original line number Diff line number Diff line
@@ -206,10 +206,16 @@ BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags) 
	 */

	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);

	struct bpf_cgroup_storage *storage;

	void *ptr;



	storage = this_cpu_read(bpf_cgroup_storage[stype]);



	return (unsigned long)&READ_ONCE(storage->buf)->data[0];

	if (stype == BPF_CGROUP_STORAGE_SHARED)

		ptr = &READ_ONCE(storage->buf)->data[0];

	else

		ptr = this_cpu_ptr(storage->percpu_buf);



	return (unsigned long)ptr;

}



const struct bpf_func_proto bpf_get_local_storage_proto = {

CRA Git | Maintained and supported by SUSTech CRA and CCSE