Commit 03653515 authored by Lorenz Bauer's avatar Lorenz Bauer Committed by Alexei Starovoitov
Browse files

net: Allow iterating sockmap and sockhash 



Add bpf_iter support for sockmap / sockhash, based on the bpf_sk_storage and
hashtable implementation. sockmap and sockhash share the same iteration
context: a pointer to an arbitrary key and a pointer to a socket. Both
pointers may be NULL, and so BPF has to perform a NULL check before accessing
them. Technically it's not possible for sockhash iteration to yield a NULL
socket, but we ignore this to be able to use a single iteration point.

Iteration will visit all keys that remain unmodified during the lifetime of
the iterator. It may or may not visit newly added ones.

Switch from using rcu_dereference_raw to plain rcu_dereference, so we gain
another guard rail if CONFIG_PROVE_RCU is enabled.

Signed-off-by: default avatarLorenz Bauer <lmb@cloudflare.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarYonghong Song <yhs@fb.com>
Link: https://lore.kernel.org/bpf/20200909162712.221874-3-lmb@cloudflare.com
parent 654785a1

net/core/sock_map.c

+278 −2
Original line number Diff line number Diff line
@@ -2,6 +2,7 @@ 
/* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */



#include <linux/bpf.h>

#include <linux/btf_ids.h>

#include <linux/filter.h>

#include <linux/errno.h>

#include <linux/file.h>
@@ -703,6 +704,109 @@ const struct bpf_func_proto bpf_msg_redirect_map_proto = { 
	.arg4_type      = ARG_ANYTHING,

};



struct sock_map_seq_info {

	struct bpf_map *map;

	struct sock *sk;

	u32 index;

};



struct bpf_iter__sockmap {

	__bpf_md_ptr(struct bpf_iter_meta *, meta);

	__bpf_md_ptr(struct bpf_map *, map);

	__bpf_md_ptr(void *, key);

	__bpf_md_ptr(struct sock *, sk);

};



DEFINE_BPF_ITER_FUNC(sockmap, struct bpf_iter_meta *meta,

		     struct bpf_map *map, void *key,

		     struct sock *sk)



static void *sock_map_seq_lookup_elem(struct sock_map_seq_info *info)

{

	if (unlikely(info->index >= info->map->max_entries))

		return NULL;



	info->sk = __sock_map_lookup_elem(info->map, info->index);



	/* can't return sk directly, since that might be NULL */

	return info;

}



static void *sock_map_seq_start(struct seq_file *seq, loff_t *pos)

{

	struct sock_map_seq_info *info = seq->private;



	if (*pos == 0)

		++*pos;



	/* pairs with sock_map_seq_stop */

	rcu_read_lock();

	return sock_map_seq_lookup_elem(info);

}



static void *sock_map_seq_next(struct seq_file *seq, void *v, loff_t *pos)

{

	struct sock_map_seq_info *info = seq->private;



	++*pos;

	++info->index;



	return sock_map_seq_lookup_elem(info);

}



static int sock_map_seq_show(struct seq_file *seq, void *v)

{

	struct sock_map_seq_info *info = seq->private;

	struct bpf_iter__sockmap ctx = {};

	struct bpf_iter_meta meta;

	struct bpf_prog *prog;



	meta.seq = seq;

	prog = bpf_iter_get_info(&meta, !v);

	if (!prog)

		return 0;



	ctx.meta = &meta;

	ctx.map = info->map;

	if (v) {

		ctx.key = &info->index;

		ctx.sk = info->sk;

	}



	return bpf_iter_run_prog(prog, &ctx);

}



static void sock_map_seq_stop(struct seq_file *seq, void *v)

{

	if (!v)

		(void)sock_map_seq_show(seq, NULL);



	/* pairs with sock_map_seq_start */

	rcu_read_unlock();

}



static const struct seq_operations sock_map_seq_ops = {

	.start	= sock_map_seq_start,

	.next	= sock_map_seq_next,

	.stop	= sock_map_seq_stop,

	.show	= sock_map_seq_show,

};



static int sock_map_init_seq_private(void *priv_data,

				     struct bpf_iter_aux_info *aux)

{

	struct sock_map_seq_info *info = priv_data;



	info->map = aux->map;

	return 0;

}



static const struct bpf_iter_seq_info sock_map_iter_seq_info = {

	.seq_ops		= &sock_map_seq_ops,

	.init_seq_private	= sock_map_init_seq_private,

	.seq_priv_size		= sizeof(struct sock_map_seq_info),

};



static int sock_map_btf_id;

const struct bpf_map_ops sock_map_ops = {

	.map_meta_equal		= bpf_map_meta_equal,
@@ -717,6 +821,7 @@ const struct bpf_map_ops sock_map_ops = { 
	.map_check_btf		= map_check_no_btf,

	.map_btf_name		= "bpf_stab",

	.map_btf_id		= &sock_map_btf_id,

	.iter_seq_info		= &sock_map_iter_seq_info,

};



struct bpf_shtab_elem {
@@ -953,7 +1058,7 @@ static int sock_hash_get_next_key(struct bpf_map *map, void *key, 
	if (!elem)

		goto find_first_elem;



	elem_next = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(&elem->node)),

	elem_next = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&elem->node)),

				     struct bpf_shtab_elem, node);

	if (elem_next) {

		memcpy(key_next, elem_next->key, key_size);
@@ -965,7 +1070,7 @@ static int sock_hash_get_next_key(struct bpf_map *map, void *key, 
find_first_elem:

	for (; i < htab->buckets_num; i++) {

		head = &sock_hash_select_bucket(htab, i)->head;

		elem_next = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),

		elem_next = hlist_entry_safe(rcu_dereference(hlist_first_rcu(head)),

					     struct bpf_shtab_elem, node);

		if (elem_next) {

			memcpy(key_next, elem_next->key, key_size);
@@ -1199,6 +1304,117 @@ const struct bpf_func_proto bpf_msg_redirect_hash_proto = { 
	.arg4_type      = ARG_ANYTHING,

};



struct sock_hash_seq_info {

	struct bpf_map *map;

	struct bpf_shtab *htab;

	u32 bucket_id;

};



static void *sock_hash_seq_find_next(struct sock_hash_seq_info *info,

				     struct bpf_shtab_elem *prev_elem)

{

	const struct bpf_shtab *htab = info->htab;

	struct bpf_shtab_bucket *bucket;

	struct bpf_shtab_elem *elem;

	struct hlist_node *node;



	/* try to find next elem in the same bucket */

	if (prev_elem) {

		node = rcu_dereference(hlist_next_rcu(&prev_elem->node));

		elem = hlist_entry_safe(node, struct bpf_shtab_elem, node);

		if (elem)

			return elem;



		/* no more elements, continue in the next bucket */

		info->bucket_id++;

	}



	for (; info->bucket_id < htab->buckets_num; info->bucket_id++) {

		bucket = &htab->buckets[info->bucket_id];

		node = rcu_dereference(hlist_first_rcu(&bucket->head));

		elem = hlist_entry_safe(node, struct bpf_shtab_elem, node);

		if (elem)

			return elem;

	}



	return NULL;

}



static void *sock_hash_seq_start(struct seq_file *seq, loff_t *pos)

{

	struct sock_hash_seq_info *info = seq->private;



	if (*pos == 0)

		++*pos;



	/* pairs with sock_hash_seq_stop */

	rcu_read_lock();

	return sock_hash_seq_find_next(info, NULL);

}



static void *sock_hash_seq_next(struct seq_file *seq, void *v, loff_t *pos)

{

	struct sock_hash_seq_info *info = seq->private;



	++*pos;

	return sock_hash_seq_find_next(info, v);

}



static int sock_hash_seq_show(struct seq_file *seq, void *v)

{

	struct sock_hash_seq_info *info = seq->private;

	struct bpf_iter__sockmap ctx = {};

	struct bpf_shtab_elem *elem = v;

	struct bpf_iter_meta meta;

	struct bpf_prog *prog;



	meta.seq = seq;

	prog = bpf_iter_get_info(&meta, !elem);

	if (!prog)

		return 0;



	ctx.meta = &meta;

	ctx.map = info->map;

	if (elem) {

		ctx.key = elem->key;

		ctx.sk = elem->sk;

	}



	return bpf_iter_run_prog(prog, &ctx);

}



static void sock_hash_seq_stop(struct seq_file *seq, void *v)

{

	if (!v)

		(void)sock_hash_seq_show(seq, NULL);



	/* pairs with sock_hash_seq_start */

	rcu_read_unlock();

}



static const struct seq_operations sock_hash_seq_ops = {

	.start	= sock_hash_seq_start,

	.next	= sock_hash_seq_next,

	.stop	= sock_hash_seq_stop,

	.show	= sock_hash_seq_show,

};



static int sock_hash_init_seq_private(void *priv_data,

				     struct bpf_iter_aux_info *aux)

{

	struct sock_hash_seq_info *info = priv_data;



	info->map = aux->map;

	info->htab = container_of(aux->map, struct bpf_shtab, map);

	return 0;

}



static const struct bpf_iter_seq_info sock_hash_iter_seq_info = {

	.seq_ops		= &sock_hash_seq_ops,

	.init_seq_private	= sock_hash_init_seq_private,

	.seq_priv_size		= sizeof(struct sock_hash_seq_info),

};



static int sock_hash_map_btf_id;

const struct bpf_map_ops sock_hash_ops = {

	.map_meta_equal		= bpf_map_meta_equal,
@@ -1213,6 +1429,7 @@ const struct bpf_map_ops sock_hash_ops = { 
	.map_check_btf		= map_check_no_btf,

	.map_btf_name		= "bpf_shtab",

	.map_btf_id		= &sock_hash_map_btf_id,

	.iter_seq_info		= &sock_hash_iter_seq_info,

};



static struct sk_psock_progs *sock_map_progs(struct bpf_map *map)
@@ -1323,3 +1540,62 @@ void sock_map_close(struct sock *sk, long timeout) 
	release_sock(sk);

	saved_close(sk, timeout);

}



static int sock_map_iter_attach_target(struct bpf_prog *prog,

				       union bpf_iter_link_info *linfo,

				       struct bpf_iter_aux_info *aux)

{

	struct bpf_map *map;

	int err = -EINVAL;



	if (!linfo->map.map_fd)

		return -EBADF;



	map = bpf_map_get_with_uref(linfo->map.map_fd);

	if (IS_ERR(map))

		return PTR_ERR(map);



	if (map->map_type != BPF_MAP_TYPE_SOCKMAP &&

	    map->map_type != BPF_MAP_TYPE_SOCKHASH)

		goto put_map;



	if (prog->aux->max_rdonly_access > map->key_size) {

		err = -EACCES;

		goto put_map;

	}



	aux->map = map;

	return 0;



put_map:

	bpf_map_put_with_uref(map);

	return err;

}



static void sock_map_iter_detach_target(struct bpf_iter_aux_info *aux)

{

	bpf_map_put_with_uref(aux->map);

}



static struct bpf_iter_reg sock_map_iter_reg = {

	.target			= "sockmap",

	.attach_target		= sock_map_iter_attach_target,

	.detach_target		= sock_map_iter_detach_target,

	.show_fdinfo		= bpf_iter_map_show_fdinfo,

	.fill_link_info		= bpf_iter_map_fill_link_info,

	.ctx_arg_info_size	= 2,

	.ctx_arg_info		= {

		{ offsetof(struct bpf_iter__sockmap, key),

		  PTR_TO_RDONLY_BUF_OR_NULL },

		{ offsetof(struct bpf_iter__sockmap, sk),

		  PTR_TO_BTF_ID_OR_NULL },

	},

};



static int __init bpf_sockmap_iter_init(void)

{

	sock_map_iter_reg.ctx_arg_info[1].btf_id =

		btf_sock_ids[BTF_SOCK_TYPE_SOCK];

	return bpf_iter_reg_target(&sock_map_iter_reg);

}

late_initcall(bpf_sockmap_iter_init);

CRA Git | Maintained and supported by SUSTech CRA and CCSE