Commit ba926603 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'xdp_redirect-bulking' 



Toke Høiland-Jørgensen says:

====================
Since commit 96360004 ("xdp: Make devmap flush_list common for all map
instances"), devmap flushing is a global operation instead of tied to a
particular map. This means that with a bit of refactoring, we can finally fix
the performance delta between the bpf_redirect_map() and bpf_redirect() helper
functions, by introducing bulking for the latter as well.

This series makes this change by moving the data structure used for the bulking
into struct net_device itself, so we can access it even when there is not
devmap. Once this is done, moving the bpf_redirect() helper to use the bulking
mechanism becomes quite trivial, and brings bpf_redirect() up to the same as
bpf_redirect_map():

                       Before:   After:
1 CPU:
bpf_redirect_map:      8.4 Mpps  8.4 Mpps  (no change)
bpf_redirect:          5.0 Mpps  8.4 Mpps  (+68%)
2 CPUs:
bpf_redirect_map:     15.9 Mpps  16.1 Mpps  (+1% or ~no change)
bpf_redirect:          9.5 Mpps  15.9 Mpps  (+67%)

After this patch series, the only semantics different between the two variants
of the bpf() helper (apart from the absence of a map argument, obviously) is
that the _map() variant will return an error if passed an invalid map index,
whereas the bpf_redirect() helper will succeed, but drop packets on
xdp_do_redirect(). This is because the helper has no reference to the calling
netdev, so unfortunately we can't do the ifindex lookup directly in the helper.

Changelog:

v3:
  - Switch two more fields to avoid a list_head spanning two cache lines
  - Include Jesper's tracepoint patch
  - Also rename xdp_do_flush_map()
  - Fix a few nits from Maciej

v2:
  - Consolidate code paths and tracepoints for map and non-map redirect variants
    (Björn)
  - Add performance data for 2-CPU test (Jesper)
  - Move fields to avoid shifting cache lines in struct net_device (Eric)
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 20f21d98 58aa94f9

drivers/net/tun.c

+2 −2
Original line number Diff line number Diff line
@@ -1718,7 +1718,7 @@ static struct sk_buff *tun_build_skb(struct tun_struct *tun, 
		if (err < 0)

			goto err_xdp;

		if (err == XDP_REDIRECT)

			xdp_do_flush_map();

			xdp_do_flush();

		if (err != XDP_PASS)

			goto out;
@@ -2549,7 +2549,7 @@ static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len) 
		}



		if (flush)

			xdp_do_flush_map();

			xdp_do_flush();



		rcu_read_unlock();

		local_bh_enable();

drivers/net/veth.c

+1 −1
Original line number Diff line number Diff line
@@ -769,7 +769,7 @@ static int veth_poll(struct napi_struct *napi, int budget) 
	if (xdp_xmit & VETH_XDP_TX)

		veth_xdp_flush(rq->dev, &bq);

	if (xdp_xmit & VETH_XDP_REDIR)

		xdp_do_flush_map();

		xdp_do_flush();

	xdp_clear_return_frame_no_direct();



	return done;

drivers/net/virtio_net.c

+1 −1
Original line number Diff line number Diff line
@@ -1432,7 +1432,7 @@ static int virtnet_poll(struct napi_struct *napi, int budget) 
		virtqueue_napi_complete(napi, rq->vq, received);



	if (xdp_xmit & VIRTIO_XDP_REDIR)

		xdp_do_flush_map();

		xdp_do_flush();



	if (xdp_xmit & VIRTIO_XDP_TX) {

		sq = virtnet_xdp_sq(vi);

include/linux/bpf.h

+11 −2
Original line number Diff line number Diff line
@@ -1056,7 +1056,9 @@ struct sk_buff; 


struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key);

struct bpf_dtab_netdev *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key);

void __dev_map_flush(void);

void __dev_flush(void);

int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,

		    struct net_device *dev_rx);

int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,

		    struct net_device *dev_rx);

int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
@@ -1169,13 +1171,20 @@ static inline struct net_device *__dev_map_hash_lookup_elem(struct bpf_map *map 
	return NULL;

}



static inline void __dev_map_flush(void)

static inline void __dev_flush(void)

{

}



struct xdp_buff;

struct bpf_dtab_netdev;



static inline

int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,

		    struct net_device *dev_rx)

{

	return 0;

}



static inline

int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,

		    struct net_device *dev_rx)

include/linux/filter.h

+8 −2
Original line number Diff line number Diff line
@@ -918,7 +918,7 @@ static inline int xdp_ok_fwd_dev(const struct net_device *fwd, 
	return 0;

}



/* The pair of xdp_do_redirect and xdp_do_flush_map MUST be called in the

/* The pair of xdp_do_redirect and xdp_do_flush MUST be called in the

 * same cpu context. Further for best results no more than a single map

 * for the do_redirect/do_flush pair should be used. This limitation is

 * because we only track one map and force a flush when the map changes.
@@ -929,7 +929,13 @@ int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb, 
int xdp_do_redirect(struct net_device *dev,

		    struct xdp_buff *xdp,

		    struct bpf_prog *prog);

void xdp_do_flush_map(void);

void xdp_do_flush(void);



/* The xdp_do_flush_map() helper has been renamed to drop the _map suffix, as

 * it is no longer only flushing maps. Keep this define for compatibility

 * until all drivers are updated - do not use xdp_do_flush_map() in new code!

 */

#define xdp_do_flush_map xdp_do_flush



void bpf_warn_invalid_xdp_action(u32 act);

CRA Git | Maintained and supported by SUSTech CRA and CCSE