Commit 1d233886 authored by Toke Høiland-Jørgensen's avatar Toke Høiland-Jørgensen Committed by Alexei Starovoitov
Browse files

xdp: Use bulking for non-map XDP_REDIRECT and consolidate code paths 



Since the bulk queue used by XDP_REDIRECT now lives in struct net_device,
we can re-use the bulking for the non-map version of the bpf_redirect()
helper. This is a simple matter of having xdp_do_redirect_slow() queue the
frame on the bulk queue instead of sending it out with __bpf_tx_xdp().

Unfortunately we can't make the bpf_redirect() helper return an error if
the ifindex doesn't exit (as bpf_redirect_map() does), because we don't
have a reference to the network namespace of the ingress device at the time
the helper is called. So we have to leave it as-is and keep the device
lookup in xdp_do_redirect_slow().

Since this leaves less reason to have the non-map redirect code in a
separate function, so we get rid of the xdp_do_redirect_slow() function
entirely. This does lose us the tracepoint disambiguation, but fortunately
the xdp_redirect and xdp_redirect_map tracepoints use the same tracepoint
entry structures. This means both can contain a map index, so we can just
amend the tracepoint definitions so we always emit the xdp_redirect(_err)
tracepoints, but with the map ID only populated if a map is present. This
means we retire the xdp_redirect_map(_err) tracepoints entirely, but keep
the definitions around in case someone is still listening for them.

With this change, the performance of the xdp_redirect sample program goes
from 5Mpps to 8.4Mpps (a 68% increase).

Since the flush functions are no longer map-specific, rename the flush()
functions to drop _map from their names. One of the renamed functions is
the xdp_do_flush_map() callback used in all the xdp-enabled drivers. To
keep from having to update all drivers, use a #define to keep the old name
working, and only update the virtual drivers in this patch.

Signed-off-by: default avatarToke Høiland-Jørgensen <toke@redhat.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarJohn Fastabend <john.fastabend@gmail.com>
Link: https://lore.kernel.org/bpf/157918768505.1458396.17518057312953572912.stgit@toke.dk
parent 75ccae62

drivers/net/tun.c

+2 −2
Original line number Diff line number Diff line
@@ -1718,7 +1718,7 @@ static struct sk_buff *tun_build_skb(struct tun_struct *tun, 
		if (err < 0)

			goto err_xdp;

		if (err == XDP_REDIRECT)

			xdp_do_flush_map();

			xdp_do_flush();

		if (err != XDP_PASS)

			goto out;
@@ -2549,7 +2549,7 @@ static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len) 
		}



		if (flush)

			xdp_do_flush_map();

			xdp_do_flush();



		rcu_read_unlock();

		local_bh_enable();

drivers/net/veth.c

+1 −1
Original line number Diff line number Diff line
@@ -769,7 +769,7 @@ static int veth_poll(struct napi_struct *napi, int budget) 
	if (xdp_xmit & VETH_XDP_TX)

		veth_xdp_flush(rq->dev, &bq);

	if (xdp_xmit & VETH_XDP_REDIR)

		xdp_do_flush_map();

		xdp_do_flush();

	xdp_clear_return_frame_no_direct();



	return done;

drivers/net/virtio_net.c

+1 −1
Original line number Diff line number Diff line
@@ -1432,7 +1432,7 @@ static int virtnet_poll(struct napi_struct *napi, int budget) 
		virtqueue_napi_complete(napi, rq->vq, received);



	if (xdp_xmit & VIRTIO_XDP_REDIR)

		xdp_do_flush_map();

		xdp_do_flush();



	if (xdp_xmit & VIRTIO_XDP_TX) {

		sq = virtnet_xdp_sq(vi);

include/linux/bpf.h

+11 −2
Original line number Diff line number Diff line
@@ -1056,7 +1056,9 @@ struct sk_buff; 


struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key);

struct bpf_dtab_netdev *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key);

void __dev_map_flush(void);

void __dev_flush(void);

int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,

		    struct net_device *dev_rx);

int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,

		    struct net_device *dev_rx);

int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
@@ -1169,13 +1171,20 @@ static inline struct net_device *__dev_map_hash_lookup_elem(struct bpf_map *map 
	return NULL;

}



static inline void __dev_map_flush(void)

static inline void __dev_flush(void)

{

}



struct xdp_buff;

struct bpf_dtab_netdev;



static inline

int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,

		    struct net_device *dev_rx)

{

	return 0;

}



static inline

int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,

		    struct net_device *dev_rx)

include/linux/filter.h

+8 −2
Original line number Diff line number Diff line
@@ -918,7 +918,7 @@ static inline int xdp_ok_fwd_dev(const struct net_device *fwd, 
	return 0;

}



/* The pair of xdp_do_redirect and xdp_do_flush_map MUST be called in the

/* The pair of xdp_do_redirect and xdp_do_flush MUST be called in the

 * same cpu context. Further for best results no more than a single map

 * for the do_redirect/do_flush pair should be used. This limitation is

 * because we only track one map and force a flush when the map changes.
@@ -929,7 +929,13 @@ int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb, 
int xdp_do_redirect(struct net_device *dev,

		    struct xdp_buff *xdp,

		    struct bpf_prog *prog);

void xdp_do_flush_map(void);

void xdp_do_flush(void);



/* The xdp_do_flush_map() helper has been renamed to drop the _map suffix, as

 * it is no longer only flushing maps. Keep this define for compatibility

 * until all drivers are updated - do not use xdp_do_flush_map() in new code!

 */

#define xdp_do_flush_map xdp_do_flush



void bpf_warn_invalid_xdp_action(u32 act);

CRA Git | Maintained and supported by SUSTech CRA and CCSE