Commit 186b3c99 authored by Jason Wang's avatar Jason Wang Committed by David S. Miller
Browse files

virtio-net: support XDP_REDIRECT 



This patch tries to add XDP_REDIRECT for virtio-net. The changes are
not complex as we could use exist XDP_TX helpers for most of the
work. The rest is passing the XDP_TX to NAPI handler for implementing
batching.

Cc: John Fastabend <john.fastabend@gmail.com>
Signed-off-by: default avatarJason Wang <jasowang@redhat.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 31240345

drivers/net/virtio_net.

0 → 100644
+0 −0

Empty file added.

drivers/net/virtio_net.c

+62 −15
Original line number Diff line number Diff line
@@ -29,6 +29,7 @@ 
#include <linux/slab.h>

#include <linux/cpu.h>

#include <linux/average.h>

#include <linux/filter.h>

#include <net/route.h>



static int napi_weight = NAPI_POLL_WEIGHT;
@@ -372,7 +373,19 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, 
	return skb;

}



static bool virtnet_xdp_xmit(struct virtnet_info *vi,

static void virtnet_xdp_flush(struct net_device *dev)

{

	struct virtnet_info *vi = netdev_priv(dev);

	struct send_queue *sq;

	unsigned int qp;



	qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();

	sq = &vi->sq[qp];



	virtqueue_kick(sq->vq);

}



static bool __virtnet_xdp_xmit(struct virtnet_info *vi,

			       struct xdp_buff *xdp)

{

	struct virtio_net_hdr_mrg_rxbuf *hdr;
@@ -407,10 +420,19 @@ static bool virtnet_xdp_xmit(struct virtnet_info *vi, 
		return false;

	}



	virtqueue_kick(sq->vq);

	return true;

}



static int virtnet_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)

{

	struct virtnet_info *vi = netdev_priv(dev);

	bool sent = __virtnet_xdp_xmit(vi, xdp);



	if (!sent)

		return -ENOSPC;

	return 0;

}



static unsigned int virtnet_get_headroom(struct virtnet_info *vi)

{

	return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;
@@ -483,7 +505,8 @@ static struct sk_buff *receive_small(struct net_device *dev, 
				     struct virtnet_info *vi,

				     struct receive_queue *rq,

				     void *buf, void *ctx,

				     unsigned int len)

				     unsigned int len,

				     bool *xdp_xmit)

{

	struct sk_buff *skb;

	struct bpf_prog *xdp_prog;
@@ -493,7 +516,7 @@ static struct sk_buff *receive_small(struct net_device *dev, 
	unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +

			      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));

	struct page *page = virt_to_head_page(buf);

	unsigned int delta = 0;

	unsigned int delta = 0, err;

	struct page *xdp_page;

	len -= vi->hdr_len;
@@ -541,8 +564,16 @@ static struct sk_buff *receive_small(struct net_device *dev, 
			delta = orig_data - xdp.data;

			break;

		case XDP_TX:

			if (unlikely(!virtnet_xdp_xmit(vi, &xdp)))

			if (unlikely(!__virtnet_xdp_xmit(vi, &xdp)))

				trace_xdp_exception(vi->dev, xdp_prog, act);

			else

				*xdp_xmit = true;

			rcu_read_unlock();

			goto xdp_xmit;

		case XDP_REDIRECT:

			err = xdp_do_redirect(dev, &xdp, xdp_prog);

			if (!err)

				*xdp_xmit = true;

			rcu_read_unlock();

			goto xdp_xmit;

		default:
@@ -603,7 +634,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, 
					 struct receive_queue *rq,

					 void *buf,

					 void *ctx,

					 unsigned int len)

					 unsigned int len,

					 bool *xdp_xmit)

{

	struct virtio_net_hdr_mrg_rxbuf *hdr = buf;

	u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
@@ -613,6 +645,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, 
	struct bpf_prog *xdp_prog;

	unsigned int truesize;

	unsigned int headroom = mergeable_ctx_to_headroom(ctx);

	int err;



	head_skb = NULL;
@@ -678,12 +711,20 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, 
			}

			break;

		case XDP_TX:

			if (unlikely(!virtnet_xdp_xmit(vi, &xdp)))

			if (unlikely(!__virtnet_xdp_xmit(vi, &xdp)))

				trace_xdp_exception(vi->dev, xdp_prog, act);

			else

				*xdp_xmit = true;

			if (unlikely(xdp_page != page))

				goto err_xdp;

			rcu_read_unlock();

			goto xdp_xmit;

		case XDP_REDIRECT:

			err = xdp_do_redirect(dev, &xdp, xdp_prog);

			if (err)

				*xdp_xmit = true;

			rcu_read_unlock();

			goto xdp_xmit;

		default:

			bpf_warn_invalid_xdp_action(act);

		case XDP_ABORTED:
@@ -788,7 +829,7 @@ xdp_xmit: 
}



static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq,

		       void *buf, unsigned int len, void **ctx)

		       void *buf, unsigned int len, void **ctx, bool *xdp_xmit)

{

	struct net_device *dev = vi->dev;

	struct sk_buff *skb;
@@ -809,11 +850,11 @@ static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq, 
	}



	if (vi->mergeable_rx_bufs)

		skb = receive_mergeable(dev, vi, rq, buf, ctx, len);

		skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit);

	else if (vi->big_packets)

		skb = receive_big(dev, vi, rq, buf, len);

	else

		skb = receive_small(dev, vi, rq, buf, ctx, len);

		skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit);



	if (unlikely(!skb))

		return 0;
@@ -1071,7 +1112,7 @@ static void refill_work(struct work_struct *work) 
	}

}



static int virtnet_receive(struct receive_queue *rq, int budget)

static int virtnet_receive(struct receive_queue *rq, int budget, bool *xdp_xmit)

{

	struct virtnet_info *vi = rq->vq->vdev->priv;

	unsigned int len, received = 0, bytes = 0;
@@ -1083,13 +1124,13 @@ static int virtnet_receive(struct receive_queue *rq, int budget) 


		while (received < budget &&

		       (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) {

			bytes += receive_buf(vi, rq, buf, len, ctx);

			bytes += receive_buf(vi, rq, buf, len, ctx, xdp_xmit);

			received++;

		}

	} else {

		while (received < budget &&

		       (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {

			bytes += receive_buf(vi, rq, buf, len, NULL);

			bytes += receive_buf(vi, rq, buf, len, NULL, xdp_xmit);

			received++;

		}

	}
@@ -1161,15 +1202,19 @@ static int virtnet_poll(struct napi_struct *napi, int budget) 
	struct receive_queue *rq =

		container_of(napi, struct receive_queue, napi);

	unsigned int received;

	bool xdp_xmit = false;



	virtnet_poll_cleantx(rq);



	received = virtnet_receive(rq, budget);

	received = virtnet_receive(rq, budget, &xdp_xmit);



	/* Out of packets? */

	if (received < budget)

		virtqueue_napi_complete(napi, rq->vq, received);



	if (xdp_xmit)

		xdp_do_flush_map();



	return received;

}
@@ -2069,6 +2114,8 @@ static const struct net_device_ops virtnet_netdev = { 
	.ndo_poll_controller = virtnet_netpoll,

#endif

	.ndo_xdp		= virtnet_xdp,

	.ndo_xdp_xmit		= virtnet_xdp_xmit,

	.ndo_xdp_flush		= virtnet_xdp_flush,

	.ndo_features_check	= passthru_features_check,

};

CRA Git | Maintained and supported by SUSTech CRA and CCSE