Commit b5aea28d authored by Magnus Karlsson's avatar Magnus Karlsson Committed by Daniel Borkmann
Browse files

xsk: Add shared umem support between queue ids 



Add support to share a umem between queue ids on the same
device. This mode can be invoked with the XDP_SHARED_UMEM bind
flag. Previously, sharing was only supported within the same
queue id and device, and you shared one set of fill and
completion rings. However, note that when sharing a umem between
queue ids, you need to create a fill ring and a completion ring
and tie them to the socket before you do the bind with the
XDP_SHARED_UMEM flag. This so that the single-producer
single-consumer semantics can be upheld.

Signed-off-by: default avatarMagnus Karlsson <magnus.karlsson@intel.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Acked-by: default avatarBjörn Töpel <bjorn.topel@intel.com>
Link: https://lore.kernel.org/bpf/1598603189-32145-12-git-send-email-magnus.karlsson@intel.com
parent 9647c57b

include/net/xsk_buff_pool.h

+2 −0
Original line number Diff line number Diff line
@@ -81,6 +81,8 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs, 
						struct xdp_umem *umem);

int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *dev,

		  u16 queue_id, u16 flags);

int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem,

			 struct net_device *dev, u16 queue_id);

void xp_destroy(struct xsk_buff_pool *pool);

void xp_release(struct xdp_buff_xsk *xskb);

void xp_get_pool(struct xsk_buff_pool *pool);

net/xdp/xsk.c

+30 −14
Original line number Diff line number Diff line
@@ -689,12 +689,6 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len) 
			goto out_unlock;

		}



		if (xs->fq_tmp || xs->cq_tmp) {

			/* Do not allow setting your own fq or cq. */

			err = -EINVAL;

			goto out_unlock;

		}



		sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd);

		if (IS_ERR(sock)) {

			err = PTR_ERR(sock);
@@ -707,15 +701,41 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len) 
			sockfd_put(sock);

			goto out_unlock;

		}

		if (umem_xs->dev != dev || umem_xs->queue_id != qid) {

		if (umem_xs->dev != dev) {

			err = -EINVAL;

			sockfd_put(sock);

			goto out_unlock;

		}



		if (umem_xs->queue_id != qid) {

			/* Share the umem with another socket on another qid */

			xs->pool = xp_create_and_assign_umem(xs,

							     umem_xs->umem);

			if (!xs->pool) {

				sockfd_put(sock);

				goto out_unlock;

			}



			err = xp_assign_dev_shared(xs->pool, umem_xs->umem,

						   dev, qid);

			if (err) {

				xp_destroy(xs->pool);

				sockfd_put(sock);

				goto out_unlock;

			}

		} else {

			/* Share the buffer pool with the other socket. */

			if (xs->fq_tmp || xs->cq_tmp) {

				/* Do not allow setting your own fq or cq. */

				err = -EINVAL;

				sockfd_put(sock);

				goto out_unlock;

			}



			xp_get_pool(umem_xs->pool);

			xs->pool = umem_xs->pool;

		}



		xdp_get_umem(umem_xs->umem);

		WRITE_ONCE(xs->umem, umem_xs->umem);

		sockfd_put(sock);
@@ -847,10 +867,6 @@ static int xsk_setsockopt(struct socket *sock, int level, int optname, 
			mutex_unlock(&xs->mutex);

			return -EBUSY;

		}

		if (!xs->umem) {

			mutex_unlock(&xs->mutex);

			return -EINVAL;

		}



		q = (optname == XDP_UMEM_FILL_RING) ? &xs->fq_tmp :

			&xs->cq_tmp;

net/xdp/xsk_buff_pool.c

+24 −2
Original line number Diff line number Diff line
@@ -123,8 +123,8 @@ static void xp_disable_drv_zc(struct xsk_buff_pool *pool) 
	}

}



int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *netdev,

		  u16 queue_id, u16 flags)

static int __xp_assign_dev(struct xsk_buff_pool *pool,

			   struct net_device *netdev, u16 queue_id, u16 flags)

{

	bool force_zc, force_copy;

	struct netdev_bpf bpf;
@@ -193,6 +193,28 @@ err_unreg_pool: 
	return err;

}



int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *dev,

		  u16 queue_id, u16 flags)

{

	return __xp_assign_dev(pool, dev, queue_id, flags);

}



int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem,

			 struct net_device *dev, u16 queue_id)

{

	u16 flags;



	/* One fill and completion ring required for each queue id. */

	if (!pool->fq || !pool->cq)

		return -EINVAL;



	flags = umem->zc ? XDP_ZEROCOPY : XDP_COPY;

	if (pool->uses_need_wakeup)

		flags |= XDP_USE_NEED_WAKEUP;



	return __xp_assign_dev(pool, dev, queue_id, flags);

}



void xp_clear_dev(struct xsk_buff_pool *pool)

{

	if (!pool->netdev)

CRA Git | Maintained and supported by SUSTech CRA and CCSE