Commit c497176c authored by Björn Töpel's avatar Björn Töpel Committed by Alexei Starovoitov
Browse files

xsk: add Rx receive functions and poll support 



Here the actual receive functions of AF_XDP are implemented, that in a
later commit, will be called from the XDP layers.

There's one set of functions for the XDP_DRV side and another for
XDP_SKB (generic).

A new XDP API, xdp_return_buff, is also introduced.

Adding xdp_return_buff, which is analogous to xdp_return_frame, but
acts upon an struct xdp_buff. The API will be used by AF_XDP in future
commits.

Support for the poll syscall is also implemented.

v2: xskq_validate_id did not update cons_tail.
    The entries variable was calculated twice in xskq_nb_avail.
    Squashed xdp_return_buff commit.

Signed-off-by: default avatarBjörn Töpel <bjorn.topel@intel.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 965a9909

include/net/xdp.h

+1 −0
Original line number Diff line number Diff line
@@ -104,6 +104,7 @@ struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp) 
}



void xdp_return_frame(struct xdp_frame *xdpf);

void xdp_return_buff(struct xdp_buff *xdp);



int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,

		     struct net_device *dev, u32 queue_index);

include/net/xdp_sock.h

+22 −0
Original line number Diff line number Diff line
@@ -31,6 +31,28 @@ struct xdp_sock { 
	u16 queue_id;

	/* Protects multiple processes in the control path */

	struct mutex mutex;

	u64 rx_dropped;

};



struct xdp_buff;

#ifdef CONFIG_XDP_SOCKETS

int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);

int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);

void xsk_flush(struct xdp_sock *xs);

#else

static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)

{

	return -ENOTSUPP;

}



static inline int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)

{

	return -ENOTSUPP;

}



static inline void xsk_flush(struct xdp_sock *xs)

{

}

#endif /* CONFIG_XDP_SOCKETS */



#endif /* _LINUX_XDP_SOCK_H */

net/core/xdp.c

+12 −3
Original line number Diff line number Diff line
@@ -308,11 +308,9 @@ err: 
}

EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model);



void xdp_return_frame(struct xdp_frame *xdpf)

static void xdp_return(void *data, struct xdp_mem_info *mem)

{

	struct xdp_mem_info *mem = &xdpf->mem;

	struct xdp_mem_allocator *xa;

	void *data = xdpf->data;

	struct page *page;



	switch (mem->type) {
@@ -339,4 +337,15 @@ void xdp_return_frame(struct xdp_frame *xdpf) 
		break;

	}

}



void xdp_return_frame(struct xdp_frame *xdpf)

{

	xdp_return(xdpf->data, &xdpf->mem);

}

EXPORT_SYMBOL_GPL(xdp_return_frame);



void xdp_return_buff(struct xdp_buff *xdp)

{

	xdp_return(xdp->data, &xdp->rxq->mem);

}

EXPORT_SYMBOL_GPL(xdp_return_buff);

net/xdp/xdp_umem.h

+18 −0
Original line number Diff line number Diff line
@@ -39,6 +39,24 @@ struct xdp_umem { 
	struct work_struct work;

};



static inline char *xdp_umem_get_data(struct xdp_umem *umem, u32 idx)

{

	u64 pg, off;

	char *data;



	pg = idx >> umem->nfpplog2;

	off = (idx & umem->nfpp_mask) << umem->frame_size_log2;



	data = page_address(umem->pgs[pg]);

	return data + off;

}



static inline char *xdp_umem_get_data_with_headroom(struct xdp_umem *umem,

						    u32 idx)

{

	return xdp_umem_get_data(umem, idx) + umem->frame_headroom;

}



bool xdp_umem_validate_queues(struct xdp_umem *umem);

int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr);

void xdp_get_umem(struct xdp_umem *umem);

net/xdp/xsk.c

+72 −1
Original line number Diff line number Diff line
@@ -41,6 +41,74 @@ static struct xdp_sock *xdp_sk(struct sock *sk) 
	return (struct xdp_sock *)sk;

}



static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)

{

	u32 *id, len = xdp->data_end - xdp->data;

	void *buffer;

	int err = 0;



	if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)

		return -EINVAL;



	id = xskq_peek_id(xs->umem->fq);

	if (!id)

		return -ENOSPC;



	buffer = xdp_umem_get_data_with_headroom(xs->umem, *id);

	memcpy(buffer, xdp->data, len);

	err = xskq_produce_batch_desc(xs->rx, *id, len,

				      xs->umem->frame_headroom);

	if (!err)

		xskq_discard_id(xs->umem->fq);



	return err;

}



int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)

{

	int err;



	err = __xsk_rcv(xs, xdp);

	if (likely(!err))

		xdp_return_buff(xdp);

	else

		xs->rx_dropped++;



	return err;

}



void xsk_flush(struct xdp_sock *xs)

{

	xskq_produce_flush_desc(xs->rx);

	xs->sk.sk_data_ready(&xs->sk);

}



int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)

{

	int err;



	err = __xsk_rcv(xs, xdp);

	if (!err)

		xsk_flush(xs);

	else

		xs->rx_dropped++;



	return err;

}



static unsigned int xsk_poll(struct file *file, struct socket *sock,

			     struct poll_table_struct *wait)

{

	unsigned int mask = datagram_poll(file, sock, wait);

	struct sock *sk = sock->sk;

	struct xdp_sock *xs = xdp_sk(sk);



	if (xs->rx && !xskq_empty_desc(xs->rx))

		mask |= POLLIN | POLLRDNORM;



	return mask;

}



static int xsk_init_queue(u32 entries, struct xsk_queue **queue,

			  bool umem_queue)

{
@@ -179,6 +247,9 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len) 
	} else if (!xs->umem || !xdp_umem_validate_queues(xs->umem)) {

		err = -EINVAL;

		goto out_unlock;

	} else {

		/* This xsk has its own umem. */

		xskq_set_umem(xs->umem->fq, &xs->umem->props);

	}



	/* Rebind? */
@@ -330,7 +401,7 @@ static const struct proto_ops xsk_proto_ops = { 
	.socketpair =	sock_no_socketpair,

	.accept =	sock_no_accept,

	.getname =	sock_no_getname,

	.poll =		sock_no_poll,

	.poll =		xsk_poll,

	.ioctl =	sock_no_ioctl,

	.listen =	sock_no_listen,

	.shutdown =	sock_no_shutdown,

CRA Git | Maintained and supported by SUSTech CRA and CCSE