Commit b9673cf5 authored by Maxim Mikityanskiy's avatar Maxim Mikityanskiy Committed by Daniel Borkmann
Browse files

net/mlx5e: Share the XDP SQ for XDP_TX between RQs 



Put the XDP SQ that is used for XDP_TX into the channel. It used to be a
part of the RQ, but with introduction of AF_XDP there will be one more
RQ that could share the same XDP SQ. This patch is a preparation for
that change.

Separate XDP_TX statistics per RQ were implemented in one of the previous
patches.

Signed-off-by: default avatarMaxim Mikityanskiy <maximmi@mellanox.com>
Signed-off-by: default avatarTariq Toukan <tariqt@mellanox.com>
Acked-by: default avatarSaeed Mahameed <saeedm@mellanox.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parent d963fa15

drivers/net/ethernet/mellanox/mlx5/core/en.h

+3 −1
Original line number Diff line number Diff line
@@ -432,6 +432,7 @@ struct mlx5e_xdp_info { 
			dma_addr_t dma_addr;

		} frame;

		struct {

			struct mlx5e_rq *rq;

			struct mlx5e_dma_info di;

		} page;

	};
@@ -644,7 +645,7 @@ struct mlx5e_rq { 


	/* XDP */

	struct bpf_prog       *xdp_prog;

	struct mlx5e_xdpsq     xdpsq;

	struct mlx5e_xdpsq    *xdpsq;

	DECLARE_BITMAP(flags, 8);

	struct page_pool      *page_pool;
@@ -663,6 +664,7 @@ struct mlx5e_rq { 
struct mlx5e_channel {

	/* data path */

	struct mlx5e_rq            rq;

	struct mlx5e_xdpsq         rq_xdpsq;

	struct mlx5e_txqsq         sq[MLX5E_MAX_NUM_TC];

	struct mlx5e_icosq         icosq;   /* internal control operations */

	bool                       xdp;

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c

+10 −10
Original line number Diff line number Diff line
@@ -54,8 +54,8 @@ int mlx5e_xdp_max_mtu(struct mlx5e_params *params) 
}



static inline bool

mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_dma_info *di,

		    struct xdp_buff *xdp)

mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,

		    struct mlx5e_dma_info *di, struct xdp_buff *xdp)

{

	struct mlx5e_xdp_xmit_data xdptxd;

	struct mlx5e_xdp_info xdpi;
@@ -75,6 +75,7 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_dma_info *di, 
	dma_sync_single_for_device(sq->pdev, dma_addr, xdptxd.len, DMA_TO_DEVICE);



	xdptxd.dma_addr = dma_addr;

	xdpi.page.rq = rq;

	xdpi.page.di = *di;



	return sq->xmit_xdp_frame(sq, &xdptxd, &xdpi);
@@ -105,7 +106,7 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di, 
		*len = xdp.data_end - xdp.data;

		return false;

	case XDP_TX:

		if (unlikely(!mlx5e_xmit_xdp_buff(&rq->xdpsq, di, &xdp)))

		if (unlikely(!mlx5e_xmit_xdp_buff(rq->xdpsq, rq, di, &xdp)))

			goto xdp_abort;

		__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */

		return true;
@@ -287,7 +288,6 @@ static bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, 


static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,

				  struct mlx5e_xdp_wqe_info *wi,

				  struct mlx5e_rq *rq,

				  bool recycle)

{

	struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo;
@@ -305,7 +305,7 @@ static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq, 
			break;

		case MLX5E_XDP_XMIT_MODE_PAGE:

			/* XDP_TX */

			mlx5e_page_release(rq, &xdpi.page.di, recycle);

			mlx5e_page_release(xdpi.page.rq, &xdpi.page.di, recycle);

			break;

		default:

			WARN_ON_ONCE(true);
@@ -313,7 +313,7 @@ static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq, 
	}

}



bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq)

bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)

{

	struct mlx5e_xdpsq *sq;

	struct mlx5_cqe64 *cqe;
@@ -358,7 +358,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) 


			sqcc += wi->num_wqebbs;



			mlx5e_free_xdpsq_desc(sq, wi, rq, true);

			mlx5e_free_xdpsq_desc(sq, wi, true);

		} while (!last_wqe);

	} while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
@@ -373,7 +373,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) 
	return (i == MLX5E_TX_CQ_POLL_BUDGET);

}



void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq)

void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq)

{

	while (sq->cc != sq->pc) {

		struct mlx5e_xdp_wqe_info *wi;
@@ -384,7 +384,7 @@ void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq) 


		sq->cc += wi->num_wqebbs;



		mlx5e_free_xdpsq_desc(sq, wi, rq, false);

		mlx5e_free_xdpsq_desc(sq, wi, false);

	}

}
@@ -450,7 +450,7 @@ int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 


void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq)

{

	struct mlx5e_xdpsq *xdpsq = &rq->xdpsq;

	struct mlx5e_xdpsq *xdpsq = rq->xdpsq;



	if (xdpsq->mpwqe.wqe)

		mlx5e_xdp_mpwqe_complete(xdpsq);

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h

+2 −2
Original line number Diff line number Diff line
@@ -42,8 +42,8 @@ 
int mlx5e_xdp_max_mtu(struct mlx5e_params *params);

bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di,

		      void *va, u16 *rx_headroom, u32 *len);

bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq);

void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq);

bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);

void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq);

void mlx5e_set_xmit_fp(struct mlx5e_xdpsq *sq, bool is_mpw);

void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq);

int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,

drivers/net/ethernet/mellanox/mlx5/core/en_main.c

+15 −11
Original line number Diff line number Diff line
@@ -418,6 +418,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, 
	rq->mdev    = mdev;

	rq->hw_mtu  = MLX5E_SW2HW_MTU(params, params->sw_mtu);

	rq->stats   = &c->priv->channel_stats[c->ix].rq;

	rq->xdpsq   = &c->rq_xdpsq;



	rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;

	if (IS_ERR(rq->xdp_prog)) {
@@ -1438,7 +1439,7 @@ err_free_xdpsq: 
	return err;

}



static void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq)

static void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)

{

	struct mlx5e_channel *c = sq->channel;
@@ -1446,7 +1447,7 @@ static void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq) 
	napi_synchronize(&c->napi);



	mlx5e_destroy_sq(c->mdev, sq->sqn);

	mlx5e_free_xdpsq_descs(sq, rq);

	mlx5e_free_xdpsq_descs(sq);

	mlx5e_free_xdpsq(sq);

}
@@ -1825,7 +1826,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, 


	/* XDP SQ CQ params are same as normal TXQ sq CQ params */

	err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation,

				     &cparam->tx_cq, &c->rq.xdpsq.cq) : 0;

				     &cparam->tx_cq, &c->rq_xdpsq.cq) : 0;

	if (err)

		goto err_close_rx_cq;
@@ -1839,9 +1840,12 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, 
	if (err)

		goto err_close_icosq;



	err = c->xdp ? mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->rq.xdpsq, false) : 0;

	if (c->xdp) {

		err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq,

				       &c->rq_xdpsq, false);

		if (err)

			goto err_close_sqs;

	}



	err = mlx5e_open_rq(c, params, &cparam->rq, &c->rq);

	if (err)
@@ -1860,7 +1864,7 @@ err_close_rq: 


err_close_xdp_sq:

	if (c->xdp)

		mlx5e_close_xdpsq(&c->rq.xdpsq, &c->rq);

		mlx5e_close_xdpsq(&c->rq_xdpsq);



err_close_sqs:

	mlx5e_close_sqs(c);
@@ -1871,7 +1875,7 @@ err_close_icosq: 
err_disable_napi:

	napi_disable(&c->napi);

	if (c->xdp)

		mlx5e_close_cq(&c->rq.xdpsq.cq);

		mlx5e_close_cq(&c->rq_xdpsq.cq);



err_close_rx_cq:

	mlx5e_close_cq(&c->rq.cq);
@@ -1916,15 +1920,15 @@ static void mlx5e_deactivate_channel(struct mlx5e_channel *c) 


static void mlx5e_close_channel(struct mlx5e_channel *c)

{

	mlx5e_close_xdpsq(&c->xdpsq, NULL);

	mlx5e_close_xdpsq(&c->xdpsq);

	mlx5e_close_rq(&c->rq);

	if (c->xdp)

		mlx5e_close_xdpsq(&c->rq.xdpsq, &c->rq);

		mlx5e_close_xdpsq(&c->rq_xdpsq);

	mlx5e_close_sqs(c);

	mlx5e_close_icosq(&c->icosq);

	napi_disable(&c->napi);

	if (c->xdp)

		mlx5e_close_cq(&c->rq.xdpsq.cq);

		mlx5e_close_cq(&c->rq_xdpsq.cq);

	mlx5e_close_cq(&c->rq.cq);

	mlx5e_close_cq(&c->xdpsq.cq);

	mlx5e_close_tx_cqs(c);

drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c

+2 −2
Original line number Diff line number Diff line
@@ -97,10 +97,10 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) 
	for (i = 0; i < c->num_tc; i++)

		busy |= mlx5e_poll_tx_cq(&c->sq[i].cq, budget);



	busy |= mlx5e_poll_xdpsq_cq(&c->xdpsq.cq, NULL);

	busy |= mlx5e_poll_xdpsq_cq(&c->xdpsq.cq);



	if (c->xdp)

		busy |= mlx5e_poll_xdpsq_cq(&rq->xdpsq.cq, rq);

		busy |= mlx5e_poll_xdpsq_cq(&c->rq_xdpsq.cq);



	if (likely(budget)) { /* budget=0 means: don't poll rx rings */

		work_done = mlx5e_poll_rx_cq(&rq->cq, budget);

CRA Git | Maintained and supported by SUSTech CRA and CCSE