Commit 3b1c667e authored by Daniel Borkmann's avatar Daniel Borkmann
Browse files

Merge branch 'bpf-af-xdp-mlx5e' 



Tariq Toukan says:

====================
This series contains improvements to the AF_XDP kernel infrastructure
and AF_XDP support in mlx5e. The infrastructure improvements are
required for mlx5e, but also some of them benefit to all drivers, and
some can be useful for other drivers that want to implement AF_XDP.

The performance testing was performed on a machine with the following
configuration:

- 24 cores of Intel Xeon E5-2620 v3 @ 2.40 GHz
- Mellanox ConnectX-5 Ex with 100 Gbit/s link

The results with retpoline disabled, single stream:

txonly: 33.3 Mpps (21.5 Mpps with queue and app pinned to the same CPU)
rxdrop: 12.2 Mpps
l2fwd: 9.4 Mpps

The results with retpoline enabled, single stream:

txonly: 21.3 Mpps (14.1 Mpps with queue and app pinned to the same CPU)
rxdrop: 9.9 Mpps
l2fwd: 6.8 Mpps

v2 changes:

Added patches for mlx5e and addressed the comments for v1. Rebased for
bpf-next.

v3 changes:

Rebased for the newer bpf-next, resolved conflicts in libbpf. Addressed
Björn's comments for coding style. Fixed a bug in error handling flow in
mlx5e_open_xsk.

v4 changes:

UAPI is not changed, XSK RX queues are exposed to the kernel. The lower
half of the available amount of RX queues are regular queues, and the
upper half are XSK RX queues. The patch "xsk: Extend channels to support
combined XSK/non-XSK traffic" was dropped. The final patch was reworked
accordingly.

Added "net/mlx5e: Attach/detach XDP program safely", as the changes
introduced in the XSK patch base on the stuff from this one.

Added "libbpf: Support drivers with non-combined channels", which aligns
the condition in libbpf with the condition in the kernel.

Rebased over the newer bpf-next.

v5 changes:

In v4, ethtool reports the number of channels as 'combined' and the
number of XSK RX queues as 'rx' for mlx5e. It was changed, so that 'rx'
is 0, and 'combined' reports the double amount of channels if there is
an active UMEM - to make libbpf happy.

The patch for libbpf was dropped. Although it's still useful and fixes
things, it raises some disagreement, so I'm dropping it - it's no longer
useful for mlx5e anymore after the change above.

v6 changes:

As Maxim is out of office, I rebased the series on behalf of him,
solved some conflicts, and re-spinned.
====================

Acked-by: default avatarBjörn Töpel <bjorn.topel@intel.com>
Tested-by: default avatarJonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parents e5c891a3 db05815b

drivers/net/ethernet/intel/i40e/i40e_xsk.c

+7 −5
Original line number Diff line number Diff line
@@ -641,8 +641,8 @@ static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget) 
	struct i40e_tx_desc *tx_desc = NULL;

	struct i40e_tx_buffer *tx_bi;

	bool work_done = true;

	struct xdp_desc desc;

	dma_addr_t dma;

	u32 len;



	while (budget-- > 0) {

		if (!unlikely(I40E_DESC_UNUSED(xdp_ring))) {
@@ -651,21 +651,23 @@ static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget) 
			break;

		}



		if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &dma, &len))

		if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc))

			break;



		dma_sync_single_for_device(xdp_ring->dev, dma, len,

		dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr);



		dma_sync_single_for_device(xdp_ring->dev, dma, desc.len,

					   DMA_BIDIRECTIONAL);



		tx_bi = &xdp_ring->tx_bi[xdp_ring->next_to_use];

		tx_bi->bytecount = len;

		tx_bi->bytecount = desc.len;



		tx_desc = I40E_TX_DESC(xdp_ring, xdp_ring->next_to_use);

		tx_desc->buffer_addr = cpu_to_le64(dma);

		tx_desc->cmd_type_offset_bsz =

			build_ctob(I40E_TX_DESC_CMD_ICRC

				   | I40E_TX_DESC_CMD_EOP,

				   0, len, 0);

				   0, desc.len, 0);



		xdp_ring->next_to_use++;

		if (xdp_ring->next_to_use == xdp_ring->count)

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c

+9 −6
Original line number Diff line number Diff line
@@ -571,8 +571,9 @@ static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) 
	union ixgbe_adv_tx_desc *tx_desc = NULL;

	struct ixgbe_tx_buffer *tx_bi;

	bool work_done = true;

	u32 len, cmd_type;

	struct xdp_desc desc;

	dma_addr_t dma;

	u32 cmd_type;



	while (budget-- > 0) {

		if (unlikely(!ixgbe_desc_unused(xdp_ring)) ||
@@ -581,14 +582,16 @@ static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) 
			break;

		}



		if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &dma, &len))

		if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc))

			break;



		dma_sync_single_for_device(xdp_ring->dev, dma, len,

		dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr);



		dma_sync_single_for_device(xdp_ring->dev, dma, desc.len,

					   DMA_BIDIRECTIONAL);



		tx_bi = &xdp_ring->tx_buffer_info[xdp_ring->next_to_use];

		tx_bi->bytecount = len;

		tx_bi->bytecount = desc.len;

		tx_bi->xdpf = NULL;

		tx_bi->gso_segs = 1;
@@ -599,10 +602,10 @@ static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) 
		cmd_type = IXGBE_ADVTXD_DTYP_DATA |

			   IXGBE_ADVTXD_DCMD_DEXT |

			   IXGBE_ADVTXD_DCMD_IFCS;

		cmd_type |= len | IXGBE_TXD_CMD;

		cmd_type |= desc.len | IXGBE_TXD_CMD;

		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);

		tx_desc->read.olinfo_status =

			cpu_to_le32(len << IXGBE_ADVTXD_PAYLEN_SHIFT);

			cpu_to_le32(desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT);



		xdp_ring->next_to_use++;

		if (xdp_ring->next_to_use == xdp_ring->count)

drivers/net/ethernet/mellanox/mlx5/core/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -24,7 +24,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ 
mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \

		en_tx.o en_rx.o en_dim.o en_txrx.o en/xdp.o en_stats.o \

		en_selftest.o en/port.o en/monitor_stats.o en/reporter_tx.o \

		en/params.o

		en/params.o en/xsk/umem.o en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o



#

# Netdev extra

drivers/net/ethernet/mellanox/mlx5/core/en.h

+140 −15
Original line number Diff line number Diff line
@@ -137,6 +137,7 @@ struct page_pool; 
#define MLX5E_MAX_NUM_CHANNELS         (MLX5E_INDIR_RQT_SIZE >> 1)

#define MLX5E_MAX_NUM_SQS              (MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC)

#define MLX5E_TX_CQ_POLL_BUDGET        128

#define MLX5E_TX_XSK_POLL_BUDGET       64

#define MLX5E_SQ_RECOVER_MIN_INTERVAL  500 /* msecs */



#define MLX5E_UMR_WQE_INLINE_SZ \
@@ -155,6 +156,11 @@ do { \ 
			    ##__VA_ARGS__);                     \

} while (0)



enum mlx5e_rq_group {

	MLX5E_RQ_GROUP_REGULAR,

	MLX5E_RQ_GROUP_XSK,

	MLX5E_NUM_RQ_GROUPS /* Keep last. */

};



static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size)

{
@@ -179,7 +185,8 @@ static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) 
/* Use this function to get max num channels after netdev was created */

static inline int mlx5e_get_netdev_max_channels(struct net_device *netdev)

{

	return min_t(unsigned int, netdev->num_rx_queues,

	return min_t(unsigned int,

		     netdev->num_rx_queues / MLX5E_NUM_RQ_GROUPS,

		     netdev->num_tx_queues);

}
@@ -250,6 +257,7 @@ struct mlx5e_params { 
	u32 lro_timeout;

	u32 pflags;

	struct bpf_prog *xdp_prog;

	struct mlx5e_xsk *xsk;

	unsigned int sw_mtu;

	int hard_mtu;

};
@@ -348,6 +356,13 @@ enum { 


struct mlx5e_sq_wqe_info {

	u8  opcode;



	/* Auxiliary data for different opcodes. */

	union {

		struct {

			struct mlx5e_rq *rq;

		} umr;

	};

};



struct mlx5e_txqsq {
@@ -392,14 +407,55 @@ struct mlx5e_txqsq { 
} ____cacheline_aligned_in_smp;



struct mlx5e_dma_info {

	struct page     *page;

	dma_addr_t addr;

	union {

		struct page *page;

		struct {

			u64 handle;

			void *data;

		} xsk;

	};

};



/* XDP packets can be transmitted in different ways. On completion, we need to

 * distinguish between them to clean up things in a proper way.

 */

enum mlx5e_xdp_xmit_mode {

	/* An xdp_frame was transmitted due to either XDP_REDIRECT from another

	 * device or XDP_TX from an XSK RQ. The frame has to be unmapped and

	 * returned.

	 */

	MLX5E_XDP_XMIT_MODE_FRAME,



	/* The xdp_frame was created in place as a result of XDP_TX from a

	 * regular RQ. No DMA remapping happened, and the page belongs to us.

	 */

	MLX5E_XDP_XMIT_MODE_PAGE,



	/* No xdp_frame was created at all, the transmit happened from a UMEM

	 * page. The UMEM Completion Ring producer pointer has to be increased.

	 */

	MLX5E_XDP_XMIT_MODE_XSK,

};



struct mlx5e_xdp_info {

	enum mlx5e_xdp_xmit_mode mode;

	union {

		struct {

			struct xdp_frame *xdpf;

			dma_addr_t dma_addr;

		} frame;

		struct {

			struct mlx5e_rq *rq;

			struct mlx5e_dma_info di;

		} page;

	};

};



struct mlx5e_xdp_xmit_data {

	dma_addr_t  dma_addr;

	void       *data;

	u32         len;

};



struct mlx5e_xdp_info_fifo {
@@ -425,8 +481,12 @@ struct mlx5e_xdp_mpwqe { 
};



struct mlx5e_xdpsq;

typedef int (*mlx5e_fp_xmit_xdp_frame_check)(struct mlx5e_xdpsq *);

typedef bool (*mlx5e_fp_xmit_xdp_frame)(struct mlx5e_xdpsq *,

					struct mlx5e_xdp_info*);

					struct mlx5e_xdp_xmit_data *,

					struct mlx5e_xdp_info *,

					int);



struct mlx5e_xdpsq {

	/* data path */
@@ -443,8 +503,10 @@ struct mlx5e_xdpsq { 
	struct mlx5e_cq            cq;



	/* read only */

	struct xdp_umem           *umem;

	struct mlx5_wq_cyc         wq;

	struct mlx5e_xdpsq_stats  *stats;

	mlx5e_fp_xmit_xdp_frame_check xmit_xdp_frame_check;

	mlx5e_fp_xmit_xdp_frame    xmit_xdp_frame;

	struct {

		struct mlx5e_xdp_wqe_info *wqe_info;
@@ -571,9 +633,11 @@ struct mlx5e_rq { 
			u8                     log_stride_sz;

			u8                     umr_in_progress;

			u8                     umr_last_bulk;

			u8                     umr_completed;

		} mpwqe;

	};

	struct {

		u16            umem_headroom;

		u16            headroom;

		u8             map_dir;   /* dma map direction */

	} buff;
@@ -600,10 +664,14 @@ struct mlx5e_rq { 


	/* XDP */

	struct bpf_prog       *xdp_prog;

	struct mlx5e_xdpsq     xdpsq;

	struct mlx5e_xdpsq    *xdpsq;

	DECLARE_BITMAP(flags, 8);

	struct page_pool      *page_pool;



	/* AF_XDP zero-copy */

	struct zero_copy_allocator zca;

	struct xdp_umem       *umem;



	/* control */

	struct mlx5_wq_ctrl    wq_ctrl;

	__be32                 mkey_be;
@@ -616,9 +684,15 @@ struct mlx5e_rq { 
	struct xdp_rxq_info    xdp_rxq;

} ____cacheline_aligned_in_smp;



enum mlx5e_channel_state {

	MLX5E_CHANNEL_STATE_XSK,

	MLX5E_CHANNEL_NUM_STATES

};



struct mlx5e_channel {

	/* data path */

	struct mlx5e_rq            rq;

	struct mlx5e_xdpsq         rq_xdpsq;

	struct mlx5e_txqsq         sq[MLX5E_MAX_NUM_TC];

	struct mlx5e_icosq         icosq;   /* internal control operations */

	bool                       xdp;
@@ -631,6 +705,13 @@ struct mlx5e_channel { 
	/* XDP_REDIRECT */

	struct mlx5e_xdpsq         xdpsq;



	/* AF_XDP zero-copy */

	struct mlx5e_rq            xskrq;

	struct mlx5e_xdpsq         xsksq;

	struct mlx5e_icosq         xskicosq;

	/* xskicosq can be accessed from any CPU - the spinlock protects it. */

	spinlock_t                 xskicosq_lock;



	/* data path - accessed per napi poll */

	struct irq_desc *irq_desc;

	struct mlx5e_ch_stats     *stats;
@@ -639,6 +720,7 @@ struct mlx5e_channel { 
	struct mlx5e_priv         *priv;

	struct mlx5_core_dev      *mdev;

	struct hwtstamp_config    *tstamp;

	DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES);

	int                        ix;

	int                        cpu;

	cpumask_var_t              xps_cpumask;
@@ -654,14 +736,17 @@ struct mlx5e_channel_stats { 
	struct mlx5e_ch_stats ch;

	struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC];

	struct mlx5e_rq_stats rq;

	struct mlx5e_rq_stats xskrq;

	struct mlx5e_xdpsq_stats rq_xdpsq;

	struct mlx5e_xdpsq_stats xdpsq;

	struct mlx5e_xdpsq_stats xsksq;

} ____cacheline_aligned_in_smp;



enum {

	MLX5E_STATE_OPENED,

	MLX5E_STATE_DESTROYING,

	MLX5E_STATE_XDP_TX_ENABLED,

	MLX5E_STATE_XDP_OPEN,

};



struct mlx5e_rqt {
@@ -694,6 +779,17 @@ struct mlx5e_modify_sq_param { 
	int rl_index;

};



struct mlx5e_xsk {

	/* UMEMs are stored separately from channels, because we don't want to

	 * lose them when channels are recreated. The kernel also stores UMEMs,

	 * but it doesn't distinguish between zero-copy and non-zero-copy UMEMs,

	 * so rely on our mechanism.

	 */

	struct xdp_umem **umems;

	u16 refcnt;

	bool ever_used;

};



struct mlx5e_priv {

	/* priv data path fields - start */

	struct mlx5e_txqsq *txq2sq[MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC];
@@ -714,6 +810,7 @@ struct mlx5e_priv { 
	struct mlx5e_tir           indir_tir[MLX5E_NUM_INDIR_TIRS];

	struct mlx5e_tir           inner_indir_tir[MLX5E_NUM_INDIR_TIRS];

	struct mlx5e_tir           direct_tir[MLX5E_MAX_NUM_CHANNELS];

	struct mlx5e_tir           xsk_tir[MLX5E_MAX_NUM_CHANNELS];

	struct mlx5e_rss_params    rss_params;

	u32                        tx_rates[MLX5E_MAX_NUM_SQS];
@@ -750,6 +847,7 @@ struct mlx5e_priv { 
	struct mlx5e_tls          *tls;

#endif

	struct devlink_health_reporter *tx_reporter;

	struct mlx5e_xsk           xsk;

};



struct mlx5e_profile {
@@ -793,11 +891,13 @@ bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev, 
				struct mlx5e_params *params);



void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info);

void mlx5e_page_release(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info,

void mlx5e_page_release_dynamic(struct mlx5e_rq *rq,

				struct mlx5e_dma_info *dma_info,

				bool recycle);

void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);

void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);

bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq);

void mlx5e_poll_ico_cq(struct mlx5e_cq *cq);

bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq);

void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);

void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
@@ -853,6 +953,30 @@ void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_rss_params *rss_params, 
void mlx5e_modify_tirs_hash(struct mlx5e_priv *priv, void *in, int inlen);

struct mlx5e_tirc_config mlx5e_tirc_get_default_config(enum mlx5e_traffic_types tt);



struct mlx5e_xsk_param;



struct mlx5e_rq_param;

int mlx5e_open_rq(struct mlx5e_channel *c, struct mlx5e_params *params,

		  struct mlx5e_rq_param *param, struct mlx5e_xsk_param *xsk,

		  struct xdp_umem *umem, struct mlx5e_rq *rq);

int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time);

void mlx5e_deactivate_rq(struct mlx5e_rq *rq);

void mlx5e_close_rq(struct mlx5e_rq *rq);



struct mlx5e_sq_param;

int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params,

		     struct mlx5e_sq_param *param, struct mlx5e_icosq *sq);

void mlx5e_close_icosq(struct mlx5e_icosq *sq);

int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,

		     struct mlx5e_sq_param *param, struct xdp_umem *umem,

		     struct mlx5e_xdpsq *sq, bool is_redirect);

void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq);



struct mlx5e_cq_param;

int mlx5e_open_cq(struct mlx5e_channel *c, struct net_dim_cq_moder moder,

		  struct mlx5e_cq_param *param, struct mlx5e_cq *cq);

void mlx5e_close_cq(struct mlx5e_cq *cq);



int mlx5e_open_locked(struct net_device *netdev);

int mlx5e_close_locked(struct net_device *netdev);
@@ -1023,10 +1147,10 @@ int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv); 
int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc);

void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc);



int mlx5e_create_direct_rqts(struct mlx5e_priv *priv);

void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv);

int mlx5e_create_direct_tirs(struct mlx5e_priv *priv);

void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv);

int mlx5e_create_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs);

void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs);

int mlx5e_create_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs);

void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs);

void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt);



int mlx5e_create_tis(struct mlx5_core_dev *mdev, int tc,
@@ -1095,6 +1219,7 @@ void mlx5e_detach_netdev(struct mlx5e_priv *priv); 
void mlx5e_destroy_netdev(struct mlx5e_priv *priv);

void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv);

void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,

			    struct mlx5e_xsk *xsk,

			    struct mlx5e_rss_params *rss_params,

			    struct mlx5e_params *params,

			    u16 max_channels, u16 mtu);

drivers/net/ethernet/mellanox/mlx5/core/en/params.c

+71 −37
Original line number Diff line number Diff line
@@ -3,65 +3,102 @@ 


#include "en/params.h"



u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params)

static inline bool mlx5e_rx_is_xdp(struct mlx5e_params *params,

				   struct mlx5e_xsk_param *xsk)

{

	u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);

	u16 linear_rq_headroom = params->xdp_prog ?

		XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;

	u32 frag_sz;

	return params->xdp_prog || xsk;

}



u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,

				 struct mlx5e_xsk_param *xsk)

{

	u16 headroom = NET_IP_ALIGN;



	if (mlx5e_rx_is_xdp(params, xsk)) {

		headroom += XDP_PACKET_HEADROOM;

		if (xsk)

			headroom += xsk->headroom;

	} else {

		headroom += MLX5_RX_HEADROOM;

	}



	return headroom;

}



u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params,

				struct mlx5e_xsk_param *xsk)

{

	u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);

	u16 linear_rq_headroom = mlx5e_get_linear_rq_headroom(params, xsk);

	u32 frag_sz = linear_rq_headroom + hw_mtu;



	linear_rq_headroom += NET_IP_ALIGN;

	/* AF_XDP doesn't build SKBs in place. */

	if (!xsk)

		frag_sz = MLX5_SKB_FRAG_SZ(frag_sz);



	frag_sz = MLX5_SKB_FRAG_SZ(linear_rq_headroom + hw_mtu);

	/* XDP in mlx5e doesn't support multiple packets per page. */

	if (mlx5e_rx_is_xdp(params, xsk))

		frag_sz = max_t(u32, frag_sz, PAGE_SIZE);



	if (params->xdp_prog && frag_sz < PAGE_SIZE)

		frag_sz = PAGE_SIZE;

	/* Even if we can go with a smaller fragment size, we must not put

	 * multiple packets into a single frame.

	 */

	if (xsk)

		frag_sz = max_t(u32, frag_sz, xsk->chunk_size);



	return frag_sz;

}



u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params)

u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params,

				struct mlx5e_xsk_param *xsk)

{

	u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params);

	u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params, xsk);



	return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);

}



bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params)

bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params,

			    struct mlx5e_xsk_param *xsk)

{

	u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);

	/* AF_XDP allocates SKBs on XDP_PASS - ensure they don't occupy more

	 * than one page. For this, check both with and without xsk.

	 */

	u32 linear_frag_sz = max(mlx5e_rx_get_linear_frag_sz(params, xsk),

				 mlx5e_rx_get_linear_frag_sz(params, NULL));



	return !params->lro_en && frag_sz <= PAGE_SIZE;

	return !params->lro_en && linear_frag_sz <= PAGE_SIZE;

}



#define MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ ((BIT(__mlx5_bit_sz(wq, log_wqe_stride_size)) - 1) + \

					  MLX5_MPWQE_LOG_STRIDE_SZ_BASE)

bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,

				  struct mlx5e_params *params)

				  struct mlx5e_params *params,

				  struct mlx5e_xsk_param *xsk)

{

	u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);

	u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params, xsk);

	s8 signed_log_num_strides_param;

	u8 log_num_strides;



	if (!mlx5e_rx_is_linear_skb(params))

	if (!mlx5e_rx_is_linear_skb(params, xsk))

		return false;



	if (order_base_2(frag_sz) > MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ)

	if (order_base_2(linear_frag_sz) > MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ)

		return false;



	if (MLX5_CAP_GEN(mdev, ext_stride_num_range))

		return true;



	log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(frag_sz);

	log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);

	signed_log_num_strides_param =

		(s8)log_num_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE;



	return signed_log_num_strides_param >= 0;

}



u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params)

u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params,

			       struct mlx5e_xsk_param *xsk)

{

	u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params);

	u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params, xsk);



	/* Numbers are unsigned, don't subtract to avoid underflow. */

	if (params->log_rq_mtu_frames <
@@ -72,33 +109,30 @@ u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params) 
}



u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,

				   struct mlx5e_params *params)

				   struct mlx5e_params *params,

				   struct mlx5e_xsk_param *xsk)

{

	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params))

		return order_base_2(mlx5e_rx_get_linear_frag_sz(params));

	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))

		return order_base_2(mlx5e_rx_get_linear_frag_sz(params, xsk));



	return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);

}



u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,

				   struct mlx5e_params *params)

				   struct mlx5e_params *params,

				   struct mlx5e_xsk_param *xsk)

{

	return MLX5_MPWRQ_LOG_WQE_SZ -

		mlx5e_mpwqe_get_log_stride_size(mdev, params);

		mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);

}



u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,

			  struct mlx5e_params *params)

			  struct mlx5e_params *params,

			  struct mlx5e_xsk_param *xsk)

{

	u16 linear_rq_headroom = params->xdp_prog ?

		XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;

	bool is_linear_skb;



	linear_rq_headroom += NET_IP_ALIGN;



	is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ?

		mlx5e_rx_is_linear_skb(params) :

		mlx5e_rx_mpwqe_is_linear_skb(mdev, params);

	bool is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ?

		mlx5e_rx_is_linear_skb(params, xsk) :

		mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk);



	return is_linear_skb ? linear_rq_headroom : 0;

	return is_linear_skb ? mlx5e_get_linear_rq_headroom(params, xsk) : 0;

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE