Commit 93761ca1 authored by Tariq Toukan's avatar Tariq Toukan Committed by Saeed Mahameed
Browse files

net/mlx5e: XDP, Avoid indirect call in TX flow 



Use INDIRECT_CALL_2() helper to avoid the cost of the indirect call
when/if CONFIG_RETPOLINE=y.

Signed-off-by: default avatarTariq Toukan <tariqt@mellanox.com>
Reviewed-by: default avatarMaxim Mikityanskiy <maximmi@mellanox.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@mellanox.com>
parent 7ed92f97

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c

+15 −12
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ 
#include <net/xdp_sock_drv.h>

#include "en/xdp.h"

#include "en/params.h"

#include <linux/indirect_call_wrapper.h>



int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk)

{
@@ -114,7 +115,8 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq, 
		xdpi.page.di    = *di;

	}



	return sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, 0);

	return INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,

			       mlx5e_xmit_xdp_frame, sq, &xdptxd, &xdpi, 0);

}



/* returns true if packet was consumed by xdp */
@@ -237,7 +239,7 @@ enum { 
	MLX5E_XDP_CHECK_START_MPWQE = 2,

};



static int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq)

INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq)

{

	if (unlikely(!sq->mpwqe.wqe)) {

		const u16 stop_room = mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS);
@@ -256,10 +258,9 @@ static int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq) 
	return MLX5E_XDP_CHECK_OK;

}



static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq,

				       struct mlx5e_xdp_xmit_data *xdptxd,

				       struct mlx5e_xdp_info *xdpi,

				       int check_result)

INDIRECT_CALLABLE_SCOPE bool

mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_xmit_data *xdptxd,

			   struct mlx5e_xdp_info *xdpi, int check_result)

{

	struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;

	struct mlx5e_xdpsq_stats *stats = sq->stats;
@@ -293,7 +294,7 @@ static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, 
	return true;

}



static int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq)

INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq)

{

	if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1))) {

		/* SQ is full, ring doorbell */
@@ -305,10 +306,9 @@ static int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq) 
	return MLX5E_XDP_CHECK_OK;

}



static bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,

				 struct mlx5e_xdp_xmit_data *xdptxd,

				 struct mlx5e_xdp_info *xdpi,

				 int check_result)

INDIRECT_CALLABLE_SCOPE bool

mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_xmit_data *xdptxd,

		     struct mlx5e_xdp_info *xdpi, int check_result)

{

	struct mlx5_wq_cyc       *wq   = &sq->wq;

	u16                       pi   = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
@@ -506,6 +506,7 @@ int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 
		struct xdp_frame *xdpf = frames[i];

		struct mlx5e_xdp_xmit_data xdptxd;

		struct mlx5e_xdp_info xdpi;

		bool ret;



		xdptxd.data = xdpf->data;

		xdptxd.len = xdpf->len;
@@ -522,7 +523,9 @@ int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 
		xdpi.frame.xdpf     = xdpf;

		xdpi.frame.dma_addr = xdptxd.dma_addr;



		if (unlikely(!sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, 0))) {

		ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,

				      mlx5e_xmit_xdp_frame, sq, &xdptxd, &xdpi, 0);

		if (unlikely(!ret)) {

			dma_unmap_single(sq->pdev, xdptxd.dma_addr,

					 xdptxd.len, DMA_TO_DEVICE);

			xdp_return_frame_rx_napi(xdpf);

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h

+13 −0
Original line number Diff line number Diff line
@@ -32,6 +32,8 @@ 
#ifndef __MLX5_EN_XDP_H__

#define __MLX5_EN_XDP_H__



#include <linux/indirect_call_wrapper.h>



#include "en.h"

#include "en/txrx.h"
@@ -70,6 +72,17 @@ void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq); 
int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,

		   u32 flags);



INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq,

							  struct mlx5e_xdp_xmit_data *xdptxd,

							  struct mlx5e_xdp_info *xdpi,

							  int check_result));

INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,

						    struct mlx5e_xdp_xmit_data *xdptxd,

						    struct mlx5e_xdp_info *xdpi,

						    int check_result));

INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq));

INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq));



static inline void mlx5e_xdp_tx_enable(struct mlx5e_priv *priv)

{

	set_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state);

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c

+9 −2
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 
#include "en/xdp.h"

#include "en/params.h"

#include <net/xdp_sock_drv.h>

#include <linux/indirect_call_wrapper.h>



int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)

{
@@ -75,8 +76,12 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget) 
	xdpi.mode = MLX5E_XDP_XMIT_MODE_XSK;



	for (; budget; budget--) {

		int check_result = sq->xmit_xdp_frame_check(sq);

		int check_result = INDIRECT_CALL_2(sq->xmit_xdp_frame_check,

						   mlx5e_xmit_xdp_frame_check_mpwqe,

						   mlx5e_xmit_xdp_frame_check,

						   sq);

		struct xdp_desc desc;

		bool ret;



		if (unlikely(check_result < 0)) {

			work_done = false;
@@ -98,7 +103,9 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget) 


		xsk_buff_raw_dma_sync_for_device(umem, xdptxd.dma_addr, xdptxd.len);



		if (unlikely(!sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, check_result))) {

		ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,

				      mlx5e_xmit_xdp_frame, sq, &xdptxd, &xdpi, check_result);

		if (unlikely(!ret)) {

			if (sq->mpwqe.wqe)

				mlx5e_xdp_mpwqe_complete(sq);

CRA Git | Maintained and supported by SUSTech CRA and CCSE