Commit 874fcf1d authored by David S. Miller's avatar David S. Miller
Browse files

Merge tag 'mlx5e-updates-2018-05-25' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux 



Saeed Mahameed says:

====================
mlx5e-updates-2018-05-25

This series includes updates for mlx5e netdev driver.

1) Allowr flow based VF vport mirroring under sriov switchdev scheme,
added support for offloading the TC mirred mirror sub-action, from
Chris Mi.

=================
From: Or Gerlitz <ogerlitz@mellanox.com>

The user will typically set the actions order such that the mirror
port (mirror VF) sees packets as the original port (VF under
mirroring) sent them or as it will receive them. In the general case,
it means that packets are potentially sent to the mirror port before
or after some actions were applied on them.

To properly do that, we follow on the exact action order as set for
the flow and make sure this will also be the case when we program the
HW offload.

If all the actions should apply before forwarding to the mirror and dest port,
mirroring is just multicasting to the two vports. Otherwise, we split
the TC flow to two HW rules, where the 1st applies only the actions
needed up to the mirror (if there are such) and the 2nd the rest of
the actions plus the forwarding to the dest vport.
=================

2) Move to order-0 only allocations (using fragmented work queues) for all
work queues used by the driver, RX and TX descriptor rings
(RQs, SQs and Completion Queues (CQs)), from Tariq Toukan.

3) Avoid resetting netdevice statistics on netdevice
state changes, from Eran Ben Elisha.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents d71dbdaa 05909bab

drivers/net/ethernet/mellanox/mlx5/core/en.h

+19 −11
Original line number Diff line number Diff line
@@ -183,6 +183,7 @@ static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) 
struct mlx5e_tx_wqe {

	struct mlx5_wqe_ctrl_seg ctrl;

	struct mlx5_wqe_eth_seg  eth;

	struct mlx5_wqe_data_seg data[0];

};



struct mlx5e_rx_wqe {
@@ -313,7 +314,7 @@ struct mlx5e_cq { 


	/* control */

	struct mlx5_core_dev      *mdev;

	struct mlx5_frag_wq_ctrl   wq_ctrl;

	struct mlx5_wq_ctrl        wq_ctrl;

} ____cacheline_aligned_in_smp;



struct mlx5e_tx_wqe_info {
@@ -357,7 +358,6 @@ struct mlx5e_txqsq { 
	/* dirtied @xmit */

	u16                        pc ____cacheline_aligned_in_smp;

	u32                        dma_fifo_pc;

	struct mlx5e_sq_stats      stats;



	struct mlx5e_cq            cq;
@@ -370,11 +370,11 @@ struct mlx5e_txqsq { 
	/* read only */

	struct mlx5_wq_cyc         wq;

	u32                        dma_fifo_mask;

	struct mlx5e_sq_stats     *stats;

	void __iomem              *uar_map;

	struct netdev_queue       *txq;

	u32                        sqn;

	u8                         min_inline_mode;

	u16                        edge;

	struct device             *pdev;

	__be32                     mkey_be;

	unsigned long              state;
@@ -439,7 +439,6 @@ struct mlx5e_icosq { 
	struct mlx5_wq_cyc         wq;

	void __iomem              *uar_map;

	u32                        sqn;

	u16                        edge;

	unsigned long              state;



	/* control path */
@@ -450,7 +449,7 @@ struct mlx5e_icosq { 
static inline bool

mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n)

{

	return (((wq->sz_m1 & (cc - pc)) >= n) || (cc == pc));

	return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) || (cc == pc);

}



struct mlx5e_dma_info {
@@ -527,7 +526,7 @@ struct mlx5e_rq { 
	struct mlx5e_channel  *channel;

	struct device         *pdev;

	struct net_device     *netdev;

	struct mlx5e_rq_stats  stats;

	struct mlx5e_rq_stats *stats;

	struct mlx5e_cq        cq;

	struct mlx5e_page_cache page_cache;

	struct hwtstamp_config *tstamp;
@@ -575,7 +574,7 @@ struct mlx5e_channel { 


	/* data path - accessed per napi poll */

	struct irq_desc *irq_desc;

	struct mlx5e_ch_stats      stats;

	struct mlx5e_ch_stats     *stats;



	/* control */

	struct mlx5e_priv         *priv;
@@ -591,6 +590,12 @@ struct mlx5e_channels { 
	struct mlx5e_params    params;

};



struct mlx5e_channel_stats {

	struct mlx5e_ch_stats ch;

	struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC];

	struct mlx5e_rq_stats rq;

} ____cacheline_aligned_in_smp;



enum mlx5e_traffic_types {

	MLX5E_TT_IPV4_TCP,

	MLX5E_TT_IPV6_TCP,
@@ -772,6 +777,8 @@ struct mlx5e_priv { 
	struct mutex               state_lock; /* Protects Interface state */

	struct mlx5e_rq            drop_rq;



	rwlock_t                   stats_lock; /* Protects channels SW stats updates */

	bool                       channels_active;

	struct mlx5e_channels      channels;

	u32                        tisn[MLX5E_MAX_NUM_TC];

	struct mlx5e_rqt           indir_rqt;
@@ -792,6 +799,8 @@ struct mlx5e_priv { 
	struct mlx5_core_dev      *mdev;

	struct net_device         *netdev;

	struct mlx5e_stats         stats;

	struct mlx5e_channel_stats channel_stats[MLX5E_MAX_NUM_CHANNELS];

	u8                         max_opened_tc;

	struct hwtstamp_config     tstamp;

	u16                        q_counter;

	u16                        drop_rq_q_counter;
@@ -956,10 +965,9 @@ static inline void mlx5e_sq_fetch_wqe(struct mlx5e_txqsq *sq, 
				      struct mlx5e_tx_wqe **wqe,

				      u16 *pi)

{

	struct mlx5_wq_cyc *wq;

	struct mlx5_wq_cyc *wq = &sq->wq;



	wq = &sq->wq;

	*pi = sq->pc & wq->sz_m1;

	*pi  = mlx5_wq_cyc_ctr2ix(wq, sq->pc);

	*wqe = mlx5_wq_cyc_get_wqe(wq, *pi);

	memset(*wqe, 0, sizeof(**wqe));

}
@@ -967,7 +975,7 @@ static inline void mlx5e_sq_fetch_wqe(struct mlx5e_txqsq *sq, 
static inline

struct mlx5e_tx_wqe *mlx5e_post_nop(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)

{

	u16                         pi   = *pc & wq->sz_m1;

	u16                         pi   = mlx5_wq_cyc_ctr2ix(wq, *pc);

	struct mlx5e_tx_wqe        *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);

	struct mlx5_wqe_ctrl_seg   *cseg = &wqe->ctrl;

drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c

+2 −2
Original line number Diff line number Diff line
@@ -174,7 +174,7 @@ mlx5e_tls_handle_ooo(struct mlx5e_tls_offload_context *context, 
	int headln;

	int i;



	sq->stats.tls_ooo++;

	sq->stats->tls_ooo++;



	if (mlx5e_tls_get_sync_data(context, tcp_seq, &info)) {

		/* We might get here if a retransmission reaches the driver
@@ -220,7 +220,7 @@ mlx5e_tls_handle_ooo(struct mlx5e_tls_offload_context *context, 
	skb_shinfo(nskb)->nr_frags = info.nr_frags;

	nskb->data_len = info.sync_len;

	nskb->len += info.sync_len;

	sq->stats.tls_resync_bytes += nskb->len;

	sq->stats->tls_resync_bytes += nskb->len;

	mlx5e_tls_complete_sync_skb(skb, nskb, tcp_seq, headln,

				    cpu_to_be64(info.rcd_sn));

	mlx5e_sq_xmit(sq, nskb, *wqe, *pi);

drivers/net/ethernet/mellanox/mlx5/core/en_main.c

+54 −34
Original line number Diff line number Diff line
@@ -423,6 +423,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, 
	rq->ix      = c->ix;

	rq->mdev    = mdev;

	rq->hw_mtu  = MLX5E_SW2HW_MTU(params, params->sw_mtu);

	rq->stats   = &c->priv->channel_stats[c->ix].rq;



	rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;

	if (IS_ERR(rq->xdp_prog)) {
@@ -646,7 +647,7 @@ static int mlx5e_create_rq(struct mlx5e_rq *rq, 
						MLX5_ADAPTER_PAGE_SHIFT);

	MLX5_SET64(wq, wq,  dbr_addr,		rq->wq_ctrl.db.dma);



	mlx5_fill_page_array(&rq->wq_ctrl.buf,

	mlx5_fill_page_frag_array(&rq->wq_ctrl.buf,

				  (__be64 *)MLX5_ADDR_OF(wq, wq, pas));



	err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
@@ -836,13 +837,15 @@ err_free_rq: 
static void mlx5e_activate_rq(struct mlx5e_rq *rq)

{

	struct mlx5e_icosq *sq = &rq->channel->icosq;

	u16 pi = sq->pc & sq->wq.sz_m1;

	struct mlx5_wq_cyc *wq = &sq->wq;

	struct mlx5e_tx_wqe *nopwqe;



	u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);



	set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);

	sq->db.ico_wqe[pi].opcode     = MLX5_OPCODE_NOP;

	nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);

	mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &nopwqe->ctrl);

	nopwqe = mlx5e_post_nop(wq, sq->sqn, &sq->pc);

	mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nopwqe->ctrl);

}



static void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
@@ -885,6 +888,7 @@ static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, 
{

	void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);

	struct mlx5_core_dev *mdev = c->mdev;

	struct mlx5_wq_cyc *wq = &sq->wq;

	int err;



	sq->pdev      = c->pdev;
@@ -894,10 +898,10 @@ static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, 
	sq->min_inline_mode = params->tx_min_inline_mode;



	param->wq.db_numa_node = cpu_to_node(c->cpu);

	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);

	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);

	if (err)

		return err;

	sq->wq.db = &sq->wq.db[MLX5_SND_DBR];

	wq->db = &wq->db[MLX5_SND_DBR];



	err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu));

	if (err)
@@ -940,23 +944,22 @@ static int mlx5e_alloc_icosq(struct mlx5e_channel *c, 
{

	void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);

	struct mlx5_core_dev *mdev = c->mdev;

	struct mlx5_wq_cyc *wq = &sq->wq;

	int err;



	sq->channel   = c;

	sq->uar_map   = mdev->mlx5e_res.bfreg.map;



	param->wq.db_numa_node = cpu_to_node(c->cpu);

	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);

	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);

	if (err)

		return err;

	sq->wq.db = &sq->wq.db[MLX5_SND_DBR];

	wq->db = &wq->db[MLX5_SND_DBR];



	err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu));

	if (err)

		goto err_sq_wq_destroy;



	sq->edge = (sq->wq.sz_m1 + 1) - MLX5E_ICOSQ_MAX_WQEBBS;



	return 0;



err_sq_wq_destroy:
@@ -1001,10 +1004,12 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, 
			     int txq_ix,

			     struct mlx5e_params *params,

			     struct mlx5e_sq_param *param,

			     struct mlx5e_txqsq *sq)

			     struct mlx5e_txqsq *sq,

			     int tc)

{

	void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);

	struct mlx5_core_dev *mdev = c->mdev;

	struct mlx5_wq_cyc *wq = &sq->wq;

	int err;



	sq->pdev      = c->pdev;
@@ -1015,6 +1020,7 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, 
	sq->txq_ix    = txq_ix;

	sq->uar_map   = mdev->mlx5e_res.bfreg.map;

	sq->min_inline_mode = params->tx_min_inline_mode;

	sq->stats     = &c->priv->channel_stats[c->ix].sq[tc];

	INIT_WORK(&sq->recover.recover_work, mlx5e_sq_recover);

	if (MLX5_IPSEC_DEV(c->priv->mdev))

		set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state);
@@ -1022,10 +1028,10 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, 
		set_bit(MLX5E_SQ_STATE_TLS, &sq->state);



	param->wq.db_numa_node = cpu_to_node(c->cpu);

	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);

	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);

	if (err)

		return err;

	sq->wq.db    = &sq->wq.db[MLX5_SND_DBR];

	wq->db    = &wq->db[MLX5_SND_DBR];



	err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu));

	if (err)
@@ -1034,8 +1040,6 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, 
	INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work);

	sq->dim.mode = params->tx_cq_moderation.cq_period_mode;



	sq->edge = (sq->wq.sz_m1 + 1) - MLX5_SEND_WQE_MAX_WQEBBS;



	return 0;



err_sq_wq_destroy:
@@ -1095,7 +1099,8 @@ static int mlx5e_create_sq(struct mlx5_core_dev *mdev, 
					  MLX5_ADAPTER_PAGE_SHIFT);

	MLX5_SET64(wq, wq, dbr_addr,      csp->wq_ctrl->db.dma);



	mlx5_fill_page_array(&csp->wq_ctrl->buf, (__be64 *)MLX5_ADDR_OF(wq, wq, pas));

	mlx5_fill_page_frag_array(&csp->wq_ctrl->buf,

				  (__be64 *)MLX5_ADDR_OF(wq, wq, pas));



	err = mlx5_core_create_sq(mdev, in, inlen, sqn);
@@ -1174,13 +1179,14 @@ static int mlx5e_open_txqsq(struct mlx5e_channel *c, 
			    int txq_ix,

			    struct mlx5e_params *params,

			    struct mlx5e_sq_param *param,

			    struct mlx5e_txqsq *sq)

			    struct mlx5e_txqsq *sq,

			    int tc)

{

	struct mlx5e_create_sq_param csp = {};

	u32 tx_rate;

	int err;



	err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq);

	err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc);

	if (err)

		return err;
@@ -1238,6 +1244,7 @@ static inline void netif_tx_disable_queue(struct netdev_queue *txq) 
static void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)

{

	struct mlx5e_channel *c = sq->channel;

	struct mlx5_wq_cyc *wq = &sq->wq;



	clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);

	/* prevent netif_tx_wake_queue */
@@ -1246,12 +1253,13 @@ static void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq) 
	netif_tx_disable_queue(sq->txq);



	/* last doorbell out, godspeed .. */

	if (mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1)) {

	if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) {

		u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);

		struct mlx5e_tx_wqe *nop;



		sq->db.wqe_info[(sq->pc & sq->wq.sz_m1)].skb = NULL;

		nop = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);

		mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &nop->ctrl);

		sq->db.wqe_info[pi].skb = NULL;

		nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc);

		mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl);

	}

}
@@ -1366,7 +1374,7 @@ static void mlx5e_sq_recover(struct work_struct *work) 
		return;



	mlx5e_reset_txqsq_cc_pc(sq);

	sq->stats.recover++;

	sq->stats->recover++;

	recover->last_recover = jiffies;

	mlx5e_activate_txqsq(sq);

}
@@ -1535,7 +1543,7 @@ static int mlx5e_alloc_cq(struct mlx5e_channel *c, 


static void mlx5e_free_cq(struct mlx5e_cq *cq)

{

	mlx5_cqwq_destroy(&cq->wq_ctrl);

	mlx5_wq_destroy(&cq->wq_ctrl);

}



static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
@@ -1551,7 +1559,7 @@ static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param) 
	int err;



	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +

		sizeof(u64) * cq->wq_ctrl.frag_buf.npages;

		sizeof(u64) * cq->wq_ctrl.buf.npages;

	in = kvzalloc(inlen, GFP_KERNEL);

	if (!in)

		return -ENOMEM;
@@ -1560,7 +1568,7 @@ static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param) 


	memcpy(cqc, param->cqc, sizeof(param->cqc));



	mlx5_fill_page_frag_array(&cq->wq_ctrl.frag_buf,

	mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,

				  (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));



	mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
@@ -1568,7 +1576,7 @@ static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param) 
	MLX5_SET(cqc,   cqc, cq_period_mode, param->cq_period_mode);

	MLX5_SET(cqc,   cqc, c_eqn,         eqn);

	MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.uar->index);

	MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.frag_buf.page_shift -

	MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -

					    MLX5_ADAPTER_PAGE_SHIFT);

	MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
@@ -1661,14 +1669,14 @@ static int mlx5e_open_sqs(struct mlx5e_channel *c, 
			  struct mlx5e_params *params,

			  struct mlx5e_channel_param *cparam)

{

	int err;

	int tc;

	struct mlx5e_priv *priv = c->priv;

	int err, tc, max_nch = priv->profile->max_nch(priv->mdev);



	for (tc = 0; tc < params->num_tc; tc++) {

		int txq_ix = c->ix + tc * params->num_channels;

		int txq_ix = c->ix + tc * max_nch;



		err = mlx5e_open_txqsq(c, c->priv->tisn[tc], txq_ix,

				       params, &cparam->sq, &c->sq[tc]);

				       params, &cparam->sq, &c->sq[tc], tc);

		if (err)

			goto err_close_sqs;

	}
@@ -1798,6 +1806,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, 
	c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);

	c->num_tc   = params->num_tc;

	c->xdp      = !!params->xdp_prog;

	c->stats    = &priv->channel_stats[ix].ch;



	mlx5_vector2eqn(priv->mdev, ix, &eqn, &irq);

	c->irq_desc = irq_to_desc(irq);
@@ -2630,7 +2639,7 @@ static void mlx5e_build_channels_tx_maps(struct mlx5e_priv *priv) 
	struct mlx5e_txqsq *sq;

	int i, tc;



	for (i = 0; i < priv->channels.num; i++)

	for (i = 0; i < priv->profile->max_nch(priv->mdev); i++)

		for (tc = 0; tc < priv->profile->max_tc; tc++)

			priv->channel_tc2txq[i][tc] = i + tc * priv->channels.num;
@@ -2654,6 +2663,9 @@ void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) 


	mlx5e_build_channels_tx_maps(priv);

	mlx5e_activate_channels(&priv->channels);

	write_lock(&priv->stats_lock);

	priv->channels_active = true;

	write_unlock(&priv->stats_lock);

	netif_tx_start_all_queues(priv->netdev);



	if (MLX5_VPORT_MANAGER(priv->mdev))
@@ -2675,6 +2687,9 @@ void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv) 
	 */

	netif_tx_stop_all_queues(priv->netdev);

	netif_tx_disable(priv->netdev);

	write_lock(&priv->stats_lock);

	priv->channels_active = false;

	write_unlock(&priv->stats_lock);

	mlx5e_deactivate_channels(&priv->channels);

}
@@ -3129,6 +3144,8 @@ static int mlx5e_setup_tc_mqprio(struct net_device *netdev, 
	if (err)

		goto out;



	priv->max_opened_tc = max_t(u8, priv->max_opened_tc,

				    new_channels.params.num_tc);

	mlx5e_switch_priv_channels(priv, &new_channels, NULL);

out:

	mutex_unlock(&priv->state_lock);
@@ -3219,6 +3236,7 @@ mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) 
		stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok);

		stats->tx_bytes   = PPORT_802_3_GET(pstats, a_octets_transmitted_ok);

	} else {

		mlx5e_grp_sw_update_stats(priv);

		stats->rx_packets = sstats->rx_packets;

		stats->rx_bytes   = sstats->rx_bytes;

		stats->tx_packets = sstats->tx_packets;
@@ -3815,7 +3833,7 @@ static bool mlx5e_tx_timeout_eq_recover(struct net_device *dev, 
		return false;



	netdev_err(dev, "Recover %d eqes on EQ 0x%x\n", eqe_count, eq->eqn);

	sq->channel->stats.eq_rearm++;

	sq->channel->stats->eq_rearm++;

	return true;

}
@@ -4239,11 +4257,13 @@ static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev, 
	priv->profile     = profile;

	priv->ppriv       = ppriv;

	priv->msglevel    = MLX5E_MSG_LEVEL;

	priv->max_opened_tc = 1;



	mlx5e_build_nic_params(mdev, &priv->channels.params,

			       profile->max_nch(mdev), netdev->mtu);



	mutex_init(&priv->state_lock);

	rwlock_init(&priv->stats_lock);



	INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);

	INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);

drivers/net/ethernet/mellanox/mlx5/core/en_rep.c

+11 −9
Original line number Diff line number Diff line
@@ -130,28 +130,28 @@ static void mlx5e_rep_update_sw_counters(struct mlx5e_priv *priv) 
	struct mlx5e_sq_stats *sq_stats;

	int i, j;



	read_lock(&priv->stats_lock);

	if (!priv->channels_active)

	        goto out;



	memset(s, 0, sizeof(*s));

	for (i = 0; i < priv->channels.num; i++) {

		struct mlx5e_channel *c = priv->channels.c[i];



		rq_stats = &c->rq.stats;

		rq_stats = c->rq.stats;



		s->rx_packets	+= rq_stats->packets;

		s->rx_bytes	+= rq_stats->bytes;



		for (j = 0; j < priv->channels.params.num_tc; j++) {

			sq_stats = &c->sq[j].stats;

			sq_stats = c->sq[j].stats;



			s->tx_packets		+= sq_stats->packets;

			s->tx_bytes		+= sq_stats->bytes;

		}

	}

}



static void mlx5e_rep_update_stats(struct mlx5e_priv *priv)

{

	mlx5e_rep_update_sw_counters(priv);

	mlx5e_rep_update_hw_counters(priv);

out:

	read_unlock(&priv->stats_lock);

}



static void mlx5e_rep_get_ethtool_stats(struct net_device *dev,
@@ -871,6 +871,8 @@ mlx5e_get_sw_stats64(const struct net_device *dev, 
	struct mlx5e_priv *priv = netdev_priv(dev);

	struct mlx5e_sw_stats *sstats = &priv->stats.sw;



	mlx5e_rep_update_sw_counters(priv);



	stats->rx_packets = sstats->rx_packets;

	stats->rx_bytes   = sstats->rx_bytes;

	stats->tx_packets = sstats->tx_packets;
@@ -1046,7 +1048,7 @@ static const struct mlx5e_profile mlx5e_rep_profile = { 
	.cleanup_rx		= mlx5e_cleanup_rep_rx,

	.init_tx		= mlx5e_init_rep_tx,

	.cleanup_tx		= mlx5e_cleanup_nic_tx,

	.update_stats           = mlx5e_rep_update_stats,

	.update_stats           = mlx5e_rep_update_hw_counters,

	.max_nch		= mlx5e_get_rep_max_num_channels,

	.update_carrier		= NULL,

	.rx_handlers.handle_rx_cqe       = mlx5e_handle_rx_cqe_rep,

drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

+77 −49
Original line number Diff line number Diff line
@@ -54,7 +54,7 @@ static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config) 
static inline void mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cqcc,

				       void *data)

{

	u32 ci = cqcc & cq->wq.fbc.sz_m1;

	u32 ci = mlx5_cqwq_ctr2ix(&cq->wq, cqcc);



	memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, ci), sizeof(struct mlx5_cqe64));

}
@@ -65,7 +65,7 @@ static inline void mlx5e_read_title_slot(struct mlx5e_rq *rq, 
	mlx5e_read_cqe_slot(cq, cqcc, &cq->title);

	cq->decmprs_left        = be32_to_cpu(cq->title.byte_cnt);

	cq->decmprs_wqe_counter = be16_to_cpu(cq->title.wqe_counter);

	rq->stats.cqe_compress_blks++;

	rq->stats->cqe_compress_blks++;

}



static inline void mlx5e_read_mini_arr_slot(struct mlx5e_cq *cq, u32 cqcc)
@@ -76,10 +76,11 @@ static inline void mlx5e_read_mini_arr_slot(struct mlx5e_cq *cq, u32 cqcc) 


static inline void mlx5e_cqes_update_owner(struct mlx5e_cq *cq, u32 cqcc, int n)

{

	struct mlx5_frag_buf_ctrl *fbc = &cq->wq.fbc;

	u8 op_own = (cqcc >> fbc->log_sz) & 1;

	u32 wq_sz = 1 << fbc->log_sz;

	u32 ci = cqcc & fbc->sz_m1;

	struct mlx5_cqwq *wq = &cq->wq;



	u8  op_own = mlx5_cqwq_get_ctr_wrap_cnt(wq, cqcc) & 1;

	u32 ci     = mlx5_cqwq_ctr2ix(wq, cqcc);

	u32 wq_sz  = mlx5_cqwq_get_size(wq);

	u32 ci_top = min_t(u32, wq_sz, ci + n);



	for (; ci < ci_top; ci++, n--) {
@@ -112,7 +113,7 @@ static inline void mlx5e_decompress_cqe(struct mlx5e_rq *rq, 
			mpwrq_get_cqe_consumed_strides(&cq->title);

	else

		cq->decmprs_wqe_counter =

			(cq->decmprs_wqe_counter + 1) & rq->wq.sz_m1;

			mlx5_wq_ll_ctr2ix(&rq->wq, cq->decmprs_wqe_counter + 1);

}



static inline void mlx5e_decompress_cqe_no_hash(struct mlx5e_rq *rq,
@@ -145,7 +146,7 @@ static inline u32 mlx5e_decompress_cqes_cont(struct mlx5e_rq *rq, 
	mlx5e_cqes_update_owner(cq, cq->wq.cc, cqcc - cq->wq.cc);

	cq->wq.cc = cqcc;

	cq->decmprs_left -= cqe_count;

	rq->stats.cqe_compress_pkts += cqe_count;

	rq->stats->cqe_compress_pkts += cqe_count;



	return cqe_count;

}
@@ -175,14 +176,15 @@ static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq, 
{

	struct mlx5e_page_cache *cache = &rq->page_cache;

	u32 tail_next = (cache->tail + 1) & (MLX5E_CACHE_SIZE - 1);

	struct mlx5e_rq_stats *stats = rq->stats;



	if (tail_next == cache->head) {

		rq->stats.cache_full++;

		stats->cache_full++;

		return false;

	}



	if (unlikely(mlx5e_page_is_reserved(dma_info->page))) {

		rq->stats.cache_waive++;

		stats->cache_waive++;

		return false;

	}
@@ -195,20 +197,21 @@ static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, 
				      struct mlx5e_dma_info *dma_info)

{

	struct mlx5e_page_cache *cache = &rq->page_cache;

	struct mlx5e_rq_stats *stats = rq->stats;



	if (unlikely(cache->head == cache->tail)) {

		rq->stats.cache_empty++;

		stats->cache_empty++;

		return false;

	}



	if (page_ref_count(cache->page_cache[cache->head].page) != 1) {

		rq->stats.cache_busy++;

		stats->cache_busy++;

		return false;

	}



	*dma_info = cache->page_cache[cache->head];

	cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1);

	rq->stats.cache_reuse++;

	stats->cache_reuse++;



	dma_sync_single_for_device(rq->pdev, dma_info->addr,

				   RQ_PAGE_SIZE(rq),
@@ -293,7 +296,7 @@ static inline void mlx5e_free_rx_wqe_reuse(struct mlx5e_rq *rq, 
					   struct mlx5e_wqe_frag_info *wi)

{

	if (mlx5e_page_reuse(rq, wi)) {

		rq->stats.page_reuse++;

		rq->stats->page_reuse++;

		return;

	}
@@ -382,6 +385,22 @@ static inline u16 mlx5e_icosq_wrap_cnt(struct mlx5e_icosq *sq) 
	return sq->pc >> MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;

}



static inline void mlx5e_fill_icosq_frag_edge(struct mlx5e_icosq *sq,

					      struct mlx5_wq_cyc *wq,

					      u16 pi, u16 frag_pi)

{

	struct mlx5e_sq_wqe_info *edge_wi, *wi = &sq->db.ico_wqe[pi];

	u8 nnops = mlx5_wq_cyc_get_frag_size(wq) - frag_pi;



	edge_wi = wi + nnops;



	/* fill sq frag edge with nops to avoid wqe wrapping two pages */

	for (; wi < edge_wi; wi++) {

		wi->opcode = MLX5_OPCODE_NOP;

		mlx5e_post_nop(wq, sq->sqn, &sq->pc);

	}

}



static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)

{

	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
@@ -390,14 +409,16 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) 
	struct mlx5_wq_cyc *wq = &sq->wq;

	struct mlx5e_umr_wqe *umr_wqe;

	u16 xlt_offset = ix << (MLX5E_LOG_ALIGNED_MPWQE_PPW - 1);

	u16 pi, frag_pi;

	int err;

	u16 pi;

	int i;



	/* fill sq edge with nops to avoid wqe wrap around */

	while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) {

		sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;

		mlx5e_post_nop(wq, sq->sqn, &sq->pc);

	pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);

	frag_pi = mlx5_wq_cyc_ctr2fragix(wq, sq->pc);



	if (unlikely(frag_pi + MLX5E_UMR_WQEBBS > mlx5_wq_cyc_get_frag_size(wq))) {

		mlx5e_fill_icosq_frag_edge(sq, wq, pi, frag_pi);

		pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);

	}



	umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);
@@ -433,7 +454,7 @@ err_unmap: 
		dma_info--;

		mlx5e_page_release(rq, dma_info, true);

	}

	rq->stats.buff_alloc_err++;

	rq->stats->buff_alloc_err++;



	return err;

}
@@ -461,7 +482,7 @@ bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) 


		err = mlx5e_alloc_rx_wqe(rq, wqe, wq->head);

		if (unlikely(err)) {

			rq->stats.buff_alloc_err++;

			rq->stats->buff_alloc_err++;

			break;

		}
@@ -482,7 +503,7 @@ static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq *cq, 
					     struct mlx5_cqe64 *cqe)

{

	struct mlx5_wq_cyc *wq = &sq->wq;

	u16 ci = be16_to_cpu(cqe->wqe_counter) & wq->sz_m1;

	u16 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));

	struct mlx5e_sq_wqe_info *icowi = &sq->db.ico_wqe[ci];



	mlx5_cqwq_pop(&cq->wq);
@@ -672,6 +693,7 @@ static inline void mlx5e_handle_csum(struct net_device *netdev, 
				     struct sk_buff *skb,

				     bool   lro)

{

	struct mlx5e_rq_stats *stats = rq->stats;

	int network_depth = 0;



	if (unlikely(!(netdev->features & NETIF_F_RXCSUM)))
@@ -679,7 +701,7 @@ static inline void mlx5e_handle_csum(struct net_device *netdev, 


	if (lro) {

		skb->ip_summed = CHECKSUM_UNNECESSARY;

		rq->stats.csum_unnecessary++;

		stats->csum_unnecessary++;

		return;

	}
@@ -697,7 +719,7 @@ static inline void mlx5e_handle_csum(struct net_device *netdev, 
		if (unlikely(netdev->features & NETIF_F_RXFCS))

			skb->csum = csum_add(skb->csum,

					     (__force __wsum)mlx5e_get_fcs(skb));

		rq->stats.csum_complete++;

		stats->csum_complete++;

		return;

	}
@@ -707,15 +729,15 @@ static inline void mlx5e_handle_csum(struct net_device *netdev, 
		if (cqe_is_tunneled(cqe)) {

			skb->csum_level = 1;

			skb->encapsulation = 1;

			rq->stats.csum_unnecessary_inner++;

			stats->csum_unnecessary_inner++;

			return;

		}

		rq->stats.csum_unnecessary++;

		stats->csum_unnecessary++;

		return;

	}

csum_none:

	skb->ip_summed = CHECKSUM_NONE;

	rq->stats.csum_none++;

	stats->csum_none++;

}



static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe,
@@ -724,6 +746,7 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, 
				      struct sk_buff *skb)

{

	u8 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24;

	struct mlx5e_rq_stats *stats = rq->stats;

	struct net_device *netdev = rq->netdev;



	skb->mac_len = ETH_HLEN;
@@ -733,9 +756,9 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, 
		/* Subtract one since we already counted this as one

		 * "regular" packet in mlx5e_complete_rx_cqe()

		 */

		rq->stats.packets += lro_num_seg - 1;

		rq->stats.lro_packets++;

		rq->stats.lro_bytes += cqe_bcnt;

		stats->packets += lro_num_seg - 1;

		stats->lro_packets++;

		stats->lro_bytes += cqe_bcnt;

	}



	if (unlikely(mlx5e_rx_hw_stamp(rq->tstamp)))
@@ -750,7 +773,7 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, 
	if (cqe_has_vlan(cqe)) {

		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),

				       be16_to_cpu(cqe->vlan_info));

		rq->stats.removed_vlan_packets++;

		stats->removed_vlan_packets++;

	}



	skb->mark = be32_to_cpu(cqe->sop_drop_qpn) & MLX5E_TC_FLOW_ID_MASK;
@@ -764,8 +787,10 @@ static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq, 
					 u32 cqe_bcnt,

					 struct sk_buff *skb)

{

	rq->stats.packets++;

	rq->stats.bytes += cqe_bcnt;

	struct mlx5e_rq_stats *stats = rq->stats;



	stats->packets++;

	stats->bytes += cqe_bcnt;

	mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb);

}
@@ -773,7 +798,7 @@ static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq) 
{

	struct mlx5_wq_cyc *wq = &sq->wq;

	struct mlx5e_tx_wqe *wqe;

	u16 pi = (sq->pc - 1) & wq->sz_m1; /* last pi */

	u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc - 1); /* last pi */



	wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
@@ -786,7 +811,7 @@ static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq, 
{

	struct mlx5e_xdpsq       *sq   = &rq->xdpsq;

	struct mlx5_wq_cyc       *wq   = &sq->wq;

	u16                       pi   = sq->pc & wq->sz_m1;

	u16                       pi   = mlx5_wq_cyc_ctr2ix(wq, sq->pc);

	struct mlx5e_tx_wqe      *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);



	struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
@@ -797,10 +822,12 @@ static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq, 
	dma_addr_t dma_addr  = di->addr + data_offset;

	unsigned int dma_len = xdp->data_end - xdp->data;



	struct mlx5e_rq_stats *stats = rq->stats;



	prefetchw(wqe);



	if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE || rq->hw_mtu < dma_len)) {

		rq->stats.xdp_drop++;

		stats->xdp_drop++;

		return false;

	}
@@ -810,7 +837,7 @@ static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq, 
			mlx5e_xmit_xdp_doorbell(sq);

			sq->db.doorbell = false;

		}

		rq->stats.xdp_tx_full++;

		stats->xdp_tx_full++;

		return false;

	}
@@ -844,7 +871,7 @@ static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq, 


	sq->db.doorbell = true;



	rq->stats.xdp_tx++;

	stats->xdp_tx++;

	return true;

}
@@ -891,7 +918,7 @@ static inline bool mlx5e_xdp_handle(struct mlx5e_rq *rq, 
	case XDP_ABORTED:

		trace_xdp_exception(rq->netdev, prog, act);

	case XDP_DROP:

		rq->stats.xdp_drop++;

		rq->stats->xdp_drop++;

		return true;

	}

}
@@ -904,7 +931,7 @@ struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va, 
	struct sk_buff *skb = build_skb(va, frag_size);



	if (unlikely(!skb)) {

		rq->stats.buff_alloc_err++;

		rq->stats->buff_alloc_err++;

		return NULL;

	}
@@ -936,7 +963,7 @@ struct sk_buff *skb_from_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, 
	wi->offset += frag_size;



	if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {

		rq->stats.wqe_err++;

		rq->stats->wqe_err++;

		return NULL;

	}
@@ -1053,7 +1080,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w 
	skb = napi_alloc_skb(rq->cq.napi,

			     ALIGN(MLX5_MPWRQ_SMALL_PACKET_THRESHOLD, sizeof(long)));

	if (unlikely(!skb)) {

		rq->stats.buff_alloc_err++;

		rq->stats->buff_alloc_err++;

		return NULL;

	}
@@ -1139,12 +1166,12 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) 
	wi->consumed_strides += cstrides;



	if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {

		rq->stats.wqe_err++;

		rq->stats->wqe_err++;

		goto mpwrq_cqe_out;

	}



	if (unlikely(mpwrq_is_filler_cqe(cqe))) {

		rq->stats.mpwqe_filler++;

		rq->stats->mpwqe_filler++;

		goto mpwrq_cqe_out;

	}
@@ -1256,7 +1283,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq) 


			last_wqe = (sqcc == wqe_counter);



			ci = sqcc & sq->wq.sz_m1;

			ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);

			di = &sq->db.di[ci];



			sqcc++;
@@ -1281,7 +1308,7 @@ void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq) 
	u16 ci;



	while (sq->cc != sq->pc) {

		ci = sq->cc & sq->wq.sz_m1;

		ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc);

		di = &sq->db.di[ci];

		sq->cc++;
@@ -1299,6 +1326,7 @@ static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq, 
					 u32 cqe_bcnt,

					 struct sk_buff *skb)

{

	struct mlx5e_rq_stats *stats = rq->stats;

	struct hwtstamp_config *tstamp;

	struct net_device *netdev;

	struct mlx5e_priv *priv;
@@ -1360,9 +1388,9 @@ static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq, 


	skb->dev = netdev;



	rq->stats.csum_complete++;

	rq->stats.packets++;

	rq->stats.bytes += cqe_bcnt;

	stats->csum_complete++;

	stats->packets++;

	stats->bytes += cqe_bcnt;

}



void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)

CRA Git | Maintained and supported by SUSTech CRA and CCSE