Commit 55454e9e authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'mlx4-XDP-tx-refactor' 



Tariq Toukan says:

====================
mlx4 XDP TX refactor

This patchset refactors the XDP forwarding case, so that
its dedicated transmit queues are managed in a complete
separation from the other regular ones.

It also adds ethtool counters for XDP cases.

Series generated against net-next commit:
22ca904a genetlink: fix error return code in genl_register_family()

Thanks,
Tariq.

v3:
* Exposed per ring counters.

v2:
* Added ethtool counters.
* Rebased, now patch 2 reverts Brenden's fix, as the bug no longer exists:
  958b3d39 ("net/mlx4_en: fixup xdp tx irq to match rx")
* Updated commit message of patch 2.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents e4ff952a 15fca2c8

drivers/net/ethernet/mellanox/mlx4/en_cq.c

+10 −18
Original line number Diff line number Diff line
@@ -65,7 +65,7 @@ int mlx4_en_create_cq(struct mlx4_en_priv *priv, 
	cq->buf_size = cq->size * mdev->dev->caps.cqe_size;



	cq->ring = ring;

	cq->is_tx = mode;

	cq->type = mode;

	cq->vector = mdev->dev->caps.num_comp_vectors;



	/* Allocate HW buffers on provided NUMA node.
@@ -104,7 +104,7 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, 
	*cq->mcq.arm_db    = 0;

	memset(cq->buf, 0, cq->buf_size);



	if (cq->is_tx == RX) {

	if (cq->type == RX) {

		if (!mlx4_is_eq_vector_valid(mdev->dev, priv->port,

					     cq->vector)) {

			cq->vector = cpumask_first(priv->rx_ring[cq->ring]->affinity_mask);
@@ -127,25 +127,17 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, 
		/* For TX we use the same irq per

		ring we assigned for the RX    */

		struct mlx4_en_cq *rx_cq;

		int xdp_index;



		/* The xdp tx irq must align with the rx ring that forwards to

		 * it, so reindex these from 0. This should only happen when

		 * tx_ring_num is not a multiple of rx_ring_num.

		 */

		xdp_index = (priv->xdp_ring_num - priv->tx_ring_num) + cq_idx;

		if (xdp_index >= 0)

			cq_idx = xdp_index;

		cq_idx = cq_idx % priv->rx_ring_num;

		rx_cq = priv->rx_cq[cq_idx];

		cq->vector = rx_cq->vector;

	}



	if (!cq->is_tx)

	if (cq->type == RX)

		cq->size = priv->rx_ring[cq->ring]->actual_size;



	if ((cq->is_tx && priv->hwtstamp_config.tx_type) ||

	    (!cq->is_tx && priv->hwtstamp_config.rx_filter))

	if ((cq->type != RX && priv->hwtstamp_config.tx_type) ||

	    (cq->type == RX && priv->hwtstamp_config.rx_filter))

		timestamp_en = 1;



	err = mlx4_cq_alloc(mdev->dev, cq->size, &cq->wqres.mtt,
@@ -154,10 +146,10 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, 
	if (err)

		goto free_eq;



	cq->mcq.comp  = cq->is_tx ? mlx4_en_tx_irq : mlx4_en_rx_irq;

	cq->mcq.comp  = cq->type != RX ? mlx4_en_tx_irq : mlx4_en_rx_irq;

	cq->mcq.event = mlx4_en_cq_event;



	if (cq->is_tx)

	if (cq->type != RX)

		netif_tx_napi_add(cq->dev, &cq->napi, mlx4_en_poll_tx_cq,

				  NAPI_POLL_WEIGHT);

	else
@@ -181,7 +173,7 @@ void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq) 


	mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size);

	if (mlx4_is_eq_vector_valid(mdev->dev, priv->port, cq->vector) &&

	    cq->is_tx == RX)

	    cq->type == RX)

		mlx4_release_eq(priv->mdev->dev, cq->vector);

	cq->vector = 0;

	cq->buf_size = 0;
@@ -193,7 +185,7 @@ void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq) 
void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq)

{

	napi_disable(&cq->napi);

	if (!cq->is_tx) {

	if (cq->type == RX) {

		napi_hash_del(&cq->napi);

		synchronize_rcu();

	}

drivers/net/ethernet/mellanox/mlx4/en_ethtool.c

+69 −32
Original line number Diff line number Diff line
@@ -49,18 +49,21 @@ 


static int mlx4_en_moderation_update(struct mlx4_en_priv *priv)

{

	int i;

	int i, t;

	int err = 0;



	for (i = 0; i < priv->tx_ring_num; i++) {

		priv->tx_cq[i]->moder_cnt = priv->tx_frames;

		priv->tx_cq[i]->moder_time = priv->tx_usecs;

	for (t = 0 ; t < MLX4_EN_NUM_TX_TYPES; t++) {

		for (i = 0; i < priv->tx_ring_num[t]; i++) {

			priv->tx_cq[t][i]->moder_cnt = priv->tx_frames;

			priv->tx_cq[t][i]->moder_time = priv->tx_usecs;

			if (priv->port_up) {

			err = mlx4_en_set_cq_moder(priv, priv->tx_cq[i]);

				err = mlx4_en_set_cq_moder(priv,

							   priv->tx_cq[t][i]);

				if (err)

					return err;

			}

		}

	}



	if (priv->adaptive_rx_coal)

		return 0;
@@ -192,6 +195,10 @@ static const char main_strings[][ETH_GSTRING_LEN] = { 
	"tx_prio_7_packets", "tx_prio_7_bytes",

	"tx_novlan_packets", "tx_novlan_bytes",



	/* xdp statistics */

	"rx_xdp_drop",

	"rx_xdp_tx",

	"rx_xdp_tx_full",

};



static const char mlx4_en_test_names[][ETH_GSTRING_LEN]= {
@@ -336,8 +343,8 @@ static int mlx4_en_get_sset_count(struct net_device *dev, int sset) 
	switch (sset) {

	case ETH_SS_STATS:

		return bitmap_iterator_count(&it) +

			(priv->tx_ring_num * 2) +

			(priv->rx_ring_num * 3);

			(priv->tx_ring_num[TX] * 2) +

			(priv->rx_ring_num * (3 + NUM_XDP_STATS));

	case ETH_SS_TEST:

		return MLX4_EN_NUM_SELF_TEST - !(priv->mdev->dev->caps.flags

					& MLX4_DEV_CAP_FLAG_UC_LOOPBACK) * 2;
@@ -397,14 +404,21 @@ static void mlx4_en_get_ethtool_stats(struct net_device *dev, 
		if (bitmap_iterator_test(&it))

			data[index++] = ((unsigned long *)&priv->pkstats)[i];



	for (i = 0; i < priv->tx_ring_num; i++) {

		data[index++] = priv->tx_ring[i]->packets;

		data[index++] = priv->tx_ring[i]->bytes;

	for (i = 0; i < NUM_XDP_STATS; i++, bitmap_iterator_inc(&it))

		if (bitmap_iterator_test(&it))

			data[index++] = ((unsigned long *)&priv->xdp_stats)[i];



	for (i = 0; i < priv->tx_ring_num[TX]; i++) {

		data[index++] = priv->tx_ring[TX][i]->packets;

		data[index++] = priv->tx_ring[TX][i]->bytes;

	}

	for (i = 0; i < priv->rx_ring_num; i++) {

		data[index++] = priv->rx_ring[i]->packets;

		data[index++] = priv->rx_ring[i]->bytes;

		data[index++] = priv->rx_ring[i]->dropped;

		data[index++] = priv->rx_ring[i]->xdp_drop;

		data[index++] = priv->rx_ring[i]->xdp_tx;

		data[index++] = priv->rx_ring[i]->xdp_tx_full;

	}

	spin_unlock_bh(&priv->stats_lock);
@@ -467,7 +481,13 @@ static void mlx4_en_get_strings(struct net_device *dev, 
				strcpy(data + (index++) * ETH_GSTRING_LEN,

				       main_strings[strings]);



		for (i = 0; i < priv->tx_ring_num; i++) {

		for (i = 0; i < NUM_XDP_STATS; i++, strings++,

		     bitmap_iterator_inc(&it))

			if (bitmap_iterator_test(&it))

				strcpy(data + (index++) * ETH_GSTRING_LEN,

				       main_strings[strings]);



		for (i = 0; i < priv->tx_ring_num[TX]; i++) {

			sprintf(data + (index++) * ETH_GSTRING_LEN,

				"tx%d_packets", i);

			sprintf(data + (index++) * ETH_GSTRING_LEN,
@@ -480,6 +500,12 @@ static void mlx4_en_get_strings(struct net_device *dev, 
				"rx%d_bytes", i);

			sprintf(data + (index++) * ETH_GSTRING_LEN,

				"rx%d_dropped", i);

			sprintf(data + (index++) * ETH_GSTRING_LEN,

				"rx%d_xdp_drop", i);

			sprintf(data + (index++) * ETH_GSTRING_LEN,

				"rx%d_xdp_tx", i);

			sprintf(data + (index++) * ETH_GSTRING_LEN,

				"rx%d_xdp_tx_full", i);

		}

		break;

	case ETH_SS_PRIV_FLAGS:
@@ -1060,7 +1086,7 @@ static int mlx4_en_set_ringparam(struct net_device *dev, 


	if (rx_size == (priv->port_up ? priv->rx_ring[0]->actual_size :

					priv->rx_ring[0]->size) &&

	    tx_size == priv->tx_ring[0]->size)

	    tx_size == priv->tx_ring[TX][0]->size)

		return 0;



	tmp = kzalloc(sizeof(*tmp), GFP_KERNEL);
@@ -1105,7 +1131,7 @@ static void mlx4_en_get_ringparam(struct net_device *dev, 
	param->tx_max_pending = MLX4_EN_MAX_TX_SIZE;

	param->rx_pending = priv->port_up ?

		priv->rx_ring[0]->actual_size : priv->rx_ring[0]->size;

	param->tx_pending = priv->tx_ring[0]->size;

	param->tx_pending = priv->tx_ring[TX][0]->size;

}



static u32 mlx4_en_get_rxfh_indir_size(struct net_device *dev)
@@ -1710,7 +1736,7 @@ static void mlx4_en_get_channels(struct net_device *dev, 
	channel->max_tx = MLX4_EN_MAX_TX_RING_P_UP;



	channel->rx_count = priv->rx_ring_num;

	channel->tx_count = priv->tx_ring_num / MLX4_EN_NUM_UP;

	channel->tx_count = priv->tx_ring_num[TX] / MLX4_EN_NUM_UP;

}



static int mlx4_en_set_channels(struct net_device *dev,
@@ -1721,6 +1747,7 @@ static int mlx4_en_set_channels(struct net_device *dev, 
	struct mlx4_en_port_profile new_prof;

	struct mlx4_en_priv *tmp;

	int port_up = 0;

	int xdp_count;

	int err = 0;



	if (channel->other_count || channel->combined_count ||
@@ -1729,20 +1756,25 @@ static int mlx4_en_set_channels(struct net_device *dev, 
	    !channel->tx_count || !channel->rx_count)

		return -EINVAL;



	if (channel->tx_count * MLX4_EN_NUM_UP <= priv->xdp_ring_num) {

		en_err(priv, "Minimum %d tx channels required with XDP on\n",

		       priv->xdp_ring_num / MLX4_EN_NUM_UP + 1);

		return -EINVAL;

	}



	tmp = kzalloc(sizeof(*tmp), GFP_KERNEL);

	if (!tmp)

		return -ENOMEM;



	mutex_lock(&mdev->state_lock);

	xdp_count = priv->tx_ring_num[TX_XDP] ? channel->rx_count : 0;

	if (channel->tx_count * MLX4_EN_NUM_UP + xdp_count > MAX_TX_RINGS) {

		err = -EINVAL;

		en_err(priv,

		       "Total number of TX and XDP rings (%d) exceeds the maximum supported (%d)\n",

		       channel->tx_count * MLX4_EN_NUM_UP + xdp_count,

		       MAX_TX_RINGS);

		goto out;

	}



	memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile));

	new_prof.num_tx_rings_p_up = channel->tx_count;

	new_prof.tx_ring_num = channel->tx_count * MLX4_EN_NUM_UP;

	new_prof.tx_ring_num[TX] = channel->tx_count * MLX4_EN_NUM_UP;

	new_prof.tx_ring_num[TX_XDP] = xdp_count;

	new_prof.rx_ring_num = channel->rx_count;



	err = mlx4_en_try_alloc_resources(priv, tmp, &new_prof);
@@ -1756,14 +1788,13 @@ static int mlx4_en_set_channels(struct net_device *dev, 


	mlx4_en_safe_replace_resources(priv, tmp);



	netif_set_real_num_tx_queues(dev, priv->tx_ring_num -

							priv->xdp_ring_num);

	netif_set_real_num_tx_queues(dev, priv->tx_ring_num[TX]);

	netif_set_real_num_rx_queues(dev, priv->rx_ring_num);



	if (dev->num_tc)

		mlx4_en_setup_tc(dev, MLX4_EN_NUM_UP);



	en_warn(priv, "Using %d TX rings\n", priv->tx_ring_num);

	en_warn(priv, "Using %d TX rings\n", priv->tx_ring_num[TX]);

	en_warn(priv, "Using %d RX rings\n", priv->rx_ring_num);



	if (port_up) {
@@ -1774,8 +1805,8 @@ static int mlx4_en_set_channels(struct net_device *dev, 


	err = mlx4_en_moderation_update(priv);

out:

	kfree(tmp);

	mutex_unlock(&mdev->state_lock);

	kfree(tmp);

	return err;

}
@@ -1823,11 +1854,15 @@ static int mlx4_en_set_priv_flags(struct net_device *dev, u32 flags) 
	int ret = 0;



	if (bf_enabled_new != bf_enabled_old) {

		int t;



		if (bf_enabled_new) {

			bool bf_supported = true;



			for (i = 0; i < priv->tx_ring_num; i++)

				bf_supported &= priv->tx_ring[i]->bf_alloced;

			for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++)

				for (i = 0; i < priv->tx_ring_num[t]; i++)

					bf_supported &=

						priv->tx_ring[t][i]->bf_alloced;



			if (!bf_supported) {

				en_err(priv, "BlueFlame is not supported\n");
@@ -1839,8 +1874,10 @@ static int mlx4_en_set_priv_flags(struct net_device *dev, u32 flags) 
			priv->pflags &= ~MLX4_EN_PRIV_FLAGS_BLUEFLAME;

		}



		for (i = 0; i < priv->tx_ring_num; i++)

			priv->tx_ring[i]->bf_enabled = bf_enabled_new;

		for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++)

			for (i = 0; i < priv->tx_ring_num[t]; i++)

				priv->tx_ring[t][i]->bf_enabled =

					bf_enabled_new;



		en_info(priv, "BlueFlame %s\n",

			bf_enabled_new ?  "Enabled" : "Disabled");

drivers/net/ethernet/mellanox/mlx4/en_main.c

+1 −1
Original line number Diff line number Diff line
@@ -169,7 +169,7 @@ static int mlx4_en_get_profile(struct mlx4_en_dev *mdev) 
		params->prof[i].tx_ppp = pfctx;

		params->prof[i].tx_ring_size = MLX4_EN_DEF_TX_RING_SIZE;

		params->prof[i].rx_ring_size = MLX4_EN_DEF_RX_RING_SIZE;

		params->prof[i].tx_ring_num = params->num_tx_rings_p_up *

		params->prof[i].tx_ring_num[TX] = params->num_tx_rings_p_up *

			MLX4_EN_NUM_UP;

		params->prof[i].rss_rings = 0;

		params->prof[i].inline_thold = inline_thold;

drivers/net/ethernet/mellanox/mlx4/en_netdev.c

+234 −148

File changed.

Preview size limit exceeded, changes collapsed.

drivers/net/ethernet/mellanox/mlx4/en_port.c

+8 −2
Original line number Diff line number Diff line
@@ -179,6 +179,9 @@ int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset) 
	priv->port_stats.rx_chksum_good = 0;

	priv->port_stats.rx_chksum_none = 0;

	priv->port_stats.rx_chksum_complete = 0;

	priv->xdp_stats.rx_xdp_drop    = 0;

	priv->xdp_stats.rx_xdp_tx      = 0;

	priv->xdp_stats.rx_xdp_tx_full = 0;

	for (i = 0; i < priv->rx_ring_num; i++) {

		stats->rx_packets += priv->rx_ring[i]->packets;

		stats->rx_bytes += priv->rx_ring[i]->bytes;
@@ -186,6 +189,9 @@ int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset) 
		priv->port_stats.rx_chksum_good += priv->rx_ring[i]->csum_ok;

		priv->port_stats.rx_chksum_none += priv->rx_ring[i]->csum_none;

		priv->port_stats.rx_chksum_complete += priv->rx_ring[i]->csum_complete;

		priv->xdp_stats.rx_xdp_drop    += priv->rx_ring[i]->xdp_drop;

		priv->xdp_stats.rx_xdp_tx      += priv->rx_ring[i]->xdp_tx;

		priv->xdp_stats.rx_xdp_tx_full += priv->rx_ring[i]->xdp_tx_full;

	}

	stats->tx_packets = 0;

	stats->tx_bytes = 0;
@@ -196,8 +202,8 @@ int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset) 
	priv->port_stats.tso_packets = 0;

	priv->port_stats.xmit_more = 0;



	for (i = 0; i < priv->tx_ring_num; i++) {

		const struct mlx4_en_tx_ring *ring = priv->tx_ring[i];

	for (i = 0; i < priv->tx_ring_num[TX]; i++) {

		const struct mlx4_en_tx_ring *ring = priv->tx_ring[TX][i];



		stats->tx_packets += ring->packets;

		stats->tx_bytes += ring->bytes;

CRA Git | Maintained and supported by SUSTech CRA and CCSE