Commit 8ef0c040 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'mlxsw-Update-main-pool-computation-and-pool-size-limits' 



Ido Schimmel says:

====================
mlxsw: Update main pool computation and pool size limits

Petr says:

In Spectrum ASICs, the shared buffer is an area of memory where packets are
kept until they can be transmitted. There are two resources associated with
shared buffer size: cap_total_buffer_size and cap_guaranteed_shared_buffer.
So far, mlxsw has been using the former as a limit when validating shared
buffer pool size configuration. However, the total size also includes
headrooms and reserved space, which really cannot be used for shared buffer
pools. Patch #1 mends this and has mlxsw use the guaranteed size.

To configure default pool sizes, mlxsw has historically hard-coded one or
two smallish pools, and one "main" pool that took most of the shared buffer
(that would be pool 0 on ingress and pool 4 on egress). During the
development of Spectrum-2, it became clear that the shared buffer size
keeps shrinking as bugs are identified and worked around. In order to
prevent having to tweak the size of pools 0 and 4 to catch up with updates
to values reported by the FW, patch #2 changes the way these pools are set.
Instead of hard-coding a fixed value, the main pool now takes whatever is
left from the guaranteed size after the smaller pool(s) are taken into
account.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 06cd9da5 bc9f6e94

drivers/net/ethernet/mellanox/mlxsw/resources.h

+2 −2
Original line number Diff line number Diff line
@@ -26,7 +26,7 @@ enum mlxsw_res_id { 
	MLXSW_RES_ID_MAX_LAG_MEMBERS,

	MLXSW_RES_ID_LOCAL_PORTS_IN_1X,

	MLXSW_RES_ID_LOCAL_PORTS_IN_2X,

	MLXSW_RES_ID_MAX_BUFFER_SIZE,

	MLXSW_RES_ID_GUARANTEED_SHARED_BUFFER,

	MLXSW_RES_ID_CELL_SIZE,

	MLXSW_RES_ID_MAX_HEADROOM_SIZE,

	MLXSW_RES_ID_ACL_MAX_TCAM_REGIONS,
@@ -82,7 +82,7 @@ static u16 mlxsw_res_ids[] = { 
	[MLXSW_RES_ID_MAX_LAG_MEMBERS] = 0x2521,

	[MLXSW_RES_ID_LOCAL_PORTS_IN_1X] = 0x2610,

	[MLXSW_RES_ID_LOCAL_PORTS_IN_2X] = 0x2611,

	[MLXSW_RES_ID_MAX_BUFFER_SIZE] = 0x2802,	/* Bytes */

	[MLXSW_RES_ID_GUARANTEED_SHARED_BUFFER] = 0x2805,	/* Bytes */

	[MLXSW_RES_ID_CELL_SIZE] = 0x2803,	/* Bytes */

	[MLXSW_RES_ID_MAX_HEADROOM_SIZE] = 0x2811,	/* Bytes */

	[MLXSW_RES_ID_ACL_MAX_TCAM_REGIONS] = 0x2901,

drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c

+38 −15
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ struct mlxsw_sp_sb_cm { 
};



#define MLXSW_SP_SB_INFI -1U

#define MLXSW_SP_SB_REST -2U



struct mlxsw_sp_sb_pm {

	u32 min_buff;
@@ -421,19 +422,16 @@ static void mlxsw_sp_sb_ports_fini(struct mlxsw_sp *mlxsw_sp) 
		.freeze_size = _freeze_size,				\

	}



#define MLXSW_SP1_SB_PR_INGRESS_SIZE	12440000

#define MLXSW_SP1_SB_PR_EGRESS_SIZE	13232000

#define MLXSW_SP1_SB_PR_CPU_SIZE	(256 * 1000)



/* Order according to mlxsw_sp1_sb_pool_dess */

static const struct mlxsw_sp_sb_pr mlxsw_sp1_sb_prs[] = {

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC,

		       MLXSW_SP1_SB_PR_INGRESS_SIZE),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),

	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,

			   MLXSW_SP1_SB_PR_EGRESS_SIZE, true, false),

	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST,

			   true, false),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
@@ -445,19 +443,16 @@ static const struct mlxsw_sp_sb_pr mlxsw_sp1_sb_prs[] = { 
			   MLXSW_SP1_SB_PR_CPU_SIZE, true, false),

};



#define MLXSW_SP2_SB_PR_INGRESS_SIZE	35297568

#define MLXSW_SP2_SB_PR_EGRESS_SIZE	35297568

#define MLXSW_SP2_SB_PR_CPU_SIZE	(256 * 1000)



/* Order according to mlxsw_sp2_sb_pool_dess */

static const struct mlxsw_sp_sb_pr mlxsw_sp2_sb_prs[] = {

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC,

		       MLXSW_SP2_SB_PR_INGRESS_SIZE),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),

	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,

			   MLXSW_SP2_SB_PR_EGRESS_SIZE, true, false),

	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST,

			   true, false),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),

	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
@@ -471,11 +466,33 @@ static const struct mlxsw_sp_sb_pr mlxsw_sp2_sb_prs[] = { 


static int mlxsw_sp_sb_prs_init(struct mlxsw_sp *mlxsw_sp,

				const struct mlxsw_sp_sb_pr *prs,

				const struct mlxsw_sp_sb_pool_des *pool_dess,

				size_t prs_len)

{

	/* Round down, unlike mlxsw_sp_bytes_cells(). */

	u32 sb_cells = mlxsw_sp->sb->sb_size / mlxsw_sp->sb->cell_size;

	u32 rest_cells[2] = {sb_cells, sb_cells};

	int i;

	int err;



	/* Calculate how much space to give to the "REST" pools in either

	 * direction.

	 */

	for (i = 0; i < prs_len; i++) {

		enum mlxsw_reg_sbxx_dir dir = pool_dess[i].dir;

		u32 size = prs[i].size;

		u32 size_cells;



		if (size == MLXSW_SP_SB_INFI || size == MLXSW_SP_SB_REST)

			continue;



		size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size);

		if (WARN_ON_ONCE(size_cells > rest_cells[dir]))

			continue;



		rest_cells[dir] -= size_cells;

	}



	for (i = 0; i < prs_len; i++) {

		u32 size = prs[i].size;

		u32 size_cells;
@@ -483,6 +500,10 @@ static int mlxsw_sp_sb_prs_init(struct mlxsw_sp *mlxsw_sp, 
		if (size == MLXSW_SP_SB_INFI) {

			err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,

						   0, true);

		} else if (size == MLXSW_SP_SB_REST) {

			size_cells = rest_cells[pool_dess[i].dir];

			err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,

						   size_cells, false);

		} else {

			size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size);

			err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,
@@ -904,7 +925,7 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp) 
	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, CELL_SIZE))

		return -EIO;



	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_BUFFER_SIZE))

	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, GUARANTEED_SHARED_BUFFER))

		return -EIO;



	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_HEADROOM_SIZE))
@@ -915,7 +936,7 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp) 
		return -ENOMEM;

	mlxsw_sp->sb->cell_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, CELL_SIZE);

	mlxsw_sp->sb->sb_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,

						   MAX_BUFFER_SIZE);

						   GUARANTEED_SHARED_BUFFER);

	max_headroom_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,

					       MAX_HEADROOM_SIZE);

	/* Round down, because this limit must not be overstepped. */
@@ -926,6 +947,7 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp) 
	if (err)

		goto err_sb_ports_init;

	err = mlxsw_sp_sb_prs_init(mlxsw_sp, mlxsw_sp->sb_vals->prs,

				   mlxsw_sp->sb_vals->pool_dess,

				   mlxsw_sp->sb_vals->pool_count);

	if (err)

		goto err_sb_prs_init;
@@ -1013,7 +1035,8 @@ int mlxsw_sp_sb_pool_set(struct mlxsw_core *mlxsw_core, 
	mode = (enum mlxsw_reg_sbpr_mode) threshold_type;

	pr = &mlxsw_sp->sb_vals->prs[pool_index];



	if (size > MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_BUFFER_SIZE)) {

	if (size > MLXSW_CORE_RES_GET(mlxsw_sp->core,

				      GUARANTEED_SHARED_BUFFER)) {

		NL_SET_ERR_MSG_MOD(extack, "Exceeded shared buffer size");

		return -EINVAL;

	}

drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c

+2 −1
Original line number Diff line number Diff line
@@ -305,7 +305,8 @@ mlxsw_sp_qdisc_red_check_params(struct mlxsw_sp_port *mlxsw_sp_port, 
			p->max);

		return -EINVAL;

	}

	if (p->max > MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_BUFFER_SIZE)) {

	if (p->max > MLXSW_CORE_RES_GET(mlxsw_sp->core,

					GUARANTEED_SHARED_BUFFER)) {

		dev_err(mlxsw_sp->bus_info->dev,

			"spectrum: RED: max value %u is too big\n", p->max);

		return -EINVAL;

CRA Git | Maintained and supported by SUSTech CRA and CCSE