Commit 16b61beb authored by John Fastabend's avatar John Fastabend Committed by Jeff Kirsher
Browse files

ixgbe: DCB set PFC high and low water marks per data sheet specs 



Currently the high and low water marks for PFC are being set
conservatively for jumbo frames. This means the RX buffers
are being underutilized in the default 1500 MTU. This patch
fixes this so that the water marks are set as described in
the data sheet considering the MTU size.

The equation used is,

RTT * 1.44 + MTU * 1.44 + MTU

Where RTT is the round trip time and MTU is the max frame size
in KB. To avoid floating point arithmetic FC_HIGH_WATER is
defined

((((RTT + MTU) * 144) + 99) / 100) + MTU

This changes how the hardware field fc.low_water and
fc.high_water are used. With this change they are no longer
storing the actual low water and high water markers but are
storing the required head room in the buffer. This simplifies
the logic and we do not need to account for the size of the
buffer when setting the thresholds.

Testing with iperf and 16 threads showed a slight uptick in
throughput over a single traffic class .1-.2Gbps and a reduction
in pause frames. Without the patch a 30 second run would show
~10-15 pause frames being transmitted with the patch ~2-5 are
seen. Test were run back to back with 82599.

Note RXPBSIZE is in KB and low and high water marks fields are
also in KB. However the FCRT* registers are 32B granularity and
right shifted 5 into the register,

(((rx_pbsize - water_mark) * 1024) / 32) << 5

is the most explicit conversion here we simplify

(rx_pbsize - water_mark) * 32 << 5 = (rx_pbsize - water_mark) << 10

This patch updates the PFC thresholds and legacy FC thresholds.

Signed-off-by: default avatarJohn Fastabend <john.r.fastabend@intel.com>
Tested-by: default avatarRoss Brattain <ross.b.brattain@intel.com>
Signed-off-by: default avatarJeff Kirsher <jeffrey.t.kirsher@intel.com>
parent 66c87bd5

drivers/net/ixgbe/ixgbe.h

+0 −2
Original line number Diff line number Diff line
@@ -61,10 +61,8 @@ 
#define IXGBE_MIN_RXD			     64



/* flow control */

#define IXGBE_DEFAULT_FCRTL		0x10000

#define IXGBE_MIN_FCRTL			   0x40

#define IXGBE_MAX_FCRTL			0x7FF80

#define IXGBE_DEFAULT_FCRTH		0x20000

#define IXGBE_MIN_FCRTH			  0x600

#define IXGBE_MAX_FCRTH			0x7FFF0

#define IXGBE_DEFAULT_FCPAUSE		 0xFFFF

drivers/net/ixgbe/ixgbe_82598.c

+12 −9
Original line number Diff line number Diff line
@@ -357,6 +357,7 @@ static s32 ixgbe_fc_enable_82598(struct ixgbe_hw *hw, s32 packetbuf_num) 
	u32 fctrl_reg;

	u32 rmcs_reg;

	u32 reg;

	u32 rx_pba_size;

	u32 link_speed = 0;

	bool link_up;
@@ -459,16 +460,18 @@ static s32 ixgbe_fc_enable_82598(struct ixgbe_hw *hw, s32 packetbuf_num) 


	/* Set up and enable Rx high/low water mark thresholds, enable XON. */

	if (hw->fc.current_mode & ixgbe_fc_tx_pause) {

		if (hw->fc.send_xon) {

			IXGBE_WRITE_REG(hw, IXGBE_FCRTL(packetbuf_num),

			                (hw->fc.low_water | IXGBE_FCRTL_XONE));

		} else {

			IXGBE_WRITE_REG(hw, IXGBE_FCRTL(packetbuf_num),

			                hw->fc.low_water);

		}

		rx_pba_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(packetbuf_num));

		rx_pba_size >>= IXGBE_RXPBSIZE_SHIFT;



		reg = (rx_pba_size - hw->fc.low_water) << 6;

		if (hw->fc.send_xon)

			reg |= IXGBE_FCRTL_XONE;

		IXGBE_WRITE_REG(hw, IXGBE_FCRTL(packetbuf_num), reg);



		reg = (rx_pba_size - hw->fc.high_water) << 10;

		reg |= IXGBE_FCRTH_FCEN;



		IXGBE_WRITE_REG(hw, IXGBE_FCRTH(packetbuf_num),

		                (hw->fc.high_water | IXGBE_FCRTH_FCEN));

		IXGBE_WRITE_REG(hw, IXGBE_FCRTH(packetbuf_num), reg);

	}



	/* Configure pause time (2 TCs per register) */

drivers/net/ixgbe/ixgbe_common.c

+12 −31
Original line number Diff line number Diff line
@@ -1595,6 +1595,7 @@ s32 ixgbe_fc_enable_generic(struct ixgbe_hw *hw, s32 packetbuf_num) 
	u32 mflcn_reg, fccfg_reg;

	u32 reg;

	u32 rx_pba_size;

	u32 fcrtl, fcrth;



#ifdef CONFIG_DCB

	if (hw->fc.requested_mode == ixgbe_fc_pfc)
@@ -1671,40 +1672,20 @@ s32 ixgbe_fc_enable_generic(struct ixgbe_hw *hw, s32 packetbuf_num) 
	IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);

	IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);



	reg = IXGBE_READ_REG(hw, IXGBE_MTQC);

	/* Thresholds are different for link flow control when in DCB mode */

	if (reg & IXGBE_MTQC_RT_ENA) {

	rx_pba_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(packetbuf_num));

	rx_pba_size >>= IXGBE_RXPBSIZE_SHIFT;



		/* Always disable XON for LFC when in DCB mode */

		reg = (rx_pba_size >> 5) & 0xFFE0;

		IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(packetbuf_num), reg);

	fcrth = (rx_pba_size - hw->fc.high_water) << 10;

	fcrtl = (rx_pba_size - hw->fc.low_water) << 10;



		reg = (rx_pba_size >> 2) & 0xFFE0;

		if (hw->fc.current_mode & ixgbe_fc_tx_pause)

			reg |= IXGBE_FCRTH_FCEN;

		IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(packetbuf_num), reg);

	} else {

		/*

		 * Set up and enable Rx high/low water mark thresholds,

		 * enable XON.

		 */

	if (hw->fc.current_mode & ixgbe_fc_tx_pause) {

			if (hw->fc.send_xon) {

				IXGBE_WRITE_REG(hw,

				              IXGBE_FCRTL_82599(packetbuf_num),

			                      (hw->fc.low_water |

				              IXGBE_FCRTL_XONE));

			} else {

				IXGBE_WRITE_REG(hw,

				              IXGBE_FCRTL_82599(packetbuf_num),

				              hw->fc.low_water);

		fcrth |= IXGBE_FCRTH_FCEN;

		if (hw->fc.send_xon)

			fcrtl |= IXGBE_FCRTL_XONE;

	}



			IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(packetbuf_num),

			               (hw->fc.high_water | IXGBE_FCRTH_FCEN));

		}

	}

	IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(packetbuf_num), fcrth);

	IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(packetbuf_num), fcrtl);



	/* Configure pause time (2 TCs per register) */

	reg = IXGBE_READ_REG(hw, IXGBE_FCTTV(packetbuf_num / 2));

drivers/net/ixgbe/ixgbe_dcb_82598.c

+4 −8
Original line number Diff line number Diff line
@@ -256,21 +256,17 @@ s32 ixgbe_dcb_config_pfc_82598(struct ixgbe_hw *hw, 
	 * for each traffic class.

	 */

	for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {

		if (dcb_config->rx_pba_cfg == pba_equal) {

			rx_pba_size = IXGBE_RXPBSIZE_64KB;

		} else {

			rx_pba_size = (i < 4) ? IXGBE_RXPBSIZE_80KB

					      : IXGBE_RXPBSIZE_48KB;

		}

		rx_pba_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));

		rx_pba_size >>= IXGBE_RXPBSIZE_SHIFT;

		reg = (rx_pba_size - hw->fc.low_water) << 10;



		reg = ((rx_pba_size >> 5) &  0xFFF0);

		if (dcb_config->tc_config[i].dcb_pfc == pfc_enabled_tx ||

		    dcb_config->tc_config[i].dcb_pfc == pfc_enabled_full)

			reg |= IXGBE_FCRTL_XONE;



		IXGBE_WRITE_REG(hw, IXGBE_FCRTL(i), reg);



		reg = ((rx_pba_size >> 2) & 0xFFF0);

		reg = (rx_pba_size - hw->fc.high_water) << 10;

		if (dcb_config->tc_config[i].dcb_pfc == pfc_enabled_tx ||

		    dcb_config->tc_config[i].dcb_pfc == pfc_enabled_full)

			reg |= IXGBE_FCRTH_FCEN;

drivers/net/ixgbe/ixgbe_dcb_82599.c

+5 −7
Original line number Diff line number Diff line
@@ -251,19 +251,17 @@ s32 ixgbe_dcb_config_pfc_82599(struct ixgbe_hw *hw, 


	/* Configure PFC Tx thresholds per TC */

	for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {

		if (dcb_config->rx_pba_cfg == pba_equal)

			rx_pba_size = IXGBE_RXPBSIZE_64KB;

		else

			rx_pba_size = (i < 4) ? IXGBE_RXPBSIZE_80KB

			                      : IXGBE_RXPBSIZE_48KB;

		rx_pba_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));

		rx_pba_size >>= IXGBE_RXPBSIZE_SHIFT;



		reg = (rx_pba_size - hw->fc.low_water) << 10;



		reg = ((rx_pba_size >> 5) & 0xFFE0);

		if (dcb_config->tc_config[i].dcb_pfc == pfc_enabled_full ||

		    dcb_config->tc_config[i].dcb_pfc == pfc_enabled_tx)

			reg |= IXGBE_FCRTL_XONE;

		IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(i), reg);



		reg = ((rx_pba_size >> 2) & 0xFFE0);

		reg = (rx_pba_size - hw->fc.high_water) << 10;

		if (dcb_config->tc_config[i].dcb_pfc == pfc_enabled_full ||

		    dcb_config->tc_config[i].dcb_pfc == pfc_enabled_tx)

			reg |= IXGBE_FCRTH_FCEN;

CRA Git | Maintained and supported by SUSTech CRA and CCSE