Commit ed6c24ed authored by Jon Mason's avatar Jon Mason
Browse files

NTB: NTB-RP support 



Add support for Non-Transparent Bridge connected to a PCI-E Root Port on
the remote system (also known as NTB-RP mode).  This allows for a NTB
enabled system to be connected to a non-NTB enabled system/slot.

Modifications to the registers and BARs/MWs on the Secondary side by the
remote system are reflected into registers on the Primary side for the
local system.  Similarly, modifications of registers and BARs/MWs on
Primary side by the local system are reflected into registers on the
Secondary side for the Remote System.  This allows communication between
the 2 sides via these registers and BARs/MWs.

Note: there is not a fix for the Xeon Errata (that was already worked
around in NTB-B2B mode) for NTB-RP mode.  Due to this limitation, NTB-RP
will not work on the Secondary side with the Xeon Errata workaround
enabled.  To get around this, disable the workaround via the
xeon_errata_workaround=0 modparm.  However, this can cause the hang
described in the errata.

Signed-off-by: default avatarJon Mason <jon.mason@intel.com>
parent 49793889

drivers/ntb/ntb_hw.c

+160 −102
Original line number Diff line number Diff line
@@ -69,7 +69,7 @@ module_param(xeon_errata_workaround, bool, 0644); 
MODULE_PARM_DESC(xeon_errata_workaround, "Workaround for the Xeon Errata");



enum {

	NTB_CONN_CLASSIC = 0,

	NTB_CONN_TRANSPARENT = 0,

	NTB_CONN_B2B,

	NTB_CONN_RP,

};
@@ -509,7 +509,8 @@ static void ntb_link_event(struct ntb_device *ndev, int link_state) 
		ndev->link_status = NTB_LINK_UP;

		event = NTB_EVENT_HW_LINK_UP;



		if (ndev->hw_type == BWD_HW)

		if (ndev->hw_type == BWD_HW ||

		    ndev->conn_type == NTB_CONN_TRANSPARENT)

			status = readw(ndev->reg_ofs.lnk_stat);

		else {

			int rc = pci_read_config_word(ndev->pdev,
@@ -649,30 +650,21 @@ static int ntb_xeon_setup(struct ntb_device *ndev) 
	if (rc)

		return rc;



	switch (val & SNB_PPD_CONN_TYPE) {

	case NTB_CONN_B2B:

		ndev->conn_type = NTB_CONN_B2B;

		break;

	case NTB_CONN_CLASSIC:

	case NTB_CONN_RP:

	default:

		dev_err(&ndev->pdev->dev, "Only B2B supported at this time\n");

		return -EINVAL;

	}



	if (val & SNB_PPD_DEV_TYPE)

		ndev->dev_type = NTB_DEV_USD;

	else

		ndev->dev_type = NTB_DEV_DSD;



	switch (val & SNB_PPD_CONN_TYPE) {

	case NTB_CONN_B2B:

		dev_info(&ndev->pdev->dev, "Conn Type = B2B\n");

		ndev->conn_type = NTB_CONN_B2B;

		ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;

		ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;

		ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;

		ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;

		ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;

	ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET;

	ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_LINK_STATUS_OFFSET;

	ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;

	ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;

		ndev->limits.max_spads = SNB_MAX_B2B_SPADS;



		/* There is a Xeon hardware errata related to writes to

		 * SDOORBELL or B2BDOORBELL in conjunction with inbound access
@@ -724,7 +716,7 @@ static int ntb_xeon_setup(struct ntb_device *ndev) 
				/* B2B_XLAT_OFFSET is a 64bit register, but can

				 * only take 32bit writes

				 */

			writel(SNB_MBAR01_USD_ADDR & 0xffffffff,

				writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,

				       ndev->reg_base + SNB_B2B_XLAT_OFFSETL);

				writel(SNB_MBAR01_DSD_ADDR >> 32,

				       ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
@@ -748,7 +740,7 @@ static int ntb_xeon_setup(struct ntb_device *ndev) 
				/* B2B_XLAT_OFFSET is a 64bit register, but can

				 * only take 32bit writes

				 */

			writel(SNB_MBAR01_USD_ADDR & 0xffffffff,

				writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,

				       ndev->reg_base + SNB_B2B_XLAT_OFFSETL);

				writel(SNB_MBAR01_USD_ADDR >> 32,

				       ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
@@ -760,8 +752,72 @@ static int ntb_xeon_setup(struct ntb_device *ndev) 
			writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base +

			       SNB_SBAR4BASE_OFFSET);

		}

		break;

	case NTB_CONN_RP:

		dev_info(&ndev->pdev->dev, "Conn Type = RP\n");

		ndev->conn_type = NTB_CONN_RP;



		if (xeon_errata_workaround) {

			dev_err(&ndev->pdev->dev, 

				"NTB-RP disabled due to hardware errata.  To disregard this warning and potentially lock-up the system, add the parameter 'xeon_errata_workaround=0'.\n");

			return -EINVAL;

		}



		/* Scratch pads need to have exclusive access from the primary

		 * or secondary side.  Halve the num spads so that each side can

		 * have an equal amount.

		 */

		ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;

		/* Note: The SDOORBELL is the cause of the errata.  You REALLY

		 * don't want to touch it.

		 */

		ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET;

		ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;

		ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;

		/* Offset the start of the spads to correspond to whether it is

		 * primary or secondary

		 */

		ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET +

					   ndev->limits.max_spads * 4;

		ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;

		ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;

		ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;

		ndev->limits.max_mw = SNB_MAX_MW;

		break;

	case NTB_CONN_TRANSPARENT:

		dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n");

		ndev->conn_type = NTB_CONN_TRANSPARENT;

		/* Scratch pads need to have exclusive access from the primary

		 * or secondary side.  Halve the num spads so that each side can

		 * have an equal amount.

		 */

		ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;

		ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;

		ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET;

		ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET;

		ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET;

		/* Offset the start of the spads to correspond to whether it is

		 * primary or secondary

		 */

		ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET +

					  ndev->limits.max_spads * 4;

		ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET;

		ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET;



		ndev->limits.max_mw = SNB_MAX_MW;

		break;

	default:

		/* Most likely caused by the remote NTB-RP device not being

		 * configured

		 */

		dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", val);

		return -EINVAL;

	}



	ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET;

	ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET;

	ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;



	ndev->limits.max_spads = SNB_MAX_B2B_SPADS;

	ndev->limits.max_db_bits = SNB_MAX_DB_BITS;

	ndev->limits.msix_cnt = SNB_MSIX_CNT;

	ndev->bits_per_vector = SNB_DB_BITS_PER_VEC;
@@ -865,8 +921,10 @@ static int ntb_device_setup(struct ntb_device *ndev) 
	dev_info(&ndev->pdev->dev, "Device Type = %s\n",

		 ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP");



	if (ndev->conn_type == NTB_CONN_B2B)

		/* Enable Bus Master and Memory Space on the secondary side */

	writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, ndev->reg_ofs.spci_cmd);

		writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,

		       ndev->reg_ofs.spci_cmd);



	return 0;

}
@@ -1360,7 +1418,7 @@ static void ntb_pci_remove(struct pci_dev *pdev) 


	/* Bring NTB link down */

	ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);

	ntb_cntl |= NTB_LINK_DISABLE;

	ntb_cntl |= NTB_CNTL_LINK_DISABLE;

	writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);



	ntb_transport_free(ndev->ntb_transport);

drivers/ntb/ntb_regs.h

+3 −2
Original line number Diff line number Diff line
@@ -46,8 +46,6 @@ 
 * Jon Mason <jon.mason@intel.com>

 */



#define NTB_LINK_ENABLE		0x0000

#define NTB_LINK_DISABLE	0x0002

#define NTB_LINK_STATUS_ACTIVE	0x2000

#define NTB_LINK_SPEED_MASK	0x000f

#define NTB_LINK_WIDTH_MASK	0x03f0
@@ -65,6 +63,7 @@ 


#define SNB_PCICMD_OFFSET	0x0504

#define SNB_DEVCTRL_OFFSET	0x0598

#define SNB_SLINK_STATUS_OFFSET	0x05A2

#define SNB_LINK_STATUS_OFFSET	0x01A2



#define SNB_PBAR2LMT_OFFSET	0x0000
@@ -147,6 +146,8 @@ 
#define BWD_LTSSMSTATEJMP_FORCEDETECT	(1 << 2)

#define BWD_IBIST_ERR_OFLOW	0x7FFF7FFF



#define NTB_CNTL_CFG_LOCK	(1 << 0)

#define NTB_CNTL_LINK_DISABLE	(1 << 1)

#define NTB_CNTL_BAR23_SNOOP	(1 << 2)

#define NTB_CNTL_BAR45_SNOOP	(1 << 6)

#define BWD_CNTL_LINK_DOWN	(1 << 16)

CRA Git | Maintained and supported by SUSTech CRA and CCSE