Commit b9548514 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'ntb-3.13' of git://github.com/jonmason/ntb 

Pull non-transparent bridge updates from Jon Mason:
 "NTB driver bug fixes to address a missed call to pci_enable_msix,
  NTB-RP Link Up issue, Xeon Doorbell errata workaround, ntb_transport
  link down race, and correct dmaengine_get/put usage.

  Also, clean-ups to remove duplicate defines and document a hardware
  errata.  Finally, some changes to improve performance"

* tag 'ntb-3.13' of git://github.com/jonmason/ntb:
  NTB: Disable interrupts and poll under high load
  NTB: Enable Snoop on Primary Side
  NTB: Document HW errata
  NTB: remove duplicate defines
  NTB: correct dmaengine_get/put usage
  NTB: Fix ntb_transport link down race
  ntb: Fix missed call to pci_enable_msix()
  NTB: Fix NTB-RP Link Up
  NTB: Xeon Doorbell errata workaround
parents b975dc36 e8aeb60c

drivers/ntb/ntb_hw.c

+103 −18
Original line number Diff line number Diff line
@@ -141,6 +141,24 @@ void ntb_unregister_event_callback(struct ntb_device *ndev) 
	ndev->event_cb = NULL;

}



static void ntb_irq_work(unsigned long data)

{

	struct ntb_db_cb *db_cb = (struct ntb_db_cb *)data;

	int rc;



	rc = db_cb->callback(db_cb->data, db_cb->db_num);

	if (rc)

		tasklet_schedule(&db_cb->irq_work);

	else {

		struct ntb_device *ndev = db_cb->ndev;

		unsigned long mask;



		mask = readw(ndev->reg_ofs.ldb_mask);

		clear_bit(db_cb->db_num * ndev->bits_per_vector, &mask);

		writew(mask, ndev->reg_ofs.ldb_mask);

	}

}



/**

 * ntb_register_db_callback() - register a callback for doorbell interrupt

 * @ndev: pointer to ntb_device instance
@@ -155,7 +173,7 @@ void ntb_unregister_event_callback(struct ntb_device *ndev) 
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.

 */

int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,

			     void *data, void (*func)(void *data, int db_num))

			     void *data, int (*func)(void *data, int db_num))

{

	unsigned long mask;
@@ -166,6 +184,10 @@ int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx, 


	ndev->db_cb[idx].callback = func;

	ndev->db_cb[idx].data = data;

	ndev->db_cb[idx].ndev = ndev;



	tasklet_init(&ndev->db_cb[idx].irq_work, ntb_irq_work,

		     (unsigned long) &ndev->db_cb[idx]);



	/* unmask interrupt */

	mask = readw(ndev->reg_ofs.ldb_mask);
@@ -194,6 +216,8 @@ void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx) 
	set_bit(idx * ndev->bits_per_vector, &mask);

	writew(mask, ndev->reg_ofs.ldb_mask);



	tasklet_disable(&ndev->db_cb[idx].irq_work);



	ndev->db_cb[idx].callback = NULL;

}
@@ -678,6 +702,7 @@ static int ntb_xeon_setup(struct ntb_device *ndev) 
				return -EINVAL;



			ndev->limits.max_mw = SNB_ERRATA_MAX_MW;

			ndev->limits.max_db_bits = SNB_MAX_DB_BITS;

			ndev->reg_ofs.spad_write = ndev->mw[1].vbase +

						   SNB_SPAD_OFFSET;

			ndev->reg_ofs.rdb = ndev->mw[1].vbase +
@@ -688,8 +713,21 @@ static int ntb_xeon_setup(struct ntb_device *ndev) 
			 */

			writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base +

			       SNB_PBAR4LMT_OFFSET);

			/* HW errata on the Limit registers.  They can only be

			 * written when the base register is 4GB aligned and

			 * < 32bit.  This should already be the case based on the

			 * driver defaults, but write the Limit registers first

			 * just in case.

			 */

		} else {

			ndev->limits.max_mw = SNB_MAX_MW;



			/* HW Errata on bit 14 of b2bdoorbell register.  Writes

			 * will not be mirrored to the remote system.  Shrink

			 * the number of bits by one, since bit 14 is the last

			 * bit.

			 */

			ndev->limits.max_db_bits = SNB_MAX_DB_BITS - 1;

			ndev->reg_ofs.spad_write = ndev->reg_base +

						   SNB_B2B_SPAD_OFFSET;

			ndev->reg_ofs.rdb = ndev->reg_base +
@@ -699,6 +737,12 @@ static int ntb_xeon_setup(struct ntb_device *ndev) 
			 * something silly

			 */

			writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET);

			/* HW errata on the Limit registers.  They can only be

			 * written when the base register is 4GB aligned and

			 * < 32bit.  This should already be the case based on the

			 * driver defaults, but write the Limit registers first

			 * just in case.

			 */

		}



		/* The Xeon errata workaround requires setting SBAR Base
@@ -769,6 +813,7 @@ static int ntb_xeon_setup(struct ntb_device *ndev) 
		 * have an equal amount.

		 */

		ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;

		ndev->limits.max_db_bits = SNB_MAX_DB_BITS;

		/* Note: The SDOORBELL is the cause of the errata.  You REALLY

		 * don't want to touch it.

		 */
@@ -793,6 +838,7 @@ static int ntb_xeon_setup(struct ntb_device *ndev) 
		 * have an equal amount.

		 */

		ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;

		ndev->limits.max_db_bits = SNB_MAX_DB_BITS;

		ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;

		ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET;

		ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET;
@@ -819,7 +865,6 @@ static int ntb_xeon_setup(struct ntb_device *ndev) 
	ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET;

	ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;



	ndev->limits.max_db_bits = SNB_MAX_DB_BITS;

	ndev->limits.msix_cnt = SNB_MSIX_CNT;

	ndev->bits_per_vector = SNB_DB_BITS_PER_VEC;
@@ -934,12 +979,16 @@ static irqreturn_t bwd_callback_msix_irq(int irq, void *data) 
{

	struct ntb_db_cb *db_cb = data;

	struct ntb_device *ndev = db_cb->ndev;

	unsigned long mask;



	dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,

		db_cb->db_num);



	if (db_cb->callback)

		db_cb->callback(db_cb->data, db_cb->db_num);

	mask = readw(ndev->reg_ofs.ldb_mask);

	set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);

	writew(mask, ndev->reg_ofs.ldb_mask);



	tasklet_schedule(&db_cb->irq_work);



	/* No need to check for the specific HB irq, any interrupt means

	 * we're connected.
@@ -955,12 +1004,16 @@ static irqreturn_t xeon_callback_msix_irq(int irq, void *data) 
{

	struct ntb_db_cb *db_cb = data;

	struct ntb_device *ndev = db_cb->ndev;

	unsigned long mask;



	dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,

		db_cb->db_num);



	if (db_cb->callback)

		db_cb->callback(db_cb->data, db_cb->db_num);

	mask = readw(ndev->reg_ofs.ldb_mask);

	set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);

	writew(mask, ndev->reg_ofs.ldb_mask);



	tasklet_schedule(&db_cb->irq_work);



	/* On Sandybridge, there are 16 bits in the interrupt register

	 * but only 4 vectors.  So, 5 bits are assigned to the first 3
@@ -986,7 +1039,7 @@ static irqreturn_t xeon_event_msix_irq(int irq, void *dev) 
		dev_err(&ndev->pdev->dev, "Error determining link status\n");



	/* bit 15 is always the link bit */

	writew(1 << ndev->limits.max_db_bits, ndev->reg_ofs.ldb);

	writew(1 << SNB_LINK_DB, ndev->reg_ofs.ldb);



	return IRQ_HANDLED;

}
@@ -1075,6 +1128,10 @@ static int ntb_setup_msix(struct ntb_device *ndev) 
			 "Only %d MSI-X vectors.  Limiting the number of queues to that number.\n",

			 rc);

		msix_entries = rc;



		rc = pci_enable_msix(pdev, ndev->msix_entries, msix_entries);

		if (rc)

			goto err1;

	}



	for (i = 0; i < msix_entries; i++) {
@@ -1176,9 +1233,10 @@ static int ntb_setup_interrupts(struct ntb_device *ndev) 
	 */

	if (ndev->hw_type == BWD_HW)

		writeq(~0, ndev->reg_ofs.ldb_mask);

	else

		writew(~(1 << ndev->limits.max_db_bits),

		       ndev->reg_ofs.ldb_mask);

	else {

		u16 var = 1 << SNB_LINK_DB;

		writew(~var, ndev->reg_ofs.ldb_mask);

	}



	rc = ntb_setup_msix(ndev);

	if (!rc)
@@ -1286,6 +1344,39 @@ static void ntb_free_debugfs(struct ntb_device *ndev) 
	}

}



static void ntb_hw_link_up(struct ntb_device *ndev)

{

	if (ndev->conn_type == NTB_CONN_TRANSPARENT)

		ntb_link_event(ndev, NTB_LINK_UP);

	else {

		u32 ntb_cntl;



		/* Let's bring the NTB link up */

		ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);

		ntb_cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);

		ntb_cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;

		ntb_cntl |= NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP;

		writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);

	}

}



static void ntb_hw_link_down(struct ntb_device *ndev)

{

	u32 ntb_cntl;



	if (ndev->conn_type == NTB_CONN_TRANSPARENT) {

		ntb_link_event(ndev, NTB_LINK_DOWN);

		return;

	}



	/* Bring NTB link down */

	ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);

	ntb_cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);

	ntb_cntl &= ~(NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP);

	ntb_cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;

	writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);

}



static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)

{

	struct ntb_device *ndev;
@@ -1374,9 +1465,7 @@ static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 
	if (rc)

		goto err6;



	/* Let's bring the NTB link up */

	writel(NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP,

	       ndev->reg_ofs.lnk_cntl);

	ntb_hw_link_up(ndev);



	return 0;
@@ -1406,12 +1495,8 @@ static void ntb_pci_remove(struct pci_dev *pdev) 
{

	struct ntb_device *ndev = pci_get_drvdata(pdev);

	int i;

	u32 ntb_cntl;



	/* Bring NTB link down */

	ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);

	ntb_cntl |= NTB_CNTL_LINK_DISABLE;

	writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);

	ntb_hw_link_down(ndev);



	ntb_transport_free(ndev->ntb_transport);

drivers/ntb/ntb_hw.h

+4 −3
Original line number Diff line number Diff line
@@ -106,10 +106,11 @@ struct ntb_mw { 
};



struct ntb_db_cb {

	void (*callback) (void *data, int db_num);

	int (*callback)(void *data, int db_num);

	unsigned int db_num;

	void *data;

	struct ntb_device *ndev;

	struct tasklet_struct irq_work;

};



struct ntb_device {
@@ -228,7 +229,7 @@ struct ntb_device *ntb_register_transport(struct pci_dev *pdev, 
void ntb_unregister_transport(struct ntb_device *ndev);

void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr);

int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,

			     void *data, void (*db_cb_func) (void *data,

			     void *data, int (*db_cb_func)(void *data,

							   int db_num));

void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx);

int ntb_register_event_callback(struct ntb_device *ndev,

drivers/ntb/ntb_regs.h

+8 −8
Original line number Diff line number Diff line
@@ -55,6 +55,7 @@ 
#define SNB_MAX_COMPAT_SPADS	16

/* Reserve the uppermost bit for link interrupt */

#define SNB_MAX_DB_BITS		15

#define SNB_LINK_DB		15

#define SNB_DB_BITS_PER_VEC	5

#define SNB_MAX_MW		2

#define SNB_ERRATA_MAX_MW	1
@@ -75,9 +76,6 @@ 
#define SNB_SBAR2XLAT_OFFSET	0x0030

#define SNB_SBAR4XLAT_OFFSET	0x0038

#define SNB_SBAR0BASE_OFFSET	0x0040

#define SNB_SBAR0BASE_OFFSET	0x0040

#define SNB_SBAR2BASE_OFFSET	0x0048

#define SNB_SBAR4BASE_OFFSET	0x0050

#define SNB_SBAR2BASE_OFFSET	0x0048

#define SNB_SBAR4BASE_OFFSET	0x0050

#define SNB_NTBCNTL_OFFSET	0x0058
@@ -147,8 +145,10 @@ 


#define NTB_CNTL_CFG_LOCK		(1 << 0)

#define NTB_CNTL_LINK_DISABLE		(1 << 1)

#define NTB_CNTL_BAR23_SNOOP	(1 << 2)

#define NTB_CNTL_BAR45_SNOOP	(1 << 6)

#define NTB_CNTL_S2P_BAR23_SNOOP	(1 << 2)

#define NTB_CNTL_P2S_BAR23_SNOOP	(1 << 4)

#define NTB_CNTL_S2P_BAR45_SNOOP	(1 << 6)

#define NTB_CNTL_P2S_BAR45_SNOOP	(1 << 8)

#define BWD_CNTL_LINK_DOWN		(1 << 16)



#define NTB_PPD_OFFSET		0x00D4

drivers/ntb/ntb_transport.c

+41 −36
Original line number Diff line number Diff line
@@ -119,7 +119,6 @@ struct ntb_transport_qp { 


	void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data,

			    void *data, int len);

	struct tasklet_struct rx_work;

	struct list_head rx_pend_q;

	struct list_head rx_free_q;

	spinlock_t ntb_rx_pend_q_lock;
@@ -584,11 +583,8 @@ static int ntb_set_mw(struct ntb_transport *nt, int num_mw, unsigned int size) 
	return 0;

}



static void ntb_qp_link_cleanup(struct work_struct *work)

static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)

{

	struct ntb_transport_qp *qp = container_of(work,

						   struct ntb_transport_qp,

						   link_cleanup);

	struct ntb_transport *nt = qp->transport;

	struct pci_dev *pdev = ntb_query_pdev(nt->ndev);
@@ -602,6 +598,16 @@ static void ntb_qp_link_cleanup(struct work_struct *work) 


	dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num);

	qp->qp_link = NTB_LINK_DOWN;

}



static void ntb_qp_link_cleanup_work(struct work_struct *work)

{

	struct ntb_transport_qp *qp = container_of(work,

						   struct ntb_transport_qp,

						   link_cleanup);

	struct ntb_transport *nt = qp->transport;



	ntb_qp_link_cleanup(qp);



	if (nt->transport_link == NTB_LINK_UP)

		schedule_delayed_work(&qp->link_work,
@@ -613,22 +619,20 @@ static void ntb_qp_link_down(struct ntb_transport_qp *qp) 
	schedule_work(&qp->link_cleanup);

}



static void ntb_transport_link_cleanup(struct work_struct *work)

static void ntb_transport_link_cleanup(struct ntb_transport *nt)

{

	struct ntb_transport *nt = container_of(work, struct ntb_transport,

						link_cleanup);

	int i;



	/* Pass along the info to any clients */

	for (i = 0; i < nt->max_qps; i++)

		if (!test_bit(i, &nt->qp_bitmap))

			ntb_qp_link_cleanup(&nt->qps[i]);



	if (nt->transport_link == NTB_LINK_DOWN)

		cancel_delayed_work_sync(&nt->link_work);

	else

		nt->transport_link = NTB_LINK_DOWN;



	/* Pass along the info to any clients */

	for (i = 0; i < nt->max_qps; i++)

		if (!test_bit(i, &nt->qp_bitmap))

			ntb_qp_link_down(&nt->qps[i]);



	/* The scratchpad registers keep the values if the remote side

	 * goes down, blast them now to give them a sane value the next

	 * time they are accessed
@@ -637,6 +641,14 @@ static void ntb_transport_link_cleanup(struct work_struct *work) 
		ntb_write_local_spad(nt->ndev, i, 0);

}



static void ntb_transport_link_cleanup_work(struct work_struct *work)

{

	struct ntb_transport *nt = container_of(work, struct ntb_transport,

						link_cleanup);



	ntb_transport_link_cleanup(nt);

}



static void ntb_transport_event_callback(void *data, enum ntb_hw_event event)

{

	struct ntb_transport *nt = data;
@@ -880,7 +892,7 @@ static int ntb_transport_init_queue(struct ntb_transport *nt, 
	}



	INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work);

	INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup);

	INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work);



	spin_lock_init(&qp->ntb_rx_pend_q_lock);

	spin_lock_init(&qp->ntb_rx_free_q_lock);
@@ -936,7 +948,7 @@ int ntb_transport_init(struct pci_dev *pdev) 
	}



	INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);

	INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup);

	INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work);



	rc = ntb_register_event_callback(nt->ndev,

					 ntb_transport_event_callback);
@@ -972,7 +984,7 @@ void ntb_transport_free(void *transport) 
	struct ntb_device *ndev = nt->ndev;

	int i;



	nt->transport_link = NTB_LINK_DOWN;

	ntb_transport_link_cleanup(nt);



	/* verify that all the qp's are freed */

	for (i = 0; i < nt->max_qps; i++) {
@@ -1188,11 +1200,14 @@ err: 
	goto out;

}



static void ntb_transport_rx(unsigned long data)

static int ntb_transport_rxc_db(void *data, int db_num)

{

	struct ntb_transport_qp *qp = (struct ntb_transport_qp *)data;

	struct ntb_transport_qp *qp = data;

	int rc, i;



	dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%s: doorbell %d received\n",

		__func__, db_num);



	/* Limit the number of packets processed in a single interrupt to

	 * provide fairness to others

	 */
@@ -1204,16 +1219,8 @@ static void ntb_transport_rx(unsigned long data) 


	if (qp->dma_chan)

		dma_async_issue_pending(qp->dma_chan);

}



static void ntb_transport_rxc_db(void *data, int db_num)

{

	struct ntb_transport_qp *qp = data;



	dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%s: doorbell %d received\n",

		__func__, db_num);



	tasklet_schedule(&qp->rx_work);

	return i;

}



static void ntb_tx_copy_callback(void *data)
@@ -1432,11 +1439,12 @@ ntb_transport_create_queue(void *data, struct pci_dev *pdev, 
	qp->tx_handler = handlers->tx_handler;

	qp->event_handler = handlers->event_handler;



	dmaengine_get();

	qp->dma_chan = dma_find_channel(DMA_MEMCPY);

	if (!qp->dma_chan)

	if (!qp->dma_chan) {

		dmaengine_put();

		dev_info(&pdev->dev, "Unable to allocate DMA channel, using CPU instead\n");

	else

		dmaengine_get();

	}



	for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {

		entry = kzalloc(sizeof(struct ntb_queue_entry), GFP_ATOMIC);
@@ -1458,25 +1466,23 @@ ntb_transport_create_queue(void *data, struct pci_dev *pdev, 
			     &qp->tx_free_q);

	}



	tasklet_init(&qp->rx_work, ntb_transport_rx, (unsigned long) qp);



	rc = ntb_register_db_callback(qp->ndev, free_queue, qp,

				      ntb_transport_rxc_db);

	if (rc)

		goto err3;

		goto err2;



	dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num);



	return qp;



err3:

	tasklet_disable(&qp->rx_work);

err2:

	while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))

		kfree(entry);

err1:

	while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))

		kfree(entry);

	if (qp->dma_chan)

		dmaengine_put();

	set_bit(free_queue, &nt->qp_bitmap);

err:

	return NULL;
@@ -1515,7 +1521,6 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp) 
	}



	ntb_unregister_db_callback(qp->ndev, qp->qp_num);

	tasklet_disable(&qp->rx_work);



	cancel_delayed_work_sync(&qp->link_work);

CRA Git | Maintained and supported by SUSTech CRA and CCSE