Commit 7ce1a3ee authored by Erez Shitrit's avatar Erez Shitrit Committed by Doug Ledford
Browse files

IB/IPoIB: Separate control from HW operation on ipoib_open/stop ndo 



This patch is preparing the netdev part at the IPoIB driver to be able
to use the ipoib_options.

It deals with the two flows from the .ndo: ipoib_open and ipoib_stop.

The code is rearranged as follows:
 * All operations which deal with the hardware resources, (for example
   change QP state, post-receive etc.) are performed in one place.
 * All operations that are control oriented (like restart multicast task,
   start the reap_ah etc.) are performed in separate place.

The functions that deal with the hardware resources now located at
__ipoib_ib_dev_open for the ipoib_open flow and __ipoib_ib_dev_stop
for ipoib_stop.

Signed-off-by: default avatarErez Shitrit <erezsh@mellanox.com>
Reviewed-by: default avatarAlex Vesker <valex@mellanox.com>
Signed-off-by: default avatarLeon Romanovsky <leon@kernel.org>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 515ed4f3

drivers/infiniband/ulp/ipoib/ipoib.h

+1 −1
Original line number Diff line number Diff line
@@ -503,7 +503,7 @@ void ipoib_dev_uninit_default(struct net_device *dev); 
int ipoib_ib_dev_open(struct net_device *dev);

void ipoib_ib_dev_up(struct net_device *dev);

void ipoib_ib_dev_down(struct net_device *dev);

void ipoib_ib_dev_stop(struct net_device *dev);

int ipoib_ib_dev_stop_default(struct net_device *dev);

void ipoib_pkey_dev_check_presence(struct net_device *dev);



int ipoib_dev_init(struct net_device *dev, struct ib_device *ca, int port);

drivers/infiniband/ulp/ipoib/ipoib_ib.c

+129 −101
Original line number Diff line number Diff line
@@ -692,24 +692,115 @@ static void ipoib_stop_ah(struct net_device *dev) 
	ipoib_flush_ah(dev);

}



void ipoib_ib_tx_timer_func(unsigned long ctx)

static int recvs_pending(struct net_device *dev)

{

	drain_tx_cq((struct net_device *)ctx);

	struct ipoib_dev_priv *priv = netdev_priv(dev);

	int pending = 0;

	int i;



	for (i = 0; i < ipoib_recvq_size; ++i)

		if (priv->rx_ring[i].skb)

			++pending;



	return pending;

}



int ipoib_ib_dev_open(struct net_device *dev)

int ipoib_ib_dev_stop_default(struct net_device *dev)

{

	struct ipoib_dev_priv *priv = netdev_priv(dev);

	int ret;

	struct ib_qp_attr qp_attr;

	unsigned long begin;

	struct ipoib_tx_buf *tx_req;

	int i;



	ipoib_pkey_dev_check_presence(dev);

	if (test_and_clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags))

		napi_disable(&priv->napi);



	if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {

		ipoib_warn(priv, "P_Key 0x%04x is %s\n", priv->pkey,

			   (!(priv->pkey & 0x7fff) ? "Invalid" : "not found"));

		return -1;

	ipoib_cm_dev_stop(dev);



	/*

	 * Move our QP to the error state and then reinitialize in

	 * when all work requests have completed or have been flushed.

	 */

	qp_attr.qp_state = IB_QPS_ERR;

	if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))

		ipoib_warn(priv, "Failed to modify QP to ERROR state\n");



	/* Wait for all sends and receives to complete */

	begin = jiffies;



	while (priv->tx_head != priv->tx_tail || recvs_pending(dev)) {

		if (time_after(jiffies, begin + 5 * HZ)) {

			ipoib_warn(priv,

				   "timing out; %d sends %d receives not completed\n",

				   priv->tx_head - priv->tx_tail,

				   recvs_pending(dev));



			/*

			 * assume the HW is wedged and just free up

			 * all our pending work requests.

			 */

			while ((int)priv->tx_tail - (int)priv->tx_head < 0) {

				tx_req = &priv->tx_ring[priv->tx_tail &

							(ipoib_sendq_size - 1)];

				ipoib_dma_unmap_tx(priv, tx_req);

				dev_kfree_skb_any(tx_req->skb);

				++priv->tx_tail;

				--priv->tx_outstanding;

			}



			for (i = 0; i < ipoib_recvq_size; ++i) {

				struct ipoib_rx_buf *rx_req;



				rx_req = &priv->rx_ring[i];

				if (!rx_req->skb)

					continue;

				ipoib_ud_dma_unmap_rx(priv,

						      priv->rx_ring[i].mapping);

				dev_kfree_skb_any(rx_req->skb);

				rx_req->skb = NULL;

			}



			goto timeout;

		}



		ipoib_drain_cq(dev);



		msleep(1);

	}



	ipoib_dbg(priv, "All sends and receives done.\n");



timeout:

	del_timer_sync(&priv->poll_timer);

	qp_attr.qp_state = IB_QPS_RESET;

	if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))

		ipoib_warn(priv, "Failed to modify QP to RESET state\n");



	ib_req_notify_cq(priv->recv_cq, IB_CQ_NEXT_COMP);



	return 0;

}



int ipoib_ib_dev_stop(struct net_device *dev)

{

	ipoib_ib_dev_stop_default(dev);



	ipoib_flush_ah(dev);



	return 0;

}



void ipoib_ib_tx_timer_func(unsigned long ctx)

{

	drain_tx_cq((struct net_device *)ctx);

}



int ipoib_ib_dev_open_default(struct net_device *dev)

{

	struct ipoib_dev_priv *priv = netdev_priv(dev);

	int ret;



	ret = ipoib_init_qp(dev);

	if (ret) {

		ipoib_warn(priv, "ipoib_init_qp returned %d\n", ret);
@@ -728,10 +819,6 @@ int ipoib_ib_dev_open(struct net_device *dev) 
		goto dev_stop;

	}



	clear_bit(IPOIB_STOP_REAPER, &priv->flags);

	queue_delayed_work(priv->wq, &priv->ah_reap_task,

			   round_jiffies_relative(HZ));



	if (!test_and_set_bit(IPOIB_FLAG_INITIALIZED, &priv->flags))

		napi_enable(&priv->napi);
@@ -743,6 +830,35 @@ dev_stop: 
	return -1;

}



int ipoib_ib_dev_open(struct net_device *dev)

{

	struct ipoib_dev_priv *priv = netdev_priv(dev);



	ipoib_pkey_dev_check_presence(dev);



	if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {

		ipoib_warn(priv, "P_Key 0x%04x is %s\n", priv->pkey,

			   (!(priv->pkey & 0x7fff) ? "Invalid" : "not found"));

		return -1;

	}



	clear_bit(IPOIB_STOP_REAPER, &priv->flags);

	queue_delayed_work(priv->wq, &priv->ah_reap_task,

			   round_jiffies_relative(HZ));



	if (ipoib_ib_dev_open_default(dev)) {

		pr_warn("%s: Failed to open dev\n", dev->name);

		goto stop_ah_reap;

	}



	return 0;



stop_ah_reap:

	set_bit(IPOIB_STOP_REAPER, &priv->flags);

	cancel_delayed_work(&priv->ah_reap_task);

	return -1;

}



void ipoib_pkey_dev_check_presence(struct net_device *dev)

{

	struct ipoib_dev_priv *priv = netdev_priv(dev);
@@ -786,19 +902,6 @@ void ipoib_ib_dev_down(struct net_device *dev) 
	ipoib_flush_paths(dev);

}



static int recvs_pending(struct net_device *dev)

{

	struct ipoib_dev_priv *priv = netdev_priv(dev);

	int pending = 0;

	int i;



	for (i = 0; i < ipoib_recvq_size; ++i)

		if (priv->rx_ring[i].skb)

			++pending;



	return pending;

}



void ipoib_drain_cq(struct net_device *dev)

{

	struct ipoib_dev_priv *priv = netdev_priv(dev);
@@ -838,81 +941,6 @@ void ipoib_drain_cq(struct net_device *dev) 
	local_bh_enable();

}



void ipoib_ib_dev_stop(struct net_device *dev)

{

	struct ipoib_dev_priv *priv = netdev_priv(dev);

	struct ib_qp_attr qp_attr;

	unsigned long begin;

	struct ipoib_tx_buf *tx_req;

	int i;



	if (test_and_clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags))

		napi_disable(&priv->napi);



	ipoib_cm_dev_stop(dev);



	/*

	 * Move our QP to the error state and then reinitialize in

	 * when all work requests have completed or have been flushed.

	 */

	qp_attr.qp_state = IB_QPS_ERR;

	if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))

		ipoib_warn(priv, "Failed to modify QP to ERROR state\n");



	/* Wait for all sends and receives to complete */

	begin = jiffies;



	while (priv->tx_head != priv->tx_tail || recvs_pending(dev)) {

		if (time_after(jiffies, begin + 5 * HZ)) {

			ipoib_warn(priv, "timing out; %d sends %d receives not completed\n",

				   priv->tx_head - priv->tx_tail, recvs_pending(dev));



			/*

			 * assume the HW is wedged and just free up

			 * all our pending work requests.

			 */

			while ((int) priv->tx_tail - (int) priv->tx_head < 0) {

				tx_req = &priv->tx_ring[priv->tx_tail &

							(ipoib_sendq_size - 1)];

				ipoib_dma_unmap_tx(priv, tx_req);

				dev_kfree_skb_any(tx_req->skb);

				++priv->tx_tail;

				--priv->tx_outstanding;

			}



			for (i = 0; i < ipoib_recvq_size; ++i) {

				struct ipoib_rx_buf *rx_req;



				rx_req = &priv->rx_ring[i];

				if (!rx_req->skb)

					continue;

				ipoib_ud_dma_unmap_rx(priv,

						      priv->rx_ring[i].mapping);

				dev_kfree_skb_any(rx_req->skb);

				rx_req->skb = NULL;

			}



			goto timeout;

		}



		ipoib_drain_cq(dev);



		msleep(1);

	}



	ipoib_dbg(priv, "All sends and receives done.\n");



timeout:

	del_timer_sync(&priv->poll_timer);

	qp_attr.qp_state = IB_QPS_RESET;

	if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))

		ipoib_warn(priv, "Failed to modify QP to RESET state\n");



	ipoib_flush_ah(dev);



	ib_req_notify_cq(priv->recv_cq, IB_CQ_NEXT_COMP);

}



/*

 * Takes whatever value which is in pkey index 0 and updates priv->pkey

 * returns 0 if the pkey value was changed.

drivers/infiniband/ulp/ipoib/ipoib_main.c

+1 −1
Original line number Diff line number Diff line
@@ -193,7 +193,7 @@ static int ipoib_stop(struct net_device *dev) 
	netif_stop_queue(dev);



	ipoib_ib_dev_down(dev);

	ipoib_ib_dev_stop(dev);

	ipoib_ib_dev_stop_default(dev);



	if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {

		struct ipoib_dev_priv *cpriv;

CRA Git | Maintained and supported by SUSTech CRA and CCSE