Commit e509be89 authored by Ralph Campbell's avatar Ralph Campbell Committed by Roland Dreier
Browse files

IB/ipath: Fix many locking issues when switching to error state 



The send DMA hardware queue voided a number of prior assumptions about
when a send is complete which led to completions being generated out of
order.  There were also a number of locking issues when switching the QP
to the error or reset states, and we implement the IB_QPS_SQD state.

Signed-off-by: default avatarRalph Campbell <ralph.campbell@qlogic.com>
Signed-off-by: default avatarRoland Dreier <rolandd@cisco.com>
parent 53dc1ca1

drivers/infiniband/hw/ipath/ipath_qp.c

+91 −92
Original line number Diff line number Diff line
@@ -242,7 +242,6 @@ static void ipath_free_qp(struct ipath_qp_table *qpt, struct ipath_qp *qp) 
{

	struct ipath_qp *q, **qpp;

	unsigned long flags;

	int fnd = 0;



	spin_lock_irqsave(&qpt->lock, flags);
@@ -253,51 +252,40 @@ static void ipath_free_qp(struct ipath_qp_table *qpt, struct ipath_qp *qp) 
			*qpp = qp->next;

			qp->next = NULL;

			atomic_dec(&qp->refcount);

			fnd = 1;

			break;

		}

	}



	spin_unlock_irqrestore(&qpt->lock, flags);



	if (!fnd)

		return;



	free_qpn(qpt, qp->ibqp.qp_num);



	wait_event(qp->wait, !atomic_read(&qp->refcount));

}



/**

 * ipath_free_all_qps - remove all QPs from the table

 * ipath_free_all_qps - check for QPs still in use

 * @qpt: the QP table to empty

 *

 * There should not be any QPs still in use.

 * Free memory for table.

 */

void ipath_free_all_qps(struct ipath_qp_table *qpt)

unsigned ipath_free_all_qps(struct ipath_qp_table *qpt)

{

	unsigned long flags;

	struct ipath_qp *qp, *nqp;

	u32 n;

	struct ipath_qp *qp;

	u32 n, qp_inuse = 0;



	for (n = 0; n < qpt->max; n++) {

	spin_lock_irqsave(&qpt->lock, flags);

	for (n = 0; n < qpt->max; n++) {

		qp = qpt->table[n];

		qpt->table[n] = NULL;

		spin_unlock_irqrestore(&qpt->lock, flags);



		while (qp) {

			nqp = qp->next;

			free_qpn(qpt, qp->ibqp.qp_num);

			if (!atomic_dec_and_test(&qp->refcount) ||

			    !ipath_destroy_qp(&qp->ibqp))

				ipath_dbg("QP memory leak!\n");

			qp = nqp;

		}

		for (; qp; qp = qp->next)

			qp_inuse++;

	}

	spin_unlock_irqrestore(&qpt->lock, flags);



	for (n = 0; n < ARRAY_SIZE(qpt->map); n++) {

	for (n = 0; n < ARRAY_SIZE(qpt->map); n++)

		if (qpt->map[n].page)

			free_page((unsigned long) qpt->map[n].page);

	}

	return qp_inuse;

}



/**
@@ -336,11 +324,12 @@ static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type) 
	qp->remote_qpn = 0;

	qp->qkey = 0;

	qp->qp_access_flags = 0;

	qp->s_busy = 0;

	atomic_set(&qp->s_dma_busy, 0);

	qp->s_flags &= IPATH_S_SIGNAL_REQ_WR;

	qp->s_hdrwords = 0;

	qp->s_wqe = NULL;

	qp->s_pkt_delay = 0;

	qp->s_draining = 0;

	qp->s_psn = 0;

	qp->r_psn = 0;

	qp->r_msn = 0;
@@ -353,7 +342,8 @@ static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type) 
	}

	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;

	qp->r_nak_state = 0;

	qp->r_wrid_valid = 0;

	qp->r_aflags = 0;

	qp->r_flags = 0;

	qp->s_rnr_timeout = 0;

	qp->s_head = 0;

	qp->s_tail = 0;
@@ -361,7 +351,6 @@ static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type) 
	qp->s_last = 0;

	qp->s_ssn = 1;

	qp->s_lsn = 0;

	qp->s_wait_credit = 0;

	memset(qp->s_ack_queue, 0, sizeof(qp->s_ack_queue));

	qp->r_head_ack_queue = 0;

	qp->s_tail_ack_queue = 0;
@@ -370,7 +359,6 @@ static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type) 
		qp->r_rq.wq->head = 0;

		qp->r_rq.wq->tail = 0;

	}

	qp->r_reuse_sge = 0;

}



/**
@@ -402,39 +390,21 @@ int ipath_error_qp(struct ipath_qp *qp, enum ib_wc_status err) 
		list_del_init(&qp->piowait);

	spin_unlock(&dev->pending_lock);



	wc.vendor_err = 0;

	wc.byte_len = 0;

	wc.imm_data = 0;

	/* Schedule the sending tasklet to drain the send work queue. */

	if (qp->s_last != qp->s_head)

		ipath_schedule_send(qp);



	memset(&wc, 0, sizeof(wc));

	wc.qp = &qp->ibqp;

	wc.src_qp = 0;

	wc.wc_flags = 0;

	wc.pkey_index = 0;

	wc.slid = 0;

	wc.sl = 0;

	wc.dlid_path_bits = 0;

	wc.port_num = 0;

	if (qp->r_wrid_valid) {

		qp->r_wrid_valid = 0;

		wc.wr_id = qp->r_wr_id;

	wc.opcode = IB_WC_RECV;



	if (test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags)) {

		wc.wr_id = qp->r_wr_id;

		wc.status = err;

		ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);

	}

	wc.status = IB_WC_WR_FLUSH_ERR;



	while (qp->s_last != qp->s_head) {

		struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);



		wc.wr_id = wqe->wr.wr_id;

		wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode];

		if (++qp->s_last >= qp->s_size)

			qp->s_last = 0;

		ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 1);

	}

	qp->s_cur = qp->s_tail = qp->s_head;

	qp->s_hdrwords = 0;

	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;



	if (qp->r_rq.wq) {

		struct ipath_rwq *wq;

		u32 head;
@@ -450,7 +420,6 @@ int ipath_error_qp(struct ipath_qp *qp, enum ib_wc_status err) 
		tail = wq->tail;

		if (tail >= qp->r_rq.size)

			tail = 0;

		wc.opcode = IB_WC_RECV;

		while (tail != head) {

			wc.wr_id = get_rwqe_ptr(&qp->r_rq, tail)->wr_id;

			if (++tail >= qp->r_rq.size)
@@ -482,11 +451,10 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 
	struct ipath_ibdev *dev = to_idev(ibqp->device);

	struct ipath_qp *qp = to_iqp(ibqp);

	enum ib_qp_state cur_state, new_state;

	unsigned long flags;

	int lastwqe = 0;

	int ret;



	spin_lock_irqsave(&qp->s_lock, flags);

	spin_lock_irq(&qp->s_lock);



	cur_state = attr_mask & IB_QP_CUR_STATE ?

		attr->cur_qp_state : qp->state;
@@ -539,16 +507,42 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 


	switch (new_state) {

	case IB_QPS_RESET:

		if (qp->state != IB_QPS_RESET) {

			qp->state = IB_QPS_RESET;

			spin_lock(&dev->pending_lock);

			if (!list_empty(&qp->timerwait))

				list_del_init(&qp->timerwait);

			if (!list_empty(&qp->piowait))

				list_del_init(&qp->piowait);

			spin_unlock(&dev->pending_lock);

			qp->s_flags &= ~IPATH_S_ANY_WAIT;

			spin_unlock_irq(&qp->s_lock);

			/* Stop the sending tasklet */

			tasklet_kill(&qp->s_task);

			wait_event(qp->wait_dma, !atomic_read(&qp->s_dma_busy));

			spin_lock_irq(&qp->s_lock);

		}

		ipath_reset_qp(qp, ibqp->qp_type);

		break;



	case IB_QPS_SQD:

		qp->s_draining = qp->s_last != qp->s_cur;

		qp->state = new_state;

		break;



	case IB_QPS_SQE:

		if (qp->ibqp.qp_type == IB_QPT_RC)

			goto inval;

		qp->state = new_state;

		break;



	case IB_QPS_ERR:

		lastwqe = ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR);

		break;



	default:

		qp->state = new_state;

		break;



	}



	if (attr_mask & IB_QP_PKEY_INDEX)
@@ -601,8 +595,7 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 
	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC)

		qp->s_max_rd_atomic = attr->max_rd_atomic;



	qp->state = new_state;

	spin_unlock_irqrestore(&qp->s_lock, flags);

	spin_unlock_irq(&qp->s_lock);



	if (lastwqe) {

		struct ib_event ev;
@@ -616,7 +609,7 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 
	goto bail;



inval:

	spin_unlock_irqrestore(&qp->s_lock, flags);

	spin_unlock_irq(&qp->s_lock);

	ret = -EINVAL;



bail:
@@ -647,7 +640,7 @@ int ipath_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 
	attr->pkey_index = qp->s_pkey_index;

	attr->alt_pkey_index = 0;

	attr->en_sqd_async_notify = 0;

	attr->sq_draining = 0;

	attr->sq_draining = qp->s_draining;

	attr->max_rd_atomic = qp->s_max_rd_atomic;

	attr->max_dest_rd_atomic = qp->r_max_rd_atomic;

	attr->min_rnr_timer = qp->r_min_rnr_timer;
@@ -837,6 +830,7 @@ struct ib_qp *ipath_create_qp(struct ib_pd *ibpd, 
		spin_lock_init(&qp->r_rq.lock);

		atomic_set(&qp->refcount, 0);

		init_waitqueue_head(&qp->wait);

		init_waitqueue_head(&qp->wait_dma);

		tasklet_init(&qp->s_task, ipath_do_send, (unsigned long)qp);

		INIT_LIST_HEAD(&qp->piowait);

		INIT_LIST_HEAD(&qp->timerwait);
@@ -930,6 +924,7 @@ bail_ip: 
	else

		vfree(qp->r_rq.wq);

	ipath_free_qp(&dev->qp_table, qp);

	free_qpn(&dev->qp_table, qp->ibqp.qp_num);

bail_qp:

	kfree(qp);

bail_swq:
@@ -951,41 +946,44 @@ int ipath_destroy_qp(struct ib_qp *ibqp) 
{

	struct ipath_qp *qp = to_iqp(ibqp);

	struct ipath_ibdev *dev = to_idev(ibqp->device);

	unsigned long flags;



	spin_lock_irqsave(&qp->s_lock, flags);

	qp->state = IB_QPS_ERR;

	spin_unlock_irqrestore(&qp->s_lock, flags);

	spin_lock(&dev->n_qps_lock);

	dev->n_qps_allocated--;

	spin_unlock(&dev->n_qps_lock);



	/* Stop the sending tasklet. */

	/* Make sure HW and driver activity is stopped. */

	spin_lock_irq(&qp->s_lock);

	if (qp->state != IB_QPS_RESET) {

		qp->state = IB_QPS_RESET;

		spin_lock(&dev->pending_lock);

		if (!list_empty(&qp->timerwait))

			list_del_init(&qp->timerwait);

		if (!list_empty(&qp->piowait))

			list_del_init(&qp->piowait);

		spin_unlock(&dev->pending_lock);

		qp->s_flags &= ~IPATH_S_ANY_WAIT;

		spin_unlock_irq(&qp->s_lock);

		/* Stop the sending tasklet */

		tasklet_kill(&qp->s_task);

		wait_event(qp->wait_dma, !atomic_read(&qp->s_dma_busy));

	} else

		spin_unlock_irq(&qp->s_lock);



	ipath_free_qp(&dev->qp_table, qp);



	if (qp->s_tx) {

		atomic_dec(&qp->refcount);

		if (qp->s_tx->txreq.flags & IPATH_SDMA_TXREQ_F_FREEBUF)

			kfree(qp->s_tx->txreq.map_addr);

		spin_lock_irq(&dev->pending_lock);

		list_add(&qp->s_tx->txreq.list, &dev->txreq_free);

		spin_unlock_irq(&dev->pending_lock);

		qp->s_tx = NULL;

	}



	/* Make sure the QP isn't on the timeout list. */

	spin_lock_irqsave(&dev->pending_lock, flags);

	if (!list_empty(&qp->timerwait))

		list_del_init(&qp->timerwait);

	if (!list_empty(&qp->piowait))

		list_del_init(&qp->piowait);

	if (qp->s_tx)

		list_add(&qp->s_tx->txreq.list, &dev->txreq_free);

	spin_unlock_irqrestore(&dev->pending_lock, flags);

	wait_event(qp->wait, !atomic_read(&qp->refcount));



	/*

	 * Make sure that the QP is not in the QPN table so receive

	 * interrupts will discard packets for this QP.  XXX Also remove QP

	 * from multicast table.

	 */

	if (atomic_read(&qp->refcount) != 0)

		ipath_free_qp(&dev->qp_table, qp);

	/* all user's cleaned up, mark it available */

	free_qpn(&dev->qp_table, qp->ibqp.qp_num);

	spin_lock(&dev->n_qps_lock);

	dev->n_qps_allocated--;

	spin_unlock(&dev->n_qps_lock);



	if (qp->ip)

		kref_put(&qp->ip->ref, ipath_release_mmap_info);
@@ -1055,9 +1053,10 @@ void ipath_get_credit(struct ipath_qp *qp, u32 aeth) 
	}



	/* Restart sending if it was blocked due to lack of credits. */

	if (qp->s_cur != qp->s_head &&

	if ((qp->s_flags & IPATH_S_WAIT_SSN_CREDIT) &&

	    qp->s_cur != qp->s_head &&

	    (qp->s_lsn == (u32) -1 ||

	     ipath_cmp24(get_swqe_ptr(qp, qp->s_cur)->ssn,

			 qp->s_lsn + 1) <= 0))

		tasklet_hi_schedule(&qp->s_task);

		ipath_schedule_send(qp);

}

drivers/infiniband/hw/ipath/ipath_rc.c

+97 −54
Original line number Diff line number Diff line
@@ -92,6 +92,10 @@ static int ipath_make_rc_ack(struct ipath_ibdev *dev, struct ipath_qp *qp, 
	u32 bth0;

	u32 bth2;



	/* Don't send an ACK if we aren't supposed to. */

	if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))

		goto bail;



	/* header size in 32-bit words LRH+BTH = (8+12)/4. */

	hwords = 5;
@@ -238,14 +242,25 @@ int ipath_make_rc_req(struct ipath_qp *qp) 
	    ipath_make_rc_ack(dev, qp, ohdr, pmtu))

		goto done;



	if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK) ||

	    qp->s_rnr_timeout || qp->s_wait_credit)

	if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)) {

		if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND))

			goto bail;

		/* We are in the error state, flush the work request. */

		if (qp->s_last == qp->s_head)

			goto bail;

		/* If DMAs are in progress, we can't flush immediately. */

		if (atomic_read(&qp->s_dma_busy)) {

			qp->s_flags |= IPATH_S_WAIT_DMA;

			goto bail;

		}

		wqe = get_swqe_ptr(qp, qp->s_last);

		ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);

		goto done;

	}



	/* Limit the number of packets sent without an ACK. */

	if (ipath_cmp24(qp->s_psn, qp->s_last_psn + IPATH_PSN_CREDIT) > 0) {

		qp->s_wait_credit = 1;

		dev->n_rc_stalls++;

	/* Leave BUSY set until RNR timeout. */

	if (qp->s_rnr_timeout) {

		qp->s_flags |= IPATH_S_WAITING;

		goto bail;

	}
@@ -257,6 +272,9 @@ int ipath_make_rc_req(struct ipath_qp *qp) 
	wqe = get_swqe_ptr(qp, qp->s_cur);

	switch (qp->s_state) {

	default:

		if (!(ib_ipath_state_ops[qp->state] &

		    IPATH_PROCESS_NEXT_SEND_OK))

			goto bail;

		/*

		 * Resend an old request or start a new one.

		 *
@@ -294,8 +312,10 @@ int ipath_make_rc_req(struct ipath_qp *qp) 
		case IB_WR_SEND_WITH_IMM:

			/* If no credit, return. */

			if (qp->s_lsn != (u32) -1 &&

			    ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0)

			    ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {

				qp->s_flags |= IPATH_S_WAIT_SSN_CREDIT;

				goto bail;

			}

			wqe->lpsn = wqe->psn;

			if (len > pmtu) {

				wqe->lpsn += (len - 1) / pmtu;
@@ -325,8 +345,10 @@ int ipath_make_rc_req(struct ipath_qp *qp) 
		case IB_WR_RDMA_WRITE_WITH_IMM:

			/* If no credit, return. */

			if (qp->s_lsn != (u32) -1 &&

			    ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0)

			    ipath_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {

				qp->s_flags |= IPATH_S_WAIT_SSN_CREDIT;

				goto bail;

			}

			ohdr->u.rc.reth.vaddr =

				cpu_to_be64(wqe->wr.wr.rdma.remote_addr);

			ohdr->u.rc.reth.rkey =
@@ -570,7 +592,11 @@ int ipath_make_rc_req(struct ipath_qp *qp) 
	ipath_make_ruc_header(dev, qp, ohdr, bth0 | (qp->s_state << 24), bth2);

done:

	ret = 1;

	goto unlock;



bail:

	qp->s_flags &= ~IPATH_S_BUSY;

unlock:

	spin_unlock_irqrestore(&qp->s_lock, flags);

	return ret;

}
@@ -606,7 +632,11 @@ static void send_rc_ack(struct ipath_qp *qp) 


	spin_unlock_irqrestore(&qp->s_lock, flags);



	/* Don't try to send ACKs if the link isn't ACTIVE */

	dd = dev->dd;

	if (!(dd->ipath_flags & IPATH_LINKACTIVE))

		goto done;



	piobuf = ipath_getpiobuf(dd, 0, NULL);

	if (!piobuf) {

		/*
@@ -668,15 +698,16 @@ static void send_rc_ack(struct ipath_qp *qp) 
	goto done;



queue_ack:

	if (ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK) {

		dev->n_rc_qacks++;

		qp->s_flags |= IPATH_S_ACK_PENDING;

		qp->s_nak_state = qp->r_nak_state;

		qp->s_ack_psn = qp->r_ack_psn;

	spin_unlock_irqrestore(&qp->s_lock, flags);



	/* Call ipath_do_rc_send() in another thread. */

	tasklet_hi_schedule(&qp->s_task);



		/* Schedule the send tasklet. */

		ipath_schedule_send(qp);

	}

	spin_unlock_irqrestore(&qp->s_lock, flags);

done:

	return;

}
@@ -735,7 +766,7 @@ static void reset_psn(struct ipath_qp *qp, u32 psn) 
	/*

	 * Set the state to restart in the middle of a request.

	 * Don't change the s_sge, s_cur_sge, or s_cur_size.

	 * See ipath_do_rc_send().

	 * See ipath_make_rc_req().

	 */

	switch (opcode) {

	case IB_WR_SEND:
@@ -801,7 +832,7 @@ void ipath_restart_rc(struct ipath_qp *qp, u32 psn) 
		dev->n_rc_resends += (qp->s_psn - psn) & IPATH_PSN_MASK;



	reset_psn(qp, psn);

	tasklet_hi_schedule(&qp->s_task);

	ipath_schedule_send(qp);



bail:

	return;
@@ -809,13 +840,7 @@ bail: 


static inline void update_last_psn(struct ipath_qp *qp, u32 psn)

{

	if (qp->s_last_psn != psn) {

	qp->s_last_psn = psn;

		if (qp->s_wait_credit) {

			qp->s_wait_credit = 0;

			tasklet_hi_schedule(&qp->s_task);

		}

	}

}



/**
@@ -915,14 +940,10 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode, 
		     wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) {

			qp->s_num_rd_atomic--;

			/* Restart sending task if fence is complete */

			if ((qp->s_flags & IPATH_S_FENCE_PENDING) &&

			    !qp->s_num_rd_atomic) {

				qp->s_flags &= ~IPATH_S_FENCE_PENDING;

				tasklet_hi_schedule(&qp->s_task);

			} else if (qp->s_flags & IPATH_S_RDMAR_PENDING) {

				qp->s_flags &= ~IPATH_S_RDMAR_PENDING;

				tasklet_hi_schedule(&qp->s_task);

			}

			if (((qp->s_flags & IPATH_S_FENCE_PENDING) &&

			     !qp->s_num_rd_atomic) ||

			    qp->s_flags & IPATH_S_RDMAR_PENDING)

				ipath_schedule_send(qp);

		}

		/* Post a send completion queue entry if requested. */

		if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) ||
@@ -956,6 +977,8 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode, 
		} else {

			if (++qp->s_last >= qp->s_size)

				qp->s_last = 0;

			if (qp->state == IB_QPS_SQD && qp->s_last == qp->s_cur)

				qp->s_draining = 0;

			if (qp->s_last == qp->s_tail)

				break;

			wqe = get_swqe_ptr(qp, qp->s_last);
@@ -979,7 +1002,7 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode, 
			 */

			if (ipath_cmp24(qp->s_psn, psn) <= 0) {

				reset_psn(qp, psn + 1);

				tasklet_hi_schedule(&qp->s_task);

				ipath_schedule_send(qp);

			}

		} else if (ipath_cmp24(qp->s_psn, psn) <= 0) {

			qp->s_state = OP(SEND_LAST);
@@ -1018,6 +1041,7 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode, 
			ib_ipath_rnr_table[(aeth >> IPATH_AETH_CREDIT_SHIFT) &

					   IPATH_AETH_CREDIT_MASK];

		ipath_insert_rnr_queue(qp);

		ipath_schedule_send(qp);

		goto bail;



	case 3:		/* NAK */
@@ -1108,6 +1132,10 @@ static inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev, 


	spin_lock_irqsave(&qp->s_lock, flags);



	/* Double check we can process this now that we hold the s_lock. */

	if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))

		goto ack_done;



	/* Ignore invalid responses. */

	if (ipath_cmp24(psn, qp->s_next_psn) >= 0)

		goto ack_done;
@@ -1343,7 +1371,12 @@ static inline int ipath_rc_rcv_error(struct ipath_ibdev *dev, 
	psn &= IPATH_PSN_MASK;

	e = NULL;

	old_req = 1;



	spin_lock_irqsave(&qp->s_lock, flags);

	/* Double check we can process this now that we hold the s_lock. */

	if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))

		goto unlock_done;



	for (i = qp->r_head_ack_queue; ; i = prev) {

		if (i == qp->s_tail_ack_queue)

			old_req = 0;
@@ -1471,7 +1504,7 @@ static inline int ipath_rc_rcv_error(struct ipath_ibdev *dev, 
		break;

	}

	qp->r_nak_state = 0;

	tasklet_hi_schedule(&qp->s_task);

	ipath_schedule_send(qp);



unlock_done:

	spin_unlock_irqrestore(&qp->s_lock, flags);
@@ -1503,18 +1536,15 @@ void ipath_rc_error(struct ipath_qp *qp, enum ib_wc_status err) 


static inline void ipath_update_ack_queue(struct ipath_qp *qp, unsigned n)

{

	unsigned long flags;

	unsigned next;



	next = n + 1;

	if (next > IPATH_MAX_RDMA_ATOMIC)

		next = 0;

	spin_lock_irqsave(&qp->s_lock, flags);

	if (n == qp->s_tail_ack_queue) {

		qp->s_tail_ack_queue = next;

		qp->s_ack_state = OP(ACKNOWLEDGE);

	}

	spin_unlock_irqrestore(&qp->s_lock, flags);

}



/**
@@ -1543,6 +1573,7 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
	int diff;

	struct ib_reth *reth;

	int header_in_data;

	unsigned long flags;



	/* Validate the SLID. See Ch. 9.6.1.5 */

	if (unlikely(be16_to_cpu(hdr->lrh[3]) != qp->remote_ah_attr.dlid))
@@ -1690,9 +1721,8 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
			goto nack_inv;

		ipath_copy_sge(&qp->r_sge, data, tlen);

		qp->r_msn++;

		if (!qp->r_wrid_valid)

		if (!test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags))

			break;

		qp->r_wrid_valid = 0;

		wc.wr_id = qp->r_wr_id;

		wc.status = IB_WC_SUCCESS;

		if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) ||
@@ -1764,9 +1794,13 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
		next = qp->r_head_ack_queue + 1;

		if (next > IPATH_MAX_RDMA_ATOMIC)

			next = 0;

		spin_lock_irqsave(&qp->s_lock, flags);

		/* Double check we can process this while holding the s_lock. */

		if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))

			goto unlock;

		if (unlikely(next == qp->s_tail_ack_queue)) {

			if (!qp->s_ack_queue[next].sent)

				goto nack_inv;

				goto nack_inv_unlck;

			ipath_update_ack_queue(qp, next);

		}

		e = &qp->s_ack_queue[qp->r_head_ack_queue];
@@ -1787,7 +1821,7 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
			ok = ipath_rkey_ok(qp, &e->rdma_sge, len, vaddr,

					   rkey, IB_ACCESS_REMOTE_READ);

			if (unlikely(!ok))

				goto nack_acc;

				goto nack_acc_unlck;

			/*

			 * Update the next expected PSN.  We add 1 later

			 * below, so only add the remainder here.
@@ -1814,13 +1848,12 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
		qp->r_psn++;

		qp->r_state = opcode;

		qp->r_nak_state = 0;

		barrier();

		qp->r_head_ack_queue = next;



		/* Call ipath_do_rc_send() in another thread. */

		tasklet_hi_schedule(&qp->s_task);

		/* Schedule the send tasklet. */

		ipath_schedule_send(qp);



		goto done;

		goto unlock;

	}



	case OP(COMPARE_SWAP):
@@ -1839,9 +1872,13 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
		next = qp->r_head_ack_queue + 1;

		if (next > IPATH_MAX_RDMA_ATOMIC)

			next = 0;

		spin_lock_irqsave(&qp->s_lock, flags);

		/* Double check we can process this while holding the s_lock. */

		if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK))

			goto unlock;

		if (unlikely(next == qp->s_tail_ack_queue)) {

			if (!qp->s_ack_queue[next].sent)

				goto nack_inv;

				goto nack_inv_unlck;

			ipath_update_ack_queue(qp, next);

		}

		if (!header_in_data)
@@ -1851,13 +1888,13 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
		vaddr = ((u64) be32_to_cpu(ateth->vaddr[0]) << 32) |

			be32_to_cpu(ateth->vaddr[1]);

		if (unlikely(vaddr & (sizeof(u64) - 1)))

			goto nack_inv;

			goto nack_inv_unlck;

		rkey = be32_to_cpu(ateth->rkey);

		/* Check rkey & NAK */

		if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge,

					    sizeof(u64), vaddr, rkey,

					    IB_ACCESS_REMOTE_ATOMIC)))

			goto nack_acc;

			goto nack_acc_unlck;

		/* Perform atomic OP and save result. */

		maddr = (atomic64_t *) qp->r_sge.sge.vaddr;

		sdata = be64_to_cpu(ateth->swap_data);
@@ -1874,13 +1911,12 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
		qp->r_psn++;

		qp->r_state = opcode;

		qp->r_nak_state = 0;

		barrier();

		qp->r_head_ack_queue = next;



		/* Call ipath_do_rc_send() in another thread. */

		tasklet_hi_schedule(&qp->s_task);

		/* Schedule the send tasklet. */

		ipath_schedule_send(qp);



		goto done;

		goto unlock;

	}



	default:
@@ -1901,19 +1937,26 @@ rnr_nak: 
	qp->r_ack_psn = qp->r_psn;

	goto send_ack;



nack_inv_unlck:

	spin_unlock_irqrestore(&qp->s_lock, flags);

nack_inv:

	ipath_rc_error(qp, IB_WC_LOC_QP_OP_ERR);

	qp->r_nak_state = IB_NAK_INVALID_REQUEST;

	qp->r_ack_psn = qp->r_psn;

	goto send_ack;



nack_acc_unlck:

	spin_unlock_irqrestore(&qp->s_lock, flags);

nack_acc:

	ipath_rc_error(qp, IB_WC_LOC_PROT_ERR);

	qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;

	qp->r_ack_psn = qp->r_psn;

send_ack:

	send_rc_ack(qp);

	goto done;



unlock:

	spin_unlock_irqrestore(&qp->s_lock, flags);

done:

	return;

}

drivers/infiniband/hw/ipath/ipath_ruc.c

+115 −53

File changed.

Preview size limit exceeded, changes collapsed.

drivers/infiniband/hw/ipath/ipath_uc.c

+38 −19
Original line number Diff line number Diff line 
/*

 * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.

 * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.

 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.

 *

 * This software is available to you under a choice of one of two
@@ -47,14 +47,30 @@ int ipath_make_uc_req(struct ipath_qp *qp) 
{

	struct ipath_other_headers *ohdr;

	struct ipath_swqe *wqe;

	unsigned long flags;

	u32 hwords;

	u32 bth0;

	u32 len;

	u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);

	int ret = 0;



	if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK))

	spin_lock_irqsave(&qp->s_lock, flags);



	if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)) {

		if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND))

			goto bail;

		/* We are in the error state, flush the work request. */

		if (qp->s_last == qp->s_head)

			goto bail;

		/* If DMAs are in progress, we can't flush immediately. */

		if (atomic_read(&qp->s_dma_busy)) {

			qp->s_flags |= IPATH_S_WAIT_DMA;

			goto bail;

		}

		wqe = get_swqe_ptr(qp, qp->s_last);

		ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);

		goto done;

	}



	ohdr = &qp->s_hdr.u.oth;

	if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
@@ -69,9 +85,12 @@ int ipath_make_uc_req(struct ipath_qp *qp) 
	qp->s_wqe = NULL;

	switch (qp->s_state) {

	default:

		if (!(ib_ipath_state_ops[qp->state] &

		    IPATH_PROCESS_NEXT_SEND_OK))

			goto bail;

		/* Check if send work queue is empty. */

		if (qp->s_cur == qp->s_head)

			goto done;

			goto bail;

		/*

		 * Start a new request.

		 */
@@ -134,7 +153,7 @@ int ipath_make_uc_req(struct ipath_qp *qp) 
			break;



		default:

			goto done;

			goto bail;

		}

		break;
@@ -194,9 +213,14 @@ int ipath_make_uc_req(struct ipath_qp *qp) 
	ipath_make_ruc_header(to_idev(qp->ibqp.device),

			      qp, ohdr, bth0 | (qp->s_state << 24),

			      qp->s_next_psn++ & IPATH_PSN_MASK);

done:

	ret = 1;

	goto unlock;



done:

bail:

	qp->s_flags &= ~IPATH_S_BUSY;

unlock:

	spin_unlock_irqrestore(&qp->s_lock, flags);

	return ret;

}
@@ -258,8 +282,7 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
	 */

	opcode = be32_to_cpu(ohdr->bth[0]) >> 24;



	wc.imm_data = 0;

	wc.wc_flags = 0;

	memset(&wc, 0, sizeof wc);



	/* Compare the PSN verses the expected PSN. */

	if (unlikely(ipath_cmp24(psn, qp->r_psn) != 0)) {
@@ -322,8 +345,8 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
	case OP(SEND_ONLY):

	case OP(SEND_ONLY_WITH_IMMEDIATE):

	send_first:

		if (qp->r_reuse_sge) {

			qp->r_reuse_sge = 0;

		if (qp->r_flags & IPATH_R_REUSE_SGE) {

			qp->r_flags &= ~IPATH_R_REUSE_SGE;

			qp->r_sge = qp->s_rdma_read_sge;

		} else if (!ipath_get_rwqe(qp, 0)) {

			dev->n_pkt_drops++;
@@ -340,13 +363,13 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
	case OP(SEND_MIDDLE):

		/* Check for invalid length PMTU or posted rwqe len. */

		if (unlikely(tlen != (hdrsize + pmtu + 4))) {

			qp->r_reuse_sge = 1;

			qp->r_flags |= IPATH_R_REUSE_SGE;

			dev->n_pkt_drops++;

			goto done;

		}

		qp->r_rcv_len += pmtu;

		if (unlikely(qp->r_rcv_len > qp->r_len)) {

			qp->r_reuse_sge = 1;

			qp->r_flags |= IPATH_R_REUSE_SGE;

			dev->n_pkt_drops++;

			goto done;

		}
@@ -372,7 +395,7 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
		/* Check for invalid length. */

		/* XXX LAST len should be >= 1 */

		if (unlikely(tlen < (hdrsize + pad + 4))) {

			qp->r_reuse_sge = 1;

			qp->r_flags |= IPATH_R_REUSE_SGE;

			dev->n_pkt_drops++;

			goto done;

		}
@@ -380,7 +403,7 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
		tlen -= (hdrsize + pad + 4);

		wc.byte_len = tlen + qp->r_rcv_len;

		if (unlikely(wc.byte_len > qp->r_len)) {

			qp->r_reuse_sge = 1;

			qp->r_flags |= IPATH_R_REUSE_SGE;

			dev->n_pkt_drops++;

			goto done;

		}
@@ -390,14 +413,10 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
		wc.wr_id = qp->r_wr_id;

		wc.status = IB_WC_SUCCESS;

		wc.opcode = IB_WC_RECV;

		wc.vendor_err = 0;

		wc.qp = &qp->ibqp;

		wc.src_qp = qp->remote_qpn;

		wc.pkey_index = 0;

		wc.slid = qp->remote_ah_attr.dlid;

		wc.sl = qp->remote_ah_attr.sl;

		wc.dlid_path_bits = 0;

		wc.port_num = 0;

		/* Signal completion event if the solicited bit is set. */

		ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,

			       (ohdr->bth[0] &
@@ -488,8 +507,8 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
			dev->n_pkt_drops++;

			goto done;

		}

		if (qp->r_reuse_sge)

			qp->r_reuse_sge = 0;

		if (qp->r_flags & IPATH_R_REUSE_SGE)

			qp->r_flags &= ~IPATH_R_REUSE_SGE;

		else if (!ipath_get_rwqe(qp, 1)) {

			dev->n_pkt_drops++;

			goto done;

drivers/infiniband/hw/ipath/ipath_ud.c

+48 −18
Original line number Diff line number Diff line
@@ -65,9 +65,9 @@ static void ipath_ud_loopback(struct ipath_qp *sqp, struct ipath_swqe *swqe) 
	u32 length;



	qp = ipath_lookup_qpn(&dev->qp_table, swqe->wr.wr.ud.remote_qpn);

	if (!qp) {

	if (!qp || !(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) {

		dev->n_pkt_drops++;

		goto send_comp;

		goto done;

	}



	rsge.sg_list = NULL;
@@ -91,14 +91,12 @@ static void ipath_ud_loopback(struct ipath_qp *sqp, struct ipath_swqe *swqe) 
	 * present on the wire.

	 */

	length = swqe->length;

	memset(&wc, 0, sizeof wc);

	wc.byte_len = length + sizeof(struct ib_grh);



	if (swqe->wr.opcode == IB_WR_SEND_WITH_IMM) {

		wc.wc_flags = IB_WC_WITH_IMM;

		wc.imm_data = swqe->wr.ex.imm_data;

	} else {

		wc.wc_flags = 0;

		wc.imm_data = 0;

	}



	/*
@@ -229,7 +227,6 @@ static void ipath_ud_loopback(struct ipath_qp *sqp, struct ipath_swqe *swqe) 
	}

	wc.status = IB_WC_SUCCESS;

	wc.opcode = IB_WC_RECV;

	wc.vendor_err = 0;

	wc.qp = &qp->ibqp;

	wc.src_qp = sqp->ibqp.qp_num;

	/* XXX do we know which pkey matched? Only needed for GSI. */
@@ -248,8 +245,7 @@ drop: 
	kfree(rsge.sg_list);

	if (atomic_dec_and_test(&qp->refcount))

		wake_up(&qp->wait);

send_comp:

	ipath_send_complete(sqp, swqe, IB_WC_SUCCESS);

done:;

}



/**
@@ -264,6 +260,7 @@ int ipath_make_ud_req(struct ipath_qp *qp) 
	struct ipath_other_headers *ohdr;

	struct ib_ah_attr *ah_attr;

	struct ipath_swqe *wqe;

	unsigned long flags;

	u32 nwords;

	u32 extra_bytes;

	u32 bth0;
@@ -271,13 +268,30 @@ int ipath_make_ud_req(struct ipath_qp *qp) 
	u16 lid;

	int ret = 0;



	if (unlikely(!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK)))

	spin_lock_irqsave(&qp->s_lock, flags);



	if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_NEXT_SEND_OK)) {

		if (!(ib_ipath_state_ops[qp->state] & IPATH_FLUSH_SEND))

			goto bail;

		/* We are in the error state, flush the work request. */

		if (qp->s_last == qp->s_head)

			goto bail;

		/* If DMAs are in progress, we can't flush immediately. */

		if (atomic_read(&qp->s_dma_busy)) {

			qp->s_flags |= IPATH_S_WAIT_DMA;

			goto bail;

		}

		wqe = get_swqe_ptr(qp, qp->s_last);

		ipath_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);

		goto done;

	}



	if (qp->s_cur == qp->s_head)

		goto bail;



	wqe = get_swqe_ptr(qp, qp->s_cur);

	if (++qp->s_cur >= qp->s_size)

		qp->s_cur = 0;



	/* Construct the header. */

	ah_attr = &to_iah(wqe->wr.wr.ud.ah)->attr;
@@ -288,10 +302,23 @@ int ipath_make_ud_req(struct ipath_qp *qp) 
			dev->n_unicast_xmit++;

	} else {

		dev->n_unicast_xmit++;

		lid = ah_attr->dlid &

			~((1 << dev->dd->ipath_lmc) - 1);

		lid = ah_attr->dlid & ~((1 << dev->dd->ipath_lmc) - 1);

		if (unlikely(lid == dev->dd->ipath_lid)) {

			/*

			 * If DMAs are in progress, we can't generate

			 * a completion for the loopback packet since

			 * it would be out of order.

			 * XXX Instead of waiting, we could queue a

			 * zero length descriptor so we get a callback.

			 */

			if (atomic_read(&qp->s_dma_busy)) {

				qp->s_flags |= IPATH_S_WAIT_DMA;

				goto bail;

			}

			spin_unlock_irqrestore(&qp->s_lock, flags);

			ipath_ud_loopback(qp, wqe);

			spin_lock_irqsave(&qp->s_lock, flags);

			ipath_send_complete(qp, wqe, IB_WC_SUCCESS);

			goto done;

		}

	}
@@ -368,11 +395,13 @@ int ipath_make_ud_req(struct ipath_qp *qp) 
	ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);



done:

	if (++qp->s_cur >= qp->s_size)

		qp->s_cur = 0;

	ret = 1;

	goto unlock;



bail:

	qp->s_flags &= ~IPATH_S_BUSY;

unlock:

	spin_unlock_irqrestore(&qp->s_lock, flags);

	return ret;

}
@@ -506,8 +535,8 @@ void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
	/*

	 * Get the next work request entry to find where to put the data.

	 */

	if (qp->r_reuse_sge)

		qp->r_reuse_sge = 0;

	if (qp->r_flags & IPATH_R_REUSE_SGE)

		qp->r_flags &= ~IPATH_R_REUSE_SGE;

	else if (!ipath_get_rwqe(qp, 0)) {

		/*

		 * Count VL15 packets dropped due to no receive buffer.
@@ -523,7 +552,7 @@ void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
	}

	/* Silently drop packets which are too big. */

	if (wc.byte_len > qp->r_len) {

		qp->r_reuse_sge = 1;

		qp->r_flags |= IPATH_R_REUSE_SGE;

		dev->n_pkt_drops++;

		goto bail;

	}
@@ -535,7 +564,8 @@ void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
		ipath_skip_sge(&qp->r_sge, sizeof(struct ib_grh));

	ipath_copy_sge(&qp->r_sge, data,

		       wc.byte_len - sizeof(struct ib_grh));

	qp->r_wrid_valid = 0;

	if (!test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags))

		goto bail;

	wc.wr_id = qp->r_wr_id;

	wc.status = IB_WC_SUCCESS;

	wc.opcode = IB_WC_RECV;

CRA Git | Maintained and supported by SUSTech CRA and CCSE