Commit 019f118b authored by Brian Welty's avatar Brian Welty Committed by Jason Gunthorpe
Browse files

IB/{hfi1, qib, rdmavt}: Move copy SGE logic into rdmavt 



This patch moves hfi1_copy_sge() into rdmavt for sharing with qib.
This patch also moves all the wss_*() functions into rdmavt as
several wss_*() functions are called from hfi1_copy_sge()

When SGE copy mode is adaptive, cacheless copy may be done in some cases
for performance reasons. In those cases, X86 cacheless copy function
is called since the drivers that use rdmavt and may set SGE copy mode
to adaptive are X86 only. For this reason, this patch adds
"depends on X86_64" to rdmavt/Kconfig.

Reviewed-by: default avatarAshutosh Dixit <ashutosh.dixit@intel.com>
Reviewed-by: default avatarMichael J. Ruhl <michael.j.ruhl@intel.com>
Reviewed-by: default avatarMike Marciniszyn <mike.marciniszyn@intel.com>
Reviewed-by: default avatarDennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: default avatarBrian Welty <brian.welty@intel.com>
Signed-off-by: default avatarHarish Chegondi <harish.chegondi@intel.com>
Signed-off-by: default avatarDennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@mellanox.com>
parent b56511c1

drivers/infiniband/hw/hfi1/init.c

+0 −6
Original line number Diff line number Diff line
@@ -1504,9 +1504,6 @@ static int __init hfi1_mod_init(void) 
	idr_init(&hfi1_unit_table);



	hfi1_dbg_init();

	ret = hfi1_wss_init();

	if (ret < 0)

		goto bail_wss;

	ret = pci_register_driver(&hfi1_pci_driver);

	if (ret < 0) {

		pr_err("Unable to register driver: error %d\n", -ret);
@@ -1515,8 +1512,6 @@ static int __init hfi1_mod_init(void) 
	goto bail; /* all OK */



bail_dev:

	hfi1_wss_exit();

bail_wss:

	hfi1_dbg_exit();

	idr_destroy(&hfi1_unit_table);

	dev_cleanup();
@@ -1533,7 +1528,6 @@ static void __exit hfi1_mod_cleanup(void) 
{

	pci_unregister_driver(&hfi1_pci_driver);

	node_affinity_destroy_all();

	hfi1_wss_exit();

	hfi1_dbg_exit();



	idr_destroy(&hfi1_unit_table);

drivers/infiniband/hw/hfi1/rc.c

+6 −4
Original line number Diff line number Diff line
@@ -1644,7 +1644,8 @@ read_middle: 
		qp->s_rdma_read_len -= pmtu;

		update_last_psn(qp, psn);

		spin_unlock_irqrestore(&qp->s_lock, flags);

		hfi1_copy_sge(&qp->s_rdma_read_sge, data, pmtu, false, false);

		rvt_copy_sge(qp, &qp->s_rdma_read_sge,

			     data, pmtu, false, false);

		goto bail;



	case OP(RDMA_READ_RESPONSE_ONLY):
@@ -1684,7 +1685,8 @@ read_last: 
		if (unlikely(tlen != qp->s_rdma_read_len))

			goto ack_len_err;

		aeth = be32_to_cpu(ohdr->u.aeth);

		hfi1_copy_sge(&qp->s_rdma_read_sge, data, tlen, false, false);

		rvt_copy_sge(qp, &qp->s_rdma_read_sge,

			     data, tlen, false, false);

		WARN_ON(qp->s_rdma_read_sge.num_sge);

		(void)do_rc_ack(qp, aeth, psn,

				 OP(RDMA_READ_RESPONSE_LAST), 0, rcd);
@@ -2144,7 +2146,7 @@ send_middle: 
		qp->r_rcv_len += pmtu;

		if (unlikely(qp->r_rcv_len > qp->r_len))

			goto nack_inv;

		hfi1_copy_sge(&qp->r_sge, data, pmtu, true, false);

		rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false);

		break;



	case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
@@ -2200,7 +2202,7 @@ send_last: 
		wc.byte_len = tlen + qp->r_rcv_len;

		if (unlikely(wc.byte_len > qp->r_len))

			goto nack_inv;

		hfi1_copy_sge(&qp->r_sge, data, tlen, true, copy_last);

		rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, copy_last);

		rvt_put_ss(&qp->r_sge);

		qp->r_msn++;

		if (!__test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))

drivers/infiniband/hw/hfi1/ruc.c

+2 −1
Original line number Diff line number Diff line
@@ -361,7 +361,8 @@ do_write: 
		if (len > sge->sge_length)

			len = sge->sge_length;

		WARN_ON_ONCE(len == 0);

		hfi1_copy_sge(&qp->r_sge, sge->vaddr, len, release, copy_last);

		rvt_copy_sge(qp, &qp->r_sge, sge->vaddr,

			     len, release, copy_last);

		sge->vaddr += len;

		sge->length -= len;

		sge->sge_length -= len;

drivers/infiniband/hw/hfi1/uc.c

+5 −5
Original line number Diff line number Diff line
@@ -426,7 +426,7 @@ send_first: 
		qp->r_rcv_len += pmtu;

		if (unlikely(qp->r_rcv_len > qp->r_len))

			goto rewind;

		hfi1_copy_sge(&qp->r_sge, data, pmtu, false, false);

		rvt_copy_sge(qp, &qp->r_sge, data, pmtu, false, false);

		break;



	case OP(SEND_LAST_WITH_IMMEDIATE):
@@ -449,7 +449,7 @@ send_last: 
		if (unlikely(wc.byte_len > qp->r_len))

			goto rewind;

		wc.opcode = IB_WC_RECV;

		hfi1_copy_sge(&qp->r_sge, data, tlen, false, false);

		rvt_copy_sge(qp, &qp->r_sge, data, tlen, false, false);

		rvt_put_ss(&qp->s_rdma_read_sge);

last_imm:

		wc.wr_id = qp->r_wr_id;
@@ -523,7 +523,7 @@ rdma_first: 
		qp->r_rcv_len += pmtu;

		if (unlikely(qp->r_rcv_len > qp->r_len))

			goto drop;

		hfi1_copy_sge(&qp->r_sge, data, pmtu, true, false);

		rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false);

		break;



	case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
@@ -550,7 +550,7 @@ rdma_last_imm: 
		}

		wc.byte_len = qp->r_len;

		wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;

		hfi1_copy_sge(&qp->r_sge, data, tlen, true, false);

		rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false);

		rvt_put_ss(&qp->r_sge);

		goto last_imm;
@@ -564,7 +564,7 @@ rdma_last: 
		tlen -= (hdrsize + extra_bytes);

		if (unlikely(tlen + qp->r_rcv_len != qp->r_len))

			goto drop;

		hfi1_copy_sge(&qp->r_sge, data, tlen, true, false);

		rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false);

		rvt_put_ss(&qp->r_sge);

		break;

drivers/infiniband/hw/hfi1/ud.c

+9 −9
Original line number Diff line number Diff line
@@ -210,7 +210,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) 
		}



		hfi1_make_grh(ibp, &grh, &grd, 0, 0);

		hfi1_copy_sge(&qp->r_sge, &grh,

		rvt_copy_sge(qp, &qp->r_sge, &grh,

			     sizeof(grh), true, false);

		wc.wc_flags |= IB_WC_GRH;

	} else {
@@ -228,7 +228,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) 
		if (len > sge->sge_length)

			len = sge->sge_length;

		WARN_ON_ONCE(len == 0);

		hfi1_copy_sge(&qp->r_sge, sge->vaddr, len, true, false);

		rvt_copy_sge(qp, &qp->r_sge, sge->vaddr, len, true, false);

		sge->vaddr += len;

		sge->length -= len;

		sge->sge_length -= len;
@@ -1019,7 +1019,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) 
		goto drop;

	}

	if (packet->grh) {

		hfi1_copy_sge(&qp->r_sge, packet->grh,

		rvt_copy_sge(qp, &qp->r_sge, packet->grh,

			     sizeof(struct ib_grh), true, false);

		wc.wc_flags |= IB_WC_GRH;

	} else if (packet->etype == RHF_RCV_TYPE_BYPASS) {
@@ -1030,13 +1030,13 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) 
		 * out when creating 16B, add back the GRH here.

		 */

		hfi1_make_ext_grh(packet, &grh, slid, dlid);

		hfi1_copy_sge(&qp->r_sge, &grh,

		rvt_copy_sge(qp, &qp->r_sge, &grh,

			     sizeof(struct ib_grh), true, false);

		wc.wc_flags |= IB_WC_GRH;

	} else {

		rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true);

	}

	hfi1_copy_sge(&qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh),

	rvt_copy_sge(qp, &qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh),

		     true, false);

	rvt_put_ss(&qp->r_sge);

	if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))

CRA Git | Maintained and supported by SUSTech CRA and CCSE