Commit 5ecef9c8 authored by Chuck Lever's avatar Chuck Lever Committed by Anna Schumaker
Browse files

xprtrdma: Introduce FRWR completion IDs 



Set up a completion ID in each rpcrdma_frwr. The ID is used to match
an incoming completion to a transport (CQ) and other MR-related
activity.

Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Signed-off-by: default avatarAnna Schumaker <Anna.Schumaker@Netapp.com>
parent b2e7467f

include/trace/events/rpcrdma.h

+4 −40
Original line number Diff line number Diff line
@@ -261,41 +261,6 @@ DECLARE_EVENT_CLASS(xprtrdma_wrch_event, 
				),					\

				TP_ARGS(task, mr, nsegs))



DECLARE_EVENT_CLASS(xprtrdma_frwr_done,

	TP_PROTO(

		const struct ib_wc *wc,

		const struct rpcrdma_frwr *frwr

	),



	TP_ARGS(wc, frwr),



	TP_STRUCT__entry(

		__field(u32, mr_id)

		__field(unsigned int, status)

		__field(unsigned int, vendor_err)

	),



	TP_fast_assign(

		__entry->mr_id = frwr->fr_mr->res.id;

		__entry->status = wc->status;

		__entry->vendor_err = __entry->status ? wc->vendor_err : 0;

	),



	TP_printk(

		"mr.id=%u: %s (%u/0x%x)",

		__entry->mr_id, rdma_show_wc_status(__entry->status),

		__entry->status, __entry->vendor_err

	)

);



#define DEFINE_FRWR_DONE_EVENT(name)					\

		DEFINE_EVENT(xprtrdma_frwr_done, name,			\

				TP_PROTO(				\

					const struct ib_wc *wc,		\

					const struct rpcrdma_frwr *frwr	\

				),					\

				TP_ARGS(wc, frwr))



TRACE_DEFINE_ENUM(DMA_BIDIRECTIONAL);

TRACE_DEFINE_ENUM(DMA_TO_DEVICE);

TRACE_DEFINE_ENUM(DMA_FROM_DEVICE);
@@ -850,11 +815,10 @@ TRACE_EVENT(xprtrdma_post_linv, 


DEFINE_COMPLETION_EVENT(xprtrdma_wc_receive);

DEFINE_COMPLETION_EVENT(xprtrdma_wc_send);



DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_fastreg);

DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li);

DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_wake);

DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_done);

DEFINE_COMPLETION_EVENT(xprtrdma_wc_fastreg);

DEFINE_COMPLETION_EVENT(xprtrdma_wc_li);

DEFINE_COMPLETION_EVENT(xprtrdma_wc_li_wake);

DEFINE_COMPLETION_EVENT(xprtrdma_wc_li_done);



TRACE_EVENT(xprtrdma_frwr_alloc,

	TP_PROTO(

net/sunrpc/xprtrdma/frwr_ops.c

+22 −7
Original line number Diff line number Diff line
@@ -363,12 +363,21 @@ static void frwr_wc_fastreg(struct ib_cq *cq, struct ib_wc *wc) 
		container_of(cqe, struct rpcrdma_frwr, fr_cqe);



	/* WARNING: Only wr_cqe and status are reliable at this point */

	trace_xprtrdma_wc_fastreg(wc, frwr);

	trace_xprtrdma_wc_fastreg(wc, &frwr->fr_cid);

	/* The MR will get recycled when the associated req is retransmitted */



	rpcrdma_flush_disconnect(cq->cq_context, wc);

}



static void frwr_cid_init(struct rpcrdma_ep *ep,

			  struct rpcrdma_frwr *frwr)

{

	struct rpc_rdma_cid *cid = &frwr->fr_cid;



	cid->ci_queue_id = ep->re_attr.send_cq->res.id;

	cid->ci_completion_id = frwr->fr_mr->res.id;

}



/**

 * frwr_send - post Send WRs containing the RPC Call message

 * @r_xprt: controlling transport instance
@@ -385,6 +394,7 @@ static void frwr_wc_fastreg(struct ib_cq *cq, struct ib_wc *wc) 
 */

int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)

{

	struct rpcrdma_ep *ep = r_xprt->rx_ep;

	struct ib_send_wr *post_wr;

	struct rpcrdma_mr *mr;
@@ -395,6 +405,7 @@ int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) 
		frwr = &mr->frwr;



		frwr->fr_cqe.done = frwr_wc_fastreg;

		frwr_cid_init(ep, frwr);

		frwr->fr_regwr.wr.next = post_wr;

		frwr->fr_regwr.wr.wr_cqe = &frwr->fr_cqe;

		frwr->fr_regwr.wr.num_sge = 0;
@@ -404,7 +415,7 @@ int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) 
		post_wr = &frwr->fr_regwr.wr;

	}



	return ib_post_send(r_xprt->rx_ep->re_id->qp, post_wr, NULL);

	return ib_post_send(ep->re_id->qp, post_wr, NULL);

}



/**
@@ -448,7 +459,7 @@ static void frwr_wc_localinv(struct ib_cq *cq, struct ib_wc *wc) 
	struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);



	/* WARNING: Only wr_cqe and status are reliable at this point */

	trace_xprtrdma_wc_li(wc, frwr);

	trace_xprtrdma_wc_li(wc, &frwr->fr_cid);

	__frwr_release_mr(wc, mr);



	rpcrdma_flush_disconnect(cq->cq_context, wc);
@@ -469,7 +480,7 @@ static void frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc) 
	struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);



	/* WARNING: Only wr_cqe and status are reliable at this point */

	trace_xprtrdma_wc_li_wake(wc, frwr);

	trace_xprtrdma_wc_li_wake(wc, &frwr->fr_cid);

	__frwr_release_mr(wc, mr);

	complete(&frwr->fr_linv_done);
@@ -490,6 +501,7 @@ static void frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc) 
void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)

{

	struct ib_send_wr *first, **prev, *last;

	struct rpcrdma_ep *ep = r_xprt->rx_ep;

	const struct ib_send_wr *bad_wr;

	struct rpcrdma_frwr *frwr;

	struct rpcrdma_mr *mr;
@@ -509,6 +521,7 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) 


		frwr = &mr->frwr;

		frwr->fr_cqe.done = frwr_wc_localinv;

		frwr_cid_init(ep, frwr);

		last = &frwr->fr_invwr;

		last->next = NULL;

		last->wr_cqe = &frwr->fr_cqe;
@@ -534,7 +547,7 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) 
	 * unless re_id->qp is a valid pointer.

	 */

	bad_wr = NULL;

	rc = ib_post_send(r_xprt->rx_ep->re_id->qp, first, &bad_wr);

	rc = ib_post_send(ep->re_id->qp, first, &bad_wr);



	/* The final LOCAL_INV WR in the chain is supposed to

	 * do the wake. If it was never posted, the wake will
@@ -574,7 +587,7 @@ static void frwr_wc_localinv_done(struct ib_cq *cq, struct ib_wc *wc) 
	struct rpcrdma_rep *rep = mr->mr_req->rl_reply;



	/* WARNING: Only wr_cqe and status are reliable at this point */

	trace_xprtrdma_wc_li_done(wc, frwr);

	trace_xprtrdma_wc_li_done(wc, &frwr->fr_cid);

	__frwr_release_mr(wc, mr);



	/* Ensure @rep is generated before __frwr_release_mr */
@@ -597,6 +610,7 @@ static void frwr_wc_localinv_done(struct ib_cq *cq, struct ib_wc *wc) 
void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)

{

	struct ib_send_wr *first, *last, **prev;

	struct rpcrdma_ep *ep = r_xprt->rx_ep;

	const struct ib_send_wr *bad_wr;

	struct rpcrdma_frwr *frwr;

	struct rpcrdma_mr *mr;
@@ -614,6 +628,7 @@ void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) 


		frwr = &mr->frwr;

		frwr->fr_cqe.done = frwr_wc_localinv;

		frwr_cid_init(ep, frwr);

		last = &frwr->fr_invwr;

		last->next = NULL;

		last->wr_cqe = &frwr->fr_cqe;
@@ -639,7 +654,7 @@ void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) 
	 * unless re_id->qp is a valid pointer.

	 */

	bad_wr = NULL;

	rc = ib_post_send(r_xprt->rx_ep->re_id->qp, first, &bad_wr);

	rc = ib_post_send(ep->re_id->qp, first, &bad_wr);

	if (!rc)

		return;

net/sunrpc/xprtrdma/xprt_rdma.h

+1 −0
Original line number Diff line number Diff line
@@ -231,6 +231,7 @@ struct rpcrdma_sendctx { 
struct rpcrdma_frwr {

	struct ib_mr			*fr_mr;

	struct ib_cqe			fr_cqe;

	struct rpc_rdma_cid		fr_cid;

	struct completion		fr_linv_done;

	union {

		struct ib_reg_wr	fr_regwr;

CRA Git | Maintained and supported by SUSTech CRA and CCSE