Commit 93bdcf9f authored by Trond Myklebust's avatar Trond Myklebust
Browse files

Merge tag 'nfs-rdma-for-4.20-1' of git://git.linux-nfs.org/projects/anna/linux-nfs 



NFS RDMA client updates for Linux 4.20

Stable bugfixes:
- Reset credit grant properly after a disconnect

Other bugfixes and cleanups:
- xprt_release_rqst_cong is called outside of transport_lock
- Create more MRs at a time and toss out old ones during recovery
- Various improvements to the RDMA connection and disconnection code:
  - Improve naming of trace events, functions, and variables
  - Add documenting comments
  - Fix metrics and stats reporting
- Fix a tracepoint sparse warning

Signed-off-by: default avatarTrond Myklebust <trond.myklebust@hammerspace.com>
parents 826799e6 470443e0

include/trace/events/rpcrdma.h

+9 −9
Original line number Diff line number Diff line
@@ -263,7 +263,7 @@ DECLARE_EVENT_CLASS(xprtrdma_mr, 
);



#define DEFINE_MR_EVENT(name) \

		DEFINE_EVENT(xprtrdma_mr, name, \

		DEFINE_EVENT(xprtrdma_mr, xprtrdma_mr_##name, \

				TP_PROTO( \

					const struct rpcrdma_mr *mr \

				), \
@@ -306,7 +306,7 @@ DECLARE_EVENT_CLASS(xprtrdma_cb_event, 
 ** Connection events

 **/



TRACE_EVENT(xprtrdma_conn_upcall,

TRACE_EVENT(xprtrdma_cm_event,

	TP_PROTO(

		const struct rpcrdma_xprt *r_xprt,

		struct rdma_cm_event *event
@@ -377,7 +377,7 @@ DEFINE_RXPRT_EVENT(xprtrdma_reinsert); 
DEFINE_RXPRT_EVENT(xprtrdma_reconnect);

DEFINE_RXPRT_EVENT(xprtrdma_inject_dsc);



TRACE_EVENT(xprtrdma_qp_error,

TRACE_EVENT(xprtrdma_qp_event,

	TP_PROTO(

		const struct rpcrdma_xprt *r_xprt,

		const struct ib_event *event
@@ -509,7 +509,7 @@ TRACE_EVENT(xprtrdma_post_send, 
	TP_STRUCT__entry(

		__field(const void *, req)

		__field(int, num_sge)

		__field(bool, signaled)

		__field(int, signaled)

		__field(int, status)

	),
@@ -651,11 +651,11 @@ DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_fastreg); 
DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li);

DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_wake);



DEFINE_MR_EVENT(xprtrdma_localinv);

DEFINE_MR_EVENT(xprtrdma_dma_map);

DEFINE_MR_EVENT(xprtrdma_dma_unmap);

DEFINE_MR_EVENT(xprtrdma_remoteinv);

DEFINE_MR_EVENT(xprtrdma_recover_mr);

DEFINE_MR_EVENT(localinv);

DEFINE_MR_EVENT(map);

DEFINE_MR_EVENT(unmap);

DEFINE_MR_EVENT(remoteinv);

DEFINE_MR_EVENT(recycle);



/**

 ** Reply events

net/sunrpc/xprt.c

+4 −10
Original line number Diff line number Diff line
@@ -834,17 +834,11 @@ void xprt_connect(struct rpc_task *task) 


static void xprt_connect_status(struct rpc_task *task)

{

	struct rpc_xprt	*xprt = task->tk_rqstp->rq_xprt;



	if (task->tk_status == 0) {

		xprt->stat.connect_count++;

		xprt->stat.connect_time += (long)jiffies - xprt->stat.connect_start;

	switch (task->tk_status) {

	case 0:

		dprintk("RPC: %5u xprt_connect_status: connection established\n",

				task->tk_pid);

		return;

	}



	switch (task->tk_status) {

		break;

	case -ECONNREFUSED:

	case -ECONNRESET:

	case -ECONNABORTED:
@@ -861,7 +855,7 @@ static void xprt_connect_status(struct rpc_task *task) 
	default:

		dprintk("RPC: %5u xprt_connect_status: error %d connecting to "

				"server %s\n", task->tk_pid, -task->tk_status,

				xprt->servername);

				task->tk_rqstp->rq_xprt->servername);

		task->tk_status = -EIO;

	}

}

net/sunrpc/xprtrdma/backchannel.c

+8 −8
Original line number Diff line number Diff line
@@ -53,9 +53,9 @@ static int rpcrdma_bc_setup_reqs(struct rpcrdma_xprt *r_xprt, 
		rqst->rq_xprt = xprt;

		INIT_LIST_HEAD(&rqst->rq_bc_list);

		__set_bit(RPC_BC_PA_IN_USE, &rqst->rq_bc_pa_state);

		spin_lock_bh(&xprt->bc_pa_lock);

		spin_lock(&xprt->bc_pa_lock);

		list_add(&rqst->rq_bc_pa_list, &xprt->bc_pa_list);

		spin_unlock_bh(&xprt->bc_pa_lock);

		spin_unlock(&xprt->bc_pa_lock);



		size = r_xprt->rx_data.inline_rsize;

		rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, GFP_KERNEL);
@@ -230,16 +230,16 @@ void xprt_rdma_bc_destroy(struct rpc_xprt *xprt, unsigned int reqs) 
	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);

	struct rpc_rqst *rqst, *tmp;



	spin_lock_bh(&xprt->bc_pa_lock);

	spin_lock(&xprt->bc_pa_lock);

	list_for_each_entry_safe(rqst, tmp, &xprt->bc_pa_list, rq_bc_pa_list) {

		list_del(&rqst->rq_bc_pa_list);

		spin_unlock_bh(&xprt->bc_pa_lock);

		spin_unlock(&xprt->bc_pa_lock);



		rpcrdma_bc_free_rqst(r_xprt, rqst);



		spin_lock_bh(&xprt->bc_pa_lock);

		spin_lock(&xprt->bc_pa_lock);

	}

	spin_unlock_bh(&xprt->bc_pa_lock);

	spin_unlock(&xprt->bc_pa_lock);

}



/**
@@ -257,9 +257,9 @@ void xprt_rdma_bc_free_rqst(struct rpc_rqst *rqst) 
	rpcrdma_recv_buffer_put(req->rl_reply);

	req->rl_reply = NULL;



	spin_lock_bh(&xprt->bc_pa_lock);

	spin_lock(&xprt->bc_pa_lock);

	list_add_tail(&rqst->rq_bc_pa_list, &xprt->bc_pa_list);

	spin_unlock_bh(&xprt->bc_pa_lock);

	spin_unlock(&xprt->bc_pa_lock);

}



/**

net/sunrpc/xprtrdma/fmr_ops.c

+60 −71
Original line number Diff line number Diff line
@@ -49,46 +49,7 @@ fmr_is_supported(struct rpcrdma_ia *ia) 
	return true;

}



static int

fmr_op_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr)

{

	static struct ib_fmr_attr fmr_attr = {

		.max_pages	= RPCRDMA_MAX_FMR_SGES,

		.max_maps	= 1,

		.page_shift	= PAGE_SHIFT

	};



	mr->fmr.fm_physaddrs = kcalloc(RPCRDMA_MAX_FMR_SGES,

				       sizeof(u64), GFP_KERNEL);

	if (!mr->fmr.fm_physaddrs)

		goto out_free;



	mr->mr_sg = kcalloc(RPCRDMA_MAX_FMR_SGES,

			    sizeof(*mr->mr_sg), GFP_KERNEL);

	if (!mr->mr_sg)

		goto out_free;



	sg_init_table(mr->mr_sg, RPCRDMA_MAX_FMR_SGES);



	mr->fmr.fm_mr = ib_alloc_fmr(ia->ri_pd, RPCRDMA_FMR_ACCESS_FLAGS,

				     &fmr_attr);

	if (IS_ERR(mr->fmr.fm_mr))

		goto out_fmr_err;



	INIT_LIST_HEAD(&mr->mr_list);

	return 0;



out_fmr_err:

	dprintk("RPC:       %s: ib_alloc_fmr returned %ld\n", __func__,

		PTR_ERR(mr->fmr.fm_mr));



out_free:

	kfree(mr->mr_sg);

	kfree(mr->fmr.fm_physaddrs);

	return -ENOMEM;

}



static int

static void

__fmr_unmap(struct rpcrdma_mr *mr)

{

	LIST_HEAD(l);
@@ -97,13 +58,16 @@ __fmr_unmap(struct rpcrdma_mr *mr) 
	list_add(&mr->fmr.fm_mr->list, &l);

	rc = ib_unmap_fmr(&l);

	list_del(&mr->fmr.fm_mr->list);

	return rc;

	if (rc)

		pr_err("rpcrdma: final ib_unmap_fmr for %p failed %i\n",

		       mr, rc);

}



/* Release an MR.

 */

static void

fmr_op_release_mr(struct rpcrdma_mr *mr)

{

	LIST_HEAD(unmap_list);

	int rc;



	kfree(mr->fmr.fm_physaddrs);
@@ -112,10 +76,7 @@ fmr_op_release_mr(struct rpcrdma_mr *mr) 
	/* In case this one was left mapped, try to unmap it

	 * to prevent dealloc_fmr from failing with EBUSY

	 */

	rc = __fmr_unmap(mr);

	if (rc)

		pr_err("rpcrdma: final ib_unmap_fmr for %p failed %i\n",

		       mr, rc);

	__fmr_unmap(mr);



	rc = ib_dealloc_fmr(mr->fmr.fm_mr);

	if (rc)
@@ -125,40 +86,68 @@ fmr_op_release_mr(struct rpcrdma_mr *mr) 
	kfree(mr);

}



/* Reset of a single FMR.

/* MRs are dynamically allocated, so simply clean up and release the MR.

 * A replacement MR will subsequently be allocated on demand.

 */

static void

fmr_op_recover_mr(struct rpcrdma_mr *mr)

fmr_mr_recycle_worker(struct work_struct *work)

{

	struct rpcrdma_mr *mr = container_of(work, struct rpcrdma_mr, mr_recycle);

	struct rpcrdma_xprt *r_xprt = mr->mr_xprt;

	int rc;



	/* ORDER: invalidate first */

	rc = __fmr_unmap(mr);

	if (rc)

		goto out_release;



	/* ORDER: then DMA unmap */

	rpcrdma_mr_unmap_and_put(mr);

	trace_xprtrdma_mr_recycle(mr);



	r_xprt->rx_stats.mrs_recovered++;

	return;



out_release:

	pr_err("rpcrdma: FMR reset failed (%d), %p released\n", rc, mr);

	r_xprt->rx_stats.mrs_orphaned++;



	trace_xprtrdma_dma_unmap(mr);

	trace_xprtrdma_mr_unmap(mr);

	ib_dma_unmap_sg(r_xprt->rx_ia.ri_device,

			mr->mr_sg, mr->mr_nents, mr->mr_dir);



	spin_lock(&r_xprt->rx_buf.rb_mrlock);

	list_del(&mr->mr_all);

	r_xprt->rx_stats.mrs_recycled++;

	spin_unlock(&r_xprt->rx_buf.rb_mrlock);



	fmr_op_release_mr(mr);

}



static int

fmr_op_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr)

{

	static struct ib_fmr_attr fmr_attr = {

		.max_pages	= RPCRDMA_MAX_FMR_SGES,

		.max_maps	= 1,

		.page_shift	= PAGE_SHIFT

	};



	mr->fmr.fm_physaddrs = kcalloc(RPCRDMA_MAX_FMR_SGES,

				       sizeof(u64), GFP_KERNEL);

	if (!mr->fmr.fm_physaddrs)

		goto out_free;



	mr->mr_sg = kcalloc(RPCRDMA_MAX_FMR_SGES,

			    sizeof(*mr->mr_sg), GFP_KERNEL);

	if (!mr->mr_sg)

		goto out_free;



	sg_init_table(mr->mr_sg, RPCRDMA_MAX_FMR_SGES);



	mr->fmr.fm_mr = ib_alloc_fmr(ia->ri_pd, RPCRDMA_FMR_ACCESS_FLAGS,

				     &fmr_attr);

	if (IS_ERR(mr->fmr.fm_mr))

		goto out_fmr_err;



	INIT_LIST_HEAD(&mr->mr_list);

	INIT_WORK(&mr->mr_recycle, fmr_mr_recycle_worker);

	return 0;



out_fmr_err:

	dprintk("RPC:       %s: ib_alloc_fmr returned %ld\n", __func__,

		PTR_ERR(mr->fmr.fm_mr));



out_free:

	kfree(mr->mr_sg);

	kfree(mr->fmr.fm_physaddrs);

	return -ENOMEM;

}



/* On success, sets:

 *	ep->rep_attr.cap.max_send_wr

 *	ep->rep_attr.cap.max_recv_wr
@@ -187,6 +176,7 @@ fmr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, 


	ia->ri_max_segs = max_t(unsigned int, 1, RPCRDMA_MAX_DATA_SEGS /

				RPCRDMA_MAX_FMR_SGES);

	ia->ri_max_segs += 2;	/* segments for head and tail buffers */

	return 0;

}
@@ -244,7 +234,7 @@ fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 
				     mr->mr_sg, i, mr->mr_dir);

	if (!mr->mr_nents)

		goto out_dmamap_err;

	trace_xprtrdma_dma_map(mr);

	trace_xprtrdma_mr_map(mr);



	for (i = 0, dma_pages = mr->fmr.fm_physaddrs; i < mr->mr_nents; i++)

		dma_pages[i] = sg_dma_address(&mr->mr_sg[i]);
@@ -305,13 +295,13 @@ fmr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs) 
	list_for_each_entry(mr, mrs, mr_list) {

		dprintk("RPC:       %s: unmapping fmr %p\n",

			__func__, &mr->fmr);

		trace_xprtrdma_localinv(mr);

		trace_xprtrdma_mr_localinv(mr);

		list_add_tail(&mr->fmr.fm_mr->list, &unmap_list);

	}

	r_xprt->rx_stats.local_inv_needed++;

	rc = ib_unmap_fmr(&unmap_list);

	if (rc)

		goto out_reset;

		goto out_release;



	/* ORDER: Now DMA unmap all of the req's MRs, and return

	 * them to the free MW list.
@@ -324,13 +314,13 @@ fmr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs) 


	return;



out_reset:

out_release:

	pr_err("rpcrdma: ib_unmap_fmr failed (%i)\n", rc);



	while (!list_empty(mrs)) {

		mr = rpcrdma_mr_pop(mrs);

		list_del(&mr->fmr.fm_mr->list);

		fmr_op_recover_mr(mr);

		rpcrdma_mr_recycle(mr);

	}

}
@@ -338,7 +328,6 @@ const struct rpcrdma_memreg_ops rpcrdma_fmr_memreg_ops = { 
	.ro_map				= fmr_op_map,

	.ro_send			= fmr_op_send,

	.ro_unmap_sync			= fmr_op_unmap_sync,

	.ro_recover_mr			= fmr_op_recover_mr,

	.ro_open			= fmr_op_open,

	.ro_maxpages			= fmr_op_maxpages,

	.ro_init_mr			= fmr_op_init_mr,

net/sunrpc/xprtrdma/frwr_ops.c

+51 −86
Original line number Diff line number Diff line
@@ -97,6 +97,44 @@ out_not_supported: 
	return false;

}



static void

frwr_op_release_mr(struct rpcrdma_mr *mr)

{

	int rc;



	rc = ib_dereg_mr(mr->frwr.fr_mr);

	if (rc)

		pr_err("rpcrdma: final ib_dereg_mr for %p returned %i\n",

		       mr, rc);

	kfree(mr->mr_sg);

	kfree(mr);

}



/* MRs are dynamically allocated, so simply clean up and release the MR.

 * A replacement MR will subsequently be allocated on demand.

 */

static void

frwr_mr_recycle_worker(struct work_struct *work)

{

	struct rpcrdma_mr *mr = container_of(work, struct rpcrdma_mr, mr_recycle);

	enum rpcrdma_frwr_state state = mr->frwr.fr_state;

	struct rpcrdma_xprt *r_xprt = mr->mr_xprt;



	trace_xprtrdma_mr_recycle(mr);



	if (state != FRWR_FLUSHED_LI) {

		trace_xprtrdma_mr_unmap(mr);

		ib_dma_unmap_sg(r_xprt->rx_ia.ri_device,

				mr->mr_sg, mr->mr_nents, mr->mr_dir);

	}



	spin_lock(&r_xprt->rx_buf.rb_mrlock);

	list_del(&mr->mr_all);

	r_xprt->rx_stats.mrs_recycled++;

	spin_unlock(&r_xprt->rx_buf.rb_mrlock);

	frwr_op_release_mr(mr);

}



static int

frwr_op_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr)

{
@@ -113,6 +151,7 @@ frwr_op_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr) 
		goto out_list_err;



	INIT_LIST_HEAD(&mr->mr_list);

	INIT_WORK(&mr->mr_recycle, frwr_mr_recycle_worker);

	sg_init_table(mr->mr_sg, depth);

	init_completion(&frwr->fr_linv_done);

	return 0;
@@ -131,79 +170,6 @@ out_list_err: 
	return rc;

}



static void

frwr_op_release_mr(struct rpcrdma_mr *mr)

{

	int rc;



	rc = ib_dereg_mr(mr->frwr.fr_mr);

	if (rc)

		pr_err("rpcrdma: final ib_dereg_mr for %p returned %i\n",

		       mr, rc);

	kfree(mr->mr_sg);

	kfree(mr);

}



static int

__frwr_mr_reset(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr)

{

	struct rpcrdma_frwr *frwr = &mr->frwr;

	int rc;



	rc = ib_dereg_mr(frwr->fr_mr);

	if (rc) {

		pr_warn("rpcrdma: ib_dereg_mr status %d, frwr %p orphaned\n",

			rc, mr);

		return rc;

	}



	frwr->fr_mr = ib_alloc_mr(ia->ri_pd, ia->ri_mrtype,

				  ia->ri_max_frwr_depth);

	if (IS_ERR(frwr->fr_mr)) {

		pr_warn("rpcrdma: ib_alloc_mr status %ld, frwr %p orphaned\n",

			PTR_ERR(frwr->fr_mr), mr);

		return PTR_ERR(frwr->fr_mr);

	}



	dprintk("RPC:       %s: recovered FRWR %p\n", __func__, frwr);

	frwr->fr_state = FRWR_IS_INVALID;

	return 0;

}



/* Reset of a single FRWR. Generate a fresh rkey by replacing the MR.

 */

static void

frwr_op_recover_mr(struct rpcrdma_mr *mr)

{

	enum rpcrdma_frwr_state state = mr->frwr.fr_state;

	struct rpcrdma_xprt *r_xprt = mr->mr_xprt;

	struct rpcrdma_ia *ia = &r_xprt->rx_ia;

	int rc;



	rc = __frwr_mr_reset(ia, mr);

	if (state != FRWR_FLUSHED_LI) {

		trace_xprtrdma_dma_unmap(mr);

		ib_dma_unmap_sg(ia->ri_device,

				mr->mr_sg, mr->mr_nents, mr->mr_dir);

	}

	if (rc)

		goto out_release;



	rpcrdma_mr_put(mr);

	r_xprt->rx_stats.mrs_recovered++;

	return;



out_release:

	pr_err("rpcrdma: FRWR reset failed %d, %p released\n", rc, mr);

	r_xprt->rx_stats.mrs_orphaned++;



	spin_lock(&r_xprt->rx_buf.rb_mrlock);

	list_del(&mr->mr_all);

	spin_unlock(&r_xprt->rx_buf.rb_mrlock);



	frwr_op_release_mr(mr);

}



/* On success, sets:

 *	ep->rep_attr.cap.max_send_wr

 *	ep->rep_attr.cap.max_recv_wr
@@ -276,6 +242,7 @@ frwr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, 


	ia->ri_max_segs = max_t(unsigned int, 1, RPCRDMA_MAX_DATA_SEGS /

				ia->ri_max_frwr_depth);

	ia->ri_max_segs += 2;	/* segments for head and tail buffers */

	return 0;

}
@@ -384,7 +351,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 
	mr = NULL;

	do {

		if (mr)

			rpcrdma_mr_defer_recovery(mr);

			rpcrdma_mr_recycle(mr);

		mr = rpcrdma_mr_get(r_xprt);

		if (!mr)

			return ERR_PTR(-EAGAIN);
@@ -417,7 +384,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 
	mr->mr_nents = ib_dma_map_sg(ia->ri_device, mr->mr_sg, i, mr->mr_dir);

	if (!mr->mr_nents)

		goto out_dmamap_err;

	trace_xprtrdma_dma_map(mr);

	trace_xprtrdma_mr_map(mr);



	ibmr = frwr->fr_mr;

	n = ib_map_mr_sg(ibmr, mr->mr_sg, mr->mr_nents, NULL, PAGE_SIZE);
@@ -451,7 +418,7 @@ out_dmamap_err: 
out_mapmr_err:

	pr_err("rpcrdma: failed to map mr %p (%d/%d)\n",

	       frwr->fr_mr, n, mr->mr_nents);

	rpcrdma_mr_defer_recovery(mr);

	rpcrdma_mr_recycle(mr);

	return ERR_PTR(-EIO);

}
@@ -499,7 +466,7 @@ frwr_op_reminv(struct rpcrdma_rep *rep, struct list_head *mrs) 
	list_for_each_entry(mr, mrs, mr_list)

		if (mr->mr_handle == rep->rr_inv_rkey) {

			list_del_init(&mr->mr_list);

			trace_xprtrdma_remoteinv(mr);

			trace_xprtrdma_mr_remoteinv(mr);

			mr->frwr.fr_state = FRWR_IS_INVALID;

			rpcrdma_mr_unmap_and_put(mr);

			break;	/* only one invalidated MR per RPC */
@@ -536,7 +503,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs) 
		mr->frwr.fr_state = FRWR_IS_INVALID;



		frwr = &mr->frwr;

		trace_xprtrdma_localinv(mr);

		trace_xprtrdma_mr_localinv(mr);



		frwr->fr_cqe.done = frwr_wc_localinv;

		last = &frwr->fr_invwr;
@@ -570,7 +537,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs) 
	if (bad_wr != first)

		wait_for_completion(&frwr->fr_linv_done);

	if (rc)

		goto reset_mrs;

		goto out_release;



	/* ORDER: Now DMA unmap all of the MRs, and return

	 * them to the free MR list.
@@ -582,22 +549,21 @@ unmap: 
	}

	return;



reset_mrs:

out_release:

	pr_err("rpcrdma: FRWR invalidate ib_post_send returned %i\n", rc);



	/* Find and reset the MRs in the LOCAL_INV WRs that did not

	/* Unmap and release the MRs in the LOCAL_INV WRs that did not

	 * get posted.

	 */

	while (bad_wr) {

		frwr = container_of(bad_wr, struct rpcrdma_frwr,

				    fr_invwr);

		mr = container_of(frwr, struct rpcrdma_mr, frwr);



		__frwr_mr_reset(ia, mr);



		bad_wr = bad_wr->next;



		list_del(&mr->mr_list);

		frwr_op_release_mr(mr);

	}

	goto unmap;

}



const struct rpcrdma_memreg_ops rpcrdma_frwr_memreg_ops = {
@@ -605,7 +571,6 @@ const struct rpcrdma_memreg_ops rpcrdma_frwr_memreg_ops = { 
	.ro_send			= frwr_op_send,

	.ro_reminv			= frwr_op_reminv,

	.ro_unmap_sync			= frwr_op_unmap_sync,

	.ro_recover_mr			= frwr_op_recover_mr,

	.ro_open			= frwr_op_open,

	.ro_maxpages			= frwr_op_maxpages,

	.ro_init_mr			= frwr_op_init_mr,

CRA Git | Maintained and supported by SUSTech CRA and CCSE