Commit 99ef4db3 authored by Chuck Lever's avatar Chuck Lever Committed by Anna Schumaker
Browse files

xprtrdma: Replace DMA_BIDIRECTIONAL 



The use of DMA_BIDIRECTIONAL is discouraged by DMA-API.txt.
Fortunately, xprtrdma now knows which direction I/O is going as
soon as it allocates each regbuf.

The RPC Call and Reply buffers are no longer the same regbuf. They
can each be labeled correctly now. The RPC Reply buffer is never
part of either a Send or Receive WR, but it can be part of Reply
chunk, which is mapped and registered via ->ro_map . So it is not
DMA mapped when it is allocated (DMA_NONE), to avoid a double-
mapping.

Since Receive buffers are no longer DMA_BIDIRECTIONAL and their
contents are never modified by the host CPU, DMA-API-HOWTO.txt
suggests that a DMA sync before posting each buffer should be
unnecessary. (See my_card_interrupt_handler).

Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Signed-off-by: default avatarAnna Schumaker <Anna.Schumaker@Netapp.com>
parent 08cf2efd

net/sunrpc/xprtrdma/backchannel.c

+3 −2
Original line number Original line Diff line number Diff line
@@ -45,13 +45,14 @@ static int rpcrdma_bc_setup_rqst(struct rpcrdma_xprt *r_xprt, 
		return PTR_ERR(req);

		return PTR_ERR(req);

	req->rl_backchannel = true;

	req->rl_backchannel = true;





	rb = rpcrdma_alloc_regbuf(ia, RPCRDMA_HDRBUF_SIZE, GFP_KERNEL);

	rb = rpcrdma_alloc_regbuf(ia, RPCRDMA_HDRBUF_SIZE,

				  DMA_TO_DEVICE, GFP_KERNEL);

	if (IS_ERR(rb))

	if (IS_ERR(rb))

		goto out_fail;

		goto out_fail;

	req->rl_rdmabuf = rb;

	req->rl_rdmabuf = rb;





	size = r_xprt->rx_data.inline_rsize;

	size = r_xprt->rx_data.inline_rsize;

	rb = rpcrdma_alloc_regbuf(ia, size, GFP_KERNEL);

	rb = rpcrdma_alloc_regbuf(ia, size, DMA_TO_DEVICE, GFP_KERNEL);

	if (IS_ERR(rb))

	if (IS_ERR(rb))

		goto out_fail;

		goto out_fail;

	req->rl_sendbuf = rb;

	req->rl_sendbuf = rb;

net/sunrpc/xprtrdma/transport.c

+4 −3
Original line number Original line Diff line number Diff line
@@ -490,7 +490,7 @@ rpcrdma_get_rdmabuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, 
	if (req->rl_rdmabuf)

	if (req->rl_rdmabuf)

		return true;

		return true;





	rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, size, flags);

	rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, size, DMA_TO_DEVICE, flags);

	if (IS_ERR(rb))

	if (IS_ERR(rb))

		return false;

		return false;
@@ -517,7 +517,8 @@ rpcrdma_get_sendbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, 
		return true;

		return true;





	min_size = max_t(size_t, size, r_xprt->rx_data.inline_wsize);

	min_size = max_t(size_t, size, r_xprt->rx_data.inline_wsize);

	rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, min_size, flags);

	rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, min_size,

				  DMA_TO_DEVICE, flags);

	if (IS_ERR(rb))

	if (IS_ERR(rb))

		return false;

		return false;
@@ -547,7 +548,7 @@ rpcrdma_get_recvbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, 
	if (req->rl_recvbuf && rdmab_length(req->rl_recvbuf) >= size)

	if (req->rl_recvbuf && rdmab_length(req->rl_recvbuf) >= size)

		return true;

		return true;





	rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, size, flags);

	rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, size, DMA_NONE, flags);

	if (IS_ERR(rb))

	if (IS_ERR(rb))

		return false;

		return false;

net/sunrpc/xprtrdma/verbs.c

+26 −29
Original line number Original line Diff line number Diff line
@@ -866,7 +866,7 @@ rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt) 
		goto out;

		goto out;





	rep->rr_rdmabuf = rpcrdma_alloc_regbuf(ia, cdata->inline_rsize,

	rep->rr_rdmabuf = rpcrdma_alloc_regbuf(ia, cdata->inline_rsize,

					       GFP_KERNEL);

					       DMA_FROM_DEVICE, GFP_KERNEL);

	if (IS_ERR(rep->rr_rdmabuf)) {

	if (IS_ERR(rep->rr_rdmabuf)) {

		rc = PTR_ERR(rep->rr_rdmabuf);

		rc = PTR_ERR(rep->rr_rdmabuf);

		goto out_free;

		goto out_free;
@@ -1172,27 +1172,24 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) 
	spin_unlock(&buffers->rb_lock);

	spin_unlock(&buffers->rb_lock);

}

}





/*

 * Wrappers for internal-use kmalloc memory registration, used by buffer code.

 */



/**

/**

 * rpcrdma_alloc_regbuf - kmalloc and register memory for SEND/RECV buffers

 * rpcrdma_alloc_regbuf - allocate and DMA-map memory for SEND/RECV buffers

 * @ia: controlling rpcrdma_ia

 * @ia: controlling rpcrdma_ia

 * @size: size of buffer to be allocated, in bytes

 * @size: size of buffer to be allocated, in bytes

 * @direction: direction of data movement

 * @flags: GFP flags

 * @flags: GFP flags

 *

 *

 * Returns pointer to private header of an area of internally

 * Returns an ERR_PTR, or a pointer to a regbuf, which is a

 * registered memory, or an ERR_PTR. The registered buffer follows

 * contiguous memory region that is DMA mapped persistently, and

 * the end of the private header.

 * is registered for local I/O.

 *

 *

 * xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for

 * xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for

 * receiving the payload of RDMA RECV operations. regbufs are not

 * receiving the payload of RDMA RECV operations. During Long Calls

 * used for RDMA READ/WRITE operations, thus are registered only for

 * or Replies they may be registered externally via ro_map.

 * LOCAL access.

 */

 */

struct rpcrdma_regbuf *

struct rpcrdma_regbuf *

rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags)

rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size,

		     enum dma_data_direction direction, gfp_t flags)

{

{

	struct rpcrdma_regbuf *rb;

	struct rpcrdma_regbuf *rb;

	struct ib_sge *iov;

	struct ib_sge *iov;
@@ -1201,15 +1198,20 @@ rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags) 
	if (rb == NULL)

	if (rb == NULL)

		goto out;

		goto out;





	rb->rg_direction = direction;

	iov = &rb->rg_iov;

	iov = &rb->rg_iov;

	iov->length = size;

	iov->lkey = ia->ri_pd->local_dma_lkey;



	if (direction != DMA_NONE) {

		iov->addr = ib_dma_map_single(ia->ri_device,

		iov->addr = ib_dma_map_single(ia->ri_device,

				      (void *)rb->rg_base, size,

					      (void *)rb->rg_base,

				      DMA_BIDIRECTIONAL);

					      rdmab_length(rb),

					      rb->rg_direction);

		if (ib_dma_mapping_error(ia->ri_device, iov->addr))

		if (ib_dma_mapping_error(ia->ri_device, iov->addr))

			goto out_free;

			goto out_free;

	}





	iov->length = size;

	iov->lkey = ia->ri_pd->local_dma_lkey;

	return rb;

	return rb;





out_free:

out_free:
@@ -1226,14 +1228,14 @@ out: 
void

void

rpcrdma_free_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)

rpcrdma_free_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)

{

{

	struct ib_sge *iov;



	if (!rb)

	if (!rb)

		return;

		return;





	iov = &rb->rg_iov;

	if (rb->rg_direction != DMA_NONE) {

	ib_dma_unmap_single(ia->ri_device,

		ib_dma_unmap_single(ia->ri_device, rdmab_addr(rb),

			    iov->addr, iov->length, DMA_BIDIRECTIONAL);

				    rdmab_length(rb), rb->rg_direction);

	}



	kfree(rb);

	kfree(rb);

}

}
@@ -1305,11 +1307,6 @@ rpcrdma_ep_post_recv(struct rpcrdma_ia *ia, 
	recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov;

	recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov;

	recv_wr.num_sge = 1;

	recv_wr.num_sge = 1;





	ib_dma_sync_single_for_cpu(ia->ri_device,

				   rdmab_addr(rep->rr_rdmabuf),

				   rdmab_length(rep->rr_rdmabuf),

				   DMA_BIDIRECTIONAL);



	rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail);

	rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail);

	if (rc)

	if (rc)

		goto out_postrecv;

		goto out_postrecv;

net/sunrpc/xprtrdma/xprt_rdma.h

+3 −1
Original line number Original line Diff line number Diff line
@@ -113,6 +113,7 @@ struct rpcrdma_ep { 




struct rpcrdma_regbuf {

struct rpcrdma_regbuf {

	struct ib_sge		rg_iov;

	struct ib_sge		rg_iov;

	enum dma_data_direction	rg_direction;

	__be32			rg_base[0] __attribute__ ((aligned(256)));

	__be32			rg_base[0] __attribute__ ((aligned(256)));

};

};
@@ -477,7 +478,8 @@ void rpcrdma_recv_buffer_put(struct rpcrdma_rep *); 
void rpcrdma_defer_mr_recovery(struct rpcrdma_mw *);

void rpcrdma_defer_mr_recovery(struct rpcrdma_mw *);





struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(struct rpcrdma_ia *,

struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(struct rpcrdma_ia *,

					    size_t, gfp_t);

					    size_t, enum dma_data_direction,

					    gfp_t);

void rpcrdma_free_regbuf(struct rpcrdma_ia *,

void rpcrdma_free_regbuf(struct rpcrdma_ia *,

			 struct rpcrdma_regbuf *);

			 struct rpcrdma_regbuf *);

CRA Git | Maintained and supported by SUSTech CRA and CCSE