Commit d6764bbd authored by Chuck Lever's avatar Chuck Lever Committed by Anna Schumaker
Browse files

xprtrdma: Refactor rpcrdma_prepare_msg_sges() 



Refactor: Replace spaghetti with code that makes it plain what needs
to be done for each rtype. This makes it easier to add features and
optimizations.

Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Signed-off-by: default avatarAnna Schumaker <Anna.Schumaker@Netapp.com>
parent dc15c3d5

net/sunrpc/xprtrdma/rpc_rdma.c

+146 −117
Original line number Original line Diff line number Diff line
@@ -589,146 +589,160 @@ static bool rpcrdma_prepare_hdr_sge(struct rpcrdma_xprt *r_xprt, 
{

{

	struct rpcrdma_sendctx *sc = req->rl_sendctx;

	struct rpcrdma_sendctx *sc = req->rl_sendctx;

	struct rpcrdma_regbuf *rb = req->rl_rdmabuf;

	struct rpcrdma_regbuf *rb = req->rl_rdmabuf;

	struct ib_sge *sge = sc->sc_sges;

	struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++];





	if (!rpcrdma_regbuf_dma_map(r_xprt, rb))

	if (!rpcrdma_regbuf_dma_map(r_xprt, rb))

		goto out_regbuf;

		return false;

	sge->addr = rdmab_addr(rb);

	sge->addr = rdmab_addr(rb);

	sge->length = len;

	sge->length = len;

	sge->lkey = rdmab_lkey(rb);

	sge->lkey = rdmab_lkey(rb);





	ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length,

	ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length,

				      DMA_TO_DEVICE);

				      DMA_TO_DEVICE);

	req->rl_wr.num_sge++;

	return true;

	return true;



out_regbuf:

	pr_err("rpcrdma: failed to DMA map a Send buffer\n");

	return false;

}

}





/* Prepare the Send SGEs. The head and tail iovec, and each entry

/* The head iovec is straightforward, as it is usually already

 * in the page list, gets its own SGE.

 * DMA-mapped. Sync the content that has changed.

 */

 */

static bool rpcrdma_prepare_msg_sges(struct rpcrdma_xprt *r_xprt,

static bool rpcrdma_prepare_head_iov(struct rpcrdma_xprt *r_xprt,

				     struct rpcrdma_req *req,

				     struct rpcrdma_req *req, unsigned int len)

				     struct xdr_buf *xdr,

				     enum rpcrdma_chunktype rtype)

{

{

	struct rpcrdma_sendctx *sc = req->rl_sendctx;

	struct rpcrdma_sendctx *sc = req->rl_sendctx;

	unsigned int sge_no, page_base, len, remaining;

	struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++];

	struct rpcrdma_regbuf *rb = req->rl_sendbuf;

	struct rpcrdma_regbuf *rb = req->rl_sendbuf;

	struct ib_sge *sge = sc->sc_sges;

	struct page *page, **ppages;





	/* The head iovec is straightforward, as it is already

	 * DMA-mapped. Sync the content that has changed.

	 */

	if (!rpcrdma_regbuf_dma_map(r_xprt, rb))

	if (!rpcrdma_regbuf_dma_map(r_xprt, rb))

		goto out_regbuf;

		return false;

	sge_no = 1;

	sge[sge_no].addr = rdmab_addr(rb);

	sge[sge_no].length = xdr->head[0].iov_len;

	sge[sge_no].lkey = rdmab_lkey(rb);

	ib_dma_sync_single_for_device(rdmab_device(rb), sge[sge_no].addr,

				      sge[sge_no].length, DMA_TO_DEVICE);



	/* If there is a Read chunk, the page list is being handled

	 * via explicit RDMA, and thus is skipped here. However, the

	 * tail iovec may include an XDR pad for the page list, as

	 * well as additional content, and may not reside in the

	 * same page as the head iovec.

	 */

	if (rtype == rpcrdma_readch) {

		len = xdr->tail[0].iov_len;



		/* Do not include the tail if it is only an XDR pad */

		if (len < 4)

			goto out;





		page = virt_to_page(xdr->tail[0].iov_base);

	sge->addr = rdmab_addr(rb);

		page_base = offset_in_page(xdr->tail[0].iov_base);

	sge->length = len;

	sge->lkey = rdmab_lkey(rb);





		/* If the content in the page list is an odd length,

	ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length,

		 * xdr_write_pages() has added a pad at the beginning

				      DMA_TO_DEVICE);

		 * of the tail iovec. Force the tail's non-pad content

	return true;

		 * to land at the next XDR position in the Send message.

		 */

		page_base += len & 3;

		len -= len & 3;

		goto map_tail;

}

}





	/* If there is a page list present, temporarily DMA map

/* If there is a page list present, DMA map and prepare an

	 * and prepare an SGE for each page to be sent.

 * SGE for each page to be sent.

 */

 */

	if (xdr->page_len) {

static bool rpcrdma_prepare_pagelist(struct rpcrdma_req *req,

				     struct xdr_buf *xdr)

{

	struct rpcrdma_sendctx *sc = req->rl_sendctx;

	struct rpcrdma_regbuf *rb = req->rl_sendbuf;

	unsigned int page_base, len, remaining;

	struct page **ppages;

	struct ib_sge *sge;



	ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT);

	ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT);

	page_base = offset_in_page(xdr->page_base);

	page_base = offset_in_page(xdr->page_base);

	remaining = xdr->page_len;

	remaining = xdr->page_len;

	while (remaining) {

	while (remaining) {

			sge_no++;

		sge = &sc->sc_sges[req->rl_wr.num_sge++];

			if (sge_no > RPCRDMA_MAX_SEND_SGES - 2)

		len = min_t(unsigned int, PAGE_SIZE - page_base, remaining);

				goto out_mapping_overflow;

		sge->addr = ib_dma_map_page(rdmab_device(rb), *ppages,



			len = min_t(u32, PAGE_SIZE - page_base, remaining);

			sge[sge_no].addr =

				ib_dma_map_page(rdmab_device(rb), *ppages,

					    page_base, len, DMA_TO_DEVICE);

					    page_base, len, DMA_TO_DEVICE);

			if (ib_dma_mapping_error(rdmab_device(rb),

		if (ib_dma_mapping_error(rdmab_device(rb), sge->addr))

						 sge[sge_no].addr))

			goto out_mapping_err;

			goto out_mapping_err;

			sge[sge_no].length = len;



			sge[sge_no].lkey = rdmab_lkey(rb);

		sge->length = len;

		sge->lkey = rdmab_lkey(rb);





		sc->sc_unmap_count++;

		sc->sc_unmap_count++;

		ppages++;

		ppages++;

		remaining -= len;

		remaining -= len;

		page_base = 0;

		page_base = 0;

	}

	}



	return true;



out_mapping_err:

	trace_xprtrdma_dma_maperr(sge->addr);

	return false;

}

}





	/* The tail iovec is not always constructed in the same

/* The tail iovec may include an XDR pad for the page list,

	 * page where the head iovec resides (see, for example,

 * as well as additional content, and may not reside in the

	 * gss_wrap_req_priv). To neatly accommodate that case,

 * same page as the head iovec.

	 * DMA map it separately.

 */

 */

	if (xdr->tail[0].iov_len) {

static bool rpcrdma_prepare_tail_iov(struct rpcrdma_req *req,

		page = virt_to_page(xdr->tail[0].iov_base);

				     struct xdr_buf *xdr,

		page_base = offset_in_page(xdr->tail[0].iov_base);

				     unsigned int page_base, unsigned int len)

		len = xdr->tail[0].iov_len;

{

	struct rpcrdma_sendctx *sc = req->rl_sendctx;

	struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++];

	struct rpcrdma_regbuf *rb = req->rl_sendbuf;

	struct page *page = virt_to_page(xdr->tail[0].iov_base);





map_tail:

	sge->addr = ib_dma_map_page(rdmab_device(rb), page, page_base, len,

		sge_no++;

		sge[sge_no].addr =

			ib_dma_map_page(rdmab_device(rb), page, page_base, len,

				    DMA_TO_DEVICE);

				    DMA_TO_DEVICE);

		if (ib_dma_mapping_error(rdmab_device(rb), sge[sge_no].addr))

	if (ib_dma_mapping_error(rdmab_device(rb), sge->addr))

		goto out_mapping_err;

		goto out_mapping_err;

		sge[sge_no].length = len;



		sge[sge_no].lkey = rdmab_lkey(rb);

	sge->length = len;

		sc->sc_unmap_count++;

	sge->lkey = rdmab_lkey(rb);

	++sc->sc_unmap_count;

	return true;



out_mapping_err:

	trace_xprtrdma_dma_maperr(sge->addr);

	return false;

}

}





out:

static bool rpcrdma_prepare_noch_mapped(struct rpcrdma_xprt *r_xprt,

	req->rl_wr.num_sge += sge_no;

					struct rpcrdma_req *req,

	if (sc->sc_unmap_count)

					struct xdr_buf *xdr)

{

	struct kvec *tail = &xdr->tail[0];



	if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len))

		return false;

	if (xdr->page_len)

		if (!rpcrdma_prepare_pagelist(req, xdr))

			return false;

	if (tail->iov_len)

		if (!rpcrdma_prepare_tail_iov(req, xdr,

					      offset_in_page(tail->iov_base),

					      tail->iov_len))

			return false;



	if (req->rl_sendctx->sc_unmap_count)

		kref_get(&req->rl_kref);

		kref_get(&req->rl_kref);

	return true;

	return true;

}





out_regbuf:

static bool rpcrdma_prepare_readch(struct rpcrdma_xprt *r_xprt,

	pr_err("rpcrdma: failed to DMA map a Send buffer\n");

				   struct rpcrdma_req *req,

				   struct xdr_buf *xdr)

{

	if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len))

		return false;

		return false;





out_mapping_overflow:

	/* If there is a Read chunk, the page list is being handled

	rpcrdma_sendctx_unmap(sc);

	 * via explicit RDMA, and thus is skipped here.

	pr_err("rpcrdma: too many Send SGEs (%u)\n", sge_no);

	 */

	return false;





out_mapping_err:

	/* Do not include the tail if it is only an XDR pad */

	rpcrdma_sendctx_unmap(sc);

	if (xdr->tail[0].iov_len > 3) {

	trace_xprtrdma_dma_maperr(sge[sge_no].addr);

		unsigned int page_base, len;



		/* If the content in the page list is an odd length,

		 * xdr_write_pages() adds a pad at the beginning of

		 * the tail iovec. Force the tail's non-pad content to

		 * land at the next XDR position in the Send message.

		 */

		page_base = offset_in_page(xdr->tail[0].iov_base);

		len = xdr->tail[0].iov_len;

		page_base += len & 3;

		len -= len & 3;

		if (!rpcrdma_prepare_tail_iov(req, xdr, page_base, len))

			return false;

			return false;

		kref_get(&req->rl_kref);

	}



	return true;

}

}





/**

/**
@@ -741,17 +755,17 @@ out_mapping_err: 
 *

 *

 * Returns 0 on success; otherwise a negative errno is returned.

 * Returns 0 on success; otherwise a negative errno is returned.

 */

 */

int

inline int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,

rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,

				     struct rpcrdma_req *req, u32 hdrlen,

				     struct rpcrdma_req *req, u32 hdrlen,

			  struct xdr_buf *xdr, enum rpcrdma_chunktype rtype)

				     struct xdr_buf *xdr,

				     enum rpcrdma_chunktype rtype)

{

{

	int ret;

	int ret;





	ret = -EAGAIN;

	ret = -EAGAIN;

	req->rl_sendctx = rpcrdma_sendctx_get_locked(r_xprt);

	req->rl_sendctx = rpcrdma_sendctx_get_locked(r_xprt);

	if (!req->rl_sendctx)

	if (!req->rl_sendctx)

		goto err;

		goto out_nosc;

	req->rl_sendctx->sc_unmap_count = 0;

	req->rl_sendctx->sc_unmap_count = 0;

	req->rl_sendctx->sc_req = req;

	req->rl_sendctx->sc_req = req;

	kref_init(&req->rl_kref);

	kref_init(&req->rl_kref);
@@ -762,13 +776,28 @@ rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt, 




	ret = -EIO;

	ret = -EIO;

	if (!rpcrdma_prepare_hdr_sge(r_xprt, req, hdrlen))

	if (!rpcrdma_prepare_hdr_sge(r_xprt, req, hdrlen))

		goto err;

		goto out_unmap;

	if (rtype != rpcrdma_areadch)



		if (!rpcrdma_prepare_msg_sges(r_xprt, req, xdr, rtype))

	switch (rtype) {

			goto err;

	case rpcrdma_noch:

		if (!rpcrdma_prepare_noch_mapped(r_xprt, req, xdr))

			goto out_unmap;

		break;

	case rpcrdma_readch:

		if (!rpcrdma_prepare_readch(r_xprt, req, xdr))

			goto out_unmap;

		break;

	case rpcrdma_areadch:

		break;

	default:

		goto out_unmap;

	}



	return 0;

	return 0;





err:

out_unmap:

	rpcrdma_sendctx_unmap(req->rl_sendctx);

out_nosc:

	trace_xprtrdma_prepsend_failed(&req->rl_slot, ret);

	trace_xprtrdma_prepsend_failed(&req->rl_slot, ret);

	return ret;

	return ret;

}

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE