Commit d901b276 authored by Jason Gunthorpe's avatar Jason Gunthorpe
Browse files

lib/scatterlist: Provide a DMA page iterator 



Commit 2db76d7c ("lib/scatterlist: sg_page_iter: support sg lists w/o
backing pages") introduced the sg_page_iter_dma_address() function without
providing a way to use it in the general case. If the sg_dma_len() is not
equal to the sg length callers cannot safely use the
for_each_sg_page/sg_page_iter_dma_address combination.

Resolve this API mistake by providing a DMA specific iterator,
for_each_sg_dma_page(), that uses the right length so
sg_page_iter_dma_address() works as expected with all sglists.

A new iterator type is introduced to provide compile-time safety against
wrongly mixing accessors and iterators.

Acked-by: Christoph Hellwig <hch@lst.de> (for scatterlist)
Acked-by: default avatarThomas Hellstrom <thellstrom@vmware.com>
Acked-by: Sakari Ailus <sakari.ailus@linux.intel.com> (ipu3-cio2)
Signed-off-by: default avatarJason Gunthorpe <jgg@mellanox.com>
parent f368ff18

.clang-format

+1 −0
Original line number Original line Diff line number Diff line
@@ -240,6 +240,7 @@ ForEachMacros: 
  - 'for_each_set_bit'

  - 'for_each_set_bit'

  - 'for_each_set_bit_from'

  - 'for_each_set_bit_from'

  - 'for_each_sg'

  - 'for_each_sg'

  - 'for_each_sg_dma_page'

  - 'for_each_sg_page'

  - 'for_each_sg_page'

  - 'for_each_sibling_event'

  - 'for_each_sibling_event'

  - '__for_each_thread'

  - '__for_each_thread'

drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c

+7 −1
Original line number Original line Diff line number Diff line
@@ -311,7 +311,13 @@ static dma_addr_t __vmw_piter_dma_addr(struct vmw_piter *viter) 




static dma_addr_t __vmw_piter_sg_addr(struct vmw_piter *viter)

static dma_addr_t __vmw_piter_sg_addr(struct vmw_piter *viter)

{

{

	return sg_page_iter_dma_address(&viter->iter);

	/*

	 * FIXME: This driver wrongly mixes DMA and CPU SG list iteration and

	 * needs revision. See

	 * https://lore.kernel.org/lkml/20190104223531.GA1705@ziepe.ca/

	 */

	return sg_page_iter_dma_address(

		container_of(&viter->iter, struct sg_dma_page_iter, base));

}

}

drivers/media/pci/intel/ipu3/ipu3-cio2.c

+2 −2
Original line number Original line Diff line number Diff line
@@ -846,7 +846,7 @@ static int cio2_vb2_buf_init(struct vb2_buffer *vb) 
	unsigned int pages = DIV_ROUND_UP(vb->planes[0].length, CIO2_PAGE_SIZE);

	unsigned int pages = DIV_ROUND_UP(vb->planes[0].length, CIO2_PAGE_SIZE);

	unsigned int lops = DIV_ROUND_UP(pages + 1, entries_per_page);

	unsigned int lops = DIV_ROUND_UP(pages + 1, entries_per_page);

	struct sg_table *sg;

	struct sg_table *sg;

	struct sg_page_iter sg_iter;

	struct sg_dma_page_iter sg_iter;

	int i, j;

	int i, j;





	if (lops <= 0 || lops > CIO2_MAX_LOPS) {

	if (lops <= 0 || lops > CIO2_MAX_LOPS) {
@@ -873,7 +873,7 @@ static int cio2_vb2_buf_init(struct vb2_buffer *vb) 
		b->offset = sg->sgl->offset;

		b->offset = sg->sgl->offset;





	i = j = 0;

	i = j = 0;

	for_each_sg_page(sg->sgl, &sg_iter, sg->nents, 0) {

	for_each_sg_dma_page (sg->sgl, &sg_iter, sg->nents, 0) {

		if (!pages--)

		if (!pages--)

			break;

			break;

		b->lop[i][j] = sg_page_iter_dma_address(&sg_iter) >> PAGE_SHIFT;

		b->lop[i][j] = sg_page_iter_dma_address(&sg_iter) >> PAGE_SHIFT;

include/linux/scatterlist.h

+40 −9
Original line number Original line Diff line number Diff line
@@ -339,12 +339,12 @@ int sg_alloc_table_chained(struct sg_table *table, int nents, 
/*

/*

 * sg page iterator

 * sg page iterator

 *

 *

 * Iterates over sg entries page-by-page.  On each successful iteration,

 * Iterates over sg entries page-by-page.  On each successful iteration, you

 * you can call sg_page_iter_page(@piter) and sg_page_iter_dma_address(@piter)

 * can call sg_page_iter_page(@piter) to get the current page and its dma

 * to get the current page and its dma address. @piter->sg will point to the

 * address. @piter->sg will point to the sg holding this page and

 * sg holding this page and @piter->sg_pgoffset to the page's page offset

 * @piter->sg_pgoffset to the page's page offset within the sg. The iteration

 * within the sg. The iteration will stop either when a maximum number of sg

 * will stop either when a maximum number of sg entries was reached or a

 * entries was reached or a terminating sg (sg_last(sg) == true) was reached.

 * terminating sg (sg_last(sg) == true) was reached.

 */

 */

struct sg_page_iter {

struct sg_page_iter {

	struct scatterlist	*sg;		/* sg holding the page */

	struct scatterlist	*sg;		/* sg holding the page */
@@ -356,7 +356,19 @@ struct sg_page_iter { 
						 * next step */

						 * next step */

};

};





/*

 * sg page iterator for DMA addresses

 *

 * This is the same as sg_page_iter however you can call

 * sg_page_iter_dma_address(@dma_iter) to get the page's DMA

 * address. sg_page_iter_page() cannot be called on this iterator.

 */

struct sg_dma_page_iter {

	struct sg_page_iter base;

};



bool __sg_page_iter_next(struct sg_page_iter *piter);

bool __sg_page_iter_next(struct sg_page_iter *piter);

bool __sg_page_iter_dma_next(struct sg_dma_page_iter *dma_iter);

void __sg_page_iter_start(struct sg_page_iter *piter,

void __sg_page_iter_start(struct sg_page_iter *piter,

			  struct scatterlist *sglist, unsigned int nents,

			  struct scatterlist *sglist, unsigned int nents,

			  unsigned long pgoffset);

			  unsigned long pgoffset);
@@ -372,11 +384,13 @@ static inline struct page *sg_page_iter_page(struct sg_page_iter *piter) 
/**

/**

 * sg_page_iter_dma_address - get the dma address of the current page held by

 * sg_page_iter_dma_address - get the dma address of the current page held by

 * the page iterator.

 * the page iterator.

 * @piter:	page iterator holding the page

 * @dma_iter:	page iterator holding the page

 */

 */

static inline dma_addr_t sg_page_iter_dma_address(struct sg_page_iter *piter)

static inline dma_addr_t

sg_page_iter_dma_address(struct sg_dma_page_iter *dma_iter)

{

{

	return sg_dma_address(piter->sg) + (piter->sg_pgoffset << PAGE_SHIFT);

	return sg_dma_address(dma_iter->base.sg) +

	       (dma_iter->base.sg_pgoffset << PAGE_SHIFT);

}

}





/**

/**
@@ -385,11 +399,28 @@ static inline dma_addr_t sg_page_iter_dma_address(struct sg_page_iter *piter) 
 * @piter:	page iterator to hold current page, sg, sg_pgoffset

 * @piter:	page iterator to hold current page, sg, sg_pgoffset

 * @nents:	maximum number of sg entries to iterate over

 * @nents:	maximum number of sg entries to iterate over

 * @pgoffset:	starting page offset

 * @pgoffset:	starting page offset

 *

 * Callers may use sg_page_iter_page() to get each page pointer.

 */

 */

#define for_each_sg_page(sglist, piter, nents, pgoffset)		   \

#define for_each_sg_page(sglist, piter, nents, pgoffset)		   \

	for (__sg_page_iter_start((piter), (sglist), (nents), (pgoffset)); \

	for (__sg_page_iter_start((piter), (sglist), (nents), (pgoffset)); \

	     __sg_page_iter_next(piter);)

	     __sg_page_iter_next(piter);)





/**

 * for_each_sg_dma_page - iterate over the pages of the given sg list

 * @sglist:	sglist to iterate over

 * @dma_iter:	page iterator to hold current page

 * @dma_nents:	maximum number of sg entries to iterate over, this is the value

 *              returned from dma_map_sg

 * @pgoffset:	starting page offset

 *

 * Callers may use sg_page_iter_dma_address() to get each page's DMA address.

 */

#define for_each_sg_dma_page(sglist, dma_iter, dma_nents, pgoffset)            \

	for (__sg_page_iter_start(&(dma_iter)->base, sglist, dma_nents,        \

				  pgoffset);                                   \

	     __sg_page_iter_dma_next(dma_iter);)



/*

/*

 * Mapping sg iterator

 * Mapping sg iterator

 *

 *

lib/scatterlist.c

+26 −0
Original line number Original line Diff line number Diff line
@@ -625,6 +625,32 @@ bool __sg_page_iter_next(struct sg_page_iter *piter) 
}

}

EXPORT_SYMBOL(__sg_page_iter_next);

EXPORT_SYMBOL(__sg_page_iter_next);





static int sg_dma_page_count(struct scatterlist *sg)

{

	return PAGE_ALIGN(sg->offset + sg_dma_len(sg)) >> PAGE_SHIFT;

}



bool __sg_page_iter_dma_next(struct sg_dma_page_iter *dma_iter)

{

	struct sg_page_iter *piter = &dma_iter->base;



	if (!piter->__nents || !piter->sg)

		return false;



	piter->sg_pgoffset += piter->__pg_advance;

	piter->__pg_advance = 1;



	while (piter->sg_pgoffset >= sg_dma_page_count(piter->sg)) {

		piter->sg_pgoffset -= sg_dma_page_count(piter->sg);

		piter->sg = sg_next(piter->sg);

		if (!--piter->__nents || !piter->sg)

			return false;

	}



	return true;

}

EXPORT_SYMBOL(__sg_page_iter_dma_next);



/**

/**

 * sg_miter_start - start mapping iteration over a sg list

 * sg_miter_start - start mapping iteration over a sg list

 * @miter: sg mapping iter to be started

 * @miter: sg mapping iter to be started

CRA Git | Maintained and supported by SUSTech CRA and CCSE