Commit 576ed913 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe
Browse files

block: use bio_add_page in bio_iov_iter_get_pages 



Replace a nasty hack with a different nasty hack to prepare for multipage
bio_vecs.  By moving the temporary page array as far up as possible in
the space allocated for the bio_vec array we can iterate forward over it
and thus use bio_add_page.  Using bio_add_page means we'll be able to
merge physically contiguous pages once support for multipath bio_vecs is
merged.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent c8765de0

block/bio.c

+20 −21
Original line number Original line Diff line number Diff line
@@ -827,6 +827,8 @@ int bio_add_page(struct bio *bio, struct page *page, 
}

}

EXPORT_SYMBOL(bio_add_page);

EXPORT_SYMBOL(bio_add_page);





#define PAGE_PTRS_PER_BVEC     (sizeof(struct bio_vec) / sizeof(struct page *))



/**

/**

 * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio

 * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio

 * @bio: bio to add pages to

 * @bio: bio to add pages to
@@ -839,38 +841,35 @@ EXPORT_SYMBOL(bio_add_page); 
 */

 */

static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)

static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)

{

{

	unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt, idx;

	unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;

	unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;

	struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;

	struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;

	struct page **pages = (struct page **)bv;

	struct page **pages = (struct page **)bv;

	ssize_t size, left;

	unsigned len, i;

	size_t offset;

	size_t offset;

	ssize_t size;



	/*

	 * Move page array up in the allocated memory for the bio vecs as far as

	 * possible so that we can start filling biovecs from the beginning

	 * without overwriting the temporary page array.

	*/

	BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2);

	pages += entries_left * (PAGE_PTRS_PER_BVEC - 1);





	size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);

	size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);

	if (unlikely(size <= 0))

	if (unlikely(size <= 0))

		return size ? size : -EFAULT;

		return size ? size : -EFAULT;

	idx = nr_pages = (size + offset + PAGE_SIZE - 1) / PAGE_SIZE;





	/*

	for (left = size, i = 0; left > 0; left -= len, i++) {

	 * Deep magic below:  We need to walk the pinned pages backwards

		struct page *page = pages[i];

	 * because we are abusing the space allocated for the bio_vecs

	 * for the page array.  Because the bio_vecs are larger than the

	 * page pointers by definition this will always work.  But it also

	 * means we can't use bio_add_page, so any changes to it's semantics

	 * need to be reflected here as well.

	 */

	bio->bi_iter.bi_size += size;

	bio->bi_vcnt += nr_pages;





	while (idx--) {

		len = min_t(size_t, PAGE_SIZE - offset, left);

		bv[idx].bv_page = pages[idx];

		if (WARN_ON_ONCE(bio_add_page(bio, page, len, offset) != len))

		bv[idx].bv_len = PAGE_SIZE;

			return -EINVAL;

		bv[idx].bv_offset = 0;

		offset = 0;

	}

	}





	bv[0].bv_offset += offset;

	bv[0].bv_len -= offset;

	bv[nr_pages - 1].bv_len -= nr_pages * PAGE_SIZE - offset - size;



	iov_iter_advance(iter, size);

	iov_iter_advance(iter, size);

	return 0;

	return 0;

}

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE