Commit 7b2c99d1 authored by Al Viro's avatar Al Viro
Browse files

new helper: iov_iter_get_pages() 



iov_iter_get_pages(iter, pages, maxsize, &start) grabs references pinning
the pages of up to maxsize of (contiguous) data from iter.  Returns the
amount of memory grabbed or -error.  In case of success, the requested
area begins at offset start in pages[0] and runs through pages[1], etc.
Less than requested amount might be returned - either because the contiguous
area in the beginning of iterator is smaller than requested, or because
the kernel failed to pin that many pages.

direct-io.c switched to using iov_iter_get_pages()

Signed-off-by: default avatarAl Viro <viro@zeniv.linux.org.uk>
parent 3320c60b

fs/direct-io.c

+38 −73
Original line number Diff line number Diff line
@@ -77,7 +77,6 @@ struct dio_submit { 
	unsigned blocks_available;	/* At block_in_file.  changes */

	int reap_counter;		/* rate limit reaping */

	sector_t final_block_in_request;/* doesn't change */

	unsigned first_block_in_page;	/* doesn't change, Used only once */

	int boundary;			/* prev block is at a boundary */

	get_block_t *get_block;		/* block mapping function */

	dio_submit_t *submit_io;	/* IO submition function */
@@ -98,19 +97,14 @@ struct dio_submit { 
	sector_t cur_page_block;	/* Where it starts */

	loff_t cur_page_fs_offset;	/* Offset in file */



	/*

	 * Page fetching state. These variables belong to dio_refill_pages().

	 */

	int curr_page;			/* changes */

	int total_pages;		/* doesn't change */

	unsigned long curr_user_address;/* changes */



	struct iov_iter *iter;

	/*

	 * Page queue.  These variables belong to dio_refill_pages() and

	 * dio_get_page().

	 */

	unsigned head;			/* next page to process */

	unsigned tail;			/* last valid page + 1 */

	size_t from, to;

};



/* dio_state communicated between submission path and end_io */
@@ -163,15 +157,10 @@ static inline unsigned dio_pages_present(struct dio_submit *sdio) 
 */

static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio)

{

	int ret;

	int nr_pages;

	ssize_t ret;



	nr_pages = min(sdio->total_pages - sdio->curr_page, DIO_PAGES);

	ret = get_user_pages_fast(

		sdio->curr_user_address,		/* Where from? */

		nr_pages,			/* How many pages? */

		dio->rw == READ,		/* Write to memory? */

		&dio->pages[0]);		/* Put results here */

	ret = iov_iter_get_pages(sdio->iter, dio->pages, DIO_PAGES * PAGE_SIZE,

				&sdio->from);



	if (ret < 0 && sdio->blocks_available && (dio->rw & WRITE)) {

		struct page *page = ZERO_PAGE(0);
@@ -186,18 +175,19 @@ static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) 
		dio->pages[0] = page;

		sdio->head = 0;

		sdio->tail = 1;

		ret = 0;

		goto out;

		sdio->from = 0;

		sdio->to = PAGE_SIZE;

		return 0;

	}



	if (ret >= 0) {

		sdio->curr_user_address += ret * PAGE_SIZE;

		sdio->curr_page += ret;

		iov_iter_advance(sdio->iter, ret);

		ret += sdio->from;

		sdio->head = 0;

		sdio->tail = ret;

		ret = 0;

		sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE;

		sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1;

		return 0;

	}

out:

	return ret;	

}
@@ -208,8 +198,9 @@ out: 
 * L1 cache.

 */

static inline struct page *dio_get_page(struct dio *dio,

		struct dio_submit *sdio)

		struct dio_submit *sdio, size_t *from, size_t *to)

{

	int n;

	if (dio_pages_present(sdio) == 0) {

		int ret;
@@ -218,7 +209,10 @@ static inline struct page *dio_get_page(struct dio *dio, 
			return ERR_PTR(ret);

		BUG_ON(dio_pages_present(sdio) == 0);

	}

	return dio->pages[sdio->head++];

	n = sdio->head++;

	*from = n ? 0 : sdio->from;

	*to = (n == sdio->tail - 1) ? sdio->to : PAGE_SIZE;

	return dio->pages[n];

}



/**
@@ -422,8 +416,8 @@ static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) 
 */

static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio)

{

	while (dio_pages_present(sdio))

		page_cache_release(dio_get_page(dio, sdio));

	while (sdio->head < sdio->tail)

		page_cache_release(dio->pages[sdio->head++]);

}



/*
@@ -912,23 +906,18 @@ static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, 
			struct buffer_head *map_bh)

{

	const unsigned blkbits = sdio->blkbits;

	const unsigned blocks_per_page = PAGE_SIZE >> blkbits;

	struct page *page;

	unsigned block_in_page;

	int ret = 0;



	/* The I/O can start at any block offset within the first page */

	block_in_page = sdio->first_block_in_page;



	while (sdio->block_in_file < sdio->final_block_in_request) {

		page = dio_get_page(dio, sdio);

		struct page *page;

		size_t from, to;

		page = dio_get_page(dio, sdio, &from, &to);

		if (IS_ERR(page)) {

			ret = PTR_ERR(page);

			goto out;

		}



		while (block_in_page < blocks_per_page) {

			unsigned offset_in_page = block_in_page << blkbits;

		while (from < to) {

			unsigned this_chunk_bytes;	/* # of bytes mapped */

			unsigned this_chunk_blocks;	/* # of blocks */

			unsigned u;
@@ -999,10 +988,9 @@ do_holes: 
					page_cache_release(page);

					goto out;

				}

				zero_user(page, block_in_page << blkbits,

						1 << blkbits);

				zero_user(page, from, 1 << blkbits);

				sdio->block_in_file++;

				block_in_page++;

				from += 1 << blkbits;

				dio->result += 1 << blkbits;

				goto next_block;

			}
@@ -1020,7 +1008,7 @@ do_holes: 
			 * can add to this page

			 */

			this_chunk_blocks = sdio->blocks_available;

			u = (PAGE_SIZE - offset_in_page) >> blkbits;

			u = (to - from) >> blkbits;

			if (this_chunk_blocks > u)

				this_chunk_blocks = u;

			u = sdio->final_block_in_request - sdio->block_in_file;
@@ -1032,7 +1020,7 @@ do_holes: 
			if (this_chunk_blocks == sdio->blocks_available)

				sdio->boundary = buffer_boundary(map_bh);

			ret = submit_page_section(dio, sdio, page,

						  offset_in_page,

						  from,

						  this_chunk_bytes,

						  sdio->next_block_for_io,

						  map_bh);
@@ -1043,9 +1031,9 @@ do_holes: 
			sdio->next_block_for_io += this_chunk_blocks;



			sdio->block_in_file += this_chunk_blocks;

			block_in_page += this_chunk_blocks;

			from += this_chunk_bytes;

			dio->result += this_chunk_bytes;

			sdio->blocks_available -= this_chunk_blocks;

			dio->result += this_chunk_blocks << blkbits;

next_block:

			BUG_ON(sdio->block_in_file > sdio->final_block_in_request);

			if (sdio->block_in_file == sdio->final_block_in_request)
@@ -1054,7 +1042,6 @@ next_block: 


		/* Drop the ref which was taken in get_user_pages() */

		page_cache_release(page);

		block_in_page = 0;

	}

out:

	return ret;
@@ -1122,7 +1109,6 @@ do_blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, 
	struct dio *dio;

	struct dio_submit sdio = { 0, };

	unsigned long user_addr;

	size_t bytes;

	struct buffer_head map_bh = { 0, };

	struct blk_plug plug;

	unsigned long align = offset | iov_iter_alignment(iter);
@@ -1234,6 +1220,10 @@ do_blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, 
	spin_lock_init(&dio->bio_lock);

	dio->refcount = 1;



	sdio.iter = iter;

	sdio.final_block_in_request =

		(offset + iov_iter_count(iter)) >> blkbits;



	/*

	 * In case of non-aligned buffers, we may need 2 more

	 * pages since we need to zero out first and last block.
@@ -1250,34 +1240,9 @@ do_blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, 


	blk_start_plug(&plug);



	for (seg = 0; seg < iter->nr_segs; seg++) {

		user_addr = (unsigned long)iter->iov[seg].iov_base;

		sdio.size += bytes = iter->iov[seg].iov_len;



		/* Index into the first page of the first block */

		sdio.first_block_in_page = (user_addr & ~PAGE_MASK) >> blkbits;

		sdio.final_block_in_request = sdio.block_in_file +

						(bytes >> blkbits);

		/* Page fetching state */

		sdio.head = 0;

		sdio.tail = 0;

		sdio.curr_page = 0;



		sdio.total_pages = 0;

		if (user_addr & (PAGE_SIZE-1)) {

			sdio.total_pages++;

			bytes -= PAGE_SIZE - (user_addr & (PAGE_SIZE - 1));

		}

		sdio.total_pages += (bytes + PAGE_SIZE - 1) / PAGE_SIZE;

		sdio.curr_user_address = user_addr;



	retval = do_direct_IO(dio, &sdio, &map_bh);



		if (retval) {

	if (retval)

		dio_cleanup(dio, &sdio);

			break;

		}

	} /* end iovec loop */



	if (retval == -ENOTBLK) {

		/*

include/linux/uio.h

+2 −0
Original line number Diff line number Diff line
@@ -71,6 +71,8 @@ size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, 
unsigned long iov_iter_alignment(const struct iov_iter *i);

void iov_iter_init(struct iov_iter *i, int direction, const struct iovec *iov,

			unsigned long nr_segs, size_t count);

ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages,

			size_t maxsize, size_t *start);



static inline size_t iov_iter_count(struct iov_iter *i)

{

mm/iov_iter.c

+27 −0
Original line number Diff line number Diff line
@@ -235,3 +235,30 @@ void iov_iter_init(struct iov_iter *i, int direction, 
	i->count = count;

}

EXPORT_SYMBOL(iov_iter_init);



ssize_t iov_iter_get_pages(struct iov_iter *i,

		   struct page **pages, size_t maxsize,

		   size_t *start)

{

	size_t offset = i->iov_offset;

	const struct iovec *iov = i->iov;

	size_t len;

	unsigned long addr;

	int n;

	int res;



	len = iov->iov_len - offset;

	if (len > i->count)

		len = i->count;

	if (len > maxsize)

		len = maxsize;

	addr = (unsigned long)iov->iov_base + offset;

	len += *start = addr & (PAGE_SIZE - 1);

	addr &= ~(PAGE_SIZE - 1);

	n = (len + PAGE_SIZE - 1) / PAGE_SIZE;

	res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);

	if (unlikely(res < 0))

		return res;

	return (res == n ? len : res * PAGE_SIZE) - *start;

}

EXPORT_SYMBOL(iov_iter_get_pages);

CRA Git | Maintained and supported by SUSTech CRA and CCSE