Commit 3e7ee3e7 authored by Jens Axboe's avatar Jens Axboe
Browse files

[PATCH] splice: fix page stealing LRU handling. 



Originally from Nick Piggin, just adapted to the newer branch.

You can't check PageLRU without holding zone->lru_lock.  The page
release code can get away with it only because the page refcount is 0 at
that point. Also, you can't reliably remove pages from the LRU unless
the refcount is 0. Ever.

Signed-off-by: default avatarNick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: default avatarJens Axboe <axboe@suse.de>
parent ad8d6f0a

fs/pipe.c

+3 −0
Original line number Original line Diff line number Diff line
@@ -95,6 +95,8 @@ static void anon_pipe_buf_release(struct pipe_inode_info *info, struct pipe_buff 
{

{

	struct page *page = buf->page;

	struct page *page = buf->page;





	buf->flags &= ~PIPE_BUF_FLAG_STOLEN;



	/*

	/*

	 * If nobody else uses this page, and we don't already have a

	 * If nobody else uses this page, and we don't already have a

	 * temporary page, let's keep track of it as a one-deep

	 * temporary page, let's keep track of it as a one-deep
@@ -124,6 +126,7 @@ static void anon_pipe_buf_unmap(struct pipe_inode_info *info, struct pipe_buffer 
static int anon_pipe_buf_steal(struct pipe_inode_info *info,

static int anon_pipe_buf_steal(struct pipe_inode_info *info,

			       struct pipe_buffer *buf)

			       struct pipe_buffer *buf)

{

{

	buf->flags |= PIPE_BUF_FLAG_STOLEN;

	return 0;

	return 0;

}

}

fs/splice.c

+11 −19
Original line number Original line Diff line number Diff line
@@ -67,16 +67,7 @@ static int page_cache_pipe_buf_steal(struct pipe_inode_info *info, 
	if (!remove_mapping(mapping, page))

	if (!remove_mapping(mapping, page))

		return 1;

		return 1;





	if (PageLRU(page)) {

	buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU;

		struct zone *zone = page_zone(page);



		spin_lock_irq(&zone->lru_lock);

		BUG_ON(!PageLRU(page));

		__ClearPageLRU(page);

		del_page_from_lru(zone, page);

		spin_unlock_irq(&zone->lru_lock);

	}



	return 0;

	return 0;

}

}
@@ -85,6 +76,7 @@ static void page_cache_pipe_buf_release(struct pipe_inode_info *info, 
{

{

	page_cache_release(buf->page);

	page_cache_release(buf->page);

	buf->page = NULL;

	buf->page = NULL;

	buf->flags &= ~(PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU);

}

}





static void *page_cache_pipe_buf_map(struct file *file,

static void *page_cache_pipe_buf_map(struct file *file,
@@ -414,11 +406,12 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, 
{

{

	struct file *file = sd->file;

	struct file *file = sd->file;

	struct address_space *mapping = file->f_mapping;

	struct address_space *mapping = file->f_mapping;

	gfp_t gfp_mask = mapping_gfp_mask(mapping);

	unsigned int offset;

	unsigned int offset;

	struct page *page;

	struct page *page;

	pgoff_t index;

	pgoff_t index;

	char *src;

	char *src;

	int ret, stolen;

	int ret;





	/*

	/*

	 * after this, page will be locked and unmapped

	 * after this, page will be locked and unmapped
@@ -429,7 +422,6 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, 




	index = sd->pos >> PAGE_CACHE_SHIFT;

	index = sd->pos >> PAGE_CACHE_SHIFT;

	offset = sd->pos & ~PAGE_CACHE_MASK;

	offset = sd->pos & ~PAGE_CACHE_MASK;

	stolen = 0;





	/*

	/*

	 * reuse buf page, if SPLICE_F_MOVE is set

	 * reuse buf page, if SPLICE_F_MOVE is set
@@ -443,15 +435,15 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, 
			goto find_page;

			goto find_page;





		page = buf->page;

		page = buf->page;

		stolen = 1;

		if (add_to_page_cache(page, mapping, index, gfp_mask))

		if (add_to_page_cache_lru(page, mapping, index,

						mapping_gfp_mask(mapping)))

			goto find_page;

			goto find_page;



		if (!(buf->flags & PIPE_BUF_FLAG_LRU))

			lru_cache_add(page);

	} else {

	} else {

find_page:

find_page:

		ret = -ENOMEM;

		ret = -ENOMEM;

		page = find_or_create_page(mapping, index,

		page = find_or_create_page(mapping, index, gfp_mask);

						mapping_gfp_mask(mapping));

		if (!page)

		if (!page)

			goto out;

			goto out;
@@ -494,7 +486,7 @@ find_page: 
	} else if (ret)

	} else if (ret)

		goto out;

		goto out;





	if (!stolen) {

	if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {

		char *dst = kmap_atomic(page, KM_USER0);

		char *dst = kmap_atomic(page, KM_USER0);





		memcpy(dst + offset, src + buf->offset, sd->len);

		memcpy(dst + offset, src + buf->offset, sd->len);
@@ -511,7 +503,7 @@ find_page: 




	balance_dirty_pages_ratelimited(mapping);

	balance_dirty_pages_ratelimited(mapping);

out:

out:

	if (!stolen) {

	if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {

		page_cache_release(page);

		page_cache_release(page);

		unlock_page(page);

		unlock_page(page);

	}

	}

include/linux/pipe_fs_i.h

+4 −0
Original line number Original line Diff line number Diff line
@@ -5,10 +5,14 @@ 




#define PIPE_BUFFERS (16)

#define PIPE_BUFFERS (16)





#define PIPE_BUF_FLAG_STOLEN	0x01

#define PIPE_BUF_FLAG_LRU	0x02



struct pipe_buffer {

struct pipe_buffer {

	struct page *page;

	struct page *page;

	unsigned int offset, len;

	unsigned int offset, len;

	struct pipe_buf_operations *ops;

	struct pipe_buf_operations *ops;

	unsigned int flags;

};

};





struct pipe_buf_operations {

struct pipe_buf_operations {

CRA Git | Maintained and supported by SUSTech CRA and CCSE