Commit 0b20fcec authored by Chao Yu's avatar Chao Yu Committed by Jaegeuk Kim
Browse files

f2fs: cache global IPU bio 



In commit 8648de2c ("f2fs: add bio cache for IPU"), we added
f2fs_submit_ipu_bio() in __write_data_page() as below:

__write_data_page()

	if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode)) {
		f2fs_submit_ipu_bio(sbi, bio, page);
		....
	}

in order to avoid below deadlock:

Thread A				Thread B
- __write_data_page (inode x, page y)
 - f2fs_do_write_data_page
  - set_page_writeback        ---- set writeback flag in page y
  - f2fs_inplace_write_data
 - f2fs_balance_fs
					 - lock gc_mutex
 - lock gc_mutex
					  - f2fs_gc
					   - do_garbage_collect
					    - gc_data_segment
					     - move_data_page
					      - f2fs_wait_on_page_writeback
					       - wait_on_page_writeback  --- wait writeback of page y

However, the bio submission breaks the merge of IPU IOs.

So in this patch let's add a global bio cache for merged IPU pages,
then f2fs_wait_on_page_writeback() is able to submit bio if a
writebacked page is cached in global bio cache.

Signed-off-by: default avatarChao Yu <yuchao0@huawei.com>
Signed-off-by: default avatarJaegeuk Kim <jaegeuk@kernel.org>
parent 688078e7

fs/f2fs/data.c

+147 −32
Original line number Diff line number Diff line
@@ -29,6 +29,7 @@ 
#define NUM_PREALLOC_POST_READ_CTXS	128



static struct kmem_cache *bio_post_read_ctx_cache;

static struct kmem_cache *bio_entry_slab;

static mempool_t *bio_post_read_ctx_pool;



static bool __is_cp_guaranteed(struct page *page)
@@ -543,6 +544,126 @@ static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, 
	return io_type_is_mergeable(io, fio);

}



static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio,

				struct page *page, enum temp_type temp)

{

	struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;

	struct bio_entry *be;



	be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS);

	be->bio = bio;

	bio_get(bio);



	if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE)

		f2fs_bug_on(sbi, 1);



	down_write(&io->bio_list_lock);

	list_add_tail(&be->list, &io->bio_list);

	up_write(&io->bio_list_lock);

}



static void del_bio_entry(struct bio_entry *be)

{

	list_del(&be->list);

	kmem_cache_free(bio_entry_slab, be);

}



static int add_ipu_page(struct f2fs_sb_info *sbi, struct bio **bio,

							struct page *page)

{

	enum temp_type temp;

	bool found = false;

	int ret = -EAGAIN;



	for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {

		struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;

		struct list_head *head = &io->bio_list;

		struct bio_entry *be;



		down_write(&io->bio_list_lock);

		list_for_each_entry(be, head, list) {

			if (be->bio != *bio)

				continue;



			found = true;



			if (bio_add_page(*bio, page, PAGE_SIZE, 0) == PAGE_SIZE) {

				ret = 0;

				break;

			}



			/* bio is full */

			del_bio_entry(be);

			__submit_bio(sbi, *bio, DATA);

			break;

		}

		up_write(&io->bio_list_lock);

	}



	if (ret) {

		bio_put(*bio);

		*bio = NULL;

	}



	return ret;

}



void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,

					struct bio **bio, struct page *page)

{

	enum temp_type temp;

	bool found = false;

	struct bio *target = bio ? *bio : NULL;



	for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {

		struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;

		struct list_head *head = &io->bio_list;

		struct bio_entry *be;



		if (list_empty(head))

			continue;



		down_read(&io->bio_list_lock);

		list_for_each_entry(be, head, list) {

			if (target)

				found = (target == be->bio);

			else

				found = __has_merged_page(be->bio, NULL,

								page, 0);

			if (found)

				break;

		}

		up_read(&io->bio_list_lock);



		if (!found)

			continue;



		found = false;



		down_write(&io->bio_list_lock);

		list_for_each_entry(be, head, list) {

			if (target)

				found = (target == be->bio);

			else

				found = __has_merged_page(be->bio, NULL,

								page, 0);

			if (found) {

				target = be->bio;

				del_bio_entry(be);

				break;

			}

		}

		up_write(&io->bio_list_lock);

	}



	if (found)

		__submit_bio(sbi, target, DATA);

	if (bio && *bio) {

		bio_put(*bio);

		*bio = NULL;

	}

}



int f2fs_merge_page_bio(struct f2fs_io_info *fio)

{

	struct bio *bio = *fio->bio;
@@ -557,19 +678,16 @@ int f2fs_merge_page_bio(struct f2fs_io_info *fio) 
	f2fs_trace_ios(fio, 0);



	if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block,

						fio->new_blkaddr)) {

		__submit_bio(fio->sbi, bio, fio->type);

		bio = NULL;

	}

						fio->new_blkaddr))

		f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL);

alloc_new:

	if (!bio) {

		bio = __bio_alloc(fio, BIO_MAX_PAGES);

		bio_set_op_attrs(bio, fio->op, fio->op_flags);

	}



	if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {

		__submit_bio(fio->sbi, bio, fio->type);

		bio = NULL;

		add_bio_entry(fio->sbi, bio, page, fio->temp);

	} else {

		if (add_ipu_page(fio->sbi, &bio, page))

			goto alloc_new;

	}
@@ -584,19 +702,6 @@ alloc_new: 
	return 0;

}



static void f2fs_submit_ipu_bio(struct f2fs_sb_info *sbi, struct bio **bio,

							struct page *page)

{

	if (!bio)

		return;



	if (!__has_merged_page(*bio, NULL, page, 0))

		return;



	__submit_bio(sbi, *bio, DATA);

	*bio = NULL;

}



void f2fs_submit_page_write(struct f2fs_io_info *fio)

{

	struct f2fs_sb_info *sbi = fio->sbi;
@@ -2215,14 +2320,12 @@ out: 


	unlock_page(page);

	if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&

					!F2FS_I(inode)->cp_task) {

		f2fs_submit_ipu_bio(sbi, bio, page);

					!F2FS_I(inode)->cp_task)

		f2fs_balance_fs(sbi, need_balance_fs);

	}



	if (unlikely(f2fs_cp_error(sbi))) {

		f2fs_submit_ipu_bio(sbi, bio, page);

		f2fs_submit_merged_write(sbi, DATA);

		f2fs_submit_merged_ipu_write(sbi, bio, NULL);

		submitted = NULL;

	}
@@ -2342,14 +2445,12 @@ continue_unlock: 
			}



			if (PageWriteback(page)) {

				if (wbc->sync_mode != WB_SYNC_NONE) {

				if (wbc->sync_mode != WB_SYNC_NONE)

					f2fs_wait_on_page_writeback(page,

							DATA, true, true);

					f2fs_submit_ipu_bio(sbi, &bio, page);

				} else {

				else

					goto continue_unlock;

			}

			}



			if (!clear_page_dirty_for_io(page))

				goto continue_unlock;
@@ -2406,7 +2507,7 @@ continue_unlock: 
								NULL, 0, DATA);

	/* submit cached bio of IPU write */

	if (bio)

		__submit_bio(sbi, bio, DATA);

		f2fs_submit_merged_ipu_write(sbi, &bio, NULL);



	return ret;

}
@@ -3211,8 +3312,22 @@ fail: 
	return -ENOMEM;

}



void __exit f2fs_destroy_post_read_processing(void)

void f2fs_destroy_post_read_processing(void)

{

	mempool_destroy(bio_post_read_ctx_pool);

	kmem_cache_destroy(bio_post_read_ctx_cache);

}



int __init f2fs_init_bio_entry_cache(void)

{

	bio_entry_slab = f2fs_kmem_cache_create("bio_entry_slab",

			sizeof(struct bio_entry));

	if (!bio_entry_slab)

		return -ENOMEM;

	return 0;

}



void __exit f2fs_destroy_bio_entry_cache(void)

{

	kmem_cache_destroy(bio_entry_slab);

}

fs/f2fs/f2fs.h

+11 −0
Original line number Diff line number Diff line
@@ -1068,6 +1068,11 @@ struct f2fs_io_info { 
	unsigned char version;		/* version of the node */

};



struct bio_entry {

	struct bio *bio;

	struct list_head list;

};



#define is_read_io(rw) ((rw) == READ)

struct f2fs_bio_info {

	struct f2fs_sb_info *sbi;	/* f2fs superblock */
@@ -1077,6 +1082,8 @@ struct f2fs_bio_info { 
	struct rw_semaphore io_rwsem;	/* blocking op for bio */

	spinlock_t io_lock;		/* serialize DATA/NODE IOs */

	struct list_head io_list;	/* track fios */

	struct list_head bio_list;	/* bio entry list head */

	struct rw_semaphore bio_list_lock;	/* lock to protect bio entry list */

};



#define FDEV(i)				(sbi->devs[i])
@@ -3195,10 +3202,14 @@ void f2fs_destroy_checkpoint_caches(void); 
 */

int f2fs_init_post_read_processing(void);

void f2fs_destroy_post_read_processing(void);

int f2fs_init_bio_entry_cache(void);

void f2fs_destroy_bio_entry_cache(void);

void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type);

void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,

				struct inode *inode, struct page *page,

				nid_t ino, enum page_type type);

void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,

					struct bio **bio, struct page *page);

void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi);

int f2fs_submit_page_bio(struct f2fs_io_info *fio);

int f2fs_merge_page_bio(struct f2fs_io_info *fio);

fs/f2fs/segment.c

+3 −0
Original line number Diff line number Diff line
@@ -3379,7 +3379,10 @@ void f2fs_wait_on_page_writeback(struct page *page, 
	if (PageWriteback(page)) {

		struct f2fs_sb_info *sbi = F2FS_P_SB(page);



		/* submit cached LFS IO */

		f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);

		/* sbumit cached IPU IO */

		f2fs_submit_merged_ipu_write(sbi, NULL, page);

		if (ordered) {

			wait_on_page_writeback(page);

			f2fs_bug_on(sbi, locked && PageWriteback(page));

fs/f2fs/super.c

+8 −0
Original line number Diff line number Diff line
@@ -3342,6 +3342,8 @@ try_onemore: 
			sbi->write_io[i][j].bio = NULL;

			spin_lock_init(&sbi->write_io[i][j].io_lock);

			INIT_LIST_HEAD(&sbi->write_io[i][j].io_list);

			INIT_LIST_HEAD(&sbi->write_io[i][j].bio_list);

			init_rwsem(&sbi->write_io[i][j].bio_list_lock);

		}

	}
@@ -3753,8 +3755,13 @@ static int __init init_f2fs_fs(void) 
	err = f2fs_init_post_read_processing();

	if (err)

		goto free_root_stats;

	err = f2fs_init_bio_entry_cache();

	if (err)

		goto free_post_read;

	return 0;



free_post_read:

	f2fs_destroy_post_read_processing();

free_root_stats:

	f2fs_destroy_root_stats();

	unregister_filesystem(&f2fs_fs_type);
@@ -3778,6 +3785,7 @@ fail: 


static void __exit exit_f2fs_fs(void)

{

	f2fs_destroy_bio_entry_cache();

	f2fs_destroy_post_read_processing();

	f2fs_destroy_root_stats();

	unregister_filesystem(&f2fs_fs_type);

CRA Git | Maintained and supported by SUSTech CRA and CCSE