Commit 004b6862 authored by Chao Yu's avatar Chao Yu Committed by Jaegeuk Kim
Browse files

f2fs: use rb-tree to track pending discard commands 



Introduce rb-tree based discard cache infrastructure to speed up lookup and
merge operation of discard entry.

Signed-off-by: default avatarChao Yu <yuchao0@huawei.com>
[Jaegeuk Kim: initialize dc to avoid build warning]
Signed-off-by: default avatarJaegeuk Kim <jaegeuk@kernel.org>
parent d40d30c5

fs/f2fs/extent_cache.c

+8 −7
Original line number Diff line number Diff line
@@ -49,7 +49,7 @@ static struct rb_entry *__lookup_rb_tree_slow(struct rb_root *root, 
	return NULL;

}



static struct rb_entry *__lookup_rb_tree(struct rb_root *root,

struct rb_entry *__lookup_rb_tree(struct rb_root *root,

				struct rb_entry *cached_re, unsigned int ofs)

{

	struct rb_entry *re;
@@ -61,7 +61,7 @@ static struct rb_entry *__lookup_rb_tree(struct rb_root *root, 
	return re;

}



static struct rb_node **__lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi,

struct rb_node **__lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi,

				struct rb_root *root, struct rb_node **parent,

				unsigned int ofs)

{
@@ -92,13 +92,14 @@ static struct rb_node **__lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi, 
 * in order to simpfy the insertion after.

 * tree must stay unchanged between lookup and insertion.

 */

static struct rb_entry *__lookup_rb_tree_ret(struct rb_root *root,

struct rb_entry *__lookup_rb_tree_ret(struct rb_root *root,

				struct rb_entry *cached_re,

				unsigned int ofs,

				struct rb_entry **prev_entry,

				struct rb_entry **next_entry,

				struct rb_node ***insert_p,

				struct rb_node **insert_parent)

				struct rb_node **insert_parent,

				bool force)

{

	struct rb_node **pnode = &root->rb_node;

	struct rb_node *parent = NULL, *tmp_node;
@@ -145,12 +146,12 @@ static struct rb_entry *__lookup_rb_tree_ret(struct rb_root *root, 
	return NULL;



lookup_neighbors:

	if (ofs == re->ofs) {

	if (ofs == re->ofs || force) {

		/* lookup prev node for merging backward later */

		tmp_node = rb_prev(&re->rb_node);

		*prev_entry = rb_entry_safe(tmp_node, struct rb_entry, rb_node);

	}

	if (ofs == re->ofs + re->len - 1) {

	if (ofs == re->ofs + re->len - 1 || force) {

		/* lookup next node for merging frontward later */

		tmp_node = rb_next(&re->rb_node);

		*next_entry = rb_entry_safe(tmp_node, struct rb_entry, rb_node);
@@ -481,7 +482,7 @@ static void f2fs_update_extent_tree_range(struct inode *inode, 
					(struct rb_entry *)et->cached_en, fofs,

					(struct rb_entry **)&prev_en,

					(struct rb_entry **)&next_en,

					&insert_p, &insert_parent);

					&insert_p, &insert_parent, false);

	if (!en)

		en = next_en;

fs/f2fs/f2fs.h

+45 −3
Original line number Diff line number Diff line
@@ -194,13 +194,26 @@ enum { 
	D_DONE,

};



struct discard_info {

	block_t lstart;			/* logical start address */

	block_t len;			/* length */

	block_t start;			/* actual start address in dev */

};



struct discard_cmd {

	struct rb_node rb_node;		/* rb node located in rb-tree */

	union {

		struct {

			block_t lstart;	/* logical start address */

			block_t len;	/* length */

			block_t start;	/* actual start address in dev */

		};

		struct discard_info di;	/* discard info */



	};

	struct list_head list;		/* command list */

	struct completion wait;		/* compleation */

	struct block_device *bdev;	/* bdev */

	block_t lstart;			/* logical start address */

	block_t start;			/* actual start address in dev */

	block_t len;			/* length */

	int state;			/* state */

	int error;			/* bio error */

};
@@ -217,6 +230,7 @@ struct discard_cmd_control { 
	atomic_t issued_discard;		/* # of issued discard */

	atomic_t issing_discard;		/* # of issing discard */

	atomic_t discard_cmd_cnt;		/* # of cached cmd count */

	struct rb_root root;			/* root of discard rb-tree */

};



/* for the list of fsync inodes, used only during recovery */
@@ -517,6 +531,24 @@ static inline void set_extent_info(struct extent_info *ei, unsigned int fofs, 
	ei->len = len;

}



static inline bool __is_discard_mergeable(struct discard_info *back,

						struct discard_info *front)

{

	return back->lstart + back->len == front->lstart;

}



static inline bool __is_discard_back_mergeable(struct discard_info *cur,

						struct discard_info *back)

{

	return __is_discard_mergeable(back, cur);

}



static inline bool __is_discard_front_mergeable(struct discard_info *cur,

						struct discard_info *front)

{

	return __is_discard_mergeable(cur, front);

}



static inline bool __is_extent_mergeable(struct extent_info *back,

						struct extent_info *front)

{
@@ -2573,6 +2605,16 @@ void f2fs_leave_shrinker(struct f2fs_sb_info *sbi); 
/*

 * extent_cache.c

 */

struct rb_entry *__lookup_rb_tree(struct rb_root *root,

				struct rb_entry *cached_re, unsigned int ofs);

struct rb_node **__lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi,

				struct rb_root *root, struct rb_node **parent,

				unsigned int ofs);

struct rb_entry *__lookup_rb_tree_ret(struct rb_root *root,

		struct rb_entry *cached_re, unsigned int ofs,

		struct rb_entry **prev_entry, struct rb_entry **next_entry,

		struct rb_node ***insert_p, struct rb_node **insert_parent,

		bool force);

unsigned int f2fs_shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink);

bool f2fs_init_extent_tree(struct inode *inode, struct f2fs_extent *i_ext);

void f2fs_drop_extent_tree(struct inode *inode);

fs/f2fs/segment.c

+183 −40
Original line number Diff line number Diff line
@@ -672,7 +672,7 @@ static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) 
	mutex_unlock(&dirty_i->seglist_lock);

}



static void __add_discard_cmd(struct f2fs_sb_info *sbi,

static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,

		struct block_device *bdev, block_t lstart,

		block_t start, block_t len)

{
@@ -689,18 +689,46 @@ static void __add_discard_cmd(struct f2fs_sb_info *sbi, 
	dc->state = D_PREP;

	dc->error = 0;

	init_completion(&dc->wait);



	mutex_lock(&dcc->cmd_lock);

	list_add_tail(&dc->list, pend_list);

	mutex_unlock(&dcc->cmd_lock);



	atomic_inc(&dcc->discard_cmd_cnt);



	return dc;

}



static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,

				struct block_device *bdev, block_t lstart,

				block_t start, block_t len,

				struct rb_node *parent, struct rb_node **p)

{

	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;

	struct discard_cmd *dc;



	dc = __create_discard_cmd(sbi, bdev, lstart, start, len);



	rb_link_node(&dc->rb_node, parent, p);

	rb_insert_color(&dc->rb_node, &dcc->root);



	return dc;

}



static void __remove_discard_cmd(struct f2fs_sb_info *sbi, struct discard_cmd *dc)

static void __detach_discard_cmd(struct discard_cmd_control *dcc,

							struct discard_cmd *dc)

{

	if (dc->state == D_DONE)

		atomic_dec(&(SM_I(sbi)->dcc_info->issing_discard));

		atomic_dec(&dcc->issing_discard);



	list_del(&dc->list);

	rb_erase(&dc->rb_node, &dcc->root);



	kmem_cache_free(discard_cmd_slab, dc);



	atomic_dec(&dcc->discard_cmd_cnt);

}



static void __remove_discard_cmd(struct f2fs_sb_info *sbi,

							struct discard_cmd *dc)

{

	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;



	if (dc->error == -EOPNOTSUPP)

		dc->error = 0;
@@ -708,9 +736,7 @@ static void __remove_discard_cmd(struct f2fs_sb_info *sbi, struct discard_cmd *d 
	if (dc->error)

		f2fs_msg(sbi->sb, KERN_INFO,

				"Issue discard failed, ret: %d", dc->error);

	list_del(&dc->list);

	kmem_cache_free(discard_cmd_slab, dc);

	atomic_dec(&SM_I(sbi)->dcc_info->discard_cmd_cnt);

	__detach_discard_cmd(dcc, dc);

}



static void f2fs_submit_discard_endio(struct bio *bio)
@@ -754,63 +780,179 @@ static void __submit_discard_cmd(struct f2fs_sb_info *sbi, 
	}

}



static int __queue_discard_cmd(struct f2fs_sb_info *sbi,

		struct block_device *bdev, block_t blkstart, block_t blklen)

static struct discard_cmd *__insert_discard_tree(struct f2fs_sb_info *sbi,

				struct block_device *bdev, block_t lstart,

				block_t start, block_t len,

				struct rb_node **insert_p,

				struct rb_node *insert_parent)

{

	block_t lblkstart = blkstart;

	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;

	struct rb_node **p = &dcc->root.rb_node;

	struct rb_node *parent = NULL;

	struct discard_cmd *dc = NULL;



	trace_f2fs_issue_discard(bdev, blkstart, blklen);

	if (insert_p && insert_parent) {

		parent = insert_parent;

		p = insert_p;

		goto do_insert;

	}



	if (sbi->s_ndevs) {

		int devi = f2fs_target_device_index(sbi, blkstart);

	p = __lookup_rb_tree_for_insert(sbi, &dcc->root, &parent, lstart);

do_insert:

	dc = __attach_discard_cmd(sbi, bdev, lstart, start, len, parent, p);

	if (!dc)

		return NULL;



		blkstart -= FDEV(devi).start_blk;

	}

	__add_discard_cmd(sbi, bdev, lblkstart, blkstart, blklen);

	wake_up(&SM_I(sbi)->dcc_info->discard_wait_queue);

	return 0;

	return dc;

}



static void __punch_discard_cmd(struct f2fs_sb_info *sbi,

				struct discard_cmd *dc, block_t blkaddr)

{

	block_t end_block = START_BLOCK(sbi, GET_SEGNO(sbi, blkaddr) + 1);

	struct discard_info di = dc->di;

	bool modified = false;



	if (dc->state == D_DONE || dc->lstart + dc->len <= end_block) {

	if (dc->state == D_DONE || dc->len == 1) {

		__remove_discard_cmd(sbi, dc);

		return;

	}



	if (blkaddr - dc->lstart < dc->lstart + dc->len - end_block) {

		dc->start += (end_block - dc->lstart);

		dc->len -= (end_block - dc->lstart);

		dc->lstart = end_block;

	} else {

	if (blkaddr > di.lstart) {

		dc->len = blkaddr - dc->lstart;

		modified = true;

	}



	if (blkaddr < di.lstart + di.len - 1) {

		if (modified) {

			__insert_discard_tree(sbi, dc->bdev, blkaddr + 1,

					di.start + blkaddr + 1 - di.lstart,

					di.lstart + di.len - 1 - blkaddr,

					NULL, NULL);

		} else {

			dc->lstart++;

			dc->len--;

			dc->start++;

		}

	}

}



/* This should be covered by global mutex, &sit_i->sentry_lock */

void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)

static void __update_discard_tree_range(struct f2fs_sb_info *sbi,

				struct block_device *bdev, block_t lstart,

				block_t start, block_t len)

{

	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;

	struct list_head *pend_list = &(dcc->discard_pend_list);

	struct list_head *wait_list = &(dcc->discard_wait_list);

	struct discard_cmd *dc, *tmp;

	struct discard_cmd *prev_dc = NULL, *next_dc = NULL;

	struct discard_cmd *dc;

	struct discard_info di = {0};

	struct rb_node **insert_p = NULL, *insert_parent = NULL;

	block_t end = lstart + len;



	mutex_lock(&dcc->cmd_lock);



	list_for_each_entry_safe(dc, tmp, pend_list, list) {

		if (dc->lstart <= blkaddr && blkaddr < dc->lstart + dc->len)

			__punch_discard_cmd(sbi, dc, blkaddr);

	dc = (struct discard_cmd *)__lookup_rb_tree_ret(&dcc->root,

					NULL, lstart,

					(struct rb_entry **)&prev_dc,

					(struct rb_entry **)&next_dc,

					&insert_p, &insert_parent, true);

	if (dc)

		prev_dc = dc;



	if (!prev_dc) {

		di.lstart = lstart;

		di.len = next_dc ? next_dc->lstart - lstart : len;

		di.len = min(di.len, len);

		di.start = start;

	}



	list_for_each_entry_safe(dc, tmp, wait_list, list) {

		if (dc->lstart <= blkaddr && blkaddr < dc->lstart + dc->len) {

	while (1) {

		struct rb_node *node;

		bool merged = false;

		struct discard_cmd *tdc = NULL;



		if (prev_dc) {

			di.lstart = prev_dc->lstart + prev_dc->len;

			if (di.lstart < lstart)

				di.lstart = lstart;

			if (di.lstart >= end)

				break;



			if (!next_dc || next_dc->lstart > end)

				di.len = end - di.lstart;

			else

				di.len = next_dc->lstart - di.lstart;

			di.start = start + di.lstart - lstart;

		}



		if (!di.len)

			goto next;



		if (prev_dc && prev_dc->state == D_PREP &&

			prev_dc->bdev == bdev &&

			__is_discard_back_mergeable(&di, &prev_dc->di)) {

			prev_dc->di.len += di.len;

			di = prev_dc->di;

			tdc = prev_dc;

			merged = true;

		}



		if (next_dc && next_dc->state == D_PREP &&

			next_dc->bdev == bdev &&

			__is_discard_front_mergeable(&di, &next_dc->di)) {

			next_dc->di.lstart = di.lstart;

			next_dc->di.len += di.len;

			next_dc->di.start = di.start;

			if (tdc)

				__remove_discard_cmd(sbi, tdc);



			merged = true;

		}



		if (!merged)

			__insert_discard_tree(sbi, bdev, di.lstart, di.start,

							di.len, NULL, NULL);

 next:

		prev_dc = next_dc;

		if (!prev_dc)

			break;



		node = rb_next(&prev_dc->rb_node);

		next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);

	}



	mutex_unlock(&dcc->cmd_lock);

}



static int __queue_discard_cmd(struct f2fs_sb_info *sbi,

		struct block_device *bdev, block_t blkstart, block_t blklen)

{

	block_t lblkstart = blkstart;



	trace_f2fs_issue_discard(bdev, blkstart, blklen);



	if (sbi->s_ndevs) {

		int devi = f2fs_target_device_index(sbi, blkstart);



		blkstart -= FDEV(devi).start_blk;

	}

	__update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);

	wake_up(&SM_I(sbi)->dcc_info->discard_wait_queue);

	return 0;

}



/* This should be covered by global mutex, &sit_i->sentry_lock */

void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)

{

	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;

	struct discard_cmd *dc;



	mutex_lock(&dcc->cmd_lock);



	dc = (struct discard_cmd *)__lookup_rb_tree(&dcc->root, NULL, blkaddr);

	if (dc) {

		if (dc->state != D_PREP)

			wait_for_completion_io(&dc->wait);

		__punch_discard_cmd(sbi, dc, blkaddr);

	}

	}



	mutex_unlock(&dcc->cmd_lock);

}
@@ -1178,6 +1320,7 @@ static int create_discard_cmd_control(struct f2fs_sb_info *sbi) 
	atomic_set(&dcc->discard_cmd_cnt, 0);

	dcc->nr_discards = 0;

	dcc->max_discards = 0;

	dcc->root = RB_ROOT;



	init_waitqueue_head(&dcc->discard_wait_queue);

	SM_I(sbi)->dcc_info = dcc;

CRA Git | Maintained and supported by SUSTech CRA and CCSE