Commit cffbc8aa authored by Dave Chinner's avatar Dave Chinner Committed by Al Viro
Browse files

fs: Convert nr_inodes and nr_unused to per-cpu counters 



The number of inodes allocated does not need to be tied to the
addition or removal of an inode to/from a list. If we are not tied
to a list lock, we could update the counters when inodes are
initialised or destroyed, but to do that we need to convert the
counters to be per-cpu (i.e. independent of a lock). This means that
we have the freedom to change the list/locking implementation
without needing to care about the counters.

Based on a patch originally from Eric Dumazet.

[AV: cleaned up a bit, fixed build breakage on weird configs

Signed-off-by: default avatarDave Chinner <dchinner@redhat.com>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarAl Viro <viro@zeniv.linux.org.uk>
parent be1a16a0

fs/fs-writeback.c

+2 −3
Original line number Diff line number Diff line
@@ -723,7 +723,7 @@ static long wb_check_old_data_flush(struct bdi_writeback *wb) 
	wb->last_old_flush = jiffies;

	nr_pages = global_page_state(NR_FILE_DIRTY) +

			global_page_state(NR_UNSTABLE_NFS) +

			(inodes_stat.nr_inodes - inodes_stat.nr_unused);

			get_nr_dirty_inodes();



	if (nr_pages) {

		struct wb_writeback_work work = {
@@ -1090,8 +1090,7 @@ void writeback_inodes_sb(struct super_block *sb) 


	WARN_ON(!rwsem_is_locked(&sb->s_umount));



	work.nr_pages = nr_dirty + nr_unstable +

			(inodes_stat.nr_inodes - inodes_stat.nr_unused);

	work.nr_pages = nr_dirty + nr_unstable + get_nr_dirty_inodes();



	bdi_queue_work(sb->s_bdi, &work);

	wait_for_completion(&done);

fs/inode.c

+45 −19
Original line number Diff line number Diff line
@@ -103,8 +103,41 @@ static DECLARE_RWSEM(iprune_sem); 
 */

struct inodes_stat_t inodes_stat;



static struct percpu_counter nr_inodes __cacheline_aligned_in_smp;

static struct percpu_counter nr_inodes_unused __cacheline_aligned_in_smp;



static struct kmem_cache *inode_cachep __read_mostly;



static inline int get_nr_inodes(void)

{

	return percpu_counter_sum_positive(&nr_inodes);

}



static inline int get_nr_inodes_unused(void)

{

	return percpu_counter_sum_positive(&nr_inodes_unused);

}



int get_nr_dirty_inodes(void)

{

	int nr_dirty = get_nr_inodes() - get_nr_inodes_unused();

	return nr_dirty > 0 ? nr_dirty : 0;



}



/*

 * Handle nr_inode sysctl

 */

#ifdef CONFIG_SYSCTL

int proc_nr_inodes(ctl_table *table, int write,

		   void __user *buffer, size_t *lenp, loff_t *ppos)

{

	inodes_stat.nr_inodes = get_nr_inodes();

	inodes_stat.nr_unused = get_nr_inodes_unused();

	return proc_dointvec(table, write, buffer, lenp, ppos);

}

#endif



static void wake_up_inode(struct inode *inode)

{

	/*
@@ -192,6 +225,8 @@ int inode_init_always(struct super_block *sb, struct inode *inode) 
	inode->i_fsnotify_mask = 0;

#endif



	percpu_counter_inc(&nr_inodes);



	return 0;

out:

	return -ENOMEM;
@@ -232,6 +267,7 @@ void __destroy_inode(struct inode *inode) 
	if (inode->i_default_acl && inode->i_default_acl != ACL_NOT_CACHED)

		posix_acl_release(inode->i_default_acl);

#endif

	percpu_counter_dec(&nr_inodes);

}

EXPORT_SYMBOL(__destroy_inode);
@@ -286,7 +322,7 @@ void __iget(struct inode *inode) 


	if (!(inode->i_state & (I_DIRTY|I_SYNC)))

		list_move(&inode->i_list, &inode_in_use);

	inodes_stat.nr_unused--;

	percpu_counter_dec(&nr_inodes_unused);

}



void end_writeback(struct inode *inode)
@@ -327,8 +363,6 @@ static void evict(struct inode *inode) 
 */

static void dispose_list(struct list_head *head)

{

	int nr_disposed = 0;



	while (!list_empty(head)) {

		struct inode *inode;
@@ -344,11 +378,7 @@ static void dispose_list(struct list_head *head) 


		wake_up_inode(inode);

		destroy_inode(inode);

		nr_disposed++;

	}

	spin_lock(&inode_lock);

	inodes_stat.nr_inodes -= nr_disposed;

	spin_unlock(&inode_lock);

}



/*
@@ -357,7 +387,7 @@ static void dispose_list(struct list_head *head) 
static int invalidate_list(struct list_head *head, struct list_head *dispose)

{

	struct list_head *next;

	int busy = 0, count = 0;

	int busy = 0;



	next = head->next;

	for (;;) {
@@ -383,13 +413,11 @@ static int invalidate_list(struct list_head *head, struct list_head *dispose) 
			list_move(&inode->i_list, dispose);

			WARN_ON(inode->i_state & I_NEW);

			inode->i_state |= I_FREEING;

			count++;

			percpu_counter_dec(&nr_inodes_unused);

			continue;

		}

		busy = 1;

	}

	/* only unused inodes may be cached with i_count zero */

	inodes_stat.nr_unused -= count;

	return busy;

}
@@ -447,7 +475,6 @@ static int can_unuse(struct inode *inode) 
static void prune_icache(int nr_to_scan)

{

	LIST_HEAD(freeable);

	int nr_pruned = 0;

	int nr_scanned;

	unsigned long reap = 0;
@@ -483,9 +510,8 @@ static void prune_icache(int nr_to_scan) 
		list_move(&inode->i_list, &freeable);

		WARN_ON(inode->i_state & I_NEW);

		inode->i_state |= I_FREEING;

		nr_pruned++;

		percpu_counter_dec(&nr_inodes_unused);

	}

	inodes_stat.nr_unused -= nr_pruned;

	if (current_is_kswapd())

		__count_vm_events(KSWAPD_INODESTEAL, reap);

	else
@@ -517,7 +543,7 @@ static int shrink_icache_memory(struct shrinker *shrink, int nr, gfp_t gfp_mask) 
			return -1;

		prune_icache(nr);

	}

	return (inodes_stat.nr_unused / 100) * sysctl_vfs_cache_pressure;

	return (get_nr_inodes_unused() / 100) * sysctl_vfs_cache_pressure;

}



static struct shrinker icache_shrinker = {
@@ -594,7 +620,6 @@ static inline void 
__inode_add_to_lists(struct super_block *sb, struct hlist_head *head,

			struct inode *inode)

{

	inodes_stat.nr_inodes++;

	list_add(&inode->i_list, &inode_in_use);

	list_add(&inode->i_sb_list, &sb->s_inodes);

	if (head)
@@ -1214,7 +1239,7 @@ static void iput_final(struct inode *inode) 
	if (!drop) {

		if (!(inode->i_state & (I_DIRTY|I_SYNC)))

			list_move(&inode->i_list, &inode_unused);

		inodes_stat.nr_unused++;

		percpu_counter_inc(&nr_inodes_unused);

		if (sb->s_flags & MS_ACTIVE) {

			spin_unlock(&inode_lock);

			return;
@@ -1226,14 +1251,13 @@ static void iput_final(struct inode *inode) 
		spin_lock(&inode_lock);

		WARN_ON(inode->i_state & I_NEW);

		inode->i_state &= ~I_WILL_FREE;

		inodes_stat.nr_unused--;

		percpu_counter_dec(&nr_inodes_unused);

		hlist_del_init(&inode->i_hash);

	}

	list_del_init(&inode->i_list);

	list_del_init(&inode->i_sb_list);

	WARN_ON(inode->i_state & I_NEW);

	inode->i_state |= I_FREEING;

	inodes_stat.nr_inodes--;

	spin_unlock(&inode_lock);

	evict(inode);

	spin_lock(&inode_lock);
@@ -1502,6 +1526,8 @@ void __init inode_init(void) 
					 SLAB_MEM_SPREAD),

					 init_once);

	register_shrinker(&icache_shrinker);

	percpu_counter_init(&nr_inodes, 0);

	percpu_counter_init(&nr_inodes_unused, 0);



	/* Hash may have been set up in inode_init_early */

	if (!hashdist)

fs/internal.h

+1 −0
Original line number Diff line number Diff line
@@ -105,4 +105,5 @@ extern void release_open_intent(struct nameidata *); 
/*

 * inode.c

 */

extern int get_nr_dirty_inodes(void);

extern int invalidate_inodes(struct super_block *);

include/linux/fs.h

+2 −1
Original line number Diff line number Diff line
@@ -2486,7 +2486,8 @@ ssize_t simple_attr_write(struct file *file, const char __user *buf, 
struct ctl_table;

int proc_nr_files(struct ctl_table *table, int write,

		  void __user *buffer, size_t *lenp, loff_t *ppos);



int proc_nr_inodes(struct ctl_table *table, int write,

		   void __user *buffer, size_t *lenp, loff_t *ppos);

int __init get_filesystem_list(char *buf);



#define ACC_MODE(x) ("\004\002\006\006"[(x)&O_ACCMODE])

kernel/sysctl.c

+2 −2
Original line number Diff line number Diff line
@@ -1340,14 +1340,14 @@ static struct ctl_table fs_table[] = { 
		.data		= &inodes_stat,

		.maxlen		= 2*sizeof(int),

		.mode		= 0444,

		.proc_handler	= proc_dointvec,

		.proc_handler	= proc_nr_inodes,

	},

	{

		.procname	= "inode-state",

		.data		= &inodes_stat,

		.maxlen		= 7*sizeof(int),

		.mode		= 0444,

		.proc_handler	= proc_dointvec,

		.proc_handler	= proc_nr_inodes,

	},

	{

		.procname	= "file-nr",

CRA Git | Maintained and supported by SUSTech CRA and CCSE