Commit d559db08 authored by KAMEZAWA Hiroyuki's avatar KAMEZAWA Hiroyuki Committed by Linus Torvalds
Browse files

mm: clean up mm_counter 



Presently, per-mm statistics counter is defined by macro in sched.h

This patch modifies it to
  - defined in mm.h as inlinf functions
  - use array instead of macro's name creation.

This patch is for reducing patch size in future patch to modify
implementation of per-mm counter.

Signed-off-by: default avatarKAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Reviewed-by: default avatarMinchan Kim <minchan.kim@gmail.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 19b629f5

fs/proc/task_mmu.c

+2 −2
Original line number Original line Diff line number Diff line
@@ -65,11 +65,11 @@ unsigned long task_vsize(struct mm_struct *mm) 
int task_statm(struct mm_struct *mm, int *shared, int *text,

int task_statm(struct mm_struct *mm, int *shared, int *text,

	       int *data, int *resident)

	       int *data, int *resident)

{

{

	*shared = get_mm_counter(mm, file_rss);

	*shared = get_mm_counter(mm, MM_FILEPAGES);

	*text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))

	*text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))

								>> PAGE_SHIFT;

								>> PAGE_SHIFT;

	*data = mm->total_vm - mm->shared_vm;

	*data = mm->total_vm - mm->shared_vm;

	*resident = *shared + get_mm_counter(mm, anon_rss);

	*resident = *shared + get_mm_counter(mm, MM_ANONPAGES);

	return mm->total_vm;

	return mm->total_vm;

}

}

include/linux/mm.h

+104 −0
Original line number Original line Diff line number Diff line
@@ -870,6 +870,110 @@ extern int mprotect_fixup(struct vm_area_struct *vma, 
 */

 */

int __get_user_pages_fast(unsigned long start, int nr_pages, int write,

int __get_user_pages_fast(unsigned long start, int nr_pages, int write,

			  struct page **pages);

			  struct page **pages);

/*

 * per-process(per-mm_struct) statistics.

 */

#if USE_SPLIT_PTLOCKS

/*

 * The mm counters are not protected by its page_table_lock,

 * so must be incremented atomically.

 */

static inline void set_mm_counter(struct mm_struct *mm, int member, long value)

{

	atomic_long_set(&mm->rss_stat.count[member], value);

}



static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)

{

	return (unsigned long)atomic_long_read(&mm->rss_stat.count[member]);

}



static inline void add_mm_counter(struct mm_struct *mm, int member, long value)

{

	atomic_long_add(value, &mm->rss_stat.count[member]);

}



static inline void inc_mm_counter(struct mm_struct *mm, int member)

{

	atomic_long_inc(&mm->rss_stat.count[member]);

}



static inline void dec_mm_counter(struct mm_struct *mm, int member)

{

	atomic_long_dec(&mm->rss_stat.count[member]);

}



#else  /* !USE_SPLIT_PTLOCKS */

/*

 * The mm counters are protected by its page_table_lock,

 * so can be incremented directly.

 */

static inline void set_mm_counter(struct mm_struct *mm, int member, long value)

{

	mm->rss_stat.count[member] = value;

}



static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)

{

	return mm->rss_stat.count[member];

}



static inline void add_mm_counter(struct mm_struct *mm, int member, long value)

{

	mm->rss_stat.count[member] += value;

}



static inline void inc_mm_counter(struct mm_struct *mm, int member)

{

	mm->rss_stat.count[member]++;

}



static inline void dec_mm_counter(struct mm_struct *mm, int member)

{

	mm->rss_stat.count[member]--;

}



#endif /* !USE_SPLIT_PTLOCKS */



static inline unsigned long get_mm_rss(struct mm_struct *mm)

{

	return get_mm_counter(mm, MM_FILEPAGES) +

		get_mm_counter(mm, MM_ANONPAGES);

}



static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)

{

	return max(mm->hiwater_rss, get_mm_rss(mm));

}



static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)

{

	return max(mm->hiwater_vm, mm->total_vm);

}



static inline void update_hiwater_rss(struct mm_struct *mm)

{

	unsigned long _rss = get_mm_rss(mm);



	if ((mm)->hiwater_rss < _rss)

		(mm)->hiwater_rss = _rss;

}



static inline void update_hiwater_vm(struct mm_struct *mm)

{

	if (mm->hiwater_vm < mm->total_vm)

		mm->hiwater_vm = mm->total_vm;

}



static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,

					 struct mm_struct *mm)

{

	unsigned long hiwater_rss = get_mm_hiwater_rss(mm);



	if (*maxrss < hiwater_rss)

		*maxrss = hiwater_rss;

}







/*

/*

 * A callback you can register to apply pressure to ageable caches.

 * A callback you can register to apply pressure to ageable caches.

include/linux/mm_types.h

+22 −11
Original line number Original line Diff line number Diff line
@@ -24,12 +24,6 @@ struct address_space; 




#define USE_SPLIT_PTLOCKS	(NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)

#define USE_SPLIT_PTLOCKS	(NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)





#if USE_SPLIT_PTLOCKS

typedef atomic_long_t mm_counter_t;

#else  /* !USE_SPLIT_PTLOCKS */

typedef unsigned long mm_counter_t;

#endif /* !USE_SPLIT_PTLOCKS */



/*

/*

 * Each physical page in the system has a struct page associated with

 * Each physical page in the system has a struct page associated with

 * it to keep track of whatever it is we are using the page for at the

 * it to keep track of whatever it is we are using the page for at the
@@ -201,6 +195,22 @@ struct core_state { 
	struct completion startup;

	struct completion startup;

};

};





enum {

	MM_FILEPAGES,

	MM_ANONPAGES,

	NR_MM_COUNTERS

};



#if USE_SPLIT_PTLOCKS

struct mm_rss_stat {

	atomic_long_t count[NR_MM_COUNTERS];

};

#else  /* !USE_SPLIT_PTLOCKS */

struct mm_rss_stat {

	unsigned long count[NR_MM_COUNTERS];

};

#endif /* !USE_SPLIT_PTLOCKS */



struct mm_struct {

struct mm_struct {

	struct vm_area_struct * mmap;		/* list of VMAs */

	struct vm_area_struct * mmap;		/* list of VMAs */

	struct rb_root mm_rb;

	struct rb_root mm_rb;
@@ -227,11 +237,6 @@ struct mm_struct { 
						 * by mmlist_lock

						 * by mmlist_lock

						 */

						 */





	/* Special counters, in some configurations protected by the

	 * page_table_lock, in other configurations by being atomic.

	 */

	mm_counter_t _file_rss;

	mm_counter_t _anon_rss;





	unsigned long hiwater_rss;	/* High-watermark of RSS usage */

	unsigned long hiwater_rss;	/* High-watermark of RSS usage */

	unsigned long hiwater_vm;	/* High-water virtual memory usage */

	unsigned long hiwater_vm;	/* High-water virtual memory usage */
@@ -244,6 +249,12 @@ struct mm_struct { 




	unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */

	unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */





	/*

	 * Special counters, in some configurations protected by the

	 * page_table_lock, in other configurations by being atomic.

	 */

	struct mm_rss_stat rss_stat;



	struct linux_binfmt *binfmt;

	struct linux_binfmt *binfmt;





	cpumask_t cpu_vm_mask;

	cpumask_t cpu_vm_mask;

include/linux/sched.h

+0 −54
Original line number Original line Diff line number Diff line
@@ -396,60 +396,6 @@ extern void arch_unmap_area_topdown(struct mm_struct *, unsigned long); 
static inline void arch_pick_mmap_layout(struct mm_struct *mm) {}

static inline void arch_pick_mmap_layout(struct mm_struct *mm) {}

#endif

#endif





#if USE_SPLIT_PTLOCKS

/*

 * The mm counters are not protected by its page_table_lock,

 * so must be incremented atomically.

 */

#define set_mm_counter(mm, member, value) atomic_long_set(&(mm)->_##member, value)

#define get_mm_counter(mm, member) ((unsigned long)atomic_long_read(&(mm)->_##member))

#define add_mm_counter(mm, member, value) atomic_long_add(value, &(mm)->_##member)

#define inc_mm_counter(mm, member) atomic_long_inc(&(mm)->_##member)

#define dec_mm_counter(mm, member) atomic_long_dec(&(mm)->_##member)



#else  /* !USE_SPLIT_PTLOCKS */

/*

 * The mm counters are protected by its page_table_lock,

 * so can be incremented directly.

 */

#define set_mm_counter(mm, member, value) (mm)->_##member = (value)

#define get_mm_counter(mm, member) ((mm)->_##member)

#define add_mm_counter(mm, member, value) (mm)->_##member += (value)

#define inc_mm_counter(mm, member) (mm)->_##member++

#define dec_mm_counter(mm, member) (mm)->_##member--



#endif /* !USE_SPLIT_PTLOCKS */



#define get_mm_rss(mm)					\

	(get_mm_counter(mm, file_rss) + get_mm_counter(mm, anon_rss))

#define update_hiwater_rss(mm)	do {			\

	unsigned long _rss = get_mm_rss(mm);		\

	if ((mm)->hiwater_rss < _rss)			\

		(mm)->hiwater_rss = _rss;		\

} while (0)

#define update_hiwater_vm(mm)	do {			\

	if ((mm)->hiwater_vm < (mm)->total_vm)		\

		(mm)->hiwater_vm = (mm)->total_vm;	\

} while (0)



static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)

{

	return max(mm->hiwater_rss, get_mm_rss(mm));

}



static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,

					 struct mm_struct *mm)

{

	unsigned long hiwater_rss = get_mm_hiwater_rss(mm);



	if (*maxrss < hiwater_rss)

		*maxrss = hiwater_rss;

}



static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)

{

	return max(mm->hiwater_vm, mm->total_vm);

}





extern void set_dumpable(struct mm_struct *mm, int value);

extern void set_dumpable(struct mm_struct *mm, int value);

extern int get_dumpable(struct mm_struct *mm);

extern int get_dumpable(struct mm_struct *mm);

kernel/fork.c

+1 −2
Original line number Original line Diff line number Diff line
@@ -455,8 +455,7 @@ static struct mm_struct * mm_init(struct mm_struct * mm, struct task_struct *p) 
		(current->mm->flags & MMF_INIT_MASK) : default_dump_filter;

		(current->mm->flags & MMF_INIT_MASK) : default_dump_filter;

	mm->core_state = NULL;

	mm->core_state = NULL;

	mm->nr_ptes = 0;

	mm->nr_ptes = 0;

	set_mm_counter(mm, file_rss, 0);

	memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));

	set_mm_counter(mm, anon_rss, 0);

	spin_lock_init(&mm->page_table_lock);

	spin_lock_init(&mm->page_table_lock);

	mm->free_area_cache = TASK_UNMAPPED_BASE;

	mm->free_area_cache = TASK_UNMAPPED_BASE;

	mm->cached_hole_size = ~0UL;

	mm->cached_hole_size = ~0UL;

CRA Git | Maintained and supported by SUSTech CRA and CCSE