Commit 6df46865 authored by Dave Hansen's avatar Dave Hansen Committed by Linus Torvalds
Browse files

mm: vmstats: track TLB flush stats on UP too 



The previous patch doing vmstats for TLB flushes ("mm: vmstats: tlb flush
counters") effectively missed UP since arch/x86/mm/tlb.c is only compiled
for SMP.

UP systems do not do remote TLB flushes, so compile those counters out on
UP.

arch/x86/kernel/cpu/mtrr/generic.c calls __flush_tlb() directly.  This is
probably an optimization since both the mtrr code and __flush_tlb() write
cr4.  It would probably be safe to make that a flush_tlb_all() (and then
get these statistics), but the mtrr code is ancient and I'm hesitant to
touch it other than to just stick in the counters.

[akpm@linux-foundation.org: tweak comments]
Signed-off-by: default avatarDave Hansen <dave.hansen@linux.intel.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 9824cf97

arch/x86/include/asm/tlbflush.h

+31 −6
Original line number Diff line number Diff line
@@ -62,6 +62,7 @@ static inline void __flush_tlb_all(void) 


static inline void __flush_tlb_one(unsigned long addr)

{

	count_vm_event(NR_TLB_LOCAL_FLUSH_ONE);

	__flush_tlb_single(addr);

}
@@ -84,14 +85,38 @@ static inline void __flush_tlb_one(unsigned long addr) 


#ifndef CONFIG_SMP



#define flush_tlb() __flush_tlb()

#define flush_tlb_all() __flush_tlb_all()

#define local_flush_tlb() __flush_tlb()

/* "_up" is for UniProcessor.

 *

 * This is a helper for other header functions.  *Not* intended to be called

 * directly.  All global TLB flushes need to either call this, or to bump the

 * vm statistics themselves.

 */

static inline void __flush_tlb_up(void)

{

	count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);

	__flush_tlb();

}



static inline void flush_tlb_all(void)

{

	count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);

	__flush_tlb_all();

}



static inline void flush_tlb(void)

{

	__flush_tlb_up();

}



static inline void local_flush_tlb(void)

{

	__flush_tlb_up();

}



static inline void flush_tlb_mm(struct mm_struct *mm)

{

	if (mm == current->active_mm)

		__flush_tlb();

		__flush_tlb_up();

}



static inline void flush_tlb_page(struct vm_area_struct *vma,
@@ -105,14 +130,14 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, 
				   unsigned long start, unsigned long end)

{

	if (vma->vm_mm == current->active_mm)

		__flush_tlb();

		__flush_tlb_up();

}



static inline void flush_tlb_mm_range(struct mm_struct *mm,

	   unsigned long start, unsigned long end, unsigned long vmflag)

{

	if (mm == current->active_mm)

		__flush_tlb();

		__flush_tlb_up();

}



static inline void native_flush_tlb_others(const struct cpumask *cpumask,

arch/x86/kernel/cpu/mtrr/generic.c

+2 −0
Original line number Diff line number Diff line
@@ -683,6 +683,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock) 
	}



	/* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */

	count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);

	__flush_tlb();



	/* Save MTRR state */
@@ -696,6 +697,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock) 
static void post_set(void) __releases(set_atomicity_lock)

{

	/* Flush TLBs (no need to flush caches - they are disabled) */

	count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);

	__flush_tlb();



	/* Intel (P6) standard MTRRs */

arch/x86/mm/tlb.c

+1 −3
Original line number Diff line number Diff line
@@ -280,11 +280,9 @@ static void do_kernel_range_flush(void *info) 
	unsigned long addr;



	/* flush range by one by one 'invlpg' */

	for (addr = f->flush_start; addr < f->flush_end; addr += PAGE_SIZE) {

		count_vm_event(NR_TLB_LOCAL_FLUSH_ONE_KERNEL);

	for (addr = f->flush_start; addr < f->flush_end; addr += PAGE_SIZE)

		__flush_tlb_single(addr);

}

}



void flush_tlb_kernel_range(unsigned long start, unsigned long end)

{

include/linux/vm_event_item.h

+2 −1
Original line number Diff line number Diff line
@@ -70,11 +70,12 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, 
		THP_ZERO_PAGE_ALLOC,

		THP_ZERO_PAGE_ALLOC_FAILED,

#endif

#ifdef CONFIG_SMP

		NR_TLB_REMOTE_FLUSH,	/* cpu tried to flush others' tlbs */

		NR_TLB_REMOTE_FLUSH_RECEIVED,/* cpu received ipi for flush */

#endif

		NR_TLB_LOCAL_FLUSH_ALL,

		NR_TLB_LOCAL_FLUSH_ONE,

		NR_TLB_LOCAL_FLUSH_ONE_KERNEL,

		NR_VM_EVENT_ITEMS

};

mm/vmstat.c

+2 −1
Original line number Diff line number Diff line
@@ -817,11 +817,12 @@ const char * const vmstat_text[] = { 
	"thp_zero_page_alloc",

	"thp_zero_page_alloc_failed",

#endif

#ifdef CONFIG_SMP

	"nr_tlb_remote_flush",

	"nr_tlb_remote_flush_received",

#endif

	"nr_tlb_local_flush_all",

	"nr_tlb_local_flush_one",

	"nr_tlb_local_flush_one_kernel",



#endif /* CONFIG_VM_EVENTS_COUNTERS */

};

CRA Git | Maintained and supported by SUSTech CRA and CCSE