Commit 5a920155 authored by Thomas Gleixner's avatar Thomas Gleixner
Browse files

x86/process: Optimize TIF_NOTSC switch 



Provide and use a toggle helper instead of doing it with a branch.

x86_64: arch/x86/kernel/process.o
text	   data	    bss	    dec	    hex
3008	   8577	     16	  11601	   2d51 Before
2976       8577      16	  11569	   2d31 After

i386: arch/x86/kernel/process.o
text	   data	    bss	    dec	    hex
2925	   8673	      8	  11606	   2d56 Before
2893	   8673       8	  11574	   2d36 After

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Andy Lutomirski <luto@kernel.org>
Link: http://lkml.kernel.org/r/20170214081104.9244-4-khuey@kylehuey.com


Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent b9894a2f

arch/x86/include/asm/tlbflush.h

+10 −0
Original line number Diff line number Diff line
@@ -110,6 +110,16 @@ static inline void cr4_clear_bits(unsigned long mask) 
	}

}



static inline void cr4_toggle_bits(unsigned long mask)

{

	unsigned long cr4;



	cr4 = this_cpu_read(cpu_tlbstate.cr4);

	cr4 ^= mask;

	this_cpu_write(cpu_tlbstate.cr4, cr4);

	__write_cr4(cr4);

}



/* Read the CR4 shadow. */

static inline unsigned long cr4_read_shadow(void)

{

arch/x86/kernel/process.c

+4 −18
Original line number Diff line number Diff line
@@ -124,11 +124,6 @@ void flush_thread(void) 
	fpu__clear(&tsk->thread.fpu);

}



static void hard_disable_TSC(void)

{

	cr4_set_bits(X86_CR4_TSD);

}



void disable_TSC(void)

{

	preempt_disable();
@@ -137,15 +132,10 @@ void disable_TSC(void) 
		 * Must flip the CPU state synchronously with

		 * TIF_NOTSC in the current running context.

		 */

		hard_disable_TSC();

		cr4_set_bits(X86_CR4_TSD);

	preempt_enable();

}



static void hard_enable_TSC(void)

{

	cr4_clear_bits(X86_CR4_TSD);

}



static void enable_TSC(void)

{

	preempt_disable();
@@ -154,7 +144,7 @@ static void enable_TSC(void) 
		 * Must flip the CPU state synchronously with

		 * TIF_NOTSC in the current running context.

		 */

		hard_enable_TSC();

		cr4_clear_bits(X86_CR4_TSD);

	preempt_enable();

}
@@ -233,12 +223,8 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p, 
		wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);

	}



	if ((tifp ^ tifn) & _TIF_NOTSC) {

		if (tifn & _TIF_NOTSC)

			hard_disable_TSC();

		else

			hard_enable_TSC();

	}

	if ((tifp ^ tifn) & _TIF_NOTSC)

		cr4_toggle_bits(X86_CR4_TSD);

}



/*

CRA Git | Maintained and supported by SUSTech CRA and CCSE