Commit 2faf153b authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Borislav Petkov
Browse files

x86/tlb: Move __flush_tlb() out of line 



cpu_tlbstate is exported because various TLB-related functions need
access to it, but cpu_tlbstate is sensitive information which should
only be accessed by well-contained kernel functions and not be directly
exposed to modules.

As a first step, move __flush_tlb() out of line and hide the native
function. The latter can be static when CONFIG_PARAVIRT is disabled.

Consolidate the namespace while at it and remove the pointless extra
wrapper in the paravirt code.

No functional change.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Reviewed-by: default avatarAlexandre Chartre <alexandre.chartre@oracle.com>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20200421092559.246130908@linutronix.de
parent 9020d395

arch/x86/include/asm/paravirt.h

+3 −1
Original line number Diff line number Diff line
@@ -47,7 +47,9 @@ static inline void slow_down_io(void) 
#endif

}



static inline void __flush_tlb(void)

void native_flush_tlb_local(void);



static inline void __flush_tlb_local(void)

{

	PVOP_VCALL0(mmu.flush_tlb_user);

}

arch/x86/include/asm/tlbflush.h

+5 −24
Original line number Diff line number Diff line
@@ -140,10 +140,11 @@ static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid) 
	return __sme_pa(pgd) | kern_pcid(asid) | CR3_NOFLUSH;

}



void flush_tlb_local(void);



#ifdef CONFIG_PARAVIRT

#include <asm/paravirt.h>

#else

#define __flush_tlb() __native_flush_tlb()

#define __flush_tlb_global()		__native_flush_tlb_global()

#define __flush_tlb_one_user(addr)	__native_flush_tlb_one_user(addr)

#endif
@@ -370,24 +371,6 @@ static inline void invalidate_user_asid(u16 asid) 
		  (unsigned long *)this_cpu_ptr(&cpu_tlbstate.user_pcid_flush_mask));

}



/*

 * flush the entire current user mapping

 */

static inline void __native_flush_tlb(void)

{

	/*

	 * Preemption or interrupts must be disabled to protect the access

	 * to the per CPU variable and to prevent being preempted between

	 * read_cr3() and write_cr3().

	 */

	WARN_ON_ONCE(preemptible());



	invalidate_user_asid(this_cpu_read(cpu_tlbstate.loaded_mm_asid));



	/* If current->mm == NULL then the read_cr3() "borrows" an mm */

	native_write_cr3(__native_read_cr3());

}



/*

 * flush everything

 */
@@ -461,7 +444,7 @@ static inline void __flush_tlb_all(void) 
		/*

		 * !PGE -> !PCID (setup_pcid()), thus every flush is total.

		 */

		__flush_tlb();

		flush_tlb_local();

	}

}
@@ -537,8 +520,6 @@ struct flush_tlb_info { 
	bool			freed_tables;

};



#define local_flush_tlb() __flush_tlb()



#define flush_tlb_mm(mm)						\

		flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL, true)

arch/x86/kernel/cpu/mtrr/generic.c

+2 −2
Original line number Diff line number Diff line
@@ -761,7 +761,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock) 


	/* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */

	count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);

	__flush_tlb();

	flush_tlb_local();



	/* Save MTRR state */

	rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
@@ -778,7 +778,7 @@ static void post_set(void) __releases(set_atomicity_lock) 
{

	/* Flush TLBs (no need to flush caches - they are disabled) */

	count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);

	__flush_tlb();

	flush_tlb_local();



	/* Intel (P6) standard MTRRs */

	mtrr_wrmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);

arch/x86/kernel/paravirt.c

+1 −6
Original line number Diff line number Diff line
@@ -160,11 +160,6 @@ unsigned paravirt_patch_insns(void *insn_buff, unsigned len, 
	return insn_len;

}



static void native_flush_tlb(void)

{

	__native_flush_tlb();

}



/*

 * Global pages have to be flushed a bit differently. Not a real

 * performance problem because this does not happen often.
@@ -359,7 +354,7 @@ struct paravirt_patch_template pv_ops = { 
#endif /* CONFIG_PARAVIRT_XXL */



	/* Mmu ops. */

	.mmu.flush_tlb_user	= native_flush_tlb,

	.mmu.flush_tlb_user	= native_flush_tlb_local,

	.mmu.flush_tlb_kernel	= native_flush_tlb_global,

	.mmu.flush_tlb_one_user	= native_flush_tlb_one_user,

	.mmu.flush_tlb_others	= native_flush_tlb_others,

arch/x86/mm/mem_encrypt.c

+1 −1
Original line number Diff line number Diff line
@@ -134,7 +134,7 @@ static void __init __sme_early_map_unmap_mem(void *vaddr, unsigned long size, 
		size = (size <= PMD_SIZE) ? 0 : size - PMD_SIZE;

	} while (size);



	__native_flush_tlb();

	flush_tlb_local();

}



void __init sme_unmap_bootdata(char *real_mode_data)

CRA Git | Maintained and supported by SUSTech CRA and CCSE