Commit 127ac915 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Borislav Petkov
Browse files

x86/tlb: Move __flush_tlb_one_user() out of line 



cpu_tlbstate is exported because various TLB-related functions need access
to it, but cpu_tlbstate is sensitive information which should only be
accessed by well-contained kernel functions and not be directly exposed to
modules.

As a third step, move _flush_tlb_one_user() out of line and hide the
native function. The latter can be static when CONFIG_PARAVIRT is
disabled.

Consolidate the name space while at it and remove the pointless extra
wrapper in the paravirt code.

No functional change.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Reviewed-by: default avatarAlexandre Chartre <alexandre.chartre@oracle.com>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20200421092559.428213098@linutronix.de
parent cd30d26c

arch/x86/include/asm/paravirt.h

+1 −0
Original line number Diff line number Diff line
@@ -49,6 +49,7 @@ static inline void slow_down_io(void) 


void native_flush_tlb_local(void);

void native_flush_tlb_global(void);

void native_flush_tlb_one_user(unsigned long addr);



static inline void __flush_tlb_local(void)

{

arch/x86/include/asm/tlbflush.h

+2 −51
Original line number Diff line number Diff line
@@ -142,11 +142,10 @@ static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid) 


void flush_tlb_local(void);

void flush_tlb_global(void);

void flush_tlb_one_user(unsigned long addr);



#ifdef CONFIG_PARAVIRT

#include <asm/paravirt.h>

#else

#define __flush_tlb_one_user(addr)	__native_flush_tlb_one_user(addr)

#endif



struct tlb_context {
@@ -345,54 +344,6 @@ static inline void cr4_set_bits_and_update_boot(unsigned long mask) 


extern void initialize_tlbstate_and_flush(void);



/*

 * Given an ASID, flush the corresponding user ASID.  We can delay this

 * until the next time we switch to it.

 *

 * See SWITCH_TO_USER_CR3.

 */

static inline void invalidate_user_asid(u16 asid)

{

	/* There is no user ASID if address space separation is off */

	if (!IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))

		return;



	/*

	 * We only have a single ASID if PCID is off and the CR3

	 * write will have flushed it.

	 */

	if (!cpu_feature_enabled(X86_FEATURE_PCID))

		return;



	if (!static_cpu_has(X86_FEATURE_PTI))

		return;



	__set_bit(kern_pcid(asid),

		  (unsigned long *)this_cpu_ptr(&cpu_tlbstate.user_pcid_flush_mask));

}



/*

 * flush one page in the user mapping

 */

static inline void __native_flush_tlb_one_user(unsigned long addr)

{

	u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);



	asm volatile("invlpg (%0)" ::"r" (addr) : "memory");



	if (!static_cpu_has(X86_FEATURE_PTI))

		return;



	/*

	 * Some platforms #GP if we call invpcid(type=1/2) before CR4.PCIDE=1.

	 * Just use invalidate_user_asid() in case we are called early.

	 */

	if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE))

		invalidate_user_asid(loaded_mm_asid);

	else

		invpcid_flush_one(user_pcid(loaded_mm_asid), addr);

}



/*

 * flush everything

 */
@@ -432,7 +383,7 @@ static inline void __flush_tlb_one_kernel(unsigned long addr) 
	 * kernel address space and for its usermode counterpart, but it does

	 * not flush it for other address spaces.

	 */

	__flush_tlb_one_user(addr);

	flush_tlb_one_user(addr);



	if (!static_cpu_has(X86_FEATURE_PTI))

		return;

arch/x86/kernel/paravirt.c

+0 −5
Original line number Diff line number Diff line
@@ -160,11 +160,6 @@ unsigned paravirt_patch_insns(void *insn_buff, unsigned len, 
	return insn_len;

}



static void native_flush_tlb_one_user(unsigned long addr)

{

	__native_flush_tlb_one_user(addr);

}



struct static_key paravirt_steal_enabled;

struct static_key paravirt_steal_rq_enabled;

arch/x86/mm/tlb.c

+55 −1
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ 
# define STATIC_NOPV			static

# define __flush_tlb_local		native_flush_tlb_local

# define __flush_tlb_global		native_flush_tlb_global

# define __flush_tlb_one_user(addr)	native_flush_tlb_one_user(addr)

#endif



/*
@@ -118,6 +119,32 @@ static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen, 
	*need_flush = true;

}



/*

 * Given an ASID, flush the corresponding user ASID.  We can delay this

 * until the next time we switch to it.

 *

 * See SWITCH_TO_USER_CR3.

 */

static inline void invalidate_user_asid(u16 asid)

{

	/* There is no user ASID if address space separation is off */

	if (!IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))

		return;



	/*

	 * We only have a single ASID if PCID is off and the CR3

	 * write will have flushed it.

	 */

	if (!cpu_feature_enabled(X86_FEATURE_PCID))

		return;



	if (!static_cpu_has(X86_FEATURE_PTI))

		return;



	__set_bit(kern_pcid(asid),

		  (unsigned long *)this_cpu_ptr(&cpu_tlbstate.user_pcid_flush_mask));

}



static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush)

{

	unsigned long new_mm_cr3;
@@ -645,7 +672,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f, 
		unsigned long addr = f->start;



		while (addr < f->end) {

			__flush_tlb_one_user(addr);

			flush_tlb_one_user(addr);

			addr += 1UL << f->stride_shift;

		}

		if (local)
@@ -891,6 +918,33 @@ unsigned long __get_current_cr3_fast(void) 
}

EXPORT_SYMBOL_GPL(__get_current_cr3_fast);



/*

 * Flush one page in the user mapping

 */

STATIC_NOPV void native_flush_tlb_one_user(unsigned long addr)

{

	u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);



	asm volatile("invlpg (%0)" ::"r" (addr) : "memory");



	if (!static_cpu_has(X86_FEATURE_PTI))

		return;



	/*

	 * Some platforms #GP if we call invpcid(type=1/2) before CR4.PCIDE=1.

	 * Just use invalidate_user_asid() in case we are called early.

	 */

	if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE))

		invalidate_user_asid(loaded_mm_asid);

	else

		invpcid_flush_one(user_pcid(loaded_mm_asid), addr);

}



void flush_tlb_one_user(unsigned long addr)

{

	__flush_tlb_one_user(addr);

}



/*

 * Flush everything

 */

arch/x86/platform/uv/tlb_uv.c

+1 −1
Original line number Diff line number Diff line
@@ -296,7 +296,7 @@ static void bau_process_message(struct msg_desc *mdp, struct bau_control *bcp, 
		flush_tlb_local();

		stat->d_alltlb++;

	} else {

		__flush_tlb_one_user(msg->address);

		flush_tlb_one_user(msg->address);

		stat->d_onetlb++;

	}

	stat->d_requestee++;

CRA Git | Maintained and supported by SUSTech CRA and CCSE