Commit 58430c5d authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Borislav Petkov
Browse files

x86/tlb: Move __flush_tlb_one_kernel() out of line 



cpu_tlbstate is exported because various TLB-related functions need
access to it, but cpu_tlbstate is sensitive information which should
only be accessed by well-contained kernel functions and not be directly
exposed to modules.

As a fourth step, move __flush_tlb_one_kernel() out of line and hide
the native function. The latter can be static when CONFIG_PARAVIRT is
disabled.

Consolidate the name space while at it and remove the pointless extra
wrapper in the paravirt code.

No functional change.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Reviewed-by: default avatarAlexandre Chartre <alexandre.chartre@oracle.com>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20200421092559.535159540@linutronix.de
parent 127ac915

arch/x86/include/asm/pgtable_32.h

+1 −1
Original line number Diff line number Diff line
@@ -60,7 +60,7 @@ void sync_initial_page_table(void); 
#define kpte_clear_flush(ptep, vaddr)		\

do {						\

	pte_clear(&init_mm, (vaddr), (ptep));	\

	__flush_tlb_one_kernel((vaddr));		\

	flush_tlb_one_kernel((vaddr));		\

} while (0)



#endif /* !__ASSEMBLY__ */

arch/x86/include/asm/tlbflush.h

+1 −40
Original line number Diff line number Diff line
@@ -143,6 +143,7 @@ static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid) 
void flush_tlb_local(void);

void flush_tlb_global(void);

void flush_tlb_one_user(unsigned long addr);

void flush_tlb_one_kernel(unsigned long addr);



#ifdef CONFIG_PARAVIRT

#include <asm/paravirt.h>
@@ -317,14 +318,6 @@ static inline void cr4_clear_bits(unsigned long mask) 
	local_irq_restore(flags);

}



/*

 * Mark all other ASIDs as invalid, preserves the current.

 */

static inline void invalidate_other_asid(void)

{

	this_cpu_write(cpu_tlbstate.invalidate_other, true);

}



/*

 * Save some of cr4 feature set we're using (e.g.  Pentium 4MB

 * enable and PPro Global page enable), so that any CPU's that boot
@@ -365,38 +358,6 @@ static inline void __flush_tlb_all(void) 
	}

}



/*

 * flush one page in the kernel mapping

 */

static inline void __flush_tlb_one_kernel(unsigned long addr)

{

	count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);



	/*

	 * If PTI is off, then __flush_tlb_one_user() is just INVLPG or its

	 * paravirt equivalent.  Even with PCID, this is sufficient: we only

	 * use PCID if we also use global PTEs for the kernel mapping, and

	 * INVLPG flushes global translations across all address spaces.

	 *

	 * If PTI is on, then the kernel is mapped with non-global PTEs, and

	 * __flush_tlb_one_user() will flush the given address for the current

	 * kernel address space and for its usermode counterpart, but it does

	 * not flush it for other address spaces.

	 */

	flush_tlb_one_user(addr);



	if (!static_cpu_has(X86_FEATURE_PTI))

		return;



	/*

	 * See above.  We need to propagate the flush to all other address

	 * spaces.  In principle, we only need to propagate it to kernelmode

	 * address spaces, but the extra bookkeeping we would need is not

	 * worth it.

	 */

	invalidate_other_asid();

}



#define TLB_FLUSH_ALL	-1UL



/*

arch/x86/mm/init_64.c

+1 −1
Original line number Diff line number Diff line
@@ -298,7 +298,7 @@ static void __set_pte_vaddr(pud_t *pud, unsigned long vaddr, pte_t new_pte) 
	 * It's enough to flush this one mapping.

	 * (PGE mappings get flushed as well)

	 */

	__flush_tlb_one_kernel(vaddr);

	flush_tlb_one_kernel(vaddr);

}



void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte)

arch/x86/mm/ioremap.c

+1 −1
Original line number Diff line number Diff line
@@ -885,5 +885,5 @@ void __init __early_set_fixmap(enum fixed_addresses idx, 
		set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));

	else

		pte_clear(&init_mm, addr, pte);

	__flush_tlb_one_kernel(addr);

	flush_tlb_one_kernel(addr);

}

arch/x86/mm/kmmio.c

+1 −1
Original line number Diff line number Diff line
@@ -173,7 +173,7 @@ static int clear_page_presence(struct kmmio_fault_page *f, bool clear) 
		return -1;

	}



	__flush_tlb_one_kernel(f->addr);

	flush_tlb_one_kernel(f->addr);

	return 0;

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE