Commit a048a07d authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Linus Torvalds
Browse files

powerpc/64s: Add support for a store forwarding barrier at kernel entry/exit 



On some CPUs we can prevent a vulnerability related to store-to-load
forwarding by preventing store forwarding between privilege domains,
by inserting a barrier in kernel entry and exit paths.

This is known to be the case on at least Power7, Power8 and Power9
powerpc CPUs.

Barriers must be inserted generally before the first load after moving
to a higher privilege, and after the last store before moving to a
lower privilege, HV and PR privilege transitions must be protected.

Barriers are added as patch sections, with all kernel/hypervisor entry
points patched, and the exit points to lower privilge levels patched
similarly to the RFI flush patching.

Firmware advertisement is not implemented yet, so CPU flush types
are hard coded.

Thanks to Michal Suchánek for bug fixes and review.

Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMauricio Faria de Oliveira <mauricfo@linux.vnet.ibm.com>
Signed-off-by: default avatarMichael Neuling <mikey@neuling.org>
Signed-off-by: default avatarMichal Suchánek <msuchanek@suse.de>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent c85061e6

arch/powerpc/include/asm/exception-64s.h

+29 −0
Original line number Original line Diff line number Diff line
@@ -74,6 +74,27 @@ 
 */

 */

#define EX_R3		EX_DAR

#define EX_R3		EX_DAR





#define STF_ENTRY_BARRIER_SLOT						\

	STF_ENTRY_BARRIER_FIXUP_SECTION;				\

	nop;								\

	nop;								\

	nop



#define STF_EXIT_BARRIER_SLOT						\

	STF_EXIT_BARRIER_FIXUP_SECTION;					\

	nop;								\

	nop;								\

	nop;								\

	nop;								\

	nop;								\

	nop



/*

 * r10 must be free to use, r13 must be paca

 */

#define INTERRUPT_TO_KERNEL						\

	STF_ENTRY_BARRIER_SLOT



/*

/*

 * Macros for annotating the expected destination of (h)rfid

 * Macros for annotating the expected destination of (h)rfid

 *

 *
@@ -90,16 +111,19 @@ 
	rfid

	rfid





#define RFI_TO_USER							\

#define RFI_TO_USER							\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	RFI_FLUSH_SLOT;							\

	rfid;								\

	rfid;								\

	b	rfi_flush_fallback

	b	rfi_flush_fallback





#define RFI_TO_USER_OR_KERNEL						\

#define RFI_TO_USER_OR_KERNEL						\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	RFI_FLUSH_SLOT;							\

	rfid;								\

	rfid;								\

	b	rfi_flush_fallback

	b	rfi_flush_fallback





#define RFI_TO_GUEST							\

#define RFI_TO_GUEST							\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	RFI_FLUSH_SLOT;							\

	rfid;								\

	rfid;								\

	b	rfi_flush_fallback

	b	rfi_flush_fallback
@@ -108,21 +132,25 @@ 
	hrfid

	hrfid





#define HRFI_TO_USER							\

#define HRFI_TO_USER							\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	RFI_FLUSH_SLOT;							\

	hrfid;								\

	hrfid;								\

	b	hrfi_flush_fallback

	b	hrfi_flush_fallback





#define HRFI_TO_USER_OR_KERNEL						\

#define HRFI_TO_USER_OR_KERNEL						\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	RFI_FLUSH_SLOT;							\

	hrfid;								\

	hrfid;								\

	b	hrfi_flush_fallback

	b	hrfi_flush_fallback





#define HRFI_TO_GUEST							\

#define HRFI_TO_GUEST							\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	RFI_FLUSH_SLOT;							\

	hrfid;								\

	hrfid;								\

	b	hrfi_flush_fallback

	b	hrfi_flush_fallback





#define HRFI_TO_UNKNOWN							\

#define HRFI_TO_UNKNOWN							\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	RFI_FLUSH_SLOT;							\

	hrfid;								\

	hrfid;								\

	b	hrfi_flush_fallback

	b	hrfi_flush_fallback
@@ -254,6 +282,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) 
#define __EXCEPTION_PROLOG_1_PRE(area)					\

#define __EXCEPTION_PROLOG_1_PRE(area)					\

	OPT_SAVE_REG_TO_PACA(area+EX_PPR, r9, CPU_FTR_HAS_PPR);		\

	OPT_SAVE_REG_TO_PACA(area+EX_PPR, r9, CPU_FTR_HAS_PPR);		\

	OPT_SAVE_REG_TO_PACA(area+EX_CFAR, r10, CPU_FTR_CFAR);		\

	OPT_SAVE_REG_TO_PACA(area+EX_CFAR, r10, CPU_FTR_CFAR);		\

	INTERRUPT_TO_KERNEL;						\

	SAVE_CTR(r10, area);						\

	SAVE_CTR(r10, area);						\

	mfcr	r9;

	mfcr	r9;

arch/powerpc/include/asm/feature-fixups.h

+19 −0
Original line number Original line Diff line number Diff line
@@ -187,6 +187,22 @@ label##3: \ 
	FTR_ENTRY_OFFSET label##1b-label##3b;		\

	FTR_ENTRY_OFFSET label##1b-label##3b;		\

	.popsection;

	.popsection;





#define STF_ENTRY_BARRIER_FIXUP_SECTION			\

953:							\

	.pushsection __stf_entry_barrier_fixup,"a";	\

	.align 2;					\

954:							\

	FTR_ENTRY_OFFSET 953b-954b;			\

	.popsection;



#define STF_EXIT_BARRIER_FIXUP_SECTION			\

955:							\

	.pushsection __stf_exit_barrier_fixup,"a";	\

	.align 2;					\

956:							\

	FTR_ENTRY_OFFSET 955b-956b;			\

	.popsection;



#define RFI_FLUSH_FIXUP_SECTION				\

#define RFI_FLUSH_FIXUP_SECTION				\

951:							\

951:							\

	.pushsection __rfi_flush_fixup,"a";		\

	.pushsection __rfi_flush_fixup,"a";		\
@@ -199,6 +215,9 @@ label##3: \ 
#ifndef __ASSEMBLY__

#ifndef __ASSEMBLY__

#include <linux/types.h>

#include <linux/types.h>





extern long stf_barrier_fallback;

extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup;

extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup;

extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup;

extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup;





void apply_feature_fixups(void);

void apply_feature_fixups(void);

arch/powerpc/include/asm/security_features.h

+11 −0
Original line number Original line Diff line number Diff line
@@ -12,6 +12,17 @@ 
extern unsigned long powerpc_security_features;

extern unsigned long powerpc_security_features;

extern bool rfi_flush;

extern bool rfi_flush;





/* These are bit flags */

enum stf_barrier_type {

	STF_BARRIER_NONE	= 0x1,

	STF_BARRIER_FALLBACK	= 0x2,

	STF_BARRIER_EIEIO	= 0x4,

	STF_BARRIER_SYNC_ORI	= 0x8,

};



void setup_stf_barrier(void);

void do_stf_barrier_fixups(enum stf_barrier_type types);



static inline void security_ftr_set(unsigned long feature)

static inline void security_ftr_set(unsigned long feature)

{

{

	powerpc_security_features |= feature;

	powerpc_security_features |= feature;

arch/powerpc/kernel/exceptions-64s.S

+17 −2
Original line number Original line Diff line number Diff line
@@ -885,7 +885,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM) 
#endif

#endif









EXC_REAL_MASKABLE(decrementer, 0x900, 0x80, IRQS_DISABLED)

EXC_REAL_OOL_MASKABLE(decrementer, 0x900, 0x80, IRQS_DISABLED)

EXC_VIRT_MASKABLE(decrementer, 0x4900, 0x80, 0x900, IRQS_DISABLED)

EXC_VIRT_MASKABLE(decrementer, 0x4900, 0x80, 0x900, IRQS_DISABLED)

TRAMP_KVM(PACA_EXGEN, 0x900)

TRAMP_KVM(PACA_EXGEN, 0x900)

EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt)

EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt)
@@ -961,6 +961,7 @@ EXC_COMMON(trap_0b_common, 0xb00, unknown_exception) 
	mtctr	r13;							\

	mtctr	r13;							\

	GET_PACA(r13);							\

	GET_PACA(r13);							\

	std	r10,PACA_EXGEN+EX_R10(r13);				\

	std	r10,PACA_EXGEN+EX_R10(r13);				\

	INTERRUPT_TO_KERNEL;						\

	KVMTEST_PR(0xc00); /* uses r10, branch to do_kvm_0xc00_system_call */ \

	KVMTEST_PR(0xc00); /* uses r10, branch to do_kvm_0xc00_system_call */ \

	HMT_MEDIUM;							\

	HMT_MEDIUM;							\

	mfctr	r9;

	mfctr	r9;
@@ -969,7 +970,8 @@ EXC_COMMON(trap_0b_common, 0xb00, unknown_exception) 
#define SYSCALL_KVMTEST							\

#define SYSCALL_KVMTEST							\

	HMT_MEDIUM;							\

	HMT_MEDIUM;							\

	mr	r9,r13;							\

	mr	r9,r13;							\

	GET_PACA(r13);

	GET_PACA(r13);							\

	INTERRUPT_TO_KERNEL;

#endif

#endif

	

	

#define LOAD_SYSCALL_HANDLER(reg)					\

#define LOAD_SYSCALL_HANDLER(reg)					\
@@ -1507,6 +1509,19 @@ masked_##_H##interrupt: \ 
	b	.;					\

	b	.;					\

	MASKED_DEC_HANDLER(_H)

	MASKED_DEC_HANDLER(_H)





TRAMP_REAL_BEGIN(stf_barrier_fallback)

	std	r9,PACA_EXRFI+EX_R9(r13)

	std	r10,PACA_EXRFI+EX_R10(r13)

	sync

	ld	r9,PACA_EXRFI+EX_R9(r13)

	ld	r10,PACA_EXRFI+EX_R10(r13)

	ori	31,31,0

	.rept 14

	b	1f

1:

	.endr

	blr



TRAMP_REAL_BEGIN(rfi_flush_fallback)

TRAMP_REAL_BEGIN(rfi_flush_fallback)

	SET_SCRATCH0(r13);

	SET_SCRATCH0(r13);

	GET_PACA(r13);

	GET_PACA(r13);

arch/powerpc/kernel/security.c

+149 −0
Original line number Original line Diff line number Diff line
@@ -8,6 +8,7 @@ 
#include <linux/device.h>

#include <linux/device.h>

#include <linux/seq_buf.h>

#include <linux/seq_buf.h>





#include <asm/debugfs.h>

#include <asm/security_features.h>

#include <asm/security_features.h>
@@ -86,3 +87,151 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, c 




	return s.len;

	return s.len;

}

}



/*

 * Store-forwarding barrier support.

 */



static enum stf_barrier_type stf_enabled_flush_types;

static bool no_stf_barrier;

bool stf_barrier;



static int __init handle_no_stf_barrier(char *p)

{

	pr_info("stf-barrier: disabled on command line.");

	no_stf_barrier = true;

	return 0;

}



early_param("no_stf_barrier", handle_no_stf_barrier);



/* This is the generic flag used by other architectures */

static int __init handle_ssbd(char *p)

{

	if (!p || strncmp(p, "auto", 5) == 0 || strncmp(p, "on", 2) == 0 ) {

		/* Until firmware tells us, we have the barrier with auto */

		return 0;

	} else if (strncmp(p, "off", 3) == 0) {

		handle_no_stf_barrier(NULL);

		return 0;

	} else

		return 1;



	return 0;

}

early_param("spec_store_bypass_disable", handle_ssbd);



/* This is the generic flag used by other architectures */

static int __init handle_no_ssbd(char *p)

{

	handle_no_stf_barrier(NULL);

	return 0;

}

early_param("nospec_store_bypass_disable", handle_no_ssbd);



static void stf_barrier_enable(bool enable)

{

	if (enable)

		do_stf_barrier_fixups(stf_enabled_flush_types);

	else

		do_stf_barrier_fixups(STF_BARRIER_NONE);



	stf_barrier = enable;

}



void setup_stf_barrier(void)

{

	enum stf_barrier_type type;

	bool enable, hv;



	hv = cpu_has_feature(CPU_FTR_HVMODE);



	/* Default to fallback in case fw-features are not available */

	if (cpu_has_feature(CPU_FTR_ARCH_300))

		type = STF_BARRIER_EIEIO;

	else if (cpu_has_feature(CPU_FTR_ARCH_207S))

		type = STF_BARRIER_SYNC_ORI;

	else if (cpu_has_feature(CPU_FTR_ARCH_206))

		type = STF_BARRIER_FALLBACK;

	else

		type = STF_BARRIER_NONE;



	enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&

		(security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR) ||

		 (security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV) && hv));



	if (type == STF_BARRIER_FALLBACK) {

		pr_info("stf-barrier: fallback barrier available\n");

	} else if (type == STF_BARRIER_SYNC_ORI) {

		pr_info("stf-barrier: hwsync barrier available\n");

	} else if (type == STF_BARRIER_EIEIO) {

		pr_info("stf-barrier: eieio barrier available\n");

	}



	stf_enabled_flush_types = type;



	if (!no_stf_barrier)

		stf_barrier_enable(enable);

}



ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)

{

	if (stf_barrier && stf_enabled_flush_types != STF_BARRIER_NONE) {

		const char *type;

		switch (stf_enabled_flush_types) {

		case STF_BARRIER_EIEIO:

			type = "eieio";

			break;

		case STF_BARRIER_SYNC_ORI:

			type = "hwsync";

			break;

		case STF_BARRIER_FALLBACK:

			type = "fallback";

			break;

		default:

			type = "unknown";

		}

		return sprintf(buf, "Mitigation: Kernel entry/exit barrier (%s)\n", type);

	}



	if (!security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV) &&

	    !security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR))

		return sprintf(buf, "Not affected\n");



	return sprintf(buf, "Vulnerable\n");

}



#ifdef CONFIG_DEBUG_FS

static int stf_barrier_set(void *data, u64 val)

{

	bool enable;



	if (val == 1)

		enable = true;

	else if (val == 0)

		enable = false;

	else

		return -EINVAL;



	/* Only do anything if we're changing state */

	if (enable != stf_barrier)

		stf_barrier_enable(enable);



	return 0;

}



static int stf_barrier_get(void *data, u64 *val)

{

	*val = stf_barrier ? 1 : 0;

	return 0;

}



DEFINE_SIMPLE_ATTRIBUTE(fops_stf_barrier, stf_barrier_get, stf_barrier_set, "%llu\n");



static __init int stf_barrier_debugfs_init(void)

{

	debugfs_create_file("stf_barrier", 0600, powerpc_debugfs_root, NULL, &fops_stf_barrier);

	return 0;

}

device_initcall(stf_barrier_debugfs_init);

#endif /* CONFIG_DEBUG_FS */

CRA Git | Maintained and supported by SUSTech CRA and CCSE