Commit c2876207 authored by Will Deacon's avatar Will Deacon
Browse files

arm64: Rewrite Spectre-v4 mitigation code 



Rewrite the Spectre-v4 mitigation handling code to follow the same
approach as that taken by Spectre-v2.

For now, report to KVM that the system is vulnerable (by forcing
'ssbd_state' to ARM64_SSBD_UNKNOWN), as this will be cleared up in
subsequent steps.

Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 9e78b659

arch/arm64/include/asm/processor.h

+2 −17
Original line number Diff line number Diff line
@@ -198,25 +198,12 @@ static inline void start_thread_common(struct pt_regs *regs, unsigned long pc) 
		regs->pmr_save = GIC_PRIO_IRQON;

}



static inline void set_ssbs_bit(struct pt_regs *regs)

{

	regs->pstate |= PSR_SSBS_BIT;

}



static inline void set_compat_ssbs_bit(struct pt_regs *regs)

{

	regs->pstate |= PSR_AA32_SSBS_BIT;

}



static inline void start_thread(struct pt_regs *regs, unsigned long pc,

				unsigned long sp)

{

	start_thread_common(regs, pc);

	regs->pstate = PSR_MODE_EL0t;



	if (arm64_get_ssbd_state() != ARM64_SSBD_FORCE_ENABLE)

		set_ssbs_bit(regs);



	spectre_v4_enable_task_mitigation(current);

	regs->sp = sp;

}
@@ -233,9 +220,7 @@ static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc, 
	regs->pstate |= PSR_AA32_E_BIT;

#endif



	if (arm64_get_ssbd_state() != ARM64_SSBD_FORCE_ENABLE)

		set_compat_ssbs_bit(regs);



	spectre_v4_enable_task_mitigation(current);

	regs->compat_sp = sp;

}

#endif

arch/arm64/include/asm/spectre.h

+5 −0
Original line number Diff line number Diff line
@@ -24,4 +24,9 @@ enum mitigation_state arm64_get_spectre_v2_state(void); 
bool has_spectre_v2(const struct arm64_cpu_capabilities *cap, int scope);

void spectre_v2_enable_mitigation(const struct arm64_cpu_capabilities *__unused);



enum mitigation_state arm64_get_spectre_v4_state(void);

bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope);

void spectre_v4_enable_mitigation(const struct arm64_cpu_capabilities *__unused);

void spectre_v4_enable_task_mitigation(struct task_struct *tsk);



#endif	/* __ASM_SPECTRE_H */

arch/arm64/kernel/cpu_errata.c

+4 −213
Original line number Diff line number Diff line
@@ -106,62 +106,7 @@ cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *cap) 
		sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);

}



DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);



int ssbd_state __read_mostly = ARM64_SSBD_KERNEL;

static bool __ssb_safe = true;



static const struct ssbd_options {

	const char	*str;

	int		state;

} ssbd_options[] = {

	{ "force-on",	ARM64_SSBD_FORCE_ENABLE, },

	{ "force-off",	ARM64_SSBD_FORCE_DISABLE, },

	{ "kernel",	ARM64_SSBD_KERNEL, },

};



static int __init ssbd_cfg(char *buf)

{

	int i;



	if (!buf || !buf[0])

		return -EINVAL;



	for (i = 0; i < ARRAY_SIZE(ssbd_options); i++) {

		int len = strlen(ssbd_options[i].str);



		if (strncmp(buf, ssbd_options[i].str, len))

			continue;



		ssbd_state = ssbd_options[i].state;

		return 0;

	}



	return -EINVAL;

}

early_param("ssbd", ssbd_cfg);



void __init arm64_update_smccc_conduit(struct alt_instr *alt,

				       __le32 *origptr, __le32 *updptr,

				       int nr_inst)

{

	u32 insn;



	BUG_ON(nr_inst != 1);



	switch (arm_smccc_1_1_get_conduit()) {

	case SMCCC_CONDUIT_HVC:

		insn = aarch64_insn_get_hvc_value();

		break;

	case SMCCC_CONDUIT_SMC:

		insn = aarch64_insn_get_smc_value();

		break;

	default:

		return;

	}



	*updptr = cpu_to_le32(insn);

}

int ssbd_state __read_mostly = ARM64_SSBD_UNKNOWN;



void __init arm64_enable_wa2_handling(struct alt_instr *alt,

				      __le32 *origptr, __le32 *updptr,
@@ -177,144 +122,6 @@ void __init arm64_enable_wa2_handling(struct alt_instr *alt, 
		*updptr = cpu_to_le32(aarch64_insn_gen_nop());

}



void arm64_set_ssbd_mitigation(bool state)

{

	int conduit;



	if (this_cpu_has_cap(ARM64_SSBS)) {

		if (state)

			asm volatile(SET_PSTATE_SSBS(0));

		else

			asm volatile(SET_PSTATE_SSBS(1));

		return;

	}



	conduit = arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, state,

				       NULL);



	WARN_ON_ONCE(conduit == SMCCC_CONDUIT_NONE);

}



static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,

				    int scope)

{

	struct arm_smccc_res res;

	bool required = true;

	s32 val;

	bool this_cpu_safe = false;

	int conduit;



	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());



	if (cpu_mitigations_off())

		ssbd_state = ARM64_SSBD_FORCE_DISABLE;



	/* delay setting __ssb_safe until we get a firmware response */

	if (is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list))

		this_cpu_safe = true;



	if (this_cpu_has_cap(ARM64_SSBS)) {

		if (!this_cpu_safe)

			__ssb_safe = false;

		required = false;

		goto out_printmsg;

	}



	conduit = arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,

				       ARM_SMCCC_ARCH_WORKAROUND_2, &res);



	if (conduit == SMCCC_CONDUIT_NONE) {

		ssbd_state = ARM64_SSBD_UNKNOWN;

		if (!this_cpu_safe)

			__ssb_safe = false;

		return false;

	}



	val = (s32)res.a0;



	switch (val) {

	case SMCCC_RET_NOT_SUPPORTED:

		ssbd_state = ARM64_SSBD_UNKNOWN;

		if (!this_cpu_safe)

			__ssb_safe = false;

		return false;



	/* machines with mixed mitigation requirements must not return this */

	case SMCCC_RET_NOT_REQUIRED:

		pr_info_once("%s mitigation not required\n", entry->desc);

		ssbd_state = ARM64_SSBD_MITIGATED;

		return false;



	case SMCCC_RET_SUCCESS:

		__ssb_safe = false;

		required = true;

		break;



	case 1:	/* Mitigation not required on this CPU */

		required = false;

		break;



	default:

		WARN_ON(1);

		if (!this_cpu_safe)

			__ssb_safe = false;

		return false;

	}



	switch (ssbd_state) {

	case ARM64_SSBD_FORCE_DISABLE:

		arm64_set_ssbd_mitigation(false);

		required = false;

		break;



	case ARM64_SSBD_KERNEL:

		if (required) {

			__this_cpu_write(arm64_ssbd_callback_required, 1);

			arm64_set_ssbd_mitigation(true);

		}

		break;



	case ARM64_SSBD_FORCE_ENABLE:

		arm64_set_ssbd_mitigation(true);

		required = true;

		break;



	default:

		WARN_ON(1);

		break;

	}



out_printmsg:

	switch (ssbd_state) {

	case ARM64_SSBD_FORCE_DISABLE:

		pr_info_once("%s disabled from command-line\n", entry->desc);

		break;



	case ARM64_SSBD_FORCE_ENABLE:

		pr_info_once("%s forced from command-line\n", entry->desc);

		break;

	}



	return required;

}



static void cpu_enable_ssbd_mitigation(const struct arm64_cpu_capabilities *cap)

{

	if (ssbd_state != ARM64_SSBD_FORCE_DISABLE)

		cap->matches(cap, SCOPE_LOCAL_CPU);

}



/* known invulnerable cores */

static const struct midr_range arm64_ssb_cpus[] = {

	MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),

	MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),

	MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),

	MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),

	MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),

	MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),

	{},

};



#ifdef CONFIG_ARM64_ERRATUM_1463225

DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);
@@ -674,12 +481,11 @@ const struct arm64_cpu_capabilities arm64_errata[] = { 
	},

#endif

	{

		.desc = "Speculative Store Bypass Disable",

		.desc = "Spectre-v4",

		.capability = ARM64_SPECTRE_V4,

		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,

		.matches = has_ssbd_mitigation,

		.cpu_enable = cpu_enable_ssbd_mitigation,

		.midr_range_list = arm64_ssb_cpus,

		.matches = has_spectre_v4,

		.cpu_enable = spectre_v4_enable_mitigation,

	},

#ifdef CONFIG_ARM64_ERRATUM_1418040

	{
@@ -732,18 +538,3 @@ const struct arm64_cpu_capabilities arm64_errata[] = { 
	{

	}

};
 


ssize_t cpu_show_spec_store_bypass(struct device *dev,

		struct device_attribute *attr, char *buf)

{

	if (__ssb_safe)

		return sprintf(buf, "Not affected\n");



	switch (ssbd_state) {

	case ARM64_SSBD_KERNEL:

	case ARM64_SSBD_FORCE_ENABLE:

		return sprintf(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n");

	}



	return sprintf(buf, "Vulnerable\n");

}

arch/arm64/kernel/cpufeature.c

+0 −41
Original line number Diff line number Diff line
@@ -1583,46 +1583,6 @@ static void cpu_has_fwb(const struct arm64_cpu_capabilities *__unused) 
	WARN_ON(val & (7 << 27 | 7 << 21));

}



static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)

{

	if (user_mode(regs))

		return 1;



	if (instr & BIT(PSTATE_Imm_shift))

		regs->pstate |= PSR_SSBS_BIT;

	else

		regs->pstate &= ~PSR_SSBS_BIT;



	arm64_skip_faulting_instruction(regs, 4);

	return 0;

}



static struct undef_hook ssbs_emulation_hook = {

	.instr_mask	= ~(1U << PSTATE_Imm_shift),

	.instr_val	= 0xd500401f | PSTATE_SSBS,

	.fn		= ssbs_emulation_handler,

};



static void cpu_enable_ssbs(const struct arm64_cpu_capabilities *__unused)

{

	static bool undef_hook_registered = false;

	static DEFINE_RAW_SPINLOCK(hook_lock);



	raw_spin_lock(&hook_lock);

	if (!undef_hook_registered) {

		register_undef_hook(&ssbs_emulation_hook);

		undef_hook_registered = true;

	}

	raw_spin_unlock(&hook_lock);



	if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) {

		sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);

		arm64_set_ssbd_mitigation(false);

	} else {

		arm64_set_ssbd_mitigation(true);

	}

}



#ifdef CONFIG_ARM64_PAN

static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused)

{
@@ -1983,7 +1943,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { 
		.field_pos = ID_AA64PFR1_SSBS_SHIFT,

		.sign = FTR_UNSIGNED,

		.min_field_value = ID_AA64PFR1_SSBS_PSTATE_ONLY,

		.cpu_enable = cpu_enable_ssbs,

	},

#ifdef CONFIG_ARM64_CNP

	{

arch/arm64/kernel/entry.S

+3 −3
Original line number Diff line number Diff line
@@ -132,8 +132,8 @@ alternative_else_nop_endif 
	 * them if required.

	 */

	.macro	apply_ssbd, state, tmp1, tmp2

alternative_cb	arm64_enable_wa2_handling

	b	.L__asm_ssbd_skip\@

alternative_cb	spectre_v4_patch_fw_mitigation_enable

	b	.L__asm_ssbd_skip\@		// Patched to NOP

alternative_cb_end

	ldr_this_cpu	\tmp2, arm64_ssbd_callback_required, \tmp1

	cbz	\tmp2,	.L__asm_ssbd_skip\@
@@ -141,7 +141,7 @@ alternative_cb_end 
	tbnz	\tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@

	mov	w0, #ARM_SMCCC_ARCH_WORKAROUND_2

	mov	w1, #\state

alternative_cb	arm64_update_smccc_conduit

alternative_cb	spectre_v4_patch_fw_mitigation_conduit

	nop					// Patched to SMC/HVC #0

alternative_cb_end

.L__asm_ssbd_skip\@:

CRA Git | Maintained and supported by SUSTech CRA and CCSE