Commit 6a036afb authored by Catalin Marinas's avatar Catalin Marinas
Browse files

Merge branch 'for-next/neoverse-n1-stale-instr' into for-next/core 

Neoverse-N1 cores with the 'COHERENT_ICACHE' feature may fetch stale
instructions when software depends on prefetch-speculation-protection
instead of explicit synchronization. [0]

The workaround is to trap I-Cache maintenance and issue an
inner-shareable TLBI. The affected cores have a Coherent I-Cache, so the
I-Cache maintenance isn't necessary. The core tells user-space it can
skip it with CTR_EL0.DIC. We also have to trap this register to hide the
bit forcing DIC-aware user-space to perform the maintenance.

To avoid trapping all cache-maintenance, this workaround depends on
a firmware component that only traps I-cache maintenance from EL0 and
performs the workaround.

For user-space, the kernel's work is to trap CTR_EL0 to hide DIC, and
produce a fake IminLine. EL3 traps the now-necessary I-Cache maintenance
and performs the inner-shareable-TLBI that makes everything better.

[0] https://developer.arm.com/docs/sden885747/latest/arm-neoverse-n1-mp050-software-developer-errata-notice

* for-next/neoverse-n1-stale-instr:
  arm64: Silence clang warning on mismatched value/register sizes
  arm64: compat: Workaround Neoverse-N1 #1542419 for compat user-space
  arm64: Fake the IminLine size on systems affected by Neoverse-N1 #1542419
  arm64: errata: Hide CTR_EL0.DIC on systems affected by Neoverse-N1 #1542419
parents ba95e9bd 27a22fbd

Documentation/arm64/silicon-errata.rst

+2 −0
Original line number Diff line number Diff line
@@ -88,6 +88,8 @@ stable kernels. 
+----------------+-----------------+-----------------+-----------------------------+

| ARM            | Neoverse-N1     | #1349291        | N/A                         |

+----------------+-----------------+-----------------+-----------------------------+

| ARM            | Neoverse-N1     | #1542419        | ARM64_ERRATUM_1542419       |

+----------------+-----------------+-----------------+-----------------------------+

| ARM            | MMU-500         | #841119,826419  | N/A                         |

+----------------+-----------------+-----------------+-----------------------------+

+----------------+-----------------+-----------------+-----------------------------+

arch/arm64/Kconfig

+16 −0
Original line number Diff line number Diff line
@@ -558,6 +558,22 @@ config ARM64_ERRATUM_1463225 


	  If unsure, say Y.



config ARM64_ERRATUM_1542419

	bool "Neoverse-N1: workaround mis-ordering of instruction fetches"

	default y

	help

	  This option adds a workaround for ARM Neoverse-N1 erratum

	  1542419.



	  Affected Neoverse-N1 cores could execute a stale instruction when

	  modified by another CPU. The workaround depends on a firmware

	  counterpart.



	  Workaround the issue by hiding the DIC feature from EL0. This

	  forces user-space to perform cache maintenance.



	  If unsure, say Y.



config CAVIUM_ERRATUM_22375

	bool "Cavium erratum 22375, 24313"

	default y

arch/arm64/include/asm/cache.h

+2 −1
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ 
#define CTR_L1IP_MASK		3

#define CTR_DMINLINE_SHIFT	16

#define CTR_IMINLINE_SHIFT	0

#define CTR_IMINLINE_MASK	0xf

#define CTR_ERG_SHIFT		20

#define CTR_CWG_SHIFT		24

#define CTR_CWG_MASK		15
@@ -18,7 +19,7 @@ 
#define CTR_DIC_SHIFT		29



#define CTR_CACHE_MINLINE_MASK	\

	(0xf << CTR_DMINLINE_SHIFT | 0xf << CTR_IMINLINE_SHIFT)

	(0xf << CTR_DMINLINE_SHIFT | CTR_IMINLINE_MASK << CTR_IMINLINE_SHIFT)



#define CTR_L1IP(ctr)		(((ctr) >> CTR_L1IP_SHIFT) & CTR_L1IP_MASK)

arch/arm64/include/asm/cpucaps.h

+2 −1
Original line number Diff line number Diff line
@@ -54,7 +54,8 @@ 
#define ARM64_WORKAROUND_1463225		44

#define ARM64_WORKAROUND_CAVIUM_TX2_219_TVM	45

#define ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM	46

#define ARM64_WORKAROUND_1542419		47



#define ARM64_NCAPS				47

#define ARM64_NCAPS				48



#endif /* __ASM_CPUCAPS_H */

arch/arm64/kernel/cpu_errata.c

+31 −1
Original line number Diff line number Diff line
@@ -88,13 +88,21 @@ has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry, 
}



static void

cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *__unused)

cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *cap)

{

	u64 mask = arm64_ftr_reg_ctrel0.strict_mask;

	bool enable_uct_trap = false;



	/* Trap CTR_EL0 access on this CPU, only if it has a mismatch */

	if ((read_cpuid_cachetype() & mask) !=

	    (arm64_ftr_reg_ctrel0.sys_val & mask))

		enable_uct_trap = true;



	/* ... or if the system is affected by an erratum */

	if (cap->capability == ARM64_WORKAROUND_1542419)

		enable_uct_trap = true;



	if (enable_uct_trap)

		sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);

}
@@ -648,6 +656,18 @@ needs_tx2_tvm_workaround(const struct arm64_cpu_capabilities *entry, 
	return false;

}



static bool __maybe_unused

has_neoverse_n1_erratum_1542419(const struct arm64_cpu_capabilities *entry,

				int scope)

{

	u32 midr = read_cpuid_id();

	bool has_dic = read_cpuid_cachetype() & BIT(CTR_DIC_SHIFT);

	const struct midr_range range = MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1);



	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());

	return is_midr_in_range(midr, &range) && has_dic;

}



#ifdef CONFIG_HARDEN_EL2_VECTORS



static const struct midr_range arm64_harden_el2_vectors[] = {
@@ -889,6 +909,16 @@ const struct arm64_cpu_capabilities arm64_errata[] = { 
		.capability = ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM,

		ERRATA_MIDR_RANGE_LIST(tx2_family_cpus),

	},

#endif

#ifdef CONFIG_ARM64_ERRATUM_1542419

	{

		/* we depend on the firmware portion for correctness */

		.desc = "ARM erratum 1542419 (kernel portion)",

		.capability = ARM64_WORKAROUND_1542419,

		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,

		.matches = has_neoverse_n1_erratum_1542419,

		.cpu_enable = cpu_enable_trap_ctr_access,

	},

#endif

	{

	}

CRA Git | Maintained and supported by SUSTech CRA and CCSE