Commit f2266504 authored by Marc Zyngier's avatar Marc Zyngier Committed by Catalin Marinas
Browse files

arm64: Relax ICC_PMR_EL1 accesses when ICC_CTLR_EL1.PMHE is clear 



The GICv3 architecture specification is incredibly misleading when it
comes to PMR and the requirement for a DSB. It turns out that this DSB
is only required if the CPU interface sends an Upstream Control
message to the redistributor in order to update the RD's view of PMR.

This message is only sent when ICC_CTLR_EL1.PMHE is set, which isn't
the case in Linux. It can still be set from EL3, so some special care
is required. But the upshot is that in the (hopefuly large) majority
of the cases, we can drop the DSB altogether.

This relies on a new static key being set if the boot CPU has PMHE
set. The drawback is that this static key has to be exported to
modules.

Cc: Will Deacon <will@kernel.org>
Cc: James Morse <james.morse@arm.com>
Cc: Julien Thierry <julien.thierry.kdev@gmail.com>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 4f5cafb5

arch/arm64/include/asm/barrier.h

+12 −0
Original line number Diff line number Diff line
@@ -29,6 +29,18 @@ 
						 SB_BARRIER_INSN"nop\n",	\

						 ARM64_HAS_SB))



#ifdef CONFIG_ARM64_PSEUDO_NMI

#define pmr_sync()						\

	do {							\

		extern struct static_key_false gic_pmr_sync;	\

								\

		if (static_branch_unlikely(&gic_pmr_sync))	\

			dsb(sy);				\

	} while(0)

#else

#define pmr_sync()	do {} while (0)

#endif



#define mb()		dsb(sy)

#define rmb()		dsb(ld)

#define wmb()		dsb(st)

arch/arm64/include/asm/daifflags.h

+2 −1
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ 
#include <linux/irqflags.h>



#include <asm/arch_gicv3.h>

#include <asm/barrier.h>

#include <asm/cpufeature.h>



#define DAIF_PROCCTX		0
@@ -65,7 +66,7 @@ static inline void local_daif_restore(unsigned long flags) 


		if (system_uses_irq_prio_masking()) {

			gic_write_pmr(GIC_PRIO_IRQON);

			dsb(sy);

			pmr_sync();

		}

	} else if (system_uses_irq_prio_masking()) {

		u64 pmr;

arch/arm64/include/asm/irqflags.h

+10 −9
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 
#define __ASM_IRQFLAGS_H



#include <asm/alternative.h>

#include <asm/barrier.h>

#include <asm/ptrace.h>

#include <asm/sysreg.h>
@@ -34,14 +35,14 @@ static inline void arch_local_irq_enable(void) 
	}



	asm volatile(ALTERNATIVE(

		"msr	daifclr, #2		// arch_local_irq_enable\n"

		"nop",

		__msr_s(SYS_ICC_PMR_EL1, "%0")

		"dsb	sy",

		"msr	daifclr, #2		// arch_local_irq_enable",

		__msr_s(SYS_ICC_PMR_EL1, "%0"),

		ARM64_HAS_IRQ_PRIO_MASKING)

		:

		: "r" ((unsigned long) GIC_PRIO_IRQON)

		: "memory");



	pmr_sync();

}



static inline void arch_local_irq_disable(void)
@@ -116,14 +117,14 @@ static inline unsigned long arch_local_irq_save(void) 
static inline void arch_local_irq_restore(unsigned long flags)

{

	asm volatile(ALTERNATIVE(

			"msr	daif, %0\n"

			"nop",

			__msr_s(SYS_ICC_PMR_EL1, "%0")

			"dsb	sy",

		"msr	daif, %0",

		__msr_s(SYS_ICC_PMR_EL1, "%0"),

		ARM64_HAS_IRQ_PRIO_MASKING)

		:

		: "r" (flags)

		: "memory");



	pmr_sync();

}



#endif /* __ASM_IRQFLAGS_H */

arch/arm64/include/asm/kvm_host.h

+1 −2
Original line number Diff line number Diff line
@@ -600,8 +600,7 @@ static inline void kvm_arm_vhe_guest_enter(void) 
	 * local_daif_mask() already sets GIC_PRIO_PSR_I_SET, we just need a

	 * dsb to ensure the redistributor is forwards EL2 IRQs to the CPU.

	 */

	if (system_uses_irq_prio_masking())

		dsb(sy);

	pmr_sync();

}



static inline void kvm_arm_vhe_guest_exit(void)

arch/arm64/kernel/entry.S

+4 −2
Original line number Diff line number Diff line
@@ -269,8 +269,10 @@ alternative_else_nop_endif 
alternative_if ARM64_HAS_IRQ_PRIO_MASKING

	ldr	x20, [sp, #S_PMR_SAVE]

	msr_s	SYS_ICC_PMR_EL1, x20

	/* Ensure priority change is seen by redistributor */

	dsb	sy

	mrs_s	x21, SYS_ICC_CTLR_EL1

	tbz	x21, #6, .L__skip_pmr_sync\@	// Check for ICC_CTLR_EL1.PMHE

	dsb	sy				// Ensure priority change is seen by redistributor

.L__skip_pmr_sync\@:

alternative_else_nop_endif



	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR

CRA Git | Maintained and supported by SUSTech CRA and CCSE