Commit e03fa291 authored by David Brazdil's avatar David Brazdil Committed by Marc Zyngier
Browse files

KVM: arm64: Duplicate hyp/tlb.c for VHE/nVHE 



tlb.c contains code for flushing the TLB, with code shared between VHE/nVHE.
Because common code is small, duplicate tlb.c and specialize each copy for
VHE/nVHE.

Signed-off-by: default avatarDavid Brazdil <dbrazdil@google.com>
Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20200625131420.71444-9-dbrazdil@google.com
parent 208243c7

arch/arm64/kernel/image-vars.h

+8 −6
Original line number Diff line number Diff line
@@ -81,12 +81,6 @@ KVM_NVHE_ALIAS(__kvm_enable_ssbs); 
/* Symbols defined in timer-sr.c (not yet compiled with nVHE build rules). */

KVM_NVHE_ALIAS(__kvm_timer_set_cntvoff);



/* Symbols defined in tlb.c (not yet compiled with nVHE build rules). */

KVM_NVHE_ALIAS(__kvm_flush_vm_context);

KVM_NVHE_ALIAS(__kvm_tlb_flush_local_vmid);

KVM_NVHE_ALIAS(__kvm_tlb_flush_vmid);

KVM_NVHE_ALIAS(__kvm_tlb_flush_vmid_ipa);



/* Symbols defined in vgic-v3-sr.c (not yet compiled with nVHE build rules). */

KVM_NVHE_ALIAS(__vgic_v3_get_ich_vtr_el2);

KVM_NVHE_ALIAS(__vgic_v3_init_lrs);
@@ -116,6 +110,14 @@ KVM_NVHE_ALIAS(__hyp_stub_vectors); 
/* IDMAP TCR_EL1.T0SZ as computed by the EL1 init code */

KVM_NVHE_ALIAS(idmap_t0sz);



/* Kernel symbol used by icache_is_vpipt(). */

KVM_NVHE_ALIAS(__icache_flags);



/* Kernel symbols needed for cpus_have_final/const_caps checks. */

KVM_NVHE_ALIAS(arm64_const_caps_ready);

KVM_NVHE_ALIAS(cpu_hwcap_keys);

KVM_NVHE_ALIAS(cpu_hwcaps);



#endif /* CONFIG_KVM */



#endif /* __ARM64_KERNEL_IMAGE_VARS_H */

arch/arm64/kvm/hyp/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ obj-$(CONFIG_KVM) += hyp.o vhe/ nvhe/ 
obj-$(CONFIG_KVM_INDIRECT_VECTORS) += smccc_wa.o



hyp-y := vgic-v3-sr.o timer-sr.o aarch32.o vgic-v2-cpuif-proxy.o sysreg-sr.o \

	 debug-sr.o entry.o switch.o fpsimd.o tlb.o

	 debug-sr.o entry.o switch.o fpsimd.o



# KVM code is run at a different exception code with a different map, so

# compiler instrumentation that inserts callbacks or checks into the code may

arch/arm64/kvm/hyp/nvhe/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -6,7 +6,7 @@ 
asflags-y := -D__KVM_NVHE_HYPERVISOR__

ccflags-y := -D__KVM_NVHE_HYPERVISOR__



obj-y := hyp-init.o ../hyp-entry.o

obj-y := tlb.o hyp-init.o ../hyp-entry.o



obj-y := $(patsubst %.o,%.hyp.o,$(obj-y))

extra-y := $(patsubst %.hyp.o,%.hyp.tmp.o,$(obj-y))

arch/arm64/kvm/hyp/tlb.c→arch/arm64/kvm/hyp/nvhe/tlb.c

+5 −91
Original line number Diff line number Diff line
@@ -4,63 +4,15 @@ 
 * Author: Marc Zyngier <marc.zyngier@arm.com>

 */



#include <linux/irqflags.h>



#include <asm/kvm_hyp.h>

#include <asm/kvm_mmu.h>

#include <asm/tlbflush.h>



struct tlb_inv_context {

	unsigned long	flags;

	u64		tcr;

	u64		sctlr;

};



static void __hyp_text __tlb_switch_to_guest_vhe(struct kvm *kvm,

						 struct tlb_inv_context *cxt)

{

	u64 val;



	local_irq_save(cxt->flags);



	if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {

		/*

		 * For CPUs that are affected by ARM errata 1165522 or 1530923,

		 * we cannot trust stage-1 to be in a correct state at that

		 * point. Since we do not want to force a full load of the

		 * vcpu state, we prevent the EL1 page-table walker to

		 * allocate new TLBs. This is done by setting the EPD bits

		 * in the TCR_EL1 register. We also need to prevent it to

		 * allocate IPA->PA walks, so we enable the S1 MMU...

		 */

		val = cxt->tcr = read_sysreg_el1(SYS_TCR);

		val |= TCR_EPD1_MASK | TCR_EPD0_MASK;

		write_sysreg_el1(val, SYS_TCR);

		val = cxt->sctlr = read_sysreg_el1(SYS_SCTLR);

		val |= SCTLR_ELx_M;

		write_sysreg_el1(val, SYS_SCTLR);

	}



	/*

	 * With VHE enabled, we have HCR_EL2.{E2H,TGE} = {1,1}, and

	 * most TLB operations target EL2/EL0. In order to affect the

	 * guest TLBs (EL1/EL0), we need to change one of these two

	 * bits. Changing E2H is impossible (goodbye TTBR1_EL2), so

	 * let's flip TGE before executing the TLB operation.

	 *

	 * ARM erratum 1165522 requires some special handling (again),

	 * as we need to make sure both stages of translation are in

	 * place before clearing TGE. __load_guest_stage2() already

	 * has an ISB in order to deal with this.

	 */

	__load_guest_stage2(kvm);

	val = read_sysreg(hcr_el2);

	val &= ~HCR_TGE;

	write_sysreg(val, hcr_el2);

	isb();

}



static void __hyp_text __tlb_switch_to_guest_nvhe(struct kvm *kvm,

static void __hyp_text __tlb_switch_to_guest(struct kvm *kvm,

					     struct tlb_inv_context *cxt)

{

	if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
@@ -84,36 +36,7 @@ static void __hyp_text __tlb_switch_to_guest_nvhe(struct kvm *kvm, 
	asm(ALTERNATIVE("isb", "nop", ARM64_WORKAROUND_SPECULATIVE_AT));

}



static void __hyp_text __tlb_switch_to_guest(struct kvm *kvm,

					     struct tlb_inv_context *cxt)

{

	if (has_vhe())

		__tlb_switch_to_guest_vhe(kvm, cxt);

	else

		__tlb_switch_to_guest_nvhe(kvm, cxt);

}



static void __hyp_text __tlb_switch_to_host_vhe(struct kvm *kvm,

						struct tlb_inv_context *cxt)

{

	/*

	 * We're done with the TLB operation, let's restore the host's

	 * view of HCR_EL2.

	 */

	write_sysreg(0, vttbr_el2);

	write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);

	isb();



	if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {

		/* Restore the registers to what they were */

		write_sysreg_el1(cxt->tcr, SYS_TCR);

		write_sysreg_el1(cxt->sctlr, SYS_SCTLR);

	}



	local_irq_restore(cxt->flags);

}



static void __hyp_text __tlb_switch_to_host_nvhe(struct kvm *kvm,

static void __hyp_text __tlb_switch_to_host(struct kvm *kvm,

					    struct tlb_inv_context *cxt)

{

	write_sysreg(0, vttbr_el2);
@@ -126,15 +49,6 @@ static void __hyp_text __tlb_switch_to_host_nvhe(struct kvm *kvm, 
	}

}



static void __hyp_text __tlb_switch_to_host(struct kvm *kvm,

					    struct tlb_inv_context *cxt)

{

	if (has_vhe())

		__tlb_switch_to_host_vhe(kvm, cxt);

	else

		__tlb_switch_to_host_nvhe(kvm, cxt);

}



void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)

{

	struct tlb_inv_context cxt;
@@ -183,7 +97,7 @@ void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa) 
	 * The moral of this story is: if you have a VPIPT I-cache, then

	 * you should be running with VHE enabled.

	 */

	if (!has_vhe() && icache_is_vpipt())

	if (icache_is_vpipt())

		__flush_icache_all();



	__tlb_switch_to_host(kvm, &cxt);

arch/arm64/kvm/hyp/vhe/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -6,7 +6,7 @@ 
asflags-y := -D__KVM_VHE_HYPERVISOR__

ccflags-y := -D__KVM_VHE_HYPERVISOR__



obj-y := ../hyp-entry.o

obj-y := tlb.o ../hyp-entry.o



# KVM code is run at a different exception code with a different map, so

# compiler instrumentation that inserts callbacks or checks into the code may

CRA Git | Maintained and supported by SUSTech CRA and CCSE