Commit 94d0e598 authored by Marc Zyngier's avatar Marc Zyngier
Browse files

arm/arm64: KVM: Perform local TLB invalidation when multiplexing vcpus on a single CPU 



Architecturally, TLBs are private to the (physical) CPU they're
associated with. But when multiple vcpus from the same VM are
being multiplexed on the same CPU, the TLBs are not private
to the vcpus (and are actually shared across the VMID).

Let's consider the following scenario:

- vcpu-0 maps PA to VA
- vcpu-1 maps PA' to VA

If run on the same physical CPU, vcpu-1 can hit TLB entries generated
by vcpu-0 accesses, and access the wrong physical page.

The solution to this is to keep a per-VM map of which vcpu ran last
on each given physical CPU, and invalidate local TLBs when switching
to a different vcpu from the same VM.

Reviewed-by: default avatarChristoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: default avatarMarc Zyngier <marc.zyngier@arm.com>
parent 07d9a380

arch/arm/include/asm/kvm_asm.h

+1 −0
Original line number Diff line number Diff line
@@ -66,6 +66,7 @@ extern char __kvm_hyp_vector[]; 
extern void __kvm_flush_vm_context(void);

extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);

extern void __kvm_tlb_flush_vmid(struct kvm *kvm);

extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu);



extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);

arch/arm/include/asm/kvm_host.h

+3 −0
Original line number Diff line number Diff line
@@ -57,6 +57,9 @@ struct kvm_arch { 
	/* VTTBR value associated with below pgd and vmid */

	u64    vttbr;



	/* The last vcpu id that ran on each physical CPU */

	int __percpu *last_vcpu_ran;



	/* Timer */

	struct arch_timer_kvm	timer;

arch/arm/include/asm/kvm_hyp.h

+1 −0
Original line number Diff line number Diff line
@@ -71,6 +71,7 @@ 
#define ICIALLUIS	__ACCESS_CP15(c7, 0, c1, 0)

#define ATS1CPR		__ACCESS_CP15(c7, 0, c8, 0)

#define TLBIALLIS	__ACCESS_CP15(c8, 0, c3, 0)

#define TLBIALL		__ACCESS_CP15(c8, 0, c7, 0)

#define TLBIALLNSNHIS	__ACCESS_CP15(c8, 4, c3, 4)

#define PRRR		__ACCESS_CP15(c10, 0, c2, 0)

#define NMRR		__ACCESS_CP15(c10, 0, c2, 1)

arch/arm/kvm/arm.c

+26 −1
Original line number Diff line number Diff line
@@ -114,11 +114,18 @@ void kvm_arch_check_processor_compat(void *rtn) 
 */

int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)

{

	int ret = 0;

	int ret, cpu;



	if (type)

		return -EINVAL;



	kvm->arch.last_vcpu_ran = alloc_percpu(typeof(*kvm->arch.last_vcpu_ran));

	if (!kvm->arch.last_vcpu_ran)

		return -ENOMEM;



	for_each_possible_cpu(cpu)

		*per_cpu_ptr(kvm->arch.last_vcpu_ran, cpu) = -1;



	ret = kvm_alloc_stage2_pgd(kvm);

	if (ret)

		goto out_fail_alloc;
@@ -141,6 +148,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 
out_free_stage2_pgd:

	kvm_free_stage2_pgd(kvm);

out_fail_alloc:

	free_percpu(kvm->arch.last_vcpu_ran);

	kvm->arch.last_vcpu_ran = NULL;

	return ret;

}
@@ -168,6 +177,9 @@ void kvm_arch_destroy_vm(struct kvm *kvm) 
{

	int i;



	free_percpu(kvm->arch.last_vcpu_ran);

	kvm->arch.last_vcpu_ran = NULL;



	for (i = 0; i < KVM_MAX_VCPUS; ++i) {

		if (kvm->vcpus[i]) {

			kvm_arch_vcpu_free(kvm->vcpus[i]);
@@ -312,6 +324,19 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 


void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)

{

	int *last_ran;



	last_ran = this_cpu_ptr(vcpu->kvm->arch.last_vcpu_ran);



	/*

	 * We might get preempted before the vCPU actually runs, but

	 * over-invalidation doesn't affect correctness.

	 */

	if (*last_ran != vcpu->vcpu_id) {

		kvm_call_hyp(__kvm_tlb_flush_local_vmid, vcpu);

		*last_ran = vcpu->vcpu_id;

	}



	vcpu->cpu = cpu;

	vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state);

arch/arm/kvm/hyp/tlb.c

+15 −0
Original line number Diff line number Diff line
@@ -55,6 +55,21 @@ void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa) 
	__kvm_tlb_flush_vmid(kvm);

}



void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu)

{

	struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm);



	/* Switch to requested VMID */

	write_sysreg(kvm->arch.vttbr, VTTBR);

	isb();



	write_sysreg(0, TLBIALL);

	dsb(nsh);

	isb();



	write_sysreg(0, VTTBR);

}



void __hyp_text __kvm_flush_vm_context(void)

{

	write_sysreg(0, TLBIALLNSNHIS);

CRA Git | Maintained and supported by SUSTech CRA and CCSE