Commit da4ad88c authored by Davidlohr Bueso's avatar Davidlohr Bueso Committed by Paolo Bonzini
Browse files

kvm: Replace vcpu->swait with rcuwait 



The use of any sort of waitqueue (simple or regular) for
wait/waking vcpus has always been an overkill and semantically
wrong. Because this is per-vcpu (which is blocked) there is
only ever a single waiting vcpu, thus no need for any sort of
queue.

As such, make use of the rcuwait primitive, with the following
considerations:

  - rcuwait already provides the proper barriers that serialize
  concurrent waiter and waker.

  - Task wakeup is done in rcu read critical region, with a
  stable task pointer.

  - Because there is no concurrency among waiters, we need
  not worry about rcuwait_wait_event() calls corrupting
  the wait->task. As a consequence, this saves the locking
  done in swait when modifying the queue. This also applies
  to per-vcore wait for powerpc kvm-hv.

The x86 tscdeadline_latency test mentioned in 8577370f
("KVM: Use simple waitqueue for vcpu->wq") shows that, on avg,
latency is reduced by around 15-20% with this change.

Cc: Paul Mackerras <paulus@ozlabs.org>
Cc: kvmarm@lists.cs.columbia.edu
Cc: linux-mips@vger.kernel.org
Reviewed-by: default avatarMarc Zyngier <maz@kernel.org>
Signed-off-by: default avatarDavidlohr Bueso <dbueso@suse.de>
Message-Id: <20200424054837.5138-6-dave@stgolabs.net>
[Avoid extra logic changes. - Paolo]
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent 191a43be

arch/mips/kvm/mips.c

+2 −4
Original line number Diff line number Diff line
@@ -283,8 +283,7 @@ static enum hrtimer_restart kvm_mips_comparecount_wakeup(struct hrtimer *timer) 
	kvm_mips_callbacks->queue_timer_int(vcpu);



	vcpu->arch.wait = 0;

	if (swq_has_sleeper(&vcpu->wq))

		swake_up_one(&vcpu->wq);

	rcuwait_wake_up(&vcpu->wait);



	return kvm_mips_count_timeout(vcpu);

}
@@ -511,8 +510,7 @@ int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, 


	dvcpu->arch.wait = 0;



	if (swq_has_sleeper(&dvcpu->wq))

		swake_up_one(&dvcpu->wq);

	rcuwait_wake_up(&dvcpu->wait);



	return 0;

}

arch/powerpc/include/asm/kvm_book3s.h

+1 −1
Original line number Diff line number Diff line
@@ -78,7 +78,7 @@ struct kvmppc_vcore { 
	struct kvm_vcpu *runnable_threads[MAX_SMT_THREADS];

	struct list_head preempt_list;

	spinlock_t lock;

	struct swait_queue_head wq;

	struct rcuwait wait;

	spinlock_t stoltb_lock;	/* protects stolen_tb and preempt_tb */

	u64 stolen_tb;

	u64 preempt_tb;

arch/powerpc/include/asm/kvm_host.h

+1 −1
Original line number Diff line number Diff line
@@ -751,7 +751,7 @@ struct kvm_vcpu_arch { 
	u8 irq_pending; /* Used by XIVE to signal pending guest irqs */

	u32 last_inst;



	struct swait_queue_head *wqp;

	struct rcuwait *waitp;

	struct kvmppc_vcore *vcore;

	int ret;

	int trap;

arch/powerpc/kvm/book3s_hv.c

+10 −13
Original line number Diff line number Diff line
@@ -230,13 +230,11 @@ static bool kvmppc_ipi_thread(int cpu) 
static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)

{

	int cpu;

	struct swait_queue_head *wqp;

	struct rcuwait *waitp;



	wqp = kvm_arch_vcpu_wq(vcpu);

	if (swq_has_sleeper(wqp)) {

		swake_up_one(wqp);

	waitp = kvm_arch_vcpu_get_wait(vcpu);

	if (rcuwait_wake_up(waitp))

		++vcpu->stat.halt_wakeup;

	}



	cpu = READ_ONCE(vcpu->arch.thread_cpu);

	if (cpu >= 0 && kvmppc_ipi_thread(cpu))
@@ -2125,7 +2123,7 @@ static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int id) 


	spin_lock_init(&vcore->lock);

	spin_lock_init(&vcore->stoltb_lock);

	init_swait_queue_head(&vcore->wq);

	rcuwait_init(&vcore->wait);

	vcore->preempt_tb = TB_NIL;

	vcore->lpcr = kvm->arch.lpcr;

	vcore->first_vcpuid = id;
@@ -3784,7 +3782,6 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) 
	ktime_t cur, start_poll, start_wait;

	int do_sleep = 1;

	u64 block_ns;

	DECLARE_SWAITQUEUE(wait);



	/* Poll for pending exceptions and ceded state */

	cur = start_poll = ktime_get();
@@ -3812,10 +3809,10 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) 
		}

	}



	prepare_to_swait_exclusive(&vc->wq, &wait, TASK_INTERRUPTIBLE);



	prepare_to_rcuwait(&vc->wait);

	set_current_state(TASK_INTERRUPTIBLE);

	if (kvmppc_vcore_check_block(vc)) {

		finish_swait(&vc->wq, &wait);

		finish_rcuwait(&vc->wait);

		do_sleep = 0;

		/* If we polled, count this as a successful poll */

		if (vc->halt_poll_ns)
@@ -3829,7 +3826,7 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) 
	trace_kvmppc_vcore_blocked(vc, 0);

	spin_unlock(&vc->lock);

	schedule();

	finish_swait(&vc->wq, &wait);

	finish_rcuwait(&vc->wait);

	spin_lock(&vc->lock);

	vc->vcore_state = VCORE_INACTIVE;

	trace_kvmppc_vcore_blocked(vc, 1);
@@ -3940,7 +3937,7 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) 
			kvmppc_start_thread(vcpu, vc);

			trace_kvm_guest_enter(vcpu);

		} else if (vc->vcore_state == VCORE_SLEEPING) {

			swake_up_one(&vc->wq);

		        rcuwait_wake_up(&vc->wait);

		}



	}
@@ -4279,7 +4276,7 @@ static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu) 
	}

	user_vrsave = mfspr(SPRN_VRSAVE);



	vcpu->arch.wqp = &vcpu->arch.vcore->wq;

	vcpu->arch.waitp = &vcpu->arch.vcore->wait;

	vcpu->arch.pgdir = kvm->mm->pgd;

	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;

arch/powerpc/kvm/powerpc.c

+1 −1
Original line number Diff line number Diff line
@@ -752,7 +752,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 
	if (err)

		goto out_vcpu_uninit;



	vcpu->arch.wqp = &vcpu->wq;

	vcpu->arch.waitp = &vcpu->wait;

	kvmppc_create_vcpu_debugfs(vcpu, vcpu->vcpu_id);

	return 0;

CRA Git | Maintained and supported by SUSTech CRA and CCSE