Commit 2ffd9e33 authored by Vitaly Kuznetsov's avatar Vitaly Kuznetsov Committed by Ingo Molnar
Browse files

x86/hyper-v: Use hypercall for remote TLB flush 



Hyper-V host can suggest us to use hypercall for doing remote TLB flush,
this is supposed to work faster than IPIs.

Implementation details: to do HvFlushVirtualAddress{Space,List} hypercalls
we need to put the input somewhere in memory and we don't really want to
have memory allocation on each call so we pre-allocate per cpu memory areas
on boot.

pv_ops patching is happening very early so we need to separate
hyperv_setup_mmu_ops() and hyper_alloc_mmu().

It is possible and easy to implement local TLB flushing too and there is
even a hint for that. However, I don't see a room for optimization on the
host side as both hypercall and native tlb flush will result in vmexit. The
hint is also not set on modern Hyper-V versions.

Signed-off-by: default avatarVitaly Kuznetsov <vkuznets@redhat.com>
Reviewed-by: default avatarAndy Shevchenko <andy.shevchenko@gmail.com>
Reviewed-by: default avatarStephen Hemminger <sthemmin@microsoft.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Haiyang Zhang <haiyangz@microsoft.com>
Cc: Jork Loeser <Jork.Loeser@microsoft.com>
Cc: K. Y. Srinivasan <kys@microsoft.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Simon Xiao <sixiao@microsoft.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: devel@linuxdriverproject.org
Link: http://lkml.kernel.org/r/20170802160921.21791-8-vkuznets@redhat.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 7415aea6

arch/x86/hyperv/Makefile

+1 −1
Original line number Original line Diff line number Diff line 
obj-y		:= hv_init.o
 
obj-y		:= hv_init.o mmu.o

arch/x86/hyperv/hv_init.c

+2 −0
Original line number Original line Diff line number Diff line
@@ -140,6 +140,8 @@ void hyperv_init(void) 
	hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg);

	hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg);

	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);

	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);





	hyper_alloc_mmu();



	/*

	/*

	 * Register Hyper-V specific clocksource.

	 * Register Hyper-V specific clocksource.

	 */

	 */

arch/x86/hyperv/mmu.c

0 → 100644
+138 −0
Original line number Original line Diff line number Diff line 
#define pr_fmt(fmt)  "Hyper-V: " fmt



#include <linux/hyperv.h>

#include <linux/log2.h>

#include <linux/slab.h>

#include <linux/types.h>



#include <asm/fpu/api.h>

#include <asm/mshyperv.h>

#include <asm/msr.h>

#include <asm/tlbflush.h>



/* HvFlushVirtualAddressSpace, HvFlushVirtualAddressList hypercalls */

struct hv_flush_pcpu {

	u64 address_space;

	u64 flags;

	u64 processor_mask;

	u64 gva_list[];

};



/* Each gva in gva_list encodes up to 4096 pages to flush */

#define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE)



static struct hv_flush_pcpu __percpu *pcpu_flush;



/*

 * Fills in gva_list starting from offset. Returns the number of items added.

 */

static inline int fill_gva_list(u64 gva_list[], int offset,

				unsigned long start, unsigned long end)

{

	int gva_n = offset;

	unsigned long cur = start, diff;



	do {

		diff = end > cur ? end - cur : 0;



		gva_list[gva_n] = cur & PAGE_MASK;

		/*

		 * Lower 12 bits encode the number of additional

		 * pages to flush (in addition to the 'cur' page).

		 */

		if (diff >= HV_TLB_FLUSH_UNIT)

			gva_list[gva_n] |= ~PAGE_MASK;

		else if (diff)

			gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT;



		cur += HV_TLB_FLUSH_UNIT;

		gva_n++;



	} while (cur < end);



	return gva_n - offset;

}



static void hyperv_flush_tlb_others(const struct cpumask *cpus,

				    const struct flush_tlb_info *info)

{

	int cpu, vcpu, gva_n, max_gvas;

	struct hv_flush_pcpu *flush;

	u64 status = U64_MAX;

	unsigned long flags;



	if (!pcpu_flush || !hv_hypercall_pg)

		goto do_native;



	if (cpumask_empty(cpus))

		return;



	local_irq_save(flags);



	flush = this_cpu_ptr(pcpu_flush);



	if (info->mm) {

		flush->address_space = virt_to_phys(info->mm->pgd);

		flush->flags = 0;

	} else {

		flush->address_space = 0;

		flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;

	}



	flush->processor_mask = 0;

	if (cpumask_equal(cpus, cpu_present_mask)) {

		flush->flags |= HV_FLUSH_ALL_PROCESSORS;

	} else {

		for_each_cpu(cpu, cpus) {

			vcpu = hv_cpu_number_to_vp_number(cpu);

			if (vcpu >= 64)

				goto do_native;



			__set_bit(vcpu, (unsigned long *)

				  &flush->processor_mask);

		}

	}



	/*

	 * We can flush not more than max_gvas with one hypercall. Flush the

	 * whole address space if we were asked to do more.

	 */

	max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]);



	if (info->end == TLB_FLUSH_ALL) {

		flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;

		status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,

					 flush, NULL);

	} else if (info->end &&

		   ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {

		status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,

					 flush, NULL);

	} else {

		gva_n = fill_gva_list(flush->gva_list, 0,

				      info->start, info->end);

		status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST,

					     gva_n, 0, flush, NULL);

	}



	local_irq_restore(flags);



	if (!(status & HV_HYPERCALL_RESULT_MASK))

		return;

do_native:

	native_flush_tlb_others(cpus, info);

}



void hyperv_setup_mmu_ops(void)

{

	if (ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED) {

		pr_info("Using hypercall for remote TLB flush\n");

		pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others;

		setup_clear_cpu_cap(X86_FEATURE_PCID);

	}

}



void hyper_alloc_mmu(void)

{

	if (ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED)

		pcpu_flush = __alloc_percpu(PAGE_SIZE, PAGE_SIZE);

}

arch/x86/include/asm/mshyperv.h

+3 −0
Original line number Original line Diff line number Diff line
@@ -307,6 +307,8 @@ static inline int hv_cpu_number_to_vp_number(int cpu_number) 
}

}





void hyperv_init(void);

void hyperv_init(void);

void hyperv_setup_mmu_ops(void);

void hyper_alloc_mmu(void);

void hyperv_report_panic(struct pt_regs *regs);

void hyperv_report_panic(struct pt_regs *regs);

bool hv_is_hypercall_page_setup(void);

bool hv_is_hypercall_page_setup(void);

void hyperv_cleanup(void);

void hyperv_cleanup(void);
@@ -314,6 +316,7 @@ void hyperv_cleanup(void); 
static inline void hyperv_init(void) {}

static inline void hyperv_init(void) {}

static inline bool hv_is_hypercall_page_setup(void) { return false; }

static inline bool hv_is_hypercall_page_setup(void) { return false; }

static inline void hyperv_cleanup(void) {}

static inline void hyperv_cleanup(void) {}

static inline void hyperv_setup_mmu_ops(void) {}

#endif /* CONFIG_HYPERV */

#endif /* CONFIG_HYPERV */





#ifdef CONFIG_HYPERV_TSCPAGE

#ifdef CONFIG_HYPERV_TSCPAGE

arch/x86/include/uapi/asm/hyperv.h

+7 −0
Original line number Original line Diff line number Diff line
@@ -242,6 +242,8 @@ 
		(~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1))

		(~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1))





/* Declare the various hypercall operations. */

/* Declare the various hypercall operations. */

#define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE	0x0002

#define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST	0x0003

#define HVCALL_NOTIFY_LONG_SPIN_WAIT		0x0008

#define HVCALL_NOTIFY_LONG_SPIN_WAIT		0x0008

#define HVCALL_POST_MESSAGE			0x005c

#define HVCALL_POST_MESSAGE			0x005c

#define HVCALL_SIGNAL_EVENT			0x005d

#define HVCALL_SIGNAL_EVENT			0x005d
@@ -259,6 +261,11 @@ 
#define HV_PROCESSOR_POWER_STATE_C2		2

#define HV_PROCESSOR_POWER_STATE_C2		2

#define HV_PROCESSOR_POWER_STATE_C3		3

#define HV_PROCESSOR_POWER_STATE_C3		3





#define HV_FLUSH_ALL_PROCESSORS			BIT(0)

#define HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES	BIT(1)

#define HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY	BIT(2)

#define HV_FLUSH_USE_EXTENDED_RANGE_FORMAT	BIT(3)



/* hypercall status code */

/* hypercall status code */

#define HV_STATUS_SUCCESS			0

#define HV_STATUS_SUCCESS			0

#define HV_STATUS_INVALID_HYPERCALL_CODE	2

#define HV_STATUS_INVALID_HYPERCALL_CODE	2

CRA Git | Maintained and supported by SUSTech CRA and CCSE