Commit 2f6474e4 authored by Thomas Gleixner's avatar Thomas Gleixner
Browse files

x86/entry: Switch XEN/PV hypercall entry to IDTENTRY 



Convert the XEN/PV hypercall to IDTENTRY:

  - Emit the ASM stub with DECLARE_IDTENTRY
  - Remove the ASM idtentry in 64-bit
  - Remove the open coded ASM entry code in 32-bit
  - Remove the old prototypes

The handler stubs need to stay in ASM code as they need corner case handling
and adjustment of the stack pointer.

Provide a new C function which invokes the entry/exit handling and calls
into the XEN handler on the interrupt stack if required.

The exit code is slightly different from the regular idtentry_exit() on
non-preemptible kernels. If the hypercall is preemptible and need_resched()
is set then XEN provides a preempt hypercall scheduling function.

Move this functionality into the entry code so it can use the existing
idtentry functionality.

[ mingo: Build fixes. ]

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Acked-by: default avatarAndy Lutomirski <luto@kernel.org>
Acked-by: default avatarJuergen Gross <jgross@suse.com>
Tested-by: default avatarJuergen Gross <jgross@suse.com>
Link: https://lore.kernel.org/r/20200521202118.055270078@linutronix.de
parent 1de16e0c

arch/x86/entry/common.c

+78 −0
Original line number Diff line number Diff line
@@ -27,6 +27,11 @@ 
#include <linux/syscalls.h>

#include <linux/uaccess.h>



#ifdef CONFIG_XEN_PV

#include <xen/xen-ops.h>

#include <xen/events.h>

#endif



#include <asm/desc.h>

#include <asm/traps.h>

#include <asm/vdso.h>
@@ -35,6 +40,7 @@ 
#include <asm/nospec-branch.h>

#include <asm/io_bitmap.h>

#include <asm/syscall.h>

#include <asm/irq_stack.h>



#define CREATE_TRACE_POINTS

#include <trace/events/syscalls.h>
@@ -680,3 +686,75 @@ void noinstr idtentry_exit_user(struct pt_regs *regs) 


	prepare_exit_to_usermode(regs);

}



#ifdef CONFIG_XEN_PV

#ifndef CONFIG_PREEMPTION

/*

 * Some hypercalls issued by the toolstack can take many 10s of

 * seconds. Allow tasks running hypercalls via the privcmd driver to

 * be voluntarily preempted even if full kernel preemption is

 * disabled.

 *

 * Such preemptible hypercalls are bracketed by

 * xen_preemptible_hcall_begin() and xen_preemptible_hcall_end()

 * calls.

 */

DEFINE_PER_CPU(bool, xen_in_preemptible_hcall);

EXPORT_SYMBOL_GPL(xen_in_preemptible_hcall);



/*

 * In case of scheduling the flag must be cleared and restored after

 * returning from schedule as the task might move to a different CPU.

 */

static __always_inline bool get_and_clear_inhcall(void)

{

	bool inhcall = __this_cpu_read(xen_in_preemptible_hcall);



	__this_cpu_write(xen_in_preemptible_hcall, false);

	return inhcall;

}



static __always_inline void restore_inhcall(bool inhcall)

{

	__this_cpu_write(xen_in_preemptible_hcall, inhcall);

}

#else

static __always_inline bool get_and_clear_inhcall(void) { return false; }

static __always_inline void restore_inhcall(bool inhcall) { }

#endif



static void __xen_pv_evtchn_do_upcall(void)

{

	irq_enter_rcu();

	inc_irq_stat(irq_hv_callback_count);



	xen_hvm_evtchn_do_upcall();



	irq_exit_rcu();

}



__visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs)

{

	struct pt_regs *old_regs;

	bool inhcall, rcu_exit;



	rcu_exit = idtentry_enter_cond_rcu(regs);

	old_regs = set_irq_regs(regs);



	instrumentation_begin();

	run_on_irqstack_cond(__xen_pv_evtchn_do_upcall, NULL, regs);

	instrumentation_begin();



	set_irq_regs(old_regs);



	inhcall = get_and_clear_inhcall();

	if (inhcall && !WARN_ON_ONCE(rcu_exit)) {

		instrumentation_begin();

		idtentry_exit_cond_resched(regs, true);

		instrumentation_end();

		restore_inhcall(inhcall);

	} else {

		idtentry_exit_cond_rcu(regs, rcu_exit);

	}

}

#endif /* CONFIG_XEN_PV */

arch/x86/entry/entry_32.S

+12 −8
Original line number Diff line number Diff line
@@ -1298,7 +1298,13 @@ SYM_CODE_END(native_iret) 
#endif



#ifdef CONFIG_XEN_PV

SYM_FUNC_START(xen_hypervisor_callback)

/*

 * See comment in entry_64.S for further explanation

 *

 * Note: This is not an actual IDT entry point. It's a XEN specific entry

 * point and therefore named to match the 64-bit trampoline counterpart.

 */

SYM_FUNC_START(xen_asm_exc_xen_hypervisor_callback)

	/*

	 * Check to see if we got the event in the critical

	 * region in xen_iret_direct, after we've reenabled
@@ -1315,14 +1321,11 @@ SYM_FUNC_START(xen_hypervisor_callback) 
	pushl	$-1				/* orig_ax = -1 => not a system call */

	SAVE_ALL

	ENCODE_FRAME_POINTER

	TRACE_IRQS_OFF



	mov	%esp, %eax

	call	xen_evtchn_do_upcall

#ifndef CONFIG_PREEMPTION

	call	xen_maybe_preempt_hcall

#endif

	jmp	ret_from_intr

SYM_FUNC_END(xen_hypervisor_callback)

	call	xen_pv_evtchn_do_upcall

	jmp	handle_exception_return

SYM_FUNC_END(xen_asm_exc_xen_hypervisor_callback)



/*

 * Hypervisor uses this for application faults while it executes.
@@ -1464,6 +1467,7 @@ SYM_CODE_START_LOCAL_NOALIGN(handle_exception) 
	movl	%esp, %eax			# pt_regs pointer

	CALL_NOSPEC edi



handle_exception_return:

#ifdef CONFIG_VM86

	movl	PT_EFLAGS(%esp), %eax		# mix EFLAGS and CS

	movb	PT_CS(%esp), %al

arch/x86/entry/entry_64.S

+6 −14
Original line number Diff line number Diff line
@@ -1067,10 +1067,6 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt 


idtentry	X86_TRAP_PF		page_fault		do_page_fault			has_error_code=1



#ifdef CONFIG_XEN_PV

idtentry	512 /* dummy */		hypervisor_callback	xen_do_hypervisor_callback	has_error_code=0

#endif



/*

 * Reload gs selector with exception handling

 * edi:  new selector
@@ -1158,9 +1154,10 @@ SYM_FUNC_END(asm_call_on_stack) 
 * So, on entry to the handler we detect whether we interrupted an

 * existing activation in its critical region -- if so, we pop the current

 * activation and restart the handler using the previous one.

 *

 * C calling convention: exc_xen_hypervisor_callback(struct *pt_regs)

 */

/* do_hypervisor_callback(struct *pt_regs) */

SYM_CODE_START_LOCAL(xen_do_hypervisor_callback)

SYM_CODE_START_LOCAL(exc_xen_hypervisor_callback)



/*

 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
@@ -1170,15 +1167,10 @@ SYM_CODE_START_LOCAL(xen_do_hypervisor_callback) 
	movq	%rdi, %rsp			/* we don't return, adjust the stack frame */

	UNWIND_HINT_REGS



	ENTER_IRQ_STACK old_rsp=%r10

	call	xen_evtchn_do_upcall

	LEAVE_IRQ_STACK

	call	xen_pv_evtchn_do_upcall



#ifndef CONFIG_PREEMPTION

	call	xen_maybe_preempt_hcall

#endif

	jmp	error_exit

SYM_CODE_END(xen_do_hypervisor_callback)

	jmp	error_return

SYM_CODE_END(exc_xen_hypervisor_callback)



/*

 * Hypervisor uses this for application faults while it executes.

arch/x86/include/asm/idtentry.h

+34 −0
Original line number Diff line number Diff line
@@ -165,6 +165,21 @@ __visible noinstr void func(struct pt_regs *regs) 
#define DEFINE_IDTENTRY_RAW_ERRORCODE(func)				\

__visible noinstr void func(struct pt_regs *regs, unsigned long error_code)



/**

 * DECLARE_IDTENTRY_XENCB - Declare functions for XEN HV callback entry point

 * @vector:	Vector number (ignored for C)

 * @func:	Function name of the entry point

 *

 * Declares three functions:

 * - The ASM entry point: asm_##func

 * - The XEN PV trap entry point: xen_##func (maybe unused)

 * - The C handler called from the ASM entry point

 *

 * Maps to DECLARE_IDTENTRY(). Distinct entry point to handle the 32/64-bit

 * difference

 */

#define DECLARE_IDTENTRY_XENCB(vector, func)				\

	DECLARE_IDTENTRY(vector, func)



#ifdef CONFIG_X86_64

/**
@@ -307,6 +322,9 @@ __visible noinstr void func(struct pt_regs *regs, \ 
# define DECLARE_IDTENTRY_DF(vector, func)				\

	idtentry_df vector asm_##func func



# define DECLARE_IDTENTRY_XENCB(vector, func)				\

	DECLARE_IDTENTRY(vector, func)



#else

# define DECLARE_IDTENTRY_MCE(vector, func)				\

	DECLARE_IDTENTRY(vector, func)
@@ -317,6 +335,9 @@ __visible noinstr void func(struct pt_regs *regs, \ 
/* No ASM emitted for DF as this goes through a C shim */

# define DECLARE_IDTENTRY_DF(vector, func)



/* No ASM emitted for XEN hypervisor callback */

# define DECLARE_IDTENTRY_XENCB(vector, func)



#endif



/* No ASM code emitted for NMI */
@@ -337,6 +358,13 @@ __visible noinstr void func(struct pt_regs *regs, \ 
 * This avoids duplicate defines and ensures that everything is consistent.

 */



/*

 * Dummy trap number so the low level ASM macro vector number checks do not

 * match which results in emitting plain IDTENTRY stubs without bells and

 * whistels.

 */

#define X86_TRAP_OTHER		0xFFFF



/* Simple exception entry points. No hardware error code */

DECLARE_IDTENTRY(X86_TRAP_DE,		exc_divide_error);

DECLARE_IDTENTRY(X86_TRAP_OF,		exc_overflow);
@@ -376,4 +404,10 @@ DECLARE_IDTENTRY_XEN(X86_TRAP_DB, debug); 
/* #DF */

DECLARE_IDTENTRY_DF(X86_TRAP_DF,	exc_double_fault);



#ifdef CONFIG_XEN_PV

DECLARE_IDTENTRY_XENCB(X86_TRAP_OTHER,	exc_xen_hypervisor_callback);

#endif



#undef X86_TRAP_OTHER



#endif

arch/x86/xen/setup.c

+3 −1
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ 
#include <asm/setup.h>

#include <asm/acpi.h>

#include <asm/numa.h>

#include <asm/idtentry.h>

#include <asm/xen/hypervisor.h>

#include <asm/xen/hypercall.h>
@@ -993,7 +994,8 @@ static void __init xen_pvmmu_arch_setup(void) 
	HYPERVISOR_vm_assist(VMASST_CMD_enable,

			     VMASST_TYPE_pae_extended_cr3);



	if (register_callback(CALLBACKTYPE_event, xen_hypervisor_callback) ||

	if (register_callback(CALLBACKTYPE_event,

			      xen_asm_exc_xen_hypervisor_callback) ||

	    register_callback(CALLBACKTYPE_failsafe, xen_failsafe_callback))

		BUG();

CRA Git | Maintained and supported by SUSTech CRA and CCSE