Commit adb334d1 authored by Ingo Molnar's avatar Ingo Molnar
Browse files

Merge branch 'WIP.x86/entry' into x86/entry, to merge the latest generic code and resolve conflicts 



Pick up and resolve the NMI entry code changes from the locking tree,
and also pick up the latest two fixes from tip:core/entry.

Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parents f3020b88 f87032ae

arch/x86/entry/common.c

+34 −0
Original line number Diff line number Diff line
@@ -198,6 +198,40 @@ SYSCALL_DEFINE0(ni_syscall) 
	return -ENOSYS;

}



noinstr bool idtentry_enter_nmi(struct pt_regs *regs)

{

	bool irq_state = lockdep_hardirqs_enabled();



	__nmi_enter();

	lockdep_hardirqs_off(CALLER_ADDR0);

	lockdep_hardirq_enter();

	rcu_nmi_enter();



	instrumentation_begin();

	trace_hardirqs_off_finish();

	ftrace_nmi_enter();

	instrumentation_end();



	return irq_state;

}



noinstr void idtentry_exit_nmi(struct pt_regs *regs, bool restore)

{

	instrumentation_begin();

	ftrace_nmi_exit();

	if (restore) {

		trace_hardirqs_on_prepare();

		lockdep_hardirqs_on_prepare(CALLER_ADDR0);

	}

	instrumentation_end();



	rcu_nmi_exit();

	lockdep_hardirq_exit();

	if (restore)

		lockdep_hardirqs_on(CALLER_ADDR0);

	__nmi_exit();

}



#ifdef CONFIG_XEN_PV

#ifndef CONFIG_PREEMPTION

/*

arch/x86/include/asm/idtentry.h

+3 −0
Original line number Diff line number Diff line
@@ -11,6 +11,9 @@ 


#include <asm/irq_stack.h>



bool idtentry_enter_nmi(struct pt_regs *regs);

void idtentry_exit_nmi(struct pt_regs *regs, bool irq_state);



/**

 * DECLARE_IDTENTRY - Declare functions for simple IDT entry points

 *		      No error code pushed by hardware

arch/x86/kernel/nmi.c

+4 −5
Original line number Diff line number Diff line
@@ -330,7 +330,6 @@ static noinstr void default_do_nmi(struct pt_regs *regs) 
	__this_cpu_write(last_nmi_rip, regs->ip);



	instrumentation_begin();

	trace_hardirqs_off_finish();



	handled = nmi_handle(NMI_LOCAL, regs);

	__this_cpu_add(nmi_stats.normal, handled);
@@ -417,8 +416,6 @@ static noinstr void default_do_nmi(struct pt_regs *regs) 
		unknown_nmi_error(reason, regs);



out:

	if (regs->flags & X86_EFLAGS_IF)

		trace_hardirqs_on_prepare();

	instrumentation_end();

}
@@ -478,6 +475,8 @@ static DEFINE_PER_CPU(unsigned long, nmi_dr7); 


DEFINE_IDTENTRY_RAW(exc_nmi)

{

	bool irq_state;



	if (IS_ENABLED(CONFIG_SMP) && arch_cpu_is_offline(smp_processor_id()))

		return;
@@ -491,14 +490,14 @@ nmi_restart: 


	this_cpu_write(nmi_dr7, local_db_save());



	nmi_enter();

	irq_state = idtentry_enter_nmi(regs);



	inc_irq_stat(__nmi_count);



	if (!ignore_nmis)

		default_do_nmi(regs);



	nmi_exit();

	idtentry_exit_nmi(regs, irq_state);



	local_db_restore(this_cpu_read(nmi_dr7));

arch/x86/kernel/traps.c

+6 −11
Original line number Diff line number Diff line
@@ -405,7 +405,7 @@ DEFINE_IDTENTRY_DF(exc_double_fault) 
	}

#endif



	nmi_enter();

	idtentry_enter_nmi(regs);

	instrumentation_begin();

	notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);
@@ -651,15 +651,12 @@ DEFINE_IDTENTRY_RAW(exc_int3) 
		instrumentation_end();

		irqentry_exit_to_user_mode(regs);

	} else {

		nmi_enter();

		bool irq_state = idtentry_enter_nmi(regs);

		instrumentation_begin();

		trace_hardirqs_off_finish();

		if (!do_int3(regs))

			die("int3", regs, 0);

		if (regs->flags & X86_EFLAGS_IF)

			trace_hardirqs_on_prepare();

		instrumentation_end();

		nmi_exit();

		idtentry_exit_nmi(regs, irq_state);

	}

}
@@ -867,9 +864,8 @@ out: 
static __always_inline void exc_debug_kernel(struct pt_regs *regs,

					     unsigned long dr6)

{

	nmi_enter();

	bool irq_state = idtentry_enter_nmi(regs);

	instrumentation_begin();

	trace_hardirqs_off_finish();



	/*

	 * If something gets miswired and we end up here for a user mode
@@ -886,10 +882,8 @@ static __always_inline void exc_debug_kernel(struct pt_regs *regs, 


	handle_debug(regs, dr6, false);



	if (regs->flags & X86_EFLAGS_IF)

		trace_hardirqs_on_prepare();

	instrumentation_end();

	nmi_exit();

	idtentry_exit_nmi(regs, irq_state);

}



static __always_inline void exc_debug_user(struct pt_regs *regs,
@@ -905,6 +899,7 @@ static __always_inline void exc_debug_user(struct pt_regs *regs, 
	instrumentation_begin();



	handle_debug(regs, dr6, true);



	instrumentation_end();

	irqentry_exit_to_user_mode(regs);

}

include/linux/hardirq.h

+19 −9
Original line number Diff line number Diff line
@@ -111,32 +111,42 @@ extern void rcu_nmi_exit(void); 
/*

 * nmi_enter() can nest up to 15 times; see NMI_BITS.

 */

#define nmi_enter()						\

#define __nmi_enter()						\

	do {							\

		lockdep_off();					\

		arch_nmi_enter();				\

		printk_nmi_enter();				\

		lockdep_off();					\

		BUG_ON(in_nmi() == NMI_MASK);			\

		__preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET);	\

		rcu_nmi_enter();				\

	} while (0)



#define nmi_enter()						\

	do {							\

		__nmi_enter();					\

		lockdep_hardirq_enter();			\

		rcu_nmi_enter();				\

		instrumentation_begin();			\

		ftrace_nmi_enter();				\

		instrumentation_end();				\

	} while (0)



#define __nmi_exit()						\

	do {							\

		BUG_ON(!in_nmi());				\

		__preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET);	\

		printk_nmi_exit();				\

		arch_nmi_exit();				\

		lockdep_on();					\

	} while (0)



#define nmi_exit()						\

	do {							\

		instrumentation_begin();			\

		ftrace_nmi_exit();				\

		instrumentation_end();				\

		lockdep_hardirq_exit();				\

		rcu_nmi_exit();					\

		BUG_ON(!in_nmi());				\

		__preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET);	\

		lockdep_on();					\

		printk_nmi_exit();				\

		arch_nmi_exit();				\

		lockdep_hardirq_exit();				\

		__nmi_exit();					\

	} while (0)



#endif /* LINUX_HARDIRQ_H */

CRA Git | Maintained and supported by SUSTech CRA and CCSE