Merge branch 'locking/nmi' into x86/entry (f87032ae) · Commits · 戴 / test

arch/x86/entry/common.c

+34 −0

Original line number	Diff line number	Diff line
		@@ -198,6 +198,40 @@ SYSCALL_DEFINE0(ni_syscall)
		return -ENOSYS;
		}

		noinstr bool idtentry_enter_nmi(struct pt_regs *regs)
		{
		bool irq_state = lockdep_hardirqs_enabled();

		__nmi_enter();
		lockdep_hardirqs_off(CALLER_ADDR0);
		lockdep_hardirq_enter();
		rcu_nmi_enter();

		instrumentation_begin();
		trace_hardirqs_off_finish();
		ftrace_nmi_enter();
		instrumentation_end();

		return irq_state;
		}

		noinstr void idtentry_exit_nmi(struct pt_regs *regs, bool restore)
		{
		instrumentation_begin();
		ftrace_nmi_exit();
		if (restore) {
		trace_hardirqs_on_prepare();
		lockdep_hardirqs_on_prepare(CALLER_ADDR0);
		}
		instrumentation_end();

		rcu_nmi_exit();
		lockdep_hardirq_exit();
		if (restore)
		lockdep_hardirqs_on(CALLER_ADDR0);
		__nmi_exit();
		}

		#ifdef CONFIG_XEN_PV
		#ifndef CONFIG_PREEMPTION
		/*

+3 −0

Original line number	Diff line number	Diff line
		@@ -11,6 +11,9 @@

		#include <asm/irq_stack.h>

		bool idtentry_enter_nmi(struct pt_regs *regs);
		void idtentry_exit_nmi(struct pt_regs *regs, bool irq_state);

		/**
		* DECLARE_IDTENTRY - Declare functions for simple IDT entry points
		* No error code pushed by hardware

+4 −5

Original line number	Diff line number	Diff line
		@@ -330,7 +330,6 @@ static noinstr void default_do_nmi(struct pt_regs *regs)
		__this_cpu_write(last_nmi_rip, regs->ip);

		instrumentation_begin();
		trace_hardirqs_off_finish();

		handled = nmi_handle(NMI_LOCAL, regs);
		__this_cpu_add(nmi_stats.normal, handled);
		@@ -417,8 +416,6 @@ static noinstr void default_do_nmi(struct pt_regs *regs)
		unknown_nmi_error(reason, regs);

		out:
		if (regs->flags & X86_EFLAGS_IF)
		trace_hardirqs_on_prepare();
		instrumentation_end();
		}

		@@ -478,6 +475,8 @@ static DEFINE_PER_CPU(unsigned long, nmi_dr7);

		DEFINE_IDTENTRY_RAW(exc_nmi)
		{
		bool irq_state;

		if (IS_ENABLED(CONFIG_SMP) && arch_cpu_is_offline(smp_processor_id()))
		return;

		@@ -491,14 +490,14 @@ nmi_restart:

		this_cpu_write(nmi_dr7, local_db_save());

		nmi_enter();
		irq_state = idtentry_enter_nmi(regs);

		inc_irq_stat(__nmi_count);

		if (!ignore_nmis)
		default_do_nmi(regs);

		nmi_exit();
		idtentry_exit_nmi(regs, irq_state);

		local_db_restore(this_cpu_read(nmi_dr7));

+6 −11

Original line number	Diff line number	Diff line
		@@ -405,7 +405,7 @@ DEFINE_IDTENTRY_DF(exc_double_fault)
		}
		#endif

		nmi_enter();
		idtentry_enter_nmi(regs);
		instrumentation_begin();
		notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);

		@@ -651,15 +651,12 @@ DEFINE_IDTENTRY_RAW(exc_int3)
		instrumentation_end();
		irqentry_exit_to_user_mode(regs);
		} else {
		nmi_enter();
		bool irq_state = idtentry_enter_nmi(regs);
		instrumentation_begin();
		trace_hardirqs_off_finish();
		if (!do_int3(regs))
		die("int3", regs, 0);
		if (regs->flags & X86_EFLAGS_IF)
		trace_hardirqs_on_prepare();
		instrumentation_end();
		nmi_exit();
		idtentry_exit_nmi(regs, irq_state);
		}
		}

		@@ -867,9 +864,8 @@ out:
		static __always_inline void exc_debug_kernel(struct pt_regs *regs,
		unsigned long dr6)
		{
		nmi_enter();
		bool irq_state = idtentry_enter_nmi(regs);
		instrumentation_begin();
		trace_hardirqs_off_finish();

		/*
		* If something gets miswired and we end up here for a user mode
		@@ -886,10 +882,8 @@ static __always_inline void exc_debug_kernel(struct pt_regs *regs,

		handle_debug(regs, dr6, false);

		if (regs->flags & X86_EFLAGS_IF)
		trace_hardirqs_on_prepare();
		instrumentation_end();
		nmi_exit();
		idtentry_exit_nmi(regs, irq_state);
		}

		static __always_inline void exc_debug_user(struct pt_regs *regs,
		@@ -905,6 +899,7 @@ static __always_inline void exc_debug_user(struct pt_regs *regs,
		instrumentation_begin();

		handle_debug(regs, dr6, true);

		instrumentation_end();
		irqentry_exit_to_user_mode(regs);
		}

+19 −9

Original line number	Diff line number	Diff line
		@@ -111,32 +111,42 @@ extern void rcu_nmi_exit(void);
		/*
		* nmi_enter() can nest up to 15 times; see NMI_BITS.
		*/
		#define nmi_enter() \
		#define __nmi_enter() \
		do { \
		lockdep_off(); \
		arch_nmi_enter(); \
		printk_nmi_enter(); \
		lockdep_off(); \
		BUG_ON(in_nmi() == NMI_MASK); \
		__preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \
		rcu_nmi_enter(); \
		} while (0)

		#define nmi_enter() \
		do { \
		__nmi_enter(); \
		lockdep_hardirq_enter(); \
		rcu_nmi_enter(); \
		instrumentation_begin(); \
		ftrace_nmi_enter(); \
		instrumentation_end(); \
		} while (0)

		#define __nmi_exit() \
		do { \
		BUG_ON(!in_nmi()); \
		__preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \
		printk_nmi_exit(); \
		arch_nmi_exit(); \
		lockdep_on(); \
		} while (0)

		#define nmi_exit() \
		do { \
		instrumentation_begin(); \
		ftrace_nmi_exit(); \
		instrumentation_end(); \
		lockdep_hardirq_exit(); \
		rcu_nmi_exit(); \
		BUG_ON(!in_nmi()); \
		__preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \
		lockdep_on(); \
		printk_nmi_exit(); \
		arch_nmi_exit(); \
		lockdep_hardirq_exit(); \
		__nmi_exit(); \
		} while (0)

		#endif /* LINUX_HARDIRQ_H */