entry: Provide generic interrupt entry/exit code (a5497bab) · Commits · 戴 / test

include/linux/entry-common.h

+62 −0

Original line number	Diff line number	Diff line
		@@ -307,4 +307,66 @@ void irqentry_enter_from_user_mode(struct pt_regs *regs);
		*/
		void irqentry_exit_to_user_mode(struct pt_regs *regs);

		#ifndef irqentry_state
		typedef struct irqentry_state {
		bool exit_rcu;
		} irqentry_state_t;
		#endif

		/**
		* irqentry_enter - Handle state tracking on ordinary interrupt entries
		* @regs: Pointer to pt_regs of interrupted context
		*
		* Invokes:
		* - lockdep irqflag state tracking as low level ASM entry disabled
		* interrupts.
		*
		* - Context tracking if the exception hit user mode.
		*
		* - The hardirq tracer to keep the state consistent as low level ASM
		* entry disabled interrupts.
		*
		* As a precondition, this requires that the entry came from user mode,
		* idle, or a kernel context in which RCU is watching.
		*
		* For kernel mode entries RCU handling is done conditional. If RCU is
		* watching then the only RCU requirement is to check whether the tick has
		* to be restarted. If RCU is not watching then rcu_irq_enter() has to be
		* invoked on entry and rcu_irq_exit() on exit.
		*
		* Avoiding the rcu_irq_enter/exit() calls is an optimization but also
		* solves the problem of kernel mode pagefaults which can schedule, which
		* is not possible after invoking rcu_irq_enter() without undoing it.
		*
		* For user mode entries irqentry_enter_from_user_mode() is invoked to
		* establish the proper context for NOHZ_FULL. Otherwise scheduling on exit
		* would not be possible.
		*
		* Returns: An opaque object that must be passed to idtentry_exit()
		*/
		irqentry_state_t noinstr irqentry_enter(struct pt_regs *regs);

		/**
		* irqentry_exit_cond_resched - Conditionally reschedule on return from interrupt
		*
		* Conditional reschedule with additional sanity checks.
		*/
		void irqentry_exit_cond_resched(void);

		/**
		* irqentry_exit - Handle return from exception that used irqentry_enter()
		* @regs: Pointer to pt_regs (exception entry regs)
		* @state: Return value from matching call to irqentry_enter()
		*
		* Depending on the return target (kernel/user) this runs the necessary
		* preemption and work checks if possible and reguired and returns to
		* the caller with interrupts disabled and no further work pending.
		*
		* This is the last action before returning to the low level ASM code which
		* just needs to return to the appropriate context.
		*
		* Counterpart to irqentry_enter().
		*/
		void noinstr irqentry_exit(struct pt_regs *regs, irqentry_state_t state);

		#endif

kernel/entry/common.c

+117 −0

Original line number	Diff line number	Diff line
		@@ -255,3 +255,120 @@ noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs)
		instrumentation_end();
		exit_to_user_mode();
		}

		irqentry_state_t noinstr irqentry_enter(struct pt_regs *regs)
		{
		irqentry_state_t ret = {
		.exit_rcu = false,
		};

		if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);
		return ret;
		}

		/*
		* If this entry hit the idle task invoke rcu_irq_enter() whether
		* RCU is watching or not.
		*
		* Interupts can nest when the first interrupt invokes softirq
		* processing on return which enables interrupts.
		*
		* Scheduler ticks in the idle task can mark quiescent state and
		* terminate a grace period, if and only if the timer interrupt is
		* not nested into another interrupt.
		*
		* Checking for __rcu_is_watching() here would prevent the nesting
		* interrupt to invoke rcu_irq_enter(). If that nested interrupt is
		* the tick then rcu_flavor_sched_clock_irq() would wrongfully
		* assume that it is the first interupt and eventually claim
		* quiescient state and end grace periods prematurely.
		*
		* Unconditionally invoke rcu_irq_enter() so RCU state stays
		* consistent.
		*
		* TINY_RCU does not support EQS, so let the compiler eliminate
		* this part when enabled.
		*/
		if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
		/*
		* If RCU is not watching then the same careful
		* sequence vs. lockdep and tracing is required
		* as in irq_enter_from_user_mode().
		*/
		lockdep_hardirqs_off(CALLER_ADDR0);
		rcu_irq_enter();
		instrumentation_begin();
		trace_hardirqs_off_finish();
		instrumentation_end();

		ret.exit_rcu = true;
		return ret;
		}

		/*
		* If RCU is watching then RCU only wants to check whether it needs
		* to restart the tick in NOHZ mode. rcu_irq_enter_check_tick()
		* already contains a warning when RCU is not watching, so no point
		* in having another one here.
		*/
		instrumentation_begin();
		rcu_irq_enter_check_tick();
		/* Use the combo lockdep/tracing function */
		trace_hardirqs_off();
		instrumentation_end();

		return ret;
		}

		void irqentry_exit_cond_resched(void)
		{
		if (!preempt_count()) {
		/* Sanity check RCU and thread stack */
		rcu_irq_exit_check_preempt();
		if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
		WARN_ON_ONCE(!on_thread_stack());
		if (need_resched())
		preempt_schedule_irq();
		}
		}

		void noinstr irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
		{
		lockdep_assert_irqs_disabled();

		/* Check whether this returns to user mode */
		if (user_mode(regs)) {
		irqentry_exit_to_user_mode(regs);
		} else if (!regs_irqs_disabled(regs)) {
		/*
		* If RCU was not watching on entry this needs to be done
		* carefully and needs the same ordering of lockdep/tracing
		* and RCU as the return to user mode path.
		*/
		if (state.exit_rcu) {
		instrumentation_begin();
		/* Tell the tracer that IRET will enable interrupts */
		trace_hardirqs_on_prepare();
		lockdep_hardirqs_on_prepare(CALLER_ADDR0);
		instrumentation_end();
		rcu_irq_exit();
		lockdep_hardirqs_on(CALLER_ADDR0);
		return;
		}

		instrumentation_begin();
		if (IS_ENABLED(CONFIG_PREEMPTION))
		irqentry_exit_cond_resched();
		/* Covers both tracing and lockdep */
		trace_hardirqs_on();
		instrumentation_end();
		} else {
		/*
		* IRQ flags state is correct already. Just tell RCU if it
		* was not watching on entry.
		*/
		if (state.exit_rcu)
		rcu_irq_exit();
		}
		}

Admin message