Commit 90b5363a authored by Peter Zijlstra (Intel)'s avatar Peter Zijlstra (Intel) Committed by Thomas Gleixner
Browse files

sched: Clean up scheduler_ipi() 



The scheduler IPI has grown weird and wonderful over the years, time
for spring cleaning.

Move all the non-trivial stuff out of it and into a regular smp function
call IPI. This then reduces the schedule_ipi() to most of it's former NOP
glory and ensures to keep the interrupt vector lean and mean.

Aside of that avoiding the full irq_enter() in the x86 IPI implementation
is incorrect as scheduler_ipi() can be instrumented. To work around that
scheduler_ipi() had an irq_enter/exit() hack when heavy work was
pending. This is gone now.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Reviewed-by: default avatarAlexandre Chartre <alexandre.chartre@oracle.com>
Link: https://lkml.kernel.org/r/20200505134058.361859938@linutronix.de
parent b1d1779e

kernel/sched/core.c

+30 −34
Original line number Diff line number Diff line
@@ -219,6 +219,13 @@ void update_rq_clock(struct rq *rq) 
	update_rq_clock_task(rq, delta);

}



static inline void

rq_csd_init(struct rq *rq, call_single_data_t *csd, smp_call_func_t func)

{

	csd->flags = 0;

	csd->func = func;

	csd->info = rq;

}



#ifdef CONFIG_SCHED_HRTICK

/*
@@ -314,16 +321,14 @@ void hrtick_start(struct rq *rq, u64 delay) 
	hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),

		      HRTIMER_MODE_REL_PINNED_HARD);

}



#endif /* CONFIG_SMP */



static void hrtick_rq_init(struct rq *rq)

{

#ifdef CONFIG_SMP

	rq->hrtick_csd.flags = 0;

	rq->hrtick_csd.func = __hrtick_start;

	rq->hrtick_csd.info = rq;

	rq_csd_init(rq, &rq->hrtick_csd, __hrtick_start);

#endif



	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);

	rq->hrtick_timer.function = hrtick;

}
@@ -650,6 +655,16 @@ static inline bool got_nohz_idle_kick(void) 
	return false;

}



static void nohz_csd_func(void *info)

{

	struct rq *rq = info;



	if (got_nohz_idle_kick()) {

		rq->idle_balance = 1;

		raise_softirq_irqoff(SCHED_SOFTIRQ);

	}

}



#else /* CONFIG_NO_HZ_COMMON */



static inline bool got_nohz_idle_kick(void)
@@ -2292,6 +2307,11 @@ void sched_ttwu_pending(void) 
	rq_unlock_irqrestore(rq, &rf);

}



static void wake_csd_func(void *info)

{

	sched_ttwu_pending();

}



void scheduler_ipi(void)

{

	/*
@@ -2300,34 +2320,6 @@ void scheduler_ipi(void) 
	 * this IPI.

	 */

	preempt_fold_need_resched();



	if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())

		return;



	/*

	 * Not all reschedule IPI handlers call irq_enter/irq_exit, since

	 * traditionally all their work was done from the interrupt return

	 * path. Now that we actually do some work, we need to make sure

	 * we do call them.

	 *

	 * Some archs already do call them, luckily irq_enter/exit nest

	 * properly.

	 *

	 * Arguably we should visit all archs and update all handlers,

	 * however a fair share of IPIs are still resched only so this would

	 * somewhat pessimize the simple resched case.

	 */

	irq_enter();

	sched_ttwu_pending();



	/*

	 * Check if someone kicked us for doing the nohz idle load balance.

	 */

	if (unlikely(got_nohz_idle_kick())) {

		this_rq()->idle_balance = 1;

		raise_softirq_irqoff(SCHED_SOFTIRQ);

	}

	irq_exit();

}



static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
@@ -2336,9 +2328,9 @@ static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags) 


	p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);



	if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) {

	if (llist_add(&p->wake_entry, &rq->wake_list)) {

		if (!set_nr_if_polling(rq->idle))

			smp_send_reschedule(cpu);

			smp_call_function_single_async(cpu, &rq->wake_csd);

		else

			trace_sched_wake_idle_without_ipi(cpu);

	}
@@ -6693,12 +6685,16 @@ void __init sched_init(void) 
		rq->avg_idle = 2*sysctl_sched_migration_cost;

		rq->max_idle_balance_cost = sysctl_sched_migration_cost;



		rq_csd_init(rq, &rq->wake_csd, wake_csd_func);



		INIT_LIST_HEAD(&rq->cfs_tasks);



		rq_attach_root(rq, &def_root_domain);

#ifdef CONFIG_NO_HZ_COMMON

		rq->last_blocked_load_update_tick = jiffies;

		atomic_set(&rq->nohz_flags, 0);



		rq_csd_init(rq, &rq->nohz_csd, nohz_csd_func);

#endif

#endif /* CONFIG_SMP */

		hrtick_rq_init(rq);

kernel/sched/fair.c

+2 −3
Original line number Diff line number Diff line
@@ -10000,12 +10000,11 @@ static void kick_ilb(unsigned int flags) 
		return;



	/*

	 * Use smp_send_reschedule() instead of resched_cpu().

	 * This way we generate a sched IPI on the target CPU which

	 * This way we generate an IPI on the target CPU which

	 * is idle. And the softirq performing nohz idle load balance

	 * will be run before returning from the IPI.

	 */

	smp_send_reschedule(ilb_cpu);

	smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd);

}



/*

kernel/sched/sched.h

+4 −2
Original line number Diff line number Diff line
@@ -889,6 +889,7 @@ struct rq { 
#ifdef CONFIG_SMP

	unsigned long		last_blocked_load_update_tick;

	unsigned int		has_blocked_load;

	call_single_data_t	nohz_csd;

#endif /* CONFIG_SMP */

	unsigned int		nohz_tick_stopped;

	atomic_t		nohz_flags;
@@ -978,7 +979,7 @@ struct rq { 


	/* This is used to determine avg_idle's max value */

	u64			max_idle_balance_cost;

#endif

#endif /* CONFIG_SMP */



#ifdef CONFIG_IRQ_TIME_ACCOUNTING

	u64			prev_irq_time;
@@ -1020,6 +1021,7 @@ struct rq { 
#endif



#ifdef CONFIG_SMP

	call_single_data_t	wake_csd;

	struct llist_head	wake_list;

#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE