Commit 5ba553ef authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched/fair: Expose newidle_balance() 



For pick_next_task_fair() it is the newidle balance that requires
dropping the rq->lock; provided we do put_prev_task() early, we can
also detect the condition for doing newidle early.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lwe@gmail.com>
Cc: Valentin Schneider <valentin.schneider@arm.com>
Cc: mingo@kernel.org
Cc: Phil Auld <pauld@redhat.com>
Cc: Julien Desfossez <jdesfossez@digitalocean.com>
Cc: Nishanth Aravamudan <naravamudan@digitalocean.com>
Link: https://lkml.kernel.org/r/9e3eb1859b946f03d7e500453a885725b68957ba.1559129225.git.vpillai@digitalocean.com
parent 03b7fad1

kernel/sched/fair.c

+8 −10
Original line number Original line Diff line number Diff line
@@ -3690,8 +3690,6 @@ static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 
	return cfs_rq->avg.load_avg;

	return cfs_rq->avg.load_avg;

}

}





static int idle_balance(struct rq *this_rq, struct rq_flags *rf);



static inline unsigned long task_util(struct task_struct *p)

static inline unsigned long task_util(struct task_struct *p)

{

{

	return READ_ONCE(p->se.avg.util_avg);

	return READ_ONCE(p->se.avg.util_avg);
@@ -6878,11 +6876,10 @@ done: __maybe_unused; 
	return p;

	return p;





idle:

idle:

	update_misfit_status(NULL, rq);

	new_tasks = newidle_balance(rq, rf);

	new_tasks = idle_balance(rq, rf);





	/*

	/*

	 * Because idle_balance() releases (and re-acquires) rq->lock, it is

	 * Because newidle_balance() releases (and re-acquires) rq->lock, it is

	 * possible for any higher priority task to appear. In that case we

	 * possible for any higher priority task to appear. In that case we

	 * must re-start the pick_next_entity() loop.

	 * must re-start the pick_next_entity() loop.

	 */

	 */
@@ -9045,10 +9042,10 @@ out_one_pinned: 
	ld_moved = 0;

	ld_moved = 0;





	/*

	/*

	 * idle_balance() disregards balance intervals, so we could repeatedly

	 * newidle_balance() disregards balance intervals, so we could

	 * reach this code, which would lead to balance_interval skyrocketting

	 * repeatedly reach this code, which would lead to balance_interval

	 * in a short amount of time. Skip the balance_interval increase logic

	 * skyrocketting in a short amount of time. Skip the balance_interval

	 * to avoid that.

	 * increase logic to avoid that.

	 */

	 */

	if (env.idle == CPU_NEWLY_IDLE)

	if (env.idle == CPU_NEWLY_IDLE)

		goto out;

		goto out;
@@ -9758,7 +9755,7 @@ static inline void nohz_newidle_balance(struct rq *this_rq) { } 
 * idle_balance is called by schedule() if this_cpu is about to become

 * idle_balance is called by schedule() if this_cpu is about to become

 * idle. Attempts to pull tasks from other CPUs.

 * idle. Attempts to pull tasks from other CPUs.

 */

 */

static int idle_balance(struct rq *this_rq, struct rq_flags *rf)

int newidle_balance(struct rq *this_rq, struct rq_flags *rf)

{

{

	unsigned long next_balance = jiffies + HZ;

	unsigned long next_balance = jiffies + HZ;

	int this_cpu = this_rq->cpu;

	int this_cpu = this_rq->cpu;
@@ -9766,6 +9763,7 @@ static int idle_balance(struct rq *this_rq, struct rq_flags *rf) 
	int pulled_task = 0;

	int pulled_task = 0;

	u64 curr_cost = 0;

	u64 curr_cost = 0;





	update_misfit_status(NULL, this_rq);

	/*

	/*

	 * We must set idle_stamp _before_ calling idle_balance(), such that we

	 * We must set idle_stamp _before_ calling idle_balance(), such that we

	 * measure the duration of idle_balance() as idle time.

	 * measure the duration of idle_balance() as idle time.

kernel/sched/sched.h

+4 −0
Original line number Original line Diff line number Diff line
@@ -1445,10 +1445,14 @@ static inline void unregister_sched_domain_sysctl(void) 
}

}

#endif

#endif





extern int newidle_balance(struct rq *this_rq, struct rq_flags *rf);



#else

#else





static inline void sched_ttwu_pending(void) { }

static inline void sched_ttwu_pending(void) { }





static inline int newidle_balance(struct rq *this_rq, struct rq_flags *rf) { return 0; }



#endif /* CONFIG_SMP */

#endif /* CONFIG_SMP */





#include "stats.h"

#include "stats.h"

CRA Git | Maintained and supported by SUSTech CRA and CCSE