Commit 99bd5e2f authored by Suresh Siddha's avatar Suresh Siddha Committed by Ingo Molnar
Browse files

sched: Fix select_idle_sibling() logic in select_task_rq_fair() 



Issues in the current select_idle_sibling() logic in select_task_rq_fair()
in the context of a task wake-up:

a) Once we select the idle sibling, we use that domain (spanning the cpu that
   the task is currently woken-up and the idle sibling that we found) in our
   wake_affine() decisions. This domain is completely different from the
   domain(we are supposed to use) that spans the cpu that the task currently
   woken-up and the cpu where the task previously ran.

b) We do select_idle_sibling() check only for the cpu that the task is
   currently woken-up on. If select_task_rq_fair() selects the previously run
   cpu for waking the task, doing a select_idle_sibling() check
   for that cpu also helps and we don't do this currently.

c) In the scenarios where the cpu that the task is woken-up is busy but
   with its HT siblings are idle, we are selecting the task be woken-up
   on the idle HT sibling instead of a core that it previously ran
   and currently completely idle. i.e., we are not taking decisions based on
   wake_affine() but directly selecting an idle sibling that can cause
   an imbalance at the SMT/MC level which will be later corrected by the
   periodic load balancer.

Fix this by first going through the load imbalance calculations using
wake_affine() and once we make a decision of woken-up cpu vs previously-ran cpu,
then choose a possible idle sibling for waking up the task on.

Signed-off-by: default avatarSuresh Siddha <suresh.b.siddha@intel.com>
Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <1270079265.7835.8.camel@sbs-t61.sc.intel.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 669c55e9

kernel/sched_fair.c

+40 −42
Original line number Diff line number Diff line
@@ -1375,31 +1375,50 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) 
/*

 * Try and locate an idle CPU in the sched_domain.

 */

static int

select_idle_sibling(struct task_struct *p, struct sched_domain *sd, int target)

static int select_idle_sibling(struct task_struct *p, int target)

{

	int cpu = smp_processor_id();

	int prev_cpu = task_cpu(p);

	struct sched_domain *sd;

	int i;



	/*

	 * If this domain spans both cpu and prev_cpu (see the SD_WAKE_AFFINE

	 * test in select_task_rq_fair) and the prev_cpu is idle then that's

	 * always a better target than the current cpu.

	 * If the task is going to be woken-up on this cpu and if it is

	 * already idle, then it is the right target.

	 */

	if (target == cpu && idle_cpu(cpu))

		return cpu;



	/*

	 * If the task is going to be woken-up on the cpu where it previously

	 * ran and if it is currently idle, then it the right target.

	 */

	if (target == cpu && !cpu_rq(prev_cpu)->cfs.nr_running)

	if (target == prev_cpu && idle_cpu(prev_cpu))

		return prev_cpu;



	/*

	 * Otherwise, iterate the domain and find an elegible idle cpu.

	 * Otherwise, iterate the domains and find an elegible idle cpu.

	 */

	for_each_domain(target, sd) {

		if (!(sd->flags & SD_SHARE_PKG_RESOURCES))

			break;



		for_each_cpu_and(i, sched_domain_span(sd), &p->cpus_allowed) {

		if (!cpu_rq(i)->cfs.nr_running) {

			if (idle_cpu(i)) {

				target = i;

				break;

			}

		}



		/*

		 * Lets stop looking for an idle sibling when we reached

		 * the domain that spans the current cpu and prev_cpu.

		 */

		if (cpumask_test_cpu(cpu, sched_domain_span(sd)) &&

		    cpumask_test_cpu(prev_cpu, sched_domain_span(sd)))

			break;

	}



	return target;

}
@@ -1421,7 +1440,7 @@ select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_ 
	int cpu = smp_processor_id();

	int prev_cpu = task_cpu(p);

	int new_cpu = cpu;

	int want_affine = 0, cpu_idle = !current->pid;

	int want_affine = 0;

	int want_sd = 1;

	int sync = wake_flags & WF_SYNC;
@@ -1459,37 +1478,14 @@ select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_ 
				want_sd = 0;

		}



		/*

		 * While iterating the domains looking for a spanning

		 * WAKE_AFFINE domain, adjust the affine target to any idle cpu

		 * in cache sharing domains along the way.

		 */

		if (want_affine) {

			int target = -1;



		/*

		 * If both cpu and prev_cpu are part of this domain,

		 * cpu is a valid SD_WAKE_AFFINE target.

		 */

			if (cpumask_test_cpu(prev_cpu, sched_domain_span(tmp)))

				target = cpu;



			/*

			 * If there's an idle sibling in this domain, make that

			 * the wake_affine target instead of the current cpu.

			 */

			if (!cpu_idle && tmp->flags & SD_SHARE_PKG_RESOURCES)

				target = select_idle_sibling(p, tmp, target);



			if (target >= 0) {

				if (tmp->flags & SD_WAKE_AFFINE) {

		if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&

		    cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {

			affine_sd = tmp;

			want_affine = 0;

					if (target != cpu)

						cpu_idle = 1;

				}

				cpu = target;

			}

		}



		if (!want_sd && !want_affine)
@@ -1520,8 +1516,10 @@ select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_ 
#endif



	if (affine_sd) {

		if (cpu_idle || cpu == prev_cpu || wake_affine(affine_sd, p, sync))

			return cpu;

		if (cpu == prev_cpu || wake_affine(affine_sd, p, sync))

			return select_idle_sibling(p, cpu);

		else

			return select_idle_sibling(p, prev_cpu);

	}



	while (sd) {

CRA Git | Maintained and supported by SUSTech CRA and CCSE