Commit 37dc6b50 authored by Preeti U Murthy's avatar Preeti U Murthy Committed by Ingo Molnar
Browse files

sched: Remove unnecessary iteration over sched domains to update nr_busy_cpus 



nr_busy_cpus parameter is used by nohz_kick_needed() to find out the
number of busy cpus in a sched domain which has SD_SHARE_PKG_RESOURCES
flag set.  Therefore instead of updating nr_busy_cpus at every level
of sched domain, since it is irrelevant, we can update this parameter
only at the parent domain of the sd which has this flag set. Introduce
a per-cpu parameter sd_busy which represents this parent domain.

In nohz_kick_needed() we directly query the nr_busy_cpus parameter
associated with the groups of sd_busy.

By associating sd_busy with the highest domain which has
SD_SHARE_PKG_RESOURCES flag set, we cover all lower level domains
which could have this flag set and trigger nohz_idle_balancing if any
of the levels have more than one busy cpu.

sd_busy is irrelevant for asymmetric load balancing. However sd_asym
has been introduced to represent the highest sched domain which has
SD_ASYM_PACKING flag set so that it can be queried directly when
required.

While we are at it, we might as well change the nohz_idle parameter to
be updated at the sd_busy domain level alone and not the base domain
level of a CPU.  This will unify the concept of busy cpus at just one
level of sched domain where it is currently used.

Signed-off-by: default avatarPreeti U <Murthy&lt;preeti@linux.vnet.ibm.com>
Signed-off-by: default avatarPeter Zijlstra <peterz@infradead.org>
Cc: svaidy@linux.vnet.ibm.com
Cc: vincent.guittot@linaro.org
Cc: bitbucket@online.de
Cc: benh@kernel.crashing.org
Cc: anton@samba.org
Cc: Morten.Rasmussen@arm.com
Cc: pjt@google.com
Cc: peterz@infradead.org
Cc: mikey@neuling.org
Link: http://lkml.kernel.org/r/20131030031252.23426.4417.stgit@preeti.in.ibm.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 2042abe7

kernel/sched/core.c

+6 −0
Original line number Diff line number Diff line
@@ -4883,6 +4883,8 @@ DEFINE_PER_CPU(struct sched_domain *, sd_llc); 
DEFINE_PER_CPU(int, sd_llc_size);

DEFINE_PER_CPU(int, sd_llc_id);

DEFINE_PER_CPU(struct sched_domain *, sd_numa);

DEFINE_PER_CPU(struct sched_domain *, sd_busy);

DEFINE_PER_CPU(struct sched_domain *, sd_asym);



static void update_top_cache_domain(int cpu)

{
@@ -4894,6 +4896,7 @@ static void update_top_cache_domain(int cpu) 
	if (sd) {

		id = cpumask_first(sched_domain_span(sd));

		size = cpumask_weight(sched_domain_span(sd));

		rcu_assign_pointer(per_cpu(sd_busy, cpu), sd->parent);

	}



	rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
@@ -4902,6 +4905,9 @@ static void update_top_cache_domain(int cpu) 


	sd = lowest_flag_domain(cpu, SD_NUMA);

	rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);



	sd = highest_flag_domain(cpu, SD_ASYM_PACKING);

	rcu_assign_pointer(per_cpu(sd_asym, cpu), sd);

}



/*

kernel/sched/fair.c

+20 −18
Original line number Diff line number Diff line
@@ -6534,15 +6534,15 @@ static inline void nohz_balance_exit_idle(int cpu) 
static inline void set_cpu_sd_state_busy(void)

{

	struct sched_domain *sd;

	int cpu = smp_processor_id();



	rcu_read_lock();

	sd = rcu_dereference_check_sched_domain(this_rq()->sd);

	sd = rcu_dereference(per_cpu(sd_busy, cpu));



	if (!sd || !sd->nohz_idle)

		goto unlock;

	sd->nohz_idle = 0;



	for (; sd; sd = sd->parent)

	atomic_inc(&sd->groups->sgp->nr_busy_cpus);

unlock:

	rcu_read_unlock();
@@ -6551,15 +6551,15 @@ unlock: 
void set_cpu_sd_state_idle(void)

{

	struct sched_domain *sd;

	int cpu = smp_processor_id();



	rcu_read_lock();

	sd = rcu_dereference_check_sched_domain(this_rq()->sd);

	sd = rcu_dereference(per_cpu(sd_busy, cpu));



	if (!sd || sd->nohz_idle)

		goto unlock;

	sd->nohz_idle = 1;



	for (; sd; sd = sd->parent)

	atomic_dec(&sd->groups->sgp->nr_busy_cpus);

unlock:

	rcu_read_unlock();
@@ -6767,6 +6767,8 @@ static inline int nohz_kick_needed(struct rq *rq, int cpu) 
{

	unsigned long now = jiffies;

	struct sched_domain *sd;

	struct sched_group_power *sgp;

	int nr_busy;



	if (unlikely(idle_cpu(cpu)))

		return 0;
@@ -6792,22 +6794,22 @@ static inline int nohz_kick_needed(struct rq *rq, int cpu) 
		goto need_kick;



	rcu_read_lock();

	for_each_domain(cpu, sd) {

		struct sched_group *sg = sd->groups;

		struct sched_group_power *sgp = sg->sgp;

		int nr_busy = atomic_read(&sgp->nr_busy_cpus);

	sd = rcu_dereference(per_cpu(sd_busy, cpu));



	if (sd) {

		sgp = sd->groups->sgp;

		nr_busy = atomic_read(&sgp->nr_busy_cpus);



		if (sd->flags & SD_SHARE_PKG_RESOURCES && nr_busy > 1)

		if (nr_busy > 1)

			goto need_kick_unlock;

	}



	sd = rcu_dereference(per_cpu(sd_asym, cpu));



		if (sd->flags & SD_ASYM_PACKING

		    && (cpumask_first_and(nohz.idle_cpus_mask,

	if (sd && (cpumask_first_and(nohz.idle_cpus_mask,

				  sched_domain_span(sd)) < cpu))

		goto need_kick_unlock;



		if (!(sd->flags & (SD_SHARE_PKG_RESOURCES | SD_ASYM_PACKING)))

			break;

	}

	rcu_read_unlock();

	return 0;

kernel/sched/sched.h

+2 −0
Original line number Diff line number Diff line
@@ -623,6 +623,8 @@ DECLARE_PER_CPU(struct sched_domain *, sd_llc); 
DECLARE_PER_CPU(int, sd_llc_size);

DECLARE_PER_CPU(int, sd_llc_id);

DECLARE_PER_CPU(struct sched_domain *, sd_numa);

DECLARE_PER_CPU(struct sched_domain *, sd_busy);

DECLARE_PER_CPU(struct sched_domain *, sd_asym);



struct sched_group_power {

	atomic_t ref;

CRA Git | Maintained and supported by SUSTech CRA and CCSE