Commit 1c1b8a7b authored by Dietmar Eggemann's avatar Dietmar Eggemann Committed by Ingo Molnar
Browse files

sched/fair: Replace source_load() & target_load() with weighted_cpuload() 



With LB_BIAS disabled, source_load() & target_load() return
weighted_cpuload(). Replace both with calls to weighted_cpuload().

The function to obtain the load index (sd->*_idx) for an sd,
get_sd_load_idx(), can be removed as well.

Finally, get rid of the sched feature LB_BIAS.

Signed-off-by: default avatarDietmar Eggemann <dietmar.eggemann@arm.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarRik van Riel <riel@surriel.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Morten Rasmussen <morten.rasmussen@arm.com>
Cc: Patrick Bellasi <patrick.bellasi@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Quentin Perret <quentin.perret@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Valentin Schneider <valentin.schneider@arm.com>
Cc: Vincent Guittot <vincent.guittot@linaro.org>
Link: https://lkml.kernel.org/r/20190527062116.11512-3-dietmar.eggemann@arm.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 5e83eafb

kernel/sched/fair.c

+4 −86
Original line number Original line Diff line number Diff line
@@ -1467,8 +1467,6 @@ bool should_numa_migrate_memory(struct task_struct *p, struct page * page, 
}

}





static unsigned long weighted_cpuload(struct rq *rq);

static unsigned long weighted_cpuload(struct rq *rq);

static unsigned long source_load(int cpu, int type);

static unsigned long target_load(int cpu, int type);





/* Cached statistics for all CPUs within a node */

/* Cached statistics for all CPUs within a node */

struct numa_stats {

struct numa_stats {
@@ -5333,45 +5331,11 @@ static struct { 




#endif /* CONFIG_NO_HZ_COMMON */

#endif /* CONFIG_NO_HZ_COMMON */





/* Used instead of source_load when we know the type == 0 */

static unsigned long weighted_cpuload(struct rq *rq)

static unsigned long weighted_cpuload(struct rq *rq)

{

{

	return cfs_rq_runnable_load_avg(&rq->cfs);

	return cfs_rq_runnable_load_avg(&rq->cfs);

}

}





/*

 * Return a low guess at the load of a migration-source CPU weighted

 * according to the scheduling class and "nice" value.

 *

 * We want to under-estimate the load of migration sources, to

 * balance conservatively.

 */

static unsigned long source_load(int cpu, int type)

{

	struct rq *rq = cpu_rq(cpu);

	unsigned long total = weighted_cpuload(rq);



	if (type == 0 || !sched_feat(LB_BIAS))

		return total;



	return min(rq->cpu_load[type-1], total);

}



/*

 * Return a high guess at the load of a migration-target CPU weighted

 * according to the scheduling class and "nice" value.

 */

static unsigned long target_load(int cpu, int type)

{

	struct rq *rq = cpu_rq(cpu);

	unsigned long total = weighted_cpuload(rq);



	if (type == 0 || !sched_feat(LB_BIAS))

		return total;



	return max(rq->cpu_load[type-1], total);

}



static unsigned long capacity_of(int cpu)

static unsigned long capacity_of(int cpu)

{

{

	return cpu_rq(cpu)->cpu_capacity;

	return cpu_rq(cpu)->cpu_capacity;
@@ -5479,7 +5443,7 @@ wake_affine_weight(struct sched_domain *sd, struct task_struct *p, 
	s64 this_eff_load, prev_eff_load;

	s64 this_eff_load, prev_eff_load;

	unsigned long task_load;

	unsigned long task_load;





	this_eff_load = target_load(this_cpu, sd->wake_idx);

	this_eff_load = weighted_cpuload(cpu_rq(this_cpu));





	if (sync) {

	if (sync) {

		unsigned long current_load = task_h_load(current);

		unsigned long current_load = task_h_load(current);
@@ -5497,7 +5461,7 @@ wake_affine_weight(struct sched_domain *sd, struct task_struct *p, 
		this_eff_load *= 100;

		this_eff_load *= 100;

	this_eff_load *= capacity_of(prev_cpu);

	this_eff_load *= capacity_of(prev_cpu);





	prev_eff_load = source_load(prev_cpu, sd->wake_idx);

	prev_eff_load = weighted_cpuload(cpu_rq(prev_cpu));

	prev_eff_load -= task_load;

	prev_eff_load -= task_load;

	if (sched_feat(WA_BIAS))

	if (sched_feat(WA_BIAS))

		prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2;

		prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2;
@@ -5558,14 +5522,10 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, 
	unsigned long this_runnable_load = ULONG_MAX;

	unsigned long this_runnable_load = ULONG_MAX;

	unsigned long min_avg_load = ULONG_MAX, this_avg_load = ULONG_MAX;

	unsigned long min_avg_load = ULONG_MAX, this_avg_load = ULONG_MAX;

	unsigned long most_spare = 0, this_spare = 0;

	unsigned long most_spare = 0, this_spare = 0;

	int load_idx = sd->forkexec_idx;

	int imbalance_scale = 100 + (sd->imbalance_pct-100)/2;

	int imbalance_scale = 100 + (sd->imbalance_pct-100)/2;

	unsigned long imbalance = scale_load_down(NICE_0_LOAD) *

	unsigned long imbalance = scale_load_down(NICE_0_LOAD) *

				(sd->imbalance_pct-100) / 100;

				(sd->imbalance_pct-100) / 100;





	if (sd_flag & SD_BALANCE_WAKE)

		load_idx = sd->wake_idx;



	do {

	do {

		unsigned long load, avg_load, runnable_load;

		unsigned long load, avg_load, runnable_load;

		unsigned long spare_cap, max_spare_cap;

		unsigned long spare_cap, max_spare_cap;
@@ -5589,12 +5549,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, 
		max_spare_cap = 0;

		max_spare_cap = 0;





		for_each_cpu(i, sched_group_span(group)) {

		for_each_cpu(i, sched_group_span(group)) {

			/* Bias balancing toward CPUs of our domain */

			load = weighted_cpuload(cpu_rq(i));

			if (local_group)

				load = source_load(i, load_idx);

			else

				load = target_load(i, load_idx);



			runnable_load += load;

			runnable_load += load;





			avg_load += cfs_rq_load_avg(&cpu_rq(i)->cfs);

			avg_load += cfs_rq_load_avg(&cpu_rq(i)->cfs);
@@ -7676,34 +7631,6 @@ static inline void init_sd_lb_stats(struct sd_lb_stats *sds) 
	};

	};

}

}





/**

 * get_sd_load_idx - Obtain the load index for a given sched domain.

 * @sd: The sched_domain whose load_idx is to be obtained.

 * @idle: The idle status of the CPU for whose sd load_idx is obtained.

 *

 * Return: The load index.

 */

static inline int get_sd_load_idx(struct sched_domain *sd,

					enum cpu_idle_type idle)

{

	int load_idx;



	switch (idle) {

	case CPU_NOT_IDLE:

		load_idx = sd->busy_idx;

		break;



	case CPU_NEWLY_IDLE:

		load_idx = sd->newidle_idx;

		break;

	default:

		load_idx = sd->idle_idx;

		break;

	}



	return load_idx;

}



static unsigned long scale_rt_capacity(struct sched_domain *sd, int cpu)

static unsigned long scale_rt_capacity(struct sched_domain *sd, int cpu)

{

{

	struct rq *rq = cpu_rq(cpu);

	struct rq *rq = cpu_rq(cpu);
@@ -7992,9 +7919,6 @@ static inline void update_sg_lb_stats(struct lb_env *env, 
				      struct sg_lb_stats *sgs,

				      struct sg_lb_stats *sgs,

				      int *sg_status)

				      int *sg_status)

{

{

	int local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(group));

	int load_idx = get_sd_load_idx(env->sd, env->idle);

	unsigned long load;

	int i, nr_running;

	int i, nr_running;





	memset(sgs, 0, sizeof(*sgs));

	memset(sgs, 0, sizeof(*sgs));
@@ -8005,13 +7929,7 @@ static inline void update_sg_lb_stats(struct lb_env *env, 
		if ((env->flags & LBF_NOHZ_STATS) && update_nohz_stats(rq, false))

		if ((env->flags & LBF_NOHZ_STATS) && update_nohz_stats(rq, false))

			env->flags |= LBF_NOHZ_AGAIN;

			env->flags |= LBF_NOHZ_AGAIN;





		/* Bias balancing toward CPUs of our domain: */

		sgs->group_load += weighted_cpuload(rq);

		if (local_group)

			load = target_load(i, load_idx);

		else

			load = source_load(i, load_idx);



		sgs->group_load += load;

		sgs->group_util += cpu_util(i);

		sgs->group_util += cpu_util(i);

		sgs->sum_nr_running += rq->cfs.h_nr_running;

		sgs->sum_nr_running += rq->cfs.h_nr_running;

kernel/sched/features.h

+0 −1
Original line number Original line Diff line number Diff line
@@ -39,7 +39,6 @@ SCHED_FEAT(WAKEUP_PREEMPTION, true) 




SCHED_FEAT(HRTICK, false)

SCHED_FEAT(HRTICK, false)

SCHED_FEAT(DOUBLE_TICK, false)

SCHED_FEAT(DOUBLE_TICK, false)

SCHED_FEAT(LB_BIAS, false)





/*

/*

 * Decrement CPU capacity based on time not spent running tasks

 * Decrement CPU capacity based on time not spent running tasks

CRA Git | Maintained and supported by SUSTech CRA and CCSE