Commit b5c0ce7b authored by Patrick Bellasi's avatar Patrick Bellasi Committed by Ingo Molnar
Browse files

sched/fair: Add lsub_positive() and use it consistently 



The following pattern:

   var -= min_t(typeof(var), var, val);

is used multiple times in fair.c.

The existing sub_positive() already captures that pattern, but it also
adds an explicit load-store to properly support lockless observations.
In other cases the pattern above is used to update local, and/or not
concurrently accessed, variables.

Let's add a simpler version of sub_positive(), targeted at local variables
updates, which gives the same readability benefits at calling sites,
without enforcing {READ,WRITE}_ONCE() barriers.

Signed-off-by: default avatarPatrick Bellasi <patrick.bellasi@arm.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Dietmar Eggemann <dietmar.eggemann@arm.com>
Cc: Juri Lelli <juri.lelli@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Morten Rasmussen <morten.rasmussen@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Quentin Perret <quentin.perret@arm.com>
Cc: Steve Muckle <smuckle@google.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Todd Kjos <tkjos@google.com>
Cc: Vincent Guittot <vincent.guittot@linaro.org>
Link: https://lore.kernel.org/lkml/20181031184527.GA3178@hirez.programming.kicks-ass.net


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 92a801e5

kernel/sched/fair.c

+17 −7
Original line number Diff line number Diff line
@@ -2734,6 +2734,17 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) 
	WRITE_ONCE(*ptr, res);					\

} while (0)



/*

 * Remove and clamp on negative, from a local variable.

 *

 * A variant of sub_positive(), which does not use explicit load-store

 * and is thus optimized for local variable updates.

 */

#define lsub_positive(_ptr, _val) do {				\

	typeof(_ptr) ptr = (_ptr);				\

	*ptr -= min_t(typeof(*ptr), *ptr, _val);		\

} while (0)



#ifdef CONFIG_SMP

static inline void

enqueue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
@@ -4639,7 +4650,7 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) 
		cfs_b->distribute_running = 0;

		throttled = !list_empty(&cfs_b->throttled_cfs_rq);



		cfs_b->runtime -= min(runtime, cfs_b->runtime);

		lsub_positive(&cfs_b->runtime, runtime);

	}



	/*
@@ -4773,7 +4784,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) 


	raw_spin_lock(&cfs_b->lock);

	if (expires == cfs_b->runtime_expires)

		cfs_b->runtime -= min(runtime, cfs_b->runtime);

		lsub_positive(&cfs_b->runtime, runtime);

	cfs_b->distribute_running = 0;

	raw_spin_unlock(&cfs_b->lock);

}
@@ -6240,7 +6251,7 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p) 
	util = READ_ONCE(cfs_rq->avg.util_avg);



	/* Discount task's util from CPU's util */

	util -= min_t(unsigned int, util, task_util(p));

	lsub_positive(&util, task_util(p));



	/*

	 * Covered cases:
@@ -6289,10 +6300,9 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p) 
		 * properly fix the execl regression and it helps in further

		 * reducing the chances for the above race.

		 */

		if (unlikely(task_on_rq_queued(p) || current == p)) {

			estimated -= min_t(unsigned int, estimated,

					   _task_util_est(p));

		}

		if (unlikely(task_on_rq_queued(p) || current == p))

			lsub_positive(&estimated, _task_util_est(p));



		util = max(util, estimated);

	}

CRA Git | Maintained and supported by SUSTech CRA and CCSE