Commit 9edfbfed authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

sched/core: Rework rq->clock update skips 



The original purpose of rq::skip_clock_update was to avoid 'costly' clock
updates for back to back wakeup-preempt pairs. The big problem with it
has always been that the rq variable is unaware of the context and
causes indiscrimiate clock skips.

Rework the entire thing and create a sense of context by only allowing
schedule() to skip clock updates. (XXX can we measure the cost of the
added store?)

By ensuring only schedule can ever skip an update, we guarantee we're
never more than 1 tick behind on the update.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: umgwanakikbuti@gmail.com
Link: http://lkml.kernel.org/r/20150105103554.432381549@infradead.org


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent cebde6d6

kernel/sched/core.c

+8 −4
Original line number Diff line number Diff line
@@ -119,7 +119,9 @@ void update_rq_clock(struct rq *rq) 
{

	s64 delta;



	if (rq->skip_clock_update > 0)

	lockdep_assert_held(&rq->lock);



	if (rq->clock_skip_update & RQCF_ACT_SKIP)

		return;



	delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
@@ -1046,7 +1048,7 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) 
	 * this case, we can save a useless back to back clock update.

	 */

	if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))

		rq->skip_clock_update = 1;

		rq_clock_skip_update(rq, true);

}



#ifdef CONFIG_SMP
@@ -2779,6 +2781,8 @@ need_resched: 
	smp_mb__before_spinlock();

	raw_spin_lock_irq(&rq->lock);



	rq->clock_skip_update <<= 1; /* promote REQ to ACT */



	switch_count = &prev->nivcsw;

	if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {

		if (unlikely(signal_pending_state(prev->state, prev))) {
@@ -2803,13 +2807,13 @@ need_resched: 
		switch_count = &prev->nvcsw;

	}



	if (task_on_rq_queued(prev) || rq->skip_clock_update < 0)

	if (task_on_rq_queued(prev))

		update_rq_clock(rq);



	next = pick_next_task(rq, prev);

	clear_tsk_need_resched(prev);

	clear_preempt_need_resched();

	rq->skip_clock_update = 0;

	rq->clock_skip_update = 0;



	if (likely(prev != next)) {

		rq->nr_switches++;

kernel/sched/fair.c

+1 −1
Original line number Diff line number Diff line
@@ -5156,7 +5156,7 @@ static void yield_task_fair(struct rq *rq) 
		 * so we don't do microscopic update in schedule()

		 * and double the fastpath cost.

		 */

		 rq->skip_clock_update = 1;

		rq_clock_skip_update(rq, true);

	}



	set_skip_buddy(se);

kernel/sched/rt.c

+6 −3
Original line number Diff line number Diff line
@@ -831,11 +831,14 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) 
				enqueue = 1;



				/*

				 * Force a clock update if the CPU was idle,

				 * lest wakeup -> unthrottle time accumulate.

				 * When we're idle and a woken (rt) task is

				 * throttled check_preempt_curr() will set

				 * skip_update and the time between the wakeup

				 * and this unthrottle will get accounted as

				 * 'runtime'.

				 */

				if (rt_rq->rt_nr_running && rq->curr == rq->idle)

					rq->skip_clock_update = -1;

					rq_clock_skip_update(rq, false);

			}

			if (rt_rq->rt_time || rt_rq->rt_nr_running)

				idle = 0;

kernel/sched/sched.h

+13 −2
Original line number Diff line number Diff line
@@ -558,8 +558,6 @@ struct rq { 
#ifdef CONFIG_NO_HZ_FULL

	unsigned long last_sched_tick;

#endif

	int skip_clock_update;



	/* capture load from *all* tasks on this cpu: */

	struct load_weight load;

	unsigned long nr_load_updates;
@@ -588,6 +586,7 @@ struct rq { 
	unsigned long next_balance;

	struct mm_struct *prev_mm;



	unsigned int clock_skip_update;

	u64 clock;

	u64 clock_task;
@@ -704,6 +703,18 @@ static inline u64 rq_clock_task(struct rq *rq) 
	return rq->clock_task;

}



#define RQCF_REQ_SKIP	0x01

#define RQCF_ACT_SKIP	0x02



static inline void rq_clock_skip_update(struct rq *rq, bool skip)

{

	lockdep_assert_held(&rq->lock);

	if (skip)

		rq->clock_skip_update |= RQCF_REQ_SKIP;

	else

		rq->clock_skip_update &= ~RQCF_REQ_SKIP;

}



#ifdef CONFIG_NUMA

enum numa_topology_type {

	NUMA_DIRECT,

CRA Git | Maintained and supported by SUSTech CRA and CCSE