Commit 4793241b authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

sched: backward looking buddy 



Impact: improve/change/fix wakeup-buddy scheduling

Currently we only have a forward looking buddy, that is, we prefer to
schedule to the task we last woke up, under the presumption that its
going to consume the data we just produced, and therefore will have
cache hot benefits.

This allows co-waking producer/consumer task pairs to run ahead of the
pack for a little while, keeping their cache warm. Without this, we
would interleave all pairs, utterly trashing the cache.

This patch introduces a backward looking buddy, that is, suppose that
in the above scenario, the consumer preempts the producer before it
can go to sleep, we will therefore miss the wakeup from consumer to
producer (its already running, after all), breaking the cycle and
reverting to the cache-trashing interleaved schedule pattern.

The backward buddy will try to schedule back to the task that woke us
up in case the forward buddy is not available, under the assumption
that the last task will be the one with the most cache hot task around
barring current.

This will basically allow a task to continue after it got preempted.

In order to avoid starvation, we allow either buddy to get wakeup_gran
ahead of the pack.

Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: default avatarMike Galbraith <efault@gmx.de>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent d95f98d0

kernel/sched.c

+4 −2
Original line number Diff line number Diff line
@@ -397,7 +397,7 @@ struct cfs_rq { 
	 * 'curr' points to currently running entity on this cfs_rq.

	 * It is set to NULL otherwise (i.e when none are currently running).

	 */

	struct sched_entity *curr, *next;

	struct sched_entity *curr, *next, *last;



	unsigned long nr_spread_over;
@@ -1805,7 +1805,9 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) 
	/*

	 * Buddy candidates are cache hot:

	 */

	if (sched_feat(CACHE_HOT_BUDDY) && (&p->se == cfs_rq_of(&p->se)->next))

	if (sched_feat(CACHE_HOT_BUDDY) &&

			(&p->se == cfs_rq_of(&p->se)->next ||

			 &p->se == cfs_rq_of(&p->se)->last))

		return 1;



	if (p->sched_class != &fair_sched_class)

kernel/sched_fair.c

+25 −7
Original line number Diff line number Diff line
@@ -341,9 +341,6 @@ static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 
		cfs_rq->rb_leftmost = next_node;

	}



	if (cfs_rq->next == se)

		cfs_rq->next = NULL;



	rb_erase(&se->run_node, &cfs_rq->tasks_timeline);

}
@@ -741,6 +738,12 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) 
#endif

	}



	if (cfs_rq->last == se)

		cfs_rq->last = NULL;



	if (cfs_rq->next == se)

		cfs_rq->next = NULL;



	if (se != cfs_rq->curr)

		__dequeue_entity(cfs_rq, se);

	account_entity_dequeue(cfs_rq, se);
@@ -798,10 +801,13 @@ static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) 
{

	struct sched_entity *se = __pick_next_entity(cfs_rq);



	if (!cfs_rq->next || wakeup_preempt_entity(cfs_rq->next, se) == 1)

		return se;



	if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, se) < 1)

		return cfs_rq->next;



	if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, se) < 1)

		return cfs_rq->last;



	return se;

}



static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
@@ -1319,10 +1325,11 @@ wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se) 
static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync)

{

	struct task_struct *curr = rq->curr;

	struct cfs_rq *cfs_rq = task_cfs_rq(curr);

	struct sched_entity *se = &curr->se, *pse = &p->se;



	if (unlikely(rt_prio(p->prio))) {

		struct cfs_rq *cfs_rq = task_cfs_rq(curr);



		update_rq_clock(rq);

		update_curr(cfs_rq);

		resched_task(curr);
@@ -1335,6 +1342,17 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync) 
	if (unlikely(se == pse))

		return;



	/*

	 * Only set the backward buddy when the current task is still on the

	 * rq. This can happen when a wakeup gets interleaved with schedule on

	 * the ->pre_schedule() or idle_balance() point, either of which can

	 * drop the rq lock.

	 *

	 * Also, during early boot the idle thread is in the fair class, for

	 * obvious reasons its a bad idea to schedule back to the idle thread.

	 */

	if (sched_feat(LAST_BUDDY) && likely(se->on_rq && curr != rq->idle))

		cfs_rq_of(se)->last = se;

	cfs_rq_of(pse)->next = pse;



	/*

kernel/sched_features.h

+1 −0
Original line number Diff line number Diff line
@@ -12,3 +12,4 @@ SCHED_FEAT(LB_BIAS, 1) 
SCHED_FEAT(LB_WAKEUP_UPDATE, 1)

SCHED_FEAT(ASYM_EFF_LOAD, 1)

SCHED_FEAT(WAKEUP_OVERLAP, 0)

SCHED_FEAT(LAST_BUDDY, 1)

CRA Git | Maintained and supported by SUSTech CRA and CCSE