Commit daa59407 authored by Byungchul Park's avatar Byungchul Park Committed by Ingo Molnar
Browse files

sched/fair: Unify switched_{from,to}_fair() and task_move_group_fair() 



By observing that switched_from_fair() detaches from a runqueue, and
switched_to_fair() attaches to a runqueue, we can see that
task_move_group_fair() is one followed by the other with flipping the
runqueue in between.

Therefore extract all the common bits and implement all three
functions in terms of them.

This should fix a few corner cases wrt. vruntime normalization; where,
when we take a task off of a runqueue we convert to an approximation
of lag by subtracting min_vruntime, and when placing a task on the a
runqueue to the reverse.

Suggested-by: default avatarPeter Zijlstra <peterz@infradead.org>
Signed-off-by: default avatarByungchul Park <byungchul.park@lge.com>
[peterz: Changelog]
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: yuyang.du@intel.com
Link: http://lkml.kernel.org/r/1440069720-27038-6-git-send-email-byungchul.park@lge.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent a9280514

kernel/sched/fair.c

+52 −77
Original line number Diff line number Diff line
@@ -7924,21 +7924,39 @@ prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio) 
		check_preempt_curr(rq, p, 0);

}



static void switched_from_fair(struct rq *rq, struct task_struct *p)

static inline bool vruntime_normalized(struct task_struct *p)

{

	struct sched_entity *se = &p->se;

	struct cfs_rq *cfs_rq = cfs_rq_of(se);



	/*

	 * Ensure the task's vruntime is normalized, so that when it's

	 * switched back to the fair class the enqueue_entity(.flags=0) will

	 * do the right thing.

	 * In both the TASK_ON_RQ_QUEUED and TASK_ON_RQ_MIGRATING cases,

	 * the dequeue_entity(.flags=0) will already have normalized the

	 * vruntime.

	 */

	if (p->on_rq)

		return true;



	/*

	 * When !on_rq, vruntime of the task has usually NOT been normalized.

	 * But there are some cases where it has already been normalized:

	 *

	 * If it's queued, then the dequeue_entity(.flags=0) will already

	 * have normalized the vruntime, if it's !queued, then only when

	 * the task is sleeping will it still have non-normalized vruntime.

	 * - A forked child which is waiting for being woken up by

	 *   wake_up_new_task().

	 * - A task which has been woken up by try_to_wake_up() and

	 *   waiting for actually being woken up by sched_ttwu_pending().

	 */

	if (!task_on_rq_queued(p) && p->state != TASK_RUNNING) {

	if (!se->sum_exec_runtime || p->state == TASK_WAKING)

		return true;



	return false;

}



static void detach_task_cfs_rq(struct task_struct *p)

{

	struct sched_entity *se = &p->se;

	struct cfs_rq *cfs_rq = cfs_rq_of(se);



	if (!vruntime_normalized(p)) {

		/*

		 * Fix up our vruntime so that the current sleep doesn't

		 * cause 'unlimited' sleep bonus.
@@ -7951,9 +7969,10 @@ static void switched_from_fair(struct rq *rq, struct task_struct *p) 
	detach_entity_load_avg(cfs_rq, se);

}



static void switched_to_fair(struct rq *rq, struct task_struct *p)

static void attach_task_cfs_rq(struct task_struct *p)

{

	struct sched_entity *se = &p->se;

	struct cfs_rq *cfs_rq = cfs_rq_of(se);



#ifdef CONFIG_FAIR_GROUP_SCHED

	/*
@@ -7964,24 +7983,22 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) 
#endif



	/* Synchronize task with its cfs_rq */

	attach_entity_load_avg(cfs_rq_of(&p->se), &p->se);

	attach_entity_load_avg(cfs_rq, se);



	if (!task_on_rq_queued(p)) {

	if (!vruntime_normalized(p))

		se->vruntime += cfs_rq->min_vruntime;

}



		/*

		 * Ensure the task has a non-normalized vruntime when it is switched

		 * back to the fair class with !queued, so that enqueue_entity() at

		 * wake-up time will do the right thing.

		 *

		 * If it's queued, then the enqueue_entity(.flags=0) makes the task

		 * has non-normalized vruntime, if it's !queued, then it still has

		 * normalized vruntime.

		 */

		if (p->state != TASK_RUNNING)

			se->vruntime += cfs_rq_of(se)->min_vruntime;

		return;

static void switched_from_fair(struct rq *rq, struct task_struct *p)

{

	detach_task_cfs_rq(p);

}



static void switched_to_fair(struct rq *rq, struct task_struct *p)

{

	attach_task_cfs_rq(p);



	if (task_on_rq_queued(p)) {

		/*

		 * We were most likely switched from sched_rt, so

		 * kick off the schedule if running, otherwise just see
@@ -7992,6 +8009,7 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) 
		else

			check_preempt_curr(rq, p, 0);

	}

}



/* Account for a task changing its policy or group.

 *
@@ -8027,57 +8045,14 @@ void init_cfs_rq(struct cfs_rq *cfs_rq) 
#ifdef CONFIG_FAIR_GROUP_SCHED

static void task_move_group_fair(struct task_struct *p, int queued)

{

	struct sched_entity *se = &p->se;

	struct cfs_rq *cfs_rq;



	/*

	 * If the task was not on the rq at the time of this cgroup movement

	 * it must have been asleep, sleeping tasks keep their ->vruntime

	 * absolute on their old rq until wakeup (needed for the fair sleeper

	 * bonus in place_entity()).

	 *

	 * If it was on the rq, we've just 'preempted' it, which does convert

	 * ->vruntime to a relative base.

	 *

	 * Make sure both cases convert their relative position when migrating

	 * to another cgroup's rq. This does somewhat interfere with the

	 * fair sleeper stuff for the first placement, but who cares.

	 */

	/*

	 * When !queued, vruntime of the task has usually NOT been normalized.

	 * But there are some cases where it has already been normalized:

	 *

	 * - Moving a forked child which is waiting for being woken up by

	 *   wake_up_new_task().

	 * - Moving a task which has been woken up by try_to_wake_up() and

	 *   waiting for actually being woken up by sched_ttwu_pending().

	 *

	 * To prevent boost or penalty in the new cfs_rq caused by delta

	 * min_vruntime between the two cfs_rqs, we skip vruntime adjustment.

	 */

	if (!queued && (!se->sum_exec_runtime || p->state == TASK_WAKING))

		queued = 1;



	cfs_rq = cfs_rq_of(se);

	if (!queued)

		se->vruntime -= cfs_rq->min_vruntime;



	/* Synchronize task with its prev cfs_rq */

	detach_entity_load_avg(cfs_rq, se);

	detach_task_cfs_rq(p);

	set_task_rq(p, task_cpu(p));



#ifdef CONFIG_SMP

	/* Tell se's cfs_rq has been changed -- migrated */

	p->se.avg.last_update_time = 0;

#endif



	se->depth = se->parent ? se->parent->depth + 1 : 0;

	cfs_rq = cfs_rq_of(se);

	if (!queued)

		se->vruntime += cfs_rq->min_vruntime;



	/* Virtually synchronize task with its new cfs_rq */

	attach_entity_load_avg(cfs_rq, se);

	attach_task_cfs_rq(p);

}



void free_fair_sched_group(struct task_group *tg)

CRA Git | Maintained and supported by SUSTech CRA and CCSE