Commit ad936d86 authored by Byungchul Park's avatar Byungchul Park Committed by Ingo Molnar
Browse files

sched/fair: Make it possible to account fair load avg consistently 



The current code accounts for the time a task was absent from the fair
class (per ATTACH_AGE_LOAD). However it does not work correctly when a
task got migrated or moved to another cgroup while outside of the fair
class.

This patch tries to address that by aging on migration. We locklessly
read the 'last_update_time' stamp from both the old and new cfs_rq,
ages the load upto the old time, and sets it to the new time.

These timestamps should in general not be more than 1 tick apart from
one another, so there is a definite bound on things.

Signed-off-by: default avatarByungchul Park <byungchul.park@lge.com>
[ Changelog, a few edits and !SMP build fix ]
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/1445616981-29904-2-git-send-email-byungchul.park@lge.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 467386fb

kernel/sched/core.c

+4 −0
Original line number Diff line number Diff line
@@ -2120,6 +2120,10 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) 
	p->se.vruntime			= 0;

	INIT_LIST_HEAD(&p->se.group_node);



#ifdef CONFIG_FAIR_GROUP_SCHED

	p->se.cfs_rq			= NULL;

#endif



#ifdef CONFIG_SCHEDSTATS

	memset(&p->se.statistics, 0, sizeof(p->se.statistics));

#endif

kernel/sched/fair.c

+46 −0
Original line number Diff line number Diff line
@@ -2715,6 +2715,52 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) 
	}

}



/*

 * Called within set_task_rq() right before setting a task's cpu. The

 * caller only guarantees p->pi_lock is held; no other assumptions,

 * including the state of rq->lock, should be made.

 */

void set_task_rq_fair(struct sched_entity *se,

		      struct cfs_rq *prev, struct cfs_rq *next)

{

	if (!sched_feat(ATTACH_AGE_LOAD))

		return;



	/*

	 * We are supposed to update the task to "current" time, then its up to

	 * date and ready to go to new CPU/cfs_rq. But we have difficulty in

	 * getting what current time is, so simply throw away the out-of-date

	 * time. This will result in the wakee task is less decayed, but giving

	 * the wakee more load sounds not bad.

	 */

	if (se->avg.last_update_time && prev) {

		u64 p_last_update_time;

		u64 n_last_update_time;



#ifndef CONFIG_64BIT

		u64 p_last_update_time_copy;

		u64 n_last_update_time_copy;



		do {

			p_last_update_time_copy = prev->load_last_update_time_copy;

			n_last_update_time_copy = next->load_last_update_time_copy;



			smp_rmb();



			p_last_update_time = prev->avg.last_update_time;

			n_last_update_time = next->avg.last_update_time;



		} while (p_last_update_time != p_last_update_time_copy ||

			 n_last_update_time != n_last_update_time_copy);

#else

		p_last_update_time = prev->avg.last_update_time;

		n_last_update_time = next->avg.last_update_time;

#endif

		__update_load_avg(p_last_update_time, cpu_of(rq_of(prev)),

				  &se->avg, 0, 0, NULL);

		se->avg.last_update_time = n_last_update_time;

	}

}

#else /* CONFIG_FAIR_GROUP_SCHED */

static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) {}

#endif /* CONFIG_FAIR_GROUP_SCHED */

kernel/sched/sched.h

+10 −1
Original line number Diff line number Diff line
@@ -335,7 +335,15 @@ extern void sched_move_task(struct task_struct *tsk); 


#ifdef CONFIG_FAIR_GROUP_SCHED

extern int sched_group_set_shares(struct task_group *tg, unsigned long shares);

#endif



#ifdef CONFIG_SMP

extern void set_task_rq_fair(struct sched_entity *se,

			     struct cfs_rq *prev, struct cfs_rq *next);

#else /* !CONFIG_SMP */

static inline void set_task_rq_fair(struct sched_entity *se,

			     struct cfs_rq *prev, struct cfs_rq *next) { }

#endif /* CONFIG_SMP */

#endif /* CONFIG_FAIR_GROUP_SCHED */



#else /* CONFIG_CGROUP_SCHED */
@@ -933,6 +941,7 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) 
#endif



#ifdef CONFIG_FAIR_GROUP_SCHED

	set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]);

	p->se.cfs_rq = tg->cfs_rq[cpu];

	p->se.parent = tg->se[cpu];

#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE