Commit c308b56b authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

sched: Fix nohz load accounting -- again! 



Various people reported nohz load tracking still being wrecked, but Doug
spotted the actual problem. We fold the nohz remainder in too soon,
causing us to loose samples and under-account.

So instead of playing catch-up up-front, always do a single load-fold
with whatever state we encounter and only then fold the nohz remainder
and play catch-up.

Reported-by: default avatarDoug Smythies <dsmythies@telus.net>
Reported-by: default avatarLesÅ=82aw Kope=C4=87 <leslaw.kopec@nasza-klasa.pl>
Reported-by: default avatarAman Gupta <aman@tmm1.net>
Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/n/tip-4v31etnhgg9kwd6ocgx3rxl8@git.kernel.org


Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 8e3fabfd

kernel/sched/core.c

+26 −27
Original line number Diff line number Diff line
@@ -2266,13 +2266,10 @@ calc_load_n(unsigned long load, unsigned long exp, 
 * Once we've updated the global active value, we need to apply the exponential

 * weights adjusted to the number of cycles missed.

 */

static void calc_global_nohz(unsigned long ticks)

static void calc_global_nohz(void)

{

	long delta, active, n;



	if (time_before(jiffies, calc_load_update))

		return;



	/*

	 * If we crossed a calc_load_update boundary, make sure to fold

	 * any pending idle changes, the respective CPUs might have
@@ -2284,10 +2281,16 @@ static void calc_global_nohz(unsigned long ticks) 
		atomic_long_add(delta, &calc_load_tasks);



	/*

	 * If we were idle for multiple load cycles, apply them.

	 * It could be the one fold was all it took, we done!

	 */

	if (ticks >= LOAD_FREQ) {

		n = ticks / LOAD_FREQ;

	if (time_before(jiffies, calc_load_update + 10))

		return;



	/*

	 * Catch-up, fold however many we are behind still

	 */

	delta = jiffies - calc_load_update - 10;

	n = 1 + (delta / LOAD_FREQ);



	active = atomic_long_read(&calc_load_tasks);

	active = active > 0 ? active * FIXED_1 : 0;
@@ -2298,18 +2301,6 @@ static void calc_global_nohz(unsigned long ticks) 


	calc_load_update += n * LOAD_FREQ;

}



	/*

	 * Its possible the remainder of the above division also crosses

	 * a LOAD_FREQ period, the regular check in calc_global_load()

	 * which comes after this will take care of that.

	 *

	 * Consider us being 11 ticks before a cycle completion, and us

	 * sleeping for 4*LOAD_FREQ + 22 ticks, then the above code will

	 * age us 4 cycles, and the test in calc_global_load() will

	 * pick up the final one.

	 */

}

#else

void calc_load_account_idle(struct rq *this_rq)

{
@@ -2320,7 +2311,7 @@ static inline long calc_load_fold_idle(void) 
	return 0;

}



static void calc_global_nohz(unsigned long ticks)

static void calc_global_nohz(void)

{

}

#endif
@@ -2348,8 +2339,6 @@ void calc_global_load(unsigned long ticks) 
{

	long active;



	calc_global_nohz(ticks);



	if (time_before(jiffies, calc_load_update + 10))

		return;
@@ -2361,6 +2350,16 @@ void calc_global_load(unsigned long ticks) 
	avenrun[2] = calc_load(avenrun[2], EXP_15, active);



	calc_load_update += LOAD_FREQ;



	/*

	 * Account one period with whatever state we found before

	 * folding in the nohz state and ageing the entire idle period.

	 *

	 * This avoids loosing a sample when we go idle between 

	 * calc_load_account_active() (10 ticks ago) and now and thus

	 * under-accounting.

	 */

	calc_global_nohz();

}



/*

CRA Git | Maintained and supported by SUSTech CRA and CCSE