Commit 35af99e6 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

sched/clock, x86: Use a static_key for sched_clock_stable 



In order to avoid the runtime condition and variable load turn
sched_clock_stable into a static_key.

Also provide a shorter implementation of local_clock() and
cpu_clock(int) when sched_clock_stable==1.

                        MAINLINE   PRE       POST

    sched_clock_stable: 1          1         1
    (cold) sched_clock: 329841     221876    215295
    (cold) local_clock: 301773     234692    220773
    (warm) sched_clock: 38375      25602     25659
    (warm) local_clock: 100371     33265     27242
    (warm) rdtsc:       27340      24214     24208
    sched_clock_stable: 0          0         0
    (cold) sched_clock: 382634     235941    237019
    (cold) local_clock: 396890     297017    294819
    (warm) sched_clock: 38194      25233     25609
    (warm) local_clock: 143452     71234     71232
    (warm) rdtsc:       27345      24245     24243

Signed-off-by: default avatarPeter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Link: http://lkml.kernel.org/n/tip-eummbdechzz37mwmpags1gjr@git.kernel.org


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent ef08f0ff

arch/x86/kernel/cpu/amd.c

+1 −1
Original line number Diff line number Diff line
@@ -487,7 +487,7 @@ static void early_init_amd(struct cpuinfo_x86 *c) 
		set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);

		set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);

		if (!check_tsc_unstable())

			sched_clock_stable = 1;

			set_sched_clock_stable();

	}



#ifdef CONFIG_X86_64

arch/x86/kernel/cpu/intel.c

+1 −1
Original line number Diff line number Diff line
@@ -93,7 +93,7 @@ static void early_init_intel(struct cpuinfo_x86 *c) 
		set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);

		set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);

		if (!check_tsc_unstable())

			sched_clock_stable = 1;

			set_sched_clock_stable();

	}



	/* Penwell and Cloverview have the TSC which doesn't sleep on S3 */

arch/x86/kernel/cpu/perf_event.c

+1 −1
Original line number Diff line number Diff line
@@ -1890,7 +1890,7 @@ void arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now) 
	userpg->cap_user_rdpmc = x86_pmu.attr_rdpmc;

	userpg->pmc_width = x86_pmu.cntval_bits;



	if (!sched_clock_stable)

	if (!sched_clock_stable())

		return;



	data = cyc2ns_read_begin();

arch/x86/kernel/tsc.c

+3 −3
Original line number Diff line number Diff line
@@ -822,7 +822,7 @@ static unsigned long long cyc2ns_suspend; 


void tsc_save_sched_clock_state(void)

{

	if (!sched_clock_stable)

	if (!sched_clock_stable())

		return;



	cyc2ns_suspend = sched_clock();
@@ -842,7 +842,7 @@ void tsc_restore_sched_clock_state(void) 
	unsigned long flags;

	int cpu;



	if (!sched_clock_stable)

	if (!sched_clock_stable())

		return;



	local_irq_save(flags);
@@ -984,7 +984,7 @@ void mark_tsc_unstable(char *reason) 
{

	if (!tsc_unstable) {

		tsc_unstable = 1;

		sched_clock_stable = 0;

		clear_sched_clock_stable();

		disable_sched_clock_irqtime();

		pr_info("Marking TSC unstable due to %s\n", reason);

		/* Change only the rating, when not registered */

include/linux/sched.h

+3 −1
Original line number Diff line number Diff line
@@ -1994,7 +1994,9 @@ static inline void sched_clock_idle_wakeup_event(u64 delta_ns) 
 * but then during bootup it turns out that sched_clock()

 * is reliable after all:

 */

extern int sched_clock_stable;

extern int sched_clock_stable(void);

extern void set_sched_clock_stable(void);

extern void clear_sched_clock_stable(void);



extern void sched_clock_tick(void);

extern void sched_clock_idle_sleep_event(void);

CRA Git | Maintained and supported by SUSTech CRA and CCSE