Commit ae67bada authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Ingo Molnar
Browse files

hrtimer: Optimize the hrtimer code by using static keys for migration_enable/nohz_active 



The hrtimer_cpu_base::migration_enable and ::nohz_active fields
were originally introduced to avoid accessing global variables
for these decisions.

Still that results in a (cache hot) load and conditional branch,
which can be avoided by using static keys.

Implement it with static keys and optimize for the most critical
case of high performance networking which tends to disable the
timer migration functionality.

No change in functionality.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Anna-Maria Gleixner <anna-maria@linutronix.de>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: keescook@chromium.org
Link: http://lkml.kernel.org/r/alpine.DEB.2.20.1801142327490.2371@nanos
Link: https://lkml.kernel.org/r/20171221104205.7269-2-anna-maria@linutronix.de


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 57957fb5

include/linux/hrtimer.h

+0 −4
Original line number Diff line number Diff line
@@ -153,8 +153,6 @@ enum hrtimer_base_type { 
 * @cpu:		cpu number

 * @active_bases:	Bitfield to mark bases with active timers

 * @clock_was_set_seq:	Sequence counter of clock was set events

 * @migration_enabled:	The migration of hrtimers to other cpus is enabled

 * @nohz_active:	The nohz functionality is enabled

 * @expires_next:	absolute time of the next event which was scheduled

 *			via clock_set_next_event()

 * @next_timer:		Pointer to the first expiring timer
@@ -178,8 +176,6 @@ struct hrtimer_cpu_base { 
	unsigned int			cpu;

	unsigned int			active_bases;

	unsigned int			clock_was_set_seq;

	bool				migration_enabled;

	bool				nohz_active;

#ifdef CONFIG_HIGH_RES_TIMERS

	unsigned int			in_hrtirq	: 1,

					hres_active	: 1,

kernel/time/hrtimer.c

+5 −12
Original line number Diff line number Diff line
@@ -178,23 +178,16 @@ hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base) 
#endif

}



#ifdef CONFIG_NO_HZ_COMMON

static inline

struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,

					 int pinned)

{

	if (pinned || !base->migration_enabled)

		return base;

#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)

	if (static_branch_likely(&timers_migration_enabled) && !pinned)

		return &per_cpu(hrtimer_bases, get_nohz_timer_target());

}

#else

static inline

struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,

					 int pinned)

{

#endif

	return base;

}

#endif



/*

 * We switch the timer base to a power-optimized selected CPU target,
@@ -969,7 +962,7 @@ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, 
		 * Kick to reschedule the next tick to handle the new timer

		 * on dynticks target.

		 */

		if (new_base->cpu_base->nohz_active)

		if (is_timers_nohz_active())

			wake_up_nohz_cpu(new_base->cpu_base->cpu);

	} else {

		hrtimer_reprogram(timer, new_base);

kernel/time/tick-internal.h

+12 −7
Original line number Diff line number Diff line
@@ -150,14 +150,19 @@ static inline void tick_nohz_init(void) { } 


#ifdef CONFIG_NO_HZ_COMMON

extern unsigned long tick_nohz_active;

#else

#define tick_nohz_active (0)

extern void timers_update_nohz(void);

extern struct static_key_false timers_nohz_active;

static inline bool is_timers_nohz_active(void)

{

	return static_branch_likely(&timers_nohz_active);

}

# ifdef CONFIG_SMP

extern struct static_key_false timers_migration_enabled;

# endif



#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)

extern void timers_update_migration(bool update_nohz);

#else

static inline void timers_update_migration(bool update_nohz) { }

#else /* CONFIG_NO_HZ_COMMON */

static inline void timers_update_nohz(void) { }

#define tick_nohz_active (0)

static inline bool is_timers_nohz_active(void) { return false; }

#endif



DECLARE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases);

kernel/time/tick-sched.c

+1 −1
Original line number Diff line number Diff line
@@ -1107,7 +1107,7 @@ static inline void tick_nohz_activate(struct tick_sched *ts, int mode) 
	ts->nohz_mode = mode;

	/* One update is enough */

	if (!test_and_set_bit(0, &tick_nohz_active))

		timers_update_migration(true);

		timers_update_nohz();

}



/**

kernel/time/timer.c

+42 −41
Original line number Diff line number Diff line
@@ -200,8 +200,6 @@ struct timer_base { 
	unsigned long		clk;

	unsigned long		next_expiry;

	unsigned int		cpu;

	bool			migration_enabled;

	bool			nohz_active;

	bool			is_idle;

	bool			must_forward_clk;

	DECLARE_BITMAP(pending_map, WHEEL_SIZE);
@@ -210,45 +208,57 @@ struct timer_base { 


static DEFINE_PER_CPU(struct timer_base, timer_bases[NR_BASES]);



#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)

#ifdef CONFIG_NO_HZ_COMMON



DEFINE_STATIC_KEY_FALSE(timers_nohz_active);

static DEFINE_MUTEX(timer_keys_mutex);



static void timer_update_keys(struct work_struct *work);

static DECLARE_WORK(timer_update_work, timer_update_keys);



#ifdef CONFIG_SMP

unsigned int sysctl_timer_migration = 1;



void timers_update_migration(bool update_nohz)

{

	bool on = sysctl_timer_migration && tick_nohz_active;

	unsigned int cpu;

DEFINE_STATIC_KEY_FALSE(timers_migration_enabled);



	/* Avoid the loop, if nothing to update */

	if (this_cpu_read(timer_bases[BASE_STD].migration_enabled) == on)

		return;

static void timers_update_migration(void)

{

	if (sysctl_timer_migration && tick_nohz_active)

		static_branch_enable(&timers_migration_enabled);

	else

		static_branch_disable(&timers_migration_enabled);

}

#else

static inline void timers_update_migration(void) { }

#endif /* !CONFIG_SMP */



	for_each_possible_cpu(cpu) {

		per_cpu(timer_bases[BASE_STD].migration_enabled, cpu) = on;

		per_cpu(timer_bases[BASE_DEF].migration_enabled, cpu) = on;

		per_cpu(hrtimer_bases.migration_enabled, cpu) = on;

		if (!update_nohz)

			continue;

		per_cpu(timer_bases[BASE_STD].nohz_active, cpu) = true;

		per_cpu(timer_bases[BASE_DEF].nohz_active, cpu) = true;

		per_cpu(hrtimer_bases.nohz_active, cpu) = true;

static void timer_update_keys(struct work_struct *work)

{

	mutex_lock(&timer_keys_mutex);

	timers_update_migration();

	static_branch_enable(&timers_nohz_active);

	mutex_unlock(&timer_keys_mutex);

}



void timers_update_nohz(void)

{

	schedule_work(&timer_update_work);

}



int timer_migration_handler(struct ctl_table *table, int write,

			    void __user *buffer, size_t *lenp,

			    loff_t *ppos)

{

	static DEFINE_MUTEX(mutex);

	int ret;



	mutex_lock(&mutex);

	mutex_lock(&timer_keys_mutex);

	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

	if (!ret && write)

		timers_update_migration(false);

	mutex_unlock(&mutex);

		timers_update_migration();

	mutex_unlock(&timer_keys_mutex);

	return ret;

}

#endif

#endif /* NO_HZ_COMMON */



static unsigned long round_jiffies_common(unsigned long j, int cpu,

		bool force_up)
@@ -534,7 +544,7 @@ __internal_add_timer(struct timer_base *base, struct timer_list *timer) 
static void

trigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer)

{

	if (!IS_ENABLED(CONFIG_NO_HZ_COMMON) || !base->nohz_active)

	if (!is_timers_nohz_active())

		return;



	/*
@@ -849,21 +859,20 @@ static inline struct timer_base *get_timer_base(u32 tflags) 
	return get_timer_cpu_base(tflags, tflags & TIMER_CPUMASK);

}



#ifdef CONFIG_NO_HZ_COMMON

static inline struct timer_base *

get_target_base(struct timer_base *base, unsigned tflags)

{

#ifdef CONFIG_SMP

	if ((tflags & TIMER_PINNED) || !base->migration_enabled)

		return get_timer_this_cpu_base(tflags);

#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)

	if (static_branch_likely(&timers_migration_enabled) &&

	    !(tflags & TIMER_PINNED))

		return get_timer_cpu_base(tflags, get_nohz_timer_target());

#else

	return get_timer_this_cpu_base(tflags);

#endif

	return get_timer_this_cpu_base(tflags);

}



static inline void forward_timer_base(struct timer_base *base)

{

#ifdef CONFIG_NO_HZ_COMMON

	unsigned long jnow;



	/*
@@ -887,16 +896,8 @@ static inline void forward_timer_base(struct timer_base *base) 
		base->clk = jnow;

	else

		base->clk = base->next_expiry;

}

#else

static inline struct timer_base *

get_target_base(struct timer_base *base, unsigned tflags)

{

	return get_timer_this_cpu_base(tflags);

}



static inline void forward_timer_base(struct timer_base *base) { }

#endif

}





/*

CRA Git | Maintained and supported by SUSTech CRA and CCSE