Commit bc7a34b8 authored by Thomas Gleixner's avatar Thomas Gleixner
Browse files

timer: Reduce timer migration overhead if disabled 



Eric reported that the timer_migration sysctl is not really nice
performance wise as it needs to check at every timer insertion whether
the feature is enabled or not. Further the check does not live in the
timer code, so we have an extra function call which checks an extra
cache line to figure out that it is disabled.

We can do better and store that information in the per cpu (hr)timer
bases. I pondered to use a static key, but that's a nightmare to
update from the nohz code and the timer base cache line is hot anyway
when we select a timer base.

The old logic enabled the timer migration unconditionally if
CONFIG_NO_HZ was set even if nohz was disabled on the kernel command
line.

With this modification, we start off with migration disabled. The user
visible sysctl is still set to enabled. If the kernel switches to NOHZ
migration is enabled, if the user did not disable it via the sysctl
prior to the switch. If nohz=off is on the kernel command line,
migration stays disabled no matter what.

Before:
  47.76%  hog       [.] main
  14.84%  [kernel]  [k] _raw_spin_lock_irqsave
   9.55%  [kernel]  [k] _raw_spin_unlock_irqrestore
   6.71%  [kernel]  [k] mod_timer
   6.24%  [kernel]  [k] lock_timer_base.isra.38
   3.76%  [kernel]  [k] detach_if_pending
   3.71%  [kernel]  [k] del_timer
   2.50%  [kernel]  [k] internal_add_timer
   1.51%  [kernel]  [k] get_nohz_timer_target
   1.28%  [kernel]  [k] __internal_add_timer
   0.78%  [kernel]  [k] timerfn
   0.48%  [kernel]  [k] wake_up_nohz_cpu

After:
  48.10%  hog       [.] main
  15.25%  [kernel]  [k] _raw_spin_lock_irqsave
   9.76%  [kernel]  [k] _raw_spin_unlock_irqrestore
   6.50%  [kernel]  [k] mod_timer
   6.44%  [kernel]  [k] lock_timer_base.isra.38
   3.87%  [kernel]  [k] detach_if_pending
   3.80%  [kernel]  [k] del_timer
   2.67%  [kernel]  [k] internal_add_timer
   1.33%  [kernel]  [k] __internal_add_timer
   0.73%  [kernel]  [k] timerfn
   0.54%  [kernel]  [k] wake_up_nohz_cpu


Reported-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Paul McKenney <paulmck@linux.vnet.ibm.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Viresh Kumar <viresh.kumar@linaro.org>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Joonwoo Park <joonwoop@codeaurora.org>
Cc: Wenbo Wang <wenbo.wang@memblaze.com>
Link: http://lkml.kernel.org/r/20150526224512.127050787@linutronix.de


Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent c74441a1

include/linux/hrtimer.h

+2 −0
Original line number Diff line number Diff line
@@ -163,6 +163,7 @@ enum hrtimer_base_type { 
 * @cpu:		cpu number

 * @active_bases:	Bitfield to mark bases with active timers

 * @clock_was_set_seq:	Sequence counter of clock was set events

 * @migration_enabled:	The migration of hrtimers to other cpus is enabled

 * @expires_next:	absolute time of the next event which was scheduled

 *			via clock_set_next_event()

 * @next_timer:		Pointer to the first expiring timer
@@ -186,6 +187,7 @@ struct hrtimer_cpu_base { 
	unsigned int			cpu;

	unsigned int			active_bases;

	unsigned int			clock_was_set_seq;

	bool				migration_enabled;

#ifdef CONFIG_HIGH_RES_TIMERS

	unsigned int			in_hrtirq	: 1,

					hres_active	: 1,

include/linux/sched.h

+1 −5
Original line number Diff line number Diff line
@@ -335,14 +335,10 @@ extern int runqueue_is_locked(int cpu); 
#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)

extern void nohz_balance_enter_idle(int cpu);

extern void set_cpu_sd_state_idle(void);

extern int get_nohz_timer_target(int pinned);

extern int get_nohz_timer_target(void);

#else

static inline void nohz_balance_enter_idle(int cpu) { }

static inline void set_cpu_sd_state_idle(void) { }

static inline int get_nohz_timer_target(int pinned)

{

	return smp_processor_id();

}

#endif



/*

include/linux/sched/sysctl.h

+0 −12
Original line number Diff line number Diff line
@@ -57,24 +57,12 @@ extern unsigned int sysctl_numa_balancing_scan_size; 
extern unsigned int sysctl_sched_migration_cost;

extern unsigned int sysctl_sched_nr_migrate;

extern unsigned int sysctl_sched_time_avg;

extern unsigned int sysctl_timer_migration;

extern unsigned int sysctl_sched_shares_window;



int sched_proc_update_handler(struct ctl_table *table, int write,

		void __user *buffer, size_t *length,

		loff_t *ppos);

#endif

#ifdef CONFIG_SCHED_DEBUG

static inline unsigned int get_sysctl_timer_migration(void)

{

	return sysctl_timer_migration;

}

#else

static inline unsigned int get_sysctl_timer_migration(void)

{

	return 1;

}

#endif



/*

 *  control realtime throttling:

include/linux/timer.h

+9 −0
Original line number Diff line number Diff line
@@ -238,6 +238,15 @@ extern void run_local_timers(void); 
struct hrtimer;

extern enum hrtimer_restart it_real_fn(struct hrtimer *);



#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)

#include <linux/sysctl.h>



extern unsigned int sysctl_timer_migration;

int timer_migration_handler(struct ctl_table *table, int write,

			    void __user *buffer, size_t *lenp,

			    loff_t *ppos);

#endif



unsigned long __round_jiffies(unsigned long j, int cpu);

unsigned long __round_jiffies_relative(unsigned long j, int cpu);

unsigned long round_jiffies(unsigned long j);

kernel/rcu/tree_plugin.h

+0 −2
Original line number Diff line number Diff line
@@ -1432,8 +1432,6 @@ module_param(rcu_idle_gp_delay, int, 0644); 
static int rcu_idle_lazy_gp_delay = RCU_IDLE_LAZY_GP_DELAY;

module_param(rcu_idle_lazy_gp_delay, int, 0644);



extern int tick_nohz_active;



/*

 * Try to advance callbacks for all flavors of RCU on the current CPU, but

 * only if it has been awhile since the last time we did so.  Afterwards,

CRA Git | Maintained and supported by SUSTech CRA and CCSE