Commit 1f251846 authored by Ingo Molnar's avatar Ingo Molnar
Browse files

Merge branch 'timers/core-v9' of... 

Merge branch 'timers/core-v9' of git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks

 into timers/nohz

Pull nohz enhancements from Frederic Weisbecker:

"Currently in nohz full configs, the tick dependency is checked
 asynchronously by nohz code from interrupt and context switch for each
 concerned subsystem with a set of function provided by these. Such
 functions are made of many conditions and details that can be heavyweight
 as they are called on fastpath: sched_can_stop_tick(),
 posix_cpu_timer_can_stop_tick(), perf_event_can_stop_tick()...

 Thomas suggested a few months ago to make that tick dependency check
 synchronous. Instead of checking subsystems details from each interrupt
 to guess if the tick can be stopped, every subsystem that may have a tick
 dependency should set itself a flag specifying the state of that
 dependency. This way we can verify if we can stop the tick with a single
 lightweight mask check on fast path.

 This conversion from a pull to a push model to implement tick dependency
 is the core feature of this patchset that is split into:

  * Nohz wide kick simplification
  * Improve nohz tracing
  * Introduce tick dependency mask
  * Migrate scheduler, posix timers, perf events and sched clock tick
    dependencies to the tick dependency mask."

Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parents e2857b8f 4f49b90a

include/linux/atomic.h

+21 −0
Original line number Diff line number Diff line
@@ -548,6 +548,27 @@ static inline int atomic_dec_if_positive(atomic_t *v) 
}

#endif



/**

 * fetch_or - perform *ptr |= mask and return old value of *ptr

 * @ptr: pointer to value

 * @mask: mask to OR on the value

 *

 * cmpxchg based fetch_or, macro so it works for different integer types

 */

#ifndef fetch_or

#define fetch_or(ptr, mask)						\

({	typeof(*(ptr)) __old, __val = *(ptr);				\

	for (;;) {							\

		__old = cmpxchg((ptr), __val, __val | (mask));		\

		if (__old == __val)					\

			break;						\

		__val = __old;						\

	}								\

	__old;								\

})

#endif





#ifdef CONFIG_GENERIC_ATOMIC64

#include <asm-generic/atomic64.h>

#endif

include/linux/perf_event.h

+0 −6
Original line number Diff line number Diff line
@@ -1109,12 +1109,6 @@ static inline void perf_event_task_tick(void) { } 
static inline int perf_event_release_kernel(struct perf_event *event)	{ return 0; }

#endif



#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_NO_HZ_FULL)

extern bool perf_event_can_stop_tick(void);

#else

static inline bool perf_event_can_stop_tick(void)			{ return true; }

#endif



#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)

extern void perf_restore_debug_store(void);

#else

include/linux/posix-timers.h

+0 −3
Original line number Diff line number Diff line
@@ -128,9 +128,6 @@ void posix_cpu_timer_schedule(struct k_itimer *timer); 
void run_posix_cpu_timers(struct task_struct *task);

void posix_cpu_timers_exit(struct task_struct *task);

void posix_cpu_timers_exit_group(struct task_struct *task);



bool posix_cpu_timers_can_stop_tick(struct task_struct *tsk);



void set_process_cpu_timer(struct task_struct *task, unsigned int clock_idx,

			   cputime_t *newval, cputime_t *oldval);

include/linux/sched.h

+8 −3
Original line number Diff line number Diff line
@@ -719,6 +719,10 @@ struct signal_struct { 
	/* Earliest-expiration cache. */

	struct task_cputime cputime_expires;



#ifdef CONFIG_NO_HZ_FULL

	unsigned long tick_dep_mask;

#endif



	struct list_head cpu_timers[3];



	struct pid *tty_old_pgrp;
@@ -1542,6 +1546,10 @@ struct task_struct { 
		VTIME_SYS,

	} vtime_snap_whence;

#endif



#ifdef CONFIG_NO_HZ_FULL

	unsigned long tick_dep_mask;

#endif

	unsigned long nvcsw, nivcsw; /* context switch counts */

	u64 start_time;		/* monotonic time in nsec */

	u64 real_start_time;	/* boot based time in nsec */
@@ -2356,10 +2364,7 @@ static inline void wake_up_nohz_cpu(int cpu) { } 
#endif



#ifdef CONFIG_NO_HZ_FULL

extern bool sched_can_stop_tick(void);

extern u64 scheduler_tick_max_deferment(void);

#else

static inline bool sched_can_stop_tick(void) { return false; }

#endif



#ifdef CONFIG_SCHED_AUTOGROUP

include/linux/tick.h

+93 −4
Original line number Diff line number Diff line
@@ -97,6 +97,19 @@ static inline void tick_broadcast_exit(void) 
	tick_broadcast_oneshot_control(TICK_BROADCAST_EXIT);

}



enum tick_dep_bits {

	TICK_DEP_BIT_POSIX_TIMER	= 0,

	TICK_DEP_BIT_PERF_EVENTS	= 1,

	TICK_DEP_BIT_SCHED		= 2,

	TICK_DEP_BIT_CLOCK_UNSTABLE	= 3

};



#define TICK_DEP_MASK_NONE		0

#define TICK_DEP_MASK_POSIX_TIMER	(1 << TICK_DEP_BIT_POSIX_TIMER)

#define TICK_DEP_MASK_PERF_EVENTS	(1 << TICK_DEP_BIT_PERF_EVENTS)

#define TICK_DEP_MASK_SCHED		(1 << TICK_DEP_BIT_SCHED)

#define TICK_DEP_MASK_CLOCK_UNSTABLE	(1 << TICK_DEP_BIT_CLOCK_UNSTABLE)



#ifdef CONFIG_NO_HZ_COMMON

extern int tick_nohz_enabled;

extern int tick_nohz_tick_stopped(void);
@@ -154,9 +167,73 @@ static inline int housekeeping_any_cpu(void) 
	return cpumask_any_and(housekeeping_mask, cpu_online_mask);

}



extern void tick_nohz_full_kick(void);

extern void tick_nohz_dep_set(enum tick_dep_bits bit);

extern void tick_nohz_dep_clear(enum tick_dep_bits bit);

extern void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit);

extern void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit);

extern void tick_nohz_dep_set_task(struct task_struct *tsk,

				   enum tick_dep_bits bit);

extern void tick_nohz_dep_clear_task(struct task_struct *tsk,

				     enum tick_dep_bits bit);

extern void tick_nohz_dep_set_signal(struct signal_struct *signal,

				     enum tick_dep_bits bit);

extern void tick_nohz_dep_clear_signal(struct signal_struct *signal,

				       enum tick_dep_bits bit);



/*

 * The below are tick_nohz_[set,clear]_dep() wrappers that optimize off-cases

 * on top of static keys.

 */

static inline void tick_dep_set(enum tick_dep_bits bit)

{

	if (tick_nohz_full_enabled())

		tick_nohz_dep_set(bit);

}



static inline void tick_dep_clear(enum tick_dep_bits bit)

{

	if (tick_nohz_full_enabled())

		tick_nohz_dep_clear(bit);

}



static inline void tick_dep_set_cpu(int cpu, enum tick_dep_bits bit)

{

	if (tick_nohz_full_cpu(cpu))

		tick_nohz_dep_set_cpu(cpu, bit);

}



static inline void tick_dep_clear_cpu(int cpu, enum tick_dep_bits bit)

{

	if (tick_nohz_full_cpu(cpu))

		tick_nohz_dep_clear_cpu(cpu, bit);

}



static inline void tick_dep_set_task(struct task_struct *tsk,

				     enum tick_dep_bits bit)

{

	if (tick_nohz_full_enabled())

		tick_nohz_dep_set_task(tsk, bit);

}

static inline void tick_dep_clear_task(struct task_struct *tsk,

				       enum tick_dep_bits bit)

{

	if (tick_nohz_full_enabled())

		tick_nohz_dep_clear_task(tsk, bit);

}

static inline void tick_dep_set_signal(struct signal_struct *signal,

				       enum tick_dep_bits bit)

{

	if (tick_nohz_full_enabled())

		tick_nohz_dep_set_signal(signal, bit);

}

static inline void tick_dep_clear_signal(struct signal_struct *signal,

					 enum tick_dep_bits bit)

{

	if (tick_nohz_full_enabled())

		tick_nohz_dep_clear_signal(signal, bit);

}



extern void tick_nohz_full_kick_cpu(int cpu);

extern void tick_nohz_full_kick_all(void);

extern void __tick_nohz_task_switch(void);

#else

static inline int housekeeping_any_cpu(void)
@@ -166,9 +243,21 @@ static inline int housekeeping_any_cpu(void) 
static inline bool tick_nohz_full_enabled(void) { return false; }

static inline bool tick_nohz_full_cpu(int cpu) { return false; }

static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask) { }



static inline void tick_dep_set(enum tick_dep_bits bit) { }

static inline void tick_dep_clear(enum tick_dep_bits bit) { }

static inline void tick_dep_set_cpu(int cpu, enum tick_dep_bits bit) { }

static inline void tick_dep_clear_cpu(int cpu, enum tick_dep_bits bit) { }

static inline void tick_dep_set_task(struct task_struct *tsk,

				     enum tick_dep_bits bit) { }

static inline void tick_dep_clear_task(struct task_struct *tsk,

				       enum tick_dep_bits bit) { }

static inline void tick_dep_set_signal(struct signal_struct *signal,

				       enum tick_dep_bits bit) { }

static inline void tick_dep_clear_signal(struct signal_struct *signal,

					 enum tick_dep_bits bit) { }



static inline void tick_nohz_full_kick_cpu(int cpu) { }

static inline void tick_nohz_full_kick(void) { }

static inline void tick_nohz_full_kick_all(void) { }

static inline void __tick_nohz_task_switch(void) { }

#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE