Commit f009a7a7 authored by Frederic Weisbecker's avatar Frederic Weisbecker Committed by Ingo Molnar
Browse files

timers/nohz: Convert tick dependency mask to atomic_t 



The tick dependency mask was intially unsigned long because this is the
type on which clear_bit() operates on and fetch_or() accepts it.

But now that we have atomic_fetch_or(), we can instead use
atomic_andnot() to clear the bit. This consolidates the type of our
tick dependency mask, reduce its size on structures and benefit from
possible architecture optimizations on atomic_t operations.

Suggested-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: default avatarFrederic Weisbecker <fweisbec@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/1458830281-4255-3-git-send-email-fweisbec@gmail.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 5acba71e

include/linux/sched.h

+2 −2
Original line number Original line Diff line number Diff line
@@ -720,7 +720,7 @@ struct signal_struct { 
	struct task_cputime cputime_expires;

	struct task_cputime cputime_expires;





#ifdef CONFIG_NO_HZ_FULL

#ifdef CONFIG_NO_HZ_FULL

	unsigned long tick_dep_mask;

	atomic_t tick_dep_mask;

#endif

#endif





	struct list_head cpu_timers[3];

	struct list_head cpu_timers[3];
@@ -1549,7 +1549,7 @@ struct task_struct { 
#endif

#endif





#ifdef CONFIG_NO_HZ_FULL

#ifdef CONFIG_NO_HZ_FULL

	unsigned long tick_dep_mask;

	atomic_t tick_dep_mask;

#endif

#endif

	unsigned long nvcsw, nivcsw; /* context switch counts */

	unsigned long nvcsw, nivcsw; /* context switch counts */

	u64 start_time;		/* monotonic time in nsec */

	u64 start_time;		/* monotonic time in nsec */

kernel/time/tick-sched.c

+30 −31
Original line number Original line Diff line number Diff line
@@ -157,52 +157,50 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) 
cpumask_var_t tick_nohz_full_mask;

cpumask_var_t tick_nohz_full_mask;

cpumask_var_t housekeeping_mask;

cpumask_var_t housekeeping_mask;

bool tick_nohz_full_running;

bool tick_nohz_full_running;

static unsigned long tick_dep_mask;

static atomic_t tick_dep_mask;





static void trace_tick_dependency(unsigned long dep)

static bool check_tick_dependency(atomic_t *dep)

{

{

	if (dep & TICK_DEP_MASK_POSIX_TIMER) {

	int val = atomic_read(dep);



	if (val & TICK_DEP_MASK_POSIX_TIMER) {

		trace_tick_stop(0, TICK_DEP_MASK_POSIX_TIMER);

		trace_tick_stop(0, TICK_DEP_MASK_POSIX_TIMER);

		return;

		return true;

	}

	}





	if (dep & TICK_DEP_MASK_PERF_EVENTS) {

	if (val & TICK_DEP_MASK_PERF_EVENTS) {

		trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS);

		trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS);

		return;

		return true;

	}

	}





	if (dep & TICK_DEP_MASK_SCHED) {

	if (val & TICK_DEP_MASK_SCHED) {

		trace_tick_stop(0, TICK_DEP_MASK_SCHED);

		trace_tick_stop(0, TICK_DEP_MASK_SCHED);

		return;

		return true;

	}

	}





	if (dep & TICK_DEP_MASK_CLOCK_UNSTABLE)

	if (val & TICK_DEP_MASK_CLOCK_UNSTABLE) {

		trace_tick_stop(0, TICK_DEP_MASK_CLOCK_UNSTABLE);

		trace_tick_stop(0, TICK_DEP_MASK_CLOCK_UNSTABLE);

		return true;

	}



	return false;

}

}





static bool can_stop_full_tick(struct tick_sched *ts)

static bool can_stop_full_tick(struct tick_sched *ts)

{

{

	WARN_ON_ONCE(!irqs_disabled());

	WARN_ON_ONCE(!irqs_disabled());





	if (tick_dep_mask) {

	if (check_tick_dependency(&tick_dep_mask))

		trace_tick_dependency(tick_dep_mask);

		return false;

		return false;

	}





	if (ts->tick_dep_mask) {

	if (check_tick_dependency(&ts->tick_dep_mask))

		trace_tick_dependency(ts->tick_dep_mask);

		return false;

		return false;

	}





	if (current->tick_dep_mask) {

	if (check_tick_dependency(&current->tick_dep_mask))

		trace_tick_dependency(current->tick_dep_mask);

		return false;

		return false;

	}





	if (current->signal->tick_dep_mask) {

	if (check_tick_dependency(&current->signal->tick_dep_mask))

		trace_tick_dependency(current->signal->tick_dep_mask);

		return false;

		return false;

	}





	return true;

	return true;

}

}
@@ -259,12 +257,12 @@ static void tick_nohz_full_kick_all(void) 
	preempt_enable();

	preempt_enable();

}

}





static void tick_nohz_dep_set_all(unsigned long *dep,

static void tick_nohz_dep_set_all(atomic_t *dep,

				  enum tick_dep_bits bit)

				  enum tick_dep_bits bit)

{

{

	unsigned long prev;

	int prev;





	prev = fetch_or(dep, BIT_MASK(bit));

	prev = atomic_fetch_or(dep, BIT(bit));

	if (!prev)

	if (!prev)

		tick_nohz_full_kick_all();

		tick_nohz_full_kick_all();

}

}
@@ -280,7 +278,7 @@ void tick_nohz_dep_set(enum tick_dep_bits bit) 




void tick_nohz_dep_clear(enum tick_dep_bits bit)

void tick_nohz_dep_clear(enum tick_dep_bits bit)

{

{

	clear_bit(bit, &tick_dep_mask);

	atomic_andnot(BIT(bit), &tick_dep_mask);

}

}





/*

/*
@@ -289,12 +287,12 @@ void tick_nohz_dep_clear(enum tick_dep_bits bit) 
 */

 */

void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit)

void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit)

{

{

	unsigned long prev;

	int prev;

	struct tick_sched *ts;

	struct tick_sched *ts;





	ts = per_cpu_ptr(&tick_cpu_sched, cpu);

	ts = per_cpu_ptr(&tick_cpu_sched, cpu);





	prev = fetch_or(&ts->tick_dep_mask, BIT_MASK(bit));

	prev = atomic_fetch_or(&ts->tick_dep_mask, BIT(bit));

	if (!prev) {

	if (!prev) {

		preempt_disable();

		preempt_disable();

		/* Perf needs local kick that is NMI safe */

		/* Perf needs local kick that is NMI safe */
@@ -313,7 +311,7 @@ void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit) 
{

{

	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);

	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);





	clear_bit(bit, &ts->tick_dep_mask);

	atomic_andnot(BIT(bit), &ts->tick_dep_mask);

}

}





/*

/*
@@ -331,7 +329,7 @@ void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit) 




void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit)

void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit)

{

{

	clear_bit(bit, &tsk->tick_dep_mask);

	atomic_andnot(BIT(bit), &tsk->tick_dep_mask);

}

}





/*

/*
@@ -345,7 +343,7 @@ void tick_nohz_dep_set_signal(struct signal_struct *sig, enum tick_dep_bits bit) 




void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)

void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)

{

{

	clear_bit(bit, &sig->tick_dep_mask);

	atomic_andnot(BIT(bit), &sig->tick_dep_mask);

}

}





/*

/*
@@ -366,7 +364,8 @@ void __tick_nohz_task_switch(void) 
	ts = this_cpu_ptr(&tick_cpu_sched);

	ts = this_cpu_ptr(&tick_cpu_sched);





	if (ts->tick_stopped) {

	if (ts->tick_stopped) {

		if (current->tick_dep_mask || current->signal->tick_dep_mask)

		if (atomic_read(&current->tick_dep_mask) ||

		    atomic_read(&current->signal->tick_dep_mask))

			tick_nohz_full_kick();

			tick_nohz_full_kick();

	}

	}

out:

out:

kernel/time/tick-sched.h

+1 −1
Original line number Original line Diff line number Diff line
@@ -60,7 +60,7 @@ struct tick_sched { 
	u64				next_timer;

	u64				next_timer;

	ktime_t				idle_expires;

	ktime_t				idle_expires;

	int				do_timer_last;

	int				do_timer_last;

	unsigned long			tick_dep_mask;

	atomic_t			tick_dep_mask;

};

};





extern struct tick_sched *tick_get_tick_sched(int cpu);

extern struct tick_sched *tick_get_tick_sched(int cpu);

CRA Git | Maintained and supported by SUSTech CRA and CCSE