Commit d4f7dae8 authored by Frederic Weisbecker's avatar Frederic Weisbecker Committed by Thomas Gleixner
Browse files

timers: Spare timer softirq until next expiry 



Now that the core timer infrastructure doesn't depend anymore on
periodic base->clk increments, even when the CPU is not in NO_HZ mode,
timer softirqs can be skipped until there are timers to expire.

Some spurious softirqs can still remain since base->next_expiry doesn't
keep track of canceled timers but this still reduces the number of softirqs
significantly: ~15 times less for HZ=1000 and ~5 times less for HZ=100.

Signed-off-by: default avatarFrederic Weisbecker <frederic@kernel.org>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Tested-by: default avatarJuri Lelli <juri.lelli@redhat.com>
Link: https://lkml.kernel.org/r/20200717140551.29076-11-frederic@kernel.org
parent 1f8a4212

kernel/time/timer.c

+8 −41
Original line number Diff line number Diff line
@@ -1458,10 +1458,10 @@ static void expire_timers(struct timer_base *base, struct hlist_head *head) 
	}

}



static int __collect_expired_timers(struct timer_base *base,

static int collect_expired_timers(struct timer_base *base,

				  struct hlist_head *heads)

{

	unsigned long clk = base->clk;

	unsigned long clk = base->clk = base->next_expiry;

	struct hlist_head *vec;

	int i, levels = 0;

	unsigned int idx;
@@ -1684,40 +1684,6 @@ void timer_clear_idle(void) 
	 */

	base->is_idle = false;

}



static int collect_expired_timers(struct timer_base *base,

				  struct hlist_head *heads)

{

	unsigned long now = READ_ONCE(jiffies);



	/*

	 * NOHZ optimization. After a long idle sleep we need to forward the

	 * base to current jiffies. Avoid a loop by searching the bitfield for

	 * the next expiring timer.

	 */

	if ((long)(now - base->clk) > 2) {

		/*

		 * If the next timer is ahead of time forward to current

		 * jiffies, otherwise forward to the next expiry time:

		 */

		if (time_after(base->next_expiry, now)) {

			/*

			 * The call site will increment base->clk and then

			 * terminate the expiry loop immediately.

			 */

			base->clk = now;

			return 0;

		}

		base->clk = base->next_expiry;

	}

	return __collect_expired_timers(base, heads);

}

#else

static inline int collect_expired_timers(struct timer_base *base,

					 struct hlist_head *heads)

{

	return __collect_expired_timers(base, heads);

}

#endif



/*
@@ -1750,7 +1716,7 @@ static inline void __run_timers(struct timer_base *base) 
	struct hlist_head heads[LVL_DEPTH];

	int levels;



	if (!time_after_eq(jiffies, base->clk))

	if (time_before(jiffies, base->next_expiry))

		return;



	timer_base_lock_expiry(base);
@@ -1763,7 +1729,8 @@ static inline void __run_timers(struct timer_base *base) 
	 */

	base->must_forward_clk = false;



	while (time_after_eq(jiffies, base->clk)) {

	while (time_after_eq(jiffies, base->clk) &&

	       time_after_eq(jiffies, base->next_expiry)) {



		levels = collect_expired_timers(base, heads);

		base->clk++;
@@ -1798,12 +1765,12 @@ void run_local_timers(void) 


	hrtimer_run_queues();

	/* Raise the softirq only if required. */

	if (time_before(jiffies, base->clk)) {

	if (time_before(jiffies, base->next_expiry)) {

		if (!IS_ENABLED(CONFIG_NO_HZ_COMMON))

			return;

		/* CPU is awake, so check the deferrable base. */

		base++;

		if (time_before(jiffies, base->clk))

		if (time_before(jiffies, base->next_expiry))

			return;

	}

	raise_softirq(TIMER_SOFTIRQ);

CRA Git | Maintained and supported by SUSTech CRA and CCSE