Commit e5d4d175 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Peter Zijlstra
Browse files

timekeeping: Split jiffies seqlock 



seqlock consists of a sequence counter and a spinlock_t which is used to
serialize the writers. spinlock_t is substituted by a "sleeping" spinlock
on PREEMPT_RT enabled kernels which breaks the usage in the timekeeping
code as the writers are executed in hard interrupt and therefore
non-preemptible context even on PREEMPT_RT.

The spinlock in seqlock cannot be unconditionally replaced by a
raw_spinlock_t as many seqlock users have nesting spinlock sections or
other code which is not suitable to run in truly atomic context on RT.

Instead of providing a raw_seqlock API for a single use case, open code the
seqlock for the jiffies use case and implement it with a raw_spinlock_t and
a sequence counter.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20200321113242.120587764@linutronix.de
parent 919e9e63

kernel/time/jiffies.c

+4 −3
Original line number Diff line number Diff line
@@ -58,7 +58,8 @@ static struct clocksource clocksource_jiffies = { 
	.max_cycles	= 10,

};



__cacheline_aligned_in_smp DEFINE_SEQLOCK(jiffies_lock);

__cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(jiffies_lock);

__cacheline_aligned_in_smp seqcount_t jiffies_seq;



#if (BITS_PER_LONG < 64)

u64 get_jiffies_64(void)
@@ -67,9 +68,9 @@ u64 get_jiffies_64(void) 
	u64 ret;



	do {

		seq = read_seqbegin(&jiffies_lock);

		seq = read_seqcount_begin(&jiffies_seq);

		ret = jiffies_64;

	} while (read_seqretry(&jiffies_lock, seq));

	} while (read_seqcount_retry(&jiffies_seq, seq));

	return ret;

}

EXPORT_SYMBOL(get_jiffies_64);

kernel/time/tick-common.c

+6 −4
Original line number Diff line number Diff line
@@ -84,13 +84,15 @@ int tick_is_oneshot_available(void) 
static void tick_periodic(int cpu)

{

	if (tick_do_timer_cpu == cpu) {

		write_seqlock(&jiffies_lock);

		raw_spin_lock(&jiffies_lock);

		write_seqcount_begin(&jiffies_seq);



		/* Keep track of the next tick event */

		tick_next_period = ktime_add(tick_next_period, tick_period);



		do_timer(1);

		write_sequnlock(&jiffies_lock);

		write_seqcount_end(&jiffies_seq);

		raw_spin_unlock(&jiffies_lock);

		update_wall_time();

	}
@@ -162,9 +164,9 @@ void tick_setup_periodic(struct clock_event_device *dev, int broadcast) 
		ktime_t next;



		do {

			seq = read_seqbegin(&jiffies_lock);

			seq = read_seqcount_begin(&jiffies_seq);

			next = tick_next_period;

		} while (read_seqretry(&jiffies_lock, seq));

		} while (read_seqcount_retry(&jiffies_seq, seq));



		clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);

kernel/time/tick-sched.c

+12 −7
Original line number Diff line number Diff line
@@ -65,7 +65,8 @@ static void tick_do_update_jiffies64(ktime_t now) 
		return;



	/* Reevaluate with jiffies_lock held */

	write_seqlock(&jiffies_lock);

	raw_spin_lock(&jiffies_lock);

	write_seqcount_begin(&jiffies_seq);



	delta = ktime_sub(now, last_jiffies_update);

	if (delta >= tick_period) {
@@ -91,10 +92,12 @@ static void tick_do_update_jiffies64(ktime_t now) 
		/* Keep the tick_next_period variable up to date */

		tick_next_period = ktime_add(last_jiffies_update, tick_period);

	} else {

		write_sequnlock(&jiffies_lock);

		write_seqcount_end(&jiffies_seq);

		raw_spin_unlock(&jiffies_lock);

		return;

	}

	write_sequnlock(&jiffies_lock);

	write_seqcount_end(&jiffies_seq);

	raw_spin_unlock(&jiffies_lock);

	update_wall_time();

}
@@ -105,12 +108,14 @@ static ktime_t tick_init_jiffy_update(void) 
{

	ktime_t period;



	write_seqlock(&jiffies_lock);

	raw_spin_lock(&jiffies_lock);

	write_seqcount_begin(&jiffies_seq);

	/* Did we start the jiffies update yet ? */

	if (last_jiffies_update == 0)

		last_jiffies_update = tick_next_period;

	period = last_jiffies_update;

	write_sequnlock(&jiffies_lock);

	write_seqcount_end(&jiffies_seq);

	raw_spin_unlock(&jiffies_lock);

	return period;

}
@@ -676,10 +681,10 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu) 


	/* Read jiffies and the time when jiffies were updated last */

	do {

		seq = read_seqbegin(&jiffies_lock);

		seq = read_seqcount_begin(&jiffies_seq);

		basemono = last_jiffies_update;

		basejiff = jiffies;

	} while (read_seqretry(&jiffies_lock, seq));

	} while (read_seqcount_retry(&jiffies_seq, seq));

	ts->last_jiffies = basejiff;

	ts->timer_expires_base = basemono;

kernel/time/timekeeping.c

+4 −2
Original line number Diff line number Diff line
@@ -2397,8 +2397,10 @@ EXPORT_SYMBOL(hardpps); 
 */

void xtime_update(unsigned long ticks)

{

	write_seqlock(&jiffies_lock);

	raw_spin_lock(&jiffies_lock);

	write_seqcount_begin(&jiffies_seq);

	do_timer(ticks);

	write_sequnlock(&jiffies_lock);

	write_seqcount_end(&jiffies_seq);

	raw_spin_unlock(&jiffies_lock);

	update_wall_time();

}

kernel/time/timekeeping.h

+2 −1
Original line number Diff line number Diff line
@@ -25,7 +25,8 @@ static inline void sched_clock_resume(void) { } 
extern void do_timer(unsigned long ticks);

extern void update_wall_time(void);



extern seqlock_t jiffies_lock;

extern raw_spinlock_t jiffies_lock;

extern seqcount_t jiffies_seq;



#define CS_NAME_LEN	32

CRA Git | Maintained and supported by SUSTech CRA and CCSE