x86/hpet: Reduce HPET counter read contention

/*

/*

 * Clock source related code

 * Clock source related code

 */

 */

#if defined(CONFIG_SMP) && defined(CONFIG_64BIT)

/*

 * Reading the HPET counter is a very slow operation. If a large number of

 * CPUs are trying to access the HPET counter simultaneously, it can cause

 * massive delay and slow down system performance dramatically. This may

 * happen when HPET is the default clock source instead of TSC. For a

 * really large system with hundreds of CPUs, the slowdown may be so

 * severe that it may actually crash the system because of a NMI watchdog

 * soft lockup, for example.

 *

 * If multiple CPUs are trying to access the HPET counter at the same time,

 * we don't actually need to read the counter multiple times. Instead, the

 * other CPUs can use the counter value read by the first CPU in the group.

 *

 * This special feature is only enabled on x86-64 systems. It is unlikely

 * that 32-bit x86 systems will have enough CPUs to require this feature

 * with its associated locking overhead. And we also need 64-bit atomic

 * read.

 *

 * The lock and the hpet value are stored together and can be read in a

 * single atomic 64-bit read. It is explicitly assumed that arch_spinlock_t

 * is 32 bits in size.

 */

union hpet_lock {

	struct {

		arch_spinlock_t lock;

		u32 value;

	};

	u64 lockval;

};

static union hpet_lock hpet __cacheline_aligned = {

	{ .lock = __ARCH_SPIN_LOCK_UNLOCKED, },

};

static cycle_t read_hpet(struct clocksource *cs)

{

	unsigned long flags;

	union hpet_lock old, new;

	BUILD_BUG_ON(sizeof(union hpet_lock) != 8);

	/*

	 * Read HPET directly if in NMI.

	 */

	if (in_nmi())

		return (cycle_t)hpet_readl(HPET_COUNTER);

	/*

	 * Read the current state of the lock and HPET value atomically.

	 */

	old.lockval = READ_ONCE(hpet.lockval);

	if (arch_spin_is_locked(&old.lock))

		goto contended;

	local_irq_save(flags);

	if (arch_spin_trylock(&hpet.lock)) {

		new.value = hpet_readl(HPET_COUNTER);

		/*

		 * Use WRITE_ONCE() to prevent store tearing.

		 */

		WRITE_ONCE(hpet.value, new.value);

		arch_spin_unlock(&hpet.lock);

		local_irq_restore(flags);

		return (cycle_t)new.value;

	}

	local_irq_restore(flags);

contended:

	/*

	 * Contended case

	 * --------------

	 * Wait until the HPET value change or the lock is free to indicate

	 * its value is up-to-date.

	 *

	 * It is possible that old.value has already contained the latest

	 * HPET value while the lock holder was in the process of releasing

	 * the lock. Checking for lock state change will enable us to return

	 * the value immediately instead of waiting for the next HPET reader

	 * to come along.

	 */

	do {

		cpu_relax();

		new.lockval = READ_ONCE(hpet.lockval);

	} while ((new.value == old.value) && arch_spin_is_locked(&new.lock));

	return (cycle_t)new.value;

}

#else

/*

 * For UP or 32-bit.

 */

static cycle_t read_hpet(struct clocksource *cs)

static cycle_t read_hpet(struct clocksource *cs)

{

{

	return (cycle_t)hpet_readl(HPET_COUNTER);

	return (cycle_t)hpet_readl(HPET_COUNTER);

}

}

#endif

static struct clocksource clocksource_hpet = {

static struct clocksource clocksource_hpet = {

	.name		= "hpet",

	.name		= "hpet",