Commit 54fdade1 authored by Xiao Guangrong's avatar Xiao Guangrong Committed by Linus Torvalds
Browse files

generic-ipi: make struct call_function_data lockless 



This patch can remove spinlock from struct call_function_data, the
reasons are below:

1: add a new interface for cpumask named cpumask_test_and_clear_cpu(),
   it can atomically test and clear specific cpu, we can use it instead
   of cpumask_test_cpu() and cpumask_clear_cpu() and no need data->lock
   to protect those in generic_smp_call_function_interrupt().

2: in smp_call_function_many(), after csd_lock() return, the current's
   cfd_data is deleted from call_function list, so it not have race
   between other cpus, then cfs_data is only used in
   smp_call_function_many() that must disable preemption and not from
   a hardware interrupthandler or from a bottom half handler to call,
   only the correspond cpu can use it, so it not have race in current
   cpu, no need cfs_data->lock to protect it.

3: after 1 and 2, cfs_data->lock is only use to protect cfs_data->refs in
   generic_smp_call_function_interrupt(), so we can define cfs_data->refs
   to atomic_t, and no need cfs_data->lock any more.

Signed-off-by: default avatarXiao Guangrong <xiaoguangrong@cn.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Jens Axboe <jens.axboe@oracle.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Peter Zijlstra <peterz@infradead.org>
Acked-by: default avatarRusty Russell <rusty@rustcorp.com.au>
[akpm@linux-foundation.org: use atomic_dec_return()]
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 5c725138

include/linux/cpumask.h

+12 −0
Original line number Diff line number Diff line
@@ -714,6 +714,18 @@ static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) 
	return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));

}



/**

 * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask

 * @cpu: cpu number (< nr_cpu_ids)

 * @cpumask: the cpumask pointer

 *

 * test_and_clear_bit wrapper for cpumasks.

 */

static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)

{

	return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));

}



/**

 * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask

 * @dstp: the cpumask pointer

kernel/smp.c

+8 −21
Original line number Diff line number Diff line
@@ -29,8 +29,7 @@ enum { 


struct call_function_data {

	struct call_single_data	csd;

	spinlock_t		lock;

	unsigned int		refs;

	atomic_t		refs;

	cpumask_var_t		cpumask;

};
@@ -39,9 +38,7 @@ struct call_single_queue { 
	spinlock_t		lock;

};



static DEFINE_PER_CPU(struct call_function_data, cfd_data) = {

	.lock			= __SPIN_LOCK_UNLOCKED(cfd_data.lock),

};

static DEFINE_PER_CPU(struct call_function_data, cfd_data);



static int

hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
@@ -196,25 +193,18 @@ void generic_smp_call_function_interrupt(void) 
	list_for_each_entry_rcu(data, &call_function.queue, csd.list) {

		int refs;



		spin_lock(&data->lock);

		if (!cpumask_test_cpu(cpu, data->cpumask)) {

			spin_unlock(&data->lock);

		if (!cpumask_test_and_clear_cpu(cpu, data->cpumask))

			continue;

		}

		cpumask_clear_cpu(cpu, data->cpumask);

		spin_unlock(&data->lock);



		data->csd.func(data->csd.info);



		spin_lock(&data->lock);

		WARN_ON(data->refs == 0);

		refs = --data->refs;

		refs = atomic_dec_return(&data->refs);

		WARN_ON(refs < 0);

		if (!refs) {

			spin_lock(&call_function.lock);

			list_del_rcu(&data->csd.list);

			spin_unlock(&call_function.lock);

		}

		spin_unlock(&data->lock);



		if (refs)

			continue;
@@ -419,23 +409,20 @@ void smp_call_function_many(const struct cpumask *mask, 
	data = &__get_cpu_var(cfd_data);

	csd_lock(&data->csd);



	spin_lock_irqsave(&data->lock, flags);

	data->csd.func = func;

	data->csd.info = info;

	cpumask_and(data->cpumask, mask, cpu_online_mask);

	cpumask_clear_cpu(this_cpu, data->cpumask);

	data->refs = cpumask_weight(data->cpumask);

	atomic_set(&data->refs, cpumask_weight(data->cpumask));



	spin_lock(&call_function.lock);

	spin_lock_irqsave(&call_function.lock, flags);

	/*

	 * Place entry at the _HEAD_ of the list, so that any cpu still

	 * observing the entry in generic_smp_call_function_interrupt()

	 * will not miss any other list entries:

	 */

	list_add_rcu(&data->csd.list, &call_function.queue);

	spin_unlock(&call_function.lock);



	spin_unlock_irqrestore(&data->lock, flags);

	spin_unlock_irqrestore(&call_function.lock, flags);



	/*

	 * Make the list addition visible before sending the ipi.

CRA Git | Maintained and supported by SUSTech CRA and CCSE