generic-ipi: remove kmalloc()

#include <linux/rcupdate.h>

#include <linux/rculist.h>

#include <linux/smp.h>

#include <linux/cpu.h>

static DEFINE_PER_CPU(struct call_single_queue, call_single_queue);

static LIST_HEAD(call_function_queue);

__cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock);

static struct {

	struct list_head	queue;

	spinlock_t		lock;

} call_function __cacheline_aligned_in_smp = {

	.queue = LIST_HEAD_INIT(call_function.queue),

	.lock  = __SPIN_LOCK_UNLOCKED(call_function.lock),

};

enum {

	CSD_FLAG_WAIT		= 0x01,

	CSD_FLAG_ALLOC		= 0x02,

	CSD_FLAG_LOCK		= 0x04,

	CSD_FLAG_LOCK		= 0x02,

};

struct call_function_data {

	struct call_single_data csd;

	spinlock_t lock;

	unsigned int refs;

	struct rcu_head rcu_head;

	unsigned long cpumask_bits[];

	cpumask_var_t cpumask;

};

struct call_single_queue {

	spinlock_t lock;

};

static DEFINE_PER_CPU(struct call_function_data, cfd_data) = {

	.lock = __SPIN_LOCK_UNLOCKED(cfd_data.lock),

};

static int

hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)

{

	long cpu = (long)hcpu;

	struct call_function_data *cfd = &per_cpu(cfd_data, cpu);

	switch (action) {

	case CPU_UP_PREPARE:

	case CPU_UP_PREPARE_FROZEN:

		if (!alloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,

				cpu_to_node(cpu)))

			return NOTIFY_BAD;

		break;

#ifdef CONFIG_CPU_HOTPLUG

	case CPU_UP_CANCELED:

	case CPU_UP_CANCELED_FROZEN:

	case CPU_DEAD:

	case CPU_DEAD_FROZEN:

		free_cpumask_var(cfd->cpumask);

		break;

#endif

	};

	return NOTIFY_OK;

}

static struct notifier_block __cpuinitdata hotplug_cfd_notifier = {

	.notifier_call = hotplug_cfd,

};

static int __cpuinit init_call_single_data(void)

{

	void *cpu = (void *)(long)smp_processor_id();

	int i;

	for_each_possible_cpu(i) {

		spin_lock_init(&q->lock);

		INIT_LIST_HEAD(&q->list);

	}

	hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu);

	register_cpu_notifier(&hotplug_cfd_notifier);

	return 0;

}

early_initcall(init_call_single_data);

static void csd_flag_wait(struct call_single_data *data)

/*

 * csd_wait/csd_complete are used for synchronous ipi calls

 */

static void csd_wait_prepare(struct call_single_data *data)

{

	/* Wait for response */

	do {

		if (!(data->flags & CSD_FLAG_WAIT))

			break;

	data->flags |= CSD_FLAG_WAIT;

}

static void csd_complete(struct call_single_data *data)

{

	if (data->flags & CSD_FLAG_WAIT) {

		/*

		 * ensure we're all done before saying we are

		 */

		smp_mb();

		data->flags &= ~CSD_FLAG_WAIT;

	}

}

static void csd_wait(struct call_single_data *data)

{

	while (data->flags & CSD_FLAG_WAIT)

		cpu_relax();

	} while (1);

}

/*

 * csd_lock/csd_unlock used to serialize access to per-cpu csd resources

 *

 * For non-synchronous ipi calls the csd can still be in use by the previous

 * function call. For multi-cpu calls its even more interesting as we'll have

 * to ensure no other cpu is observing our csd.

 */

static void csd_lock(struct call_single_data *data)

{

	while (data->flags & CSD_FLAG_LOCK)

		cpu_relax();

	data->flags = CSD_FLAG_LOCK;

	/*

	 * prevent CPU from reordering the above assignment to ->flags

	 * with any subsequent assignments to other fields of the

	 * specified call_single_data structure.

	 */

	smp_mb();

}

static void csd_unlock(struct call_single_data *data)

{

	WARN_ON(!(data->flags & CSD_FLAG_LOCK));

	/*

	 * ensure we're all done before releasing data

	 */

	smp_mb();

	data->flags &= ~CSD_FLAG_LOCK;

}

/*

		arch_send_call_function_single_ipi(cpu);

	if (wait)

		csd_flag_wait(data);

}

static void rcu_free_call_data(struct rcu_head *head)

{

	struct call_function_data *data;

	data = container_of(head, struct call_function_data, rcu_head);

	kfree(data);

		csd_wait(data);

}

/*

	 * It's ok to use list_for_each_rcu() here even though we may delete

	 * 'pos', since list_del_rcu() doesn't clear ->next

	 */

	rcu_read_lock();

	list_for_each_entry_rcu(data, &call_function_queue, csd.list) {

	list_for_each_entry_rcu(data, &call_function.queue, csd.list) {

		int refs;

		if (!cpumask_test_cpu(cpu, to_cpumask(data->cpumask_bits)))

		spin_lock(&data->lock);

		if (!cpumask_test_cpu(cpu, data->cpumask)) {

			spin_unlock(&data->lock);

			continue;

		}

		cpumask_clear_cpu(cpu, data->cpumask);

		spin_unlock(&data->lock);

		data->csd.func(data->csd.info);

		spin_lock(&data->lock);

		cpumask_clear_cpu(cpu, to_cpumask(data->cpumask_bits));

		WARN_ON(data->refs == 0);

		data->refs--;

		refs = data->refs;

		refs = --data->refs;

		if (!refs) {

			spin_lock(&call_function.lock);

			list_del_rcu(&data->csd.list);

			spin_unlock(&call_function.lock);

		}

		spin_unlock(&data->lock);

		if (refs)

			continue;

		spin_lock(&call_function_lock);

		list_del_rcu(&data->csd.list);

		spin_unlock(&call_function_lock);

		if (data->csd.flags & CSD_FLAG_WAIT) {

			/*

			 * serialize stores to data with the flag clear

			 * and wakeup

			 */

			smp_wmb();

			data->csd.flags &= ~CSD_FLAG_WAIT;

		csd_complete(&data->csd);

		csd_unlock(&data->csd);

	}

		if (data->csd.flags & CSD_FLAG_ALLOC)

			call_rcu(&data->rcu_head, rcu_free_call_data);

	}

	rcu_read_unlock();

	put_cpu();

}

		data->func(data->info);

		if (data_flags & CSD_FLAG_WAIT) {

			smp_wmb();

			data->flags &= ~CSD_FLAG_WAIT;

		} else if (data_flags & CSD_FLAG_LOCK) {

			smp_wmb();

			data->flags &= ~CSD_FLAG_LOCK;

		} else if (data_flags & CSD_FLAG_ALLOC)

			kfree(data);

		if (data_flags & CSD_FLAG_WAIT)

			csd_complete(data);

		/*

		 * Unlocked CSDs are valid through generic_exec_single()

		 */

		if (data_flags & CSD_FLAG_LOCK)

			csd_unlock(data);

	}

}

int smp_call_function_single(int cpu, void (*func) (void *info), void *info,

			     int wait)

{

	struct call_single_data d;

	struct call_single_data d = {

		.flags = 0,

	};

	unsigned long flags;

	/* prevent preemption and reschedule on another processor,

	   as well as CPU removal */

			/*

			 * We are calling a function on a single CPU

			 * and we are not going to wait for it to finish.

			 * We first try to allocate the data, but if we

			 * fail, we fall back to use a per cpu data to pass

			 * the information to that CPU. Since all callers

			 * of this code will use the same data, we must

			 * synchronize the callers to prevent a new caller

			 * from corrupting the data before the callee

			 * can access it.

			 * We use a per cpu data to pass the information to

			 * that CPU. Since all callers of this code will

			 * use the same data, we must synchronize the

			 * callers to prevent a new caller from corrupting

			 * the data before the callee can access it.

			 *

			 * The CSD_FLAG_LOCK is used to let us know when

			 * the IPI handler is done with the data.

			 * will make sure the callee is done with the

			 * data before a new caller will use it.

			 */

			data = kmalloc(sizeof(*data), GFP_ATOMIC);

			if (data)

				data->flags = CSD_FLAG_ALLOC;

			else {

				data = &per_cpu(csd_data, me);

				while (data->flags & CSD_FLAG_LOCK)

					cpu_relax();

				data->flags = CSD_FLAG_LOCK;

			}

			data = &__get_cpu_var(csd_data);

			csd_lock(data);

		} else {

			data = &d;

			data->flags = CSD_FLAG_WAIT;

			csd_wait_prepare(data);

		}

		data->func = func;

{

	struct call_function_data *data;

	unsigned long flags;

	int cpu, next_cpu;

	int cpu, next_cpu, me = smp_processor_id();

	/* Can deadlock when called with interrupts disabled */

	WARN_ON(irqs_disabled());

	/* So, what's a CPU they want?  Ignoring this one. */

	cpu = cpumask_first_and(mask, cpu_online_mask);

	if (cpu == smp_processor_id())

	if (cpu == me)

		cpu = cpumask_next_and(cpu, mask, cpu_online_mask);

	/* No online cpus?  We're done. */

	if (cpu >= nr_cpu_ids)

	/* Do we have another CPU which isn't us? */

	next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);

	if (next_cpu == smp_processor_id())

	if (next_cpu == me)

		next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);

	/* Fastpath: do that cpu by itself. */

		return;

	}

	data = kmalloc(sizeof(*data) + cpumask_size(), GFP_ATOMIC);

	if (unlikely(!data)) {

		/* Slow path. */

		for_each_online_cpu(cpu) {

			if (cpu == smp_processor_id())

				continue;

			if (cpumask_test_cpu(cpu, mask))

				smp_call_function_single(cpu, func, info, wait);

		}

		return;

	}

	data = &__get_cpu_var(cfd_data);

	csd_lock(&data->csd);

	spin_lock_init(&data->lock);

	data->csd.flags = CSD_FLAG_ALLOC;

	spin_lock_irqsave(&data->lock, flags);

	if (wait)

		data->csd.flags |= CSD_FLAG_WAIT;

		csd_wait_prepare(&data->csd);

	data->csd.func = func;

	data->csd.info = info;

	cpumask_and(to_cpumask(data->cpumask_bits), mask, cpu_online_mask);

	cpumask_clear_cpu(smp_processor_id(), to_cpumask(data->cpumask_bits));

	data->refs = cpumask_weight(to_cpumask(data->cpumask_bits));

	cpumask_and(data->cpumask, mask, cpu_online_mask);

	cpumask_clear_cpu(me, data->cpumask);

	data->refs = cpumask_weight(data->cpumask);

	spin_lock_irqsave(&call_function_lock, flags);

	list_add_tail_rcu(&data->csd.list, &call_function_queue);

	spin_unlock_irqrestore(&call_function_lock, flags);

	spin_lock(&call_function.lock);

	/*

	 * Place entry at the _HEAD_ of the list, so that any cpu still

	 * observing the entry in generic_smp_call_function_interrupt() will

	 * not miss any other list entries.

	 */

	list_add_rcu(&data->csd.list, &call_function.queue);

	spin_unlock(&call_function.lock);

	spin_unlock_irqrestore(&data->lock, flags);

	/*

	 * Make the list addition visible before sending the ipi.

	smp_mb();

	/* Send a message to all CPUs in the map */

	arch_send_call_function_ipi_mask(to_cpumask(data->cpumask_bits));

	arch_send_call_function_ipi_mask(data->cpumask);

	/* optionally wait for the CPUs to complete */

	if (wait)

		csd_flag_wait(&data->csd);

		csd_wait(&data->csd);

}

EXPORT_SYMBOL(smp_call_function_many);

void ipi_call_lock(void)

{

	spin_lock(&call_function_lock);

	spin_lock(&call_function.lock);

}

void ipi_call_unlock(void)

{

	spin_unlock(&call_function_lock);

	spin_unlock(&call_function.lock);

}

void ipi_call_lock_irq(void)

{

	spin_lock_irq(&call_function_lock);

	spin_lock_irq(&call_function.lock);

}

void ipi_call_unlock_irq(void)

{

	spin_unlock_irq(&call_function_lock);

	spin_unlock_irq(&call_function.lock);

}