Commit a63fbed7 authored by Thomas Gleixner's avatar Thomas Gleixner
Browse files

perf/tracing/cpuhotplug: Fix locking order 



perf, tracing, kprobes and jump_labels have a gazillion of ways to create
dependency lock chains. Some of those involve nested invocations of
get_online_cpus().

The conversion of the hotplug locking to a percpu rwsem requires to avoid
such nested calls. sys_perf_event_open() protects most of the syscall logic
against cpu hotplug. This causes nested calls and lock inversions versus
ftrace and kprobes in various interesting ways.

It's impossible to move the hotplug locking to the outer end of all call
chains in the involved facilities, so the hotplug protection in
sys_perf_event_open() needs to be solved differently.

Introduce 'pmus_mutex' which protects a perf private online cpumask. This
mutex is taken when the mask is updated in the cpu hotplug callbacks and
can be taken in sys_perf_event_open() to protect the swhash setup/teardown
code and when the final judgement about a valid event has to be made.

[ tglx: Produced changelog and fixed the swhash interaction ]

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Acked-by: default avatarIngo Molnar <mingo@kernel.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Sebastian Siewior <bigeasy@linutronix.de>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Link: http://lkml.kernel.org/r/20170524081548.930941109@linutronix.de
parent fdaf0a51

include/linux/perf_event.h

+2 −0
Original line number Original line Diff line number Diff line
@@ -801,6 +801,8 @@ struct perf_cpu_context { 




	struct list_head		sched_cb_entry;

	struct list_head		sched_cb_entry;

	int				sched_cb_usage;

	int				sched_cb_usage;



	int				online;

};

};





struct perf_output_handle {

struct perf_output_handle {

kernel/events/core.c

+76 −30
Original line number Original line Diff line number Diff line
@@ -389,6 +389,7 @@ static atomic_t nr_switch_events __read_mostly; 
static LIST_HEAD(pmus);

static LIST_HEAD(pmus);

static DEFINE_MUTEX(pmus_lock);

static DEFINE_MUTEX(pmus_lock);

static struct srcu_struct pmus_srcu;

static struct srcu_struct pmus_srcu;

static cpumask_var_t perf_online_mask;





/*

/*

 * perf event paranoia level:

 * perf event paranoia level:
@@ -3812,14 +3813,6 @@ find_get_context(struct pmu *pmu, struct task_struct *task, 
		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))

		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))

			return ERR_PTR(-EACCES);

			return ERR_PTR(-EACCES);





		/*

		 * We could be clever and allow to attach a event to an

		 * offline CPU and activate it when the CPU comes up, but

		 * that's for later.

		 */

		if (!cpu_online(cpu))

			return ERR_PTR(-ENODEV);



		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);

		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);

		ctx = &cpuctx->ctx;

		ctx = &cpuctx->ctx;

		get_ctx(ctx);

		get_ctx(ctx);
@@ -7703,7 +7696,8 @@ static int swevent_hlist_get_cpu(int cpu) 
	int err = 0;

	int err = 0;





	mutex_lock(&swhash->hlist_mutex);

	mutex_lock(&swhash->hlist_mutex);

	if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {

	if (!swevent_hlist_deref(swhash) &&

	    cpumask_test_cpu(cpu, perf_online_mask)) {

		struct swevent_hlist *hlist;

		struct swevent_hlist *hlist;





		hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);

		hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
@@ -7724,7 +7718,7 @@ static int swevent_hlist_get(void) 
{

{

	int err, cpu, failed_cpu;

	int err, cpu, failed_cpu;





	get_online_cpus();

	mutex_lock(&pmus_lock);

	for_each_possible_cpu(cpu) {

	for_each_possible_cpu(cpu) {

		err = swevent_hlist_get_cpu(cpu);

		err = swevent_hlist_get_cpu(cpu);

		if (err) {

		if (err) {
@@ -7732,8 +7726,7 @@ static int swevent_hlist_get(void) 
			goto fail;

			goto fail;

		}

		}

	}

	}

	put_online_cpus();

	mutex_unlock(&pmus_lock);



	return 0;

	return 0;

fail:

fail:

	for_each_possible_cpu(cpu) {

	for_each_possible_cpu(cpu) {
@@ -7741,8 +7734,7 @@ fail: 
			break;

			break;

		swevent_hlist_put_cpu(cpu);

		swevent_hlist_put_cpu(cpu);

	}

	}



	mutex_unlock(&pmus_lock);

	put_online_cpus();

	return err;

	return err;

}

}
@@ -8920,7 +8912,7 @@ perf_event_mux_interval_ms_store(struct device *dev, 
	pmu->hrtimer_interval_ms = timer;

	pmu->hrtimer_interval_ms = timer;





	/* update all cpuctx for this PMU */

	/* update all cpuctx for this PMU */

	get_online_cpus();

	cpus_read_lock();

	for_each_online_cpu(cpu) {

	for_each_online_cpu(cpu) {

		struct perf_cpu_context *cpuctx;

		struct perf_cpu_context *cpuctx;

		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);

		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
@@ -8929,7 +8921,7 @@ perf_event_mux_interval_ms_store(struct device *dev, 
		cpu_function_call(cpu,

		cpu_function_call(cpu,

			(remote_function_f)perf_mux_hrtimer_restart, cpuctx);

			(remote_function_f)perf_mux_hrtimer_restart, cpuctx);

	}

	}

	put_online_cpus();

	cpus_read_unlock();

	mutex_unlock(&mux_interval_mutex);

	mutex_unlock(&mux_interval_mutex);





	return count;

	return count;
@@ -9059,6 +9051,7 @@ skip_type: 
		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);

		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);

		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);

		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);

		cpuctx->ctx.pmu = pmu;

		cpuctx->ctx.pmu = pmu;

		cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask);





		__perf_mux_hrtimer_init(cpuctx, cpu);

		__perf_mux_hrtimer_init(cpuctx, cpu);

	}

	}
@@ -9882,12 +9875,10 @@ SYSCALL_DEFINE5(perf_event_open, 
		goto err_task;

		goto err_task;

	}

	}





	get_online_cpus();



	if (task) {

	if (task) {

		err = mutex_lock_interruptible(&task->signal->cred_guard_mutex);

		err = mutex_lock_interruptible(&task->signal->cred_guard_mutex);

		if (err)

		if (err)

			goto err_cpus;

			goto err_cred;





		/*

		/*

		 * Reuse ptrace permission checks for now.

		 * Reuse ptrace permission checks for now.
@@ -10073,6 +10064,23 @@ SYSCALL_DEFINE5(perf_event_open, 
		goto err_locked;

		goto err_locked;

	}

	}





	if (!task) {

		/*

		 * Check if the @cpu we're creating an event for is online.

		 *

		 * We use the perf_cpu_context::ctx::mutex to serialize against

		 * the hotplug notifiers. See perf_event_{init,exit}_cpu().

		 */

		struct perf_cpu_context *cpuctx =

			container_of(ctx, struct perf_cpu_context, ctx);



		if (!cpuctx->online) {

			err = -ENODEV;

			goto err_locked;

		}

	}





	/*

	/*

	 * Must be under the same ctx::mutex as perf_install_in_context(),

	 * Must be under the same ctx::mutex as perf_install_in_context(),

	 * because we need to serialize with concurrent event creation.

	 * because we need to serialize with concurrent event creation.
@@ -10162,8 +10170,6 @@ SYSCALL_DEFINE5(perf_event_open, 
		put_task_struct(task);

		put_task_struct(task);

	}

	}





	put_online_cpus();



	mutex_lock(&current->perf_event_mutex);

	mutex_lock(&current->perf_event_mutex);

	list_add_tail(&event->owner_entry, &current->perf_event_list);

	list_add_tail(&event->owner_entry, &current->perf_event_list);

	mutex_unlock(&current->perf_event_mutex);

	mutex_unlock(&current->perf_event_mutex);
@@ -10197,8 +10203,6 @@ err_alloc: 
err_cred:

err_cred:

	if (task)

	if (task)

		mutex_unlock(&task->signal->cred_guard_mutex);

		mutex_unlock(&task->signal->cred_guard_mutex);

err_cpus:

	put_online_cpus();

err_task:

err_task:

	if (task)

	if (task)

		put_task_struct(task);

		put_task_struct(task);
@@ -10253,6 +10257,21 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, 
		goto err_unlock;

		goto err_unlock;

	}

	}





	if (!task) {

		/*

		 * Check if the @cpu we're creating an event for is online.

		 *

		 * We use the perf_cpu_context::ctx::mutex to serialize against

		 * the hotplug notifiers. See perf_event_{init,exit}_cpu().

		 */

		struct perf_cpu_context *cpuctx =

			container_of(ctx, struct perf_cpu_context, ctx);

		if (!cpuctx->online) {

			err = -ENODEV;

			goto err_unlock;

		}

	}



	if (!exclusive_event_installable(event, ctx)) {

	if (!exclusive_event_installable(event, ctx)) {

		err = -EBUSY;

		err = -EBUSY;

		goto err_unlock;

		goto err_unlock;
@@ -10920,6 +10939,8 @@ static void __init perf_event_init_all_cpus(void) 
	struct swevent_htable *swhash;

	struct swevent_htable *swhash;

	int cpu;

	int cpu;





	zalloc_cpumask_var(&perf_online_mask, GFP_KERNEL);



	for_each_possible_cpu(cpu) {

	for_each_possible_cpu(cpu) {

		swhash = &per_cpu(swevent_htable, cpu);

		swhash = &per_cpu(swevent_htable, cpu);

		mutex_init(&swhash->hlist_mutex);

		mutex_init(&swhash->hlist_mutex);
@@ -10935,7 +10956,7 @@ static void __init perf_event_init_all_cpus(void) 
	}

	}

}

}





int perf_event_init_cpu(unsigned int cpu)

void perf_swevent_init_cpu(unsigned int cpu)

{

{

	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);

	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
@@ -10948,7 +10969,6 @@ int perf_event_init_cpu(unsigned int cpu) 
		rcu_assign_pointer(swhash->swevent_hlist, hlist);

		rcu_assign_pointer(swhash->swevent_hlist, hlist);

	}

	}

	mutex_unlock(&swhash->hlist_mutex);

	mutex_unlock(&swhash->hlist_mutex);

	return 0;

}

}





#if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC_CORE

#if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC_CORE
@@ -10966,19 +10986,22 @@ static void __perf_event_exit_context(void *__info) 




static void perf_event_exit_cpu_context(int cpu)

static void perf_event_exit_cpu_context(int cpu)

{

{

	struct perf_cpu_context *cpuctx;

	struct perf_event_context *ctx;

	struct perf_event_context *ctx;

	struct pmu *pmu;

	struct pmu *pmu;

	int idx;





	idx = srcu_read_lock(&pmus_srcu);

	mutex_lock(&pmus_lock);

	list_for_each_entry_rcu(pmu, &pmus, entry) {

	list_for_each_entry(pmu, &pmus, entry) {

		ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;

		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);

		ctx = &cpuctx->ctx;





		mutex_lock(&ctx->mutex);

		mutex_lock(&ctx->mutex);

		smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);

		smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);

		cpuctx->online = 0;

		mutex_unlock(&ctx->mutex);

		mutex_unlock(&ctx->mutex);

	}

	}

	srcu_read_unlock(&pmus_srcu, idx);

	cpumask_clear_cpu(cpu, perf_online_mask);

	mutex_unlock(&pmus_lock);

}

}

#else

#else
@@ -10986,6 +11009,29 @@ static void perf_event_exit_cpu_context(int cpu) { } 




#endif

#endif





int perf_event_init_cpu(unsigned int cpu)

{

	struct perf_cpu_context *cpuctx;

	struct perf_event_context *ctx;

	struct pmu *pmu;



	perf_swevent_init_cpu(cpu);



	mutex_lock(&pmus_lock);

	cpumask_set_cpu(cpu, perf_online_mask);

	list_for_each_entry(pmu, &pmus, entry) {

		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);

		ctx = &cpuctx->ctx;



		mutex_lock(&ctx->mutex);

		cpuctx->online = 1;

		mutex_unlock(&ctx->mutex);

	}

	mutex_unlock(&pmus_lock);



	return 0;

}



int perf_event_exit_cpu(unsigned int cpu)

int perf_event_exit_cpu(unsigned int cpu)

{

{

	perf_event_exit_cpu_context(cpu);

	perf_event_exit_cpu_context(cpu);

CRA Git | Maintained and supported by SUSTech CRA and CCSE