Commit ce71b9df authored by Frederic Weisbecker's avatar Frederic Weisbecker Committed by Ingo Molnar
Browse files

tracing: Use the perf recursion protection from trace event 



When we commit a trace to perf, we first check if we are
recursing in the same buffer so that we don't mess-up the buffer
with a recursing trace. But later on, we do the same check from
perf to avoid commit recursion. The recursion check is desired
early before we touch the buffer but we want to do this check
only once.

Then export the recursion protection from perf and use it from
the trace events before submitting a trace.

v2: Put appropriate Reported-by tag

Reported-by: default avatarPeter Zijlstra <peterz@infradead.org>
Signed-off-by: default avatarFrederic Weisbecker <fweisbec@gmail.com>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Masami Hiramatsu <mhiramat@redhat.com>
Cc: Jason Baron <jbaron@redhat.com>
LKML-Reference: <1258864015-10579-1-git-send-email-fweisbec@gmail.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent e2561368

include/linux/ftrace_event.h

+2 −7
Original line number Diff line number Diff line
@@ -137,13 +137,8 @@ struct ftrace_event_call { 


#define FTRACE_MAX_PROFILE_SIZE	2048



struct perf_trace_buf {

	char	buf[FTRACE_MAX_PROFILE_SIZE];

	int	recursion;

};



extern struct perf_trace_buf	*perf_trace_buf;

extern struct perf_trace_buf	*perf_trace_buf_nmi;

extern char *perf_trace_buf;

extern char *perf_trace_buf_nmi;



#define MAX_FILTER_PRED		32

#define MAX_FILTER_STR_VAL	256	/* Should handle KSYM_SYMBOL_LEN */

include/linux/perf_event.h

+4 −0
Original line number Diff line number Diff line
@@ -874,6 +874,8 @@ extern int perf_output_begin(struct perf_output_handle *handle, 
extern void perf_output_end(struct perf_output_handle *handle);

extern void perf_output_copy(struct perf_output_handle *handle,

			     const void *buf, unsigned int len);

extern int perf_swevent_get_recursion_context(int **recursion);

extern void perf_swevent_put_recursion_context(int *recursion);

#else

static inline void

perf_event_task_sched_in(struct task_struct *task, int cpu)		{ }
@@ -902,6 +904,8 @@ static inline void perf_event_mmap(struct vm_area_struct *vma) { } 
static inline void perf_event_comm(struct task_struct *tsk)		{ }

static inline void perf_event_fork(struct task_struct *tsk)		{ }

static inline void perf_event_init(void)				{ }

static int perf_swevent_get_recursion_context(int **recursion)	{ return -1; }

static void perf_swevent_put_recursion_context(int *recursion)		{ }



#endif

include/trace/ftrace.h

+12 −11
Original line number Diff line number Diff line
@@ -724,16 +724,19 @@ __attribute__((section("_ftrace_events"))) event_##call = { \ 
static void ftrace_profile_##call(proto)				\

{									\

	struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\

	extern int perf_swevent_get_recursion_context(int **recursion); \

	extern void perf_swevent_put_recursion_context(int *recursion); \

	struct ftrace_event_call *event_call = &event_##call;		\

	extern void perf_tp_event(int, u64, u64, void *, int);		\

	struct ftrace_raw_##call *entry;				\

	struct perf_trace_buf *trace_buf;				\

	u64 __addr = 0, __count = 1;					\

	unsigned long irq_flags;					\

	struct trace_entry *ent;					\

	int __entry_size;						\

	int __data_size;						\

	char *trace_buf;						\

	char *raw_data;							\

	int *recursion;							\

	int __cpu;							\

	int pc;								\

									\
@@ -749,6 +752,10 @@ static void ftrace_profile_##call(proto) \ 
		return;							\

									\

	local_irq_save(irq_flags);					\

									\

	if (perf_swevent_get_recursion_context(&recursion))		\

		goto end_recursion;						\

									\

	__cpu = smp_processor_id();					\

									\

	if (in_nmi())							\
@@ -759,13 +766,7 @@ static void ftrace_profile_##call(proto) \ 
	if (!trace_buf)							\

		goto end;						\

									\

	trace_buf = per_cpu_ptr(trace_buf, __cpu);			\

	if (trace_buf->recursion++)					\

		goto end_recursion;					\

									\

	barrier();							\

									\

	raw_data = trace_buf->buf;					\

	raw_data = per_cpu_ptr(trace_buf, __cpu);			\

									\

	*(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL;		\

	entry = (struct ftrace_raw_##call *)raw_data;			\
@@ -780,9 +781,9 @@ static void ftrace_profile_##call(proto) \ 
	perf_tp_event(event_call->id, __addr, __count, entry,		\

			     __entry_size);				\

									\

end_recursion:								\

	trace_buf->recursion--;						\

end:								\

	perf_swevent_put_recursion_context(recursion);			\

end_recursion:									\

	local_irq_restore(irq_flags);					\

									\

}

kernel/perf_event.c

+45 −23
Original line number Diff line number Diff line
@@ -3880,34 +3880,42 @@ static void perf_swevent_ctx_event(struct perf_event_context *ctx, 
	}

}



static int *perf_swevent_recursion_context(struct perf_cpu_context *cpuctx)

/*

 * Must be called with preemption disabled

 */

int perf_swevent_get_recursion_context(int **recursion)

{

	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);



	if (in_nmi())

		return &cpuctx->recursion[3];

		*recursion = &cpuctx->recursion[3];

	else if (in_irq())

		*recursion = &cpuctx->recursion[2];

	else if (in_softirq())

		*recursion = &cpuctx->recursion[1];

	else

		*recursion = &cpuctx->recursion[0];



	if (in_irq())

		return &cpuctx->recursion[2];

	if (**recursion)

		return -1;



	if (in_softirq())

		return &cpuctx->recursion[1];

	(**recursion)++;



	return &cpuctx->recursion[0];

	return 0;

}



static void do_perf_sw_event(enum perf_type_id type, u32 event_id,

void perf_swevent_put_recursion_context(int *recursion)

{

	(*recursion)--;

}



static void __do_perf_sw_event(enum perf_type_id type, u32 event_id,

			       u64 nr, int nmi,

			       struct perf_sample_data *data,

			       struct pt_regs *regs)

{

	struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);

	int *recursion = perf_swevent_recursion_context(cpuctx);

	struct perf_event_context *ctx;



	if (*recursion)

		goto out;



	(*recursion)++;

	barrier();

	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);



	rcu_read_lock();

	perf_swevent_ctx_event(&cpuctx->ctx, type, event_id,
@@ -3920,12 +3928,25 @@ static void do_perf_sw_event(enum perf_type_id type, u32 event_id, 
	if (ctx)

		perf_swevent_ctx_event(ctx, type, event_id, nr, nmi, data, regs);

	rcu_read_unlock();

}



	barrier();

	(*recursion)--;

static void do_perf_sw_event(enum perf_type_id type, u32 event_id,

				    u64 nr, int nmi,

				    struct perf_sample_data *data,

				    struct pt_regs *regs)

{

	int *recursion;



	preempt_disable();



	if (perf_swevent_get_recursion_context(&recursion))

		goto out;



	__do_perf_sw_event(type, event_id, nr, nmi, data, regs);



	perf_swevent_put_recursion_context(recursion);

out:

	put_cpu_var(perf_cpu_context);

	preempt_enable();

}



void __perf_sw_event(u32 event_id, u64 nr, int nmi,
@@ -4159,7 +4180,8 @@ void perf_tp_event(int event_id, u64 addr, u64 count, void *record, 
	if (!regs)

		regs = task_pt_regs(current);



	do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,

	/* Trace events already protected against recursion */

	__do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,

				&data, regs);

}

EXPORT_SYMBOL_GPL(perf_tp_event);

kernel/trace/trace_event_profile.c

+8 −6
Original line number Diff line number Diff line
@@ -9,31 +9,33 @@ 
#include "trace.h"





struct perf_trace_buf *perf_trace_buf;

char *perf_trace_buf;

EXPORT_SYMBOL_GPL(perf_trace_buf);



struct perf_trace_buf *perf_trace_buf_nmi;

char *perf_trace_buf_nmi;

EXPORT_SYMBOL_GPL(perf_trace_buf_nmi);



typedef typeof(char [FTRACE_MAX_PROFILE_SIZE]) perf_trace_t ;



/* Count the events in use (per event id, not per instance) */

static int	total_profile_count;



static int ftrace_profile_enable_event(struct ftrace_event_call *event)

{

	struct perf_trace_buf *buf;

	char *buf;

	int ret = -ENOMEM;



	if (atomic_inc_return(&event->profile_count))

		return 0;



	if (!total_profile_count) {

		buf = alloc_percpu(struct perf_trace_buf);

		buf = (char *)alloc_percpu(perf_trace_t);

		if (!buf)

			goto fail_buf;



		rcu_assign_pointer(perf_trace_buf, buf);



		buf = alloc_percpu(struct perf_trace_buf);

		buf = (char *)alloc_percpu(perf_trace_t);

		if (!buf)

			goto fail_buf_nmi;
@@ -79,7 +81,7 @@ int ftrace_profile_enable(int event_id) 


static void ftrace_profile_disable_event(struct ftrace_event_call *event)

{

	struct perf_trace_buf *buf, *nmi_buf;

	char *buf, *nmi_buf;



	if (!atomic_add_negative(-1, &event->profile_count))

		return;

CRA Git | Maintained and supported by SUSTech CRA and CCSE