Commit 1e1dcd93 authored by Alexei Starovoitov's avatar Alexei Starovoitov Committed by David S. Miller
Browse files

perf: split perf_trace_buf_prepare into alloc and update parts 



split allows to move expensive update of 'struct trace_entry' to later phase.
Repurpose unused 1st argument of perf_tp_event() to indicate event type.

While splitting use temp variable 'rctx' instead of '*rctx' to avoid
unnecessary loads done by the compiler due to -fno-strict-aliasing

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent e93735be

include/linux/perf_event.h

+1 −1
Original line number Original line Diff line number Diff line
@@ -1016,7 +1016,7 @@ static inline bool perf_paranoid_kernel(void) 
}

}





extern void perf_event_init(void);

extern void perf_event_init(void);

extern void perf_tp_event(u64 addr, u64 count, void *record,

extern void perf_tp_event(u16 event_type, u64 count, void *record,

			  int entry_size, struct pt_regs *regs,

			  int entry_size, struct pt_regs *regs,

			  struct hlist_head *head, int rctx,

			  struct hlist_head *head, int rctx,

			  struct task_struct *task);

			  struct task_struct *task);

include/linux/trace_events.h

+4 −4
Original line number Original line Diff line number Diff line
@@ -605,15 +605,15 @@ extern void perf_trace_del(struct perf_event *event, int flags); 
extern int  ftrace_profile_set_filter(struct perf_event *event, int event_id,

extern int  ftrace_profile_set_filter(struct perf_event *event, int event_id,

				     char *filter_str);

				     char *filter_str);

extern void ftrace_profile_free_filter(struct perf_event *event);

extern void ftrace_profile_free_filter(struct perf_event *event);

extern void *perf_trace_buf_prepare(int size, unsigned short type,

void perf_trace_buf_update(void *record, u16 type);

				    struct pt_regs **regs, int *rctxp);

void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp);





static inline void

static inline void

perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,

perf_trace_buf_submit(void *raw_data, int size, int rctx, u16 type,

		       u64 count, struct pt_regs *regs, void *head,

		       u64 count, struct pt_regs *regs, void *head,

		       struct task_struct *task)

		       struct task_struct *task)

{

{

	perf_tp_event(addr, count, raw_data, size, regs, head, rctx, task);

	perf_tp_event(type, count, raw_data, size, regs, head, rctx, task);

}

}

#endif

#endif

include/trace/perf.h

+4 −4
Original line number Original line Diff line number Diff line
@@ -53,8 +53,7 @@ perf_trace_##call(void *__data, proto) \ 
			     sizeof(u64));				\

			     sizeof(u64));				\

	__entry_size -= sizeof(u32);					\

	__entry_size -= sizeof(u32);					\

									\

									\

	entry = perf_trace_buf_prepare(__entry_size,			\

	entry = perf_trace_buf_alloc(__entry_size, &__regs, &rctx);	\

			event_call->event.type, &__regs, &rctx);	\

	if (!entry)							\

	if (!entry)							\

		return;							\

		return;							\

									\

									\
@@ -64,8 +63,9 @@ perf_trace_##call(void *__data, proto) \ 
									\

									\

	{ assign; }							\

	{ assign; }							\

									\

									\

	perf_trace_buf_submit(entry, __entry_size, rctx, 0,		\

	perf_trace_buf_submit(entry, __entry_size, rctx,		\

		__count, __regs, head, __task);				\

			      event_call->event.type, __count, __regs,	\

			      head, __task);				\

}

}





/*

/*

kernel/events/core.c

+4 −2
Original line number Original line Diff line number Diff line
@@ -6987,7 +6987,7 @@ static int perf_tp_event_match(struct perf_event *event, 
	return 1;

	return 1;

}

}





void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,

void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,

		   struct pt_regs *regs, struct hlist_head *head, int rctx,

		   struct pt_regs *regs, struct hlist_head *head, int rctx,

		   struct task_struct *task)

		   struct task_struct *task)

{

{
@@ -6999,9 +6999,11 @@ void perf_tp_event(u64 addr, u64 count, void *record, int entry_size, 
		.data = record,

		.data = record,

	};

	};





	perf_sample_data_init(&data, addr, 0);

	perf_sample_data_init(&data, 0, 0);

	data.raw = &raw;

	data.raw = &raw;





	perf_trace_buf_update(record, event_type);



	hlist_for_each_entry_rcu(event, head, hlist_entry) {

	hlist_for_each_entry_rcu(event, head, hlist_entry) {

		if (perf_tp_event_match(event, &data, regs))

		if (perf_tp_event_match(event, &data, regs))

			perf_swevent_event(event, count, &data, regs);

			perf_swevent_event(event, count, &data, regs);

kernel/trace/trace_event_perf.c

+20 −19
Original line number Original line Diff line number Diff line
@@ -260,13 +260,10 @@ void perf_trace_del(struct perf_event *p_event, int flags) 
	tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event);

	tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event);

}

}





void *perf_trace_buf_prepare(int size, unsigned short type,

void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp)

			     struct pt_regs **regs, int *rctxp)

{

{

	struct trace_entry *entry;

	unsigned long flags;

	char *raw_data;

	char *raw_data;

	int pc;

	int rctx;





	BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));

	BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
@@ -274,28 +271,32 @@ void *perf_trace_buf_prepare(int size, unsigned short type, 
		      "perf buffer not large enough"))

		      "perf buffer not large enough"))

		return NULL;

		return NULL;





	pc = preempt_count();

	*rctxp = rctx = perf_swevent_get_recursion_context();



	if (rctx < 0)

	*rctxp = perf_swevent_get_recursion_context();

	if (*rctxp < 0)

		return NULL;

		return NULL;





	if (regs)

	if (regs)

		*regs = this_cpu_ptr(&__perf_regs[*rctxp]);

		*regs = this_cpu_ptr(&__perf_regs[rctx]);

	raw_data = this_cpu_ptr(perf_trace_buf[*rctxp]);

	raw_data = this_cpu_ptr(perf_trace_buf[rctx]);





	/* zero the dead bytes from align to not leak stack to user */

	/* zero the dead bytes from align to not leak stack to user */

	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));

	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));

	return raw_data;

}

EXPORT_SYMBOL_GPL(perf_trace_buf_alloc);

NOKPROBE_SYMBOL(perf_trace_buf_alloc);



void perf_trace_buf_update(void *record, u16 type)

{

	struct trace_entry *entry = record;

	int pc = preempt_count();

	unsigned long flags;





	entry = (struct trace_entry *)raw_data;

	local_save_flags(flags);

	local_save_flags(flags);

	tracing_generic_entry_update(entry, flags, pc);

	tracing_generic_entry_update(entry, flags, pc);

	entry->type = type;

	entry->type = type;



	return raw_data;

}

}

EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);

NOKPROBE_SYMBOL(perf_trace_buf_update);

NOKPROBE_SYMBOL(perf_trace_buf_prepare);





#ifdef CONFIG_FUNCTION_TRACER

#ifdef CONFIG_FUNCTION_TRACER

static void

static void
@@ -319,13 +320,13 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip, 
	memset(&regs, 0, sizeof(regs));

	memset(&regs, 0, sizeof(regs));

	perf_fetch_caller_regs(&regs);

	perf_fetch_caller_regs(&regs);





	entry = perf_trace_buf_prepare(ENTRY_SIZE, TRACE_FN, NULL, &rctx);

	entry = perf_trace_buf_alloc(ENTRY_SIZE, NULL, &rctx);

	if (!entry)

	if (!entry)

		return;

		return;





	entry->ip = ip;

	entry->ip = ip;

	entry->parent_ip = parent_ip;

	entry->parent_ip = parent_ip;

	perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, 0,

	perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, TRACE_FN,

			      1, &regs, head, NULL);

			      1, &regs, head, NULL);





#undef ENTRY_SIZE

#undef ENTRY_SIZE

CRA Git | Maintained and supported by SUSTech CRA and CCSE