Commit d45ae1f7 authored by Steven Rostedt (VMware)'s avatar Steven Rostedt (VMware)
Browse files

tracing: Process constants for (un)likely() profiler 



When running the likely/unlikely profiler, one of the results did not look
accurate. It noted that the unlikely() in link_path_walk() was 100%
incorrect. When I added a trace_printk() to see what was happening there, it
became 80% correct! Looking deeper into what whas happening, I found that
gcc split that if statement into two paths. One where the if statement
became a constant, the other path a variable. The other path had the if
statement always hit (making the unlikely there, always false), but since
the #define unlikely() has:

  #define unlikely() (__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 0))

Where constants are ignored by the branch profiler, the "constant" path
made by the compiler was ignored, even though it was hit 80% of the time.

By just passing the constant value to the __branch_check__() function and
tracing it out of line (as always correct, as likely/unlikely isn't a factor
for constants), then we get back the accurate readings of branches that were
optimized by gcc causing part of the execution to become constant.

Signed-off-by: default avatarSteven Rostedt (VMware) <rostedt@goodmis.org>
parent 6496bb72

include/linux/compiler.h

+8 −6
Original line number Diff line number Diff line
@@ -107,12 +107,13 @@ struct ftrace_branch_data { 
 */

#if defined(CONFIG_TRACE_BRANCH_PROFILING) \

    && !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__)

void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect);

void ftrace_likely_update(struct ftrace_branch_data *f, int val,

			  int expect, int is_constant);



#define likely_notrace(x)	__builtin_expect(!!(x), 1)

#define unlikely_notrace(x)	__builtin_expect(!!(x), 0)



#define __branch_check__(x, expect) ({					\

#define __branch_check__(x, expect, is_constant) ({			\

			int ______r;					\

			static struct ftrace_branch_data		\

				__attribute__((__aligned__(4)))		\
@@ -122,8 +123,9 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); 
				.file = __FILE__,			\

				.line = __LINE__,			\

			};						\

			______r = likely_notrace(x);			\

			ftrace_likely_update(&______f, ______r, expect); \

			______r = __builtin_expect(!!(x), expect);	\

			ftrace_likely_update(&______f, ______r,		\

					     expect, is_constant);	\

			______r;					\

		})
@@ -133,10 +135,10 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); 
 * written by Daniel Walker.

 */

# ifndef likely

#  define likely(x)	(__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 1))

#  define likely(x)	(__branch_check__(x, 1, __builtin_constant_p(x)))

# endif

# ifndef unlikely

#  define unlikely(x)	(__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 0))

#  define unlikely(x)	(__branch_check__(x, 0, __builtin_constant_p(x)))

# endif



#ifdef CONFIG_PROFILE_ALL_BRANCHES

kernel/trace/trace_branch.c

+5 −1
Original line number Diff line number Diff line
@@ -200,8 +200,12 @@ void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) 
}

#endif /* CONFIG_BRANCH_TRACER */



void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect)

void ftrace_likely_update(struct ftrace_branch_data *f, int val,

			  int expect, int is_constant)

{

	/* A constant is always correct */

	if (is_constant)

		val = expect;

	/*

	 * I would love to have a trace point here instead, but the

	 * trace point code is so inundated with unlikely and likely

CRA Git | Maintained and supported by SUSTech CRA and CCSE