Commit 26198c21 authored by Ingo Molnar's avatar Ingo Molnar
Browse files

Merge branch 'tip/perf/core' of... 

Merge branch 'tip/perf/core' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace

 into perf/core

Pull ftrace updates from Steve Rostedt:

" This patch series extends ftrace function tracing utility to be
  more dynamic for its users. It allows for data passing to the callback
  functions, as well as reading regs as if a breakpoint were to trigger
  at function entry.

  The main goal of this patch series was to allow kprobes to use ftrace
  as an optimized probe point when a probe is placed on an ftrace nop.
  With lots of help from Masami Hiramatsu, and going through lots of
  iterations, we finally came up with a good solution. "

Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parents 194f8dcb e5253896

arch/x86/include/asm/ftrace.h

+30 −19
Original line number Diff line number Diff line
@@ -3,27 +3,33 @@ 


#ifdef __ASSEMBLY__



	.macro MCOUNT_SAVE_FRAME

	/* taken from glibc */

	subq $0x38, %rsp

	movq %rax, (%rsp)

	movq %rcx, 8(%rsp)

	movq %rdx, 16(%rsp)

	movq %rsi, 24(%rsp)

	movq %rdi, 32(%rsp)

	movq %r8, 40(%rsp)

	movq %r9, 48(%rsp)

	/* skip is set if the stack was already partially adjusted */

	.macro MCOUNT_SAVE_FRAME skip=0

	 /*

	  * We add enough stack to save all regs.

	  */

	subq $(SS+8-\skip), %rsp

	movq %rax, RAX(%rsp)

	movq %rcx, RCX(%rsp)

	movq %rdx, RDX(%rsp)

	movq %rsi, RSI(%rsp)

	movq %rdi, RDI(%rsp)

	movq %r8, R8(%rsp)

	movq %r9, R9(%rsp)

	 /* Move RIP to its proper location */

	movq SS+8(%rsp), %rdx

	movq %rdx, RIP(%rsp)

	.endm



	.macro MCOUNT_RESTORE_FRAME

	movq 48(%rsp), %r9

	movq 40(%rsp), %r8

	movq 32(%rsp), %rdi

	movq 24(%rsp), %rsi

	movq 16(%rsp), %rdx

	movq 8(%rsp), %rcx

	movq (%rsp), %rax

	addq $0x38, %rsp

	.macro MCOUNT_RESTORE_FRAME skip=0

	movq R9(%rsp), %r9

	movq R8(%rsp), %r8

	movq RDI(%rsp), %rdi

	movq RSI(%rsp), %rsi

	movq RDX(%rsp), %rdx

	movq RCX(%rsp), %rcx

	movq RAX(%rsp), %rax

	addq $(SS+8-\skip), %rsp

	.endm



#endif
@@ -32,6 +38,11 @@ 
#define MCOUNT_ADDR		((long)(mcount))

#define MCOUNT_INSN_SIZE	5 /* sizeof mcount call */



#ifdef CONFIG_DYNAMIC_FTRACE

#define ARCH_SUPPORTS_FTRACE_OPS 1

#define ARCH_SUPPORTS_FTRACE_SAVE_REGS

#endif



#ifndef __ASSEMBLY__

extern void mcount(void);

extern atomic_t modifying_ftrace_code;

arch/x86/include/asm/kprobes.h

+1 −0
Original line number Diff line number Diff line
@@ -27,6 +27,7 @@ 
#include <asm/insn.h>



#define  __ARCH_WANT_KPROBES_INSN_SLOT

#define  ARCH_SUPPORTS_KPROBES_ON_FTRACE



struct pt_regs;

struct kprobe;

arch/x86/kernel/entry_32.S

+71 −4
Original line number Diff line number Diff line
@@ -1109,17 +1109,21 @@ ENTRY(ftrace_caller) 
	pushl %eax

	pushl %ecx

	pushl %edx

	movl 0xc(%esp), %eax

	pushl $0	/* Pass NULL as regs pointer */

	movl 4*4(%esp), %eax

	movl 0x4(%ebp), %edx

	leal function_trace_op, %ecx

	subl $MCOUNT_INSN_SIZE, %eax



.globl ftrace_call

ftrace_call:

	call ftrace_stub



	addl $4,%esp	/* skip NULL pointer */

	popl %edx

	popl %ecx

	popl %eax

ftrace_ret:

#ifdef CONFIG_FUNCTION_GRAPH_TRACER

.globl ftrace_graph_call

ftrace_graph_call:
@@ -1131,6 +1135,72 @@ ftrace_stub: 
	ret

END(ftrace_caller)



ENTRY(ftrace_regs_caller)

	pushf	/* push flags before compare (in cs location) */

	cmpl $0, function_trace_stop

	jne ftrace_restore_flags



	/*

	 * i386 does not save SS and ESP when coming from kernel.

	 * Instead, to get sp, &regs->sp is used (see ptrace.h).

	 * Unfortunately, that means eflags must be at the same location

	 * as the current return ip is. We move the return ip into the

	 * ip location, and move flags into the return ip location.

	 */

	pushl 4(%esp)	/* save return ip into ip slot */

	subl $MCOUNT_INSN_SIZE, (%esp)	/* Adjust ip */



	pushl $0	/* Load 0 into orig_ax */

	pushl %gs

	pushl %fs

	pushl %es

	pushl %ds

	pushl %eax

	pushl %ebp

	pushl %edi

	pushl %esi

	pushl %edx

	pushl %ecx

	pushl %ebx



	movl 13*4(%esp), %eax	/* Get the saved flags */

	movl %eax, 14*4(%esp)	/* Move saved flags into regs->flags location */

				/* clobbering return ip */

	movl $__KERNEL_CS,13*4(%esp)



	movl 12*4(%esp), %eax	/* Load ip (1st parameter) */

	movl 0x4(%ebp), %edx	/* Load parent ip (2nd parameter) */

	leal function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */

	pushl %esp		/* Save pt_regs as 4th parameter */



GLOBAL(ftrace_regs_call)

	call ftrace_stub



	addl $4, %esp		/* Skip pt_regs */

	movl 14*4(%esp), %eax	/* Move flags back into cs */

	movl %eax, 13*4(%esp)	/* Needed to keep addl from modifying flags */

	movl 12*4(%esp), %eax	/* Get return ip from regs->ip */

	addl $MCOUNT_INSN_SIZE, %eax

	movl %eax, 14*4(%esp)	/* Put return ip back for ret */



	popl %ebx

	popl %ecx

	popl %edx

	popl %esi

	popl %edi

	popl %ebp

	popl %eax

	popl %ds

	popl %es

	popl %fs

	popl %gs

	addl $8, %esp		/* Skip orig_ax and ip */

	popf			/* Pop flags at end (no addl to corrupt flags) */

	jmp ftrace_ret



ftrace_restore_flags:

	popf

	jmp  ftrace_stub

#else /* ! CONFIG_DYNAMIC_FTRACE */



ENTRY(mcount)
@@ -1171,9 +1241,6 @@ END(mcount) 


#ifdef CONFIG_FUNCTION_GRAPH_TRACER

ENTRY(ftrace_graph_caller)

	cmpl $0, function_trace_stop

	jne ftrace_stub



	pushl %eax

	pushl %ecx

	pushl %edx

arch/x86/kernel/entry_64.S

+86 −10
Original line number Diff line number Diff line
@@ -73,20 +73,34 @@ ENTRY(mcount) 
	retq

END(mcount)



/* skip is set if stack has been adjusted */

.macro ftrace_caller_setup skip=0

	MCOUNT_SAVE_FRAME \skip



	/* Load the ftrace_ops into the 3rd parameter */

	leaq function_trace_op, %rdx



	/* Load ip into the first parameter */

	movq RIP(%rsp), %rdi

	subq $MCOUNT_INSN_SIZE, %rdi

	/* Load the parent_ip into the second parameter */

	movq 8(%rbp), %rsi

.endm



ENTRY(ftrace_caller)

	/* Check if tracing was disabled (quick check) */

	cmpl $0, function_trace_stop

	jne  ftrace_stub



	MCOUNT_SAVE_FRAME



	movq 0x38(%rsp), %rdi

	movq 8(%rbp), %rsi

	subq $MCOUNT_INSN_SIZE, %rdi

	ftrace_caller_setup

	/* regs go into 4th parameter (but make it NULL) */

	movq $0, %rcx



GLOBAL(ftrace_call)

	call ftrace_stub



	MCOUNT_RESTORE_FRAME

ftrace_return:



#ifdef CONFIG_FUNCTION_GRAPH_TRACER

GLOBAL(ftrace_graph_call)
@@ -97,6 +111,71 @@ GLOBAL(ftrace_stub) 
	retq

END(ftrace_caller)



ENTRY(ftrace_regs_caller)

	/* Save the current flags before compare (in SS location)*/

	pushfq



	/* Check if tracing was disabled (quick check) */

	cmpl $0, function_trace_stop

	jne  ftrace_restore_flags



	/* skip=8 to skip flags saved in SS */

	ftrace_caller_setup 8



	/* Save the rest of pt_regs */

	movq %r15, R15(%rsp)

	movq %r14, R14(%rsp)

	movq %r13, R13(%rsp)

	movq %r12, R12(%rsp)

	movq %r11, R11(%rsp)

	movq %r10, R10(%rsp)

	movq %rbp, RBP(%rsp)

	movq %rbx, RBX(%rsp)

	/* Copy saved flags */

	movq SS(%rsp), %rcx

	movq %rcx, EFLAGS(%rsp)

	/* Kernel segments */

	movq $__KERNEL_DS, %rcx

	movq %rcx, SS(%rsp)

	movq $__KERNEL_CS, %rcx

	movq %rcx, CS(%rsp)

	/* Stack - skipping return address */

	leaq SS+16(%rsp), %rcx

	movq %rcx, RSP(%rsp)



	/* regs go into 4th parameter */

	leaq (%rsp), %rcx



GLOBAL(ftrace_regs_call)

	call ftrace_stub



	/* Copy flags back to SS, to restore them */

	movq EFLAGS(%rsp), %rax

	movq %rax, SS(%rsp)



	/* restore the rest of pt_regs */

	movq R15(%rsp), %r15

	movq R14(%rsp), %r14

	movq R13(%rsp), %r13

	movq R12(%rsp), %r12

	movq R10(%rsp), %r10

	movq RBP(%rsp), %rbp

	movq RBX(%rsp), %rbx



	/* skip=8 to skip flags saved in SS */

	MCOUNT_RESTORE_FRAME 8



	/* Restore flags */

	popfq



	jmp ftrace_return

ftrace_restore_flags:

	popfq

	jmp  ftrace_stub



END(ftrace_regs_caller)





#else /* ! CONFIG_DYNAMIC_FTRACE */

ENTRY(mcount)

	cmpl $0, function_trace_stop
@@ -119,7 +198,7 @@ GLOBAL(ftrace_stub) 
trace:

	MCOUNT_SAVE_FRAME



	movq 0x38(%rsp), %rdi

	movq RIP(%rsp), %rdi

	movq 8(%rbp), %rsi

	subq $MCOUNT_INSN_SIZE, %rdi
@@ -134,13 +213,10 @@ END(mcount) 


#ifdef CONFIG_FUNCTION_GRAPH_TRACER

ENTRY(ftrace_graph_caller)

	cmpl $0, function_trace_stop

	jne ftrace_stub



	MCOUNT_SAVE_FRAME



	leaq 8(%rbp), %rdi

	movq 0x38(%rsp), %rsi

	movq RIP(%rsp), %rsi

	movq (%rbp), %rdx

	subq $MCOUNT_INSN_SIZE, %rsi

arch/x86/kernel/ftrace.c

+69 −4
Original line number Diff line number Diff line
@@ -206,6 +206,21 @@ static int 
ftrace_modify_code(unsigned long ip, unsigned const char *old_code,

		   unsigned const char *new_code);



/*

 * Should never be called:

 *  As it is only called by __ftrace_replace_code() which is called by

 *  ftrace_replace_code() that x86 overrides, and by ftrace_update_code()

 *  which is called to turn mcount into nops or nops into function calls

 *  but not to convert a function from not using regs to one that uses

 *  regs, which ftrace_modify_call() is for.

 */

int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,

				 unsigned long addr)

{

	WARN_ON(1);

	return -EINVAL;

}



int ftrace_update_ftrace_func(ftrace_func_t func)

{

	unsigned long ip = (unsigned long)(&ftrace_call);
@@ -220,6 +235,14 @@ int ftrace_update_ftrace_func(ftrace_func_t func) 


	ret = ftrace_modify_code(ip, old, new);



	/* Also update the regs callback function */

	if (!ret) {

		ip = (unsigned long)(&ftrace_regs_call);

		memcpy(old, &ftrace_regs_call, MCOUNT_INSN_SIZE);

		new = ftrace_call_replace(ip, (unsigned long)func);

		ret = ftrace_modify_code(ip, old, new);

	}



	atomic_dec(&modifying_ftrace_code);



	return ret;
@@ -299,6 +322,32 @@ static int add_brk_on_nop(struct dyn_ftrace *rec) 
	return add_break(rec->ip, old);

}



/*

 * If the record has the FTRACE_FL_REGS set, that means that it

 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS

 * is not not set, then it wants to convert to the normal callback.

 */

static unsigned long get_ftrace_addr(struct dyn_ftrace *rec)

{

	if (rec->flags & FTRACE_FL_REGS)

		return (unsigned long)FTRACE_REGS_ADDR;

	else

		return (unsigned long)FTRACE_ADDR;

}



/*

 * The FTRACE_FL_REGS_EN is set when the record already points to

 * a function that saves all the regs. Basically the '_EN' version

 * represents the current state of the function.

 */

static unsigned long get_ftrace_old_addr(struct dyn_ftrace *rec)

{

	if (rec->flags & FTRACE_FL_REGS_EN)

		return (unsigned long)FTRACE_REGS_ADDR;

	else

		return (unsigned long)FTRACE_ADDR;

}



static int add_breakpoints(struct dyn_ftrace *rec, int enable)

{

	unsigned long ftrace_addr;
@@ -306,7 +355,7 @@ static int add_breakpoints(struct dyn_ftrace *rec, int enable) 


	ret = ftrace_test_record(rec, enable);



	ftrace_addr = (unsigned long)FTRACE_ADDR;

	ftrace_addr = get_ftrace_addr(rec);



	switch (ret) {

	case FTRACE_UPDATE_IGNORE:
@@ -316,6 +365,10 @@ static int add_breakpoints(struct dyn_ftrace *rec, int enable) 
		/* converting nop to call */

		return add_brk_on_nop(rec);



	case FTRACE_UPDATE_MODIFY_CALL_REGS:

	case FTRACE_UPDATE_MODIFY_CALL:

		ftrace_addr = get_ftrace_old_addr(rec);

		/* fall through */

	case FTRACE_UPDATE_MAKE_NOP:

		/* converting a call to a nop */

		return add_brk_on_call(rec, ftrace_addr);
@@ -360,13 +413,21 @@ static int remove_breakpoint(struct dyn_ftrace *rec) 
		 * If not, don't touch the breakpoint, we make just create

		 * a disaster.

		 */

		ftrace_addr = (unsigned long)FTRACE_ADDR;

		ftrace_addr = get_ftrace_addr(rec);

		nop = ftrace_call_replace(ip, ftrace_addr);



		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)

			goto update;



		/* Check both ftrace_addr and ftrace_old_addr */

		ftrace_addr = get_ftrace_old_addr(rec);

		nop = ftrace_call_replace(ip, ftrace_addr);



		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)

			return -EINVAL;

	}



 update:

	return probe_kernel_write((void *)ip, &nop[0], 1);

}
@@ -405,12 +466,14 @@ static int add_update(struct dyn_ftrace *rec, int enable) 


	ret = ftrace_test_record(rec, enable);



	ftrace_addr = (unsigned long)FTRACE_ADDR;

	ftrace_addr  = get_ftrace_addr(rec);



	switch (ret) {

	case FTRACE_UPDATE_IGNORE:

		return 0;



	case FTRACE_UPDATE_MODIFY_CALL_REGS:

	case FTRACE_UPDATE_MODIFY_CALL:

	case FTRACE_UPDATE_MAKE_CALL:

		/* converting nop to call */

		return add_update_call(rec, ftrace_addr);
@@ -455,12 +518,14 @@ static int finish_update(struct dyn_ftrace *rec, int enable) 


	ret = ftrace_update_record(rec, enable);



	ftrace_addr = (unsigned long)FTRACE_ADDR;

	ftrace_addr = get_ftrace_addr(rec);



	switch (ret) {

	case FTRACE_UPDATE_IGNORE:

		return 0;



	case FTRACE_UPDATE_MODIFY_CALL_REGS:

	case FTRACE_UPDATE_MODIFY_CALL:

	case FTRACE_UPDATE_MAKE_CALL:

		/* converting nop to call */

		return finish_update_call(rec, ftrace_addr);

CRA Git | Maintained and supported by SUSTech CRA and CCSE