Merge branch 'uprobes/core' of... (a03b1e1c) · Commits · 戴 / test

arch/x86/include/asm/traps.h

+0 −1

Original line number	Diff line number	Diff line
		@@ -98,7 +98,6 @@ static inline int get_si_code(unsigned long condition)

		extern int panic_on_unrecovered_nmi;

		void math_error(struct pt_regs *, int, int);
		void math_emulate(struct math_emu_info *);
		#ifndef CONFIG_X86_32
		asmlinkage void smp_thermal_interrupt(void);

arch/x86/include/asm/uprobes.h

+0 −3

Original line number	Diff line number	Diff line
		@@ -50,9 +50,6 @@ struct arch_uprobe {
		u8 opc1;
		} branch;
		struct {
		#ifdef CONFIG_X86_64
		long riprel_target;
		#endif
		u8 fixups;
		u8 ilen;
		} def;

arch/x86/kernel/traps.c

+62 −48

Original line number	Diff line number	Diff line
		@@ -23,6 +23,7 @@
		#include <linux/kernel.h>
		#include <linux/module.h>
		#include <linux/ptrace.h>
		#include <linux/uprobes.h>
		#include <linux/string.h>
		#include <linux/delay.h>
		#include <linux/errno.h>
		@@ -136,6 +137,37 @@ do_trap_no_signal(struct task_struct tsk, int trapnr, char str,
		return -1;
		}

		static siginfo_t fill_trap_info(struct pt_regs regs, int signr, int trapnr,
		siginfo_t *info)
		{
		unsigned long siaddr;
		int sicode;

		switch (trapnr) {
		default:
		return SEND_SIG_PRIV;

		case X86_TRAP_DE:
		sicode = FPE_INTDIV;
		siaddr = uprobe_get_trap_addr(regs);
		break;
		case X86_TRAP_UD:
		sicode = ILL_ILLOPN;
		siaddr = uprobe_get_trap_addr(regs);
		break;
		case X86_TRAP_AC:
		sicode = BUS_ADRALN;
		siaddr = 0;
		break;
		}

		info->si_signo = signr;
		info->si_errno = 0;
		info->si_code = sicode;
		info->si_addr = (void __user *)siaddr;
		return info;
		}

		static void __kprobes
		do_trap(int trapnr, int signr, char str, struct pt_regs regs,
		long error_code, siginfo_t *info)
		@@ -168,60 +200,42 @@ do_trap(int trapnr, int signr, char str, struct pt_regs regs,
		}
		#endif

		if (info)
		force_sig_info(signr, info, tsk);
		else
		force_sig(signr, tsk);
		force_sig_info(signr, info ?: SEND_SIG_PRIV, tsk);
		}

		static void do_error_trap(struct pt_regs regs, long error_code, char str,
		unsigned long trapnr, int signr)
		{
		enum ctx_state prev_state = exception_enter();
		siginfo_t info;

		if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=
		NOTIFY_STOP) {
		conditional_sti(regs);
		do_trap(trapnr, signr, str, regs, error_code,
		fill_trap_info(regs, signr, trapnr, &info));
		}

		exception_exit(prev_state);
		}

		#define DO_ERROR(trapnr, signr, str, name) \
		dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
		{ \
		enum ctx_state prev_state; \
		\
		prev_state = exception_enter(); \
		if (notify_die(DIE_TRAP, str, regs, error_code, \
		trapnr, signr) == NOTIFY_STOP) { \
		exception_exit(prev_state); \
		return; \
		} \
		conditional_sti(regs); \
		do_trap(trapnr, signr, str, regs, error_code, NULL); \
		exception_exit(prev_state); \
		}

		#define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr) \
		dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
		{ \
		siginfo_t info; \
		enum ctx_state prev_state; \
		\
		info.si_signo = signr; \
		info.si_errno = 0; \
		info.si_code = sicode; \
		info.si_addr = (void __user *)siaddr; \
		prev_state = exception_enter(); \
		if (notify_die(DIE_TRAP, str, regs, error_code, \
		trapnr, signr) == NOTIFY_STOP) { \
		exception_exit(prev_state); \
		return; \
		} \
		conditional_sti(regs); \
		do_trap(trapnr, signr, str, regs, error_code, &info); \
		exception_exit(prev_state); \
		}

		DO_ERROR_INFO(X86_TRAP_DE, SIGFPE, "divide error", divide_error, FPE_INTDIV, regs->ip )
		do_error_trap(regs, error_code, str, trapnr, signr); \
		}

		DO_ERROR(X86_TRAP_DE, SIGFPE, "divide error", divide_error)
		DO_ERROR(X86_TRAP_OF, SIGSEGV, "overflow", overflow)
		DO_ERROR(X86_TRAP_BR, SIGSEGV, "bounds", bounds)
		DO_ERROR_INFO(X86_TRAP_UD, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN, regs->ip )
		DO_ERROR(X86_TRAP_UD, SIGILL, "invalid opcode", invalid_op)
		DO_ERROR(X86_TRAP_OLD_MF, SIGFPE, "coprocessor segment overrun",coprocessor_segment_overrun)
		DO_ERROR(X86_TRAP_TS, SIGSEGV, "invalid TSS", invalid_TSS)
		DO_ERROR(X86_TRAP_NP, SIGBUS, "segment not present", segment_not_present)
		#ifdef CONFIG_X86_32
		DO_ERROR(X86_TRAP_SS, SIGBUS, "stack segment", stack_segment)
		#endif
		DO_ERROR_INFO(X86_TRAP_AC, SIGBUS, "alignment check", alignment_check, BUS_ADRALN, 0 )
		DO_ERROR(X86_TRAP_AC, SIGBUS, "alignment check", alignment_check)

		#ifdef CONFIG_X86_64
		/* Runs on IST stack */
		@@ -305,7 +319,7 @@ do_general_protection(struct pt_regs *regs, long error_code)
		pr_cont("\n");
		}

		force_sig(SIGSEGV, tsk);
		force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk);
		exit:
		exception_exit(prev_state);
		}
		@@ -488,7 +502,7 @@ exit:
		* the correct behaviour even in the presence of the asynchronous
		* IRQ13 behaviour
		*/
		void math_error(struct pt_regs *regs, int error_code, int trapnr)
		static void math_error(struct pt_regs *regs, int error_code, int trapnr)
		{
		struct task_struct *task = current;
		siginfo_t info;
		@@ -518,7 +532,7 @@ void math_error(struct pt_regs *regs, int error_code, int trapnr)
		task->thread.error_code = error_code;
		info.si_signo = SIGFPE;
		info.si_errno = 0;
		info.si_addr = (void __user *)regs->ip;
		info.si_addr = (void __user *)uprobe_get_trap_addr(regs);
		if (trapnr == X86_TRAP_MF) {
		unsigned short cwd, swd;
		/*
		@@ -645,7 +659,7 @@ void math_state_restore(void)
		*/
		if (unlikely(restore_fpu_checking(tsk))) {
		drop_init_fpu(tsk);
		force_sig(SIGSEGV, tsk);
		force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk);
		return;
		}

arch/x86/kernel/uprobes.c

+138 −75

Original line number	Diff line number	Diff line
		@@ -41,8 +41,11 @@
		/* Instruction will modify TF, don't change it */
		#define UPROBE_FIX_SETF 0x04

		#define UPROBE_FIX_RIP_AX 0x08
		#define UPROBE_FIX_RIP_CX 0x10
		#define UPROBE_FIX_RIP_SI 0x08
		#define UPROBE_FIX_RIP_DI 0x10
		#define UPROBE_FIX_RIP_BX 0x20
		#define UPROBE_FIX_RIP_MASK \
		(UPROBE_FIX_RIP_SI \| UPROBE_FIX_RIP_DI \| UPROBE_FIX_RIP_BX)

		#define UPROBE_TRAP_NR UINT_MAX

		@@ -251,9 +254,9 @@ static inline bool is_64bit_mm(struct mm_struct *mm)
		* If arch_uprobe->insn doesn't use rip-relative addressing, return
		* immediately. Otherwise, rewrite the instruction so that it accesses
		* its memory operand indirectly through a scratch register. Set
		* def->fixups and def->riprel_target accordingly. (The contents of the
		* scratch register will be saved before we single-step the modified
		* instruction, and restored afterward).
		* def->fixups accordingly. (The contents of the scratch register
		* will be saved before we single-step the modified instruction,
		* and restored afterward).
		*
		* We do this because a rip-relative instruction can access only a
		* relatively small area (+/- 2 GB from the instruction), and the XOL
		@@ -264,72 +267,142 @@ static inline bool is_64bit_mm(struct mm_struct *mm)
		*
		* Some useful facts about rip-relative instructions:
		*
		* - There's always a modrm byte.
		* - There's always a modrm byte with bit layout "00 reg 101".
		* - There's never a SIB byte.
		* - The displacement is always 4 bytes.
		* - REX.B=1 bit in REX prefix, which normally extends r/m field,
		* has no effect on rip-relative mode. It doesn't make modrm byte
		* with r/m=101 refer to register 1101 = R13.
		*/
		static void riprel_analyze(struct arch_uprobe auprobe, struct insn insn)
		{
		u8 *cursor;
		u8 reg;
		u8 reg2;

		if (!insn_rip_relative(insn))
		return;

		/*
		* insn_rip_relative() would have decoded rex_prefix, modrm.
		* insn_rip_relative() would have decoded rex_prefix, vex_prefix, modrm.
		* Clear REX.b bit (extension of MODRM.rm field):
		* we want to encode rax/rcx, not r8/r9.
		* we want to encode low numbered reg, not r8+.
		*/
		if (insn->rex_prefix.nbytes) {
		cursor = auprobe->insn + insn_offset_rex_prefix(insn);
		cursor &= 0xfe; / Clearing REX.B bit */
		/* REX byte has 0100wrxb layout, clearing REX.b bit */
		*cursor &= 0xfe;
		}
		/*
		* Similar treatment for VEX3 prefix.
		* TODO: add XOP/EVEX treatment when insn decoder supports them
		*/
		if (insn->vex_prefix.nbytes == 3) {
		/*
		* vex2: c5 rvvvvLpp (has no b bit)
		* vex3/xop: c4/8f rxbmmmmm wvvvvLpp
		* evex: 62 rxbR00mm wvvvv1pp zllBVaaa
		* (evex will need setting of both b and x since
		* in non-sib encoding evex.x is 4th bit of MODRM.rm)
		* Setting VEX3.b (setting because it has inverted meaning):
		*/
		cursor = auprobe->insn + insn_offset_vex_prefix(insn) + 1;
		*cursor \|= 0x20;
		}

		/*
		* Point cursor at the modrm byte. The next 4 bytes are the
		* displacement. Beyond the displacement, for some instructions,
		* is the immediate operand.
		* Convert from rip-relative addressing to register-relative addressing
		* via a scratch register.
		*
		* This is tricky since there are insns with modrm byte
		* which also use registers not encoded in modrm byte:
		* [i]div/[i]mul: implicitly use dx:ax
		* shift ops: implicitly use cx
		* cmpxchg: implicitly uses ax
		* cmpxchg8/16b: implicitly uses dx:ax and bx:cx
		* Encoding: 0f c7/1 modrm
		* The code below thinks that reg=1 (cx), chooses si as scratch.
		* mulx: implicitly uses dx: mulx r/m,r1,r2 does r1:r2 = dx * r/m.
		* First appeared in Haswell (BMI2 insn). It is vex-encoded.
		* Example where none of bx,cx,dx can be used as scratch reg:
		* c4 e2 63 f6 0d disp32 mulx disp32(%rip),%ebx,%ecx
		* [v]pcmpistri: implicitly uses cx, xmm0
		* [v]pcmpistrm: implicitly uses xmm0
		* [v]pcmpestri: implicitly uses ax, dx, cx, xmm0
		* [v]pcmpestrm: implicitly uses ax, dx, xmm0
		* Evil SSE4.2 string comparison ops from hell.
		* maskmovq/[v]maskmovdqu: implicitly uses (ds:rdi) as destination.
		* Encoding: 0f f7 modrm, 66 0f f7 modrm, vex-encoded: c5 f9 f7 modrm.
		* Store op1, byte-masked by op2 msb's in each byte, to (ds:rdi).
		* AMD says it has no 3-operand form (vex.vvvv must be 1111)
		* and that it can have only register operands, not mem
		* (its modrm byte must have mode=11).
		* If these restrictions will ever be lifted,
		* we'll need code to prevent selection of di as scratch reg!
		*
		* Summary: I don't know any insns with modrm byte which
		* use SI register implicitly. DI register is used only
		* by one insn (maskmovq) and BX register is used
		* only by one too (cmpxchg8b).
		* BP is stack-segment based (may be a problem?).
		* AX, DX, CX are off-limits (many implicit users).
		* SP is unusable (it's stack pointer - think about "pop mem";
		* also, rsp+disp32 needs sib encoding -> insn length change).
		*/
		cursor = auprobe->insn + insn_offset_modrm(insn);

		reg = MODRM_REG(insn); /* Fetch modrm.reg */
		reg2 = 0xff; /* Fetch vex.vvvv */
		if (insn->vex_prefix.nbytes == 2)
		reg2 = insn->vex_prefix.bytes[1];
		else if (insn->vex_prefix.nbytes == 3)
		reg2 = insn->vex_prefix.bytes[2];
		/*
		* Convert from rip-relative addressing to indirect addressing
		* via a scratch register. Change the r/m field from 0x5 (%rip)
		* to 0x0 (%rax) or 0x1 (%rcx), and squeeze out the offset field.
		* TODO: add XOP, EXEV vvvv reading.
		*
		* vex.vvvv field is in bits 6-3, bits are inverted.
		* But in 32-bit mode, high-order bit may be ignored.
		* Therefore, let's consider only 3 low-order bits.
		*/
		reg = MODRM_REG(insn);
		if (reg == 0) {
		reg2 = ((reg2 >> 3) & 0x7) ^ 0x7;
		/*
		* The register operand (if any) is either the A register
		* (%rax, %eax, etc.) or (if the 0x4 bit is set in the
		* REX prefix) %r8. In any case, we know the C register
		* is NOT the register operand, so we use %rcx (register
		* #1) for the scratch register.
		* Register numbering is ax,cx,dx,bx, sp,bp,si,di, r8..r15.
		*
		* Choose scratch reg. Order is important: must not select bx
		* if we can use si (cmpxchg8b case!)
		*/
		auprobe->def.fixups \|= UPROBE_FIX_RIP_CX;
		/* Change modrm from 00 000 101 to 00 000 001. */
		*cursor = 0x1;
		if (reg != 6 && reg2 != 6) {
		reg2 = 6;
		auprobe->def.fixups \|= UPROBE_FIX_RIP_SI;
		} else if (reg != 7 && reg2 != 7) {
		reg2 = 7;
		auprobe->def.fixups \|= UPROBE_FIX_RIP_DI;
		/* TODO (paranoia): force maskmovq to not use di */
		} else {
		/* Use %rax (register #0) for the scratch register. */
		auprobe->def.fixups \|= UPROBE_FIX_RIP_AX;
		/* Change modrm from 00 xxx 101 to 00 xxx 000 */
		*cursor = (reg << 3);
		}

		/* Target address = address of next instruction + (signed) offset */
		auprobe->def.riprel_target = (long)insn->length + insn->displacement.value;

		/* Displacement field is gone; slide immediate field (if any) over. */
		if (insn->immediate.nbytes) {
		cursor++;
		memmove(cursor, cursor + insn->displacement.nbytes, insn->immediate.nbytes);
		reg2 = 3;
		auprobe->def.fixups \|= UPROBE_FIX_RIP_BX;
		}
		/*
		* Point cursor at the modrm byte. The next 4 bytes are the
		* displacement. Beyond the displacement, for some instructions,
		* is the immediate operand.
		*/
		cursor = auprobe->insn + insn_offset_modrm(insn);
		/*
		* Change modrm from "00 reg 101" to "10 reg reg2". Example:
		* 89 05 disp32 mov %eax,disp32(%rip) becomes
		* 89 86 disp32 mov %eax,disp32(%rsi)
		*/
		*cursor = 0x80 \| (reg << 3) \| reg2;
		}

		static inline unsigned long *
		scratch_reg(struct arch_uprobe auprobe, struct pt_regs regs)
		{
		return (auprobe->def.fixups & UPROBE_FIX_RIP_AX) ? &regs->ax : &regs->cx;
		if (auprobe->def.fixups & UPROBE_FIX_RIP_SI)
		return &regs->si;
		if (auprobe->def.fixups & UPROBE_FIX_RIP_DI)
		return &regs->di;
		return &regs->bx;
		}

		/*
		@@ -338,31 +411,22 @@ scratch_reg(struct arch_uprobe auprobe, struct pt_regs regs)
		*/
		static void riprel_pre_xol(struct arch_uprobe auprobe, struct pt_regs regs)
		{
		if (auprobe->def.fixups & (UPROBE_FIX_RIP_AX \| UPROBE_FIX_RIP_CX)) {
		if (auprobe->def.fixups & UPROBE_FIX_RIP_MASK) {
		struct uprobe_task *utask = current->utask;
		unsigned long *sr = scratch_reg(auprobe, regs);

		utask->autask.saved_scratch_register = *sr;
		*sr = utask->vaddr + auprobe->def.riprel_target;
		*sr = utask->vaddr + auprobe->def.ilen;
		}
		}

		static void riprel_post_xol(struct arch_uprobe auprobe, struct pt_regs regs,
		long *correction)
		static void riprel_post_xol(struct arch_uprobe auprobe, struct pt_regs regs)
		{
		if (auprobe->def.fixups & (UPROBE_FIX_RIP_AX \| UPROBE_FIX_RIP_CX)) {
		if (auprobe->def.fixups & UPROBE_FIX_RIP_MASK) {
		struct uprobe_task *utask = current->utask;
		unsigned long *sr = scratch_reg(auprobe, regs);

		*sr = utask->autask.saved_scratch_register;
		/*
		* The original instruction includes a displacement, and so
		* is 4 bytes longer than what we've just single-stepped.
		* Caller may need to apply other fixups to handle stuff
		* like "jmpq ...(%rip)" and "callq ...(%rip)".
		*/
		if (correction)
		*correction += 4;
		}
		}
		#else /* 32-bit: */
		@@ -379,8 +443,7 @@ static void riprel_analyze(struct arch_uprobe auprobe, struct insn insn)
		static void riprel_pre_xol(struct arch_uprobe auprobe, struct pt_regs regs)
		{
		}
		static void riprel_post_xol(struct arch_uprobe auprobe, struct pt_regs regs,
		long *correction)
		static void riprel_post_xol(struct arch_uprobe auprobe, struct pt_regs regs)
		{
		}
		#endif /* CONFIG_X86_64 */
		@@ -414,13 +477,30 @@ static int push_ret_address(struct pt_regs *regs, unsigned long ip)
		return 0;
		}

		/*
		* We have to fix things up as follows:
		*
		* Typically, the new ip is relative to the copied instruction. We need
		* to make it relative to the original instruction (FIX_IP). Exceptions
		* are return instructions and absolute or indirect jump or call instructions.
		*
		* If the single-stepped instruction was a call, the return address that
		* is atop the stack is the address following the copied instruction. We
		* need to make it the address following the original instruction (FIX_CALL).
		*
		* If the original instruction was a rip-relative instruction such as
		* "movl %edx,0xnnnn(%rip)", we have instead executed an equivalent
		* instruction using a scratch register -- e.g., "movl %edx,0xnnnn(%rsi)".
		* We need to restore the contents of the scratch register
		* (FIX_RIP_reg).
		*/
		static int default_post_xol_op(struct arch_uprobe auprobe, struct pt_regs regs)
		{
		struct uprobe_task *utask = current->utask;
		long correction = (long)(utask->vaddr - utask->xol_vaddr);

		riprel_post_xol(auprobe, regs, &correction);
		riprel_post_xol(auprobe, regs);
		if (auprobe->def.fixups & UPROBE_FIX_IP) {
		long correction = utask->vaddr - utask->xol_vaddr;
		regs->ip += correction;
		} else if (auprobe->def.fixups & UPROBE_FIX_CALL) {
		regs->sp += sizeof_long();
		@@ -436,7 +516,7 @@ static int default_post_xol_op(struct arch_uprobe auprobe, struct pt_regs regs

		static void default_abort_op(struct arch_uprobe auprobe, struct pt_regs regs)
		{
		riprel_post_xol(auprobe, regs, NULL);
		riprel_post_xol(auprobe, regs);
		}

		static struct uprobe_xol_ops default_xol_ops = {
		@@ -720,23 +800,6 @@ bool arch_uprobe_xol_was_trapped(struct task_struct *t)
		* single-step, we single-stepped a copy of the instruction.
		*
		* This function prepares to resume execution after the single-step.
		* We have to fix things up as follows:
		*
		* Typically, the new ip is relative to the copied instruction. We need
		* to make it relative to the original instruction (FIX_IP). Exceptions
		* are return instructions and absolute or indirect jump or call instructions.
		*
		* If the single-stepped instruction was a call, the return address that
		* is atop the stack is the address following the copied instruction. We
		* need to make it the address following the original instruction (FIX_CALL).
		*
		* If the original instruction was a rip-relative instruction such as
		* "movl %edx,0xnnnn(%rip)", we have instead executed an equivalent
		* instruction using a scratch register -- e.g., "movl %edx,(%rax)".
		* We need to restore the contents of the scratch register and adjust
		* the ip, keeping in mind that the instruction we executed is 4 bytes
		* shorter than the original instruction (since we squeezed out the offset
		* field). (FIX_RIP_AX or FIX_RIP_CX)
		*/
		int arch_uprobe_post_xol(struct arch_uprobe auprobe, struct pt_regs regs)
		{

include/linux/uprobes.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -102,6 +102,7 @@ extern int __weak set_orig_insn(struct arch_uprobe aup, struct mm_struct mm, u
		extern bool __weak is_swbp_insn(uprobe_opcode_t *insn);
		extern bool __weak is_trap_insn(uprobe_opcode_t *insn);
		extern unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs);
		extern unsigned long uprobe_get_trap_addr(struct pt_regs *regs);
		extern int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr, uprobe_opcode_t);
		extern int uprobe_register(struct inode inode, loff_t offset, struct uprobe_consumer uc);
		extern int uprobe_apply(struct inode inode, loff_t offset, struct uprobe_consumer uc, bool);
		@@ -130,6 +131,9 @@ extern bool __weak arch_uprobe_ignore(struct arch_uprobe aup, struct pt_regs r
		#else /* !CONFIG_UPROBES */
		struct uprobes_state {
		};

		#define uprobe_get_trap_addr(regs) instruction_pointer(regs)

		static inline int
		uprobe_register(struct inode inode, loff_t offset, struct uprobe_consumer uc)
		{

Admin message