Commit 1c689070 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'perf-kprobes-2020-10-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull perf/kprobes updates from Ingo Molnar:
 "This prepares to unify the kretprobe trampoline handler and make
  kretprobe lockless (those patches are still work in progress)"

* tag 'perf-kprobes-2020-10-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  kprobes: Fix to check probe enabled before disarm_kprobe_ftrace()
  kprobes: Make local functions static
  kprobes: Free kretprobe_instance with RCU callback
  kprobes: Remove NMI context check
  sparc: kprobes: Use generic kretprobe trampoline handler
  sh: kprobes: Use generic kretprobe trampoline handler
  s390: kprobes: Use generic kretprobe trampoline handler
  powerpc: kprobes: Use generic kretprobe trampoline handler
  parisc: kprobes: Use generic kretprobe trampoline handler
  mips: kprobes: Use generic kretprobe trampoline handler
  ia64: kprobes: Use generic kretprobe trampoline handler
  csky: kprobes: Use generic kretprobe trampoline handler
  arc: kprobes: Use generic kretprobe trampoline handler
  arm64: kprobes: Use generic kretprobe trampoline handler
  arm: kprobes: Use generic kretprobe trampoline handler
  x86/kprobes: Use generic kretprobe trampoline handler
  kprobes: Add generic kretprobe trampoline handler
parents 3bff6112 bcb53209

arch/arc/kernel/kprobes.c

+2 −52
Original line number Diff line number Diff line
@@ -388,6 +388,7 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, 
{



	ri->ret_addr = (kprobe_opcode_t *) regs->blink;

	ri->fp = NULL;



	/* Replace the return addr with trampoline addr */

	regs->blink = (unsigned long)&kretprobe_trampoline;
@@ -396,58 +397,7 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, 
static int __kprobes trampoline_probe_handler(struct kprobe *p,

					      struct pt_regs *regs)

{

	struct kretprobe_instance *ri = NULL;

	struct hlist_head *head, empty_rp;

	struct hlist_node *tmp;

	unsigned long flags, orig_ret_address = 0;

	unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;



	INIT_HLIST_HEAD(&empty_rp);

	kretprobe_hash_lock(current, &head, &flags);



	/*

	 * It is possible to have multiple instances associated with a given

	 * task either because an multiple functions in the call path

	 * have a return probe installed on them, and/or more than one return

	 * return probe was registered for a target function.

	 *

	 * We can handle this because:

	 *     - instances are always inserted at the head of the list

	 *     - when multiple return probes are registered for the same

	 *       function, the first instance's ret_addr will point to the

	 *       real return address, and all the rest will point to

	 *       kretprobe_trampoline

	 */

	hlist_for_each_entry_safe(ri, tmp, head, hlist) {

		if (ri->task != current)

			/* another task is sharing our hash bucket */

			continue;



		if (ri->rp && ri->rp->handler)

			ri->rp->handler(ri, regs);



		orig_ret_address = (unsigned long)ri->ret_addr;

		recycle_rp_inst(ri, &empty_rp);



		if (orig_ret_address != trampoline_address) {

			/*

			 * This is the real return address. Any other

			 * instances associated with this task are for

			 * other calls deeper on the call stack

			 */

			break;

		}

	}



	kretprobe_assert(ri, orig_ret_address, trampoline_address);

	regs->ret = orig_ret_address;



	kretprobe_hash_unlock(current, &flags);



	hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {

		hlist_del(&ri->hlist);

		kfree(ri);

	}

	regs->ret = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL);



	/* By returning a non zero value, we are telling the kprobe handler

	 * that we don't want the post_handler to run

arch/arm/probes/kprobes/core.c

+3 −75
Original line number Diff line number Diff line
@@ -413,87 +413,15 @@ void __naked __kprobes kretprobe_trampoline(void) 
/* Called from kretprobe_trampoline */

static __used __kprobes void *trampoline_handler(struct pt_regs *regs)

{

	struct kretprobe_instance *ri = NULL;

	struct hlist_head *head, empty_rp;

	struct hlist_node *tmp;

	unsigned long flags, orig_ret_address = 0;

	unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;

	kprobe_opcode_t *correct_ret_addr = NULL;



	INIT_HLIST_HEAD(&empty_rp);

	kretprobe_hash_lock(current, &head, &flags);



	/*

	 * It is possible to have multiple instances associated with a given

	 * task either because multiple functions in the call path have

	 * a return probe installed on them, and/or more than one return

	 * probe was registered for a target function.

	 *

	 * We can handle this because:

	 *     - instances are always inserted at the head of the list

	 *     - when multiple return probes are registered for the same

	 *       function, the first instance's ret_addr will point to the

	 *       real return address, and all the rest will point to

	 *       kretprobe_trampoline

	 */

	hlist_for_each_entry_safe(ri, tmp, head, hlist) {

		if (ri->task != current)

			/* another task is sharing our hash bucket */

			continue;



		orig_ret_address = (unsigned long)ri->ret_addr;



		if (orig_ret_address != trampoline_address)

			/*

			 * This is the real return address. Any other

			 * instances associated with this task are for

			 * other calls deeper on the call stack

			 */

			break;

	}



	kretprobe_assert(ri, orig_ret_address, trampoline_address);



	correct_ret_addr = ri->ret_addr;

	hlist_for_each_entry_safe(ri, tmp, head, hlist) {

		if (ri->task != current)

			/* another task is sharing our hash bucket */

			continue;



		orig_ret_address = (unsigned long)ri->ret_addr;

		if (ri->rp && ri->rp->handler) {

			__this_cpu_write(current_kprobe, &ri->rp->kp);

			get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;

			ri->ret_addr = correct_ret_addr;

			ri->rp->handler(ri, regs);

			__this_cpu_write(current_kprobe, NULL);

		}



		recycle_rp_inst(ri, &empty_rp);



		if (orig_ret_address != trampoline_address)

			/*

			 * This is the real return address. Any other

			 * instances associated with this task are for

			 * other calls deeper on the call stack

			 */

			break;

	}



	kretprobe_hash_unlock(current, &flags);



	hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {

		hlist_del(&ri->hlist);

		kfree(ri);

	}



	return (void *)orig_ret_address;

	return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline,

						    (void *)regs->ARM_fp);

}



void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,

				      struct pt_regs *regs)

{

	ri->ret_addr = (kprobe_opcode_t *)regs->ARM_lr;

	ri->fp = (void *)regs->ARM_fp;



	/* Replace the return addr with trampoline addr. */

	regs->ARM_lr = (unsigned long)&kretprobe_trampoline;

arch/arm64/kernel/probes/kprobes.c

+3 −75
Original line number Diff line number Diff line
@@ -464,87 +464,15 @@ int __init arch_populate_kprobe_blacklist(void) 


void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs)

{

	struct kretprobe_instance *ri = NULL;

	struct hlist_head *head, empty_rp;

	struct hlist_node *tmp;

	unsigned long flags, orig_ret_address = 0;

	unsigned long trampoline_address =

		(unsigned long)&kretprobe_trampoline;

	kprobe_opcode_t *correct_ret_addr = NULL;



	INIT_HLIST_HEAD(&empty_rp);

	kretprobe_hash_lock(current, &head, &flags);



	/*

	 * It is possible to have multiple instances associated with a given

	 * task either because multiple functions in the call path have

	 * return probes installed on them, and/or more than one

	 * return probe was registered for a target function.

	 *

	 * We can handle this because:

	 *     - instances are always pushed into the head of the list

	 *     - when multiple return probes are registered for the same

	 *	 function, the (chronologically) first instance's ret_addr

	 *	 will be the real return address, and all the rest will

	 *	 point to kretprobe_trampoline.

	 */

	hlist_for_each_entry_safe(ri, tmp, head, hlist) {

		if (ri->task != current)

			/* another task is sharing our hash bucket */

			continue;



		orig_ret_address = (unsigned long)ri->ret_addr;



		if (orig_ret_address != trampoline_address)

			/*

			 * This is the real return address. Any other

			 * instances associated with this task are for

			 * other calls deeper on the call stack

			 */

			break;

	}



	kretprobe_assert(ri, orig_ret_address, trampoline_address);



	correct_ret_addr = ri->ret_addr;

	hlist_for_each_entry_safe(ri, tmp, head, hlist) {

		if (ri->task != current)

			/* another task is sharing our hash bucket */

			continue;



		orig_ret_address = (unsigned long)ri->ret_addr;

		if (ri->rp && ri->rp->handler) {

			__this_cpu_write(current_kprobe, &ri->rp->kp);

			get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;

			ri->ret_addr = correct_ret_addr;

			ri->rp->handler(ri, regs);

			__this_cpu_write(current_kprobe, NULL);

		}



		recycle_rp_inst(ri, &empty_rp);



		if (orig_ret_address != trampoline_address)

			/*

			 * This is the real return address. Any other

			 * instances associated with this task are for

			 * other calls deeper on the call stack

			 */

			break;

	}



	kretprobe_hash_unlock(current, &flags);



	hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {

		hlist_del(&ri->hlist);

		kfree(ri);

	}

	return (void *)orig_ret_address;

	return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline,

					(void *)kernel_stack_pointer(regs));

}



void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,

				      struct pt_regs *regs)

{

	ri->ret_addr = (kprobe_opcode_t *)regs->regs[30];

	ri->fp = (void *)kernel_stack_pointer(regs);



	/* replace return addr (x30) with trampoline */

	regs->regs[30] = (long)&kretprobe_trampoline;

arch/csky/kernel/probes/kprobes.c

+2 −75
Original line number Diff line number Diff line
@@ -404,87 +404,14 @@ int __init arch_populate_kprobe_blacklist(void) 


void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs)

{

	struct kretprobe_instance *ri = NULL;

	struct hlist_head *head, empty_rp;

	struct hlist_node *tmp;

	unsigned long flags, orig_ret_address = 0;

	unsigned long trampoline_address =

		(unsigned long)&kretprobe_trampoline;

	kprobe_opcode_t *correct_ret_addr = NULL;



	INIT_HLIST_HEAD(&empty_rp);

	kretprobe_hash_lock(current, &head, &flags);



	/*

	 * It is possible to have multiple instances associated with a given

	 * task either because multiple functions in the call path have

	 * return probes installed on them, and/or more than one

	 * return probe was registered for a target function.

	 *

	 * We can handle this because:

	 *     - instances are always pushed into the head of the list

	 *     - when multiple return probes are registered for the same

	 *	 function, the (chronologically) first instance's ret_addr

	 *	 will be the real return address, and all the rest will

	 *	 point to kretprobe_trampoline.

	 */

	hlist_for_each_entry_safe(ri, tmp, head, hlist) {

		if (ri->task != current)

			/* another task is sharing our hash bucket */

			continue;



		orig_ret_address = (unsigned long)ri->ret_addr;



		if (orig_ret_address != trampoline_address)

			/*

			 * This is the real return address. Any other

			 * instances associated with this task are for

			 * other calls deeper on the call stack

			 */

			break;

	}



	kretprobe_assert(ri, orig_ret_address, trampoline_address);



	correct_ret_addr = ri->ret_addr;

	hlist_for_each_entry_safe(ri, tmp, head, hlist) {

		if (ri->task != current)

			/* another task is sharing our hash bucket */

			continue;



		orig_ret_address = (unsigned long)ri->ret_addr;

		if (ri->rp && ri->rp->handler) {

			__this_cpu_write(current_kprobe, &ri->rp->kp);

			get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;

			ri->ret_addr = correct_ret_addr;

			ri->rp->handler(ri, regs);

			__this_cpu_write(current_kprobe, NULL);

		}



		recycle_rp_inst(ri, &empty_rp);



		if (orig_ret_address != trampoline_address)

			/*

			 * This is the real return address. Any other

			 * instances associated with this task are for

			 * other calls deeper on the call stack

			 */

			break;

	}



	kretprobe_hash_unlock(current, &flags);



	hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {

		hlist_del(&ri->hlist);

		kfree(ri);

	}

	return (void *)orig_ret_address;

	return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL);

}



void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,

				      struct pt_regs *regs)

{

	ri->ret_addr = (kprobe_opcode_t *)regs->lr;

	ri->fp = NULL;

	regs->lr = (unsigned long) &kretprobe_trampoline;

}

arch/ia64/kernel/kprobes.c

+2 −75
Original line number Diff line number Diff line
@@ -396,83 +396,9 @@ static void kretprobe_trampoline(void) 
{

}



/*

 * At this point the target function has been tricked into

 * returning into our trampoline.  Lookup the associated instance

 * and then:

 *    - call the handler function

 *    - cleanup by marking the instance as unused

 *    - long jump back to the original return address

 */

int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)

{

	struct kretprobe_instance *ri = NULL;

	struct hlist_head *head, empty_rp;

	struct hlist_node *tmp;

	unsigned long flags, orig_ret_address = 0;

	unsigned long trampoline_address =

		((struct fnptr *)kretprobe_trampoline)->ip;



	INIT_HLIST_HEAD(&empty_rp);

	kretprobe_hash_lock(current, &head, &flags);



	/*

	 * It is possible to have multiple instances associated with a given

	 * task either because an multiple functions in the call path

	 * have a return probe installed on them, and/or more than one return

	 * return probe was registered for a target function.

	 *

	 * We can handle this because:

	 *     - instances are always inserted at the head of the list

	 *     - when multiple return probes are registered for the same

	 *       function, the first instance's ret_addr will point to the

	 *       real return address, and all the rest will point to

	 *       kretprobe_trampoline

	 */

	hlist_for_each_entry_safe(ri, tmp, head, hlist) {

		if (ri->task != current)

			/* another task is sharing our hash bucket */

			continue;



		orig_ret_address = (unsigned long)ri->ret_addr;

		if (orig_ret_address != trampoline_address)

			/*

			 * This is the real return address. Any other

			 * instances associated with this task are for

			 * other calls deeper on the call stack

			 */

			break;

	}



	regs->cr_iip = orig_ret_address;



	hlist_for_each_entry_safe(ri, tmp, head, hlist) {

		if (ri->task != current)

			/* another task is sharing our hash bucket */

			continue;



		if (ri->rp && ri->rp->handler)

			ri->rp->handler(ri, regs);



		orig_ret_address = (unsigned long)ri->ret_addr;

		recycle_rp_inst(ri, &empty_rp);



		if (orig_ret_address != trampoline_address)

			/*

			 * This is the real return address. Any other

			 * instances associated with this task are for

			 * other calls deeper on the call stack

			 */

			break;

	}

	kretprobe_assert(ri, orig_ret_address, trampoline_address);



	kretprobe_hash_unlock(current, &flags);



	hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {

		hlist_del(&ri->hlist);

		kfree(ri);

	}

	regs->cr_iip = __kretprobe_trampoline_handler(regs, kretprobe_trampoline, NULL);

	/*

	 * By returning a non-zero value, we are telling

	 * kprobe_handler() that we don't want the post_handler
@@ -485,6 +411,7 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, 
				      struct pt_regs *regs)

{

	ri->ret_addr = (kprobe_opcode_t *)regs->b0;

	ri->fp = NULL;



	/* Replace the return addr with trampoline addr */

	regs->b0 = ((struct fnptr *)kretprobe_trampoline)->ip;

CRA Git | Maintained and supported by SUSTech CRA and CCSE