Commit 4610ee1d authored by Masami Hiramatsu's avatar Masami Hiramatsu Committed by Ingo Molnar
Browse files

kprobes: Introduce generic insn_slot framework 



Make insn_slot framework support various size slots.
Current insn_slot just supports one-size instruction buffer
slot. However, kprobes jump optimization needs larger size
buffers.

Signed-off-by: default avatarMasami Hiramatsu <mhiramat@redhat.com>
Cc: systemtap <systemtap@sources.redhat.com>
Cc: DLE <dle-develop@lists.sourceforge.net>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Cc: Jim Keniston <jkenisto@us.ibm.com>
Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Anders Kaseorg <andersk@ksplice.com>
Cc: Tim Abbott <tabbott@ksplice.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Jason Baron <jbaron@redhat.com>
Cc: Mathieu Desnoyers <compudj@krystal.dyndns.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
LKML-Reference: <20100225133358.6725.82430.stgit@localhost6.localdomain6>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Cc: Jim Keniston <jkenisto@us.ibm.com>
Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Anders Kaseorg <andersk@ksplice.com>
Cc: Tim Abbott <tabbott@ksplice.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Jason Baron <jbaron@redhat.com>
Cc: Mathieu Desnoyers <compudj@krystal.dyndns.org>
parent d498f763

kernel/kprobes.c

+65 −39
Original line number Diff line number Diff line
@@ -105,57 +105,74 @@ static struct kprobe_blackpoint kprobe_blacklist[] = { 
 * stepping on the instruction on a vmalloced/kmalloced/data page

 * is a recipe for disaster

 */

#define INSNS_PER_PAGE	(PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))



struct kprobe_insn_page {

	struct list_head list;

	kprobe_opcode_t *insns;		/* Page of instruction slots */

	char slot_used[INSNS_PER_PAGE];

	int nused;

	int ngarbage;

	char slot_used[];

};



#define KPROBE_INSN_PAGE_SIZE(slots)			\

	(offsetof(struct kprobe_insn_page, slot_used) +	\

	 (sizeof(char) * (slots)))



struct kprobe_insn_cache {

	struct list_head pages;	/* list of kprobe_insn_page */

	size_t insn_size;	/* size of instruction slot */

	int nr_garbage;

};



static int slots_per_page(struct kprobe_insn_cache *c)

{

	return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));

}



enum kprobe_slot_state {

	SLOT_CLEAN = 0,

	SLOT_DIRTY = 1,

	SLOT_USED = 2,

};



static DEFINE_MUTEX(kprobe_insn_mutex);	/* Protects kprobe_insn_pages */

static LIST_HEAD(kprobe_insn_pages);

static int kprobe_garbage_slots;

static int collect_garbage_slots(void);

static DEFINE_MUTEX(kprobe_insn_mutex);	/* Protects kprobe_insn_slots */

static struct kprobe_insn_cache kprobe_insn_slots = {

	.pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),

	.insn_size = MAX_INSN_SIZE,

	.nr_garbage = 0,

};

static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c);



/**

 * __get_insn_slot() - Find a slot on an executable page for an instruction.

 * We allocate an executable page if there's no room on existing ones.

 */

static kprobe_opcode_t __kprobes *__get_insn_slot(void)

static kprobe_opcode_t __kprobes *__get_insn_slot(struct kprobe_insn_cache *c)

{

	struct kprobe_insn_page *kip;



 retry:

	list_for_each_entry(kip, &kprobe_insn_pages, list) {

		if (kip->nused < INSNS_PER_PAGE) {

	list_for_each_entry(kip, &c->pages, list) {

		if (kip->nused < slots_per_page(c)) {

			int i;

			for (i = 0; i < INSNS_PER_PAGE; i++) {

			for (i = 0; i < slots_per_page(c); i++) {

				if (kip->slot_used[i] == SLOT_CLEAN) {

					kip->slot_used[i] = SLOT_USED;

					kip->nused++;

					return kip->insns + (i * MAX_INSN_SIZE);

					return kip->insns + (i * c->insn_size);

				}

			}

			/* Surprise!  No unused slots.  Fix kip->nused. */

			kip->nused = INSNS_PER_PAGE;

			/* kip->nused is broken. Fix it. */

			kip->nused = slots_per_page(c);

			WARN_ON(1);

		}

	}



	/* If there are any garbage slots, collect it and try again. */

	if (kprobe_garbage_slots && collect_garbage_slots() == 0) {

	if (c->nr_garbage && collect_garbage_slots(c) == 0)

		goto retry;

	}

	/* All out of space.  Need to allocate a new page. Use slot 0. */

	kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL);



	/* All out of space.  Need to allocate a new page. */

	kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);

	if (!kip)

		return NULL;
@@ -170,20 +187,23 @@ static kprobe_opcode_t __kprobes *__get_insn_slot(void) 
		return NULL;

	}

	INIT_LIST_HEAD(&kip->list);

	list_add(&kip->list, &kprobe_insn_pages);

	memset(kip->slot_used, SLOT_CLEAN, INSNS_PER_PAGE);

	memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));

	kip->slot_used[0] = SLOT_USED;

	kip->nused = 1;

	kip->ngarbage = 0;

	list_add(&kip->list, &c->pages);

	return kip->insns;

}





kprobe_opcode_t __kprobes *get_insn_slot(void)

{

	kprobe_opcode_t *ret;

	kprobe_opcode_t *ret = NULL;



	mutex_lock(&kprobe_insn_mutex);

	ret = __get_insn_slot();

	ret = __get_insn_slot(&kprobe_insn_slots);

	mutex_unlock(&kprobe_insn_mutex);



	return ret;

}
@@ -199,7 +219,7 @@ static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx) 
		 * so as not to have to set it up again the

		 * next time somebody inserts a probe.

		 */

		if (!list_is_singular(&kprobe_insn_pages)) {

		if (!list_is_singular(&kip->list)) {

			list_del(&kip->list);

			module_free(NULL, kip->insns);

			kfree(kip);
@@ -209,49 +229,55 @@ static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx) 
	return 0;

}



static int __kprobes collect_garbage_slots(void)

static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c)

{

	struct kprobe_insn_page *kip, *next;



	/* Ensure no-one is interrupted on the garbages */

	synchronize_sched();



	list_for_each_entry_safe(kip, next, &kprobe_insn_pages, list) {

	list_for_each_entry_safe(kip, next, &c->pages, list) {

		int i;

		if (kip->ngarbage == 0)

			continue;

		kip->ngarbage = 0;	/* we will collect all garbages */

		for (i = 0; i < INSNS_PER_PAGE; i++) {

		for (i = 0; i < slots_per_page(c); i++) {

			if (kip->slot_used[i] == SLOT_DIRTY &&

			    collect_one_slot(kip, i))

				break;

		}

	}

	kprobe_garbage_slots = 0;

	c->nr_garbage = 0;

	return 0;

}



void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)

static void __kprobes __free_insn_slot(struct kprobe_insn_cache *c,

				       kprobe_opcode_t *slot, int dirty)

{

	struct kprobe_insn_page *kip;



	mutex_lock(&kprobe_insn_mutex);

	list_for_each_entry(kip, &kprobe_insn_pages, list) {

		if (kip->insns <= slot &&

		    slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) {

			int i = (slot - kip->insns) / MAX_INSN_SIZE;

	list_for_each_entry(kip, &c->pages, list) {

		long idx = ((long)slot - (long)kip->insns) / c->insn_size;

		if (idx >= 0 && idx < slots_per_page(c)) {

			WARN_ON(kip->slot_used[idx] != SLOT_USED);

			if (dirty) {

				kip->slot_used[i] = SLOT_DIRTY;

				kip->slot_used[idx] = SLOT_DIRTY;

				kip->ngarbage++;

				if (++c->nr_garbage > slots_per_page(c))

					collect_garbage_slots(c);

			} else

				collect_one_slot(kip, i);

			break;

				collect_one_slot(kip, idx);

			return;

		}

	}

	/* Could not free this slot. */

	WARN_ON(1);

}



	if (dirty && ++kprobe_garbage_slots > INSNS_PER_PAGE)

		collect_garbage_slots();



void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)

{

	mutex_lock(&kprobe_insn_mutex);

	__free_insn_slot(&kprobe_insn_slots, slot, dirty);

	mutex_unlock(&kprobe_insn_mutex);

}

#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE