Commit c1961a46 authored by Hugh Dickins's avatar Hugh Dickins Committed by Ingo Molnar
Browse files

x86/events/intel/ds: Map debug buffers in cpu_entry_area 



The BTS and PEBS buffers both have their virtual addresses programmed into
the hardware.  This means that any access to them is performed via the page
tables.  The times that the hardware accesses these are entirely dependent
on how the performance monitoring hardware events are set up.  In other
words, there is no way for the kernel to tell when the hardware might
access these buffers.

To avoid perf crashes, place 'debug_store' allocate pages and map them into
the cpu_entry_area.

The PEBS fixup buffer does not need this treatment.

[ tglx: Got rid of the kaiser_add_mapping() complication ]

Signed-off-by: default avatarHugh Dickins <hughd@google.com>
Signed-off-by: default avatarDave Hansen <dave.hansen@linux.intel.com>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: David Laight <David.Laight@aculab.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Eduardo Valentin <eduval@amazon.com>
Cc: Greg KH <gregkh@linuxfoundation.org>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will.deacon@arm.com>
Cc: aliguori@amazon.com
Cc: daniel.gruss@iaik.tugraz.at
Cc: keescook@google.com
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 10043e02

arch/x86/events/intel/ds.c

+80 −45
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ 
#include <linux/types.h>

#include <linux/slab.h>



#include <asm/cpu_entry_area.h>

#include <asm/perf_event.h>

#include <asm/insn.h>
@@ -280,17 +281,52 @@ void fini_debug_store_on_cpu(int cpu) 


static DEFINE_PER_CPU(void *, insn_buffer);



static int alloc_pebs_buffer(int cpu)

static void ds_update_cea(void *cea, void *addr, size_t size, pgprot_t prot)

{

	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;

	phys_addr_t pa;

	size_t msz = 0;



	pa = virt_to_phys(addr);

	for (; msz < size; msz += PAGE_SIZE, pa += PAGE_SIZE, cea += PAGE_SIZE)

		cea_set_pte(cea, pa, prot);

}



static void ds_clear_cea(void *cea, size_t size)

{

	size_t msz = 0;



	for (; msz < size; msz += PAGE_SIZE, cea += PAGE_SIZE)

		cea_set_pte(cea, 0, PAGE_NONE);

}



static void *dsalloc_pages(size_t size, gfp_t flags, int cpu)

{

	unsigned int order = get_order(size);

	int node = cpu_to_node(cpu);

	int max;

	void *buffer, *ibuffer;

	struct page *page;



	page = __alloc_pages_node(node, flags | __GFP_ZERO, order);

	return page ? page_address(page) : NULL;

}



static void dsfree_pages(const void *buffer, size_t size)

{

	if (buffer)

		free_pages((unsigned long)buffer, get_order(size));

}



static int alloc_pebs_buffer(int cpu)

{

	struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);

	struct debug_store *ds = hwev->ds;

	size_t bsiz = x86_pmu.pebs_buffer_size;

	int max, node = cpu_to_node(cpu);

	void *buffer, *ibuffer, *cea;



	if (!x86_pmu.pebs)

		return 0;



	buffer = kzalloc_node(x86_pmu.pebs_buffer_size, GFP_KERNEL, node);

	buffer = dsalloc_pages(bsiz, GFP_KERNEL, cpu);

	if (unlikely(!buffer))

		return -ENOMEM;
@@ -301,25 +337,27 @@ static int alloc_pebs_buffer(int cpu) 
	if (x86_pmu.intel_cap.pebs_format < 2) {

		ibuffer = kzalloc_node(PEBS_FIXUP_SIZE, GFP_KERNEL, node);

		if (!ibuffer) {

			kfree(buffer);

			dsfree_pages(buffer, bsiz);

			return -ENOMEM;

		}

		per_cpu(insn_buffer, cpu) = ibuffer;

	}



	max = x86_pmu.pebs_buffer_size / x86_pmu.pebs_record_size;



	ds->pebs_buffer_base = (u64)(unsigned long)buffer;

	hwev->ds_pebs_vaddr = buffer;

	/* Update the cpu entry area mapping */

	cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;

	ds->pebs_buffer_base = (unsigned long) cea;

	ds_update_cea(cea, buffer, bsiz, PAGE_KERNEL);

	ds->pebs_index = ds->pebs_buffer_base;

	ds->pebs_absolute_maximum = ds->pebs_buffer_base +

		max * x86_pmu.pebs_record_size;



	max = x86_pmu.pebs_record_size * (bsiz / x86_pmu.pebs_record_size);

	ds->pebs_absolute_maximum = ds->pebs_buffer_base + max;

	return 0;

}



static void release_pebs_buffer(int cpu)

{

	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;

	struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);

	struct debug_store *ds = hwev->ds;

	void *cea;



	if (!ds || !x86_pmu.pebs)

		return;
@@ -327,73 +365,70 @@ static void release_pebs_buffer(int cpu) 
	kfree(per_cpu(insn_buffer, cpu));

	per_cpu(insn_buffer, cpu) = NULL;



	kfree((void *)(unsigned long)ds->pebs_buffer_base);

	/* Clear the fixmap */

	cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;

	ds_clear_cea(cea, x86_pmu.pebs_buffer_size);

	ds->pebs_buffer_base = 0;

	dsfree_pages(hwev->ds_pebs_vaddr, x86_pmu.pebs_buffer_size);

	hwev->ds_pebs_vaddr = NULL;

}



static int alloc_bts_buffer(int cpu)

{

	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;

	int node = cpu_to_node(cpu);

	int max, thresh;

	void *buffer;

	struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);

	struct debug_store *ds = hwev->ds;

	void *buffer, *cea;

	int max;



	if (!x86_pmu.bts)

		return 0;



	buffer = kzalloc_node(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, node);

	buffer = dsalloc_pages(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, cpu);

	if (unlikely(!buffer)) {

		WARN_ONCE(1, "%s: BTS buffer allocation failure\n", __func__);

		return -ENOMEM;

	}



	max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE;

	thresh = max / 16;



	ds->bts_buffer_base = (u64)(unsigned long)buffer;

	hwev->ds_bts_vaddr = buffer;

	/* Update the fixmap */

	cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer;

	ds->bts_buffer_base = (unsigned long) cea;

	ds_update_cea(cea, buffer, BTS_BUFFER_SIZE, PAGE_KERNEL);

	ds->bts_index = ds->bts_buffer_base;

	ds->bts_absolute_maximum = ds->bts_buffer_base +

		max * BTS_RECORD_SIZE;

	ds->bts_interrupt_threshold = ds->bts_absolute_maximum -

		thresh * BTS_RECORD_SIZE;



	max = BTS_RECORD_SIZE * (BTS_BUFFER_SIZE / BTS_RECORD_SIZE);

	ds->bts_absolute_maximum = ds->bts_buffer_base + max;

	ds->bts_interrupt_threshold = ds->bts_absolute_maximum - (max / 16);

	return 0;

}



static void release_bts_buffer(int cpu)

{

	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;

	struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);

	struct debug_store *ds = hwev->ds;

	void *cea;



	if (!ds || !x86_pmu.bts)

		return;



	kfree((void *)(unsigned long)ds->bts_buffer_base);

	/* Clear the fixmap */

	cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer;

	ds_clear_cea(cea, BTS_BUFFER_SIZE);

	ds->bts_buffer_base = 0;

	dsfree_pages(hwev->ds_bts_vaddr, BTS_BUFFER_SIZE);

	hwev->ds_bts_vaddr = NULL;

}



static int alloc_ds_buffer(int cpu)

{

	int node = cpu_to_node(cpu);

	struct debug_store *ds;



	ds = kzalloc_node(sizeof(*ds), GFP_KERNEL, node);

	if (unlikely(!ds))

		return -ENOMEM;

	struct debug_store *ds = &get_cpu_entry_area(cpu)->cpu_debug_store;



	memset(ds, 0, sizeof(*ds));

	per_cpu(cpu_hw_events, cpu).ds = ds;



	return 0;

}



static void release_ds_buffer(int cpu)

{

	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;



	if (!ds)

		return;



	per_cpu(cpu_hw_events, cpu).ds = NULL;

	kfree(ds);

}



void release_ds_buffers(void)

arch/x86/events/perf_event.h

+2 −0
Original line number Diff line number Diff line
@@ -199,6 +199,8 @@ struct cpu_hw_events { 
	 * Intel DebugStore bits

	 */

	struct debug_store	*ds;

	void			*ds_pebs_vaddr;

	void			*ds_bts_vaddr;

	u64			pebs_enabled;

	int			n_pebs;

	int			n_large_pebs;

CRA Git | Maintained and supported by SUSTech CRA and CCSE