Commit c450c8f5 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Borislav Petkov
Browse files

x86/dumpstack/64: Speedup in_exception_stack() 



The current implementation of in_exception_stack() iterates over the
exception stacks array. Most of the time this is an useless exercise, but
even for the actual use cases (perf and ftrace) it takes at least 2
iterations to get to the NMI stack.

As the exception stacks and the guard pages are page aligned the loop can
be avoided completely.

Add a initial check whether the stack pointer is inside the full exception
stack area and leave early if not.

Create a lookup table which describes the stack area. The table index is
the page offset from the beginning of the exception stacks. So for any
given stack pointer the page offset is computed and a lookup in the
description table is performed. If it is inside a guard page, return. If
not, use the descriptor to fill in the info structure.

The table is filled at compile time and for the !KASAN case the interesting
page descriptors exactly fit into a single cache line. Just the last guard
page descriptor is in the next cacheline, but that should not be accessed
in the regular case.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Acked-by: default avatarJosh Poimboeuf <jpoimboe@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Sean Christopherson <sean.j.christopherson@intel.com>
Cc: x86-ml <x86@kernel.org>
Link: https://lkml.kernel.org/r/20190414160145.543320386@linutronix.de
parent 2a594d4c

arch/x86/kernel/dumpstack_64.c

+51 −31
Original line number Diff line number Diff line
@@ -50,54 +50,74 @@ const char *stack_type_name(enum stack_type type) 
	return NULL;

}



struct estack_layout {

	unsigned int	begin;

	unsigned int	end;

/**

 * struct estack_pages - Page descriptor for exception stacks

 * @offs:	Offset from the start of the exception stack area

 * @size:	Size of the exception stack

 * @type:	Type to store in the stack_info struct

 */

struct estack_pages {

	u32	offs;

	u16	size;

	u16	type;

};



#define	ESTACK_ENTRY(x)	{						  \

	.begin	= offsetof(struct cea_exception_stacks, x## _stack),	  \

	.end	= offsetof(struct cea_exception_stacks, x## _stack_guard) \

	}

#define EPAGERANGE(st)							\

	[PFN_DOWN(CEA_ESTACK_OFFS(st)) ...				\

	 PFN_DOWN(CEA_ESTACK_OFFS(st) + CEA_ESTACK_SIZE(st) - 1)] = {	\

		.offs	= CEA_ESTACK_OFFS(st),				\

		.size	= CEA_ESTACK_SIZE(st),				\

		.type	= STACK_TYPE_EXCEPTION + ESTACK_ ##st, }



static const struct estack_layout layout[] = {

	[ ESTACK_DF	]	= ESTACK_ENTRY(DF),

	[ ESTACK_NMI	]	= ESTACK_ENTRY(NMI),

	[ ESTACK_DB2	]	= { .begin = 0, .end = 0},

	[ ESTACK_DB1	]	= ESTACK_ENTRY(DB1),

	[ ESTACK_DB	]	= ESTACK_ENTRY(DB),

	[ ESTACK_MCE	]	= ESTACK_ENTRY(MCE),

/*

 * Array of exception stack page descriptors. If the stack is larger than

 * PAGE_SIZE, all pages covering a particular stack will have the same

 * info. The guard pages including the not mapped DB2 stack are zeroed

 * out.

 */

static const

struct estack_pages estack_pages[CEA_ESTACK_PAGES] ____cacheline_aligned = {

	EPAGERANGE(DF),

	EPAGERANGE(NMI),

	EPAGERANGE(DB1),

	EPAGERANGE(DB),

	EPAGERANGE(MCE),

};



static bool in_exception_stack(unsigned long *stack, struct stack_info *info)

{

	unsigned long estacks, begin, end, stk = (unsigned long)stack;

	unsigned long begin, end, stk = (unsigned long)stack;

	const struct estack_pages *ep;

	struct pt_regs *regs;

	unsigned int k;



	BUILD_BUG_ON(N_EXCEPTION_STACKS != 6);



	estacks = (unsigned long)__this_cpu_read(cea_exception_stacks);

	begin = (unsigned long)__this_cpu_read(cea_exception_stacks);

	end = begin + sizeof(struct cea_exception_stacks);

	/* Bail if @stack is outside the exception stack area. */

	if (stk < begin || stk >= end)

		return false;



	for (k = 0; k < N_EXCEPTION_STACKS; k++) {

		begin = estacks + layout[k].begin;

		end   = estacks + layout[k].end;

		regs  = (struct pt_regs *)end - 1;

	/* Calc page offset from start of exception stacks */

	k = (stk - begin) >> PAGE_SHIFT;

	/* Lookup the page descriptor */

	ep = &estack_pages[k];

	/* Guard page? */

	if (!ep->size)

		return false;



		if (stk < begin || stk >= end)

			continue;

	begin += (unsigned long)ep->offs;

	end = begin + (unsigned long)ep->size;

	regs = (struct pt_regs *)end - 1;



		info->type	= STACK_TYPE_EXCEPTION + k;

	info->type	= ep->type;

	info->begin	= (unsigned long *)begin;

	info->end	= (unsigned long *)end;

	info->next_sp	= (unsigned long *)regs->sp;



	return true;

}



	return false;

}



static bool in_irq_stack(unsigned long *stack, struct stack_info *info)

{

	unsigned long *end   = (unsigned long *)this_cpu_read(irq_stack_ptr);

CRA Git | Maintained and supported by SUSTech CRA and CCSE