Commit e6401c13 authored by Andy Lutomirski's avatar Andy Lutomirski Committed by Borislav Petkov
Browse files

x86/irq/64: Split the IRQ stack into its own pages 



Currently, the IRQ stack is hardcoded as the first page of the percpu
area, and the stack canary lives on the IRQ stack. The former gets in
the way of adding an IRQ stack guard page, and the latter is a potential
weakness in the stack canary mechanism.

Split the IRQ stack into its own private percpu pages.

[ tglx: Make 64 and 32 bit share struct irq_stack ]

Signed-off-by: default avatarAndy Lutomirski <luto@kernel.org>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Cc: Brijesh Singh <brijesh.singh@amd.com>
Cc: "Chang S. Bae" <chang.seok.bae@intel.com>
Cc: Dominik Brodowski <linux@dominikbrodowski.net>
Cc: Feng Tang <feng.tang@intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jan Beulich <JBeulich@suse.com>
Cc: Jiri Kosina <jkosina@suse.cz>
Cc: Joerg Roedel <jroedel@suse.de>
Cc: Jordan Borgner <mail@jordan-borgner.de>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Maran Wilson <maran.wilson@oracle.com>
Cc: Masahiro Yamada <yamada.masahiro@socionext.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@linux.vnet.ibm.com>
Cc: Nick Desaulniers <ndesaulniers@google.com>
Cc: Nicolai Stange <nstange@suse.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Pu Wen <puwen@hygon.cn>
Cc: "Rafael Ávila de Espíndola" <rafael@espindo.la>
Cc: Sean Christopherson <sean.j.christopherson@intel.com>
Cc: Stefano Stabellini <sstabellini@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: x86-ml <x86@kernel.org>
Cc: xen-devel@lists.xenproject.org
Link: https://lkml.kernel.org/r/20190414160146.267376656@linutronix.de
parent 0ac26104

arch/x86/entry/entry_64.S

+2 −2
Original line number Diff line number Diff line
@@ -298,7 +298,7 @@ ENTRY(__switch_to_asm) 


#ifdef CONFIG_STACKPROTECTOR

	movq	TASK_stack_canary(%rsi), %rbx

	movq	%rbx, PER_CPU_VAR(irq_stack_union)+stack_canary_offset

	movq	%rbx, PER_CPU_VAR(fixed_percpu_data) + stack_canary_offset

#endif



#ifdef CONFIG_RETPOLINE
@@ -430,7 +430,7 @@ END(irq_entries_start) 
	 * it before we actually move ourselves to the IRQ stack.

	 */



	movq	\old_rsp, PER_CPU_VAR(irq_stack_union + IRQ_STACK_SIZE - 8)

	movq	\old_rsp, PER_CPU_VAR(irq_stack_backing_store + IRQ_STACK_SIZE - 8)

	movq	PER_CPU_VAR(hardirq_stack_ptr), %rsp



#ifdef CONFIG_DEBUG_ENTRY

arch/x86/include/asm/processor.h

+14 −18
Original line number Diff line number Diff line
@@ -367,6 +367,13 @@ DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw); 
#define __KERNEL_TSS_LIMIT	\

	(IO_BITMAP_OFFSET + IO_BITMAP_BYTES + sizeof(unsigned long) - 1)



/* Per CPU interrupt stacks */

struct irq_stack {

	char		stack[IRQ_STACK_SIZE];

} __aligned(IRQ_STACK_SIZE);



DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);



#ifdef CONFIG_X86_32

DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);

#else
@@ -375,28 +382,24 @@ DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack); 
#endif



#ifdef CONFIG_X86_64

union irq_stack_union {

	char irq_stack[IRQ_STACK_SIZE];

struct fixed_percpu_data {

	/*

	 * GCC hardcodes the stack canary as %gs:40.  Since the

	 * irq_stack is the object at %gs:0, we reserve the bottom

	 * 48 bytes of the irq stack for the canary.

	 */

	struct {

	char		gs_base[40];

	unsigned long	stack_canary;

};

};



DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible;

DECLARE_INIT_PER_CPU(irq_stack_union);

DECLARE_PER_CPU_FIRST(struct fixed_percpu_data, fixed_percpu_data) __visible;

DECLARE_INIT_PER_CPU(fixed_percpu_data);



static inline unsigned long cpu_kernelmode_gs_base(int cpu)

{

	return (unsigned long)per_cpu(irq_stack_union.gs_base, cpu);

	return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);

}



DECLARE_PER_CPU(char *, hardirq_stack_ptr);

DECLARE_PER_CPU(unsigned int, irq_count);

extern asmlinkage void ignore_sysret(void);
@@ -418,14 +421,7 @@ struct stack_canary { 
};

DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);

#endif

/*

 * per-CPU IRQ handling stacks

 */

struct irq_stack {

	char			stack[IRQ_STACK_SIZE];

} __aligned(IRQ_STACK_SIZE);



DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);

/* Per CPU softirq stack pointer */

DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr);

#endif	/* X86_64 */

arch/x86/include/asm/stackprotector.h

+3 −3
Original line number Diff line number Diff line
@@ -13,7 +13,7 @@ 
 * On x86_64, %gs is shared by percpu area and stack canary.  All

 * percpu symbols are zero based and %gs points to the base of percpu

 * area.  The first occupant of the percpu area is always

 * irq_stack_union which contains stack_canary at offset 40.  Userland

 * fixed_percpu_data which contains stack_canary at offset 40.  Userland

 * %gs is always saved and restored on kernel entry and exit using

 * swapgs, so stack protector doesn't add any complexity there.

 *
@@ -64,7 +64,7 @@ static __always_inline void boot_init_stack_canary(void) 
	u64 tsc;



#ifdef CONFIG_X86_64

	BUILD_BUG_ON(offsetof(union irq_stack_union, stack_canary) != 40);

	BUILD_BUG_ON(offsetof(struct fixed_percpu_data, stack_canary) != 40);

#endif

	/*

	 * We both use the random pool and the current TSC as a source
@@ -79,7 +79,7 @@ static __always_inline void boot_init_stack_canary(void) 


	current->stack_canary = canary;

#ifdef CONFIG_X86_64

	this_cpu_write(irq_stack_union.stack_canary, canary);

	this_cpu_write(fixed_percpu_data.stack_canary, canary);

#else

	this_cpu_write(stack_canary.canary, canary);

#endif

arch/x86/kernel/asm-offsets_64.c

+1 −1
Original line number Diff line number Diff line
@@ -73,7 +73,7 @@ int main(void) 
	BLANK();



#ifdef CONFIG_STACKPROTECTOR

	DEFINE(stack_canary_offset, offsetof(union irq_stack_union, stack_canary));

	DEFINE(stack_canary_offset, offsetof(struct fixed_percpu_data, stack_canary));

	BLANK();

#endif

arch/x86/kernel/cpu/common.c

+4 −4
Original line number Diff line number Diff line
@@ -1498,9 +1498,9 @@ static __init int setup_clearcpuid(char *arg) 
__setup("clearcpuid=", setup_clearcpuid);



#ifdef CONFIG_X86_64

DEFINE_PER_CPU_FIRST(union irq_stack_union,

		     irq_stack_union) __aligned(PAGE_SIZE) __visible;

EXPORT_PER_CPU_SYMBOL_GPL(irq_stack_union);

DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,

		     fixed_percpu_data) __aligned(PAGE_SIZE) __visible;

EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);



/*

 * The following percpu variables are hot.  Align current_task to
@@ -1510,7 +1510,7 @@ DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned = 
	&init_task;

EXPORT_PER_CPU_SYMBOL(current_task);



DEFINE_PER_CPU(char *, hardirq_stack_ptr);

DEFINE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);

DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1;



DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;

CRA Git | Maintained and supported by SUSTech CRA and CCSE