Commit 37a8f7c3 authored by Andy Lutomirski's avatar Andy Lutomirski Committed by Thomas Gleixner
Browse files

x86/asm: Move 'status' from thread_struct to thread_info 



The TS_COMPAT bit is very hot and is accessed from code paths that mostly
also touch thread_info::flags.  Move it into struct thread_info to improve
cache locality.

The only reason it was in thread_struct is that there was a brief period
during which arch-specific fields were not allowed in struct thread_info.

Linus suggested further changing:

  ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);

to:

  if (unlikely(ti->status & (TS_COMPAT|TS_I386_REGS_POKED)))
          ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);

on the theory that frequently dirtying the cacheline even in pure 64-bit
code that never needs to modify status hurts performance.  That could be a
reasonable followup patch, but I suspect it matters less on top of this
patch.

Suggested-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: default avatarAndy Lutomirski <luto@kernel.org>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Reviewed-by: default avatarIngo Molnar <mingo@kernel.org>
Acked-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Kernel Hardening <kernel-hardening@lists.openwall.com>
Link: https://lkml.kernel.org/r/03148bcc1b217100e6e8ecf6a5468c45cf4304b6.1517164461.git.luto@kernel.org
parent d1f77320

arch/x86/entry/common.c

+2 −2
Original line number Diff line number Diff line
@@ -206,7 +206,7 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs) 
	 * special case only applies after poking regs and before the

	 * very next return to user mode.

	 */

	current->thread.status &= ~(TS_COMPAT|TS_I386_REGS_POKED);

	ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);

#endif



	user_enter_irqoff();
@@ -304,7 +304,7 @@ static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs) 
	unsigned int nr = (unsigned int)regs->orig_ax;



#ifdef CONFIG_IA32_EMULATION

	current->thread.status |= TS_COMPAT;

	ti->status |= TS_COMPAT;

#endif



	if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY) {

arch/x86/include/asm/processor.h

+0 −2
Original line number Diff line number Diff line
@@ -460,8 +460,6 @@ struct thread_struct { 
	unsigned short		gsindex;

#endif



	u32			status;		/* thread synchronous flags */



#ifdef CONFIG_X86_64

	unsigned long		fsbase;

	unsigned long		gsbase;

arch/x86/include/asm/syscall.h

+3 −3
Original line number Diff line number Diff line
@@ -60,7 +60,7 @@ static inline long syscall_get_error(struct task_struct *task, 
	 * TS_COMPAT is set for 32-bit syscall entries and then

	 * remains set until we return to user mode.

	 */

	if (task->thread.status & (TS_COMPAT|TS_I386_REGS_POKED))

	if (task->thread_info.status & (TS_COMPAT|TS_I386_REGS_POKED))

		/*

		 * Sign-extend the value so (int)-EFOO becomes (long)-EFOO

		 * and will match correctly in comparisons.
@@ -116,7 +116,7 @@ static inline void syscall_get_arguments(struct task_struct *task, 
					 unsigned long *args)

{

# ifdef CONFIG_IA32_EMULATION

	if (task->thread.status & TS_COMPAT)

	if (task->thread_info.status & TS_COMPAT)

		switch (i) {

		case 0:

			if (!n--) break;
@@ -177,7 +177,7 @@ static inline void syscall_set_arguments(struct task_struct *task, 
					 const unsigned long *args)

{

# ifdef CONFIG_IA32_EMULATION

	if (task->thread.status & TS_COMPAT)

	if (task->thread_info.status & TS_COMPAT)

		switch (i) {

		case 0:

			if (!n--) break;

arch/x86/include/asm/thread_info.h

+2 −1
Original line number Diff line number Diff line
@@ -55,6 +55,7 @@ struct task_struct; 


struct thread_info {

	unsigned long		flags;		/* low level flags */

	u32			status;		/* thread synchronous flags */

};



#define INIT_THREAD_INFO(tsk)			\
@@ -221,7 +222,7 @@ static inline int arch_within_stack_frames(const void * const stack, 
#define in_ia32_syscall() true

#else

#define in_ia32_syscall() (IS_ENABLED(CONFIG_IA32_EMULATION) && \

			   current->thread.status & TS_COMPAT)

			   current_thread_info()->status & TS_COMPAT)

#endif



/*

arch/x86/kernel/process_64.c

+2 −2
Original line number Diff line number Diff line
@@ -557,7 +557,7 @@ static void __set_personality_x32(void) 
	 * Pretend to come from a x32 execve.

	 */

	task_pt_regs(current)->orig_ax = __NR_x32_execve | __X32_SYSCALL_BIT;

	current->thread.status &= ~TS_COMPAT;

	current_thread_info()->status &= ~TS_COMPAT;

#endif

}
@@ -571,7 +571,7 @@ static void __set_personality_ia32(void) 
	current->personality |= force_personality32;

	/* Prepare the first "return" to user space */

	task_pt_regs(current)->orig_ax = __NR_ia32_execve;

	current->thread.status |= TS_COMPAT;

	current_thread_info()->status |= TS_COMPAT;

#endif

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE