x86/entry: Consolidate 32/64 bit syscall entry (0b085e68) · Commits · 戴 / test

arch/x86/entry/common.c

+41 −52

Original line number	Original line	Diff line number	Diff line
	@@ -366,8 +366,7 @@ __visible noinstr void syscall_return_slowpath(struct pt_regs *regs)
	exit_to_user_mode();		exit_to_user_mode();
	}		}

	#ifdef CONFIG_X86_64		static noinstr long syscall_enter(struct pt_regs *regs, unsigned long nr)
	__visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
	{		{
	struct thread_info *ti;		struct thread_info *ti;

	@@ -379,6 +378,16 @@ __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
	if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY)		if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY)
	nr = syscall_trace_enter(regs);		nr = syscall_trace_enter(regs);

			instrumentation_end();
			return nr;
			}

			#ifdef CONFIG_X86_64
			__visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
			{
			nr = syscall_enter(regs, nr);

			instrumentation_begin();
	if (likely(nr < NR_syscalls)) {		if (likely(nr < NR_syscalls)) {
	nr = array_index_nospec(nr, NR_syscalls);		nr = array_index_nospec(nr, NR_syscalls);
	regs->ax = sys_call_table[nr](regs);		regs->ax = sys_call_table[nr](regs);
	@@ -390,64 +399,53 @@ __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
	regs->ax = x32_sys_call_table[nr](regs);		regs->ax = x32_sys_call_table[nr](regs);
	#endif		#endif
	}		}
	__syscall_return_slowpath(regs);

	instrumentation_end();		instrumentation_end();
	exit_to_user_mode();		syscall_return_slowpath(regs);
	}		}
	#endif		#endif

	#if defined(CONFIG_X86_32) \|\| defined(CONFIG_IA32_EMULATION)		#if defined(CONFIG_X86_32) \|\| defined(CONFIG_IA32_EMULATION)
	/*		static __always_inline unsigned int syscall_32_enter(struct pt_regs *regs)
	* Does a 32-bit syscall. Called with IRQs on in CONTEXT_KERNEL. Does
	* all entry and exit work and returns with IRQs off. This function is
	* extremely hot in workloads that use it, and it's usually called from
	* do_fast_syscall_32, so forcibly inline it to improve performance.
	*/
	static void do_syscall_32_irqs_on(struct pt_regs *regs)
	{		{
	struct thread_info *ti = current_thread_info();		if (IS_ENABLED(CONFIG_IA32_EMULATION))
	unsigned int nr = (unsigned int)regs->orig_ax;		current_thread_info()->status \|= TS_COMPAT;

	#ifdef CONFIG_IA32_EMULATION
	ti->status \|= TS_COMPAT;
	#endif

	if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY) {
	/*		/*
	* Subtlety here: if ptrace pokes something larger than		* Subtlety here: if ptrace pokes something larger than 2^32-1 into
	* 2^32-1 into orig_ax, this truncates it. This may or		* orig_ax, the unsigned int return value truncates it. This may
	* may not be necessary, but it matches the old asm		* or may not be necessary, but it matches the old asm behavior.
	* behavior.
	*/		*/
	nr = syscall_trace_enter(regs);		return syscall_enter(regs, (unsigned int)regs->orig_ax);
	}		}

			/*
			* Invoke a 32-bit syscall. Called with IRQs on in CONTEXT_KERNEL.
			*/
			static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs,
			unsigned int nr)
			{
	if (likely(nr < IA32_NR_syscalls)) {		if (likely(nr < IA32_NR_syscalls)) {
			instrumentation_begin();
	nr = array_index_nospec(nr, IA32_NR_syscalls);		nr = array_index_nospec(nr, IA32_NR_syscalls);
	regs->ax = ia32_sys_call_table[nr](regs);		regs->ax = ia32_sys_call_table[nr](regs);
			instrumentation_end();
	}		}

	__syscall_return_slowpath(regs);
	}		}

	/* Handles int $0x80 */		/* Handles int $0x80 */
	__visible noinstr void do_int80_syscall_32(struct pt_regs *regs)		__visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
	{		{
	enter_from_user_mode(regs);		unsigned int nr = syscall_32_enter(regs);
	instrumentation_begin();

	local_irq_enable();		do_syscall_32_irqs_on(regs, nr);
	do_syscall_32_irqs_on(regs);		syscall_return_slowpath(regs);

	instrumentation_end();
	exit_to_user_mode();
	}		}

	static bool __do_fast_syscall_32(struct pt_regs *regs)		static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
	{		{
			unsigned int nr = syscall_32_enter(regs);
	int res;		int res;

			instrumentation_begin();
	/* Fetch EBP from where the vDSO stashed it. */		/* Fetch EBP from where the vDSO stashed it. */
	if (IS_ENABLED(CONFIG_X86_64)) {		if (IS_ENABLED(CONFIG_X86_64)) {
	/*		/*
	@@ -460,17 +458,18 @@ static bool __do_fast_syscall_32(struct pt_regs *regs)
	res = get_user((u32 )&regs->bp,		res = get_user((u32 )&regs->bp,
	(u32 __user __force *)(unsigned long)(u32)regs->sp);		(u32 __user __force *)(unsigned long)(u32)regs->sp);
	}		}
			instrumentation_end();

	if (res) {		if (res) {
	/* User code screwed up. */		/* User code screwed up. */
	regs->ax = -EFAULT;		regs->ax = -EFAULT;
	local_irq_disable();		syscall_return_slowpath(regs);
	__prepare_exit_to_usermode(regs);
	return false;		return false;
	}		}

	/* Now this is just like a normal syscall. */		/* Now this is just like a normal syscall. */
	do_syscall_32_irqs_on(regs);		do_syscall_32_irqs_on(regs, nr);
			syscall_return_slowpath(regs);
	return true;		return true;
	}		}

	@@ -483,7 +482,6 @@ __visible noinstr long do_fast_syscall_32(struct pt_regs *regs)
	*/		*/
	unsigned long landing_pad = (unsigned long)current->mm->context.vdso +		unsigned long landing_pad = (unsigned long)current->mm->context.vdso +
	vdso_image_32.sym_int80_landing_pad;		vdso_image_32.sym_int80_landing_pad;
	bool success;

	/*		/*
	* SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward		* SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward
	@@ -492,17 +490,8 @@ __visible noinstr long do_fast_syscall_32(struct pt_regs *regs)
	*/		*/
	regs->ip = landing_pad;		regs->ip = landing_pad;

	enter_from_user_mode(regs);		/* Invoke the syscall. If it failed, keep it simple: use IRET. */
	instrumentation_begin();		if (!__do_fast_syscall_32(regs))

	local_irq_enable();
	success = __do_fast_syscall_32(regs);

	instrumentation_end();
	exit_to_user_mode();

	/* If it failed, keep it simple: use IRET. */
	if (!success)
	return 0;		return 0;

	#ifdef CONFIG_X86_64		#ifdef CONFIG_X86_64

Admin message