s390: run user space and KVM guests with modified branch prediction (6b73044b) · Commits · 戴 / test

arch/s390/include/asm/processor.h

+3 −0

Original line number	Original line	Diff line number	Diff line
	@@ -378,6 +378,9 @@ extern void memcpy_absolute(void , void , size_t);
	memcpy_absolute(&(dest), &__tmp, sizeof(__tmp)); \		memcpy_absolute(&(dest), &__tmp, sizeof(__tmp)); \
	} while (0)		} while (0)

			extern int s390_isolate_bp(void);
			extern int s390_isolate_bp_guest(void);

	#endif /* __ASSEMBLY__ */		#endif /* __ASSEMBLY__ */

	#endif /* __ASM_S390_PROCESSOR_H */		#endif /* __ASM_S390_PROCESSOR_H */

arch/s390/include/asm/thread_info.h

+4 −0

arch/s390/kernel/entry.S

+46 −5

Original line number	Original line	Diff line number	Diff line
	@@ -107,6 +107,7 @@ _PIF_WORK = (_PIF_PER_TRAP \| _PIF_SYSCALL_RESTART)
	aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)		aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	j 3f		j 3f
	1: UPDATE_VTIME %r14,%r15,\timer		1: UPDATE_VTIME %r14,%r15,\timer
			BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
	2: lg %r15,__LC_ASYNC_STACK # load async stack		2: lg %r15,__LC_ASYNC_STACK # load async stack
	3: la %r11,STACK_FRAME_OVERHEAD(%r15)		3: la %r11,STACK_FRAME_OVERHEAD(%r15)
	.endm		.endm
	@@ -187,6 +188,40 @@ _PIF_WORK = (_PIF_PER_TRAP \| _PIF_SYSCALL_RESTART)
	.popsection		.popsection
	.endm		.endm

			.macro BPENTER tif_ptr,tif_mask
			.pushsection .altinstr_replacement, "ax"
			662: .word 0xc004, 0x0000, 0x0000 # 6 byte nop
			.word 0xc004, 0x0000, 0x0000 # 6 byte nop
			.popsection
			664: TSTMSK \tif_ptr,\tif_mask
			jz . + 8
			.long 0xb2e8d000
			.pushsection .altinstructions, "a"
			.long 664b - .
			.long 662b - .
			.word 82
			.byte 12
			.byte 12
			.popsection
			.endm

			.macro BPEXIT tif_ptr,tif_mask
			TSTMSK \tif_ptr,\tif_mask
			.pushsection .altinstr_replacement, "ax"
			662: jnz . + 8
			.long 0xb2e8d000
			.popsection
			664: jz . + 8
			.long 0xb2e8c000
			.pushsection .altinstructions, "a"
			.long 664b - .
			.long 662b - .
			.word 82
			.byte 8
			.byte 8
			.popsection
			.endm

	.section .kprobes.text, "ax"		.section .kprobes.text, "ax"
	.Ldummy:		.Ldummy:
	/*		/*
	@@ -240,9 +275,11 @@ ENTRY(__switch_to)
	*/		*/
	ENTRY(sie64a)		ENTRY(sie64a)
	stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers		stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers
			lg %r12,__LC_CURRENT
	stg %r2,__SF_EMPTY(%r15) # save control block pointer		stg %r2,__SF_EMPTY(%r15) # save control block pointer
	stg %r3,__SF_EMPTY+8(%r15) # save guest register save area		stg %r3,__SF_EMPTY+8(%r15) # save guest register save area
	xc __SF_EMPTY+16(8,%r15),__SF_EMPTY+16(%r15) # reason code = 0		xc __SF_EMPTY+16(8,%r15),__SF_EMPTY+16(%r15) # reason code = 0
			mvc __SF_EMPTY+24(8,%r15),__TI_flags(%r12) # copy thread flags
	TSTMSK __LC_CPU_FLAGS,_CIF_FPU # load guest fp/vx registers ?		TSTMSK __LC_CPU_FLAGS,_CIF_FPU # load guest fp/vx registers ?
	jno .Lsie_load_guest_gprs		jno .Lsie_load_guest_gprs
	brasl %r14,load_fpu_regs # load guest fp/vx regs		brasl %r14,load_fpu_regs # load guest fp/vx regs
	@@ -259,11 +296,12 @@ ENTRY(sie64a)
	jnz .Lsie_skip		jnz .Lsie_skip
	TSTMSK __LC_CPU_FLAGS,_CIF_FPU		TSTMSK __LC_CPU_FLAGS,_CIF_FPU
	jo .Lsie_skip # exit if fp/vx regs changed		jo .Lsie_skip # exit if fp/vx regs changed
	BPON		BPEXIT __SF_EMPTY+24(%r15),(_TIF_ISOLATE_BP\|_TIF_ISOLATE_BP_GUEST)
	.Lsie_entry:		.Lsie_entry:
	sie 0(%r14)		sie 0(%r14)
	.Lsie_exit:		.Lsie_exit:
	BPOFF		BPOFF
			BPENTER __SF_EMPTY+24(%r15),(_TIF_ISOLATE_BP\|_TIF_ISOLATE_BP_GUEST)
	.Lsie_skip:		.Lsie_skip:
	ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE		ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
	lctlg %c1,%c1,__LC_USER_ASCE # load primary asce		lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
	@@ -324,6 +362,7 @@ ENTRY(system_call)
	la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs		la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
	.Lsysc_vtime:		.Lsysc_vtime:
	UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER		UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
			BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
	stmg %r0,%r7,__PT_R0(%r11)		stmg %r0,%r7,__PT_R0(%r11)
	# clear user controlled register to prevent speculative use		# clear user controlled register to prevent speculative use
	xgr %r0,%r0		xgr %r0,%r0
	@@ -362,7 +401,7 @@ ENTRY(system_call)
	jnz .Lsysc_work # check for work		jnz .Lsysc_work # check for work
	TSTMSK __LC_CPU_FLAGS,_CIF_WORK		TSTMSK __LC_CPU_FLAGS,_CIF_WORK
	jnz .Lsysc_work		jnz .Lsysc_work
	BPON		BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
	.Lsysc_restore:		.Lsysc_restore:
	lg %r14,__LC_VDSO_PER_CPU		lg %r14,__LC_VDSO_PER_CPU
	lmg %r0,%r10,__PT_R0(%r11)		lmg %r0,%r10,__PT_R0(%r11)
	@@ -597,6 +636,7 @@ ENTRY(pgm_check_handler)
	aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)		aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	j 4f		j 4f
	2: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER		2: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
			BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
	lg %r15,__LC_KERNEL_STACK		lg %r15,__LC_KERNEL_STACK
	lgr %r14,%r12		lgr %r14,%r12
	aghi %r14,__TASK_thread # pointer to thread_struct		aghi %r14,__TASK_thread # pointer to thread_struct
	@@ -729,7 +769,7 @@ ENTRY(io_int_handler)
	mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)		mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
	tm __PT_PSW+1(%r11),0x01 # returning to user ?		tm __PT_PSW+1(%r11),0x01 # returning to user ?
	jno .Lio_exit_kernel		jno .Lio_exit_kernel
	BPON		BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
	.Lio_exit_timer:		.Lio_exit_timer:
	stpt __LC_EXIT_TIMER		stpt __LC_EXIT_TIMER
	mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER		mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
	@@ -1165,7 +1205,7 @@ ENTRY(mcck_int_handler)
	mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW		mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
	tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?		tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
	jno 0f		jno 0f
	BPON		BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
	stpt __LC_EXIT_TIMER		stpt __LC_EXIT_TIMER
	mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER		mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
	0: lmg %r11,%r15,__PT_R11(%r11)		0: lmg %r11,%r15,__PT_R11(%r11)
	@@ -1292,7 +1332,8 @@ cleanup_critical:
	clg %r9,BASED(.Lsie_crit_mcck_length)		clg %r9,BASED(.Lsie_crit_mcck_length)
	jh 1f		jh 1f
	oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST		oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
	1: lg %r9,__SF_EMPTY(%r15) # get control block pointer		1: BPENTER __SF_EMPTY+24(%r15),(_TIF_ISOLATE_BP\|_TIF_ISOLATE_BP_GUEST)
			lg %r9,__SF_EMPTY(%r15) # get control block pointer
	ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE		ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
	lctlg %c1,%c1,__LC_USER_ASCE # load primary asce		lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
	larl %r9,sie_exit # skip forward to sie_exit		larl %r9,sie_exit # skip forward to sie_exit

arch/s390/kernel/processor.c

+18 −0

Original line number	Original line	Diff line number	Diff line
	@@ -197,3 +197,21 @@ const struct seq_operations cpuinfo_op = {
	.stop = c_stop,		.stop = c_stop,
	.show = show_cpuinfo,		.show = show_cpuinfo,
	};		};

			int s390_isolate_bp(void)
			{
			if (!test_facility(82))
			return -EOPNOTSUPP;
			set_thread_flag(TIF_ISOLATE_BP);
			return 0;
			}
			EXPORT_SYMBOL(s390_isolate_bp);

			int s390_isolate_bp_guest(void)
			{
			if (!test_facility(82))
			return -EOPNOTSUPP;
			set_thread_flag(TIF_ISOLATE_BP_GUEST);
			return 0;
			}
			EXPORT_SYMBOL(s390_isolate_bp_guest);

Original line number	Original line	Diff line number	Diff line
	@@ -58,6 +58,8 @@ int arch_dup_task_struct(struct task_struct dst, struct task_struct src);
	#define TIF_GUARDED_STORAGE 4 /* load guarded storage control block */		#define TIF_GUARDED_STORAGE 4 /* load guarded storage control block */
	#define TIF_PATCH_PENDING 5 /* pending live patching update */		#define TIF_PATCH_PENDING 5 /* pending live patching update */
	#define TIF_PGSTE 6 /* New mm's will use 4K page tables */		#define TIF_PGSTE 6 /* New mm's will use 4K page tables */
			#define TIF_ISOLATE_BP 8 /* Run process with isolated BP */
			#define TIF_ISOLATE_BP_GUEST 9 /* Run KVM guests with isolated BP */

	#define TIF_31BIT 16 /* 32bit process */		#define TIF_31BIT 16 /* 32bit process */
	#define TIF_MEMDIE 17 /* is terminating due to OOM killer */		#define TIF_MEMDIE 17 /* is terminating due to OOM killer */
	@@ -78,6 +80,8 @@ int arch_dup_task_struct(struct task_struct dst, struct task_struct src);
	#define _TIF_UPROBE _BITUL(TIF_UPROBE)		#define _TIF_UPROBE _BITUL(TIF_UPROBE)
	#define _TIF_GUARDED_STORAGE _BITUL(TIF_GUARDED_STORAGE)		#define _TIF_GUARDED_STORAGE _BITUL(TIF_GUARDED_STORAGE)
	#define _TIF_PATCH_PENDING _BITUL(TIF_PATCH_PENDING)		#define _TIF_PATCH_PENDING _BITUL(TIF_PATCH_PENDING)
			#define _TIF_ISOLATE_BP _BITUL(TIF_ISOLATE_BP)
			#define _TIF_ISOLATE_BP_GUEST _BITUL(TIF_ISOLATE_BP_GUEST)

	#define _TIF_31BIT _BITUL(TIF_31BIT)		#define _TIF_31BIT _BITUL(TIF_31BIT)
	#define _TIF_SINGLE_STEP _BITUL(TIF_SINGLE_STEP)		#define _TIF_SINGLE_STEP _BITUL(TIF_SINGLE_STEP)

Admin message