Commit c742b31c authored by Martin Schwidefsky's avatar Martin Schwidefsky
Browse files

[PATCH] fast vdso implementation for CLOCK_THREAD_CPUTIME_ID 



The extract cpu time instruction (ectg) instruction allows the user
process to get the current thread cputime without calling into the
kernel. The code that uses the instruction needs to switch to the
access registers mode to get access to the per-cpu info page that
contains the two base values that are needed to calculate the current
cputime from the CPU timer with the ectg instruction.

Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
parent 9cfb9b3c

arch/s390/include/asm/lowcore.h

+8 −1
Original line number Diff line number Diff line
@@ -106,8 +106,10 @@ 
#define __LC_IPLDEV                     0xDB8

#define __LC_CURRENT			0xDD8

#define __LC_INT_CLOCK			0xDE8

#define __LC_VDSO_PER_CPU		0xE38

#endif /* __s390x__ */



#define __LC_PASTE			0xE40



#define __LC_PANIC_MAGIC		0xE00

#ifndef __s390x__
@@ -381,7 +383,12 @@ struct _lowcore 
        /* whether the kernel died with panic() or not */

        __u32        panic_magic;              /* 0xe00 */



	__u8         pad13[0x11b8-0xe04];      /* 0xe04 */

	/* Per cpu primary space access list */

	__u8	     pad_0xe04[0xe3c-0xe04];   /* 0xe04 */

	__u32	     vdso_per_cpu_data;	       /* 0xe3c */

	__u32	     paste[16];		       /* 0xe40 */



	__u8	     pad13[0x11b8-0xe80];      /* 0xe80 */



	/* 64 bit extparam used for pfault, diag 250 etc  */

	__u64        ext_params2;               /* 0x11B8 */

arch/s390/include/asm/vdso.h

+13 −2
Original line number Diff line number Diff line
@@ -12,9 +12,9 @@ 
#ifndef __ASSEMBLY__



/*

 * Note about this structure:

 * Note about the vdso_data and vdso_per_cpu_data structures:

 *

 * NEVER USE THIS IN USERSPACE CODE DIRECTLY. The layout of this

 * NEVER USE THEM IN USERSPACE CODE DIRECTLY. The layout of the

 * structure is supposed to be known only to the function in the vdso

 * itself and may change without notice.

 */
@@ -28,10 +28,21 @@ struct vdso_data { 
	__u64 wtom_clock_nsec;		/*				0x28 */

	__u32 tz_minuteswest;		/* Minutes west of Greenwich	0x30 */

	__u32 tz_dsttime;		/* Type of dst correction	0x34 */

	__u32 ectg_available;

};



struct vdso_per_cpu_data {

	__u64 ectg_timer_base;

	__u64 ectg_user_time;

};



extern struct vdso_data *vdso_data;



#ifdef CONFIG_64BIT

int vdso_alloc_per_cpu(int cpu, struct _lowcore *lowcore);

void vdso_free_per_cpu(int cpu, struct _lowcore *lowcore);

#endif



#endif /* __ASSEMBLY__ */



#endif /* __KERNEL__ */

arch/s390/kernel/asm-offsets.c

+5 −0
Original line number Diff line number Diff line
@@ -48,6 +48,11 @@ int main(void) 
	DEFINE(__VDSO_WTOM_SEC, offsetof(struct vdso_data, wtom_clock_sec));

	DEFINE(__VDSO_WTOM_NSEC, offsetof(struct vdso_data, wtom_clock_nsec));

	DEFINE(__VDSO_TIMEZONE, offsetof(struct vdso_data, tz_minuteswest));

	DEFINE(__VDSO_ECTG_OK, offsetof(struct vdso_data, ectg_available));

	DEFINE(__VDSO_ECTG_BASE,

	       offsetof(struct vdso_per_cpu_data, ectg_timer_base));

	DEFINE(__VDSO_ECTG_USER,

	       offsetof(struct vdso_per_cpu_data, ectg_user_time));

	/* constants used by the vdso */

	DEFINE(CLOCK_REALTIME, CLOCK_REALTIME);

	DEFINE(CLOCK_MONOTONIC, CLOCK_MONOTONIC);

arch/s390/kernel/entry64.S

+24 −21
Original line number Diff line number Diff line
@@ -177,8 +177,11 @@ _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ 
	.if !\sync

	ni	\psworg+1,0xfd		# clear wait state bit

	.endif

	lmg	%r0,%r15,SP_R0(%r15)	# load gprs 0-15 of user

	lg	%r14,__LC_VDSO_PER_CPU

	lmg	%r0,%r13,SP_R0(%r15)	# load gprs 0-13 of user

	stpt	__LC_EXIT_TIMER

	mvc	__VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER

	lmg	%r14,%r15,SP_R14(%r15)	# load grps 14-15 of user

	lpswe	\psworg			# back to caller

	.endm
@@ -980,23 +983,23 @@ cleanup_sysc_return: 


cleanup_sysc_leave:

	clc	8(8,%r12),BASED(cleanup_sysc_leave_insn)

	je	2f

	mvc	__LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER

	je	3f

	clc	8(8,%r12),BASED(cleanup_sysc_leave_insn+8)

	je	2f

	mvc	__LC_RETURN_PSW(16),SP_PSW(%r15)

	jhe	0f

	mvc	__LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER

0:	mvc	__LC_RETURN_PSW(16),SP_PSW(%r15)

	cghi	%r12,__LC_MCK_OLD_PSW

	jne	0f

	jne	1f

	mvc	__LC_SAVE_AREA+64(32),SP_R12(%r15)

	j	1f

0:	mvc	__LC_SAVE_AREA+32(32),SP_R12(%r15)

1:	lmg	%r0,%r11,SP_R0(%r15)

	j	2f

1:	mvc	__LC_SAVE_AREA+32(32),SP_R12(%r15)

2:	lmg	%r0,%r11,SP_R0(%r15)

	lg	%r15,SP_R15(%r15)

2:	la	%r12,__LC_RETURN_PSW

3:	la	%r12,__LC_RETURN_PSW

	br	%r14

cleanup_sysc_leave_insn:

	.quad	sysc_done - 4

	.quad	sysc_done - 8

	.quad	sysc_done - 16



cleanup_io_return:

	mvc	__LC_RETURN_PSW(8),0(%r12)
@@ -1006,23 +1009,23 @@ cleanup_io_return: 


cleanup_io_leave:

	clc	8(8,%r12),BASED(cleanup_io_leave_insn)

	je	2f

	mvc	__LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER

	je	3f

	clc	8(8,%r12),BASED(cleanup_io_leave_insn+8)

	je	2f

	mvc	__LC_RETURN_PSW(16),SP_PSW(%r15)

	jhe	0f

	mvc	__LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER

0:	mvc	__LC_RETURN_PSW(16),SP_PSW(%r15)

	cghi	%r12,__LC_MCK_OLD_PSW

	jne	0f

	jne	1f

	mvc	__LC_SAVE_AREA+64(32),SP_R12(%r15)

	j	1f

0:	mvc	__LC_SAVE_AREA+32(32),SP_R12(%r15)

1:	lmg	%r0,%r11,SP_R0(%r15)

	j	2f

1:	mvc	__LC_SAVE_AREA+32(32),SP_R12(%r15)

2:	lmg	%r0,%r11,SP_R0(%r15)

	lg	%r15,SP_R15(%r15)

2:	la	%r12,__LC_RETURN_PSW

3:	la	%r12,__LC_RETURN_PSW

	br	%r14

cleanup_io_leave_insn:

	.quad	io_done - 4

	.quad	io_done - 8

	.quad	io_done - 16



/*

 * Integer constants

arch/s390/kernel/head64.S

+2 −0
Original line number Diff line number Diff line
@@ -87,6 +87,8 @@ startup_continue: 
	lg	%r12,.Lparmaddr-.LPG1(%r13)	# pointer to parameter area

					# move IPL device to lowcore

	mvc	__LC_IPLDEV(4),IPL_DEVICE+4-PARMAREA(%r12)

	lghi	%r0,__LC_PASTE

	stg	%r0,__LC_VDSO_PER_CPU

#

# Setup stack

#

CRA Git | Maintained and supported by SUSTech CRA and CCSE