Commit 6a61671b authored by Frederic Weisbecker's avatar Frederic Weisbecker
Browse files

cputime: Safely read cputime of full dynticks CPUs 



While remotely reading the cputime of a task running in a
full dynticks CPU, the values stored in utime/stime fields
of struct task_struct may be stale. Its values may be those
of the last kernel <-> user transition time snapshot and
we need to add the tickless time spent since this snapshot.

To fix this, flush the cputime of the dynticks CPUs on
kernel <-> user transition and record the time / context
where we did this. Then on top of this snapshot and the current
time, perform the fixup on the reader side from task_times()
accessors.

Signed-off-by: default avatarFrederic Weisbecker <fweisbec@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Li Zhong <zhong@linux.vnet.ibm.com>
Cc: Namhyung Kim <namhyung.kim@lge.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
[fixed kvm module related build errors]
Signed-off-by: default avatarSedat Dilek <sedat.dilek@gmail.com>
parent c11f11fc

arch/s390/kernel/vtime.c

+3 −3
Original line number Diff line number Diff line
@@ -127,7 +127,7 @@ void vtime_account_user(struct task_struct *tsk) 
 * Update process times based on virtual cpu times stored by entry.S

 * to the lowcore fields user_timer, system_timer & steal_clock.

 */

void vtime_account(struct task_struct *tsk)

void vtime_account_irq_enter(struct task_struct *tsk)

{

	struct thread_info *ti = task_thread_info(tsk);

	u64 timer, system;
@@ -145,10 +145,10 @@ void vtime_account(struct task_struct *tsk) 


	virt_timer_forward(system);

}

EXPORT_SYMBOL_GPL(vtime_account);

EXPORT_SYMBOL_GPL(vtime_account_irq_enter);



void vtime_account_system(struct task_struct *tsk)

__attribute__((alias("vtime_account")));

__attribute__((alias("vtime_account_irq_enter")));

EXPORT_SYMBOL_GPL(vtime_account_system);



void __kprobes vtime_stop_cpu(void)

include/linux/hardirq.h

+2 −2
Original line number Diff line number Diff line
@@ -153,7 +153,7 @@ extern void rcu_nmi_exit(void); 
 */

#define __irq_enter()					\

	do {						\

		vtime_account_irq_enter(current);	\

		account_irq_enter_time(current);	\

		add_preempt_count(HARDIRQ_OFFSET);	\

		trace_hardirq_enter();			\

	} while (0)
@@ -169,7 +169,7 @@ extern void irq_enter(void); 
#define __irq_exit()					\

	do {						\

		trace_hardirq_exit();			\

		vtime_account_irq_exit(current);	\

		account_irq_exit_time(current);		\

		sub_preempt_count(HARDIRQ_OFFSET);	\

	} while (0)

include/linux/init_task.h

+11 −0
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ 
#include <linux/pid_namespace.h>

#include <linux/user_namespace.h>

#include <linux/securebits.h>

#include <linux/seqlock.h>

#include <net/net_namespace.h>



#ifdef CONFIG_SMP
@@ -141,6 +142,15 @@ extern struct task_group root_task_group; 
# define INIT_PERF_EVENTS(tsk)

#endif



#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN

# define INIT_VTIME(tsk)						\

	.vtime_seqlock = __SEQLOCK_UNLOCKED(tsk.vtime_seqlock),	\

	.vtime_snap = 0,				\

	.vtime_snap_whence = VTIME_SYS,

#else

# define INIT_VTIME(tsk)

#endif



#define INIT_TASK_COMM "swapper"



/*
@@ -210,6 +220,7 @@ extern struct task_group root_task_group; 
	INIT_TRACE_RECURSION						\

	INIT_TASK_RCU_PREEMPT(tsk)					\

	INIT_CPUSET_SEQ							\

	INIT_VTIME(tsk)							\

}

include/linux/kvm_host.h

+18 −2
Original line number Diff line number Diff line
@@ -741,7 +741,7 @@ static inline int kvm_deassign_device(struct kvm *kvm, 
}

#endif /* CONFIG_IOMMU_API */



static inline void guest_enter(void)

static inline void __guest_enter(void)

{

	/*

	 * This is running in ioctl context so we can avoid
@@ -751,7 +751,7 @@ static inline void guest_enter(void) 
	current->flags |= PF_VCPU;

}



static inline void guest_exit(void)

static inline void __guest_exit(void)

{

	/*

	 * This is running in ioctl context so we can avoid
@@ -761,6 +761,22 @@ static inline void guest_exit(void) 
	current->flags &= ~PF_VCPU;

}



#ifdef CONFIG_CONTEXT_TRACKING

extern void guest_enter(void);

extern void guest_exit(void);



#else /* !CONFIG_CONTEXT_TRACKING */

static inline void guest_enter(void)

{

	__guest_enter();

}



static inline void guest_exit(void)

{

	__guest_exit();

}

#endif /* !CONFIG_CONTEXT_TRACKING */



static inline void kvm_guest_enter(void)

{

	unsigned long flags;

include/linux/sched.h

+22 −5
Original line number Diff line number Diff line
@@ -1367,6 +1367,15 @@ struct task_struct { 
	cputime_t gtime;

#ifndef CONFIG_VIRT_CPU_ACCOUNTING

	struct cputime prev_cputime;

#endif

#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN

	seqlock_t vtime_seqlock;

	unsigned long long vtime_snap;

	enum {

		VTIME_SLEEPING = 0,

		VTIME_USER,

		VTIME_SYS,

	} vtime_snap_whence;

#endif

	unsigned long nvcsw, nivcsw; /* context switch counts */

	struct timespec start_time; 		/* monotonic time */
@@ -1792,11 +1801,13 @@ static inline void put_task_struct(struct task_struct *t) 
		__put_task_struct(t);

}



static inline cputime_t task_gtime(struct task_struct *t)

{

	return t->gtime;

}



#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN

extern void task_cputime(struct task_struct *t,

			 cputime_t *utime, cputime_t *stime);

extern void task_cputime_scaled(struct task_struct *t,

				cputime_t *utimescaled, cputime_t *stimescaled);

extern cputime_t task_gtime(struct task_struct *t);

#else

static inline void task_cputime(struct task_struct *t,

				cputime_t *utime, cputime_t *stime)

{
@@ -1815,6 +1826,12 @@ static inline void task_cputime_scaled(struct task_struct *t, 
	if (stimescaled)

		*stimescaled = t->stimescaled;

}



static inline cputime_t task_gtime(struct task_struct *t)

{

	return t->gtime;

}

#endif

extern void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st);

extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st);

CRA Git | Maintained and supported by SUSTech CRA and CCSE