Commit ff303e66 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

perf: Fix software migrate events 



Stephane asked about PERF_COUNT_SW_CPU_MIGRATIONS and I realized it
was borken:

 > The problem is that the task isn't actually scheduled while its being
 > migrated (obviously), and if its not scheduled, the counters aren't
 > scheduled either, so there's no observing of the fact.
 >
 > A further problem with migrations is that many migrations happen from
 > softirq context, which is nested inside the 'random' task context of
 > whoemever happens to run at that time, similarly for the wakeup
 > migrations triggered from (soft)irq context. All those end up being
 > accounted in the task that's currently running, eg. your 'ls'.

The below cures this by marking a task as migrated and accounting it
on the subsequent sched_in().

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Stephane Eranian <eranian@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 1836ac85

include/linux/perf_event.h

+24 −0
Original line number Diff line number Diff line
@@ -798,11 +798,33 @@ perf_sw_event_sched(u32 event_id, u64 nr, u64 addr) 


extern struct static_key_deferred perf_sched_events;



static __always_inline bool

perf_sw_migrate_enabled(void)

{

	if (static_key_false(&perf_swevent_enabled[PERF_COUNT_SW_CPU_MIGRATIONS]))

		return true;

	return false;

}



static inline void perf_event_task_migrate(struct task_struct *task)

{

	if (perf_sw_migrate_enabled())

		task->sched_migrated = 1;

}



static inline void perf_event_task_sched_in(struct task_struct *prev,

					    struct task_struct *task)

{

	if (static_key_false(&perf_sched_events.key))

		__perf_event_task_sched_in(prev, task);



	if (perf_sw_migrate_enabled() && task->sched_migrated) {

		struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]);



		perf_fetch_caller_regs(regs);

		___perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, regs, 0);

		task->sched_migrated = 0;

	}

}



static inline void perf_event_task_sched_out(struct task_struct *prev,
@@ -925,6 +947,8 @@ perf_aux_output_skip(struct perf_output_handle *handle, 
static inline void *

perf_get_aux(struct perf_output_handle *handle)				{ return NULL; }

static inline void

perf_event_task_migrate(struct task_struct *task)			{ }

static inline void

perf_event_task_sched_in(struct task_struct *prev,

			 struct task_struct *task)			{ }

static inline void

include/linux/sched.h

+4 −3
Original line number Diff line number Diff line
@@ -1356,9 +1356,6 @@ struct task_struct { 
#endif



	struct mm_struct *mm, *active_mm;

#ifdef CONFIG_COMPAT_BRK

	unsigned brk_randomized:1;

#endif

	/* per-thread vma caching */

	u32 vmacache_seqnum;

	struct vm_area_struct *vmacache[VMACACHE_SIZE];
@@ -1381,10 +1378,14 @@ struct task_struct { 
	/* Revert to default priority/policy when forking */

	unsigned sched_reset_on_fork:1;

	unsigned sched_contributes_to_load:1;

	unsigned sched_migrated:1;



#ifdef CONFIG_MEMCG_KMEM

	unsigned memcg_kmem_skip_account:1;

#endif

#ifdef CONFIG_COMPAT_BRK

	unsigned brk_randomized:1;

#endif



	unsigned long atomic_flags; /* Flags needing atomic access. */

kernel/sched/core.c

+1 −1
Original line number Diff line number Diff line
@@ -1049,7 +1049,7 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 
		if (p->sched_class->migrate_task_rq)

			p->sched_class->migrate_task_rq(p, new_cpu);

		p->se.nr_migrations++;

		perf_sw_event_sched(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 0);

		perf_event_task_migrate(p);

	}



	__set_task_cpu(p, new_cpu);

CRA Git | Maintained and supported by SUSTech CRA and CCSE