Commit b5dd77c8 authored by Rik van Riel's avatar Rik van Riel Committed by Ingo Molnar
Browse files

sched/numa: Scale scan period with tasks in group and shared/private 



Running 80 tasks in the same group, or as threads of the same process,
results in the memory getting scanned 80x as fast as it would be if a
single task was using the memory.

This really hurts some workloads.

Scale the scan period by the number of tasks in the numa group, and
the shared / private ratio, so the average rate at which memory in
the group is scanned corresponds roughly to the rate at which a single
task would scan its memory.

Signed-off-by: default avatarRik van Riel <riel@redhat.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarMel Gorman <mgorman@suse.de>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: jhladky@redhat.com
Cc: lvenanci@redhat.com
Link: http://lkml.kernel.org/r/20170731192847.23050-3-riel@redhat.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 37ec97de

kernel/sched/fair.c

+86 −25
Original line number Diff line number Diff line
@@ -1071,6 +1071,29 @@ unsigned int sysctl_numa_balancing_scan_size = 256; 
/* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */

unsigned int sysctl_numa_balancing_scan_delay = 1000;



struct numa_group {

	atomic_t refcount;



	spinlock_t lock; /* nr_tasks, tasks */

	int nr_tasks;

	pid_t gid;

	int active_nodes;



	struct rcu_head rcu;

	unsigned long total_faults;

	unsigned long max_faults_cpu;

	/*

	 * Faults_cpu is used to decide whether memory should move

	 * towards the CPU. As a consequence, these stats are weighted

	 * more by CPU use than by memory faults.

	 */

	unsigned long *faults_cpu;

	unsigned long faults[0];

};



static inline unsigned long group_faults_priv(struct numa_group *ng);

static inline unsigned long group_faults_shared(struct numa_group *ng);



static unsigned int task_nr_scan_windows(struct task_struct *p)

{

	unsigned long rss = 0;
@@ -1107,13 +1130,47 @@ static unsigned int task_scan_min(struct task_struct *p) 
	return max_t(unsigned int, floor, scan);

}



static unsigned int task_scan_start(struct task_struct *p)

{

	unsigned long smin = task_scan_min(p);

	unsigned long period = smin;



	/* Scale the maximum scan period with the amount of shared memory. */

	if (p->numa_group) {

		struct numa_group *ng = p->numa_group;

		unsigned long shared = group_faults_shared(ng);

		unsigned long private = group_faults_priv(ng);



		period *= atomic_read(&ng->refcount);

		period *= shared + 1;

		period /= private + shared + 1;

	}



	return max(smin, period);

}



static unsigned int task_scan_max(struct task_struct *p)

{

	unsigned int smin = task_scan_min(p);

	unsigned int smax;

	unsigned long smin = task_scan_min(p);

	unsigned long smax;



	/* Watch for min being lower than max due to floor calculations */

	smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p);



	/* Scale the maximum scan period with the amount of shared memory. */

	if (p->numa_group) {

		struct numa_group *ng = p->numa_group;

		unsigned long shared = group_faults_shared(ng);

		unsigned long private = group_faults_priv(ng);

		unsigned long period = smax;



		period *= atomic_read(&ng->refcount);

		period *= shared + 1;

		period /= private + shared + 1;



		smax = max(smax, period);

	}



	return max(smin, smax);

}
@@ -1129,26 +1186,6 @@ static void account_numa_dequeue(struct rq *rq, struct task_struct *p) 
	rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p));

}



struct numa_group {

	atomic_t refcount;



	spinlock_t lock; /* nr_tasks, tasks */

	int nr_tasks;

	pid_t gid;

	int active_nodes;



	struct rcu_head rcu;

	unsigned long total_faults;

	unsigned long max_faults_cpu;

	/*

	 * Faults_cpu is used to decide whether memory should move

	 * towards the CPU. As a consequence, these stats are weighted

	 * more by CPU use than by memory faults.

	 */

	unsigned long *faults_cpu;

	unsigned long faults[0];

};



/* Shared or private faults. */

#define NR_NUMA_HINT_FAULT_TYPES 2
@@ -1198,6 +1235,30 @@ static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) 
		group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 1)];

}



static inline unsigned long group_faults_priv(struct numa_group *ng)

{

	unsigned long faults = 0;

	int node;



	for_each_online_node(node) {

		faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)];

	}



	return faults;

}



static inline unsigned long group_faults_shared(struct numa_group *ng)

{

	unsigned long faults = 0;

	int node;



	for_each_online_node(node) {

		faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)];

	}



	return faults;

}



/*

 * A node triggering more than 1/3 as many NUMA faults as the maximum is

 * considered part of a numa group's pseudo-interleaving set. Migrations
@@ -1808,7 +1869,7 @@ static int task_numa_migrate(struct task_struct *p) 
	 * Reset the scan period if the task is being rescheduled on an

	 * alternative node to recheck if the tasks is now properly placed.

	 */

	p->numa_scan_period = task_scan_min(p);

	p->numa_scan_period = task_scan_start(p);



	if (env.best_task == NULL) {

		ret = migrate_task_to(p, env.best_cpu);
@@ -2459,7 +2520,7 @@ void task_numa_work(struct callback_head *work) 


	if (p->numa_scan_period == 0) {

		p->numa_scan_period_max = task_scan_max(p);

		p->numa_scan_period = task_scan_min(p);

		p->numa_scan_period = task_scan_start(p);

	}



	next_scan = now + msecs_to_jiffies(p->numa_scan_period);
@@ -2587,7 +2648,7 @@ void task_tick_numa(struct rq *rq, struct task_struct *curr) 


	if (now > curr->node_stamp + period) {

		if (!curr->node_stamp)

			curr->numa_scan_period = task_scan_min(curr);

			curr->numa_scan_period = task_scan_start(curr);

		curr->node_stamp += period;



		if (!time_before(jiffies, curr->mm->numa_next_scan)) {

CRA Git | Maintained and supported by SUSTech CRA and CCSE