Commit 143e1e28 authored by Vincent Guittot's avatar Vincent Guittot Committed by Ingo Molnar
Browse files

sched: Rework sched_domain topology definition 



We replace the old way to configure the scheduler topology with a new method
which enables a platform to declare additionnal level (if needed).

We still have a default topology table definition that can be used by platform
that don't want more level than the SMT, MC, CPU and NUMA ones. This table can
be overwritten by an arch which either wants to add new level where a load
balance make sense like BOOK or powergating level or wants to change the flags
configuration of some levels.

For each level, we need a function pointer that returns cpumask for each cpu,
a function pointer that returns the flags for the level and a name. Only flags
that describe topology, can be set by an architecture. The current topology
flags are:

 SD_SHARE_CPUPOWER
 SD_SHARE_PKG_RESOURCES
 SD_NUMA
 SD_ASYM_PACKING

Then, each level must be a subset on the next one. The build sequence of the
sched_domain will take care of removing useless levels like those with 1 CPU
and those with the same CPU span and no more relevant information for
load balancing than its children.

Signed-off-by: default avatarVincent Guittot <vincent.guittot@linaro.org>
Tested-by: default avatarDietmar Eggemann <dietmar.eggemann@arm.com>
Reviewed-by: default avatarPreeti U Murthy <preeti@linux.vnet.ibm.com>
Reviewed-by: default avatarDietmar Eggemann <dietmar.eggemann@arm.com>
Signed-off-by: default avatarPeter Zijlstra <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Hanjun Guo <hanjun.guo@linaro.org>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Jason Low <jason.low2@hp.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: linux390@de.ibm.com
Cc: linux-ia64@vger.kernel.org
Cc: linux-s390@vger.kernel.org
Link: http://lkml.kernel.org/r/1397209481-28542-2-git-send-email-vincent.guittot@linaro.org


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 107437fe

arch/ia64/include/asm/topology.h

+0 −24
Original line number Diff line number Diff line
@@ -46,30 +46,6 @@ 


void build_cpu_to_node_map(void);



#define SD_CPU_INIT (struct sched_domain) {		\

	.parent			= NULL,			\

	.child			= NULL,			\

	.groups			= NULL,			\

	.min_interval		= 1,			\

	.max_interval		= 4,			\

	.busy_factor		= 64,			\

	.imbalance_pct		= 125,			\

	.cache_nice_tries	= 2,			\

	.busy_idx		= 2,			\

	.idle_idx		= 1,			\

	.newidle_idx		= 0,			\

	.wake_idx		= 0,			\

	.forkexec_idx		= 0,			\

	.flags			= SD_LOAD_BALANCE	\

				| SD_BALANCE_NEWIDLE	\

				| SD_BALANCE_EXEC	\

				| SD_BALANCE_FORK	\

				| SD_WAKE_AFFINE,	\

	.last_balance		= jiffies,		\

	.balance_interval	= 1,			\

	.nr_balance_failed	= 0,			\

}



#endif /* CONFIG_NUMA */



#ifdef CONFIG_SMP

arch/s390/include/asm/topology.h

+0 −2
Original line number Diff line number Diff line
@@ -64,8 +64,6 @@ static inline void s390_init_cpu_topology(void) 
};

#endif



#define SD_BOOK_INIT	SD_CPU_INIT



#include <asm-generic/topology.h>



#endif /* _ASM_S390_TOPOLOGY_H */

arch/tile/include/asm/topology.h

+0 −33
Original line number Diff line number Diff line
@@ -44,39 +44,6 @@ static inline const struct cpumask *cpumask_of_node(int node) 
/* For now, use numa node -1 for global allocation. */

#define pcibus_to_node(bus)		((void)(bus), -1)



/*

 * TILE architecture has many cores integrated in one processor, so we need

 * setup bigger balance_interval for both CPU/NODE scheduling domains to

 * reduce process scheduling costs.

 */



/* sched_domains SD_CPU_INIT for TILE architecture */

#define SD_CPU_INIT (struct sched_domain) {				\

	.min_interval		= 4,					\

	.max_interval		= 128,					\

	.busy_factor		= 64,					\

	.imbalance_pct		= 125,					\

	.cache_nice_tries	= 1,					\

	.busy_idx		= 2,					\

	.idle_idx		= 1,					\

	.newidle_idx		= 0,					\

	.wake_idx		= 0,					\

	.forkexec_idx		= 0,					\

									\

	.flags			= 1*SD_LOAD_BALANCE			\

				| 1*SD_BALANCE_NEWIDLE			\

				| 1*SD_BALANCE_EXEC			\

				| 1*SD_BALANCE_FORK			\

				| 0*SD_BALANCE_WAKE			\

				| 0*SD_WAKE_AFFINE			\

				| 0*SD_SHARE_CPUPOWER			\

				| 0*SD_SHARE_PKG_RESOURCES		\

				| 0*SD_SERIALIZE			\

				,					\

	.last_balance		= jiffies,				\

	.balance_interval	= 32,					\

}



/* By definition, we create nodes based on online memory. */

#define node_has_online_mem(nid) 1

include/linux/sched.h

+53 −0
Original line number Diff line number Diff line
@@ -879,6 +879,27 @@ enum cpu_idle_type { 


extern int __weak arch_sd_sibiling_asym_packing(void);



#ifdef CONFIG_SCHED_SMT

static inline const int cpu_smt_flags(void)

{

	return SD_SHARE_CPUPOWER | SD_SHARE_PKG_RESOURCES;

}

#endif



#ifdef CONFIG_SCHED_MC

static inline const int cpu_core_flags(void)

{

	return SD_SHARE_PKG_RESOURCES;

}

#endif



#ifdef CONFIG_NUMA

static inline const int cpu_numa_flags(void)

{

	return SD_NUMA;

}

#endif



struct sched_domain_attr {

	int relax_domain_level;

};
@@ -985,6 +1006,38 @@ void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms); 


bool cpus_share_cache(int this_cpu, int that_cpu);



typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);

typedef const int (*sched_domain_flags_f)(void);



#define SDTL_OVERLAP	0x01



struct sd_data {

	struct sched_domain **__percpu sd;

	struct sched_group **__percpu sg;

	struct sched_group_power **__percpu sgp;

};



struct sched_domain_topology_level {

	sched_domain_mask_f mask;

	sched_domain_flags_f sd_flags;

	int		    flags;

	int		    numa_level;

	struct sd_data      data;

#ifdef CONFIG_SCHED_DEBUG

	char                *name;

#endif

};



extern struct sched_domain_topology_level *sched_domain_topology;



extern void set_sched_topology(struct sched_domain_topology_level *tl);



#ifdef CONFIG_SCHED_DEBUG

# define SD_INIT_NAME(type)		.name = #type

#else

# define SD_INIT_NAME(type)

#endif



#else /* CONFIG_SMP */



struct sched_domain_attr;

include/linux/topology.h

+13 −115
Original line number Diff line number Diff line
@@ -66,121 +66,6 @@ int arch_update_cpu_topology(void); 
#define PENALTY_FOR_NODE_WITH_CPUS	(1)

#endif



/*

 * Below are the 3 major initializers used in building sched_domains:

 * SD_SIBLING_INIT, for SMT domains

 * SD_CPU_INIT, for SMP domains

 *

 * Any architecture that cares to do any tuning to these values should do so

 * by defining their own arch-specific initializer in include/asm/topology.h.

 * A definition there will automagically override these default initializers

 * and allow arch-specific performance tuning of sched_domains.

 * (Only non-zero and non-null fields need be specified.)

 */



#ifdef CONFIG_SCHED_SMT

/* MCD - Do we really need this?  It is always on if CONFIG_SCHED_SMT is,

 * so can't we drop this in favor of CONFIG_SCHED_SMT?

 */

#define ARCH_HAS_SCHED_WAKE_IDLE

/* Common values for SMT siblings */

#ifndef SD_SIBLING_INIT

#define SD_SIBLING_INIT (struct sched_domain) {				\

	.min_interval		= 1,					\

	.max_interval		= 2,					\

	.busy_factor		= 64,					\

	.imbalance_pct		= 110,					\

									\

	.flags			= 1*SD_LOAD_BALANCE			\

				| 1*SD_BALANCE_NEWIDLE			\

				| 1*SD_BALANCE_EXEC			\

				| 1*SD_BALANCE_FORK			\

				| 0*SD_BALANCE_WAKE			\

				| 1*SD_WAKE_AFFINE			\

				| 1*SD_SHARE_CPUPOWER			\

				| 1*SD_SHARE_PKG_RESOURCES		\

				| 0*SD_SERIALIZE			\

				| 0*SD_PREFER_SIBLING			\

				| arch_sd_sibling_asym_packing()	\

				,					\

	.last_balance		= jiffies,				\

	.balance_interval	= 1,					\

	.smt_gain		= 1178,	/* 15% */			\

	.max_newidle_lb_cost	= 0,					\

	.next_decay_max_lb_cost	= jiffies,				\

}

#endif

#endif /* CONFIG_SCHED_SMT */



#ifdef CONFIG_SCHED_MC

/* Common values for MC siblings. for now mostly derived from SD_CPU_INIT */

#ifndef SD_MC_INIT

#define SD_MC_INIT (struct sched_domain) {				\

	.min_interval		= 1,					\

	.max_interval		= 4,					\

	.busy_factor		= 64,					\

	.imbalance_pct		= 125,					\

	.cache_nice_tries	= 1,					\

	.busy_idx		= 2,					\

	.wake_idx		= 0,					\

	.forkexec_idx		= 0,					\

									\

	.flags			= 1*SD_LOAD_BALANCE			\

				| 1*SD_BALANCE_NEWIDLE			\

				| 1*SD_BALANCE_EXEC			\

				| 1*SD_BALANCE_FORK			\

				| 0*SD_BALANCE_WAKE			\

				| 1*SD_WAKE_AFFINE			\

				| 0*SD_SHARE_CPUPOWER			\

				| 1*SD_SHARE_PKG_RESOURCES		\

				| 0*SD_SERIALIZE			\

				,					\

	.last_balance		= jiffies,				\

	.balance_interval	= 1,					\

	.max_newidle_lb_cost	= 0,					\

	.next_decay_max_lb_cost	= jiffies,				\

}

#endif

#endif /* CONFIG_SCHED_MC */



/* Common values for CPUs */

#ifndef SD_CPU_INIT

#define SD_CPU_INIT (struct sched_domain) {				\

	.min_interval		= 1,					\

	.max_interval		= 4,					\

	.busy_factor		= 64,					\

	.imbalance_pct		= 125,					\

	.cache_nice_tries	= 1,					\

	.busy_idx		= 2,					\

	.idle_idx		= 1,					\

	.newidle_idx		= 0,					\

	.wake_idx		= 0,					\

	.forkexec_idx		= 0,					\

									\

	.flags			= 1*SD_LOAD_BALANCE			\

				| 1*SD_BALANCE_NEWIDLE			\

				| 1*SD_BALANCE_EXEC			\

				| 1*SD_BALANCE_FORK			\

				| 0*SD_BALANCE_WAKE			\

				| 1*SD_WAKE_AFFINE			\

				| 0*SD_SHARE_CPUPOWER			\

				| 0*SD_SHARE_PKG_RESOURCES		\

				| 0*SD_SERIALIZE			\

				| 1*SD_PREFER_SIBLING			\

				,					\

	.last_balance		= jiffies,				\

	.balance_interval	= 1,					\

	.max_newidle_lb_cost	= 0,					\

	.next_decay_max_lb_cost	= jiffies,				\

}

#endif



#ifdef CONFIG_SCHED_BOOK

#ifndef SD_BOOK_INIT

#error Please define an appropriate SD_BOOK_INIT in include/asm/topology.h!!!

#endif

#endif /* CONFIG_SCHED_BOOK */



#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID

DECLARE_PER_CPU(int, numa_node);
@@ -295,4 +180,17 @@ static inline int cpu_to_mem(int cpu) 
#define topology_core_cpumask(cpu)		cpumask_of(cpu)

#endif



#ifdef CONFIG_SCHED_SMT

static inline const struct cpumask *cpu_smt_mask(int cpu)

{

	return topology_thread_cpumask(cpu);

}

#endif



static inline const struct cpumask *cpu_cpu_mask(int cpu)

{

	return cpumask_of_node(cpu_to_node(cpu));

}





#endif /* _LINUX_TOPOLOGY_H */

CRA Git | Maintained and supported by SUSTech CRA and CCSE