sched: Only queue remote wakeups when crossing cache boundaries (518cd623) · Commits · 戴 / test

kernel/sched/core.c

+33 −1

Original line number	Diff line number	Diff line
		@@ -1511,6 +1511,11 @@ static int ttwu_activate_remote(struct task_struct *p, int wake_flags)

		}
		#endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */

		static inline int ttwu_share_cache(int this_cpu, int that_cpu)
		{
		return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
		}
		#endif /* CONFIG_SMP */

		static void ttwu_queue(struct task_struct *p, int cpu)
		@@ -1518,7 +1523,7 @@ static void ttwu_queue(struct task_struct *p, int cpu)
		struct rq *rq = cpu_rq(cpu);

		#if defined(CONFIG_SMP)
		if (sched_feat(TTWU_QUEUE) && cpu != smp_processor_id()) {
		if (sched_feat(TTWU_QUEUE) && !ttwu_share_cache(smp_processor_id(), cpu)) {
		sched_clock_cpu(cpu); /* sync clocks x-cpu */
		ttwu_queue_remote(p, cpu);
		return;
		@@ -5743,6 +5748,31 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu)
		destroy_sched_domain(sd, cpu);
		}

		/*
		* Keep a special pointer to the highest sched_domain that has
		* SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
		* allows us to avoid some pointer chasing select_idle_sibling().
		*
		* Also keep a unique ID per domain (we use the first cpu number in
		* the cpumask of the domain), this allows us to quickly tell if
		* two cpus are in the same cache domain, see ttwu_share_cache().
		*/
		DEFINE_PER_CPU(struct sched_domain *, sd_llc);
		DEFINE_PER_CPU(int, sd_llc_id);

		static void update_top_cache_domain(int cpu)
		{
		struct sched_domain *sd;
		int id = cpu;

		sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
		if (sd)
		id = cpumask_first(sched_domain_span(sd));

		rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
		per_cpu(sd_llc_id, cpu) = id;
		}

		/*
		* Attach the domain 'sd' to 'cpu' as its base domain. Callers must
		* hold the hotplug lock.
		@@ -5782,6 +5812,8 @@ cpu_attach_domain(struct sched_domain sd, struct root_domain rd, int cpu)
		tmp = rq->sd;
		rcu_assign_pointer(rq->sd, sd);
		destroy_sched_domains(tmp, cpu);

		update_top_cache_domain(cpu);
		}

		/* cpus with isolated domains */

kernel/sched/fair.c

+1 −23

Original line number	Diff line number	Diff line
		@@ -2644,28 +2644,6 @@ find_idlest_cpu(struct sched_group group, struct task_struct p, int this_cpu)
		return idlest;
		}

		/**
		* highest_flag_domain - Return highest sched_domain containing flag.
		* @cpu: The cpu whose highest level of sched domain is to
		* be returned.
		* @flag: The flag to check for the highest sched_domain
		* for the given cpu.
		*
		* Returns the highest sched_domain of a cpu which contains the given flag.
		*/
		static inline struct sched_domain *highest_flag_domain(int cpu, int flag)
		{
		struct sched_domain sd, hsd = NULL;

		for_each_domain(cpu, sd) {
		if (!(sd->flags & flag))
		break;
		hsd = sd;
		}

		return hsd;
		}

		/*
		* Try and locate an idle CPU in the sched_domain.
		*/
		@@ -2696,7 +2674,7 @@ static int select_idle_sibling(struct task_struct *p, int target)
		*/
		rcu_read_lock();

		sd = highest_flag_domain(target, SD_SHARE_PKG_RESOURCES);
		sd = rcu_dereference(per_cpu(sd_llc, target));
		for_each_lower_domain(sd) {
		sg = sd->groups;
		do {

kernel/sched/sched.h

+36 −6

Original line number	Diff line number	Diff line
		@@ -487,6 +487,14 @@ static inline int cpu_of(struct rq *rq)

		DECLARE_PER_CPU(struct rq, runqueues);

		#define cpu_rq(cpu) (&per_cpu(runqueues, (cpu)))
		#define this_rq() (&__get_cpu_var(runqueues))
		#define task_rq(p) cpu_rq(task_cpu(p))
		#define cpu_curr(cpu) (cpu_rq(cpu)->curr)
		#define raw_rq() (&__raw_get_cpu_var(runqueues))

		#ifdef CONFIG_SMP

		#define rcu_dereference_check_sched_domain(p) \
		rcu_dereference_check((p), \
		lockdep_is_held(&sched_domains_mutex))
		@@ -499,15 +507,37 @@ DECLARE_PER_CPU(struct rq, runqueues);
		* preempt-disabled sections.
		*/
		#define for_each_domain(cpu, __sd) \
		for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
		for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); \
		__sd; __sd = __sd->parent)

		#define for_each_lower_domain(sd) for (; sd; sd = sd->child)

		#define cpu_rq(cpu) (&per_cpu(runqueues, (cpu)))
		#define this_rq() (&__get_cpu_var(runqueues))
		#define task_rq(p) cpu_rq(task_cpu(p))
		#define cpu_curr(cpu) (cpu_rq(cpu)->curr)
		#define raw_rq() (&__raw_get_cpu_var(runqueues))
		/**
		* highest_flag_domain - Return highest sched_domain containing flag.
		* @cpu: The cpu whose highest level of sched domain is to
		* be returned.
		* @flag: The flag to check for the highest sched_domain
		* for the given cpu.
		*
		* Returns the highest sched_domain of a cpu which contains the given flag.
		*/
		static inline struct sched_domain *highest_flag_domain(int cpu, int flag)
		{
		struct sched_domain sd, hsd = NULL;

		for_each_domain(cpu, sd) {
		if (!(sd->flags & flag))
		break;
		hsd = sd;
		}

		return hsd;
		}

		DECLARE_PER_CPU(struct sched_domain *, sd_llc);
		DECLARE_PER_CPU(int, sd_llc_id);

		#endif /* CONFIG_SMP */

		#include "stats.h"
		#include "auto_group.h"

Admin message