Commit 08f511fd authored by Rafael J. Wysocki's avatar Rafael J. Wysocki
Browse files

cpufreq: Reduce cpufreq_update_util() overhead a bit 



Use the observation that cpufreq_update_util() is only called
by the scheduler with rq->lock held, so the callers of
cpufreq_set_update_util_data() can use synchronize_sched()
instead of synchronize_rcu() to wait for cpufreq_update_util()
to complete.  Moreover, if they are updated to do that,
rcu_read_(un)lock() calls in cpufreq_update_util() might be
replaced with rcu_read_(un)lock_sched(), respectively, but
those aren't really necessary, because the scheduler calls
that function from RCU-sched read-side critical sections
already.

In addition to that, if cpufreq_set_update_util_data() checks
the func field in the struct update_util_data before setting
the per-CPU pointer to it, the data->func check may be dropped
from cpufreq_update_util() as well.

Make the above changes to reduce the overhead from
cpufreq_update_util() in the scheduler paths invoking it
and to make the cleanup after removing its callbacks less
heavy-weight somewhat.

Signed-off-by: default avatarRafael J. Wysocki <rafael.j.wysocki@intel.com>
Acked-by: default avatarViresh Kumar <viresh.kumar@linaro.org>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
parent e6f03657

drivers/cpufreq/cpufreq.c

+17 −8
Original line number Original line Diff line number Diff line
@@ -115,12 +115,15 @@ static DEFINE_PER_CPU(struct update_util_data *, cpufreq_update_util_data); 
 * to call from cpufreq_update_util().  That function will be called from an RCU

 * to call from cpufreq_update_util().  That function will be called from an RCU

 * read-side critical section, so it must not sleep.

 * read-side critical section, so it must not sleep.

 *

 *

 * Callers must use RCU callbacks to free any memory that might be accessed

 * Callers must use RCU-sched callbacks to free any memory that might be

 * via the old update_util_data pointer or invoke synchronize_rcu() right after

 * accessed via the old update_util_data pointer or invoke synchronize_sched()

 * this function to avoid use-after-free.

 * right after this function to avoid use-after-free.

 */

 */

void cpufreq_set_update_util_data(int cpu, struct update_util_data *data)

void cpufreq_set_update_util_data(int cpu, struct update_util_data *data)

{

{

	if (WARN_ON(data && !data->func))

		return;



	rcu_assign_pointer(per_cpu(cpufreq_update_util_data, cpu), data);

	rcu_assign_pointer(per_cpu(cpufreq_update_util_data, cpu), data);

}

}

EXPORT_SYMBOL_GPL(cpufreq_set_update_util_data);

EXPORT_SYMBOL_GPL(cpufreq_set_update_util_data);
@@ -133,18 +136,24 @@ EXPORT_SYMBOL_GPL(cpufreq_set_update_util_data); 
 *

 *

 * This function is called by the scheduler on every invocation of

 * This function is called by the scheduler on every invocation of

 * update_load_avg() on the CPU whose utilization is being updated.

 * update_load_avg() on the CPU whose utilization is being updated.

 *

 * It can only be called from RCU-sched read-side critical sections.

 */

 */

void cpufreq_update_util(u64 time, unsigned long util, unsigned long max)

void cpufreq_update_util(u64 time, unsigned long util, unsigned long max)

{

{

	struct update_util_data *data;

	struct update_util_data *data;





	rcu_read_lock();

#ifdef CONFIG_LOCKDEP

	WARN_ON(debug_locks && !rcu_read_lock_sched_held());

#endif





	data = rcu_dereference(*this_cpu_ptr(&cpufreq_update_util_data));

	data = rcu_dereference_sched(*this_cpu_ptr(&cpufreq_update_util_data));

	if (data && data->func)

	/*

	 * If this isn't inside of an RCU-sched read-side critical section, data

	 * may become NULL after the check below.

	 */

	if (data)

		data->func(data, time, util, max);

		data->func(data, time, util, max);



	rcu_read_unlock();

}

}





/* Flag to suspend/resume CPUFreq governors */

/* Flag to suspend/resume CPUFreq governors */

drivers/cpufreq/cpufreq_governor.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -280,7 +280,7 @@ static inline void gov_clear_update_util(struct cpufreq_policy *policy) 
	for_each_cpu(i, policy->cpus)

	for_each_cpu(i, policy->cpus)

		cpufreq_set_update_util_data(i, NULL);

		cpufreq_set_update_util_data(i, NULL);





	synchronize_rcu();

	synchronize_sched();

}

}





static void gov_cancel_work(struct cpufreq_policy *policy)

static void gov_cancel_work(struct cpufreq_policy *policy)

drivers/cpufreq/intel_pstate.c

+2 −2
Original line number Original line Diff line number Diff line
@@ -1168,7 +1168,7 @@ static void intel_pstate_stop_cpu(struct cpufreq_policy *policy) 
	pr_debug("intel_pstate: CPU %d exiting\n", cpu_num);

	pr_debug("intel_pstate: CPU %d exiting\n", cpu_num);





	cpufreq_set_update_util_data(cpu_num, NULL);

	cpufreq_set_update_util_data(cpu_num, NULL);

	synchronize_rcu();

	synchronize_sched();





	if (hwp_active)

	if (hwp_active)

		return;

		return;
@@ -1426,7 +1426,7 @@ out: 
	for_each_online_cpu(cpu) {

	for_each_online_cpu(cpu) {

		if (all_cpu_data[cpu]) {

		if (all_cpu_data[cpu]) {

			cpufreq_set_update_util_data(cpu, NULL);

			cpufreq_set_update_util_data(cpu, NULL);

			synchronize_rcu();

			synchronize_sched();

			kfree(all_cpu_data[cpu]);

			kfree(all_cpu_data[cpu]);

		}

		}

	}

	}

CRA Git | Maintained and supported by SUSTech CRA and CCSE