Commit 126c2092 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

sched: Add rq::ttwu_pending 



In preparation of removing rq->wake_list, replace the
!list_empty(rq->wake_list) with rq->ttwu_pending. This is not fully
equivalent as this new variable is racy.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20200526161908.070399698@infradead.org
parent 4b44a21d

kernel/sched/core.c

+11 −2
Original line number Diff line number Diff line
@@ -2275,13 +2275,21 @@ static int ttwu_remote(struct task_struct *p, int wake_flags) 
void sched_ttwu_pending(void)

{

	struct rq *rq = this_rq();

	struct llist_node *llist = llist_del_all(&rq->wake_list);

	struct llist_node *llist;

	struct task_struct *p, *t;

	struct rq_flags rf;



	llist = llist_del_all(&rq->wake_list);

	if (!llist)

		return;



	/*

	 * rq::ttwu_pending racy indication of out-standing wakeups.

	 * Races such that false-negatives are possible, since they

	 * are shorter lived that false-positives would be.

	 */

	WRITE_ONCE(rq->ttwu_pending, 0);



	rq_lock_irqsave(rq, &rf);

	update_rq_clock(rq);
@@ -2318,6 +2326,7 @@ static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags 


	p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);



	WRITE_ONCE(rq->ttwu_pending, 1);

	if (llist_add(&p->wake_entry, &rq->wake_list)) {

		if (!set_nr_if_polling(rq->idle))

			smp_call_function_single_async(cpu, &rq->wake_csd);
@@ -4705,7 +4714,7 @@ int idle_cpu(int cpu) 
		return 0;



#ifdef CONFIG_SMP

	if (!llist_empty(&rq->wake_list))

	if (rq->ttwu_pending)

		return 0;

#endif

kernel/sched/debug.c

+0 −1
Original line number Diff line number Diff line
@@ -638,7 +638,6 @@ do { \ 


	P(nr_running);

	P(nr_switches);

	P(nr_load_updates);

	P(nr_uninterruptible);

	PN(next_balance);

	SEQ_printf(m, "  .%-30s: %ld\n", "curr->pid", (long)(task_pid_nr(rq->curr)));

kernel/sched/fair.c

+1 −1
Original line number Diff line number Diff line
@@ -8590,7 +8590,7 @@ static int idle_cpu_without(int cpu, struct task_struct *p) 
	 */



#ifdef CONFIG_SMP

	if (!llist_empty(&rq->wake_list))

	if (rq->ttwu_pending)

		return 0;

#endif

kernel/sched/sched.h

+3 −1
Original line number Diff line number Diff line
@@ -895,7 +895,9 @@ struct rq { 
	atomic_t		nohz_flags;

#endif /* CONFIG_NO_HZ_COMMON */



	unsigned long		nr_load_updates;

#ifdef CONFIG_SMP

	unsigned int		ttwu_pending;

#endif

	u64			nr_switches;



#ifdef CONFIG_UCLAMP_TASK

CRA Git | Maintained and supported by SUSTech CRA and CCSE