Commit 6cf539a8 authored by Marco Elver's avatar Marco Elver Committed by Paul E. McKenney
Browse files

rcu: Fix data-race due to atomic_t copy-by-value 



This fixes a data-race where `atomic_t dynticks` is copied by value. The
copy is performed non-atomically, resulting in a data-race if `dynticks`
is updated concurrently.

This data-race was found with KCSAN:
==================================================================
BUG: KCSAN: data-race in dyntick_save_progress_counter / rcu_irq_enter

write to 0xffff989dbdbe98e0 of 4 bytes by task 10 on cpu 3:
 atomic_add_return include/asm-generic/atomic-instrumented.h:78 [inline]
 rcu_dynticks_snap kernel/rcu/tree.c:310 [inline]
 dyntick_save_progress_counter+0x43/0x1b0 kernel/rcu/tree.c:984
 force_qs_rnp+0x183/0x200 kernel/rcu/tree.c:2286
 rcu_gp_fqs kernel/rcu/tree.c:1601 [inline]
 rcu_gp_fqs_loop+0x71/0x880 kernel/rcu/tree.c:1653
 rcu_gp_kthread+0x22c/0x3b0 kernel/rcu/tree.c:1799
 kthread+0x1b5/0x200 kernel/kthread.c:255
 <snip>

read to 0xffff989dbdbe98e0 of 4 bytes by task 154 on cpu 7:
 rcu_nmi_enter_common kernel/rcu/tree.c:828 [inline]
 rcu_irq_enter+0xda/0x240 kernel/rcu/tree.c:870
 irq_enter+0x5/0x50 kernel/softirq.c:347
 <snip>

Reported by Kernel Concurrency Sanitizer on:
CPU: 7 PID: 154 Comm: kworker/7:1H Not tainted 5.3.0+ #5
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014
Workqueue: kblockd blk_mq_run_work_fn
==================================================================

Signed-off-by: default avatarMarco Elver <elver@google.com>
Cc: Paul E. McKenney <paulmck@kernel.org>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Joel Fernandes <joel@joelfernandes.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: rcu@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
Reviewed-by: default avatarJoel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 9f08cf08

include/trace/events/rcu.h

+2 −2
Original line number Diff line number Diff line
@@ -449,7 +449,7 @@ TRACE_EVENT_RCU(rcu_fqs, 
 */

TRACE_EVENT_RCU(rcu_dyntick,



	TP_PROTO(const char *polarity, long oldnesting, long newnesting, atomic_t dynticks),

	TP_PROTO(const char *polarity, long oldnesting, long newnesting, int dynticks),



	TP_ARGS(polarity, oldnesting, newnesting, dynticks),
@@ -464,7 +464,7 @@ TRACE_EVENT_RCU(rcu_dyntick, 
		__entry->polarity = polarity;

		__entry->oldnesting = oldnesting;

		__entry->newnesting = newnesting;

		__entry->dynticks = atomic_read(&dynticks);

		__entry->dynticks = dynticks;

	),



	TP_printk("%s %lx %lx %#3x", __entry->polarity,

kernel/rcu/tree.c

+6 −5
Original line number Diff line number Diff line
@@ -577,7 +577,7 @@ static void rcu_eqs_enter(bool user) 
	}



	lockdep_assert_irqs_disabled();

	trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, rdp->dynticks);

	trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks));

	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));

	rdp = this_cpu_ptr(&rcu_data);

	do_nocb_deferred_wakeup(rdp);
@@ -650,14 +650,15 @@ static __always_inline void rcu_nmi_exit_common(bool irq) 
	 * leave it in non-RCU-idle state.

	 */

	if (rdp->dynticks_nmi_nesting != 1) {

		trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2, rdp->dynticks);

		trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2,

				  atomic_read(&rdp->dynticks));

		WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */

			   rdp->dynticks_nmi_nesting - 2);

		return;

	}



	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */

	trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, rdp->dynticks);

	trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks));

	WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */



	if (irq)
@@ -744,7 +745,7 @@ static void rcu_eqs_exit(bool user) 
	rcu_dynticks_task_exit();

	rcu_dynticks_eqs_exit();

	rcu_cleanup_after_idle();

	trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, rdp->dynticks);

	trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks));

	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));

	WRITE_ONCE(rdp->dynticks_nesting, 1);

	WARN_ON_ONCE(rdp->dynticks_nmi_nesting);
@@ -833,7 +834,7 @@ static __always_inline void rcu_nmi_enter_common(bool irq) 
	}

	trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),

			  rdp->dynticks_nmi_nesting,

			  rdp->dynticks_nmi_nesting + incby, rdp->dynticks);

			  rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks));

	WRITE_ONCE(rdp->dynticks_nmi_nesting, /* Prevent store tearing. */

		   rdp->dynticks_nmi_nesting + incby);

	barrier();

CRA Git | Maintained and supported by SUSTech CRA and CCSE