Commit 76c6927c authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu/nocb: Allow lockless use of rcu_segcblist_restempty() 



Currently, rcu_segcblist_restempty() assumes that the callback list
is not being changed by other CPUs, but upcoming changes will require
it to operate locklessly.  This commit therefore adds the needed
READ_ONCE() calls, along with the WRITE_ONCE() calls when updating
the callback list.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.ibm.com>
parent ca5c8258

kernel/rcu/rcu_segcblist.c

+15 −15
Original line number Diff line number Diff line
@@ -147,8 +147,8 @@ void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, 
		rsclp->len_lazy++;

	smp_mb(); /* Ensure counts are updated before callback is enqueued. */

	rhp->next = NULL;

	*rsclp->tails[RCU_NEXT_TAIL] = rhp;

	rsclp->tails[RCU_NEXT_TAIL] = &rhp->next;

	WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rhp);

	WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], &rhp->next);

}



/*
@@ -176,9 +176,9 @@ bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, 
	for (i = RCU_NEXT_TAIL; i > RCU_DONE_TAIL; i--)

		if (rsclp->tails[i] != rsclp->tails[i - 1])

			break;

	*rsclp->tails[i] = rhp;

	WRITE_ONCE(*rsclp->tails[i], rhp);

	for (; i <= RCU_NEXT_TAIL; i++)

		rsclp->tails[i] = &rhp->next;

		WRITE_ONCE(rsclp->tails[i], &rhp->next);

	return true;

}
@@ -214,11 +214,11 @@ void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, 
		return; /* Nothing to do. */

	*rclp->tail = rsclp->head;

	rsclp->head = *rsclp->tails[RCU_DONE_TAIL];

	*rsclp->tails[RCU_DONE_TAIL] = NULL;

	WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);

	rclp->tail = rsclp->tails[RCU_DONE_TAIL];

	for (i = RCU_CBLIST_NSEGS - 1; i >= RCU_DONE_TAIL; i--)

		if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL])

			rsclp->tails[i] = &rsclp->head;

			WRITE_ONCE(rsclp->tails[i], &rsclp->head);

}



/*
@@ -237,9 +237,9 @@ void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, 
		return; /* Nothing to do. */

	*rclp->tail = *rsclp->tails[RCU_DONE_TAIL];

	rclp->tail = rsclp->tails[RCU_NEXT_TAIL];

	*rsclp->tails[RCU_DONE_TAIL] = NULL;

	WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);

	for (i = RCU_DONE_TAIL + 1; i < RCU_CBLIST_NSEGS; i++)

		rsclp->tails[i] = rsclp->tails[RCU_DONE_TAIL];

		WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_DONE_TAIL]);

}



/*
@@ -271,7 +271,7 @@ void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, 
	rsclp->head = rclp->head;

	for (i = RCU_DONE_TAIL; i < RCU_CBLIST_NSEGS; i++)

		if (&rsclp->head == rsclp->tails[i])

			rsclp->tails[i] = rclp->tail;

			WRITE_ONCE(rsclp->tails[i], rclp->tail);

		else

			break;

	rclp->head = NULL;
@@ -287,8 +287,8 @@ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, 
{

	if (!rclp->head)

		return; /* Nothing to do. */

	*rsclp->tails[RCU_NEXT_TAIL] = rclp->head;

	rsclp->tails[RCU_NEXT_TAIL] = rclp->tail;

	WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head);

	WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail);

	rclp->head = NULL;

	rclp->tail = &rclp->head;

}
@@ -312,7 +312,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) 
	for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) {

		if (ULONG_CMP_LT(seq, rsclp->gp_seq[i]))

			break;

		rsclp->tails[RCU_DONE_TAIL] = rsclp->tails[i];

		WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]);

	}



	/* If no callbacks moved, nothing more need be done. */
@@ -321,7 +321,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) 


	/* Clean up tail pointers that might have been misordered above. */

	for (j = RCU_WAIT_TAIL; j < i; j++)

		rsclp->tails[j] = rsclp->tails[RCU_DONE_TAIL];

		WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]);



	/*

	 * Callbacks moved, so clean up the misordered ->tails[] pointers
@@ -332,7 +332,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) 
	for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) {

		if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL])

			break;  /* No more callbacks. */

		rsclp->tails[j] = rsclp->tails[i];

		WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]);

		rsclp->gp_seq[j] = rsclp->gp_seq[i];

	}

}
@@ -397,7 +397,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) 
	 * structure other than in the RCU_NEXT_TAIL segment.

	 */

	for (; i < RCU_NEXT_TAIL; i++) {

		rsclp->tails[i] = rsclp->tails[RCU_NEXT_TAIL];

		WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]);

		rsclp->gp_seq[i] = seq;

	}

	return true;

kernel/rcu/rcu_segcblist.h

+1 −1
Original line number Diff line number Diff line
@@ -79,7 +79,7 @@ static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) 
 */

static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)

{

	return !*rsclp->tails[seg];

	return !READ_ONCE(*READ_ONCE(rsclp->tails[seg]));

}



void rcu_segcblist_init(struct rcu_segcblist *rsclp);

CRA Git | Maintained and supported by SUSTech CRA and CCSE