Commit ebf0d100 authored by Jens Axboe's avatar Jens Axboe
Browse files

task_work: only grab task signal lock when needed 



If JOBCTL_TASK_WORK is already set on the targeted task, then we need
not go through {lock,unlock}_task_sighand() to set it again and queue
a signal wakeup. This is safe as we're checking it _after_ adding the
new task_work with cmpxchg().

The ordering is as follows:

task_work_add()				get_signal()
--------------------------------------------------------------
STORE(task->task_works, new_work);	STORE(task->jobctl);
mb();					mb();
LOAD(task->jobctl);			LOAD(task->task_works);

This speeds up TWA_SIGNAL handling quite a bit, which is important now
that io_uring is relying on it for all task_work deliveries.

Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Jann Horn <jannh@google.com>
Acked-by: default avatarOleg Nesterov <oleg@redhat.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent f254ac04

kernel/signal.c

+15 −1
Original line number Diff line number Diff line
@@ -2541,7 +2541,21 @@ bool get_signal(struct ksignal *ksig) 


relock:

	spin_lock_irq(&sighand->siglock);

	current->jobctl &= ~JOBCTL_TASK_WORK;

	/*

	 * Make sure we can safely read ->jobctl() in task_work add. As Oleg

	 * states:

	 *

	 * It pairs with mb (implied by cmpxchg) before READ_ONCE. So we

	 * roughly have

	 *

	 *	task_work_add:				get_signal:

	 *	STORE(task->task_works, new_work);	STORE(task->jobctl);

	 *	mb();					mb();

	 *	LOAD(task->jobctl);			LOAD(task->task_works);

	 *

	 * and we can rely on STORE-MB-LOAD [ in task_work_add].

	 */

	smp_store_mb(current->jobctl, current->jobctl & ~JOBCTL_TASK_WORK);

	if (unlikely(current->task_works)) {

		spin_unlock_irq(&sighand->siglock);

		task_work_run();

kernel/task_work.c

+7 −1
Original line number Diff line number Diff line
@@ -42,7 +42,13 @@ task_work_add(struct task_struct *task, struct callback_head *work, int notify) 
		set_notify_resume(task);

		break;

	case TWA_SIGNAL:

		if (lock_task_sighand(task, &flags)) {

		/*

		 * Only grab the sighand lock if we don't already have some

		 * task_work pending. This pairs with the smp_store_mb()

		 * in get_signal(), see comment there.

		 */

		if (!(READ_ONCE(task->jobctl) & JOBCTL_TASK_WORK) &&

		    lock_task_sighand(task, &flags)) {

			task->jobctl |= JOBCTL_TASK_WORK;

			signal_wake_up(task, 0);

			unlock_task_sighand(task, &flags);

CRA Git | Maintained and supported by SUSTech CRA and CCSE