workqueue: Convert the pool::lock and wq_mayday_lock to raw_spinlock_t

/* struct worker is defined in workqueue_internal.h */

struct worker_pool {

	spinlock_t		lock;		/* the pool lock */

	raw_spinlock_t		lock;		/* the pool lock */

	int			cpu;		/* I: the associated cpu */

	int			node;		/* I: the associated node ID */

	int			id;		/* I: pool ID */

static DEFINE_MUTEX(wq_pool_mutex);	/* protects pools and workqueues list */

static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */

static DEFINE_SPINLOCK(wq_mayday_lock);	/* protects wq->maydays list */

static DEFINE_RAW_SPINLOCK(wq_mayday_lock);	/* protects wq->maydays list */

/* wait for manager to go away */

static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);

 * Wake up the first idle worker of @pool.

 *

 * CONTEXT:

 * spin_lock_irq(pool->lock).

 * raw_spin_lock_irq(pool->lock).

 */

static void wake_up_worker(struct worker_pool *pool)

{

		return;

	worker->sleeping = 1;

	spin_lock_irq(&pool->lock);

	raw_spin_lock_irq(&pool->lock);

	/*

	 * The counterpart of the following dec_and_test, implied mb,

		if (next)

			wake_up_process(next->task);

	}

	spin_unlock_irq(&pool->lock);

	raw_spin_unlock_irq(&pool->lock);

}

/**

 * the scheduler to get a worker's last known identity.

 *

 * CONTEXT:

 * spin_lock_irq(rq->lock)

 * raw_spin_lock_irq(rq->lock)

 *

 * This function is called during schedule() when a kworker is going

 * to sleep. It's used by psi to identify aggregation workers during

 * Set @flags in @worker->flags and adjust nr_running accordingly.

 *

 * CONTEXT:

 * spin_lock_irq(pool->lock)

 * raw_spin_lock_irq(pool->lock)

 */

static inline void worker_set_flags(struct worker *worker, unsigned int flags)

{

 * Clear @flags in @worker->flags and adjust nr_running accordingly.

 *

 * CONTEXT:

 * spin_lock_irq(pool->lock)

 * raw_spin_lock_irq(pool->lock)

 */

static inline void worker_clr_flags(struct worker *worker, unsigned int flags)

{

 * actually occurs, it should be easy to locate the culprit work function.

 *

 * CONTEXT:

 * spin_lock_irq(pool->lock).

 * raw_spin_lock_irq(pool->lock).

 *

 * Return:

 * Pointer to worker which is executing @work if found, %NULL

 * nested inside outer list_for_each_entry_safe().

 *

 * CONTEXT:

 * spin_lock_irq(pool->lock).

 * raw_spin_lock_irq(pool->lock).

 */

static void move_linked_works(struct work_struct *work, struct list_head *head,

			      struct work_struct **nextp)

		 * As both pwqs and pools are RCU protected, the

		 * following lock operations are safe.

		 */

		spin_lock_irq(&pwq->pool->lock);

		raw_spin_lock_irq(&pwq->pool->lock);

		put_pwq(pwq);

		spin_unlock_irq(&pwq->pool->lock);

		raw_spin_unlock_irq(&pwq->pool->lock);

	}

}

 * decrement nr_in_flight of its pwq and handle workqueue flushing.

 *

 * CONTEXT:

 * spin_lock_irq(pool->lock).

 * raw_spin_lock_irq(pool->lock).

 */

static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)

{

	if (!pool)

		goto fail;

	spin_lock(&pool->lock);

	raw_spin_lock(&pool->lock);

	/*

	 * work->data is guaranteed to point to pwq only while the work

	 * item is queued on pwq->wq, and both updating work->data to point

		/* work->data points to pwq iff queued, point to pool */

		set_work_pool_and_keep_pending(work, pool->id);

		spin_unlock(&pool->lock);

		raw_spin_unlock(&pool->lock);

		rcu_read_unlock();

		return 1;

	}

	spin_unlock(&pool->lock);

	raw_spin_unlock(&pool->lock);

fail:

	rcu_read_unlock();

	local_irq_restore(*flags);

 * work_struct flags.

 *

 * CONTEXT:

 * spin_lock_irq(pool->lock).

 * raw_spin_lock_irq(pool->lock).

 */

static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,

			struct list_head *head, unsigned int extra_flags)

	if (last_pool && last_pool != pwq->pool) {

		struct worker *worker;

		spin_lock(&last_pool->lock);

		raw_spin_lock(&last_pool->lock);

		worker = find_worker_executing_work(last_pool, work);

			pwq = worker->current_pwq;

		} else {

			/* meh... not running there, queue here */

			spin_unlock(&last_pool->lock);

			spin_lock(&pwq->pool->lock);

			raw_spin_unlock(&last_pool->lock);

			raw_spin_lock(&pwq->pool->lock);

		}

	} else {

		spin_lock(&pwq->pool->lock);

		raw_spin_lock(&pwq->pool->lock);

	}

	/*

	 */

	if (unlikely(!pwq->refcnt)) {

		if (wq->flags & WQ_UNBOUND) {

			spin_unlock(&pwq->pool->lock);

			raw_spin_unlock(&pwq->pool->lock);

			cpu_relax();

			goto retry;

		}

	insert_work(pwq, work, worklist, work_flags);

out:

	spin_unlock(&pwq->pool->lock);

	raw_spin_unlock(&pwq->pool->lock);

	rcu_read_unlock();

}

 * necessary.

 *

 * LOCKING:

 * spin_lock_irq(pool->lock).

 * raw_spin_lock_irq(pool->lock).

 */

static void worker_enter_idle(struct worker *worker)

{

 * @worker is leaving idle state.  Update stats.

 *

 * LOCKING:

 * spin_lock_irq(pool->lock).

 * raw_spin_lock_irq(pool->lock).

 */

static void worker_leave_idle(struct worker *worker)

{

	worker_attach_to_pool(worker, pool);

	/* start the newly created worker */

	spin_lock_irq(&pool->lock);

	raw_spin_lock_irq(&pool->lock);

	worker->pool->nr_workers++;

	worker_enter_idle(worker);

	wake_up_process(worker->task);

	spin_unlock_irq(&pool->lock);

	raw_spin_unlock_irq(&pool->lock);

	return worker;

 * be idle.

 *

 * CONTEXT:

 * spin_lock_irq(pool->lock).

 * raw_spin_lock_irq(pool->lock).

 */

static void destroy_worker(struct worker *worker)

{

{

	struct worker_pool *pool = from_timer(pool, t, idle_timer);

	spin_lock_irq(&pool->lock);

	raw_spin_lock_irq(&pool->lock);

	while (too_many_workers(pool)) {

		struct worker *worker;

		destroy_worker(worker);

	}

	spin_unlock_irq(&pool->lock);

	raw_spin_unlock_irq(&pool->lock);

}

static void send_mayday(struct work_struct *work)

	struct worker_pool *pool = from_timer(pool, t, mayday_timer);

	struct work_struct *work;

	spin_lock_irq(&pool->lock);

	spin_lock(&wq_mayday_lock);		/* for wq->maydays */

	raw_spin_lock_irq(&pool->lock);

	raw_spin_lock(&wq_mayday_lock);		/* for wq->maydays */

	if (need_to_create_worker(pool)) {

		/*

			send_mayday(work);

	}

	spin_unlock(&wq_mayday_lock);

	spin_unlock_irq(&pool->lock);

	raw_spin_unlock(&wq_mayday_lock);

	raw_spin_unlock_irq(&pool->lock);

	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);

}

 * may_start_working() %true.

 *

 * LOCKING:

 * spin_lock_irq(pool->lock) which may be released and regrabbed

 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed

 * multiple times.  Does GFP_KERNEL allocations.  Called only from

 * manager.

 */

__acquires(&pool->lock)

{

restart:

	spin_unlock_irq(&pool->lock);

	raw_spin_unlock_irq(&pool->lock);

	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */

	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);

	}

	del_timer_sync(&pool->mayday_timer);

	spin_lock_irq(&pool->lock);

	raw_spin_lock_irq(&pool->lock);

	/*

	 * This is necessary even after a new worker was just successfully

	 * created as @pool->lock was dropped and the new worker might have

 * and may_start_working() is true.

 *

 * CONTEXT:

 * spin_lock_irq(pool->lock) which may be released and regrabbed

 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed

 * multiple times.  Does GFP_KERNEL allocations.

 *

 * Return:

 * call this function to process a work.

 *

 * CONTEXT:

 * spin_lock_irq(pool->lock) which is released and regrabbed.

 * raw_spin_lock_irq(pool->lock) which is released and regrabbed.

 */

static void process_one_work(struct worker *worker, struct work_struct *work)

__releases(&pool->lock)

	 */

	set_work_pool_and_clear_pending(work, pool->id);

	spin_unlock_irq(&pool->lock);

	raw_spin_unlock_irq(&pool->lock);

	lock_map_acquire(&pwq->wq->lockdep_map);

	lock_map_acquire(&lockdep_map);

	 */

	cond_resched();

	spin_lock_irq(&pool->lock);

	raw_spin_lock_irq(&pool->lock);

	/* clear cpu intensive status */

	if (unlikely(cpu_intensive))

 * fetches a work from the top and executes it.

 *

 * CONTEXT:

 * spin_lock_irq(pool->lock) which may be released and regrabbed

 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed

 * multiple times.

 */

static void process_scheduled_works(struct worker *worker)

	/* tell the scheduler that this is a workqueue worker */

	set_pf_worker(true);

woke_up:

	spin_lock_irq(&pool->lock);

	raw_spin_lock_irq(&pool->lock);

	/* am I supposed to die? */

	if (unlikely(worker->flags & WORKER_DIE)) {

		spin_unlock_irq(&pool->lock);

		raw_spin_unlock_irq(&pool->lock);

		WARN_ON_ONCE(!list_empty(&worker->entry));

		set_pf_worker(false);

	 */

	worker_enter_idle(worker);

	__set_current_state(TASK_IDLE);

	spin_unlock_irq(&pool->lock);

	raw_spin_unlock_irq(&pool->lock);

	schedule();

	goto woke_up;

}

	should_stop = kthread_should_stop();

	/* see whether any pwq is asking for help */

	spin_lock_irq(&wq_mayday_lock);

	raw_spin_lock_irq(&wq_mayday_lock);

	while (!list_empty(&wq->maydays)) {

		struct pool_workqueue *pwq = list_first_entry(&wq->maydays,

		__set_current_state(TASK_RUNNING);

		list_del_init(&pwq->mayday_node);

		spin_unlock_irq(&wq_mayday_lock);

		raw_spin_unlock_irq(&wq_mayday_lock);

		worker_attach_to_pool(rescuer, pool);

		spin_lock_irq(&pool->lock);

		raw_spin_lock_irq(&pool->lock);

		/*

		 * Slurp in all works issued via this workqueue and

			 * incur MAYDAY_INTERVAL delay inbetween.

			 */

			if (need_to_create_worker(pool)) {

				spin_lock(&wq_mayday_lock);

				raw_spin_lock(&wq_mayday_lock);

				/*

				 * Queue iff we aren't racing destruction

				 * and somebody else hasn't queued it already.

					get_pwq(pwq);

					list_add_tail(&pwq->mayday_node, &wq->maydays);

				}

				spin_unlock(&wq_mayday_lock);

				raw_spin_unlock(&wq_mayday_lock);

			}

		}

		if (need_more_worker(pool))

			wake_up_worker(pool);

		spin_unlock_irq(&pool->lock);

		raw_spin_unlock_irq(&pool->lock);

		worker_detach_from_pool(rescuer);

		spin_lock_irq(&wq_mayday_lock);

		raw_spin_lock_irq(&wq_mayday_lock);

	}

	spin_unlock_irq(&wq_mayday_lock);

	raw_spin_unlock_irq(&wq_mayday_lock);

	if (should_stop) {

		__set_current_state(TASK_RUNNING);

 * underneath us, so we can't reliably determine pwq from @target.

 *

 * CONTEXT:

 * spin_lock_irq(pool->lock).

 * raw_spin_lock_irq(pool->lock).

 */

static void insert_wq_barrier(struct pool_workqueue *pwq,

			      struct wq_barrier *barr,

	for_each_pwq(pwq, wq) {

		struct worker_pool *pool = pwq->pool;

		spin_lock_irq(&pool->lock);

		raw_spin_lock_irq(&pool->lock);

		if (flush_color >= 0) {

			WARN_ON_ONCE(pwq->flush_color != -1);

			pwq->work_color = work_color;

		}

		spin_unlock_irq(&pool->lock);

		raw_spin_unlock_irq(&pool->lock);

	}

	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))

	for_each_pwq(pwq, wq) {

		bool drained;

		spin_lock_irq(&pwq->pool->lock);

		raw_spin_lock_irq(&pwq->pool->lock);

		drained = !pwq->nr_active && list_empty(&pwq->delayed_works);

		spin_unlock_irq(&pwq->pool->lock);

		raw_spin_unlock_irq(&pwq->pool->lock);

		if (drained)

			continue;

		return false;

	}

	spin_lock_irq(&pool->lock);

	raw_spin_lock_irq(&pool->lock);

	/* see the comment in try_to_grab_pending() with the same code */

	pwq = get_work_pwq(work);

	if (pwq) {

	check_flush_dependency(pwq->wq, work);

	insert_wq_barrier(pwq, barr, work, worker);

	spin_unlock_irq(&pool->lock);

	raw_spin_unlock_irq(&pool->lock);

	/*

	 * Force a lock recursion deadlock when using flush_work() inside a

	rcu_read_unlock();

	return true;

already_gone:

	spin_unlock_irq(&pool->lock);

	raw_spin_unlock_irq(&pool->lock);

	rcu_read_unlock();

	return false;

}

 */

static int init_worker_pool(struct worker_pool *pool)

{

	spin_lock_init(&pool->lock);

	raw_spin_lock_init(&pool->lock);

	pool->id = -1;

	pool->cpu = -1;

	pool->node = NUMA_NO_NODE;

/* This returns with the lock held on success (pool manager is inactive). */

static bool wq_manager_inactive(struct worker_pool *pool)

{

	spin_lock_irq(&pool->lock);

	raw_spin_lock_irq(&pool->lock);

	if (pool->flags & POOL_MANAGER_ACTIVE) {

		spin_unlock_irq(&pool->lock);

		raw_spin_unlock_irq(&pool->lock);

		return false;

	}

	return true;

	while ((worker = first_idle_worker(pool)))

		destroy_worker(worker);

	WARN_ON(pool->nr_workers || pool->nr_idle);

	spin_unlock_irq(&pool->lock);

	raw_spin_unlock_irq(&pool->lock);

	mutex_lock(&wq_pool_attach_mutex);

	if (!list_empty(&pool->workers))

		return;

	/* this function can be called during early boot w/ irq disabled */

	spin_lock_irqsave(&pwq->pool->lock, flags);

	raw_spin_lock_irqsave(&pwq->pool->lock, flags);

	/*

	 * During [un]freezing, the caller is responsible for ensuring that

		pwq->max_active = 0;

	}

	spin_unlock_irqrestore(&pwq->pool->lock, flags);

	raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);

}

/* initialize newly alloced @pwq which is associated with @wq and @pool */

use_dfl_pwq:

	mutex_lock(&wq->mutex);

	spin_lock_irq(&wq->dfl_pwq->pool->lock);

	raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);

	get_pwq(wq->dfl_pwq);

	spin_unlock_irq(&wq->dfl_pwq->pool->lock);

	raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);

	old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);

out_unlock:

	mutex_unlock(&wq->mutex);

		struct worker *rescuer = wq->rescuer;

		/* this prevents new queueing */

		spin_lock_irq(&wq_mayday_lock);

		raw_spin_lock_irq(&wq_mayday_lock);

		wq->rescuer = NULL;

		spin_unlock_irq(&wq_mayday_lock);

		raw_spin_unlock_irq(&wq_mayday_lock);

		/* rescuer will empty maydays list before exiting */

		kthread_stop(rescuer->task);

	mutex_lock(&wq_pool_mutex);

	mutex_lock(&wq->mutex);

	for_each_pwq(pwq, wq) {

		spin_lock_irq(&pwq->pool->lock);

		raw_spin_lock_irq(&pwq->pool->lock);

		if (WARN_ON(pwq_busy(pwq))) {

			pr_warn("%s: %s has the following busy pwq\n",

				__func__, wq->name);

			show_pwq(pwq);

			spin_unlock_irq(&pwq->pool->lock);

			raw_spin_unlock_irq(&pwq->pool->lock);

			mutex_unlock(&wq->mutex);

			mutex_unlock(&wq_pool_mutex);

			show_workqueue_state();

			return;

		}

		spin_unlock_irq(&pwq->pool->lock);

		raw_spin_unlock_irq(&pwq->pool->lock);

	}

	mutex_unlock(&wq->mutex);

	mutex_unlock(&wq_pool_mutex);

	rcu_read_lock();

	pool = get_work_pool(work);

	if (pool) {

		spin_lock_irqsave(&pool->lock, flags);

		raw_spin_lock_irqsave(&pool->lock, flags);

		if (find_worker_executing_work(pool, work))

			ret |= WORK_BUSY_RUNNING;

		spin_unlock_irqrestore(&pool->lock, flags);

		raw_spin_unlock_irqrestore(&pool->lock, flags);

	}

	rcu_read_unlock();

		pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);

		for_each_pwq(pwq, wq) {

			spin_lock_irqsave(&pwq->pool->lock, flags);

			raw_spin_lock_irqsave(&pwq->pool->lock, flags);

			if (pwq->nr_active || !list_empty(&pwq->delayed_works))

				show_pwq(pwq);

			spin_unlock_irqrestore(&pwq->pool->lock, flags);

			raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);

			/*

			 * We could be printing a lot from atomic context, e.g.

			 * sysrq-t -> show_workqueue_state(). Avoid triggering

		struct worker *worker;

		bool first = true;

		spin_lock_irqsave(&pool->lock, flags);

		raw_spin_lock_irqsave(&pool->lock, flags);

		if (pool->nr_workers == pool->nr_idle)

			goto next_pool;

		}

		pr_cont("\n");

	next_pool:

		spin_unlock_irqrestore(&pool->lock, flags);

		raw_spin_unlock_irqrestore(&pool->lock, flags);

		/*

		 * We could be printing a lot from atomic context, e.g.

		 * sysrq-t -> show_workqueue_state(). Avoid triggering

		struct worker_pool *pool = worker->pool;

		if (pool) {

			spin_lock_irq(&pool->lock);

			raw_spin_lock_irq(&pool->lock);

			/*

			 * ->desc tracks information (wq name or

			 * set_worker_desc()) for the latest execution.  If

					scnprintf(buf + off, size - off, "-%s",

						  worker->desc);

			}

			spin_unlock_irq(&pool->lock);

			raw_spin_unlock_irq(&pool->lock);

		}

	}

	for_each_cpu_worker_pool(pool, cpu) {