Commit 5b95e1af authored by Lai Jiangshan's avatar Lai Jiangshan Committed by Tejun Heo
Browse files

workqueue: wq_pool_mutex protects the attrs-installation 



Current wq_pool_mutex doesn't proctect the attrs-installation, it results
that ->unbound_attrs, ->numa_pwq_tbl[] and ->dfl_pwq can only be accessed
under wq->mutex and causes some inconveniences. Example, wq_update_unbound_numa()
has to acquire wq->mutex before fetching the wq->unbound_attrs->no_numa
and the old_pwq.

attrs-installation is a short operation, so this change will no cause any
latency for other operations which also acquire the wq_pool_mutex.

The only unprotected attrs-installation code is in apply_workqueue_attrs(),
so this patch touches code less than comments.

It is also a preparation patch for next several patches which read
wq->unbound_attrs, wq->numa_pwq_tbl[] and wq->dfl_pwq with
only wq_pool_mutex held.

Signed-off-by: default avatarLai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent b749b1b6

kernel/workqueue.c

+20 −7
Original line number Diff line number Diff line
@@ -127,6 +127,11 @@ enum { 
 *

 * PR: wq_pool_mutex protected for writes.  Sched-RCU protected for reads.

 *

 * PW: wq_pool_mutex and wq->mutex protected for writes.  Either for reads.

 *

 * PWR: wq_pool_mutex and wq->mutex protected for writes.  Either or

 *      sched-RCU for reads.

 *

 * WQ: wq->mutex protected.

 *

 * WR: wq->mutex protected for writes.  Sched-RCU protected for reads.
@@ -247,8 +252,8 @@ struct workqueue_struct { 
	int			nr_drainers;	/* WQ: drain in progress */

	int			saved_max_active; /* WQ: saved pwq max_active */



	struct workqueue_attrs	*unbound_attrs;	/* WQ: only for unbound wqs */

	struct pool_workqueue	*dfl_pwq;	/* WQ: only for unbound wqs */

	struct workqueue_attrs	*unbound_attrs;	/* PW: only for unbound wqs */

	struct pool_workqueue	*dfl_pwq;	/* PW: only for unbound wqs */



#ifdef CONFIG_SYSFS

	struct wq_device	*wq_dev;	/* I: for sysfs interface */
@@ -268,7 +273,7 @@ struct workqueue_struct { 
	/* hot fields used during command issue, aligned to cacheline */

	unsigned int		flags ____cacheline_aligned; /* WQ: WQ_* flags */

	struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */

	struct pool_workqueue __rcu *numa_pwq_tbl[]; /* FR: unbound pwqs indexed by node */

	struct pool_workqueue __rcu *numa_pwq_tbl[]; /* PWR: unbound pwqs indexed by node */

};



static struct kmem_cache *pwq_cache;
@@ -349,6 +354,12 @@ static void workqueue_sysfs_unregister(struct workqueue_struct *wq); 
			   lockdep_is_held(&wq->mutex),			\

			   "sched RCU or wq->mutex should be held")



#define assert_rcu_or_wq_mutex_or_pool_mutex(wq)			\

	rcu_lockdep_assert(rcu_read_lock_sched_held() ||		\

			   lockdep_is_held(&wq->mutex) ||		\

			   lockdep_is_held(&wq_pool_mutex),		\

			   "sched RCU, wq->mutex or wq_pool_mutex should be held")



#define for_each_cpu_worker_pool(pool, cpu)				\

	for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0];		\

	     (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
@@ -553,7 +564,8 @@ static int worker_pool_assign_id(struct worker_pool *pool) 
 * @wq: the target workqueue

 * @node: the node ID

 *

 * This must be called either with pwq_lock held or sched RCU read locked.

 * This must be called with any of wq_pool_mutex, wq->mutex or sched RCU

 * read locked.

 * If the pwq needs to be used beyond the locking in effect, the caller is

 * responsible for guaranteeing that the pwq stays online.

 *
@@ -562,7 +574,7 @@ static int worker_pool_assign_id(struct worker_pool *pool) 
static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,

						  int node)

{

	assert_rcu_or_wq_mutex(wq);

	assert_rcu_or_wq_mutex_or_pool_mutex(wq);

	return rcu_dereference_raw(wq->numa_pwq_tbl[node]);

}
@@ -3479,6 +3491,7 @@ static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq, 
{

	struct pool_workqueue *old_pwq;



	lockdep_assert_held(&wq_pool_mutex);

	lockdep_assert_held(&wq->mutex);



	/* link_pwq() can handle duplicate calls */
@@ -3644,10 +3657,9 @@ int apply_workqueue_attrs(struct workqueue_struct *wq, 
	 * pwqs accordingly.

	 */

	get_online_cpus();



	mutex_lock(&wq_pool_mutex);



	ctx = apply_wqattrs_prepare(wq, attrs);

	mutex_unlock(&wq_pool_mutex);



	/* the ctx has been prepared successfully, let's commit it */

	if (ctx) {
@@ -3655,6 +3667,7 @@ int apply_workqueue_attrs(struct workqueue_struct *wq, 
		ret = 0;

	}



	mutex_unlock(&wq_pool_mutex);

	put_online_cpus();



	apply_wqattrs_cleanup(ctx);

CRA Git | Maintained and supported by SUSTech CRA and CCSE