workqueue: wq_pool_mutex protects the attrs-installation

[ Upstream commit 5b95e1af8d17d85a17728f6de7dbff538e6e3c49 ]

Current wq_pool_mutex doesn't proctect the attrs-installation, it results
that ->unbound_attrs, ->numa_pwq_tbl[] and ->dfl_pwq can only be accessed
under wq->mutex and causes some inconveniences. Example, wq_update_unbound_numa()
has to acquire wq->mutex before fetching the wq->unbound_attrs->no_numa
and the old_pwq.

attrs-installation is a short operation, so this change will no cause any
latency for other operations which also acquire the wq_pool_mutex.

The only unprotected attrs-installation code is in apply_workqueue_attrs(),
so this patch touches code less than comments.

It is also a preparation patch for next several patches which read
wq->unbound_attrs, wq->numa_pwq_tbl[] and wq->dfl_pwq with
only wq_pool_mutex held.

Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 26ff249240160b6fb79ee07a71691555ab91bb9a..53b80c011341f3dd60b7b1b404d9a7a2e9041448 100644 (file)
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -127,6 +127,11 @@ enum {
  *
  * PR: wq_pool_mutex protected for writes.  Sched-RCU protected for reads.
  *
+ * PW: wq_pool_mutex and wq->mutex protected for writes.  Either for reads.
+ *
+ * PWR: wq_pool_mutex and wq->mutex protected for writes.  Either or
+ *      sched-RCU for reads.
+ *
  * WQ: wq->mutex protected.
  *
  * WR: wq->mutex protected for writes.  Sched-RCU protected for reads.
@@ -247,8 +252,8 @@ struct workqueue_struct {
        int                     nr_drainers;    /* WQ: drain in progress */
        int                     saved_max_active; /* WQ: saved pwq max_active */
 
-       struct workqueue_attrs  *unbound_attrs; /* WQ: only for unbound wqs */
-       struct pool_workqueue   *dfl_pwq;       /* WQ: only for unbound wqs */
+       struct workqueue_attrs  *unbound_attrs; /* PW: only for unbound wqs */
+       struct pool_workqueue   *dfl_pwq;       /* PW: only for unbound wqs */
 
 #ifdef CONFIG_SYSFS
        struct wq_device        *wq_dev;        /* I: for sysfs interface */
@@ -268,7 +273,7 @@ struct workqueue_struct {
        /* hot fields used during command issue, aligned to cacheline */
        unsigned int            flags ____cacheline_aligned; /* WQ: WQ_* flags */
        struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
-       struct pool_workqueue __rcu *numa_pwq_tbl[]; /* FR: unbound pwqs indexed by node */
+       struct pool_workqueue __rcu *numa_pwq_tbl[]; /* PWR: unbound pwqs indexed by node */
 };
 
 static struct kmem_cache *pwq_cache;
@@ -347,6 +352,12 @@ static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
                           lockdep_is_held(&wq->mutex),                 \
                           "sched RCU or wq->mutex should be held")
 
+#define assert_rcu_or_wq_mutex_or_pool_mutex(wq)                       \
+       rcu_lockdep_assert(rcu_read_lock_sched_held() ||                \
+                          lockdep_is_held(&wq->mutex) ||               \
+                          lockdep_is_held(&wq_pool_mutex),             \
+                          "sched RCU, wq->mutex or wq_pool_mutex should be held")
+
 #define for_each_cpu_worker_pool(pool, cpu)                            \
        for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0];               \
             (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
@@ -551,7 +562,8 @@ static int worker_pool_assign_id(struct worker_pool *pool)
  * @wq: the target workqueue
  * @node: the node ID
  *
- * This must be called either with pwq_lock held or sched RCU read locked.
+ * This must be called with any of wq_pool_mutex, wq->mutex or sched RCU
+ * read locked.
  * If the pwq needs to be used beyond the locking in effect, the caller is
  * responsible for guaranteeing that the pwq stays online.
  *
@@ -560,7 +572,7 @@ static int worker_pool_assign_id(struct worker_pool *pool)
 static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,
                                                  int node)
 {
-       assert_rcu_or_wq_mutex(wq);
+       assert_rcu_or_wq_mutex_or_pool_mutex(wq);
        return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
 }
 
@@ -3477,6 +3489,7 @@ static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,
 {
        struct pool_workqueue *old_pwq;
 
+       lockdep_assert_held(&wq_pool_mutex);
        lockdep_assert_held(&wq->mutex);
 
        /* link_pwq() can handle duplicate calls */
@@ -3631,10 +3644,9 @@ int apply_workqueue_attrs(struct workqueue_struct *wq,
         * pwqs accordingly.
         */
        get_online_cpus();
-
        mutex_lock(&wq_pool_mutex);
+
        ctx = apply_wqattrs_prepare(wq, attrs);
-       mutex_unlock(&wq_pool_mutex);
 
        /* the ctx has been prepared successfully, let's commit it */
        if (ctx) {
@@ -3642,6 +3654,7 @@ int apply_workqueue_attrs(struct workqueue_struct *wq,
                ret = 0;
        }
 
+       mutex_unlock(&wq_pool_mutex);
        put_online_cpus();
 
        apply_wqattrs_cleanup(ctx);