vfs: make inode_sb_list_lock per sb

When doing fs_mark tests I was noticing other things getting starved out of
doing operations while the fs was unmounting.  This is because we protect all
super_block's s_inodes list with a global lock, which is kind of a bummer.
There doesn't seem to be any reason we do this so make it a per-sb lock.  This
makes sure that we don't add latency to anybody trying to add/remove inodes from
the per sb list while somebody else is unmounting or evicting inodes.  Thanks,

Signed-off-by: Josef Bacik <jbacik@fb.com>
(cherry picked from commit e4a58b71ff981b098ac3371f4d573dc6a90006ce)

diff --git a/fs/block_dev.c b/fs/block_dev.c
index 1d9c9f3754f860b2f9ca19a61d1e89d3eaa9c47c..8aa739144a76c4f2f61f8a7d249d5b1bd402f0a9 100644 (file)
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -1724,7 +1724,7 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
 {
        struct inode *inode, *old_inode = NULL;
 
-       spin_lock(&inode_sb_list_lock);
+       spin_lock(&blockdev_superblock->s_inodes_list_lock);
        list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {
                struct address_space *mapping = inode->i_mapping;
 
@@ -1736,13 +1736,13 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
                }
                __iget(inode);
                spin_unlock(&inode->i_lock);
-               spin_unlock(&inode_sb_list_lock);
+               spin_unlock(&blockdev_superblock->s_inodes_list_lock);
                /*
                 * We hold a reference to 'inode' so it couldn't have been
                 * removed from s_inodes list while we dropped the
-                * inode_sb_list_lock.  We cannot iput the inode now as we can
+                * sb->s_inodes_list_lock.  We cannot iput the inode now as we can
                 * be holding the last reference and we cannot iput it under
-                * inode_sb_list_lock. So we keep the reference and iput it
+                * sb->s_inodes_list_lock. So we keep the reference and iput it
                 * later.
                 */
                iput(old_inode);
@@ -1750,8 +1750,8 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
 
                func(I_BDEV(inode), arg);
 
-               spin_lock(&inode_sb_list_lock);
+               spin_lock(&blockdev_superblock->s_inodes_list_lock);
        }
-       spin_unlock(&inode_sb_list_lock);
+       spin_unlock(&blockdev_superblock->s_inodes_list_lock);
        iput(old_inode);
 }

diff --git a/fs/drop_caches.c b/fs/drop_caches.c
index 1de7294aad208db5fb5b749ee7fc4c5a7d048f4e..15a224fa3b3c082ba4d39835ee03b666094dfc56 100644 (file)
--- a/fs/drop_caches.c
+++ b/fs/drop_caches.c
@@ -17,7 +17,7 @@ static void drop_pagecache_sb(struct super_block *sb, void *unused)
 {
        struct inode *inode, *toput_inode = NULL;
 
-       spin_lock(&inode_sb_list_lock);
+       spin_lock(&sb->s_inodes_list_lock);
        list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
                spin_lock(&inode->i_lock);
                if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) ||
@@ -27,13 +27,13 @@ static void drop_pagecache_sb(struct super_block *sb, void *unused)
                }
                __iget(inode);
                spin_unlock(&inode->i_lock);
-               spin_unlock(&inode_sb_list_lock);
+               spin_unlock(&sb->s_inodes_list_lock);
                invalidate_mapping_pages(inode->i_mapping, 0, -1);
                iput(toput_inode);
                toput_inode = inode;
-               spin_lock(&inode_sb_list_lock);
+               spin_lock(&sb->s_inodes_list_lock);
        }
-       spin_unlock(&inode_sb_list_lock);
+       spin_unlock(&sb->s_inodes_list_lock);
        iput(toput_inode);
 }
 

diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index 2d609a5fbfea96e6ebe8dc65ebf64cd64dca0c8d..6731c2cf09dd110ae7bf7f2c94aff31ce9e06688 100644 (file)
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -1249,7 +1249,7 @@ static void wait_sb_inodes(struct super_block *sb)
         */
        WARN_ON(!rwsem_is_locked(&sb->s_umount));
 
-       spin_lock(&inode_sb_list_lock);
+       spin_lock(&sb->s_inodes_list_lock);
 
        /*
         * Data integrity sync. Must wait for all pages under writeback,
@@ -1269,14 +1269,14 @@ static void wait_sb_inodes(struct super_block *sb)
                }
                __iget(inode);
                spin_unlock(&inode->i_lock);
-               spin_unlock(&inode_sb_list_lock);
+               spin_unlock(&sb->s_inodes_list_lock);
 
                /*
                 * We hold a reference to 'inode' so it couldn't have been
                 * removed from s_inodes list while we dropped the
-                * inode_sb_list_lock.  We cannot iput the inode now as we can
+                * sb->s_inodes_list_lock.  We cannot iput the inode now as we can
                 * be holding the last reference and we cannot iput it under
-                * inode_sb_list_lock. So we keep the reference and iput it
+                * sb->s_inodes_list_lock. So we keep the reference and iput it
                 * later.
                 */
                iput(old_inode);
@@ -1286,9 +1286,9 @@ static void wait_sb_inodes(struct super_block *sb)
 
                cond_resched();
 
-               spin_lock(&inode_sb_list_lock);
+               spin_lock(&sb->s_inodes_list_lock);
        }
-       spin_unlock(&inode_sb_list_lock);
+       spin_unlock(&sb->s_inodes_list_lock);
        iput(old_inode);
 }
 

diff --git a/fs/inode.c b/fs/inode.c
index eea75db8e06d83ccf2125e0e811854803a3cda99..30ee8d0e017716ae1553a4ae7939d5e6881bdc9b 100644 (file)
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -27,7 +27,7 @@
  *   inode->i_state, inode->i_hash, __iget()
  * Inode LRU list locks protect:
  *   inode->i_sb->s_inode_lru, inode->i_lru
- * inode_sb_list_lock protects:
+ * sb->s_inodes_list_lock protects:
  *   sb->s_inodes, inode->i_sb_list
  * bdi->wb.list_lock protects:
  *   bdi->wb.b_{dirty,io,more_io}, inode->i_wb_list
@@ -36,7 +36,7 @@
  *
  * Lock ordering:
  *
- * inode_sb_list_lock
+ * sb->s_inodes_list_lock
  *   inode->i_lock
  *     Inode LRU list locks
  *
@@ -44,7 +44,7 @@
  *   inode->i_lock
  *
  * inode_hash_lock
- *   inode_sb_list_lock
+ *   sb->s_inodes_list_lock
  *   inode->i_lock
  *
  * iunique_lock
@@ -56,8 +56,6 @@ static unsigned int i_hash_shift __read_mostly;
 static struct hlist_head *inode_hashtable __read_mostly;
 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
 
-__cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_sb_list_lock);
-
 /*
  * Empty aops. Can be used for the cases where the user does not
  * define any of the address_space operations.
@@ -433,18 +431,18 @@ static void inode_lru_list_del(struct inode *inode)
  */
 void inode_sb_list_add(struct inode *inode)
 {
-       spin_lock(&inode_sb_list_lock);
+       spin_lock(&inode->i_sb->s_inodes_list_lock);
        list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
-       spin_unlock(&inode_sb_list_lock);
+       spin_unlock(&inode->i_sb->s_inodes_list_lock);
 }
 EXPORT_SYMBOL_GPL(inode_sb_list_add);
 
 static inline void inode_sb_list_del(struct inode *inode)
 {
        if (!list_empty(&inode->i_sb_list)) {
-               spin_lock(&inode_sb_list_lock);
+               spin_lock(&inode->i_sb->s_inodes_list_lock);
                list_del_init(&inode->i_sb_list);
-               spin_unlock(&inode_sb_list_lock);
+               spin_unlock(&inode->i_sb->s_inodes_list_lock);
        }
 }
 
@@ -602,7 +600,7 @@ void evict_inodes(struct super_block *sb)
        struct inode *inode, *next;
        LIST_HEAD(dispose);
 
-       spin_lock(&inode_sb_list_lock);
+       spin_lock(&sb->s_inodes_list_lock);
        list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
                if (atomic_read(&inode->i_count))
                        continue;
@@ -617,9 +615,9 @@ void evict_inodes(struct super_block *sb)
                inode_lru_list_del(inode);
                spin_unlock(&inode->i_lock);
                list_add(&inode->i_lru, &dispose);
-               cond_resched_lock(&inode_sb_list_lock);
+               cond_resched_lock(&sb->s_inodes_list_lock);
        }
-       spin_unlock(&inode_sb_list_lock);
+       spin_unlock(&sb->s_inodes_list_lock);
 
        dispose_list(&dispose);
 }
@@ -640,7 +638,7 @@ int invalidate_inodes(struct super_block *sb, bool kill_dirty)
        struct inode *inode, *next;
        LIST_HEAD(dispose);
 
-       spin_lock(&inode_sb_list_lock);
+       spin_lock(&sb->s_inodes_list_lock);
        list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
                spin_lock(&inode->i_lock);
                if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
@@ -663,7 +661,7 @@ int invalidate_inodes(struct super_block *sb, bool kill_dirty)
                spin_unlock(&inode->i_lock);
                list_add(&inode->i_lru, &dispose);
        }
-       spin_unlock(&inode_sb_list_lock);
+       spin_unlock(&sb->s_inodes_list_lock);
 
        dispose_list(&dispose);
 
@@ -896,7 +894,7 @@ struct inode *new_inode(struct super_block *sb)
 {
        struct inode *inode;
 
-       spin_lock_prefetch(&inode_sb_list_lock);
+       spin_lock_prefetch(&sb->s_inodes_list_lock);
 
        inode = new_inode_pseudo(sb);
        if (inode)

diff --git a/fs/internal.h b/fs/internal.h
index 757ba2abf21e6c205867960d5fa2ee3b2f77d46f..a23e5df67ce00031d736467c767120ccee506a1e 100644 (file)
--- a/fs/internal.h
+++ b/fs/internal.h
@@ -110,7 +110,6 @@ extern int open_check_o_direct(struct file *f);
 /*
  * inode.c
  */
-extern spinlock_t inode_sb_list_lock;
 extern long prune_icache_sb(struct super_block *sb, unsigned long nr_to_scan,
                            int nid);
 extern void inode_add_lru(struct inode *inode);

diff --git a/fs/notify/inode_mark.c b/fs/notify/inode_mark.c
index dfbf5447eea4cea8fdf664ff5b0232f8a61e68b0..3f15479b72ee34f2ece0db4ff988dcc0b2bb5804 100644 (file)
--- a/fs/notify/inode_mark.c
+++ b/fs/notify/inode_mark.c
@@ -240,17 +240,17 @@ out:
 
 /**
  * fsnotify_unmount_inodes - an sb is unmounting.  handle any watched inodes.
- * @list: list of inodes being unmounted (sb->s_inodes)
+ * @sb: the sb being unmounted.
  *
  * Called during unmount with no locks held, so needs to be safe against
- * concurrent modifiers. We temporarily drop inode_sb_list_lock and CAN block.
+ * concurrent modifiers. We temporarily drop sb->s_inodes_list_lock and CAN block.
  */
-void fsnotify_unmount_inodes(struct list_head *list)
+void fsnotify_unmount_inodes(struct super_block *sb)
 {
        struct inode *inode, *next_i, *need_iput = NULL;
 
-       spin_lock(&inode_sb_list_lock);
-       list_for_each_entry_safe(inode, next_i, list, i_sb_list) {
+       spin_lock(&sb->s_inodes_list_lock);
+       list_for_each_entry_safe(inode, next_i, &sb->s_inodes, i_sb_list) {
                struct inode *need_iput_tmp;
 
                /*
@@ -286,7 +286,7 @@ void fsnotify_unmount_inodes(struct list_head *list)
                spin_unlock(&inode->i_lock);
 
                /* In case the dropping of a reference would nuke next_i. */
-               while (&next_i->i_sb_list != list) {
+               while (&next_i->i_sb_list != &sb->s_inodes) {
                        spin_lock(&next_i->i_lock);
                        if (!(next_i->i_state & (I_FREEING | I_WILL_FREE)) &&
                                                atomic_read(&next_i->i_count)) {
@@ -301,12 +301,12 @@ void fsnotify_unmount_inodes(struct list_head *list)
                }
 
                /*
-                * We can safely drop inode_sb_list_lock here because either
+                * We can safely drop sb->s_inodes_list_lock here because either
                 * we actually hold references on both inode and next_i or
                 * end of list.  Also no new inodes will be added since the
                 * umount has begun.
                 */
-               spin_unlock(&inode_sb_list_lock);
+               spin_unlock(&sb->s_inodes_list_lock);
 
                if (need_iput_tmp)
                        iput(need_iput_tmp);
@@ -318,7 +318,7 @@ void fsnotify_unmount_inodes(struct list_head *list)
 
                iput(inode);
 
-               spin_lock(&inode_sb_list_lock);
+               spin_lock(&sb->s_inodes_list_lock);
        }
-       spin_unlock(&inode_sb_list_lock);
+       spin_unlock(&sb->s_inodes_list_lock);
 }

diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c
index 6b4527216a7fe2c60ead15d57383648f25d4d3ef..c9be87e0f3bf8432ae93cd8ad4f4895462288d36 100644 (file)
--- a/fs/quota/dquot.c
+++ b/fs/quota/dquot.c
@@ -915,7 +915,7 @@ static void add_dquot_ref(struct super_block *sb, int type)
        int reserved = 0;
 #endif
 
-       spin_lock(&inode_sb_list_lock);
+       spin_lock(&sb->s_inodes_list_lock);
        list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
                spin_lock(&inode->i_lock);
                if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) ||
@@ -926,7 +926,7 @@ static void add_dquot_ref(struct super_block *sb, int type)
                }
                __iget(inode);
                spin_unlock(&inode->i_lock);
-               spin_unlock(&inode_sb_list_lock);
+               spin_unlock(&sb->s_inodes_list_lock);
 
 #ifdef CONFIG_QUOTA_DEBUG
                if (unlikely(inode_get_rsv_space(inode) > 0))
@@ -938,15 +938,15 @@ static void add_dquot_ref(struct super_block *sb, int type)
                /*
                 * We hold a reference to 'inode' so it couldn't have been
                 * removed from s_inodes list while we dropped the
-                * inode_sb_list_lock We cannot iput the inode now as we can be
+                * sb->s_inodes_list_lock We cannot iput the inode now as we can be
                 * holding the last reference and we cannot iput it under
-                * inode_sb_list_lock. So we keep the reference and iput it
+                * sb->s_inodes_list_lock. So we keep the reference and iput it
                 * later.
                 */
                old_inode = inode;
-               spin_lock(&inode_sb_list_lock);
+               spin_lock(&sb->s_inodes_list_lock);
        }
-       spin_unlock(&inode_sb_list_lock);
+       spin_unlock(&sb->s_inodes_list_lock);
        iput(old_inode);
 
 #ifdef CONFIG_QUOTA_DEBUG
@@ -1014,7 +1014,7 @@ static void remove_dquot_ref(struct super_block *sb, int type,
        struct inode *inode;
        int reserved = 0;
 
-       spin_lock(&inode_sb_list_lock);
+       spin_lock(&sb->s_inodes_list_lock);
        list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
                /*
                 *  We have to scan also I_NEW inodes because they can already
@@ -1030,7 +1030,7 @@ static void remove_dquot_ref(struct super_block *sb, int type,
                }
                spin_unlock(&dq_data_lock);
        }
-       spin_unlock(&inode_sb_list_lock);
+       spin_unlock(&sb->s_inodes_list_lock);
 #ifdef CONFIG_QUOTA_DEBUG
        if (reserved) {
                printk(KERN_WARNING "VFS (%s): Writes happened after quota"

diff --git a/fs/super.c b/fs/super.c
index eae088f6aaaeb6e8a3380adad19fb66feebe9905..3b56fba2dba60165ffba7e31295a7ced4fa0e0a4 100644 (file)
--- a/fs/super.c
+++ b/fs/super.c
@@ -190,6 +190,7 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags)
        INIT_HLIST_NODE(&s->s_instances);
        INIT_HLIST_BL_HEAD(&s->s_anon);
        INIT_LIST_HEAD(&s->s_inodes);
+       spin_lock_init(&s->s_inodes_list_lock);
 
        if (list_lru_init(&s->s_dentry_lru))
                goto fail;
@@ -394,7 +395,7 @@ void generic_shutdown_super(struct super_block *sb)
                sync_filesystem(sb);
                sb->s_flags &= ~MS_ACTIVE;
 
-               fsnotify_unmount_inodes(&sb->s_inodes);
+               fsnotify_unmount_inodes(sb);
 
                evict_inodes(sb);
 

diff --git a/include/linux/fs.h b/include/linux/fs.h
index 9ab779e8a63ccd7785637dd9017fc3bc69ff260f..91b498b989e11eab42741ef5ca2ff9c943c26f13 100644 (file)
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -1217,6 +1217,7 @@ struct super_block {
 #endif
        const struct xattr_handler **s_xattr;
 
+       spinlock_t              s_inodes_list_lock ____cacheline_aligned_in_smp;
        struct list_head        s_inodes;       /* all inodes */
        struct hlist_bl_head    s_anon;         /* anonymous dentries for (nfs) exporting */
        struct list_head        s_mounts;       /* list of mounts; _not_ for fs use */

diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h
index ca060d7c4fa63e3395c4754ed8e8f5ab32ceff30..ce596409ffad948a617dfa86e2a7572744d8ed67 100644 (file)
--- a/include/linux/fsnotify_backend.h
+++ b/include/linux/fsnotify_backend.h
@@ -372,7 +372,7 @@ extern void fsnotify_clear_marks_by_group_flags(struct fsnotify_group *group, un
 extern void fsnotify_clear_marks_by_group(struct fsnotify_group *group);
 extern void fsnotify_get_mark(struct fsnotify_mark *mark);
 extern void fsnotify_put_mark(struct fsnotify_mark *mark);
-extern void fsnotify_unmount_inodes(struct list_head *list);
+extern void fsnotify_unmount_inodes(struct super_block *sb);
 
 /* put here because inotify does some weird stuff when destroying watches */
 extern void fsnotify_init_event(struct fsnotify_event *event,
@@ -408,7 +408,7 @@ static inline u32 fsnotify_get_cookie(void)
        return 0;
 }
 
-static inline void fsnotify_unmount_inodes(struct list_head *list)
+static inline void fsnotify_unmount_inodes(struct super_block *sb)
 {}
 
 #endif /* CONFIG_FSNOTIFY */