[PATCH] remove inode.i_wait

Remove i_wait from struct inode and hash it instead.

This is a pure space-saving exercise - 12 bytes from struct
inode on x86.

NFS was using i_wait for its own purposes.  Add a wait_queue_head_t to
the fs-private inode for that.  This change has been acked by Trond.

diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index dae51694ae438ae2fbfa079bdf7ca68f220611fb..c9549228ee97e0b99b16db182ffa72777d821706 100644 (file)
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -176,8 +176,7 @@ static void __sync_single_inode(struct inode *inode, int wait, int *nr_to_write)
                        }
                }
        }
-       if (waitqueue_active(&inode->i_wait))
-               wake_up(&inode->i_wait);
+       wake_up_inode(inode);
 }
 
 /*

diff --git a/fs/inode.c b/fs/inode.c
index 9fca16d59ee89adf9aa6900defddfc320936ca6f..17cc8dfb1ac9f06e2cdc273d1e1496a6a6ba1943 100644 (file)
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -14,6 +14,9 @@
 #include <linux/writeback.h>
 #include <linux/module.h>
 #include <linux/backing-dev.h>
+#include <linux/wait.h>
+#include <linux/hash.h>
+
 /*
  * This is needed for the following functions:
  *  - inode_has_buffers
@@ -149,7 +152,6 @@ static void destroy_inode(struct inode *inode)
 void inode_init_once(struct inode *inode)
 {
        memset(inode, 0, sizeof(*inode));
-       init_waitqueue_head(&inode->i_wait);
        INIT_LIST_HEAD(&inode->i_hash);
        INIT_LIST_HEAD(&inode->i_data.clean_pages);
        INIT_LIST_HEAD(&inode->i_data.dirty_pages);
@@ -176,21 +178,6 @@ static void init_once(void * foo, kmem_cache_t * cachep, unsigned long flags)
                inode_init_once(inode);
 }
 
-void __wait_on_inode(struct inode * inode)
-{
-       DECLARE_WAITQUEUE(wait, current);
-
-       add_wait_queue(&inode->i_wait, &wait);
-repeat:
-       set_current_state(TASK_UNINTERRUPTIBLE);
-       if (inode->i_state & I_LOCK) {
-               schedule();
-               goto repeat;
-       }
-       remove_wait_queue(&inode->i_wait, &wait);
-       current->state = TASK_RUNNING;
-}
-
 /*
  * inode_lock must be held
  */
@@ -538,7 +525,7 @@ void unlock_new_inode(struct inode *inode)
         * that haven't tested I_LOCK).
         */
        inode->i_state &= ~(I_LOCK|I_NEW);
-       wake_up(&inode->i_wait);
+       wake_up_inode(inode);
 }
 
 
@@ -899,59 +886,6 @@ int bmap(struct inode * inode, int block)
        return res;
 }
 
-/*
- * Initialize the hash tables.
- */
-void __init inode_init(unsigned long mempages)
-{
-       struct list_head *head;
-       unsigned long order;
-       unsigned int nr_hash;
-       int i;
-
-       mempages >>= (14 - PAGE_SHIFT);
-       mempages *= sizeof(struct list_head);
-       for (order = 0; ((1UL << order) << PAGE_SHIFT) < mempages; order++)
-               ;
-
-       do {
-               unsigned long tmp;
-
-               nr_hash = (1UL << order) * PAGE_SIZE /
-                       sizeof(struct list_head);
-               i_hash_mask = (nr_hash - 1);
-
-               tmp = nr_hash;
-               i_hash_shift = 0;
-               while ((tmp >>= 1UL) != 0UL)
-                       i_hash_shift++;
-
-               inode_hashtable = (struct list_head *)
-                       __get_free_pages(GFP_ATOMIC, order);
-       } while (inode_hashtable == NULL && --order >= 0);
-
-       printk("Inode-cache hash table entries: %d (order: %ld, %ld bytes)\n",
-                       nr_hash, order, (PAGE_SIZE << order));
-
-       if (!inode_hashtable)
-               panic("Failed to allocate inode hash table\n");
-
-       head = inode_hashtable;
-       i = nr_hash;
-       do {
-               INIT_LIST_HEAD(head);
-               head++;
-               i--;
-       } while (i);
-
-       /* inode slab cache */
-       inode_cachep = kmem_cache_create("inode_cache", sizeof(struct inode),
-                                        0, SLAB_HWCACHE_ALIGN, init_once,
-                                        NULL);
-       if (!inode_cachep)
-               panic("cannot create inode slab cache");
-}
-
 static inline void do_atime_update(struct inode *inode)
 {
        unsigned long time = CURRENT_TIME;
@@ -1044,3 +978,104 @@ void remove_dquot_ref(struct super_block *sb, int type)
 }
 
 #endif
+
+/*
+ * Hashed waitqueues for wait_on_inode().  The table is pretty small - the
+ * kernel doesn't lock many inodes at the same time.
+ */
+#define I_WAIT_TABLE_ORDER     3
+static struct i_wait_queue_head {
+       wait_queue_head_t wqh;
+} ____cacheline_aligned_in_smp i_wait_queue_heads[1<<I_WAIT_TABLE_ORDER];
+
+/*
+ * Return the address of the waitqueue_head to be used for this inode
+ */
+static wait_queue_head_t *i_waitq_head(struct inode *inode)
+{
+       return &i_wait_queue_heads[hash_ptr(inode, I_WAIT_TABLE_ORDER)].wqh;
+}
+
+void __wait_on_inode(struct inode *inode)
+{
+       DECLARE_WAITQUEUE(wait, current);
+       wait_queue_head_t *wq = i_waitq_head(inode);
+
+       add_wait_queue(wq, &wait);
+repeat:
+       set_current_state(TASK_UNINTERRUPTIBLE);
+       if (inode->i_state & I_LOCK) {
+               schedule();
+               goto repeat;
+       }
+       remove_wait_queue(wq, &wait);
+       current->state = TASK_RUNNING;
+}
+
+void wake_up_inode(struct inode *inode)
+{
+       wait_queue_head_t *wq = i_waitq_head(inode);
+
+       /*
+        * Prevent speculative execution through spin_unlock(&inode_lock);
+        */
+       smp_mb();
+       if (waitqueue_active(wq))
+               wake_up_all(wq);
+}
+
+/*
+ * Initialize the waitqueues and inode hash table.
+ */
+void __init inode_init(unsigned long mempages)
+{
+       struct list_head *head;
+       unsigned long order;
+       unsigned int nr_hash;
+       int i;
+
+       for (i = 0; i < ARRAY_SIZE(i_wait_queue_heads); i++)
+               init_waitqueue_head(&i_wait_queue_heads[i].wqh);
+
+       mempages >>= (14 - PAGE_SHIFT);
+       mempages *= sizeof(struct list_head);
+       for (order = 0; ((1UL << order) << PAGE_SHIFT) < mempages; order++)
+               ;
+
+       do {
+               unsigned long tmp;
+
+               nr_hash = (1UL << order) * PAGE_SIZE /
+                       sizeof(struct list_head);
+               i_hash_mask = (nr_hash - 1);
+
+               tmp = nr_hash;
+               i_hash_shift = 0;
+               while ((tmp >>= 1UL) != 0UL)
+                       i_hash_shift++;
+
+               inode_hashtable = (struct list_head *)
+                       __get_free_pages(GFP_ATOMIC, order);
+       } while (inode_hashtable == NULL && --order >= 0);
+
+       printk("Inode-cache hash table entries: %d (order: %ld, %ld bytes)\n",
+                       nr_hash, order, (PAGE_SIZE << order));
+
+       if (!inode_hashtable)
+               panic("Failed to allocate inode hash table\n");
+
+       head = inode_hashtable;
+       i = nr_hash;
+       do {
+               INIT_LIST_HEAD(head);
+               head++;
+               i--;
+       } while (i);
+
+       /* inode slab cache */
+       inode_cachep = kmem_cache_create("inode_cache", sizeof(struct inode),
+                                        0, SLAB_HWCACHE_ALIGN, init_once,
+                                        NULL);
+       if (!inode_cachep)
+               panic("cannot create inode slab cache");
+}

diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index 6f0e6fc0340f2d35e370bcfe99b6fa6a2526e5f9..9d61c8b75258e7dc7ba5a763f5d71c4bf75dcad8 100644 (file)
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -799,11 +799,14 @@ int
 nfs_wait_on_inode(struct inode *inode, int flag)
 {
        struct rpc_clnt *clnt = NFS_CLIENT(inode);
+       struct nfs_inode *nfsi = NFS_I(inode);
+
        int error;
        if (!(NFS_FLAGS(inode) & flag))
                return 0;
        atomic_inc(&inode->i_count);
-       error = nfs_wait_event(clnt, inode->i_wait, !(NFS_FLAGS(inode) & flag));
+       error = nfs_wait_event(clnt, nfsi->nfs_i_wait,
+                               !(NFS_FLAGS(inode) & flag));
        iput(inode);
        return error;
 }
@@ -922,7 +925,7 @@ __nfs_revalidate_inode(struct nfs_server *server, struct inode *inode)
        NFS_FLAGS(inode) &= ~NFS_INO_STALE;
 out:
        NFS_FLAGS(inode) &= ~NFS_INO_REVALIDATING;
-       wake_up(&inode->i_wait);
+       wake_up(&NFS_I(inode)->nfs_i_wait);
  out_nowait:
        unlock_kernel();
        return status;
@@ -1258,6 +1261,7 @@ static void init_once(void * foo, kmem_cache_t * cachep, unsigned long flags)
                nfsi->ndirty = 0;
                nfsi->ncommit = 0;
                nfsi->npages = 0;
+               init_waitqueue_head(&nfsi->nfs_i_wait);
        }
 }
  

diff --git a/include/linux/fs.h b/include/linux/fs.h
index 31a691491a9fd0142006bfb02ea9f8c1b31cea4e..1b01db32cc355dbdf98e850906f8392c2310cad9 100644 (file)
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -383,7 +383,6 @@ struct inode {
        struct inode_operations *i_op;
        struct file_operations  *i_fop; /* former ->i_op->default_file_ops */
        struct super_block      *i_sb;
-       wait_queue_head_t       i_wait;
        struct file_lock        *i_flock;
        struct address_space    *i_mapping;
        struct address_space    i_data;

diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
index 027986b2f3364434060324f066a9f97b663acae1..64c3b5ef2d8808c722473a96f831fe531217dda6 100644 (file)
--- a/include/linux/nfs_fs.h
+++ b/include/linux/nfs_fs.h
@@ -13,6 +13,7 @@
 #include <linux/in.h>
 #include <linux/mm.h>
 #include <linux/pagemap.h>
+#include <linux/wait.h>
 
 #include <linux/nfs_fs_sb.h>
 
@@ -159,6 +160,8 @@ struct nfs_inode {
        /* Credentials for shared mmap */
        struct rpc_cred         *mm_cred;
 
+       wait_queue_head_t       nfs_i_wait;
+
        struct inode            vfs_inode;
 };
 

diff --git a/include/linux/writeback.h b/include/linux/writeback.h
index 5f035892d26f7fa923f05fca11f06c2cf08b2fa5..cf706c783eda71e22880beeb9d330ad49b180379 100644 (file)
--- a/include/linux/writeback.h
+++ b/include/linux/writeback.h
@@ -31,6 +31,7 @@ static inline int current_is_pdflush(void)
 
 void writeback_unlocked_inodes(int *nr_to_write, int sync_mode,
                                unsigned long *older_than_this);
+void wake_up_inode(struct inode *inode);
 void __wait_on_inode(struct inode * inode);
 void sync_inodes_sb(struct super_block *, int wait);
 void sync_inodes(int wait);