[PATCH] use spinlocking in the ext2 block allocator

From Alex Tomas and myself

ext2 currently uses lock_super() to protect the filesystem's in-core block
allocation bitmaps.

On big SMP machines the contention on that semaphore is causing high context
switch rates, large amounts of idle time and reduced throughput.

The context switch rate can also worsen block allocation: if several tasks
are trying to allocate blocks inside the same blockgroup for different files,
madly rotating between those tasks will cause the files' blocks to be
intermingled.

On SDET and dbench-style worloads (lots of tasks doing lots of allocation)
this patch (and a similar one for the inode allocator) improve throughout on
an 8-way by ~15%.  On 16-way NUMAQ the speedup is 150%.

What wedo isto remove the lock altogether and just rely on the atomic
semantics of test_and_set_bit(): if the allocator sees a block was free it
runs test_and_set_bit().  If that fails, then we raced and the allocator will
go and look for another block.

Of course, we don't really use test_and_set_bit() because that
isn'tendian-dependent.  New atomic endian-independent functions are
introduced: ext2_set_bit_atomic() and ext2_clear_bit_atomic().  We do not
need ext2_test_bit_atomic(), since even if ext2_test_bit() returns the wrong
result, that error will be detected and naturally handled in the subsequent
ext2_set_bit_atomic().

For little-endian machines the new atomic ops map directly onto the
test_and_set_bit(), etc.

For big-endian machines we provide the architecture's impementation with the
address of a spinlock whcih can be taken around the nonatomic ext2_set_bit().
 The spinlocks are hashed, and the hash is scaled according to the machine
size.  Architectures are free to implement optimised versions of
ext2_set_bit_atomic() and ext2_clear_bit_atomic().

diff --git a/fs/ext2/balloc.c b/fs/ext2/balloc.c
index 40868d403f0b133e7479ee8c5d7a223105142a8a..b63ebc25a245e6f767d964d71ad35ea905ba676a 100644 (file)
--- a/fs/ext2/balloc.c
+++ b/fs/ext2/balloc.c
@@ -94,12 +94,19 @@ error_out:
        return bh;
 }
 
-static inline int reserve_blocks(struct super_block *sb, int count)
+/*
+ * Set sb->s_dirt here because the superblock was "logically" altered.  We
+ * need to recalculate its free blocks count and flush it out.
+ */
+static int reserve_blocks(struct super_block *sb, int count)
 {
-       struct ext2_sb_info * sbi = EXT2_SB(sb);
-       struct ext2_super_block * es = sbi->s_es;
-       unsigned free_blocks = le32_to_cpu(es->s_free_blocks_count);
-       unsigned root_blocks = le32_to_cpu(es->s_r_blocks_count);
+       struct ext2_sb_info *sbi = EXT2_SB(sb);
+       struct ext2_super_block *es = sbi->s_es;
+       unsigned free_blocks;
+       unsigned root_blocks;
+
+       free_blocks = percpu_counter_read_positive(&sbi->s_freeblocks_counter);
+       root_blocks = le32_to_cpu(es->s_r_blocks_count);
 
        if (free_blocks < count)
                count = free_blocks;
@@ -117,46 +124,51 @@ static inline int reserve_blocks(struct super_block *sb, int count)
                        return 0;
        }
 
-       es->s_free_blocks_count = cpu_to_le32(free_blocks - count);
-       mark_buffer_dirty(sbi->s_sbh);
+       percpu_counter_mod(&sbi->s_freeblocks_counter, -count);
        sb->s_dirt = 1;
        return count;
 }
 
-static inline void release_blocks(struct super_block *sb, int count)
+static void release_blocks(struct super_block *sb, int count)
 {
        if (count) {
-               struct ext2_sb_info * sbi = EXT2_SB(sb);
-               struct ext2_super_block * es = sbi->s_es;
-               unsigned free_blocks = le32_to_cpu(es->s_free_blocks_count);
-               es->s_free_blocks_count = cpu_to_le32(free_blocks + count);
-               mark_buffer_dirty(sbi->s_sbh);
+               struct ext2_sb_info *sbi = EXT2_SB(sb);
+
+               percpu_counter_mod(&sbi->s_freeblocks_counter, count);
                sb->s_dirt = 1;
        }
 }
 
-static inline int group_reserve_blocks(struct ext2_group_desc *desc,
-                                   struct buffer_head *bh, int count)
+static int group_reserve_blocks(struct ext2_sb_info *sbi, int group_no,
+       struct ext2_group_desc *desc, struct buffer_head *bh, int count)
 {
        unsigned free_blocks;
 
        if (!desc->bg_free_blocks_count)
                return 0;
 
+       spin_lock(sb_bgl_lock(sbi, group_no));
        free_blocks = le16_to_cpu(desc->bg_free_blocks_count);
        if (free_blocks < count)
                count = free_blocks;
        desc->bg_free_blocks_count = cpu_to_le16(free_blocks - count);
+       spin_unlock(sb_bgl_lock(sbi, group_no));
        mark_buffer_dirty(bh);
        return count;
 }
 
-static inline void group_release_blocks(struct ext2_group_desc *desc,
-                                   struct buffer_head *bh, int count)
+static void group_release_blocks(struct super_block *sb, int group_no,
+       struct ext2_group_desc *desc, struct buffer_head *bh, int count)
 {
        if (count) {
-               unsigned free_blocks = le16_to_cpu(desc->bg_free_blocks_count);
+               struct ext2_sb_info *sbi = EXT2_SB(sb);
+               unsigned free_blocks;
+
+               spin_lock(sb_bgl_lock(sbi, group_no));
+               free_blocks = le16_to_cpu(desc->bg_free_blocks_count);
                desc->bg_free_blocks_count = cpu_to_le16(free_blocks + count);
+               spin_unlock(sb_bgl_lock(sbi, group_no));
+               sb->s_dirt = 1;
                mark_buffer_dirty(bh);
        }
 }
@@ -172,12 +184,11 @@ void ext2_free_blocks (struct inode * inode, unsigned long block,
        unsigned long i;
        unsigned long overflow;
        struct super_block * sb = inode->i_sb;
+       struct ext2_sb_info * sbi = EXT2_SB(sb);
        struct ext2_group_desc * desc;
-       struct ext2_super_block * es;
+       struct ext2_super_block * es = sbi->s_es;
        unsigned freed = 0, group_freed;
 
-       lock_super (sb);
-       es = EXT2_SB(sb)->s_es;
        if (block < le32_to_cpu(es->s_first_data_block) ||
            block + count < block ||
            block + count > le32_to_cpu(es->s_blocks_count)) {
@@ -215,16 +226,17 @@ do_more:
        if (in_range (le32_to_cpu(desc->bg_block_bitmap), block, count) ||
            in_range (le32_to_cpu(desc->bg_inode_bitmap), block, count) ||
            in_range (block, le32_to_cpu(desc->bg_inode_table),
-                     EXT2_SB(sb)->s_itb_per_group) ||
+                     sbi->s_itb_per_group) ||
            in_range (block + count - 1, le32_to_cpu(desc->bg_inode_table),
-                     EXT2_SB(sb)->s_itb_per_group))
+                     sbi->s_itb_per_group))
                ext2_error (sb, "ext2_free_blocks",
                            "Freeing blocks in system zones - "
                            "Block = %lu, count = %lu",
                            block, count);
 
        for (i = 0, group_freed = 0; i < count; i++) {
-               if (!ext2_clear_bit(bit + i, bitmap_bh->b_data))
+               if (!ext2_clear_bit_atomic(sb_bgl_lock(sbi, block_group),
+                                       bit + i, (void *) bitmap_bh->b_data))
                        ext2_error (sb, "ext2_free_blocks",
                                      "bit already cleared for block %lu",
                                      block + i);
@@ -236,7 +248,7 @@ do_more:
        if (sb->s_flags & MS_SYNCHRONOUS)
                sync_dirty_buffer(bitmap_bh);
 
-       group_release_blocks(desc, bh2, group_freed);
+       group_release_blocks(sb, block_group, desc, bh2, group_freed);
        freed += group_freed;
 
        if (overflow) {
@@ -247,17 +259,18 @@ do_more:
 error_return:
        brelse(bitmap_bh);
        release_blocks(sb, freed);
-       unlock_super (sb);
        DQUOT_FREE_BLOCK(inode, freed);
 }
 
-static int grab_block(char *map, unsigned size, int goal)
+static int grab_block(spinlock_t *lock, char *map, unsigned size, int goal)
 {
        int k;
        char *p, *r;
 
        if (!ext2_test_bit(goal, map))
                goto got_it;
+
+repeat:
        if (goal) {
                /*
                 * The goal was occupied; search forward for a free 
@@ -297,7 +310,8 @@ static int grab_block(char *map, unsigned size, int goal)
        }
        return -1;
 got_it:
-       ext2_set_bit(goal, map);
+       if (ext2_set_bit_atomic(lock, goal, (void *) map)) 
+               goto repeat;    
        return goal;
 }
 
@@ -309,15 +323,15 @@ got_it:
  * bitmap, and then for any free bit if that fails.
  * This function also updates quota and i_blocks field.
  */
-int ext2_new_block (struct inode * inode, unsigned long goal,
-    u32 * prealloc_count, u32 * prealloc_block, int * err)
+int ext2_new_block(struct inode *inode, unsigned long goal,
+                       u32 *prealloc_count, u32 *prealloc_block, int *err)
 {
        struct buffer_head *bitmap_bh = NULL;
        struct buffer_head *gdp_bh;     /* bh2 */
        struct ext2_group_desc *desc;
        int group_no;                   /* i */
        int ret_block;                  /* j */
-       int bit;                /* k */
+       int bit;                        /* k */
        int target_block;               /* tmp */
        int block = 0;
        struct super_block *sb = inode->i_sb;
@@ -341,13 +355,10 @@ int ext2_new_block (struct inode * inode, unsigned long goal,
                prealloc_goal--;
 
        dq_alloc = prealloc_goal + 1;
-
-       lock_super (sb);
-
        es_alloc = reserve_blocks(sb, dq_alloc);
        if (!es_alloc) {
                *err = -ENOSPC;
-               goto out_unlock;
+               goto out_dquot;
        }
 
        ext2_debug ("goal=%lu.\n", goal);
@@ -357,10 +368,16 @@ int ext2_new_block (struct inode * inode, unsigned long goal,
                goal = le32_to_cpu(es->s_first_data_block);
        group_no = (goal - le32_to_cpu(es->s_first_data_block)) / group_size;
        desc = ext2_get_group_desc (sb, group_no, &gdp_bh);
-       if (!desc)
+       if (!desc) {
+               /*
+                * gdp_bh may still be uninitialised.  But group_release_blocks
+                * will not touch it because group_alloc is zero.
+                */
                goto io_error;
+       }
 
-       group_alloc = group_reserve_blocks(desc, gdp_bh, es_alloc);
+       group_alloc = group_reserve_blocks(sbi, group_no, desc,
+                                       gdp_bh, es_alloc);
        if (group_alloc) {
                ret_block = ((goal - le32_to_cpu(es->s_first_data_block)) %
                                        group_size);
@@ -371,11 +388,11 @@ int ext2_new_block (struct inode * inode, unsigned long goal,
                
                ext2_debug("goal is at %d:%d.\n", group_no, ret_block);
 
-               ret_block = grab_block(bitmap_bh->b_data,
-                               group_size, ret_block);
+               ret_block = grab_block(sb_bgl_lock(sbi, group_no),
+                               bitmap_bh->b_data, group_size, ret_block);
                if (ret_block >= 0)
                        goto got_block;
-               group_release_blocks(desc, gdp_bh, group_alloc);
+               group_release_blocks(sb, group_no, desc, gdp_bh, group_alloc);
                group_alloc = 0;
        }
 
@@ -393,7 +410,8 @@ int ext2_new_block (struct inode * inode, unsigned long goal,
                desc = ext2_get_group_desc(sb, group_no, &gdp_bh);
                if (!desc)
                        goto io_error;
-               group_alloc = group_reserve_blocks(desc, gdp_bh, es_alloc);
+               group_alloc = group_reserve_blocks(sbi, group_no, desc,
+                                               gdp_bh, es_alloc);
        }
        if (!group_alloc) {
                *err = -ENOSPC;
@@ -404,7 +422,8 @@ int ext2_new_block (struct inode * inode, unsigned long goal,
        if (!bitmap_bh)
                goto io_error;
 
-       ret_block = grab_block(bitmap_bh->b_data, group_size, 0);
+       ret_block = grab_block(sb_bgl_lock(sbi, group_no), bitmap_bh->b_data,
+                               group_size, 0);
        if (ret_block < 0) {
                ext2_error (sb, "ext2_new_block",
                        "Free blocks count corrupted for block group %d",
@@ -452,7 +471,9 @@ got_block:
                unsigned n;
 
                for (n = 0; n < group_alloc && ++ret_block < group_size; n++) {
-                       if (ext2_set_bit(ret_block, bitmap_bh->b_data))
+                       if (ext2_set_bit_atomic(sb_bgl_lock(sbi, group_no),
+                                               ret_block,
+                                               (void*) bitmap_bh->b_data))
                                break;
                }
                *prealloc_block = block + 1;
@@ -471,10 +492,9 @@ got_block:
 
        *err = 0;
 out_release:
-       group_release_blocks(desc, gdp_bh, group_alloc);
+       group_release_blocks(sb, group_no, desc, gdp_bh, group_alloc);
        release_blocks(sb, es_alloc);
-out_unlock:
-       unlock_super (sb);
+out_dquot:
        DQUOT_FREE_BLOCK(inode, dq_alloc);
 out:
        brelse(bitmap_bh);
@@ -487,11 +507,11 @@ io_error:
 
 unsigned long ext2_count_free_blocks (struct super_block * sb)
 {
-#ifdef EXT2FS_DEBUG
-       struct ext2_super_block * es;
-       unsigned long desc_count, bitmap_count, x;
        struct ext2_group_desc * desc;
+       unsigned long desc_count = 0;
        int i;
+#ifdef EXT2FS_DEBUG
+       unsigned long bitmap_count, x;
        
        lock_super (sb);
        es = EXT2_SB(sb)->s_es;
@@ -519,13 +539,18 @@ unsigned long ext2_count_free_blocks (struct super_block * sb)
        unlock_super (sb);
        return bitmap_count;
 #else
-       return le32_to_cpu(EXT2_SB(sb)->s_es->s_free_blocks_count);
+        for (i = 0; i < EXT2_SB(sb)->s_groups_count; i++) {
+                desc = ext2_get_group_desc (sb, i, NULL);
+                if (!desc)
+                        continue;
+                desc_count += le16_to_cpu(desc->bg_free_blocks_count);
+       }
+       return desc_count;
 #endif
 }
 
-static inline int block_in_use (unsigned long block,
-                               struct super_block * sb,
-                               unsigned char * map)
+static inline int
+block_in_use(unsigned long block, struct super_block *sb, unsigned char *map)
 {
        return ext2_test_bit ((block - le32_to_cpu(EXT2_SB(sb)->s_es->s_first_data_block)) %
                         EXT2_BLOCKS_PER_GROUP(sb), map);

diff --git a/fs/ext2/ialloc.c b/fs/ext2/ialloc.c
index 87b2d99f4a7c778d74bac5692fcf64faf5be6852..642b0b8cad177a148a75818d75588e9f3b89d98e 100644 (file)
--- a/fs/ext2/ialloc.c
+++ b/fs/ext2/ialloc.c
@@ -278,7 +278,8 @@ static int find_group_orlov(struct super_block *sb, struct inode *parent)
        int ngroups = sbi->s_groups_count;
        int inodes_per_group = EXT2_INODES_PER_GROUP(sb);
        int avefreei = le32_to_cpu(es->s_free_inodes_count) / ngroups;
-       int avefreeb = le32_to_cpu(es->s_free_blocks_count) / ngroups;
+       int free_blocks = percpu_counter_read(&sbi->s_freeblocks_counter);
+       int avefreeb = free_blocks / ngroups;
        int blocks_per_dir;
        int ndirs = sbi->s_dir_count;
        int max_debt, max_dirs, min_blocks, min_inodes;
@@ -320,8 +321,7 @@ static int find_group_orlov(struct super_block *sb, struct inode *parent)
                goto fallback;
        }
 
-       blocks_per_dir = (le32_to_cpu(es->s_blocks_count) -
-                         le32_to_cpu(es->s_free_blocks_count)) / ndirs;
+       blocks_per_dir = (le32_to_cpu(es->s_blocks_count) - free_blocks) / ndirs;
 
        max_dirs = ndirs / ngroups + inodes_per_group / 16;
        min_inodes = avefreei - inodes_per_group / 4;
@@ -500,6 +500,7 @@ repeat:
        es->s_free_inodes_count =
                cpu_to_le32(le32_to_cpu(es->s_free_inodes_count) - 1);
 
+       spin_lock(sb_bgl_lock(EXT2_SB(sb), group));
        if (S_ISDIR(mode)) {
                if (EXT2_SB(sb)->s_debts[group] < 255)
                        EXT2_SB(sb)->s_debts[group]++;
@@ -507,6 +508,7 @@ repeat:
                if (EXT2_SB(sb)->s_debts[group])
                        EXT2_SB(sb)->s_debts[group]--;
        }
+       spin_unlock(sb_bgl_lock(EXT2_SB(sb), group));
 
        mark_buffer_dirty(EXT2_SB(sb)->s_sbh);
        sb->s_dirt = 1;

diff --git a/fs/ext2/super.c b/fs/ext2/super.c
index 7265ad05c1f750f9f2b54b645e0bf9aba25fdf77..f066727983f967eb7b6188f22474d06e2c7244df 100644 (file)
--- a/fs/ext2/super.c
+++ b/fs/ext2/super.c
@@ -769,6 +769,8 @@ static int ext2_fill_super(struct super_block *sb, void *data, int silent)
                printk ("EXT2-fs: not enough memory\n");
                goto failed_mount;
        }
+       percpu_counter_init(&sbi->s_freeblocks_counter);
+       bgl_lock_init(&sbi->s_blockgroup_lock);
        sbi->s_debts = kmalloc(sbi->s_groups_count * sizeof(*sbi->s_debts),
                               GFP_KERNEL);
        if (!sbi->s_debts) {
@@ -814,6 +816,8 @@ static int ext2_fill_super(struct super_block *sb, void *data, int silent)
                ext2_warning(sb, __FUNCTION__,
                        "mounting ext3 filesystem as ext2\n");
        ext2_setup_super (sb, es, sb->s_flags & MS_RDONLY);
+       percpu_counter_mod(&sbi->s_freeblocks_counter,
+                               ext2_count_free_blocks(sb));
        return 0;
 failed_mount2:
        for (i = 0; i < db_count; i++)
@@ -840,6 +844,7 @@ static void ext2_commit_super (struct super_block * sb,
 
 static void ext2_sync_super(struct super_block *sb, struct ext2_super_block *es)
 {
+       es->s_free_blocks_count = cpu_to_le32(ext2_count_free_blocks(sb));
        es->s_wtime = cpu_to_le32(get_seconds());
        mark_buffer_dirty(EXT2_SB(sb)->s_sbh);
        sync_dirty_buffer(EXT2_SB(sb)->s_sbh);
@@ -868,6 +873,7 @@ void ext2_write_super (struct super_block * sb)
                        ext2_debug ("setting valid to 0\n");
                        es->s_state = cpu_to_le16(le16_to_cpu(es->s_state) &
                                                  ~EXT2_VALID_FS);
+                       es->s_free_blocks_count = cpu_to_le32(ext2_count_free_blocks(sb));
                        es->s_mtime = cpu_to_le32(get_seconds());
                        ext2_sync_super(sb, es);
                } else
@@ -965,7 +971,7 @@ static int ext2_statfs (struct super_block * sb, struct statfs * buf)
        buf->f_type = EXT2_SUPER_MAGIC;
        buf->f_bsize = sb->s_blocksize;
        buf->f_blocks = le32_to_cpu(sbi->s_es->s_blocks_count) - overhead;
-       buf->f_bfree = ext2_count_free_blocks (sb);
+       buf->f_bfree = ext2_count_free_blocks(sb);
        buf->f_bavail = buf->f_bfree - le32_to_cpu(sbi->s_es->s_r_blocks_count);
        if (buf->f_bfree < le32_to_cpu(sbi->s_es->s_r_blocks_count))
                buf->f_bavail = 0;

diff --git a/include/asm-alpha/bitops.h b/include/asm-alpha/bitops.h
index 14879b4f33b5595ea6780a0382c337496be2f067..25263777a35f1376e62aea7ac9161136fa0d4c3c 100644 (file)
--- a/include/asm-alpha/bitops.h
+++ b/include/asm-alpha/bitops.h
@@ -487,7 +487,9 @@ sched_find_first_bit(unsigned long b[3])
 
 
 #define ext2_set_bit                 __test_and_set_bit
+#define ext2_set_bit_atomic(l,n,a)   test_and_set_bit(n,a)
 #define ext2_clear_bit               __test_and_clear_bit
+#define ext2_clear_bit_atomic(l,n,a) test_and_clear_bit(n,a)
 #define ext2_test_bit                test_bit
 #define ext2_find_first_zero_bit     find_first_zero_bit
 #define ext2_find_next_zero_bit      find_next_zero_bit

diff --git a/include/asm-arm/bitops.h b/include/asm-arm/bitops.h
index 622bde7dbf603b4e92b91c8369b93fef49865c1a..819c89592d62c0208abf08c9d5806d262046da3c 100644 (file)
--- a/include/asm-arm/bitops.h
+++ b/include/asm-arm/bitops.h
@@ -357,8 +357,12 @@ static inline int sched_find_first_bit(unsigned long *b)
  */
 #define ext2_set_bit(nr,p)                     \
                __test_and_set_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
+#define ext2_set_bit_atomic(lock,nr,p)          \
+                test_and_set_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
 #define ext2_clear_bit(nr,p)                   \
                __test_and_clear_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
+#define ext2_clear_bit_atomic(lock,nr,p)        \
+                test_and_clear_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
 #define ext2_test_bit(nr,p)                    \
                __test_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p))
 #define ext2_find_first_zero_bit(p,sz)         \

diff --git a/include/asm-cris/bitops.h b/include/asm-cris/bitops.h
index 4d9856538d92c4fa8b9c5a8735e49a4ae9c4560c..563cc9f6a9986664908f13edb97da77ba57d25a0 100644 (file)
--- a/include/asm-cris/bitops.h
+++ b/include/asm-cris/bitops.h
@@ -360,7 +360,9 @@ static inline int find_next_zero_bit (void * addr, int size, int offset)
 #define hweight8(x) generic_hweight8(x)
 
 #define ext2_set_bit                 test_and_set_bit
+#define ext2_set_bit_atomic(l,n,a)   test_and_set_bit(n,a)
 #define ext2_clear_bit               test_and_clear_bit
+#define ext2_clear_bit_atomic(l,n,a) test_and_clear_bit(n,a)
 #define ext2_test_bit                test_bit
 #define ext2_find_first_zero_bit     find_first_zero_bit
 #define ext2_find_next_zero_bit      find_next_zero_bit

diff --git a/include/asm-i386/bitops.h b/include/asm-i386/bitops.h
index 130cf24045856f81da6a840ffeb66ecbf352b96a..f854ed3540388093e1a3a104cf95e903aa51e820 100644 (file)
--- a/include/asm-i386/bitops.h
+++ b/include/asm-i386/bitops.h
@@ -479,8 +479,12 @@ static __inline__ int ffs(int x)
 
 #define ext2_set_bit(nr,addr) \
        __test_and_set_bit((nr),(unsigned long*)addr)
+#define ext2_set_bit_atomic(lock,nr,addr) \
+        test_and_set_bit((nr),(unsigned long*)addr)
 #define ext2_clear_bit(nr, addr) \
        __test_and_clear_bit((nr),(unsigned long*)addr)
+#define ext2_clear_bit_atomic(lock,nr, addr) \
+               test_and_clear_bit((nr),(unsigned long*)addr)
 #define ext2_test_bit(nr, addr)      test_bit((nr),(unsigned long*)addr)
 #define ext2_find_first_zero_bit(addr, size) \
        find_first_zero_bit((unsigned long*)addr, size)

diff --git a/include/asm-ia64/bitops.h b/include/asm-ia64/bitops.h
index 8a61b49df8b13917fb66d7cc1a90401af3c48c44..af58934491f07efec37e5a843e140e65fa9feacb 100644 (file)
--- a/include/asm-ia64/bitops.h
+++ b/include/asm-ia64/bitops.h
@@ -453,7 +453,9 @@ find_next_bit (void *addr, unsigned long size, unsigned long offset)
 #define __clear_bit(nr, addr)        clear_bit(nr, addr)
 
 #define ext2_set_bit                 test_and_set_bit
+#define ext2_set_atomic(l,n,a)      test_and_set_bit(n,a)
 #define ext2_clear_bit               test_and_clear_bit
+#define ext2_clear_atomic(l,n,a)     test_and_clear_bit(n,a)
 #define ext2_test_bit                test_bit
 #define ext2_find_first_zero_bit     find_first_zero_bit
 #define ext2_find_next_zero_bit      find_next_zero_bit

diff --git a/include/asm-m68k/bitops.h b/include/asm-m68k/bitops.h
index fb9146e78ecc8ae77e47c345f87ef683d9814c5c..0940cb3c1cff6a4a91770f4f56cea1384b500e6d 100644 (file)
--- a/include/asm-m68k/bitops.h
+++ b/include/asm-m68k/bitops.h
@@ -365,6 +365,24 @@ ext2_clear_bit (int nr, volatile void *vaddr)
        return retval;
 }
 
+#define ext2_set_bit_atomic(lock, nr, addr)            \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_set_bit((nr), (addr));       \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
+#define ext2_clear_bit_atomic(lock, nr, addr)          \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_clear_bit((nr), (addr));     \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
 extern __inline__ int
 ext2_test_bit (int nr, const volatile void *vaddr)
 {

diff --git a/include/asm-m68knommu/bitops.h b/include/asm-m68knommu/bitops.h
index 29c2ffab0913ac0eb3a348cb3be1854fc3f602a0..351f6fdeae3426c6bdd08472ef6459c420ca0911 100644 (file)
--- a/include/asm-m68knommu/bitops.h
+++ b/include/asm-m68knommu/bitops.h
@@ -402,6 +402,24 @@ extern __inline__ int ext2_clear_bit(int nr, volatile void * addr)
        return retval;
 }
 
+#define ext2_set_bit_atomic(lock, nr, addr)            \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_set_bit((nr), (addr));       \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
+#define ext2_clear_bit_atomic(lock, nr, addr)          \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_clear_bit((nr), (addr));     \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
 extern __inline__ int ext2_test_bit(int nr, const volatile void * addr)
 {
        int     mask;

diff --git a/include/asm-mips/bitops.h b/include/asm-mips/bitops.h
index a8d32325118974f16148faf57de38b37536dda54..d76387979544e0d3cd4258477c348a41d6363a59 100644 (file)
--- a/include/asm-mips/bitops.h
+++ b/include/asm-mips/bitops.h
@@ -824,6 +824,24 @@ extern __inline__ int ext2_clear_bit(int nr, void * addr)
        return retval;
 }
 
+#define ext2_set_bit_atomic(lock, nr, addr)            \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_set_bit((nr), (addr));       \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
+#define ext2_clear_bit_atomic(lock, nr, addr)          \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_clear_bit((nr), (addr));     \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
 extern __inline__ int ext2_test_bit(int nr, const void * addr)
 {
        int                     mask;
@@ -890,7 +908,9 @@ found_middle:
 
 /* Native ext2 byte ordering, just collapse using defines. */
 #define ext2_set_bit(nr, addr) test_and_set_bit((nr), (addr))
+#define ext2_set_bit_atomic(lock, nr, addr) test_and_set_bit((nr), (addr))
 #define ext2_clear_bit(nr, addr) test_and_clear_bit((nr), (addr))
+#define ext2_clear_bit_atomic(lock, nr, addr) test_and_clear_bit((nr), (addr))
 #define ext2_test_bit(nr, addr) test_bit((nr), (addr))
 #define ext2_find_first_zero_bit(addr, size) find_first_zero_bit((addr), (size))
 #define ext2_find_next_zero_bit(addr, size, offset) \

diff --git a/include/asm-mips64/bitops.h b/include/asm-mips64/bitops.h
index dc1675415acc1f8ca9c45bb32278c0def566fe20..b30ea151511aa3d614d314b51e211ac13ddf86f0 100644 (file)
--- a/include/asm-mips64/bitops.h
+++ b/include/asm-mips64/bitops.h
@@ -531,6 +531,24 @@ ext2_clear_bit(int nr, void * addr)
        return retval;
 }
 
+#define ext2_set_bit_atomic(lock, nr, addr)            \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_set_bit((nr), (addr));       \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
+#define ext2_clear_bit_atomic(lock, nr, addr)          \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_clear_bit((nr), (addr));     \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
 extern inline int
 ext2_test_bit(int nr, const void * addr)
 {
@@ -599,7 +617,9 @@ found_middle:
 
 /* Native ext2 byte ordering, just collapse using defines. */
 #define ext2_set_bit(nr, addr) test_and_set_bit((nr), (addr))
+#define ext2_set_bit_atomic(lock, nr, addr) test_and_set_bit((nr), (addr))
 #define ext2_clear_bit(nr, addr) test_and_clear_bit((nr), (addr))
+#define ext2_clear_bit_atomic(lock, nr, addr) test_and_clear_bit((nr), (addr))
 #define ext2_test_bit(nr, addr) test_bit((nr), (addr))
 #define ext2_find_first_zero_bit(addr, size) find_first_zero_bit((addr), (size))
 #define ext2_find_next_zero_bit(addr, size, offset) \

diff --git a/include/asm-parisc/bitops.h b/include/asm-parisc/bitops.h
index c2eed7aa218bd597f61684cc9c68e936890cf2d8..711e7d154afa213f39d20eca1bb469f55a86b69d 100644 (file)
--- a/include/asm-parisc/bitops.h
+++ b/include/asm-parisc/bitops.h
@@ -389,10 +389,14 @@ found_middle:
  */
 #ifdef __LP64__
 #define ext2_set_bit(nr, addr)         test_and_set_bit((nr) ^ 0x38, addr)
+#define ext2_set_bit_atomic(l,nr,addr)  test_and_set_bit((nr) ^ 0x38, addr)
 #define ext2_clear_bit(nr, addr)       test_and_clear_bit((nr) ^ 0x38, addr)
+#define ext2_clear_bit_atomic(l,nr,addr) test_and_clear_bit((nr) ^ 0x38, addr)
 #else
 #define ext2_set_bit(nr, addr)         test_and_set_bit((nr) ^ 0x18, addr)
+#define ext2_set_bit_atomic(l,nr,addr)  test_and_set_bit((nr) ^ 0x18, addr)
 #define ext2_clear_bit(nr, addr)       test_and_clear_bit((nr) ^ 0x18, addr)
+#define ext2_clear_bit_atomic(l,nr,addr) test_and_clear_bit((nr) ^ 0x18, addr)
 #endif
 
 #endif /* __KERNEL__ */

diff --git a/include/asm-ppc/bitops.h b/include/asm-ppc/bitops.h
index da32a543c856c51661fb048971692fbf786f0a37..68e4a5963fb325bbe732e17a525f087e36b247cb 100644 (file)
--- a/include/asm-ppc/bitops.h
+++ b/include/asm-ppc/bitops.h
@@ -392,7 +392,9 @@ found_middle:
 
 
 #define ext2_set_bit(nr, addr) __test_and_set_bit((nr) ^ 0x18, (unsigned long *)(addr))
+#define ext2_set_bit_atomic(lock, nr, addr)  test_and_set_bit((nr) ^ 0x18, (unsigned long *)(addr))
 #define ext2_clear_bit(nr, addr) __test_and_clear_bit((nr) ^ 0x18, (unsigned long *)(addr))
+#define ext2_clear_bit_atomic(lock, nr, addr) test_and_clear_bit((nr) ^ 0x18, (unsigned long *)(addr))
 
 static __inline__ int ext2_test_bit(int nr, __const__ void * addr)
 {

diff --git a/include/asm-ppc64/bitops.h b/include/asm-ppc64/bitops.h
index 05c66df61a0a96aebb07e7714257bcfbfbed6780..6bd06208e3e251e3a8eb1119f4c830bf5faa444f 100644 (file)
--- a/include/asm-ppc64/bitops.h
+++ b/include/asm-ppc64/bitops.h
@@ -338,6 +338,25 @@ static __inline__ int __test_and_clear_le_bit(unsigned long nr, unsigned long *a
        __test_and_set_le_bit((nr),(unsigned long*)addr)
 #define ext2_clear_bit(nr, addr) \
        __test_and_clear_le_bit((nr),(unsigned long*)addr)
+
+#define ext2_set_bit_atomic(lock, nr, addr)            \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_set_bit((nr), (addr));       \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
+#define ext2_clear_bit_atomic(lock, nr, addr)          \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_clear_bit((nr), (addr));     \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
 #define ext2_test_bit(nr, addr)      test_le_bit((nr),(unsigned long*)addr)
 #define ext2_find_first_zero_bit(addr, size) \
        find_first_zero_le_bit((unsigned long*)addr, size)

diff --git a/include/asm-s390/bitops.h b/include/asm-s390/bitops.h
index e680985331bd10a9cb319a97772443ef955e7f5a..bef5812337a316da2230ad15f7989b743320f486 100644 (file)
--- a/include/asm-s390/bitops.h
+++ b/include/asm-s390/bitops.h
@@ -805,8 +805,12 @@ extern __inline__ int fls(int x)
 
 #define ext2_set_bit(nr, addr)       \
        test_and_set_bit((nr)^24, (unsigned long *)addr)
+#define ext2_set_bit_atomic(lock, nr, addr)       \
+               test_and_set_bit((nr)^24, (unsigned long *)addr)
 #define ext2_clear_bit(nr, addr)     \
        test_and_clear_bit((nr)^24, (unsigned long *)addr)
+#define ext2_clear_bit_atomic(lock, nr, addr)     \
+               test_and_clear_bit((nr)^24, (unsigned long *)addr)
 #define ext2_test_bit(nr, addr)      \
        test_bit((nr)^24, (unsigned long *)addr)
 

diff --git a/include/asm-s390x/bitops.h b/include/asm-s390x/bitops.h
index c8bb2d04b7cea7ce163750a594794fa87357de87..18911f98504aa21de90ad0878d78a7cd81c6f6e6 100644 (file)
--- a/include/asm-s390x/bitops.h
+++ b/include/asm-s390x/bitops.h
@@ -838,8 +838,12 @@ extern __inline__ int fls(int x)
 
 #define ext2_set_bit(nr, addr)       \
        test_and_set_bit((nr)^56, (unsigned long *)addr)
+#define ext2_set_bit_atomic(lock, nr, addr)       \
+               test_and_set_bit((nr)^56, (unsigned long *)addr)
 #define ext2_clear_bit(nr, addr)     \
        test_and_clear_bit((nr)^56, (unsigned long *)addr)
+#define ext2_clear_bit_atomic(lock, nr, addr)     \
+               test_and_clear_bit((nr)^56, (unsigned long *)addr)
 #define ext2_test_bit(nr, addr)      \
        test_bit((nr)^56, (unsigned long *)addr)
 

diff --git a/include/asm-sh/bitops.h b/include/asm-sh/bitops.h
index a897f7bdd4ee944b6f8f2d98c2ced89653333869..856ce04fc424a711e5de846c6cf9a1ec8b414207 100644 (file)
--- a/include/asm-sh/bitops.h
+++ b/include/asm-sh/bitops.h
@@ -344,6 +344,24 @@ found_middle:
 }
 #endif
 
+#define ext2_set_bit_atomic(lock, nr, addr)            \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_set_bit((nr), (addr));       \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
+#define ext2_clear_bit_atomic(lock, nr, addr)          \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_clear_bit((nr), (addr));     \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
 /* Bitmap functions for the minix filesystem.  */
 #define minix_test_and_set_bit(nr,addr) test_and_set_bit(nr,addr)
 #define minix_set_bit(nr,addr) set_bit(nr,addr)

diff --git a/include/asm-sparc/bitops.h b/include/asm-sparc/bitops.h
index 5b9707927b4354611b0fe32a67d9e4445e8dc852..5b3226834489d55812aca7cf8285b999240f0bb8 100644 (file)
--- a/include/asm-sparc/bitops.h
+++ b/include/asm-sparc/bitops.h
@@ -455,6 +455,25 @@ found_middle:
 
 #define ext2_set_bit                   __test_and_set_le_bit
 #define ext2_clear_bit                 __test_and_clear_le_bit
+
+#define ext2_set_bit_atomic(lock, nr, addr)            \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_set_bit((nr), (addr));       \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
+#define ext2_clear_bit_atomic(lock, nr, addr)          \
+       ({                                              \
+               int ret;                                \
+               spin_lock(lock);                        \
+               ret = ext2_clear_bit((nr), (addr));     \
+               spin_unlock(lock);                      \
+               ret;                                    \
+       })
+
 #define ext2_test_bit                  test_le_bit
 #define ext2_find_first_zero_bit       find_first_zero_le_bit
 #define ext2_find_next_zero_bit                find_next_zero_le_bit

diff --git a/include/asm-sparc64/bitops.h b/include/asm-sparc64/bitops.h
index 0cccb55ec8c91c9277035d0fadb89b5472f71fa7..79adf5b5384b21790a2a3dcd3e68c24c031d8c78 100644 (file)
--- a/include/asm-sparc64/bitops.h
+++ b/include/asm-sparc64/bitops.h
@@ -351,7 +351,9 @@ found_middle:
 #ifdef __KERNEL__
 
 #define ext2_set_bit(nr,addr)          test_and_set_le_bit((nr),(unsigned long *)(addr))
+#define ext2_set_bit_atomic(lock,nr,addr) test_and_set_le_bit((nr),(unsigned long *)(addr))
 #define ext2_clear_bit(nr,addr)                test_and_clear_le_bit((nr),(unsigned long *)(addr))
+#define ext2_clear_bit_atomic(lock,nr,addr) test_and_clear_le_bit((nr),(unsigned long *)(addr))
 #define ext2_test_bit(nr,addr)         test_le_bit((nr),(unsigned long *)(addr))
 #define ext2_find_first_zero_bit(addr, size) \
        find_first_zero_le_bit((unsigned long *)(addr), (size))

diff --git a/include/asm-v850/bitops.h b/include/asm-v850/bitops.h
index 21c0b3c1c0a3d5c90cc78f5495cb8af6996c3ab1..13c558d48d4f04172441240f291c215368766153 100644 (file)
--- a/include/asm-v850/bitops.h
+++ b/include/asm-v850/bitops.h
@@ -252,7 +252,9 @@ static inline int sched_find_first_bit(unsigned long *b)
 #define hweight8(x)                    generic_hweight8 (x)
 
 #define ext2_set_bit                   test_and_set_bit
+#define ext2_set_bit_atomic(l,n,a)      test_and_set_bit(n,a)
 #define ext2_clear_bit                 test_and_clear_bit
+#define ext2_clear_bit_atomic(l,n,a)    test_and_clear_bit(n,a)
 #define ext2_test_bit                  test_bit
 #define ext2_find_first_zero_bit       find_first_zero_bit
 #define ext2_find_next_zero_bit                find_next_zero_bit

diff --git a/include/asm-x86_64/bitops.h b/include/asm-x86_64/bitops.h
index 7b3578dd2da982242f105579c3566b0df7f469fd..493aa82d889c16927420698afb7589bf987bdd9d 100644 (file)
--- a/include/asm-x86_64/bitops.h
+++ b/include/asm-x86_64/bitops.h
@@ -487,8 +487,12 @@ static __inline__ int ffs(int x)
 
 #define ext2_set_bit(nr,addr) \
        __test_and_set_bit((nr),(unsigned long*)addr)
+#define ext2_set_bit_atomic(lock,nr,addr) \
+               test_and_set_bit((nr),(unsigned long*)addr)
 #define ext2_clear_bit(nr, addr) \
        __test_and_clear_bit((nr),(unsigned long*)addr)
+#define ext2_clear_bit_atomic(lock,nr,addr) \
+               test_and_clear_bit((nr),(unsigned long*)addr)
 #define ext2_test_bit(nr, addr)      test_bit((nr),(unsigned long*)addr)
 #define ext2_find_first_zero_bit(addr, size) \
        find_first_zero_bit((unsigned long*)addr, size)

diff --git a/include/linux/ext2_fs_sb.h b/include/linux/ext2_fs_sb.h
index 3c07d4ecf898daa0022cfb96b82479e37407412f..f6139acdac5cff29fbfea7bc6441c6f1c51aa96c 100644 (file)
--- a/include/linux/ext2_fs_sb.h
+++ b/include/linux/ext2_fs_sb.h
@@ -16,6 +16,9 @@
 #ifndef _LINUX_EXT2_FS_SB
 #define _LINUX_EXT2_FS_SB
 
+#include <linux/blockgroup_lock.h>
+#include <linux/percpu_counter.h>
+
 /*
  * second extended-fs super-block data in memory
  */
@@ -45,6 +48,8 @@ struct ext2_sb_info {
        u32 s_next_generation;
        unsigned long s_dir_count;
        u8 *s_debts;
+       struct percpu_counter s_freeblocks_counter;
+       struct blockgroup_lock s_blockgroup_lock;
 };
 
 #endif /* _LINUX_EXT2_FS_SB */