[PATCH] rmap 22 flush_dcache_mmap_lock

From: Hugh Dickins <hugh@veritas.com>

arm and parisc __flush_dcache_page have been scanning the i_mmap(_shared) list
without locking or disabling preemption.  That may be even more unsafe now
it's a prio tree instead of a list.

It looks like we cannot use i_shared_lock for this protection: most uses of
flush_dcache_page are okay, and only one would need lock ordering fixed
(get_user_pages holds page_table_lock across flush_dcache_page); but there's a
few (e.g.  in net and ntfs) which look as if they're using it in I/O
completion - and it would be restrictive to disallow it there.

So, on arm and parisc only, define flush_dcache_mmap_lock(mapping) as
spin_lock_irq(&(mapping)->tree_lock); on i386 (and other arches left to the
next patch) define it away to nothing; and use where needed.

While updating locking hierarchy in filemap.c, remove two layers of the fossil
record from add_to_page_cache comment: no longer used for swap.

I believe all the #includes will work out, but have only built i386.  I can
see several things about this patch which might cause revulsion: the name
flush_dcache_mmap_lock?  the reuse of the page radix_tree's tree_lock for this
different purpose?  spin_lock_irqsave instead?  can't we somehow get
i_shared_lock to handle the problem?

diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c
index 5ff9c9f8323ff7bf75961b04fb80ed3f78a3ad08..81485b5b4de4ddd72164f9ed8166432ad212c098 100644 (file)
--- a/arch/arm/mm/fault-armv.c
+++ b/arch/arm/mm/fault-armv.c
@@ -94,6 +94,8 @@ void __flush_dcache_page(struct page *page)
         * and invalidate any user data.
         */
        pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
+
+       flush_dcache_mmap_lock(mapping);
        while ((mpnt = vma_prio_tree_next(mpnt, &mapping->i_mmap,
                                        &iter, pgoff, pgoff)) != NULL) {
                /*
@@ -106,6 +108,7 @@ void __flush_dcache_page(struct page *page)
                offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
                flush_cache_page(mpnt, mpnt->vm_start + offset);
        }
+       flush_dcache_mmap_unlock(mapping);
 }
 
 static void
@@ -129,6 +132,7 @@ make_coherent(struct vm_area_struct *vma, unsigned long addr, struct page *page,
         * space, then we need to handle them specially to maintain
         * cache coherency.
         */
+       flush_dcache_mmap_lock(mapping);
        while ((mpnt = vma_prio_tree_next(mpnt, &mapping->i_mmap,
                                        &iter, pgoff, pgoff)) != NULL) {
                /*
@@ -143,6 +147,7 @@ make_coherent(struct vm_area_struct *vma, unsigned long addr, struct page *page,
                offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
                aliases += adjust_pte(mpnt, mpnt->vm_start + offset);
        }
+       flush_dcache_mmap_unlock(mapping);
        if (aliases)
                adjust_pte(vma, addr);
        else

diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
index 8f0bd9e72482ee114ed9e73d7d8769c6a672b284..9fc0614401223e72cc94726b92f654b51d85d29b 100644 (file)
--- a/arch/parisc/kernel/cache.c
+++ b/arch/parisc/kernel/cache.c
@@ -249,6 +249,7 @@ void __flush_dcache_page(struct page *page)
         * declared as MAP_PRIVATE or MAP_SHARED), so we only need
         * to flush one address here for them all to become coherent */
 
+       flush_dcache_mmap_lock(mapping);
        while ((mpnt = vma_prio_tree_next(mpnt, &mapping->i_mmap,
                                        &iter, pgoff, pgoff)) != NULL) {
                offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
@@ -266,8 +267,9 @@ void __flush_dcache_page(struct page *page)
 
                __flush_cache_page(mpnt, addr);
 
-               return;
+               break;
        }
+       flush_dcache_mmap_unlock(mapping);
 }
 EXPORT_SYMBOL(__flush_dcache_page);
 

diff --git a/include/asm-arm/cacheflush.h b/include/asm-arm/cacheflush.h
index 0e053301c09f185eec4d503c7e42101a61af9dd8..224c8d729775e8fbbd2266360f4356d2640ddda1 100644 (file)
--- a/include/asm-arm/cacheflush.h
+++ b/include/asm-arm/cacheflush.h
@@ -303,6 +303,11 @@ static inline void flush_dcache_page(struct page *page)
                __flush_dcache_page(page);
 }
 
+#define flush_dcache_mmap_lock(mapping) \
+       spin_lock_irq(&(mapping)->tree_lock)
+#define flush_dcache_mmap_unlock(mapping) \
+       spin_unlock_irq(&(mapping)->tree_lock)
+
 #define flush_icache_user_range(vma,page,addr,len) \
        flush_dcache_page(page)
 

diff --git a/include/asm-i386/cacheflush.h b/include/asm-i386/cacheflush.h
index 9f78028b23e44c2063adba212f38a4d3cd22f948..183361ebe0bc1c59857d2d6f945f2618c4ad15e4 100644 (file)
--- a/include/asm-i386/cacheflush.h
+++ b/include/asm-i386/cacheflush.h
@@ -10,6 +10,8 @@
 #define flush_cache_range(vma, start, end)     do { } while (0)
 #define flush_cache_page(vma, vmaddr)          do { } while (0)
 #define flush_dcache_page(page)                        do { } while (0)
+#define flush_dcache_mmap_lock(mapping)                do { } while (0)
+#define flush_dcache_mmap_unlock(mapping)      do { } while (0)
 #define flush_icache_range(start, end)         do { } while (0)
 #define flush_icache_page(vma,pg)              do { } while (0)
 #define flush_icache_user_range(vma,pg,adr,len)        do { } while (0)

diff --git a/include/asm-parisc/cacheflush.h b/include/asm-parisc/cacheflush.h
index 84ba7dbe6493fc2d798a3bbcf9b3e9d26d2d93c6..a5700c29f18a868d9f2732e7ba7644cb96d0dc80 100644 (file)
--- a/include/asm-parisc/cacheflush.h
+++ b/include/asm-parisc/cacheflush.h
@@ -78,6 +78,11 @@ static inline void flush_dcache_page(struct page *page)
        }
 }
 
+#define flush_dcache_mmap_lock(mapping) \
+       spin_lock_irq(&(mapping)->tree_lock)
+#define flush_dcache_mmap_unlock(mapping) \
+       spin_unlock_irq(&(mapping)->tree_lock)
+
 #define flush_icache_page(vma,page)    do { flush_kernel_dcache_page(page_address(page)); flush_kernel_icache_page(page_address(page)); } while (0)
 
 #define flush_icache_range(s,e)                do { flush_kernel_dcache_range_asm(s,e); flush_kernel_icache_range_asm(s,e); } while (0)

diff --git a/kernel/fork.c b/kernel/fork.c
index 3eb6ca91d29ac62324725043825def597b5fec4c..ef85a909e171f8ed0e4c612d2fc0d7b51eac75de 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -332,7 +332,9 @@ static inline int dup_mmap(struct mm_struct * mm, struct mm_struct * oldmm)
       
                        /* insert tmp into the share list, just after mpnt */
                        spin_lock(&file->f_mapping->i_mmap_lock);
+                       flush_dcache_mmap_lock(mapping);
                        vma_prio_tree_add(tmp, mpnt);
+                       flush_dcache_mmap_unlock(mapping);
                        spin_unlock(&file->f_mapping->i_mmap_lock);
                }
 

diff --git a/mm/filemap.c b/mm/filemap.c
index 804d96e88fd78a9ec31662ef1661af2c3d3f9dd5..6e59faaa6afb06215f55c7de69eec5002242babf 100644 (file)
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -65,7 +65,9 @@
  *    ->i_mmap_lock            (truncate->unmap_mapping_range)
  *
  *  ->mmap_sem
- *    ->i_mmap_lock            (various places)
+ *    ->i_mmap_lock
+ *      ->page_table_lock      (various places, mainly in mmap.c)
+ *        ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock)
  *
  *  ->mmap_sem
  *    ->lock_page              (access_process_vm)

diff --git a/mm/fremap.c b/mm/fremap.c
index c624d3d6970fe72ba801098c21d1fa01655b976e..eb056db90eb4e8924767e9fb0a7de212264f455e 100644 (file)
--- a/mm/fremap.c
+++ b/mm/fremap.c
@@ -202,11 +202,13 @@ asmlinkage long sys_remap_file_pages(unsigned long start, unsigned long size,
                    !(vma->vm_flags & VM_NONLINEAR)) {
                        mapping = vma->vm_file->f_mapping;
                        spin_lock(&mapping->i_mmap_lock);
+                       flush_dcache_mmap_lock(mapping);
                        vma->vm_flags |= VM_NONLINEAR;
                        vma_prio_tree_remove(vma, &mapping->i_mmap);
                        vma_prio_tree_init(vma);
                        list_add_tail(&vma->shared.vm_set.list,
                                        &mapping->i_mmap_nonlinear);
+                       flush_dcache_mmap_unlock(mapping);
                        spin_unlock(&mapping->i_mmap_lock);
                }
 

diff --git a/mm/mmap.c b/mm/mmap.c
index baa5d6c1600ce90f987285bb677fe3d2e9c9fd28..ad64fb0367916020f7b6be2a4a8b31ab4f73a5ec 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -25,6 +25,7 @@
 
 #include <asm/uaccess.h>
 #include <asm/pgalloc.h>
+#include <asm/cacheflush.h>
 #include <asm/tlb.h>
 
 /*
@@ -74,10 +75,12 @@ static inline void __remove_shared_vm_struct(struct vm_area_struct *vma,
        if (vma->vm_flags & VM_SHARED)
                mapping->i_mmap_writable--;
 
+       flush_dcache_mmap_lock(mapping);
        if (unlikely(vma->vm_flags & VM_NONLINEAR))
                list_del_init(&vma->shared.vm_set.list);
        else
                vma_prio_tree_remove(vma, &mapping->i_mmap);
+       flush_dcache_mmap_unlock(mapping);
 }
 
 /*
@@ -266,11 +269,13 @@ static inline void __vma_link_file(struct vm_area_struct *vma)
                if (vma->vm_flags & VM_SHARED)
                        mapping->i_mmap_writable++;
 
+               flush_dcache_mmap_lock(mapping);
                if (unlikely(vma->vm_flags & VM_NONLINEAR))
                        list_add_tail(&vma->shared.vm_set.list,
                                        &mapping->i_mmap_nonlinear);
                else
                        vma_prio_tree_insert(vma, &mapping->i_mmap);
+               flush_dcache_mmap_unlock(mapping);
        }
 }
 
@@ -350,14 +355,17 @@ void vma_adjust(struct vm_area_struct *vma, unsigned long start,
        }
        spin_lock(&mm->page_table_lock);
 
-       if (root)
+       if (root) {
+               flush_dcache_mmap_lock(mapping);
                vma_prio_tree_remove(vma, root);
+       }
        vma->vm_start = start;
        vma->vm_end = end;
        vma->vm_pgoff = pgoff;
        if (root) {
                vma_prio_tree_init(vma);
                vma_prio_tree_insert(vma, root);
+               flush_dcache_mmap_unlock(mapping);
        }
 
        if (next) {