[PATCH] s390: tlb flush optimization.

From: Martin Schwidefsky <schwidefsky@de.ibm.com>

On the s/390 architecture we still have the issue with tlb flushing and the
ipte instruction.  We can optimize the tlb flushing a lot with some minor
interface changes between the arch backend and the memory management core.
In the end the whole thing is about the Invalidate Page Table Entry (ipte)
instruction.  The instruction sets the invalid bit in the pte and removes the
tlb for the page on all cpus for the virtual to physical mapping of the page
in a particular address space.  The nice thing is that only the tlb for this
page gets removed, all the other tlbs stay valid.  The reason we can't use
ipte to implement flush_tlb_page() is one of the requirements of the
instruction: the pte that should get flushed needs to be *valid*.

I'd like to add the following four functions to the mm interface:

  * ptep_establish: Establish a new mapping. This sets a pte entry to a
    page table and flushes the tlb of the old entry on all cpus if it
    exists. This is more or less what establish_pte in mm/memory.c does
    right now but without the update_mmu_cache call.

  * ptep_test_and_clear_and_flush_young. Do what ptep_test_and_clear_young
    does and flush the tlb.

  * ptep_test_and_clear_and_flush_dirty. Do what ptep_test_and_clear_dirty
    does and flush the tlb.

  * ptep_get_and_clear_and_flush: Do what ptep_get_and_clear does and
    flush the tlb.

The s/390 specific functions in include/pgtable.h define their own optimized
version of these four functions by use of the ipte.

I avoid the definition of these function for every architecture I added them
to include/asm-generic/pgtable.h.  Since i386/x86 and others don't include
this header yet and define their own version of the functions found there I
#ifdef'd all functions in include/asm-generic/pgtable.h to be able to pick
the ones that are needed for each architecture (see patch for details).

With the new functions in place it is easy to do the optimization, e.g.  the
sequence

         ptep_get_and_clear(ptep);
         flush_tlb_page(vma, address);

gets replace by

         ptep_get_and_clear_and_flush(vma, address, ptep);

The old sequence still works but it is suboptimal on s/390.

diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
index b98102522d0d03c386b2d27550ddbaff9064b578..f8f99849bfa51197f3aeadb1c1ae7e791f89d425 100644 (file)
--- a/include/asm-generic/pgtable.h
+++ b/include/asm-generic/pgtable.h
@@ -1,6 +1,23 @@
 #ifndef _ASM_GENERIC_PGTABLE_H
 #define _ASM_GENERIC_PGTABLE_H
 
+#ifndef __HAVE_ARCH_PTEP_ESTABLISH
+/*
+ * Establish a new mapping:
+ *  - flush the old one
+ *  - update the page tables
+ *  - inform the TLB about the new one
+ *
+ * We hold the mm semaphore for reading and vma->vm_mm->page_table_lock
+ */
+#define ptep_establish(__vma, __address, __ptep, __entry)              \
+do {                                                                   \
+       set_pte(__ptep, __entry);                                       \
+       flush_tlb_page(__vma, __address);                               \
+} while (0)
+#endif
+
+#ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
 static inline int ptep_test_and_clear_young(pte_t *ptep)
 {
        pte_t pte = *ptep;
@@ -9,7 +26,19 @@ static inline int ptep_test_and_clear_young(pte_t *ptep)
        set_pte(ptep, pte_mkold(pte));
        return 1;
 }
+#endif
+
+#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
+#define ptep_clear_flush_young(__vma, __address, __ptep)               \
+({                                                                     \
+       int __young = ptep_test_and_clear_young(__ptep);                \
+       if (__young)                                                    \
+               flush_tlb_page(__vma, __address);                       \
+       __young;                                                        \
+})
+#endif
 
+#ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
 static inline int ptep_test_and_clear_dirty(pte_t *ptep)
 {
        pte_t pte = *ptep;
@@ -18,26 +47,54 @@ static inline int ptep_test_and_clear_dirty(pte_t *ptep)
        set_pte(ptep, pte_mkclean(pte));
        return 1;
 }
+#endif
 
+#ifndef __HAVE_ARCH_PTEP_CLEAR_DIRTY_FLUSH
+#define ptep_clear_flush_dirty(__vma, __address, __ptep)               \
+({                                                                     \
+       int __dirty = ptep_test_and_clear_dirty(__ptep);                \
+       if (__dirty)                                                    \
+               flush_tlb_page(__vma, __address);                       \
+       __dirty;                                                        \
+})
+#endif
+
+#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR
 static inline pte_t ptep_get_and_clear(pte_t *ptep)
 {
        pte_t pte = *ptep;
        pte_clear(ptep);
        return pte;
 }
+#endif
+
+#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
+#define ptep_clear_flush(__vma, __address, __ptep)                     \
+({                                                                     \
+       pte_t __pte = ptep_get_and_clear(__ptep);                       \
+       flush_tlb_page(__vma, __address);                               \
+       __pte;                                                          \
+})
+#endif
 
+#ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT
 static inline void ptep_set_wrprotect(pte_t *ptep)
 {
        pte_t old_pte = *ptep;
        set_pte(ptep, pte_wrprotect(old_pte));
 }
+#endif
 
+#ifndef __HAVE_ARCH_PTEP_MKDIRTY
 static inline void ptep_mkdirty(pte_t *ptep)
 {
        pte_t old_pte = *ptep;
        set_pte(ptep, pte_mkdirty(old_pte));
 }
+#endif
 
+#ifndef __HAVE_ARCH_PTE_SAME
 #define pte_same(A,B)  (pte_val(A) == pte_val(B))
+#endif
 
 #endif /* _ASM_GENERIC_PGTABLE_H */

diff --git a/include/asm-i386/pgtable.h b/include/asm-i386/pgtable.h
index 741069aeadb9ce6f8a24b29000c8542802b8d075..0a21b7a01b2918aee22fd489a809670258ea50d1 100644 (file)
--- a/include/asm-i386/pgtable.h
+++ b/include/asm-i386/pgtable.h
@@ -341,4 +341,12 @@ typedef pte_t *pte_addr_t;
 
 #define io_remap_page_range remap_page_range
 
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
+#define __HAVE_ARCH_PTEP_SET_WRPROTECT
+#define __HAVE_ARCH_PTEP_MKDIRTY
+#define __HAVE_ARCH_PTE_SAME
+#include <asm-generic/pgtable.h>
+
 #endif /* _I386_PGTABLE_H */

diff --git a/include/asm-ia64/pgtable.h b/include/asm-ia64/pgtable.h
index 7561d3d8f6324f4e8c15ac6fb0816bf06c360642..86d7f7f91bfbde8ac0f13583f6da09af9c74f952 100644 (file)
--- a/include/asm-ia64/pgtable.h
+++ b/include/asm-ia64/pgtable.h
@@ -511,4 +511,12 @@ extern void update_mmu_cache (struct vm_area_struct *vma, unsigned long vaddr, p
 #define FIXADDR_USER_START     GATE_ADDR
 #define FIXADDR_USER_END       (GATE_ADDR + 2*PERCPU_PAGE_SIZE)
 
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
+#define __HAVE_ARCH_PTEP_SET_WRPROTECT
+#define __HAVE_ARCH_PTEP_MKDIRTY
+#define __HAVE_ARCH_PTE_SAME
+#include <asm-generic/pgtable.h>
+
 #endif /* _ASM_IA64_PGTABLE_H */

diff --git a/include/asm-parisc/pgtable.h b/include/asm-parisc/pgtable.h
index b7f2daf59ecb4108b018d563fdcb1c6495b7a07b..5d0b71ba386a698c661499d9837dffd3edcb95c8 100644 (file)
--- a/include/asm-parisc/pgtable.h
+++ b/include/asm-parisc/pgtable.h
@@ -460,4 +460,12 @@ typedef pte_t *pte_addr_t;
 
 #define HAVE_ARCH_UNMAPPED_AREA
 
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
+#define __HAVE_ARCH_PTEP_SET_WRPROTECT
+#define __HAVE_ARCH_PTEP_MKDIRTY
+#define __HAVE_ARCH_PTE_SAME
+#include <asm-generic/pgtable.h>
+
 #endif /* _PARISC_PGTABLE_H */

diff --git a/include/asm-ppc/pgtable.h b/include/asm-ppc/pgtable.h
index ad364a9fe357d139a543d97d1d8a6af9360b8cb8..53fa036d620a4deb7ec17aeaff21328ab9eedf3e 100644 (file)
--- a/include/asm-ppc/pgtable.h
+++ b/include/asm-ppc/pgtable.h
@@ -661,5 +661,14 @@ extern void kernel_set_cachemode (unsigned long address, unsigned long size,
 typedef pte_t *pte_addr_t;
 
 #endif /* !__ASSEMBLY__ */
+
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
+#define __HAVE_ARCH_PTEP_SET_WRPROTECT
+#define __HAVE_ARCH_PTEP_MKDIRTY
+#define __HAVE_ARCH_PTE_SAME
+#include <asm-generic/pgtable.h>
+
 #endif /* _PPC_PGTABLE_H */
 #endif /* __KERNEL__ */

diff --git a/include/asm-ppc64/pgtable.h b/include/asm-ppc64/pgtable.h
index 719c6531b5528e240dff2924d1c4de0f02268ff6..5cefc49da027ecef8ec365d1e9bae6c2fbcde31b 100644 (file)
--- a/include/asm-ppc64/pgtable.h
+++ b/include/asm-ppc64/pgtable.h
@@ -455,4 +455,13 @@ static inline pte_t *find_linux_pte(pgd_t *pgdir, unsigned long ea)
 }
 
 #endif /* __ASSEMBLY__ */
+
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
+#define __HAVE_ARCH_PTEP_SET_WRPROTECT
+#define __HAVE_ARCH_PTEP_MKDIRTY
+#define __HAVE_ARCH_PTE_SAME
+#include <asm-generic/pgtable.h>
+
 #endif /* _PPC64_PGTABLE_H */

diff --git a/include/asm-s390/pgalloc.h b/include/asm-s390/pgalloc.h
index ac2aba7a92995c60cc271c3f02cc51f63a32f097..39697ffa96f06588f287f5ec1d0a62cf39cb8ba7 100644 (file)
--- a/include/asm-s390/pgalloc.h
+++ b/include/asm-s390/pgalloc.h
@@ -159,29 +159,4 @@ static inline void pte_free(struct page *pte)
  */
 #define set_pgdir(addr,entry) do { } while(0)
 
-static inline pte_t ptep_invalidate(struct vm_area_struct *vma, 
-                                    unsigned long address, pte_t *ptep)
-{
-       pte_t pte = *ptep;
-#ifndef __s390x__
-       if (!(pte_val(pte) & _PAGE_INVALID)) {
-               /* S390 has 1mb segments, we are emulating 4MB segments */
-               pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00);
-               __asm__ __volatile__ ("ipte %0,%1" : : "a" (pto), "a" (address));
-       }
-#else /* __s390x__ */
-       if (!(pte_val(pte) & _PAGE_INVALID)) 
-               __asm__ __volatile__ ("ipte %0,%1" : : "a" (ptep), "a" (address));
-#endif /* __s390x__ */
-       pte_clear(ptep);
-       return pte;
-}
-
-static inline void ptep_establish(struct vm_area_struct *vma, 
-                                  unsigned long address, pte_t *ptep, pte_t entry)
-{
-       ptep_invalidate(vma, address, ptep);
-       set_pte(ptep, entry);
-}
-
 #endif /* _S390_PGALLOC_H */

diff --git a/include/asm-s390/pgtable.h b/include/asm-s390/pgtable.h
index 776e2b1c887b30ec1ac45fbf0fe156a27ce0585f..1e07cabb521bc4f5e9cb07b9ade805b8f57dc437 100644 (file)
--- a/include/asm-s390/pgtable.h
+++ b/include/asm-s390/pgtable.h
@@ -33,6 +33,8 @@
 #include <asm/processor.h>
 #include <linux/threads.h>
 
+struct vm_area_struct; /* forward declaration (include/linux/mm.h) */
+
 extern pgd_t swapper_pg_dir[] __attribute__ ((aligned (4096)));
 extern void paging_init(void);
 
@@ -493,22 +495,22 @@ extern inline pte_t pte_mkdirty(pte_t pte)
         * sske instruction is slow. It is faster to let the
         * next instruction set the dirty bit.
         */
-       pte_val(pte) &= ~ _PAGE_ISCLEAN;
        return pte;
 }
 
 extern inline pte_t pte_mkold(pte_t pte)
 {
-       asm volatile ("rrbe 0,%0" : : "a" (pte_val(pte)) : "cc" );
+       /* S/390 doesn't keep its dirty/referenced bit in the pte.
+        * There is no point in clearing the real referenced bit.
+        */
        return pte;
 }
 
 extern inline pte_t pte_mkyoung(pte_t pte)
 {
-       /* To set the referenced bit we read the first word from the real
-        * page with a special instruction: load using real address (lura).
-        * Isn't S/390 a nice architecture ?! */
-       asm volatile ("lura 0,%0" : : "a" (pte_val(pte) & PAGE_MASK) : "0" );
+       /* S/390 doesn't keep its dirty/referenced bit in the pte.
+        * There is no point in setting the real referenced bit.
+        */
        return pte;
 }
 
@@ -523,6 +525,14 @@ static inline int ptep_test_and_clear_young(pte_t *ptep)
        return ccode & 2;
 }
 
+static inline int
+ptep_clear_flush_young(struct vm_area_struct *vma,
+                       unsigned long address, pte_t *ptep)
+{
+       /* No need to flush TLB; bits are in storage key */
+       return ptep_test_and_clear_young(ptep);
+}
+
 static inline int ptep_test_and_clear_dirty(pte_t *ptep)
 {
        int skey;
@@ -539,6 +549,14 @@ static inline int ptep_test_and_clear_dirty(pte_t *ptep)
        return 1;
 }
 
+static inline int
+ptep_clear_flush_dirty(struct vm_area_struct *vma,
+                       unsigned long address, pte_t *ptep)
+{
+       /* No need to flush TLB; bits are in storage key */
+       return ptep_test_and_clear_dirty(ptep);
+}
+
 static inline pte_t ptep_get_and_clear(pte_t *ptep)
 {
        pte_t pte = *ptep;
@@ -546,6 +564,25 @@ static inline pte_t ptep_get_and_clear(pte_t *ptep)
        return pte;
 }
 
+static inline pte_t
+ptep_clear_flush(struct vm_area_struct *vma,
+                unsigned long address, pte_t *ptep)
+{
+       pte_t pte = *ptep;
+#ifndef __s390x__
+       if (!(pte_val(pte) & _PAGE_INVALID)) {
+               /* S390 has 1mb segments, we are emulating 4MB segments */
+               pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00);
+               __asm__ __volatile__ ("ipte %0,%1" : : "a" (pto), "a" (address));
+       }
+#else /* __s390x__ */
+       if (!(pte_val(pte) & _PAGE_INVALID)) 
+               __asm__ __volatile__ ("ipte %0,%1" : : "a" (ptep), "a" (address));
+#endif /* __s390x__ */
+       pte_clear(ptep);
+       return pte;
+}
+
 static inline void ptep_set_wrprotect(pte_t *ptep)
 {
        pte_t old_pte = *ptep;
@@ -557,6 +594,14 @@ static inline void ptep_mkdirty(pte_t *ptep)
        pte_mkdirty(*ptep);
 }
 
+static inline void
+ptep_establish(struct vm_area_struct *vma, 
+              unsigned long address, pte_t *ptep, pte_t entry)
+{
+       ptep_clear_flush(vma, address, ptep);
+       set_pte(ptep, entry);
+}
+
 /*
  * Conversion functions: convert a page and protection to a page entry,
  * and a page entry and page directory to the page they refer to.
@@ -727,5 +772,17 @@ typedef pte_t *pte_addr_t;
 # define HAVE_ARCH_UNMAPPED_AREA
 #endif /* __s390x__ */
 
+#define __HAVE_ARCH_PTEP_ESTABLISH
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_ARCH_PTEP_CLEAR_DIRTY_FLUSH
+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
+#define __HAVE_ARCH_PTEP_CLEAR_FLUSH
+#define __HAVE_ARCH_PTEP_SET_WRPROTECT
+#define __HAVE_ARCH_PTEP_MKDIRTY
+#define __HAVE_ARCH_PTE_SAME
+#include <asm-generic/pgtable.h>
+
 #endif /* _S390_PAGE_H */
 

diff --git a/include/asm-sh/pgtable.h b/include/asm-sh/pgtable.h
index 962729a5d15307827df790b8fb5976b248c8b2f5..aaebf907c4f6f7e6597a585cb9777d00aeccca92 100644 (file)
--- a/include/asm-sh/pgtable.h
+++ b/include/asm-sh/pgtable.h
@@ -280,5 +280,13 @@ typedef pte_t *pte_addr_t;
 extern unsigned int kobjsize(const void *objp);
 #endif /* !CONFIG_MMU */
 
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
+#define __HAVE_ARCH_PTEP_SET_WRPROTECT
+#define __HAVE_ARCH_PTEP_MKDIRTY
+#define __HAVE_ARCH_PTE_SAME
+#include <asm-generic/pgtable.h>
+
 #endif /* __ASM_SH_PAGE_H */
 

diff --git a/include/asm-x86_64/pgtable.h b/include/asm-x86_64/pgtable.h
index e48a738d9604da1cabd6b41f7bdae8b49fc12a2d..9a935fadb286931db95388cf520259fd49a750bd 100644 (file)
--- a/include/asm-x86_64/pgtable.h
+++ b/include/asm-x86_64/pgtable.h
@@ -411,4 +411,12 @@ extern int kern_addr_valid(unsigned long addr);
 #define        kc_offset_to_vaddr(o) \
    (((o) & (1UL << (__VIRTUAL_MASK_SHIFT-1))) ? ((o) | (~__VIRTUAL_MASK)) : (o))
 
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
+#define __HAVE_ARCH_PTEP_SET_WRPROTECT
+#define __HAVE_ARCH_PTEP_MKDIRTY
+#define __HAVE_ARCH_PTE_SAME
+#include <asm-generic/pgtable.h>
+
 #endif /* _X86_64_PGTABLE_H */

diff --git a/mm/memory.c b/mm/memory.c
index c0fa1f577a7292540d37067a7a7eb29a35fd0e58..b7ecb9abf26b11146db2c460368478414f7f160b 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -951,29 +951,18 @@ int remap_page_range(struct vm_area_struct *vma, unsigned long from, unsigned lo
 
 EXPORT_SYMBOL(remap_page_range);
 
-/*
- * Establish a new mapping:
- *  - flush the old one
- *  - update the page tables
- *  - inform the TLB about the new one
- *
- * We hold the mm semaphore for reading and vma->vm_mm->page_table_lock
- */
-static inline void establish_pte(struct vm_area_struct * vma, unsigned long address, pte_t *page_table, pte_t entry)
-{
-       set_pte(page_table, entry);
-       flush_tlb_page(vma, address);
-       update_mmu_cache(vma, address, entry);
-}
-
 /*
  * We hold the mm semaphore for reading and vma->vm_mm->page_table_lock
  */
 static inline void break_cow(struct vm_area_struct * vma, struct page * new_page, unsigned long address, 
                pte_t *page_table)
 {
+       pte_t entry;
+
        flush_cache_page(vma, address);
-       establish_pte(vma, address, page_table, pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot))));
+       entry = pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot)));
+       ptep_establish(vma, address, page_table, entry);
+       update_mmu_cache(vma, address, entry);
 }
 
 /*
@@ -1002,6 +991,7 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct * vma,
        struct page *old_page, *new_page;
        unsigned long pfn = pte_pfn(pte);
        struct pte_chain *pte_chain;
+       pte_t entry;
 
        if (unlikely(!pfn_valid(pfn))) {
                /*
@@ -1022,8 +1012,9 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct * vma,
                unlock_page(old_page);
                if (reuse) {
                        flush_cache_page(vma, address);
-                       establish_pte(vma, address, page_table,
-                               pte_mkyoung(pte_mkdirty(pte_mkwrite(pte))));
+                       entry = pte_mkyoung(pte_mkdirty(pte_mkwrite(pte)));
+                       ptep_establish(vma, address, page_table, entry);
+                       update_mmu_cache(vma, address, entry);
                        pte_unmap(page_table);
                        spin_unlock(&mm->page_table_lock);
                        return VM_FAULT_MINOR;
@@ -1580,7 +1571,8 @@ static inline int handle_pte_fault(struct mm_struct *mm,
                entry = pte_mkdirty(entry);
        }
        entry = pte_mkyoung(entry);
-       establish_pte(vma, address, pte, entry);
+       ptep_establish(vma, address, pte, entry);
+       update_mmu_cache(vma, address, entry);
        pte_unmap(pte);
        spin_unlock(&mm->page_table_lock);
        return VM_FAULT_MINOR;

diff --git a/mm/mremap.c b/mm/mremap.c
index 9c0850a568008979831ad18190004f60d02620f3..5a93274fbd16891118d42067f45b072fe99e3e3c 100644 (file)
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -80,8 +80,8 @@ static inline pte_t *alloc_one_pte_map(struct mm_struct *mm, unsigned long addr)
 }
 
 static int
-copy_one_pte(struct mm_struct *mm, pte_t *src, pte_t *dst,
-               struct pte_chain **pte_chainp)
+copy_one_pte(struct vm_area_struct *vma, unsigned long old_addr,
+            pte_t *src, pte_t *dst, struct pte_chain **pte_chainp)
 {
        int error = 0;
        pte_t pte;
@@ -93,7 +93,7 @@ copy_one_pte(struct mm_struct *mm, pte_t *src, pte_t *dst,
        if (!pte_none(*src)) {
                if (page)
                        page_remove_rmap(page, src);
-               pte = ptep_get_and_clear(src);
+               pte = ptep_clear_flush(vma, old_addr, src);
                if (!dst) {
                        /* No dest?  We must put it back. */
                        dst = src;
@@ -135,11 +135,15 @@ move_one_page(struct vm_area_struct *vma, unsigned long old_addr,
                dst = alloc_one_pte_map(mm, new_addr);
                if (src == NULL)
                        src = get_one_pte_map_nested(mm, old_addr);
-               error = copy_one_pte(mm, src, dst, &pte_chain);
+               error = copy_one_pte(vma, old_addr, src, dst, &pte_chain);
                pte_unmap_nested(src);
                pte_unmap(dst);
-       }
-       flush_tlb_page(vma, old_addr);
+       } else
+               /*
+                * Why do we need this flush ? If there is no pte for
+                * old_addr, then there must not be a pte for it as well.
+                */
+               flush_tlb_page(vma, old_addr);
        spin_unlock(&mm->page_table_lock);
        pte_chain_free(pte_chain);
 out:

diff --git a/mm/msync.c b/mm/msync.c
index 130608545f44158748ffa634476cb628c0363b90..2a8d75816f241a188720275f74320b01166bef3d 100644 (file)
--- a/mm/msync.c
+++ b/mm/msync.c
@@ -30,10 +30,9 @@ static int filemap_sync_pte(pte_t *ptep, struct vm_area_struct *vma,
                unsigned long pfn = pte_pfn(pte);
                if (pfn_valid(pfn)) {
                        page = pfn_to_page(pfn);
-                       if (!PageReserved(page) && ptep_test_and_clear_dirty(ptep)) {
-                               flush_tlb_page(vma, address);
+                       if (!PageReserved(page) &&
+                           ptep_clear_flush_dirty(vma, address, ptep))
                                set_page_dirty(page);
-                       }
                }
        }
        return 0;

diff --git a/mm/rmap.c b/mm/rmap.c
index c4434f218e4de9688e5c314ad45c1b2a5582824f..f38f238dc03b5b862b9567e6f5aea60bdc1532b7 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -329,8 +329,7 @@ static int try_to_unmap_one(struct page * page, pte_addr_t paddr)
 
        /* Nuke the page table entry. */
        flush_cache_page(vma, address);
-       pte = ptep_get_and_clear(ptep);
-       flush_tlb_page(vma, address);
+       pte = ptep_clear_flush(vma, address, ptep);
 
        if (PageSwapCache(page)) {
                /*