Split ptep_establish into "establish" and "update_access_flags"

ptep_establish() is used to establish a new mapping at COW time,
and it always replaces a non-writable page mapping with a totally
new page mapping that is dirty (and likely writable, although ptrace
may cause a non-writable new mapping). Because it was nonwritable,
we don't have to worry about losing concurrent dirty page bit updates.

ptep_update_access_flags() leaves the same page mapping, but updates
the accessed/dirty/writable bits (it only ever sets them, and never
removes any permissions). Often easier, but it may race with a dirty
bit update on another CPU.

Booted on x86 and ppc64.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
index e9a387cdd37ab52d2c0167f25bce1627f07bcaec..ee74be68c19e96f34bfe3b9d5b483ac81908558a 100644 (file)
--- a/include/asm-generic/pgtable.h
+++ b/include/asm-generic/pgtable.h
@@ -2,26 +2,38 @@
 #define _ASM_GENERIC_PGTABLE_H
 
 #ifndef __HAVE_ARCH_PTEP_ESTABLISH
-
-#ifndef ptep_update_dirty_accessed
-#define ptep_update_dirty_accessed(__ptep, __entry, __dirty) set_pte(__ptep, __entry)
-#endif
-
 /*
  * Establish a new mapping:
  *  - flush the old one
  *  - update the page tables
  *  - inform the TLB about the new one
  *
- * We hold the mm semaphore for reading and vma->vm_mm->page_table_lock
+ * We hold the mm semaphore for reading and vma->vm_mm->page_table_lock.
+ *
+ * Note: the old pte is known to not be writable, so we don't need to
+ * worry about dirty bits etc getting lost.
  */
-#define ptep_establish(__vma, __address, __ptep, __entry, __dirty)     \
-do {                                                                   \
-       ptep_update_dirty_accessed(__ptep, __entry, __dirty);           \
+#define ptep_establish(__vma, __address, __ptep, __entry)              \
+do {                                                                   \
+       set_pte(__ptep, __entry);                                       \
        flush_tlb_page(__vma, __address);                               \
 } while (0)
 #endif
 
+#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
+/*
+ * Largely same as above, but only sets the access flags (dirty,
+ * accessed, and writable). Furthermore, we know it always gets set
+ * to a "more permissive" setting, which allows most architectures
+ * to optimize this.
+ */
+#define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \
+do {                                                                     \
+       set_pte(__ptep, __entry);                                         \
+       flush_tlb_page(__vma, __address);                                 \
+} while (0)
+#endif
+
 #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
 static inline int ptep_test_and_clear_young(pte_t *ptep)
 {

diff --git a/include/asm-i386/pgtable.h b/include/asm-i386/pgtable.h
index c408168b22e294c8bde8d64bff8cd35292783d14..299c7587140a98e6f64e66b13a5066b00ff65626 100644 (file)
--- a/include/asm-i386/pgtable.h
+++ b/include/asm-i386/pgtable.h
@@ -325,9 +325,13 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
  * bit at the same time.
  */
 #define update_mmu_cache(vma,address,pte) do { } while (0)
-#define ptep_update_dirty_accessed(__ptep, __entry, __dirty)   \
-       do {                                                    \
-               if (__dirty) set_pte(__ptep, __entry);          \
+#define  __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
+#define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \
+       do {                                                              \
+               if (__dirty) {                                            \
+                       (__ptep)->pte_low = (__entry).pte_low;            \
+                       flush_tlb_page(__vma, __address);                 \
+               }                                                         \
        } while (0)
 
 /* Encode and de-code a swap entry */

diff --git a/include/asm-ppc/pgtable.h b/include/asm-ppc/pgtable.h
index 8eefa54b86febf3392222089366ee35c84d34939..976796d2f4c6e901118e2cc24c22b72fa899287f 100644 (file)
--- a/include/asm-ppc/pgtable.h
+++ b/include/asm-ppc/pgtable.h
@@ -548,6 +548,16 @@ static inline void ptep_mkdirty(pte_t *ptep)
        pte_update(ptep, 0, _PAGE_DIRTY);
 }
 
+#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
+static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry, int dirty)
+{
+       unsigned long bits = pte_val(entry) &
+               (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW);
+       pte_update(ptep, 0, bits);
+}
+#define  ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \
+        __ptep_set_access_flags(__ptep, __entry, __dirty)
+
 /*
  * Macro to mark a page protection value as "uncacheable".
  */

diff --git a/include/asm-ppc64/pgtable.h b/include/asm-ppc64/pgtable.h
index 5903d4fe3b5e0b6e439f3f9a2ec92c73fb2e2b36..cb7f53d6c8e5dbde1b8b9696f98e70c96da8b0a9 100644 (file)
--- a/include/asm-ppc64/pgtable.h
+++ b/include/asm-ppc64/pgtable.h
@@ -306,7 +306,10 @@ static inline unsigned long pte_update(pte_t *p, unsigned long clr)
        return old;
 }
 
-/* PTE updating functions */
+/* PTE updating functions, this function puts the PTE in the
+ * batch, doesn't actually triggers the hash flush immediately,
+ * you need to call flush_tlb_pending() to do that.
+ */
 extern void hpte_update(pte_t *ptep, unsigned long pte, int wrprot);
 
 static inline int ptep_test_and_clear_young(pte_t *ptep)
@@ -318,7 +321,7 @@ static inline int ptep_test_and_clear_young(pte_t *ptep)
        old = pte_update(ptep, _PAGE_ACCESSED);
        if (old & _PAGE_HASHPTE) {
                hpte_update(ptep, old, 0);
-               flush_tlb_pending();    /* XXX generic code doesn't flush */
+               flush_tlb_pending();
        }
        return (old & _PAGE_ACCESSED) != 0;
 }
@@ -396,11 +399,37 @@ static inline void pte_clear(pte_t * ptep)
  */
 static inline void set_pte(pte_t *ptep, pte_t pte)
 {
-       if (pte_present(*ptep))
+       if (pte_present(*ptep)) {
                pte_clear(ptep);
+               flush_tlb_pending();
+       }
        *ptep = __pte(pte_val(pte)) & ~_PAGE_HPTEFLAGS;
 }
 
+/* Set the dirty and/or accessed bits atomically in a linux PTE, this
+ * function doesn't need to flush the hash entry
+ */
+#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
+static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry, int dirty)
+{
+       unsigned long bits = pte_val(entry) &
+               (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW);
+       unsigned long old, tmp;
+
+       __asm__ __volatile__(
+       "1:     ldarx   %0,0,%4\n\
+               andi.   %1,%0,%6\n\
+               bne-    1b \n\
+               or      %0,%3,%0\n\
+               stdcx.  %0,0,%4\n\
+               bne-    1b"
+       :"=&r" (old), "=&r" (tmp), "=m" (*ptep)
+       :"r" (bits), "r" (ptep), "m" (ptep), "i" (_PAGE_BUSY)
+       :"cc");
+}
+#define  ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \
+        __ptep_set_access_flags(__ptep, __entry, __dirty)
+
 /*
  * Macro to mark a page protection value as "uncacheable".
  */

diff --git a/include/asm-s390/pgtable.h b/include/asm-s390/pgtable.h
index 959c67a9592c6b0b3602ea2042e3001e7df548d5..ebd93e484dda6f91a50c1604a850ddda16ad108d 100644 (file)
--- a/include/asm-s390/pgtable.h
+++ b/include/asm-s390/pgtable.h
@@ -581,7 +581,7 @@ static inline void ptep_mkdirty(pte_t *ptep)
 static inline void
 ptep_establish(struct vm_area_struct *vma, 
               unsigned long address, pte_t *ptep,
-              pte_t entry, int dirty)
+              pte_t entry)
 {
        ptep_clear_flush(vma, address, ptep);
        set_pte(ptep, entry);

diff --git a/mm/memory.c b/mm/memory.c
index 2ae115e4bd0927df4eb4417a64a5cbebdcbdfe1d..578070758905275564ad2233b594244fe90d2a75 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1004,7 +1004,7 @@ static inline void break_cow(struct vm_area_struct * vma, struct page * new_page
        flush_cache_page(vma, address);
        entry = maybe_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot)),
                              vma);
-       ptep_establish(vma, address, page_table, entry, 1);
+       ptep_establish(vma, address, page_table, entry);
        update_mmu_cache(vma, address, entry);
 }
 
@@ -1056,7 +1056,7 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct * vma,
                        flush_cache_page(vma, address);
                        entry = maybe_mkwrite(pte_mkyoung(pte_mkdirty(pte)),
                                              vma);
-                       ptep_establish(vma, address, page_table, entry, 1);
+                       ptep_set_access_flags(vma, address, page_table, entry, 1);
                        update_mmu_cache(vma, address, entry);
                        pte_unmap(page_table);
                        spin_unlock(&mm->page_table_lock);
@@ -1646,7 +1646,7 @@ static inline int handle_pte_fault(struct mm_struct *mm,
                entry = pte_mkdirty(entry);
        }
        entry = pte_mkyoung(entry);
-       ptep_establish(vma, address, pte, entry, write_access);
+       ptep_set_access_flags(vma, address, pte, entry, write_access);
        update_mmu_cache(vma, address, entry);
        pte_unmap(pte);
        spin_unlock(&mm->page_table_lock);