From 6314b8e29bf462a34dde8f05d3eb1ec1a893a65c Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Tue, 25 May 2004 17:56:23 -0700 Subject: [PATCH] Split ptep_establish into "establish" and "update_access_flags" ptep_establish() is used to establish a new mapping at COW time, and it always replaces a non-writable page mapping with a totally new page mapping that is dirty (and likely writable, although ptrace may cause a non-writable new mapping). Because it was nonwritable, we don't have to worry about losing concurrent dirty page bit updates. ptep_update_access_flags() leaves the same page mapping, but updates the accessed/dirty/writable bits (it only ever sets them, and never removes any permissions). Often easier, but it may race with a dirty bit update on another CPU. Booted on x86 and ppc64. Signed-off-by: Benjamin Herrenschmidt Signed-off-by: Linus Torvalds --- include/asm-generic/pgtable.h | 30 +++++++++++++++++++++--------- include/asm-i386/pgtable.h | 10 +++++++--- include/asm-ppc/pgtable.h | 10 ++++++++++ include/asm-ppc64/pgtable.h | 35 ++++++++++++++++++++++++++++++++--- include/asm-s390/pgtable.h | 2 +- mm/memory.c | 6 +++--- 6 files changed, 74 insertions(+), 19 deletions(-) diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h index e9a387cdd37ab..ee74be68c19e9 100644 --- a/include/asm-generic/pgtable.h +++ b/include/asm-generic/pgtable.h @@ -2,26 +2,38 @@ #define _ASM_GENERIC_PGTABLE_H #ifndef __HAVE_ARCH_PTEP_ESTABLISH - -#ifndef ptep_update_dirty_accessed -#define ptep_update_dirty_accessed(__ptep, __entry, __dirty) set_pte(__ptep, __entry) -#endif - /* * Establish a new mapping: * - flush the old one * - update the page tables * - inform the TLB about the new one * - * We hold the mm semaphore for reading and vma->vm_mm->page_table_lock + * We hold the mm semaphore for reading and vma->vm_mm->page_table_lock. + * + * Note: the old pte is known to not be writable, so we don't need to + * worry about dirty bits etc getting lost. */ -#define ptep_establish(__vma, __address, __ptep, __entry, __dirty) \ -do { \ - ptep_update_dirty_accessed(__ptep, __entry, __dirty); \ +#define ptep_establish(__vma, __address, __ptep, __entry) \ +do { \ + set_pte(__ptep, __entry); \ flush_tlb_page(__vma, __address); \ } while (0) #endif +#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +/* + * Largely same as above, but only sets the access flags (dirty, + * accessed, and writable). Furthermore, we know it always gets set + * to a "more permissive" setting, which allows most architectures + * to optimize this. + */ +#define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \ +do { \ + set_pte(__ptep, __entry); \ + flush_tlb_page(__vma, __address); \ +} while (0) +#endif + #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG static inline int ptep_test_and_clear_young(pte_t *ptep) { diff --git a/include/asm-i386/pgtable.h b/include/asm-i386/pgtable.h index c408168b22e29..299c7587140a9 100644 --- a/include/asm-i386/pgtable.h +++ b/include/asm-i386/pgtable.h @@ -325,9 +325,13 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) * bit at the same time. */ #define update_mmu_cache(vma,address,pte) do { } while (0) -#define ptep_update_dirty_accessed(__ptep, __entry, __dirty) \ - do { \ - if (__dirty) set_pte(__ptep, __entry); \ +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +#define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \ + do { \ + if (__dirty) { \ + (__ptep)->pte_low = (__entry).pte_low; \ + flush_tlb_page(__vma, __address); \ + } \ } while (0) /* Encode and de-code a swap entry */ diff --git a/include/asm-ppc/pgtable.h b/include/asm-ppc/pgtable.h index 8eefa54b86feb..976796d2f4c6e 100644 --- a/include/asm-ppc/pgtable.h +++ b/include/asm-ppc/pgtable.h @@ -548,6 +548,16 @@ static inline void ptep_mkdirty(pte_t *ptep) pte_update(ptep, 0, _PAGE_DIRTY); } +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry, int dirty) +{ + unsigned long bits = pte_val(entry) & + (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW); + pte_update(ptep, 0, bits); +} +#define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \ + __ptep_set_access_flags(__ptep, __entry, __dirty) + /* * Macro to mark a page protection value as "uncacheable". */ diff --git a/include/asm-ppc64/pgtable.h b/include/asm-ppc64/pgtable.h index 5903d4fe3b5e0..cb7f53d6c8e5d 100644 --- a/include/asm-ppc64/pgtable.h +++ b/include/asm-ppc64/pgtable.h @@ -306,7 +306,10 @@ static inline unsigned long pte_update(pte_t *p, unsigned long clr) return old; } -/* PTE updating functions */ +/* PTE updating functions, this function puts the PTE in the + * batch, doesn't actually triggers the hash flush immediately, + * you need to call flush_tlb_pending() to do that. + */ extern void hpte_update(pte_t *ptep, unsigned long pte, int wrprot); static inline int ptep_test_and_clear_young(pte_t *ptep) @@ -318,7 +321,7 @@ static inline int ptep_test_and_clear_young(pte_t *ptep) old = pte_update(ptep, _PAGE_ACCESSED); if (old & _PAGE_HASHPTE) { hpte_update(ptep, old, 0); - flush_tlb_pending(); /* XXX generic code doesn't flush */ + flush_tlb_pending(); } return (old & _PAGE_ACCESSED) != 0; } @@ -396,11 +399,37 @@ static inline void pte_clear(pte_t * ptep) */ static inline void set_pte(pte_t *ptep, pte_t pte) { - if (pte_present(*ptep)) + if (pte_present(*ptep)) { pte_clear(ptep); + flush_tlb_pending(); + } *ptep = __pte(pte_val(pte)) & ~_PAGE_HPTEFLAGS; } +/* Set the dirty and/or accessed bits atomically in a linux PTE, this + * function doesn't need to flush the hash entry + */ +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry, int dirty) +{ + unsigned long bits = pte_val(entry) & + (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW); + unsigned long old, tmp; + + __asm__ __volatile__( + "1: ldarx %0,0,%4\n\ + andi. %1,%0,%6\n\ + bne- 1b \n\ + or %0,%3,%0\n\ + stdcx. %0,0,%4\n\ + bne- 1b" + :"=&r" (old), "=&r" (tmp), "=m" (*ptep) + :"r" (bits), "r" (ptep), "m" (ptep), "i" (_PAGE_BUSY) + :"cc"); +} +#define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \ + __ptep_set_access_flags(__ptep, __entry, __dirty) + /* * Macro to mark a page protection value as "uncacheable". */ diff --git a/include/asm-s390/pgtable.h b/include/asm-s390/pgtable.h index 959c67a9592c6..ebd93e484dda6 100644 --- a/include/asm-s390/pgtable.h +++ b/include/asm-s390/pgtable.h @@ -581,7 +581,7 @@ static inline void ptep_mkdirty(pte_t *ptep) static inline void ptep_establish(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, - pte_t entry, int dirty) + pte_t entry) { ptep_clear_flush(vma, address, ptep); set_pte(ptep, entry); diff --git a/mm/memory.c b/mm/memory.c index 2ae115e4bd092..5780707589052 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1004,7 +1004,7 @@ static inline void break_cow(struct vm_area_struct * vma, struct page * new_page flush_cache_page(vma, address); entry = maybe_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot)), vma); - ptep_establish(vma, address, page_table, entry, 1); + ptep_establish(vma, address, page_table, entry); update_mmu_cache(vma, address, entry); } @@ -1056,7 +1056,7 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct * vma, flush_cache_page(vma, address); entry = maybe_mkwrite(pte_mkyoung(pte_mkdirty(pte)), vma); - ptep_establish(vma, address, page_table, entry, 1); + ptep_set_access_flags(vma, address, page_table, entry, 1); update_mmu_cache(vma, address, entry); pte_unmap(page_table); spin_unlock(&mm->page_table_lock); @@ -1646,7 +1646,7 @@ static inline int handle_pte_fault(struct mm_struct *mm, entry = pte_mkdirty(entry); } entry = pte_mkyoung(entry); - ptep_establish(vma, address, pte, entry, write_access); + ptep_set_access_flags(vma, address, pte, entry, write_access); update_mmu_cache(vma, address, entry); pte_unmap(pte); spin_unlock(&mm->page_table_lock); -- 2.53.0