[PATCH] rmaplock: PageAnon in mapping

First of a batch of five patches to eliminate rmap's page_map_lock, replace
its trylocking by spinlocking, and use anon_vma to speed up swapoff.

Patches updated from the originals against 2.6.7-mm7: nothing new so I won't
spam the list, but including Manfred's SLAB_DESTROY_BY_RCU fixes, and omitting
the unuse_process mmap_sem fix already in 2.6.8-rc3.

This patch:

Replace the PG_anon page->flags bit by setting the lower bit of the pointer in
page->mapping when it's anon_vma: PAGE_MAPPING_ANON bit.

We're about to eliminate the locking which kept the flags and mapping in
synch: it's much easier to work on a local copy of page->mapping, than worry
about whether flags and mapping are in synch (though I imagine it could be
done, at greater cost, with some barriers).

Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 88b3cace0def6f2a5bcf3e20596113d225e20603..ff1aa78f9775fc2bddc78c0b64dcb2795d8cae8f 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -211,11 +211,12 @@ struct page {
                                         * if PagePrivate set; used for
                                         * swp_entry_t if PageSwapCache
                                         */
-       struct address_space *mapping;  /* If PG_anon clear, points to
+       struct address_space *mapping;  /* If low bit clear, points to
                                         * inode address_space, or NULL.
                                         * If page mapped as anonymous
-                                        * memory, PG_anon is set, and
-                                        * it points to anon_vma object.
+                                        * memory, low bit is set, and
+                                        * it points to anon_vma object:
+                                        * see PAGE_MAPPING_ANON below.
                                         */
        pgoff_t index;                  /* Our offset within mapping. */
        struct list_head lru;           /* Pageout list, eg. active_list
@@ -439,24 +440,32 @@ void page_address_init(void);
 
 /*
  * On an anonymous page mapped into a user virtual memory area,
- * page->mapping points to its anon_vma, not to a struct address_space.
+ * page->mapping points to its anon_vma, not to a struct address_space;
+ * with the PAGE_MAPPING_ANON bit set to distinguish it.
  *
  * Please note that, confusingly, "page_mapping" refers to the inode
  * address_space which maps the page from disk; whereas "page_mapped"
  * refers to user virtual address space into which the page is mapped.
  */
+#define PAGE_MAPPING_ANON      1
+
 extern struct address_space swapper_space;
 static inline struct address_space *page_mapping(struct page *page)
 {
-       struct address_space *mapping = NULL;
+       struct address_space *mapping = page->mapping;
 
        if (unlikely(PageSwapCache(page)))
                mapping = &swapper_space;
-       else if (likely(!PageAnon(page)))
-               mapping = page->mapping;
+       else if (unlikely((unsigned long)mapping & PAGE_MAPPING_ANON))
+               mapping = NULL;
        return mapping;
 }
 
+static inline int PageAnon(struct page *page)
+{
+       return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
+}
+
 /*
  * Return the pagecache index of the passed page.  Regular pagecache pages
  * use ->index whereas swapcache pages use ->private

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index c6f5063f0addf34d682a731b19cf4c81971945e3..0cc396d8b149cd73e6634633f1d4511a09918b29 100644 (file)
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -76,8 +76,6 @@
 #define PG_reclaim             18      /* To be reclaimed asap */
 #define PG_compound            19      /* Part of a compound page */
 
-#define PG_anon                        20      /* Anonymous: anon_vma in mapping */
-
 
 /*
  * Global page accounting.  One instance per CPU.  Only unsigned longs are
@@ -292,10 +290,6 @@ extern unsigned long __read_page_state(unsigned offset);
 #define SetPageCompound(page)  set_bit(PG_compound, &(page)->flags)
 #define ClearPageCompound(page)        clear_bit(PG_compound, &(page)->flags)
 
-#define PageAnon(page)         test_bit(PG_anon, &(page)->flags)
-#define SetPageAnon(page)      set_bit(PG_anon, &(page)->flags)
-#define ClearPageAnon(page)    clear_bit(PG_anon, &(page)->flags)
-
 #ifdef CONFIG_SWAP
 #define PageSwapCache(page)    test_bit(PG_swapcache, &(page)->flags)
 #define SetPageSwapCache(page) set_bit(PG_swapcache, &(page)->flags)

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index aa5cfa69e584a514c6c72a7e4f18c9479f9095fe..01812f28bd23ccf8101d8c629ac6d3d92eed7a63 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -88,7 +88,6 @@ static void bad_page(const char *function, struct page *page)
                        1 << PG_active  |
                        1 << PG_dirty   |
                        1 << PG_maplock |
-                       1 << PG_anon    |
                        1 << PG_swapcache |
                        1 << PG_writeback);
        set_page_count(page, 0);
@@ -230,7 +229,6 @@ static inline void free_pages_check(const char *function, struct page *page)
                        1 << PG_reclaim |
                        1 << PG_slab    |
                        1 << PG_maplock |
-                       1 << PG_anon    |
                        1 << PG_swapcache |
                        1 << PG_writeback )))
                bad_page(function, page);
@@ -353,7 +351,6 @@ static void prep_new_page(struct page *page, int order)
                        1 << PG_dirty   |
                        1 << PG_reclaim |
                        1 << PG_maplock |
-                       1 << PG_anon    |
                        1 << PG_swapcache |
                        1 << PG_writeback )))
                bad_page(__FUNCTION__, page);

diff --git a/mm/rmap.c b/mm/rmap.c
index 1cb3353daa16d3fffb9ab9bf7aa386c705b86797..a7c1ae8ff4048e396c756eafa0f1dc049007575d 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -168,7 +168,6 @@ static inline void clear_page_anon(struct page *page)
 {
        BUG_ON(!page->mapping);
        page->mapping = NULL;
-       ClearPageAnon(page);
 }
 
 /*
@@ -246,7 +245,7 @@ out:
 static inline int page_referenced_anon(struct page *page)
 {
        unsigned int mapcount = page->mapcount;
-       struct anon_vma *anon_vma = (struct anon_vma *) page->mapping;
+       struct anon_vma *anon_vma = (void *) page->mapping - PAGE_MAPPING_ANON;
        struct vm_area_struct *vma;
        int referenced = 0;
 
@@ -346,31 +345,18 @@ void page_add_anon_rmap(struct page *page,
        BUG_ON(PageReserved(page));
        BUG_ON(!anon_vma);
 
+       anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
        index = (address - vma->vm_start) >> PAGE_SHIFT;
        index += vma->vm_pgoff;
        index >>= PAGE_CACHE_SHIFT - PAGE_SHIFT;
 
-       /*
-        * Setting and clearing PG_anon must always happen inside
-        * page_map_lock to avoid races between mapping and
-        * unmapping on different processes of the same
-        * shared cow swapcache page. And while we take the
-        * page_map_lock PG_anon cannot change from under us.
-        * Actually PG_anon cannot change under fork either
-        * since fork holds a reference on the page so it cannot
-        * be unmapped under fork and in turn copy_page_range is
-        * allowed to read PG_anon outside the page_map_lock.
-        */
        page_map_lock(page);
        if (!page->mapcount) {
-               BUG_ON(PageAnon(page));
                BUG_ON(page->mapping);
-               SetPageAnon(page);
                page->index = index;
                page->mapping = (struct address_space *) anon_vma;
                inc_page_state(nr_mapped);
        } else {
-               BUG_ON(!PageAnon(page));
                BUG_ON(page->index != index);
                BUG_ON(page->mapping != (struct address_space *) anon_vma);
        }
@@ -629,7 +615,7 @@ out_unlock:
 
 static inline int try_to_unmap_anon(struct page *page)
 {
-       struct anon_vma *anon_vma = (struct anon_vma *) page->mapping;
+       struct anon_vma *anon_vma = (void *) page->mapping - PAGE_MAPPING_ANON;
        struct vm_area_struct *vma;
        int ret = SWAP_AGAIN;