#include <asm/tlbflush.h>
+/*
+ * struct anonmm: to track a bundle of anonymous memory mappings.
+ *
+ * Could be embedded in mm_struct, but mm_struct is rather heavyweight,
+ * and we may need the anonmm to stay around long after the mm_struct
+ * and its pgd have been freed: because pages originally faulted into
+ * that mm have been duped into forked mms, and still need tracking.
+ */
+struct anonmm {
+ atomic_t count; /* ref count, including 1 per page */
+ spinlock_t lock; /* head's locks list; others unused */
+ struct mm_struct *mm; /* assoc mm_struct, NULL when gone */
+ struct anonmm *head; /* exec starts new chain from head */
+ struct list_head list; /* chain of associated anonmms */
+};
+static kmem_cache_t *anonmm_cachep;
+
+/**
+ ** Functions for creating and destroying struct anonmm.
+ **/
+
+void __init init_rmap(void)
+{
+ anonmm_cachep = kmem_cache_create("anonmm",
+ sizeof(struct anonmm), 0, SLAB_PANIC, NULL, NULL);
+}
+
+int exec_rmap(struct mm_struct *mm)
+{
+ struct anonmm *anonmm;
+
+ anonmm = kmem_cache_alloc(anonmm_cachep, SLAB_KERNEL);
+ if (unlikely(!anonmm))
+ return -ENOMEM;
+
+ atomic_set(&anonmm->count, 2); /* ref by mm and head */
+ anonmm->lock = SPIN_LOCK_UNLOCKED; /* this lock is used */
+ anonmm->mm = mm;
+ anonmm->head = anonmm;
+ INIT_LIST_HEAD(&anonmm->list);
+ mm->anonmm = anonmm;
+ return 0;
+}
+
+int dup_rmap(struct mm_struct *mm, struct mm_struct *oldmm)
+{
+ struct anonmm *anonmm;
+ struct anonmm *anonhd = oldmm->anonmm->head;
+
+ anonmm = kmem_cache_alloc(anonmm_cachep, SLAB_KERNEL);
+ if (unlikely(!anonmm))
+ return -ENOMEM;
+
+ /*
+ * copy_mm calls us before dup_mmap has reset the mm fields,
+ * so reset rss ourselves before adding to anonhd's list,
+ * to keep away from this mm until it's worth examining.
+ */
+ mm->rss = 0;
+
+ atomic_set(&anonmm->count, 1); /* ref by mm */
+ anonmm->lock = SPIN_LOCK_UNLOCKED; /* this lock is not used */
+ anonmm->mm = mm;
+ anonmm->head = anonhd;
+ spin_lock(&anonhd->lock);
+ atomic_inc(&anonhd->count); /* ref by anonmm's head */
+ list_add_tail(&anonmm->list, &anonhd->list);
+ spin_unlock(&anonhd->lock);
+ mm->anonmm = anonmm;
+ return 0;
+}
+
+void exit_rmap(struct mm_struct *mm)
+{
+ struct anonmm *anonmm = mm->anonmm;
+ struct anonmm *anonhd = anonmm->head;
+ int anonhd_count;
+
+ mm->anonmm = NULL;
+ spin_lock(&anonhd->lock);
+ anonmm->mm = NULL;
+ if (atomic_dec_and_test(&anonmm->count)) {
+ BUG_ON(anonmm == anonhd);
+ list_del(&anonmm->list);
+ kmem_cache_free(anonmm_cachep, anonmm);
+ if (atomic_dec_and_test(&anonhd->count))
+ BUG();
+ }
+ anonhd_count = atomic_read(&anonhd->count);
+ spin_unlock(&anonhd->lock);
+ if (anonhd_count == 1) {
+ BUG_ON(anonhd->mm);
+ BUG_ON(!list_empty(&anonhd->list));
+ kmem_cache_free(anonmm_cachep, anonhd);
+ }
+}
+
+static void free_anonmm(struct anonmm *anonmm)
+{
+ struct anonmm *anonhd = anonmm->head;
+
+ BUG_ON(anonmm->mm);
+ BUG_ON(anonmm == anonhd);
+ spin_lock(&anonhd->lock);
+ list_del(&anonmm->list);
+ if (atomic_dec_and_test(&anonhd->count))
+ BUG();
+ spin_unlock(&anonhd->lock);
+ kmem_cache_free(anonmm_cachep, anonmm);
+}
+
static inline void clear_page_anon(struct page *page)
{
+ struct anonmm *anonmm = (struct anonmm *) page->mapping;
+
page->mapping = NULL;
ClearPageAnon(page);
+ if (atomic_dec_and_test(&anonmm->count))
+ free_anonmm(anonmm);
}
/**
static inline int page_referenced_anon(struct page *page)
{
- return 1; /* until next patch */
+ unsigned int mapcount = page->mapcount;
+ struct anonmm *anonmm = (struct anonmm *) page->mapping;
+ struct anonmm *anonhd = anonmm->head;
+ struct anonmm *new_anonmm = anonmm;
+ struct list_head *seek_head;
+ int referenced = 0;
+ int failed = 0;
+
+ spin_lock(&anonhd->lock);
+ /*
+ * First try the indicated mm, it's the most likely.
+ * Make a note to migrate the page if this mm is extinct.
+ */
+ if (!anonmm->mm)
+ new_anonmm = NULL;
+ else if (anonmm->mm->rss) {
+ referenced += page_referenced_one(page,
+ anonmm->mm, page->index, &mapcount, &failed);
+ if (!mapcount)
+ goto out;
+ }
+
+ /*
+ * Then down the rest of the list, from that as the head. Stop
+ * when we reach anonhd? No: although a page cannot get dup'ed
+ * into an older mm, once swapped, its indicated mm may not be
+ * the oldest, just the first into which it was faulted back.
+ * If original mm now extinct, note first to contain the page.
+ */
+ seek_head = &anonmm->list;
+ list_for_each_entry(anonmm, seek_head, list) {
+ if (!anonmm->mm || !anonmm->mm->rss)
+ continue;
+ referenced += page_referenced_one(page,
+ anonmm->mm, page->index, &mapcount, &failed);
+ if (!new_anonmm && mapcount < page->mapcount)
+ new_anonmm = anonmm;
+ if (!mapcount) {
+ anonmm = (struct anonmm *) page->mapping;
+ if (new_anonmm == anonmm)
+ goto out;
+ goto migrate;
+ }
+ }
+
+ WARN_ON(!failed);
+out:
+ spin_unlock(&anonhd->lock);
+ return referenced;
+
+migrate:
+ /*
+ * Migrate pages away from an extinct mm, so that its anonmm
+ * can be freed in due course: we could leave this to happen
+ * through the natural attrition of try_to_unmap, but that
+ * would miss locked pages and frequently referenced pages.
+ */
+ spin_unlock(&anonhd->lock);
+ page->mapping = (void *) new_anonmm;
+ atomic_inc(&new_anonmm->count);
+ if (atomic_dec_and_test(&anonmm->count))
+ free_anonmm(anonmm);
+ return referenced;
}
/**
void fastcall page_add_anon_rmap(struct page *page,
struct mm_struct *mm, unsigned long address)
{
+ struct anonmm *anonmm = mm->anonmm;
+
BUG_ON(PageReserved(page));
page_map_lock(page);
BUG_ON(page->mapping);
SetPageAnon(page);
page->index = address & PAGE_MASK;
- page->mapping = (void *) mm; /* until next patch */
+ page->mapping = (void *) anonmm;
+ atomic_inc(&anonmm->count);
inc_page_state(nr_mapped);
}
page->mapcount++;
(*mapcount)--;
+ if (!vma) {
+ vma = find_vma(mm, address);
+ /* unmap_vmas drops page_table_lock with vma unlinked */
+ if (!vma)
+ goto out_unmap;
+ }
+
/*
* If the page is mlock()d, we cannot swap it out.
* If it's recently referenced (perhaps page_referenced
if (pte_dirty(pteval))
set_page_dirty(page);
+ if (PageAnon(page)) {
+ swp_entry_t entry = { .val = page->private };
+ /*
+ * Store the swap location in the pte.
+ * See handle_pte_fault() ...
+ */
+ BUG_ON(!PageSwapCache(page));
+ swap_duplicate(entry);
+ set_pte(pte, swp_entry_to_pte(entry));
+ BUG_ON(pte_file(*pte));
+ }
+
mm->rss--;
BUG_ON(!page->mapcount);
page->mapcount--;
static inline int try_to_unmap_anon(struct page *page)
{
- return SWAP_FAIL; /* until next patch */
+ unsigned int mapcount = page->mapcount;
+ struct anonmm *anonmm = (struct anonmm *) page->mapping;
+ struct anonmm *anonhd = anonmm->head;
+ struct list_head *seek_head;
+ int ret = SWAP_AGAIN;
+
+ spin_lock(&anonhd->lock);
+ /*
+ * First try the indicated mm, it's the most likely.
+ */
+ if (anonmm->mm && anonmm->mm->rss) {
+ ret = try_to_unmap_one(page,
+ anonmm->mm, page->index, &mapcount, NULL);
+ if (ret == SWAP_FAIL || !mapcount)
+ goto out;
+ }
+
+ /*
+ * Then down the rest of the list, from that as the head. Stop
+ * when we reach anonhd? No: although a page cannot get dup'ed
+ * into an older mm, once swapped, its indicated mm may not be
+ * the oldest, just the first into which it was faulted back.
+ */
+ seek_head = &anonmm->list;
+ list_for_each_entry(anonmm, seek_head, list) {
+ if (!anonmm->mm || !anonmm->mm->rss)
+ continue;
+ ret = try_to_unmap_one(page,
+ anonmm->mm, page->index, &mapcount, NULL);
+ if (ret == SWAP_FAIL || !mapcount)
+ goto out;
+ }
+out:
+ spin_unlock(&anonhd->lock);
+ return ret;
}
/**
projects / linux / commitdiff