[PATCH] numa api: Add VMA hooks for policy

From: Andi Kleen <ak@suse.de>

NUMA API adds a policy to each VMA.  During VMA creattion, merging and
splitting these policies must be handled properly.  This patch adds the calls
to this.

It is a nop when CONFIG_NUMA is not defined.

diff --git a/arch/ia64/ia32/binfmt_elf32.c b/arch/ia64/ia32/binfmt_elf32.c
index 679e68afd653dd381fc607c7170fe3701c098a57..44952a0ef8cda92b0ee549fcb4b7af3927200970 100644 (file)
--- a/arch/ia64/ia32/binfmt_elf32.c
+++ b/arch/ia64/ia32/binfmt_elf32.c
@@ -104,6 +104,7 @@ ia64_elf32_init (struct pt_regs *regs)
                vma->vm_pgoff = 0;
                vma->vm_file = NULL;
                vma->vm_private_data = NULL;
+               mpol_set_vma_default(vma);
                down_write(&current->mm->mmap_sem);
                {
                        insert_vm_struct(current->mm, vma);
@@ -190,6 +191,7 @@ ia32_setup_arg_pages (struct linux_binprm *bprm, int executable_stack)
                mpnt->vm_pgoff = 0;
                mpnt->vm_file = NULL;
                mpnt->vm_private_data = 0;
+               mpol_set_vma_default(mpnt);
                insert_vm_struct(current->mm, mpnt);
                current->mm->total_vm = (mpnt->vm_end - mpnt->vm_start) >> PAGE_SHIFT;
        }

diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c
index 0a591d316b6dfeb274a78c8f7e90f310b94fbeb0..3365f23ffc914ff8e60875eaa6c5c5502e907fab 100644 (file)
--- a/arch/ia64/kernel/perfmon.c
+++ b/arch/ia64/kernel/perfmon.c
@@ -2321,6 +2321,7 @@ pfm_smpl_buffer_alloc(struct task_struct *task, pfm_context_t *ctx, unsigned lon
        vma->vm_ops          = NULL;
        vma->vm_pgoff        = 0;
        vma->vm_file         = NULL;
+       mpol_set_vma_default(vma);
        vma->vm_private_data = NULL; 
 
        /*

diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c
index a764a25c2b982aa93f5d5bf6889815ae102cfcd5..673d73bfa7b76d720e82b049363db203bb08d1c0 100644 (file)
--- a/arch/ia64/mm/init.c
+++ b/arch/ia64/mm/init.c
@@ -132,6 +132,7 @@ ia64_init_addr_space (void)
                vma->vm_pgoff = 0;
                vma->vm_file = NULL;
                vma->vm_private_data = NULL;
+               mpol_set_vma_default(vma);
                insert_vm_struct(current->mm, vma);
        }
 
@@ -144,6 +145,7 @@ ia64_init_addr_space (void)
                        vma->vm_end = PAGE_SIZE;
                        vma->vm_page_prot = __pgprot(pgprot_val(PAGE_READONLY) | _PAGE_MA_NAT);
                        vma->vm_flags = VM_READ | VM_MAYREAD | VM_IO | VM_RESERVED;
+                       mpol_set_vma_default(vma);
                        insert_vm_struct(current->mm, vma);
                }
        }

diff --git a/arch/m68k/atari/stram.c b/arch/m68k/atari/stram.c
index ae6e059a715d7cb820326b08ea4869c7187a1c05..b03d031634969408b79a6ed0f905fbe3c89ceb75 100644 (file)
--- a/arch/m68k/atari/stram.c
+++ b/arch/m68k/atari/stram.c
@@ -752,7 +752,7 @@ static int unswap_by_read(unsigned short *map, unsigned long max,
                        /* Get a page for the entry, using the existing
                           swap cache page if there is one.  Otherwise,
                           get a clean page and read the swap into it. */
-                       page = read_swap_cache_async(entry);
+                       page = read_swap_cache_async(entry, NULL, 0);
                        if (!page) {
                                swap_free(entry);
                                return -ENOMEM;

diff --git a/arch/s390/kernel/compat_exec.c b/arch/s390/kernel/compat_exec.c
index d8e378cbed03ddc361bbccea93e861028def62f0..3ee7f93a34a191dc54a4a2d48941400ab96b6788 100644 (file)
--- a/arch/s390/kernel/compat_exec.c
+++ b/arch/s390/kernel/compat_exec.c
@@ -69,6 +69,7 @@ int setup_arg_pages32(struct linux_binprm *bprm, int executable_stack)
                mpnt->vm_ops = NULL;
                mpnt->vm_pgoff = 0;
                mpnt->vm_file = NULL;
+               mpol_set_vma_default(mpnt);
                INIT_LIST_HEAD(&mpnt->shared);
                mpnt->vm_private_data = (void *) 0;
                insert_vm_struct(mm, mpnt);

diff --git a/arch/x86_64/ia32/ia32_binfmt.c b/arch/x86_64/ia32/ia32_binfmt.c
index 92817f18e39b919c8cf0d28c14f91558b2696ef5..af8d595da912a04a3860fcb2fd6b4cdece04d9ba 100644 (file)
--- a/arch/x86_64/ia32/ia32_binfmt.c
+++ b/arch/x86_64/ia32/ia32_binfmt.c
@@ -365,6 +365,7 @@ int setup_arg_pages(struct linux_binprm *bprm, int executable_stack)
                mpnt->vm_ops = NULL;
                mpnt->vm_pgoff = 0;
                mpnt->vm_file = NULL;
+               mpol_set_vma_default(mpnt);
                INIT_LIST_HEAD(&mpnt->shared);
                mpnt->vm_private_data = (void *) 0;
                insert_vm_struct(mm, mpnt);

diff --git a/fs/exec.c b/fs/exec.c
index b7e544a0b5edbc2fd4598d574cb81d1bf7b3551a..0e6fdcf20984884ab92c17c0a57024f7a94cc39f 100644 (file)
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -427,6 +427,7 @@ int setup_arg_pages(struct linux_binprm *bprm, int executable_stack)
                mpnt->vm_ops = NULL;
                mpnt->vm_pgoff = 0;
                mpnt->vm_file = NULL;
+               mpol_set_vma_default(mpnt);
                INIT_LIST_HEAD(&mpnt->shared);
                mpnt->vm_private_data = (void *) 0;
                insert_vm_struct(mm, mpnt);

diff --git a/kernel/exit.c b/kernel/exit.c
index d439e608f3435b21fd3c79523f885e3f4309b8bd..5ac2db963f0b2fcdcdf61337cf5250101854d43b 100644 (file)
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -791,6 +791,7 @@ asmlinkage NORET_TYPE void do_exit(long code)
        __exit_fs(tsk);
        exit_namespace(tsk);
        exit_thread();
+       mpol_free(tsk->mempolicy);
 
        if (tsk->signal->leader)
                disassociate_ctty(1);

diff --git a/kernel/fork.c b/kernel/fork.c
index 3cb3bc41b0c0858ee45f812df7fdf987a747a5f4..2297ac8d0a0fb96432c770f5bddb2389477bf1b8 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -271,6 +271,7 @@ static inline int dup_mmap(struct mm_struct * mm, struct mm_struct * oldmm)
        struct rb_node **rb_link, *rb_parent;
        int retval;
        unsigned long charge = 0;
+       struct mempolicy *pol;
 
        down_write(&oldmm->mmap_sem);
        flush_cache_mm(current->mm);
@@ -312,6 +313,11 @@ static inline int dup_mmap(struct mm_struct * mm, struct mm_struct * oldmm)
                if (!tmp)
                        goto fail_nomem;
                *tmp = *mpnt;
+               pol = mpol_copy(vma_policy(mpnt));
+               retval = PTR_ERR(pol);
+               if (IS_ERR(pol))
+                       goto fail_nomem_policy;
+               vma_set_policy(tmp, pol);
                tmp->vm_flags &= ~VM_LOCKED;
                tmp->vm_mm = mm;
                tmp->vm_next = NULL;
@@ -358,6 +364,8 @@ out:
        flush_tlb_mm(current->mm);
        up_write(&oldmm->mmap_sem);
        return retval;
+fail_nomem_policy:
+       kmem_cache_free(vm_area_cachep, tmp);
 fail_nomem:
        retval = -ENOMEM;
 fail:
@@ -964,10 +972,16 @@ struct task_struct *copy_process(unsigned long clone_flags,
        p->security = NULL;
        p->io_context = NULL;
        p->audit_context = NULL;
+       p->mempolicy = mpol_copy(p->mempolicy);
+       if (IS_ERR(p->mempolicy)) {
+               retval = PTR_ERR(p->mempolicy);
+               p->mempolicy = NULL;
+               goto bad_fork_cleanup;
+       }
 
        retval = -ENOMEM;
        if ((retval = security_task_alloc(p)))
-               goto bad_fork_cleanup;
+               goto bad_fork_cleanup_policy;
        if ((retval = audit_alloc(p)))
                goto bad_fork_cleanup_security;
        /* copy all the process information */
@@ -1113,6 +1127,8 @@ bad_fork_cleanup_audit:
        audit_free(p);
 bad_fork_cleanup_security:
        security_task_free(p);
+bad_fork_cleanup_policy:
+       mpol_free(p->mempolicy);
 bad_fork_cleanup:
        if (p->pid > 0)
                free_pidmap(p->pid);

diff --git a/mm/mmap.c b/mm/mmap.c
index cafc5fc389b44d904ea24d73185eeaf2e1df51bc..0a570ac3b4de8003ccc986c14d23b5eb2131928d 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -387,7 +387,8 @@ static struct vm_area_struct *vma_merge(struct mm_struct *mm,
                        struct vm_area_struct *prev,
                        struct rb_node *rb_parent, unsigned long addr, 
                        unsigned long end, unsigned long vm_flags,
-                       struct file *file, unsigned long pgoff)
+                       struct file *file, unsigned long pgoff,
+                       struct mempolicy *policy)
 {
        spinlock_t *lock = &mm->page_table_lock;
        struct inode *inode = file ? file->f_dentry->d_inode : NULL;
@@ -411,6 +412,7 @@ static struct vm_area_struct *vma_merge(struct mm_struct *mm,
         * Can it merge with the predecessor?
         */
        if (prev->vm_end == addr &&
+                       mpol_equal(vma_policy(prev), policy) &&
                        can_vma_merge_after(prev, vm_flags, file, pgoff)) {
                struct vm_area_struct *next;
                int need_up = 0;
@@ -428,6 +430,7 @@ static struct vm_area_struct *vma_merge(struct mm_struct *mm,
                 */
                next = prev->vm_next;
                if (next && prev->vm_end == next->vm_start &&
+                               vma_mpol_equal(prev, next) &&
                                can_vma_merge_before(next, vm_flags, file,
                                        pgoff, (end - addr) >> PAGE_SHIFT)) {
                        prev->vm_end = next->vm_end;
@@ -440,6 +443,7 @@ static struct vm_area_struct *vma_merge(struct mm_struct *mm,
                                fput(file);
 
                        mm->map_count--;
+                       mpol_free(vma_policy(next));
                        kmem_cache_free(vm_area_cachep, next);
                        return prev;
                }
@@ -455,6 +459,8 @@ static struct vm_area_struct *vma_merge(struct mm_struct *mm,
        prev = prev->vm_next;
        if (prev) {
  merge_next:
+               if (!mpol_equal(policy, vma_policy(prev)))
+                       return 0;
                if (!can_vma_merge_before(prev, vm_flags, file,
                                pgoff, (end - addr) >> PAGE_SHIFT))
                        return NULL;
@@ -631,7 +637,7 @@ munmap_back:
        /* Can we just expand an old anonymous mapping? */
        if (!file && !(vm_flags & VM_SHARED) && rb_parent)
                if (vma_merge(mm, prev, rb_parent, addr, addr + len,
-                                       vm_flags, NULL, 0))
+                                       vm_flags, NULL, pgoff, NULL))
                        goto out;
 
        /*
@@ -654,6 +660,7 @@ munmap_back:
        vma->vm_file = NULL;
        vma->vm_private_data = NULL;
        vma->vm_next = NULL;
+       mpol_set_vma_default(vma);
        INIT_LIST_HEAD(&vma->shared);
 
        if (file) {
@@ -693,7 +700,9 @@ munmap_back:
        addr = vma->vm_start;
 
        if (!file || !rb_parent || !vma_merge(mm, prev, rb_parent, addr,
-                               addr + len, vma->vm_flags, file, pgoff)) {
+                                             vma->vm_end,
+                                             vma->vm_flags, file, pgoff,
+                                             vma_policy(vma))) {
                vma_link(mm, vma, prev, rb_link, rb_parent);
                if (correct_wcount)
                        atomic_inc(&inode->i_writecount);
@@ -703,6 +712,7 @@ munmap_back:
                                atomic_inc(&inode->i_writecount);
                        fput(file);
                }
+               mpol_free(vma_policy(vma));
                kmem_cache_free(vm_area_cachep, vma);
        }
 out:   
@@ -1118,6 +1128,7 @@ static void unmap_vma(struct mm_struct *mm, struct vm_area_struct *area)
 
        remove_shared_vm_struct(area);
 
+       mpol_free(vma_policy(area));
        if (area->vm_ops && area->vm_ops->close)
                area->vm_ops->close(area);
        if (area->vm_file)
@@ -1200,6 +1211,7 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
 int split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
              unsigned long addr, int new_below)
 {
+       struct mempolicy *pol;
        struct vm_area_struct *new;
        struct address_space *mapping = NULL;
 
@@ -1222,6 +1234,13 @@ int split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
                new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
        }
 
+       pol = mpol_copy(vma_policy(vma));
+       if (IS_ERR(pol)) {
+               kmem_cache_free(vm_area_cachep, new);
+               return PTR_ERR(pol);
+       }
+       vma_set_policy(new, pol);
+
        if (new->vm_file)
                get_file(new->vm_file);
 
@@ -1391,7 +1410,7 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
 
        /* Can we just expand an old anonymous mapping? */
        if (rb_parent && vma_merge(mm, prev, rb_parent, addr, addr + len,
-                                       flags, NULL, 0))
+                                       flags, NULL, 0, NULL))
                goto out;
 
        /*
@@ -1412,6 +1431,7 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
        vma->vm_pgoff = 0;
        vma->vm_file = NULL;
        vma->vm_private_data = NULL;
+       mpol_set_vma_default(vma);
        INIT_LIST_HEAD(&vma->shared);
 
        vma_link(mm, vma, prev, rb_link, rb_parent);
@@ -1472,6 +1492,7 @@ void exit_mmap(struct mm_struct *mm)
                }
                if (vma->vm_file)
                        fput(vma->vm_file);
+               mpol_free(vma_policy(vma));
                kmem_cache_free(vm_area_cachep, vma);
                vma = next;
        }
@@ -1508,7 +1529,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
 
        find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
        new_vma = vma_merge(mm, prev, rb_parent, addr, addr + len,
-                       vma->vm_flags, vma->vm_file, pgoff);
+                       vma->vm_flags, vma->vm_file, pgoff, vma_policy(vma));
        if (new_vma) {
                /*
                 * Source vma may have been merged into new_vma

diff --git a/mm/mprotect.c b/mm/mprotect.c
index dcc9418d9f4e39bb5991210792229965adf04f47..d7c3e33a855f506141141dc927876c39e1ecb6cb 100644 (file)
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -125,6 +125,8 @@ mprotect_attempt_merge(struct vm_area_struct *vma, struct vm_area_struct *prev,
                return 0;
        if (vma->vm_file || (vma->vm_flags & VM_SHARED))
                return 0;
+       if (!vma_mpol_equal(vma, prev))
+               return 0;
 
        /*
         * If the whole area changes to the protection of the previous one
@@ -136,6 +138,7 @@ mprotect_attempt_merge(struct vm_area_struct *vma, struct vm_area_struct *prev,
                __vma_unlink(mm, vma, prev);
                spin_unlock(&mm->page_table_lock);
 
+               mpol_free(vma_policy(vma));
                kmem_cache_free(vm_area_cachep, vma);
                mm->map_count--;
                return 1;
@@ -317,12 +320,14 @@ sys_mprotect(unsigned long start, size_t len, unsigned long prot)
 
        if (next && prev->vm_end == next->vm_start &&
                        can_vma_merge(next, prev->vm_flags) &&
+               vma_mpol_equal(prev, next) &&
                        !prev->vm_file && !(prev->vm_flags & VM_SHARED)) {
                spin_lock(&prev->vm_mm->page_table_lock);
                prev->vm_end = next->vm_end;
                __vma_unlink(prev->vm_mm, next, prev);
                spin_unlock(&prev->vm_mm->page_table_lock);
 
+               mpol_free(vma_policy(next));
                kmem_cache_free(vm_area_cachep, next);
                prev->vm_mm->map_count--;
        }