[PATCH] percpu: convert global page accounting

Convert global page state accounting to use per-cpu storage

(I think this code remains a little buggy, btw.  Note how I do

	per_cpu(page_states, cpu).member += (delta);

This gets done at interrupt time and hence is assuming that
the "+=" operation on a ulong is atomic wrt interrupts on
all architectures. How do we feel about that assumption?)

diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index 939f16910233f7bc35a5ee1fe1517129b304faa4..c340b447a96339e2214b194d38b5eac89a003260 100644 (file)
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -86,4 +86,6 @@ extern void FASTCALL(free_pages(unsigned long addr, unsigned int order));
 #define __free_page(page) __free_pages((page), 0)
 #define free_page(addr) free_pages((addr),0)
 
+void page_alloc_init(void);
+
 #endif /* __LINUX_GFP_H */

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 5c770f49787aed3b0603327cceb9790303c58a5a..282902bb9816f8fa655808d0d5e6cd91974e59e3 100644 (file)
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -5,6 +5,8 @@
 #ifndef PAGE_FLAGS_H
 #define PAGE_FLAGS_H
 
+#include <linux/percpu.h>
+
 /*
  * Various page->flags bits:
  *
@@ -73,7 +75,7 @@
  * Global page accounting.  One instance per CPU.  Only unsigned longs are
  * allowed.
  */
-extern struct page_state {
+struct page_state {
        unsigned long nr_dirty;
        unsigned long nr_writeback;
        unsigned long nr_pagecache;
@@ -103,7 +105,9 @@ extern struct page_state {
        unsigned long kswapd_steal;
        unsigned long pageoutrun;
        unsigned long allocstall;
-} ____cacheline_aligned_in_smp page_states[NR_CPUS];
+};
+
+DECLARE_PER_CPU(struct page_state, page_states);
 
 extern void get_page_state(struct page_state *ret);
 extern void get_full_page_state(struct page_state *ret);
@@ -111,7 +115,7 @@ extern void get_full_page_state(struct page_state *ret);
 #define mod_page_state(member, delta)                                  \
        do {                                                            \
                int cpu = get_cpu();                                    \
-               page_states[cpu].member += (delta);                     \
+               per_cpu(page_states, cpu).member += (delta);            \
                put_cpu();                                              \
        } while (0)
 

diff --git a/init/main.c b/init/main.c
index 97d88c50366b9a71aa00cbcf96a2af2491f3e7e4..d75bf0acb1ebb1f45beaf3c6a115af790e5e67a6 100644 (file)
--- a/init/main.c
+++ b/init/main.c
@@ -26,6 +26,7 @@
 #include <linux/hdreg.h>
 #include <linux/bootmem.h>
 #include <linux/tty.h>
+#include <linux/gfp.h>
 #include <linux/percpu.h>
 #include <linux/kernel_stat.h>
 #include <linux/security.h>
@@ -388,6 +389,7 @@ asmlinkage void __init start_kernel(void)
        setup_arch(&command_line);
        setup_per_cpu_areas();
        build_all_zonelists();
+       page_alloc_init();
        printk("Kernel command line: %s\n", saved_command_line);
        parse_options(command_line);
        trap_init();

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 848b1ed8f0012d54536abbad25b04b235212e9aa..0e7425d56652fb2d074eb92615d43e5418da7557 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -25,6 +25,7 @@
 #include <linux/pagevec.h>
 #include <linux/blkdev.h>
 #include <linux/slab.h>
+#include <linux/notifier.h>
 
 struct pglist_data *pgdat_list;
 unsigned long totalram_pages;
@@ -573,8 +574,8 @@ unsigned int nr_free_highpages (void)
  * The result is unavoidably approximate - it can change
  * during and after execution of this function.
  */
-struct page_state page_states[NR_CPUS] __cacheline_aligned;
-EXPORT_SYMBOL(page_states);
+DEFINE_PER_CPU(struct page_state, page_states) = {0};
+EXPORT_PER_CPU_SYMBOL(page_states);
 
 void __get_page_state(struct page_state *ret, int nr)
 {
@@ -587,7 +588,7 @@ void __get_page_state(struct page_state *ret, int nr)
                if (!cpu_online(cpu))
                        continue;
 
-               in = (unsigned long *)(page_states + cpu);
+               in = (unsigned long *)&per_cpu(page_states, cpu);
                out = (unsigned long *)ret;
                for (off = 0; off < nr; off++)
                        *out++ += *in++;
@@ -1197,3 +1198,33 @@ struct seq_operations vmstat_op = {
 };
 
 #endif /* CONFIG_PROC_FS */
+
+static void __devinit init_page_alloc_cpu(int cpu)
+{
+       struct page_state *ps = &per_cpu(page_states, cpu);
+       memset(ps, 0, sizeof(*ps));
+}
+       
+static int __devinit page_alloc_cpu_notify(struct notifier_block *self, 
+                               unsigned long action, void *hcpu)
+{
+       int cpu = (unsigned long)hcpu;
+       switch(action) {
+       case CPU_UP_PREPARE:
+               init_page_alloc_cpu(cpu);
+               break;
+       default:
+               break;
+       }
+       return NOTIFY_OK;
+}
+
+static struct notifier_block __devinitdata page_alloc_nb = {
+       .notifier_call  = page_alloc_cpu_notify,
+};
+
+void __init page_alloc_init(void)
+{
+       init_page_alloc_cpu(smp_processor_id());
+       register_cpu_notifier(&page_alloc_nb);
+}