mm: page_alloc: use jump labels to avoid checking number_of_cpusets

commit 664eeddeef6539247691197c1ac124d4aa872ab6 upstream.

If cpusets are not in use then we still check a global variable on every
page allocation.  Use jump labels to avoid the overhead.

Signed-off-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.cz>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Hugh Dickins <hughd@google.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Theodore Ts'o <tytso@mit.edu>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Mel Gorman <mgorman@suse.de>
Signed-off-by: Jiri Slaby <jslaby@suse.cz>

diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h
index 6f30672f5f925819d266a48f72142393e7429b67..a7ebb89ae9fb588a372e25068e93971c4269070e 100644 (file)
--- a/include/linux/cpuset.h
+++ b/include/linux/cpuset.h
@@ -12,10 +12,31 @@
 #include <linux/cpumask.h>
 #include <linux/nodemask.h>
 #include <linux/mm.h>
+#include <linux/jump_label.h>
 
 #ifdef CONFIG_CPUSETS
 
-extern int number_of_cpusets;  /* How many cpusets are defined in system? */
+extern struct static_key cpusets_enabled_key;
+static inline bool cpusets_enabled(void)
+{
+       return static_key_false(&cpusets_enabled_key);
+}
+
+static inline int nr_cpusets(void)
+{
+       /* jump label reference count + the top-level cpuset */
+       return static_key_count(&cpusets_enabled_key) + 1;
+}
+
+static inline void cpuset_inc(void)
+{
+       static_key_slow_inc(&cpusets_enabled_key);
+}
+
+static inline void cpuset_dec(void)
+{
+       static_key_slow_dec(&cpusets_enabled_key);
+}
 
 extern int cpuset_init(void);
 extern void cpuset_init_smp(void);
@@ -32,13 +53,13 @@ extern int __cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask);
 
 static inline int cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)
 {
-       return number_of_cpusets <= 1 ||
+       return nr_cpusets() <= 1 ||
                __cpuset_node_allowed_softwall(node, gfp_mask);
 }
 
 static inline int cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask)
 {
-       return number_of_cpusets <= 1 ||
+       return nr_cpusets() <= 1 ||
                __cpuset_node_allowed_hardwall(node, gfp_mask);
 }
 
@@ -120,6 +141,8 @@ static inline void set_mems_allowed(nodemask_t nodemask)
 
 #else /* !CONFIG_CPUSETS */
 
+static inline bool cpusets_enabled(void) { return false; }
+
 static inline int cpuset_init(void) { return 0; }
 static inline void cpuset_init_smp(void) {}
 

diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index 15ae2b7e38462348f6dec221d2eef22ab0737b77..c8289138cad4dd3c040e42e246114271bca95b91 100644 (file)
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -61,12 +61,7 @@
 #include <linux/cgroup.h>
 #include <linux/wait.h>
 
-/*
- * Tracks how many cpusets are currently defined in system.
- * When there is only one cpuset (the root cpuset) we can
- * short circuit some hooks.
- */
-int number_of_cpusets __read_mostly;
+struct static_key cpusets_enabled_key __read_mostly = STATIC_KEY_INIT_FALSE;
 
 /* See "Frequency meter" comments, below. */
 
@@ -611,7 +606,7 @@ static int generate_sched_domains(cpumask_var_t **domains,
                goto done;
        }
 
-       csa = kmalloc(number_of_cpusets * sizeof(cp), GFP_KERNEL);
+       csa = kmalloc(nr_cpusets() * sizeof(cp), GFP_KERNEL);
        if (!csa)
                goto done;
        csn = 0;
@@ -1986,7 +1981,7 @@ static int cpuset_css_online(struct cgroup_subsys_state *css)
        if (is_spread_slab(parent))
                set_bit(CS_SPREAD_SLAB, &cs->flags);
 
-       number_of_cpusets++;
+       cpuset_inc();
 
        if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
                goto out_unlock;
@@ -2037,7 +2032,7 @@ static void cpuset_css_offline(struct cgroup_subsys_state *css)
        if (is_sched_load_balance(cs))
                update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);
 
-       number_of_cpusets--;
+       cpuset_dec();
        clear_bit(CS_ONLINE, &cs->flags);
 
        mutex_unlock(&cpuset_mutex);
@@ -2092,7 +2087,6 @@ int __init cpuset_init(void)
        if (!alloc_cpumask_var(&cpus_attach, GFP_KERNEL))
                BUG();
 
-       number_of_cpusets = 1;
        return 0;
 }
 

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 68e0349f95e478b10180d7a27dd23ebcd0c7b468..5e0f1375b1b891aab9a04d9d5bd8dc841f35aa63 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1920,7 +1920,8 @@ zonelist_scan:
                if (IS_ENABLED(CONFIG_NUMA) && zlc_active &&
                        !zlc_zone_worth_trying(zonelist, z, allowednodes))
                                continue;
-               if ((alloc_flags & ALLOC_CPUSET) &&
+               if (cpusets_enabled() &&
+                       (alloc_flags & ALLOC_CPUSET) &&
                        !cpuset_zone_allowed_softwall(zone, gfp_mask))
                                continue;
                BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);