[PATCH] sched_domains: use cpu_possible_map

From: Nick Piggin <piggin@cyberone.com.au>

This changes sched domains to contain all possible CPUs, and check for
online as needed.  It's in order to play nicely with CPU hotplug.

diff --git a/arch/i386/kernel/smpboot.c b/arch/i386/kernel/smpboot.c
index b86eb597588651f94cd2b307fafd56778cf9d7de..d3a90d1b92e2ce6ad6d7ae18f70b4696b6cfd0fc 100644 (file)
--- a/arch/i386/kernel/smpboot.c
+++ b/arch/i386/kernel/smpboot.c
@@ -1138,7 +1138,7 @@ __init void arch_init_sched_domains(void)
        struct sched_group *first_cpu = NULL, *last_cpu = NULL;
 
        /* Set up domains */
-       for_each_cpu_mask(i, cpu_online_map) {
+       for_each_cpu(i) {
                struct sched_domain *cpu_domain = cpu_sched_domain(i);
                struct sched_domain *phys_domain = &per_cpu(phys_domains, i);
                struct sched_domain *node_domain = &per_cpu(node_domains, i);
@@ -1152,11 +1152,11 @@ __init void arch_init_sched_domains(void)
                phys_domain->span = nodemask;
 
                *node_domain = SD_NODE_INIT;
-               node_domain->span = cpu_online_map;
+               node_domain->span = cpu_possible_map;
        }
 
        /* Set up CPU (sibling) groups */
-       for_each_cpu_mask(i, cpu_online_map) {
+       for_each_cpu(i) {
                struct sched_domain *cpu_domain = cpu_sched_domain(i);
                int j;
                first_cpu = last_cpu = NULL;
@@ -1184,7 +1184,7 @@ __init void arch_init_sched_domains(void)
                int j;
                cpumask_t nodemask;
                struct sched_group *node = &sched_group_nodes[i];
-               cpus_and(nodemask, node_to_cpumask(i), cpu_online_map);
+               cpus_and(nodemask, node_to_cpumask(i), cpu_possible_map);
 
                if (cpus_empty(nodemask))
                        continue;
@@ -1220,7 +1220,7 @@ __init void arch_init_sched_domains(void)
        for (i = 0; i < MAX_NUMNODES; i++) {
                struct sched_group *cpu = &sched_group_nodes[i];
                cpumask_t nodemask;
-               cpus_and(nodemask, node_to_cpumask(i), cpu_online_map);
+               cpus_and(nodemask, node_to_cpumask(i), cpu_possible_map);
 
                if (cpus_empty(nodemask))
                        continue;
@@ -1237,7 +1237,7 @@ __init void arch_init_sched_domains(void)
        last_cpu->next = first_cpu;
 
        mb();
-       for_each_cpu_mask(i, cpu_online_map) {
+       for_each_cpu(i) {
                int node = cpu_to_node(i);
                struct sched_domain *cpu_domain = cpu_sched_domain(i);
                struct sched_domain *phys_domain = &per_cpu(phys_domains, i);
@@ -1264,7 +1264,7 @@ __init void arch_init_sched_domains(void)
        struct sched_group *first_cpu = NULL, *last_cpu = NULL;
 
        /* Set up domains */
-       for_each_cpu_mask(i, cpu_online_map) {
+       for_each_cpu(i) {
                struct sched_domain *cpu_domain = cpu_sched_domain(i);
                struct sched_domain *phys_domain = &per_cpu(phys_domains, i);
 
@@ -1272,11 +1272,11 @@ __init void arch_init_sched_domains(void)
                cpu_domain->span = cpu_sibling_map[i];
 
                *phys_domain = SD_CPU_INIT;
-               phys_domain->span = cpu_online_map;
+               phys_domain->span = cpu_possible_map;
        }
 
        /* Set up CPU (sibling) groups */
-       for_each_cpu_mask(i, cpu_online_map) {
+       for_each_cpu(i) {
                struct sched_domain *cpu_domain = cpu_sched_domain(i);
                int j;
                first_cpu = last_cpu = NULL;
@@ -1302,7 +1302,7 @@ __init void arch_init_sched_domains(void)
 
        first_cpu = last_cpu = NULL;
        /* Set up physical groups */
-       for_each_cpu_mask(i, cpu_online_map) {
+       for_each_cpu(i) {
                struct sched_domain *cpu_domain = cpu_sched_domain(i);
                struct sched_group *cpu = &sched_group_phys[i];
 
@@ -1322,7 +1322,7 @@ __init void arch_init_sched_domains(void)
        last_cpu->next = first_cpu;
 
        mb();
-       for_each_cpu_mask(i, cpu_online_map) {
+       for_each_cpu(i) {
                struct sched_domain *cpu_domain = cpu_sched_domain(i);
                struct sched_domain *phys_domain = &per_cpu(phys_domains, i);
                struct sched_group *cpu_group = &sched_group_cpus[i];

diff --git a/kernel/sched.c b/kernel/sched.c
index 7abd3a019fd966aec7672a71ba90506b6c75b0a9..09979ac4b676bf2f95f400b69612993bad3f0287 100644 (file)
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -647,6 +647,7 @@ static inline unsigned long get_high_cpu_load(int cpu, int update)
 #if defined(CONFIG_SMP) && defined(ARCH_HAS_SCHED_WAKE_BALANCE)
 static int sched_balance_wake(int cpu, task_t *p)
 {
+       cpumask_t tmp;
        struct sched_domain *domain;
        int i;
 
@@ -657,10 +658,8 @@ static int sched_balance_wake(int cpu, task_t *p)
        if (!(domain->flags & SD_FLAG_WAKE))
                return cpu;
 
-       for_each_cpu_mask(i, domain->span) {
-               if (!cpu_online(i))
-                       continue;
-
+       cpus_and(tmp, domain->span, cpu_online_map);
+       for_each_cpu_mask(i, tmp) {
                if (!cpu_isset(i, p->cpus_allowed))
                        continue;
 
@@ -1149,16 +1148,15 @@ out:
  */
 static int sched_best_cpu(struct task_struct *p, struct sched_domain *domain)
 {
+       cpumask_t tmp;
        int i, min_load, this_cpu, best_cpu;
 
        best_cpu = this_cpu = task_cpu(p);
        min_load = INT_MAX;
 
-       for_each_online_cpu(i) {
+       cpus_and(tmp, domain->span, cpu_online_map);
+       for_each_cpu_mask(i, tmp) {
                unsigned long load;
-               if (!cpu_isset(i, domain->span))
-                       continue;
-
                if (i == this_cpu)
                        load = get_low_cpu_load(i, 0);
                else
@@ -1373,6 +1371,7 @@ find_busiest_group(struct sched_domain *domain, int this_cpu,
                modify = 1;
 
        do {
+               cpumask_t tmp;
                unsigned long load;
                int local_group;
                int i, nr_cpus = 0;
@@ -1381,10 +1380,8 @@ find_busiest_group(struct sched_domain *domain, int this_cpu,
 
                /* Tally up the load of all CPUs in the group */
                avg_load = 0;
-               for_each_cpu_mask(i, group->cpumask) {
-                       if (!cpu_online(i))
-                               continue;
-
+               cpus_and(tmp, group->cpumask, cpu_online_map);
+               for_each_cpu_mask(i, tmp) {
                        /* Bias balancing toward cpus of our domain */
                        if (local_group) {
                                load = get_high_cpu_load(i, modify);
@@ -1496,16 +1493,15 @@ out_balanced:
  */
 static runqueue_t *find_busiest_queue(struct sched_group *group)
 {
+       cpumask_t tmp;
        int i;
        unsigned long max_load = 0;
        runqueue_t *busiest = NULL;
 
-       for_each_cpu_mask(i, group->cpumask) {
+       cpus_and(tmp, group->cpumask, cpu_online_map);
+       for_each_cpu_mask(i, tmp) {
                unsigned long load;
 
-               if (!cpu_online(i))
-                       continue;
-
                load = get_low_cpu_load(i, 0);
 
                if (load > max_load) {
@@ -1694,16 +1690,15 @@ static void active_load_balance(runqueue_t *busiest, int busiest_cpu)
 
        group = sd->groups;
        do {
+               cpumask_t tmp;
                runqueue_t *rq;
                int push_cpu = 0, nr = 0;
 
                if (group == busy_group)
                        goto next_group;
 
-               for_each_cpu_mask(i, group->cpumask) {
-                       if (!cpu_online(i))
-                               continue;
-
+               cpus_and(tmp, group->cpumask, cpu_online_map);
+               for_each_cpu_mask(i, tmp) {
                        if (!idle_cpu(i))
                                goto next_group;
                        push_cpu = i;
@@ -3298,17 +3293,17 @@ static void __init arch_init_sched_domains(void)
        struct sched_group *first_node = NULL, *last_node = NULL;
 
        /* Set up domains */
-       for_each_cpu_mask(i, cpu_online_map) {
+       for_each_cpu(i) {
                int node = cpu_to_node(i);
                cpumask_t nodemask = node_to_cpumask(node);
                struct sched_domain *node_domain = &per_cpu(node_domains, i);
                struct sched_domain *cpu_domain = cpu_sched_domain(i);
 
                *node_domain = SD_NODE_INIT;
-               node_domain->span = cpu_online_map;
+               node_domain->span = cpu_possible_map;
 
                *cpu_domain = SD_CPU_INIT;
-               cpus_and(cpu_domain->span, nodemask, cpu_online_map);
+               cpus_and(cpu_domain->span, nodemask, cpu_possible_map);
                cpu_domain->parent = node_domain;
        }
 
@@ -3320,7 +3315,7 @@ static void __init arch_init_sched_domains(void)
                struct sched_group *node = &sched_group_nodes[i];
                cpumask_t tmp = node_to_cpumask(i);
 
-               cpus_and(nodemask, tmp, cpu_online_map);
+               cpus_and(nodemask, tmp, cpu_possible_map);
 
                if (cpus_empty(nodemask))
                        continue;
@@ -3352,7 +3347,7 @@ static void __init arch_init_sched_domains(void)
        last_node->next = first_node;
 
        mb();
-       for_each_cpu_mask(i, cpu_online_map) {
+       for_each_cpu(i) {
                struct sched_domain *node_domain = &per_cpu(node_domains, i);
                struct sched_domain *cpu_domain = cpu_sched_domain(i);
                node_domain->groups = &sched_group_nodes[cpu_to_node(i)];
@@ -3367,15 +3362,15 @@ static void __init arch_init_sched_domains(void)
        struct sched_group *first_cpu = NULL, *last_cpu = NULL;
 
        /* Set up domains */
-       for_each_cpu_mask(i, cpu_online_map) {
+       for_each_cpu(i) {
                struct sched_domain *cpu_domain = cpu_sched_domain(i);
 
                *cpu_domain = SD_CPU_INIT;
-               cpu_domain->span = cpu_online_map;
+               cpu_domain->span = cpu_possible_map;
        }
 
        /* Set up CPU groups */
-       for_each_cpu_mask(i, cpu_online_map) {
+       for_each_cpu_mask(i, cpu_possible_map) {
                struct sched_group *cpu = &sched_group_cpus[i];
 
                cpus_clear(cpu->cpumask);
@@ -3391,7 +3386,7 @@ static void __init arch_init_sched_domains(void)
        last_cpu->next = first_cpu;
 
        mb();
-       for_each_cpu_mask(i, cpu_online_map) {
+       for_each_cpu(i) {
                struct sched_domain *cpu_domain = cpu_sched_domain(i);
                cpu_domain->groups = &sched_group_cpus[i];
        }