sched/rt: Up the root domain ref count when passing it around via IPIs

commit 364f56653708ba8bcdefd4f0da2a42904baa8eeb upstream.

When issuing an IPI RT push, where an IPI is sent to each CPU that has more
than one RT task scheduled on it, it references the root domain's rto_mask,
that contains all the CPUs within the root domain that has more than one RT
task in the runable state. The problem is, after the IPIs are initiated, the
rq->lock is released. This means that the root domain that is associated to
the run queue could be freed while the IPIs are going around.

Add a sched_get_rd() and a sched_put_rd() that will increment and decrement
the root domain's ref count respectively. This way when initiating the IPIs,
the scheduler will up the root domain's ref count before releasing the
rq->lock, ensuring that the root domain does not go away until the IPI round
is complete.

Reported-by: Pavan Kondeti <pkondeti@codeaurora.org>
Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Fixes: 4bdced5c9a292 ("sched/rt: Simplify the IPI based RT balancing logic")
Link: http://lkml.kernel.org/r/CAEU1=PkiHO35Dzna8EQqNSKW1fr1y1zRQ5y66X117MG06sQtNA@mail.gmail.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index e5066955cc3a152149df120020192cd9a908c1ce..bce3a7ad4253b954f08a749b8aee2194d00d8592 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5864,6 +5864,19 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
                call_rcu_sched(&old_rd->rcu, free_rootdomain);
 }
 
+void sched_get_rd(struct root_domain *rd)
+{
+       atomic_inc(&rd->refcount);
+}
+
+void sched_put_rd(struct root_domain *rd)
+{
+       if (!atomic_dec_and_test(&rd->refcount))
+               return;
+
+       call_rcu_sched(&rd->rcu, free_rootdomain);
+}
+
 static int init_rootdomain(struct root_domain *rd)
 {
        memset(rd, 0, sizeof(*rd));

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 27b6209a2928b6c42114b8e3cd47049d8028410c..f6d68ddfa2f320f87d5aade13b15185398b43876 100644 (file)
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1978,8 +1978,11 @@ static void tell_cpu_to_push(struct rq *rq)
 
        rto_start_unlock(&rq->rd->rto_loop_start);
 
-       if (cpu >= 0)
+       if (cpu >= 0) {
+               /* Make sure the rd does not get freed while pushing */
+               sched_get_rd(rq->rd);
                irq_work_queue_on(&rq->rd->rto_push_work, cpu);
+       }
 }
 
 /* Called from hardirq context */
@@ -2009,8 +2012,10 @@ void rto_push_irq_work_func(struct irq_work *work)
 
        raw_spin_unlock(&rd->rto_lock);
 
-       if (cpu < 0)
+       if (cpu < 0) {
+               sched_put_rd(rd);
                return;
+       }
 
        /* Try the next RT overloaded CPU */
        irq_work_queue_on(&rd->rto_push_work, cpu);

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index cff985feb6e7dd3b5c33b1bd387a33a4d623fa81..f564a1d2c9d5323ff5c4a446e9560e32e42de7f0 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -590,6 +590,8 @@ struct root_domain {
 };
 
 extern struct root_domain def_root_domain;
+extern void sched_get_rd(struct root_domain *rd);
+extern void sched_put_rd(struct root_domain *rd);
 
 #ifdef HAVE_RT_PUSH_IPI
 extern void rto_push_irq_work_func(struct irq_work *work);