[PATCH] fix preempt-issues with smp_call_function()

Patch from Thomas Schlichter <schlicht@uni-mannheim.de>

Based on a patch from Dave Jones.

It converts a large number of instances of:

	smp_call_function(foo);
	foo();

into

	on_each_cpu(foo);

and in doing so fixes up the preempt-unsafeness of the first version.

diff --git a/arch/alpha/kernel/process.c b/arch/alpha/kernel/process.c
index 85b45eee28683fd53d17951c9a8c74c97d727563..768e2fb81deedff79bee99affa02d1de87a6c20c 100644 (file)
--- a/arch/alpha/kernel/process.c
+++ b/arch/alpha/kernel/process.c
@@ -155,10 +155,7 @@ common_shutdown(int mode, char *restart_cmd)
        struct halt_info args;
        args.mode = mode;
        args.restart_cmd = restart_cmd;
-#ifdef CONFIG_SMP
-       smp_call_function(common_shutdown_1, &args, 1, 0);
-#endif
-       common_shutdown_1(&args);
+       on_each_cpu(common_shutdown_1, &args, 1, 0);
 }
 
 void

diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c
index a1ed1762893472ee73b0cc51d8f491ff32c08669..05e25b55e72cbade4332b1ef23812a0ebda4cdae 100644 (file)
--- a/arch/alpha/kernel/smp.c
+++ b/arch/alpha/kernel/smp.c
@@ -899,10 +899,8 @@ void
 smp_imb(void)
 {
        /* Must wait other processors to flush their icache before continue. */
-       if (smp_call_function(ipi_imb, NULL, 1, 1))
+       if (on_each_cpu(ipi_imb, NULL, 1, 1))
                printk(KERN_CRIT "smp_imb: timed out\n");
-
-       imb();
 }
 
 static void
@@ -916,11 +914,9 @@ flush_tlb_all(void)
 {
        /* Although we don't have any data to pass, we do want to
           synchronize with the other processors.  */
-       if (smp_call_function(ipi_flush_tlb_all, NULL, 1, 1)) {
+       if (on_each_cpu(ipi_flush_tlb_all, NULL, 1, 1)) {
                printk(KERN_CRIT "flush_tlb_all: timed out\n");
        }
-
-       tbia();
 }
 
 #define asn_locked() (cpu_data[smp_processor_id()].asn_lock)
@@ -938,6 +934,8 @@ ipi_flush_tlb_mm(void *x)
 void
 flush_tlb_mm(struct mm_struct *mm)
 {
+       preempt_disable();
+
        if (mm == current->active_mm) {
                flush_tlb_current(mm);
                if (atomic_read(&mm->mm_users) <= 1) {
@@ -948,6 +946,7 @@ flush_tlb_mm(struct mm_struct *mm)
                                if (mm->context[cpu])
                                        mm->context[cpu] = 0;
                        }
+                       preempt_enable();
                        return;
                }
        }
@@ -955,6 +954,8 @@ flush_tlb_mm(struct mm_struct *mm)
        if (smp_call_function(ipi_flush_tlb_mm, mm, 1, 1)) {
                printk(KERN_CRIT "flush_tlb_mm: timed out\n");
        }
+
+       preempt_enable();
 }
 
 struct flush_tlb_page_struct {
@@ -981,6 +982,8 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
        struct flush_tlb_page_struct data;
        struct mm_struct *mm = vma->vm_mm;
 
+       preempt_disable();
+
        if (mm == current->active_mm) {
                flush_tlb_current_page(mm, vma, addr);
                if (atomic_read(&mm->mm_users) <= 1) {
@@ -991,6 +994,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
                                if (mm->context[cpu])
                                        mm->context[cpu] = 0;
                        }
+                       preempt_enable();
                        return;
                }
        }
@@ -1002,6 +1006,8 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
        if (smp_call_function(ipi_flush_tlb_page, &data, 1, 1)) {
                printk(KERN_CRIT "flush_tlb_page: timed out\n");
        }
+
+       preempt_enable();
 }
 
 void
@@ -1030,6 +1036,8 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
        if ((vma->vm_flags & VM_EXEC) == 0)
                return;
 
+       preempt_disable();
+
        if (mm == current->active_mm) {
                __load_new_mm_context(mm);
                if (atomic_read(&mm->mm_users) <= 1) {
@@ -1040,6 +1048,7 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
                                if (mm->context[cpu])
                                        mm->context[cpu] = 0;
                        }
+                       preempt_enable();
                        return;
                }
        }
@@ -1047,6 +1056,8 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
        if (smp_call_function(ipi_flush_icache_page, mm, 1, 1)) {
                printk(KERN_CRIT "flush_icache_page: timed out\n");
        }
+
+       preempt_enable();
 }
 \f
 #ifdef CONFIG_DEBUG_SPINLOCK

diff --git a/arch/i386/kernel/io_apic.c b/arch/i386/kernel/io_apic.c
index 7f2892e1e35e78e0674ce8e81bd8a40becdb4613..8bc3f9fee17c6cfc80397e6f2c4ecd84658cf095 100644 (file)
--- a/arch/i386/kernel/io_apic.c
+++ b/arch/i386/kernel/io_apic.c
@@ -1376,8 +1376,7 @@ void /*__init*/ print_local_APIC(void * dummy)
 
 void print_all_local_APICs (void)
 {
-       smp_call_function(print_local_APIC, NULL, 1, 1);
-       print_local_APIC(NULL);
+       on_each_cpu(print_local_APIC, NULL, 1, 1);
 }
 
 void /*__init*/ print_PIC(void)
@@ -1843,8 +1842,7 @@ static void setup_nmi (void)
         */ 
        printk(KERN_INFO "activating NMI Watchdog ...");
 
-       smp_call_function(enable_NMI_through_LVT0, NULL, 1, 1);
-       enable_NMI_through_LVT0(NULL);
+       on_each_cpu(enable_NMI_through_LVT0, NULL, 1, 1);
 
        printk(" done.\n");
 }

diff --git a/arch/i386/kernel/ldt.c b/arch/i386/kernel/ldt.c
index 8388fed9f9c278bc5992fc028d5c94701a595226..71a710a64cc8019695c3133baa8928902389e055 100644 (file)
--- a/arch/i386/kernel/ldt.c
+++ b/arch/i386/kernel/ldt.c
@@ -55,12 +55,14 @@ static int alloc_ldt(mm_context_t *pc, int mincount, int reload)
        wmb();
 
        if (reload) {
-               load_LDT(pc);
 #ifdef CONFIG_SMP
                preempt_disable();
+               load_LDT(pc);
                if (current->mm->cpu_vm_mask != (1 << smp_processor_id()))
                        smp_call_function(flush_ldt, 0, 1, 1);
                preempt_enable();
+#else
+               load_LDT(pc);
 #endif
        }
        if (oldsize) {

diff --git a/arch/i386/kernel/microcode.c b/arch/i386/kernel/microcode.c
index 3e7b429f7848f90465114b2dbcdc59d61ec49400..0d51b9dc9144197ff26ce7c754bc813b693155ff 100644 (file)
--- a/arch/i386/kernel/microcode.c
+++ b/arch/i386/kernel/microcode.c
@@ -183,11 +183,10 @@ static int do_microcode_update(void)
        int i, error = 0, err;
        struct microcode *m;
 
-       if (smp_call_function(do_update_one, NULL, 1, 1) != 0) {
+       if (on_each_cpu(do_update_one, NULL, 1, 1) != 0) {
                printk(KERN_ERR "microcode: IPI timeout, giving up\n");
                return -EIO;
        }
-       do_update_one(NULL);
 
        for (i=0; i<NR_CPUS; i++) {
                err = update_req[i].err;

diff --git a/arch/i386/kernel/smp.c b/arch/i386/kernel/smp.c
index 55d21ff02c89766b32d8e9f3edd882089fe7178e..7044332208860e45eeb78de451a31cfdd7e0d206 100644 (file)
--- a/arch/i386/kernel/smp.c
+++ b/arch/i386/kernel/smp.c
@@ -436,7 +436,7 @@ void flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
        preempt_enable();
 }
 
-static inline void do_flush_tlb_all_local(void)
+static void do_flush_tlb_all(void* info)
 {
        unsigned long cpu = smp_processor_id();
 
@@ -445,18 +445,9 @@ static inline void do_flush_tlb_all_local(void)
                leave_mm(cpu);
 }
 
-static void flush_tlb_all_ipi(void* info)
-{
-       do_flush_tlb_all_local();
-}
-
 void flush_tlb_all(void)
 {
-       preempt_disable();
-       smp_call_function (flush_tlb_all_ipi,0,1,1);
-
-       do_flush_tlb_all_local();
-       preempt_enable();
+       on_each_cpu(do_flush_tlb_all, 0, 1, 1);
 }
 
 /*

diff --git a/arch/i386/kernel/sysenter.c b/arch/i386/kernel/sysenter.c
index 5aa698f5447c3dc29e4471fc1992f97af73dfbec..2345e5da7d2abfb3a298f3183e7a3776df5e9eef 100644 (file)
--- a/arch/i386/kernel/sysenter.c
+++ b/arch/i386/kernel/sysenter.c
@@ -95,8 +95,7 @@ static int __init sysenter_setup(void)
                return 0;
 
        memcpy((void *) page, sysent, sizeof(sysent));
-       enable_sep_cpu(NULL);
-       smp_call_function(enable_sep_cpu, NULL, 1, 1);
+       on_each_cpu(enable_sep_cpu, NULL, 1, 1);
        return 0;
 }
 

diff --git a/arch/i386/mach-voyager/voyager_smp.c b/arch/i386/mach-voyager/voyager_smp.c
index 6463639d5b2659e1d9c3250c77900f981e0c8831..6a74600e0b09680d6086dd0624f4969d2a58c23d 100644 (file)
--- a/arch/i386/mach-voyager/voyager_smp.c
+++ b/arch/i386/mach-voyager/voyager_smp.c
@@ -1209,8 +1209,8 @@ smp_vic_cpi_interrupt(struct pt_regs regs)
                smp_call_function_interrupt();
 }
 
-static inline void
-do_flush_tlb_all_local(void)
+static void
+do_flush_tlb_all(void* info)
 {
        unsigned long cpu = smp_processor_id();
 
@@ -1220,20 +1220,11 @@ do_flush_tlb_all_local(void)
 }
 
 
-static void
-flush_tlb_all_function(void* info)
-{
-       do_flush_tlb_all_local();
-}
-
 /* flush the TLB of every active CPU in the system */
 void
 flush_tlb_all(void)
 {
-       preempt_disable();
-       smp_call_function (flush_tlb_all_function, 0, 1, 1);
-       do_flush_tlb_all_local();
-       preempt_enable();
+       on_each_cpu(do_flush_tlb_all, 0, 1, 1);
 }
 
 /* used to set up the trampoline for other CPUs when the memory manager

diff --git a/arch/i386/mm/pageattr.c b/arch/i386/mm/pageattr.c
index b4e286cec50f91f6a133fc36e3b8eefb679774e1..7bb9f7ebe469dd3a14e955a381c618af46204a4c 100644 (file)
--- a/arch/i386/mm/pageattr.c
+++ b/arch/i386/mm/pageattr.c
@@ -130,11 +130,8 @@ __change_page_attr(struct page *page, pgprot_t prot, struct page **oldpage)
 } 
 
 static inline void flush_map(void)
-{      
-#ifdef CONFIG_SMP 
-       smp_call_function(flush_kernel_map, NULL, 1, 1);
-#endif 
-       flush_kernel_map(NULL);
+{
+       on_each_cpu(flush_kernel_map, NULL, 1, 1);
 }
 
 struct deferred_page { 

diff --git a/arch/i386/oprofile/nmi_int.c b/arch/i386/oprofile/nmi_int.c
index e426bbfb37188af9827a6be9b39a403fddd8e83e..c6c30b22a0e2acad12a162e4c8c02f622546f801 100644 (file)
--- a/arch/i386/oprofile/nmi_int.c
+++ b/arch/i386/oprofile/nmi_int.c
@@ -95,8 +95,7 @@ static int nmi_setup(void)
         * without actually triggering any NMIs as this will
         * break the core code horrifically.
         */
-       smp_call_function(nmi_cpu_setup, NULL, 0, 1);
-       nmi_cpu_setup(0);
+       on_each_cpu(nmi_cpu_setup, NULL, 0, 1);
        set_nmi_callback(nmi_callback);
        oprofile_pmdev = set_nmi_pm_callback(oprofile_pm_callback);
        return 0;
@@ -148,8 +147,7 @@ static void nmi_shutdown(void)
 {
        unset_nmi_pm_callback(oprofile_pmdev);
        unset_nmi_callback();
-       smp_call_function(nmi_cpu_shutdown, NULL, 0, 1);
-       nmi_cpu_shutdown(0);
+       on_each_cpu(nmi_cpu_shutdown, NULL, 0, 1);
 }
 
  
@@ -162,8 +160,7 @@ static void nmi_cpu_start(void * dummy)
 
 static int nmi_start(void)
 {
-       smp_call_function(nmi_cpu_start, NULL, 0, 1);
-       nmi_cpu_start(0);
+       on_each_cpu(nmi_cpu_start, NULL, 0, 1);
        return 0;
 }
  
@@ -177,8 +174,7 @@ static void nmi_cpu_stop(void * dummy)
  
 static void nmi_stop(void)
 {
-       smp_call_function(nmi_cpu_stop, NULL, 0, 1);
-       nmi_cpu_stop(0);
+       on_each_cpu(nmi_cpu_stop, NULL, 0, 1);
 }
 
 

diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c
index 9532fe7ff1166620c6ba5991691e25405e534ef5..a3eff7b9fd665b1ffdc26e101c14e374e164143d 100644 (file)
--- a/arch/ia64/kernel/smp.c
+++ b/arch/ia64/kernel/smp.c
@@ -206,18 +206,18 @@ smp_send_reschedule_all (void)
 void
 smp_flush_tlb_all (void)
 {
-       smp_call_function((void (*)(void *))local_flush_tlb_all, 0, 1, 1);
-       local_flush_tlb_all();
+       on_each_cpu((void (*)(void *))local_flush_tlb_all, 0, 1, 1);
 }
 
 void
 smp_flush_tlb_mm (struct mm_struct *mm)
 {
-       local_finish_flush_tlb_mm(mm);
-
        /* this happens for the common case of a single-threaded fork():  */
        if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
+       {
+               local_finish_flush_tlb_mm(mm);
                return;
+       }
 
        /*
         * We could optimize this further by using mm->cpu_vm_mask to track which CPUs
@@ -226,7 +226,7 @@ smp_flush_tlb_mm (struct mm_struct *mm)
         * anyhow, and once a CPU is interrupted, the cost of local_flush_tlb_all() is
         * rather trivial.
         */
-       smp_call_function((void (*)(void *))local_finish_flush_tlb_mm, mm, 1, 1);
+       on_each_cpu((void (*)(void *))local_finish_flush_tlb_mm, mm, 1, 1);
 }
 
 /*

diff --git a/arch/mips64/kernel/smp.c b/arch/mips64/kernel/smp.c
index 4494b7d96ca6910e093f5cb33aab75a663ad1603..9ccb54fa72b5a6df4cd74893f71726928dd0c9a1 100644 (file)
--- a/arch/mips64/kernel/smp.c
+++ b/arch/mips64/kernel/smp.c
@@ -195,8 +195,7 @@ static void flush_tlb_all_ipi(void *info)
 
 void flush_tlb_all(void)
 {
-       smp_call_function(flush_tlb_all_ipi, 0, 1, 1);
-       _flush_tlb_all();
+       on_each_cpu(flush_tlb_all_ipi, 0, 1, 1);
 }
 
 static void flush_tlb_mm_ipi(void *mm)
@@ -219,6 +218,8 @@ static void flush_tlb_mm_ipi(void *mm)
 
 void flush_tlb_mm(struct mm_struct *mm)
 {
+       preempt_disable();
+
        if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
                smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1, 1);
        } else {
@@ -228,6 +229,8 @@ void flush_tlb_mm(struct mm_struct *mm)
                                CPU_CONTEXT(i, mm) = 0;
        }
        _flush_tlb_mm(mm);
+
+       preempt_enable();
 }
 
 struct flush_tlb_data {
@@ -246,6 +249,8 @@ static void flush_tlb_range_ipi(void *info)
 
 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
 {
+       preempt_disable();
+
        if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
                struct flush_tlb_data fd;
 
@@ -260,6 +265,8 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned l
                                CPU_CONTEXT(i, mm) = 0;
        }
        _flush_tlb_range(mm, start, end);
+
+       preempt_enable();
 }
 
 static void flush_tlb_page_ipi(void *info)
@@ -271,6 +278,8 @@ static void flush_tlb_page_ipi(void *info)
 
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 {
+       preempt_disable();
+
        if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
                struct flush_tlb_data fd;
 
@@ -284,5 +293,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
                                CPU_CONTEXT(i, vma->vm_mm) = 0;
        }
        _flush_tlb_page(vma, page);
+
+       preempt_enable();
 }
 

diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
index 646e4c1a61e340f9cc628803e3dcd6110ee94823..bf551a9745afee99f9269ab41fd235ca6b4bcfc2 100644 (file)
--- a/arch/parisc/kernel/cache.c
+++ b/arch/parisc/kernel/cache.c
@@ -39,8 +39,7 @@ static struct pdc_btlb_info btlb_info;
 void
 flush_data_cache(void)
 {
-       smp_call_function((void (*)(void *))flush_data_cache_local, NULL, 1, 1);
-       flush_data_cache_local();
+       on_each_cpu((void (*)(void *))flush_data_cache_local, NULL, 1, 1);
 }
 #endif
 

diff --git a/arch/parisc/kernel/irq.c b/arch/parisc/kernel/irq.c
index d618a3bec3cc265594d4e37eaab1365ead6abcb1..87d7e7cbf4bfe03e40dbc1eebf2be5015fb2819c 100644 (file)
--- a/arch/parisc/kernel/irq.c
+++ b/arch/parisc/kernel/irq.c
@@ -61,20 +61,17 @@ static volatile unsigned long cpu_eiem = 0;
 
 static spinlock_t irq_lock = SPIN_LOCK_UNLOCKED;  /* protect IRQ regions */
 
-#ifdef CONFIG_SMP
 static void cpu_set_eiem(void *info)
 {
        set_eiem((unsigned long) info);
 }
-#endif
 
 static inline void disable_cpu_irq(void *unused, int irq)
 {
        unsigned long eirr_bit = EIEM_MASK(irq);
 
        cpu_eiem &= ~eirr_bit;
-       set_eiem(cpu_eiem);
-        smp_call_function(cpu_set_eiem, (void *) cpu_eiem, 1, 1);
+        on_each_cpu(cpu_set_eiem, (void *) cpu_eiem, 1, 1);
 }
 
 static void enable_cpu_irq(void *unused, int irq)
@@ -83,8 +80,7 @@ static void enable_cpu_irq(void *unused, int irq)
 
        mtctl(eirr_bit, 23);    /* clear EIRR bit before unmasking */
        cpu_eiem |= eirr_bit;
-        smp_call_function(cpu_set_eiem, (void *) cpu_eiem, 1, 1);
-       set_eiem(cpu_eiem);
+        on_each_cpu(cpu_set_eiem, (void *) cpu_eiem, 1, 1);
 }
 
 /* mask and disable are the same at the CPU level
@@ -100,8 +96,7 @@ static inline void unmask_cpu_irq(void *unused, int irq)
        ** handle *any* unmasked pending interrupts.
        ** ie We don't need to check for pending interrupts here.
        */
-        smp_call_function(cpu_set_eiem, (void *) cpu_eiem, 1, 1);
-       set_eiem(cpu_eiem);
+        on_each_cpu(cpu_set_eiem, (void *) cpu_eiem, 1, 1);
 }
 
 /*

diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c
index f31e00df6bad3e5fa8daf31bd249a6ac2668ed9c..84eea09ef6c65e6826b82472c25b4d0cc004bd24 100644 (file)
--- a/arch/parisc/kernel/smp.c
+++ b/arch/parisc/kernel/smp.c
@@ -401,7 +401,7 @@ static int __init maxcpus(char *str)
 __setup("maxcpus=", maxcpus);
 
 /*
- * Flush all other CPU's tlb and then mine.  Do this with smp_call_function()
+ * Flush all other CPU's tlb and then mine.  Do this with on_each_cpu()
  * as we want to ensure all TLB's flushed before proceeding.
  */
 
@@ -410,8 +410,7 @@ extern void flush_tlb_all_local(void);
 void
 smp_flush_tlb_all(void)
 {
-       smp_call_function((void (*)(void *))flush_tlb_all_local, NULL, 1, 1);
-       flush_tlb_all_local();
+       on_each_cpu((void (*)(void *))flush_tlb_all_local, NULL, 1, 1);
 }
 
 

diff --git a/arch/parisc/mm/init.c b/arch/parisc/mm/init.c
index 0a78eef7169f2854a465aa71ad10dbb0bf520c43..3b382ef901daca55a6d735329cabbc2fd401d9a6 100644 (file)
--- a/arch/parisc/mm/init.c
+++ b/arch/parisc/mm/init.c
@@ -974,8 +974,7 @@ void flush_tlb_all(void)
            do_recycle++;
        }
        spin_unlock(&sid_lock);
-       smp_call_function((void (*)(void *))flush_tlb_all_local, NULL, 1, 1);
-       flush_tlb_all_local();
+       on_each_cpu((void (*)(void *))flush_tlb_all_local, NULL, 1, 1);
        if (do_recycle) {
            spin_lock(&sid_lock);
            recycle_sids(recycle_ndirty,recycle_dirty_array);

diff --git a/arch/ppc/kernel/temp.c b/arch/ppc/kernel/temp.c
index e096d841767b68e04f514912a500f685a9492a4b..fba7fb3518312139f9f58d09fc89de9cf4cfaf65 100644 (file)
--- a/arch/ppc/kernel/temp.c
+++ b/arch/ppc/kernel/temp.c
@@ -194,10 +194,7 @@ static void tau_timeout_smp(unsigned long unused)
 
        /* schedule ourselves to be run again */
        mod_timer(&tau_timer, jiffies + shrink_timer) ;
-#ifdef CONFIG_SMP
-       smp_call_function(tau_timeout, NULL, 1, 0);
-#endif
-       tau_timeout(NULL);
+       on_each_cpu(tau_timeout, NULL, 1, 0);
 }
 
 /*
@@ -239,10 +236,7 @@ int __init TAU_init(void)
        tau_timer.expires = jiffies + shrink_timer;
        add_timer(&tau_timer);
        
-#ifdef CONFIG_SMP
-       smp_call_function(TAU_init_smp, NULL, 1, 0);
-#endif
-       TAU_init_smp(NULL);
+       on_each_cpu(TAU_init_smp, NULL, 1, 0);
        
        printk("Thermal assist unit ");
 #ifdef CONFIG_TAU_INT

diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index e02e5c8c47fe7316e1a93936144a6df447b56374..2781e9e5ef7de6c81a388e1809c680113ed8bbf5 100644 (file)
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -228,8 +228,7 @@ static void do_machine_restart(void * __unused)
 void machine_restart_smp(char * __unused) 
 {
        cpu_restart_map = cpu_online_map;
-        smp_call_function(do_machine_restart, NULL, 0, 0);
-       do_machine_restart(NULL);
+        on_each_cpu(do_machine_restart, NULL, 0, 0);
 }
 
 static void do_machine_halt(void * __unused)
@@ -247,8 +246,7 @@ static void do_machine_halt(void * __unused)
 
 void machine_halt_smp(void)
 {
-        smp_call_function(do_machine_halt, NULL, 0, 0);
-       do_machine_halt(NULL);
+        on_each_cpu(do_machine_halt, NULL, 0, 0);
 }
 
 static void do_machine_power_off(void * __unused)
@@ -266,8 +264,7 @@ static void do_machine_power_off(void * __unused)
 
 void machine_power_off_smp(void)
 {
-        smp_call_function(do_machine_power_off, NULL, 0, 0);
-       do_machine_power_off(NULL);
+        on_each_cpu(do_machine_power_off, NULL, 0, 0);
 }
 
 /*
@@ -339,8 +336,7 @@ void smp_ptlb_callback(void *info)
 
 void smp_ptlb_all(void)
 {
-        smp_call_function(smp_ptlb_callback, NULL, 0, 1);
-       local_flush_tlb();
+        on_each_cpu(smp_ptlb_callback, NULL, 0, 1);
 }
 
 /*
@@ -400,8 +396,10 @@ void smp_ctl_set_bit(int cr, int bit) {
        parms.end_ctl = cr;
        parms.orvals[cr] = 1 << bit;
        parms.andvals[cr] = 0xFFFFFFFF;
+       preempt_disable();
        smp_call_function(smp_ctl_bit_callback, &parms, 0, 1);
         __ctl_set_bit(cr, bit);
+       preempt_enable();
 }
 
 /*
@@ -414,8 +412,10 @@ void smp_ctl_clear_bit(int cr, int bit) {
        parms.end_ctl = cr;
        parms.orvals[cr] = 0x00000000;
        parms.andvals[cr] = ~(1 << bit);
+       preempt_disable();
        smp_call_function(smp_ctl_bit_callback, &parms, 0, 1);
         __ctl_clear_bit(cr, bit);
+       preempt_enable();
 }
 
 /*

diff --git a/arch/s390x/kernel/smp.c b/arch/s390x/kernel/smp.c
index 8abfde52cb9a869b66fda3a6ce3ec47d5b645175..469a9f647d440686471cd3dfac541b4447a89577 100644 (file)
--- a/arch/s390x/kernel/smp.c
+++ b/arch/s390x/kernel/smp.c
@@ -227,8 +227,7 @@ static void do_machine_restart(void * __unused)
 void machine_restart_smp(char * __unused) 
 {
        cpu_restart_map = cpu_online_map;
-        smp_call_function(do_machine_restart, NULL, 0, 0);
-       do_machine_restart(NULL);
+        on_each_cpu(do_machine_restart, NULL, 0, 0);
 }
 
 static void do_machine_halt(void * __unused)
@@ -246,8 +245,7 @@ static void do_machine_halt(void * __unused)
 
 void machine_halt_smp(void)
 {
-        smp_call_function(do_machine_halt, NULL, 0, 0);
-       do_machine_halt(NULL);
+        on_each_cpu(do_machine_halt, NULL, 0, 0);
 }
 
 static void do_machine_power_off(void * __unused)
@@ -265,8 +263,7 @@ static void do_machine_power_off(void * __unused)
 
 void machine_power_off_smp(void)
 {
-        smp_call_function(do_machine_power_off, NULL, 0, 0);
-       do_machine_power_off(NULL);
+        on_each_cpu(do_machine_power_off, NULL, 0, 0);
 }
 
 /*
@@ -383,8 +380,10 @@ void smp_ctl_set_bit(int cr, int bit) {
        parms.end_ctl = cr;
        parms.orvals[cr] = 1 << bit;
        parms.andvals[cr] = -1L;
+       preempt_disable();
        smp_call_function(smp_ctl_bit_callback, &parms, 0, 1);
         __ctl_set_bit(cr, bit);
+       preempt_enable();
 }
 
 /*
@@ -397,8 +396,10 @@ void smp_ctl_clear_bit(int cr, int bit) {
        parms.end_ctl = cr;
        parms.orvals[cr] = 0;
        parms.andvals[cr] = ~(1L << bit);
+       preempt_disable();
        smp_call_function(smp_ctl_bit_callback, &parms, 0, 1);
         __ctl_clear_bit(cr, bit);
+       preempt_enable();
 }
 
 

diff --git a/arch/x86_64/kernel/bluesmoke.c b/arch/x86_64/kernel/bluesmoke.c
index 4a60fa6003a595937c080ca5d1fd9f6f3e61a14c..831e848a7ace612d01202ed9184ca00f4101c6b0 100644 (file)
--- a/arch/x86_64/kernel/bluesmoke.c
+++ b/arch/x86_64/kernel/bluesmoke.c
@@ -111,11 +111,7 @@ static void mce_checkregs (void *info)
 {
        u32 low, high;
        int i;
-       unsigned int *cpu = info;
 
-       BUG_ON (*cpu != smp_processor_id());
-
-       preempt_disable();
        for (i=0; i<banks; i++) {
                rdmsr(MSR_IA32_MC0_STATUS+i*4, low, high);
 
@@ -130,20 +126,12 @@ static void mce_checkregs (void *info)
                        wmb();
                }
        }
-       preempt_enable();
 }
 
 
 static void mce_timerfunc (unsigned long data)
 {
-       unsigned int i;
-
-       for (i=0; i<smp_num_cpus; i++) {
-               if (i == smp_processor_id())
-                       mce_checkregs(&i);
-               else
-                       smp_call_function (mce_checkregs, &i, 1, 1);
-       }
+       on_each_cpu (mce_checkregs, NULL, 1, 1);
 
        /* Refresh the timer. */
        mce_timer.expires = jiffies + MCE_RATE;

diff --git a/arch/x86_64/kernel/io_apic.c b/arch/x86_64/kernel/io_apic.c
index 02be6caeecc84126aadc04602b5adeb53946bde4..06783f127a3de6c5d83d2d4e547a311e5a9f9583 100644 (file)
--- a/arch/x86_64/kernel/io_apic.c
+++ b/arch/x86_64/kernel/io_apic.c
@@ -928,8 +928,7 @@ void /*__init*/ print_local_APIC(void * dummy)
 
 void print_all_local_APICs (void)
 {
-       smp_call_function(print_local_APIC, NULL, 1, 1);
-       print_local_APIC(NULL);
+       on_each_cpu(print_local_APIC, NULL, 1, 1);
 }
 
 void /*__init*/ print_PIC(void)

diff --git a/arch/x86_64/kernel/ldt.c b/arch/x86_64/kernel/ldt.c
index e4d13043c168603a3b01e3ec6227aeeab3b075c6..91abd555afd3675418f3db13b2e1137d7f9b30d9 100644 (file)
--- a/arch/x86_64/kernel/ldt.c
+++ b/arch/x86_64/kernel/ldt.c
@@ -60,12 +60,14 @@ static int alloc_ldt(mm_context_t *pc, int mincount, int reload)
        pc->size = mincount;
        wmb();
        if (reload) {
-               load_LDT(pc);
 #ifdef CONFIG_SMP
                preempt_disable();
+               load_LDT(pc);
                if (current->mm->cpu_vm_mask != (1<<smp_processor_id()))
                        smp_call_function(flush_ldt, 0, 1, 1);
                preempt_enable();
+#else
+               load_LDT(pc);
 #endif
        }
        if (oldsize) {

diff --git a/arch/x86_64/kernel/smp.c b/arch/x86_64/kernel/smp.c
index 557bfa36539930895c4c9df8a1e0e1ce880aa877..c5d1957b1f9feac87082b9cc93de28cc328b52ea 100644 (file)
--- a/arch/x86_64/kernel/smp.c
+++ b/arch/x86_64/kernel/smp.c
@@ -328,7 +328,7 @@ void flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
        preempt_enable();
 }
 
-static inline void do_flush_tlb_all_local(void)
+static void do_flush_tlb_all(void* info)
 {
        unsigned long cpu = smp_processor_id();
 
@@ -337,18 +337,9 @@ static inline void do_flush_tlb_all_local(void)
                leave_mm(cpu);
 }
 
-static void flush_tlb_all_ipi(void* info)
-{
-       do_flush_tlb_all_local();
-}
-
 void flush_tlb_all(void)
 {
-       preempt_disable();
-       smp_call_function (flush_tlb_all_ipi,0,1,1);
-
-       do_flush_tlb_all_local();
-       preempt_enable();
+       on_each_cpu(do_flush_tlb_all, 0, 1, 1);
 }
 
 void smp_kdb_stop(void)

diff --git a/arch/x86_64/mm/pageattr.c b/arch/x86_64/mm/pageattr.c
index fabd1f60560df634f5cb30685db57adfdcf4ebeb..4eaf4e5df48df9da1b0ec5250f888fb61d6b50b0 100644 (file)
--- a/arch/x86_64/mm/pageattr.c
+++ b/arch/x86_64/mm/pageattr.c
@@ -123,12 +123,7 @@ __change_page_attr(unsigned long address, struct page *page, pgprot_t prot,
 
 static inline void flush_map(unsigned long address)
 {      
-       preempt_disable();
-#ifdef CONFIG_SMP 
-       smp_call_function(flush_kernel_map, (void *)address, 1, 1);
-#endif 
-       flush_kernel_map((void *)address);
-       preempt_enable();
+       on_each_cpu(flush_kernel_map, (void *)address, 1, 1);
 }
 
 struct deferred_page { 

diff --git a/drivers/char/agp/agp.h b/drivers/char/agp/agp.h
index c1ab6b0a10cdfa464715741c1af77ca996f0c3ef..3f7aaa76a5bb8b8d858309e0edefcbadfad987a3 100644 (file)
--- a/drivers/char/agp/agp.h
+++ b/drivers/char/agp/agp.h
@@ -42,9 +42,8 @@ static void ipi_handler(void *null)
 
 static void __attribute__((unused)) global_cache_flush(void)
 {
-       if (smp_call_function(ipi_handler, NULL, 1, 1) != 0)
+       if (on_each_cpu(ipi_handler, NULL, 1, 1) != 0)
                panic(PFX "timed out waiting for the other CPUs!\n");
-       flush_agp_cache();
 }
 #else
 static inline void global_cache_flush(void)

diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c
index 60f5a3bd4fe60ba61afe1aeb25ae7d2f7a460bd3..cfef881fb52f4ef55d0c586f75e22e24ca9f3975 100644 (file)
--- a/drivers/s390/char/sclp.c
+++ b/drivers/s390/char/sclp.c
@@ -481,8 +481,7 @@ static void
 do_machine_quiesce(void)
 {
        cpu_quiesce_map = cpu_online_map;
-       smp_call_function(do_load_quiesce_psw, NULL, 0, 0);
-       do_load_quiesce_psw(NULL);
+       on_each_cpu(do_load_quiesce_psw, NULL, 0, 0);
 }
 #else
 static void

diff --git a/fs/buffer.c b/fs/buffer.c
index 9258e2d6b6fe7b08a207706e21b0363528a18c54..7a04adf9c1740ed84e91b5e6ce126a374c08ddff 100644 (file)
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -1403,10 +1403,7 @@ static void invalidate_bh_lru(void *arg)
        
 static void invalidate_bh_lrus(void)
 {
-       preempt_disable();
-       invalidate_bh_lru(NULL);
-       smp_call_function(invalidate_bh_lru, NULL, 1, 1);
-       preempt_enable();
+       on_each_cpu(invalidate_bh_lru, NULL, 1, 1);
 }
 
 void set_bh_page(struct buffer_head *bh,

diff --git a/include/asm-parisc/cacheflush.h b/include/asm-parisc/cacheflush.h
index 325dd36a28498f8f5e4aad2dac031eb0beb8bc95..b7012daae8609c986b2b27fd4e7322f4afe9ede7 100644 (file)
--- a/include/asm-parisc/cacheflush.h
+++ b/include/asm-parisc/cacheflush.h
@@ -25,16 +25,10 @@ flush_page_to_ram(struct page *page)
 
 extern void flush_cache_all_local(void);
 
-#ifdef CONFIG_SMP
 static inline void flush_cache_all(void)
 {
-       smp_call_function((void (*)(void *))flush_cache_all_local, NULL, 1, 1);
-       flush_cache_all_local();
+       on_each_cpu((void (*)(void *))flush_cache_all_local, NULL, 1, 1);
 }
-#else
-#define flush_cache_all flush_cache_all_local
-#endif
-
 
 /* The following value needs to be tuned and probably scaled with the
  * cache size.

diff --git a/include/linux/smp.h b/include/linux/smp.h
index ce016a544cd4d8bae01a430851dd377993da8847..81a906924efe5bb721e48ac74cff425507d9ed77 100644 (file)
--- a/include/linux/smp.h
+++ b/include/linux/smp.h
@@ -10,9 +10,10 @@
 
 #ifdef CONFIG_SMP
 
+#include <linux/preempt.h>
 #include <linux/kernel.h>
 #include <linux/compiler.h>
-#include <linux/threads.h>
+#include <linux/thread_info.h>
 #include <asm/smp.h>
 #include <asm/bug.h>
 
@@ -53,6 +54,21 @@ extern void smp_cpus_done(unsigned int max_cpus);
 extern int smp_call_function (void (*func) (void *info), void *info,
                              int retry, int wait);
 
+/*
+ * Call a function on all processors
+ */
+static inline int on_each_cpu(void (*func) (void *info), void *info,
+                             int retry, int wait)
+{
+       int ret = 0;
+
+       preempt_disable();
+       ret = smp_call_function(func, info, retry, wait);
+       func(info);
+       preempt_enable();
+       return ret;
+}
+
 /*
  * True once the per process idle is forked
  */
@@ -96,6 +112,7 @@ void smp_prepare_boot_cpu(void);
 #define hard_smp_processor_id()                        0
 #define smp_threads_ready                      1
 #define smp_call_function(func,info,retry,wait)        ({ 0; })
+#define on_each_cpu(func,info,retry,wait)      ({ func(info); 0; })
 static inline void smp_send_reschedule(int cpu) { }
 static inline void smp_send_reschedule_all(void) { }
 #define cpu_online_map                         1

diff --git a/mm/slab.c b/mm/slab.c
index c199cf890d58452fce2d67207beeb3fe6648cec6..d5f5a178f6337a10ac9dd8cdaf2827377df89c52 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -1116,12 +1116,16 @@ static inline void check_spinlock_acquired(kmem_cache_t *cachep)
 static void smp_call_function_all_cpus(void (*func) (void *arg), void *arg)
 {
        check_irq_on();
+       preempt_disable();
+
        local_irq_disable();
        func(arg);
        local_irq_enable();
 
        if (smp_call_function(func, arg, 1, 1))
                BUG();
+
+       preempt_enable();
 }
 
 static void free_block (kmem_cache_t* cachep, void** objpp, int len);