x86/mm: Remove the UP asm/tlbflush.h code, always use the (formerly) SMP code

commit ce4a4e565f5264909a18c733b864c3f74467f69e upstream.

The UP asm/tlbflush.h generates somewhat nicer code than the SMP version.
Aside from that, it's fallen quite a bit behind the SMP code:

 - flush_tlb_mm_range() didn't flush individual pages if the range
   was small.

 - The lazy TLB code was much weaker.  This usually wouldn't matter,
   but, if a kernel thread flushed its lazy "active_mm" more than
   once (due to reclaim or similar), it wouldn't be unlazied and
   would instead pointlessly flush repeatedly.

 - Tracepoints were missing.

Aside from that, simply having the UP code around was a maintanence
burden, since it means that any change to the TLB flush code had to
make sure not to break it.

Simplify everything by deleting the UP code.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Cc: Borislav Petkov <bpetkov@suse.de>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Nadav Amit <nadav.amit@gmail.com>
Cc: Nadav Amit <namit@vmware.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-mm@kvack.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
[Hugh Dickins: Backported to 3.18]
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Ben Hutchings <ben@decadent.org.uk>

diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index f1ac82834e3127d986bbbd320bee70bb6b78ce9e..07e1325ac8db25e00fd6550b55bbd0db26856c58 100644 (file)
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -21,8 +21,8 @@ typedef struct {
 #ifdef CONFIG_SMP
        unsigned int irq_resched_count;
        unsigned int irq_call_count;
-       unsigned int irq_tlb_count;
 #endif
+       unsigned int irq_tlb_count;
 #ifdef CONFIG_X86_THERMAL_VECTOR
        unsigned int irq_thermal_count;
 #endif

diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h
index b6b7bc3f5d261fc87993ae21b932360e299b2d13..3bc5c0b13cd7a75183233f7984d7dfff8890f164 100644 (file)
--- a/arch/x86/include/asm/mmu.h
+++ b/arch/x86/include/asm/mmu.h
@@ -20,12 +20,6 @@ typedef struct {
        void __user *vdso;
 } mm_context_t;
 
-#ifdef CONFIG_SMP
 void leave_mm(int cpu);
-#else
-static inline void leave_mm(int cpu)
-{
-}
-#endif
 
 #endif /* _ASM_X86_MMU_H */

diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 972b74bea6bd5474220eef970714662c5b3d932c..740e65890eaa0ac3d86b0d6ff30389516f4c9c6c 100644 (file)
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -68,10 +68,8 @@ void destroy_context(struct mm_struct *mm);
 
 static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
 {
-#ifdef CONFIG_SMP
        if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
                this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
-#endif
 }
 
 extern void switch_mm(struct mm_struct *prev, struct mm_struct *next,

diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index bb3a7de4ffdf3c0c797b28e90bb198b940648539..e8bb37958e19aaa2c86a77a53a02cb0cdd431765 100644 (file)
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -6,6 +6,7 @@
 
 #include <asm/processor.h>
 #include <asm/special_insns.h>
+#include <asm/smp.h>
 
 static inline void __invpcid(unsigned long pcid, unsigned long addr,
                             unsigned long type)
@@ -183,79 +184,6 @@ static inline void __flush_tlb_one(unsigned long addr)
  * and page-granular flushes are available only on i486 and up.
  */
 
-#ifndef CONFIG_SMP
-
-/* "_up" is for UniProcessor.
- *
- * This is a helper for other header functions.  *Not* intended to be called
- * directly.  All global TLB flushes need to either call this, or to bump the
- * vm statistics themselves.
- */
-static inline void __flush_tlb_up(void)
-{
-       count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
-       __flush_tlb();
-}
-
-static inline void flush_tlb_all(void)
-{
-       count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
-       __flush_tlb_all();
-}
-
-static inline void local_flush_tlb(void)
-{
-       __flush_tlb_up();
-}
-
-static inline void flush_tlb_mm(struct mm_struct *mm)
-{
-       if (mm == current->active_mm)
-               __flush_tlb_up();
-}
-
-static inline void flush_tlb_page(struct vm_area_struct *vma,
-                                 unsigned long addr)
-{
-       if (vma->vm_mm == current->active_mm)
-               __flush_tlb_one(addr);
-}
-
-static inline void flush_tlb_range(struct vm_area_struct *vma,
-                                  unsigned long start, unsigned long end)
-{
-       if (vma->vm_mm == current->active_mm)
-               __flush_tlb_up();
-}
-
-static inline void flush_tlb_mm_range(struct mm_struct *mm,
-          unsigned long start, unsigned long end, unsigned long vmflag)
-{
-       if (mm == current->active_mm)
-               __flush_tlb_up();
-}
-
-static inline void native_flush_tlb_others(const struct cpumask *cpumask,
-                                          struct mm_struct *mm,
-                                          unsigned long start,
-                                          unsigned long end)
-{
-}
-
-static inline void reset_lazy_tlbstate(void)
-{
-}
-
-static inline void flush_tlb_kernel_range(unsigned long start,
-                                         unsigned long end)
-{
-       flush_tlb_all();
-}
-
-#else  /* SMP */
-
-#include <asm/smp.h>
-
 #define local_flush_tlb() __flush_tlb()
 
 #define flush_tlb_mm(mm)       flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL)
@@ -292,8 +220,6 @@ static inline void reset_lazy_tlbstate(void)
        this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
 }
 
-#endif /* SMP */
-
 #ifndef CONFIG_PARAVIRT
 #define flush_tlb_others(mask, mm, start, end) \
        native_flush_tlb_others(mask, mm, start, end)

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 9c5b1f29f3f2b001e486ba03a488790b756598db..63ac57ef6f923bec8a825c071aebaf68c129ef90 100644 (file)
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -18,7 +18,7 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
                        = { &init_mm, 0, };
 
 /*
- *     Smarter SMP flushing macros.
+ *     TLB flushing, formerly SMP-only
  *             c/o Linus Torvalds.
  *
  *     These mean you can really definitely utterly forget about
@@ -31,8 +31,6 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
  *     Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi
  */
 
-#ifdef CONFIG_SMP
-
 struct flush_tlb_info {
        struct mm_struct *flush_mm;
        unsigned long flush_start;
@@ -55,8 +53,6 @@ void leave_mm(int cpu)
 }
 EXPORT_SYMBOL_GPL(leave_mm);
 
-#endif /* CONFIG_SMP */
-
 void switch_mm(struct mm_struct *prev, struct mm_struct *next,
               struct task_struct *tsk)
 {
@@ -73,10 +69,8 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
        unsigned cpu = smp_processor_id();
 
        if (likely(prev != next)) {
-#ifdef CONFIG_SMP
                this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
                this_cpu_write(cpu_tlbstate.active_mm, next);
-#endif
                cpumask_set_cpu(cpu, mm_cpumask(next));
 
                /*
@@ -115,9 +109,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
                /* Load the LDT, if the LDT is different: */
                if (unlikely(prev->context.ldt != next->context.ldt))
                        load_mm_ldt(next);
-       }
-#ifdef CONFIG_SMP
-         else {
+       } else {
                this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
                BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next);
 
@@ -142,11 +134,8 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
                        load_mm_ldt(next);
                }
        }
-#endif
 }
 
-#ifdef CONFIG_SMP
-
 /*
  * The flush IPI assumes that a thread switch happens in this order:
  * [cpu0: the cpu that switches]
@@ -387,5 +376,3 @@ static int __init create_tlb_single_page_flush_ceiling(void)
        return 0;
 }
 late_initcall(create_tlb_single_page_flush_ceiling);
-
-#endif /* CONFIG_SMP */