mm, x86: Account for TLB flushes only when debugging

commit ec65993443736a5091b68e80ff1734548944a4b8 upstream.

Bisection between 3.11 and 3.12 fingered commit 9824cf97 ("mm:
vmstats: tlb flush counters") to cause overhead problems.

The counters are undeniably useful but how often do we really
need to debug TLB flush related issues?  It does not justify
taking the penalty everywhere so make it a debugging option.

Signed-off-by: Mel Gorman <mgorman@suse.de>
Tested-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: Alex Shi <alex.shi@linaro.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/n/tip-XzxjntugxuwpxXhcrxqqh53b@git.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Signed-off-by: Mel Gorman <mgorman@suse.de>
Signed-off-by: Jiri Slaby <jslaby@suse.cz>

diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index e6d90babc245c1c2713bea64d436fb2a736b83e6..04905bfc508b9925c697687b7c1d5754827ba417 100644 (file)
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -62,7 +62,7 @@ static inline void __flush_tlb_all(void)
 
 static inline void __flush_tlb_one(unsigned long addr)
 {
-       count_vm_event(NR_TLB_LOCAL_FLUSH_ONE);
+       count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
        __flush_tlb_single(addr);
 }
 
@@ -93,13 +93,13 @@ static inline void __flush_tlb_one(unsigned long addr)
  */
 static inline void __flush_tlb_up(void)
 {
-       count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
+       count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
        __flush_tlb();
 }
 
 static inline void flush_tlb_all(void)
 {
-       count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
+       count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
        __flush_tlb_all();
 }
 

diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index ce2d0a2c3e4ff56819152574eefded2e8d64ea69..0e25a1bc5ab5cfbbf21484ce268ad17ed48844ec 100644 (file)
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -683,7 +683,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
        }
 
        /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
-       count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
+       count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
        __flush_tlb();
 
        /* Save MTRR state */
@@ -697,7 +697,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
 static void post_set(void) __releases(set_atomicity_lock)
 {
        /* Flush TLBs (no need to flush caches - they are disabled) */
-       count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
+       count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
        __flush_tlb();
 
        /* Intel (P6) standard MTRRs */

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index ae699b3bbac84a920042349c1fc8605a8f93aba0..05446c1cccfe02bbbb75ad51a36a8835afe89dba 100644 (file)
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -103,7 +103,7 @@ static void flush_tlb_func(void *info)
        if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm))
                return;
 
-       count_vm_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
+       count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
        if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
                if (f->flush_end == TLB_FLUSH_ALL)
                        local_flush_tlb();
@@ -131,7 +131,7 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
        info.flush_start = start;
        info.flush_end = end;
 
-       count_vm_event(NR_TLB_REMOTE_FLUSH);
+       count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
        if (is_uv_system()) {
                unsigned int cpu;
 
@@ -151,7 +151,7 @@ void flush_tlb_current_task(void)
 
        preempt_disable();
 
-       count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
+       count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
        local_flush_tlb();
        if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
                flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
@@ -215,7 +215,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
 
        /* tlb_flushall_shift is on balance point, details in commit log */
        if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift) {
-               count_vm_event(NR_TLB_LOCAL_FLUSH_ALL);
+               count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
                local_flush_tlb();
        } else {
                if (has_large_page(mm, start, end)) {
@@ -224,7 +224,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
                }
                /* flush range by one by one 'invlpg' */
                for (addr = start; addr < end;  addr += PAGE_SIZE) {
-                       count_vm_event(NR_TLB_LOCAL_FLUSH_ONE);
+                       count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
                        __flush_tlb_single(addr);
                }
 
@@ -262,7 +262,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
 
 static void do_flush_tlb_all(void *info)
 {
-       count_vm_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
+       count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
        __flush_tlb_all();
        if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
                leave_mm(smp_processor_id());
@@ -270,7 +270,7 @@ static void do_flush_tlb_all(void *info)
 
 void flush_tlb_all(void)
 {
-       count_vm_event(NR_TLB_REMOTE_FLUSH);
+       count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
        on_each_cpu(do_flush_tlb_all, NULL, 1);
 }
 

diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index c557c6d096def9a93bb5b00af4df87166539e048..3a712e2e7d762f479fc91377f1e37a9186172dc9 100644 (file)
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -71,12 +71,14 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
                THP_ZERO_PAGE_ALLOC,
                THP_ZERO_PAGE_ALLOC_FAILED,
 #endif
+#ifdef CONFIG_DEBUG_TLBFLUSH
 #ifdef CONFIG_SMP
                NR_TLB_REMOTE_FLUSH,    /* cpu tried to flush others' tlbs */
                NR_TLB_REMOTE_FLUSH_RECEIVED,/* cpu received ipi for flush */
-#endif
+#endif /* CONFIG_SMP */
                NR_TLB_LOCAL_FLUSH_ALL,
                NR_TLB_LOCAL_FLUSH_ONE,
+#endif /* CONFIG_DEBUG_TLBFLUSH */
                NR_VM_EVENT_ITEMS
 };
 

diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index a67b384157689ec9fda2abfec0173122fc98a4a8..67ce70c8279be201e8ecb33fed569845a5386257 100644 (file)
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -83,6 +83,14 @@ static inline void vm_events_fold_cpu(int cpu)
 #define count_vm_numa_events(x, y) do { (void)(y); } while (0)
 #endif /* CONFIG_NUMA_BALANCING */
 
+#ifdef CONFIG_DEBUG_TLBFLUSH
+#define count_vm_tlb_event(x)     count_vm_event(x)
+#define count_vm_tlb_events(x, y)  count_vm_events(x, y)
+#else
+#define count_vm_tlb_event(x)     do {} while (0)
+#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
+#endif
+
 #define __count_zone_vm_events(item, zone, delta) \
                __count_vm_events(item##_NORMAL - ZONE_NORMAL + \
                zone_idx(zone), delta)

diff --git a/mm/vmstat.c b/mm/vmstat.c
index 5a442a723d79926aad4b96b02523d08ebf68f938..8e8acfa0836d099a99ac23e791fe594ac3a551b8 100644 (file)
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -851,12 +851,14 @@ const char * const vmstat_text[] = {
        "thp_zero_page_alloc",
        "thp_zero_page_alloc_failed",
 #endif
+#ifdef CONFIG_DEBUG_TLBFLUSH
 #ifdef CONFIG_SMP
        "nr_tlb_remote_flush",
        "nr_tlb_remote_flush_received",
-#endif
+#endif /* CONFIG_SMP */
        "nr_tlb_local_flush_all",
        "nr_tlb_local_flush_one",
+#endif /* CONFIG_DEBUG_TLBFLUSH */
 
 #endif /* CONFIG_VM_EVENTS_COUNTERS */
 };