[ARM] Make TLB instruction selection more fine-grained

This eliminates some unnecessary branches and instructions in the
TLB flush macros where configurations allow for this to occur.

diff --git a/include/asm-arm/proc-armv/tlbflush.h b/include/asm-arm/proc-armv/tlbflush.h
index fd758b2e90f2af31b934219209ffba81e92aa9e8..d063ede9c50f71c181f9ac4ed1fd7b697eb607e7 100644 (file)
--- a/include/asm-arm/proc-armv/tlbflush.h
+++ b/include/asm-arm/proc-armv/tlbflush.h
@@ -1,7 +1,7 @@
 /*
  *  linux/include/asm-arm/proc-armv/tlbflush.h
  *
- *  Copyright (C) 1999-2002 Russell King
+ *  Copyright (C) 1999-2003 Russell King
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -38,21 +38,31 @@
 #define v3_tlb_flags   (TLB_V3_FULL | TLB_V3_PAGE)
 
 #if defined(CONFIG_CPU_ARM610) || defined(CONFIG_CPU_ARM710)
+# define v3_possible_flags     v3_tlb_flags
+# define v3_always_flags       v3_tlb_flags
 # ifdef _TLB
 #  define MULTI_TLB 1
 # else
 #  define _TLB v3
 # endif
+#else
+# define v3_possible_flags     0
+# define v3_always_flags       (-1UL)
 #endif
 
 #define v4_tlb_flags   (TLB_V4_U_FULL | TLB_V4_U_PAGE)
 
 #if defined(CONFIG_CPU_ARM720T)
+# define v4_possible_flags     v4_tlb_flags
+# define v4_always_flags       v4_tlb_flags
 # ifdef _TLB
 #  define MULTI_TLB 1
 # else
 #  define _TLB v4
 # endif
+#else
+# define v4_possible_flags     0
+# define v4_always_flags       (-1UL)
 #endif
 
 #define v4wbi_tlb_flags        (TLB_WB | \
@@ -62,11 +72,16 @@
 #if defined(CONFIG_CPU_ARM920T) || defined(CONFIG_CPU_ARM922T) || \
     defined(CONFIG_CPU_ARM926T) || defined(CONFIG_CPU_ARM1020) || \
     defined(CONFIG_CPU_XSCALE)
+# define v4wbi_possible_flags  v4wbi_tlb_flags
+# define v4wbi_always_flags    v4wbi_tlb_flags
 # ifdef _TLB
 #  define MULTI_TLB 1
 # else
 #  define _TLB v4wbi
 # endif
+#else
+# define v4wbi_possible_flags  0
+# define v4wbi_always_flags    (-1UL)
 #endif
 
 #define v4wb_tlb_flags (TLB_WB | \
@@ -74,11 +89,16 @@
                         TLB_V4_D_PAGE)
 
 #if defined(CONFIG_CPU_SA110) || defined(CONFIG_CPU_SA1100)
+# define v4wb_possible_flags   v4wb_tlb_flags
+# define v4wb_always_flags     v4wb_tlb_flags
 # ifdef _TLB
 #  define MULTI_TLB 1
 # else
 #  define _TLB v4wb
 # endif
+#else
+# define v4wb_possible_flags   0
+# define v4wb_always_flags     (-1UL)
 #endif
 
 #ifndef _TLB
@@ -98,23 +118,23 @@ struct cpu_tlb_fns {
  */
 #ifdef MULTI_TLB
 
-extern struct cpu_tlb_fns cpu_tlb;
-
 #define __cpu_flush_user_tlb_range     cpu_tlb.flush_user_range
 #define __cpu_flush_kern_tlb_range     cpu_tlb.flush_kern_range
-#define __cpu_tlb_flags                        cpu_tlb.tlb_flags
 
 #else
 
 #define __cpu_flush_user_tlb_range     __glue(_TLB,_flush_user_tlb_range)
 #define __cpu_flush_kern_tlb_range     __glue(_TLB,_flush_kern_tlb_range)
-#define __cpu_tlb_flags                        __glue(_TLB,_tlb_flags)
 
 extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
 extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long);
 
 #endif
 
+extern struct cpu_tlb_fns cpu_tlb;
+
+#define __cpu_tlb_flags                        cpu_tlb.tlb_flags
+
 /*
  *     TLB Management
  *     ==============
@@ -158,11 +178,34 @@ extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long);
  *             - kaddr - Kernel virtual memory address
  */
 
-#define tlb_flag(f)    (__cpu_tlb_flags & (f))
+/*
+ * We optimise the code below by:
+ *  - building a set of TLB flags that might be set in __cpu_tlb_flags
+ *  - building a set of TLB flags that will always be set in __cpu_tlb_flags
+ *  - if we're going to need __cpu_tlb_flags, access it once and only once
+ *
+ * This allows us to build optimal assembly for the single-CPU type case,
+ * and as close to optimal given the compiler constrants for multi-CPU
+ * case.  We could do better for the multi-CPU case if the compiler
+ * implemented the "%?" method, but this has been discontinued due to too
+ * many people getting it wrong.
+ */
+#define possible_tlb_flags     (v3_possible_flags | \
+                                v4_possible_flags | \
+                                v4wbi_possible_flags | \
+                                v4wb_possible_flags)
+
+#define always_tlb_flags       (v3_always_flags & \
+                                v4_always_flags & \
+                                v4wbi_always_flags & \
+                                v4wb_always_flags)
+
+#define tlb_flag(f)    ((always_tlb_flags & (f)) || (__tlb_flag & possible_tlb_flags & (f)))
 
 static inline void flush_tlb_all(void)
 {
        const int zero = 0;
+       const unsigned int __tlb_flag = __cpu_tlb_flags;
 
        if (tlb_flag(TLB_WB))
                asm("mcr%? p15, 0, %0, c7, c10, 4" : : "r" (zero));
@@ -180,6 +223,7 @@ static inline void flush_tlb_all(void)
 static inline void flush_tlb_mm(struct mm_struct *mm)
 {
        const int zero = 0;
+       const unsigned int __tlb_flag = __cpu_tlb_flags;
 
        if (tlb_flag(TLB_WB))
                asm("mcr%? p15, 0, %0, c7, c10, 4" : : "r" (zero));
@@ -200,6 +244,7 @@ static inline void
 flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 {
        const int zero = 0;
+       const unsigned int __tlb_flag = __cpu_tlb_flags;
 
        uaddr &= PAGE_MASK;
 
@@ -223,6 +268,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 static inline void flush_tlb_kernel_page(unsigned long kaddr)
 {
        const int zero = 0;
+       const unsigned int __tlb_flag = __cpu_tlb_flags;
 
        kaddr &= PAGE_MASK;
 
@@ -241,6 +287,10 @@ static inline void flush_tlb_kernel_page(unsigned long kaddr)
                asm("mcr%? p15, 0, %0, c8, c5, 0" : : "r" (zero));
 }
 
+#undef tlb_flag
+#undef always_tlb_flags
+#undef possible_tlb_flags
+
 /*
  * Convert calls to our calling convention.
  */