From d7181b4b753c8a5a6893036eeabe431724b21bd0 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 27 Apr 2003 19:51:24 +0100 Subject: [PATCH] [ARM] Inline PMD entry cache handling The common case is building a kernel for one CPU type, and we are able to allow GCC to optimise any the PMD entry cache handling assembly which will never be used. --- arch/arm/mach-sa1100/assabet.c | 2 ++ arch/arm/mm/mm-armv.c | 2 +- arch/arm/mm/proc-arm1020.S | 18 ------------- arch/arm/mm/proc-arm6_7.S | 13 --------- arch/arm/mm/proc-arm720.S | 11 -------- arch/arm/mm/proc-arm920.S | 15 ----------- arch/arm/mm/proc-arm922.S | 15 ----------- arch/arm/mm/proc-arm926.S | 17 ------------ arch/arm/mm/proc-sa110.S | 17 ------------ arch/arm/mm/proc-syms.c | 1 - arch/arm/mm/proc-xscale.S | 16 ----------- include/asm-arm/cpu-multi32.h | 5 ---- include/asm-arm/cpu-single.h | 2 -- include/asm-arm/proc-armv/pgalloc.h | 5 ++-- include/asm-arm/proc-armv/pgtable.h | 20 +++++++++----- include/asm-arm/proc-armv/tlbflush.h | 40 ++++++++++++++++++++++++++-- 16 files changed, 57 insertions(+), 142 deletions(-) diff --git a/arch/arm/mach-sa1100/assabet.c b/arch/arm/mach-sa1100/assabet.c index e57c2ccc10cda..b8312f498f633 100644 --- a/arch/arm/mach-sa1100/assabet.c +++ b/arch/arm/mach-sa1100/assabet.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #include #include diff --git a/arch/arm/mm/mm-armv.c b/arch/arm/mm/mm-armv.c index fc7ea269281d7..3017f7b1481f7 100644 --- a/arch/arm/mm/mm-armv.c +++ b/arch/arm/mm/mm-armv.c @@ -247,7 +247,7 @@ alloc_init_page(unsigned long virt, unsigned long phys, unsigned int prot_l1, pg pmdval = __pa(ptep) | prot_l1; pmdp[0] = __pmd(pmdval); pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t)); - cpu_flush_pmd(pmdp); + flush_pmd_entry(pmdp); } ptep = pte_offset_kernel(pmdp, virt); diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S index adadfd9fb4107..046d5ca42998e 100644 --- a/arch/arm/mm/proc-arm1020.S +++ b/arch/arm/mm/proc-arm1020.S @@ -364,23 +364,6 @@ ENTRY(cpu_arm1020_set_pgd) mcr p15, 0, r1, c8, c7, 0 @ invalidate I & D TLBs mov pc, lr -/* - * cpu_arm1020_flush_pmd(pmdp) - * - * Set a level 1 translation table entry, and clean it out of - * any caches such that the MMUs can load it correctly. - * - * pmdp: pointer to PMD entry - */ - .align 5 -ENTRY(cpu_arm1020_flush_pmd) -#ifndef CONFIG_CPU_DCACHE_DISABLE - mcr p15, 0, r0, c7, c10, 4 - mcr p15, 0, r0, c7, c10, 1 @ clean D entry (drain is done by TLB fns) -#endif - mcr p15, 0, r0, c7, c10, 4 @ drain WB - mov pc, lr - /* * cpu_arm1020_set_pte(ptep, pte) * @@ -511,7 +494,6 @@ arm1020_processor_functions: /* pgtable */ .word cpu_arm1020_set_pgd - .word cpu_arm1020_flush_pmd .word cpu_arm1020_set_pte .size arm1020_processor_functions, . - arm1020_processor_functions diff --git a/arch/arm/mm/proc-arm6_7.S b/arch/arm/mm/proc-arm6_7.S index 76b6eed5366cc..ef62add1cda68 100644 --- a/arch/arm/mm/proc-arm6_7.S +++ b/arch/arm/mm/proc-arm6_7.S @@ -233,17 +233,6 @@ ENTRY(cpu_arm7_set_pgd) mcr p15, 0, r1, c5, c0, 0 @ flush TLBs mov pc, lr -/* - * Function: arm6_flush_pmd(pmdp) - * - * Params : r0 = Address to set - * - * Purpose : Set a PMD and flush it out of any WB cache - */ -ENTRY(cpu_arm6_flush_pmd) -ENTRY(cpu_arm7_flush_pmd) - mov pc, lr - /* * Function: arm6_7_set_pte(pte_t *ptep, pte_t pte) * Params : r0 = Address to set @@ -346,7 +335,6 @@ ENTRY(arm6_processor_functions) /* pgtable */ .word cpu_arm6_set_pgd - .word cpu_arm6_flush_pmd .word cpu_arm6_set_pte .size arm6_processor_functions, . - arm6_processor_functions @@ -380,7 +368,6 @@ ENTRY(arm7_processor_functions) /* pgtable */ .word cpu_arm7_set_pgd - .word cpu_arm7_flush_pmd .word cpu_arm7_set_pte .size arm7_processor_functions, . - arm7_processor_functions diff --git a/arch/arm/mm/proc-arm720.S b/arch/arm/mm/proc-arm720.S index 391dc601d84d6..5fd31b10f2bbe 100644 --- a/arch/arm/mm/proc-arm720.S +++ b/arch/arm/mm/proc-arm720.S @@ -114,16 +114,6 @@ ENTRY(cpu_arm720_set_pgd) mcr p15, 0, r1, c8, c7, 0 @ flush TLB (v4) mov pc, lr -/* - * Function: arm720_flush_pmd(pmdp) - * - * Params : r0 = Address to set - * - * Purpose : Set a PMD and flush it out of any WB cache - */ -ENTRY(cpu_arm720_flush_pmd) - mov pc, lr - /* * Function: arm720_set_pte(pte_t *ptep, pte_t pte) * Params : r0 = Address to set @@ -216,7 +206,6 @@ ENTRY(arm720_processor_functions) /* pgtable */ .word cpu_arm720_set_pgd - .word cpu_arm720_flush_pmd .word cpu_arm720_set_pte .size arm720_processor_functions, . - arm720_processor_functions diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S index cbeaaf9a08532..2b6baae17adea 100644 --- a/arch/arm/mm/proc-arm920.S +++ b/arch/arm/mm/proc-arm920.S @@ -367,20 +367,6 @@ ENTRY(cpu_arm920_set_pgd) mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs mov pc, lr -/* - * cpu_arm920_flush_pmd(pmdp) - * - * Set a level 1 translation table entry, and clean it out of - * any caches such that the MMUs can load it correctly. - * - * pmdp: pointer to PMD entry - */ - .align 5 -ENTRY(cpu_arm920_flush_pmd) - mcr p15, 0, r0, c7, c10, 1 @ clean D entry - mcr p15, 0, r0, c7, c10, 4 @ drain WB - mov pc, lr - /* * cpu_arm920_set_pte(ptep, pte) * @@ -499,7 +485,6 @@ arm920_processor_functions: /* pgtable */ .word cpu_arm920_set_pgd - .word cpu_arm920_flush_pmd .word cpu_arm920_set_pte .size arm920_processor_functions, . - arm920_processor_functions diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S index f115e476bc7b2..a4cc0be63b63d 100644 --- a/arch/arm/mm/proc-arm922.S +++ b/arch/arm/mm/proc-arm922.S @@ -368,20 +368,6 @@ ENTRY(cpu_arm922_set_pgd) mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs mov pc, lr -/* - * cpu_arm922_flush_pmd(pmdp) - * - * Set a level 1 translation table entry, and clean it out of - * any caches such that the MMUs can load it correctly. - * - * pmdp: pointer to PMD entry - */ - .align 5 -ENTRY(cpu_arm922_flush_pmd) - mcr p15, 0, r0, c7, c10, 1 @ clean D entry - mcr p15, 0, r0, c7, c10, 4 @ drain WB - mov pc, lr - /* * cpu_arm922_set_pte(ptep, pte) * @@ -498,7 +484,6 @@ arm922_processor_functions: /* pgtable */ .word cpu_arm922_set_pgd - .word cpu_arm922_flush_pmd .word cpu_arm922_set_pte .size arm922_processor_functions, . - arm922_processor_functions diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S index a2cdcdd3d9f3c..fdaf694e81519 100644 --- a/arch/arm/mm/proc-arm926.S +++ b/arch/arm/mm/proc-arm926.S @@ -349,22 +349,6 @@ ENTRY(cpu_arm926_set_pgd) mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs mov pc, lr -/* - * cpu_arm926_flush_pmd(pmdp) - * - * Set a level 1 translation table entry, and clean it out of - * any caches such that the MMUs can load it correctly. - * - * pmdp: pointer to PMD entry - */ - .align 5 -ENTRY(cpu_arm926_flush_pmd) -#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH - mcr p15, 0, r0, c7, c10, 1 @ clean D entry -#endif - mcr p15, 0, r0, c7, c10, 4 @ drain WB - mov pc, lr - /* * cpu_arm926_set_pte(ptep, pte) * @@ -496,7 +480,6 @@ arm926_processor_functions: /* pgtable */ .word cpu_arm926_set_pgd - .word cpu_arm926_flush_pmd .word cpu_arm926_set_pte .size arm926_processor_functions, . - arm926_processor_functions diff --git a/arch/arm/mm/proc-sa110.S b/arch/arm/mm/proc-sa110.S index 7242fe7bfe70b..ffc12970010b9 100644 --- a/arch/arm/mm/proc-sa110.S +++ b/arch/arm/mm/proc-sa110.S @@ -424,21 +424,6 @@ ENTRY(cpu_sa1100_set_pgd) mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs mov pc, lr -/* - * cpu_sa110_flush_pmd(pmdp) - * - * Set a level 1 translation table entry, and clean it out of - * any caches such that the MMUs can load it correctly. - * - * pmdp: pointer to PMD entry - */ - .align 5 -ENTRY(cpu_sa110_flush_pmd) -ENTRY(cpu_sa1100_flush_pmd) - mcr p15, 0, r0, c7, c10, 1 @ clean D entry - mcr p15, 0, r0, c7, c10, 4 @ drain WB - mov pc, lr - /* * cpu_sa110_set_pte(ptep, pte) * @@ -540,7 +525,6 @@ ENTRY(sa110_processor_functions) /* pgtable */ .word cpu_sa110_set_pgd - .word cpu_sa110_flush_pmd .word cpu_sa110_set_pte .size sa110_processor_functions, . - sa110_processor_functions @@ -573,7 +557,6 @@ ENTRY(sa1100_processor_functions) /* pgtable */ .word cpu_sa1100_set_pgd - .word cpu_sa1100_flush_pmd .word cpu_sa1100_set_pte .size sa1100_processor_functions, . - sa1100_processor_functions diff --git a/arch/arm/mm/proc-syms.c b/arch/arm/mm/proc-syms.c index 47c5448e619f0..697fb05c70c47 100644 --- a/arch/arm/mm/proc-syms.c +++ b/arch/arm/mm/proc-syms.c @@ -27,7 +27,6 @@ EXPORT_SYMBOL(cpu_dcache_invalidate_range); EXPORT_SYMBOL(cpu_icache_invalidate_range); EXPORT_SYMBOL(cpu_icache_invalidate_page); EXPORT_SYMBOL(cpu_set_pgd); -EXPORT_SYMBOL(cpu_flush_pmd); EXPORT_SYMBOL(cpu_set_pte); #else EXPORT_SYMBOL(processor); diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S index ba13ebc13e2eb..71855af07d412 100644 --- a/arch/arm/mm/proc-xscale.S +++ b/arch/arm/mm/proc-xscale.S @@ -572,21 +572,6 @@ ENTRY(cpu_xscale_set_pgd) mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs cpwait_ret lr, ip -/* - * cpu_xscale_flush_pmd(pmdp) - * - * Set a level 1 translation table entry, and clean it out of - * any caches such that the MMUs can load it correctly. - * - * pmdp: pointer to PMD entry - */ - .align 5 -ENTRY(cpu_xscale_flush_pmd) - mov ip, #0 - mcr p15, 0, r0, c7, c10, 1 @ Clean D cache line - mcr p15, 0, ip, c7, c10, 4 @ Drain Write (& Fill) Buffer - mov pc, lr - /* * cpu_xscale_set_pte(ptep, pte) * @@ -719,7 +704,6 @@ ENTRY(xscale_processor_functions) /* pgtable */ .word cpu_xscale_set_pgd - .word cpu_xscale_flush_pmd .word cpu_xscale_set_pte .size xscale_processor_functions, . - xscale_processor_functions diff --git a/include/asm-arm/cpu-multi32.h b/include/asm-arm/cpu-multi32.h index 0ed88c812944b..59835af6fa00e 100644 --- a/include/asm-arm/cpu-multi32.h +++ b/include/asm-arm/cpu-multi32.h @@ -93,10 +93,6 @@ extern struct processor { * Set the page table */ void (*set_pgd)(unsigned long pgd_phys, struct mm_struct *mm); - /* - * Set a PMD (handling IMP bit 4) - */ - void (*flush_pmd)(pmd_t *pmdp); /* * Set a PTE */ @@ -126,7 +122,6 @@ extern const struct processor sa110_processor_functions; #define cpu_icache_invalidate_page(vp) processor.icache.invalidate_page(vp) #define cpu_set_pgd(pgd,mm) processor.pgtable.set_pgd(pgd,mm) -#define cpu_flush_pmd(pmdp) processor.pgtable.flush_pmd(pmdp) #define cpu_set_pte(ptep, pte) processor.pgtable.set_pte(ptep, pte) #define cpu_switch_mm(pgd,mm) cpu_set_pgd(__virt_to_phys((unsigned long)(pgd)),mm) diff --git a/include/asm-arm/cpu-single.h b/include/asm-arm/cpu-single.h index 843e3a03db587..738b61dc2553c 100644 --- a/include/asm-arm/cpu-single.h +++ b/include/asm-arm/cpu-single.h @@ -36,7 +36,6 @@ #define cpu_icache_invalidate_range __cpu_fn(CPU_NAME,_icache_invalidate_range) #define cpu_icache_invalidate_page __cpu_fn(CPU_NAME,_icache_invalidate_page) #define cpu_set_pgd __cpu_fn(CPU_NAME,_set_pgd) -#define cpu_flush_pmd __cpu_fn(CPU_NAME,_flush_pmd) #define cpu_set_pte __cpu_fn(CPU_NAME,_set_pte) #ifndef __ASSEMBLY__ @@ -65,7 +64,6 @@ extern void cpu_icache_invalidate_range(unsigned long start, unsigned long end); extern void cpu_icache_invalidate_page(void *virt_page); extern void cpu_set_pgd(unsigned long pgd_phys, struct mm_struct *mm); -extern void cpu_flush_pmd(pmd_t *pmdp); extern void cpu_set_pte(pte_t *ptep, pte_t pte); extern volatile void cpu_reset(unsigned long addr); diff --git a/include/asm-arm/proc-armv/pgalloc.h b/include/asm-arm/proc-armv/pgalloc.h index 3263c346ccba8..0e65ab7362e43 100644 --- a/include/asm-arm/proc-armv/pgalloc.h +++ b/include/asm-arm/proc-armv/pgalloc.h @@ -6,6 +6,7 @@ * Page table allocation/freeing primitives for 32-bit ARM processors. */ #include +#include #include "pgtable.h" /* @@ -92,7 +93,7 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) pmdval = __pa(pte_ptr) | _PAGE_KERNEL_TABLE; pmdp[0] = __pmd(pmdval); pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t)); - cpu_flush_pmd(pmdp); + flush_pmd_entry(pmdp); } static inline void @@ -105,5 +106,5 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep) pmdval = page_to_pfn(ptep) << PAGE_SHIFT | _PAGE_USER_TABLE; pmdp[0] = __pmd(pmdval); pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t)); - cpu_flush_pmd(pmdp); + flush_pmd_entry(pmdp); } diff --git a/include/asm-arm/proc-armv/pgtable.h b/include/asm-arm/proc-armv/pgtable.h index 53f2b3da4d16d..616d80d69b1f8 100644 --- a/include/asm-arm/proc-armv/pgtable.h +++ b/include/asm-arm/proc-armv/pgtable.h @@ -51,6 +51,7 @@ #define PMD_SECT_TEX(x) ((x) << 12) /* v5 */ #define PMD_SECT_UNCACHED (0) +#define PMD_SECT_BUFFERED (PMD_SECT_BUFFERABLE) #define PMD_SECT_WT (PMD_SECT_CACHEABLE) #define PMD_SECT_WB (PMD_SECT_CACHEABLE | PMD_SECT_BUFFERABLE) #define PMD_SECT_MINICACHE (PMD_SECT_TEX(1) | PMD_SECT_CACHEABLE) @@ -120,14 +121,19 @@ #define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_KERNEL)) #define pmd_bad(pmd) (pmd_val(pmd) & 2) -#define set_pmd(pmdp,pmd) do { *pmdp = pmd; cpu_flush_pmd(pmdp); } while (0) -static inline void pmd_clear(pmd_t *pmdp) -{ - pmdp[0] = __pmd(0); - pmdp[1] = __pmd(0); - cpu_flush_pmd(pmdp); -} +#define set_pmd(pmdp,pmd) \ + do { \ + *pmdp = pmd; \ + flush_pmd_entry(pmdp); \ + } while (0) + +#define pmd_clear(pmdp) \ + do { \ + pmdp[0] = __pmd(0); \ + pmdp[1] = __pmd(0); \ + clean_pmd_entry(pmdp); \ + } while (0) static inline pte_t *pmd_page_kernel(pmd_t pmd) { diff --git a/include/asm-arm/proc-armv/tlbflush.h b/include/asm-arm/proc-armv/tlbflush.h index d063ede9c50f7..278c0624c11f6 100644 --- a/include/asm-arm/proc-armv/tlbflush.h +++ b/include/asm-arm/proc-armv/tlbflush.h @@ -20,6 +20,7 @@ #define TLB_V4_D_FULL (1 << 10) #define TLB_V4_I_FULL (1 << 11) +#define TLB_DCLEAN (1 << 30) #define TLB_WB (1 << 31) /* @@ -65,7 +66,7 @@ # define v4_always_flags (-1UL) #endif -#define v4wbi_tlb_flags (TLB_WB | \ +#define v4wbi_tlb_flags (TLB_WB | TLB_DCLEAN | \ TLB_V4_I_FULL | TLB_V4_D_FULL | \ TLB_V4_I_PAGE | TLB_V4_D_PAGE) @@ -84,7 +85,7 @@ # define v4wbi_always_flags (-1UL) #endif -#define v4wb_tlb_flags (TLB_WB | \ +#define v4wb_tlb_flags (TLB_WB | TLB_DCLEAN | \ TLB_V4_I_FULL | TLB_V4_D_FULL | \ TLB_V4_D_PAGE) @@ -287,6 +288,41 @@ static inline void flush_tlb_kernel_page(unsigned long kaddr) asm("mcr%? p15, 0, %0, c8, c5, 0" : : "r" (zero)); } +/* + * flush_pmd_entry + * + * Flush a PMD entry (word aligned, or double-word aligned) to + * RAM if the TLB for the CPU we are running on requires this. + * This is typically used when we are creating PMD entries. + * + * clean_pmd_entry + * + * Clean (but don't drain the write buffer) if the CPU requires + * these operations. This is typically used when we are removing + * PMD entries. + */ +static inline void flush_pmd_entry(pmd_t *pmd) +{ + const unsigned int zero = 0; + const unsigned int __tlb_flag = __cpu_tlb_flags; + + if (tlb_flag(TLB_DCLEAN)) + asm("mcr%? p15, 0, %0, c7, c10, 1 @ flush_pmd" + : : "r" (pmd)); + if (tlb_flag(TLB_WB)) + asm("mcr%? p15, 0, %0, c7, c10, 4 @ flush_pmd" + : : "r" (zero)); +} + +static inline void clean_pmd_entry(pmd_t *pmd) +{ + const unsigned int __tlb_flag = __cpu_tlb_flags; + + if (tlb_flag(TLB_DCLEAN)) + asm("mcr%? p15, 0, %0, c7, c10, 1 @ flush_pmd" + : : "r" (pmd)); +} + #undef tlb_flag #undef always_tlb_flags #undef possible_tlb_flags -- 2.53.0