From 6e04296b6d79c5ca4a68f94ea7439b857f1e2de0 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Fri, 11 Jun 2004 23:01:09 -0700 Subject: [PATCH] [SPARC64]: Preemption fixes, use get_cpu() et al. where applicable. --- arch/sparc64/kernel/chmc.c | 5 ++++- arch/sparc64/kernel/irq.c | 4 ++++ arch/sparc64/kernel/smp.c | 27 +++++++++++++++++++++------ arch/sparc64/lib/debuglocks.c | 31 ++++++++++++++++++++++++------- arch/sparc64/mm/init.c | 19 +++++++++++++------ 5 files changed, 66 insertions(+), 20 deletions(-) diff --git a/arch/sparc64/kernel/chmc.c b/arch/sparc64/kernel/chmc.c index 6e39a101f7d82..2296525af8551 100644 --- a/arch/sparc64/kernel/chmc.c +++ b/arch/sparc64/kernel/chmc.c @@ -243,8 +243,9 @@ int chmc_getunumber(int syndrome_code, static u64 read_mcreg(struct mctrl_info *mp, unsigned long offset) { unsigned long ret; + int this_cpu = get_cpu(); - if (mp->portid == smp_processor_id()) { + if (mp->portid == this_cpu) { __asm__ __volatile__("ldxa [%1] %2, %0" : "=r" (ret) : "r" (offset), "i" (ASI_MCU_CTRL_REG)); @@ -254,6 +255,8 @@ static u64 read_mcreg(struct mctrl_info *mp, unsigned long offset) : "r" (mp->regs + offset), "i" (ASI_PHYS_BYPASS_EC_E)); } + put_cpu(); + return ret; } diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c index 9a48757d33956..b7f6a1eb922fd 100644 --- a/arch/sparc64/kernel/irq.c +++ b/arch/sparc64/kernel/irq.c @@ -174,6 +174,8 @@ void enable_irq(unsigned int irq) if (imap == 0UL) return; + preempt_disable(); + if (tlb_type == cheetah || tlb_type == cheetah_plus) { unsigned long ver; @@ -214,6 +216,8 @@ void enable_irq(unsigned int irq) * Things like FFB can now be handled via the new IRQ mechanism. */ upa_writel(tid | IMAP_VALID, imap); + + preempt_enable(); } /* This now gets passed true ino's as well. */ diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c index 040323b3855e2..64b873212243f 100644 --- a/arch/sparc64/kernel/smp.c +++ b/arch/sparc64/kernel/smp.c @@ -547,15 +547,18 @@ retry: static void smp_cross_call_masked(unsigned long *func, u32 ctx, u64 data1, u64 data2, cpumask_t mask) { u64 data0 = (((u64)ctx)<<32 | (((u64)func) & 0xffffffff)); + int this_cpu = get_cpu(); cpus_and(mask, mask, cpu_online_map); - cpu_clear(smp_processor_id(), mask); + cpu_clear(this_cpu, mask); if (tlb_type == spitfire) spitfire_xcall_deliver(data0, data1, data2, mask); else cheetah_xcall_deliver(data0, data1, data2, mask); /* NOTE: Caller runs local copy on master. */ + + put_cpu(); } extern unsigned long xcall_sync_tick; @@ -685,11 +688,12 @@ static __inline__ void __local_flush_dcache_page(struct page *page) void smp_flush_dcache_page_impl(struct page *page, int cpu) { cpumask_t mask = cpumask_of_cpu(cpu); + int this_cpu = get_cpu(); #ifdef CONFIG_DEBUG_DCFLUSH atomic_inc(&dcpage_flushes); #endif - if (cpu == smp_processor_id()) { + if (cpu == this_cpu) { __local_flush_dcache_page(page); } else if (cpu_online(cpu)) { u64 data0; @@ -714,14 +718,17 @@ void smp_flush_dcache_page_impl(struct page *page, int cpu) atomic_inc(&dcpage_flushes_xcall); #endif } + + put_cpu(); } void flush_dcache_page_all(struct mm_struct *mm, struct page *page) { cpumask_t mask = cpu_online_map; u64 data0; + int this_cpu = get_cpu(); - cpu_clear(smp_processor_id(), mask); + cpu_clear(this_cpu, mask); #ifdef CONFIG_DEBUG_DCFLUSH atomic_inc(&dcpage_flushes); @@ -747,6 +754,8 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page) #endif flush_self: __local_flush_dcache_page(page); + + put_cpu(); } void smp_receive_signal(int cpu) @@ -842,7 +851,7 @@ void smp_flush_tlb_mm(struct mm_struct *mm) { u32 ctx = CTX_HWBITS(mm->context); - int cpu = smp_processor_id(); + int cpu = get_cpu(); if (atomic_read(&mm->mm_users) == 1) { /* See smp_flush_tlb_page for info about this. */ @@ -856,6 +865,8 @@ void smp_flush_tlb_mm(struct mm_struct *mm) local_flush_and_out: __flush_tlb_mm(ctx, SECONDARY_CONTEXT); + + put_cpu(); } } @@ -863,7 +874,7 @@ void smp_flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long end) { u32 ctx = CTX_HWBITS(mm->context); - int cpu = smp_processor_id(); + int cpu = get_cpu(); start &= PAGE_MASK; end = PAGE_ALIGN(end); @@ -880,6 +891,8 @@ void smp_flush_tlb_range(struct mm_struct *mm, unsigned long start, local_flush_and_out: __flush_tlb_range(ctx, start, SECONDARY_CONTEXT, end, PAGE_SIZE, (end-start)); + + put_cpu(); } void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end) @@ -898,7 +911,7 @@ void smp_flush_tlb_page(struct mm_struct *mm, unsigned long page) { { u32 ctx = CTX_HWBITS(mm->context); - int cpu = smp_processor_id(); + int cpu = get_cpu(); page &= PAGE_MASK; if (mm == current->active_mm && @@ -938,6 +951,8 @@ void smp_flush_tlb_page(struct mm_struct *mm, unsigned long page) local_flush_and_out: __flush_tlb_page(ctx, page, SECONDARY_CONTEXT); + + put_cpu(); } } diff --git a/arch/sparc64/lib/debuglocks.c b/arch/sparc64/lib/debuglocks.c index 7b68a4debb35f..9e083e27d2db9 100644 --- a/arch/sparc64/lib/debuglocks.c +++ b/arch/sparc64/lib/debuglocks.c @@ -55,7 +55,7 @@ void _do_spin_lock(spinlock_t *lock, char *str) { unsigned long caller, val; int stuck = INIT_STUCK; - int cpu = smp_processor_id(); + int cpu = get_cpu(); int shown = 0; GET_CALLER(caller); @@ -80,12 +80,14 @@ again: lock->owner_cpu = cpu; current->thread.smp_lock_count++; current->thread.smp_lock_pc = ((unsigned int)caller); + + put_cpu(); } int _spin_trylock(spinlock_t *lock) { unsigned long val, caller; - int cpu = smp_processor_id(); + int cpu = get_cpu(); GET_CALLER(caller); __asm__ __volatile__("ldstub [%1], %0" @@ -99,6 +101,9 @@ int _spin_trylock(spinlock_t *lock) current->thread.smp_lock_count++; current->thread.smp_lock_pc = ((unsigned int)caller); } + + put_cpu(); + return val == 0; } @@ -117,7 +122,7 @@ void _do_read_lock (rwlock_t *rw, char *str) { unsigned long caller, val; int stuck = INIT_STUCK; - int cpu = smp_processor_id(); + int cpu = get_cpu(); int shown = 0; GET_CALLER(caller); @@ -148,13 +153,15 @@ wlock_again: rw->reader_pc[cpu] = ((unsigned int)caller); current->thread.smp_lock_count++; current->thread.smp_lock_pc = ((unsigned int)caller); + + put_cpu(); } void _do_read_unlock (rwlock_t *rw, char *str) { unsigned long caller, val; int stuck = INIT_STUCK; - int cpu = smp_processor_id(); + int cpu = get_cpu(); int shown = 0; GET_CALLER(caller); @@ -181,13 +188,15 @@ runlock_again: } goto runlock_again; } + + put_cpu(); } void _do_write_lock (rwlock_t *rw, char *str) { unsigned long caller, val; int stuck = INIT_STUCK; - int cpu = smp_processor_id(); + int cpu = get_cpu(); int shown = 0; GET_CALLER(caller); @@ -263,6 +272,8 @@ wlock_again: rw->writer_cpu = cpu; current->thread.smp_lock_count++; current->thread.smp_lock_pc = ((unsigned int)caller); + + put_cpu(); } void _do_write_unlock(rwlock_t *rw) @@ -302,7 +313,7 @@ wlock_again: int _do_write_trylock (rwlock_t *rw, char *str) { unsigned long caller, val; - int cpu = smp_processor_id(); + int cpu = get_cpu(); GET_CALLER(caller); @@ -322,8 +333,10 @@ int _do_write_trylock (rwlock_t *rw, char *str) : "0" (&(rw->lock)) : "g3", "g5", "g7", "memory"); - if (val) + if (val) { + put_cpu(); return 0; + } if ((rw->lock & ((1UL<<63)-1UL)) != 0UL) { /* Readers still around, drop the write @@ -342,6 +355,8 @@ int _do_write_trylock (rwlock_t *rw, char *str) : "r" (&(rw->lock)) : "g3", "g5", "g7", "cc", "memory"); + put_cpu(); + return 0; } @@ -351,6 +366,8 @@ int _do_write_trylock (rwlock_t *rw, char *str) current->thread.smp_lock_count++; current->thread.smp_lock_pc = ((unsigned int)caller); + put_cpu(); + return 1; } diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c index 1e9519db059df..043861f313c2b 100644 --- a/arch/sparc64/mm/init.c +++ b/arch/sparc64/mm/init.c @@ -152,9 +152,9 @@ __inline__ void flush_dcache_page_impl(struct page *page) #define dcache_dirty_cpu(page) \ (((page)->flags >> 24) & (NR_CPUS - 1UL)) -static __inline__ void set_dcache_dirty(struct page *page) +static __inline__ void set_dcache_dirty(struct page *page, int this_cpu) { - unsigned long mask = smp_processor_id(); + unsigned long mask = this_cpu; unsigned long non_cpu_bits = ~((NR_CPUS - 1UL) << 24UL); mask = (mask << 24) | (1UL << PG_dcache_dirty); __asm__ __volatile__("1:\n\t" @@ -206,16 +206,19 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t p (page = pfn_to_page(pfn), page_mapping(page)) && ((pg_flags = page->flags) & (1UL << PG_dcache_dirty))) { int cpu = ((pg_flags >> 24) & (NR_CPUS - 1UL)); + int this_cpu = get_cpu(); /* This is just to optimize away some function calls * in the SMP case. */ - if (cpu == smp_processor_id()) + if (cpu == this_cpu) flush_dcache_page_impl(page); else smp_flush_dcache_page_impl(page, cpu); clear_dcache_dirty_cpu(page, cpu); + + put_cpu(); } if (get_thread_fault_code()) __update_mmu_cache(vma->vm_mm->context & TAG_CONTEXT_BITS, @@ -227,14 +230,15 @@ void flush_dcache_page(struct page *page) struct address_space *mapping = page_mapping(page); int dirty = test_bit(PG_dcache_dirty, &page->flags); int dirty_cpu = dcache_dirty_cpu(page); + int this_cpu = get_cpu(); if (mapping && !mapping_mapped(mapping)) { if (dirty) { - if (dirty_cpu == smp_processor_id()) - return; + if (dirty_cpu == this_cpu) + goto out; smp_flush_dcache_page_impl(page, dirty_cpu); } - set_dcache_dirty(page); + set_dcache_dirty(page, this_cpu); } else { /* We could delay the flush for the !page_mapping * case too. But that case is for exec env/arg @@ -243,6 +247,9 @@ void flush_dcache_page(struct page *page) */ flush_dcache_page_impl(page); } + +out: + put_cpu(); } /* When shared+writable mmaps of files go away, we lose all dirty -- 2.53.0