[PATCH] out-of-line locks / sparc64

Signed-off-by: Zwane Mwaikambo <zwane@fsmlabs.com>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/arch/sparc64/kernel/time.c b/arch/sparc64/kernel/time.c
index c83de2ecfb3465993c0d64d64a4c04d734814c77..ecd881fa633349d2c15f7ee83424e7cb04b47d45 100644 (file)
--- a/arch/sparc64/kernel/time.c
+++ b/arch/sparc64/kernel/time.c
@@ -68,6 +68,19 @@ struct sparc64_tick_ops *tick_ops;
 
 #define TICK_PRIV_BIT  (1UL << 63)
 
+#ifdef CONFIG_SMP
+unsigned long profile_pc(struct pt_regs *regs)
+{
+       unsigned long pc = instruction_pointer(regs);
+
+       if (pc >= (unsigned long)&__lock_text_start &&
+           pc <= (unsigned long)&__lock_text_end)
+               return regs->u_regs[UREG_RETPC];
+       return pc;
+}
+EXPORT_SYMBOL(profile_pc);
+#endif
+
 static void tick_disable_protection(void)
 {
        /* Set things up so user can access tick register for profiling

diff --git a/arch/sparc64/kernel/vmlinux.lds.S b/arch/sparc64/kernel/vmlinux.lds.S
index 136da98fca3b1c9d754963d87f7f61efdd9cff8e..a710d38d1a91f899e5c56707db206c47c20c23ac 100644 (file)
--- a/arch/sparc64/kernel/vmlinux.lds.S
+++ b/arch/sparc64/kernel/vmlinux.lds.S
@@ -16,6 +16,7 @@ SECTIONS
   {
     *(.text)
     SCHED_TEXT
+    LOCK_TEXT
     *(.gnu.warning)
   } =0
   _etext = .;

diff --git a/arch/sparc64/lib/Makefile b/arch/sparc64/lib/Makefile
index 22c5b59fd8b97dc551efada1dba2b481411ec918..2be56f34abade53ff8719951823b246c4da13ffd 100644 (file)
--- a/arch/sparc64/lib/Makefile
+++ b/arch/sparc64/lib/Makefile
@@ -8,11 +8,11 @@ EXTRA_CFLAGS := -Werror
 lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \
         memscan.o strncpy_from_user.o strlen_user.o memcmp.o checksum.o \
         VISbzero.o VISmemset.o VIScsum.o VIScsumcopy.o \
-        VIScsumcopyusr.o VISsave.o atomic.o rwlock.o bitops.o \
+        VIScsumcopyusr.o VISsave.o atomic.o bitops.o \
         U1memcpy.o U1copy_from_user.o U1copy_to_user.o \
         U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \
         copy_in_user.o user_fixup.o memmove.o \
-        mcount.o ipcsum.o rwsem.o xor.o splock.o find_bit.o delay.o
+        mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o
 
 lib-$(CONFIG_DEBUG_SPINLOCK) += debuglocks.o
 lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o

diff --git a/arch/sparc64/lib/debuglocks.c b/arch/sparc64/lib/debuglocks.c
index 9e083e27d2db966bcc8fa4f7227d033e084992e9..9fb4dedc89e13934f84bf093bde4ca2145f01825 100644 (file)
--- a/arch/sparc64/lib/debuglocks.c
+++ b/arch/sparc64/lib/debuglocks.c
@@ -84,7 +84,7 @@ again:
        put_cpu();
 }
 
-int _spin_trylock(spinlock_t *lock)
+int _do_spin_trylock(spinlock_t *lock)
 {
        unsigned long val, caller;
        int cpu = get_cpu();

diff --git a/arch/sparc64/lib/rwlock.S b/arch/sparc64/lib/rwlock.S
deleted file mode 100644 (file)
index 8d8ecec..0000000
--- a/arch/sparc64/lib/rwlock.S
+++ /dev/null
@@ -1,85 +0,0 @@
-/* $Id: rwlock.S,v 1.4 2000/09/09 00:00:34 davem Exp $
- * rwlocks.S: These things are too big to do inline.
- *
- * Copyright (C) 1999 David S. Miller (davem@redhat.com)
- */
-
-       .text
-       .align  64
-
-       /* The non-contention read lock usage is 2 cache lines. */
-
-       .globl  __read_lock, __read_unlock
-__read_lock: /* %o0 = lock_ptr */
-       ldsw            [%o0], %g5
-       brlz,pn         %g5, __read_wait_for_writer
-4:      add            %g5, 1, %g7
-       cas             [%o0], %g5, %g7
-       cmp             %g5, %g7
-       bne,pn          %icc, __read_lock
-        membar         #StoreLoad | #StoreStore
-99:    retl
-        nop
-__read_unlock: /* %o0 = lock_ptr */
-       lduw            [%o0], %g5
-       sub             %g5, 1, %g7
-       cas             [%o0], %g5, %g7
-       cmp             %g5, %g7
-       be,pt           %xcc, 99b
-        membar         #StoreLoad | #StoreStore
-       ba,a,pt         %xcc, __read_unlock
-
-__read_wait_for_writer:
-       ldsw            [%o0], %g5
-       brlz,pt         %g5, __read_wait_for_writer
-        membar         #LoadLoad
-       ba,a,pt         %xcc, 4b
-__write_wait_for_any:
-       lduw            [%o0], %g5
-       brnz,pt         %g5, __write_wait_for_any
-        membar         #LoadLoad
-       ba,a,pt         %xcc, 4f
-
-       .align          64
-       .globl          __write_unlock
-__write_unlock: /* %o0 = lock_ptr */
-       membar          #LoadStore | #StoreStore
-       retl
-        stw            %g0, [%o0]
-
-       .globl          __write_lock
-__write_lock: /* %o0 = lock_ptr */
-       sethi           %hi(0x80000000), %g2
-
-1:     lduw            [%o0], %g5
-       brnz,pn         %g5, __write_wait_for_any
-4:      or             %g5, %g2, %g7
-       cas             [%o0], %g5, %g7
-
-       cmp             %g5, %g7
-       be,pt           %icc, 99b
-        membar         #StoreLoad | #StoreStore
-       ba,a,pt         %xcc, 1b
-
-       .globl          __write_trylock
-__write_trylock: /* %o0 = lock_ptr */
-       sethi           %hi(0x80000000), %g2
-1:     lduw            [%o0], %g5
-       brnz,pn         %g5, __write_trylock_fail
-4:      or             %g5, %g2, %g7
-
-       cas             [%o0], %g5, %g7
-       cmp             %g5, %g7
-       be,pt           %icc, __write_trylock_succeed
-        membar         #StoreLoad | #StoreStore
-
-       ba,pt           %xcc, 1b
-        nop
-__write_trylock_succeed:
-       retl
-        mov            1, %o0
-
-__write_trylock_fail:
-       retl
-        mov            0, %o0
-

diff --git a/arch/sparc64/lib/splock.S b/arch/sparc64/lib/splock.S
deleted file mode 100644 (file)
index acfdae1..0000000
--- a/arch/sparc64/lib/splock.S
+++ /dev/null
@@ -1,39 +0,0 @@
-/* splock.S: Spinlock primitives too large to inline.
- *
- * Copyright (C) 2004 David S. Miller (davem@redhat.com)
- */
-
-       .text
-       .align  64
-
-       .globl          _raw_spin_lock
-       .type           _raw_spin_lock,#function
-_raw_spin_lock:                /* %o0 = lock_ptr */
-1:     ldstub          [%o0], %g7
-       brnz,pn         %g7, 2f
-        membar         #StoreLoad | #StoreStore
-       retl
-        nop
-2:     ldub            [%o0], %g7
-       brnz,pt         %g7, 2b
-        membar         #LoadLoad
-       ba,a,pt         %xcc, 1b
-       .size           _raw_spin_lock, .-_raw_spin_lock
-
-       .globl          _raw_spin_lock_flags
-       .type           _raw_spin_lock_flags,#function
-_raw_spin_lock_flags:  /* %o0 = lock_ptr, %o1 = irq_flags */
-1:     ldstub          [%o0], %g7
-       brnz,pn         %g7, 2f
-        membar         #StoreLoad | #StoreStore
-       retl
-        nop
-
-2:     rdpr            %pil, %g2               ! Save PIL
-       wrpr            %o1, %pil               ! Set previous PIL
-3:     ldub            [%o0], %g7              ! Spin on lock set
-       brnz,pt         %g7, 3b
-        membar         #LoadLoad
-       ba,pt           %xcc, 1b                ! Retry lock acquire
-        wrpr           %g2, %pil               ! Restore PIL
-       .size           _raw_spin_lock_flags, .-_raw_spin_lock_flags

diff --git a/include/asm-sparc64/ptrace.h b/include/asm-sparc64/ptrace.h
index 7b6766770848b07e5baa9ea062dc5e8105205d4f..2d2b5a113d242345fc5909fefb62c12af7596b78 100644 (file)
--- a/include/asm-sparc64/ptrace.h
+++ b/include/asm-sparc64/ptrace.h
@@ -98,7 +98,11 @@ struct sparc_trapf {
        set_thread_flag(TIF_SYSCALL_SUCCESS)
 #define user_mode(regs) (!((regs)->tstate & TSTATE_PRIV))
 #define instruction_pointer(regs) ((regs)->tpc)
+#ifdef CONFIG_SMP
+extern unsigned long profile_pc(struct pt_regs *);
+#else
 #define profile_pc(regs) instruction_pointer(regs)
+#endif
 extern void show_regs(struct pt_regs *);
 #endif
 

diff --git a/include/asm-sparc64/spinlock.h b/include/asm-sparc64/spinlock.h
index a995805ba2ab6f018a1a996b94a66e4e0c375a04..cc39ef61245338b520fb690104ee42862a71877d 100644 (file)
--- a/include/asm-sparc64/spinlock.h
+++ b/include/asm-sparc64/spinlock.h
@@ -41,8 +41,20 @@ typedef unsigned char spinlock_t;
 do {   membar("#LoadLoad");    \
 } while(*((volatile unsigned char *)lock))
 
-/* arch/sparc64/lib/spinlock.S */
-extern void _raw_spin_lock(spinlock_t *lock);
+static __inline__ void _raw_spin_lock(spinlock_t *lock)
+{
+
+       __asm__ __volatile__("1: ldstub [%0], %%g7\n\t"
+                            "brnz,pn   %%g7, 2f\n\t"
+                            "membar    #StoreLoad | #StoreStore\n\t"
+                            "b 3f\n\t"
+                            "2: ldub   [%0], %%g7\n\t"
+                            "brnz,pt   %%g7, 2b\n\t"
+                            "membar    #LoadLoad\n\t"
+                            "ba,a,pt   %%xcc, 1b\n\t"
+                            "3:\n\t"
+                            : : "r" (lock) : "memory");
+}
 
 static __inline__ int _raw_spin_trylock(spinlock_t *lock)
 {
@@ -64,7 +76,22 @@ static __inline__ void _raw_spin_unlock(spinlock_t *lock)
                             : "memory");
 }
 
-extern void _raw_spin_lock_flags(spinlock_t *lock, unsigned long flags);
+static __inline__ void _raw_spin_lock_flags(spinlock_t *lock, unsigned long flags)
+{
+       __asm__ __volatile__ ("1:ldstub [%0], %%g7\n\t"
+                             "brnz,pn  %%g7, 2f\n\t"
+                             "membar   #StoreLoad | #StoreStore\n\t"
+                             "b 4f\n\t"
+                             "2: rdpr  %%pil, %%g2     ! Save PIL\n\t"
+                             "wrpr     %1, %%pil       ! Set previous PIL\n\t"
+                             "3:ldub   [%0], %%g7      ! Spin on lock set\n\t"
+                             "brnz,pt  %%g7, 3b\n\t"
+                             "membar   #LoadLoad\n\t"
+                             "ba,pt    %%xcc, 1b       ! Retry lock acquire\n\t"
+                             "wrpr     %%g2, %%pil     ! Restore PIL\n\t"
+                             "4:\n\t"
+                               : : "r"(lock), "r"(flags) : "memory");
+}
 
 #else /* !(CONFIG_DEBUG_SPINLOCK) */
 
@@ -86,9 +113,9 @@ do { \
 
 extern void _do_spin_lock (spinlock_t *lock, char *str);
 extern void _do_spin_unlock (spinlock_t *lock);
-extern int _spin_trylock (spinlock_t *lock);
+extern int _do_spin_trylock (spinlock_t *lock);
 
-#define _raw_spin_trylock(lp)  _spin_trylock(lp)
+#define _raw_spin_trylock(lp)  _do_spin_trylock(lp)
 #define _raw_spin_lock(lock)   _do_spin_lock(lock, "spin_lock")
 #define _raw_spin_unlock(lock) _do_spin_unlock(lock)
 #define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
@@ -104,11 +131,86 @@ typedef unsigned int rwlock_t;
 #define rwlock_init(lp) do { *(lp) = RW_LOCK_UNLOCKED; } while(0)
 #define rwlock_is_locked(x) (*(x) != RW_LOCK_UNLOCKED)
 
-extern void __read_lock(rwlock_t *);
-extern void __read_unlock(rwlock_t *);
-extern void __write_lock(rwlock_t *);
-extern void __write_unlock(rwlock_t *);
-extern int __write_trylock(rwlock_t *);
+static void __inline__ __read_lock(rwlock_t *lock)
+{
+       __asm__ __volatile__ ("b 1f\n\t"
+                             "99:\n\t"
+                             "ldsw     [%0], %%g5\n\t"
+                             "brlz,pt  %%g5, 99b\n\t"
+                             "membar   #LoadLoad\n\t"
+                             "ba,a,pt  %%xcc, 4f\n\t"
+                             "1: ldsw  [%0], %%g5\n\t"
+                             "brlz,pn  %%g5, 99b\n\t"
+                             "4:add    %%g5, 1, %%g7\n\t"
+                             "cas      [%0], %%g5, %%g7\n\t"
+                             "cmp      %%g5, %%g7\n\t"
+                             "bne,pn   %%icc, 1b\n\t"
+                             "membar   #StoreLoad | #StoreStore\n\t"
+                               : : "r"(lock) : "memory");
+}
+
+static void __inline__ __read_unlock(rwlock_t *lock)
+{
+       __asm__ __volatile__ ("1: lduw  [%0], %%g5\n\t"
+                             "sub      %%g5, 1, %%g7\n\t"
+                             "cas      [%0], %%g5, %%g7\n\t"
+                             "cmp      %%g5, %%g7\n\t"
+                             "be,pt    %%xcc, 2f\n\t"
+                             "membar   #StoreLoad | #StoreStore\n\t"
+                             "ba,a,pt  %%xcc, 1b\n\t"
+                             "2:\n\t"
+                               : : "r" (lock) : "memory");
+}
+
+static void __inline__ __write_lock(rwlock_t *lock)
+{
+       __asm__ __volatile__ ("sethi    %%hi(0x80000000), %%g2\n\t"
+                             "b 1f\n\t"
+                             "99:\n\t"
+                             "lduw     [%0], %%g5\n\t"
+                             "brnz,pt  %%g5, 99b\n\t"
+                             "membar   #LoadLoad\n\t"
+                             "ba,a,pt  %%xcc, 4f\n\t"
+                             "1: lduw  [%0], %%g5\n\t"
+                             "brnz,pn  %%g5, 99b\n\t"
+                             "4: or    %%g5, %%g2, %%g7\n\t"
+                             "cas      [%0], %%g5, %%g7\n\t"
+                             "cmp      %%g5, %%g7\n\t"
+                             "be,pt    %%icc, 2f\n\t"
+                             "membar   #StoreLoad | #StoreStore\n\t"
+                             "ba,a,pt  %%xcc, 1b\n\t"
+                             "2:\n\t"
+                               : : "r"(lock) : "memory");
+}
+
+static void __inline__ __write_unlock(rwlock_t *lock)
+{
+       __asm__ __volatile__ ("membar   #LoadStore | #StoreStore\n\t"
+                             "retl\n\t"
+                             "stw      %%g0, [%0]\n\t"
+                               : : "r"(lock) : "memory");
+}
+
+static int __inline__ __write_trylock(rwlock_t *lock)
+{
+       __asm__ __volatile__ ("sethi    %%hi(0x80000000), %%g2\n\t"
+                             "1: lduw  [%0], %%g5\n\t"
+                             "brnz,pn  %%g5, 100f\n\t"
+                             "4: or    %%g5, %%g2, %%g7\n\t"
+                             "cas      [%0], %%g5, %%g7\n\t"
+                             "cmp      %%g5, %%g7\n\t"
+                             "be,pt    %%icc, 99f\n\t"
+                             "membar   #StoreLoad | #StoreStore\n\t"
+                             "ba,pt    %%xcc, 1b\n\t"
+                             "99:\n\t"
+                             "retl\n\t"
+                             "mov      1, %0\n\t"
+                             "100:\n\t"
+                             "retl\n\t"
+                             "mov      0, %0\n\t"
+                               : : "r"(lock) : "memory");
+       return rwlock_is_locked(lock);
+}
 
 #define _raw_read_lock(p)      __read_lock(p)
 #define _raw_read_unlock(p)    __read_unlock(p)