[PATCH] Fix compat shmget overflow

This fixes an incorrect sign extension in the compat layer that breaks
32bit shmget that are >2GB.  sys_shmget has a signed size_t size argument,
and the int size argument coming from 32bit user space would get sign
extended to 64bit, which is wrong.

I fixed it on all compat architectures, except PPC64 which was already ok.

It was originally debugged and fixed by Karl Rister @ IBM for SLES9 on x86-64.

Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/arch/ia64/ia32/sys_ia32.c b/arch/ia64/ia32/sys_ia32.c
index a251e353c686e5a30358afa9b1b5e85034b03d48..f2eccc24c3e727bf2cbb915b5e9dc7900ac2e8f5 100644 (file)
--- a/arch/ia64/ia32/sys_ia32.c
+++ b/arch/ia64/ia32/sys_ia32.c
@@ -1415,7 +1415,7 @@ sys32_ipc(u32 call, int first, int second, int third, u32 ptr, u32 fifth)
              case SHMDT:
                return sys_shmdt(compat_ptr(ptr));
              case SHMGET:
-               return sys_shmget(first, second, third);
+               return sys_shmget(first, (unsigned)second, third);
              case SHMCTL:
                return compat_sys_shmctl(first, second, compat_ptr(ptr));
 

diff --git a/arch/mips/kernel/linux32.c b/arch/mips/kernel/linux32.c
index b76b71fb44046c9cf6b5c564837d050cc60a8347..9476a1cb3655d82b68a6644dc5d1de9c2b9c6762 100644 (file)
--- a/arch/mips/kernel/linux32.c
+++ b/arch/mips/kernel/linux32.c
@@ -1115,7 +1115,7 @@ sys32_ipc (u32 call, int first, int second, int third, u32 ptr, u32 fifth)
                err = sys_shmdt ((char *)A(ptr));
                break;
        case SHMGET:
-               err = sys_shmget (first, second, third);
+               err = sys_shmget (first, (unsigned)second, third);
                break;
        case SHMCTL:
                err = do_sys32_shmctl (first, second, (void *)AA(ptr));

diff --git a/arch/s390/kernel/compat_linux.c b/arch/s390/kernel/compat_linux.c
index 8b214856abd9f376d968969eb1e71d081c36eedd..1bf2951d9c115ed75f403b6ef58c682d9e304f13 100644 (file)
--- a/arch/s390/kernel/compat_linux.c
+++ b/arch/s390/kernel/compat_linux.c
@@ -331,7 +331,7 @@ asmlinkage long sys32_ipc(u32 call, int first, int second, int third, u32 ptr)
        case SHMDT:
                return sys_shmdt(compat_ptr(ptr));
        case SHMGET:
-               return sys_shmget(first, second, third);
+               return sys_shmget(first, (unsigned)second, third);
        case SHMCTL:
                return compat_sys_shmctl(first, second, compat_ptr(ptr));
        }

diff --git a/arch/sparc64/kernel/sys_sparc32.c b/arch/sparc64/kernel/sys_sparc32.c
index 47716ef702ccfcf1bbb77c529b19001f905c3824..49bb881db49dfb7928e2a610c852eeee2bae9b5e 100644 (file)
--- a/arch/sparc64/kernel/sys_sparc32.c
+++ b/arch/sparc64/kernel/sys_sparc32.c
@@ -835,7 +835,7 @@ asmlinkage long compat_sys_ipc(u32 call, int first, int second, int third, compa
                        err = sys_shmdt(ptr);
                        goto out;
                case SHMGET:
-                       err = sys_shmget(first, second, third);
+                       err = sys_shmget(first, (unsigned)second, third);
                        goto out;
                case SHMCTL:
                        err = do_sys32_shmctl(first, second, ptr);

diff --git a/arch/x86_64/ia32/ipc32.c b/arch/x86_64/ia32/ipc32.c
index 6ba85551143a73be31b5954051ac2ad3a6684c39..369151dc3213e3b152bd6d135dc391b5c40ec623 100644 (file)
--- a/arch/x86_64/ia32/ipc32.c
+++ b/arch/x86_64/ia32/ipc32.c
@@ -49,7 +49,7 @@ sys32_ipc(u32 call, int first, int second, int third,
              case SHMDT:
                return sys_shmdt(compat_ptr(ptr));
              case SHMGET:
-               return sys_shmget(first, second, third);
+               return sys_shmget(first, (unsigned)second, third);
              case SHMCTL:
                return compat_sys_shmctl(first, second, compat_ptr(ptr));
        }