x86/alternatives: Add instruction padding

commit 4332195c5615bf748624094ce4ff6797e475024d upstream.

Up until now we have always paid attention to make sure the length of
the new instruction replacing the old one is at least less or equal to
the length of the old instruction. If the new instruction is longer, at
the time it replaces the old instruction it will overwrite the beginning
of the next instruction in the kernel image and cause your pants to
catch fire.

So instead of having to pay attention, teach the alternatives framework
to pad shorter old instructions with NOPs at buildtime - but only in the
case when

  len(old instruction(s)) < len(new instruction(s))

and add nothing in the >= case. (In that case we do add_nops() when
patching).

This way the alternatives user shouldn't have to care about instruction
sizes and simply use the macros.

Add asm ALTERNATIVE* flavor macros too, while at it.

Also, we need to save the pad length in a separate struct alt_instr
member for NOP optimization and the way to do that reliably is to carry
the pad length instead of trying to detect whether we're looking at
single-byte NOPs or at pathological instruction offsets like e9 90 90 90
90, for example, which is a valid instruction.

Thanks to Michael Matz for the great help with toolchain questions.

Signed-off-by: Borislav Petkov <bp@suse.de>
Cc: Hugh Dickins <hughd@google.com>
[bwh: Backported to 3.16: adjust context]
Signed-off-by: Ben Hutchings <ben@decadent.org.uk>

diff --git a/arch/x86/include/asm/alternative-asm.h b/arch/x86/include/asm/alternative-asm.h
index 372231c22a47a46b1417e5c6739d88eb927f89fd..524bddce0b7608813c8a85c590c3fb70f5bf6ce1 100644 (file)
--- a/arch/x86/include/asm/alternative-asm.h
+++ b/arch/x86/include/asm/alternative-asm.h
@@ -18,12 +18,53 @@
        .endm
 #endif
 
-.macro altinstruction_entry orig alt feature orig_len alt_len
+.macro altinstruction_entry orig alt feature orig_len alt_len pad_len
        .long \orig - .
        .long \alt - .
        .word \feature
        .byte \orig_len
        .byte \alt_len
+       .byte \pad_len
+.endm
+
+.macro ALTERNATIVE oldinstr, newinstr, feature
+140:
+       \oldinstr
+141:
+       .skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90
+142:
+
+       .pushsection .altinstructions,"a"
+       altinstruction_entry 140b,143f,\feature,142b-140b,144f-143f,142b-141b
+       .popsection
+
+       .pushsection .altinstr_replacement,"ax"
+143:
+       \newinstr
+144:
+       .popsection
+.endm
+
+.macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2
+140:
+       \oldinstr
+141:
+       .skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90
+       .skip -(((145f-144f)-(144f-143f)-(141b-140b)) > 0) * ((145f-144f)-(144f-143f)-(141b-140b)),0x90
+142:
+
+       .pushsection .altinstructions,"a"
+       altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f,142b-141b
+       altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f,142b-141b
+       .popsection
+
+       .pushsection .altinstr_replacement,"ax"
+143:
+       \newinstr1
+144:
+       \newinstr2
+145:
+       .popsection
 .endm
 
 #endif  /*  __ASSEMBLY__  */

diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 0a3f9c9f98d5ccf446980cfcf71b7055bb79ade8..34310c03708adb9ff8e10c071f945880b981e202 100644 (file)
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -48,8 +48,9 @@ struct alt_instr {
        s32 repl_offset;        /* offset to replacement instruction */
        u16 cpuid;              /* cpuid bit set for replacement */
        u8  instrlen;           /* length of original instruction */
-       u8  replacementlen;     /* length of new instruction, <= instrlen */
-};
+       u8  replacementlen;     /* length of new instruction */
+       u8  padlen;             /* length of build-time padding */
+} __packed;
 
 extern void alternative_instructions(void);
 extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
@@ -76,50 +77,61 @@ static inline int alternatives_text_reserved(void *start, void *end)
 }
 #endif /* CONFIG_SMP */
 
-#define OLDINSTR(oldinstr)     "661:\n\t" oldinstr "\n662:\n"
+#define b_replacement(num)     "664"#num
+#define e_replacement(num)     "665"#num
 
-#define b_replacement(number)  "663"#number
-#define e_replacement(number)  "664"#number
+#define alt_end_marker         "663"
+#define alt_slen               "662b-661b"
+#define alt_pad_len            alt_end_marker"b-662b"
+#define alt_total_slen         alt_end_marker"b-661b"
+#define alt_rlen(num)          e_replacement(num)"f-"b_replacement(num)"f"
 
-#define alt_slen "662b-661b"
-#define alt_rlen(number) e_replacement(number)"f-"b_replacement(number)"f"
+#define __OLDINSTR(oldinstr, num)                                      \
+       "661:\n\t" oldinstr "\n662:\n"                                  \
+       ".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * "          \
+               "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"
 
-#define ALTINSTR_ENTRY(feature, number)                                              \
+#define OLDINSTR(oldinstr, num)                                                \
+       __OLDINSTR(oldinstr, num)                                       \
+       alt_end_marker ":\n"
+
+/*
+ * Pad the second replacement alternative with additional NOPs if it is
+ * additionally longer than the first replacement alternative.
+ */
+#define OLDINSTR_2(oldinstr, num1, num2)                                       \
+       __OLDINSTR(oldinstr, num1)                                              \
+       ".skip -(((" alt_rlen(num2) ")-(" alt_rlen(num1) ")-(662b-661b)) > 0) * " \
+               "((" alt_rlen(num2) ")-(" alt_rlen(num1) ")-(662b-661b)),0x90\n"  \
+       alt_end_marker ":\n"
+
+#define ALTINSTR_ENTRY(feature, num)                                         \
        " .long 661b - .\n"                             /* label           */ \
-       " .long " b_replacement(number)"f - .\n"        /* new instruction */ \
+       " .long " b_replacement(num)"f - .\n"           /* new instruction */ \
        " .word " __stringify(feature) "\n"             /* feature bit     */ \
-       " .byte " alt_slen "\n"                         /* source len      */ \
-       " .byte " alt_rlen(number) "\n"                 /* replacement len */
+       " .byte " alt_total_slen "\n"                   /* source len      */ \
+       " .byte " alt_rlen(num) "\n"                    /* replacement len */ \
+       " .byte " alt_pad_len "\n"                      /* pad len */
 
-#define DISCARD_ENTRY(number)                          /* rlen <= slen */    \
-       " .byte 0xff + (" alt_rlen(number) ") - (" alt_slen ")\n"
-
-#define ALTINSTR_REPLACEMENT(newinstr, feature, number)        /* replacement */     \
-       b_replacement(number)":\n\t" newinstr "\n" e_replacement(number) ":\n\t"
+#define ALTINSTR_REPLACEMENT(newinstr, feature, num)   /* replacement */     \
+       b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n\t"
 
 /* alternative assembly primitive: */
 #define ALTERNATIVE(oldinstr, newinstr, feature)                       \
-       OLDINSTR(oldinstr)                                              \
+       OLDINSTR(oldinstr, 1)                                           \
        ".pushsection .altinstructions,\"a\"\n"                         \
        ALTINSTR_ENTRY(feature, 1)                                      \
        ".popsection\n"                                                 \
-       ".pushsection .discard,\"aw\",@progbits\n"                      \
-       DISCARD_ENTRY(1)                                                \
-       ".popsection\n"                                                 \
        ".pushsection .altinstr_replacement, \"ax\"\n"                  \
        ALTINSTR_REPLACEMENT(newinstr, feature, 1)                      \
        ".popsection"
 
 #define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\
-       OLDINSTR(oldinstr)                                              \
+       OLDINSTR_2(oldinstr, 1, 2)                                      \
        ".pushsection .altinstructions,\"a\"\n"                         \
        ALTINSTR_ENTRY(feature1, 1)                                     \
        ALTINSTR_ENTRY(feature2, 2)                                     \
        ".popsection\n"                                                 \
-       ".pushsection .discard,\"aw\",@progbits\n"                      \
-       DISCARD_ENTRY(1)                                                \
-       DISCARD_ENTRY(2)                                                \
-       ".popsection\n"                                                 \
        ".pushsection .altinstr_replacement, \"ax\"\n"                  \
        ALTINSTR_REPLACEMENT(newinstr1, feature1, 1)                    \
        ALTINSTR_REPLACEMENT(newinstr2, feature2, 2)                    \
@@ -146,6 +158,9 @@ static inline int alternatives_text_reserved(void *start, void *end)
 #define alternative(oldinstr, newinstr, feature)                       \
        asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")
 
+#define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \
+       asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory")
+
 /*
  * Alternative inline assembly with input.
  *

diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index e2b71980cecd24cee56ea525a446badae930bbcf..a932a1fbe66ce25f2f34f3d04f218fc891abd427 100644 (file)
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -390,6 +390,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit)
                         " .word %P0\n"         /* 1: do replace */
                         " .byte 2b - 1b\n"     /* source len */
                         " .byte 0\n"           /* replacement len */
+                        " .byte 0\n"           /* pad len */
                         ".previous\n"
                         /* skipping size check since replacement size = 0 */
                         : : "i" (X86_FEATURE_ALWAYS) : : t_warn);
@@ -404,6 +405,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit)
                         " .word %P0\n"         /* feature bit */
                         " .byte 2b - 1b\n"     /* source len */
                         " .byte 0\n"           /* replacement len */
+                        " .byte 0\n"           /* pad len */
                         ".previous\n"
                         /* skipping size check since replacement size = 0 */
                         : : "i" (bit) : : t_no);
@@ -429,6 +431,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit)
                             " .word %P1\n"             /* feature bit */
                             " .byte 2b - 1b\n"         /* source len */
                             " .byte 4f - 3f\n"         /* replacement len */
+                            " .byte 0\n"               /* pad len */
                             ".previous\n"
                             ".section .discard,\"aw\",@progbits\n"
                             " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
@@ -463,23 +466,28 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
  */
                asm_volatile_goto("1: .byte 0xe9\n .long %l[t_dynamic] - 2f\n"
                         "2:\n"
+                        ".skip -(((5f-4f) - (2b-1b)) > 0) * "
+                                "((5f-4f) - (2b-1b)),0x90\n"
+                        "3:\n"
                         ".section .altinstructions,\"a\"\n"
                         " .long 1b - .\n"              /* src offset */
-                        " .long 3f - .\n"              /* repl offset */
+                        " .long 4f - .\n"              /* repl offset */
                         " .word %P1\n"                 /* always replace */
-                        " .byte 2b - 1b\n"             /* src len */
-                        " .byte 4f - 3f\n"             /* repl len */
+                        " .byte 3b - 1b\n"             /* src len */
+                        " .byte 5f - 4f\n"             /* repl len */
+                        " .byte 3b - 2b\n"             /* pad len */
                         ".previous\n"
                         ".section .altinstr_replacement,\"ax\"\n"
-                        "3: .byte 0xe9\n .long %l[t_no] - 2b\n"
-                        "4:\n"
+                        "4: .byte 0xe9\n .long %l[t_no] - 2b\n"
+                        "5:\n"
                         ".previous\n"
                         ".section .altinstructions,\"a\"\n"
                         " .long 1b - .\n"              /* src offset */
                         " .long 0\n"                   /* no replacement */
                         " .word %P0\n"                 /* feature bit */
-                        " .byte 2b - 1b\n"             /* src len */
+                        " .byte 3b - 1b\n"             /* src len */
                         " .byte 0\n"                   /* repl len */
+                        " .byte 0\n"                   /* pad len */
                         ".previous\n"
                         : : "i" (bit), "i" (X86_FEATURE_ALWAYS)
                         : : t_dynamic, t_no);
@@ -499,6 +507,7 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
                             " .word %P2\n"             /* always replace */
                             " .byte 2b - 1b\n"         /* source len */
                             " .byte 4f - 3f\n"         /* replacement len */
+                            " .byte 0\n"               /* pad len */
                             ".previous\n"
                             ".section .discard,\"aw\",@progbits\n"
                             " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
@@ -513,6 +522,7 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
                             " .word %P1\n"             /* feature bit */
                             " .byte 4b - 3b\n"         /* src len */
                             " .byte 6f - 5f\n"         /* repl len */
+                            " .byte 0\n"               /* pad len */
                             ".previous\n"
                             ".section .discard,\"aw\",@progbits\n"
                             " .byte 0xff + (6f-5f) - (4b-3b)\n" /* size check */

diff --git a/arch/x86/include/asm/smap.h b/arch/x86/include/asm/smap.h
index 8d3120f4e27053b3fae6d6334fe7a3a6dee33e82..c56cb4f37be93eccc514acffa3ae443bca94bf05 100644 (file)
--- a/arch/x86/include/asm/smap.h
+++ b/arch/x86/include/asm/smap.h
@@ -33,7 +33,7 @@
        662: __ASM_CLAC ;                                               \
        .popsection ;                                                   \
        .pushsection .altinstructions, "a" ;                            \
-       altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3 ;       \
+       altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3, 0 ;    \
        .popsection
 
 #define ASM_STAC                                                       \
@@ -42,7 +42,7 @@
        662: __ASM_STAC ;                                               \
        .popsection ;                                                   \
        .pushsection .altinstructions, "a" ;                            \
-       altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3 ;       \
+       altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3, 0 ;    \
        .popsection
 
 #else /* CONFIG_X86_SMAP */

diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 1e86e85bcf5874bd02969925c1aed8f5b226b42b..c99b0f13a90ee366a0c7cfdb52ad411dfcfa789c 100644 (file)
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -270,7 +270,6 @@ void __init_or_module apply_alternatives(struct alt_instr *start,
        for (a = start; a < end; a++) {
                instr = (u8 *)&a->instr_offset + a->instr_offset;
                replacement = (u8 *)&a->repl_offset + a->repl_offset;
-               BUG_ON(a->replacementlen > a->instrlen);
                BUG_ON(a->instrlen > sizeof(insnbuf));
                BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
                if (!boot_cpu_has(a->cpuid))
@@ -290,8 +289,9 @@ void __init_or_module apply_alternatives(struct alt_instr *start,
                        DPRINTK("Fix CALL offset: 0x%x", *(s32 *)(insnbuf + 1));
                }
 
-               add_nops(insnbuf + a->replacementlen,
-                        a->instrlen - a->replacementlen);
+               if (a->instrlen > a->replacementlen)
+                       add_nops(insnbuf + a->replacementlen,
+                                a->instrlen - a->replacementlen);
 
                text_poke_early(instr, insnbuf, a->instrlen);
        }

diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 2c123171944a76899eb0b725a432a597286a6f59..bfcc300d8b3c27d81b138761cea5aa9d4a11e87d 100644 (file)
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -821,7 +821,7 @@ ENTRY(simd_coprocessor_error)
 661:   pushl_cfi $do_general_protection
 662:
 .section .altinstructions,"a"
-       altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f
+       altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f, 0
 .previous
 .section .altinstr_replacement,"ax"
 663:   pushl $do_simd_coprocessor_error

diff --git a/arch/x86/lib/clear_page_64.S b/arch/x86/lib/clear_page_64.S
index f2145cfa12a66830e834718340c4cc88e64a731a..38e57faefd71eabb5bd78fcb86bb8280dfa78b26 100644 (file)
--- a/arch/x86/lib/clear_page_64.S
+++ b/arch/x86/lib/clear_page_64.S
@@ -67,7 +67,7 @@ ENDPROC(clear_page)
        .previous
        .section .altinstructions,"a"
        altinstruction_entry clear_page,1b,X86_FEATURE_REP_GOOD,\
-                            .Lclear_page_end-clear_page, 2b-1b
+                            .Lclear_page_end-clear_page, 2b-1b, 0
        altinstruction_entry clear_page,2b,X86_FEATURE_ERMS,   \
-                            .Lclear_page_end-clear_page,3b-2b
+                            .Lclear_page_end-clear_page,3b-2b, 0
        .previous

diff --git a/arch/x86/lib/copy_page_64.S b/arch/x86/lib/copy_page_64.S
index 176cca67212b7072687f42450ef56018c03ebaaa..f1ffdbb077552382324256987eb515e1f814c83a 100644 (file)
--- a/arch/x86/lib/copy_page_64.S
+++ b/arch/x86/lib/copy_page_64.S
@@ -106,5 +106,5 @@ ENDPROC(copy_page)
        .previous
        .section .altinstructions,"a"
        altinstruction_entry copy_page, 1b, X86_FEATURE_REP_GOOD,       \
-               .Lcopy_page_end-copy_page, 2b-1b
+               .Lcopy_page_end-copy_page, 2b-1b, 0
        .previous

diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S
index dee945d555941a078f40049b24b8b4731aba6f2c..5027613d96608252e8862237f9a57b816492e187 100644 (file)
--- a/arch/x86/lib/copy_user_64.S
+++ b/arch/x86/lib/copy_user_64.S
@@ -39,8 +39,8 @@
        .previous
 
        .section .altinstructions,"a"
-       altinstruction_entry 0b,2b,\feature1,5,5
-       altinstruction_entry 0b,3b,\feature2,5,5
+       altinstruction_entry 0b,2b,\feature1,5,5,0
+       altinstruction_entry 0b,3b,\feature2,5,5,0
        .previous
        .endm
 

diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S
index 56313a3261888d0e7eb866788c24f806acf5d47a..f7766e8a497dc8c0d29875a14667ec87b7567569 100644 (file)
--- a/arch/x86/lib/memcpy_64.S
+++ b/arch/x86/lib/memcpy_64.S
@@ -199,8 +199,8 @@ ENDPROC(__memcpy)
         * only outcome...
         */
        .section .altinstructions, "a"
-       altinstruction_entry memcpy,.Lmemcpy_c,X86_FEATURE_REP_GOOD,\
-                            .Lmemcpy_e-.Lmemcpy_c,.Lmemcpy_e-.Lmemcpy_c
-       altinstruction_entry memcpy,.Lmemcpy_c_e,X86_FEATURE_ERMS, \
-                            .Lmemcpy_e_e-.Lmemcpy_c_e,.Lmemcpy_e_e-.Lmemcpy_c_e
+       altinstruction_entry __memcpy,.Lmemcpy_c,X86_FEATURE_REP_GOOD,\
+                            .Lmemcpy_e-.Lmemcpy_c,.Lmemcpy_e-.Lmemcpy_c,0
+       altinstruction_entry __memcpy,.Lmemcpy_c_e,X86_FEATURE_ERMS, \
+                            .Lmemcpy_e_e-.Lmemcpy_c_e,.Lmemcpy_e_e-.Lmemcpy_c_e,0
        .previous

diff --git a/arch/x86/lib/memmove_64.S b/arch/x86/lib/memmove_64.S
index 65268a6104f45e09d5e45f75cb63edf6527164a0..d290a599996c6bfa2b1849134e3347bdcd550ed8 100644 (file)
--- a/arch/x86/lib/memmove_64.S
+++ b/arch/x86/lib/memmove_64.S
@@ -218,6 +218,6 @@ ENTRY(memmove)
        altinstruction_entry .Lmemmove_begin_forward,           \
                .Lmemmove_begin_forward_efs,X86_FEATURE_ERMS,   \
                .Lmemmove_end_forward-.Lmemmove_begin_forward,  \
-               .Lmemmove_end_forward_efs-.Lmemmove_begin_forward_efs
+               .Lmemmove_end_forward_efs-.Lmemmove_begin_forward_efs,0
        .previous
 ENDPROC(memmove)

diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S
index 2dcb3808cbdab6c91b9fbbf0d58790466780bb55..aba5027380431afab60b241238f5e296cc6d9ba3 100644 (file)
--- a/arch/x86/lib/memset_64.S
+++ b/arch/x86/lib/memset_64.S
@@ -147,8 +147,8 @@ ENDPROC(__memset)
          * feature to implement the right patch order.
         */
        .section .altinstructions,"a"
-       altinstruction_entry memset,.Lmemset_c,X86_FEATURE_REP_GOOD,\
-                            .Lfinal-memset,.Lmemset_e-.Lmemset_c
-       altinstruction_entry memset,.Lmemset_c_e,X86_FEATURE_ERMS, \
-                            .Lfinal-memset,.Lmemset_e_e-.Lmemset_c_e
+       altinstruction_entry __memset,.Lmemset_c,X86_FEATURE_REP_GOOD,\
+                            .Lfinal-__memset,.Lmemset_e-.Lmemset_c,0
+       altinstruction_entry __memset,.Lmemset_c_e,X86_FEATURE_ERMS, \
+                            .Lfinal-__memset,.Lmemset_e_e-.Lmemset_c_e,0
        .previous