From e3db48525a4d5d49c5297e30571d4bd94b73c828 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Sun, 9 Mar 2003 22:41:27 -0800 Subject: [PATCH] Use a fixed per-cpu SYSENTER_MSR_ESP value by having the sysenter entry routine load the real ESP0 off that per-cpu stack. Make this even faster by putting the sysenter stack in the per-CPU TSS, so that we can use the tss->esp0 value directly (which we have to update on task switches anyway). CAREFUL! This needs very subtle code for debug and NMI exceptions, to make sure we don't run with the sysenter stack in any real kernel code! --- arch/i386/kernel/entry.S | 32 ++++++++++++++++++++++++++++++++ arch/i386/kernel/sysenter.c | 3 ++- include/asm-i386/processor.h | 12 ++++++------ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/arch/i386/kernel/entry.S b/arch/i386/kernel/entry.S index cd0dd133a572d..a1224def2c665 100644 --- a/arch/i386/kernel/entry.S +++ b/arch/i386/kernel/entry.S @@ -72,6 +72,12 @@ DF_MASK = 0x00000400 NT_MASK = 0x00004000 VM_MASK = 0x00020000 +/* + * ESP0 is at offset 4. 0x100 is the size of the TSS, and + * also thus the top-of-stack pointer offset of SYSENTER_ESP + */ +TSS_ESP0_OFFSET = (4 - 0x100) + #ifdef CONFIG_PREEMPT #define preempt_stop cli #else @@ -229,6 +235,8 @@ need_resched: # sysenter call handler stub ENTRY(sysenter_entry) + movl TSS_ESP0_OFFSET(%esp),%esp +sysenter_past_esp: sti pushl $(__USER_DS) pushl %ebp @@ -458,12 +466,36 @@ device_not_available_emulate: addl $4, %esp jmp ret_from_exception +/* + * Debug traps and NMI can happen at the one SYSENTER instruction + * that sets up the real kernel stack. Check here, since we can't + * allow the wrong stack to be used. + * + * "TSS_ESP0_OFFSET+12" is because the NMI/debug handler will have + * already pushed 3 words if it hits on the sysenter instruction: + * eflags, cs and eip. + * + * We just load the right stack, and push the three (known) values + * by hand onto the new stack - while updating the return eip past + * the instruction that would have done it for sysenter. + */ +#define CHECK_SYSENTER_EIP \ + cmpl $sysenter_entry,(%esp); \ + jne 1f; \ + movl TSS_ESP0_OFFSET+12(%esp),%esp; \ + pushfl; \ + pushl $__KERNEL_CS; \ + pushl $sysenter_past_esp; \ +1: + ENTRY(debug) + CHECK_SYSENTER_EIP pushl $0 pushl $do_debug jmp error_code ENTRY(nmi) + CHECK_SYSENTER_EIP pushl %eax SAVE_ALL movl %esp, %edx diff --git a/arch/i386/kernel/sysenter.c b/arch/i386/kernel/sysenter.c index 8d86b0b7071d4..db58ed5e8afdc 100644 --- a/arch/i386/kernel/sysenter.c +++ b/arch/i386/kernel/sysenter.c @@ -41,8 +41,9 @@ void enable_sep_cpu(void *info) struct tss_struct *tss = init_tss + cpu; tss->ss1 = __KERNEL_CS; + tss->esp1 = sizeof(struct tss_struct) + (unsigned long) tss; wrmsr(MSR_IA32_SYSENTER_CS, __KERNEL_CS, 0); - wrmsr(MSR_IA32_SYSENTER_ESP, tss->esp0, 0); + wrmsr(MSR_IA32_SYSENTER_ESP, tss->esp1, 0); wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long) sysenter_entry, 0); printk("Enabling SEP on CPU %d\n", cpu); diff --git a/include/asm-i386/processor.h b/include/asm-i386/processor.h index 02dd0857088ab..099223f882f59 100644 --- a/include/asm-i386/processor.h +++ b/include/asm-i386/processor.h @@ -404,6 +404,8 @@ struct thread_struct { #define INIT_TSS { \ .esp0 = sizeof(init_stack) + (long)&init_stack, \ .ss0 = __KERNEL_DS, \ + .esp1 = sizeof(init_tss[0]) + (long)&init_tss[0], \ + .ss1 = __KERNEL_CS, \ .ldt = GDT_ENTRY_LDT, \ .bitmap = INVALID_IO_BITMAP_OFFSET, \ .io_bitmap = { [ 0 ... IO_BITMAP_SIZE ] = ~0 }, \ @@ -412,12 +414,10 @@ struct thread_struct { static inline void load_esp0(struct tss_struct *tss, unsigned long esp0) { tss->esp0 = esp0; - if (cpu_has_sep) { - if (tss->ss1 != __KERNEL_CS) { - tss->ss1 = __KERNEL_CS; - wrmsr(MSR_IA32_SYSENTER_CS, __KERNEL_CS, 0); - } - wrmsr(MSR_IA32_SYSENTER_ESP, esp0, 0); + /* This can only happen when SEP is enabled, no need to test "SEP"arately */ + if (tss->ss1 != __KERNEL_CS) { + tss->ss1 = __KERNEL_CS; + wrmsr(MSR_IA32_SYSENTER_CS, __KERNEL_CS, 0); } } -- 2.53.0