From 0855040e79e53064f73b5c34fb7b1ccd196f9b52 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Thu, 16 Sep 2004 22:03:06 -0700 Subject: [PATCH] [PATCH] x86-64: copy ioperm bitmaps more efficiently at context switch Copy ioperm bitmaps more efficiently at context switch time. Only copy upto the highest used port. Originally from Ingo Molnar Signed-off-by: Andi Kleen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/x86_64/kernel/ioport.c | 34 +++++++++++++++++++------- arch/x86_64/kernel/process.c | 44 +++++++++++++++++++--------------- arch/x86_64/kernel/setup64.c | 8 ++++--- include/asm-x86_64/processor.h | 1 + 4 files changed, 57 insertions(+), 30 deletions(-) diff --git a/arch/x86_64/kernel/ioport.c b/arch/x86_64/kernel/ioport.c index 306f62ff71742..cde0e868efe32 100644 --- a/arch/x86_64/kernel/ioport.c +++ b/arch/x86_64/kernel/ioport.c @@ -28,12 +28,12 @@ static void set_bitmap(unsigned long *bitmap, unsigned int base, unsigned int ex clear_bit(i, bitmap); } - /* * this changes the io permissions bitmap in the current task. */ asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int turn_on) { + unsigned int i, max_long, bytes, bytes_updated; struct thread_struct * t = ¤t->thread; struct tss_struct * tss; unsigned long *bitmap; @@ -59,16 +59,34 @@ asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int turn_on) /* * do it in the per-thread copy and in the TSS ... + * + * Disable preemption via get_cpu() - we must not switch away + * because the ->io_bitmap_max value must match the bitmap + * contents: */ - set_bitmap(t->io_bitmap_ptr, from, num, !turn_on); tss = &per_cpu(init_tss, get_cpu()); - if (tss->io_bitmap_base == IO_BITMAP_OFFSET) { /* already active? */ - set_bitmap(tss->io_bitmap, from, num, !turn_on); - } else { - memcpy(tss->io_bitmap, t->io_bitmap_ptr, IO_BITMAP_BYTES); - tss->io_bitmap_base = IO_BITMAP_OFFSET; /* Activate it in the TSS */ - } + + set_bitmap(t->io_bitmap_ptr, from, num, !turn_on); + + /* + * Search for a (possibly new) maximum. This is simple and stupid, + * to keep it obviously correct: + */ + max_long = 0; + for (i = 0; i < IO_BITMAP_LONGS; i++) + if (t->io_bitmap_ptr[i] != ~0UL) + max_long = i; + + bytes = (max_long + 1) * sizeof(long); + bytes_updated = max(bytes, t->io_bitmap_max); + + t->io_bitmap_max = bytes; + + /* Update the TSS: */ + memcpy(tss->io_bitmap, t->io_bitmap_ptr, bytes_updated); + put_cpu(); + return 0; } diff --git a/arch/x86_64/kernel/process.c b/arch/x86_64/kernel/process.c index aed85c3fe7415..cd22f3c11a085 100644 --- a/arch/x86_64/kernel/process.c +++ b/arch/x86_64/kernel/process.c @@ -250,11 +250,17 @@ void show_regs(struct pt_regs *regs) void exit_thread(void) { struct task_struct *me = current; + struct thread_struct *t = &me->thread; if (me->thread.io_bitmap_ptr) { - struct tss_struct *tss = &per_cpu(init_tss,get_cpu()); - kfree(me->thread.io_bitmap_ptr); - me->thread.io_bitmap_ptr = NULL; - tss->io_bitmap_base = INVALID_IO_BITMAP_OFFSET; + struct tss_struct *tss = &per_cpu(init_tss, get_cpu()); + + kfree(t->io_bitmap_ptr); + t->io_bitmap_ptr = NULL; + /* + * Careful, clear this in the TSS too: + */ + memset(tss->io_bitmap, 0xff, t->io_bitmap_max); + t->io_bitmap_max = 0; put_cpu(); } } @@ -362,8 +368,10 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp, if (unlikely(me->thread.io_bitmap_ptr != NULL)) { p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL); - if (!p->thread.io_bitmap_ptr) + if (!p->thread.io_bitmap_ptr) { + p->thread.io_bitmap_max = 0; return -ENOMEM; + } memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr, IO_BITMAP_BYTES); } @@ -382,8 +390,10 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp, } err = 0; out: - if (err && p->thread.io_bitmap_ptr) + if (err && p->thread.io_bitmap_ptr) { kfree(p->thread.io_bitmap_ptr); + p->thread.io_bitmap_max = 0; + } return err; } @@ -490,22 +500,18 @@ struct task_struct *__switch_to(struct task_struct *prev_p, struct task_struct * * Handle the IO bitmap */ if (unlikely(prev->io_bitmap_ptr || next->io_bitmap_ptr)) { - if (next->io_bitmap_ptr) { + if (next->io_bitmap_ptr) /* - * 2 cachelines copy ... not good, but not that - * bad either. Anyone got something better? - * This only affects processes which use ioperm(). - */ - memcpy(tss->io_bitmap, next->io_bitmap_ptr, IO_BITMAP_BYTES); - tss->io_bitmap_base = IO_BITMAP_OFFSET; - } else { + * Copy the relevant range of the IO bitmap. + * Normally this is 128 bytes or less: + */ + memcpy(tss->io_bitmap, next->io_bitmap_ptr, + max(prev->io_bitmap_max, next->io_bitmap_max)); + else { /* - * a bitmap offset pointing outside of the TSS limit - * causes a nicely controllable SIGSEGV if a process - * tries to use a port IO instruction. The first - * sys_ioperm() call sets up the bitmap properly. + * Clear any possible leftover bits: */ - tss->io_bitmap_base = INVALID_IO_BITMAP_OFFSET; + memset(tss->io_bitmap, 0xff, prev->io_bitmap_max); } } diff --git a/arch/x86_64/kernel/setup64.c b/arch/x86_64/kernel/setup64.c index b0cdd5d67738f..6693fb0e45587 100644 --- a/arch/x86_64/kernel/setup64.c +++ b/arch/x86_64/kernel/setup64.c @@ -241,6 +241,7 @@ void __init cpu_init (void) unsigned long v; char *estacks = NULL; struct task_struct *me; + int i; /* CPU 0 is initialised in head64.c */ if (cpu != 0) { @@ -304,12 +305,13 @@ void __init cpu_init (void) t->ist[v] = (unsigned long)estacks; } - t->io_bitmap_base = INVALID_IO_BITMAP_OFFSET; + t->io_bitmap_base = offsetof(struct tss_struct, io_bitmap); /* - * This is required because the CPU will access up to + * <= is required because the CPU will access up to * 8 bits beyond the end of the IO permission bitmap. */ - t->io_bitmap[IO_BITMAP_LONGS] = ~0UL; + for (i = 0; i <= IO_BITMAP_LONGS; i++) + t->io_bitmap[i] = ~0UL; atomic_inc(&init_mm.mm_count); me->active_mm = &init_mm; diff --git a/include/asm-x86_64/processor.h b/include/asm-x86_64/processor.h index aca8b34bad4fb..b42eec0e195dd 100644 --- a/include/asm-x86_64/processor.h +++ b/include/asm-x86_64/processor.h @@ -254,6 +254,7 @@ struct thread_struct { switch faster for a limited number of ioperm using tasks. -AK */ int ioperm; unsigned long *io_bitmap_ptr; + unsigned io_bitmap_max; /* cached TLS descriptors. */ u64 tls_array[GDT_ENTRY_TLS_ENTRIES]; } __attribute__((aligned(16))); -- 2.53.0