From 87e5358740f940ba9f76781f3f249ca5eb262e96 Mon Sep 17 00:00:00 2001 From: Peter Chubb Date: Tue, 4 Mar 2003 02:06:48 -0800 Subject: [PATCH] [PATCH] ia64: Preemption patch against ~2.5.60 Latest premption patch. --- arch/ia64/Kconfig | 18 ++++++ arch/ia64/hp/sim/simserial.c | 42 ++++++------- arch/ia64/ia32/ia32_support.c | 3 +- arch/ia64/kernel/entry.S | 25 +++++++- arch/ia64/kernel/irq.c | 4 +- arch/ia64/kernel/palinfo.c | 4 +- arch/ia64/kernel/perfmon.c | 107 ++++++++++++++++++++++++++++----- arch/ia64/kernel/smp.c | 28 ++++++++- arch/ia64/mm/fault.c | 2 +- arch/ia64/mm/tlb.c | 10 ++- include/asm-ia64/hardirq.h | 10 +-- include/asm-ia64/system.h | 2 +- include/asm-ia64/thread_info.h | 3 +- 13 files changed, 204 insertions(+), 54 deletions(-) diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index e5c3fe0b6cabd..6c2c631979a33 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -424,6 +424,18 @@ config SMP If you don't know what to do here, say N. +config PREEMPT + bool "Preemptible Kernel" + help + This option reduces the latency of the kernel when reacting to + real-time or interactive events by allowing a low priority process to + be preempted even if it is in kernel mode executing a system call. + This allows applications to run more reliably even when the system is + under load. + + Say Y here if you are building a kernel for a desktop, embedded + or real-time system. Say N if you are unsure. + config IA32_SUPPORT bool "Support running of Linux/x86 binaries" help @@ -875,6 +887,12 @@ config DEBUG_SPINLOCK best used in conjunction with the NMI watchdog so that spinlock deadlocks are also debuggable. +config DEBUG_SPINLOCK_SLEEP + bool "Sleep-inside-spinlock checking" + help + If you say Y here, various routines which may sleep will become very + noisy if they are called with a spinlock held. + config IA64_DEBUG_CMPXCHG bool "Turn on compare-and-exchange bug checking (slow!)" depends on DEBUG_KERNEL diff --git a/arch/ia64/hp/sim/simserial.c b/arch/ia64/hp/sim/simserial.c index 2ca3be2678a88..f5cedfd3a10b6 100644 --- a/arch/ia64/hp/sim/simserial.c +++ b/arch/ia64/hp/sim/simserial.c @@ -63,7 +63,6 @@ extern void ia64_ssc_connect_irq (long intr, long irq); static char *serial_name = "SimSerial driver"; static char *serial_version = "0.6"; -static spinlock_t serial_lock = SPIN_LOCK_UNLOCKED; /* * This has been extracted from asm/serial.h. We need one eventually but @@ -235,14 +234,14 @@ static void rs_put_char(struct tty_struct *tty, unsigned char ch) if (!tty || !info->xmit.buf) return; - spin_lock_irqsave(&serial_lock, flags); + local_irq_save(flags); if (CIRC_SPACE(info->xmit.head, info->xmit.tail, SERIAL_XMIT_SIZE) == 0) { - spin_unlock_irqrestore(&serial_lock, flags); + local_irq_restore(flags); return; } info->xmit.buf[info->xmit.head] = ch; info->xmit.head = (info->xmit.head + 1) & (SERIAL_XMIT_SIZE-1); - spin_unlock_irqrestore(&serial_lock, flags); + local_irq_restore(flags); } static _INLINE_ void transmit_chars(struct async_struct *info, int *intr_done) @@ -250,7 +249,8 @@ static _INLINE_ void transmit_chars(struct async_struct *info, int *intr_done) int count; unsigned long flags; - spin_lock_irqsave(&serial_lock, flags); + + local_irq_save(flags); if (info->x_char) { char c = info->x_char; @@ -293,7 +293,7 @@ static _INLINE_ void transmit_chars(struct async_struct *info, int *intr_done) info->xmit.tail += count; } out: - spin_unlock_irqrestore(&serial_lock, flags); + local_irq_restore(flags); } static void rs_flush_chars(struct tty_struct *tty) @@ -334,7 +334,7 @@ static int rs_write(struct tty_struct * tty, int from_user, break; } - spin_lock_irqsave(&serial_lock, flags); + local_irq_save(flags); { c1 = CIRC_SPACE_TO_END(info->xmit.head, info->xmit.tail, SERIAL_XMIT_SIZE); @@ -344,7 +344,7 @@ static int rs_write(struct tty_struct * tty, int from_user, info->xmit.head = ((info->xmit.head + c) & (SERIAL_XMIT_SIZE-1)); } - spin_unlock_irqrestore(&serial_lock, flags); + local_irq_restore(flags); buf += c; count -= c; @@ -352,7 +352,7 @@ static int rs_write(struct tty_struct * tty, int from_user, } up(&tmp_buf_sem); } else { - spin_lock_irqsave(&serial_lock, flags); + local_irq_save(flags); while (1) { c = CIRC_SPACE_TO_END(info->xmit.head, info->xmit.tail, SERIAL_XMIT_SIZE); if (count < c) @@ -367,7 +367,7 @@ static int rs_write(struct tty_struct * tty, int from_user, count -= c; ret += c; } - spin_unlock_irqrestore(&serial_lock, flags); + local_irq_restore(flags); } /* * Hey, we transmit directly from here in our case @@ -398,9 +398,9 @@ static void rs_flush_buffer(struct tty_struct *tty) struct async_struct *info = (struct async_struct *)tty->driver_data; unsigned long flags; - spin_lock_irqsave(&serial_lock, flags); + local_irq_save(flags); info->xmit.head = info->xmit.tail = 0; - spin_unlock_irqrestore(&serial_lock, flags); + local_irq_restore(flags); wake_up_interruptible(&tty->write_wait); @@ -573,7 +573,7 @@ static void shutdown(struct async_struct * info) state->irq); #endif - spin_lock_irqsave(&serial_lock, flags); + local_irq_save(flags); { /* * First unlink the serial port from the IRQ chain... @@ -611,7 +611,7 @@ static void shutdown(struct async_struct * info) info->flags &= ~ASYNC_INITIALIZED; } - spin_unlock_irqrestore(&serial_lock, flags); + local_irq_restore(flags); } /* @@ -634,13 +634,13 @@ static void rs_close(struct tty_struct *tty, struct file * filp) state = info->state; - spin_lock_irqsave(&serial_lock, flags); + local_irq_save(flags); if (tty_hung_up_p(filp)) { #ifdef SIMSERIAL_DEBUG printk("rs_close: hung_up\n"); #endif MOD_DEC_USE_COUNT; - spin_unlock_irqrestore(&serial_lock, flags); + local_irq_restore(flags); return; } #ifdef SIMSERIAL_DEBUG @@ -665,11 +665,11 @@ static void rs_close(struct tty_struct *tty, struct file * filp) } if (state->count) { MOD_DEC_USE_COUNT; - spin_unlock_irqrestore(&serial_lock, flags); + local_irq_restore(flags); return; } info->flags |= ASYNC_CLOSING; - spin_unlock_irqrestore(&serial_lock, flags); + local_irq_restore(flags); /* * Now we wait for the transmit buffer to clear; and we notify @@ -776,7 +776,7 @@ startup(struct async_struct *info) if (!page) return -ENOMEM; - spin_lock_irqsave(&serial_lock, flags); + local_irq_save(flags); if (info->flags & ASYNC_INITIALIZED) { free_page(page); @@ -857,11 +857,11 @@ startup(struct async_struct *info) } info->flags |= ASYNC_INITIALIZED; - spin_unlock_irqrestore(&serial_lock, flags); + local_irq_restore(flags); return 0; errout: - spin_unlock_irqrestore(&serial_lock, flags); + local_irq_restore(flags); return retval; } diff --git a/arch/ia64/ia32/ia32_support.c b/arch/ia64/ia32/ia32_support.c index 1280c806e43b8..8b6db81cf0209 100644 --- a/arch/ia64/ia32/ia32_support.c +++ b/arch/ia64/ia32/ia32_support.c @@ -93,7 +93,7 @@ ia32_load_state (struct task_struct *t) { unsigned long eflag, fsr, fcr, fir, fdr, csd, ssd, tssd; struct pt_regs *regs = ia64_task_regs(t); - int nr = smp_processor_id(); /* LDT and TSS depend on CPU number: */ + int nr = get_cpu(); /* LDT and TSS depend on CPU number: */ eflag = t->thread.eflag; fsr = t->thread.fsr; @@ -119,6 +119,7 @@ ia32_load_state (struct task_struct *t) regs->r17 = (_TSS(nr) << 48) | (_LDT(nr) << 32) | (__u32) regs->r17; regs->r30 = load_desc(_LDT(nr)); /* LDTD */ + put_cpu(); } /* diff --git a/arch/ia64/kernel/entry.S b/arch/ia64/kernel/entry.S index 37ea13286776d..3fcb8205843a3 100644 --- a/arch/ia64/kernel/entry.S +++ b/arch/ia64/kernel/entry.S @@ -586,10 +586,21 @@ GLOBAL_ENTRY(ia64_leave_kernel) // work.need_resched etc. mustn't get changed by this CPU before it returns to // user- or fsys-mode: (pUStk) cmp.eq.unc p6,p0=r0,r0 // p6 <- pUStk +#ifdef CONFIG_PREEMPT + rsm psr.i // disable interrupts + adds r17=TI_FLAGS+IA64_TASK_SIZE,r13 +(pKStk) adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13 + ;; +(pKStk) ld4 r21=[r20] // preempt_count ->r21 + ;; +(pKStk) cmp4.eq p6,p0=r21,r0 // p6 <- preempt_count == 0 + ;; +#else // CONFIG_PREEMPT (pUStk) rsm psr.i ;; (pUStk) adds r17=TI_FLAGS+IA64_TASK_SIZE,r13 ;; +#endif // CONFIG_PREEMPT .work_processed: (p6) ld4 r18=[r17] // load current_thread_info()->flags adds r2=PT(R8)+16,r12 @@ -810,15 +821,27 @@ skip_rbs_switch: .work_pending: tbit.z p6,p0=r18,TIF_NEED_RESCHED // current_thread_info()->need_resched==0? (p6) br.cond.sptk.few .notify +#ifdef CONFIG_PREEMPT +(pKStk) dep r21=-1,r0,PREEMPT_ACTIVE_BIT,1 + ;; +(pKStk) st4 [r20]=r21 + ssm psr.i // enable interrupts +#endif + #if __GNUC__ < 3 br.call.spnt.many rp=invoke_schedule #else br.call.spnt.many rp=schedule #endif .ret9: cmp.eq p6,p0=r0,r0 // p6 <- 1 - rsm psr.i + rsm psr.i // disable interrupts ;; adds r17=TI_FLAGS+IA64_TASK_SIZE,r13 +#if CONFIG_PREEMPT +(pKStk) adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13 + ;; +(pKStk) st4 [r20]=r0 // preempt_count() <- 0 +#endif br.cond.sptk.many .work_processed // re-check .notify: diff --git a/arch/ia64/kernel/irq.c b/arch/ia64/kernel/irq.c index 91ac809868237..5925397f4566f 100644 --- a/arch/ia64/kernel/irq.c +++ b/arch/ia64/kernel/irq.c @@ -340,12 +340,14 @@ unsigned int do_IRQ(unsigned long irq, struct pt_regs *regs) * 0 return value means that this irq is already being * handled by some other CPU. (or is disabled) */ - int cpu = smp_processor_id(); + int cpu; irq_desc_t *desc = irq_desc(irq); struct irqaction * action; unsigned int status; irq_enter(); + cpu = smp_processor_id(); + kstat_cpu(cpu).irqs[irq]++; if (desc->status & IRQ_PER_CPU) { diff --git a/arch/ia64/kernel/palinfo.c b/arch/ia64/kernel/palinfo.c index cc0896a2e9f00..344c5ab855226 100644 --- a/arch/ia64/kernel/palinfo.c +++ b/arch/ia64/kernel/palinfo.c @@ -894,11 +894,13 @@ palinfo_read_entry(char *page, char **start, off_t off, int count, int *eof, voi * in SMP mode, we may need to call another CPU to get correct * information. PAL, by definition, is processor specific */ - if (f->req_cpu == smp_processor_id()) + if (f->req_cpu == get_cpu()) len = (*palinfo_entries[f->func_id].proc_read)(page); else len = palinfo_handle_smp(f, page); + put_cpu(); + if (len <= off+count) *eof = 1; *start = page + off; diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c index a43eb97f839c7..2bb7e9bf444a6 100644 --- a/arch/ia64/kernel/perfmon.c +++ b/arch/ia64/kernel/perfmon.c @@ -1523,6 +1523,7 @@ pfm_write_pmds(struct task_struct *task, pfm_context_t *ctx, void *arg, int coun * Cannot do anything before PMU is enabled */ if (!CTX_IS_ENABLED(ctx)) return -EINVAL; + preempt_disable(); /* XXX: ctx locking may be required here */ @@ -1599,10 +1600,12 @@ pfm_write_pmds(struct task_struct *task, pfm_context_t *ctx, void *arg, int coun ctx->ctx_used_pmds[0], ctx->ctx_soft_pmds[cnum].reset_pmds[0])); } - + preempt_enable(); return 0; abort_mission: + preempt_enable(); + /* * for now, we have only one possibility for error */ @@ -1647,6 +1650,7 @@ pfm_read_pmds(struct task_struct *task, pfm_context_t *ctx, void *arg, int count DBprintk(("ctx_last_cpu=%d for [%d]\n", atomic_read(&ctx->ctx_last_cpu), task->pid)); for (i = 0; i < count; i++, req++) { + int me; #if __GNUC__ < 3 foo = __get_user(cnum, &req->reg_num); if (foo) return -EFAULT; @@ -1674,13 +1678,16 @@ pfm_read_pmds(struct task_struct *task, pfm_context_t *ctx, void *arg, int count * PMU state is still in the local live register due to lazy ctxsw. * If true, then we read directly from the registers. */ - if (atomic_read(&ctx->ctx_last_cpu) == smp_processor_id()){ + me = get_cpu(); + if (atomic_read(&ctx->ctx_last_cpu) == me){ ia64_srlz_d(); val = ia64_get_pmd(cnum); DBprintk(("reading pmd[%u]=0x%lx from hw\n", cnum, val)); } else { val = th->pmd[cnum]; } + + if (PMD_IS_COUNTING(cnum)) { /* * XXX: need to check for overflow @@ -1702,6 +1709,8 @@ pfm_read_pmds(struct task_struct *task, pfm_context_t *ctx, void *arg, int count PFM_REG_RETFLAG_SET(reg_flags, ret); + put_cpu(); + DBprintk(("read pmd[%u] ret=%d value=0x%lx pmc=0x%lx\n", cnum, ret, val, ia64_get_pmc(cnum))); @@ -1839,6 +1848,7 @@ pfm_restart(struct task_struct *task, pfm_context_t *ctx, void *arg, int count, ctx->ctx_fl_frozen, ctx->ctx_ovfl_regs[0])); + preempt_disable(); pfm_reset_regs(ctx, ctx->ctx_ovfl_regs, PFM_PMD_LONG_RESET); ctx->ctx_ovfl_regs[0] = 0UL; @@ -1857,6 +1867,8 @@ pfm_restart(struct task_struct *task, pfm_context_t *ctx, void *arg, int count, /* simply unfreeze */ pfm_unfreeze_pmu(); + preempt_enable(); + return 0; } /* restart on another task */ @@ -1914,6 +1926,7 @@ pfm_stop(struct task_struct *task, pfm_context_t *ctx, void *arg, int count, ctx->ctx_fl_system, PMU_OWNER(), current)); + preempt_disable(); /* simply stop monitoring but not the PMU */ if (ctx->ctx_fl_system) { @@ -1941,6 +1954,7 @@ pfm_stop(struct task_struct *task, pfm_context_t *ctx, void *arg, int count, */ ia64_psr(regs)->up = 0; } + preempt_enable(); return 0; } @@ -1953,6 +1967,7 @@ pfm_disable(struct task_struct *task, pfm_context_t *ctx, void *arg, int count, if (!CTX_IS_ENABLED(ctx)) return -EINVAL; + preempt_disable(); /* * stop monitoring, freeze PMU, and save state in context * this call will clear IA64_THREAD_PM_VALID for per-task sessions. @@ -1973,6 +1988,7 @@ pfm_disable(struct task_struct *task, pfm_context_t *ctx, void *arg, int count, DBprintk(("enabling psr.sp for [%d]\n", current->pid)); ctx->ctx_flags.state = PFM_CTX_DISABLED; + preempt_enable(); return 0; } @@ -2322,6 +2338,7 @@ pfm_start(struct task_struct *task, pfm_context_t *ctx, void *arg, int count, return -EINVAL; } + preempt_disable(); if (ctx->ctx_fl_system) { PFM_CPUINFO_SET(PFM_CPUINFO_DCR_PP); @@ -2339,6 +2356,7 @@ pfm_start(struct task_struct *task, pfm_context_t *ctx, void *arg, int count, } else { if ((task->thread.flags & IA64_THREAD_PM_VALID) == 0) { + preempt_enable(); printk(KERN_DEBUG "perfmon: pfm_start task flag not set for [%d]\n", task->pid); return -EINVAL; @@ -2352,6 +2370,7 @@ pfm_start(struct task_struct *task, pfm_context_t *ctx, void *arg, int count, ia64_srlz_i(); } + preempt_enable(); return 0; } @@ -2359,9 +2378,13 @@ static int pfm_enable(struct task_struct *task, pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs) { + int me; + /* we don't quite support this right now */ if (task != current) return -EINVAL; + me = get_cpu(); /* make sure we're not migrated or preempted */ + if (ctx->ctx_fl_system == 0 && PMU_OWNER() && PMU_OWNER() != current) pfm_lazy_save_regs(PMU_OWNER()); @@ -2405,11 +2428,13 @@ pfm_enable(struct task_struct *task, pfm_context_t *ctx, void *arg, int count, SET_PMU_OWNER(task); ctx->ctx_flags.state = PFM_CTX_ENABLED; - atomic_set(&ctx->ctx_last_cpu, smp_processor_id()); + atomic_set(&ctx->ctx_last_cpu, me); /* simply unfreeze */ pfm_unfreeze_pmu(); + put_cpu(); + return 0; } @@ -2826,7 +2851,7 @@ pfm_record_sample(struct task_struct *task, pfm_context_t *ctx, unsigned long ov * initialize entry header */ h->pid = current->pid; - h->cpu = smp_processor_id(); + h->cpu = get_cpu(); h->last_reset_value = ovfl_mask ? ctx->ctx_soft_pmds[ffz(~ovfl_mask)].lval : 0UL; h->ip = regs ? regs->cr_iip | ((regs->cr_ipsr >> 41) & 0x3): 0x0UL; h->regs = ovfl_mask; /* which registers overflowed */ @@ -2853,7 +2878,7 @@ pfm_record_sample(struct task_struct *task, pfm_context_t *ctx, unsigned long ov DBprintk_ovfl(("e=%p pmd%d =0x%lx\n", (void *)e, j, *e)); e++; } - pfm_stats[smp_processor_id()].pfm_recorded_samples_count++; + pfm_stats[h->cpu].pfm_recorded_samples_count++; /* * make the new entry visible to user, needs to be atomic @@ -2870,9 +2895,11 @@ pfm_record_sample(struct task_struct *task, pfm_context_t *ctx, unsigned long ov /* * XXX: must reset buffer in blocking mode and lost notified */ - pfm_stats[smp_processor_id()].pfm_full_smpl_buffer_count++; + pfm_stats[h->cpu].pfm_full_smpl_buffer_count++; + put_cpu(); return 1; } + put_cpu(); return 0; } @@ -2904,6 +2931,8 @@ pfm_overflow_handler(struct task_struct *task, pfm_context_t *ctx, u64 pmc0, str * valid one, i.e. the one that caused the interrupt. */ + preempt_disable(); + t = &task->thread; /* @@ -2913,6 +2942,7 @@ pfm_overflow_handler(struct task_struct *task, pfm_context_t *ctx, u64 pmc0, str if ((t->flags & IA64_THREAD_PM_VALID) == 0 && ctx->ctx_fl_system == 0) { printk(KERN_DEBUG "perfmon: Spurious overflow interrupt: process %d not " "using perfmon\n", task->pid); + preempt_enable_no_resched(); return 0x1; } /* @@ -2921,6 +2951,7 @@ pfm_overflow_handler(struct task_struct *task, pfm_context_t *ctx, u64 pmc0, str if ((pmc0 & 0x1) == 0) { printk(KERN_DEBUG "perfmon: pid %d pmc0=0x%lx assumption error for freeze bit\n", task->pid, pmc0); + preempt_enable_no_resched(); return 0x0; } @@ -3003,6 +3034,7 @@ pfm_overflow_handler(struct task_struct *task, pfm_context_t *ctx, u64 pmc0, str if (ovfl_notify == 0UL) { if (ovfl_pmds) pfm_reset_regs(ctx, &ovfl_pmds, PFM_PMD_SHORT_RESET); + preempt_enable_no_resched(); return 0x0UL; } @@ -3038,6 +3070,7 @@ pfm_overflow_handler(struct task_struct *task, pfm_context_t *ctx, u64 pmc0, str t->pfm_ovfl_block_reset, ctx->ctx_fl_trap_reason)); + preempt_enable_no_resched(); return 0x1UL; } @@ -3048,13 +3081,14 @@ pfm_interrupt_handler(int irq, void *arg, struct pt_regs *regs) struct task_struct *task; pfm_context_t *ctx; - pfm_stats[smp_processor_id()].pfm_ovfl_intr_count++; + pfm_stats[get_cpu()].pfm_ovfl_intr_count++; /* * if an alternate handler is registered, just bypass the default one */ if (pfm_alternate_intr_handler) { (*pfm_alternate_intr_handler->handler)(irq, arg, regs); + put_cpu(); return; } @@ -3079,6 +3113,7 @@ pfm_interrupt_handler(int irq, void *arg, struct pt_regs *regs) if (!ctx) { printk(KERN_DEBUG "perfmon: Spurious overflow interrupt: process %d has " "no PFM context\n", task->pid); + put_cpu(); return; } @@ -3104,6 +3139,7 @@ pfm_interrupt_handler(int irq, void *arg, struct pt_regs *regs) } else { pfm_stats[smp_processor_id()].pfm_spurious_ovfl_intr_count++; } + put_cpu_no_resched(); } /* for debug only */ @@ -3174,6 +3210,7 @@ pfm_syst_wide_update_task(struct task_struct *task, unsigned long info, int is_c unsigned long dcr; unsigned long dcr_pp; + preempt_disable(); dcr_pp = info & PFM_CPUINFO_DCR_PP ? 1 : 0; /* @@ -3184,6 +3221,7 @@ pfm_syst_wide_update_task(struct task_struct *task, unsigned long info, int is_c regs = (struct pt_regs *)((unsigned long) task + IA64_STK_OFFSET); regs--; ia64_psr(regs)->pp = is_ctxswin ? dcr_pp : 0; + preempt_enable(); return; } /* @@ -3199,6 +3237,7 @@ pfm_syst_wide_update_task(struct task_struct *task, unsigned long info, int is_c ia64_set_dcr(dcr & ~IA64_DCR_PP); pfm_clear_psr_pp(); ia64_srlz_i(); + preempt_enable(); return; } /* @@ -3212,6 +3251,7 @@ pfm_syst_wide_update_task(struct task_struct *task, unsigned long info, int is_c pfm_set_psr_pp(); ia64_srlz_i(); } + preempt_enable(); } void @@ -3222,6 +3262,8 @@ pfm_save_regs (struct task_struct *task) u64 psr; int i; + preempt_disable(); + ctx = task->thread.pfm_context; @@ -3275,6 +3317,7 @@ pfm_save_regs (struct task_struct *task) */ atomic_set(&ctx->ctx_last_cpu, -1); #endif + preempt_enable(); } static void @@ -3285,6 +3328,7 @@ pfm_lazy_save_regs (struct task_struct *task) unsigned long mask; int i; + preempt_disable(); DBprintk(("on [%d] by [%d]\n", task->pid, current->pid)); t = &task->thread; @@ -3311,6 +3355,7 @@ pfm_lazy_save_regs (struct task_struct *task) /* not owned by this CPU */ atomic_set(&ctx->ctx_last_cpu, -1); + preempt_enable(); } void @@ -3323,11 +3368,14 @@ pfm_load_regs (struct task_struct *task) u64 psr; int i; + preempt_disable(); + owner = PMU_OWNER(); ctx = task->thread.pfm_context; t = &task->thread; if (ctx == NULL) { + preempt_enable(); printk("perfmon: pfm_load_regs: null ctx for [%d]\n", task->pid); return; } @@ -3366,7 +3414,7 @@ pfm_load_regs (struct task_struct *task) psr = ctx->ctx_saved_psr; pfm_set_psr_l(psr); - + preempt_enable(); return; } @@ -3428,6 +3476,7 @@ pfm_load_regs (struct task_struct *task) * restore the psr we changed in pfm_save_regs() */ psr = ctx->ctx_saved_psr; + preempt_enable(); pfm_set_psr_l(psr); } @@ -3445,6 +3494,7 @@ pfm_reset_pmu(struct task_struct *task) printk("perfmon: invalid task in pfm_reset_pmu()\n"); return; } + preempt_disable(); /* Let's make sure the PMU is frozen */ pfm_freeze_pmu(); @@ -3527,6 +3577,7 @@ pfm_reset_pmu(struct task_struct *task) ctx->ctx_used_dbrs[0] = 0UL; ia64_srlz_d(); + preempt_enable(); } /* @@ -3556,6 +3607,7 @@ pfm_flush_regs (struct task_struct *task) */ if (ctx->ctx_flags.state == PFM_CTX_DISABLED) return; + preempt_disable(); /* * stop monitoring: * This is the only way to stop monitoring without destroying overflow @@ -3683,7 +3735,7 @@ pfm_flush_regs (struct task_struct *task) * indicates that context has been saved */ atomic_set(&ctx->ctx_last_cpu, -1); - + preempt_enable(); } @@ -3706,6 +3758,7 @@ pfm_inherit(struct task_struct *task, struct pt_regs *regs) ctx = task->thread.pfm_context; thread = &task->thread; + preempt_disable(); /* * make sure child cannot mess up the monitoring session */ @@ -3760,6 +3813,8 @@ pfm_inherit(struct task_struct *task, struct pt_regs *regs) */ ia64_psr(regs)->up = 0; + preempt_enable(); + /* copy_thread() clears IA64_THREAD_PM_VALID */ return 0; } @@ -3865,6 +3920,8 @@ pfm_inherit(struct task_struct *task, struct pt_regs *regs) thread->flags |= IA64_THREAD_PM_VALID; } + preempt_enable(); + return 0; } @@ -3886,6 +3943,7 @@ pfm_context_exit(struct task_struct *task) /* * check sampling buffer */ + preempt_disable(); if (ctx->ctx_psb) { pfm_smpl_buffer_desc_t *psb = ctx->ctx_psb; @@ -3978,6 +4036,7 @@ pfm_context_exit(struct task_struct *task) } UNLOCK_CTX(ctx); + preempt_enable(); pfm_unreserve_session(task, ctx->ctx_fl_system, 1UL << ctx->ctx_cpu); @@ -4156,17 +4215,27 @@ pfm_install_alternate_syswide_subsystem(pfm_intr_handler_desc_t *hdl) { int ret; + /* some sanity checks */ - if (hdl == NULL || hdl->handler == NULL) return -EINVAL; + if (hdl == NULL || hdl->handler == NULL) { + return -EINVAL; + } /* do the easy test first */ - if (pfm_alternate_intr_handler) return -EBUSY; + if (pfm_alternate_intr_handler) { + return -EBUSY; + } + preempt_disable(); /* reserve our session */ ret = pfm_reserve_session(NULL, 1, cpu_online_map); - if (ret) return ret; + if (ret) { + preempt_enable(); + return ret; + } if (pfm_alternate_intr_handler) { + preempt_enable(); printk(KERN_DEBUG "perfmon: install_alternate, intr_handler not NULL " "after reserve\n"); return -EINVAL; @@ -4174,17 +4243,21 @@ pfm_install_alternate_syswide_subsystem(pfm_intr_handler_desc_t *hdl) pfm_alternate_intr_handler = hdl; + preempt_enable(); return 0; } int pfm_remove_alternate_syswide_subsystem(pfm_intr_handler_desc_t *hdl) { - if (hdl == NULL) return -EINVAL; + if (hdl == NULL) + return -EINVAL; /* cannot remove someone else's handler! */ - if (pfm_alternate_intr_handler != hdl) return -EINVAL; + if (pfm_alternate_intr_handler != hdl) + return -EINVAL; + preempt_disable(); pfm_alternate_intr_handler = NULL; /* @@ -4192,6 +4265,8 @@ pfm_remove_alternate_syswide_subsystem(pfm_intr_handler_desc_t *hdl) */ pfm_unreserve_session(NULL, 1, cpu_online_map); + preempt_enable(); + return 0; } @@ -4272,8 +4347,9 @@ void pfm_init_percpu(void) { int i; + int me = get_cpu(); - if (smp_processor_id() == 0) + if (me == 0) register_percpu_irq(IA64_PERFMON_VECTOR, &perfmon_irqaction); ia64_set_pmv(IA64_PERFMON_VECTOR); @@ -4297,6 +4373,7 @@ pfm_init_percpu(void) if (PMD_IS_IMPL(i) == 0) continue; ia64_set_pmd(i, 0UL); } + put_cpu(); pfm_freeze_pmu(); } diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c index 9532fe7ff1166..db36b260808d6 100644 --- a/arch/ia64/kernel/smp.c +++ b/arch/ia64/kernel/smp.c @@ -90,7 +90,7 @@ stop_this_cpu (void) void handle_IPI (int irq, void *dev_id, struct pt_regs *regs) { - int this_cpu = smp_processor_id(); + int this_cpu = get_cpu(); unsigned long *pending_ipis = &__get_cpu_var(ipi_operation); unsigned long ops; @@ -146,8 +146,12 @@ handle_IPI (int irq, void *dev_id, struct pt_regs *regs) } while (ops); mb(); /* Order data access and bit testing. */ } + put_cpu(); } +/* + * Called with preeemption disabled + */ static inline void send_IPI_single (int dest_cpu, int op) { @@ -155,6 +159,9 @@ send_IPI_single (int dest_cpu, int op) platform_send_ipi(dest_cpu, IA64_IPI_VECTOR, IA64_IPI_DM_INT, 0); } +/* + * Called with preeemption disabled + */ static inline void send_IPI_allbutself (int op) { @@ -166,6 +173,9 @@ send_IPI_allbutself (int op) } } +/* + * Called with preeemption disabled + */ static inline void send_IPI_all (int op) { @@ -176,12 +186,18 @@ send_IPI_all (int op) send_IPI_single(i, op); } +/* + * Called with preeemption disabled + */ static inline void send_IPI_self (int op) { send_IPI_single(smp_processor_id(), op); } +/* + * Called with preeemption disabled + */ void smp_send_reschedule (int cpu) { @@ -197,12 +213,15 @@ void smp_send_reschedule_all (void) { int i; + int cpu = get_cpu(); /* disable preemption */ for (i = 0; i < NR_CPUS; i++) - if (cpu_online(i) && i != smp_processor_id()) + if (cpu_online(i) && i != cpu) smp_send_reschedule(i); + put_cpu(); } + void smp_flush_tlb_all (void) { @@ -247,9 +266,11 @@ smp_call_function_single (int cpuid, void (*func) (void *info), void *info, int { struct call_data_struct data; int cpus = 1; + int me = get_cpu(); /* prevent preemption and reschedule on another processor */ - if (cpuid == smp_processor_id()) { + if (cpuid == me) { printk("%s: trying to call self\n", __FUNCTION__); + put_cpu(); return -EBUSY; } @@ -276,6 +297,7 @@ smp_call_function_single (int cpuid, void (*func) (void *info), void *info, int call_data = NULL; spin_unlock_bh(&call_lock); + put_cpu(); return 0; } diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index 18f5024c6e594..ebffa17119b53 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c @@ -55,7 +55,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re /* * If we're in an interrupt or have no user context, we must not take the fault.. */ - if (in_interrupt() || !mm) + if (in_atomic() || !mm) goto no_context; down_read(&mm->mmap_sem); diff --git a/arch/ia64/mm/tlb.c b/arch/ia64/mm/tlb.c index 4a647ce99fd9a..af5955328e355 100644 --- a/arch/ia64/mm/tlb.c +++ b/arch/ia64/mm/tlb.c @@ -81,9 +81,13 @@ wrap_mmu_context (struct mm_struct *mm) } read_unlock(&tasklist_lock); /* can't call flush_tlb_all() here because of race condition with O(1) scheduler [EF] */ - for (i = 0; i < NR_CPUS; ++i) - if (i != smp_processor_id()) - per_cpu(ia64_need_tlb_flush, i) = 1; + { + int cpu = get_cpu(); /* prevent preemption/migration */ + for (i = 0; i < NR_CPUS; ++i) + if (i != cpu) + per_cpu(ia64_need_tlb_flush, i) = 1; + put_cpu(); + } local_flush_tlb_all(); } diff --git a/include/asm-ia64/hardirq.h b/include/asm-ia64/hardirq.h index ad0a0c56a9c1b..54be084906ee4 100644 --- a/include/asm-ia64/hardirq.h +++ b/include/asm-ia64/hardirq.h @@ -32,18 +32,18 @@ * * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) - * - bits 16-31 are the hardirq count (max # of hardirqs: 65536) + * - bits 16-29 are the hardirq count (max # of hardirqs: 16384) * * - (bit 63 is the PREEMPT_ACTIVE flag---not currently implemented.) * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 - * HARDIRQ_MASK: 0xffff0000 + * HARDIRQ_MASK: 0x3fff0000 */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 -#define HARDIRQ_BITS 16 +#define HARDIRQ_BITS 14 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) @@ -83,13 +83,13 @@ #define hardirq_trylock() (!in_interrupt()) #define hardirq_endlock() do { } while (0) -#define in_atomic() (preempt_count() != 0) #define irq_enter() (preempt_count() += HARDIRQ_OFFSET) #if CONFIG_PREEMPT -# error CONFIG_PREEMT currently not supported. +# define in_atomic() ((preempt_count() & ~PREEMPT_ACTIVE) != kernel_locked()) # define IRQ_EXIT_OFFSET (HARDIRQ_OFFSET-1) #else +# define in_atomic() (preempt_count() != 0) # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET #endif diff --git a/include/asm-ia64/system.h b/include/asm-ia64/system.h index 357758463c875..ef05383039b69 100644 --- a/include/asm-ia64/system.h +++ b/include/asm-ia64/system.h @@ -206,7 +206,7 @@ extern void ia64_load_extra (struct task_struct *task); #ifdef CONFIG_PERFMON DECLARE_PER_CPU(unsigned long, pfm_syst_info); -# define PERFMON_IS_SYSWIDE() (get_cpu_var(pfm_syst_info) & 0x1) +# define PERFMON_IS_SYSWIDE() (__get_cpu_var(pfm_syst_info) & 0x1) #else # define PERFMON_IS_SYSWIDE() (0) #endif diff --git a/include/asm-ia64/thread_info.h b/include/asm-ia64/thread_info.h index 23399d0d53109..3109fa8169da7 100644 --- a/include/asm-ia64/thread_info.h +++ b/include/asm-ia64/thread_info.h @@ -15,7 +15,8 @@ #define TI_ADDR_LIMIT 0x10 #define TI_PRE_COUNT 0x18 -#define PREEMPT_ACTIVE 0x4000000 +#define PREEMPT_ACTIVE_BIT 30 +#define PREEMPT_ACTIVE (1 << PREEMPT_ACTIVE_BIT) #ifndef __ASSEMBLY__ -- 2.53.0