VERSION = 2
PATCHLEVEL = 4
SUBLEVEL = 7
-EXTRAVERSION =-pre3
+EXTRAVERSION =-pre5
KERNELRELEASE=$(VERSION).$(PATCHLEVEL).$(SUBLEVEL)$(EXTRAVERSION)
.align 3
ret_from_sys_call:
cmovne $26,0,$19 /* $19 = 0 => non-restartable */
-#ifdef CONFIG_SMP
- ldl $3,TASK_PROCESSOR($8)
- sll $3,L1_CACHE_SHIFT,$3
-#endif
- lda $4,irq_stat
-#ifdef CONFIG_SMP
- addq $3,$4,$4
-#endif
- ldq $4,0($4) /* __softirq_pending */
- bne $4,handle_softirq
-ret_from_softirq:
ldq $0,SP_OFF($30)
and $0,8,$0
beq $0,restore_all
mov $31,$26 /* tell "ret_from_sys_call" we can restart */
br ret_from_sys_call
- .align 3
-handle_softirq:
- subq $30,16,$30
- stq $19,0($30) /* save syscall nr */
- stq $20,8($30) /* and error indication (a3) */
- jsr $26,do_softirq
- ldq $19,0($30)
- ldq $20,8($30)
- addq $30,16,$30
- br ret_from_softirq
-
.align 3
syscall_error:
/*
BUG();
if (atomic_read(&conf->active_stripes)==0)
BUG();
- if (test_bit(STRIPE_DELAYED, &sh->state))
- list_add_tail(&sh->lru, &conf->delayed_list);
- else if (test_bit(STRIPE_HANDLE, &sh->state)) {
- list_add_tail(&sh->lru, &conf->handle_list);
+ if (test_bit(STRIPE_HANDLE, &sh->state)) {
+ if (test_bit(STRIPE_DELAYED, &sh->state))
+ list_add_tail(&sh->lru, &conf->delayed_list);
+ else
+ list_add_tail(&sh->lru, &conf->handle_list);
md_wakeup_thread(conf->thread);
} else {
if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
raid5_activate_delayed(conf);
- if (conf->plugged) {
- conf->plugged = 0;
- md_wakeup_thread(conf->thread);
- }
+ conf->plugged = 0;
+ md_wakeup_thread(conf->thread);
+
spin_unlock_irqrestore(&conf->device_lock, flags);
}
dentry = fhp->fh_dentry;
if (!dentry->d_inode)
goto out_negative;
- if (fhp->fh_handle.fh_fileid_type != 0)
- goto out_uptodate;
if (fhp->fh_handle.fh_version != 1) {
_fh_update_old(dentry, fhp->fh_export, &fhp->fh_handle);
} else {
+ if (fhp->fh_handle.fh_fileid_type != 0)
+ goto out_uptodate;
datap = fhp->fh_handle.fh_auth+
fhp->fh_handle.fh_size/4 -1;
fhp->fh_handle.fh_fileid_type =
unsigned int __local_irq_count;
unsigned int __local_bh_count;
unsigned int __syscall_count;
+ struct task_struct * __ksoftirqd_task;
} ____cacheline_aligned irq_cpustat_t;
#include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */
extern inline void cpu_bh_disable(int cpu)
{
local_bh_count(cpu)++;
- mb();
+ barrier();
}
-extern inline void cpu_bh_enable(int cpu)
+extern inline void __cpu_bh_enable(int cpu)
{
- mb();
+ barrier();
local_bh_count(cpu)--;
}
-#define local_bh_enable() cpu_bh_enable(smp_processor_id())
-#define __local_bh_enable local_bh_enable
+#define __local_bh_enable() __cpu_bh_enable(smp_processor_id())
#define local_bh_disable() cpu_bh_disable(smp_processor_id())
+#define local_bh_enable() \
+do { \
+ int cpu; \
+ \
+ barrier(); \
+ cpu = smp_processor_id(); \
+ if (!--local_bh_count(cpu) && softirq_pending(cpu)) \
+ do_softirq(); \
+} while (0)
+
#define in_softirq() (local_bh_count(smp_processor_id()) != 0)
-#define __cpu_raise_softirq(cpu,nr) set_bit((nr), &softirq_pending(cpu))
+#define __cpu_raise_softirq(cpu, nr) set_bit(nr, &softirq_pending(cpu))
#endif /* _ALPHA_SOFTIRQ_H */
#define local_bh_disable() cpu_bh_disable(smp_processor_id())
#define __local_bh_enable() __cpu_bh_enable(smp_processor_id())
#define __cpu_raise_softirq(cpu,nr) set_bit((nr), &softirq_pending(cpu))
-#define raise_softirq(nr) __cpu_raise_softirq(smp_processor_id(), (nr))
#define in_softirq() (local_bh_count(smp_processor_id()) != 0)
unsigned int __local_irq_count;
unsigned int __local_bh_count;
unsigned int __syscall_count;
+ struct task_struct * __ksoftirqd_task; /* waitqueue is too large */
unsigned int __nmi_count; /* arch dependent */
} ____cacheline_aligned irq_cpustat_t;
#define local_bh_disable() cpu_bh_disable(smp_processor_id())
#define __local_bh_enable() __cpu_bh_enable(smp_processor_id())
-#define __cpu_raise_softirq(cpu,nr) set_bit((nr), &softirq_pending(cpu));
-#define raise_softirq(nr) __cpu_raise_softirq(smp_processor_id(), (nr))
#define in_softirq() (local_bh_count(smp_processor_id()) != 0)
do { \
unsigned int *ptr = &local_bh_count(smp_processor_id()); \
\
+ barrier(); \
if (!--*ptr) \
__asm__ __volatile__ ( \
"cmpl $0, -8(%0);" \
/* no registers clobbered */ ); \
} while (0)
+#define __cpu_raise_softirq(cpu, nr) __set_bit(nr, &softirq_pending(cpu))
+
#endif /* __ASM_SOFTIRQ_H */
} while (0)
#define __cpu_raise_softirq(cpu, nr) set_bit((nr), &softirq_pending(cpu));
-#define raise_softirq(nr) __cpu_raise_softirq(smp_processor_id(), (nr))
#define in_softirq() (local_bh_count(smp_processor_id()) != 0)
} while (0)
#define __cpu_raise_softirq(cpu, nr) set_bit((nr), &softirq_pending(cpu));
-#define raise_softirq(nr) __cpu_raise_softirq(smp_processor_id(), (nr))
#define in_softirq() (local_bh_count(smp_processor_id()) != 0)
#endif
unsigned int __local_bh_count;
unsigned int __syscall_count;
+ struct task_struct * __ksoftirqd_task;
} ____cacheline_aligned irq_cpustat_t;
#include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */
__sti(); \
} \
} while (0)
-#define __cpu_raise_softirq(cpu, nr) (softirq_pending(cpu) |= (1<<nr))
-#define raise_softirq(nr) \
+#define __do_cpu_raise_softirq(cpu, nr) (softirq_pending(cpu) |= (1<<nr))
+#define __cpu_raise_softirq(cpu, nr) \
do { unsigned long flags; \
local_irq_save(flags); \
- __cpu_raise_softirq(smp_processor_id(), nr); \
+ __do_cpu_raise_softirq(cpu, nr); \
local_irq_restore(flags); \
} while (0)
#define in_softirq() (local_bh_count(smp_processor_id()) != 0)
#endif
unsigned int __local_bh_count;
unsigned int __syscall_count;
+ struct task_struct * __ksoftirqd_task;
} ____cacheline_aligned irq_cpustat_t;
#include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */
__sti(); \
} \
} while (0)
-#define __cpu_raise_softirq(cpu, nr) (softirq_pending(cpu) |= (1<<nr))
-#define raise_softirq(nr) \
+
+#define __do_cpu_raise_softirq(cpu, nr) (softirq_pending(cpu) |= (1<<nr))
+#define __cpu_raise_softirq(cpu,nr) \
do { unsigned long flags; \
local_irq_save(flags); \
- __cpu_raise_softirq(smp_processor_id(), nr); \
+ __do_cpu_raise_softirq(cpu, nr); \
local_irq_restore(flags); \
} while (0)
#define in_softirq() (local_bh_count(smp_processor_id()) != 0)
asmlinkage void do_softirq(void);
extern void open_softirq(int nr, void (*action)(struct softirq_action*), void *data);
-
extern void softirq_init(void);
+extern void FASTCALL(cpu_raise_softirq(unsigned int cpu, unsigned int nr));
+extern void FASTCALL(raise_softirq(unsigned int nr));
extern struct tasklet_head tasklet_hi_vec[NR_CPUS];
#define tasklet_trylock(t) (!test_and_set_bit(TASKLET_STATE_RUN, &(t)->state))
-#define tasklet_unlock(t) clear_bit(TASKLET_STATE_RUN, &(t)->state)
+#define tasklet_unlock(t) do { smp_mb__before_clear_bit(); clear_bit(TASKLET_STATE_RUN, &(t)->state); } while(0)
#define tasklet_unlock_wait(t) while (test_bit(TASKLET_STATE_RUN, &(t)->state)) { barrier(); }
extern void tasklet_schedule(struct tasklet_struct *t);
#define local_irq_count(cpu) __IRQ_STAT((cpu), __local_irq_count)
#define local_bh_count(cpu) __IRQ_STAT((cpu), __local_bh_count)
#define syscall_count(cpu) __IRQ_STAT((cpu), __syscall_count)
+#define ksoftirqd_task(cpu) __IRQ_STAT((cpu), __ksoftirqd_task)
/* arch dependent irq_stat fields */
#define nmi_count(cpu) __IRQ_STAT((cpu), __nmi_count) /* i386, ia64 */
local_irq_save(flags);
dev->next_sched = softnet_data[cpu].output_queue;
softnet_data[cpu].output_queue = dev;
- __cpu_raise_softirq(cpu, NET_TX_SOFTIRQ);
+ cpu_raise_softirq(cpu, NET_TX_SOFTIRQ);
local_irq_restore(flags);
}
}
local_irq_save(flags);
skb->next = softnet_data[cpu].completion_queue;
softnet_data[cpu].completion_queue = skb;
- __cpu_raise_softirq(cpu, NET_TX_SOFTIRQ);
+ cpu_raise_softirq(cpu, NET_TX_SOFTIRQ);
local_irq_restore(flags);
}
}
EXPORT_SYMBOL(tasklet_kill);
EXPORT_SYMBOL(__run_task_queue);
EXPORT_SYMBOL(do_softirq);
+EXPORT_SYMBOL(raise_softirq);
+EXPORT_SYMBOL(cpu_raise_softirq);
EXPORT_SYMBOL(tasklet_schedule);
EXPORT_SYMBOL(tasklet_hi_schedule);
release_kernel_lock(prev, this_cpu);
- /* Do "administrative" work here while we don't hold any locks */
- if (softirq_pending(this_cpu))
- goto handle_softirq;
-handle_softirq_back:
-
/*
* 'sched_data' is protected by the fact that we can run
* only one process per CPU.
goto repeat_schedule;
still_running:
+ if (!(prev->cpus_allowed & (1UL << this_cpu)))
+ goto still_running_back;
c = goodness(prev, this_cpu, prev->active_mm);
next = prev;
goto still_running_back;
-handle_softirq:
- do_softirq();
- goto handle_softirq_back;
-
move_rr_last:
if (!prev->counter) {
prev->counter = NICE_TO_TICKS(prev->nice);
static struct softirq_action softirq_vec[32] __cacheline_aligned;
+/*
+ * we cannot loop indefinitely here to avoid userspace starvation,
+ * but we also don't want to introduce a worst case 1/HZ latency
+ * to the pending events, so lets the scheduler to balance
+ * the softirq load for us.
+ */
+static inline void wakeup_softirqd(unsigned cpu)
+{
+ struct task_struct * tsk = ksoftirqd_task(cpu);
+
+ if (tsk && tsk->state != TASK_RUNNING)
+ wake_up_process(tsk);
+}
+
asmlinkage void do_softirq()
{
int cpu = smp_processor_id();
__u32 pending;
+ long flags;
+ __u32 mask;
if (in_interrupt())
return;
- local_irq_disable();
+ local_irq_save(flags);
pending = softirq_pending(cpu);
if (pending) {
struct softirq_action *h;
+ mask = ~pending;
local_bh_disable();
restart:
/* Reset the pending bitmask before enabling irqs */
local_irq_disable();
pending = softirq_pending(cpu);
- if (pending)
+ if (pending & mask) {
+ mask &= ~pending;
goto restart;
+ }
__local_bh_enable();
+
+ if (pending)
+ wakeup_softirqd(cpu);
}
- local_irq_enable();
+ local_irq_restore(flags);
+}
+
+inline void cpu_raise_softirq(unsigned int cpu, unsigned int nr)
+{
+ __cpu_raise_softirq(cpu, nr);
+
+ /*
+ * If we're in an interrupt or bh, we're done
+ * (this also catches bh-disabled code). We will
+ * actually run the softirq once we return from
+ * the irq or bh.
+ *
+ * Otherwise we wake up ksoftirqd to make sure we
+ * schedule the softirq soon.
+ */
+ if (!(local_irq_count(cpu) | local_bh_count(cpu)))
+ wakeup_softirqd(cpu);
}
+void raise_softirq(unsigned int nr)
+{
+ cpu_raise_softirq(smp_processor_id(), nr);
+}
void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
{
* If nobody is running it then add it to this CPU's
* tasklet queue.
*/
- if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state) &&
- tasklet_trylock(t)) {
+ if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
t->next = tasklet_vec[cpu].list;
tasklet_vec[cpu].list = t;
- __cpu_raise_softirq(cpu, TASKLET_SOFTIRQ);
+ cpu_raise_softirq(cpu, TASKLET_SOFTIRQ);
tasklet_unlock(t);
}
local_irq_restore(flags);
cpu = smp_processor_id();
local_irq_save(flags);
- if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state) &&
- tasklet_trylock(t)) {
+ if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
t->next = tasklet_hi_vec[cpu].list;
tasklet_hi_vec[cpu].list = t;
- __cpu_raise_softirq(cpu, HI_SOFTIRQ);
+ cpu_raise_softirq(cpu, HI_SOFTIRQ);
tasklet_unlock(t);
}
local_irq_restore(flags);
local_irq_disable();
list = tasklet_vec[cpu].list;
tasklet_vec[cpu].list = NULL;
+ local_irq_enable();
while (list) {
struct tasklet_struct *t = list;
list = list->next;
- /*
- * A tasklet is only added to the queue while it's
- * locked, so no other CPU can have this tasklet
- * pending:
- */
if (!tasklet_trylock(t))
BUG();
-repeat:
- if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
- BUG();
if (!atomic_read(&t->count)) {
- local_irq_enable();
+ if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
+ BUG();
t->func(t->data);
- local_irq_disable();
- /*
- * One more run if the tasklet got reactivated:
- */
- if (test_bit(TASKLET_STATE_SCHED, &t->state))
- goto repeat;
+ tasklet_unlock(t);
+ continue;
}
tasklet_unlock(t);
- if (test_bit(TASKLET_STATE_SCHED, &t->state))
- tasklet_schedule(t);
+
+ local_irq_disable();
+ t->next = tasklet_vec[cpu].list;
+ tasklet_vec[cpu].list = t;
+ cpu_raise_softirq(cpu, TASKLET_SOFTIRQ);
+ local_irq_enable();
}
- local_irq_enable();
}
local_irq_disable();
list = tasklet_hi_vec[cpu].list;
tasklet_hi_vec[cpu].list = NULL;
+ local_irq_enable();
while (list) {
struct tasklet_struct *t = list;
if (!tasklet_trylock(t))
BUG();
-repeat:
- if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
- BUG();
if (!atomic_read(&t->count)) {
- local_irq_enable();
+ if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
+ BUG();
t->func(t->data);
- local_irq_disable();
- if (test_bit(TASKLET_STATE_SCHED, &t->state))
- goto repeat;
+ tasklet_unlock(t);
+ continue;
}
tasklet_unlock(t);
- if (test_bit(TASKLET_STATE_SCHED, &t->state))
- tasklet_hi_schedule(t);
+
+ local_irq_disable();
+ t->next = tasklet_hi_vec[cpu].list;
+ tasklet_hi_vec[cpu].list = t;
+ cpu_raise_softirq(cpu, HI_SOFTIRQ);
+ local_irq_enable();
}
- local_irq_enable();
}
f(data);
}
}
+
+static int ksoftirqd(void * __bind_cpu)
+{
+ int bind_cpu = *(int *) __bind_cpu;
+ int cpu = cpu_logical_map(bind_cpu);
+
+ daemonize();
+ current->nice = 19;
+ sigfillset(¤t->blocked);
+
+ /* Migrate to the right CPU */
+ current->cpus_allowed = 1UL << cpu;
+ while (smp_processor_id() != cpu)
+ schedule();
+
+ sprintf(current->comm, "ksoftirqd_CPU%d", bind_cpu);
+
+ __set_current_state(TASK_INTERRUPTIBLE);
+ mb();
+
+ ksoftirqd_task(cpu) = current;
+
+ for (;;) {
+ if (!softirq_pending(cpu))
+ schedule();
+
+ __set_current_state(TASK_RUNNING);
+
+ while (softirq_pending(cpu)) {
+ do_softirq();
+ if (current->need_resched)
+ schedule();
+ }
+
+ __set_current_state(TASK_INTERRUPTIBLE);
+ }
+}
+
+static __init int spawn_ksoftirqd(void)
+{
+ int cpu;
+
+ for (cpu = 0; cpu < smp_num_cpus; cpu++) {
+ if (kernel_thread(ksoftirqd, (void *) &cpu,
+ CLONE_FS | CLONE_FILES | CLONE_SIGNAL) < 0)
+ printk("spawn_ksoftirqd() failed for cpu %d\n", cpu);
+ else {
+ while (!ksoftirqd_task(cpu_logical_map(cpu))) {
+ current->policy |= SCHED_YIELD;
+ schedule();
+ }
+ }
+ }
+
+ return 0;
+}
+
+__initcall(spawn_ksoftirqd);
enqueue:
dev_hold(skb->dev);
__skb_queue_tail(&queue->input_pkt_queue,skb);
+
+ /* Runs from irqs or BH's, no need to wake BH */
__cpu_raise_softirq(this_cpu, NET_RX_SOFTIRQ);
local_irq_restore(flags);
#ifndef OFFLINE_SAMPLE
local_irq_disable();
netdev_rx_stat[this_cpu].time_squeeze++;
+
+ /* This already runs in BH context, no need to wake up BH's */
__cpu_raise_softirq(this_cpu, NET_RX_SOFTIRQ);
local_irq_enable();
projects / linux / commitdiff