powerpc/pmac: Add missing unlocks in error path
[linux-2.6/next.git] / kernel / softirq.c
blob7c1a67ef027431e8f82d341ed7e729b9c0eec044
1 /*
2 * linux/kernel/softirq.c
4 * Copyright (C) 1992 Linus Torvalds
6 * Distribute under GPLv2.
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
10 * Remote softirq infrastructure is by Jens Axboe.
13 #include <linux/module.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/init.h>
17 #include <linux/mm.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/ftrace.h>
25 #include <linux/smp.h>
26 #include <linux/tick.h>
28 #define CREATE_TRACE_POINTS
29 #include <trace/events/irq.h>
31 #include <asm/irq.h>
33 - No shared variables, all the data are CPU local.
34 - If a softirq needs serialization, let it serialize itself
35 by its own spinlocks.
36 - Even if softirq is serialized, only local cpu is marked for
37 execution. Hence, we get something sort of weak cpu binding.
38 Though it is still not clear, will it result in better locality
39 or will not.
41 Examples:
42 - NET RX softirq. It is multithreaded and does not require
43 any global serialization.
44 - NET TX softirq. It kicks software netdevice queues, hence
45 it is logically serialized per device, but this serialization
46 is invisible to common code.
47 - Tasklets: serialized wrt itself.
50 #ifndef __ARCH_IRQ_STAT
51 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
52 EXPORT_SYMBOL(irq_stat);
53 #endif
55 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
57 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
59 char *softirq_to_name[NR_SOFTIRQS] = {
60 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
61 "TASKLET", "SCHED", "HRTIMER", "RCU"
65 * we cannot loop indefinitely here to avoid userspace starvation,
66 * but we also don't want to introduce a worst case 1/HZ latency
67 * to the pending events, so lets the scheduler to balance
68 * the softirq load for us.
70 void wakeup_softirqd(void)
72 /* Interrupts are disabled: no need to stop preemption */
73 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
75 if (tsk && tsk->state != TASK_RUNNING)
76 wake_up_process(tsk);
80 * This one is for softirq.c-internal use,
81 * where hardirqs are disabled legitimately:
83 #ifdef CONFIG_TRACE_IRQFLAGS
84 static void __local_bh_disable(unsigned long ip)
86 unsigned long flags;
88 WARN_ON_ONCE(in_irq());
90 raw_local_irq_save(flags);
92 * The preempt tracer hooks into add_preempt_count and will break
93 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
94 * is set and before current->softirq_enabled is cleared.
95 * We must manually increment preempt_count here and manually
96 * call the trace_preempt_off later.
98 preempt_count() += SOFTIRQ_OFFSET;
100 * Were softirqs turned off above:
102 if (softirq_count() == SOFTIRQ_OFFSET)
103 trace_softirqs_off(ip);
104 raw_local_irq_restore(flags);
106 if (preempt_count() == SOFTIRQ_OFFSET)
107 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
109 #else /* !CONFIG_TRACE_IRQFLAGS */
110 static inline void __local_bh_disable(unsigned long ip)
112 add_preempt_count(SOFTIRQ_OFFSET);
113 barrier();
115 #endif /* CONFIG_TRACE_IRQFLAGS */
117 void local_bh_disable(void)
119 __local_bh_disable((unsigned long)__builtin_return_address(0));
122 EXPORT_SYMBOL(local_bh_disable);
125 * Special-case - softirqs can safely be enabled in
126 * cond_resched_softirq(), or by __do_softirq(),
127 * without processing still-pending softirqs:
129 void _local_bh_enable(void)
131 WARN_ON_ONCE(in_irq());
132 WARN_ON_ONCE(!irqs_disabled());
134 if (softirq_count() == SOFTIRQ_OFFSET)
135 trace_softirqs_on((unsigned long)__builtin_return_address(0));
136 sub_preempt_count(SOFTIRQ_OFFSET);
139 EXPORT_SYMBOL(_local_bh_enable);
141 static inline void _local_bh_enable_ip(unsigned long ip)
143 WARN_ON_ONCE(in_irq() || irqs_disabled());
144 #ifdef CONFIG_TRACE_IRQFLAGS
145 local_irq_disable();
146 #endif
148 * Are softirqs going to be turned on now:
150 if (softirq_count() == SOFTIRQ_OFFSET)
151 trace_softirqs_on(ip);
153 * Keep preemption disabled until we are done with
154 * softirq processing:
156 sub_preempt_count(SOFTIRQ_OFFSET - 1);
158 if (unlikely(!in_interrupt() && local_softirq_pending()))
159 do_softirq();
161 dec_preempt_count();
162 #ifdef CONFIG_TRACE_IRQFLAGS
163 local_irq_enable();
164 #endif
165 preempt_check_resched();
168 void local_bh_enable(void)
170 _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
172 EXPORT_SYMBOL(local_bh_enable);
174 void local_bh_enable_ip(unsigned long ip)
176 _local_bh_enable_ip(ip);
178 EXPORT_SYMBOL(local_bh_enable_ip);
181 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
182 * and we fall back to softirqd after that.
184 * This number has been established via experimentation.
185 * The two things to balance is latency against fairness -
186 * we want to handle softirqs as soon as possible, but they
187 * should not be able to lock up the box.
189 #define MAX_SOFTIRQ_RESTART 10
191 asmlinkage void __do_softirq(void)
193 struct softirq_action *h;
194 __u32 pending;
195 int max_restart = MAX_SOFTIRQ_RESTART;
196 int cpu;
198 pending = local_softirq_pending();
199 account_system_vtime(current);
201 __local_bh_disable((unsigned long)__builtin_return_address(0));
202 lockdep_softirq_enter();
204 cpu = smp_processor_id();
205 restart:
206 /* Reset the pending bitmask before enabling irqs */
207 set_softirq_pending(0);
209 local_irq_enable();
211 h = softirq_vec;
213 do {
214 if (pending & 1) {
215 int prev_count = preempt_count();
216 kstat_incr_softirqs_this_cpu(h - softirq_vec);
218 trace_softirq_entry(h, softirq_vec);
219 h->action(h);
220 trace_softirq_exit(h, softirq_vec);
221 if (unlikely(prev_count != preempt_count())) {
222 printk(KERN_ERR "huh, entered softirq %td %s %p"
223 "with preempt_count %08x,"
224 " exited with %08x?\n", h - softirq_vec,
225 softirq_to_name[h - softirq_vec],
226 h->action, prev_count, preempt_count());
227 preempt_count() = prev_count;
230 rcu_bh_qs(cpu);
232 h++;
233 pending >>= 1;
234 } while (pending);
236 local_irq_disable();
238 pending = local_softirq_pending();
239 if (pending && --max_restart)
240 goto restart;
242 if (pending)
243 wakeup_softirqd();
245 lockdep_softirq_exit();
247 account_system_vtime(current);
248 _local_bh_enable();
251 #ifndef __ARCH_HAS_DO_SOFTIRQ
253 asmlinkage void do_softirq(void)
255 __u32 pending;
256 unsigned long flags;
258 if (in_interrupt())
259 return;
261 local_irq_save(flags);
263 pending = local_softirq_pending();
265 if (pending)
266 __do_softirq();
268 local_irq_restore(flags);
271 #endif
274 * Enter an interrupt context.
276 void irq_enter(void)
278 int cpu = smp_processor_id();
280 rcu_irq_enter();
281 if (idle_cpu(cpu) && !in_interrupt()) {
282 __irq_enter();
283 tick_check_idle(cpu);
284 } else
285 __irq_enter();
288 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
289 # define invoke_softirq() __do_softirq()
290 #else
291 # define invoke_softirq() do_softirq()
292 #endif
295 * Exit an interrupt context. Process softirqs if needed and possible:
297 void irq_exit(void)
299 account_system_vtime(current);
300 trace_hardirq_exit();
301 sub_preempt_count(IRQ_EXIT_OFFSET);
302 if (!in_interrupt() && local_softirq_pending())
303 invoke_softirq();
305 rcu_irq_exit();
306 #ifdef CONFIG_NO_HZ
307 /* Make sure that timer wheel updates are propagated */
308 if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
309 tick_nohz_stop_sched_tick(0);
310 #endif
311 preempt_enable_no_resched();
315 * This function must run with irqs disabled!
317 inline void raise_softirq_irqoff(unsigned int nr)
319 __raise_softirq_irqoff(nr);
322 * If we're in an interrupt or softirq, we're done
323 * (this also catches softirq-disabled code). We will
324 * actually run the softirq once we return from
325 * the irq or softirq.
327 * Otherwise we wake up ksoftirqd to make sure we
328 * schedule the softirq soon.
330 if (!in_interrupt())
331 wakeup_softirqd();
334 void raise_softirq(unsigned int nr)
336 unsigned long flags;
338 local_irq_save(flags);
339 raise_softirq_irqoff(nr);
340 local_irq_restore(flags);
343 void open_softirq(int nr, void (*action)(struct softirq_action *))
345 softirq_vec[nr].action = action;
349 * Tasklets
351 struct tasklet_head
353 struct tasklet_struct *head;
354 struct tasklet_struct **tail;
357 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
358 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
360 void __tasklet_schedule(struct tasklet_struct *t)
362 unsigned long flags;
364 local_irq_save(flags);
365 t->next = NULL;
366 *__get_cpu_var(tasklet_vec).tail = t;
367 __get_cpu_var(tasklet_vec).tail = &(t->next);
368 raise_softirq_irqoff(TASKLET_SOFTIRQ);
369 local_irq_restore(flags);
372 EXPORT_SYMBOL(__tasklet_schedule);
374 void __tasklet_hi_schedule(struct tasklet_struct *t)
376 unsigned long flags;
378 local_irq_save(flags);
379 t->next = NULL;
380 *__get_cpu_var(tasklet_hi_vec).tail = t;
381 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
382 raise_softirq_irqoff(HI_SOFTIRQ);
383 local_irq_restore(flags);
386 EXPORT_SYMBOL(__tasklet_hi_schedule);
388 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
390 BUG_ON(!irqs_disabled());
392 t->next = __get_cpu_var(tasklet_hi_vec).head;
393 __get_cpu_var(tasklet_hi_vec).head = t;
394 __raise_softirq_irqoff(HI_SOFTIRQ);
397 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
399 static void tasklet_action(struct softirq_action *a)
401 struct tasklet_struct *list;
403 local_irq_disable();
404 list = __get_cpu_var(tasklet_vec).head;
405 __get_cpu_var(tasklet_vec).head = NULL;
406 __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
407 local_irq_enable();
409 while (list) {
410 struct tasklet_struct *t = list;
412 list = list->next;
414 if (tasklet_trylock(t)) {
415 if (!atomic_read(&t->count)) {
416 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
417 BUG();
418 t->func(t->data);
419 tasklet_unlock(t);
420 continue;
422 tasklet_unlock(t);
425 local_irq_disable();
426 t->next = NULL;
427 *__get_cpu_var(tasklet_vec).tail = t;
428 __get_cpu_var(tasklet_vec).tail = &(t->next);
429 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
430 local_irq_enable();
434 static void tasklet_hi_action(struct softirq_action *a)
436 struct tasklet_struct *list;
438 local_irq_disable();
439 list = __get_cpu_var(tasklet_hi_vec).head;
440 __get_cpu_var(tasklet_hi_vec).head = NULL;
441 __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
442 local_irq_enable();
444 while (list) {
445 struct tasklet_struct *t = list;
447 list = list->next;
449 if (tasklet_trylock(t)) {
450 if (!atomic_read(&t->count)) {
451 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
452 BUG();
453 t->func(t->data);
454 tasklet_unlock(t);
455 continue;
457 tasklet_unlock(t);
460 local_irq_disable();
461 t->next = NULL;
462 *__get_cpu_var(tasklet_hi_vec).tail = t;
463 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
464 __raise_softirq_irqoff(HI_SOFTIRQ);
465 local_irq_enable();
470 void tasklet_init(struct tasklet_struct *t,
471 void (*func)(unsigned long), unsigned long data)
473 t->next = NULL;
474 t->state = 0;
475 atomic_set(&t->count, 0);
476 t->func = func;
477 t->data = data;
480 EXPORT_SYMBOL(tasklet_init);
482 void tasklet_kill(struct tasklet_struct *t)
484 if (in_interrupt())
485 printk("Attempt to kill tasklet from interrupt\n");
487 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
488 do {
489 yield();
490 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
492 tasklet_unlock_wait(t);
493 clear_bit(TASKLET_STATE_SCHED, &t->state);
496 EXPORT_SYMBOL(tasklet_kill);
499 * tasklet_hrtimer
503 * The trampoline is called when the hrtimer expires. It schedules a tasklet
504 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
505 * hrtimer callback, but from softirq context.
507 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
509 struct tasklet_hrtimer *ttimer =
510 container_of(timer, struct tasklet_hrtimer, timer);
512 tasklet_hi_schedule(&ttimer->tasklet);
513 return HRTIMER_NORESTART;
517 * Helper function which calls the hrtimer callback from
518 * tasklet/softirq context
520 static void __tasklet_hrtimer_trampoline(unsigned long data)
522 struct tasklet_hrtimer *ttimer = (void *)data;
523 enum hrtimer_restart restart;
525 restart = ttimer->function(&ttimer->timer);
526 if (restart != HRTIMER_NORESTART)
527 hrtimer_restart(&ttimer->timer);
531 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
532 * @ttimer: tasklet_hrtimer which is initialized
533 * @function: hrtimer callback funtion which gets called from softirq context
534 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
535 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
537 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
538 enum hrtimer_restart (*function)(struct hrtimer *),
539 clockid_t which_clock, enum hrtimer_mode mode)
541 hrtimer_init(&ttimer->timer, which_clock, mode);
542 ttimer->timer.function = __hrtimer_tasklet_trampoline;
543 tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
544 (unsigned long)ttimer);
545 ttimer->function = function;
547 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
550 * Remote softirq bits
553 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
554 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
556 static void __local_trigger(struct call_single_data *cp, int softirq)
558 struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
560 list_add_tail(&cp->list, head);
562 /* Trigger the softirq only if the list was previously empty. */
563 if (head->next == &cp->list)
564 raise_softirq_irqoff(softirq);
567 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
568 static void remote_softirq_receive(void *data)
570 struct call_single_data *cp = data;
571 unsigned long flags;
572 int softirq;
574 softirq = cp->priv;
576 local_irq_save(flags);
577 __local_trigger(cp, softirq);
578 local_irq_restore(flags);
581 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
583 if (cpu_online(cpu)) {
584 cp->func = remote_softirq_receive;
585 cp->info = cp;
586 cp->flags = 0;
587 cp->priv = softirq;
589 __smp_call_function_single(cpu, cp, 0);
590 return 0;
592 return 1;
594 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
595 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
597 return 1;
599 #endif
602 * __send_remote_softirq - try to schedule softirq work on a remote cpu
603 * @cp: private SMP call function data area
604 * @cpu: the remote cpu
605 * @this_cpu: the currently executing cpu
606 * @softirq: the softirq for the work
608 * Attempt to schedule softirq work on a remote cpu. If this cannot be
609 * done, the work is instead queued up on the local cpu.
611 * Interrupts must be disabled.
613 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
615 if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
616 __local_trigger(cp, softirq);
618 EXPORT_SYMBOL(__send_remote_softirq);
621 * send_remote_softirq - try to schedule softirq work on a remote cpu
622 * @cp: private SMP call function data area
623 * @cpu: the remote cpu
624 * @softirq: the softirq for the work
626 * Like __send_remote_softirq except that disabling interrupts and
627 * computing the current cpu is done for the caller.
629 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
631 unsigned long flags;
632 int this_cpu;
634 local_irq_save(flags);
635 this_cpu = smp_processor_id();
636 __send_remote_softirq(cp, cpu, this_cpu, softirq);
637 local_irq_restore(flags);
639 EXPORT_SYMBOL(send_remote_softirq);
641 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
642 unsigned long action, void *hcpu)
645 * If a CPU goes away, splice its entries to the current CPU
646 * and trigger a run of the softirq
648 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
649 int cpu = (unsigned long) hcpu;
650 int i;
652 local_irq_disable();
653 for (i = 0; i < NR_SOFTIRQS; i++) {
654 struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
655 struct list_head *local_head;
657 if (list_empty(head))
658 continue;
660 local_head = &__get_cpu_var(softirq_work_list[i]);
661 list_splice_init(head, local_head);
662 raise_softirq_irqoff(i);
664 local_irq_enable();
667 return NOTIFY_OK;
670 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
671 .notifier_call = remote_softirq_cpu_notify,
674 void __init softirq_init(void)
676 int cpu;
678 for_each_possible_cpu(cpu) {
679 int i;
681 per_cpu(tasklet_vec, cpu).tail =
682 &per_cpu(tasklet_vec, cpu).head;
683 per_cpu(tasklet_hi_vec, cpu).tail =
684 &per_cpu(tasklet_hi_vec, cpu).head;
685 for (i = 0; i < NR_SOFTIRQS; i++)
686 INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
689 register_hotcpu_notifier(&remote_softirq_cpu_notifier);
691 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
692 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
695 static int run_ksoftirqd(void * __bind_cpu)
697 set_current_state(TASK_INTERRUPTIBLE);
699 while (!kthread_should_stop()) {
700 preempt_disable();
701 if (!local_softirq_pending()) {
702 preempt_enable_no_resched();
703 schedule();
704 preempt_disable();
707 __set_current_state(TASK_RUNNING);
709 while (local_softirq_pending()) {
710 /* Preempt disable stops cpu going offline.
711 If already offline, we'll be on wrong CPU:
712 don't process */
713 if (cpu_is_offline((long)__bind_cpu))
714 goto wait_to_die;
715 do_softirq();
716 preempt_enable_no_resched();
717 cond_resched();
718 preempt_disable();
719 rcu_sched_qs((long)__bind_cpu);
721 preempt_enable();
722 set_current_state(TASK_INTERRUPTIBLE);
724 __set_current_state(TASK_RUNNING);
725 return 0;
727 wait_to_die:
728 preempt_enable();
729 /* Wait for kthread_stop */
730 set_current_state(TASK_INTERRUPTIBLE);
731 while (!kthread_should_stop()) {
732 schedule();
733 set_current_state(TASK_INTERRUPTIBLE);
735 __set_current_state(TASK_RUNNING);
736 return 0;
739 #ifdef CONFIG_HOTPLUG_CPU
741 * tasklet_kill_immediate is called to remove a tasklet which can already be
742 * scheduled for execution on @cpu.
744 * Unlike tasklet_kill, this function removes the tasklet
745 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
747 * When this function is called, @cpu must be in the CPU_DEAD state.
749 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
751 struct tasklet_struct **i;
753 BUG_ON(cpu_online(cpu));
754 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
756 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
757 return;
759 /* CPU is dead, so no lock needed. */
760 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
761 if (*i == t) {
762 *i = t->next;
763 /* If this was the tail element, move the tail ptr */
764 if (*i == NULL)
765 per_cpu(tasklet_vec, cpu).tail = i;
766 return;
769 BUG();
772 static void takeover_tasklets(unsigned int cpu)
774 /* CPU is dead, so no lock needed. */
775 local_irq_disable();
777 /* Find end, append list for that CPU. */
778 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
779 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
780 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
781 per_cpu(tasklet_vec, cpu).head = NULL;
782 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
784 raise_softirq_irqoff(TASKLET_SOFTIRQ);
786 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
787 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
788 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
789 per_cpu(tasklet_hi_vec, cpu).head = NULL;
790 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
792 raise_softirq_irqoff(HI_SOFTIRQ);
794 local_irq_enable();
796 #endif /* CONFIG_HOTPLUG_CPU */
798 static int __cpuinit cpu_callback(struct notifier_block *nfb,
799 unsigned long action,
800 void *hcpu)
802 int hotcpu = (unsigned long)hcpu;
803 struct task_struct *p;
805 switch (action) {
806 case CPU_UP_PREPARE:
807 case CPU_UP_PREPARE_FROZEN:
808 p = kthread_create(run_ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
809 if (IS_ERR(p)) {
810 printk("ksoftirqd for %i failed\n", hotcpu);
811 return NOTIFY_BAD;
813 kthread_bind(p, hotcpu);
814 per_cpu(ksoftirqd, hotcpu) = p;
815 break;
816 case CPU_ONLINE:
817 case CPU_ONLINE_FROZEN:
818 wake_up_process(per_cpu(ksoftirqd, hotcpu));
819 break;
820 #ifdef CONFIG_HOTPLUG_CPU
821 case CPU_UP_CANCELED:
822 case CPU_UP_CANCELED_FROZEN:
823 if (!per_cpu(ksoftirqd, hotcpu))
824 break;
825 /* Unbind so it can run. Fall thru. */
826 kthread_bind(per_cpu(ksoftirqd, hotcpu),
827 cpumask_any(cpu_online_mask));
828 case CPU_DEAD:
829 case CPU_DEAD_FROZEN: {
830 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
832 p = per_cpu(ksoftirqd, hotcpu);
833 per_cpu(ksoftirqd, hotcpu) = NULL;
834 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
835 kthread_stop(p);
836 takeover_tasklets(hotcpu);
837 break;
839 #endif /* CONFIG_HOTPLUG_CPU */
841 return NOTIFY_OK;
844 static struct notifier_block __cpuinitdata cpu_nfb = {
845 .notifier_call = cpu_callback
848 static __init int spawn_ksoftirqd(void)
850 void *cpu = (void *)(long)smp_processor_id();
851 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
853 BUG_ON(err == NOTIFY_BAD);
854 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
855 register_cpu_notifier(&cpu_nfb);
856 return 0;
858 early_initcall(spawn_ksoftirqd);
860 #ifdef CONFIG_SMP
862 * Call a function on all processors
864 int on_each_cpu(void (*func) (void *info), void *info, int wait)
866 int ret = 0;
868 preempt_disable();
869 ret = smp_call_function(func, info, wait);
870 local_irq_disable();
871 func(info);
872 local_irq_enable();
873 preempt_enable();
874 return ret;
876 EXPORT_SYMBOL(on_each_cpu);
877 #endif
880 * [ These __weak aliases are kept in a separate compilation unit, so that
881 * GCC does not inline them incorrectly. ]
884 int __init __weak early_irq_init(void)
886 return 0;
889 int __init __weak arch_probe_nr_irqs(void)
891 return 0;
894 int __init __weak arch_early_irq_init(void)
896 return 0;
899 int __weak arch_init_chip_data(struct irq_desc *desc, int node)
901 return 0;