kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/kernel_stat.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/init.h>
  13 #include <linux/mm.h>
  14 #include <linux/notifier.h>
  15 #include <linux/percpu.h>
  16 #include <linux/cpu.h>
  17 #include <linux/freezer.h>
  18 #include <linux/kthread.h>
  19 #include <linux/rcupdate.h>
  20 #include <linux/smp.h>
  21 #include <linux/tick.h>
  22
  23 #include <asm/irq.h>
  24 /*
  25    - No shared variables, all the data are CPU local.
  26    - If a softirq needs serialization, let it serialize itself
  27      by its own spinlocks.
  28    - Even if softirq is serialized, only local cpu is marked for
  29      execution. Hence, we get something sort of weak cpu binding.
  30      Though it is still not clear, will it result in better locality
  31      or will not.
  32
  33    Examples:
  34    - NET RX softirq. It is multithreaded and does not require
  35      any global serialization.
  36    - NET TX softirq. It kicks software netdevice queues, hence
  37      it is logically serialized per device, but this serialization
  38      is invisible to common code.
  39    - Tasklets: serialized wrt itself.
  40  */
  41
  42 #ifndef __ARCH_IRQ_STAT
  43 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  44 EXPORT_SYMBOL(irq_stat);
  45 #endif
  46
  47 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
  48
  49 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  50
  51 /*
  52  * we cannot loop indefinitely here to avoid userspace starvation,
  53  * but we also don't want to introduce a worst case 1/HZ latency
  54  * to the pending events, so lets the scheduler to balance
  55  * the softirq load for us.
  56  */
  57 static inline void wakeup_softirqd(void)
  58 {
  59         /* Interrupts are disabled: no need to stop preemption */
  60         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  61
  62         if (tsk && tsk->state != TASK_RUNNING)
  63                 wake_up_process(tsk);
  64 }
  65
  66 /*
  67  * This one is for softirq.c-internal use,
  68  * where hardirqs are disabled legitimately:
  69  */
  70 #ifdef CONFIG_TRACE_IRQFLAGS
  71 static void __local_bh_disable(unsigned long ip)
  72 {
  73         unsigned long flags;
  74
  75         WARN_ON_ONCE(in_irq());
  76
  77         raw_local_irq_save(flags);
  78         add_preempt_count(SOFTIRQ_OFFSET);
  79         /*
  80          * Were softirqs turned off above:
  81          */
  82         if (softirq_count() == SOFTIRQ_OFFSET)
  83                 trace_softirqs_off(ip);
  84         raw_local_irq_restore(flags);
  85 }
  86 #else /* !CONFIG_TRACE_IRQFLAGS */
  87 static inline void __local_bh_disable(unsigned long ip)
  88 {
  89         add_preempt_count(SOFTIRQ_OFFSET);
  90         barrier();
  91 }
  92 #endif /* CONFIG_TRACE_IRQFLAGS */
  93
  94 void local_bh_disable(void)
  95 {
  96         __local_bh_disable((unsigned long)__builtin_return_address(0));
  97 }
  98
  99 EXPORT_SYMBOL(local_bh_disable);
 100
 101 void __local_bh_enable(void)
 102 {
 103         WARN_ON_ONCE(in_irq());
 104
 105         /*
 106          * softirqs should never be enabled by __local_bh_enable(),
 107          * it always nests inside local_bh_enable() sections:
 108          */
 109         WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
 110
 111         sub_preempt_count(SOFTIRQ_OFFSET);
 112 }
 113 EXPORT_SYMBOL_GPL(__local_bh_enable);
 114
 115 /*
 116  * Special-case - softirqs can safely be enabled in
 117  * cond_resched_softirq(), or by __do_softirq(),
 118  * without processing still-pending softirqs:
 119  */
 120 void _local_bh_enable(void)
 121 {
 122         WARN_ON_ONCE(in_irq());
 123         WARN_ON_ONCE(!irqs_disabled());
 124
 125         if (softirq_count() == SOFTIRQ_OFFSET)
 126                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 127         sub_preempt_count(SOFTIRQ_OFFSET);
 128 }
 129
 130 EXPORT_SYMBOL(_local_bh_enable);
 131
 132 void local_bh_enable(void)
 133 {
 134 #ifdef CONFIG_TRACE_IRQFLAGS
 135         unsigned long flags;
 136
 137         WARN_ON_ONCE(in_irq());
 138 #endif
 139         WARN_ON_ONCE(irqs_disabled());
 140
 141 #ifdef CONFIG_TRACE_IRQFLAGS
 142         local_irq_save(flags);
 143 #endif
 144         /*
 145          * Are softirqs going to be turned on now:
 146          */
 147         if (softirq_count() == SOFTIRQ_OFFSET)
 148                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 149         /*
 150          * Keep preemption disabled until we are done with
 151          * softirq processing:
 152          */
 153         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 154
 155         if (unlikely(!in_interrupt() && local_softirq_pending()))
 156                 do_softirq();
 157
 158         dec_preempt_count();
 159 #ifdef CONFIG_TRACE_IRQFLAGS
 160         local_irq_restore(flags);
 161 #endif
 162         preempt_check_resched();
 163 }
 164 EXPORT_SYMBOL(local_bh_enable);
 165
 166 void local_bh_enable_ip(unsigned long ip)
 167 {
 168 #ifdef CONFIG_TRACE_IRQFLAGS
 169         unsigned long flags;
 170
 171         WARN_ON_ONCE(in_irq());
 172
 173         local_irq_save(flags);
 174 #endif
 175         /*
 176          * Are softirqs going to be turned on now:
 177          */
 178         if (softirq_count() == SOFTIRQ_OFFSET)
 179                 trace_softirqs_on(ip);
 180         /*
 181          * Keep preemption disabled until we are done with
 182          * softirq processing:
 183          */
 184         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 185
 186         if (unlikely(!in_interrupt() && local_softirq_pending()))
 187                 do_softirq();
 188
 189         dec_preempt_count();
 190 #ifdef CONFIG_TRACE_IRQFLAGS
 191         local_irq_restore(flags);
 192 #endif
 193         preempt_check_resched();
 194 }
 195 EXPORT_SYMBOL(local_bh_enable_ip);
 196
 197 /*
 198  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 199  * and we fall back to softirqd after that.
 200  *
 201  * This number has been established via experimentation.
 202  * The two things to balance is latency against fairness -
 203  * we want to handle softirqs as soon as possible, but they
 204  * should not be able to lock up the box.
 205  */
 206 #define MAX_SOFTIRQ_RESTART 10
 207
 208 asmlinkage void __do_softirq(void)
 209 {
 210         struct softirq_action *h;
 211         __u32 pending;
 212         int max_restart = MAX_SOFTIRQ_RESTART;
 213         int cpu;
 214
 215         pending = local_softirq_pending();
 216         account_system_vtime(current);
 217
 218         __local_bh_disable((unsigned long)__builtin_return_address(0));
 219         trace_softirq_enter();
 220
 221         cpu = smp_processor_id();
 222 restart:
 223         /* Reset the pending bitmask before enabling irqs */
 224         set_softirq_pending(0);
 225
 226         local_irq_enable();
 227
 228         h = softirq_vec;
 229
 230         do {
 231                 if (pending & 1) {
 232                         h->action(h);
 233                         rcu_bh_qsctr_inc(cpu);
 234                 }
 235                 h++;
 236                 pending >>= 1;
 237         } while (pending);
 238
 239         local_irq_disable();
 240
 241         pending = local_softirq_pending();
 242         if (pending && --max_restart)
 243                 goto restart;
 244
 245         if (pending)
 246                 wakeup_softirqd();
 247
 248         trace_softirq_exit();
 249
 250         account_system_vtime(current);
 251         _local_bh_enable();
 252 }
 253
 254 #ifndef __ARCH_HAS_DO_SOFTIRQ
 255
 256 asmlinkage void do_softirq(void)
 257 {
 258         __u32 pending;
 259         unsigned long flags;
 260
 261         if (in_interrupt())
 262                 return;
 263
 264         local_irq_save(flags);
 265
 266         pending = local_softirq_pending();
 267
 268         if (pending)
 269                 __do_softirq();
 270
 271         local_irq_restore(flags);
 272 }
 273
 274 #endif
 275
 276 /*
 277  * Enter an interrupt context.
 278  */
 279 void irq_enter(void)
 280 {
 281 #ifdef CONFIG_NO_HZ
 282         int cpu = smp_processor_id();
 283         if (idle_cpu(cpu) && !in_interrupt())
 284                 tick_nohz_stop_idle(cpu);
 285 #endif
 286         __irq_enter();
 287 #ifdef CONFIG_NO_HZ
 288         if (idle_cpu(cpu))
 289                 tick_nohz_update_jiffies();
 290 #endif
 291 }
 292
 293 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 294 # define invoke_softirq()       __do_softirq()
 295 #else
 296 # define invoke_softirq()       do_softirq()
 297 #endif
 298
 299 /*
 300  * Exit an interrupt context. Process softirqs if needed and possible:
 301  */
 302 void irq_exit(void)
 303 {
 304         account_system_vtime(current);
 305         trace_hardirq_exit();
 306         sub_preempt_count(IRQ_EXIT_OFFSET);
 307         if (!in_interrupt() && local_softirq_pending())
 308                 invoke_softirq();
 309
 310 #ifdef CONFIG_NO_HZ
 311         /* Make sure that timer wheel updates are propagated */
 312         if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
 313                 tick_nohz_stop_sched_tick();
 314 #endif
 315         preempt_enable_no_resched();
 316 }
 317
 318 /*
 319  * This function must run with irqs disabled!
 320  */
 321 inline fastcall void raise_softirq_irqoff(unsigned int nr)
 322 {
 323         __raise_softirq_irqoff(nr);
 324
 325         /*
 326          * If we're in an interrupt or softirq, we're done
 327          * (this also catches softirq-disabled code). We will
 328          * actually run the softirq once we return from
 329          * the irq or softirq.
 330          *
 331          * Otherwise we wake up ksoftirqd to make sure we
 332          * schedule the softirq soon.
 333          */
 334         if (!in_interrupt())
 335                 wakeup_softirqd();
 336 }
 337
 338 void fastcall raise_softirq(unsigned int nr)
 339 {
 340         unsigned long flags;
 341
 342         local_irq_save(flags);
 343         raise_softirq_irqoff(nr);
 344         local_irq_restore(flags);
 345 }
 346
 347 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
 348 {
 349         softirq_vec[nr].data = data;
 350         softirq_vec[nr].action = action;
 351 }
 352
 353 /* Tasklets */
 354 struct tasklet_head
 355 {
 356         struct tasklet_struct *list;
 357 };
 358
 359 /* Some compilers disobey section attribute on statics when not
 360    initialized -- RR */
 361 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
 362 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
 363
 364 void fastcall __tasklet_schedule(struct tasklet_struct *t)
 365 {
 366         unsigned long flags;
 367
 368         local_irq_save(flags);
 369         t->next = __get_cpu_var(tasklet_vec).list;
 370         __get_cpu_var(tasklet_vec).list = t;
 371         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 372         local_irq_restore(flags);
 373 }
 374
 375 EXPORT_SYMBOL(__tasklet_schedule);
 376
 377 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
 378 {
 379         unsigned long flags;
 380
 381         local_irq_save(flags);
 382         t->next = __get_cpu_var(tasklet_hi_vec).list;
 383         __get_cpu_var(tasklet_hi_vec).list = t;
 384         raise_softirq_irqoff(HI_SOFTIRQ);
 385         local_irq_restore(flags);
 386 }
 387
 388 EXPORT_SYMBOL(__tasklet_hi_schedule);
 389
 390 static void tasklet_action(struct softirq_action *a)
 391 {
 392         struct tasklet_struct *list;
 393
 394         local_irq_disable();
 395         list = __get_cpu_var(tasklet_vec).list;
 396         __get_cpu_var(tasklet_vec).list = NULL;
 397         local_irq_enable();
 398
 399         while (list) {
 400                 struct tasklet_struct *t = list;
 401
 402                 list = list->next;
 403
 404                 if (tasklet_trylock(t)) {
 405                         if (!atomic_read(&t->count)) {
 406                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 407                                         BUG();
 408                                 t->func(t->data);
 409                                 tasklet_unlock(t);
 410                                 continue;
 411                         }
 412                         tasklet_unlock(t);
 413                 }
 414
 415                 local_irq_disable();
 416                 t->next = __get_cpu_var(tasklet_vec).list;
 417                 __get_cpu_var(tasklet_vec).list = t;
 418                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 419                 local_irq_enable();
 420         }
 421 }
 422
 423 static void tasklet_hi_action(struct softirq_action *a)
 424 {
 425         struct tasklet_struct *list;
 426
 427         local_irq_disable();
 428         list = __get_cpu_var(tasklet_hi_vec).list;
 429         __get_cpu_var(tasklet_hi_vec).list = NULL;
 430         local_irq_enable();
 431
 432         while (list) {
 433                 struct tasklet_struct *t = list;
 434
 435                 list = list->next;
 436
 437                 if (tasklet_trylock(t)) {
 438                         if (!atomic_read(&t->count)) {
 439                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 440                                         BUG();
 441                                 t->func(t->data);
 442                                 tasklet_unlock(t);
 443                                 continue;
 444                         }
 445                         tasklet_unlock(t);
 446                 }
 447
 448                 local_irq_disable();
 449                 t->next = __get_cpu_var(tasklet_hi_vec).list;
 450                 __get_cpu_var(tasklet_hi_vec).list = t;
 451                 __raise_softirq_irqoff(HI_SOFTIRQ);
 452                 local_irq_enable();
 453         }
 454 }
 455
 456
 457 void tasklet_init(struct tasklet_struct *t,
 458                   void (*func)(unsigned long), unsigned long data)
 459 {
 460         t->next = NULL;
 461         t->state = 0;
 462         atomic_set(&t->count, 0);
 463         t->func = func;
 464         t->data = data;
 465 }
 466
 467 EXPORT_SYMBOL(tasklet_init);
 468
 469 void tasklet_kill(struct tasklet_struct *t)
 470 {
 471         if (in_interrupt())
 472                 printk("Attempt to kill tasklet from interrupt\n");
 473
 474         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 475                 do
 476                         yield();
 477                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 478         }
 479         tasklet_unlock_wait(t);
 480         clear_bit(TASKLET_STATE_SCHED, &t->state);
 481 }
 482
 483 EXPORT_SYMBOL(tasklet_kill);
 484
 485 void __init softirq_init(void)
 486 {
 487         open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
 488         open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
 489 }
 490
 491 static int ksoftirqd(void * __bind_cpu)
 492 {
 493         set_current_state(TASK_INTERRUPTIBLE);
 494
 495         while (!kthread_should_stop()) {
 496                 preempt_disable();
 497                 if (!local_softirq_pending()) {
 498                         preempt_enable_no_resched();
 499                         schedule();
 500                         preempt_disable();
 501                 }
 502
 503                 __set_current_state(TASK_RUNNING);
 504
 505                 while (local_softirq_pending()) {
 506                         /* Preempt disable stops cpu going offline.
 507                            If already offline, we'll be on wrong CPU:
 508                            don't process */
 509                         if (cpu_is_offline((long)__bind_cpu))
 510                                 goto wait_to_die;
 511                         do_softirq();
 512                         preempt_enable_no_resched();
 513                         cond_resched();
 514                         preempt_disable();
 515                 }
 516                 preempt_enable();
 517                 set_current_state(TASK_INTERRUPTIBLE);
 518         }
 519         __set_current_state(TASK_RUNNING);
 520         return 0;
 521
 522 wait_to_die:
 523         preempt_enable();
 524         /* Wait for kthread_stop */
 525         set_current_state(TASK_INTERRUPTIBLE);
 526         while (!kthread_should_stop()) {
 527                 schedule();
 528                 set_current_state(TASK_INTERRUPTIBLE);
 529         }
 530         __set_current_state(TASK_RUNNING);
 531         return 0;
 532 }
 533
 534 #ifdef CONFIG_HOTPLUG_CPU
 535 /*
 536  * tasklet_kill_immediate is called to remove a tasklet which can already be
 537  * scheduled for execution on @cpu.
 538  *
 539  * Unlike tasklet_kill, this function removes the tasklet
 540  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 541  *
 542  * When this function is called, @cpu must be in the CPU_DEAD state.
 543  */
 544 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 545 {
 546         struct tasklet_struct **i;
 547
 548         BUG_ON(cpu_online(cpu));
 549         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 550
 551         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 552                 return;
 553
 554         /* CPU is dead, so no lock needed. */
 555         for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
 556                 if (*i == t) {
 557                         *i = t->next;
 558                         return;
 559                 }
 560         }
 561         BUG();
 562 }
 563
 564 static void takeover_tasklets(unsigned int cpu)
 565 {
 566         struct tasklet_struct **i;
 567
 568         /* CPU is dead, so no lock needed. */
 569         local_irq_disable();
 570
 571         /* Find end, append list for that CPU. */
 572         for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
 573         *i = per_cpu(tasklet_vec, cpu).list;
 574         per_cpu(tasklet_vec, cpu).list = NULL;
 575         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 576
 577         for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
 578         *i = per_cpu(tasklet_hi_vec, cpu).list;
 579         per_cpu(tasklet_hi_vec, cpu).list = NULL;
 580         raise_softirq_irqoff(HI_SOFTIRQ);
 581
 582         local_irq_enable();
 583 }
 584 #endif /* CONFIG_HOTPLUG_CPU */
 585
 586 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 587                                   unsigned long action,
 588                                   void *hcpu)
 589 {
 590         int hotcpu = (unsigned long)hcpu;
 591         struct task_struct *p;
 592
 593         switch (action) {
 594         case CPU_UP_PREPARE:
 595         case CPU_UP_PREPARE_FROZEN:
 596                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 597                 if (IS_ERR(p)) {
 598                         printk("ksoftirqd for %i failed\n", hotcpu);
 599                         return NOTIFY_BAD;
 600                 }
 601                 kthread_bind(p, hotcpu);
 602                 per_cpu(ksoftirqd, hotcpu) = p;
 603                 break;
 604         case CPU_ONLINE:
 605         case CPU_ONLINE_FROZEN:
 606                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 607                 break;
 608 #ifdef CONFIG_HOTPLUG_CPU
 609         case CPU_UP_CANCELED:
 610         case CPU_UP_CANCELED_FROZEN:
 611                 if (!per_cpu(ksoftirqd, hotcpu))
 612                         break;
 613                 /* Unbind so it can run.  Fall thru. */
 614                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 615                              any_online_cpu(cpu_online_map));
 616         case CPU_DEAD:
 617         case CPU_DEAD_FROZEN: {
 618                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 619
 620                 p = per_cpu(ksoftirqd, hotcpu);
 621                 per_cpu(ksoftirqd, hotcpu) = NULL;
 622                 sched_setscheduler(p, SCHED_FIFO, &param);
 623                 kthread_stop(p);
 624                 takeover_tasklets(hotcpu);
 625                 break;
 626         }
 627 #endif /* CONFIG_HOTPLUG_CPU */
 628         }
 629         return NOTIFY_OK;
 630 }
 631
 632 static struct notifier_block __cpuinitdata cpu_nfb = {
 633         .notifier_call = cpu_callback
 634 };
 635
 636 __init int spawn_ksoftirqd(void)
 637 {
 638         void *cpu = (void *)(long)smp_processor_id();
 639         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 640
 641         BUG_ON(err == NOTIFY_BAD);
 642         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 643         register_cpu_notifier(&cpu_nfb);
 644         return 0;
 645 }
 646
 647 #ifdef CONFIG_SMP
 648 /*
 649  * Call a function on all processors
 650  */
 651 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
 652 {
 653         int ret = 0;
 654
 655         preempt_disable();
 656         ret = smp_call_function(func, info, retry, wait);
 657         local_irq_disable();
 658         func(info);
 659         local_irq_enable();
 660         preempt_enable();
 661         return ret;
 662 }
 663 EXPORT_SYMBOL(on_each_cpu);
 664 #endif