kernel/rcu/tiny.c

   1 /*
   2  * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, you can access it online at
  16  * http://www.gnu.org/licenses/gpl-2.0.html.
  17  *
  18  * Copyright IBM Corporation, 2008
  19  *
  20  * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
  21  *
  22  * For detailed explanation of Read-Copy Update mechanism see -
  23  *              Documentation/RCU
  24  */
  25 #include <linux/completion.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/notifier.h>
  28 #include <linux/rcupdate_wait.h>
  29 #include <linux/kernel.h>
  30 #include <linux/export.h>
  31 #include <linux/mutex.h>
  32 #include <linux/sched.h>
  33 #include <linux/types.h>
  34 #include <linux/init.h>
  35 #include <linux/time.h>
  36 #include <linux/cpu.h>
  37 #include <linux/prefetch.h>
  38
  39 #include "rcu.h"
  40
  41 /* Global control variables for rcupdate callback mechanism. */
  42 struct rcu_ctrlblk {
  43         struct rcu_head *rcucblist;     /* List of pending callbacks (CBs). */
  44         struct rcu_head **donetail;     /* ->next pointer of last "done" CB. */
  45         struct rcu_head **curtail;      /* ->next pointer of last CB. */
  46 };
  47
  48 /* Definition for rcupdate control block. */
  49 static struct rcu_ctrlblk rcu_ctrlblk = {
  50         .donetail       = &rcu_ctrlblk.rcucblist,
  51         .curtail        = &rcu_ctrlblk.rcucblist,
  52 };
  53
  54 void rcu_barrier(void)
  55 {
  56         wait_rcu_gp(call_rcu);
  57 }
  58 EXPORT_SYMBOL(rcu_barrier);
  59
  60 /* Record an rcu quiescent state.  */
  61 void rcu_qs(void)
  62 {
  63         unsigned long flags;
  64
  65         local_irq_save(flags);
  66         if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) {
  67                 rcu_ctrlblk.donetail = rcu_ctrlblk.curtail;
  68                 raise_softirq(RCU_SOFTIRQ);
  69         }
  70         local_irq_restore(flags);
  71 }
  72
  73 /*
  74  * Check to see if the scheduling-clock interrupt came from an extended
  75  * quiescent state, and, if so, tell RCU about it.  This function must
  76  * be called from hardirq context.  It is normally called from the
  77  * scheduling-clock interrupt.
  78  */
  79 void rcu_check_callbacks(int user)
  80 {
  81         if (user) {
  82                 rcu_qs();
  83         } else if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) {
  84                 set_tsk_need_resched(current);
  85                 set_preempt_need_resched();
  86         }
  87 }
  88
  89 /* Invoke the RCU callbacks whose grace period has elapsed.  */
  90 static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused)
  91 {
  92         struct rcu_head *next, *list;
  93         unsigned long flags;
  94
  95         /* Move the ready-to-invoke callbacks to a local list. */
  96         local_irq_save(flags);
  97         if (rcu_ctrlblk.donetail == &rcu_ctrlblk.rcucblist) {
  98                 /* No callbacks ready, so just leave. */
  99                 local_irq_restore(flags);
 100                 return;
 101         }
 102         list = rcu_ctrlblk.rcucblist;
 103         rcu_ctrlblk.rcucblist = *rcu_ctrlblk.donetail;
 104         *rcu_ctrlblk.donetail = NULL;
 105         if (rcu_ctrlblk.curtail == rcu_ctrlblk.donetail)
 106                 rcu_ctrlblk.curtail = &rcu_ctrlblk.rcucblist;
 107         rcu_ctrlblk.donetail = &rcu_ctrlblk.rcucblist;
 108         local_irq_restore(flags);
 109
 110         /* Invoke the callbacks on the local list. */
 111         while (list) {
 112                 next = list->next;
 113                 prefetch(next);
 114                 debug_rcu_head_unqueue(list);
 115                 local_bh_disable();
 116                 __rcu_reclaim("", list);
 117                 local_bh_enable();
 118                 list = next;
 119         }
 120 }
 121
 122 /*
 123  * Wait for a grace period to elapse.  But it is illegal to invoke
 124  * synchronize_rcu() from within an RCU read-side critical section.
 125  * Therefore, any legal call to synchronize_rcu() is a quiescent
 126  * state, and so on a UP system, synchronize_rcu() need do nothing.
 127  * (But Lai Jiangshan points out the benefits of doing might_sleep()
 128  * to reduce latency.)
 129  *
 130  * Cool, huh?  (Due to Josh Triplett.)
 131  */
 132 void synchronize_rcu(void)
 133 {
 134         RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
 135                          lock_is_held(&rcu_lock_map) ||
 136                          lock_is_held(&rcu_sched_lock_map),
 137                          "Illegal synchronize_rcu() in RCU read-side critical section");
 138 }
 139 EXPORT_SYMBOL_GPL(synchronize_rcu);
 140
 141 /*
 142  * Post an RCU callback to be invoked after the end of an RCU grace
 143  * period.  But since we have but one CPU, that would be after any
 144  * quiescent state.
 145  */
 146 void call_rcu(struct rcu_head *head, rcu_callback_t func)
 147 {
 148         unsigned long flags;
 149
 150         debug_rcu_head_queue(head);
 151         head->func = func;
 152         head->next = NULL;
 153
 154         local_irq_save(flags);
 155         *rcu_ctrlblk.curtail = head;
 156         rcu_ctrlblk.curtail = &head->next;
 157         local_irq_restore(flags);
 158
 159         if (unlikely(is_idle_task(current))) {
 160                 /* force scheduling for rcu_qs() */
 161                 resched_cpu(0);
 162         }
 163 }
 164 EXPORT_SYMBOL_GPL(call_rcu);
 165
 166 void __init rcu_init(void)
 167 {
 168         open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
 169         rcu_early_boot_tests();
 170         srcu_init();
 171 }