kernel/sched/wait_bit.c

   1 /*
   2  * The implementation of the wait_bit*() and related waiting APIs:
   3  */
   4 #include "sched.h"
   5
   6 #define WAIT_TABLE_BITS 8
   7 #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)
   8
   9 static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;
  10
  11 wait_queue_head_t *bit_waitqueue(void *word, int bit)
  12 {
  13         const int shift = BITS_PER_LONG == 32 ? 5 : 6;
  14         unsigned long val = (unsigned long)word << shift | bit;
  15
  16         return bit_wait_table + hash_long(val, WAIT_TABLE_BITS);
  17 }
  18 EXPORT_SYMBOL(bit_waitqueue);
  19
  20 int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg)
  21 {
  22         struct wait_bit_key *key = arg;
  23         struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
  24
  25         if (wait_bit->key.flags != key->flags ||
  26                         wait_bit->key.bit_nr != key->bit_nr ||
  27                         test_bit(key->bit_nr, key->flags))
  28                 return 0;
  29
  30         return autoremove_wake_function(wq_entry, mode, sync, key);
  31 }
  32 EXPORT_SYMBOL(wake_bit_function);
  33
  34 /*
  35  * To allow interruptible waiting and asynchronous (i.e. nonblocking)
  36  * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are
  37  * permitted return codes. Nonzero return codes halt waiting and return.
  38  */
  39 int __sched
  40 __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
  41               wait_bit_action_f *action, unsigned mode)
  42 {
  43         int ret = 0;
  44
  45         do {
  46                 prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
  47                 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags))
  48                         ret = (*action)(&wbq_entry->key, mode);
  49         } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);
  50
  51         finish_wait(wq_head, &wbq_entry->wq_entry);
  52
  53         return ret;
  54 }
  55 EXPORT_SYMBOL(__wait_on_bit);
  56
  57 int __sched out_of_line_wait_on_bit(void *word, int bit,
  58                                     wait_bit_action_f *action, unsigned mode)
  59 {
  60         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
  61         DEFINE_WAIT_BIT(wq_entry, word, bit);
  62
  63         return __wait_on_bit(wq_head, &wq_entry, action, mode);
  64 }
  65 EXPORT_SYMBOL(out_of_line_wait_on_bit);
  66
  67 int __sched out_of_line_wait_on_bit_timeout(
  68         void *word, int bit, wait_bit_action_f *action,
  69         unsigned mode, unsigned long timeout)
  70 {
  71         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
  72         DEFINE_WAIT_BIT(wq_entry, word, bit);
  73
  74         wq_entry.key.timeout = jiffies + timeout;
  75
  76         return __wait_on_bit(wq_head, &wq_entry, action, mode);
  77 }
  78 EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout);
  79
  80 int __sched
  81 __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
  82                         wait_bit_action_f *action, unsigned mode)
  83 {
  84         int ret = 0;
  85
  86         for (;;) {
  87                 prepare_to_wait_exclusive(wq_head, &wbq_entry->wq_entry, mode);
  88                 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
  89                         ret = action(&wbq_entry->key, mode);
  90                         /*
  91                          * See the comment in prepare_to_wait_event().
  92                          * finish_wait() does not necessarily takes wwq_head->lock,
  93                          * but test_and_set_bit() implies mb() which pairs with
  94                          * smp_mb__after_atomic() before wake_up_page().
  95                          */
  96                         if (ret)
  97                                 finish_wait(wq_head, &wbq_entry->wq_entry);
  98                 }
  99                 if (!test_and_set_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
 100                         if (!ret)
 101                                 finish_wait(wq_head, &wbq_entry->wq_entry);
 102                         return 0;
 103                 } else if (ret) {
 104                         return ret;
 105                 }
 106         }
 107 }
 108 EXPORT_SYMBOL(__wait_on_bit_lock);
 109
 110 int __sched out_of_line_wait_on_bit_lock(void *word, int bit,
 111                                          wait_bit_action_f *action, unsigned mode)
 112 {
 113         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
 114         DEFINE_WAIT_BIT(wq_entry, word, bit);
 115
 116         return __wait_on_bit_lock(wq_head, &wq_entry, action, mode);
 117 }
 118 EXPORT_SYMBOL(out_of_line_wait_on_bit_lock);
 119
 120 void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit)
 121 {
 122         struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
 123
 124         if (waitqueue_active(wq_head))
 125                 __wake_up(wq_head, TASK_NORMAL, 1, &key);
 126 }
 127 EXPORT_SYMBOL(__wake_up_bit);
 128
 129 /**
 130  * wake_up_bit - wake up a waiter on a bit
 131  * @word: the word being waited on, a kernel virtual address
 132  * @bit: the bit of the word being waited on
 133  *
 134  * There is a standard hashed waitqueue table for generic use. This
 135  * is the part of the hashtable's accessor API that wakes up waiters
 136  * on a bit. For instance, if one were to have waiters on a bitflag,
 137  * one would call wake_up_bit() after clearing the bit.
 138  *
 139  * In order for this to function properly, as it uses waitqueue_active()
 140  * internally, some kind of memory barrier must be done prior to calling
 141  * this. Typically, this will be smp_mb__after_atomic(), but in some
 142  * cases where bitflags are manipulated non-atomically under a lock, one
 143  * may need to use a less regular barrier, such fs/inode.c's smp_mb(),
 144  * because spin_unlock() does not guarantee a memory barrier.
 145  */
 146 void wake_up_bit(void *word, int bit)
 147 {
 148         __wake_up_bit(bit_waitqueue(word, bit), word, bit);
 149 }
 150 EXPORT_SYMBOL(wake_up_bit);
 151
 152 wait_queue_head_t *__var_waitqueue(void *p)
 153 {
 154         return bit_wait_table + hash_ptr(p, WAIT_TABLE_BITS);
 155 }
 156 EXPORT_SYMBOL(__var_waitqueue);
 157
 158 static int
 159 var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode,
 160                   int sync, void *arg)
 161 {
 162         struct wait_bit_key *key = arg;
 163         struct wait_bit_queue_entry *wbq_entry =
 164                 container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
 165
 166         if (wbq_entry->key.flags != key->flags ||
 167             wbq_entry->key.bit_nr != key->bit_nr)
 168                 return 0;
 169
 170         return autoremove_wake_function(wq_entry, mode, sync, key);
 171 }
 172
 173 void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, void *var, int flags)
 174 {
 175         *wbq_entry = (struct wait_bit_queue_entry){
 176                 .key = {
 177                         .flags  = (var),
 178                         .bit_nr = -1,
 179                 },
 180                 .wq_entry = {
 181                         .private = current,
 182                         .func    = var_wake_function,
 183                         .entry   = LIST_HEAD_INIT(wbq_entry->wq_entry.entry),
 184                 },
 185         };
 186 }
 187 EXPORT_SYMBOL(init_wait_var_entry);
 188
 189 void wake_up_var(void *var)
 190 {
 191         __wake_up_bit(__var_waitqueue(var), var, -1);
 192 }
 193 EXPORT_SYMBOL(wake_up_var);
 194
 195 __sched int bit_wait(struct wait_bit_key *word, int mode)
 196 {
 197         schedule();
 198         if (signal_pending_state(mode, current))
 199                 return -EINTR;
 200
 201         return 0;
 202 }
 203 EXPORT_SYMBOL(bit_wait);
 204
 205 __sched int bit_wait_io(struct wait_bit_key *word, int mode)
 206 {
 207         io_schedule();
 208         if (signal_pending_state(mode, current))
 209                 return -EINTR;
 210
 211         return 0;
 212 }
 213 EXPORT_SYMBOL(bit_wait_io);
 214
 215 __sched int bit_wait_timeout(struct wait_bit_key *word, int mode)
 216 {
 217         unsigned long now = READ_ONCE(jiffies);
 218
 219         if (time_after_eq(now, word->timeout))
 220                 return -EAGAIN;
 221         schedule_timeout(word->timeout - now);
 222         if (signal_pending_state(mode, current))
 223                 return -EINTR;
 224
 225         return 0;
 226 }
 227 EXPORT_SYMBOL_GPL(bit_wait_timeout);
 228
 229 __sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode)
 230 {
 231         unsigned long now = READ_ONCE(jiffies);
 232
 233         if (time_after_eq(now, word->timeout))
 234                 return -EAGAIN;
 235         io_schedule_timeout(word->timeout - now);
 236         if (signal_pending_state(mode, current))
 237                 return -EINTR;
 238
 239         return 0;
 240 }
 241 EXPORT_SYMBOL_GPL(bit_wait_io_timeout);
 242
 243 void __init wait_bit_init(void)
 244 {
 245         int i;
 246
 247         for (i = 0; i < WAIT_TABLE_SIZE; i++)
 248                 init_waitqueue_head(bit_wait_table + i);
 249 }