kernel/sched/wait_bit.c

   1 /*
   2  * The implementation of the wait_bit*() and related waiting APIs:
   3  */
   4 #include <linux/wait_bit.h>
   5 #include <linux/sched/signal.h>
   6 #include <linux/sched/debug.h>
   7 #include <linux/hash.h>
   8
   9 #define WAIT_TABLE_BITS 8
  10 #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)
  11
  12 static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;
  13
  14 wait_queue_head_t *bit_waitqueue(void *word, int bit)
  15 {
  16         const int shift = BITS_PER_LONG == 32 ? 5 : 6;
  17         unsigned long val = (unsigned long)word << shift | bit;
  18
  19         return bit_wait_table + hash_long(val, WAIT_TABLE_BITS);
  20 }
  21 EXPORT_SYMBOL(bit_waitqueue);
  22
  23 int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg)
  24 {
  25         struct wait_bit_key *key = arg;
  26         struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
  27
  28         if (wait_bit->key.flags != key->flags ||
  29                         wait_bit->key.bit_nr != key->bit_nr ||
  30                         test_bit(key->bit_nr, key->flags))
  31                 return 0;
  32         else
  33                 return autoremove_wake_function(wq_entry, mode, sync, key);
  34 }
  35 EXPORT_SYMBOL(wake_bit_function);
  36
  37 /*
  38  * To allow interruptible waiting and asynchronous (i.e. nonblocking)
  39  * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are
  40  * permitted return codes. Nonzero return codes halt waiting and return.
  41  */
  42 int __sched
  43 __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
  44               wait_bit_action_f *action, unsigned mode)
  45 {
  46         int ret = 0;
  47
  48         do {
  49                 prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
  50                 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags))
  51                         ret = (*action)(&wbq_entry->key, mode);
  52         } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);
  53         finish_wait(wq_head, &wbq_entry->wq_entry);
  54         return ret;
  55 }
  56 EXPORT_SYMBOL(__wait_on_bit);
  57
  58 int __sched out_of_line_wait_on_bit(void *word, int bit,
  59                                     wait_bit_action_f *action, unsigned mode)
  60 {
  61         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
  62         DEFINE_WAIT_BIT(wq_entry, word, bit);
  63
  64         return __wait_on_bit(wq_head, &wq_entry, action, mode);
  65 }
  66 EXPORT_SYMBOL(out_of_line_wait_on_bit);
  67
  68 int __sched out_of_line_wait_on_bit_timeout(
  69         void *word, int bit, wait_bit_action_f *action,
  70         unsigned mode, unsigned long timeout)
  71 {
  72         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
  73         DEFINE_WAIT_BIT(wq_entry, word, bit);
  74
  75         wq_entry.key.timeout = jiffies + timeout;
  76         return __wait_on_bit(wq_head, &wq_entry, action, mode);
  77 }
  78 EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout);
  79
  80 int __sched
  81 __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
  82                         wait_bit_action_f *action, unsigned mode)
  83 {
  84         int ret = 0;
  85
  86         for (;;) {
  87                 prepare_to_wait_exclusive(wq_head, &wbq_entry->wq_entry, mode);
  88                 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
  89                         ret = action(&wbq_entry->key, mode);
  90                         /*
  91                          * See the comment in prepare_to_wait_event().
  92                          * finish_wait() does not necessarily takes wwq_head->lock,
  93                          * but test_and_set_bit() implies mb() which pairs with
  94                          * smp_mb__after_atomic() before wake_up_page().
  95                          */
  96                         if (ret)
  97                                 finish_wait(wq_head, &wbq_entry->wq_entry);
  98                 }
  99                 if (!test_and_set_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
 100                         if (!ret)
 101                                 finish_wait(wq_head, &wbq_entry->wq_entry);
 102                         return 0;
 103                 } else if (ret) {
 104                         return ret;
 105                 }
 106         }
 107 }
 108 EXPORT_SYMBOL(__wait_on_bit_lock);
 109
 110 int __sched out_of_line_wait_on_bit_lock(void *word, int bit,
 111                                          wait_bit_action_f *action, unsigned mode)
 112 {
 113         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
 114         DEFINE_WAIT_BIT(wq_entry, word, bit);
 115
 116         return __wait_on_bit_lock(wq_head, &wq_entry, action, mode);
 117 }
 118 EXPORT_SYMBOL(out_of_line_wait_on_bit_lock);
 119
 120 void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit)
 121 {
 122         struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
 123         if (waitqueue_active(wq_head))
 124                 __wake_up(wq_head, TASK_NORMAL, 1, &key);
 125 }
 126 EXPORT_SYMBOL(__wake_up_bit);
 127
 128 /**
 129  * wake_up_bit - wake up a waiter on a bit
 130  * @word: the word being waited on, a kernel virtual address
 131  * @bit: the bit of the word being waited on
 132  *
 133  * There is a standard hashed waitqueue table for generic use. This
 134  * is the part of the hashtable's accessor API that wakes up waiters
 135  * on a bit. For instance, if one were to have waiters on a bitflag,
 136  * one would call wake_up_bit() after clearing the bit.
 137  *
 138  * In order for this to function properly, as it uses waitqueue_active()
 139  * internally, some kind of memory barrier must be done prior to calling
 140  * this. Typically, this will be smp_mb__after_atomic(), but in some
 141  * cases where bitflags are manipulated non-atomically under a lock, one
 142  * may need to use a less regular barrier, such fs/inode.c's smp_mb(),
 143  * because spin_unlock() does not guarantee a memory barrier.
 144  */
 145 void wake_up_bit(void *word, int bit)
 146 {
 147         __wake_up_bit(bit_waitqueue(word, bit), word, bit);
 148 }
 149 EXPORT_SYMBOL(wake_up_bit);
 150
 151 /*
 152  * Manipulate the atomic_t address to produce a better bit waitqueue table hash
 153  * index (we're keying off bit -1, but that would produce a horrible hash
 154  * value).
 155  */
 156 static inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p)
 157 {
 158         if (BITS_PER_LONG == 64) {
 159                 unsigned long q = (unsigned long)p;
 160                 return bit_waitqueue((void *)(q & ~1), q & 1);
 161         }
 162         return bit_waitqueue(p, 0);
 163 }
 164
 165 static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync,
 166                                   void *arg)
 167 {
 168         struct wait_bit_key *key = arg;
 169         struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
 170         atomic_t *val = key->flags;
 171
 172         if (wait_bit->key.flags != key->flags ||
 173             wait_bit->key.bit_nr != key->bit_nr ||
 174             atomic_read(val) != 0)
 175                 return 0;
 176         return autoremove_wake_function(wq_entry, mode, sync, key);
 177 }
 178
 179 /*
 180  * To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting,
 181  * the actions of __wait_on_atomic_t() are permitted return codes.  Nonzero
 182  * return codes halt waiting and return.
 183  */
 184 static __sched
 185 int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
 186                        wait_atomic_t_action_f action, unsigned int mode)
 187 {
 188         atomic_t *val;
 189         int ret = 0;
 190
 191         do {
 192                 prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
 193                 val = wbq_entry->key.flags;
 194                 if (atomic_read(val) == 0)
 195                         break;
 196                 ret = (*action)(val, mode);
 197         } while (!ret && atomic_read(val) != 0);
 198         finish_wait(wq_head, &wbq_entry->wq_entry);
 199         return ret;
 200 }
 201
 202 #define DEFINE_WAIT_ATOMIC_T(name, p)                                   \
 203         struct wait_bit_queue_entry name = {                            \
 204                 .key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p),              \
 205                 .wq_entry = {                                           \
 206                         .private        = current,                      \
 207                         .func           = wake_atomic_t_function,       \
 208                         .entry          =                               \
 209                                 LIST_HEAD_INIT((name).wq_entry.entry),  \
 210                 },                                                      \
 211         }
 212
 213 __sched int out_of_line_wait_on_atomic_t(atomic_t *p,
 214                                          wait_atomic_t_action_f action,
 215                                          unsigned int mode)
 216 {
 217         struct wait_queue_head *wq_head = atomic_t_waitqueue(p);
 218         DEFINE_WAIT_ATOMIC_T(wq_entry, p);
 219
 220         return __wait_on_atomic_t(wq_head, &wq_entry, action, mode);
 221 }
 222 EXPORT_SYMBOL(out_of_line_wait_on_atomic_t);
 223
 224 __sched int atomic_t_wait(atomic_t *counter, unsigned int mode)
 225 {
 226         schedule();
 227         if (signal_pending_state(mode, current))
 228                 return -EINTR;
 229         return 0;
 230 }
 231 EXPORT_SYMBOL(atomic_t_wait);
 232
 233 /**
 234  * wake_up_atomic_t - Wake up a waiter on a atomic_t
 235  * @p: The atomic_t being waited on, a kernel virtual address
 236  *
 237  * Wake up anyone waiting for the atomic_t to go to zero.
 238  *
 239  * Abuse the bit-waker function and its waitqueue hash table set (the atomic_t
 240  * check is done by the waiter's wake function, not the by the waker itself).
 241  */
 242 void wake_up_atomic_t(atomic_t *p)
 243 {
 244         __wake_up_bit(atomic_t_waitqueue(p), p, WAIT_ATOMIC_T_BIT_NR);
 245 }
 246 EXPORT_SYMBOL(wake_up_atomic_t);
 247
 248 __sched int bit_wait(struct wait_bit_key *word, int mode)
 249 {
 250         schedule();
 251         if (signal_pending_state(mode, current))
 252                 return -EINTR;
 253         return 0;
 254 }
 255 EXPORT_SYMBOL(bit_wait);
 256
 257 __sched int bit_wait_io(struct wait_bit_key *word, int mode)
 258 {
 259         io_schedule();
 260         if (signal_pending_state(mode, current))
 261                 return -EINTR;
 262         return 0;
 263 }
 264 EXPORT_SYMBOL(bit_wait_io);
 265
 266 __sched int bit_wait_timeout(struct wait_bit_key *word, int mode)
 267 {
 268         unsigned long now = READ_ONCE(jiffies);
 269         if (time_after_eq(now, word->timeout))
 270                 return -EAGAIN;
 271         schedule_timeout(word->timeout - now);
 272         if (signal_pending_state(mode, current))
 273                 return -EINTR;
 274         return 0;
 275 }
 276 EXPORT_SYMBOL_GPL(bit_wait_timeout);
 277
 278 __sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode)
 279 {
 280         unsigned long now = READ_ONCE(jiffies);
 281         if (time_after_eq(now, word->timeout))
 282                 return -EAGAIN;
 283         io_schedule_timeout(word->timeout - now);
 284         if (signal_pending_state(mode, current))
 285                 return -EINTR;
 286         return 0;
 287 }
 288 EXPORT_SYMBOL_GPL(bit_wait_io_timeout);
 289
 290 void __init wait_bit_init(void)
 291 {
 292         int i;
 293
 294         for (i = 0; i < WAIT_TABLE_SIZE; i++)
 295                 init_waitqueue_head(bit_wait_table + i);
 296 }