include/linux/wait_bit.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _LINUX_WAIT_BIT_H
   3 #define _LINUX_WAIT_BIT_H
   4
   5 /*
   6  * Linux wait-bit related types and methods:
   7  */
   8 #include <linux/wait.h>
   9
  10 struct wait_bit_key {
  11         unsigned long           *flags;
  12         int                     bit_nr;
  13         unsigned long           timeout;
  14 };
  15
  16 struct wait_bit_queue_entry {
  17         struct wait_bit_key     key;
  18         struct wait_queue_entry wq_entry;
  19 };
  20
  21 #define __WAIT_BIT_KEY_INITIALIZER(word, bit)                                   \
  22         { .flags = word, .bit_nr = bit, }
  23
  24 typedef int wait_bit_action_f(struct wait_bit_key *key, int mode);
  25
  26 void __wake_up_bit(struct wait_queue_head *wq_head, unsigned long *word, int bit);
  27 int __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode);
  28 int __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode);
  29 void wake_up_bit(unsigned long *word, int bit);
  30 int out_of_line_wait_on_bit(unsigned long *word, int, wait_bit_action_f *action, unsigned int mode);
  31 int out_of_line_wait_on_bit_timeout(unsigned long *word, int, wait_bit_action_f *action, unsigned int mode, unsigned long timeout);
  32 int out_of_line_wait_on_bit_lock(unsigned long *word, int, wait_bit_action_f *action, unsigned int mode);
  33 struct wait_queue_head *bit_waitqueue(unsigned long *word, int bit);
  34 extern void __init wait_bit_init(void);
  35
  36 int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
  37
  38 #define DEFINE_WAIT_BIT(name, word, bit)                                        \
  39         struct wait_bit_queue_entry name = {                                    \
  40                 .key = __WAIT_BIT_KEY_INITIALIZER(word, bit),                   \
  41                 .wq_entry = {                                                   \
  42                         .private        = current,                              \
  43                         .func           = wake_bit_function,                    \
  44                         .entry          =                                       \
  45                                 LIST_HEAD_INIT((name).wq_entry.entry),          \
  46                 },                                                              \
  47         }
  48
  49 extern int bit_wait(struct wait_bit_key *key, int mode);
  50 extern int bit_wait_io(struct wait_bit_key *key, int mode);
  51 extern int bit_wait_timeout(struct wait_bit_key *key, int mode);
  52
  53 /**
  54  * wait_on_bit - wait for a bit to be cleared
  55  * @word: the address containing the bit being waited on
  56  * @bit: the bit at that address being waited on
  57  * @mode: the task state to sleep in
  58  *
  59  * Wait for the given bit in an unsigned long or bitmap (see DECLARE_BITMAP())
  60  * to be cleared.  The clearing of the bit must be signalled with
  61  * wake_up_bit(), often as clear_and_wake_up_bit().
  62  *
  63  * The process will wait on a waitqueue selected by hash from a shared
  64  * pool.  It will only be woken on a wake_up for the target bit, even
  65  * if other processes on the same queue are waiting for other bits.
  66  *
  67  * Returned value will be zero if the bit was cleared in which case the
  68  * call has ACQUIRE semantics, or %-EINTR if the process received a
  69  * signal and the mode permitted wake up on that signal.
  70  */
  71 static inline int
  72 wait_on_bit(unsigned long *word, int bit, unsigned mode)
  73 {
  74         might_sleep();
  75         if (!test_bit_acquire(bit, word))
  76                 return 0;
  77         return out_of_line_wait_on_bit(word, bit,
  78                                        bit_wait,
  79                                        mode);
  80 }
  81
  82 /**
  83  * wait_on_bit_io - wait for a bit to be cleared
  84  * @word: the address containing the bit being waited on
  85  * @bit: the bit at that address being waited on
  86  * @mode: the task state to sleep in
  87  *
  88  * Wait for the given bit in an unsigned long or bitmap (see DECLARE_BITMAP())
  89  * to be cleared.  The clearing of the bit must be signalled with
  90  * wake_up_bit(), often as clear_and_wake_up_bit().
  91  *
  92  * This is similar to wait_on_bit(), but calls io_schedule() instead of
  93  * schedule() for the actual waiting.
  94  *
  95  * Returned value will be zero if the bit was cleared in which case the
  96  * call has ACQUIRE semantics, or %-EINTR if the process received a
  97  * signal and the mode permitted wake up on that signal.
  98  */
  99 static inline int
 100 wait_on_bit_io(unsigned long *word, int bit, unsigned mode)
 101 {
 102         might_sleep();
 103         if (!test_bit_acquire(bit, word))
 104                 return 0;
 105         return out_of_line_wait_on_bit(word, bit,
 106                                        bit_wait_io,
 107                                        mode);
 108 }
 109
 110 /**
 111  * wait_on_bit_timeout - wait for a bit to be cleared or a timeout to elapse
 112  * @word: the address containing the bit being waited on
 113  * @bit: the bit at that address being waited on
 114  * @mode: the task state to sleep in
 115  * @timeout: timeout, in jiffies
 116  *
 117  * Wait for the given bit in an unsigned long or bitmap (see
 118  * DECLARE_BITMAP()) to be cleared, or for a timeout to expire.  The
 119  * clearing of the bit must be signalled with wake_up_bit(), often as
 120  * clear_and_wake_up_bit().
 121  *
 122  * This is similar to wait_on_bit(), except it also takes a timeout
 123  * parameter.
 124  *
 125  * Returned value will be zero if the bit was cleared in which case the
 126  * call has ACQUIRE semantics, or %-EINTR if the process received a
 127  * signal and the mode permitted wake up on that signal, or %-EAGAIN if the
 128  * timeout elapsed.
 129  */
 130 static inline int
 131 wait_on_bit_timeout(unsigned long *word, int bit, unsigned mode,
 132                     unsigned long timeout)
 133 {
 134         might_sleep();
 135         if (!test_bit_acquire(bit, word))
 136                 return 0;
 137         return out_of_line_wait_on_bit_timeout(word, bit,
 138                                                bit_wait_timeout,
 139                                                mode, timeout);
 140 }
 141
 142 /**
 143  * wait_on_bit_action - wait for a bit to be cleared
 144  * @word: the address containing the bit waited on
 145  * @bit: the bit at that address being waited on
 146  * @action: the function used to sleep, which may take special actions
 147  * @mode: the task state to sleep in
 148  *
 149  * Wait for the given bit in an unsigned long or bitmap (see DECLARE_BITMAP())
 150  * to be cleared.  The clearing of the bit must be signalled with
 151  * wake_up_bit(), often as clear_and_wake_up_bit().
 152  *
 153  * This is similar to wait_on_bit(), but calls @action() instead of
 154  * schedule() for the actual waiting.
 155  *
 156  * Returned value will be zero if the bit was cleared in which case the
 157  * call has ACQUIRE semantics, or the error code returned by @action if
 158  * that call returned non-zero.
 159  */
 160 static inline int
 161 wait_on_bit_action(unsigned long *word, int bit, wait_bit_action_f *action,
 162                    unsigned mode)
 163 {
 164         might_sleep();
 165         if (!test_bit_acquire(bit, word))
 166                 return 0;
 167         return out_of_line_wait_on_bit(word, bit, action, mode);
 168 }
 169
 170 /**
 171  * wait_on_bit_lock - wait for a bit to be cleared, then set it
 172  * @word: the address containing the bit being waited on
 173  * @bit: the bit of the word being waited on and set
 174  * @mode: the task state to sleep in
 175  *
 176  * Wait for the given bit in an unsigned long or bitmap (see
 177  * DECLARE_BITMAP()) to be cleared.  The clearing of the bit must be
 178  * signalled with wake_up_bit(), often as clear_and_wake_up_bit().  As
 179  * soon as it is clear, atomically set it and return.
 180  *
 181  * This is similar to wait_on_bit(), but sets the bit before returning.
 182  *
 183  * Returned value will be zero if the bit was successfully set in which
 184  * case the call has the same memory sequencing semantics as
 185  * test_and_clear_bit(), or %-EINTR if the process received a signal and
 186  * the mode permitted wake up on that signal.
 187  */
 188 static inline int
 189 wait_on_bit_lock(unsigned long *word, int bit, unsigned mode)
 190 {
 191         might_sleep();
 192         if (!test_and_set_bit(bit, word))
 193                 return 0;
 194         return out_of_line_wait_on_bit_lock(word, bit, bit_wait, mode);
 195 }
 196
 197 /**
 198  * wait_on_bit_lock_io - wait for a bit to be cleared, then set it
 199  * @word: the address containing the bit being waited on
 200  * @bit: the bit of the word being waited on and set
 201  * @mode: the task state to sleep in
 202  *
 203  * Wait for the given bit in an unsigned long or bitmap (see
 204  * DECLARE_BITMAP()) to be cleared.  The clearing of the bit must be
 205  * signalled with wake_up_bit(), often as clear_and_wake_up_bit().  As
 206  * soon as it is clear, atomically set it and return.
 207  *
 208  * This is similar to wait_on_bit_lock(), but calls io_schedule() instead
 209  * of schedule().
 210  *
 211  * Returns zero if the bit was (eventually) found to be clear and was
 212  * set.  Returns non-zero if a signal was delivered to the process and
 213  * the @mode allows that signal to wake the process.
 214  */
 215 static inline int
 216 wait_on_bit_lock_io(unsigned long *word, int bit, unsigned mode)
 217 {
 218         might_sleep();
 219         if (!test_and_set_bit(bit, word))
 220                 return 0;
 221         return out_of_line_wait_on_bit_lock(word, bit, bit_wait_io, mode);
 222 }
 223
 224 /**
 225  * wait_on_bit_lock_action - wait for a bit to be cleared, then set it
 226  * @word: the address containing the bit being waited on
 227  * @bit: the bit of the word being waited on and set
 228  * @action: the function used to sleep, which may take special actions
 229  * @mode: the task state to sleep in
 230  *
 231  * This is similar to wait_on_bit_lock(), but calls @action() instead of
 232  * schedule() for the actual waiting.
 233  *
 234  * Returned value will be zero if the bit was successfully set in which
 235  * case the call has the same memory sequencing semantics as
 236  * test_and_clear_bit(), or the error code returned by @action if that
 237  * call returned non-zero.
 238  */
 239 static inline int
 240 wait_on_bit_lock_action(unsigned long *word, int bit, wait_bit_action_f *action,
 241                         unsigned mode)
 242 {
 243         might_sleep();
 244         if (!test_and_set_bit(bit, word))
 245                 return 0;
 246         return out_of_line_wait_on_bit_lock(word, bit, action, mode);
 247 }
 248
 249 extern void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, void *var, int flags);
 250 extern void wake_up_var(void *var);
 251 extern wait_queue_head_t *__var_waitqueue(void *p);
 252
 253 #define ___wait_var_event(var, condition, state, exclusive, ret, cmd)   \
 254 ({                                                                      \
 255         __label__ __out;                                                \
 256         struct wait_queue_head *__wq_head = __var_waitqueue(var);       \
 257         struct wait_bit_queue_entry __wbq_entry;                        \
 258         long __ret = ret; /* explicit shadow */                         \
 259                                                                         \
 260         init_wait_var_entry(&__wbq_entry, var,                          \
 261                             exclusive ? WQ_FLAG_EXCLUSIVE : 0);         \
 262         for (;;) {                                                      \
 263                 long __int = prepare_to_wait_event(__wq_head,           \
 264                                                    &__wbq_entry.wq_entry, \
 265                                                    state);              \
 266                 if (condition)                                          \
 267                         break;                                          \
 268                                                                         \
 269                 if (___wait_is_interruptible(state) && __int) {         \
 270                         __ret = __int;                                  \
 271                         goto __out;                                     \
 272                 }                                                       \
 273                                                                         \
 274                 cmd;                                                    \
 275         }                                                               \
 276         finish_wait(__wq_head, &__wbq_entry.wq_entry);                  \
 277 __out:  __ret;                                                          \
 278 })
 279
 280 #define __wait_var_event(var, condition)                                \
 281         ___wait_var_event(var, condition, TASK_UNINTERRUPTIBLE, 0, 0,   \
 282                           schedule())
 283 #define __wait_var_event_io(var, condition)                             \
 284         ___wait_var_event(var, condition, TASK_UNINTERRUPTIBLE, 0, 0,   \
 285                           io_schedule())
 286
 287 /**
 288  * wait_var_event - wait for a variable to be updated and notified
 289  * @var: the address of variable being waited on
 290  * @condition: the condition to wait for
 291  *
 292  * Wait for a @condition to be true, only re-checking when a wake up is
 293  * received for the given @var (an arbitrary kernel address which need
 294  * not be directly related to the given condition, but usually is).
 295  *
 296  * The process will wait on a waitqueue selected by hash from a shared
 297  * pool.  It will only be woken on a wake_up for the given address.
 298  *
 299  * The condition should normally use smp_load_acquire() or a similarly
 300  * ordered access to ensure that any changes to memory made before the
 301  * condition became true will be visible after the wait completes.
 302  */
 303 #define wait_var_event(var, condition)                                  \
 304 do {                                                                    \
 305         might_sleep();                                                  \
 306         if (condition)                                                  \
 307                 break;                                                  \
 308         __wait_var_event(var, condition);                               \
 309 } while (0)
 310
 311 /**
 312  * wait_var_event_io - wait for a variable to be updated and notified
 313  * @var: the address of variable being waited on
 314  * @condition: the condition to wait for
 315  *
 316  * Wait for an IO related @condition to be true, only re-checking when a
 317  * wake up is received for the given @var (an arbitrary kernel address
 318  * which need not be directly related to the given condition, but
 319  * usually is).
 320  *
 321  * The process will wait on a waitqueue selected by hash from a shared
 322  * pool.  It will only be woken on a wake_up for the given address.
 323  *
 324  * This is similar to wait_var_event(), but calls io_schedule() instead
 325  * of schedule().
 326  *
 327  * The condition should normally use smp_load_acquire() or a similarly
 328  * ordered access to ensure that any changes to memory made before the
 329  * condition became true will be visible after the wait completes.
 330  */
 331 #define wait_var_event_io(var, condition)                               \
 332 do {                                                                    \
 333         might_sleep();                                                  \
 334         if (condition)                                                  \
 335                 break;                                                  \
 336         __wait_var_event_io(var, condition);                            \
 337 } while (0)
 338
 339 #define __wait_var_event_killable(var, condition)                       \
 340         ___wait_var_event(var, condition, TASK_KILLABLE, 0, 0,          \
 341                           schedule())
 342
 343 /**
 344  * wait_var_event_killable - wait for a variable to be updated and notified
 345  * @var: the address of variable being waited on
 346  * @condition: the condition to wait for
 347  *
 348  * Wait for a @condition to be true or a fatal signal to be received,
 349  * only re-checking the condition when a wake up is received for the given
 350  * @var (an arbitrary kernel address which need not be directly related
 351  * to the given condition, but usually is).
 352  *
 353  * This is similar to wait_var_event() but returns a value which is
 354  * 0 if the condition became true, or %-ERESTARTSYS if a fatal signal
 355  * was received.
 356  *
 357  * The condition should normally use smp_load_acquire() or a similarly
 358  * ordered access to ensure that any changes to memory made before the
 359  * condition became true will be visible after the wait completes.
 360  */
 361 #define wait_var_event_killable(var, condition)                         \
 362 ({                                                                      \
 363         int __ret = 0;                                                  \
 364         might_sleep();                                                  \
 365         if (!(condition))                                               \
 366                 __ret = __wait_var_event_killable(var, condition);      \
 367         __ret;                                                          \
 368 })
 369
 370 #define __wait_var_event_timeout(var, condition, timeout)               \
 371         ___wait_var_event(var, ___wait_cond_timeout(condition),         \
 372                           TASK_UNINTERRUPTIBLE, 0, timeout,             \
 373                           __ret = schedule_timeout(__ret))
 374
 375 /**
 376  * wait_var_event_timeout - wait for a variable to be updated or a timeout to expire
 377  * @var: the address of variable being waited on
 378  * @condition: the condition to wait for
 379  * @timeout: maximum time to wait in jiffies
 380  *
 381  * Wait for a @condition to be true or a timeout to expire, only
 382  * re-checking the condition when a wake up is received for the given
 383  * @var (an arbitrary kernel address which need not be directly related
 384  * to the given condition, but usually is).
 385  *
 386  * This is similar to wait_var_event() but returns a value which is 0 if
 387  * the timeout expired and the condition was still false, or the
 388  * remaining time left in the timeout (but at least 1) if the condition
 389  * was found to be true.
 390  *
 391  * The condition should normally use smp_load_acquire() or a similarly
 392  * ordered access to ensure that any changes to memory made before the
 393  * condition became true will be visible after the wait completes.
 394  */
 395 #define wait_var_event_timeout(var, condition, timeout)                 \
 396 ({                                                                      \
 397         long __ret = timeout;                                           \
 398         might_sleep();                                                  \
 399         if (!___wait_cond_timeout(condition))                           \
 400                 __ret = __wait_var_event_timeout(var, condition, timeout); \
 401         __ret;                                                          \
 402 })
 403
 404 #define __wait_var_event_interruptible(var, condition)                  \
 405         ___wait_var_event(var, condition, TASK_INTERRUPTIBLE, 0, 0,     \
 406                           schedule())
 407
 408 /**
 409  * wait_var_event_killable - wait for a variable to be updated and notified
 410  * @var: the address of variable being waited on
 411  * @condition: the condition to wait for
 412  *
 413  * Wait for a @condition to be true or a signal to be received, only
 414  * re-checking the condition when a wake up is received for the given
 415  * @var (an arbitrary kernel address which need not be directly related
 416  * to the given condition, but usually is).
 417  *
 418  * This is similar to wait_var_event() but returns a value which is 0 if
 419  * the condition became true, or %-ERESTARTSYS if a signal was received.
 420  *
 421  * The condition should normally use smp_load_acquire() or a similarly
 422  * ordered access to ensure that any changes to memory made before the
 423  * condition became true will be visible after the wait completes.
 424  */
 425 #define wait_var_event_interruptible(var, condition)                    \
 426 ({                                                                      \
 427         int __ret = 0;                                                  \
 428         might_sleep();                                                  \
 429         if (!(condition))                                               \
 430                 __ret = __wait_var_event_interruptible(var, condition); \
 431         __ret;                                                          \
 432 })
 433
 434 /**
 435  * wait_var_event_any_lock - wait for a variable to be updated under a lock
 436  * @var: the address of the variable being waited on
 437  * @condition: condition to wait for
 438  * @lock: the object that is locked to protect updates to the variable
 439  * @type: prefix on lock and unlock operations
 440  * @state: waiting state, %TASK_UNINTERRUPTIBLE etc.
 441  *
 442  * Wait for a condition which can only be reliably tested while holding
 443  * a lock.  The variables assessed in the condition will normal be updated
 444  * under the same lock, and the wake up should be signalled with
 445  * wake_up_var_locked() under the same lock.
 446  *
 447  * This is similar to wait_var_event(), but assumes a lock is held
 448  * while calling this function and while updating the variable.
 449  *
 450  * This must be called while the given lock is held and the lock will be
 451  * dropped when schedule() is called to wait for a wake up, and will be
 452  * reclaimed before testing the condition again.  The functions used to
 453  * unlock and lock the object are constructed by appending _unlock and _lock
 454  * to @type.
 455  *
 456  * Return %-ERESTARTSYS if a signal arrives which is allowed to interrupt
 457  * the wait according to @state.
 458  */
 459 #define wait_var_event_any_lock(var, condition, lock, type, state)      \
 460 ({                                                                      \
 461         int __ret = 0;                                                  \
 462         if (!(condition))                                               \
 463                 __ret = ___wait_var_event(var, condition, state, 0, 0,  \
 464                                           type ## _unlock(lock);        \
 465                                           schedule();                   \
 466                                           type ## _lock(lock));         \
 467         __ret;                                                          \
 468 })
 469
 470 /**
 471  * wait_var_event_spinlock - wait for a variable to be updated under a spinlock
 472  * @var: the address of the variable being waited on
 473  * @condition: condition to wait for
 474  * @lock: the spinlock which protects updates to the variable
 475  *
 476  * Wait for a condition which can only be reliably tested while holding
 477  * a spinlock.  The variables assessed in the condition will normal be updated
 478  * under the same spinlock, and the wake up should be signalled with
 479  * wake_up_var_locked() under the same spinlock.
 480  *
 481  * This is similar to wait_var_event(), but assumes a spinlock is held
 482  * while calling this function and while updating the variable.
 483  *
 484  * This must be called while the given lock is held and the lock will be
 485  * dropped when schedule() is called to wait for a wake up, and will be
 486  * reclaimed before testing the condition again.
 487  */
 488 #define wait_var_event_spinlock(var, condition, lock)                   \
 489         wait_var_event_any_lock(var, condition, lock, spin, TASK_UNINTERRUPTIBLE)
 490
 491 /**
 492  * wait_var_event_mutex - wait for a variable to be updated under a mutex
 493  * @var: the address of the variable being waited on
 494  * @condition: condition to wait for
 495  * @mutex: the mutex which protects updates to the variable
 496  *
 497  * Wait for a condition which can only be reliably tested while holding
 498  * a mutex.  The variables assessed in the condition will normal be
 499  * updated under the same mutex, and the wake up should be signalled
 500  * with wake_up_var_locked() under the same mutex.
 501  *
 502  * This is similar to wait_var_event(), but assumes a mutex is held
 503  * while calling this function and while updating the variable.
 504  *
 505  * This must be called while the given mutex is held and the mutex will be
 506  * dropped when schedule() is called to wait for a wake up, and will be
 507  * reclaimed before testing the condition again.
 508  */
 509 #define wait_var_event_mutex(var, condition, lock)                      \
 510         wait_var_event_any_lock(var, condition, lock, mutex, TASK_UNINTERRUPTIBLE)
 511
 512 /**
 513  * wake_up_var_protected - wake up waiters for a variable asserting that it is safe
 514  * @var: the address of the variable being waited on
 515  * @cond: the condition which afirms this is safe
 516  *
 517  * When waking waiters which use wait_var_event_any_lock() the waker must be
 518  * holding the reelvant lock to avoid races.  This version of wake_up_var()
 519  * asserts that the relevant lock is held and so no barrier is needed.
 520  * The @cond is only tested when CONFIG_LOCKDEP is enabled.
 521  */
 522 #define wake_up_var_protected(var, cond)                                \
 523 do {                                                                    \
 524         lockdep_assert(cond);                                           \
 525         wake_up_var(var);                                               \
 526 } while (0)
 527
 528 /**
 529  * wake_up_var_locked - wake up waiters for a variable while holding a spinlock or mutex
 530  * @var: the address of the variable being waited on
 531  * @lock: The spinlock or mutex what protects the variable
 532  *
 533  * Send a wake up for the given variable which should be waited for with
 534  * wait_var_event_spinlock() or wait_var_event_mutex().  Unlike wake_up_var(),
 535  * no extra barriers are needed as the locking provides sufficient sequencing.
 536  */
 537 #define wake_up_var_locked(var, lock)                                   \
 538         wake_up_var_protected(var, lockdep_is_held(lock))
 539
 540 /**
 541  * clear_and_wake_up_bit - clear a bit and wake up anyone waiting on that bit
 542  * @bit: the bit of the word being waited on
 543  * @word: the address containing the bit being waited on
 544  *
 545  * The designated bit is cleared and any tasks waiting in wait_on_bit()
 546  * or similar will be woken.  This call has RELEASE semantics so that
 547  * any changes to memory made before this call are guaranteed to be visible
 548  * after the corresponding wait_on_bit() completes.
 549  */
 550 static inline void clear_and_wake_up_bit(int bit, unsigned long *word)
 551 {
 552         clear_bit_unlock(bit, word);
 553         /* See wake_up_bit() for which memory barrier you need to use. */
 554         smp_mb__after_atomic();
 555         wake_up_bit(word, bit);
 556 }
 557
 558 /**
 559  * test_and_clear_wake_up_bit - clear a bit if it was set: wake up anyone waiting on that bit
 560  * @bit: the bit of the word being waited on
 561  * @word: the address of memory containing that bit
 562  *
 563  * If the bit is set and can be atomically cleared, any tasks waiting in
 564  * wait_on_bit() or similar will be woken.  This call has the same
 565  * complete ordering semantics as test_and_clear_bit().  Any changes to
 566  * memory made before this call are guaranteed to be visible after the
 567  * corresponding wait_on_bit() completes.
 568  *
 569  * Returns %true if the bit was successfully set and the wake up was sent.
 570  */
 571 static inline bool test_and_clear_wake_up_bit(int bit, unsigned long *word)
 572 {
 573         if (!test_and_clear_bit(bit, word))
 574                 return false;
 575         /* no extra barrier required */
 576         wake_up_bit(word, bit);
 577         return true;
 578 }
 579
 580 /**
 581  * atomic_dec_and_wake_up - decrement an atomic_t and if zero, wake up waiters
 582  * @var: the variable to dec and test
 583  *
 584  * Decrements the atomic variable and if it reaches zero, send a wake_up to any
 585  * processes waiting on the variable.
 586  *
 587  * This function has the same complete ordering semantics as atomic_dec_and_test.
 588  *
 589  * Returns %true is the variable reaches zero and the wake up was sent.
 590  */
 591
 592 static inline bool atomic_dec_and_wake_up(atomic_t *var)
 593 {
 594         if (!atomic_dec_and_test(var))
 595                 return false;
 596         /* No extra barrier required */
 597         wake_up_var(var);
 598         return true;
 599 }
 600
 601 /**
 602  * store_release_wake_up - update a variable and send a wake_up
 603  * @var: the address of the variable to be updated and woken
 604  * @val: the value to store in the variable.
 605  *
 606  * Store the given value in the variable send a wake up to any tasks
 607  * waiting on the variable.  All necessary barriers are included to ensure
 608  * the task calling wait_var_event() sees the new value and all values
 609  * written to memory before this call.
 610  */
 611 #define store_release_wake_up(var, val)                                 \
 612 do {                                                                    \
 613         smp_store_release(var, val);                                    \
 614         smp_mb();                                                       \
 615         wake_up_var(var);                                               \
 616 } while (0)
 617
 618 #endif /* _LINUX_WAIT_BIT_H */