kernel/sched/completion.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Generic wait-for-completion handler;
   4  *
   5  * It differs from semaphores in that their default case is the opposite,
   6  * wait_for_completion default blocks whereas semaphore default non-block. The
   7  * interface also makes it easy to 'complete' multiple waiting threads,
   8  * something which isn't entirely natural for semaphores.
   9  *
  10  * But more importantly, the primitive documents the usage. Semaphores would
  11  * typically be used for exclusion which gives rise to priority inversion.
  12  * Waiting for completion is a typically sync point, but not an exclusion point.
  13  */
  14 #include "sched.h"
  15
  16 /**
  17  * complete: - signals a single thread waiting on this completion
  18  * @x:  holds the state of this particular completion
  19  *
  20  * This will wake up a single thread waiting on this completion. Threads will be
  21  * awakened in the same order in which they were queued.
  22  *
  23  * See also complete_all(), wait_for_completion() and related routines.
  24  *
  25  * If this function wakes up a task, it executes a full memory barrier before
  26  * accessing the task state.
  27  */
  28 void complete(struct completion *x)
  29 {
  30         unsigned long flags;
  31
  32         raw_spin_lock_irqsave(&x->wait.lock, flags);
  33
  34         if (x->done != UINT_MAX)
  35                 x->done++;
  36         swake_up_locked(&x->wait);
  37         raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  38 }
  39 EXPORT_SYMBOL(complete);
  40
  41 /**
  42  * complete_all: - signals all threads waiting on this completion
  43  * @x:  holds the state of this particular completion
  44  *
  45  * This will wake up all threads waiting on this particular completion event.
  46  *
  47  * If this function wakes up a task, it executes a full memory barrier before
  48  * accessing the task state.
  49  *
  50  * Since complete_all() sets the completion of @x permanently to done
  51  * to allow multiple waiters to finish, a call to reinit_completion()
  52  * must be used on @x if @x is to be used again. The code must make
  53  * sure that all waiters have woken and finished before reinitializing
  54  * @x. Also note that the function completion_done() can not be used
  55  * to know if there are still waiters after complete_all() has been called.
  56  */
  57 void complete_all(struct completion *x)
  58 {
  59         unsigned long flags;
  60
  61         lockdep_assert_RT_in_threaded_ctx();
  62
  63         raw_spin_lock_irqsave(&x->wait.lock, flags);
  64         x->done = UINT_MAX;
  65         swake_up_all_locked(&x->wait);
  66         raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  67 }
  68 EXPORT_SYMBOL(complete_all);
  69
  70 static inline long __sched
  71 do_wait_for_common(struct completion *x,
  72                    long (*action)(long), long timeout, int state)
  73 {
  74         if (!x->done) {
  75                 DECLARE_SWAITQUEUE(wait);
  76
  77                 do {
  78                         if (signal_pending_state(state, current)) {
  79                                 timeout = -ERESTARTSYS;
  80                                 break;
  81                         }
  82                         __prepare_to_swait(&x->wait, &wait);
  83                         __set_current_state(state);
  84                         raw_spin_unlock_irq(&x->wait.lock);
  85                         timeout = action(timeout);
  86                         raw_spin_lock_irq(&x->wait.lock);
  87                 } while (!x->done && timeout);
  88                 __finish_swait(&x->wait, &wait);
  89                 if (!x->done)
  90                         return timeout;
  91         }
  92         if (x->done != UINT_MAX)
  93                 x->done--;
  94         return timeout ?: 1;
  95 }
  96
  97 static inline long __sched
  98 __wait_for_common(struct completion *x,
  99                   long (*action)(long), long timeout, int state)
 100 {
 101         might_sleep();
 102
 103         complete_acquire(x);
 104
 105         raw_spin_lock_irq(&x->wait.lock);
 106         timeout = do_wait_for_common(x, action, timeout, state);
 107         raw_spin_unlock_irq(&x->wait.lock);
 108
 109         complete_release(x);
 110
 111         return timeout;
 112 }
 113
 114 static long __sched
 115 wait_for_common(struct completion *x, long timeout, int state)
 116 {
 117         return __wait_for_common(x, schedule_timeout, timeout, state);
 118 }
 119
 120 static long __sched
 121 wait_for_common_io(struct completion *x, long timeout, int state)
 122 {
 123         return __wait_for_common(x, io_schedule_timeout, timeout, state);
 124 }
 125
 126 /**
 127  * wait_for_completion: - waits for completion of a task
 128  * @x:  holds the state of this particular completion
 129  *
 130  * This waits to be signaled for completion of a specific task. It is NOT
 131  * interruptible and there is no timeout.
 132  *
 133  * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
 134  * and interrupt capability. Also see complete().
 135  */
 136 void __sched wait_for_completion(struct completion *x)
 137 {
 138         wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
 139 }
 140 EXPORT_SYMBOL(wait_for_completion);
 141
 142 /**
 143  * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
 144  * @x:  holds the state of this particular completion
 145  * @timeout:  timeout value in jiffies
 146  *
 147  * This waits for either a completion of a specific task to be signaled or for a
 148  * specified timeout to expire. The timeout is in jiffies. It is not
 149  * interruptible.
 150  *
 151  * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
 152  * till timeout) if completed.
 153  */
 154 unsigned long __sched
 155 wait_for_completion_timeout(struct completion *x, unsigned long timeout)
 156 {
 157         return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
 158 }
 159 EXPORT_SYMBOL(wait_for_completion_timeout);
 160
 161 /**
 162  * wait_for_completion_io: - waits for completion of a task
 163  * @x:  holds the state of this particular completion
 164  *
 165  * This waits to be signaled for completion of a specific task. It is NOT
 166  * interruptible and there is no timeout. The caller is accounted as waiting
 167  * for IO (which traditionally means blkio only).
 168  */
 169 void __sched wait_for_completion_io(struct completion *x)
 170 {
 171         wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
 172 }
 173 EXPORT_SYMBOL(wait_for_completion_io);
 174
 175 /**
 176  * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
 177  * @x:  holds the state of this particular completion
 178  * @timeout:  timeout value in jiffies
 179  *
 180  * This waits for either a completion of a specific task to be signaled or for a
 181  * specified timeout to expire. The timeout is in jiffies. It is not
 182  * interruptible. The caller is accounted as waiting for IO (which traditionally
 183  * means blkio only).
 184  *
 185  * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
 186  * till timeout) if completed.
 187  */
 188 unsigned long __sched
 189 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
 190 {
 191         return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
 192 }
 193 EXPORT_SYMBOL(wait_for_completion_io_timeout);
 194
 195 /**
 196  * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
 197  * @x:  holds the state of this particular completion
 198  *
 199  * This waits for completion of a specific task to be signaled. It is
 200  * interruptible.
 201  *
 202  * Return: -ERESTARTSYS if interrupted, 0 if completed.
 203  */
 204 int __sched wait_for_completion_interruptible(struct completion *x)
 205 {
 206         long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
 207         if (t == -ERESTARTSYS)
 208                 return t;
 209         return 0;
 210 }
 211 EXPORT_SYMBOL(wait_for_completion_interruptible);
 212
 213 /**
 214  * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
 215  * @x:  holds the state of this particular completion
 216  * @timeout:  timeout value in jiffies
 217  *
 218  * This waits for either a completion of a specific task to be signaled or for a
 219  * specified timeout to expire. It is interruptible. The timeout is in jiffies.
 220  *
 221  * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
 222  * or number of jiffies left till timeout) if completed.
 223  */
 224 long __sched
 225 wait_for_completion_interruptible_timeout(struct completion *x,
 226                                           unsigned long timeout)
 227 {
 228         return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
 229 }
 230 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
 231
 232 /**
 233  * wait_for_completion_killable: - waits for completion of a task (killable)
 234  * @x:  holds the state of this particular completion
 235  *
 236  * This waits to be signaled for completion of a specific task. It can be
 237  * interrupted by a kill signal.
 238  *
 239  * Return: -ERESTARTSYS if interrupted, 0 if completed.
 240  */
 241 int __sched wait_for_completion_killable(struct completion *x)
 242 {
 243         long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
 244         if (t == -ERESTARTSYS)
 245                 return t;
 246         return 0;
 247 }
 248 EXPORT_SYMBOL(wait_for_completion_killable);
 249
 250 /**
 251  * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
 252  * @x:  holds the state of this particular completion
 253  * @timeout:  timeout value in jiffies
 254  *
 255  * This waits for either a completion of a specific task to be
 256  * signaled or for a specified timeout to expire. It can be
 257  * interrupted by a kill signal. The timeout is in jiffies.
 258  *
 259  * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
 260  * or number of jiffies left till timeout) if completed.
 261  */
 262 long __sched
 263 wait_for_completion_killable_timeout(struct completion *x,
 264                                      unsigned long timeout)
 265 {
 266         return wait_for_common(x, timeout, TASK_KILLABLE);
 267 }
 268 EXPORT_SYMBOL(wait_for_completion_killable_timeout);
 269
 270 /**
 271  *      try_wait_for_completion - try to decrement a completion without blocking
 272  *      @x:     completion structure
 273  *
 274  *      Return: 0 if a decrement cannot be done without blocking
 275  *               1 if a decrement succeeded.
 276  *
 277  *      If a completion is being used as a counting completion,
 278  *      attempt to decrement the counter without blocking. This
 279  *      enables us to avoid waiting if the resource the completion
 280  *      is protecting is not available.
 281  */
 282 bool try_wait_for_completion(struct completion *x)
 283 {
 284         unsigned long flags;
 285         bool ret = true;
 286
 287         /*
 288          * Since x->done will need to be locked only
 289          * in the non-blocking case, we check x->done
 290          * first without taking the lock so we can
 291          * return early in the blocking case.
 292          */
 293         if (!READ_ONCE(x->done))
 294                 return false;
 295
 296         raw_spin_lock_irqsave(&x->wait.lock, flags);
 297         if (!x->done)
 298                 ret = false;
 299         else if (x->done != UINT_MAX)
 300                 x->done--;
 301         raw_spin_unlock_irqrestore(&x->wait.lock, flags);
 302         return ret;
 303 }
 304 EXPORT_SYMBOL(try_wait_for_completion);
 305
 306 /**
 307  *      completion_done - Test to see if a completion has any waiters
 308  *      @x:     completion structure
 309  *
 310  *      Return: 0 if there are waiters (wait_for_completion() in progress)
 311  *               1 if there are no waiters.
 312  *
 313  *      Note, this will always return true if complete_all() was called on @X.
 314  */
 315 bool completion_done(struct completion *x)
 316 {
 317         unsigned long flags;
 318
 319         if (!READ_ONCE(x->done))
 320                 return false;
 321
 322         /*
 323          * If ->done, we need to wait for complete() to release ->wait.lock
 324          * otherwise we can end up freeing the completion before complete()
 325          * is done referencing it.
 326          */
 327         raw_spin_lock_irqsave(&x->wait.lock, flags);
 328         raw_spin_unlock_irqrestore(&x->wait.lock, flags);
 329         return true;
 330 }
 331 EXPORT_SYMBOL(completion_done);