kernel/sched/completion.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Generic wait-for-completion handler;
   5  *
   6  * It differs from semaphores in that their default case is the opposite,
   7  * wait_for_completion default blocks whereas semaphore default non-block. The
   8  * interface also makes it easy to 'complete' multiple waiting threads,
   9  * something which isn't entirely natural for semaphores.
  10  *
  11  * But more importantly, the primitive documents the usage. Semaphores would
  12  * typically be used for exclusion which gives rise to priority inversion.
  13  * Waiting for completion is a typically sync point, but not an exclusion point.
  14  */
  15
  16 static void complete_with_flags(struct completion *x, int wake_flags)
  17 {
  18         unsigned long flags;
  19
  20         raw_spin_lock_irqsave(&x->wait.lock, flags);
  21
  22         if (x->done != UINT_MAX)
  23                 x->done++;
  24         swake_up_locked(&x->wait, wake_flags);
  25         raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  26 }
  27
  28 void complete_on_current_cpu(struct completion *x)
  29 {
  30         return complete_with_flags(x, WF_CURRENT_CPU);
  31 }
  32
  33 /**
  34  * complete: - signals a single thread waiting on this completion
  35  * @x:  holds the state of this particular completion
  36  *
  37  * This will wake up a single thread waiting on this completion. Threads will be
  38  * awakened in the same order in which they were queued.
  39  *
  40  * See also complete_all(), wait_for_completion() and related routines.
  41  *
  42  * If this function wakes up a task, it executes a full memory barrier before
  43  * accessing the task state.
  44  */
  45 void complete(struct completion *x)
  46 {
  47         complete_with_flags(x, 0);
  48 }
  49 EXPORT_SYMBOL(complete);
  50
  51 /**
  52  * complete_all: - signals all threads waiting on this completion
  53  * @x:  holds the state of this particular completion
  54  *
  55  * This will wake up all threads waiting on this particular completion event.
  56  *
  57  * If this function wakes up a task, it executes a full memory barrier before
  58  * accessing the task state.
  59  *
  60  * Since complete_all() sets the completion of @x permanently to done
  61  * to allow multiple waiters to finish, a call to reinit_completion()
  62  * must be used on @x if @x is to be used again. The code must make
  63  * sure that all waiters have woken and finished before reinitializing
  64  * @x. Also note that the function completion_done() can not be used
  65  * to know if there are still waiters after complete_all() has been called.
  66  */
  67 void complete_all(struct completion *x)
  68 {
  69         unsigned long flags;
  70
  71         lockdep_assert_RT_in_threaded_ctx();
  72
  73         raw_spin_lock_irqsave(&x->wait.lock, flags);
  74         x->done = UINT_MAX;
  75         swake_up_all_locked(&x->wait);
  76         raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  77 }
  78 EXPORT_SYMBOL(complete_all);
  79
  80 static inline long __sched
  81 do_wait_for_common(struct completion *x,
  82                    long (*action)(long), long timeout, int state)
  83 {
  84         if (!x->done) {
  85                 DECLARE_SWAITQUEUE(wait);
  86
  87                 do {
  88                         if (signal_pending_state(state, current)) {
  89                                 timeout = -ERESTARTSYS;
  90                                 break;
  91                         }
  92                         __prepare_to_swait(&x->wait, &wait);
  93                         __set_current_state(state);
  94                         raw_spin_unlock_irq(&x->wait.lock);
  95                         timeout = action(timeout);
  96                         raw_spin_lock_irq(&x->wait.lock);
  97                 } while (!x->done && timeout);
  98                 __finish_swait(&x->wait, &wait);
  99                 if (!x->done)
 100                         return timeout;
 101         }
 102         if (x->done != UINT_MAX)
 103                 x->done--;
 104         return timeout ?: 1;
 105 }
 106
 107 static inline long __sched
 108 __wait_for_common(struct completion *x,
 109                   long (*action)(long), long timeout, int state)
 110 {
 111         might_sleep();
 112
 113         complete_acquire(x);
 114
 115         raw_spin_lock_irq(&x->wait.lock);
 116         timeout = do_wait_for_common(x, action, timeout, state);
 117         raw_spin_unlock_irq(&x->wait.lock);
 118
 119         complete_release(x);
 120
 121         return timeout;
 122 }
 123
 124 static long __sched
 125 wait_for_common(struct completion *x, long timeout, int state)
 126 {
 127         return __wait_for_common(x, schedule_timeout, timeout, state);
 128 }
 129
 130 static long __sched
 131 wait_for_common_io(struct completion *x, long timeout, int state)
 132 {
 133         return __wait_for_common(x, io_schedule_timeout, timeout, state);
 134 }
 135
 136 /**
 137  * wait_for_completion: - waits for completion of a task
 138  * @x:  holds the state of this particular completion
 139  *
 140  * This waits to be signaled for completion of a specific task. It is NOT
 141  * interruptible and there is no timeout.
 142  *
 143  * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
 144  * and interrupt capability. Also see complete().
 145  */
 146 void __sched wait_for_completion(struct completion *x)
 147 {
 148         wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
 149 }
 150 EXPORT_SYMBOL(wait_for_completion);
 151
 152 /**
 153  * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
 154  * @x:  holds the state of this particular completion
 155  * @timeout:  timeout value in jiffies
 156  *
 157  * This waits for either a completion of a specific task to be signaled or for a
 158  * specified timeout to expire. The timeout is in jiffies. It is not
 159  * interruptible.
 160  *
 161  * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
 162  * till timeout) if completed.
 163  */
 164 unsigned long __sched
 165 wait_for_completion_timeout(struct completion *x, unsigned long timeout)
 166 {
 167         return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
 168 }
 169 EXPORT_SYMBOL(wait_for_completion_timeout);
 170
 171 /**
 172  * wait_for_completion_io: - waits for completion of a task
 173  * @x:  holds the state of this particular completion
 174  *
 175  * This waits to be signaled for completion of a specific task. It is NOT
 176  * interruptible and there is no timeout. The caller is accounted as waiting
 177  * for IO (which traditionally means blkio only).
 178  */
 179 void __sched wait_for_completion_io(struct completion *x)
 180 {
 181         wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
 182 }
 183 EXPORT_SYMBOL(wait_for_completion_io);
 184
 185 /**
 186  * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
 187  * @x:  holds the state of this particular completion
 188  * @timeout:  timeout value in jiffies
 189  *
 190  * This waits for either a completion of a specific task to be signaled or for a
 191  * specified timeout to expire. The timeout is in jiffies. It is not
 192  * interruptible. The caller is accounted as waiting for IO (which traditionally
 193  * means blkio only).
 194  *
 195  * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
 196  * till timeout) if completed.
 197  */
 198 unsigned long __sched
 199 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
 200 {
 201         return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
 202 }
 203 EXPORT_SYMBOL(wait_for_completion_io_timeout);
 204
 205 /**
 206  * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
 207  * @x:  holds the state of this particular completion
 208  *
 209  * This waits for completion of a specific task to be signaled. It is
 210  * interruptible.
 211  *
 212  * Return: -ERESTARTSYS if interrupted, 0 if completed.
 213  */
 214 int __sched wait_for_completion_interruptible(struct completion *x)
 215 {
 216         long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
 217
 218         if (t == -ERESTARTSYS)
 219                 return t;
 220         return 0;
 221 }
 222 EXPORT_SYMBOL(wait_for_completion_interruptible);
 223
 224 /**
 225  * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
 226  * @x:  holds the state of this particular completion
 227  * @timeout:  timeout value in jiffies
 228  *
 229  * This waits for either a completion of a specific task to be signaled or for a
 230  * specified timeout to expire. It is interruptible. The timeout is in jiffies.
 231  *
 232  * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
 233  * or number of jiffies left till timeout) if completed.
 234  */
 235 long __sched
 236 wait_for_completion_interruptible_timeout(struct completion *x,
 237                                           unsigned long timeout)
 238 {
 239         return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
 240 }
 241 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
 242
 243 /**
 244  * wait_for_completion_killable: - waits for completion of a task (killable)
 245  * @x:  holds the state of this particular completion
 246  *
 247  * This waits to be signaled for completion of a specific task. It can be
 248  * interrupted by a kill signal.
 249  *
 250  * Return: -ERESTARTSYS if interrupted, 0 if completed.
 251  */
 252 int __sched wait_for_completion_killable(struct completion *x)
 253 {
 254         long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
 255
 256         if (t == -ERESTARTSYS)
 257                 return t;
 258         return 0;
 259 }
 260 EXPORT_SYMBOL(wait_for_completion_killable);
 261
 262 int __sched wait_for_completion_state(struct completion *x, unsigned int state)
 263 {
 264         long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, state);
 265
 266         if (t == -ERESTARTSYS)
 267                 return t;
 268         return 0;
 269 }
 270 EXPORT_SYMBOL(wait_for_completion_state);
 271
 272 /**
 273  * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
 274  * @x:  holds the state of this particular completion
 275  * @timeout:  timeout value in jiffies
 276  *
 277  * This waits for either a completion of a specific task to be
 278  * signaled or for a specified timeout to expire. It can be
 279  * interrupted by a kill signal. The timeout is in jiffies.
 280  *
 281  * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
 282  * or number of jiffies left till timeout) if completed.
 283  */
 284 long __sched
 285 wait_for_completion_killable_timeout(struct completion *x,
 286                                      unsigned long timeout)
 287 {
 288         return wait_for_common(x, timeout, TASK_KILLABLE);
 289 }
 290 EXPORT_SYMBOL(wait_for_completion_killable_timeout);
 291
 292 /**
 293  *      try_wait_for_completion - try to decrement a completion without blocking
 294  *      @x:     completion structure
 295  *
 296  *      Return: 0 if a decrement cannot be done without blocking
 297  *               1 if a decrement succeeded.
 298  *
 299  *      If a completion is being used as a counting completion,
 300  *      attempt to decrement the counter without blocking. This
 301  *      enables us to avoid waiting if the resource the completion
 302  *      is protecting is not available.
 303  */
 304 bool try_wait_for_completion(struct completion *x)
 305 {
 306         unsigned long flags;
 307         bool ret = true;
 308
 309         /*
 310          * Since x->done will need to be locked only
 311          * in the non-blocking case, we check x->done
 312          * first without taking the lock so we can
 313          * return early in the blocking case.
 314          */
 315         if (!READ_ONCE(x->done))
 316                 return false;
 317
 318         raw_spin_lock_irqsave(&x->wait.lock, flags);
 319         if (!x->done)
 320                 ret = false;
 321         else if (x->done != UINT_MAX)
 322                 x->done--;
 323         raw_spin_unlock_irqrestore(&x->wait.lock, flags);
 324         return ret;
 325 }
 326 EXPORT_SYMBOL(try_wait_for_completion);
 327
 328 /**
 329  *      completion_done - Test to see if a completion has any waiters
 330  *      @x:     completion structure
 331  *
 332  *      Return: 0 if there are waiters (wait_for_completion() in progress)
 333  *               1 if there are no waiters.
 334  *
 335  *      Note, this will always return true if complete_all() was called on @X.
 336  */
 337 bool completion_done(struct completion *x)
 338 {
 339         unsigned long flags;
 340
 341         if (!READ_ONCE(x->done))
 342                 return false;
 343
 344         /*
 345          * If ->done, we need to wait for complete() to release ->wait.lock
 346          * otherwise we can end up freeing the completion before complete()
 347          * is done referencing it.
 348          */
 349         raw_spin_lock_irqsave(&x->wait.lock, flags);
 350         raw_spin_unlock_irqrestore(&x->wait.lock, flags);
 351         return true;
 352 }
 353 EXPORT_SYMBOL(completion_done);