1 // SPDX-License-Identifier: GPL-2.0-only
4 * The implementation of the wait_bit*() and related waiting APIs:
7 #define WAIT_TABLE_BITS 8
8 #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)
10 static wait_queue_head_t bit_wait_table
[WAIT_TABLE_SIZE
] __cacheline_aligned
;
12 wait_queue_head_t
*bit_waitqueue(void *word
, int bit
)
14 const int shift
= BITS_PER_LONG
== 32 ? 5 : 6;
15 unsigned long val
= (unsigned long)word
<< shift
| bit
;
17 return bit_wait_table
+ hash_long(val
, WAIT_TABLE_BITS
);
19 EXPORT_SYMBOL(bit_waitqueue
);
21 int wake_bit_function(struct wait_queue_entry
*wq_entry
, unsigned mode
, int sync
, void *arg
)
23 struct wait_bit_key
*key
= arg
;
24 struct wait_bit_queue_entry
*wait_bit
= container_of(wq_entry
, struct wait_bit_queue_entry
, wq_entry
);
26 if (wait_bit
->key
.flags
!= key
->flags
||
27 wait_bit
->key
.bit_nr
!= key
->bit_nr
||
28 test_bit(key
->bit_nr
, key
->flags
))
31 return autoremove_wake_function(wq_entry
, mode
, sync
, key
);
33 EXPORT_SYMBOL(wake_bit_function
);
36 * To allow interruptible waiting and asynchronous (i.e. non-blocking)
37 * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are
38 * permitted return codes. Nonzero return codes halt waiting and return.
41 __wait_on_bit(struct wait_queue_head
*wq_head
, struct wait_bit_queue_entry
*wbq_entry
,
42 wait_bit_action_f
*action
, unsigned mode
)
47 prepare_to_wait(wq_head
, &wbq_entry
->wq_entry
, mode
);
48 if (test_bit(wbq_entry
->key
.bit_nr
, wbq_entry
->key
.flags
))
49 ret
= (*action
)(&wbq_entry
->key
, mode
);
50 } while (test_bit_acquire(wbq_entry
->key
.bit_nr
, wbq_entry
->key
.flags
) && !ret
);
52 finish_wait(wq_head
, &wbq_entry
->wq_entry
);
56 EXPORT_SYMBOL(__wait_on_bit
);
58 int __sched
out_of_line_wait_on_bit(void *word
, int bit
,
59 wait_bit_action_f
*action
, unsigned mode
)
61 struct wait_queue_head
*wq_head
= bit_waitqueue(word
, bit
);
62 DEFINE_WAIT_BIT(wq_entry
, word
, bit
);
64 return __wait_on_bit(wq_head
, &wq_entry
, action
, mode
);
66 EXPORT_SYMBOL(out_of_line_wait_on_bit
);
68 int __sched
out_of_line_wait_on_bit_timeout(
69 void *word
, int bit
, wait_bit_action_f
*action
,
70 unsigned mode
, unsigned long timeout
)
72 struct wait_queue_head
*wq_head
= bit_waitqueue(word
, bit
);
73 DEFINE_WAIT_BIT(wq_entry
, word
, bit
);
75 wq_entry
.key
.timeout
= jiffies
+ timeout
;
77 return __wait_on_bit(wq_head
, &wq_entry
, action
, mode
);
79 EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout
);
82 __wait_on_bit_lock(struct wait_queue_head
*wq_head
, struct wait_bit_queue_entry
*wbq_entry
,
83 wait_bit_action_f
*action
, unsigned mode
)
88 prepare_to_wait_exclusive(wq_head
, &wbq_entry
->wq_entry
, mode
);
89 if (test_bit(wbq_entry
->key
.bit_nr
, wbq_entry
->key
.flags
)) {
90 ret
= action(&wbq_entry
->key
, mode
);
92 * See the comment in prepare_to_wait_event().
93 * finish_wait() does not necessarily takes wwq_head->lock,
94 * but test_and_set_bit() implies mb() which pairs with
95 * smp_mb__after_atomic() before wake_up_page().
98 finish_wait(wq_head
, &wbq_entry
->wq_entry
);
100 if (!test_and_set_bit(wbq_entry
->key
.bit_nr
, wbq_entry
->key
.flags
)) {
102 finish_wait(wq_head
, &wbq_entry
->wq_entry
);
109 EXPORT_SYMBOL(__wait_on_bit_lock
);
111 int __sched
out_of_line_wait_on_bit_lock(void *word
, int bit
,
112 wait_bit_action_f
*action
, unsigned mode
)
114 struct wait_queue_head
*wq_head
= bit_waitqueue(word
, bit
);
115 DEFINE_WAIT_BIT(wq_entry
, word
, bit
);
117 return __wait_on_bit_lock(wq_head
, &wq_entry
, action
, mode
);
119 EXPORT_SYMBOL(out_of_line_wait_on_bit_lock
);
121 void __wake_up_bit(struct wait_queue_head
*wq_head
, void *word
, int bit
)
123 struct wait_bit_key key
= __WAIT_BIT_KEY_INITIALIZER(word
, bit
);
125 if (waitqueue_active(wq_head
))
126 __wake_up(wq_head
, TASK_NORMAL
, 1, &key
);
128 EXPORT_SYMBOL(__wake_up_bit
);
131 * wake_up_bit - wake up a waiter on a bit
132 * @word: the word being waited on, a kernel virtual address
133 * @bit: the bit of the word being waited on
135 * There is a standard hashed waitqueue table for generic use. This
136 * is the part of the hash-table's accessor API that wakes up waiters
137 * on a bit. For instance, if one were to have waiters on a bitflag,
138 * one would call wake_up_bit() after clearing the bit.
140 * In order for this to function properly, as it uses waitqueue_active()
141 * internally, some kind of memory barrier must be done prior to calling
142 * this. Typically, this will be smp_mb__after_atomic(), but in some
143 * cases where bitflags are manipulated non-atomically under a lock, one
144 * may need to use a less regular barrier, such fs/inode.c's smp_mb(),
145 * because spin_unlock() does not guarantee a memory barrier.
147 void wake_up_bit(void *word
, int bit
)
149 __wake_up_bit(bit_waitqueue(word
, bit
), word
, bit
);
151 EXPORT_SYMBOL(wake_up_bit
);
153 wait_queue_head_t
*__var_waitqueue(void *p
)
155 return bit_wait_table
+ hash_ptr(p
, WAIT_TABLE_BITS
);
157 EXPORT_SYMBOL(__var_waitqueue
);
160 var_wake_function(struct wait_queue_entry
*wq_entry
, unsigned int mode
,
163 struct wait_bit_key
*key
= arg
;
164 struct wait_bit_queue_entry
*wbq_entry
=
165 container_of(wq_entry
, struct wait_bit_queue_entry
, wq_entry
);
167 if (wbq_entry
->key
.flags
!= key
->flags
||
168 wbq_entry
->key
.bit_nr
!= key
->bit_nr
)
171 return autoremove_wake_function(wq_entry
, mode
, sync
, key
);
174 void init_wait_var_entry(struct wait_bit_queue_entry
*wbq_entry
, void *var
, int flags
)
176 *wbq_entry
= (struct wait_bit_queue_entry
){
184 .func
= var_wake_function
,
185 .entry
= LIST_HEAD_INIT(wbq_entry
->wq_entry
.entry
),
189 EXPORT_SYMBOL(init_wait_var_entry
);
191 void wake_up_var(void *var
)
193 __wake_up_bit(__var_waitqueue(var
), var
, -1);
195 EXPORT_SYMBOL(wake_up_var
);
197 __sched
int bit_wait(struct wait_bit_key
*word
, int mode
)
200 if (signal_pending_state(mode
, current
))
205 EXPORT_SYMBOL(bit_wait
);
207 __sched
int bit_wait_io(struct wait_bit_key
*word
, int mode
)
210 if (signal_pending_state(mode
, current
))
215 EXPORT_SYMBOL(bit_wait_io
);
217 __sched
int bit_wait_timeout(struct wait_bit_key
*word
, int mode
)
219 unsigned long now
= READ_ONCE(jiffies
);
221 if (time_after_eq(now
, word
->timeout
))
223 schedule_timeout(word
->timeout
- now
);
224 if (signal_pending_state(mode
, current
))
229 EXPORT_SYMBOL_GPL(bit_wait_timeout
);
231 __sched
int bit_wait_io_timeout(struct wait_bit_key
*word
, int mode
)
233 unsigned long now
= READ_ONCE(jiffies
);
235 if (time_after_eq(now
, word
->timeout
))
237 io_schedule_timeout(word
->timeout
- now
);
238 if (signal_pending_state(mode
, current
))
243 EXPORT_SYMBOL_GPL(bit_wait_io_timeout
);
245 void __init
wait_bit_init(void)
249 for (i
= 0; i
< WAIT_TABLE_SIZE
; i
++)
250 init_waitqueue_head(bit_wait_table
+ i
);