1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/kdebug.h>
3 #include <linux/kprobes.h>
4 #include <linux/export.h>
5 #include <linux/notifier.h>
6 #include <linux/rcupdate.h>
7 #include <linux/vmalloc.h>
8 #include <linux/reboot.h>
11 * Notifier list for kernel code which wants to be called
12 * at shutdown. This is used to stop any idling DMA operations
15 BLOCKING_NOTIFIER_HEAD(reboot_notifier_list
);
18 * Notifier chain core routines. The exported routines below
19 * are layered on top of these, with appropriate locking added.
22 static int notifier_chain_register(struct notifier_block
**nl
,
23 struct notifier_block
*n
)
25 while ((*nl
) != NULL
) {
26 WARN_ONCE(((*nl
) == n
), "double register detected");
27 if (n
->priority
> (*nl
)->priority
)
32 rcu_assign_pointer(*nl
, n
);
36 static int notifier_chain_cond_register(struct notifier_block
**nl
,
37 struct notifier_block
*n
)
39 while ((*nl
) != NULL
) {
42 if (n
->priority
> (*nl
)->priority
)
47 rcu_assign_pointer(*nl
, n
);
51 static int notifier_chain_unregister(struct notifier_block
**nl
,
52 struct notifier_block
*n
)
54 while ((*nl
) != NULL
) {
56 rcu_assign_pointer(*nl
, n
->next
);
65 * notifier_call_chain - Informs the registered notifiers about an event.
66 * @nl: Pointer to head of the blocking notifier chain
67 * @val: Value passed unmodified to notifier function
68 * @v: Pointer passed unmodified to notifier function
69 * @nr_to_call: Number of notifier functions to be called. Don't care
70 * value of this parameter is -1.
71 * @nr_calls: Records the number of notifications sent. Don't care
72 * value of this field is NULL.
73 * @returns: notifier_call_chain returns the value returned by the
74 * last notifier function called.
76 static int notifier_call_chain(struct notifier_block
**nl
,
77 unsigned long val
, void *v
,
78 int nr_to_call
, int *nr_calls
)
80 int ret
= NOTIFY_DONE
;
81 struct notifier_block
*nb
, *next_nb
;
83 nb
= rcu_dereference_raw(*nl
);
85 while (nb
&& nr_to_call
) {
86 next_nb
= rcu_dereference_raw(nb
->next
);
88 #ifdef CONFIG_DEBUG_NOTIFIERS
89 if (unlikely(!func_ptr_is_kernel_text(nb
->notifier_call
))) {
90 WARN(1, "Invalid notifier called!");
95 ret
= nb
->notifier_call(nb
, val
, v
);
100 if (ret
& NOTIFY_STOP_MASK
)
107 NOKPROBE_SYMBOL(notifier_call_chain
);
110 * Atomic notifier chain routines. Registration and unregistration
111 * use a spinlock, and call_chain is synchronized by RCU (no locks).
115 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
116 * @nh: Pointer to head of the atomic notifier chain
117 * @n: New entry in notifier chain
119 * Adds a notifier to an atomic notifier chain.
121 * Currently always returns zero.
123 int atomic_notifier_chain_register(struct atomic_notifier_head
*nh
,
124 struct notifier_block
*n
)
129 spin_lock_irqsave(&nh
->lock
, flags
);
130 ret
= notifier_chain_register(&nh
->head
, n
);
131 spin_unlock_irqrestore(&nh
->lock
, flags
);
134 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register
);
137 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
138 * @nh: Pointer to head of the atomic notifier chain
139 * @n: Entry to remove from notifier chain
141 * Removes a notifier from an atomic notifier chain.
143 * Returns zero on success or %-ENOENT on failure.
145 int atomic_notifier_chain_unregister(struct atomic_notifier_head
*nh
,
146 struct notifier_block
*n
)
151 spin_lock_irqsave(&nh
->lock
, flags
);
152 ret
= notifier_chain_unregister(&nh
->head
, n
);
153 spin_unlock_irqrestore(&nh
->lock
, flags
);
157 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister
);
160 * __atomic_notifier_call_chain - Call functions in an atomic notifier chain
161 * @nh: Pointer to head of the atomic notifier chain
162 * @val: Value passed unmodified to notifier function
163 * @v: Pointer passed unmodified to notifier function
164 * @nr_to_call: See the comment for notifier_call_chain.
165 * @nr_calls: See the comment for notifier_call_chain.
167 * Calls each function in a notifier chain in turn. The functions
168 * run in an atomic context, so they must not block.
169 * This routine uses RCU to synchronize with changes to the chain.
171 * If the return value of the notifier can be and'ed
172 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
173 * will return immediately, with the return value of
174 * the notifier function which halted execution.
175 * Otherwise the return value is the return value
176 * of the last notifier function called.
178 int __atomic_notifier_call_chain(struct atomic_notifier_head
*nh
,
179 unsigned long val
, void *v
,
180 int nr_to_call
, int *nr_calls
)
185 ret
= notifier_call_chain(&nh
->head
, val
, v
, nr_to_call
, nr_calls
);
189 EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain
);
190 NOKPROBE_SYMBOL(__atomic_notifier_call_chain
);
192 int atomic_notifier_call_chain(struct atomic_notifier_head
*nh
,
193 unsigned long val
, void *v
)
195 return __atomic_notifier_call_chain(nh
, val
, v
, -1, NULL
);
197 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain
);
198 NOKPROBE_SYMBOL(atomic_notifier_call_chain
);
201 * Blocking notifier chain routines. All access to the chain is
202 * synchronized by an rwsem.
206 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
207 * @nh: Pointer to head of the blocking notifier chain
208 * @n: New entry in notifier chain
210 * Adds a notifier to a blocking notifier chain.
211 * Must be called in process context.
213 * Currently always returns zero.
215 int blocking_notifier_chain_register(struct blocking_notifier_head
*nh
,
216 struct notifier_block
*n
)
221 * This code gets used during boot-up, when task switching is
222 * not yet working and interrupts must remain disabled. At
223 * such times we must not call down_write().
225 if (unlikely(system_state
== SYSTEM_BOOTING
))
226 return notifier_chain_register(&nh
->head
, n
);
228 down_write(&nh
->rwsem
);
229 ret
= notifier_chain_register(&nh
->head
, n
);
230 up_write(&nh
->rwsem
);
233 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register
);
236 * blocking_notifier_chain_cond_register - Cond add notifier to a blocking notifier chain
237 * @nh: Pointer to head of the blocking notifier chain
238 * @n: New entry in notifier chain
240 * Adds a notifier to a blocking notifier chain, only if not already
241 * present in the chain.
242 * Must be called in process context.
244 * Currently always returns zero.
246 int blocking_notifier_chain_cond_register(struct blocking_notifier_head
*nh
,
247 struct notifier_block
*n
)
251 down_write(&nh
->rwsem
);
252 ret
= notifier_chain_cond_register(&nh
->head
, n
);
253 up_write(&nh
->rwsem
);
256 EXPORT_SYMBOL_GPL(blocking_notifier_chain_cond_register
);
259 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
260 * @nh: Pointer to head of the blocking notifier chain
261 * @n: Entry to remove from notifier chain
263 * Removes a notifier from a blocking notifier chain.
264 * Must be called from process context.
266 * Returns zero on success or %-ENOENT on failure.
268 int blocking_notifier_chain_unregister(struct blocking_notifier_head
*nh
,
269 struct notifier_block
*n
)
274 * This code gets used during boot-up, when task switching is
275 * not yet working and interrupts must remain disabled. At
276 * such times we must not call down_write().
278 if (unlikely(system_state
== SYSTEM_BOOTING
))
279 return notifier_chain_unregister(&nh
->head
, n
);
281 down_write(&nh
->rwsem
);
282 ret
= notifier_chain_unregister(&nh
->head
, n
);
283 up_write(&nh
->rwsem
);
286 EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister
);
289 * __blocking_notifier_call_chain - Call functions in a blocking notifier chain
290 * @nh: Pointer to head of the blocking notifier chain
291 * @val: Value passed unmodified to notifier function
292 * @v: Pointer passed unmodified to notifier function
293 * @nr_to_call: See comment for notifier_call_chain.
294 * @nr_calls: See comment for notifier_call_chain.
296 * Calls each function in a notifier chain in turn. The functions
297 * run in a process context, so they are allowed to block.
299 * If the return value of the notifier can be and'ed
300 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
301 * will return immediately, with the return value of
302 * the notifier function which halted execution.
303 * Otherwise the return value is the return value
304 * of the last notifier function called.
306 int __blocking_notifier_call_chain(struct blocking_notifier_head
*nh
,
307 unsigned long val
, void *v
,
308 int nr_to_call
, int *nr_calls
)
310 int ret
= NOTIFY_DONE
;
313 * We check the head outside the lock, but if this access is
314 * racy then it does not matter what the result of the test
315 * is, we re-check the list after having taken the lock anyway:
317 if (rcu_access_pointer(nh
->head
)) {
318 down_read(&nh
->rwsem
);
319 ret
= notifier_call_chain(&nh
->head
, val
, v
, nr_to_call
,
325 EXPORT_SYMBOL_GPL(__blocking_notifier_call_chain
);
327 int blocking_notifier_call_chain(struct blocking_notifier_head
*nh
,
328 unsigned long val
, void *v
)
330 return __blocking_notifier_call_chain(nh
, val
, v
, -1, NULL
);
332 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain
);
335 * Raw notifier chain routines. There is no protection;
336 * the caller must provide it. Use at your own risk!
340 * raw_notifier_chain_register - Add notifier to a raw notifier chain
341 * @nh: Pointer to head of the raw notifier chain
342 * @n: New entry in notifier chain
344 * Adds a notifier to a raw notifier chain.
345 * All locking must be provided by the caller.
347 * Currently always returns zero.
349 int raw_notifier_chain_register(struct raw_notifier_head
*nh
,
350 struct notifier_block
*n
)
352 return notifier_chain_register(&nh
->head
, n
);
354 EXPORT_SYMBOL_GPL(raw_notifier_chain_register
);
357 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
358 * @nh: Pointer to head of the raw notifier chain
359 * @n: Entry to remove from notifier chain
361 * Removes a notifier from a raw notifier chain.
362 * All locking must be provided by the caller.
364 * Returns zero on success or %-ENOENT on failure.
366 int raw_notifier_chain_unregister(struct raw_notifier_head
*nh
,
367 struct notifier_block
*n
)
369 return notifier_chain_unregister(&nh
->head
, n
);
371 EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister
);
374 * __raw_notifier_call_chain - Call functions in a raw notifier chain
375 * @nh: Pointer to head of the raw notifier chain
376 * @val: Value passed unmodified to notifier function
377 * @v: Pointer passed unmodified to notifier function
378 * @nr_to_call: See comment for notifier_call_chain.
379 * @nr_calls: See comment for notifier_call_chain
381 * Calls each function in a notifier chain in turn. The functions
382 * run in an undefined context.
383 * All locking must be provided by the caller.
385 * If the return value of the notifier can be and'ed
386 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
387 * will return immediately, with the return value of
388 * the notifier function which halted execution.
389 * Otherwise the return value is the return value
390 * of the last notifier function called.
392 int __raw_notifier_call_chain(struct raw_notifier_head
*nh
,
393 unsigned long val
, void *v
,
394 int nr_to_call
, int *nr_calls
)
396 return notifier_call_chain(&nh
->head
, val
, v
, nr_to_call
, nr_calls
);
398 EXPORT_SYMBOL_GPL(__raw_notifier_call_chain
);
400 int raw_notifier_call_chain(struct raw_notifier_head
*nh
,
401 unsigned long val
, void *v
)
403 return __raw_notifier_call_chain(nh
, val
, v
, -1, NULL
);
405 EXPORT_SYMBOL_GPL(raw_notifier_call_chain
);
409 * SRCU notifier chain routines. Registration and unregistration
410 * use a mutex, and call_chain is synchronized by SRCU (no locks).
414 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
415 * @nh: Pointer to head of the SRCU notifier chain
416 * @n: New entry in notifier chain
418 * Adds a notifier to an SRCU notifier chain.
419 * Must be called in process context.
421 * Currently always returns zero.
423 int srcu_notifier_chain_register(struct srcu_notifier_head
*nh
,
424 struct notifier_block
*n
)
429 * This code gets used during boot-up, when task switching is
430 * not yet working and interrupts must remain disabled. At
431 * such times we must not call mutex_lock().
433 if (unlikely(system_state
== SYSTEM_BOOTING
))
434 return notifier_chain_register(&nh
->head
, n
);
436 mutex_lock(&nh
->mutex
);
437 ret
= notifier_chain_register(&nh
->head
, n
);
438 mutex_unlock(&nh
->mutex
);
441 EXPORT_SYMBOL_GPL(srcu_notifier_chain_register
);
444 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
445 * @nh: Pointer to head of the SRCU notifier chain
446 * @n: Entry to remove from notifier chain
448 * Removes a notifier from an SRCU notifier chain.
449 * Must be called from process context.
451 * Returns zero on success or %-ENOENT on failure.
453 int srcu_notifier_chain_unregister(struct srcu_notifier_head
*nh
,
454 struct notifier_block
*n
)
459 * This code gets used during boot-up, when task switching is
460 * not yet working and interrupts must remain disabled. At
461 * such times we must not call mutex_lock().
463 if (unlikely(system_state
== SYSTEM_BOOTING
))
464 return notifier_chain_unregister(&nh
->head
, n
);
466 mutex_lock(&nh
->mutex
);
467 ret
= notifier_chain_unregister(&nh
->head
, n
);
468 mutex_unlock(&nh
->mutex
);
469 synchronize_srcu(&nh
->srcu
);
472 EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister
);
475 * __srcu_notifier_call_chain - Call functions in an SRCU notifier chain
476 * @nh: Pointer to head of the SRCU notifier chain
477 * @val: Value passed unmodified to notifier function
478 * @v: Pointer passed unmodified to notifier function
479 * @nr_to_call: See comment for notifier_call_chain.
480 * @nr_calls: See comment for notifier_call_chain
482 * Calls each function in a notifier chain in turn. The functions
483 * run in a process context, so they are allowed to block.
485 * If the return value of the notifier can be and'ed
486 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
487 * will return immediately, with the return value of
488 * the notifier function which halted execution.
489 * Otherwise the return value is the return value
490 * of the last notifier function called.
492 int __srcu_notifier_call_chain(struct srcu_notifier_head
*nh
,
493 unsigned long val
, void *v
,
494 int nr_to_call
, int *nr_calls
)
499 idx
= srcu_read_lock(&nh
->srcu
);
500 ret
= notifier_call_chain(&nh
->head
, val
, v
, nr_to_call
, nr_calls
);
501 srcu_read_unlock(&nh
->srcu
, idx
);
504 EXPORT_SYMBOL_GPL(__srcu_notifier_call_chain
);
506 int srcu_notifier_call_chain(struct srcu_notifier_head
*nh
,
507 unsigned long val
, void *v
)
509 return __srcu_notifier_call_chain(nh
, val
, v
, -1, NULL
);
511 EXPORT_SYMBOL_GPL(srcu_notifier_call_chain
);
514 * srcu_init_notifier_head - Initialize an SRCU notifier head
515 * @nh: Pointer to head of the srcu notifier chain
517 * Unlike other sorts of notifier heads, SRCU notifier heads require
518 * dynamic initialization. Be sure to call this routine before
519 * calling any of the other SRCU notifier routines for this head.
521 * If an SRCU notifier head is deallocated, it must first be cleaned
522 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
523 * per-cpu data (used by the SRCU mechanism) will leak.
525 void srcu_init_notifier_head(struct srcu_notifier_head
*nh
)
527 mutex_init(&nh
->mutex
);
528 if (init_srcu_struct(&nh
->srcu
) < 0)
532 EXPORT_SYMBOL_GPL(srcu_init_notifier_head
);
534 #endif /* CONFIG_SRCU */
536 static ATOMIC_NOTIFIER_HEAD(die_chain
);
538 int notrace
notify_die(enum die_val val
, const char *str
,
539 struct pt_regs
*regs
, long err
, int trap
, int sig
)
541 struct die_args args
= {
549 RCU_LOCKDEP_WARN(!rcu_is_watching(),
550 "notify_die called but RCU thinks we're quiescent");
551 return atomic_notifier_call_chain(&die_chain
, val
, &args
);
553 NOKPROBE_SYMBOL(notify_die
);
555 int register_die_notifier(struct notifier_block
*nb
)
557 vmalloc_sync_mappings();
558 return atomic_notifier_chain_register(&die_chain
, nb
);
560 EXPORT_SYMBOL_GPL(register_die_notifier
);
562 int unregister_die_notifier(struct notifier_block
*nb
)
564 return atomic_notifier_chain_unregister(&die_chain
, nb
);
566 EXPORT_SYMBOL_GPL(unregister_die_notifier
);