include/asm-generic/barrier.h

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 /*
   3  * Generic barrier definitions.
   4  *
   5  * It should be possible to use these on really simple architectures,
   6  * but it serves more as a starting point for new ports.
   7  *
   8  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   9  * Written by David Howells (dhowells@redhat.com)
  10  */
  11 #ifndef __ASM_GENERIC_BARRIER_H
  12 #define __ASM_GENERIC_BARRIER_H
  13
  14 #ifndef __ASSEMBLY__
  15
  16 #include <linux/compiler.h>
  17 #include <asm/rwonce.h>
  18
  19 #ifndef nop
  20 #define nop()   asm volatile ("nop")
  21 #endif
  22
  23 /*
  24  * Force strict CPU ordering. And yes, this is required on UP too when we're
  25  * talking to devices.
  26  *
  27  * Fall back to compiler barriers if nothing better is provided.
  28  */
  29
  30 #ifndef mb
  31 #define mb()    barrier()
  32 #endif
  33
  34 #ifndef rmb
  35 #define rmb()   mb()
  36 #endif
  37
  38 #ifndef wmb
  39 #define wmb()   mb()
  40 #endif
  41
  42 #ifndef dma_rmb
  43 #define dma_rmb()       rmb()
  44 #endif
  45
  46 #ifndef dma_wmb
  47 #define dma_wmb()       wmb()
  48 #endif
  49
  50 #ifndef __smp_mb
  51 #define __smp_mb()      mb()
  52 #endif
  53
  54 #ifndef __smp_rmb
  55 #define __smp_rmb()     rmb()
  56 #endif
  57
  58 #ifndef __smp_wmb
  59 #define __smp_wmb()     wmb()
  60 #endif
  61
  62 #ifdef CONFIG_SMP
  63
  64 #ifndef smp_mb
  65 #define smp_mb()        __smp_mb()
  66 #endif
  67
  68 #ifndef smp_rmb
  69 #define smp_rmb()       __smp_rmb()
  70 #endif
  71
  72 #ifndef smp_wmb
  73 #define smp_wmb()       __smp_wmb()
  74 #endif
  75
  76 #else   /* !CONFIG_SMP */
  77
  78 #ifndef smp_mb
  79 #define smp_mb()        barrier()
  80 #endif
  81
  82 #ifndef smp_rmb
  83 #define smp_rmb()       barrier()
  84 #endif
  85
  86 #ifndef smp_wmb
  87 #define smp_wmb()       barrier()
  88 #endif
  89
  90 #endif  /* CONFIG_SMP */
  91
  92 #ifndef __smp_store_mb
  93 #define __smp_store_mb(var, value)  do { WRITE_ONCE(var, value); __smp_mb(); } while (0)
  94 #endif
  95
  96 #ifndef __smp_mb__before_atomic
  97 #define __smp_mb__before_atomic()       __smp_mb()
  98 #endif
  99
 100 #ifndef __smp_mb__after_atomic
 101 #define __smp_mb__after_atomic()        __smp_mb()
 102 #endif
 103
 104 #ifndef __smp_store_release
 105 #define __smp_store_release(p, v)                                       \
 106 do {                                                                    \
 107         compiletime_assert_atomic_type(*p);                             \
 108         __smp_mb();                                                     \
 109         WRITE_ONCE(*p, v);                                              \
 110 } while (0)
 111 #endif
 112
 113 #ifndef __smp_load_acquire
 114 #define __smp_load_acquire(p)                                           \
 115 ({                                                                      \
 116         __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p);               \
 117         compiletime_assert_atomic_type(*p);                             \
 118         __smp_mb();                                                     \
 119         (typeof(*p))___p1;                                              \
 120 })
 121 #endif
 122
 123 #ifdef CONFIG_SMP
 124
 125 #ifndef smp_store_mb
 126 #define smp_store_mb(var, value)  __smp_store_mb(var, value)
 127 #endif
 128
 129 #ifndef smp_mb__before_atomic
 130 #define smp_mb__before_atomic() __smp_mb__before_atomic()
 131 #endif
 132
 133 #ifndef smp_mb__after_atomic
 134 #define smp_mb__after_atomic()  __smp_mb__after_atomic()
 135 #endif
 136
 137 #ifndef smp_store_release
 138 #define smp_store_release(p, v) __smp_store_release(p, v)
 139 #endif
 140
 141 #ifndef smp_load_acquire
 142 #define smp_load_acquire(p) __smp_load_acquire(p)
 143 #endif
 144
 145 #else   /* !CONFIG_SMP */
 146
 147 #ifndef smp_store_mb
 148 #define smp_store_mb(var, value)  do { WRITE_ONCE(var, value); barrier(); } while (0)
 149 #endif
 150
 151 #ifndef smp_mb__before_atomic
 152 #define smp_mb__before_atomic() barrier()
 153 #endif
 154
 155 #ifndef smp_mb__after_atomic
 156 #define smp_mb__after_atomic()  barrier()
 157 #endif
 158
 159 #ifndef smp_store_release
 160 #define smp_store_release(p, v)                                         \
 161 do {                                                                    \
 162         compiletime_assert_atomic_type(*p);                             \
 163         barrier();                                                      \
 164         WRITE_ONCE(*p, v);                                              \
 165 } while (0)
 166 #endif
 167
 168 #ifndef smp_load_acquire
 169 #define smp_load_acquire(p)                                             \
 170 ({                                                                      \
 171         __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p);               \
 172         compiletime_assert_atomic_type(*p);                             \
 173         barrier();                                                      \
 174         (typeof(*p))___p1;                                              \
 175 })
 176 #endif
 177
 178 #endif  /* CONFIG_SMP */
 179
 180 /* Barriers for virtual machine guests when talking to an SMP host */
 181 #define virt_mb() __smp_mb()
 182 #define virt_rmb() __smp_rmb()
 183 #define virt_wmb() __smp_wmb()
 184 #define virt_store_mb(var, value) __smp_store_mb(var, value)
 185 #define virt_mb__before_atomic() __smp_mb__before_atomic()
 186 #define virt_mb__after_atomic() __smp_mb__after_atomic()
 187 #define virt_store_release(p, v) __smp_store_release(p, v)
 188 #define virt_load_acquire(p) __smp_load_acquire(p)
 189
 190 /**
 191  * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency
 192  *
 193  * A control dependency provides a LOAD->STORE order, the additional RMB
 194  * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
 195  * aka. (load)-ACQUIRE.
 196  *
 197  * Architectures that do not do load speculation can have this be barrier().
 198  */
 199 #ifndef smp_acquire__after_ctrl_dep
 200 #define smp_acquire__after_ctrl_dep()           smp_rmb()
 201 #endif
 202
 203 /**
 204  * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
 205  * @ptr: pointer to the variable to wait on
 206  * @cond: boolean expression to wait for
 207  *
 208  * Equivalent to using READ_ONCE() on the condition variable.
 209  *
 210  * Due to C lacking lambda expressions we load the value of *ptr into a
 211  * pre-named variable @VAL to be used in @cond.
 212  */
 213 #ifndef smp_cond_load_relaxed
 214 #define smp_cond_load_relaxed(ptr, cond_expr) ({                \
 215         typeof(ptr) __PTR = (ptr);                              \
 216         __unqual_scalar_typeof(*ptr) VAL;                       \
 217         for (;;) {                                              \
 218                 VAL = READ_ONCE(*__PTR);                        \
 219                 if (cond_expr)                                  \
 220                         break;                                  \
 221                 cpu_relax();                                    \
 222         }                                                       \
 223         (typeof(*ptr))VAL;                                      \
 224 })
 225 #endif
 226
 227 /**
 228  * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
 229  * @ptr: pointer to the variable to wait on
 230  * @cond: boolean expression to wait for
 231  *
 232  * Equivalent to using smp_load_acquire() on the condition variable but employs
 233  * the control dependency of the wait to reduce the barrier on many platforms.
 234  */
 235 #ifndef smp_cond_load_acquire
 236 #define smp_cond_load_acquire(ptr, cond_expr) ({                \
 237         __unqual_scalar_typeof(*ptr) _val;                      \
 238         _val = smp_cond_load_relaxed(ptr, cond_expr);           \
 239         smp_acquire__after_ctrl_dep();                          \
 240         (typeof(*ptr))_val;                                     \
 241 })
 242 #endif
 243
 244 /*
 245  * pmem_wmb() ensures that all stores for which the modification
 246  * are written to persistent storage by preceding instructions have
 247  * updated persistent storage before any data  access or data transfer
 248  * caused by subsequent instructions is initiated.
 249  */
 250 #ifndef pmem_wmb
 251 #define pmem_wmb()      wmb()
 252 #endif
 253
 254 #endif /* !__ASSEMBLY__ */
 255 #endif /* __ASM_GENERIC_BARRIER_H */