include/asm-generic/barrier.h

   1 /*
   2  * Generic barrier definitions.
   3  *
   4  * It should be possible to use these on really simple architectures,
   5  * but it serves more as a starting point for new ports.
   6  *
   7  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   8  * Written by David Howells (dhowells@redhat.com)
   9  *
  10  * This program is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU General Public Licence
  12  * as published by the Free Software Foundation; either version
  13  * 2 of the Licence, or (at your option) any later version.
  14  */
  15 #ifndef __ASM_GENERIC_BARRIER_H
  16 #define __ASM_GENERIC_BARRIER_H
  17
  18 #ifndef __ASSEMBLY__
  19
  20 #include <linux/compiler.h>
  21
  22 #ifndef nop
  23 #define nop()   asm volatile ("nop")
  24 #endif
  25
  26 /*
  27  * Force strict CPU ordering. And yes, this is required on UP too when we're
  28  * talking to devices.
  29  *
  30  * Fall back to compiler barriers if nothing better is provided.
  31  */
  32
  33 #ifndef mb
  34 #define mb()    barrier()
  35 #endif
  36
  37 #ifndef rmb
  38 #define rmb()   mb()
  39 #endif
  40
  41 #ifndef wmb
  42 #define wmb()   mb()
  43 #endif
  44
  45 #ifndef dma_rmb
  46 #define dma_rmb()       rmb()
  47 #endif
  48
  49 #ifndef dma_wmb
  50 #define dma_wmb()       wmb()
  51 #endif
  52
  53 #ifndef read_barrier_depends
  54 #define read_barrier_depends()          do { } while (0)
  55 #endif
  56
  57 #ifndef __smp_mb
  58 #define __smp_mb()      mb()
  59 #endif
  60
  61 #ifndef __smp_rmb
  62 #define __smp_rmb()     rmb()
  63 #endif
  64
  65 #ifndef __smp_wmb
  66 #define __smp_wmb()     wmb()
  67 #endif
  68
  69 #ifndef __smp_read_barrier_depends
  70 #define __smp_read_barrier_depends()    read_barrier_depends()
  71 #endif
  72
  73 #ifdef CONFIG_SMP
  74
  75 #ifndef smp_mb
  76 #define smp_mb()        __smp_mb()
  77 #endif
  78
  79 #ifndef smp_rmb
  80 #define smp_rmb()       __smp_rmb()
  81 #endif
  82
  83 #ifndef smp_wmb
  84 #define smp_wmb()       __smp_wmb()
  85 #endif
  86
  87 #ifndef smp_read_barrier_depends
  88 #define smp_read_barrier_depends()      __smp_read_barrier_depends()
  89 #endif
  90
  91 #else   /* !CONFIG_SMP */
  92
  93 #ifndef smp_mb
  94 #define smp_mb()        barrier()
  95 #endif
  96
  97 #ifndef smp_rmb
  98 #define smp_rmb()       barrier()
  99 #endif
 100
 101 #ifndef smp_wmb
 102 #define smp_wmb()       barrier()
 103 #endif
 104
 105 #ifndef smp_read_barrier_depends
 106 #define smp_read_barrier_depends()      do { } while (0)
 107 #endif
 108
 109 #endif  /* CONFIG_SMP */
 110
 111 #ifndef __smp_store_mb
 112 #define __smp_store_mb(var, value)  do { WRITE_ONCE(var, value); __smp_mb(); } while (0)
 113 #endif
 114
 115 #ifndef __smp_mb__before_atomic
 116 #define __smp_mb__before_atomic()       __smp_mb()
 117 #endif
 118
 119 #ifndef __smp_mb__after_atomic
 120 #define __smp_mb__after_atomic()        __smp_mb()
 121 #endif
 122
 123 #ifndef __smp_store_release
 124 #define __smp_store_release(p, v)                                       \
 125 do {                                                                    \
 126         compiletime_assert_atomic_type(*p);                             \
 127         __smp_mb();                                                     \
 128         WRITE_ONCE(*p, v);                                              \
 129 } while (0)
 130 #endif
 131
 132 #ifndef __smp_load_acquire
 133 #define __smp_load_acquire(p)                                           \
 134 ({                                                                      \
 135         typeof(*p) ___p1 = READ_ONCE(*p);                               \
 136         compiletime_assert_atomic_type(*p);                             \
 137         __smp_mb();                                                     \
 138         ___p1;                                                          \
 139 })
 140 #endif
 141
 142 #ifdef CONFIG_SMP
 143
 144 #ifndef smp_store_mb
 145 #define smp_store_mb(var, value)  __smp_store_mb(var, value)
 146 #endif
 147
 148 #ifndef smp_mb__before_atomic
 149 #define smp_mb__before_atomic() __smp_mb__before_atomic()
 150 #endif
 151
 152 #ifndef smp_mb__after_atomic
 153 #define smp_mb__after_atomic()  __smp_mb__after_atomic()
 154 #endif
 155
 156 #ifndef smp_store_release
 157 #define smp_store_release(p, v) __smp_store_release(p, v)
 158 #endif
 159
 160 #ifndef smp_load_acquire
 161 #define smp_load_acquire(p) __smp_load_acquire(p)
 162 #endif
 163
 164 #else   /* !CONFIG_SMP */
 165
 166 #ifndef smp_store_mb
 167 #define smp_store_mb(var, value)  do { WRITE_ONCE(var, value); barrier(); } while (0)
 168 #endif
 169
 170 #ifndef smp_mb__before_atomic
 171 #define smp_mb__before_atomic() barrier()
 172 #endif
 173
 174 #ifndef smp_mb__after_atomic
 175 #define smp_mb__after_atomic()  barrier()
 176 #endif
 177
 178 #ifndef smp_store_release
 179 #define smp_store_release(p, v)                                         \
 180 do {                                                                    \
 181         compiletime_assert_atomic_type(*p);                             \
 182         barrier();                                                      \
 183         WRITE_ONCE(*p, v);                                              \
 184 } while (0)
 185 #endif
 186
 187 #ifndef smp_load_acquire
 188 #define smp_load_acquire(p)                                             \
 189 ({                                                                      \
 190         typeof(*p) ___p1 = READ_ONCE(*p);                               \
 191         compiletime_assert_atomic_type(*p);                             \
 192         barrier();                                                      \
 193         ___p1;                                                          \
 194 })
 195 #endif
 196
 197 #endif  /* CONFIG_SMP */
 198
 199 /* Barriers for virtual machine guests when talking to an SMP host */
 200 #define virt_mb() __smp_mb()
 201 #define virt_rmb() __smp_rmb()
 202 #define virt_wmb() __smp_wmb()
 203 #define virt_read_barrier_depends() __smp_read_barrier_depends()
 204 #define virt_store_mb(var, value) __smp_store_mb(var, value)
 205 #define virt_mb__before_atomic() __smp_mb__before_atomic()
 206 #define virt_mb__after_atomic() __smp_mb__after_atomic()
 207 #define virt_store_release(p, v) __smp_store_release(p, v)
 208 #define virt_load_acquire(p) __smp_load_acquire(p)
 209
 210 /**
 211  * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency
 212  *
 213  * A control dependency provides a LOAD->STORE order, the additional RMB
 214  * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
 215  * aka. (load)-ACQUIRE.
 216  *
 217  * Architectures that do not do load speculation can have this be barrier().
 218  */
 219 #ifndef smp_acquire__after_ctrl_dep
 220 #define smp_acquire__after_ctrl_dep()           smp_rmb()
 221 #endif
 222
 223 /**
 224  * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
 225  * @ptr: pointer to the variable to wait on
 226  * @cond: boolean expression to wait for
 227  *
 228  * Equivalent to using READ_ONCE() on the condition variable.
 229  *
 230  * Due to C lacking lambda expressions we load the value of *ptr into a
 231  * pre-named variable @VAL to be used in @cond.
 232  */
 233 #ifndef smp_cond_load_relaxed
 234 #define smp_cond_load_relaxed(ptr, cond_expr) ({                \
 235         typeof(ptr) __PTR = (ptr);                              \
 236         typeof(*ptr) VAL;                                       \
 237         for (;;) {                                              \
 238                 VAL = READ_ONCE(*__PTR);                        \
 239                 if (cond_expr)                                  \
 240                         break;                                  \
 241                 cpu_relax();                                    \
 242         }                                                       \
 243         VAL;                                                    \
 244 })
 245 #endif
 246
 247 /**
 248  * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
 249  * @ptr: pointer to the variable to wait on
 250  * @cond: boolean expression to wait for
 251  *
 252  * Equivalent to using smp_load_acquire() on the condition variable but employs
 253  * the control dependency of the wait to reduce the barrier on many platforms.
 254  */
 255 #ifndef smp_cond_load_acquire
 256 #define smp_cond_load_acquire(ptr, cond_expr) ({                \
 257         typeof(*ptr) _val;                                      \
 258         _val = smp_cond_load_relaxed(ptr, cond_expr);           \
 259         smp_acquire__after_ctrl_dep();                          \
 260         _val;                                                   \
 261 })
 262 #endif
 263
 264 #endif /* !__ASSEMBLY__ */
 265 #endif /* __ASM_GENERIC_BARRIER_H */