include/asm-generic/percpu.h

   1 #ifndef _ASM_GENERIC_PERCPU_H_
   2 #define _ASM_GENERIC_PERCPU_H_
   3
   4 #include <linux/compiler.h>
   5 #include <linux/threads.h>
   6 #include <linux/percpu-defs.h>
   7
   8 #ifdef CONFIG_SMP
   9
  10 /*
  11  * per_cpu_offset() is the offset that has to be added to a
  12  * percpu variable to get to the instance for a certain processor.
  13  *
  14  * Most arches use the __per_cpu_offset array for those offsets but
  15  * some arches have their own ways of determining the offset (x86_64, s390).
  16  */
  17 #ifndef __per_cpu_offset
  18 extern unsigned long __per_cpu_offset[NR_CPUS];
  19
  20 #define per_cpu_offset(x) (__per_cpu_offset[x])
  21 #endif
  22
  23 /*
  24  * Determine the offset for the currently active processor.
  25  * An arch may define __my_cpu_offset to provide a more effective
  26  * means of obtaining the offset to the per cpu variables of the
  27  * current processor.
  28  */
  29 #ifndef __my_cpu_offset
  30 #define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
  31 #endif
  32 #ifdef CONFIG_DEBUG_PREEMPT
  33 #define my_cpu_offset per_cpu_offset(smp_processor_id())
  34 #else
  35 #define my_cpu_offset __my_cpu_offset
  36 #endif
  37
  38 /*
  39  * Arch may define arch_raw_cpu_ptr() to provide more efficient address
  40  * translations for raw_cpu_ptr().
  41  */
  42 #ifndef arch_raw_cpu_ptr
  43 #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
  44 #endif
  45
  46 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
  47 extern void setup_per_cpu_areas(void);
  48 #endif
  49
  50 #endif  /* SMP */
  51
  52 #ifndef PER_CPU_BASE_SECTION
  53 #ifdef CONFIG_SMP
  54 #define PER_CPU_BASE_SECTION ".data..percpu"
  55 #else
  56 #define PER_CPU_BASE_SECTION ".data"
  57 #endif
  58 #endif
  59
  60 #ifndef PER_CPU_ATTRIBUTES
  61 #define PER_CPU_ATTRIBUTES
  62 #endif
  63
  64 #ifndef PER_CPU_DEF_ATTRIBUTES
  65 #define PER_CPU_DEF_ATTRIBUTES
  66 #endif
  67
  68 #define raw_cpu_generic_to_op(pcp, val, op)                             \
  69 do {                                                                    \
  70         *raw_cpu_ptr(&(pcp)) op val;                                    \
  71 } while (0)
  72
  73 #define raw_cpu_generic_add_return(pcp, val)                            \
  74 ({                                                                      \
  75         raw_cpu_add(pcp, val);                                          \
  76         raw_cpu_read(pcp);                                              \
  77 })
  78
  79 #define raw_cpu_generic_xchg(pcp, nval)                                 \
  80 ({                                                                      \
  81         typeof(pcp) __ret;                                              \
  82         __ret = raw_cpu_read(pcp);                                      \
  83         raw_cpu_write(pcp, nval);                                       \
  84         __ret;                                                          \
  85 })
  86
  87 #define raw_cpu_generic_cmpxchg(pcp, oval, nval)                        \
  88 ({                                                                      \
  89         typeof(pcp) __ret;                                              \
  90         __ret = raw_cpu_read(pcp);                                      \
  91         if (__ret == (oval))                                            \
  92                 raw_cpu_write(pcp, nval);                               \
  93         __ret;                                                          \
  94 })
  95
  96 #define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
  97 ({                                                                      \
  98         int __ret = 0;                                                  \
  99         if (raw_cpu_read(pcp1) == (oval1) &&                            \
 100                          raw_cpu_read(pcp2)  == (oval2)) {              \
 101                 raw_cpu_write(pcp1, nval1);                             \
 102                 raw_cpu_write(pcp2, nval2);                             \
 103                 __ret = 1;                                              \
 104         }                                                               \
 105         (__ret);                                                        \
 106 })
 107
 108 #define this_cpu_generic_read(pcp)                                      \
 109 ({                                                                      \
 110         typeof(pcp) __ret;                                              \
 111         preempt_disable();                                              \
 112         __ret = *this_cpu_ptr(&(pcp));                                  \
 113         preempt_enable();                                               \
 114         __ret;                                                          \
 115 })
 116
 117 #define this_cpu_generic_to_op(pcp, val, op)                            \
 118 do {                                                                    \
 119         unsigned long __flags;                                          \
 120         raw_local_irq_save(__flags);                                    \
 121         *raw_cpu_ptr(&(pcp)) op val;                                    \
 122         raw_local_irq_restore(__flags);                                 \
 123 } while (0)
 124
 125 #define this_cpu_generic_add_return(pcp, val)                           \
 126 ({                                                                      \
 127         typeof(pcp) __ret;                                              \
 128         unsigned long __flags;                                          \
 129         raw_local_irq_save(__flags);                                    \
 130         raw_cpu_add(pcp, val);                                          \
 131         __ret = raw_cpu_read(pcp);                                      \
 132         raw_local_irq_restore(__flags);                                 \
 133         __ret;                                                          \
 134 })
 135
 136 #define this_cpu_generic_xchg(pcp, nval)                                \
 137 ({                                                                      \
 138         typeof(pcp) __ret;                                              \
 139         unsigned long __flags;                                          \
 140         raw_local_irq_save(__flags);                                    \
 141         __ret = raw_cpu_read(pcp);                                      \
 142         raw_cpu_write(pcp, nval);                                       \
 143         raw_local_irq_restore(__flags);                                 \
 144         __ret;                                                          \
 145 })
 146
 147 #define this_cpu_generic_cmpxchg(pcp, oval, nval)                       \
 148 ({                                                                      \
 149         typeof(pcp) __ret;                                              \
 150         unsigned long __flags;                                          \
 151         raw_local_irq_save(__flags);                                    \
 152         __ret = raw_cpu_read(pcp);                                      \
 153         if (__ret == (oval))                                            \
 154                 raw_cpu_write(pcp, nval);                               \
 155         raw_local_irq_restore(__flags);                                 \
 156         __ret;                                                          \
 157 })
 158
 159 #define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 160 ({                                                                      \
 161         int __ret;                                                      \
 162         unsigned long __flags;                                          \
 163         raw_local_irq_save(__flags);                                    \
 164         __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2,              \
 165                         oval1, oval2, nval1, nval2);                    \
 166         raw_local_irq_restore(__flags);                                 \
 167         __ret;                                                          \
 168 })
 169
 170 #ifndef raw_cpu_read_1
 171 #define raw_cpu_read_1(pcp)             (*raw_cpu_ptr(&(pcp)))
 172 #endif
 173 #ifndef raw_cpu_read_2
 174 #define raw_cpu_read_2(pcp)             (*raw_cpu_ptr(&(pcp)))
 175 #endif
 176 #ifndef raw_cpu_read_4
 177 #define raw_cpu_read_4(pcp)             (*raw_cpu_ptr(&(pcp)))
 178 #endif
 179 #ifndef raw_cpu_read_8
 180 #define raw_cpu_read_8(pcp)             (*raw_cpu_ptr(&(pcp)))
 181 #endif
 182
 183 #ifndef raw_cpu_write_1
 184 #define raw_cpu_write_1(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 185 #endif
 186 #ifndef raw_cpu_write_2
 187 #define raw_cpu_write_2(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 188 #endif
 189 #ifndef raw_cpu_write_4
 190 #define raw_cpu_write_4(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 191 #endif
 192 #ifndef raw_cpu_write_8
 193 #define raw_cpu_write_8(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 194 #endif
 195
 196 #ifndef raw_cpu_add_1
 197 #define raw_cpu_add_1(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 198 #endif
 199 #ifndef raw_cpu_add_2
 200 #define raw_cpu_add_2(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 201 #endif
 202 #ifndef raw_cpu_add_4
 203 #define raw_cpu_add_4(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 204 #endif
 205 #ifndef raw_cpu_add_8
 206 #define raw_cpu_add_8(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 207 #endif
 208
 209 #ifndef raw_cpu_and_1
 210 #define raw_cpu_and_1(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 211 #endif
 212 #ifndef raw_cpu_and_2
 213 #define raw_cpu_and_2(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 214 #endif
 215 #ifndef raw_cpu_and_4
 216 #define raw_cpu_and_4(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 217 #endif
 218 #ifndef raw_cpu_and_8
 219 #define raw_cpu_and_8(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 220 #endif
 221
 222 #ifndef raw_cpu_or_1
 223 #define raw_cpu_or_1(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 224 #endif
 225 #ifndef raw_cpu_or_2
 226 #define raw_cpu_or_2(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 227 #endif
 228 #ifndef raw_cpu_or_4
 229 #define raw_cpu_or_4(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 230 #endif
 231 #ifndef raw_cpu_or_8
 232 #define raw_cpu_or_8(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 233 #endif
 234
 235 #ifndef raw_cpu_add_return_1
 236 #define raw_cpu_add_return_1(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 237 #endif
 238 #ifndef raw_cpu_add_return_2
 239 #define raw_cpu_add_return_2(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 240 #endif
 241 #ifndef raw_cpu_add_return_4
 242 #define raw_cpu_add_return_4(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 243 #endif
 244 #ifndef raw_cpu_add_return_8
 245 #define raw_cpu_add_return_8(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 246 #endif
 247
 248 #ifndef raw_cpu_xchg_1
 249 #define raw_cpu_xchg_1(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 250 #endif
 251 #ifndef raw_cpu_xchg_2
 252 #define raw_cpu_xchg_2(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 253 #endif
 254 #ifndef raw_cpu_xchg_4
 255 #define raw_cpu_xchg_4(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 256 #endif
 257 #ifndef raw_cpu_xchg_8
 258 #define raw_cpu_xchg_8(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 259 #endif
 260
 261 #ifndef raw_cpu_cmpxchg_1
 262 #define raw_cpu_cmpxchg_1(pcp, oval, nval) \
 263         raw_cpu_generic_cmpxchg(pcp, oval, nval)
 264 #endif
 265 #ifndef raw_cpu_cmpxchg_2
 266 #define raw_cpu_cmpxchg_2(pcp, oval, nval) \
 267         raw_cpu_generic_cmpxchg(pcp, oval, nval)
 268 #endif
 269 #ifndef raw_cpu_cmpxchg_4
 270 #define raw_cpu_cmpxchg_4(pcp, oval, nval) \
 271         raw_cpu_generic_cmpxchg(pcp, oval, nval)
 272 #endif
 273 #ifndef raw_cpu_cmpxchg_8
 274 #define raw_cpu_cmpxchg_8(pcp, oval, nval) \
 275         raw_cpu_generic_cmpxchg(pcp, oval, nval)
 276 #endif
 277
 278 #ifndef raw_cpu_cmpxchg_double_1
 279 #define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 280         raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 281 #endif
 282 #ifndef raw_cpu_cmpxchg_double_2
 283 #define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 284         raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 285 #endif
 286 #ifndef raw_cpu_cmpxchg_double_4
 287 #define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 288         raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 289 #endif
 290 #ifndef raw_cpu_cmpxchg_double_8
 291 #define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 292         raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 293 #endif
 294
 295 #ifndef this_cpu_read_1
 296 #define this_cpu_read_1(pcp)            this_cpu_generic_read(pcp)
 297 #endif
 298 #ifndef this_cpu_read_2
 299 #define this_cpu_read_2(pcp)            this_cpu_generic_read(pcp)
 300 #endif
 301 #ifndef this_cpu_read_4
 302 #define this_cpu_read_4(pcp)            this_cpu_generic_read(pcp)
 303 #endif
 304 #ifndef this_cpu_read_8
 305 #define this_cpu_read_8(pcp)            this_cpu_generic_read(pcp)
 306 #endif
 307
 308 #ifndef this_cpu_write_1
 309 #define this_cpu_write_1(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 310 #endif
 311 #ifndef this_cpu_write_2
 312 #define this_cpu_write_2(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 313 #endif
 314 #ifndef this_cpu_write_4
 315 #define this_cpu_write_4(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 316 #endif
 317 #ifndef this_cpu_write_8
 318 #define this_cpu_write_8(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 319 #endif
 320
 321 #ifndef this_cpu_add_1
 322 #define this_cpu_add_1(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 323 #endif
 324 #ifndef this_cpu_add_2
 325 #define this_cpu_add_2(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 326 #endif
 327 #ifndef this_cpu_add_4
 328 #define this_cpu_add_4(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 329 #endif
 330 #ifndef this_cpu_add_8
 331 #define this_cpu_add_8(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 332 #endif
 333
 334 #ifndef this_cpu_and_1
 335 #define this_cpu_and_1(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 336 #endif
 337 #ifndef this_cpu_and_2
 338 #define this_cpu_and_2(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 339 #endif
 340 #ifndef this_cpu_and_4
 341 #define this_cpu_and_4(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 342 #endif
 343 #ifndef this_cpu_and_8
 344 #define this_cpu_and_8(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 345 #endif
 346
 347 #ifndef this_cpu_or_1
 348 #define this_cpu_or_1(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 349 #endif
 350 #ifndef this_cpu_or_2
 351 #define this_cpu_or_2(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 352 #endif
 353 #ifndef this_cpu_or_4
 354 #define this_cpu_or_4(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 355 #endif
 356 #ifndef this_cpu_or_8
 357 #define this_cpu_or_8(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 358 #endif
 359
 360 #ifndef this_cpu_add_return_1
 361 #define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
 362 #endif
 363 #ifndef this_cpu_add_return_2
 364 #define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
 365 #endif
 366 #ifndef this_cpu_add_return_4
 367 #define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
 368 #endif
 369 #ifndef this_cpu_add_return_8
 370 #define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
 371 #endif
 372
 373 #ifndef this_cpu_xchg_1
 374 #define this_cpu_xchg_1(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 375 #endif
 376 #ifndef this_cpu_xchg_2
 377 #define this_cpu_xchg_2(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 378 #endif
 379 #ifndef this_cpu_xchg_4
 380 #define this_cpu_xchg_4(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 381 #endif
 382 #ifndef this_cpu_xchg_8
 383 #define this_cpu_xchg_8(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 384 #endif
 385
 386 #ifndef this_cpu_cmpxchg_1
 387 #define this_cpu_cmpxchg_1(pcp, oval, nval) \
 388         this_cpu_generic_cmpxchg(pcp, oval, nval)
 389 #endif
 390 #ifndef this_cpu_cmpxchg_2
 391 #define this_cpu_cmpxchg_2(pcp, oval, nval) \
 392         this_cpu_generic_cmpxchg(pcp, oval, nval)
 393 #endif
 394 #ifndef this_cpu_cmpxchg_4
 395 #define this_cpu_cmpxchg_4(pcp, oval, nval) \
 396         this_cpu_generic_cmpxchg(pcp, oval, nval)
 397 #endif
 398 #ifndef this_cpu_cmpxchg_8
 399 #define this_cpu_cmpxchg_8(pcp, oval, nval) \
 400         this_cpu_generic_cmpxchg(pcp, oval, nval)
 401 #endif
 402
 403 #ifndef this_cpu_cmpxchg_double_1
 404 #define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 405         this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 406 #endif
 407 #ifndef this_cpu_cmpxchg_double_2
 408 #define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 409         this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 410 #endif
 411 #ifndef this_cpu_cmpxchg_double_4
 412 #define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 413         this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 414 #endif
 415 #ifndef this_cpu_cmpxchg_double_8
 416 #define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 417         this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 418 #endif
 419
 420 #endif /* _ASM_GENERIC_PERCPU_H_ */