arch/arm64/include/asm/percpu.h

   1 /*
   2  * Copyright (C) 2013 ARM Ltd.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  15  */
  16 #ifndef __ASM_PERCPU_H
  17 #define __ASM_PERCPU_H
  18
  19 #ifdef CONFIG_SMP
  20
  21 static inline void set_my_cpu_offset(unsigned long off)
  22 {
  23         asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory");
  24 }
  25
  26 static inline unsigned long __my_cpu_offset(void)
  27 {
  28         unsigned long off;
  29
  30         /*
  31          * We want to allow caching the value, so avoid using volatile and
  32          * instead use a fake stack read to hazard against barrier().
  33          */
  34         asm("mrs %0, tpidr_el1" : "=r" (off) :
  35                 "Q" (*(const unsigned long *)current_stack_pointer));
  36
  37         return off;
  38 }
  39 #define __my_cpu_offset __my_cpu_offset()
  40
  41 #else   /* !CONFIG_SMP */
  42
  43 #define set_my_cpu_offset(x)    do { } while (0)
  44
  45 #endif /* CONFIG_SMP */
  46
  47 #define PERCPU_OP(op, asm_op)                                           \
  48 static inline unsigned long __percpu_##op(void *ptr,                    \
  49                         unsigned long val, int size)                    \
  50 {                                                                       \
  51         unsigned long loop, ret;                                        \
  52                                                                         \
  53         switch (size) {                                                 \
  54         case 1:                                                         \
  55                 do {                                                    \
  56                         asm ("//__per_cpu_" #op "_1\n"                  \
  57                         "ldxrb    %w[ret], %[ptr]\n"                    \
  58                         #asm_op " %w[ret], %w[ret], %w[val]\n"          \
  59                         "stxrb    %w[loop], %w[ret], %[ptr]\n"          \
  60                         : [loop] "=&r" (loop), [ret] "=&r" (ret),       \
  61                           [ptr] "+Q"(*(u8 *)ptr)                        \
  62                         : [val] "Ir" (val));                            \
  63                 } while (loop);                                         \
  64                 break;                                                  \
  65         case 2:                                                         \
  66                 do {                                                    \
  67                         asm ("//__per_cpu_" #op "_2\n"                  \
  68                         "ldxrh    %w[ret], %[ptr]\n"                    \
  69                         #asm_op " %w[ret], %w[ret], %w[val]\n"          \
  70                         "stxrh    %w[loop], %w[ret], %[ptr]\n"          \
  71                         : [loop] "=&r" (loop), [ret] "=&r" (ret),       \
  72                           [ptr]  "+Q"(*(u16 *)ptr)                      \
  73                         : [val] "Ir" (val));                            \
  74                 } while (loop);                                         \
  75                 break;                                                  \
  76         case 4:                                                         \
  77                 do {                                                    \
  78                         asm ("//__per_cpu_" #op "_4\n"                  \
  79                         "ldxr     %w[ret], %[ptr]\n"                    \
  80                         #asm_op " %w[ret], %w[ret], %w[val]\n"          \
  81                         "stxr     %w[loop], %w[ret], %[ptr]\n"          \
  82                         : [loop] "=&r" (loop), [ret] "=&r" (ret),       \
  83                           [ptr] "+Q"(*(u32 *)ptr)                       \
  84                         : [val] "Ir" (val));                            \
  85                 } while (loop);                                         \
  86                 break;                                                  \
  87         case 8:                                                         \
  88                 do {                                                    \
  89                         asm ("//__per_cpu_" #op "_8\n"                  \
  90                         "ldxr     %[ret], %[ptr]\n"                     \
  91                         #asm_op " %[ret], %[ret], %[val]\n"             \
  92                         "stxr     %w[loop], %[ret], %[ptr]\n"           \
  93                         : [loop] "=&r" (loop), [ret] "=&r" (ret),       \
  94                           [ptr] "+Q"(*(u64 *)ptr)                       \
  95                         : [val] "Ir" (val));                            \
  96                 } while (loop);                                         \
  97                 break;                                                  \
  98         default:                                                        \
  99                 BUILD_BUG();                                            \
 100         }                                                               \
 101                                                                         \
 102         return ret;                                                     \
 103 }
 104
 105 PERCPU_OP(add, add)
 106 PERCPU_OP(and, and)
 107 PERCPU_OP(or, orr)
 108 #undef PERCPU_OP
 109
 110 static inline unsigned long __percpu_read(void *ptr, int size)
 111 {
 112         unsigned long ret;
 113
 114         switch (size) {
 115         case 1:
 116                 ret = ACCESS_ONCE(*(u8 *)ptr);
 117                 break;
 118         case 2:
 119                 ret = ACCESS_ONCE(*(u16 *)ptr);
 120                 break;
 121         case 4:
 122                 ret = ACCESS_ONCE(*(u32 *)ptr);
 123                 break;
 124         case 8:
 125                 ret = ACCESS_ONCE(*(u64 *)ptr);
 126                 break;
 127         default:
 128                 BUILD_BUG();
 129         }
 130
 131         return ret;
 132 }
 133
 134 static inline void __percpu_write(void *ptr, unsigned long val, int size)
 135 {
 136         switch (size) {
 137         case 1:
 138                 ACCESS_ONCE(*(u8 *)ptr) = (u8)val;
 139                 break;
 140         case 2:
 141                 ACCESS_ONCE(*(u16 *)ptr) = (u16)val;
 142                 break;
 143         case 4:
 144                 ACCESS_ONCE(*(u32 *)ptr) = (u32)val;
 145                 break;
 146         case 8:
 147                 ACCESS_ONCE(*(u64 *)ptr) = (u64)val;
 148                 break;
 149         default:
 150                 BUILD_BUG();
 151         }
 152 }
 153
 154 static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
 155                                                 int size)
 156 {
 157         unsigned long ret, loop;
 158
 159         switch (size) {
 160         case 1:
 161                 do {
 162                         asm ("//__percpu_xchg_1\n"
 163                         "ldxrb %w[ret], %[ptr]\n"
 164                         "stxrb %w[loop], %w[val], %[ptr]\n"
 165                         : [loop] "=&r"(loop), [ret] "=&r"(ret),
 166                           [ptr] "+Q"(*(u8 *)ptr)
 167                         : [val] "r" (val));
 168                 } while (loop);
 169                 break;
 170         case 2:
 171                 do {
 172                         asm ("//__percpu_xchg_2\n"
 173                         "ldxrh %w[ret], %[ptr]\n"
 174                         "stxrh %w[loop], %w[val], %[ptr]\n"
 175                         : [loop] "=&r"(loop), [ret] "=&r"(ret),
 176                           [ptr] "+Q"(*(u16 *)ptr)
 177                         : [val] "r" (val));
 178                 } while (loop);
 179                 break;
 180         case 4:
 181                 do {
 182                         asm ("//__percpu_xchg_4\n"
 183                         "ldxr %w[ret], %[ptr]\n"
 184                         "stxr %w[loop], %w[val], %[ptr]\n"
 185                         : [loop] "=&r"(loop), [ret] "=&r"(ret),
 186                           [ptr] "+Q"(*(u32 *)ptr)
 187                         : [val] "r" (val));
 188                 } while (loop);
 189                 break;
 190         case 8:
 191                 do {
 192                         asm ("//__percpu_xchg_8\n"
 193                         "ldxr %[ret], %[ptr]\n"
 194                         "stxr %w[loop], %[val], %[ptr]\n"
 195                         : [loop] "=&r"(loop), [ret] "=&r"(ret),
 196                           [ptr] "+Q"(*(u64 *)ptr)
 197                         : [val] "r" (val));
 198                 } while (loop);
 199                 break;
 200         default:
 201                 BUILD_BUG();
 202         }
 203
 204         return ret;
 205 }
 206
 207 #define _percpu_read(pcp)                                               \
 208 ({                                                                      \
 209         typeof(pcp) __retval;                                           \
 210         preempt_disable();                                              \
 211         __retval = (typeof(pcp))__percpu_read(raw_cpu_ptr(&(pcp)),      \
 212                                               sizeof(pcp));             \
 213         preempt_enable();                                               \
 214         __retval;                                                       \
 215 })
 216
 217 #define _percpu_write(pcp, val)                                         \
 218 do {                                                                    \
 219         preempt_disable();                                              \
 220         __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val),       \
 221                                 sizeof(pcp));                           \
 222         preempt_enable();                                               \
 223 } while(0)                                                              \
 224
 225 #define _pcp_protect(operation, pcp, val)                       \
 226 ({                                                              \
 227         typeof(pcp) __retval;                                   \
 228         preempt_disable();                                      \
 229         __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)),  \
 230                                           (val), sizeof(pcp));  \
 231         preempt_enable();                                       \
 232         __retval;                                               \
 233 })
 234
 235 #define _percpu_add(pcp, val) \
 236         _pcp_protect(__percpu_add, pcp, val)
 237
 238 #define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
 239
 240 #define _percpu_and(pcp, val) \
 241         _pcp_protect(__percpu_and, pcp, val)
 242
 243 #define _percpu_or(pcp, val) \
 244         _pcp_protect(__percpu_or, pcp, val)
 245
 246 #define _percpu_xchg(pcp, val) (typeof(pcp)) \
 247         _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val))
 248
 249 #define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
 250 #define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
 251 #define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
 252 #define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
 253
 254 #define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
 255 #define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
 256 #define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
 257 #define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
 258
 259 #define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
 260 #define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
 261 #define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
 262 #define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
 263
 264 #define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
 265 #define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
 266 #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
 267 #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
 268
 269 #define this_cpu_read_1(pcp) _percpu_read(pcp)
 270 #define this_cpu_read_2(pcp) _percpu_read(pcp)
 271 #define this_cpu_read_4(pcp) _percpu_read(pcp)
 272 #define this_cpu_read_8(pcp) _percpu_read(pcp)
 273
 274 #define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
 275 #define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
 276 #define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
 277 #define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
 278
 279 #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
 280 #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
 281 #define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
 282 #define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
 283
 284 #include <asm-generic/percpu.h>
 285
 286 #endif /* __ASM_PERCPU_H */