arch/arm64/include/asm/cmpxchg.h

   1 /*
   2  * Based on arch/arm/include/asm/cmpxchg.h
   3  *
   4  * Copyright (C) 2012 ARM Ltd.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18 #ifndef __ASM_CMPXCHG_H
  19 #define __ASM_CMPXCHG_H
  20
  21 #include <linux/bug.h>
  22
  23 #include <asm/barrier.h>
  24
  25 static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
  26 {
  27         unsigned long ret, tmp;
  28
  29         switch (size) {
  30         case 1:
  31                 asm volatile("//        __xchg1\n"
  32                 "1:     ldxrb   %w0, %2\n"
  33                 "       stlxrb  %w1, %w3, %2\n"
  34                 "       cbnz    %w1, 1b\n"
  35                         : "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr)
  36                         : "r" (x)
  37                         : "memory");
  38                 break;
  39         case 2:
  40                 asm volatile("//        __xchg2\n"
  41                 "1:     ldxrh   %w0, %2\n"
  42                 "       stlxrh  %w1, %w3, %2\n"
  43                 "       cbnz    %w1, 1b\n"
  44                         : "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr)
  45                         : "r" (x)
  46                         : "memory");
  47                 break;
  48         case 4:
  49                 asm volatile("//        __xchg4\n"
  50                 "1:     ldxr    %w0, %2\n"
  51                 "       stlxr   %w1, %w3, %2\n"
  52                 "       cbnz    %w1, 1b\n"
  53                         : "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr)
  54                         : "r" (x)
  55                         : "memory");
  56                 break;
  57         case 8:
  58                 asm volatile("//        __xchg8\n"
  59                 "1:     ldxr    %0, %2\n"
  60                 "       stlxr   %w1, %3, %2\n"
  61                 "       cbnz    %w1, 1b\n"
  62                         : "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr)
  63                         : "r" (x)
  64                         : "memory");
  65                 break;
  66         default:
  67                 BUILD_BUG();
  68         }
  69
  70         smp_mb();
  71         return ret;
  72 }
  73
  74 #define xchg(ptr,x) \
  75 ({ \
  76         __typeof__(*(ptr)) __ret; \
  77         __ret = (__typeof__(*(ptr))) \
  78                 __xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \
  79         __ret; \
  80 })
  81
  82 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
  83                                       unsigned long new, int size)
  84 {
  85         unsigned long oldval = 0, res;
  86
  87         switch (size) {
  88         case 1:
  89                 do {
  90                         asm volatile("// __cmpxchg1\n"
  91                         "       ldxrb   %w1, %2\n"
  92                         "       mov     %w0, #0\n"
  93                         "       cmp     %w1, %w3\n"
  94                         "       b.ne    1f\n"
  95                         "       stxrb   %w0, %w4, %2\n"
  96                         "1:\n"
  97                                 : "=&r" (res), "=&r" (oldval), "+Q" (*(u8 *)ptr)
  98                                 : "Ir" (old), "r" (new)
  99                                 : "cc");
 100                 } while (res);
 101                 break;
 102
 103         case 2:
 104                 do {
 105                         asm volatile("// __cmpxchg2\n"
 106                         "       ldxrh   %w1, %2\n"
 107                         "       mov     %w0, #0\n"
 108                         "       cmp     %w1, %w3\n"
 109                         "       b.ne    1f\n"
 110                         "       stxrh   %w0, %w4, %2\n"
 111                         "1:\n"
 112                                 : "=&r" (res), "=&r" (oldval), "+Q" (*(u16 *)ptr)
 113                                 : "Ir" (old), "r" (new)
 114                                 : "cc");
 115                 } while (res);
 116                 break;
 117
 118         case 4:
 119                 do {
 120                         asm volatile("// __cmpxchg4\n"
 121                         "       ldxr    %w1, %2\n"
 122                         "       mov     %w0, #0\n"
 123                         "       cmp     %w1, %w3\n"
 124                         "       b.ne    1f\n"
 125                         "       stxr    %w0, %w4, %2\n"
 126                         "1:\n"
 127                                 : "=&r" (res), "=&r" (oldval), "+Q" (*(u32 *)ptr)
 128                                 : "Ir" (old), "r" (new)
 129                                 : "cc");
 130                 } while (res);
 131                 break;
 132
 133         case 8:
 134                 do {
 135                         asm volatile("// __cmpxchg8\n"
 136                         "       ldxr    %1, %2\n"
 137                         "       mov     %w0, #0\n"
 138                         "       cmp     %1, %3\n"
 139                         "       b.ne    1f\n"
 140                         "       stxr    %w0, %4, %2\n"
 141                         "1:\n"
 142                                 : "=&r" (res), "=&r" (oldval), "+Q" (*(u64 *)ptr)
 143                                 : "Ir" (old), "r" (new)
 144                                 : "cc");
 145                 } while (res);
 146                 break;
 147
 148         default:
 149                 BUILD_BUG();
 150         }
 151
 152         return oldval;
 153 }
 154
 155 static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
 156                                          unsigned long new, int size)
 157 {
 158         unsigned long ret;
 159
 160         smp_mb();
 161         ret = __cmpxchg(ptr, old, new, size);
 162         smp_mb();
 163
 164         return ret;
 165 }
 166
 167 #define cmpxchg(ptr, o, n) \
 168 ({ \
 169         __typeof__(*(ptr)) __ret; \
 170         __ret = (__typeof__(*(ptr))) \
 171                 __cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
 172                              sizeof(*(ptr))); \
 173         __ret; \
 174 })
 175
 176 #define cmpxchg_local(ptr, o, n) \
 177 ({ \
 178         __typeof__(*(ptr)) __ret; \
 179         __ret = (__typeof__(*(ptr))) \
 180                 __cmpxchg((ptr), (unsigned long)(o), \
 181                           (unsigned long)(n), sizeof(*(ptr))); \
 182         __ret; \
 183 })
 184
 185 #define cmpxchg64(ptr,o,n)              cmpxchg((ptr),(o),(n))
 186 #define cmpxchg64_local(ptr,o,n)        cmpxchg_local((ptr),(o),(n))
 187
 188 #define cmpxchg64_relaxed(ptr,o,n)      cmpxchg_local((ptr),(o),(n))
 189
 190 #endif  /* __ASM_CMPXCHG_H */