arch/xtensa/include/asm/cmpxchg.h

   1 /*
   2  * Atomic xchg and cmpxchg operations.
   3  *
   4  * This file is subject to the terms and conditions of the GNU General Public
   5  * License.  See the file "COPYING" in the main directory of this archive
   6  * for more details.
   7  *
   8  * Copyright (C) 2001 - 2005 Tensilica Inc.
   9  */
  10
  11 #ifndef _XTENSA_CMPXCHG_H
  12 #define _XTENSA_CMPXCHG_H
  13
  14 #ifndef __ASSEMBLY__
  15
  16 #include <linux/bits.h>
  17 #include <linux/stringify.h>
  18
  19 /*
  20  * cmpxchg
  21  */
  22
  23 static inline unsigned long
  24 __cmpxchg_u32(volatile int *p, int old, int new)
  25 {
  26 #if XCHAL_HAVE_EXCLUSIVE
  27         unsigned long tmp, result;
  28
  29         __asm__ __volatile__(
  30                         "1:     l32ex   %[result], %[addr]\n"
  31                         "       bne     %[result], %[cmp], 2f\n"
  32                         "       mov     %[tmp], %[new]\n"
  33                         "       s32ex   %[tmp], %[addr]\n"
  34                         "       getex   %[tmp]\n"
  35                         "       beqz    %[tmp], 1b\n"
  36                         "2:\n"
  37                         : [result] "=&a" (result), [tmp] "=&a" (tmp)
  38                         : [new] "a" (new), [addr] "a" (p), [cmp] "a" (old)
  39                         : "memory"
  40                         );
  41
  42         return result;
  43 #elif XCHAL_HAVE_S32C1I
  44         __asm__ __volatile__(
  45                         "       wsr     %[cmp], scompare1\n"
  46                         "       s32c1i  %[new], %[mem]\n"
  47                         : [new] "+a" (new), [mem] "+m" (*p)
  48                         : [cmp] "a" (old)
  49                         : "memory"
  50                         );
  51
  52         return new;
  53 #else
  54         __asm__ __volatile__(
  55                         "       rsil    a15, "__stringify(TOPLEVEL)"\n"
  56                         "       l32i    %[old], %[mem]\n"
  57                         "       bne     %[old], %[cmp], 1f\n"
  58                         "       s32i    %[new], %[mem]\n"
  59                         "1:\n"
  60                         "       wsr     a15, ps\n"
  61                         "       rsync\n"
  62                         : [old] "=&a" (old), [mem] "+m" (*p)
  63                         : [cmp] "a" (old), [new] "r" (new)
  64                         : "a15", "memory");
  65         return old;
  66 #endif
  67 }
  68 /* This function doesn't exist, so you'll get a linker error
  69  * if something tries to do an invalid cmpxchg(). */
  70
  71 extern void __cmpxchg_called_with_bad_pointer(void);
  72
  73 static __inline__ unsigned long
  74 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
  75 {
  76         switch (size) {
  77         case 4:  return __cmpxchg_u32(ptr, old, new);
  78         default: __cmpxchg_called_with_bad_pointer();
  79                  return old;
  80         }
  81 }
  82
  83 #define cmpxchg(ptr,o,n)                                                      \
  84         ({ __typeof__(*(ptr)) _o_ = (o);                                      \
  85            __typeof__(*(ptr)) _n_ = (n);                                      \
  86            (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,          \
  87                                         (unsigned long)_n_, sizeof (*(ptr))); \
  88         })
  89
  90 #include <asm-generic/cmpxchg-local.h>
  91
  92 static inline unsigned long __cmpxchg_local(volatile void *ptr,
  93                                       unsigned long old,
  94                                       unsigned long new, int size)
  95 {
  96         switch (size) {
  97         case 4:
  98                 return __cmpxchg_u32(ptr, old, new);
  99         default:
 100                 return __cmpxchg_local_generic(ptr, old, new, size);
 101         }
 102
 103         return old;
 104 }
 105
 106 /*
 107  * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
 108  * them available.
 109  */
 110 #define cmpxchg_local(ptr, o, n)                                               \
 111         ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\
 112                         (unsigned long)(n), sizeof(*(ptr))))
 113 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
 114 #define cmpxchg64(ptr, o, n)    cmpxchg64_local((ptr), (o), (n))
 115
 116 /*
 117  * xchg_u32
 118  *
 119  * Note that a15 is used here because the register allocation
 120  * done by the compiler is not guaranteed and a window overflow
 121  * may not occur between the rsil and wsr instructions. By using
 122  * a15 in the rsil, the machine is guaranteed to be in a state
 123  * where no register reference will cause an overflow.
 124  */
 125
 126 static inline unsigned long xchg_u32(volatile int * m, unsigned long val)
 127 {
 128 #if XCHAL_HAVE_EXCLUSIVE
 129         unsigned long tmp, result;
 130
 131         __asm__ __volatile__(
 132                         "1:     l32ex   %[result], %[addr]\n"
 133                         "       mov     %[tmp], %[val]\n"
 134                         "       s32ex   %[tmp], %[addr]\n"
 135                         "       getex   %[tmp]\n"
 136                         "       beqz    %[tmp], 1b\n"
 137                         : [result] "=&a" (result), [tmp] "=&a" (tmp)
 138                         : [val] "a" (val), [addr] "a" (m)
 139                         : "memory"
 140                         );
 141
 142         return result;
 143 #elif XCHAL_HAVE_S32C1I
 144         unsigned long tmp, result;
 145         __asm__ __volatile__(
 146                         "1:     l32i    %[tmp], %[mem]\n"
 147                         "       mov     %[result], %[val]\n"
 148                         "       wsr     %[tmp], scompare1\n"
 149                         "       s32c1i  %[result], %[mem]\n"
 150                         "       bne     %[result], %[tmp], 1b\n"
 151                         : [result] "=&a" (result), [tmp] "=&a" (tmp),
 152                           [mem] "+m" (*m)
 153                         : [val] "a" (val)
 154                         : "memory"
 155                         );
 156         return result;
 157 #else
 158         unsigned long tmp;
 159         __asm__ __volatile__(
 160                         "       rsil    a15, "__stringify(TOPLEVEL)"\n"
 161                         "       l32i    %[tmp], %[mem]\n"
 162                         "       s32i    %[val], %[mem]\n"
 163                         "       wsr     a15, ps\n"
 164                         "       rsync\n"
 165                         : [tmp] "=&a" (tmp), [mem] "+m" (*m)
 166                         : [val] "a" (val)
 167                         : "a15", "memory");
 168         return tmp;
 169 #endif
 170 }
 171
 172 #define xchg(ptr,x) \
 173         ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
 174
 175 static inline u32 xchg_small(volatile void *ptr, u32 x, int size)
 176 {
 177         int off = (unsigned long)ptr % sizeof(u32);
 178         volatile u32 *p = ptr - off;
 179 #ifdef __BIG_ENDIAN
 180         int bitoff = (sizeof(u32) - size - off) * BITS_PER_BYTE;
 181 #else
 182         int bitoff = off * BITS_PER_BYTE;
 183 #endif
 184         u32 bitmask = ((0x1 << size * BITS_PER_BYTE) - 1) << bitoff;
 185         u32 oldv, newv;
 186         u32 ret;
 187
 188         do {
 189                 oldv = READ_ONCE(*p);
 190                 ret = (oldv & bitmask) >> bitoff;
 191                 newv = (oldv & ~bitmask) | (x << bitoff);
 192         } while (__cmpxchg_u32(p, oldv, newv) != oldv);
 193
 194         return ret;
 195 }
 196
 197 /*
 198  * This only works if the compiler isn't horribly bad at optimizing.
 199  * gcc-2.5.8 reportedly can't handle this, but I define that one to
 200  * be dead anyway.
 201  */
 202
 203 extern void __xchg_called_with_bad_pointer(void);
 204
 205 static __inline__ unsigned long
 206 __xchg(unsigned long x, volatile void * ptr, int size)
 207 {
 208         switch (size) {
 209         case 1:
 210                 return xchg_small(ptr, x, 1);
 211         case 2:
 212                 return xchg_small(ptr, x, 2);
 213         case 4:
 214                 return xchg_u32(ptr, x);
 215         default:
 216                 __xchg_called_with_bad_pointer();
 217                 return x;
 218         }
 219 }
 220
 221 #endif /* __ASSEMBLY__ */
 222
 223 #endif /* _XTENSA_CMPXCHG_H */