arch/alpha/include/asm/xchg.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ALPHA_CMPXCHG_H
   3 #error Do not include xchg.h directly!
   4 #else
   5 /*
   6  * xchg/xchg_local and cmpxchg/cmpxchg_local share the same code
   7  * except that local version do not have the expensive memory barrier.
   8  * So this file is included twice from asm/cmpxchg.h.
   9  */
  10
  11 /*
  12  * Atomic exchange.
  13  * Since it can be used to implement critical sections
  14  * it must clobber "memory" (also for interrupts in UP).
  15  *
  16  * The leading and the trailing memory barriers guarantee that these
  17  * operations are fully ordered.
  18  *
  19  */
  20
  21 static inline unsigned long
  22 ____xchg(_u8, volatile char *m, unsigned long val)
  23 {
  24         unsigned long ret, tmp, addr64;
  25
  26         smp_mb();
  27         __asm__ __volatile__(
  28         "       andnot  %4,7,%3\n"
  29         "       insbl   %1,%4,%1\n"
  30         "1:     ldq_l   %2,0(%3)\n"
  31         "       extbl   %2,%4,%0\n"
  32         "       mskbl   %2,%4,%2\n"
  33         "       or      %1,%2,%2\n"
  34         "       stq_c   %2,0(%3)\n"
  35         "       beq     %2,2f\n"
  36         ".subsection 2\n"
  37         "2:     br      1b\n"
  38         ".previous"
  39         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
  40         : "r" ((long)m), "1" (val) : "memory");
  41         smp_mb();
  42
  43         return ret;
  44 }
  45
  46 static inline unsigned long
  47 ____xchg(_u16, volatile short *m, unsigned long val)
  48 {
  49         unsigned long ret, tmp, addr64;
  50
  51         smp_mb();
  52         __asm__ __volatile__(
  53         "       andnot  %4,7,%3\n"
  54         "       inswl   %1,%4,%1\n"
  55         "1:     ldq_l   %2,0(%3)\n"
  56         "       extwl   %2,%4,%0\n"
  57         "       mskwl   %2,%4,%2\n"
  58         "       or      %1,%2,%2\n"
  59         "       stq_c   %2,0(%3)\n"
  60         "       beq     %2,2f\n"
  61         ".subsection 2\n"
  62         "2:     br      1b\n"
  63         ".previous"
  64         : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
  65         : "r" ((long)m), "1" (val) : "memory");
  66         smp_mb();
  67
  68         return ret;
  69 }
  70
  71 static inline unsigned long
  72 ____xchg(_u32, volatile int *m, unsigned long val)
  73 {
  74         unsigned long dummy;
  75
  76         smp_mb();
  77         __asm__ __volatile__(
  78         "1:     ldl_l %0,%4\n"
  79         "       bis $31,%3,%1\n"
  80         "       stl_c %1,%2\n"
  81         "       beq %1,2f\n"
  82         ".subsection 2\n"
  83         "2:     br 1b\n"
  84         ".previous"
  85         : "=&r" (val), "=&r" (dummy), "=m" (*m)
  86         : "rI" (val), "m" (*m) : "memory");
  87         smp_mb();
  88
  89         return val;
  90 }
  91
  92 static inline unsigned long
  93 ____xchg(_u64, volatile long *m, unsigned long val)
  94 {
  95         unsigned long dummy;
  96
  97         smp_mb();
  98         __asm__ __volatile__(
  99         "1:     ldq_l %0,%4\n"
 100         "       bis $31,%3,%1\n"
 101         "       stq_c %1,%2\n"
 102         "       beq %1,2f\n"
 103         ".subsection 2\n"
 104         "2:     br 1b\n"
 105         ".previous"
 106         : "=&r" (val), "=&r" (dummy), "=m" (*m)
 107         : "rI" (val), "m" (*m) : "memory");
 108         smp_mb();
 109
 110         return val;
 111 }
 112
 113 /* This function doesn't exist, so you'll get a linker error
 114    if something tries to do an invalid xchg().  */
 115 extern void __xchg_called_with_bad_pointer(void);
 116
 117 static __always_inline unsigned long
 118 ____xchg(, volatile void *ptr, unsigned long x, int size)
 119 {
 120         switch (size) {
 121                 case 1:
 122                         return ____xchg(_u8, ptr, x);
 123                 case 2:
 124                         return ____xchg(_u16, ptr, x);
 125                 case 4:
 126                         return ____xchg(_u32, ptr, x);
 127                 case 8:
 128                         return ____xchg(_u64, ptr, x);
 129         }
 130         __xchg_called_with_bad_pointer();
 131         return x;
 132 }
 133
 134 /*
 135  * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 136  * store NEW in MEM.  Return the initial value in MEM.  Success is
 137  * indicated by comparing RETURN with OLD.
 138  *
 139  * The leading and the trailing memory barriers guarantee that these
 140  * operations are fully ordered.
 141  *
 142  * The trailing memory barrier is placed in SMP unconditionally, in
 143  * order to guarantee that dependency ordering is preserved when a
 144  * dependency is headed by an unsuccessful operation.
 145  */
 146
 147 static inline unsigned long
 148 ____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new)
 149 {
 150         unsigned long prev, tmp, cmp, addr64;
 151
 152         smp_mb();
 153         __asm__ __volatile__(
 154         "       andnot  %5,7,%4\n"
 155         "       insbl   %1,%5,%1\n"
 156         "1:     ldq_l   %2,0(%4)\n"
 157         "       extbl   %2,%5,%0\n"
 158         "       cmpeq   %0,%6,%3\n"
 159         "       beq     %3,2f\n"
 160         "       mskbl   %2,%5,%2\n"
 161         "       or      %1,%2,%2\n"
 162         "       stq_c   %2,0(%4)\n"
 163         "       beq     %2,3f\n"
 164         "2:\n"
 165         ".subsection 2\n"
 166         "3:     br      1b\n"
 167         ".previous"
 168         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 169         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
 170         smp_mb();
 171
 172         return prev;
 173 }
 174
 175 static inline unsigned long
 176 ____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new)
 177 {
 178         unsigned long prev, tmp, cmp, addr64;
 179
 180         smp_mb();
 181         __asm__ __volatile__(
 182         "       andnot  %5,7,%4\n"
 183         "       inswl   %1,%5,%1\n"
 184         "1:     ldq_l   %2,0(%4)\n"
 185         "       extwl   %2,%5,%0\n"
 186         "       cmpeq   %0,%6,%3\n"
 187         "       beq     %3,2f\n"
 188         "       mskwl   %2,%5,%2\n"
 189         "       or      %1,%2,%2\n"
 190         "       stq_c   %2,0(%4)\n"
 191         "       beq     %2,3f\n"
 192         "2:\n"
 193         ".subsection 2\n"
 194         "3:     br      1b\n"
 195         ".previous"
 196         : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 197         : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
 198         smp_mb();
 199
 200         return prev;
 201 }
 202
 203 static inline unsigned long
 204 ____cmpxchg(_u32, volatile int *m, int old, int new)
 205 {
 206         unsigned long prev, cmp;
 207
 208         smp_mb();
 209         __asm__ __volatile__(
 210         "1:     ldl_l %0,%5\n"
 211         "       cmpeq %0,%3,%1\n"
 212         "       beq %1,2f\n"
 213         "       mov %4,%1\n"
 214         "       stl_c %1,%2\n"
 215         "       beq %1,3f\n"
 216         "2:\n"
 217         ".subsection 2\n"
 218         "3:     br 1b\n"
 219         ".previous"
 220         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
 221         : "r"((long) old), "r"(new), "m"(*m) : "memory");
 222         smp_mb();
 223
 224         return prev;
 225 }
 226
 227 static inline unsigned long
 228 ____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new)
 229 {
 230         unsigned long prev, cmp;
 231
 232         smp_mb();
 233         __asm__ __volatile__(
 234         "1:     ldq_l %0,%5\n"
 235         "       cmpeq %0,%3,%1\n"
 236         "       beq %1,2f\n"
 237         "       mov %4,%1\n"
 238         "       stq_c %1,%2\n"
 239         "       beq %1,3f\n"
 240         "2:\n"
 241         ".subsection 2\n"
 242         "3:     br 1b\n"
 243         ".previous"
 244         : "=&r"(prev), "=&r"(cmp), "=m"(*m)
 245         : "r"((long) old), "r"(new), "m"(*m) : "memory");
 246         smp_mb();
 247
 248         return prev;
 249 }
 250
 251 /* This function doesn't exist, so you'll get a linker error
 252    if something tries to do an invalid cmpxchg().  */
 253 extern void __cmpxchg_called_with_bad_pointer(void);
 254
 255 static __always_inline unsigned long
 256 ____cmpxchg(, volatile void *ptr, unsigned long old, unsigned long new,
 257               int size)
 258 {
 259         switch (size) {
 260                 case 1:
 261                         return ____cmpxchg(_u8, ptr, old, new);
 262                 case 2:
 263                         return ____cmpxchg(_u16, ptr, old, new);
 264                 case 4:
 265                         return ____cmpxchg(_u32, ptr, old, new);
 266                 case 8:
 267                         return ____cmpxchg(_u64, ptr, old, new);
 268         }
 269         __cmpxchg_called_with_bad_pointer();
 270         return old;
 271 }
 272
 273 #endif