?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
uwb: Use kcalloc instead of kzalloc to allocate array
[zen-stable.git] / arch / x86 / include / asm / cmpxchg.h
blob5d3acdf5a7a682d96681f78c86548a18203aaf98
1 #ifndef ASM_X86_CMPXCHG_H
2 #define ASM_X86_CMPXCHG_H
3
4 #include <linux/compiler.h>
5 #include <asm/alternative.h> /* Provides LOCK_PREFIX */
6
7 /*
8 * Non-existant functions to indicate usage errors at link time
9 * (or compile-time if the compiler implements __compiletime_error().
10 */
11 extern void __xchg_wrong_size(void)
12 __compiletime_error("Bad argument size for xchg");
13 extern void __cmpxchg_wrong_size(void)
14 __compiletime_error("Bad argument size for cmpxchg");
15 extern void __xadd_wrong_size(void)
16 __compiletime_error("Bad argument size for xadd");
17
18 /*
19 * Constants for operation sizes. On 32-bit, the 64-bit size it set to
20 * -1 because sizeof will never return -1, thereby making those switch
21 * case statements guaranteeed dead code which the compiler will
22 * eliminate, and allowing the "missing symbol in the default case" to
23 * indicate a usage error.
24 */
25 #define __X86_CASE_B 1
26 #define __X86_CASE_W 2
27 #define __X86_CASE_L 4
28 #ifdef CONFIG_64BIT
29 #define __X86_CASE_Q 8
30 #else
31 #define __X86_CASE_Q -1 /* sizeof will never return -1 */
32 #endif
33
34 /*
35 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
36 * Since this is generally used to protect other memory information, we
37 * use "asm volatile" and "memory" clobbers to prevent gcc from moving
38 * information around.
39 */
40 #define __xchg(x, ptr, size) \
41 ({ \
42 __typeof(*(ptr)) __x = (x); \
43 switch (size) { \
44 case __X86_CASE_B: \
45 { \
46 volatile u8 *__ptr = (volatile u8 *)(ptr); \
47 asm volatile("xchgb %0,%1" \
48 : "=q" (__x), "+m" (*__ptr) \
49 : "0" (__x) \
50 : "memory"); \
51 break; \
52 } \
53 case __X86_CASE_W: \
54 { \
55 volatile u16 *__ptr = (volatile u16 *)(ptr); \
56 asm volatile("xchgw %0,%1" \
57 : "=r" (__x), "+m" (*__ptr) \
58 : "0" (__x) \
59 : "memory"); \
60 break; \
61 } \
62 case __X86_CASE_L: \
63 { \
64 volatile u32 *__ptr = (volatile u32 *)(ptr); \
65 asm volatile("xchgl %0,%1" \
66 : "=r" (__x), "+m" (*__ptr) \
67 : "0" (__x) \
68 : "memory"); \
69 break; \
70 } \
71 case __X86_CASE_Q: \
72 { \
73 volatile u64 *__ptr = (volatile u64 *)(ptr); \
74 asm volatile("xchgq %0,%1" \
75 : "=r" (__x), "+m" (*__ptr) \
76 : "0" (__x) \
77 : "memory"); \
78 break; \
79 } \
80 default: \
81 __xchg_wrong_size(); \
82 } \
83 __x; \
84 })
85
86 #define xchg(ptr, v) \
87 __xchg((v), (ptr), sizeof(*ptr))
88
89 /*
90 * Atomic compare and exchange. Compare OLD with MEM, if identical,
91 * store NEW in MEM. Return the initial value in MEM. Success is
92 * indicated by comparing RETURN with OLD.
93 */
94 #define __raw_cmpxchg(ptr, old, new, size, lock) \
95 ({ \
96 __typeof__(*(ptr)) __ret; \
97 __typeof__(*(ptr)) __old = (old); \
98 __typeof__(*(ptr)) __new = (new); \
99 switch (size) { \
100 case __X86_CASE_B: \
101 { \
102 volatile u8 *__ptr = (volatile u8 *)(ptr); \
103 asm volatile(lock "cmpxchgb %2,%1" \
104 : "=a" (__ret), "+m" (*__ptr) \
105 : "q" (__new), "0" (__old) \
106 : "memory"); \
107 break; \
108 } \
109 case __X86_CASE_W: \
110 { \
111 volatile u16 *__ptr = (volatile u16 *)(ptr); \
112 asm volatile(lock "cmpxchgw %2,%1" \
113 : "=a" (__ret), "+m" (*__ptr) \
114 : "r" (__new), "0" (__old) \
115 : "memory"); \
116 break; \
117 } \
118 case __X86_CASE_L: \
119 { \
120 volatile u32 *__ptr = (volatile u32 *)(ptr); \
121 asm volatile(lock "cmpxchgl %2,%1" \
122 : "=a" (__ret), "+m" (*__ptr) \
123 : "r" (__new), "0" (__old) \
124 : "memory"); \
125 break; \
126 } \
127 case __X86_CASE_Q: \
128 { \
129 volatile u64 *__ptr = (volatile u64 *)(ptr); \
130 asm volatile(lock "cmpxchgq %2,%1" \
131 : "=a" (__ret), "+m" (*__ptr) \
132 : "r" (__new), "0" (__old) \
133 : "memory"); \
134 break; \
135 } \
136 default: \
137 __cmpxchg_wrong_size(); \
138 } \
139 __ret; \
140 })
141
142 #define __cmpxchg(ptr, old, new, size) \
143 __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
144
145 #define __sync_cmpxchg(ptr, old, new, size) \
146 __raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
147
148 #define __cmpxchg_local(ptr, old, new, size) \
149 __raw_cmpxchg((ptr), (old), (new), (size), "")
150
151 #ifdef CONFIG_X86_32
152 # include "cmpxchg_32.h"
153 #else
154 # include "cmpxchg_64.h"
155 #endif
156
157 #ifdef __HAVE_ARCH_CMPXCHG
158 #define cmpxchg(ptr, old, new) \
159 __cmpxchg((ptr), (old), (new), sizeof(*ptr))
160
161 #define sync_cmpxchg(ptr, old, new) \
162 __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr))
163
164 #define cmpxchg_local(ptr, old, new) \
165 __cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
166 #endif
167
168 #define __xadd(ptr, inc, lock) \
169 ({ \
170 __typeof__ (*(ptr)) __ret = (inc); \
171 switch (sizeof(*(ptr))) { \
172 case __X86_CASE_B: \
173 asm volatile (lock "xaddb %b0, %1\n" \
174 : "+r" (__ret), "+m" (*(ptr)) \
175 : : "memory", "cc"); \
176 break; \
177 case __X86_CASE_W: \
178 asm volatile (lock "xaddw %w0, %1\n" \
179 : "+r" (__ret), "+m" (*(ptr)) \
180 : : "memory", "cc"); \
181 break; \
182 case __X86_CASE_L: \
183 asm volatile (lock "xaddl %0, %1\n" \
184 : "+r" (__ret), "+m" (*(ptr)) \
185 : : "memory", "cc"); \
186 break; \
187 case __X86_CASE_Q: \
188 asm volatile (lock "xaddq %q0, %1\n" \
189 : "+r" (__ret), "+m" (*(ptr)) \
190 : : "memory", "cc"); \
191 break; \
192 default: \
193 __xadd_wrong_size(); \
194 } \
195 __ret; \
196 })
197
198 /*
199 * xadd() adds "inc" to "*ptr" and atomically returns the previous
200 * value of "*ptr".
201 *
202 * xadd() is locked when multiple CPUs are online
203 * xadd_sync() is always locked
204 * xadd_local() is never locked
205 */
206 #define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX)
207 #define xadd_sync(ptr, inc) __xadd((ptr), (inc), "lock; ")
208 #define xadd_local(ptr, inc) __xadd((ptr), (inc), "")
209
210 #endif /* ASM_X86_CMPXCHG_H */