?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/dtor/input
[linux-btrfs-devel.git] / arch / x86 / include / asm / cmpxchg_32.h
blob3deb7250624c268c3b8ae819f6ef2c8081abc8c4
1 #ifndef _ASM_X86_CMPXCHG_32_H
2 #define _ASM_X86_CMPXCHG_32_H
3
4 #include <linux/bitops.h> /* for LOCK_PREFIX */
5
6 /*
7 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
8 * you need to test for the feature in boot_cpu_data.
9 */
10
11 extern void __xchg_wrong_size(void);
12
13 /*
14 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
15 * Since this is generally used to protect other memory information, we
16 * use "asm volatile" and "memory" clobbers to prevent gcc from moving
17 * information around.
18 */
19 #define __xchg(x, ptr, size) \
20 ({ \
21 __typeof(*(ptr)) __x = (x); \
22 switch (size) { \
23 case 1: \
24 { \
25 volatile u8 *__ptr = (volatile u8 *)(ptr); \
26 asm volatile("xchgb %0,%1" \
27 : "=q" (__x), "+m" (*__ptr) \
28 : "0" (__x) \
29 : "memory"); \
30 break; \
31 } \
32 case 2: \
33 { \
34 volatile u16 *__ptr = (volatile u16 *)(ptr); \
35 asm volatile("xchgw %0,%1" \
36 : "=r" (__x), "+m" (*__ptr) \
37 : "0" (__x) \
38 : "memory"); \
39 break; \
40 } \
41 case 4: \
42 { \
43 volatile u32 *__ptr = (volatile u32 *)(ptr); \
44 asm volatile("xchgl %0,%1" \
45 : "=r" (__x), "+m" (*__ptr) \
46 : "0" (__x) \
47 : "memory"); \
48 break; \
49 } \
50 default: \
51 __xchg_wrong_size(); \
52 } \
53 __x; \
54 })
55
56 #define xchg(ptr, v) \
57 __xchg((v), (ptr), sizeof(*ptr))
58
59 /*
60 * CMPXCHG8B only writes to the target if we had the previous
61 * value in registers, otherwise it acts as a read and gives us the
62 * "new previous" value. That is why there is a loop. Preloading
63 * EDX:EAX is a performance optimization: in the common case it means
64 * we need only one locked operation.
65 *
66 * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
67 * least an FPU save and/or %cr0.ts manipulation.
68 *
69 * cmpxchg8b must be used with the lock prefix here to allow the
70 * instruction to be executed atomically. We need to have the reader
71 * side to see the coherent 64bit value.
72 */
73 static inline void set_64bit(volatile u64 *ptr, u64 value)
74 {
75 u32 low = value;
76 u32 high = value >> 32;
77 u64 prev = *ptr;
78
79 asm volatile("\n1:\t"
80 LOCK_PREFIX "cmpxchg8b %0\n\t"
81 "jnz 1b"
82 : "=m" (*ptr), "+A" (prev)
83 : "b" (low), "c" (high)
84 : "memory");
85 }
86
87 extern void __cmpxchg_wrong_size(void);
88
89 /*
90 * Atomic compare and exchange. Compare OLD with MEM, if identical,
91 * store NEW in MEM. Return the initial value in MEM. Success is
92 * indicated by comparing RETURN with OLD.
93 */
94 #define __raw_cmpxchg(ptr, old, new, size, lock) \
95 ({ \
96 __typeof__(*(ptr)) __ret; \
97 __typeof__(*(ptr)) __old = (old); \
98 __typeof__(*(ptr)) __new = (new); \
99 switch (size) { \
100 case 1: \
101 { \
102 volatile u8 *__ptr = (volatile u8 *)(ptr); \
103 asm volatile(lock "cmpxchgb %2,%1" \
104 : "=a" (__ret), "+m" (*__ptr) \
105 : "q" (__new), "0" (__old) \
106 : "memory"); \
107 break; \
108 } \
109 case 2: \
110 { \
111 volatile u16 *__ptr = (volatile u16 *)(ptr); \
112 asm volatile(lock "cmpxchgw %2,%1" \
113 : "=a" (__ret), "+m" (*__ptr) \
114 : "r" (__new), "0" (__old) \
115 : "memory"); \
116 break; \
117 } \
118 case 4: \
119 { \
120 volatile u32 *__ptr = (volatile u32 *)(ptr); \
121 asm volatile(lock "cmpxchgl %2,%1" \
122 : "=a" (__ret), "+m" (*__ptr) \
123 : "r" (__new), "0" (__old) \
124 : "memory"); \
125 break; \
126 } \
127 default: \
128 __cmpxchg_wrong_size(); \
129 } \
130 __ret; \
131 })
132
133 #define __cmpxchg(ptr, old, new, size) \
134 __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
135
136 #define __sync_cmpxchg(ptr, old, new, size) \
137 __raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
138
139 #define __cmpxchg_local(ptr, old, new, size) \
140 __raw_cmpxchg((ptr), (old), (new), (size), "")
141
142 #ifdef CONFIG_X86_CMPXCHG
143 #define __HAVE_ARCH_CMPXCHG 1
144
145 #define cmpxchg(ptr, old, new) \
146 __cmpxchg((ptr), (old), (new), sizeof(*ptr))
147
148 #define sync_cmpxchg(ptr, old, new) \
149 __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr))
150
151 #define cmpxchg_local(ptr, old, new) \
152 __cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
153 #endif
154
155 #ifdef CONFIG_X86_CMPXCHG64
156 #define cmpxchg64(ptr, o, n) \
157 ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
158 (unsigned long long)(n)))
159 #define cmpxchg64_local(ptr, o, n) \
160 ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
161 (unsigned long long)(n)))
162 #endif
163
164 static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
165 {
166 u64 prev;
167 asm volatile(LOCK_PREFIX "cmpxchg8b %1"
168 : "=A" (prev),
169 "+m" (*ptr)
170 : "b" ((u32)new),
171 "c" ((u32)(new >> 32)),
172 "0" (old)
173 : "memory");
174 return prev;
175 }
176
177 static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
178 {
179 u64 prev;
180 asm volatile("cmpxchg8b %1"
181 : "=A" (prev),
182 "+m" (*ptr)
183 : "b" ((u32)new),
184 "c" ((u32)(new >> 32)),
185 "0" (old)
186 : "memory");
187 return prev;
188 }
189
190 #ifndef CONFIG_X86_CMPXCHG
191 /*
192 * Building a kernel capable running on 80386. It may be necessary to
193 * simulate the cmpxchg on the 80386 CPU. For that purpose we define
194 * a function for each of the sizes we support.
195 */
196
197 extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
198 extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
199 extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
200
201 static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
202 unsigned long new, int size)
203 {
204 switch (size) {
205 case 1:
206 return cmpxchg_386_u8(ptr, old, new);
207 case 2:
208 return cmpxchg_386_u16(ptr, old, new);
209 case 4:
210 return cmpxchg_386_u32(ptr, old, new);
211 }
212 return old;
213 }
214
215 #define cmpxchg(ptr, o, n) \
216 ({ \
217 __typeof__(*(ptr)) __ret; \
218 if (likely(boot_cpu_data.x86 > 3)) \
219 __ret = (__typeof__(*(ptr)))__cmpxchg((ptr), \
220 (unsigned long)(o), (unsigned long)(n), \
221 sizeof(*(ptr))); \
222 else \
223 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \
224 (unsigned long)(o), (unsigned long)(n), \
225 sizeof(*(ptr))); \
226 __ret; \
227 })
228 #define cmpxchg_local(ptr, o, n) \
229 ({ \
230 __typeof__(*(ptr)) __ret; \
231 if (likely(boot_cpu_data.x86 > 3)) \
232 __ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr), \
233 (unsigned long)(o), (unsigned long)(n), \
234 sizeof(*(ptr))); \
235 else \
236 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \
237 (unsigned long)(o), (unsigned long)(n), \
238 sizeof(*(ptr))); \
239 __ret; \
240 })
241 #endif
242
243 #ifndef CONFIG_X86_CMPXCHG64
244 /*
245 * Building a kernel capable running on 80386 and 80486. It may be necessary
246 * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
247 */
248
249 #define cmpxchg64(ptr, o, n) \
250 ({ \
251 __typeof__(*(ptr)) __ret; \
252 __typeof__(*(ptr)) __old = (o); \
253 __typeof__(*(ptr)) __new = (n); \
254 alternative_io(LOCK_PREFIX_HERE \
255 "call cmpxchg8b_emu", \
256 "lock; cmpxchg8b (%%esi)" , \
257 X86_FEATURE_CX8, \
258 "=A" (__ret), \
259 "S" ((ptr)), "0" (__old), \
260 "b" ((unsigned int)__new), \
261 "c" ((unsigned int)(__new>>32)) \
262 : "memory"); \
263 __ret; })
264
265
266 #define cmpxchg64_local(ptr, o, n) \
267 ({ \
268 __typeof__(*(ptr)) __ret; \
269 __typeof__(*(ptr)) __old = (o); \
270 __typeof__(*(ptr)) __new = (n); \
271 alternative_io("call cmpxchg8b_emu", \
272 "cmpxchg8b (%%esi)" , \
273 X86_FEATURE_CX8, \
274 "=A" (__ret), \
275 "S" ((ptr)), "0" (__old), \
276 "b" ((unsigned int)__new), \
277 "c" ((unsigned int)(__new>>32)) \
278 : "memory"); \
279 __ret; })
280
281 #endif
282
283 #define cmpxchg8b(ptr, o1, o2, n1, n2) \
284 ({ \
285 char __ret; \
286 __typeof__(o2) __dummy; \
287 __typeof__(*(ptr)) __old1 = (o1); \
288 __typeof__(o2) __old2 = (o2); \
289 __typeof__(*(ptr)) __new1 = (n1); \
290 __typeof__(o2) __new2 = (n2); \
291 asm volatile(LOCK_PREFIX "cmpxchg8b %2; setz %1" \
292 : "=d"(__dummy), "=a" (__ret), "+m" (*ptr)\
293 : "a" (__old1), "d"(__old2), \
294 "b" (__new1), "c" (__new2) \
295 : "memory"); \
296 __ret; })
297
298
299 #define cmpxchg8b_local(ptr, o1, o2, n1, n2) \
300 ({ \
301 char __ret; \
302 __typeof__(o2) __dummy; \
303 __typeof__(*(ptr)) __old1 = (o1); \
304 __typeof__(o2) __old2 = (o2); \
305 __typeof__(*(ptr)) __new1 = (n1); \
306 __typeof__(o2) __new2 = (n2); \
307 asm volatile("cmpxchg8b %2; setz %1" \
308 : "=d"(__dummy), "=a"(__ret), "+m" (*ptr)\
309 : "a" (__old), "d"(__old2), \
310 "b" (__new1), "c" (__new2), \
311 : "memory"); \
312 __ret; })
313
314
315 #define cmpxchg_double(ptr, o1, o2, n1, n2) \
316 ({ \
317 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
318 VM_BUG_ON((unsigned long)(ptr) % 8); \
319 cmpxchg8b((ptr), (o1), (o2), (n1), (n2)); \
320 })
321
322 #define cmpxchg_double_local(ptr, o1, o2, n1, n2) \
323 ({ \
324 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
325 VM_BUG_ON((unsigned long)(ptr) % 8); \
326 cmpxchg16b_local((ptr), (o1), (o2), (n1), (n2)); \
327 })
328
329 #define system_has_cmpxchg_double() cpu_has_cx8
330
331 #endif /* _ASM_X86_CMPXCHG_32_H */
Linux Btrfs Development