nfsd4: typo logical vs bitwise negate for want_mask
[linux-btrfs-devel.git] / arch / x86 / include / asm / bitops.h
blob1775d6e5920e1b8f2a463600a048a9e3b097d978
1 #ifndef _ASM_X86_BITOPS_H
2 #define _ASM_X86_BITOPS_H
4 /*
5 * Copyright 1992, Linus Torvalds.
7 * Note: inlines with more than a single statement should be marked
8 * __always_inline to avoid problems with older gcc's inlining heuristics.
9 */
11 #ifndef _LINUX_BITOPS_H
12 #error only <linux/bitops.h> can be included directly
13 #endif
15 #include <linux/compiler.h>
16 #include <asm/alternative.h>
19 * These have to be done with inline assembly: that way the bit-setting
20 * is guaranteed to be atomic. All bit operations return 0 if the bit
21 * was cleared before the operation and != 0 if it was not.
23 * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
26 #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
27 /* Technically wrong, but this avoids compilation errors on some gcc
28 versions. */
29 #define BITOP_ADDR(x) "=m" (*(volatile long *) (x))
30 #else
31 #define BITOP_ADDR(x) "+m" (*(volatile long *) (x))
32 #endif
34 #define ADDR BITOP_ADDR(addr)
37 * We do the locked ops that don't return the old value as
38 * a mask operation on a byte.
40 #define IS_IMMEDIATE(nr) (__builtin_constant_p(nr))
41 #define CONST_MASK_ADDR(nr, addr) BITOP_ADDR((void *)(addr) + ((nr)>>3))
42 #define CONST_MASK(nr) (1 << ((nr) & 7))
44 /**
45 * set_bit - Atomically set a bit in memory
46 * @nr: the bit to set
47 * @addr: the address to start counting from
49 * This function is atomic and may not be reordered. See __set_bit()
50 * if you do not require the atomic guarantees.
52 * Note: there are no guarantees that this function will not be reordered
53 * on non x86 architectures, so if you are writing portable code,
54 * make sure not to rely on its reordering guarantees.
56 * Note that @nr may be almost arbitrarily large; this function is not
57 * restricted to acting on a single-word quantity.
59 static __always_inline void
60 set_bit(unsigned int nr, volatile unsigned long *addr)
62 if (IS_IMMEDIATE(nr)) {
63 asm volatile(LOCK_PREFIX "orb %1,%0"
64 : CONST_MASK_ADDR(nr, addr)
65 : "iq" ((u8)CONST_MASK(nr))
66 : "memory");
67 } else {
68 asm volatile(LOCK_PREFIX "bts %1,%0"
69 : BITOP_ADDR(addr) : "Ir" (nr) : "memory");
73 /**
74 * __set_bit - Set a bit in memory
75 * @nr: the bit to set
76 * @addr: the address to start counting from
78 * Unlike set_bit(), this function is non-atomic and may be reordered.
79 * If it's called on the same region of memory simultaneously, the effect
80 * may be that only one operation succeeds.
82 static inline void __set_bit(int nr, volatile unsigned long *addr)
84 asm volatile("bts %1,%0" : ADDR : "Ir" (nr) : "memory");
87 /**
88 * clear_bit - Clears a bit in memory
89 * @nr: Bit to clear
90 * @addr: Address to start counting from
92 * clear_bit() is atomic and may not be reordered. However, it does
93 * not contain a memory barrier, so if it is used for locking purposes,
94 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
95 * in order to ensure changes are visible on other processors.
97 static __always_inline void
98 clear_bit(int nr, volatile unsigned long *addr)
100 if (IS_IMMEDIATE(nr)) {
101 asm volatile(LOCK_PREFIX "andb %1,%0"
102 : CONST_MASK_ADDR(nr, addr)
103 : "iq" ((u8)~CONST_MASK(nr)));
104 } else {
105 asm volatile(LOCK_PREFIX "btr %1,%0"
106 : BITOP_ADDR(addr)
107 : "Ir" (nr));
112 * clear_bit_unlock - Clears a bit in memory
113 * @nr: Bit to clear
114 * @addr: Address to start counting from
116 * clear_bit() is atomic and implies release semantics before the memory
117 * operation. It can be used for an unlock.
119 static inline void clear_bit_unlock(unsigned nr, volatile unsigned long *addr)
121 barrier();
122 clear_bit(nr, addr);
125 static inline void __clear_bit(int nr, volatile unsigned long *addr)
127 asm volatile("btr %1,%0" : ADDR : "Ir" (nr));
131 * __clear_bit_unlock - Clears a bit in memory
132 * @nr: Bit to clear
133 * @addr: Address to start counting from
135 * __clear_bit() is non-atomic and implies release semantics before the memory
136 * operation. It can be used for an unlock if no other CPUs can concurrently
137 * modify other bits in the word.
139 * No memory barrier is required here, because x86 cannot reorder stores past
140 * older loads. Same principle as spin_unlock.
142 static inline void __clear_bit_unlock(unsigned nr, volatile unsigned long *addr)
144 barrier();
145 __clear_bit(nr, addr);
148 #define smp_mb__before_clear_bit() barrier()
149 #define smp_mb__after_clear_bit() barrier()
152 * __change_bit - Toggle a bit in memory
153 * @nr: the bit to change
154 * @addr: the address to start counting from
156 * Unlike change_bit(), this function is non-atomic and may be reordered.
157 * If it's called on the same region of memory simultaneously, the effect
158 * may be that only one operation succeeds.
160 static inline void __change_bit(int nr, volatile unsigned long *addr)
162 asm volatile("btc %1,%0" : ADDR : "Ir" (nr));
166 * change_bit - Toggle a bit in memory
167 * @nr: Bit to change
168 * @addr: Address to start counting from
170 * change_bit() is atomic and may not be reordered.
171 * Note that @nr may be almost arbitrarily large; this function is not
172 * restricted to acting on a single-word quantity.
174 static inline void change_bit(int nr, volatile unsigned long *addr)
176 if (IS_IMMEDIATE(nr)) {
177 asm volatile(LOCK_PREFIX "xorb %1,%0"
178 : CONST_MASK_ADDR(nr, addr)
179 : "iq" ((u8)CONST_MASK(nr)));
180 } else {
181 asm volatile(LOCK_PREFIX "btc %1,%0"
182 : BITOP_ADDR(addr)
183 : "Ir" (nr));
188 * test_and_set_bit - Set a bit and return its old value
189 * @nr: Bit to set
190 * @addr: Address to count from
192 * This operation is atomic and cannot be reordered.
193 * It also implies a memory barrier.
195 static inline int test_and_set_bit(int nr, volatile unsigned long *addr)
197 int oldbit;
199 asm volatile(LOCK_PREFIX "bts %2,%1\n\t"
200 "sbb %0,%0" : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
202 return oldbit;
206 * test_and_set_bit_lock - Set a bit and return its old value for lock
207 * @nr: Bit to set
208 * @addr: Address to count from
210 * This is the same as test_and_set_bit on x86.
212 static __always_inline int
213 test_and_set_bit_lock(int nr, volatile unsigned long *addr)
215 return test_and_set_bit(nr, addr);
219 * __test_and_set_bit - Set a bit and return its old value
220 * @nr: Bit to set
221 * @addr: Address to count from
223 * This operation is non-atomic and can be reordered.
224 * If two examples of this operation race, one can appear to succeed
225 * but actually fail. You must protect multiple accesses with a lock.
227 static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
229 int oldbit;
231 asm("bts %2,%1\n\t"
232 "sbb %0,%0"
233 : "=r" (oldbit), ADDR
234 : "Ir" (nr));
235 return oldbit;
239 * test_and_clear_bit - Clear a bit and return its old value
240 * @nr: Bit to clear
241 * @addr: Address to count from
243 * This operation is atomic and cannot be reordered.
244 * It also implies a memory barrier.
246 static inline int test_and_clear_bit(int nr, volatile unsigned long *addr)
248 int oldbit;
250 asm volatile(LOCK_PREFIX "btr %2,%1\n\t"
251 "sbb %0,%0"
252 : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
254 return oldbit;
258 * __test_and_clear_bit - Clear a bit and return its old value
259 * @nr: Bit to clear
260 * @addr: Address to count from
262 * This operation is non-atomic and can be reordered.
263 * If two examples of this operation race, one can appear to succeed
264 * but actually fail. You must protect multiple accesses with a lock.
266 static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
268 int oldbit;
270 asm volatile("btr %2,%1\n\t"
271 "sbb %0,%0"
272 : "=r" (oldbit), ADDR
273 : "Ir" (nr));
274 return oldbit;
277 /* WARNING: non atomic and it can be reordered! */
278 static inline int __test_and_change_bit(int nr, volatile unsigned long *addr)
280 int oldbit;
282 asm volatile("btc %2,%1\n\t"
283 "sbb %0,%0"
284 : "=r" (oldbit), ADDR
285 : "Ir" (nr) : "memory");
287 return oldbit;
291 * test_and_change_bit - Change a bit and return its old value
292 * @nr: Bit to change
293 * @addr: Address to count from
295 * This operation is atomic and cannot be reordered.
296 * It also implies a memory barrier.
298 static inline int test_and_change_bit(int nr, volatile unsigned long *addr)
300 int oldbit;
302 asm volatile(LOCK_PREFIX "btc %2,%1\n\t"
303 "sbb %0,%0"
304 : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
306 return oldbit;
309 static __always_inline int constant_test_bit(unsigned int nr, const volatile unsigned long *addr)
311 return ((1UL << (nr % BITS_PER_LONG)) &
312 (addr[nr / BITS_PER_LONG])) != 0;
315 static inline int variable_test_bit(int nr, volatile const unsigned long *addr)
317 int oldbit;
319 asm volatile("bt %2,%1\n\t"
320 "sbb %0,%0"
321 : "=r" (oldbit)
322 : "m" (*(unsigned long *)addr), "Ir" (nr));
324 return oldbit;
327 #if 0 /* Fool kernel-doc since it doesn't do macros yet */
329 * test_bit - Determine whether a bit is set
330 * @nr: bit number to test
331 * @addr: Address to start counting from
333 static int test_bit(int nr, const volatile unsigned long *addr);
334 #endif
336 #define test_bit(nr, addr) \
337 (__builtin_constant_p((nr)) \
338 ? constant_test_bit((nr), (addr)) \
339 : variable_test_bit((nr), (addr)))
342 * __ffs - find first set bit in word
343 * @word: The word to search
345 * Undefined if no bit exists, so code should check against 0 first.
347 static inline unsigned long __ffs(unsigned long word)
349 asm("bsf %1,%0"
350 : "=r" (word)
351 : "rm" (word));
352 return word;
356 * ffz - find first zero bit in word
357 * @word: The word to search
359 * Undefined if no zero exists, so code should check against ~0UL first.
361 static inline unsigned long ffz(unsigned long word)
363 asm("bsf %1,%0"
364 : "=r" (word)
365 : "r" (~word));
366 return word;
370 * __fls: find last set bit in word
371 * @word: The word to search
373 * Undefined if no set bit exists, so code should check against 0 first.
375 static inline unsigned long __fls(unsigned long word)
377 asm("bsr %1,%0"
378 : "=r" (word)
379 : "rm" (word));
380 return word;
383 #ifdef __KERNEL__
385 * ffs - find first set bit in word
386 * @x: the word to search
388 * This is defined the same way as the libc and compiler builtin ffs
389 * routines, therefore differs in spirit from the other bitops.
391 * ffs(value) returns 0 if value is 0 or the position of the first
392 * set bit if value is nonzero. The first (least significant) bit
393 * is at position 1.
395 static inline int ffs(int x)
397 int r;
398 #ifdef CONFIG_X86_CMOV
399 asm("bsfl %1,%0\n\t"
400 "cmovzl %2,%0"
401 : "=r" (r) : "rm" (x), "r" (-1));
402 #else
403 asm("bsfl %1,%0\n\t"
404 "jnz 1f\n\t"
405 "movl $-1,%0\n"
406 "1:" : "=r" (r) : "rm" (x));
407 #endif
408 return r + 1;
412 * fls - find last set bit in word
413 * @x: the word to search
415 * This is defined in a similar way as the libc and compiler builtin
416 * ffs, but returns the position of the most significant set bit.
418 * fls(value) returns 0 if value is 0 or the position of the last
419 * set bit if value is nonzero. The last (most significant) bit is
420 * at position 32.
422 static inline int fls(int x)
424 int r;
425 #ifdef CONFIG_X86_CMOV
426 asm("bsrl %1,%0\n\t"
427 "cmovzl %2,%0"
428 : "=&r" (r) : "rm" (x), "rm" (-1));
429 #else
430 asm("bsrl %1,%0\n\t"
431 "jnz 1f\n\t"
432 "movl $-1,%0\n"
433 "1:" : "=r" (r) : "rm" (x));
434 #endif
435 return r + 1;
437 #endif /* __KERNEL__ */
439 #undef ADDR
441 #ifdef __KERNEL__
443 #include <asm-generic/bitops/find.h>
445 #include <asm-generic/bitops/sched.h>
447 #define ARCH_HAS_FAST_MULTIPLIER 1
449 #include <asm/arch_hweight.h>
451 #include <asm-generic/bitops/const_hweight.h>
453 #endif /* __KERNEL__ */
455 #include <asm-generic/bitops/fls64.h>
457 #ifdef __KERNEL__
459 #include <asm-generic/bitops/le.h>
461 #include <asm-generic/bitops/ext2-atomic-setbit.h>
463 #endif /* __KERNEL__ */
464 #endif /* _ASM_X86_BITOPS_H */