include/asm-x86/bitops.h

   1 #ifndef _ASM_X86_BITOPS_H
   2 #define _ASM_X86_BITOPS_H
   3
   4 /*
   5  * Copyright 1992, Linus Torvalds.
   6  */
   7
   8 #ifndef _LINUX_BITOPS_H
   9 #error only <linux/bitops.h> can be included directly
  10 #endif
  11
  12 #include <linux/compiler.h>
  13 #include <asm/alternative.h>
  14
  15 /*
  16  * These have to be done with inline assembly: that way the bit-setting
  17  * is guaranteed to be atomic. All bit operations return 0 if the bit
  18  * was cleared before the operation and != 0 if it was not.
  19  *
  20  * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
  21  */
  22
  23 #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
  24 /* Technically wrong, but this avoids compilation errors on some gcc
  25    versions. */
  26 #define ADDR "=m" (*(volatile long *)addr)
  27 #define BIT_ADDR "=m" (((volatile int *)addr)[nr >> 5])
  28 #else
  29 #define ADDR "+m" (*(volatile long *) addr)
  30 #define BIT_ADDR "+m" (((volatile int *)addr)[nr >> 5])
  31 #endif
  32 #define BASE_ADDR "m" (*(volatile int *)addr)
  33
  34 /**
  35  * set_bit - Atomically set a bit in memory
  36  * @nr: the bit to set
  37  * @addr: the address to start counting from
  38  *
  39  * This function is atomic and may not be reordered.  See __set_bit()
  40  * if you do not require the atomic guarantees.
  41  *
  42  * Note: there are no guarantees that this function will not be reordered
  43  * on non x86 architectures, so if you are writing portable code,
  44  * make sure not to rely on its reordering guarantees.
  45  *
  46  * Note that @nr may be almost arbitrarily large; this function is not
  47  * restricted to acting on a single-word quantity.
  48  */
  49 static inline void set_bit(int nr, volatile void *addr)
  50 {
  51         asm volatile(LOCK_PREFIX "bts %1,%0" : ADDR : "Ir" (nr) : "memory");
  52 }
  53
  54 /**
  55  * __set_bit - Set a bit in memory
  56  * @nr: the bit to set
  57  * @addr: the address to start counting from
  58  *
  59  * Unlike set_bit(), this function is non-atomic and may be reordered.
  60  * If it's called on the same region of memory simultaneously, the effect
  61  * may be that only one operation succeeds.
  62  */
  63 static inline void __set_bit(int nr, volatile void *addr)
  64 {
  65         asm volatile("bts %1,%0"
  66                      : ADDR
  67                      : "Ir" (nr) : "memory");
  68 }
  69
  70
  71 /**
  72  * clear_bit - Clears a bit in memory
  73  * @nr: Bit to clear
  74  * @addr: Address to start counting from
  75  *
  76  * clear_bit() is atomic and may not be reordered.  However, it does
  77  * not contain a memory barrier, so if it is used for locking purposes,
  78  * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
  79  * in order to ensure changes are visible on other processors.
  80  */
  81 static inline void clear_bit(int nr, volatile void *addr)
  82 {
  83         asm volatile(LOCK_PREFIX "btr %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
  84 }
  85
  86 /*
  87  * clear_bit_unlock - Clears a bit in memory
  88  * @nr: Bit to clear
  89  * @addr: Address to start counting from
  90  *
  91  * clear_bit() is atomic and implies release semantics before the memory
  92  * operation. It can be used for an unlock.
  93  */
  94 static inline void clear_bit_unlock(unsigned nr, volatile void *addr)
  95 {
  96         barrier();
  97         clear_bit(nr, addr);
  98 }
  99
 100 static inline void __clear_bit(int nr, volatile void *addr)
 101 {
 102         asm volatile("btr %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
 103 }
 104
 105 /*
 106  * __clear_bit_unlock - Clears a bit in memory
 107  * @nr: Bit to clear
 108  * @addr: Address to start counting from
 109  *
 110  * __clear_bit() is non-atomic and implies release semantics before the memory
 111  * operation. It can be used for an unlock if no other CPUs can concurrently
 112  * modify other bits in the word.
 113  *
 114  * No memory barrier is required here, because x86 cannot reorder stores past
 115  * older loads. Same principle as spin_unlock.
 116  */
 117 static inline void __clear_bit_unlock(unsigned nr, volatile void *addr)
 118 {
 119         barrier();
 120         __clear_bit(nr, addr);
 121 }
 122
 123 #define smp_mb__before_clear_bit()      barrier()
 124 #define smp_mb__after_clear_bit()       barrier()
 125
 126 /**
 127  * __change_bit - Toggle a bit in memory
 128  * @nr: the bit to change
 129  * @addr: the address to start counting from
 130  *
 131  * Unlike change_bit(), this function is non-atomic and may be reordered.
 132  * If it's called on the same region of memory simultaneously, the effect
 133  * may be that only one operation succeeds.
 134  */
 135 static inline void __change_bit(int nr, volatile void *addr)
 136 {
 137         asm volatile("btc %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
 138 }
 139
 140 /**
 141  * change_bit - Toggle a bit in memory
 142  * @nr: Bit to change
 143  * @addr: Address to start counting from
 144  *
 145  * change_bit() is atomic and may not be reordered.
 146  * Note that @nr may be almost arbitrarily large; this function is not
 147  * restricted to acting on a single-word quantity.
 148  */
 149 static inline void change_bit(int nr, volatile void *addr)
 150 {
 151         asm volatile(LOCK_PREFIX "btc %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
 152 }
 153
 154 /**
 155  * test_and_set_bit - Set a bit and return its old value
 156  * @nr: Bit to set
 157  * @addr: Address to count from
 158  *
 159  * This operation is atomic and cannot be reordered.
 160  * It also implies a memory barrier.
 161  */
 162 static inline int test_and_set_bit(int nr, volatile void *addr)
 163 {
 164         int oldbit;
 165
 166         asm volatile(LOCK_PREFIX "bts %2,%1\n\t"
 167                      "sbb %0,%0" : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
 168
 169         return oldbit;
 170 }
 171
 172 /**
 173  * test_and_set_bit_lock - Set a bit and return its old value for lock
 174  * @nr: Bit to set
 175  * @addr: Address to count from
 176  *
 177  * This is the same as test_and_set_bit on x86.
 178  */
 179 static inline int test_and_set_bit_lock(int nr, volatile void *addr)
 180 {
 181         return test_and_set_bit(nr, addr);
 182 }
 183
 184 /**
 185  * __test_and_set_bit - Set a bit and return its old value
 186  * @nr: Bit to set
 187  * @addr: Address to count from
 188  *
 189  * This operation is non-atomic and can be reordered.
 190  * If two examples of this operation race, one can appear to succeed
 191  * but actually fail.  You must protect multiple accesses with a lock.
 192  */
 193 static inline int __test_and_set_bit(int nr, volatile void *addr)
 194 {
 195         int oldbit;
 196
 197         asm volatile("bts %2,%3\n\t"
 198                      "sbb %0,%0"
 199                      : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR);
 200         return oldbit;
 201 }
 202
 203 /**
 204  * test_and_clear_bit - Clear a bit and return its old value
 205  * @nr: Bit to clear
 206  * @addr: Address to count from
 207  *
 208  * This operation is atomic and cannot be reordered.
 209  * It also implies a memory barrier.
 210  */
 211 static inline int test_and_clear_bit(int nr, volatile void *addr)
 212 {
 213         int oldbit;
 214
 215         asm volatile(LOCK_PREFIX "btr %2,%1\n\t"
 216                      "sbb %0,%0"
 217                      : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
 218
 219         return oldbit;
 220 }
 221
 222 /**
 223  * __test_and_clear_bit - Clear a bit and return its old value
 224  * @nr: Bit to clear
 225  * @addr: Address to count from
 226  *
 227  * This operation is non-atomic and can be reordered.
 228  * If two examples of this operation race, one can appear to succeed
 229  * but actually fail.  You must protect multiple accesses with a lock.
 230  */
 231 static inline int __test_and_clear_bit(int nr, volatile void *addr)
 232 {
 233         int oldbit;
 234
 235         asm volatile("btr %2,%3\n\t"
 236                      "sbb %0,%0"
 237                      : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR);
 238         return oldbit;
 239 }
 240
 241 /* WARNING: non atomic and it can be reordered! */
 242 static inline int __test_and_change_bit(int nr, volatile void *addr)
 243 {
 244         int oldbit;
 245
 246         asm volatile("btc %2,%3\n\t"
 247                      "sbb %0,%0"
 248                      : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR);
 249
 250         return oldbit;
 251 }
 252
 253 /**
 254  * test_and_change_bit - Change a bit and return its old value
 255  * @nr: Bit to change
 256  * @addr: Address to count from
 257  *
 258  * This operation is atomic and cannot be reordered.
 259  * It also implies a memory barrier.
 260  */
 261 static inline int test_and_change_bit(int nr, volatile void *addr)
 262 {
 263         int oldbit;
 264
 265         asm volatile(LOCK_PREFIX "btc %2,%1\n\t"
 266                      "sbb %0,%0"
 267                      : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
 268
 269         return oldbit;
 270 }
 271
 272 static inline int constant_test_bit(int nr, const volatile void *addr)
 273 {
 274         return ((1UL << (nr % BITS_PER_LONG)) &
 275                 (((unsigned long *)addr)[nr / BITS_PER_LONG])) != 0;
 276 }
 277
 278 static inline int variable_test_bit(int nr, volatile const void *addr)
 279 {
 280         int oldbit;
 281
 282         asm volatile("bt %2,%3\n\t"
 283                      "sbb %0,%0"
 284                      : "=r" (oldbit)
 285                      : "m" (((volatile const int *)addr)[nr >> 5]),
 286                        "Ir" (nr), BASE_ADDR);
 287
 288         return oldbit;
 289 }
 290
 291 #if 0 /* Fool kernel-doc since it doesn't do macros yet */
 292 /**
 293  * test_bit - Determine whether a bit is set
 294  * @nr: bit number to test
 295  * @addr: Address to start counting from
 296  */
 297 static int test_bit(int nr, const volatile unsigned long *addr);
 298 #endif
 299
 300 #define test_bit(nr,addr)                       \
 301         (__builtin_constant_p(nr) ?             \
 302          constant_test_bit((nr),(addr)) :       \
 303          variable_test_bit((nr),(addr)))
 304
 305 #undef BASE_ADDR
 306 #undef BIT_ADDR
 307 #undef ADDR
 308
 309 #ifdef CONFIG_X86_32
 310 # include "bitops_32.h"
 311 #else
 312 # include "bitops_64.h"
 313 #endif
 314
 315 #endif  /* _ASM_X86_BITOPS_H */