include/asm-ppc64/bitops.h

   1 /*
   2  * PowerPC64 atomic bit operations.
   3  * Dave Engebretsen, Todd Inglett, Don Reed, Pat McCarthy, Peter Bergner,
   4  * Anton Blanchard
   5  *
   6  * Originally taken from the 32b PPC code.  Modified to use 64b values for
   7  * the various counters & memory references.
   8  *
   9  * Bitops are odd when viewed on big-endian systems.  They were designed
  10  * on little endian so the size of the bitset doesn't matter (low order bytes
  11  * come first) as long as the bit in question is valid.
  12  *
  13  * Bits are "tested" often using the C expression (val & (1<<nr)) so we do
  14  * our best to stay compatible with that.  The assumption is that val will
  15  * be unsigned long for such tests.  As such, we assume the bits are stored
  16  * as an array of unsigned long (the usual case is a single unsigned long,
  17  * of course).  Here's an example bitset with bit numbering:
  18  *
  19  *   |63..........0|127........64|195.......128|255.......196|
  20  *
  21  * This leads to a problem. If an int, short or char is passed as a bitset
  22  * it will be a bad memory reference since we want to store in chunks
  23  * of unsigned long (64 bits here) size.
  24  *
  25  * There are a few little-endian macros used mostly for filesystem bitmaps,
  26  * these work on similar bit arrays layouts, but byte-oriented:
  27  *
  28  *   |7...0|15...8|23...16|31...24|39...32|47...40|55...48|63...56|
  29  *
  30  * The main difference is that bit 3-5 in the bit number field needs to be
  31  * reversed compared to the big-endian bit fields. This can be achieved
  32  * by XOR with 0b111000 (0x38).
  33  *
  34  * This program is free software; you can redistribute it and/or
  35  * modify it under the terms of the GNU General Public License
  36  * as published by the Free Software Foundation; either version
  37  * 2 of the License, or (at your option) any later version.
  38  */
  39
  40 #ifndef _PPC64_BITOPS_H
  41 #define _PPC64_BITOPS_H
  42
  43 #ifdef __KERNEL__
  44
  45 #include <asm/memory.h>
  46
  47 /*
  48  * clear_bit doesn't imply a memory barrier
  49  */
  50 #define smp_mb__before_clear_bit()      smp_mb()
  51 #define smp_mb__after_clear_bit()       smp_mb()
  52
  53 static __inline__ int test_bit(unsigned long nr, __const__ volatile unsigned long *addr)
  54 {
  55         return (1UL & (addr[nr >> 6] >> (nr & 63)));
  56 }
  57
  58 static __inline__ void set_bit(unsigned long nr, volatile unsigned long *addr)
  59 {
  60         unsigned long old;
  61         unsigned long mask = 1UL << (nr & 0x3f);
  62         unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
  63
  64         __asm__ __volatile__(
  65 "1:     ldarx   %0,0,%3         # set_bit\n\
  66         or      %0,%0,%2\n\
  67         stdcx.  %0,0,%3\n\
  68         bne-    1b"
  69         : "=&r" (old), "=m" (*p)
  70         : "r" (mask), "r" (p), "m" (*p)
  71         : "cc");
  72 }
  73
  74 static __inline__ void clear_bit(unsigned long nr, volatile unsigned long *addr)
  75 {
  76         unsigned long old;
  77         unsigned long mask = 1UL << (nr & 0x3f);
  78         unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
  79
  80         __asm__ __volatile__(
  81 "1:     ldarx   %0,0,%3         # clear_bit\n\
  82         andc    %0,%0,%2\n\
  83         stdcx.  %0,0,%3\n\
  84         bne-    1b"
  85         : "=&r" (old), "=m" (*p)
  86         : "r" (mask), "r" (p), "m" (*p)
  87         : "cc");
  88 }
  89
  90 static __inline__ void change_bit(unsigned long nr, volatile unsigned long *addr)
  91 {
  92         unsigned long old;
  93         unsigned long mask = 1UL << (nr & 0x3f);
  94         unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
  95
  96         __asm__ __volatile__(
  97 "1:     ldarx   %0,0,%3         # change_bit\n\
  98         xor     %0,%0,%2\n\
  99         stdcx.  %0,0,%3\n\
 100         bne-    1b"
 101         : "=&r" (old), "=m" (*p)
 102         : "r" (mask), "r" (p), "m" (*p)
 103         : "cc");
 104 }
 105
 106 static __inline__ int test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
 107 {
 108         unsigned long old, t;
 109         unsigned long mask = 1UL << (nr & 0x3f);
 110         unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
 111
 112         __asm__ __volatile__(
 113         EIEIO_ON_SMP
 114 "1:     ldarx   %0,0,%3         # test_and_set_bit\n\
 115         or      %1,%0,%2 \n\
 116         stdcx.  %1,0,%3 \n\
 117         bne-    1b"
 118         ISYNC_ON_SMP
 119         : "=&r" (old), "=&r" (t)
 120         : "r" (mask), "r" (p)
 121         : "cc", "memory");
 122
 123         return (old & mask) != 0;
 124 }
 125
 126 static __inline__ int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
 127 {
 128         unsigned long old, t;
 129         unsigned long mask = 1UL << (nr & 0x3f);
 130         unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
 131
 132         __asm__ __volatile__(
 133         EIEIO_ON_SMP
 134 "1:     ldarx   %0,0,%3         # test_and_clear_bit\n\
 135         andc    %1,%0,%2\n\
 136         stdcx.  %1,0,%3\n\
 137         bne-    1b"
 138         ISYNC_ON_SMP
 139         : "=&r" (old), "=&r" (t)
 140         : "r" (mask), "r" (p)
 141         : "cc", "memory");
 142
 143         return (old & mask) != 0;
 144 }
 145
 146 static __inline__ int test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
 147 {
 148         unsigned long old, t;
 149         unsigned long mask = 1UL << (nr & 0x3f);
 150         unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
 151
 152         __asm__ __volatile__(
 153         EIEIO_ON_SMP
 154 "1:     ldarx   %0,0,%3         # test_and_change_bit\n\
 155         xor     %1,%0,%2\n\
 156         stdcx.  %1,0,%3\n\
 157         bne-    1b"
 158         ISYNC_ON_SMP
 159         : "=&r" (old), "=&r" (t)
 160         : "r" (mask), "r" (p)
 161         : "cc", "memory");
 162
 163         return (old & mask) != 0;
 164 }
 165
 166 static __inline__ void set_bits(unsigned long mask, unsigned long *addr)
 167 {
 168         unsigned long old;
 169
 170         __asm__ __volatile__(
 171 "1:     ldarx   %0,0,%3         # set_bit\n\
 172         or      %0,%0,%2\n\
 173         stdcx.  %0,0,%3\n\
 174         bne-    1b"
 175         : "=&r" (old), "=m" (*addr)
 176         : "r" (mask), "r" (addr), "m" (*addr)
 177         : "cc");
 178 }
 179
 180 /*
 181  * non-atomic versions
 182  */
 183 static __inline__ void __set_bit(unsigned long nr, volatile unsigned long *addr)
 184 {
 185         unsigned long mask = 1UL << (nr & 0x3f);
 186         unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
 187
 188         *p |= mask;
 189 }
 190
 191 static __inline__ void __clear_bit(unsigned long nr, volatile unsigned long *addr)
 192 {
 193         unsigned long mask = 1UL << (nr & 0x3f);
 194         unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
 195
 196         *p &= ~mask;
 197 }
 198
 199 static __inline__ void __change_bit(unsigned long nr, volatile unsigned long *addr)
 200 {
 201         unsigned long mask = 1UL << (nr & 0x3f);
 202         unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
 203
 204         *p ^= mask;
 205 }
 206
 207 static __inline__ int __test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
 208 {
 209         unsigned long mask = 1UL << (nr & 0x3f);
 210         unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
 211         unsigned long old = *p;
 212
 213         *p = old | mask;
 214         return (old & mask) != 0;
 215 }
 216
 217 static __inline__ int __test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
 218 {
 219         unsigned long mask = 1UL << (nr & 0x3f);
 220         unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
 221         unsigned long old = *p;
 222
 223         *p = old & ~mask;
 224         return (old & mask) != 0;
 225 }
 226
 227 static __inline__ int __test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
 228 {
 229         unsigned long mask = 1UL << (nr & 0x3f);
 230         unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
 231         unsigned long old = *p;
 232
 233         *p = old ^ mask;
 234         return (old & mask) != 0;
 235 }
 236
 237 /*
 238  * Return the zero-based bit position (from RIGHT TO LEFT, 63 -> 0) of the
 239  * most significant (left-most) 1-bit in a double word.
 240  */
 241 static __inline__ int __ilog2(unsigned long x)
 242 {
 243         int lz;
 244
 245         asm ("cntlzd %0,%1" : "=r" (lz) : "r" (x));
 246         return 63 - lz;
 247 }
 248
 249 /*
 250  * Determines the bit position of the least significant (rightmost) 0 bit
 251  * in the specified double word. The returned bit position will be zero-based,
 252  * starting from the right side (63 - 0).
 253  */
 254 static __inline__ unsigned long ffz(unsigned long x)
 255 {
 256         /* no zero exists anywhere in the 8 byte area. */
 257         if ((x = ~x) == 0)
 258                 return 64;
 259
 260         /*
 261          * Calculate the bit position of the least signficant '1' bit in x
 262          * (since x has been changed this will actually be the least signficant
 263          * '0' bit in * the original x).  Note: (x & -x) gives us a mask that
 264          * is the least significant * (RIGHT-most) 1-bit of the value in x.
 265          */
 266         return __ilog2(x & -x);
 267 }
 268
 269 static __inline__ int __ffs(unsigned long x)
 270 {
 271         return __ilog2(x & -x);
 272 }
 273
 274 /*
 275  * ffs: find first bit set. This is defined the same way as
 276  * the libc and compiler builtin ffs routines, therefore
 277  * differs in spirit from the above ffz (man ffs).
 278  */
 279 static __inline__ int ffs(int x)
 280 {
 281         unsigned long i = (unsigned long)x;
 282         return __ilog2(i & -i) + 1;
 283 }
 284
 285 /*
 286  * fls: find last (most-significant) bit set.
 287  * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
 288  */
 289 #define fls(x) generic_fls(x)
 290
 291 /*
 292  * hweightN: returns the hamming weight (i.e. the number
 293  * of bits set) of a N-bit word
 294  */
 295 #define hweight64(x) generic_hweight64(x)
 296 #define hweight32(x) generic_hweight32(x)
 297 #define hweight16(x) generic_hweight16(x)
 298 #define hweight8(x) generic_hweight8(x)
 299
 300 extern unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, unsigned long offset);
 301 #define find_first_zero_bit(addr, size) \
 302         find_next_zero_bit((addr), (size), 0)
 303
 304 extern unsigned long find_next_bit(const unsigned long *addr, unsigned long size, unsigned long offset);
 305 #define find_first_bit(addr, size) \
 306         find_next_bit((addr), (size), 0)
 307
 308 extern unsigned long find_next_zero_le_bit(const unsigned long *addr, unsigned long size, unsigned long offset);
 309 #define find_first_zero_le_bit(addr, size) \
 310         find_next_zero_le_bit((addr), (size), 0)
 311
 312 static __inline__ int test_le_bit(unsigned long nr, __const__ unsigned long * addr)
 313 {
 314         __const__ unsigned char *ADDR = (__const__ unsigned char *) addr;
 315         return (ADDR[nr >> 3] >> (nr & 7)) & 1;
 316 }
 317
 318 #define test_and_clear_le_bit(nr, addr) \
 319         test_and_clear_bit((nr) ^ 0x38, (addr))
 320 #define test_and_set_le_bit(nr, addr) \
 321         test_and_set_bit((nr) ^ 0x38, (addr))
 322
 323 /*
 324  * non-atomic versions
 325  */
 326
 327 #define __set_le_bit(nr, addr) \
 328         __set_bit((nr) ^ 0x38, (addr))
 329 #define __clear_le_bit(nr, addr) \
 330         __clear_bit((nr) ^ 0x38, (addr))
 331 #define __test_and_clear_le_bit(nr, addr) \
 332         __test_and_clear_bit((nr) ^ 0x38, (addr))
 333 #define __test_and_set_le_bit(nr, addr) \
 334         __test_and_set_bit((nr) ^ 0x38, (addr))
 335
 336 #define ext2_set_bit(nr,addr) \
 337         __test_and_set_le_bit((nr), (unsigned long*)addr)
 338 #define ext2_clear_bit(nr, addr) \
 339         __test_and_clear_le_bit((nr), (unsigned long*)addr)
 340
 341 #define ext2_set_bit_atomic(lock, nr, addr) \
 342         test_and_set_le_bit((nr), (unsigned long*)addr)
 343 #define ext2_clear_bit_atomic(lock, nr, addr) \
 344         test_and_clear_le_bit((nr), (unsigned long*)addr)
 345
 346
 347 #define ext2_test_bit(nr, addr)      test_le_bit((nr),(unsigned long*)addr)
 348 #define ext2_find_first_zero_bit(addr, size) \
 349         find_first_zero_le_bit((unsigned long*)addr, size)
 350 #define ext2_find_next_zero_bit(addr, size, off) \
 351         find_next_zero_le_bit((unsigned long*)addr, size, off)
 352
 353 #define minix_test_and_set_bit(nr,addr)         test_and_set_bit(nr,addr)
 354 #define minix_set_bit(nr,addr)                  set_bit(nr,addr)
 355 #define minix_test_and_clear_bit(nr,addr)       test_and_clear_bit(nr,addr)
 356 #define minix_test_bit(nr,addr)                 test_bit(nr,addr)
 357 #define minix_find_first_zero_bit(addr,size)    find_first_zero_bit(addr,size)
 358
 359 #endif /* __KERNEL__ */
 360 #endif /* _PPC64_BITOPS_H */