arch/s390/include/asm/bitops.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  *    Copyright IBM Corp. 1999,2013
   4  *
   5  *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
   6  *
   7  * The description below was taken in large parts from the powerpc
   8  * bitops header file:
   9  * Within a word, bits are numbered LSB first.  Lot's of places make
  10  * this assumption by directly testing bits with (val & (1<<nr)).
  11  * This can cause confusion for large (> 1 word) bitmaps on a
  12  * big-endian system because, unlike little endian, the number of each
  13  * bit depends on the word size.
  14  *
  15  * The bitop functions are defined to work on unsigned longs, so the bits
  16  * end up numbered:
  17  *   |63..............0|127............64|191...........128|255...........192|
  18  *
  19  * We also have special functions which work with an MSB0 encoding.
  20  * The bits are numbered:
  21  *   |0..............63|64............127|128...........191|192...........255|
  22  *
  23  * The main difference is that bit 0-63 in the bit number field needs to be
  24  * reversed compared to the LSB0 encoded bit fields. This can be achieved by
  25  * XOR with 0x3f.
  26  *
  27  */
  28
  29 #ifndef _S390_BITOPS_H
  30 #define _S390_BITOPS_H
  31
  32 #ifndef _LINUX_BITOPS_H
  33 #error only <linux/bitops.h> can be included directly
  34 #endif
  35
  36 #include <linux/typecheck.h>
  37 #include <linux/compiler.h>
  38 #include <asm/atomic_ops.h>
  39 #include <asm/barrier.h>
  40
  41 #define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG)
  42
  43 static inline unsigned long *
  44 __bitops_word(unsigned long nr, volatile unsigned long *ptr)
  45 {
  46         unsigned long addr;
  47
  48         addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);
  49         return (unsigned long *)addr;
  50 }
  51
  52 static inline unsigned char *
  53 __bitops_byte(unsigned long nr, volatile unsigned long *ptr)
  54 {
  55         return ((unsigned char *)ptr) + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
  56 }
  57
  58 static inline void set_bit(unsigned long nr, volatile unsigned long *ptr)
  59 {
  60         unsigned long *addr = __bitops_word(nr, ptr);
  61         unsigned long mask;
  62
  63 #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
  64         if (__builtin_constant_p(nr)) {
  65                 unsigned char *caddr = __bitops_byte(nr, ptr);
  66
  67                 asm volatile(
  68                         "oi     %0,%b1\n"
  69                         : "+Q" (*caddr)
  70                         : "i" (1 << (nr & 7))
  71                         : "cc", "memory");
  72                 return;
  73         }
  74 #endif
  75         mask = 1UL << (nr & (BITS_PER_LONG - 1));
  76         __atomic64_or(mask, (long *)addr);
  77 }
  78
  79 static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
  80 {
  81         unsigned long *addr = __bitops_word(nr, ptr);
  82         unsigned long mask;
  83
  84 #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
  85         if (__builtin_constant_p(nr)) {
  86                 unsigned char *caddr = __bitops_byte(nr, ptr);
  87
  88                 asm volatile(
  89                         "ni     %0,%b1\n"
  90                         : "+Q" (*caddr)
  91                         : "i" (~(1 << (nr & 7)))
  92                         : "cc", "memory");
  93                 return;
  94         }
  95 #endif
  96         mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
  97         __atomic64_and(mask, (long *)addr);
  98 }
  99
 100 static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
 101 {
 102         unsigned long *addr = __bitops_word(nr, ptr);
 103         unsigned long mask;
 104
 105 #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
 106         if (__builtin_constant_p(nr)) {
 107                 unsigned char *caddr = __bitops_byte(nr, ptr);
 108
 109                 asm volatile(
 110                         "xi     %0,%b1\n"
 111                         : "+Q" (*caddr)
 112                         : "i" (1 << (nr & 7))
 113                         : "cc", "memory");
 114                 return;
 115         }
 116 #endif
 117         mask = 1UL << (nr & (BITS_PER_LONG - 1));
 118         __atomic64_xor(mask, (long *)addr);
 119 }
 120
 121 static inline int
 122 test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
 123 {
 124         unsigned long *addr = __bitops_word(nr, ptr);
 125         unsigned long old, mask;
 126
 127         mask = 1UL << (nr & (BITS_PER_LONG - 1));
 128         old = __atomic64_or_barrier(mask, (long *)addr);
 129         return (old & mask) != 0;
 130 }
 131
 132 static inline int
 133 test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
 134 {
 135         unsigned long *addr = __bitops_word(nr, ptr);
 136         unsigned long old, mask;
 137
 138         mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
 139         old = __atomic64_and_barrier(mask, (long *)addr);
 140         return (old & ~mask) != 0;
 141 }
 142
 143 static inline int
 144 test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
 145 {
 146         unsigned long *addr = __bitops_word(nr, ptr);
 147         unsigned long old, mask;
 148
 149         mask = 1UL << (nr & (BITS_PER_LONG - 1));
 150         old = __atomic64_xor_barrier(mask, (long *)addr);
 151         return (old & mask) != 0;
 152 }
 153
 154 static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
 155 {
 156         unsigned char *addr = __bitops_byte(nr, ptr);
 157
 158         *addr |= 1 << (nr & 7);
 159 }
 160
 161 static inline void
 162 __clear_bit(unsigned long nr, volatile unsigned long *ptr)
 163 {
 164         unsigned char *addr = __bitops_byte(nr, ptr);
 165
 166         *addr &= ~(1 << (nr & 7));
 167 }
 168
 169 static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
 170 {
 171         unsigned char *addr = __bitops_byte(nr, ptr);
 172
 173         *addr ^= 1 << (nr & 7);
 174 }
 175
 176 static inline int
 177 __test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
 178 {
 179         unsigned char *addr = __bitops_byte(nr, ptr);
 180         unsigned char ch;
 181
 182         ch = *addr;
 183         *addr |= 1 << (nr & 7);
 184         return (ch >> (nr & 7)) & 1;
 185 }
 186
 187 static inline int
 188 __test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
 189 {
 190         unsigned char *addr = __bitops_byte(nr, ptr);
 191         unsigned char ch;
 192
 193         ch = *addr;
 194         *addr &= ~(1 << (nr & 7));
 195         return (ch >> (nr & 7)) & 1;
 196 }
 197
 198 static inline int
 199 __test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
 200 {
 201         unsigned char *addr = __bitops_byte(nr, ptr);
 202         unsigned char ch;
 203
 204         ch = *addr;
 205         *addr ^= 1 << (nr & 7);
 206         return (ch >> (nr & 7)) & 1;
 207 }
 208
 209 static inline int test_bit(unsigned long nr, const volatile unsigned long *ptr)
 210 {
 211         const volatile unsigned char *addr;
 212
 213         addr = ((const volatile unsigned char *)ptr);
 214         addr += (nr ^ (BITS_PER_LONG - 8)) >> 3;
 215         return (*addr >> (nr & 7)) & 1;
 216 }
 217
 218 static inline int test_and_set_bit_lock(unsigned long nr,
 219                                         volatile unsigned long *ptr)
 220 {
 221         if (test_bit(nr, ptr))
 222                 return 1;
 223         return test_and_set_bit(nr, ptr);
 224 }
 225
 226 static inline void clear_bit_unlock(unsigned long nr,
 227                                     volatile unsigned long *ptr)
 228 {
 229         smp_mb__before_atomic();
 230         clear_bit(nr, ptr);
 231 }
 232
 233 static inline void __clear_bit_unlock(unsigned long nr,
 234                                       volatile unsigned long *ptr)
 235 {
 236         smp_mb();
 237         __clear_bit(nr, ptr);
 238 }
 239
 240 /*
 241  * Functions which use MSB0 bit numbering.
 242  * The bits are numbered:
 243  *   |0..............63|64............127|128...........191|192...........255|
 244  */
 245 unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size);
 246 unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
 247                                 unsigned long offset);
 248
 249 #define for_each_set_bit_inv(bit, addr, size)                           \
 250         for ((bit) = find_first_bit_inv((addr), (size));                \
 251              (bit) < (size);                                            \
 252              (bit) = find_next_bit_inv((addr), (size), (bit) + 1))
 253
 254 static inline void set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 255 {
 256         return set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 257 }
 258
 259 static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 260 {
 261         return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 262 }
 263
 264 static inline int test_and_clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 265 {
 266         return test_and_clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 267 }
 268
 269 static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 270 {
 271         return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 272 }
 273
 274 static inline void __clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 275 {
 276         return __clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 277 }
 278
 279 static inline int test_bit_inv(unsigned long nr,
 280                                const volatile unsigned long *ptr)
 281 {
 282         return test_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 283 }
 284
 285 #ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
 286
 287 /**
 288  * __flogr - find leftmost one
 289  * @word - The word to search
 290  *
 291  * Returns the bit number of the most significant bit set,
 292  * where the most significant bit has bit number 0.
 293  * If no bit is set this function returns 64.
 294  */
 295 static inline unsigned char __flogr(unsigned long word)
 296 {
 297         if (__builtin_constant_p(word)) {
 298                 unsigned long bit = 0;
 299
 300                 if (!word)
 301                         return 64;
 302                 if (!(word & 0xffffffff00000000UL)) {
 303                         word <<= 32;
 304                         bit += 32;
 305                 }
 306                 if (!(word & 0xffff000000000000UL)) {
 307                         word <<= 16;
 308                         bit += 16;
 309                 }
 310                 if (!(word & 0xff00000000000000UL)) {
 311                         word <<= 8;
 312                         bit += 8;
 313                 }
 314                 if (!(word & 0xf000000000000000UL)) {
 315                         word <<= 4;
 316                         bit += 4;
 317                 }
 318                 if (!(word & 0xc000000000000000UL)) {
 319                         word <<= 2;
 320                         bit += 2;
 321                 }
 322                 if (!(word & 0x8000000000000000UL)) {
 323                         word <<= 1;
 324                         bit += 1;
 325                 }
 326                 return bit;
 327         } else {
 328                 register unsigned long bit asm("4") = word;
 329                 register unsigned long out asm("5");
 330
 331                 asm volatile(
 332                         "       flogr   %[bit],%[bit]\n"
 333                         : [bit] "+d" (bit), [out] "=d" (out) : : "cc");
 334                 return bit;
 335         }
 336 }
 337
 338 /**
 339  * __ffs - find first bit in word.
 340  * @word: The word to search
 341  *
 342  * Undefined if no bit exists, so code should check against 0 first.
 343  */
 344 static inline unsigned long __ffs(unsigned long word)
 345 {
 346         return __flogr(-word & word) ^ (BITS_PER_LONG - 1);
 347 }
 348
 349 /**
 350  * ffs - find first bit set
 351  * @word: the word to search
 352  *
 353  * This is defined the same way as the libc and
 354  * compiler builtin ffs routines (man ffs).
 355  */
 356 static inline int ffs(int word)
 357 {
 358         unsigned long mask = 2 * BITS_PER_LONG - 1;
 359         unsigned int val = (unsigned int)word;
 360
 361         return (1 + (__flogr(-val & val) ^ (BITS_PER_LONG - 1))) & mask;
 362 }
 363
 364 /**
 365  * __fls - find last (most-significant) set bit in a long word
 366  * @word: the word to search
 367  *
 368  * Undefined if no set bit exists, so code should check against 0 first.
 369  */
 370 static inline unsigned long __fls(unsigned long word)
 371 {
 372         return __flogr(word) ^ (BITS_PER_LONG - 1);
 373 }
 374
 375 /**
 376  * fls64 - find last set bit in a 64-bit word
 377  * @word: the word to search
 378  *
 379  * This is defined in a similar way as the libc and compiler builtin
 380  * ffsll, but returns the position of the most significant set bit.
 381  *
 382  * fls64(value) returns 0 if value is 0 or the position of the last
 383  * set bit if value is nonzero. The last (most significant) bit is
 384  * at position 64.
 385  */
 386 static inline int fls64(unsigned long word)
 387 {
 388         unsigned long mask = 2 * BITS_PER_LONG - 1;
 389
 390         return (1 + (__flogr(word) ^ (BITS_PER_LONG - 1))) & mask;
 391 }
 392
 393 /**
 394  * fls - find last (most-significant) bit set
 395  * @word: the word to search
 396  *
 397  * This is defined the same way as ffs.
 398  * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
 399  */
 400 static inline int fls(unsigned int word)
 401 {
 402         return fls64(word);
 403 }
 404
 405 #else /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
 406
 407 #include <asm-generic/bitops/__ffs.h>
 408 #include <asm-generic/bitops/ffs.h>
 409 #include <asm-generic/bitops/__fls.h>
 410 #include <asm-generic/bitops/fls.h>
 411 #include <asm-generic/bitops/fls64.h>
 412
 413 #endif /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
 414
 415 #include <asm-generic/bitops/ffz.h>
 416 #include <asm-generic/bitops/find.h>
 417 #include <asm-generic/bitops/hweight.h>
 418 #include <asm-generic/bitops/sched.h>
 419 #include <asm-generic/bitops/le.h>
 420 #include <asm-generic/bitops/ext2-atomic-setbit.h>
 421
 422 #endif /* _S390_BITOPS_H */