include/asm-s390/bitops.h

   1 #ifndef _S390_BITOPS_H
   2 #define _S390_BITOPS_H
   3
   4 /*
   5  *  include/asm-s390/bitops.h
   6  *
   7  *  S390 version
   8  *    Copyright (C) 1999 IBM Deutschland Entwicklung GmbH, IBM Corporation
   9  *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
  10  *
  11  *  Derived from "include/asm-i386/bitops.h"
  12  *    Copyright (C) 1992, Linus Torvalds
  13  *
  14  */
  15
  16 #ifdef __KERNEL__
  17
  18 #ifndef _LINUX_BITOPS_H
  19 #error only <linux/bitops.h> can be included directly
  20 #endif
  21
  22 #include <linux/compiler.h>
  23
  24 /*
  25  * 32 bit bitops format:
  26  * bit 0 is the LSB of *addr; bit 31 is the MSB of *addr;
  27  * bit 32 is the LSB of *(addr+4). That combined with the
  28  * big endian byte order on S390 give the following bit
  29  * order in memory:
  30  *    1f 1e 1d 1c 1b 1a 19 18 17 16 15 14 13 12 11 10 \
  31  *    0f 0e 0d 0c 0b 0a 09 08 07 06 05 04 03 02 01 00
  32  * after that follows the next long with bit numbers
  33  *    3f 3e 3d 3c 3b 3a 39 38 37 36 35 34 33 32 31 30
  34  *    2f 2e 2d 2c 2b 2a 29 28 27 26 25 24 23 22 21 20
  35  * The reason for this bit ordering is the fact that
  36  * in the architecture independent code bits operations
  37  * of the form "flags |= (1 << bitnr)" are used INTERMIXED
  38  * with operation of the form "set_bit(bitnr, flags)".
  39  *
  40  * 64 bit bitops format:
  41  * bit 0 is the LSB of *addr; bit 63 is the MSB of *addr;
  42  * bit 64 is the LSB of *(addr+8). That combined with the
  43  * big endian byte order on S390 give the following bit
  44  * order in memory:
  45  *    3f 3e 3d 3c 3b 3a 39 38 37 36 35 34 33 32 31 30
  46  *    2f 2e 2d 2c 2b 2a 29 28 27 26 25 24 23 22 21 20
  47  *    1f 1e 1d 1c 1b 1a 19 18 17 16 15 14 13 12 11 10
  48  *    0f 0e 0d 0c 0b 0a 09 08 07 06 05 04 03 02 01 00
  49  * after that follows the next long with bit numbers
  50  *    7f 7e 7d 7c 7b 7a 79 78 77 76 75 74 73 72 71 70
  51  *    6f 6e 6d 6c 6b 6a 69 68 67 66 65 64 63 62 61 60
  52  *    5f 5e 5d 5c 5b 5a 59 58 57 56 55 54 53 52 51 50
  53  *    4f 4e 4d 4c 4b 4a 49 48 47 46 45 44 43 42 41 40
  54  * The reason for this bit ordering is the fact that
  55  * in the architecture independent code bits operations
  56  * of the form "flags |= (1 << bitnr)" are used INTERMIXED
  57  * with operation of the form "set_bit(bitnr, flags)".
  58  */
  59
  60 /* bitmap tables from arch/S390/kernel/bitmap.S */
  61 extern const char _oi_bitmap[];
  62 extern const char _ni_bitmap[];
  63 extern const char _zb_findmap[];
  64 extern const char _sb_findmap[];
  65
  66 #ifndef __s390x__
  67
  68 #define __BITOPS_ALIGN          3
  69 #define __BITOPS_WORDSIZE       32
  70 #define __BITOPS_OR             "or"
  71 #define __BITOPS_AND            "nr"
  72 #define __BITOPS_XOR            "xr"
  73
  74 #if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
  75
  76 #define __BITOPS_LOOP(__old, __new, __addr, __val, __op_string) \
  77         asm volatile(                                           \
  78                 "       l       %0,%2\n"                        \
  79                 "0:     lr      %1,%0\n"                        \
  80                 __op_string "   %1,%3\n"                        \
  81                 "       cs      %0,%1,%2\n"                     \
  82                 "       jl      0b"                             \
  83                 : "=&d" (__old), "=&d" (__new),                 \
  84                   "=Q" (*(unsigned long *) __addr)              \
  85                 : "d" (__val), "Q" (*(unsigned long *) __addr)  \
  86                 : "cc");
  87
  88 #else /* __GNUC__ */
  89
  90 #define __BITOPS_LOOP(__old, __new, __addr, __val, __op_string) \
  91         asm volatile(                                           \
  92                 "       l       %0,0(%4)\n"                     \
  93                 "0:     lr      %1,%0\n"                        \
  94                 __op_string "   %1,%3\n"                        \
  95                 "       cs      %0,%1,0(%4)\n"                  \
  96                 "       jl      0b"                             \
  97                 : "=&d" (__old), "=&d" (__new),                 \
  98                   "=m" (*(unsigned long *) __addr)              \
  99                 : "d" (__val), "a" (__addr),                    \
 100                   "m" (*(unsigned long *) __addr) : "cc");
 101
 102 #endif /* __GNUC__ */
 103
 104 #else /* __s390x__ */
 105
 106 #define __BITOPS_ALIGN          7
 107 #define __BITOPS_WORDSIZE       64
 108 #define __BITOPS_OR             "ogr"
 109 #define __BITOPS_AND            "ngr"
 110 #define __BITOPS_XOR            "xgr"
 111
 112 #if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
 113
 114 #define __BITOPS_LOOP(__old, __new, __addr, __val, __op_string) \
 115         asm volatile(                                           \
 116                 "       lg      %0,%2\n"                        \
 117                 "0:     lgr     %1,%0\n"                        \
 118                 __op_string "   %1,%3\n"                        \
 119                 "       csg     %0,%1,%2\n"                     \
 120                 "       jl      0b"                             \
 121                 : "=&d" (__old), "=&d" (__new),                 \
 122                   "=Q" (*(unsigned long *) __addr)              \
 123                 : "d" (__val), "Q" (*(unsigned long *) __addr)  \
 124                 : "cc");
 125
 126 #else /* __GNUC__ */
 127
 128 #define __BITOPS_LOOP(__old, __new, __addr, __val, __op_string) \
 129         asm volatile(                                           \
 130                 "       lg      %0,0(%4)\n"                     \
 131                 "0:     lgr     %1,%0\n"                        \
 132                 __op_string "   %1,%3\n"                        \
 133                 "       csg     %0,%1,0(%4)\n"                  \
 134                 "       jl      0b"                             \
 135                 : "=&d" (__old), "=&d" (__new),                 \
 136                   "=m" (*(unsigned long *) __addr)              \
 137                 : "d" (__val), "a" (__addr),                    \
 138                   "m" (*(unsigned long *) __addr) : "cc");
 139
 140
 141 #endif /* __GNUC__ */
 142
 143 #endif /* __s390x__ */
 144
 145 #define __BITOPS_WORDS(bits) (((bits)+__BITOPS_WORDSIZE-1)/__BITOPS_WORDSIZE)
 146 #define __BITOPS_BARRIER() asm volatile("" : : : "memory")
 147
 148 #ifdef CONFIG_SMP
 149 /*
 150  * SMP safe set_bit routine based on compare and swap (CS)
 151  */
 152 static inline void set_bit_cs(unsigned long nr, volatile unsigned long *ptr)
 153 {
 154         unsigned long addr, old, new, mask;
 155
 156         addr = (unsigned long) ptr;
 157         /* calculate address for CS */
 158         addr += (nr ^ (nr & (__BITOPS_WORDSIZE - 1))) >> 3;
 159         /* make OR mask */
 160         mask = 1UL << (nr & (__BITOPS_WORDSIZE - 1));
 161         /* Do the atomic update. */
 162         __BITOPS_LOOP(old, new, addr, mask, __BITOPS_OR);
 163 }
 164
 165 /*
 166  * SMP safe clear_bit routine based on compare and swap (CS)
 167  */
 168 static inline void clear_bit_cs(unsigned long nr, volatile unsigned long *ptr)
 169 {
 170         unsigned long addr, old, new, mask;
 171
 172         addr = (unsigned long) ptr;
 173         /* calculate address for CS */
 174         addr += (nr ^ (nr & (__BITOPS_WORDSIZE - 1))) >> 3;
 175         /* make AND mask */
 176         mask = ~(1UL << (nr & (__BITOPS_WORDSIZE - 1)));
 177         /* Do the atomic update. */
 178         __BITOPS_LOOP(old, new, addr, mask, __BITOPS_AND);
 179 }
 180
 181 /*
 182  * SMP safe change_bit routine based on compare and swap (CS)
 183  */
 184 static inline void change_bit_cs(unsigned long nr, volatile unsigned long *ptr)
 185 {
 186         unsigned long addr, old, new, mask;
 187
 188         addr = (unsigned long) ptr;
 189         /* calculate address for CS */
 190         addr += (nr ^ (nr & (__BITOPS_WORDSIZE - 1))) >> 3;
 191         /* make XOR mask */
 192         mask = 1UL << (nr & (__BITOPS_WORDSIZE - 1));
 193         /* Do the atomic update. */
 194         __BITOPS_LOOP(old, new, addr, mask, __BITOPS_XOR);
 195 }
 196
 197 /*
 198  * SMP safe test_and_set_bit routine based on compare and swap (CS)
 199  */
 200 static inline int
 201 test_and_set_bit_cs(unsigned long nr, volatile unsigned long *ptr)
 202 {
 203         unsigned long addr, old, new, mask;
 204
 205         addr = (unsigned long) ptr;
 206         /* calculate address for CS */
 207         addr += (nr ^ (nr & (__BITOPS_WORDSIZE - 1))) >> 3;
 208         /* make OR/test mask */
 209         mask = 1UL << (nr & (__BITOPS_WORDSIZE - 1));
 210         /* Do the atomic update. */
 211         __BITOPS_LOOP(old, new, addr, mask, __BITOPS_OR);
 212         __BITOPS_BARRIER();
 213         return (old & mask) != 0;
 214 }
 215
 216 /*
 217  * SMP safe test_and_clear_bit routine based on compare and swap (CS)
 218  */
 219 static inline int
 220 test_and_clear_bit_cs(unsigned long nr, volatile unsigned long *ptr)
 221 {
 222         unsigned long addr, old, new, mask;
 223
 224         addr = (unsigned long) ptr;
 225         /* calculate address for CS */
 226         addr += (nr ^ (nr & (__BITOPS_WORDSIZE - 1))) >> 3;
 227         /* make AND/test mask */
 228         mask = ~(1UL << (nr & (__BITOPS_WORDSIZE - 1)));
 229         /* Do the atomic update. */
 230         __BITOPS_LOOP(old, new, addr, mask, __BITOPS_AND);
 231         __BITOPS_BARRIER();
 232         return (old ^ new) != 0;
 233 }
 234
 235 /*
 236  * SMP safe test_and_change_bit routine based on compare and swap (CS)
 237  */
 238 static inline int
 239 test_and_change_bit_cs(unsigned long nr, volatile unsigned long *ptr)
 240 {
 241         unsigned long addr, old, new, mask;
 242
 243         addr = (unsigned long) ptr;
 244         /* calculate address for CS */
 245         addr += (nr ^ (nr & (__BITOPS_WORDSIZE - 1))) >> 3;
 246         /* make XOR/test mask */
 247         mask = 1UL << (nr & (__BITOPS_WORDSIZE - 1));
 248         /* Do the atomic update. */
 249         __BITOPS_LOOP(old, new, addr, mask, __BITOPS_XOR);
 250         __BITOPS_BARRIER();
 251         return (old & mask) != 0;
 252 }
 253 #endif /* CONFIG_SMP */
 254
 255 /*
 256  * fast, non-SMP set_bit routine
 257  */
 258 static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
 259 {
 260         unsigned long addr;
 261
 262         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 263         asm volatile(
 264                 "       oc      0(1,%1),0(%2)"
 265                 : "=m" (*(char *) addr) : "a" (addr),
 266                   "a" (_oi_bitmap + (nr & 7)), "m" (*(char *) addr) : "cc" );
 267 }
 268
 269 static inline void
 270 __constant_set_bit(const unsigned long nr, volatile unsigned long *ptr)
 271 {
 272         unsigned long addr;
 273
 274         addr = ((unsigned long) ptr) + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 275         *(unsigned char *) addr |= 1 << (nr & 7);
 276 }
 277
 278 #define set_bit_simple(nr,addr) \
 279 (__builtin_constant_p((nr)) ? \
 280  __constant_set_bit((nr),(addr)) : \
 281  __set_bit((nr),(addr)) )
 282
 283 /*
 284  * fast, non-SMP clear_bit routine
 285  */
 286 static inline void
 287 __clear_bit(unsigned long nr, volatile unsigned long *ptr)
 288 {
 289         unsigned long addr;
 290
 291         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 292         asm volatile(
 293                 "       nc      0(1,%1),0(%2)"
 294                 : "=m" (*(char *) addr) : "a" (addr),
 295                   "a" (_ni_bitmap + (nr & 7)), "m" (*(char *) addr) : "cc");
 296 }
 297
 298 static inline void
 299 __constant_clear_bit(const unsigned long nr, volatile unsigned long *ptr)
 300 {
 301         unsigned long addr;
 302
 303         addr = ((unsigned long) ptr) + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 304         *(unsigned char *) addr &= ~(1 << (nr & 7));
 305 }
 306
 307 #define clear_bit_simple(nr,addr) \
 308 (__builtin_constant_p((nr)) ? \
 309  __constant_clear_bit((nr),(addr)) : \
 310  __clear_bit((nr),(addr)) )
 311
 312 /*
 313  * fast, non-SMP change_bit routine
 314  */
 315 static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
 316 {
 317         unsigned long addr;
 318
 319         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 320         asm volatile(
 321                 "       xc      0(1,%1),0(%2)"
 322                 :  "=m" (*(char *) addr) : "a" (addr),
 323                    "a" (_oi_bitmap + (nr & 7)), "m" (*(char *) addr) : "cc" );
 324 }
 325
 326 static inline void
 327 __constant_change_bit(const unsigned long nr, volatile unsigned long *ptr)
 328 {
 329         unsigned long addr;
 330
 331         addr = ((unsigned long) ptr) + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 332         *(unsigned char *) addr ^= 1 << (nr & 7);
 333 }
 334
 335 #define change_bit_simple(nr,addr) \
 336 (__builtin_constant_p((nr)) ? \
 337  __constant_change_bit((nr),(addr)) : \
 338  __change_bit((nr),(addr)) )
 339
 340 /*
 341  * fast, non-SMP test_and_set_bit routine
 342  */
 343 static inline int
 344 test_and_set_bit_simple(unsigned long nr, volatile unsigned long *ptr)
 345 {
 346         unsigned long addr;
 347         unsigned char ch;
 348
 349         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 350         ch = *(unsigned char *) addr;
 351         asm volatile(
 352                 "       oc      0(1,%1),0(%2)"
 353                 : "=m" (*(char *) addr)
 354                 : "a" (addr), "a" (_oi_bitmap + (nr & 7)),
 355                   "m" (*(char *) addr) : "cc", "memory");
 356         return (ch >> (nr & 7)) & 1;
 357 }
 358 #define __test_and_set_bit(X,Y)         test_and_set_bit_simple(X,Y)
 359
 360 /*
 361  * fast, non-SMP test_and_clear_bit routine
 362  */
 363 static inline int
 364 test_and_clear_bit_simple(unsigned long nr, volatile unsigned long *ptr)
 365 {
 366         unsigned long addr;
 367         unsigned char ch;
 368
 369         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 370         ch = *(unsigned char *) addr;
 371         asm volatile(
 372                 "       nc      0(1,%1),0(%2)"
 373                 : "=m" (*(char *) addr)
 374                 : "a" (addr), "a" (_ni_bitmap + (nr & 7)),
 375                   "m" (*(char *) addr) : "cc", "memory");
 376         return (ch >> (nr & 7)) & 1;
 377 }
 378 #define __test_and_clear_bit(X,Y)       test_and_clear_bit_simple(X,Y)
 379
 380 /*
 381  * fast, non-SMP test_and_change_bit routine
 382  */
 383 static inline int
 384 test_and_change_bit_simple(unsigned long nr, volatile unsigned long *ptr)
 385 {
 386         unsigned long addr;
 387         unsigned char ch;
 388
 389         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 390         ch = *(unsigned char *) addr;
 391         asm volatile(
 392                 "       xc      0(1,%1),0(%2)"
 393                 : "=m" (*(char *) addr)
 394                 : "a" (addr), "a" (_oi_bitmap + (nr & 7)),
 395                   "m" (*(char *) addr) : "cc", "memory");
 396         return (ch >> (nr & 7)) & 1;
 397 }
 398 #define __test_and_change_bit(X,Y)      test_and_change_bit_simple(X,Y)
 399
 400 #ifdef CONFIG_SMP
 401 #define set_bit             set_bit_cs
 402 #define clear_bit           clear_bit_cs
 403 #define change_bit          change_bit_cs
 404 #define test_and_set_bit    test_and_set_bit_cs
 405 #define test_and_clear_bit  test_and_clear_bit_cs
 406 #define test_and_change_bit test_and_change_bit_cs
 407 #else
 408 #define set_bit             set_bit_simple
 409 #define clear_bit           clear_bit_simple
 410 #define change_bit          change_bit_simple
 411 #define test_and_set_bit    test_and_set_bit_simple
 412 #define test_and_clear_bit  test_and_clear_bit_simple
 413 #define test_and_change_bit test_and_change_bit_simple
 414 #endif
 415
 416
 417 /*
 418  * This routine doesn't need to be atomic.
 419  */
 420
 421 static inline int __test_bit(unsigned long nr, const volatile unsigned long *ptr)
 422 {
 423         unsigned long addr;
 424         unsigned char ch;
 425
 426         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 427         ch = *(volatile unsigned char *) addr;
 428         return (ch >> (nr & 7)) & 1;
 429 }
 430
 431 static inline int
 432 __constant_test_bit(unsigned long nr, const volatile unsigned long *addr) {
 433     return (((volatile char *) addr)
 434             [(nr^(__BITOPS_WORDSIZE-8))>>3] & (1<<(nr&7))) != 0;
 435 }
 436
 437 #define test_bit(nr,addr) \
 438 (__builtin_constant_p((nr)) ? \
 439  __constant_test_bit((nr),(addr)) : \
 440  __test_bit((nr),(addr)) )
 441
 442 /*
 443  * Optimized find bit helper functions.
 444  */
 445
 446 /**
 447  * __ffz_word_loop - find byte offset of first long != -1UL
 448  * @addr: pointer to array of unsigned long
 449  * @size: size of the array in bits
 450  */
 451 static inline unsigned long __ffz_word_loop(const unsigned long *addr,
 452                                             unsigned long size)
 453 {
 454         typedef struct { long _[__BITOPS_WORDS(size)]; } addrtype;
 455         unsigned long bytes = 0;
 456
 457         asm volatile(
 458 #ifndef __s390x__
 459                 "       ahi     %1,31\n"
 460                 "       srl     %1,5\n"
 461                 "0:     c       %2,0(%0,%3)\n"
 462                 "       jne     1f\n"
 463                 "       la      %0,4(%0)\n"
 464                 "       brct    %1,0b\n"
 465                 "1:\n"
 466 #else
 467                 "       aghi    %1,63\n"
 468                 "       srlg    %1,%1,6\n"
 469                 "0:     cg      %2,0(%0,%3)\n"
 470                 "       jne     1f\n"
 471                 "       la      %0,8(%0)\n"
 472                 "       brct    %1,0b\n"
 473                 "1:\n"
 474 #endif
 475                 : "+&a" (bytes), "+&d" (size)
 476                 : "d" (-1UL), "a" (addr), "m" (*(addrtype *) addr)
 477                 : "cc" );
 478         return bytes;
 479 }
 480
 481 /**
 482  * __ffs_word_loop - find byte offset of first long != 0UL
 483  * @addr: pointer to array of unsigned long
 484  * @size: size of the array in bits
 485  */
 486 static inline unsigned long __ffs_word_loop(const unsigned long *addr,
 487                                             unsigned long size)
 488 {
 489         typedef struct { long _[__BITOPS_WORDS(size)]; } addrtype;
 490         unsigned long bytes = 0;
 491
 492         asm volatile(
 493 #ifndef __s390x__
 494                 "       ahi     %1,31\n"
 495                 "       srl     %1,5\n"
 496                 "0:     c       %2,0(%0,%3)\n"
 497                 "       jne     1f\n"
 498                 "       la      %0,4(%0)\n"
 499                 "       brct    %1,0b\n"
 500                 "1:\n"
 501 #else
 502                 "       aghi    %1,63\n"
 503                 "       srlg    %1,%1,6\n"
 504                 "0:     cg      %2,0(%0,%3)\n"
 505                 "       jne     1f\n"
 506                 "       la      %0,8(%0)\n"
 507                 "       brct    %1,0b\n"
 508                 "1:\n"
 509 #endif
 510                 : "+&a" (bytes), "+&a" (size)
 511                 : "d" (0UL), "a" (addr), "m" (*(addrtype *) addr)
 512                 : "cc" );
 513         return bytes;
 514 }
 515
 516 /**
 517  * __ffz_word - add number of the first unset bit
 518  * @nr: base value the bit number is added to
 519  * @word: the word that is searched for unset bits
 520  */
 521 static inline unsigned long __ffz_word(unsigned long nr, unsigned long word)
 522 {
 523 #ifdef __s390x__
 524         if (likely((word & 0xffffffff) == 0xffffffff)) {
 525                 word >>= 32;
 526                 nr += 32;
 527         }
 528 #endif
 529         if (likely((word & 0xffff) == 0xffff)) {
 530                 word >>= 16;
 531                 nr += 16;
 532         }
 533         if (likely((word & 0xff) == 0xff)) {
 534                 word >>= 8;
 535                 nr += 8;
 536         }
 537         return nr + _zb_findmap[(unsigned char) word];
 538 }
 539
 540 /**
 541  * __ffs_word - add number of the first set bit
 542  * @nr: base value the bit number is added to
 543  * @word: the word that is searched for set bits
 544  */
 545 static inline unsigned long __ffs_word(unsigned long nr, unsigned long word)
 546 {
 547 #ifdef __s390x__
 548         if (likely((word & 0xffffffff) == 0)) {
 549                 word >>= 32;
 550                 nr += 32;
 551         }
 552 #endif
 553         if (likely((word & 0xffff) == 0)) {
 554                 word >>= 16;
 555                 nr += 16;
 556         }
 557         if (likely((word & 0xff) == 0)) {
 558                 word >>= 8;
 559                 nr += 8;
 560         }
 561         return nr + _sb_findmap[(unsigned char) word];
 562 }
 563
 564
 565 /**
 566  * __load_ulong_be - load big endian unsigned long
 567  * @p: pointer to array of unsigned long
 568  * @offset: byte offset of source value in the array
 569  */
 570 static inline unsigned long __load_ulong_be(const unsigned long *p,
 571                                             unsigned long offset)
 572 {
 573         p = (unsigned long *)((unsigned long) p + offset);
 574         return *p;
 575 }
 576
 577 /**
 578  * __load_ulong_le - load little endian unsigned long
 579  * @p: pointer to array of unsigned long
 580  * @offset: byte offset of source value in the array
 581  */
 582 static inline unsigned long __load_ulong_le(const unsigned long *p,
 583                                             unsigned long offset)
 584 {
 585         unsigned long word;
 586
 587         p = (unsigned long *)((unsigned long) p + offset);
 588 #ifndef __s390x__
 589         asm volatile(
 590                 "       ic      %0,0(%1)\n"
 591                 "       icm     %0,2,1(%1)\n"
 592                 "       icm     %0,4,2(%1)\n"
 593                 "       icm     %0,8,3(%1)"
 594                 : "=&d" (word) : "a" (p), "m" (*p) : "cc");
 595 #else
 596         asm volatile(
 597                 "       lrvg    %0,%1"
 598                 : "=d" (word) : "m" (*p) );
 599 #endif
 600         return word;
 601 }
 602
 603 /*
 604  * The various find bit functions.
 605  */
 606
 607 /*
 608  * ffz - find first zero in word.
 609  * @word: The word to search
 610  *
 611  * Undefined if no zero exists, so code should check against ~0UL first.
 612  */
 613 static inline unsigned long ffz(unsigned long word)
 614 {
 615         return __ffz_word(0, word);
 616 }
 617
 618 /**
 619  * __ffs - find first bit in word.
 620  * @word: The word to search
 621  *
 622  * Undefined if no bit exists, so code should check against 0 first.
 623  */
 624 static inline unsigned long __ffs (unsigned long word)
 625 {
 626         return __ffs_word(0, word);
 627 }
 628
 629 /**
 630  * ffs - find first bit set
 631  * @x: the word to search
 632  *
 633  * This is defined the same way as
 634  * the libc and compiler builtin ffs routines, therefore
 635  * differs in spirit from the above ffz (man ffs).
 636  */
 637 static inline int ffs(int x)
 638 {
 639         if (!x)
 640                 return 0;
 641         return __ffs_word(1, x);
 642 }
 643
 644 /**
 645  * find_first_zero_bit - find the first zero bit in a memory region
 646  * @addr: The address to start the search at
 647  * @size: The maximum size to search
 648  *
 649  * Returns the bit-number of the first zero bit, not the number of the byte
 650  * containing a bit.
 651  */
 652 static inline unsigned long find_first_zero_bit(const unsigned long *addr,
 653                                                 unsigned long size)
 654 {
 655         unsigned long bytes, bits;
 656
 657         if (!size)
 658                 return 0;
 659         bytes = __ffz_word_loop(addr, size);
 660         bits = __ffz_word(bytes*8, __load_ulong_be(addr, bytes));
 661         return (bits < size) ? bits : size;
 662 }
 663
 664 /**
 665  * find_first_bit - find the first set bit in a memory region
 666  * @addr: The address to start the search at
 667  * @size: The maximum size to search
 668  *
 669  * Returns the bit-number of the first set bit, not the number of the byte
 670  * containing a bit.
 671  */
 672 static inline unsigned long find_first_bit(const unsigned long * addr,
 673                                            unsigned long size)
 674 {
 675         unsigned long bytes, bits;
 676
 677         if (!size)
 678                 return 0;
 679         bytes = __ffs_word_loop(addr, size);
 680         bits = __ffs_word(bytes*8, __load_ulong_be(addr, bytes));
 681         return (bits < size) ? bits : size;
 682 }
 683
 684 /**
 685  * find_next_zero_bit - find the first zero bit in a memory region
 686  * @addr: The address to base the search on
 687  * @offset: The bitnumber to start searching at
 688  * @size: The maximum size to search
 689  */
 690 static inline int find_next_zero_bit (const unsigned long * addr,
 691                                       unsigned long size,
 692                                       unsigned long offset)
 693 {
 694         const unsigned long *p;
 695         unsigned long bit, set;
 696
 697         if (offset >= size)
 698                 return size;
 699         bit = offset & (__BITOPS_WORDSIZE - 1);
 700         offset -= bit;
 701         size -= offset;
 702         p = addr + offset / __BITOPS_WORDSIZE;
 703         if (bit) {
 704                 /*
 705                  * __ffz_word returns __BITOPS_WORDSIZE
 706                  * if no zero bit is present in the word.
 707                  */
 708                 set = __ffz_word(0, *p >> bit) + bit;
 709                 if (set >= size)
 710                         return size + offset;
 711                 if (set < __BITOPS_WORDSIZE)
 712                         return set + offset;
 713                 offset += __BITOPS_WORDSIZE;
 714                 size -= __BITOPS_WORDSIZE;
 715                 p++;
 716         }
 717         return offset + find_first_zero_bit(p, size);
 718 }
 719
 720 /**
 721  * find_next_bit - find the first set bit in a memory region
 722  * @addr: The address to base the search on
 723  * @offset: The bitnumber to start searching at
 724  * @size: The maximum size to search
 725  */
 726 static inline int find_next_bit (const unsigned long * addr,
 727                                  unsigned long size,
 728                                  unsigned long offset)
 729 {
 730         const unsigned long *p;
 731         unsigned long bit, set;
 732
 733         if (offset >= size)
 734                 return size;
 735         bit = offset & (__BITOPS_WORDSIZE - 1);
 736         offset -= bit;
 737         size -= offset;
 738         p = addr + offset / __BITOPS_WORDSIZE;
 739         if (bit) {
 740                 /*
 741                  * __ffs_word returns __BITOPS_WORDSIZE
 742                  * if no one bit is present in the word.
 743                  */
 744                 set = __ffs_word(0, *p & (~0UL << bit));
 745                 if (set >= size)
 746                         return size + offset;
 747                 if (set < __BITOPS_WORDSIZE)
 748                         return set + offset;
 749                 offset += __BITOPS_WORDSIZE;
 750                 size -= __BITOPS_WORDSIZE;
 751                 p++;
 752         }
 753         return offset + find_first_bit(p, size);
 754 }
 755
 756 /*
 757  * Every architecture must define this function. It's the fastest
 758  * way of searching a 140-bit bitmap where the first 100 bits are
 759  * unlikely to be set. It's guaranteed that at least one of the 140
 760  * bits is cleared.
 761  */
 762 static inline int sched_find_first_bit(unsigned long *b)
 763 {
 764         return find_first_bit(b, 140);
 765 }
 766
 767 #include <asm-generic/bitops/fls.h>
 768 #include <asm-generic/bitops/fls64.h>
 769
 770 #include <asm-generic/bitops/hweight.h>
 771 #include <asm-generic/bitops/lock.h>
 772
 773 /*
 774  * ATTENTION: intel byte ordering convention for ext2 and minix !!
 775  * bit 0 is the LSB of addr; bit 31 is the MSB of addr;
 776  * bit 32 is the LSB of (addr+4).
 777  * That combined with the little endian byte order of Intel gives the
 778  * following bit order in memory:
 779  *    07 06 05 04 03 02 01 00 15 14 13 12 11 10 09 08 \
 780  *    23 22 21 20 19 18 17 16 31 30 29 28 27 26 25 24
 781  */
 782
 783 #define ext2_set_bit(nr, addr)       \
 784         __test_and_set_bit((nr)^(__BITOPS_WORDSIZE - 8), (unsigned long *)addr)
 785 #define ext2_set_bit_atomic(lock, nr, addr)       \
 786         test_and_set_bit((nr)^(__BITOPS_WORDSIZE - 8), (unsigned long *)addr)
 787 #define ext2_clear_bit(nr, addr)     \
 788         __test_and_clear_bit((nr)^(__BITOPS_WORDSIZE - 8), (unsigned long *)addr)
 789 #define ext2_clear_bit_atomic(lock, nr, addr)     \
 790         test_and_clear_bit((nr)^(__BITOPS_WORDSIZE - 8), (unsigned long *)addr)
 791 #define ext2_test_bit(nr, addr)      \
 792         test_bit((nr)^(__BITOPS_WORDSIZE - 8), (unsigned long *)addr)
 793
 794 static inline int ext2_find_first_zero_bit(void *vaddr, unsigned int size)
 795 {
 796         unsigned long bytes, bits;
 797
 798         if (!size)
 799                 return 0;
 800         bytes = __ffz_word_loop(vaddr, size);
 801         bits = __ffz_word(bytes*8, __load_ulong_le(vaddr, bytes));
 802         return (bits < size) ? bits : size;
 803 }
 804
 805 static inline int ext2_find_next_zero_bit(void *vaddr, unsigned long size,
 806                                           unsigned long offset)
 807 {
 808         unsigned long *addr = vaddr, *p;
 809         unsigned long bit, set;
 810
 811         if (offset >= size)
 812                 return size;
 813         bit = offset & (__BITOPS_WORDSIZE - 1);
 814         offset -= bit;
 815         size -= offset;
 816         p = addr + offset / __BITOPS_WORDSIZE;
 817         if (bit) {
 818                 /*
 819                  * s390 version of ffz returns __BITOPS_WORDSIZE
 820                  * if no zero bit is present in the word.
 821                  */
 822                 set = ffz(__load_ulong_le(p, 0) >> bit) + bit;
 823                 if (set >= size)
 824                         return size + offset;
 825                 if (set < __BITOPS_WORDSIZE)
 826                         return set + offset;
 827                 offset += __BITOPS_WORDSIZE;
 828                 size -= __BITOPS_WORDSIZE;
 829                 p++;
 830         }
 831         return offset + ext2_find_first_zero_bit(p, size);
 832 }
 833
 834 static inline unsigned long ext2_find_first_bit(void *vaddr,
 835                                                 unsigned long size)
 836 {
 837         unsigned long bytes, bits;
 838
 839         if (!size)
 840                 return 0;
 841         bytes = __ffs_word_loop(vaddr, size);
 842         bits = __ffs_word(bytes*8, __load_ulong_le(vaddr, bytes));
 843         return (bits < size) ? bits : size;
 844 }
 845
 846 static inline int ext2_find_next_bit(void *vaddr, unsigned long size,
 847                                      unsigned long offset)
 848 {
 849         unsigned long *addr = vaddr, *p;
 850         unsigned long bit, set;
 851
 852         if (offset >= size)
 853                 return size;
 854         bit = offset & (__BITOPS_WORDSIZE - 1);
 855         offset -= bit;
 856         size -= offset;
 857         p = addr + offset / __BITOPS_WORDSIZE;
 858         if (bit) {
 859                 /*
 860                  * s390 version of ffz returns __BITOPS_WORDSIZE
 861                  * if no zero bit is present in the word.
 862                  */
 863                 set = ffs(__load_ulong_le(p, 0) >> bit) + bit;
 864                 if (set >= size)
 865                         return size + offset;
 866                 if (set < __BITOPS_WORDSIZE)
 867                         return set + offset;
 868                 offset += __BITOPS_WORDSIZE;
 869                 size -= __BITOPS_WORDSIZE;
 870                 p++;
 871         }
 872         return offset + ext2_find_first_bit(p, size);
 873 }
 874
 875 #include <asm-generic/bitops/minix.h>
 876
 877 #endif /* __KERNEL__ */
 878
 879 #endif /* _S390_BITOPS_H */