?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
blk: rq_data_dir() should not return a boolean
[cris-mirror.git] / arch / mips / include / asm / bitops.h
blobce9666cf14995ce1b713cccce029244ca5ba24b5
1 /*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (c) 1999, 2000 Silicon Graphics, Inc.
8 */
9 #ifndef _ASM_BITOPS_H
10 #define _ASM_BITOPS_H
11
12 #ifndef _LINUX_BITOPS_H
13 #error only <linux/bitops.h> can be included directly
14 #endif
15
16 #include <linux/compiler.h>
17 #include <linux/types.h>
18 #include <asm/barrier.h>
19 #include <asm/byteorder.h> /* sigh ... */
20 #include <asm/compiler.h>
21 #include <asm/cpu-features.h>
22 #include <asm/sgidefs.h>
23 #include <asm/war.h>
24
25 #if _MIPS_SZLONG == 32
26 #define SZLONG_LOG 5
27 #define SZLONG_MASK 31UL
28 #define __LL "ll "
29 #define __SC "sc "
30 #define __INS "ins "
31 #define __EXT "ext "
32 #elif _MIPS_SZLONG == 64
33 #define SZLONG_LOG 6
34 #define SZLONG_MASK 63UL
35 #define __LL "lld "
36 #define __SC "scd "
37 #define __INS "dins "
38 #define __EXT "dext "
39 #endif
40
41 /*
42 * These are the "slower" versions of the functions and are in bitops.c.
43 * These functions call raw_local_irq_{save,restore}().
44 */
45 void __mips_set_bit(unsigned long nr, volatile unsigned long *addr);
46 void __mips_clear_bit(unsigned long nr, volatile unsigned long *addr);
47 void __mips_change_bit(unsigned long nr, volatile unsigned long *addr);
48 int __mips_test_and_set_bit(unsigned long nr,
49 volatile unsigned long *addr);
50 int __mips_test_and_set_bit_lock(unsigned long nr,
51 volatile unsigned long *addr);
52 int __mips_test_and_clear_bit(unsigned long nr,
53 volatile unsigned long *addr);
54 int __mips_test_and_change_bit(unsigned long nr,
55 volatile unsigned long *addr);
56
57
58 /*
59 * set_bit - Atomically set a bit in memory
60 * @nr: the bit to set
61 * @addr: the address to start counting from
62 *
63 * This function is atomic and may not be reordered. See __set_bit()
64 * if you do not require the atomic guarantees.
65 * Note that @nr may be almost arbitrarily large; this function is not
66 * restricted to acting on a single-word quantity.
67 */
68 static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
69 {
70 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
71 int bit = nr & SZLONG_MASK;
72 unsigned long temp;
73
74 if (kernel_uses_llsc && R10000_LLSC_WAR) {
75 __asm__ __volatile__(
76 " .set arch=r4000 \n"
77 "1: " __LL "%0, %1 # set_bit \n"
78 " or %0, %2 \n"
79 " " __SC "%0, %1 \n"
80 " beqzl %0, 1b \n"
81 " .set mips0 \n"
82 : "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*m)
83 : "ir" (1UL << bit), GCC_OFF_SMALL_ASM() (*m));
84 #if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
85 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
86 do {
87 __asm__ __volatile__(
88 " " __LL "%0, %1 # set_bit \n"
89 " " __INS "%0, %3, %2, 1 \n"
90 " " __SC "%0, %1 \n"
91 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
92 : "ir" (bit), "r" (~0));
93 } while (unlikely(!temp));
94 #endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
95 } else if (kernel_uses_llsc) {
96 do {
97 __asm__ __volatile__(
98 " .set "MIPS_ISA_ARCH_LEVEL" \n"
99 " " __LL "%0, %1 # set_bit \n"
100 " or %0, %2 \n"
101 " " __SC "%0, %1 \n"
102 " .set mips0 \n"
103 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
104 : "ir" (1UL << bit));
105 } while (unlikely(!temp));
106 } else
107 __mips_set_bit(nr, addr);
108 }
109
110 /*
111 * clear_bit - Clears a bit in memory
112 * @nr: Bit to clear
113 * @addr: Address to start counting from
114 *
115 * clear_bit() is atomic and may not be reordered. However, it does
116 * not contain a memory barrier, so if it is used for locking purposes,
117 * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic()
118 * in order to ensure changes are visible on other processors.
119 */
120 static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
121 {
122 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
123 int bit = nr & SZLONG_MASK;
124 unsigned long temp;
125
126 if (kernel_uses_llsc && R10000_LLSC_WAR) {
127 __asm__ __volatile__(
128 " .set arch=r4000 \n"
129 "1: " __LL "%0, %1 # clear_bit \n"
130 " and %0, %2 \n"
131 " " __SC "%0, %1 \n"
132 " beqzl %0, 1b \n"
133 " .set mips0 \n"
134 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
135 : "ir" (~(1UL << bit)));
136 #if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
137 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
138 do {
139 __asm__ __volatile__(
140 " " __LL "%0, %1 # clear_bit \n"
141 " " __INS "%0, $0, %2, 1 \n"
142 " " __SC "%0, %1 \n"
143 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
144 : "ir" (bit));
145 } while (unlikely(!temp));
146 #endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
147 } else if (kernel_uses_llsc) {
148 do {
149 __asm__ __volatile__(
150 " .set "MIPS_ISA_ARCH_LEVEL" \n"
151 " " __LL "%0, %1 # clear_bit \n"
152 " and %0, %2 \n"
153 " " __SC "%0, %1 \n"
154 " .set mips0 \n"
155 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
156 : "ir" (~(1UL << bit)));
157 } while (unlikely(!temp));
158 } else
159 __mips_clear_bit(nr, addr);
160 }
161
162 /*
163 * clear_bit_unlock - Clears a bit in memory
164 * @nr: Bit to clear
165 * @addr: Address to start counting from
166 *
167 * clear_bit() is atomic and implies release semantics before the memory
168 * operation. It can be used for an unlock.
169 */
170 static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
171 {
172 smp_mb__before_atomic();
173 clear_bit(nr, addr);
174 }
175
176 /*
177 * change_bit - Toggle a bit in memory
178 * @nr: Bit to change
179 * @addr: Address to start counting from
180 *
181 * change_bit() is atomic and may not be reordered.
182 * Note that @nr may be almost arbitrarily large; this function is not
183 * restricted to acting on a single-word quantity.
184 */
185 static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
186 {
187 int bit = nr & SZLONG_MASK;
188
189 if (kernel_uses_llsc && R10000_LLSC_WAR) {
190 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
191 unsigned long temp;
192
193 __asm__ __volatile__(
194 " .set arch=r4000 \n"
195 "1: " __LL "%0, %1 # change_bit \n"
196 " xor %0, %2 \n"
197 " " __SC "%0, %1 \n"
198 " beqzl %0, 1b \n"
199 " .set mips0 \n"
200 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
201 : "ir" (1UL << bit));
202 } else if (kernel_uses_llsc) {
203 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
204 unsigned long temp;
205
206 do {
207 __asm__ __volatile__(
208 " .set "MIPS_ISA_ARCH_LEVEL" \n"
209 " " __LL "%0, %1 # change_bit \n"
210 " xor %0, %2 \n"
211 " " __SC "%0, %1 \n"
212 " .set mips0 \n"
213 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
214 : "ir" (1UL << bit));
215 } while (unlikely(!temp));
216 } else
217 __mips_change_bit(nr, addr);
218 }
219
220 /*
221 * test_and_set_bit - Set a bit and return its old value
222 * @nr: Bit to set
223 * @addr: Address to count from
224 *
225 * This operation is atomic and cannot be reordered.
226 * It also implies a memory barrier.
227 */
228 static inline int test_and_set_bit(unsigned long nr,
229 volatile unsigned long *addr)
230 {
231 int bit = nr & SZLONG_MASK;
232 unsigned long res;
233
234 smp_mb__before_llsc();
235
236 if (kernel_uses_llsc && R10000_LLSC_WAR) {
237 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
238 unsigned long temp;
239
240 __asm__ __volatile__(
241 " .set arch=r4000 \n"
242 "1: " __LL "%0, %1 # test_and_set_bit \n"
243 " or %2, %0, %3 \n"
244 " " __SC "%2, %1 \n"
245 " beqzl %2, 1b \n"
246 " and %2, %0, %3 \n"
247 " .set mips0 \n"
248 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
249 : "r" (1UL << bit)
250 : "memory");
251 } else if (kernel_uses_llsc) {
252 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
253 unsigned long temp;
254
255 do {
256 __asm__ __volatile__(
257 " .set "MIPS_ISA_ARCH_LEVEL" \n"
258 " " __LL "%0, %1 # test_and_set_bit \n"
259 " or %2, %0, %3 \n"
260 " " __SC "%2, %1 \n"
261 " .set mips0 \n"
262 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
263 : "r" (1UL << bit)
264 : "memory");
265 } while (unlikely(!res));
266
267 res = temp & (1UL << bit);
268 } else
269 res = __mips_test_and_set_bit(nr, addr);
270
271 smp_llsc_mb();
272
273 return res != 0;
274 }
275
276 /*
277 * test_and_set_bit_lock - Set a bit and return its old value
278 * @nr: Bit to set
279 * @addr: Address to count from
280 *
281 * This operation is atomic and implies acquire ordering semantics
282 * after the memory operation.
283 */
284 static inline int test_and_set_bit_lock(unsigned long nr,
285 volatile unsigned long *addr)
286 {
287 int bit = nr & SZLONG_MASK;
288 unsigned long res;
289
290 if (kernel_uses_llsc && R10000_LLSC_WAR) {
291 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
292 unsigned long temp;
293
294 __asm__ __volatile__(
295 " .set arch=r4000 \n"
296 "1: " __LL "%0, %1 # test_and_set_bit \n"
297 " or %2, %0, %3 \n"
298 " " __SC "%2, %1 \n"
299 " beqzl %2, 1b \n"
300 " and %2, %0, %3 \n"
301 " .set mips0 \n"
302 : "=&r" (temp), "+m" (*m), "=&r" (res)
303 : "r" (1UL << bit)
304 : "memory");
305 } else if (kernel_uses_llsc) {
306 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
307 unsigned long temp;
308
309 do {
310 __asm__ __volatile__(
311 " .set "MIPS_ISA_ARCH_LEVEL" \n"
312 " " __LL "%0, %1 # test_and_set_bit \n"
313 " or %2, %0, %3 \n"
314 " " __SC "%2, %1 \n"
315 " .set mips0 \n"
316 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
317 : "r" (1UL << bit)
318 : "memory");
319 } while (unlikely(!res));
320
321 res = temp & (1UL << bit);
322 } else
323 res = __mips_test_and_set_bit_lock(nr, addr);
324
325 smp_llsc_mb();
326
327 return res != 0;
328 }
329 /*
330 * test_and_clear_bit - Clear a bit and return its old value
331 * @nr: Bit to clear
332 * @addr: Address to count from
333 *
334 * This operation is atomic and cannot be reordered.
335 * It also implies a memory barrier.
336 */
337 static inline int test_and_clear_bit(unsigned long nr,
338 volatile unsigned long *addr)
339 {
340 int bit = nr & SZLONG_MASK;
341 unsigned long res;
342
343 smp_mb__before_llsc();
344
345 if (kernel_uses_llsc && R10000_LLSC_WAR) {
346 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
347 unsigned long temp;
348
349 __asm__ __volatile__(
350 " .set arch=r4000 \n"
351 "1: " __LL "%0, %1 # test_and_clear_bit \n"
352 " or %2, %0, %3 \n"
353 " xor %2, %3 \n"
354 " " __SC "%2, %1 \n"
355 " beqzl %2, 1b \n"
356 " and %2, %0, %3 \n"
357 " .set mips0 \n"
358 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
359 : "r" (1UL << bit)
360 : "memory");
361 #if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
362 } else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
363 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
364 unsigned long temp;
365
366 do {
367 __asm__ __volatile__(
368 " " __LL "%0, %1 # test_and_clear_bit \n"
369 " " __EXT "%2, %0, %3, 1 \n"
370 " " __INS "%0, $0, %3, 1 \n"
371 " " __SC "%0, %1 \n"
372 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
373 : "ir" (bit)
374 : "memory");
375 } while (unlikely(!temp));
376 #endif
377 } else if (kernel_uses_llsc) {
378 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
379 unsigned long temp;
380
381 do {
382 __asm__ __volatile__(
383 " .set "MIPS_ISA_ARCH_LEVEL" \n"
384 " " __LL "%0, %1 # test_and_clear_bit \n"
385 " or %2, %0, %3 \n"
386 " xor %2, %3 \n"
387 " " __SC "%2, %1 \n"
388 " .set mips0 \n"
389 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
390 : "r" (1UL << bit)
391 : "memory");
392 } while (unlikely(!res));
393
394 res = temp & (1UL << bit);
395 } else
396 res = __mips_test_and_clear_bit(nr, addr);
397
398 smp_llsc_mb();
399
400 return res != 0;
401 }
402
403 /*
404 * test_and_change_bit - Change a bit and return its old value
405 * @nr: Bit to change
406 * @addr: Address to count from
407 *
408 * This operation is atomic and cannot be reordered.
409 * It also implies a memory barrier.
410 */
411 static inline int test_and_change_bit(unsigned long nr,
412 volatile unsigned long *addr)
413 {
414 int bit = nr & SZLONG_MASK;
415 unsigned long res;
416
417 smp_mb__before_llsc();
418
419 if (kernel_uses_llsc && R10000_LLSC_WAR) {
420 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
421 unsigned long temp;
422
423 __asm__ __volatile__(
424 " .set arch=r4000 \n"
425 "1: " __LL "%0, %1 # test_and_change_bit \n"
426 " xor %2, %0, %3 \n"
427 " " __SC "%2, %1 \n"
428 " beqzl %2, 1b \n"
429 " and %2, %0, %3 \n"
430 " .set mips0 \n"
431 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
432 : "r" (1UL << bit)
433 : "memory");
434 } else if (kernel_uses_llsc) {
435 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
436 unsigned long temp;
437
438 do {
439 __asm__ __volatile__(
440 " .set "MIPS_ISA_ARCH_LEVEL" \n"
441 " " __LL "%0, %1 # test_and_change_bit \n"
442 " xor %2, %0, %3 \n"
443 " " __SC "\t%2, %1 \n"
444 " .set mips0 \n"
445 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
446 : "r" (1UL << bit)
447 : "memory");
448 } while (unlikely(!res));
449
450 res = temp & (1UL << bit);
451 } else
452 res = __mips_test_and_change_bit(nr, addr);
453
454 smp_llsc_mb();
455
456 return res != 0;
457 }
458
459 #include <asm-generic/bitops/non-atomic.h>
460
461 /*
462 * __clear_bit_unlock - Clears a bit in memory
463 * @nr: Bit to clear
464 * @addr: Address to start counting from
465 *
466 * __clear_bit() is non-atomic and implies release semantics before the memory
467 * operation. It can be used for an unlock if no other CPUs can concurrently
468 * modify other bits in the word.
469 */
470 static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
471 {
472 smp_mb__before_llsc();
473 __clear_bit(nr, addr);
474 }
475
476 /*
477 * Return the bit position (0..63) of the most significant 1 bit in a word
478 * Returns -1 if no 1 bit exists
479 */
480 static inline unsigned long __fls(unsigned long word)
481 {
482 int num;
483
484 if (BITS_PER_LONG == 32 && !__builtin_constant_p(word) &&
485 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
486 __asm__(
487 " .set push \n"
488 " .set "MIPS_ISA_LEVEL" \n"
489 " clz %0, %1 \n"
490 " .set pop \n"
491 : "=r" (num)
492 : "r" (word));
493
494 return 31 - num;
495 }
496
497 if (BITS_PER_LONG == 64 && !__builtin_constant_p(word) &&
498 __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
499 __asm__(
500 " .set push \n"
501 " .set "MIPS_ISA_LEVEL" \n"
502 " dclz %0, %1 \n"
503 " .set pop \n"
504 : "=r" (num)
505 : "r" (word));
506
507 return 63 - num;
508 }
509
510 num = BITS_PER_LONG - 1;
511
512 #if BITS_PER_LONG == 64
513 if (!(word & (~0ul << 32))) {
514 num -= 32;
515 word <<= 32;
516 }
517 #endif
518 if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
519 num -= 16;
520 word <<= 16;
521 }
522 if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
523 num -= 8;
524 word <<= 8;
525 }
526 if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
527 num -= 4;
528 word <<= 4;
529 }
530 if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
531 num -= 2;
532 word <<= 2;
533 }
534 if (!(word & (~0ul << (BITS_PER_LONG-1))))
535 num -= 1;
536 return num;
537 }
538
539 /*
540 * __ffs - find first bit in word.
541 * @word: The word to search
542 *
543 * Returns 0..SZLONG-1
544 * Undefined if no bit exists, so code should check against 0 first.
545 */
546 static inline unsigned long __ffs(unsigned long word)
547 {
548 return __fls(word & -word);
549 }
550
551 /*
552 * fls - find last bit set.
553 * @word: The word to search
554 *
555 * This is defined the same way as ffs.
556 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
557 */
558 static inline int fls(int x)
559 {
560 int r;
561
562 if (!__builtin_constant_p(x) &&
563 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
564 __asm__(
565 " .set push \n"
566 " .set "MIPS_ISA_LEVEL" \n"
567 " clz %0, %1 \n"
568 " .set pop \n"
569 : "=r" (x)
570 : "r" (x));
571
572 return 32 - x;
573 }
574
575 r = 32;
576 if (!x)
577 return 0;
578 if (!(x & 0xffff0000u)) {
579 x <<= 16;
580 r -= 16;
581 }
582 if (!(x & 0xff000000u)) {
583 x <<= 8;
584 r -= 8;
585 }
586 if (!(x & 0xf0000000u)) {
587 x <<= 4;
588 r -= 4;
589 }
590 if (!(x & 0xc0000000u)) {
591 x <<= 2;
592 r -= 2;
593 }
594 if (!(x & 0x80000000u)) {
595 x <<= 1;
596 r -= 1;
597 }
598 return r;
599 }
600
601 #include <asm-generic/bitops/fls64.h>
602
603 /*
604 * ffs - find first bit set.
605 * @word: The word to search
606 *
607 * This is defined the same way as
608 * the libc and compiler builtin ffs routines, therefore
609 * differs in spirit from the above ffz (man ffs).
610 */
611 static inline int ffs(int word)
612 {
613 if (!word)
614 return 0;
615
616 return fls(word & -word);
617 }
618
619 #include <asm-generic/bitops/ffz.h>
620 #include <asm-generic/bitops/find.h>
621
622 #ifdef __KERNEL__
623
624 #include <asm-generic/bitops/sched.h>
625
626 #include <asm/arch_hweight.h>
627 #include <asm-generic/bitops/const_hweight.h>
628
629 #include <asm-generic/bitops/le.h>
630 #include <asm-generic/bitops/ext2-atomic.h>
631
632 #endif /* __KERNEL__ */
633
634 #endif /* _ASM_BITOPS_H */
CRIS linux kernel tree