?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
First Support on Ginger and OMAP TI
[linux-ginger.git] / arch / mips / include / asm / bitops.h
blob84a383806b2cd823d9acbb362df004ae27c15854
1 /*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (c) 1999, 2000 Silicon Graphics, Inc.
8 */
9 #ifndef _ASM_BITOPS_H
10 #define _ASM_BITOPS_H
11
12 #ifndef _LINUX_BITOPS_H
13 #error only <linux/bitops.h> can be included directly
14 #endif
15
16 #include <linux/compiler.h>
17 #include <linux/irqflags.h>
18 #include <linux/types.h>
19 #include <asm/barrier.h>
20 #include <asm/bug.h>
21 #include <asm/byteorder.h> /* sigh ... */
22 #include <asm/cpu-features.h>
23 #include <asm/sgidefs.h>
24 #include <asm/war.h>
25
26 #if _MIPS_SZLONG == 32
27 #define SZLONG_LOG 5
28 #define SZLONG_MASK 31UL
29 #define __LL "ll "
30 #define __SC "sc "
31 #define __INS "ins "
32 #define __EXT "ext "
33 #elif _MIPS_SZLONG == 64
34 #define SZLONG_LOG 6
35 #define SZLONG_MASK 63UL
36 #define __LL "lld "
37 #define __SC "scd "
38 #define __INS "dins "
39 #define __EXT "dext "
40 #endif
41
42 /*
43 * clear_bit() doesn't provide any barrier for the compiler.
44 */
45 #define smp_mb__before_clear_bit() smp_llsc_mb()
46 #define smp_mb__after_clear_bit() smp_llsc_mb()
47
48 /*
49 * set_bit - Atomically set a bit in memory
50 * @nr: the bit to set
51 * @addr: the address to start counting from
52 *
53 * This function is atomic and may not be reordered. See __set_bit()
54 * if you do not require the atomic guarantees.
55 * Note that @nr may be almost arbitrarily large; this function is not
56 * restricted to acting on a single-word quantity.
57 */
58 static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
59 {
60 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
61 unsigned short bit = nr & SZLONG_MASK;
62 unsigned long temp;
63
64 if (kernel_uses_llsc && R10000_LLSC_WAR) {
65 __asm__ __volatile__(
66 " .set mips3 \n"
67 "1: " __LL "%0, %1 # set_bit \n"
68 " or %0, %2 \n"
69 " " __SC "%0, %1 \n"
70 " beqzl %0, 1b \n"
71 " .set mips0 \n"
72 : "=&r" (temp), "=m" (*m)
73 : "ir" (1UL << bit), "m" (*m));
74 #ifdef CONFIG_CPU_MIPSR2
75 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
76 __asm__ __volatile__(
77 "1: " __LL "%0, %1 # set_bit \n"
78 " " __INS "%0, %4, %2, 1 \n"
79 " " __SC "%0, %1 \n"
80 " beqz %0, 2f \n"
81 " .subsection 2 \n"
82 "2: b 1b \n"
83 " .previous \n"
84 : "=&r" (temp), "=m" (*m)
85 : "ir" (bit), "m" (*m), "r" (~0));
86 #endif /* CONFIG_CPU_MIPSR2 */
87 } else if (kernel_uses_llsc) {
88 __asm__ __volatile__(
89 " .set mips3 \n"
90 "1: " __LL "%0, %1 # set_bit \n"
91 " or %0, %2 \n"
92 " " __SC "%0, %1 \n"
93 " beqz %0, 2f \n"
94 " .subsection 2 \n"
95 "2: b 1b \n"
96 " .previous \n"
97 " .set mips0 \n"
98 : "=&r" (temp), "=m" (*m)
99 : "ir" (1UL << bit), "m" (*m));
100 } else {
101 volatile unsigned long *a = addr;
102 unsigned long mask;
103 unsigned long flags;
104
105 a += nr >> SZLONG_LOG;
106 mask = 1UL << bit;
107 raw_local_irq_save(flags);
108 *a |= mask;
109 raw_local_irq_restore(flags);
110 }
111 }
112
113 /*
114 * clear_bit - Clears a bit in memory
115 * @nr: Bit to clear
116 * @addr: Address to start counting from
117 *
118 * clear_bit() is atomic and may not be reordered. However, it does
119 * not contain a memory barrier, so if it is used for locking purposes,
120 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
121 * in order to ensure changes are visible on other processors.
122 */
123 static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
124 {
125 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
126 unsigned short bit = nr & SZLONG_MASK;
127 unsigned long temp;
128
129 if (kernel_uses_llsc && R10000_LLSC_WAR) {
130 __asm__ __volatile__(
131 " .set mips3 \n"
132 "1: " __LL "%0, %1 # clear_bit \n"
133 " and %0, %2 \n"
134 " " __SC "%0, %1 \n"
135 " beqzl %0, 1b \n"
136 " .set mips0 \n"
137 : "=&r" (temp), "=m" (*m)
138 : "ir" (~(1UL << bit)), "m" (*m));
139 #ifdef CONFIG_CPU_MIPSR2
140 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
141 __asm__ __volatile__(
142 "1: " __LL "%0, %1 # clear_bit \n"
143 " " __INS "%0, $0, %2, 1 \n"
144 " " __SC "%0, %1 \n"
145 " beqz %0, 2f \n"
146 " .subsection 2 \n"
147 "2: b 1b \n"
148 " .previous \n"
149 : "=&r" (temp), "=m" (*m)
150 : "ir" (bit), "m" (*m));
151 #endif /* CONFIG_CPU_MIPSR2 */
152 } else if (kernel_uses_llsc) {
153 __asm__ __volatile__(
154 " .set mips3 \n"
155 "1: " __LL "%0, %1 # clear_bit \n"
156 " and %0, %2 \n"
157 " " __SC "%0, %1 \n"
158 " beqz %0, 2f \n"
159 " .subsection 2 \n"
160 "2: b 1b \n"
161 " .previous \n"
162 " .set mips0 \n"
163 : "=&r" (temp), "=m" (*m)
164 : "ir" (~(1UL << bit)), "m" (*m));
165 } else {
166 volatile unsigned long *a = addr;
167 unsigned long mask;
168 unsigned long flags;
169
170 a += nr >> SZLONG_LOG;
171 mask = 1UL << bit;
172 raw_local_irq_save(flags);
173 *a &= ~mask;
174 raw_local_irq_restore(flags);
175 }
176 }
177
178 /*
179 * clear_bit_unlock - Clears a bit in memory
180 * @nr: Bit to clear
181 * @addr: Address to start counting from
182 *
183 * clear_bit() is atomic and implies release semantics before the memory
184 * operation. It can be used for an unlock.
185 */
186 static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
187 {
188 smp_mb__before_clear_bit();
189 clear_bit(nr, addr);
190 }
191
192 /*
193 * change_bit - Toggle a bit in memory
194 * @nr: Bit to change
195 * @addr: Address to start counting from
196 *
197 * change_bit() is atomic and may not be reordered.
198 * Note that @nr may be almost arbitrarily large; this function is not
199 * restricted to acting on a single-word quantity.
200 */
201 static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
202 {
203 unsigned short bit = nr & SZLONG_MASK;
204
205 if (kernel_uses_llsc && R10000_LLSC_WAR) {
206 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
207 unsigned long temp;
208
209 __asm__ __volatile__(
210 " .set mips3 \n"
211 "1: " __LL "%0, %1 # change_bit \n"
212 " xor %0, %2 \n"
213 " " __SC "%0, %1 \n"
214 " beqzl %0, 1b \n"
215 " .set mips0 \n"
216 : "=&r" (temp), "=m" (*m)
217 : "ir" (1UL << bit), "m" (*m));
218 } else if (kernel_uses_llsc) {
219 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
220 unsigned long temp;
221
222 __asm__ __volatile__(
223 " .set mips3 \n"
224 "1: " __LL "%0, %1 # change_bit \n"
225 " xor %0, %2 \n"
226 " " __SC "%0, %1 \n"
227 " beqz %0, 2f \n"
228 " .subsection 2 \n"
229 "2: b 1b \n"
230 " .previous \n"
231 " .set mips0 \n"
232 : "=&r" (temp), "=m" (*m)
233 : "ir" (1UL << bit), "m" (*m));
234 } else {
235 volatile unsigned long *a = addr;
236 unsigned long mask;
237 unsigned long flags;
238
239 a += nr >> SZLONG_LOG;
240 mask = 1UL << bit;
241 raw_local_irq_save(flags);
242 *a ^= mask;
243 raw_local_irq_restore(flags);
244 }
245 }
246
247 /*
248 * test_and_set_bit - Set a bit and return its old value
249 * @nr: Bit to set
250 * @addr: Address to count from
251 *
252 * This operation is atomic and cannot be reordered.
253 * It also implies a memory barrier.
254 */
255 static inline int test_and_set_bit(unsigned long nr,
256 volatile unsigned long *addr)
257 {
258 unsigned short bit = nr & SZLONG_MASK;
259 unsigned long res;
260
261 smp_llsc_mb();
262
263 if (kernel_uses_llsc && R10000_LLSC_WAR) {
264 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
265 unsigned long temp;
266
267 __asm__ __volatile__(
268 " .set mips3 \n"
269 "1: " __LL "%0, %1 # test_and_set_bit \n"
270 " or %2, %0, %3 \n"
271 " " __SC "%2, %1 \n"
272 " beqzl %2, 1b \n"
273 " and %2, %0, %3 \n"
274 " .set mips0 \n"
275 : "=&r" (temp), "=m" (*m), "=&r" (res)
276 : "r" (1UL << bit), "m" (*m)
277 : "memory");
278 } else if (kernel_uses_llsc) {
279 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
280 unsigned long temp;
281
282 __asm__ __volatile__(
283 " .set push \n"
284 " .set noreorder \n"
285 " .set mips3 \n"
286 "1: " __LL "%0, %1 # test_and_set_bit \n"
287 " or %2, %0, %3 \n"
288 " " __SC "%2, %1 \n"
289 " beqz %2, 2f \n"
290 " and %2, %0, %3 \n"
291 " .subsection 2 \n"
292 "2: b 1b \n"
293 " nop \n"
294 " .previous \n"
295 " .set pop \n"
296 : "=&r" (temp), "=m" (*m), "=&r" (res)
297 : "r" (1UL << bit), "m" (*m)
298 : "memory");
299 } else {
300 volatile unsigned long *a = addr;
301 unsigned long mask;
302 unsigned long flags;
303
304 a += nr >> SZLONG_LOG;
305 mask = 1UL << bit;
306 raw_local_irq_save(flags);
307 res = (mask & *a);
308 *a |= mask;
309 raw_local_irq_restore(flags);
310 }
311
312 smp_llsc_mb();
313
314 return res != 0;
315 }
316
317 /*
318 * test_and_set_bit_lock - Set a bit and return its old value
319 * @nr: Bit to set
320 * @addr: Address to count from
321 *
322 * This operation is atomic and implies acquire ordering semantics
323 * after the memory operation.
324 */
325 static inline int test_and_set_bit_lock(unsigned long nr,
326 volatile unsigned long *addr)
327 {
328 unsigned short bit = nr & SZLONG_MASK;
329 unsigned long res;
330
331 if (kernel_uses_llsc && R10000_LLSC_WAR) {
332 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
333 unsigned long temp;
334
335 __asm__ __volatile__(
336 " .set mips3 \n"
337 "1: " __LL "%0, %1 # test_and_set_bit \n"
338 " or %2, %0, %3 \n"
339 " " __SC "%2, %1 \n"
340 " beqzl %2, 1b \n"
341 " and %2, %0, %3 \n"
342 " .set mips0 \n"
343 : "=&r" (temp), "=m" (*m), "=&r" (res)
344 : "r" (1UL << bit), "m" (*m)
345 : "memory");
346 } else if (kernel_uses_llsc) {
347 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
348 unsigned long temp;
349
350 __asm__ __volatile__(
351 " .set push \n"
352 " .set noreorder \n"
353 " .set mips3 \n"
354 "1: " __LL "%0, %1 # test_and_set_bit \n"
355 " or %2, %0, %3 \n"
356 " " __SC "%2, %1 \n"
357 " beqz %2, 2f \n"
358 " and %2, %0, %3 \n"
359 " .subsection 2 \n"
360 "2: b 1b \n"
361 " nop \n"
362 " .previous \n"
363 " .set pop \n"
364 : "=&r" (temp), "=m" (*m), "=&r" (res)
365 : "r" (1UL << bit), "m" (*m)
366 : "memory");
367 } else {
368 volatile unsigned long *a = addr;
369 unsigned long mask;
370 unsigned long flags;
371
372 a += nr >> SZLONG_LOG;
373 mask = 1UL << bit;
374 raw_local_irq_save(flags);
375 res = (mask & *a);
376 *a |= mask;
377 raw_local_irq_restore(flags);
378 }
379
380 smp_llsc_mb();
381
382 return res != 0;
383 }
384 /*
385 * test_and_clear_bit - Clear a bit and return its old value
386 * @nr: Bit to clear
387 * @addr: Address to count from
388 *
389 * This operation is atomic and cannot be reordered.
390 * It also implies a memory barrier.
391 */
392 static inline int test_and_clear_bit(unsigned long nr,
393 volatile unsigned long *addr)
394 {
395 unsigned short bit = nr & SZLONG_MASK;
396 unsigned long res;
397
398 smp_llsc_mb();
399
400 if (kernel_uses_llsc && R10000_LLSC_WAR) {
401 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
402 unsigned long temp;
403
404 __asm__ __volatile__(
405 " .set mips3 \n"
406 "1: " __LL "%0, %1 # test_and_clear_bit \n"
407 " or %2, %0, %3 \n"
408 " xor %2, %3 \n"
409 " " __SC "%2, %1 \n"
410 " beqzl %2, 1b \n"
411 " and %2, %0, %3 \n"
412 " .set mips0 \n"
413 : "=&r" (temp), "=m" (*m), "=&r" (res)
414 : "r" (1UL << bit), "m" (*m)
415 : "memory");
416 #ifdef CONFIG_CPU_MIPSR2
417 } else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
418 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
419 unsigned long temp;
420
421 __asm__ __volatile__(
422 "1: " __LL "%0, %1 # test_and_clear_bit \n"
423 " " __EXT "%2, %0, %3, 1 \n"
424 " " __INS "%0, $0, %3, 1 \n"
425 " " __SC "%0, %1 \n"
426 " beqz %0, 2f \n"
427 " .subsection 2 \n"
428 "2: b 1b \n"
429 " .previous \n"
430 : "=&r" (temp), "=m" (*m), "=&r" (res)
431 : "ir" (bit), "m" (*m)
432 : "memory");
433 #endif
434 } else if (kernel_uses_llsc) {
435 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
436 unsigned long temp;
437
438 __asm__ __volatile__(
439 " .set push \n"
440 " .set noreorder \n"
441 " .set mips3 \n"
442 "1: " __LL "%0, %1 # test_and_clear_bit \n"
443 " or %2, %0, %3 \n"
444 " xor %2, %3 \n"
445 " " __SC "%2, %1 \n"
446 " beqz %2, 2f \n"
447 " and %2, %0, %3 \n"
448 " .subsection 2 \n"
449 "2: b 1b \n"
450 " nop \n"
451 " .previous \n"
452 " .set pop \n"
453 : "=&r" (temp), "=m" (*m), "=&r" (res)
454 : "r" (1UL << bit), "m" (*m)
455 : "memory");
456 } else {
457 volatile unsigned long *a = addr;
458 unsigned long mask;
459 unsigned long flags;
460
461 a += nr >> SZLONG_LOG;
462 mask = 1UL << bit;
463 raw_local_irq_save(flags);
464 res = (mask & *a);
465 *a &= ~mask;
466 raw_local_irq_restore(flags);
467 }
468
469 smp_llsc_mb();
470
471 return res != 0;
472 }
473
474 /*
475 * test_and_change_bit - Change a bit and return its old value
476 * @nr: Bit to change
477 * @addr: Address to count from
478 *
479 * This operation is atomic and cannot be reordered.
480 * It also implies a memory barrier.
481 */
482 static inline int test_and_change_bit(unsigned long nr,
483 volatile unsigned long *addr)
484 {
485 unsigned short bit = nr & SZLONG_MASK;
486 unsigned long res;
487
488 smp_llsc_mb();
489
490 if (kernel_uses_llsc && R10000_LLSC_WAR) {
491 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
492 unsigned long temp;
493
494 __asm__ __volatile__(
495 " .set mips3 \n"
496 "1: " __LL "%0, %1 # test_and_change_bit \n"
497 " xor %2, %0, %3 \n"
498 " " __SC "%2, %1 \n"
499 " beqzl %2, 1b \n"
500 " and %2, %0, %3 \n"
501 " .set mips0 \n"
502 : "=&r" (temp), "=m" (*m), "=&r" (res)
503 : "r" (1UL << bit), "m" (*m)
504 : "memory");
505 } else if (kernel_uses_llsc) {
506 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
507 unsigned long temp;
508
509 __asm__ __volatile__(
510 " .set push \n"
511 " .set noreorder \n"
512 " .set mips3 \n"
513 "1: " __LL "%0, %1 # test_and_change_bit \n"
514 " xor %2, %0, %3 \n"
515 " " __SC "\t%2, %1 \n"
516 " beqz %2, 2f \n"
517 " and %2, %0, %3 \n"
518 " .subsection 2 \n"
519 "2: b 1b \n"
520 " nop \n"
521 " .previous \n"
522 " .set pop \n"
523 : "=&r" (temp), "=m" (*m), "=&r" (res)
524 : "r" (1UL << bit), "m" (*m)
525 : "memory");
526 } else {
527 volatile unsigned long *a = addr;
528 unsigned long mask;
529 unsigned long flags;
530
531 a += nr >> SZLONG_LOG;
532 mask = 1UL << bit;
533 raw_local_irq_save(flags);
534 res = (mask & *a);
535 *a ^= mask;
536 raw_local_irq_restore(flags);
537 }
538
539 smp_llsc_mb();
540
541 return res != 0;
542 }
543
544 #include <asm-generic/bitops/non-atomic.h>
545
546 /*
547 * __clear_bit_unlock - Clears a bit in memory
548 * @nr: Bit to clear
549 * @addr: Address to start counting from
550 *
551 * __clear_bit() is non-atomic and implies release semantics before the memory
552 * operation. It can be used for an unlock if no other CPUs can concurrently
553 * modify other bits in the word.
554 */
555 static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
556 {
557 smp_mb();
558 __clear_bit(nr, addr);
559 }
560
561 /*
562 * Return the bit position (0..63) of the most significant 1 bit in a word
563 * Returns -1 if no 1 bit exists
564 */
565 static inline unsigned long __fls(unsigned long word)
566 {
567 int num;
568
569 if (BITS_PER_LONG == 32 &&
570 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
571 __asm__(
572 " .set push \n"
573 " .set mips32 \n"
574 " clz %0, %1 \n"
575 " .set pop \n"
576 : "=r" (num)
577 : "r" (word));
578
579 return 31 - num;
580 }
581
582 if (BITS_PER_LONG == 64 &&
583 __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
584 __asm__(
585 " .set push \n"
586 " .set mips64 \n"
587 " dclz %0, %1 \n"
588 " .set pop \n"
589 : "=r" (num)
590 : "r" (word));
591
592 return 63 - num;
593 }
594
595 num = BITS_PER_LONG - 1;
596
597 #if BITS_PER_LONG == 64
598 if (!(word & (~0ul << 32))) {
599 num -= 32;
600 word <<= 32;
601 }
602 #endif
603 if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
604 num -= 16;
605 word <<= 16;
606 }
607 if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
608 num -= 8;
609 word <<= 8;
610 }
611 if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
612 num -= 4;
613 word <<= 4;
614 }
615 if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
616 num -= 2;
617 word <<= 2;
618 }
619 if (!(word & (~0ul << (BITS_PER_LONG-1))))
620 num -= 1;
621 return num;
622 }
623
624 /*
625 * __ffs - find first bit in word.
626 * @word: The word to search
627 *
628 * Returns 0..SZLONG-1
629 * Undefined if no bit exists, so code should check against 0 first.
630 */
631 static inline unsigned long __ffs(unsigned long word)
632 {
633 return __fls(word & -word);
634 }
635
636 /*
637 * fls - find last bit set.
638 * @word: The word to search
639 *
640 * This is defined the same way as ffs.
641 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
642 */
643 static inline int fls(int x)
644 {
645 int r;
646
647 if (__builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
648 __asm__("clz %0, %1" : "=r" (x) : "r" (x));
649
650 return 32 - x;
651 }
652
653 r = 32;
654 if (!x)
655 return 0;
656 if (!(x & 0xffff0000u)) {
657 x <<= 16;
658 r -= 16;
659 }
660 if (!(x & 0xff000000u)) {
661 x <<= 8;
662 r -= 8;
663 }
664 if (!(x & 0xf0000000u)) {
665 x <<= 4;
666 r -= 4;
667 }
668 if (!(x & 0xc0000000u)) {
669 x <<= 2;
670 r -= 2;
671 }
672 if (!(x & 0x80000000u)) {
673 x <<= 1;
674 r -= 1;
675 }
676 return r;
677 }
678
679 #include <asm-generic/bitops/fls64.h>
680
681 /*
682 * ffs - find first bit set.
683 * @word: The word to search
684 *
685 * This is defined the same way as
686 * the libc and compiler builtin ffs routines, therefore
687 * differs in spirit from the above ffz (man ffs).
688 */
689 static inline int ffs(int word)
690 {
691 if (!word)
692 return 0;
693
694 return fls(word & -word);
695 }
696
697 #include <asm-generic/bitops/ffz.h>
698 #include <asm-generic/bitops/find.h>
699
700 #ifdef __KERNEL__
701
702 #include <asm-generic/bitops/sched.h>
703 #include <asm-generic/bitops/hweight.h>
704 #include <asm-generic/bitops/ext2-non-atomic.h>
705 #include <asm-generic/bitops/ext2-atomic.h>
706 #include <asm-generic/bitops/minix.h>
707
708 #endif /* __KERNEL__ */
709
710 #endif /* _ASM_BITOPS_H */
Linux for Ginger Game Console