2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Inline assembly cache operations.
8 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
9 * Copyright (C) 1997 - 2002 Ralf Baechle (ralf@gnu.org)
10 * Copyright (C) 2004 Ralf Baechle (ralf@linux-mips.org)
12 #ifndef _ASM_R4KCACHE_H
13 #define _ASM_R4KCACHE_H
16 #include <asm/cacheops.h>
19 * This macro return a properly sign-extended address suitable as base address
20 * for indexed cache operations. Two issues here:
22 * - The MIPS32 and MIPS64 specs permit an implementation to directly derive
23 * the index bits from the virtual address. This breaks with tradition
24 * set by the R4000. To keep unpleasant surprises from happening we pick
25 * an address in KSEG0 / CKSEG0.
26 * - We need a properly sign extended address for 64-bit code. To get away
27 * without ifdefs we let the compiler do it by a type cast.
29 #define INDEX_BASE CKSEG0
31 #define cache_op(op,addr) \
32 __asm__ __volatile__( \
34 " .set noreorder \n" \
35 " .set mips3\n\t \n" \
39 : "i" (op), "m" (*(unsigned char *)(addr)))
41 static inline void flush_icache_line_indexed(unsigned long addr
)
43 cache_op(Index_Invalidate_I
, addr
);
46 static inline void flush_dcache_line_indexed(unsigned long addr
)
48 cache_op(Index_Writeback_Inv_D
, addr
);
51 static inline void flush_scache_line_indexed(unsigned long addr
)
53 cache_op(Index_Writeback_Inv_SD
, addr
);
56 static inline void flush_icache_line(unsigned long addr
)
58 cache_op(Hit_Invalidate_I
, addr
);
61 static inline void flush_dcache_line(unsigned long addr
)
63 cache_op(Hit_Writeback_Inv_D
, addr
);
66 static inline void invalidate_dcache_line(unsigned long addr
)
68 cache_op(Hit_Invalidate_D
, addr
);
71 static inline void invalidate_scache_line(unsigned long addr
)
73 cache_op(Hit_Invalidate_SD
, addr
);
76 static inline void flush_scache_line(unsigned long addr
)
78 cache_op(Hit_Writeback_Inv_SD
, addr
);
82 * The next two are for badland addresses like signal trampolines.
84 static inline void protected_flush_icache_line(unsigned long addr
)
90 "1: cache %0, (%1) \n"
92 " .section __ex_table,\"a\" \n"
93 " "STR(PTR
)" 1b, 2b \n"
96 : "i" (Hit_Invalidate_I
), "r" (addr
));
100 * R10000 / R12000 hazard - these processors don't support the Hit_Writeback_D
101 * cacheop so we use Hit_Writeback_Inv_D which is supported by all R4000-style
102 * caches. We're talking about one cacheline unnecessarily getting invalidated
103 * here so the penalty isn't overly hard.
105 static inline void protected_writeback_dcache_line(unsigned long addr
)
107 __asm__
__volatile__(
111 "1: cache %0, (%1) \n"
113 " .section __ex_table,\"a\" \n"
114 " "STR(PTR
)" 1b, 2b \n"
117 : "i" (Hit_Writeback_Inv_D
), "r" (addr
));
120 static inline void protected_writeback_scache_line(unsigned long addr
)
122 __asm__
__volatile__(
126 "1: cache %0, (%1) \n"
128 " .section __ex_table,\"a\" \n"
129 " "STR(PTR
)" 1b, 2b \n"
132 : "i" (Hit_Writeback_Inv_SD
), "r" (addr
));
136 * This one is RM7000-specific
138 static inline void invalidate_tcache_page(unsigned long addr
)
140 cache_op(Page_Invalidate_T
, addr
);
143 #define cache16_unroll32(base,op) \
144 __asm__ __volatile__( \
146 " .set noreorder \n" \
148 " cache %1, 0x000(%0); cache %1, 0x010(%0) \n" \
149 " cache %1, 0x020(%0); cache %1, 0x030(%0) \n" \
150 " cache %1, 0x040(%0); cache %1, 0x050(%0) \n" \
151 " cache %1, 0x060(%0); cache %1, 0x070(%0) \n" \
152 " cache %1, 0x080(%0); cache %1, 0x090(%0) \n" \
153 " cache %1, 0x0a0(%0); cache %1, 0x0b0(%0) \n" \
154 " cache %1, 0x0c0(%0); cache %1, 0x0d0(%0) \n" \
155 " cache %1, 0x0e0(%0); cache %1, 0x0f0(%0) \n" \
156 " cache %1, 0x100(%0); cache %1, 0x110(%0) \n" \
157 " cache %1, 0x120(%0); cache %1, 0x130(%0) \n" \
158 " cache %1, 0x140(%0); cache %1, 0x150(%0) \n" \
159 " cache %1, 0x160(%0); cache %1, 0x170(%0) \n" \
160 " cache %1, 0x180(%0); cache %1, 0x190(%0) \n" \
161 " cache %1, 0x1a0(%0); cache %1, 0x1b0(%0) \n" \
162 " cache %1, 0x1c0(%0); cache %1, 0x1d0(%0) \n" \
163 " cache %1, 0x1e0(%0); cache %1, 0x1f0(%0) \n" \
169 static inline void blast_dcache16(void)
171 unsigned long start
= INDEX_BASE
;
172 unsigned long end
= start
+ current_cpu_data
.dcache
.waysize
;
173 unsigned long ws_inc
= 1UL << current_cpu_data
.dcache
.waybit
;
174 unsigned long ws_end
= current_cpu_data
.dcache
.ways
<<
175 current_cpu_data
.dcache
.waybit
;
176 unsigned long ws
, addr
;
178 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
179 for (addr
= start
; addr
< end
; addr
+= 0x200)
180 cache16_unroll32(addr
|ws
,Index_Writeback_Inv_D
);
183 static inline void blast_dcache16_page(unsigned long page
)
185 unsigned long start
= page
;
186 unsigned long end
= start
+ PAGE_SIZE
;
189 cache16_unroll32(start
,Hit_Writeback_Inv_D
);
191 } while (start
< end
);
194 static inline void blast_dcache16_page_indexed(unsigned long page
)
196 unsigned long start
= page
;
197 unsigned long end
= start
+ PAGE_SIZE
;
198 unsigned long ws_inc
= 1UL << current_cpu_data
.dcache
.waybit
;
199 unsigned long ws_end
= current_cpu_data
.dcache
.ways
<<
200 current_cpu_data
.dcache
.waybit
;
201 unsigned long ws
, addr
;
203 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
204 for (addr
= start
; addr
< end
; addr
+= 0x200)
205 cache16_unroll32(addr
|ws
,Index_Writeback_Inv_D
);
208 static inline void blast_icache16(void)
210 unsigned long start
= INDEX_BASE
;
211 unsigned long end
= start
+ current_cpu_data
.icache
.waysize
;
212 unsigned long ws_inc
= 1UL << current_cpu_data
.icache
.waybit
;
213 unsigned long ws_end
= current_cpu_data
.icache
.ways
<<
214 current_cpu_data
.icache
.waybit
;
215 unsigned long ws
, addr
;
217 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
218 for (addr
= start
; addr
< end
; addr
+= 0x200)
219 cache16_unroll32(addr
|ws
,Index_Invalidate_I
);
222 static inline void blast_icache16_page(unsigned long page
)
224 unsigned long start
= page
;
225 unsigned long end
= start
+ PAGE_SIZE
;
228 cache16_unroll32(start
,Hit_Invalidate_I
);
230 } while (start
< end
);
233 static inline void blast_icache16_page_indexed(unsigned long page
)
235 unsigned long start
= page
;
236 unsigned long end
= start
+ PAGE_SIZE
;
237 unsigned long ws_inc
= 1UL << current_cpu_data
.icache
.waybit
;
238 unsigned long ws_end
= current_cpu_data
.icache
.ways
<<
239 current_cpu_data
.icache
.waybit
;
240 unsigned long ws
, addr
;
242 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
243 for (addr
= start
; addr
< end
; addr
+= 0x200)
244 cache16_unroll32(addr
|ws
,Index_Invalidate_I
);
247 static inline void blast_scache16(void)
249 unsigned long start
= INDEX_BASE
;
250 unsigned long end
= start
+ current_cpu_data
.scache
.waysize
;
251 unsigned long ws_inc
= 1UL << current_cpu_data
.scache
.waybit
;
252 unsigned long ws_end
= current_cpu_data
.scache
.ways
<<
253 current_cpu_data
.scache
.waybit
;
254 unsigned long ws
, addr
;
256 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
257 for (addr
= start
; addr
< end
; addr
+= 0x200)
258 cache16_unroll32(addr
|ws
,Index_Writeback_Inv_SD
);
261 static inline void blast_scache16_page(unsigned long page
)
263 unsigned long start
= page
;
264 unsigned long end
= page
+ PAGE_SIZE
;
267 cache16_unroll32(start
,Hit_Writeback_Inv_SD
);
269 } while (start
< end
);
272 static inline void blast_scache16_page_indexed(unsigned long page
)
274 unsigned long start
= page
;
275 unsigned long end
= start
+ PAGE_SIZE
;
276 unsigned long ws_inc
= 1UL << current_cpu_data
.scache
.waybit
;
277 unsigned long ws_end
= current_cpu_data
.scache
.ways
<<
278 current_cpu_data
.scache
.waybit
;
279 unsigned long ws
, addr
;
281 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
282 for (addr
= start
; addr
< end
; addr
+= 0x200)
283 cache16_unroll32(addr
|ws
,Index_Writeback_Inv_SD
);
286 #define cache32_unroll32(base,op) \
287 __asm__ __volatile__( \
289 " .set noreorder \n" \
291 " cache %1, 0x000(%0); cache %1, 0x020(%0) \n" \
292 " cache %1, 0x040(%0); cache %1, 0x060(%0) \n" \
293 " cache %1, 0x080(%0); cache %1, 0x0a0(%0) \n" \
294 " cache %1, 0x0c0(%0); cache %1, 0x0e0(%0) \n" \
295 " cache %1, 0x100(%0); cache %1, 0x120(%0) \n" \
296 " cache %1, 0x140(%0); cache %1, 0x160(%0) \n" \
297 " cache %1, 0x180(%0); cache %1, 0x1a0(%0) \n" \
298 " cache %1, 0x1c0(%0); cache %1, 0x1e0(%0) \n" \
299 " cache %1, 0x200(%0); cache %1, 0x220(%0) \n" \
300 " cache %1, 0x240(%0); cache %1, 0x260(%0) \n" \
301 " cache %1, 0x280(%0); cache %1, 0x2a0(%0) \n" \
302 " cache %1, 0x2c0(%0); cache %1, 0x2e0(%0) \n" \
303 " cache %1, 0x300(%0); cache %1, 0x320(%0) \n" \
304 " cache %1, 0x340(%0); cache %1, 0x360(%0) \n" \
305 " cache %1, 0x380(%0); cache %1, 0x3a0(%0) \n" \
306 " cache %1, 0x3c0(%0); cache %1, 0x3e0(%0) \n" \
312 static inline void blast_dcache32(void)
314 unsigned long start
= INDEX_BASE
;
315 unsigned long end
= start
+ current_cpu_data
.dcache
.waysize
;
316 unsigned long ws_inc
= 1UL << current_cpu_data
.dcache
.waybit
;
317 unsigned long ws_end
= current_cpu_data
.dcache
.ways
<<
318 current_cpu_data
.dcache
.waybit
;
319 unsigned long ws
, addr
;
321 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
322 for (addr
= start
; addr
< end
; addr
+= 0x400)
323 cache32_unroll32(addr
|ws
,Index_Writeback_Inv_D
);
326 static inline void blast_dcache32_page(unsigned long page
)
328 unsigned long start
= page
;
329 unsigned long end
= start
+ PAGE_SIZE
;
332 cache32_unroll32(start
,Hit_Writeback_Inv_D
);
334 } while (start
< end
);
337 static inline void blast_dcache32_page_indexed(unsigned long page
)
339 unsigned long start
= page
;
340 unsigned long end
= start
+ PAGE_SIZE
;
341 unsigned long ws_inc
= 1UL << current_cpu_data
.dcache
.waybit
;
342 unsigned long ws_end
= current_cpu_data
.dcache
.ways
<<
343 current_cpu_data
.dcache
.waybit
;
344 unsigned long ws
, addr
;
346 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
347 for (addr
= start
; addr
< end
; addr
+= 0x400)
348 cache32_unroll32(addr
|ws
,Index_Writeback_Inv_D
);
351 static inline void blast_icache32(void)
353 unsigned long start
= INDEX_BASE
;
354 unsigned long end
= start
+ current_cpu_data
.icache
.waysize
;
355 unsigned long ws_inc
= 1UL << current_cpu_data
.icache
.waybit
;
356 unsigned long ws_end
= current_cpu_data
.icache
.ways
<<
357 current_cpu_data
.icache
.waybit
;
358 unsigned long ws
, addr
;
360 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
361 for (addr
= start
; addr
< end
; addr
+= 0x400)
362 cache32_unroll32(addr
|ws
,Index_Invalidate_I
);
365 static inline void blast_icache32_page(unsigned long page
)
367 unsigned long start
= page
;
368 unsigned long end
= start
+ PAGE_SIZE
;
371 cache32_unroll32(start
,Hit_Invalidate_I
);
373 } while (start
< end
);
376 static inline void blast_icache32_page_indexed(unsigned long page
)
378 unsigned long start
= page
;
379 unsigned long end
= start
+ PAGE_SIZE
;
380 unsigned long ws_inc
= 1UL << current_cpu_data
.icache
.waybit
;
381 unsigned long ws_end
= current_cpu_data
.icache
.ways
<<
382 current_cpu_data
.icache
.waybit
;
383 unsigned long ws
, addr
;
385 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
386 for (addr
= start
; addr
< end
; addr
+= 0x400)
387 cache32_unroll32(addr
|ws
,Index_Invalidate_I
);
390 static inline void blast_scache32(void)
392 unsigned long start
= INDEX_BASE
;
393 unsigned long end
= start
+ current_cpu_data
.scache
.waysize
;
394 unsigned long ws_inc
= 1UL << current_cpu_data
.scache
.waybit
;
395 unsigned long ws_end
= current_cpu_data
.scache
.ways
<<
396 current_cpu_data
.scache
.waybit
;
397 unsigned long ws
, addr
;
399 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
400 for (addr
= start
; addr
< end
; addr
+= 0x400)
401 cache32_unroll32(addr
|ws
,Index_Writeback_Inv_SD
);
404 static inline void blast_scache32_page(unsigned long page
)
406 unsigned long start
= page
;
407 unsigned long end
= page
+ PAGE_SIZE
;
410 cache32_unroll32(start
,Hit_Writeback_Inv_SD
);
412 } while (start
< end
);
415 static inline void blast_scache32_page_indexed(unsigned long page
)
417 unsigned long start
= page
;
418 unsigned long end
= start
+ PAGE_SIZE
;
419 unsigned long ws_inc
= 1UL << current_cpu_data
.scache
.waybit
;
420 unsigned long ws_end
= current_cpu_data
.scache
.ways
<<
421 current_cpu_data
.scache
.waybit
;
422 unsigned long ws
, addr
;
424 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
425 for (addr
= start
; addr
< end
; addr
+= 0x400)
426 cache32_unroll32(addr
|ws
,Index_Writeback_Inv_SD
);
429 #define cache64_unroll32(base,op) \
430 __asm__ __volatile__( \
432 " .set noreorder \n" \
434 " cache %1, 0x000(%0); cache %1, 0x040(%0) \n" \
435 " cache %1, 0x080(%0); cache %1, 0x0c0(%0) \n" \
436 " cache %1, 0x100(%0); cache %1, 0x140(%0) \n" \
437 " cache %1, 0x180(%0); cache %1, 0x1c0(%0) \n" \
438 " cache %1, 0x200(%0); cache %1, 0x240(%0) \n" \
439 " cache %1, 0x280(%0); cache %1, 0x2c0(%0) \n" \
440 " cache %1, 0x300(%0); cache %1, 0x340(%0) \n" \
441 " cache %1, 0x380(%0); cache %1, 0x3c0(%0) \n" \
442 " cache %1, 0x400(%0); cache %1, 0x440(%0) \n" \
443 " cache %1, 0x480(%0); cache %1, 0x4c0(%0) \n" \
444 " cache %1, 0x500(%0); cache %1, 0x540(%0) \n" \
445 " cache %1, 0x580(%0); cache %1, 0x5c0(%0) \n" \
446 " cache %1, 0x600(%0); cache %1, 0x640(%0) \n" \
447 " cache %1, 0x680(%0); cache %1, 0x6c0(%0) \n" \
448 " cache %1, 0x700(%0); cache %1, 0x740(%0) \n" \
449 " cache %1, 0x780(%0); cache %1, 0x7c0(%0) \n" \
455 static inline void blast_icache64(void)
457 unsigned long start
= INDEX_BASE
;
458 unsigned long end
= start
+ current_cpu_data
.icache
.waysize
;
459 unsigned long ws_inc
= 1UL << current_cpu_data
.icache
.waybit
;
460 unsigned long ws_end
= current_cpu_data
.icache
.ways
<<
461 current_cpu_data
.icache
.waybit
;
462 unsigned long ws
, addr
;
464 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
465 for (addr
= start
; addr
< end
; addr
+= 0x800)
466 cache64_unroll32(addr
|ws
,Index_Invalidate_I
);
469 static inline void blast_icache64_page(unsigned long page
)
471 unsigned long start
= page
;
472 unsigned long end
= start
+ PAGE_SIZE
;
475 cache64_unroll32(start
,Hit_Invalidate_I
);
477 } while (start
< end
);
480 static inline void blast_icache64_page_indexed(unsigned long page
)
482 unsigned long start
= page
;
483 unsigned long end
= start
+ PAGE_SIZE
;
484 unsigned long ws_inc
= 1UL << current_cpu_data
.icache
.waybit
;
485 unsigned long ws_end
= current_cpu_data
.icache
.ways
<<
486 current_cpu_data
.icache
.waybit
;
487 unsigned long ws
, addr
;
489 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
490 for (addr
= start
; addr
< end
; addr
+= 0x800)
491 cache64_unroll32(addr
|ws
,Index_Invalidate_I
);
494 static inline void blast_scache64(void)
496 unsigned long start
= INDEX_BASE
;
497 unsigned long end
= start
+ current_cpu_data
.scache
.waysize
;
498 unsigned long ws_inc
= 1UL << current_cpu_data
.scache
.waybit
;
499 unsigned long ws_end
= current_cpu_data
.scache
.ways
<<
500 current_cpu_data
.scache
.waybit
;
501 unsigned long ws
, addr
;
503 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
504 for (addr
= start
; addr
< end
; addr
+= 0x800)
505 cache64_unroll32(addr
|ws
,Index_Writeback_Inv_SD
);
508 static inline void blast_scache64_page(unsigned long page
)
510 unsigned long start
= page
;
511 unsigned long end
= page
+ PAGE_SIZE
;
514 cache64_unroll32(start
,Hit_Writeback_Inv_SD
);
516 } while (start
< end
);
519 static inline void blast_scache64_page_indexed(unsigned long page
)
521 unsigned long start
= page
;
522 unsigned long end
= start
+ PAGE_SIZE
;
523 unsigned long ws_inc
= 1UL << current_cpu_data
.scache
.waybit
;
524 unsigned long ws_end
= current_cpu_data
.scache
.ways
<<
525 current_cpu_data
.scache
.waybit
;
526 unsigned long ws
, addr
;
528 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
529 for (addr
= start
; addr
< end
; addr
+= 0x800)
530 cache64_unroll32(addr
|ws
,Index_Writeback_Inv_SD
);
533 #define cache128_unroll32(base,op) \
534 __asm__ __volatile__( \
536 " .set noreorder \n" \
538 " cache %1, 0x000(%0); cache %1, 0x080(%0) \n" \
539 " cache %1, 0x100(%0); cache %1, 0x180(%0) \n" \
540 " cache %1, 0x200(%0); cache %1, 0x280(%0) \n" \
541 " cache %1, 0x300(%0); cache %1, 0x380(%0) \n" \
542 " cache %1, 0x400(%0); cache %1, 0x480(%0) \n" \
543 " cache %1, 0x500(%0); cache %1, 0x580(%0) \n" \
544 " cache %1, 0x600(%0); cache %1, 0x680(%0) \n" \
545 " cache %1, 0x700(%0); cache %1, 0x780(%0) \n" \
546 " cache %1, 0x800(%0); cache %1, 0x880(%0) \n" \
547 " cache %1, 0x900(%0); cache %1, 0x980(%0) \n" \
548 " cache %1, 0xa00(%0); cache %1, 0xa80(%0) \n" \
549 " cache %1, 0xb00(%0); cache %1, 0xb80(%0) \n" \
550 " cache %1, 0xc00(%0); cache %1, 0xc80(%0) \n" \
551 " cache %1, 0xd00(%0); cache %1, 0xd80(%0) \n" \
552 " cache %1, 0xe00(%0); cache %1, 0xe80(%0) \n" \
553 " cache %1, 0xf00(%0); cache %1, 0xf80(%0) \n" \
559 static inline void blast_scache128(void)
561 unsigned long start
= INDEX_BASE
;
562 unsigned long end
= start
+ current_cpu_data
.scache
.waysize
;
563 unsigned long ws_inc
= 1UL << current_cpu_data
.scache
.waybit
;
564 unsigned long ws_end
= current_cpu_data
.scache
.ways
<<
565 current_cpu_data
.scache
.waybit
;
566 unsigned long ws
, addr
;
568 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
569 for (addr
= start
; addr
< end
; addr
+= 0x1000)
570 cache128_unroll32(addr
|ws
,Index_Writeback_Inv_SD
);
573 static inline void blast_scache128_page(unsigned long page
)
575 unsigned long start
= page
;
576 unsigned long end
= page
+ PAGE_SIZE
;
579 cache128_unroll32(start
,Hit_Writeback_Inv_SD
);
581 } while (start
< end
);
584 static inline void blast_scache128_page_indexed(unsigned long page
)
586 unsigned long start
= page
;
587 unsigned long end
= start
+ PAGE_SIZE
;
588 unsigned long ws_inc
= 1UL << current_cpu_data
.scache
.waybit
;
589 unsigned long ws_end
= current_cpu_data
.scache
.ways
<<
590 current_cpu_data
.scache
.waybit
;
591 unsigned long ws
, addr
;
593 for (ws
= 0; ws
< ws_end
; ws
+= ws_inc
)
594 for (addr
= start
; addr
< end
; addr
+= 0x1000)
595 cache128_unroll32(addr
|ws
,Index_Writeback_Inv_SD
);
598 #endif /* _ASM_R4KCACHE_H */