?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
[cris-mirror.git] / arch / microblaze / kernel / cpu / cache.c
blob0bde47e4fa694264256c1e8d2498436acc66320b
1 /*
2 * Cache control for MicroBlaze cache memories
3 *
4 * Copyright (C) 2007-2009 Michal Simek <monstr@monstr.eu>
5 * Copyright (C) 2007-2009 PetaLogix
6 * Copyright (C) 2007-2009 John Williams <john.williams@petalogix.com>
7 *
8 * This file is subject to the terms and conditions of the GNU General
9 * Public License. See the file COPYING in the main directory of this
10 * archive for more details.
11 */
12
13 #include <asm/cacheflush.h>
14 #include <linux/cache.h>
15 #include <asm/cpuinfo.h>
16 #include <asm/pvr.h>
17
18 static inline void __enable_icache_msr(void)
19 {
20 __asm__ __volatile__ (" msrset r0, %0;" \
21 "nop;" \
22 : : "i" (MSR_ICE) : "memory");
23 }
24
25 static inline void __disable_icache_msr(void)
26 {
27 __asm__ __volatile__ (" msrclr r0, %0;" \
28 "nop;" \
29 : : "i" (MSR_ICE) : "memory");
30 }
31
32 static inline void __enable_dcache_msr(void)
33 {
34 __asm__ __volatile__ (" msrset r0, %0;" \
35 "nop;" \
36 : : "i" (MSR_DCE) : "memory");
37 }
38
39 static inline void __disable_dcache_msr(void)
40 {
41 __asm__ __volatile__ (" msrclr r0, %0;" \
42 "nop; " \
43 : : "i" (MSR_DCE) : "memory");
44 }
45
46 static inline void __enable_icache_nomsr(void)
47 {
48 __asm__ __volatile__ (" mfs r12, rmsr;" \
49 "nop;" \
50 "ori r12, r12, %0;" \
51 "mts rmsr, r12;" \
52 "nop;" \
53 : : "i" (MSR_ICE) : "memory", "r12");
54 }
55
56 static inline void __disable_icache_nomsr(void)
57 {
58 __asm__ __volatile__ (" mfs r12, rmsr;" \
59 "nop;" \
60 "andi r12, r12, ~%0;" \
61 "mts rmsr, r12;" \
62 "nop;" \
63 : : "i" (MSR_ICE) : "memory", "r12");
64 }
65
66 static inline void __enable_dcache_nomsr(void)
67 {
68 __asm__ __volatile__ (" mfs r12, rmsr;" \
69 "nop;" \
70 "ori r12, r12, %0;" \
71 "mts rmsr, r12;" \
72 "nop;" \
73 : : "i" (MSR_DCE) : "memory", "r12");
74 }
75
76 static inline void __disable_dcache_nomsr(void)
77 {
78 __asm__ __volatile__ (" mfs r12, rmsr;" \
79 "nop;" \
80 "andi r12, r12, ~%0;" \
81 "mts rmsr, r12;" \
82 "nop;" \
83 : : "i" (MSR_DCE) : "memory", "r12");
84 }
85
86
87 /* Helper macro for computing the limits of cache range loops
88 *
89 * End address can be unaligned which is OK for C implementation.
90 * ASM implementation align it in ASM macros
91 */
92 #define CACHE_LOOP_LIMITS(start, end, cache_line_length, cache_size) \
93 do { \
94 int align = ~(cache_line_length - 1); \
95 end = min(start + cache_size, end); \
96 start &= align; \
97 } while (0)
98
99 /*
100 * Helper macro to loop over the specified cache_size/line_length and
101 * execute 'op' on that cacheline
102 */
103 #define CACHE_ALL_LOOP(cache_size, line_length, op) \
104 do { \
105 unsigned int len = cache_size - line_length; \
106 int step = -line_length; \
107 WARN_ON(step >= 0); \
108 \
109 __asm__ __volatile__ (" 1: " #op " %0, r0;" \
110 "bgtid %0, 1b;" \
111 "addk %0, %0, %1;" \
112 : : "r" (len), "r" (step) \
113 : "memory"); \
114 } while (0)
115
116 /* Used for wdc.flush/clear which can use rB for offset which is not possible
117 * to use for simple wdc or wic.
118 *
119 * start address is cache aligned
120 * end address is not aligned, if end is aligned then I have to subtract
121 * cacheline length because I can't flush/invalidate the next cacheline.
122 * If is not, I align it because I will flush/invalidate whole line.
123 */
124 #define CACHE_RANGE_LOOP_2(start, end, line_length, op) \
125 do { \
126 int step = -line_length; \
127 int align = ~(line_length - 1); \
128 int count; \
129 end = ((end & align) == end) ? end - line_length : end & align; \
130 count = end - start; \
131 WARN_ON(count < 0); \
132 \
133 __asm__ __volatile__ (" 1: " #op " %0, %1;" \
134 "bgtid %1, 1b;" \
135 "addk %1, %1, %2;" \
136 : : "r" (start), "r" (count), \
137 "r" (step) : "memory"); \
138 } while (0)
139
140 /* It is used only first parameter for OP - for wic, wdc */
141 #define CACHE_RANGE_LOOP_1(start, end, line_length, op) \
142 do { \
143 unsigned int volatile temp = 0; \
144 unsigned int align = ~(line_length - 1); \
145 end = ((end & align) == end) ? end - line_length : end & align; \
146 WARN_ON(end < start); \
147 \
148 __asm__ __volatile__ (" 1: " #op " %1, r0;" \
149 "cmpu %0, %1, %2;" \
150 "bgtid %0, 1b;" \
151 "addk %1, %1, %3;" \
152 : : "r" (temp), "r" (start), "r" (end), \
153 "r" (line_length) : "memory"); \
154 } while (0)
155
156 #define ASM_LOOP
157
158 static void __flush_icache_range_msr_irq(unsigned long start, unsigned long end)
159 {
160 unsigned long flags;
161 #ifndef ASM_LOOP
162 int i;
163 #endif
164 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
165 (unsigned int)start, (unsigned int) end);
166
167 CACHE_LOOP_LIMITS(start, end,
168 cpuinfo.icache_line_length, cpuinfo.icache_size);
169
170 local_irq_save(flags);
171 __disable_icache_msr();
172
173 #ifdef ASM_LOOP
174 CACHE_RANGE_LOOP_1(start, end, cpuinfo.icache_line_length, wic);
175 #else
176 for (i = start; i < end; i += cpuinfo.icache_line_length)
177 __asm__ __volatile__ ("wic %0, r0;" \
178 : : "r" (i));
179 #endif
180 __enable_icache_msr();
181 local_irq_restore(flags);
182 }
183
184 static void __flush_icache_range_nomsr_irq(unsigned long start,
185 unsigned long end)
186 {
187 unsigned long flags;
188 #ifndef ASM_LOOP
189 int i;
190 #endif
191 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
192 (unsigned int)start, (unsigned int) end);
193
194 CACHE_LOOP_LIMITS(start, end,
195 cpuinfo.icache_line_length, cpuinfo.icache_size);
196
197 local_irq_save(flags);
198 __disable_icache_nomsr();
199
200 #ifdef ASM_LOOP
201 CACHE_RANGE_LOOP_1(start, end, cpuinfo.icache_line_length, wic);
202 #else
203 for (i = start; i < end; i += cpuinfo.icache_line_length)
204 __asm__ __volatile__ ("wic %0, r0;" \
205 : : "r" (i));
206 #endif
207
208 __enable_icache_nomsr();
209 local_irq_restore(flags);
210 }
211
212 static void __flush_icache_range_noirq(unsigned long start,
213 unsigned long end)
214 {
215 #ifndef ASM_LOOP
216 int i;
217 #endif
218 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
219 (unsigned int)start, (unsigned int) end);
220
221 CACHE_LOOP_LIMITS(start, end,
222 cpuinfo.icache_line_length, cpuinfo.icache_size);
223 #ifdef ASM_LOOP
224 CACHE_RANGE_LOOP_1(start, end, cpuinfo.icache_line_length, wic);
225 #else
226 for (i = start; i < end; i += cpuinfo.icache_line_length)
227 __asm__ __volatile__ ("wic %0, r0;" \
228 : : "r" (i));
229 #endif
230 }
231
232 static void __flush_icache_all_msr_irq(void)
233 {
234 unsigned long flags;
235 #ifndef ASM_LOOP
236 int i;
237 #endif
238 pr_debug("%s\n", __func__);
239
240 local_irq_save(flags);
241 __disable_icache_msr();
242 #ifdef ASM_LOOP
243 CACHE_ALL_LOOP(cpuinfo.icache_size, cpuinfo.icache_line_length, wic);
244 #else
245 for (i = 0; i < cpuinfo.icache_size;
246 i += cpuinfo.icache_line_length)
247 __asm__ __volatile__ ("wic %0, r0;" \
248 : : "r" (i));
249 #endif
250 __enable_icache_msr();
251 local_irq_restore(flags);
252 }
253
254 static void __flush_icache_all_nomsr_irq(void)
255 {
256 unsigned long flags;
257 #ifndef ASM_LOOP
258 int i;
259 #endif
260 pr_debug("%s\n", __func__);
261
262 local_irq_save(flags);
263 __disable_icache_nomsr();
264 #ifdef ASM_LOOP
265 CACHE_ALL_LOOP(cpuinfo.icache_size, cpuinfo.icache_line_length, wic);
266 #else
267 for (i = 0; i < cpuinfo.icache_size;
268 i += cpuinfo.icache_line_length)
269 __asm__ __volatile__ ("wic %0, r0;" \
270 : : "r" (i));
271 #endif
272 __enable_icache_nomsr();
273 local_irq_restore(flags);
274 }
275
276 static void __flush_icache_all_noirq(void)
277 {
278 #ifndef ASM_LOOP
279 int i;
280 #endif
281 pr_debug("%s\n", __func__);
282 #ifdef ASM_LOOP
283 CACHE_ALL_LOOP(cpuinfo.icache_size, cpuinfo.icache_line_length, wic);
284 #else
285 for (i = 0; i < cpuinfo.icache_size;
286 i += cpuinfo.icache_line_length)
287 __asm__ __volatile__ ("wic %0, r0;" \
288 : : "r" (i));
289 #endif
290 }
291
292 static void __invalidate_dcache_all_msr_irq(void)
293 {
294 unsigned long flags;
295 #ifndef ASM_LOOP
296 int i;
297 #endif
298 pr_debug("%s\n", __func__);
299
300 local_irq_save(flags);
301 __disable_dcache_msr();
302 #ifdef ASM_LOOP
303 CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length, wdc);
304 #else
305 for (i = 0; i < cpuinfo.dcache_size;
306 i += cpuinfo.dcache_line_length)
307 __asm__ __volatile__ ("wdc %0, r0;" \
308 : : "r" (i));
309 #endif
310 __enable_dcache_msr();
311 local_irq_restore(flags);
312 }
313
314 static void __invalidate_dcache_all_nomsr_irq(void)
315 {
316 unsigned long flags;
317 #ifndef ASM_LOOP
318 int i;
319 #endif
320 pr_debug("%s\n", __func__);
321
322 local_irq_save(flags);
323 __disable_dcache_nomsr();
324 #ifdef ASM_LOOP
325 CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length, wdc);
326 #else
327 for (i = 0; i < cpuinfo.dcache_size;
328 i += cpuinfo.dcache_line_length)
329 __asm__ __volatile__ ("wdc %0, r0;" \
330 : : "r" (i));
331 #endif
332 __enable_dcache_nomsr();
333 local_irq_restore(flags);
334 }
335
336 static void __invalidate_dcache_all_noirq_wt(void)
337 {
338 #ifndef ASM_LOOP
339 int i;
340 #endif
341 pr_debug("%s\n", __func__);
342 #ifdef ASM_LOOP
343 CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length, wdc);
344 #else
345 for (i = 0; i < cpuinfo.dcache_size;
346 i += cpuinfo.dcache_line_length)
347 __asm__ __volatile__ ("wdc %0, r0;" \
348 : : "r" (i));
349 #endif
350 }
351
352 /*
353 * FIXME It is blindly invalidation as is expected
354 * but can't be called on noMMU in microblaze_cache_init below
355 *
356 * MS: noMMU kernel won't boot if simple wdc is used
357 * The reason should be that there are discared data which kernel needs
358 */
359 static void __invalidate_dcache_all_wb(void)
360 {
361 #ifndef ASM_LOOP
362 int i;
363 #endif
364 pr_debug("%s\n", __func__);
365 #ifdef ASM_LOOP
366 CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length,
367 wdc);
368 #else
369 for (i = 0; i < cpuinfo.dcache_size;
370 i += cpuinfo.dcache_line_length)
371 __asm__ __volatile__ ("wdc %0, r0;" \
372 : : "r" (i));
373 #endif
374 }
375
376 static void __invalidate_dcache_range_wb(unsigned long start,
377 unsigned long end)
378 {
379 #ifndef ASM_LOOP
380 int i;
381 #endif
382 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
383 (unsigned int)start, (unsigned int) end);
384
385 CACHE_LOOP_LIMITS(start, end,
386 cpuinfo.dcache_line_length, cpuinfo.dcache_size);
387 #ifdef ASM_LOOP
388 CACHE_RANGE_LOOP_2(start, end, cpuinfo.dcache_line_length, wdc.clear);
389 #else
390 for (i = start; i < end; i += cpuinfo.dcache_line_length)
391 __asm__ __volatile__ ("wdc.clear %0, r0;" \
392 : : "r" (i));
393 #endif
394 }
395
396 static void __invalidate_dcache_range_nomsr_wt(unsigned long start,
397 unsigned long end)
398 {
399 #ifndef ASM_LOOP
400 int i;
401 #endif
402 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
403 (unsigned int)start, (unsigned int) end);
404 CACHE_LOOP_LIMITS(start, end,
405 cpuinfo.dcache_line_length, cpuinfo.dcache_size);
406
407 #ifdef ASM_LOOP
408 CACHE_RANGE_LOOP_1(start, end, cpuinfo.dcache_line_length, wdc);
409 #else
410 for (i = start; i < end; i += cpuinfo.dcache_line_length)
411 __asm__ __volatile__ ("wdc %0, r0;" \
412 : : "r" (i));
413 #endif
414 }
415
416 static void __invalidate_dcache_range_msr_irq_wt(unsigned long start,
417 unsigned long end)
418 {
419 unsigned long flags;
420 #ifndef ASM_LOOP
421 int i;
422 #endif
423 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
424 (unsigned int)start, (unsigned int) end);
425 CACHE_LOOP_LIMITS(start, end,
426 cpuinfo.dcache_line_length, cpuinfo.dcache_size);
427
428 local_irq_save(flags);
429 __disable_dcache_msr();
430
431 #ifdef ASM_LOOP
432 CACHE_RANGE_LOOP_1(start, end, cpuinfo.dcache_line_length, wdc);
433 #else
434 for (i = start; i < end; i += cpuinfo.dcache_line_length)
435 __asm__ __volatile__ ("wdc %0, r0;" \
436 : : "r" (i));
437 #endif
438
439 __enable_dcache_msr();
440 local_irq_restore(flags);
441 }
442
443 static void __invalidate_dcache_range_nomsr_irq(unsigned long start,
444 unsigned long end)
445 {
446 unsigned long flags;
447 #ifndef ASM_LOOP
448 int i;
449 #endif
450 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
451 (unsigned int)start, (unsigned int) end);
452
453 CACHE_LOOP_LIMITS(start, end,
454 cpuinfo.dcache_line_length, cpuinfo.dcache_size);
455
456 local_irq_save(flags);
457 __disable_dcache_nomsr();
458
459 #ifdef ASM_LOOP
460 CACHE_RANGE_LOOP_1(start, end, cpuinfo.dcache_line_length, wdc);
461 #else
462 for (i = start; i < end; i += cpuinfo.dcache_line_length)
463 __asm__ __volatile__ ("wdc %0, r0;" \
464 : : "r" (i));
465 #endif
466
467 __enable_dcache_nomsr();
468 local_irq_restore(flags);
469 }
470
471 static void __flush_dcache_all_wb(void)
472 {
473 #ifndef ASM_LOOP
474 int i;
475 #endif
476 pr_debug("%s\n", __func__);
477 #ifdef ASM_LOOP
478 CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length,
479 wdc.flush);
480 #else
481 for (i = 0; i < cpuinfo.dcache_size;
482 i += cpuinfo.dcache_line_length)
483 __asm__ __volatile__ ("wdc.flush %0, r0;" \
484 : : "r" (i));
485 #endif
486 }
487
488 static void __flush_dcache_range_wb(unsigned long start, unsigned long end)
489 {
490 #ifndef ASM_LOOP
491 int i;
492 #endif
493 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
494 (unsigned int)start, (unsigned int) end);
495
496 CACHE_LOOP_LIMITS(start, end,
497 cpuinfo.dcache_line_length, cpuinfo.dcache_size);
498 #ifdef ASM_LOOP
499 CACHE_RANGE_LOOP_2(start, end, cpuinfo.dcache_line_length, wdc.flush);
500 #else
501 for (i = start; i < end; i += cpuinfo.dcache_line_length)
502 __asm__ __volatile__ ("wdc.flush %0, r0;" \
503 : : "r" (i));
504 #endif
505 }
506
507 /* struct for wb caches and for wt caches */
508 struct scache *mbc;
509
510 /* new wb cache model */
511 static const struct scache wb_msr = {
512 .ie = __enable_icache_msr,
513 .id = __disable_icache_msr,
514 .ifl = __flush_icache_all_noirq,
515 .iflr = __flush_icache_range_noirq,
516 .iin = __flush_icache_all_noirq,
517 .iinr = __flush_icache_range_noirq,
518 .de = __enable_dcache_msr,
519 .dd = __disable_dcache_msr,
520 .dfl = __flush_dcache_all_wb,
521 .dflr = __flush_dcache_range_wb,
522 .din = __invalidate_dcache_all_wb,
523 .dinr = __invalidate_dcache_range_wb,
524 };
525
526 /* There is only difference in ie, id, de, dd functions */
527 static const struct scache wb_nomsr = {
528 .ie = __enable_icache_nomsr,
529 .id = __disable_icache_nomsr,
530 .ifl = __flush_icache_all_noirq,
531 .iflr = __flush_icache_range_noirq,
532 .iin = __flush_icache_all_noirq,
533 .iinr = __flush_icache_range_noirq,
534 .de = __enable_dcache_nomsr,
535 .dd = __disable_dcache_nomsr,
536 .dfl = __flush_dcache_all_wb,
537 .dflr = __flush_dcache_range_wb,
538 .din = __invalidate_dcache_all_wb,
539 .dinr = __invalidate_dcache_range_wb,
540 };
541
542 /* Old wt cache model with disabling irq and turn off cache */
543 static const struct scache wt_msr = {
544 .ie = __enable_icache_msr,
545 .id = __disable_icache_msr,
546 .ifl = __flush_icache_all_msr_irq,
547 .iflr = __flush_icache_range_msr_irq,
548 .iin = __flush_icache_all_msr_irq,
549 .iinr = __flush_icache_range_msr_irq,
550 .de = __enable_dcache_msr,
551 .dd = __disable_dcache_msr,
552 .dfl = __invalidate_dcache_all_msr_irq,
553 .dflr = __invalidate_dcache_range_msr_irq_wt,
554 .din = __invalidate_dcache_all_msr_irq,
555 .dinr = __invalidate_dcache_range_msr_irq_wt,
556 };
557
558 static const struct scache wt_nomsr = {
559 .ie = __enable_icache_nomsr,
560 .id = __disable_icache_nomsr,
561 .ifl = __flush_icache_all_nomsr_irq,
562 .iflr = __flush_icache_range_nomsr_irq,
563 .iin = __flush_icache_all_nomsr_irq,
564 .iinr = __flush_icache_range_nomsr_irq,
565 .de = __enable_dcache_nomsr,
566 .dd = __disable_dcache_nomsr,
567 .dfl = __invalidate_dcache_all_nomsr_irq,
568 .dflr = __invalidate_dcache_range_nomsr_irq,
569 .din = __invalidate_dcache_all_nomsr_irq,
570 .dinr = __invalidate_dcache_range_nomsr_irq,
571 };
572
573 /* New wt cache model for newer Microblaze versions */
574 static const struct scache wt_msr_noirq = {
575 .ie = __enable_icache_msr,
576 .id = __disable_icache_msr,
577 .ifl = __flush_icache_all_noirq,
578 .iflr = __flush_icache_range_noirq,
579 .iin = __flush_icache_all_noirq,
580 .iinr = __flush_icache_range_noirq,
581 .de = __enable_dcache_msr,
582 .dd = __disable_dcache_msr,
583 .dfl = __invalidate_dcache_all_noirq_wt,
584 .dflr = __invalidate_dcache_range_nomsr_wt,
585 .din = __invalidate_dcache_all_noirq_wt,
586 .dinr = __invalidate_dcache_range_nomsr_wt,
587 };
588
589 static const struct scache wt_nomsr_noirq = {
590 .ie = __enable_icache_nomsr,
591 .id = __disable_icache_nomsr,
592 .ifl = __flush_icache_all_noirq,
593 .iflr = __flush_icache_range_noirq,
594 .iin = __flush_icache_all_noirq,
595 .iinr = __flush_icache_range_noirq,
596 .de = __enable_dcache_nomsr,
597 .dd = __disable_dcache_nomsr,
598 .dfl = __invalidate_dcache_all_noirq_wt,
599 .dflr = __invalidate_dcache_range_nomsr_wt,
600 .din = __invalidate_dcache_all_noirq_wt,
601 .dinr = __invalidate_dcache_range_nomsr_wt,
602 };
603
604 /* CPU version code for 7.20.c - see arch/microblaze/kernel/cpu/cpuinfo.c */
605 #define CPUVER_7_20_A 0x0c
606 #define CPUVER_7_20_D 0x0f
607
608 void microblaze_cache_init(void)
609 {
610 if (cpuinfo.use_instr & PVR2_USE_MSR_INSTR) {
611 if (cpuinfo.dcache_wb) {
612 pr_info("wb_msr\n");
613 mbc = (struct scache *)&wb_msr;
614 if (cpuinfo.ver_code <= CPUVER_7_20_D) {
615 /* MS: problem with signal handling - hw bug */
616 pr_info("WB won't work properly\n");
617 }
618 } else {
619 if (cpuinfo.ver_code >= CPUVER_7_20_A) {
620 pr_info("wt_msr_noirq\n");
621 mbc = (struct scache *)&wt_msr_noirq;
622 } else {
623 pr_info("wt_msr\n");
624 mbc = (struct scache *)&wt_msr;
625 }
626 }
627 } else {
628 if (cpuinfo.dcache_wb) {
629 pr_info("wb_nomsr\n");
630 mbc = (struct scache *)&wb_nomsr;
631 if (cpuinfo.ver_code <= CPUVER_7_20_D) {
632 /* MS: problem with signal handling - hw bug */
633 pr_info("WB won't work properly\n");
634 }
635 } else {
636 if (cpuinfo.ver_code >= CPUVER_7_20_A) {
637 pr_info("wt_nomsr_noirq\n");
638 mbc = (struct scache *)&wt_nomsr_noirq;
639 } else {
640 pr_info("wt_nomsr\n");
641 mbc = (struct scache *)&wt_nomsr;
642 }
643 }
644 }
645 /*
646 * FIXME Invalidation is done in U-BOOT
647 * WT cache: Data is already written to main memory
648 * WB cache: Discard data on noMMU which caused that kernel doesn't boot
649 */
650 /* invalidate_dcache(); */
651 enable_dcache();
652
653 invalidate_icache();
654 enable_icache();
655 }
CRIS linux kernel tree