| blob | 4e314b90c75d699edda82589c8a5a983bcc5dc6e |
1 /*
2 * This file contains miscellaneous low-level functions.
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 *
5 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
6 * and Paul Mackerras.
7 * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com)
8 * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com)
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
14 *
15 */
17 #include <linux/sys.h>
18 #include <asm/unistd.h>
19 #include <asm/errno.h>
20 #include <asm/processor.h>
21 #include <asm/page.h>
22 #include <asm/cache.h>
23 #include <asm/ppc_asm.h>
24 #include <asm/asm-offsets.h>
25 #include <asm/cputable.h>
26 #include <asm/thread_info.h>
27 #include <asm/kexec.h>
28 #include <asm/ptrace.h>
30 .text
32 _GLOBAL(call_do_softirq)
33 mflr r0
34 std r0,16(r1)
35 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
36 mr r1,r3
37 bl __do_softirq
38 ld r1,0(r1)
39 ld r0,16(r1)
40 mtlr r0
41 blr
43 _GLOBAL(call_do_irq)
44 mflr r0
45 std r0,16(r1)
46 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
47 mr r1,r4
48 bl __do_irq
49 ld r1,0(r1)
50 ld r0,16(r1)
51 mtlr r0
52 blr
54 .section ".toc","aw"
55 PPC64_CACHES:
56 .tc ppc64_caches[TC],ppc64_caches
57 .section ".text"
59 /*
60 * Write any modified data cache blocks out to memory
61 * and invalidate the corresponding instruction cache blocks.
62 *
63 * flush_icache_range(unsigned long start, unsigned long stop)
64 *
65 * flush all bytes from start through stop-1 inclusive
66 */
68 _KPROBE(flush_icache_range)
69 BEGIN_FTR_SECTION
70 PURGE_PREFETCHED_INS
71 blr
72 END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
73 /*
74 * Flush the data cache to memory
75 *
76 * Different systems have different cache line sizes
77 * and in some cases i-cache and d-cache line sizes differ from
78 * each other.
79 */
80 ld r10,PPC64_CACHES@toc(r2)
81 lwz r7,DCACHEL1LINESIZE(r10)/* Get cache line size */
82 addi r5,r7,-1
83 andc r6,r3,r5 /* round low to line bdy */
84 subf r8,r6,r4 /* compute length */
85 add r8,r8,r5 /* ensure we get enough */
86 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of cache line size */
87 srw. r8,r8,r9 /* compute line count */
88 beqlr /* nothing to do? */
89 mtctr r8
90 1: dcbst 0,r6
91 add r6,r6,r7
92 bdnz 1b
93 sync
95 /* Now invalidate the instruction cache */
97 lwz r7,ICACHEL1LINESIZE(r10) /* Get Icache line size */
98 addi r5,r7,-1
99 andc r6,r3,r5 /* round low to line bdy */
100 subf r8,r6,r4 /* compute length */
101 add r8,r8,r5
102 lwz r9,ICACHEL1LOGLINESIZE(r10) /* Get log-2 of Icache line size */
103 srw. r8,r8,r9 /* compute line count */
104 beqlr /* nothing to do? */
105 mtctr r8
106 2: icbi 0,r6
107 add r6,r6,r7
108 bdnz 2b
109 isync
110 blr
111 .previous .text
112 /*
113 * Like above, but only do the D-cache.
114 *
115 * flush_dcache_range(unsigned long start, unsigned long stop)
116 *
117 * flush all bytes from start to stop-1 inclusive
118 */
119 _GLOBAL(flush_dcache_range)
121 /*
122 * Flush the data cache to memory
123 *
124 * Different systems have different cache line sizes
125 */
126 ld r10,PPC64_CACHES@toc(r2)
127 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */
128 addi r5,r7,-1
129 andc r6,r3,r5 /* round low to line bdy */
130 subf r8,r6,r4 /* compute length */
131 add r8,r8,r5 /* ensure we get enough */
132 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of dcache line size */
133 srw. r8,r8,r9 /* compute line count */
134 beqlr /* nothing to do? */
135 mtctr r8
136 0: dcbst 0,r6
137 add r6,r6,r7
138 bdnz 0b
139 sync
140 blr
142 /*
143 * Like above, but works on non-mapped physical addresses.
144 * Use only for non-LPAR setups ! It also assumes real mode
145 * is cacheable. Used for flushing out the DART before using
146 * it as uncacheable memory
147 *
148 * flush_dcache_phys_range(unsigned long start, unsigned long stop)
149 *
150 * flush all bytes from start to stop-1 inclusive
151 */
152 _GLOBAL(flush_dcache_phys_range)
153 ld r10,PPC64_CACHES@toc(r2)
154 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */
155 addi r5,r7,-1
156 andc r6,r3,r5 /* round low to line bdy */
157 subf r8,r6,r4 /* compute length */
158 add r8,r8,r5 /* ensure we get enough */
159 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of dcache line size */
160 srw. r8,r8,r9 /* compute line count */
161 beqlr /* nothing to do? */
162 mfmsr r5 /* Disable MMU Data Relocation */
163 ori r0,r5,MSR_DR
164 xori r0,r0,MSR_DR
165 sync
166 mtmsr r0
167 sync
168 isync
169 mtctr r8
170 0: dcbst 0,r6
171 add r6,r6,r7
172 bdnz 0b
173 sync
174 isync
175 mtmsr r5 /* Re-enable MMU Data Relocation */
176 sync
177 isync
178 blr
180 _GLOBAL(flush_inval_dcache_range)
181 ld r10,PPC64_CACHES@toc(r2)
182 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */
183 addi r5,r7,-1
184 andc r6,r3,r5 /* round low to line bdy */
185 subf r8,r6,r4 /* compute length */
186 add r8,r8,r5 /* ensure we get enough */
187 lwz r9,DCACHEL1LOGLINESIZE(r10)/* Get log-2 of dcache line size */
188 srw. r8,r8,r9 /* compute line count */
189 beqlr /* nothing to do? */
190 sync
191 isync
192 mtctr r8
193 0: dcbf 0,r6
194 add r6,r6,r7
195 bdnz 0b
196 sync
197 isync
198 blr
201 /*
202 * Flush a particular page from the data cache to RAM.
203 * Note: this is necessary because the instruction cache does *not*
204 * snoop from the data cache.
205 *
206 * void __flush_dcache_icache(void *page)
207 */
208 _GLOBAL(__flush_dcache_icache)
209 /*
210 * Flush the data cache to memory
211 *
212 * Different systems have different cache line sizes
213 */
215 BEGIN_FTR_SECTION
216 PURGE_PREFETCHED_INS
217 blr
218 END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
220 /* Flush the dcache */
221 ld r7,PPC64_CACHES@toc(r2)
222 clrrdi r3,r3,PAGE_SHIFT /* Page align */
223 lwz r4,DCACHEL1LINESPERPAGE(r7) /* Get # dcache lines per page */
224 lwz r5,DCACHEL1LINESIZE(r7) /* Get dcache line size */
225 mr r6,r3
226 mtctr r4
227 0: dcbst 0,r6
228 add r6,r6,r5
229 bdnz 0b
230 sync
232 /* Now invalidate the icache */
234 lwz r4,ICACHEL1LINESPERPAGE(r7) /* Get # icache lines per page */
235 lwz r5,ICACHEL1LINESIZE(r7) /* Get icache line size */
236 mtctr r4
237 1: icbi 0,r3
238 add r3,r3,r5
239 bdnz 1b
240 isync
241 blr
243 _GLOBAL(__bswapdi2)
244 srdi r8,r3,32
245 rlwinm r7,r3,8,0xffffffff
246 rlwimi r7,r3,24,0,7
247 rlwinm r9,r8,8,0xffffffff
248 rlwimi r7,r3,24,16,23
249 rlwimi r9,r8,24,0,7
250 rlwimi r9,r8,24,16,23
251 sldi r7,r7,32
252 or r3,r7,r9
253 blr
256 #ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX
257 _GLOBAL(rmci_on)
258 sync
259 isync
260 li r3,0x100
261 rldicl r3,r3,32,0
262 mfspr r5,SPRN_HID4
263 or r5,r5,r3
264 sync
265 mtspr SPRN_HID4,r5
266 isync
267 slbia
268 isync
269 sync
270 blr
272 _GLOBAL(rmci_off)
273 sync
274 isync
275 li r3,0x100
276 rldicl r3,r3,32,0
277 mfspr r5,SPRN_HID4
278 andc r5,r5,r3
279 sync
280 mtspr SPRN_HID4,r5
281 isync
282 slbia
283 isync
284 sync
285 blr
286 #endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */
288 #if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE)
290 /*
291 * Do an IO access in real mode
292 */
293 _GLOBAL(real_readb)
294 mfmsr r7
295 ori r0,r7,MSR_DR
296 xori r0,r0,MSR_DR
297 sync
298 mtmsrd r0
299 sync
300 isync
301 mfspr r6,SPRN_HID4
302 rldicl r5,r6,32,0
303 ori r5,r5,0x100
304 rldicl r5,r5,32,0
305 sync
306 mtspr SPRN_HID4,r5
307 isync
308 slbia
309 isync
310 lbz r3,0(r3)
311 sync
312 mtspr SPRN_HID4,r6
313 isync
314 slbia
315 isync
316 mtmsrd r7
317 sync
318 isync
319 blr
321 /*
322 * Do an IO access in real mode
323 */
324 _GLOBAL(real_writeb)
325 mfmsr r7
326 ori r0,r7,MSR_DR
327 xori r0,r0,MSR_DR
328 sync
329 mtmsrd r0
330 sync
331 isync
332 mfspr r6,SPRN_HID4
333 rldicl r5,r6,32,0
334 ori r5,r5,0x100
335 rldicl r5,r5,32,0
336 sync
337 mtspr SPRN_HID4,r5
338 isync
339 slbia
340 isync
341 stb r3,0(r4)
342 sync
343 mtspr SPRN_HID4,r6
344 isync
345 slbia
346 isync
347 mtmsrd r7
348 sync
349 isync
350 blr
351 #endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */
353 #ifdef CONFIG_PPC_PASEMI
355 _GLOBAL(real_205_readb)
356 mfmsr r7
357 ori r0,r7,MSR_DR
358 xori r0,r0,MSR_DR
359 sync
360 mtmsrd r0
361 sync
362 isync
363 LBZCIX(R3,R0,R3)
364 isync
365 mtmsrd r7
366 sync
367 isync
368 blr
370 _GLOBAL(real_205_writeb)
371 mfmsr r7
372 ori r0,r7,MSR_DR
373 xori r0,r0,MSR_DR
374 sync
375 mtmsrd r0
376 sync
377 isync
378 STBCIX(R3,R0,R4)
379 isync
380 mtmsrd r7
381 sync
382 isync
383 blr
385 #endif /* CONFIG_PPC_PASEMI */
388 #if defined(CONFIG_CPU_FREQ_PMAC64) || defined(CONFIG_CPU_FREQ_MAPLE)
389 /*
390 * SCOM access functions for 970 (FX only for now)
391 *
392 * unsigned long scom970_read(unsigned int address);
393 * void scom970_write(unsigned int address, unsigned long value);
394 *
395 * The address passed in is the 24 bits register address. This code
396 * is 970 specific and will not check the status bits, so you should
397 * know what you are doing.
398 */
399 _GLOBAL(scom970_read)
400 /* interrupts off */
401 mfmsr r4
402 ori r0,r4,MSR_EE
403 xori r0,r0,MSR_EE
404 mtmsrd r0,1
406 /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
407 * (including parity). On current CPUs they must be 0'd,
408 * and finally or in RW bit
409 */
410 rlwinm r3,r3,8,0,15
411 ori r3,r3,0x8000
413 /* do the actual scom read */
414 sync
415 mtspr SPRN_SCOMC,r3
416 isync
417 mfspr r3,SPRN_SCOMD
418 isync
419 mfspr r0,SPRN_SCOMC
420 isync
422 /* XXX: fixup result on some buggy 970's (ouch ! we lost a bit, bah
423 * that's the best we can do). Not implemented yet as we don't use
424 * the scom on any of the bogus CPUs yet, but may have to be done
425 * ultimately
426 */
428 /* restore interrupts */
429 mtmsrd r4,1
430 blr
433 _GLOBAL(scom970_write)
434 /* interrupts off */
435 mfmsr r5
436 ori r0,r5,MSR_EE
437 xori r0,r0,MSR_EE
438 mtmsrd r0,1
440 /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
441 * (including parity). On current CPUs they must be 0'd.
442 */
444 rlwinm r3,r3,8,0,15
446 sync
447 mtspr SPRN_SCOMD,r4 /* write data */
448 isync
449 mtspr SPRN_SCOMC,r3 /* write command */
450 isync
451 mfspr 3,SPRN_SCOMC
452 isync
454 /* restore interrupts */
455 mtmsrd r5,1
456 blr
457 #endif /* CONFIG_CPU_FREQ_PMAC64 || CONFIG_CPU_FREQ_MAPLE */
459 /* kexec_wait(phys_cpu)
460 *
461 * wait for the flag to change, indicating this kernel is going away but
462 * the slave code for the next one is at addresses 0 to 100.
463 *
464 * This is used by all slaves, even those that did not find a matching
465 * paca in the secondary startup code.
466 *
467 * Physical (hardware) cpu id should be in r3.
468 */
469 _GLOBAL(kexec_wait)
470 bl 1f
471 1: mflr r5
472 addi r5,r5,kexec_flag-1b
474 99: HMT_LOW
475 #ifdef CONFIG_KEXEC /* use no memory without kexec */
476 lwz r4,0(r5)
477 cmpwi 0,r4,0
478 bnea 0x60
479 #endif
480 b 99b
482 /* this can be in text because we won't change it until we are
483 * running in real anyways
484 */
485 kexec_flag:
486 .long 0
489 #ifdef CONFIG_KEXEC
491 /* kexec_smp_wait(void)
492 *
493 * call with interrupts off
494 * note: this is a terminal routine, it does not save lr
495 *
496 * get phys id from paca
497 * switch to real mode
498 * mark the paca as no longer used
499 * join other cpus in kexec_wait(phys_id)
500 */
501 _GLOBAL(kexec_smp_wait)
502 lhz r3,PACAHWCPUID(r13)
503 bl real_mode
505 li r4,KEXEC_STATE_REAL_MODE
506 stb r4,PACAKEXECSTATE(r13)
507 SYNC
509 b kexec_wait
511 /*
512 * switch to real mode (turn mmu off)
513 * we use the early kernel trick that the hardware ignores bits
514 * 0 and 1 (big endian) of the effective address in real mode
515 *
516 * don't overwrite r3 here, it is live for kexec_wait above.
517 */
518 real_mode: /* assume normal blr return */
519 1: li r9,MSR_RI
520 li r10,MSR_DR|MSR_IR
521 mflr r11 /* return address to SRR0 */
522 mfmsr r12
523 andc r9,r12,r9
524 andc r10,r12,r10
526 mtmsrd r9,1
527 mtspr SPRN_SRR1,r10
528 mtspr SPRN_SRR0,r11
529 rfid
532 /*
533 * kexec_sequence(newstack, start, image, control, clear_all())
534 *
535 * does the grungy work with stack switching and real mode switches
536 * also does simple calls to other code
537 */
539 _GLOBAL(kexec_sequence)
540 mflr r0
541 std r0,16(r1)
543 /* switch stacks to newstack -- &kexec_stack.stack */
544 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
545 mr r1,r3
547 li r0,0
548 std r0,16(r1)
550 /* save regs for local vars on new stack.
551 * yes, we won't go back, but ...
552 */
553 std r31,-8(r1)
554 std r30,-16(r1)
555 std r29,-24(r1)
556 std r28,-32(r1)
557 std r27,-40(r1)
558 std r26,-48(r1)
559 std r25,-56(r1)
561 stdu r1,-STACK_FRAME_OVERHEAD-64(r1)
563 /* save args into preserved regs */
564 mr r31,r3 /* newstack (both) */
565 mr r30,r4 /* start (real) */
566 mr r29,r5 /* image (virt) */
567 mr r28,r6 /* control, unused */
568 mr r27,r7 /* clear_all() fn desc */
569 mr r26,r8 /* spare */
570 lhz r25,PACAHWCPUID(r13) /* get our phys cpu from paca */
572 /* disable interrupts, we are overwriting kernel data next */
573 mfmsr r3
574 rlwinm r3,r3,0,17,15
575 mtmsrd r3,1
577 /* copy dest pages, flush whole dest image */
578 mr r3,r29
579 bl kexec_copy_flush /* (image) */
581 /* turn off mmu */
582 bl real_mode
584 /* copy 0x100 bytes starting at start to 0 */
585 li r3,0
586 mr r4,r30 /* start, aka phys mem offset */
587 li r5,0x100
588 li r6,0
589 bl copy_and_flush /* (dest, src, copy limit, start offset) */
590 1: /* assume normal blr return */
592 /* release other cpus to the new kernel secondary start at 0x60 */
593 mflr r5
594 li r6,1
595 stw r6,kexec_flag-1b(5)
597 /* clear out hardware hash page table and tlb */
598 #if !defined(_CALL_ELF) || _CALL_ELF != 2
599 ld r12,0(r27) /* deref function descriptor */
600 #else
601 mr r12,r27
602 #endif
603 mtctr r12
604 bctrl /* ppc_md.hpte_clear_all(void); */
606 /*
607 * kexec image calling is:
608 * the first 0x100 bytes of the entry point are copied to 0
609 *
610 * all slaves branch to slave = 0x60 (absolute)
611 * slave(phys_cpu_id);
612 *
613 * master goes to start = entry point
614 * start(phys_cpu_id, start, 0);
615 *
616 *
617 * a wrapper is needed to call existing kernels, here is an approximate
618 * description of one method:
619 *
620 * v2: (2.6.10)
621 * start will be near the boot_block (maybe 0x100 bytes before it?)
622 * it will have a 0x60, which will b to boot_block, where it will wait
623 * and 0 will store phys into struct boot-block and load r3 from there,
624 * copy kernel 0-0x100 and tell slaves to back down to 0x60 again
625 *
626 * v1: (2.6.9)
627 * boot block will have all cpus scanning device tree to see if they
628 * are the boot cpu ?????
629 * other device tree differences (prop sizes, va vs pa, etc)...
630 */
631 mr r3,r25 # my phys cpu
632 mr r4,r30 # start, aka phys mem offset
633 mtlr 4
634 li r5,0
635 blr /* image->start(physid, image->start, 0); */
636 #endif /* CONFIG_KEXEC */
638 #ifdef CONFIG_MODULES
639 #if defined(_CALL_ELF) && _CALL_ELF == 2
641 #ifdef CONFIG_MODVERSIONS
642 .weak __crc_TOC.
643 .section "___kcrctab+TOC.","a"
644 .globl __kcrctab_TOC.
645 __kcrctab_TOC.:
646 .llong __crc_TOC.
647 #endif
649 /*
650 * Export a fake .TOC. since both modpost and depmod will complain otherwise.
651 * Both modpost and depmod strip the leading . so we do the same here.
652 */
653 .section "__ksymtab_strings","a"
654 __kstrtab_TOC.:
655 .asciz "TOC."
657 .section "___ksymtab+TOC.","a"
658 /* This symbol name is important: it's used by modpost to find exported syms */
659 .globl __ksymtab_TOC.
660 __ksymtab_TOC.:
661 .llong 0 /* .value */
662 .llong __kstrtab_TOC.
663 #endif /* ELFv2 */
664 #endif /* MODULES */