| blob | f8e971ba94f5b66231da5194f125efd35020b5d6 |
1 /*
2 * Kernel execution entry point code.
3 *
4 * Copyright (c) 1995-1996 Gary Thomas <gdt@linuxppc.org>
5 * Initial PowerPC version.
6 * Copyright (c) 1996 Cort Dougan <cort@cs.nmt.edu>
7 * Rewritten for PReP
8 * Copyright (c) 1996 Paul Mackerras <paulus@cs.anu.edu.au>
9 * Low-level exception handers, MMU support, and rewrite.
10 * Copyright (c) 1997 Dan Malek <dmalek@jlc.net>
11 * PowerPC 8xx modifications.
12 * Copyright (c) 1998-1999 TiVo, Inc.
13 * PowerPC 403GCX modifications.
14 * Copyright (c) 1999 Grant Erickson <grant@lcse.umn.edu>
15 * PowerPC 403GCX/405GP modifications.
16 * Copyright 2000 MontaVista Software Inc.
17 * PPC405 modifications
18 * PowerPC 403GCX/405GP modifications.
19 * Author: MontaVista Software, Inc.
20 * frank_rowand@mvista.com or source@mvista.com
21 * debbie_chu@mvista.com
22 * Copyright 2002-2005 MontaVista Software, Inc.
23 * PowerPC 44x support, Matt Porter <mporter@kernel.crashing.org>
24 *
25 * This program is free software; you can redistribute it and/or modify it
26 * under the terms of the GNU General Public License as published by the
27 * Free Software Foundation; either version 2 of the License, or (at your
28 * option) any later version.
29 */
31 #include <linux/init.h>
32 #include <asm/processor.h>
33 #include <asm/page.h>
34 #include <asm/mmu.h>
35 #include <asm/pgtable.h>
36 #include <asm/cputable.h>
37 #include <asm/thread_info.h>
38 #include <asm/ppc_asm.h>
39 #include <asm/asm-offsets.h>
40 #include <asm/ptrace.h>
41 #include <asm/synch.h>
42 #include "head_booke.h"
45 /* As with the other PowerPC ports, it is expected that when code
46 * execution begins here, the following registers contain valid, yet
47 * optional, information:
48 *
49 * r3 - Board info structure pointer (DRAM, frequency, MAC address, etc.)
50 * r4 - Starting address of the init RAM disk
51 * r5 - Ending address of the init RAM disk
52 * r6 - Start of kernel command line string (e.g. "mem=128")
53 * r7 - End of kernel command line string
54 *
55 */
56 __HEAD
57 _ENTRY(_stext);
58 _ENTRY(_start);
59 /*
60 * Reserve a word at a fixed location to store the address
61 * of abatron_pteptrs
62 */
63 nop
64 /*
65 * Save parameters we are passed
66 */
67 mr r31,r3
68 mr r30,r4
69 mr r29,r5
70 mr r28,r6
71 mr r27,r7
72 li r24,0 /* CPU number */
74 bl init_cpu_state
76 /*
77 * This is where the main kernel code starts.
78 */
80 /* ptr to current */
81 lis r2,init_task@h
82 ori r2,r2,init_task@l
84 /* ptr to current thread */
85 addi r4,r2,THREAD /* init task's THREAD */
86 mtspr SPRN_SPRG_THREAD,r4
88 /* stack */
89 lis r1,init_thread_union@h
90 ori r1,r1,init_thread_union@l
91 li r0,0
92 stwu r0,THREAD_SIZE-STACK_FRAME_OVERHEAD(r1)
94 bl early_init
96 #ifdef CONFIG_RELOCATABLE
97 /*
98 * r25 will contain RPN/ERPN for the start address of memory
99 *
100 * Add the difference between KERNELBASE and PAGE_OFFSET to the
101 * start of physical memory to get kernstart_addr.
102 */
103 lis r3,kernstart_addr@ha
104 la r3,kernstart_addr@l(r3)
106 lis r4,KERNELBASE@h
107 ori r4,r4,KERNELBASE@l
108 lis r5,PAGE_OFFSET@h
109 ori r5,r5,PAGE_OFFSET@l
110 subf r4,r5,r4
112 rlwinm r6,r25,0,28,31 /* ERPN */
113 rlwinm r7,r25,0,0,3 /* RPN - assuming 256 MB page size */
114 add r7,r7,r4
116 stw r6,0(r3)
117 stw r7,4(r3)
118 #endif
120 /*
121 * Decide what sort of machine this is and initialize the MMU.
122 */
123 mr r3,r31
124 mr r4,r30
125 mr r5,r29
126 mr r6,r28
127 mr r7,r27
128 bl machine_init
129 bl MMU_init
131 /* Setup PTE pointers for the Abatron bdiGDB */
132 lis r6, swapper_pg_dir@h
133 ori r6, r6, swapper_pg_dir@l
134 lis r5, abatron_pteptrs@h
135 ori r5, r5, abatron_pteptrs@l
136 lis r4, KERNELBASE@h
137 ori r4, r4, KERNELBASE@l
138 stw r5, 0(r4) /* Save abatron_pteptrs at a fixed location */
139 stw r6, 0(r5)
141 /* Clear the Machine Check Syndrome Register */
142 li r0,0
143 mtspr SPRN_MCSR,r0
145 /* Let's move on */
146 lis r4,start_kernel@h
147 ori r4,r4,start_kernel@l
148 lis r3,MSR_KERNEL@h
149 ori r3,r3,MSR_KERNEL@l
150 mtspr SPRN_SRR0,r4
151 mtspr SPRN_SRR1,r3
152 rfi /* change context and jump to start_kernel */
154 /*
155 * Interrupt vector entry code
156 *
157 * The Book E MMUs are always on so we don't need to handle
158 * interrupts in real mode as with previous PPC processors. In
159 * this case we handle interrupts in the kernel virtual address
160 * space.
161 *
162 * Interrupt vectors are dynamically placed relative to the
163 * interrupt prefix as determined by the address of interrupt_base.
164 * The interrupt vectors offsets are programmed using the labels
165 * for each interrupt vector entry.
166 *
167 * Interrupt vectors must be aligned on a 16 byte boundary.
168 * We align on a 32 byte cache line boundary for good measure.
169 */
171 interrupt_base:
172 /* Critical Input Interrupt */
173 CRITICAL_EXCEPTION(0x0100, CriticalInput, unknown_exception)
175 /* Machine Check Interrupt */
176 CRITICAL_EXCEPTION(0x0200, MachineCheck, machine_check_exception)
177 MCHECK_EXCEPTION(0x0210, MachineCheckA, machine_check_exception)
179 /* Data Storage Interrupt */
180 DATA_STORAGE_EXCEPTION
182 /* Instruction Storage Interrupt */
183 INSTRUCTION_STORAGE_EXCEPTION
185 /* External Input Interrupt */
186 EXCEPTION(0x0500, ExternalInput, do_IRQ, EXC_XFER_LITE)
188 /* Alignment Interrupt */
189 ALIGNMENT_EXCEPTION
191 /* Program Interrupt */
192 PROGRAM_EXCEPTION
194 /* Floating Point Unavailable Interrupt */
195 #ifdef CONFIG_PPC_FPU
196 FP_UNAVAILABLE_EXCEPTION
197 #else
198 EXCEPTION(0x2010, FloatingPointUnavailable, unknown_exception, EXC_XFER_EE)
199 #endif
200 /* System Call Interrupt */
201 START_EXCEPTION(SystemCall)
202 NORMAL_EXCEPTION_PROLOG
203 EXC_XFER_EE_LITE(0x0c00, DoSyscall)
205 /* Auxiliary Processor Unavailable Interrupt */
206 EXCEPTION(0x2020, AuxillaryProcessorUnavailable, unknown_exception, EXC_XFER_EE)
208 /* Decrementer Interrupt */
209 DECREMENTER_EXCEPTION
211 /* Fixed Internal Timer Interrupt */
212 /* TODO: Add FIT support */
213 EXCEPTION(0x1010, FixedIntervalTimer, unknown_exception, EXC_XFER_EE)
215 /* Watchdog Timer Interrupt */
216 /* TODO: Add watchdog support */
217 #ifdef CONFIG_BOOKE_WDT
218 CRITICAL_EXCEPTION(0x1020, WatchdogTimer, WatchdogException)
219 #else
220 CRITICAL_EXCEPTION(0x1020, WatchdogTimer, unknown_exception)
221 #endif
223 /* Data TLB Error Interrupt */
224 START_EXCEPTION(DataTLBError44x)
225 mtspr SPRN_SPRG_WSCRATCH0, r10 /* Save some working registers */
226 mtspr SPRN_SPRG_WSCRATCH1, r11
227 mtspr SPRN_SPRG_WSCRATCH2, r12
228 mtspr SPRN_SPRG_WSCRATCH3, r13
229 mfcr r11
230 mtspr SPRN_SPRG_WSCRATCH4, r11
231 mfspr r10, SPRN_DEAR /* Get faulting address */
233 /* If we are faulting a kernel address, we have to use the
234 * kernel page tables.
235 */
236 lis r11, PAGE_OFFSET@h
237 cmplw r10, r11
238 blt+ 3f
239 lis r11, swapper_pg_dir@h
240 ori r11, r11, swapper_pg_dir@l
242 mfspr r12,SPRN_MMUCR
243 rlwinm r12,r12,0,0,23 /* Clear TID */
245 b 4f
247 /* Get the PGD for the current thread */
248 3:
249 mfspr r11,SPRN_SPRG_THREAD
250 lwz r11,PGDIR(r11)
252 /* Load PID into MMUCR TID */
253 mfspr r12,SPRN_MMUCR
254 mfspr r13,SPRN_PID /* Get PID */
255 rlwimi r12,r13,0,24,31 /* Set TID */
257 4:
258 mtspr SPRN_MMUCR,r12
260 /* Mask of required permission bits. Note that while we
261 * do copy ESR:ST to _PAGE_RW position as trying to write
262 * to an RO page is pretty common, we don't do it with
263 * _PAGE_DIRTY. We could do it, but it's a fairly rare
264 * event so I'd rather take the overhead when it happens
265 * rather than adding an instruction here. We should measure
266 * whether the whole thing is worth it in the first place
267 * as we could avoid loading SPRN_ESR completely in the first
268 * place...
269 *
270 * TODO: Is it worth doing that mfspr & rlwimi in the first
271 * place or can we save a couple of instructions here ?
272 */
273 mfspr r12,SPRN_ESR
274 li r13,_PAGE_PRESENT|_PAGE_ACCESSED
275 rlwimi r13,r12,10,30,30
277 /* Load the PTE */
278 /* Compute pgdir/pmd offset */
279 rlwinm r12, r10, PPC44x_PGD_OFF_SHIFT, PPC44x_PGD_OFF_MASK_BIT, 29
280 lwzx r11, r12, r11 /* Get pgd/pmd entry */
281 rlwinm. r12, r11, 0, 0, 20 /* Extract pt base address */
282 beq 2f /* Bail if no table */
284 /* Compute pte address */
285 rlwimi r12, r10, PPC44x_PTE_ADD_SHIFT, PPC44x_PTE_ADD_MASK_BIT, 28
286 lwz r11, 0(r12) /* Get high word of pte entry */
287 lwz r12, 4(r12) /* Get low word of pte entry */
289 lis r10,tlb_44x_index@ha
291 andc. r13,r13,r12 /* Check permission */
293 /* Load the next available TLB index */
294 lwz r13,tlb_44x_index@l(r10)
296 bne 2f /* Bail if permission mismach */
298 /* Increment, rollover, and store TLB index */
299 addi r13,r13,1
301 /* Compare with watermark (instruction gets patched) */
302 .globl tlb_44x_patch_hwater_D
303 tlb_44x_patch_hwater_D:
304 cmpwi 0,r13,1 /* reserve entries */
305 ble 5f
306 li r13,0
307 5:
308 /* Store the next available TLB index */
309 stw r13,tlb_44x_index@l(r10)
311 /* Re-load the faulting address */
312 mfspr r10,SPRN_DEAR
314 /* Jump to common tlb load */
315 b finish_tlb_load_44x
317 2:
318 /* The bailout. Restore registers to pre-exception conditions
319 * and call the heavyweights to help us out.
320 */
321 mfspr r11, SPRN_SPRG_RSCRATCH4
322 mtcr r11
323 mfspr r13, SPRN_SPRG_RSCRATCH3
324 mfspr r12, SPRN_SPRG_RSCRATCH2
325 mfspr r11, SPRN_SPRG_RSCRATCH1
326 mfspr r10, SPRN_SPRG_RSCRATCH0
327 b DataStorage
329 /* Instruction TLB Error Interrupt */
330 /*
331 * Nearly the same as above, except we get our
332 * information from different registers and bailout
333 * to a different point.
334 */
335 START_EXCEPTION(InstructionTLBError44x)
336 mtspr SPRN_SPRG_WSCRATCH0, r10 /* Save some working registers */
337 mtspr SPRN_SPRG_WSCRATCH1, r11
338 mtspr SPRN_SPRG_WSCRATCH2, r12
339 mtspr SPRN_SPRG_WSCRATCH3, r13
340 mfcr r11
341 mtspr SPRN_SPRG_WSCRATCH4, r11
342 mfspr r10, SPRN_SRR0 /* Get faulting address */
344 /* If we are faulting a kernel address, we have to use the
345 * kernel page tables.
346 */
347 lis r11, PAGE_OFFSET@h
348 cmplw r10, r11
349 blt+ 3f
350 lis r11, swapper_pg_dir@h
351 ori r11, r11, swapper_pg_dir@l
353 mfspr r12,SPRN_MMUCR
354 rlwinm r12,r12,0,0,23 /* Clear TID */
356 b 4f
358 /* Get the PGD for the current thread */
359 3:
360 mfspr r11,SPRN_SPRG_THREAD
361 lwz r11,PGDIR(r11)
363 /* Load PID into MMUCR TID */
364 mfspr r12,SPRN_MMUCR
365 mfspr r13,SPRN_PID /* Get PID */
366 rlwimi r12,r13,0,24,31 /* Set TID */
368 4:
369 mtspr SPRN_MMUCR,r12
371 /* Make up the required permissions */
372 li r13,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC
374 /* Compute pgdir/pmd offset */
375 rlwinm r12, r10, PPC44x_PGD_OFF_SHIFT, PPC44x_PGD_OFF_MASK_BIT, 29
376 lwzx r11, r12, r11 /* Get pgd/pmd entry */
377 rlwinm. r12, r11, 0, 0, 20 /* Extract pt base address */
378 beq 2f /* Bail if no table */
380 /* Compute pte address */
381 rlwimi r12, r10, PPC44x_PTE_ADD_SHIFT, PPC44x_PTE_ADD_MASK_BIT, 28
382 lwz r11, 0(r12) /* Get high word of pte entry */
383 lwz r12, 4(r12) /* Get low word of pte entry */
385 lis r10,tlb_44x_index@ha
387 andc. r13,r13,r12 /* Check permission */
389 /* Load the next available TLB index */
390 lwz r13,tlb_44x_index@l(r10)
392 bne 2f /* Bail if permission mismach */
394 /* Increment, rollover, and store TLB index */
395 addi r13,r13,1
397 /* Compare with watermark (instruction gets patched) */
398 .globl tlb_44x_patch_hwater_I
399 tlb_44x_patch_hwater_I:
400 cmpwi 0,r13,1 /* reserve entries */
401 ble 5f
402 li r13,0
403 5:
404 /* Store the next available TLB index */
405 stw r13,tlb_44x_index@l(r10)
407 /* Re-load the faulting address */
408 mfspr r10,SPRN_SRR0
410 /* Jump to common TLB load point */
411 b finish_tlb_load_44x
413 2:
414 /* The bailout. Restore registers to pre-exception conditions
415 * and call the heavyweights to help us out.
416 */
417 mfspr r11, SPRN_SPRG_RSCRATCH4
418 mtcr r11
419 mfspr r13, SPRN_SPRG_RSCRATCH3
420 mfspr r12, SPRN_SPRG_RSCRATCH2
421 mfspr r11, SPRN_SPRG_RSCRATCH1
422 mfspr r10, SPRN_SPRG_RSCRATCH0
423 b InstructionStorage
425 /*
426 * Both the instruction and data TLB miss get to this
427 * point to load the TLB.
428 * r10 - EA of fault
429 * r11 - PTE high word value
430 * r12 - PTE low word value
431 * r13 - TLB index
432 * MMUCR - loaded with proper value when we get here
433 * Upon exit, we reload everything and RFI.
434 */
435 finish_tlb_load_44x:
436 /* Combine RPN & ERPN an write WS 0 */
437 rlwimi r11,r12,0,0,31-PAGE_SHIFT
438 tlbwe r11,r13,PPC44x_TLB_XLAT
440 /*
441 * Create WS1. This is the faulting address (EPN),
442 * page size, and valid flag.
443 */
444 li r11,PPC44x_TLB_VALID | PPC44x_TLBE_SIZE
445 /* Insert valid and page size */
446 rlwimi r10,r11,0,PPC44x_PTE_ADD_MASK_BIT,31
447 tlbwe r10,r13,PPC44x_TLB_PAGEID /* Write PAGEID */
449 /* And WS 2 */
450 li r10,0xf85 /* Mask to apply from PTE */
451 rlwimi r10,r12,29,30,30 /* DIRTY -> SW position */
452 and r11,r12,r10 /* Mask PTE bits to keep */
453 andi. r10,r12,_PAGE_USER /* User page ? */
454 beq 1f /* nope, leave U bits empty */
455 rlwimi r11,r11,3,26,28 /* yes, copy S bits to U */
456 1: tlbwe r11,r13,PPC44x_TLB_ATTRIB /* Write ATTRIB */
458 /* Done...restore registers and get out of here.
459 */
460 mfspr r11, SPRN_SPRG_RSCRATCH4
461 mtcr r11
462 mfspr r13, SPRN_SPRG_RSCRATCH3
463 mfspr r12, SPRN_SPRG_RSCRATCH2
464 mfspr r11, SPRN_SPRG_RSCRATCH1
465 mfspr r10, SPRN_SPRG_RSCRATCH0
466 rfi /* Force context change */
468 /* TLB error interrupts for 476
469 */
470 #ifdef CONFIG_PPC_47x
471 START_EXCEPTION(DataTLBError47x)
472 mtspr SPRN_SPRG_WSCRATCH0,r10 /* Save some working registers */
473 mtspr SPRN_SPRG_WSCRATCH1,r11
474 mtspr SPRN_SPRG_WSCRATCH2,r12
475 mtspr SPRN_SPRG_WSCRATCH3,r13
476 mfcr r11
477 mtspr SPRN_SPRG_WSCRATCH4,r11
478 mfspr r10,SPRN_DEAR /* Get faulting address */
480 /* If we are faulting a kernel address, we have to use the
481 * kernel page tables.
482 */
483 lis r11,PAGE_OFFSET@h
484 cmplw cr0,r10,r11
485 blt+ 3f
486 lis r11,swapper_pg_dir@h
487 ori r11,r11, swapper_pg_dir@l
488 li r12,0 /* MMUCR = 0 */
489 b 4f
491 /* Get the PGD for the current thread and setup MMUCR */
492 3: mfspr r11,SPRN_SPRG3
493 lwz r11,PGDIR(r11)
494 mfspr r12,SPRN_PID /* Get PID */
495 4: mtspr SPRN_MMUCR,r12 /* Set MMUCR */
497 /* Mask of required permission bits. Note that while we
498 * do copy ESR:ST to _PAGE_RW position as trying to write
499 * to an RO page is pretty common, we don't do it with
500 * _PAGE_DIRTY. We could do it, but it's a fairly rare
501 * event so I'd rather take the overhead when it happens
502 * rather than adding an instruction here. We should measure
503 * whether the whole thing is worth it in the first place
504 * as we could avoid loading SPRN_ESR completely in the first
505 * place...
506 *
507 * TODO: Is it worth doing that mfspr & rlwimi in the first
508 * place or can we save a couple of instructions here ?
509 */
510 mfspr r12,SPRN_ESR
511 li r13,_PAGE_PRESENT|_PAGE_ACCESSED
512 rlwimi r13,r12,10,30,30
514 /* Load the PTE */
515 /* Compute pgdir/pmd offset */
516 rlwinm r12,r10,PPC44x_PGD_OFF_SHIFT,PPC44x_PGD_OFF_MASK_BIT,29
517 lwzx r11,r12,r11 /* Get pgd/pmd entry */
519 /* Word 0 is EPN,V,TS,DSIZ */
520 li r12,PPC47x_TLB0_VALID | PPC47x_TLBE_SIZE
521 rlwimi r10,r12,0,32-PAGE_SHIFT,31 /* Insert valid and page size*/
522 li r12,0
523 tlbwe r10,r12,0
525 /* XXX can we do better ? Need to make sure tlbwe has established
526 * latch V bit in MMUCR0 before the PTE is loaded further down */
527 #ifdef CONFIG_SMP
528 isync
529 #endif
531 rlwinm. r12,r11,0,0,20 /* Extract pt base address */
532 /* Compute pte address */
533 rlwimi r12,r10,PPC44x_PTE_ADD_SHIFT,PPC44x_PTE_ADD_MASK_BIT,28
534 beq 2f /* Bail if no table */
535 lwz r11,0(r12) /* Get high word of pte entry */
537 /* XXX can we do better ? maybe insert a known 0 bit from r11 into the
538 * bottom of r12 to create a data dependency... We can also use r10
539 * as destination nowadays
540 */
541 #ifdef CONFIG_SMP
542 lwsync
543 #endif
544 lwz r12,4(r12) /* Get low word of pte entry */
546 andc. r13,r13,r12 /* Check permission */
548 /* Jump to common tlb load */
549 beq finish_tlb_load_47x
551 2: /* The bailout. Restore registers to pre-exception conditions
552 * and call the heavyweights to help us out.
553 */
554 mfspr r11,SPRN_SPRG_RSCRATCH4
555 mtcr r11
556 mfspr r13,SPRN_SPRG_RSCRATCH3
557 mfspr r12,SPRN_SPRG_RSCRATCH2
558 mfspr r11,SPRN_SPRG_RSCRATCH1
559 mfspr r10,SPRN_SPRG_RSCRATCH0
560 b DataStorage
562 /* Instruction TLB Error Interrupt */
563 /*
564 * Nearly the same as above, except we get our
565 * information from different registers and bailout
566 * to a different point.
567 */
568 START_EXCEPTION(InstructionTLBError47x)
569 mtspr SPRN_SPRG_WSCRATCH0,r10 /* Save some working registers */
570 mtspr SPRN_SPRG_WSCRATCH1,r11
571 mtspr SPRN_SPRG_WSCRATCH2,r12
572 mtspr SPRN_SPRG_WSCRATCH3,r13
573 mfcr r11
574 mtspr SPRN_SPRG_WSCRATCH4,r11
575 mfspr r10,SPRN_SRR0 /* Get faulting address */
577 /* If we are faulting a kernel address, we have to use the
578 * kernel page tables.
579 */
580 lis r11,PAGE_OFFSET@h
581 cmplw cr0,r10,r11
582 blt+ 3f
583 lis r11,swapper_pg_dir@h
584 ori r11,r11, swapper_pg_dir@l
585 li r12,0 /* MMUCR = 0 */
586 b 4f
588 /* Get the PGD for the current thread and setup MMUCR */
589 3: mfspr r11,SPRN_SPRG_THREAD
590 lwz r11,PGDIR(r11)
591 mfspr r12,SPRN_PID /* Get PID */
592 4: mtspr SPRN_MMUCR,r12 /* Set MMUCR */
594 /* Make up the required permissions */
595 li r13,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC
597 /* Load PTE */
598 /* Compute pgdir/pmd offset */
599 rlwinm r12,r10,PPC44x_PGD_OFF_SHIFT,PPC44x_PGD_OFF_MASK_BIT,29
600 lwzx r11,r12,r11 /* Get pgd/pmd entry */
602 /* Word 0 is EPN,V,TS,DSIZ */
603 li r12,PPC47x_TLB0_VALID | PPC47x_TLBE_SIZE
604 rlwimi r10,r12,0,32-PAGE_SHIFT,31 /* Insert valid and page size*/
605 li r12,0
606 tlbwe r10,r12,0
608 /* XXX can we do better ? Need to make sure tlbwe has established
609 * latch V bit in MMUCR0 before the PTE is loaded further down */
610 #ifdef CONFIG_SMP
611 isync
612 #endif
614 rlwinm. r12,r11,0,0,20 /* Extract pt base address */
615 /* Compute pte address */
616 rlwimi r12,r10,PPC44x_PTE_ADD_SHIFT,PPC44x_PTE_ADD_MASK_BIT,28
617 beq 2f /* Bail if no table */
619 lwz r11,0(r12) /* Get high word of pte entry */
620 /* XXX can we do better ? maybe insert a known 0 bit from r11 into the
621 * bottom of r12 to create a data dependency... We can also use r10
622 * as destination nowadays
623 */
624 #ifdef CONFIG_SMP
625 lwsync
626 #endif
627 lwz r12,4(r12) /* Get low word of pte entry */
629 andc. r13,r13,r12 /* Check permission */
631 /* Jump to common TLB load point */
632 beq finish_tlb_load_47x
634 2: /* The bailout. Restore registers to pre-exception conditions
635 * and call the heavyweights to help us out.
636 */
637 mfspr r11, SPRN_SPRG_RSCRATCH4
638 mtcr r11
639 mfspr r13, SPRN_SPRG_RSCRATCH3
640 mfspr r12, SPRN_SPRG_RSCRATCH2
641 mfspr r11, SPRN_SPRG_RSCRATCH1
642 mfspr r10, SPRN_SPRG_RSCRATCH0
643 b InstructionStorage
645 /*
646 * Both the instruction and data TLB miss get to this
647 * point to load the TLB.
648 * r10 - free to use
649 * r11 - PTE high word value
650 * r12 - PTE low word value
651 * r13 - free to use
652 * MMUCR - loaded with proper value when we get here
653 * Upon exit, we reload everything and RFI.
654 */
655 finish_tlb_load_47x:
656 /* Combine RPN & ERPN an write WS 1 */
657 rlwimi r11,r12,0,0,31-PAGE_SHIFT
658 tlbwe r11,r13,1
660 /* And make up word 2 */
661 li r10,0xf85 /* Mask to apply from PTE */
662 rlwimi r10,r12,29,30,30 /* DIRTY -> SW position */
663 and r11,r12,r10 /* Mask PTE bits to keep */
664 andi. r10,r12,_PAGE_USER /* User page ? */
665 beq 1f /* nope, leave U bits empty */
666 rlwimi r11,r11,3,26,28 /* yes, copy S bits to U */
667 1: tlbwe r11,r13,2
669 /* Done...restore registers and get out of here.
670 */
671 mfspr r11, SPRN_SPRG_RSCRATCH4
672 mtcr r11
673 mfspr r13, SPRN_SPRG_RSCRATCH3
674 mfspr r12, SPRN_SPRG_RSCRATCH2
675 mfspr r11, SPRN_SPRG_RSCRATCH1
676 mfspr r10, SPRN_SPRG_RSCRATCH0
677 rfi
679 #endif /* CONFIG_PPC_47x */
681 /* Debug Interrupt */
682 /*
683 * This statement needs to exist at the end of the IVPR
684 * definition just in case you end up taking a debug
685 * exception within another exception.
686 */
687 DEBUG_CRIT_EXCEPTION
689 /*
690 * Global functions
691 */
693 /*
694 * Adjust the machine check IVOR on 440A cores
695 */
696 _GLOBAL(__fixup_440A_mcheck)
697 li r3,MachineCheckA@l
698 mtspr SPRN_IVOR1,r3
699 sync
700 blr
702 /*
703 * extern void giveup_altivec(struct task_struct *prev)
704 *
705 * The 44x core does not have an AltiVec unit.
706 */
707 _GLOBAL(giveup_altivec)
708 blr
710 /*
711 * extern void giveup_fpu(struct task_struct *prev)
712 *
713 * The 44x core does not have an FPU.
714 */
715 #ifndef CONFIG_PPC_FPU
716 _GLOBAL(giveup_fpu)
717 blr
718 #endif
720 _GLOBAL(set_context)
722 #ifdef CONFIG_BDI_SWITCH
723 /* Context switch the PTE pointer for the Abatron BDI2000.
724 * The PGDIR is the second parameter.
725 */
726 lis r5, abatron_pteptrs@h
727 ori r5, r5, abatron_pteptrs@l
728 stw r4, 0x4(r5)
729 #endif
730 mtspr SPRN_PID,r3
731 isync /* Force context change */
732 blr
734 /*
735 * Init CPU state. This is called at boot time or for secondary CPUs
736 * to setup initial TLB entries, setup IVORs, etc...
737 *
738 */
739 _GLOBAL(init_cpu_state)
740 mflr r22
741 #ifdef CONFIG_PPC_47x
742 /* We use the PVR to differenciate 44x cores from 476 */
743 mfspr r3,SPRN_PVR
744 srwi r3,r3,16
745 cmplwi cr0,r3,PVR_476@h
746 beq head_start_47x
747 cmplwi cr0,r3,PVR_476_ISS@h
748 beq head_start_47x
749 #endif /* CONFIG_PPC_47x */
751 /*
752 * In case the firmware didn't do it, we apply some workarounds
753 * that are good for all 440 core variants here
754 */
755 mfspr r3,SPRN_CCR0
756 rlwinm r3,r3,0,0,27 /* disable icache prefetch */
757 isync
758 mtspr SPRN_CCR0,r3
759 isync
760 sync
762 /*
763 * Set up the initial MMU state for 44x
764 *
765 * We are still executing code at the virtual address
766 * mappings set by the firmware for the base of RAM.
767 *
768 * We first invalidate all TLB entries but the one
769 * we are running from. We then load the KERNELBASE
770 * mappings so we can begin to use kernel addresses
771 * natively and so the interrupt vector locations are
772 * permanently pinned (necessary since Book E
773 * implementations always have translation enabled).
774 *
775 * TODO: Use the known TLB entry we are running from to
776 * determine which physical region we are located
777 * in. This can be used to determine where in RAM
778 * (on a shared CPU system) or PCI memory space
779 * (on a DRAMless system) we are located.
780 * For now, we assume a perfect world which means
781 * we are located at the base of DRAM (physical 0).
782 */
784 /*
785 * Search TLB for entry that we are currently using.
786 * Invalidate all entries but the one we are using.
787 */
788 /* Load our current PID->MMUCR TID and MSR IS->MMUCR STS */
789 mfspr r3,SPRN_PID /* Get PID */
790 mfmsr r4 /* Get MSR */
791 andi. r4,r4,MSR_IS@l /* TS=1? */
792 beq wmmucr /* If not, leave STS=0 */
793 oris r3,r3,PPC44x_MMUCR_STS@h /* Set STS=1 */
794 wmmucr: mtspr SPRN_MMUCR,r3 /* Put MMUCR */
795 sync
797 bl invstr /* Find our address */
798 invstr: mflr r5 /* Make it accessible */
799 tlbsx r23,0,r5 /* Find entry we are in */
800 li r4,0 /* Start at TLB entry 0 */
801 li r3,0 /* Set PAGEID inval value */
802 1: cmpw r23,r4 /* Is this our entry? */
803 beq skpinv /* If so, skip the inval */
804 tlbwe r3,r4,PPC44x_TLB_PAGEID /* If not, inval the entry */
805 skpinv: addi r4,r4,1 /* Increment */
806 cmpwi r4,64 /* Are we done? */
807 bne 1b /* If not, repeat */
808 isync /* If so, context change */
810 /*
811 * Configure and load pinned entry into TLB slot 63.
812 */
814 lis r3,PAGE_OFFSET@h
815 ori r3,r3,PAGE_OFFSET@l
817 /* Kernel is at the base of RAM */
818 li r4, 0 /* Load the kernel physical address */
820 /* Load the kernel PID = 0 */
821 li r0,0
822 mtspr SPRN_PID,r0
823 sync
825 /* Initialize MMUCR */
826 li r5,0
827 mtspr SPRN_MMUCR,r5
828 sync
830 /* pageid fields */
831 clrrwi r3,r3,10 /* Mask off the effective page number */
832 ori r3,r3,PPC44x_TLB_VALID | PPC44x_TLB_256M
834 /* xlat fields */
835 clrrwi r4,r4,10 /* Mask off the real page number */
836 /* ERPN is 0 for first 4GB page */
838 /* attrib fields */
839 /* Added guarded bit to protect against speculative loads/stores */
840 li r5,0
841 ori r5,r5,(PPC44x_TLB_SW | PPC44x_TLB_SR | PPC44x_TLB_SX | PPC44x_TLB_G)
843 li r0,63 /* TLB slot 63 */
845 tlbwe r3,r0,PPC44x_TLB_PAGEID /* Load the pageid fields */
846 tlbwe r4,r0,PPC44x_TLB_XLAT /* Load the translation fields */
847 tlbwe r5,r0,PPC44x_TLB_ATTRIB /* Load the attrib/access fields */
849 /* Force context change */
850 mfmsr r0
851 mtspr SPRN_SRR1, r0
852 lis r0,3f@h
853 ori r0,r0,3f@l
854 mtspr SPRN_SRR0,r0
855 sync
856 rfi
858 /* If necessary, invalidate original entry we used */
859 3: cmpwi r23,63
860 beq 4f
861 li r6,0
862 tlbwe r6,r23,PPC44x_TLB_PAGEID
863 isync
865 4:
866 #ifdef CONFIG_PPC_EARLY_DEBUG_44x
867 /* Add UART mapping for early debug. */
869 /* pageid fields */
870 lis r3,PPC44x_EARLY_DEBUG_VIRTADDR@h
871 ori r3,r3,PPC44x_TLB_VALID|PPC44x_TLB_TS|PPC44x_TLB_64K
873 /* xlat fields */
874 lis r4,CONFIG_PPC_EARLY_DEBUG_44x_PHYSLOW@h
875 ori r4,r4,CONFIG_PPC_EARLY_DEBUG_44x_PHYSHIGH
877 /* attrib fields */
878 li r5,(PPC44x_TLB_SW|PPC44x_TLB_SR|PPC44x_TLB_I|PPC44x_TLB_G)
879 li r0,62 /* TLB slot 0 */
881 tlbwe r3,r0,PPC44x_TLB_PAGEID
882 tlbwe r4,r0,PPC44x_TLB_XLAT
883 tlbwe r5,r0,PPC44x_TLB_ATTRIB
885 /* Force context change */
886 isync
887 #endif /* CONFIG_PPC_EARLY_DEBUG_44x */
889 /* Establish the interrupt vector offsets */
890 SET_IVOR(0, CriticalInput);
891 SET_IVOR(1, MachineCheck);
892 SET_IVOR(2, DataStorage);
893 SET_IVOR(3, InstructionStorage);
894 SET_IVOR(4, ExternalInput);
895 SET_IVOR(5, Alignment);
896 SET_IVOR(6, Program);
897 SET_IVOR(7, FloatingPointUnavailable);
898 SET_IVOR(8, SystemCall);
899 SET_IVOR(9, AuxillaryProcessorUnavailable);
900 SET_IVOR(10, Decrementer);
901 SET_IVOR(11, FixedIntervalTimer);
902 SET_IVOR(12, WatchdogTimer);
903 SET_IVOR(13, DataTLBError44x);
904 SET_IVOR(14, InstructionTLBError44x);
905 SET_IVOR(15, DebugCrit);
907 b head_start_common
910 #ifdef CONFIG_PPC_47x
912 #ifdef CONFIG_SMP
914 /* Entry point for secondary 47x processors */
915 _GLOBAL(start_secondary_47x)
916 mr r24,r3 /* CPU number */
918 bl init_cpu_state
920 /* Now we need to bolt the rest of kernel memory which
921 * is done in C code. We must be careful because our task
922 * struct or our stack can (and will probably) be out
923 * of reach of the initial 256M TLB entry, so we use a
924 * small temporary stack in .bss for that. This works
925 * because only one CPU at a time can be in this code
926 */
927 lis r1,temp_boot_stack@h
928 ori r1,r1,temp_boot_stack@l
929 addi r1,r1,1024-STACK_FRAME_OVERHEAD
930 li r0,0
931 stw r0,0(r1)
932 bl mmu_init_secondary
934 /* Now we can get our task struct and real stack pointer */
936 /* Get current_thread_info and current */
937 lis r1,secondary_ti@ha
938 lwz r1,secondary_ti@l(r1)
939 lwz r2,TI_TASK(r1)
941 /* Current stack pointer */
942 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
943 li r0,0
944 stw r0,0(r1)
946 /* Kernel stack for exception entry in SPRG3 */
947 addi r4,r2,THREAD /* init task's THREAD */
948 mtspr SPRN_SPRG3,r4
950 b start_secondary
952 #endif /* CONFIG_SMP */
954 /*
955 * Set up the initial MMU state for 44x
956 *
957 * We are still executing code at the virtual address
958 * mappings set by the firmware for the base of RAM.
959 */
961 head_start_47x:
962 /* Load our current PID->MMUCR TID and MSR IS->MMUCR STS */
963 mfspr r3,SPRN_PID /* Get PID */
964 mfmsr r4 /* Get MSR */
965 andi. r4,r4,MSR_IS@l /* TS=1? */
966 beq 1f /* If not, leave STS=0 */
967 oris r3,r3,PPC47x_MMUCR_STS@h /* Set STS=1 */
968 1: mtspr SPRN_MMUCR,r3 /* Put MMUCR */
969 sync
971 /* Find the entry we are running from */
972 bl 1f
973 1: mflr r23
974 tlbsx r23,0,r23
975 tlbre r24,r23,0
976 tlbre r25,r23,1
977 tlbre r26,r23,2
979 /*
980 * Cleanup time
981 */
983 /* Initialize MMUCR */
984 li r5,0
985 mtspr SPRN_MMUCR,r5
986 sync
988 clear_all_utlb_entries:
990 #; Set initial values.
992 addis r3,0,0x8000
993 addi r4,0,0
994 addi r5,0,0
995 b clear_utlb_entry
997 #; Align the loop to speed things up.
999 .align 6
1001 clear_utlb_entry:
1003 tlbwe r4,r3,0
1004 tlbwe r5,r3,1
1005 tlbwe r5,r3,2
1006 addis r3,r3,0x2000
1007 cmpwi r3,0
1008 bne clear_utlb_entry
1009 addis r3,0,0x8000
1010 addis r4,r4,0x100
1011 cmpwi r4,0
1012 bne clear_utlb_entry
1014 #; Restore original entry.
1016 oris r23,r23,0x8000 /* specify the way */
1017 tlbwe r24,r23,0
1018 tlbwe r25,r23,1
1019 tlbwe r26,r23,2
1021 /*
1022 * Configure and load pinned entry into TLB for the kernel core
1023 */
1025 lis r3,PAGE_OFFSET@h
1026 ori r3,r3,PAGE_OFFSET@l
1028 /* Load the kernel PID = 0 */
1029 li r0,0
1030 mtspr SPRN_PID,r0
1031 sync
1033 /* Word 0 */
1034 clrrwi r3,r3,12 /* Mask off the effective page number */
1035 ori r3,r3,PPC47x_TLB0_VALID | PPC47x_TLB0_256M
1037 /* Word 1 - use r25. RPN is the same as the original entry */
1039 /* Word 2 */
1040 li r5,0
1041 ori r5,r5,PPC47x_TLB2_S_RWX
1042 #ifdef CONFIG_SMP
1043 ori r5,r5,PPC47x_TLB2_M
1044 #endif
1046 /* We write to way 0 and bolted 0 */
1047 lis r0,0x8800
1048 tlbwe r3,r0,0
1049 tlbwe r25,r0,1
1050 tlbwe r5,r0,2
1052 /*
1053 * Configure SSPCR, ISPCR and USPCR for now to search everything, we can fix
1054 * them up later
1055 */
1056 LOAD_REG_IMMEDIATE(r3, 0x9abcdef0)
1057 mtspr SPRN_SSPCR,r3
1058 mtspr SPRN_USPCR,r3
1059 LOAD_REG_IMMEDIATE(r3, 0x12345670)
1060 mtspr SPRN_ISPCR,r3
1062 /* Force context change */
1063 mfmsr r0
1064 mtspr SPRN_SRR1, r0
1065 lis r0,3f@h
1066 ori r0,r0,3f@l
1067 mtspr SPRN_SRR0,r0
1068 sync
1069 rfi
1071 /* Invalidate original entry we used */
1072 3:
1073 rlwinm r24,r24,0,21,19 /* clear the "valid" bit */
1074 tlbwe r24,r23,0
1075 addi r24,0,0
1076 tlbwe r24,r23,1
1077 tlbwe r24,r23,2
1078 isync /* Clear out the shadow TLB entries */
1080 #ifdef CONFIG_PPC_EARLY_DEBUG_44x
1081 /* Add UART mapping for early debug. */
1083 /* Word 0 */
1084 lis r3,PPC44x_EARLY_DEBUG_VIRTADDR@h
1085 ori r3,r3,PPC47x_TLB0_VALID | PPC47x_TLB0_TS | PPC47x_TLB0_1M
1087 /* Word 1 */
1088 lis r4,CONFIG_PPC_EARLY_DEBUG_44x_PHYSLOW@h
1089 ori r4,r4,CONFIG_PPC_EARLY_DEBUG_44x_PHYSHIGH
1091 /* Word 2 */
1092 li r5,(PPC47x_TLB2_S_RW | PPC47x_TLB2_IMG)
1094 /* Bolted in way 0, bolt slot 5, we -hope- we don't hit the same
1095 * congruence class as the kernel, we need to make sure of it at
1096 * some point
1097 */
1098 lis r0,0x8d00
1099 tlbwe r3,r0,0
1100 tlbwe r4,r0,1
1101 tlbwe r5,r0,2
1103 /* Force context change */
1104 isync
1105 #endif /* CONFIG_PPC_EARLY_DEBUG_44x */
1107 /* Establish the interrupt vector offsets */
1108 SET_IVOR(0, CriticalInput);
1109 SET_IVOR(1, MachineCheckA);
1110 SET_IVOR(2, DataStorage);
1111 SET_IVOR(3, InstructionStorage);
1112 SET_IVOR(4, ExternalInput);
1113 SET_IVOR(5, Alignment);
1114 SET_IVOR(6, Program);
1115 SET_IVOR(7, FloatingPointUnavailable);
1116 SET_IVOR(8, SystemCall);
1117 SET_IVOR(9, AuxillaryProcessorUnavailable);
1118 SET_IVOR(10, Decrementer);
1119 SET_IVOR(11, FixedIntervalTimer);
1120 SET_IVOR(12, WatchdogTimer);
1121 SET_IVOR(13, DataTLBError47x);
1122 SET_IVOR(14, InstructionTLBError47x);
1123 SET_IVOR(15, DebugCrit);
1125 /* We configure icbi to invalidate 128 bytes at a time since the
1126 * current 32-bit kernel code isn't too happy with icache != dcache
1127 * block size
1128 */
1129 mfspr r3,SPRN_CCR0
1130 oris r3,r3,0x0020
1131 mtspr SPRN_CCR0,r3
1132 isync
1134 #endif /* CONFIG_PPC_47x */
1136 /*
1137 * Here we are back to code that is common between 44x and 47x
1138 *
1139 * We proceed to further kernel initialization and return to the
1140 * main kernel entry
1141 */
1142 head_start_common:
1143 /* Establish the interrupt vector base */
1144 lis r4,interrupt_base@h /* IVPR only uses the high 16-bits */
1145 mtspr SPRN_IVPR,r4
1147 /*
1148 * If the kernel was loaded at a non-zero 256 MB page, we need to
1149 * mask off the most significant 4 bits to get the relative address
1150 * from the start of physical memory
1151 */
1152 rlwinm r22,r22,0,4,31
1153 addis r22,r22,PAGE_OFFSET@h
1154 mtlr r22
1155 isync
1156 blr
1158 /*
1159 * We put a few things here that have to be page-aligned. This stuff
1160 * goes at the beginning of the data segment, which is page-aligned.
1161 */
1162 .data
1163 .align PAGE_SHIFT
1164 .globl sdata
1165 sdata:
1166 .globl empty_zero_page
1167 empty_zero_page:
1168 .space PAGE_SIZE
1170 /*
1171 * To support >32-bit physical addresses, we use an 8KB pgdir.
1172 */
1173 .globl swapper_pg_dir
1174 swapper_pg_dir:
1175 .space PGD_TABLE_SIZE
1177 /*
1178 * Room for two PTE pointers, usually the kernel and current user pointers
1179 * to their respective root page table.
1180 */
1181 abatron_pteptrs:
1182 .space 8
1184 #ifdef CONFIG_SMP
1185 .align 12
1186 temp_boot_stack:
1187 .space 1024
1188 #endif /* CONFIG_SMP */