| blob | 11b4f6d9ffcecfb3fde2d60cd4c05dcee66ad832 |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 *
5 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
6 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
7 * Adapted for Power Macintosh by Paul Mackerras.
8 * Low-level exception handlers and MMU support
9 * rewritten by Paul Mackerras.
10 * Copyright (C) 1996 Paul Mackerras.
11 *
12 * Adapted for 64bit PowerPC by Dave Engebretsen, Peter Bergner, and
13 * Mike Corrigan {engebret|bergner|mikejc}@us.ibm.com
14 *
15 * This file contains the low-level support and setup for the
16 * PowerPC-64 platform, including trap and interrupt dispatch.
17 *
18 * This program is free software; you can redistribute it and/or
19 * modify it under the terms of the GNU General Public License
20 * as published by the Free Software Foundation; either version
21 * 2 of the License, or (at your option) any later version.
22 */
24 #include <linux/threads.h>
25 #include <asm/reg.h>
26 #include <asm/page.h>
27 #include <asm/mmu.h>
28 #include <asm/ppc_asm.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/bug.h>
31 #include <asm/cputable.h>
32 #include <asm/setup.h>
33 #include <asm/hvcall.h>
34 #include <asm/iseries/lpar_map.h>
35 #include <asm/thread_info.h>
36 #include <asm/firmware.h>
37 #include <asm/page_64.h>
38 #include <asm/exception.h>
40 #define DO_SOFT_DISABLE
42 /*
43 * We layout physical memory as follows:
44 * 0x0000 - 0x00ff : Secondary processor spin code
45 * 0x0100 - 0x2fff : pSeries Interrupt prologs
46 * 0x3000 - 0x5fff : interrupt support, iSeries and common interrupt prologs
47 * 0x6000 - 0x6fff : Initial (CPU0) segment table
48 * 0x7000 - 0x7fff : FWNMI data area
49 * 0x8000 - : Early init and support code
50 */
52 /*
53 * SPRG Usage
54 *
55 * Register Definition
56 *
57 * SPRG0 reserved for hypervisor
58 * SPRG1 temp - used to save gpr
59 * SPRG2 temp - used to save gpr
60 * SPRG3 virt addr of paca
61 */
63 /*
64 * Entering into this code we make the following assumptions:
65 * For pSeries:
66 * 1. The MMU is off & open firmware is running in real mode.
67 * 2. The kernel is entered at __start
68 *
69 * For iSeries:
70 * 1. The MMU is on (as it always is for iSeries)
71 * 2. The kernel is entered at system_reset_iSeries
72 */
74 .text
75 .globl _stext
76 _stext:
77 _GLOBAL(__start)
78 /* NOP this out unconditionally */
79 BEGIN_FTR_SECTION
80 b .__start_initialization_multiplatform
81 END_FTR_SECTION(0, 1)
83 /* Catch branch to 0 in real mode */
84 trap
86 /* Secondary processors spin on this value until it goes to 1. */
87 .globl __secondary_hold_spinloop
88 __secondary_hold_spinloop:
89 .llong 0x0
91 /* Secondary processors write this value with their cpu # */
92 /* after they enter the spin loop immediately below. */
93 .globl __secondary_hold_acknowledge
94 __secondary_hold_acknowledge:
95 .llong 0x0
97 #ifdef CONFIG_PPC_ISERIES
98 /*
99 * At offset 0x20, there is a pointer to iSeries LPAR data.
100 * This is required by the hypervisor
101 */
102 . = 0x20
103 .llong hvReleaseData-KERNELBASE
104 #endif /* CONFIG_PPC_ISERIES */
106 . = 0x60
107 /*
108 * The following code is used to hold secondary processors
109 * in a spin loop after they have entered the kernel, but
110 * before the bulk of the kernel has been relocated. This code
111 * is relocated to physical address 0x60 before prom_init is run.
112 * All of it must fit below the first exception vector at 0x100.
113 */
114 _GLOBAL(__secondary_hold)
115 mfmsr r24
116 ori r24,r24,MSR_RI
117 mtmsrd r24 /* RI on */
119 /* Grab our physical cpu number */
120 mr r24,r3
122 /* Tell the master cpu we're here */
123 /* Relocation is off & we are located at an address less */
124 /* than 0x100, so only need to grab low order offset. */
125 std r24,__secondary_hold_acknowledge@l(0)
126 sync
128 /* All secondary cpus wait here until told to start. */
129 100: ld r4,__secondary_hold_spinloop@l(0)
130 cmpdi 0,r4,1
131 bne 100b
133 #if defined(CONFIG_SMP) || defined(CONFIG_KEXEC)
134 LOAD_REG_IMMEDIATE(r4, .generic_secondary_smp_init)
135 mtctr r4
136 mr r3,r24
137 bctr
138 #else
139 BUG_OPCODE
140 #endif
142 /* This value is used to mark exception frames on the stack. */
143 .section ".toc","aw"
144 exception_marker:
145 .tc ID_72656773_68657265[TC],0x7265677368657265
146 .text
148 /*
149 * This is the start of the interrupt handlers for pSeries
150 * This code runs with relocation off.
151 */
152 . = 0x100
153 .globl __start_interrupts
154 __start_interrupts:
156 STD_EXCEPTION_PSERIES(0x100, system_reset)
158 . = 0x200
159 _machine_check_pSeries:
160 HMT_MEDIUM
161 mtspr SPRN_SPRG1,r13 /* save r13 */
162 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
164 . = 0x300
165 .globl data_access_pSeries
166 data_access_pSeries:
167 HMT_MEDIUM
168 mtspr SPRN_SPRG1,r13
169 BEGIN_FTR_SECTION
170 mtspr SPRN_SPRG2,r12
171 mfspr r13,SPRN_DAR
172 mfspr r12,SPRN_DSISR
173 srdi r13,r13,60
174 rlwimi r13,r12,16,0x20
175 mfcr r12
176 cmpwi r13,0x2c
177 beq do_stab_bolted_pSeries
178 mtcrf 0x80,r12
179 mfspr r12,SPRN_SPRG2
180 END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
181 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common)
183 . = 0x380
184 .globl data_access_slb_pSeries
185 data_access_slb_pSeries:
186 HMT_MEDIUM
187 mtspr SPRN_SPRG1,r13
188 mfspr r13,SPRN_SPRG3 /* get paca address into r13 */
189 std r3,PACA_EXSLB+EX_R3(r13)
190 mfspr r3,SPRN_DAR
191 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
192 mfcr r9
193 #ifdef __DISABLED__
194 /* Keep that around for when we re-implement dynamic VSIDs */
195 cmpdi r3,0
196 bge slb_miss_user_pseries
197 #endif /* __DISABLED__ */
198 std r10,PACA_EXSLB+EX_R10(r13)
199 std r11,PACA_EXSLB+EX_R11(r13)
200 std r12,PACA_EXSLB+EX_R12(r13)
201 mfspr r10,SPRN_SPRG1
202 std r10,PACA_EXSLB+EX_R13(r13)
203 mfspr r12,SPRN_SRR1 /* and SRR1 */
204 b .slb_miss_realmode /* Rel. branch works in real mode */
206 STD_EXCEPTION_PSERIES(0x400, instruction_access)
208 . = 0x480
209 .globl instruction_access_slb_pSeries
210 instruction_access_slb_pSeries:
211 HMT_MEDIUM
212 mtspr SPRN_SPRG1,r13
213 mfspr r13,SPRN_SPRG3 /* get paca address into r13 */
214 std r3,PACA_EXSLB+EX_R3(r13)
215 mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
216 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
217 mfcr r9
218 #ifdef __DISABLED__
219 /* Keep that around for when we re-implement dynamic VSIDs */
220 cmpdi r3,0
221 bge slb_miss_user_pseries
222 #endif /* __DISABLED__ */
223 std r10,PACA_EXSLB+EX_R10(r13)
224 std r11,PACA_EXSLB+EX_R11(r13)
225 std r12,PACA_EXSLB+EX_R12(r13)
226 mfspr r10,SPRN_SPRG1
227 std r10,PACA_EXSLB+EX_R13(r13)
228 mfspr r12,SPRN_SRR1 /* and SRR1 */
229 b .slb_miss_realmode /* Rel. branch works in real mode */
231 MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt)
232 STD_EXCEPTION_PSERIES(0x600, alignment)
233 STD_EXCEPTION_PSERIES(0x700, program_check)
234 STD_EXCEPTION_PSERIES(0x800, fp_unavailable)
235 MASKABLE_EXCEPTION_PSERIES(0x900, decrementer)
236 STD_EXCEPTION_PSERIES(0xa00, trap_0a)
237 STD_EXCEPTION_PSERIES(0xb00, trap_0b)
239 . = 0xc00
240 .globl system_call_pSeries
241 system_call_pSeries:
242 HMT_MEDIUM
243 mr r9,r13
244 mfmsr r10
245 mfspr r13,SPRN_SPRG3
246 mfspr r11,SPRN_SRR0
247 clrrdi r12,r13,32
248 oris r12,r12,system_call_common@h
249 ori r12,r12,system_call_common@l
250 mtspr SPRN_SRR0,r12
251 ori r10,r10,MSR_IR|MSR_DR|MSR_RI
252 mfspr r12,SPRN_SRR1
253 mtspr SPRN_SRR1,r10
254 rfid
255 b . /* prevent speculative execution */
257 STD_EXCEPTION_PSERIES(0xd00, single_step)
258 STD_EXCEPTION_PSERIES(0xe00, trap_0e)
260 /* We need to deal with the Altivec unavailable exception
261 * here which is at 0xf20, thus in the middle of the
262 * prolog code of the PerformanceMonitor one. A little
263 * trickery is thus necessary
264 */
265 . = 0xf00
266 b performance_monitor_pSeries
268 STD_EXCEPTION_PSERIES(0xf20, altivec_unavailable)
270 #ifdef CONFIG_CBE_RAS
271 HSTD_EXCEPTION_PSERIES(0x1200, cbe_system_error)
272 #endif /* CONFIG_CBE_RAS */
273 STD_EXCEPTION_PSERIES(0x1300, instruction_breakpoint)
274 #ifdef CONFIG_CBE_RAS
275 HSTD_EXCEPTION_PSERIES(0x1600, cbe_maintenance)
276 #endif /* CONFIG_CBE_RAS */
277 STD_EXCEPTION_PSERIES(0x1700, altivec_assist)
278 #ifdef CONFIG_CBE_RAS
279 HSTD_EXCEPTION_PSERIES(0x1800, cbe_thermal)
280 #endif /* CONFIG_CBE_RAS */
282 . = 0x3000
284 /*** pSeries interrupt support ***/
286 /* moved from 0xf00 */
287 STD_EXCEPTION_PSERIES(., performance_monitor)
289 /*
290 * An interrupt came in while soft-disabled; clear EE in SRR1,
291 * clear paca->hard_enabled and return.
292 */
293 masked_interrupt:
294 stb r10,PACAHARDIRQEN(r13)
295 mtcrf 0x80,r9
296 ld r9,PACA_EXGEN+EX_R9(r13)
297 mfspr r10,SPRN_SRR1
298 rldicl r10,r10,48,1 /* clear MSR_EE */
299 rotldi r10,r10,16
300 mtspr SPRN_SRR1,r10
301 ld r10,PACA_EXGEN+EX_R10(r13)
302 mfspr r13,SPRN_SPRG1
303 rfid
304 b .
306 .align 7
307 do_stab_bolted_pSeries:
308 mtcrf 0x80,r12
309 mfspr r12,SPRN_SPRG2
310 EXCEPTION_PROLOG_PSERIES(PACA_EXSLB, .do_stab_bolted)
312 /*
313 * We have some room here we use that to put
314 * the peries slb miss user trampoline code so it's reasonably
315 * away from slb_miss_user_common to avoid problems with rfid
316 *
317 * This is used for when the SLB miss handler has to go virtual,
318 * which doesn't happen for now anymore but will once we re-implement
319 * dynamic VSIDs for shared page tables
320 */
321 #ifdef __DISABLED__
322 slb_miss_user_pseries:
323 std r10,PACA_EXGEN+EX_R10(r13)
324 std r11,PACA_EXGEN+EX_R11(r13)
325 std r12,PACA_EXGEN+EX_R12(r13)
326 mfspr r10,SPRG1
327 ld r11,PACA_EXSLB+EX_R9(r13)
328 ld r12,PACA_EXSLB+EX_R3(r13)
329 std r10,PACA_EXGEN+EX_R13(r13)
330 std r11,PACA_EXGEN+EX_R9(r13)
331 std r12,PACA_EXGEN+EX_R3(r13)
332 clrrdi r12,r13,32
333 mfmsr r10
334 mfspr r11,SRR0 /* save SRR0 */
335 ori r12,r12,slb_miss_user_common@l /* virt addr of handler */
336 ori r10,r10,MSR_IR|MSR_DR|MSR_RI
337 mtspr SRR0,r12
338 mfspr r12,SRR1 /* and SRR1 */
339 mtspr SRR1,r10
340 rfid
341 b . /* prevent spec. execution */
342 #endif /* __DISABLED__ */
344 #ifdef CONFIG_PPC_PSERIES
345 /*
346 * Vectors for the FWNMI option. Share common code.
347 */
348 .globl system_reset_fwnmi
349 .align 7
350 system_reset_fwnmi:
351 HMT_MEDIUM
352 mtspr SPRN_SPRG1,r13 /* save r13 */
353 EXCEPTION_PROLOG_PSERIES_FORCE_64BIT(PACA_EXGEN, system_reset_common)
355 .globl machine_check_fwnmi
356 .align 7
357 machine_check_fwnmi:
358 HMT_MEDIUM
359 mtspr SPRN_SPRG1,r13 /* save r13 */
360 EXCEPTION_PROLOG_PSERIES_FORCE_64BIT(PACA_EXMC, machine_check_common)
362 #endif /* CONFIG_PPC_PSERIES */
364 /*** Common interrupt handlers ***/
366 STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception)
368 /*
369 * Machine check is different because we use a different
370 * save area: PACA_EXMC instead of PACA_EXGEN.
371 */
372 .align 7
373 .globl machine_check_common
374 machine_check_common:
375 EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
376 FINISH_NAP
377 DISABLE_INTS
378 bl .save_nvgprs
379 addi r3,r1,STACK_FRAME_OVERHEAD
380 bl .machine_check_exception
381 b .ret_from_except
383 STD_EXCEPTION_COMMON_LITE(0x900, decrementer, .timer_interrupt)
384 STD_EXCEPTION_COMMON(0xa00, trap_0a, .unknown_exception)
385 STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception)
386 STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception)
387 STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception)
388 STD_EXCEPTION_COMMON_IDLE(0xf00, performance_monitor, .performance_monitor_exception)
389 STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception)
390 #ifdef CONFIG_ALTIVEC
391 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception)
392 #else
393 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception)
394 #endif
395 #ifdef CONFIG_CBE_RAS
396 STD_EXCEPTION_COMMON(0x1200, cbe_system_error, .cbe_system_error_exception)
397 STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, .cbe_maintenance_exception)
398 STD_EXCEPTION_COMMON(0x1800, cbe_thermal, .cbe_thermal_exception)
399 #endif /* CONFIG_CBE_RAS */
401 /*
402 * Here we have detected that the kernel stack pointer is bad.
403 * R9 contains the saved CR, r13 points to the paca,
404 * r10 contains the (bad) kernel stack pointer,
405 * r11 and r12 contain the saved SRR0 and SRR1.
406 * We switch to using an emergency stack, save the registers there,
407 * and call kernel_bad_stack(), which panics.
408 */
409 bad_stack:
410 ld r1,PACAEMERGSP(r13)
411 subi r1,r1,64+INT_FRAME_SIZE
412 std r9,_CCR(r1)
413 std r10,GPR1(r1)
414 std r11,_NIP(r1)
415 std r12,_MSR(r1)
416 mfspr r11,SPRN_DAR
417 mfspr r12,SPRN_DSISR
418 std r11,_DAR(r1)
419 std r12,_DSISR(r1)
420 mflr r10
421 mfctr r11
422 mfxer r12
423 std r10,_LINK(r1)
424 std r11,_CTR(r1)
425 std r12,_XER(r1)
426 SAVE_GPR(0,r1)
427 SAVE_GPR(2,r1)
428 SAVE_4GPRS(3,r1)
429 SAVE_2GPRS(7,r1)
430 SAVE_10GPRS(12,r1)
431 SAVE_10GPRS(22,r1)
432 lhz r12,PACA_TRAP_SAVE(r13)
433 std r12,_TRAP(r1)
434 addi r11,r1,INT_FRAME_SIZE
435 std r11,0(r1)
436 li r12,0
437 std r12,0(r11)
438 ld r2,PACATOC(r13)
439 1: addi r3,r1,STACK_FRAME_OVERHEAD
440 bl .kernel_bad_stack
441 b 1b
443 /*
444 * Return from an exception with minimal checks.
445 * The caller is assumed to have done EXCEPTION_PROLOG_COMMON.
446 * If interrupts have been enabled, or anything has been
447 * done that might have changed the scheduling status of
448 * any task or sent any task a signal, you should use
449 * ret_from_except or ret_from_except_lite instead of this.
450 */
451 fast_exc_return_irq: /* restores irq state too */
452 ld r3,SOFTE(r1)
453 ld r12,_MSR(r1)
454 stb r3,PACASOFTIRQEN(r13) /* restore paca->soft_enabled */
455 rldicl r4,r12,49,63 /* get MSR_EE to LSB */
456 stb r4,PACAHARDIRQEN(r13) /* restore paca->hard_enabled */
457 b 1f
459 .globl fast_exception_return
460 fast_exception_return:
461 ld r12,_MSR(r1)
462 1: ld r11,_NIP(r1)
463 andi. r3,r12,MSR_RI /* check if RI is set */
464 beq- unrecov_fer
466 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
467 andi. r3,r12,MSR_PR
468 beq 2f
469 ACCOUNT_CPU_USER_EXIT(r3, r4)
470 2:
471 #endif
473 ld r3,_CCR(r1)
474 ld r4,_LINK(r1)
475 ld r5,_CTR(r1)
476 ld r6,_XER(r1)
477 mtcr r3
478 mtlr r4
479 mtctr r5
480 mtxer r6
481 REST_GPR(0, r1)
482 REST_8GPRS(2, r1)
484 mfmsr r10
485 rldicl r10,r10,48,1 /* clear EE */
486 rldicr r10,r10,16,61 /* clear RI (LE is 0 already) */
487 mtmsrd r10,1
489 mtspr SPRN_SRR1,r12
490 mtspr SPRN_SRR0,r11
491 REST_4GPRS(10, r1)
492 ld r1,GPR1(r1)
493 rfid
494 b . /* prevent speculative execution */
496 unrecov_fer:
497 bl .save_nvgprs
498 1: addi r3,r1,STACK_FRAME_OVERHEAD
499 bl .unrecoverable_exception
500 b 1b
502 /*
503 * Here r13 points to the paca, r9 contains the saved CR,
504 * SRR0 and SRR1 are saved in r11 and r12,
505 * r9 - r13 are saved in paca->exgen.
506 */
507 .align 7
508 .globl data_access_common
509 data_access_common:
510 mfspr r10,SPRN_DAR
511 std r10,PACA_EXGEN+EX_DAR(r13)
512 mfspr r10,SPRN_DSISR
513 stw r10,PACA_EXGEN+EX_DSISR(r13)
514 EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
515 ld r3,PACA_EXGEN+EX_DAR(r13)
516 lwz r4,PACA_EXGEN+EX_DSISR(r13)
517 li r5,0x300
518 b .do_hash_page /* Try to handle as hpte fault */
520 .align 7
521 .globl instruction_access_common
522 instruction_access_common:
523 EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
524 ld r3,_NIP(r1)
525 andis. r4,r12,0x5820
526 li r5,0x400
527 b .do_hash_page /* Try to handle as hpte fault */
529 /*
530 * Here is the common SLB miss user that is used when going to virtual
531 * mode for SLB misses, that is currently not used
532 */
533 #ifdef __DISABLED__
534 .align 7
535 .globl slb_miss_user_common
536 slb_miss_user_common:
537 mflr r10
538 std r3,PACA_EXGEN+EX_DAR(r13)
539 stw r9,PACA_EXGEN+EX_CCR(r13)
540 std r10,PACA_EXGEN+EX_LR(r13)
541 std r11,PACA_EXGEN+EX_SRR0(r13)
542 bl .slb_allocate_user
544 ld r10,PACA_EXGEN+EX_LR(r13)
545 ld r3,PACA_EXGEN+EX_R3(r13)
546 lwz r9,PACA_EXGEN+EX_CCR(r13)
547 ld r11,PACA_EXGEN+EX_SRR0(r13)
548 mtlr r10
549 beq- slb_miss_fault
551 andi. r10,r12,MSR_RI /* check for unrecoverable exception */
552 beq- unrecov_user_slb
553 mfmsr r10
555 .machine push
556 .machine "power4"
557 mtcrf 0x80,r9
558 .machine pop
560 clrrdi r10,r10,2 /* clear RI before setting SRR0/1 */
561 mtmsrd r10,1
563 mtspr SRR0,r11
564 mtspr SRR1,r12
566 ld r9,PACA_EXGEN+EX_R9(r13)
567 ld r10,PACA_EXGEN+EX_R10(r13)
568 ld r11,PACA_EXGEN+EX_R11(r13)
569 ld r12,PACA_EXGEN+EX_R12(r13)
570 ld r13,PACA_EXGEN+EX_R13(r13)
571 rfid
572 b .
574 slb_miss_fault:
575 EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN)
576 ld r4,PACA_EXGEN+EX_DAR(r13)
577 li r5,0
578 std r4,_DAR(r1)
579 std r5,_DSISR(r1)
580 b handle_page_fault
582 unrecov_user_slb:
583 EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN)
584 DISABLE_INTS
585 bl .save_nvgprs
586 1: addi r3,r1,STACK_FRAME_OVERHEAD
587 bl .unrecoverable_exception
588 b 1b
590 #endif /* __DISABLED__ */
593 /*
594 * r13 points to the PACA, r9 contains the saved CR,
595 * r12 contain the saved SRR1, SRR0 is still ready for return
596 * r3 has the faulting address
597 * r9 - r13 are saved in paca->exslb.
598 * r3 is saved in paca->slb_r3
599 * We assume we aren't going to take any exceptions during this procedure.
600 */
601 _GLOBAL(slb_miss_realmode)
602 mflr r10
604 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
605 std r10,PACA_EXSLB+EX_LR(r13) /* save LR */
607 bl .slb_allocate_realmode
609 /* All done -- return from exception. */
611 ld r10,PACA_EXSLB+EX_LR(r13)
612 ld r3,PACA_EXSLB+EX_R3(r13)
613 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
614 #ifdef CONFIG_PPC_ISERIES
615 BEGIN_FW_FTR_SECTION
616 ld r11,PACALPPACAPTR(r13)
617 ld r11,LPPACASRR0(r11) /* get SRR0 value */
618 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
619 #endif /* CONFIG_PPC_ISERIES */
621 mtlr r10
623 andi. r10,r12,MSR_RI /* check for unrecoverable exception */
624 beq- unrecov_slb
626 .machine push
627 .machine "power4"
628 mtcrf 0x80,r9
629 mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
630 .machine pop
632 #ifdef CONFIG_PPC_ISERIES
633 BEGIN_FW_FTR_SECTION
634 mtspr SPRN_SRR0,r11
635 mtspr SPRN_SRR1,r12
636 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
637 #endif /* CONFIG_PPC_ISERIES */
638 ld r9,PACA_EXSLB+EX_R9(r13)
639 ld r10,PACA_EXSLB+EX_R10(r13)
640 ld r11,PACA_EXSLB+EX_R11(r13)
641 ld r12,PACA_EXSLB+EX_R12(r13)
642 ld r13,PACA_EXSLB+EX_R13(r13)
643 rfid
644 b . /* prevent speculative execution */
646 unrecov_slb:
647 EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
648 DISABLE_INTS
649 bl .save_nvgprs
650 1: addi r3,r1,STACK_FRAME_OVERHEAD
651 bl .unrecoverable_exception
652 b 1b
654 .align 7
655 .globl hardware_interrupt_common
656 .globl hardware_interrupt_entry
657 hardware_interrupt_common:
658 EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN)
659 FINISH_NAP
660 hardware_interrupt_entry:
661 DISABLE_INTS
662 BEGIN_FTR_SECTION
663 bl .ppc64_runlatch_on
664 END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
665 addi r3,r1,STACK_FRAME_OVERHEAD
666 bl .do_IRQ
667 b .ret_from_except_lite
669 #ifdef CONFIG_PPC_970_NAP
670 power4_fixup_nap:
671 andc r9,r9,r10
672 std r9,TI_LOCAL_FLAGS(r11)
673 ld r10,_LINK(r1) /* make idle task do the */
674 std r10,_NIP(r1) /* equivalent of a blr */
675 blr
676 #endif
678 .align 7
679 .globl alignment_common
680 alignment_common:
681 mfspr r10,SPRN_DAR
682 std r10,PACA_EXGEN+EX_DAR(r13)
683 mfspr r10,SPRN_DSISR
684 stw r10,PACA_EXGEN+EX_DSISR(r13)
685 EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
686 ld r3,PACA_EXGEN+EX_DAR(r13)
687 lwz r4,PACA_EXGEN+EX_DSISR(r13)
688 std r3,_DAR(r1)
689 std r4,_DSISR(r1)
690 bl .save_nvgprs
691 addi r3,r1,STACK_FRAME_OVERHEAD
692 ENABLE_INTS
693 bl .alignment_exception
694 b .ret_from_except
696 .align 7
697 .globl program_check_common
698 program_check_common:
699 EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
700 bl .save_nvgprs
701 addi r3,r1,STACK_FRAME_OVERHEAD
702 ENABLE_INTS
703 bl .program_check_exception
704 b .ret_from_except
706 .align 7
707 .globl fp_unavailable_common
708 fp_unavailable_common:
709 EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
710 bne 1f /* if from user, just load it up */
711 bl .save_nvgprs
712 addi r3,r1,STACK_FRAME_OVERHEAD
713 ENABLE_INTS
714 bl .kernel_fp_unavailable_exception
715 BUG_OPCODE
716 1: b .load_up_fpu
718 .align 7
719 .globl altivec_unavailable_common
720 altivec_unavailable_common:
721 EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
722 #ifdef CONFIG_ALTIVEC
723 BEGIN_FTR_SECTION
724 bne .load_up_altivec /* if from user, just load it up */
725 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
726 #endif
727 bl .save_nvgprs
728 addi r3,r1,STACK_FRAME_OVERHEAD
729 ENABLE_INTS
730 bl .altivec_unavailable_exception
731 b .ret_from_except
733 #ifdef CONFIG_ALTIVEC
734 /*
735 * load_up_altivec(unused, unused, tsk)
736 * Disable VMX for the task which had it previously,
737 * and save its vector registers in its thread_struct.
738 * Enables the VMX for use in the kernel on return.
739 * On SMP we know the VMX is free, since we give it up every
740 * switch (ie, no lazy save of the vector registers).
741 * On entry: r13 == 'current' && last_task_used_altivec != 'current'
742 */
743 _STATIC(load_up_altivec)
744 mfmsr r5 /* grab the current MSR */
745 oris r5,r5,MSR_VEC@h
746 mtmsrd r5 /* enable use of VMX now */
747 isync
749 /*
750 * For SMP, we don't do lazy VMX switching because it just gets too
751 * horrendously complex, especially when a task switches from one CPU
752 * to another. Instead we call giveup_altvec in switch_to.
753 * VRSAVE isn't dealt with here, that is done in the normal context
754 * switch code. Note that we could rely on vrsave value to eventually
755 * avoid saving all of the VREGs here...
756 */
757 #ifndef CONFIG_SMP
758 ld r3,last_task_used_altivec@got(r2)
759 ld r4,0(r3)
760 cmpdi 0,r4,0
761 beq 1f
762 /* Save VMX state to last_task_used_altivec's THREAD struct */
763 addi r4,r4,THREAD
764 SAVE_32VRS(0,r5,r4)
765 mfvscr vr0
766 li r10,THREAD_VSCR
767 stvx vr0,r10,r4
768 /* Disable VMX for last_task_used_altivec */
769 ld r5,PT_REGS(r4)
770 ld r4,_MSR-STACK_FRAME_OVERHEAD(r5)
771 lis r6,MSR_VEC@h
772 andc r4,r4,r6
773 std r4,_MSR-STACK_FRAME_OVERHEAD(r5)
774 1:
775 #endif /* CONFIG_SMP */
776 /* Hack: if we get an altivec unavailable trap with VRSAVE
777 * set to all zeros, we assume this is a broken application
778 * that fails to set it properly, and thus we switch it to
779 * all 1's
780 */
781 mfspr r4,SPRN_VRSAVE
782 cmpdi 0,r4,0
783 bne+ 1f
784 li r4,-1
785 mtspr SPRN_VRSAVE,r4
786 1:
787 /* enable use of VMX after return */
788 ld r4,PACACURRENT(r13)
789 addi r5,r4,THREAD /* Get THREAD */
790 oris r12,r12,MSR_VEC@h
791 std r12,_MSR(r1)
792 li r4,1
793 li r10,THREAD_VSCR
794 stw r4,THREAD_USED_VR(r5)
795 lvx vr0,r10,r5
796 mtvscr vr0
797 REST_32VRS(0,r4,r5)
798 #ifndef CONFIG_SMP
799 /* Update last_task_used_math to 'current' */
800 subi r4,r5,THREAD /* Back to 'current' */
801 std r4,0(r3)
802 #endif /* CONFIG_SMP */
803 /* restore registers and return */
804 b fast_exception_return
805 #endif /* CONFIG_ALTIVEC */
807 /*
808 * Hash table stuff
809 */
810 .align 7
811 _GLOBAL(do_hash_page)
812 std r3,_DAR(r1)
813 std r4,_DSISR(r1)
815 andis. r0,r4,0xa450 /* weird error? */
816 bne- handle_page_fault /* if not, try to insert a HPTE */
817 BEGIN_FTR_SECTION
818 andis. r0,r4,0x0020 /* Is it a segment table fault? */
819 bne- do_ste_alloc /* If so handle it */
820 END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
822 /*
823 * We need to set the _PAGE_USER bit if MSR_PR is set or if we are
824 * accessing a userspace segment (even from the kernel). We assume
825 * kernel addresses always have the high bit set.
826 */
827 rlwinm r4,r4,32-25+9,31-9,31-9 /* DSISR_STORE -> _PAGE_RW */
828 rotldi r0,r3,15 /* Move high bit into MSR_PR posn */
829 orc r0,r12,r0 /* MSR_PR | ~high_bit */
830 rlwimi r4,r0,32-13,30,30 /* becomes _PAGE_USER access bit */
831 ori r4,r4,1 /* add _PAGE_PRESENT */
832 rlwimi r4,r5,22+2,31-2,31-2 /* Set _PAGE_EXEC if trap is 0x400 */
834 /*
835 * On iSeries, we soft-disable interrupts here, then
836 * hard-enable interrupts so that the hash_page code can spin on
837 * the hash_table_lock without problems on a shared processor.
838 */
839 DISABLE_INTS
841 /*
842 * r3 contains the faulting address
843 * r4 contains the required access permissions
844 * r5 contains the trap number
845 *
846 * at return r3 = 0 for success
847 */
848 bl .hash_page /* build HPTE if possible */
849 cmpdi r3,0 /* see if hash_page succeeded */
851 #ifdef DO_SOFT_DISABLE
852 BEGIN_FW_FTR_SECTION
853 /*
854 * If we had interrupts soft-enabled at the point where the
855 * DSI/ISI occurred, and an interrupt came in during hash_page,
856 * handle it now.
857 * We jump to ret_from_except_lite rather than fast_exception_return
858 * because ret_from_except_lite will check for and handle pending
859 * interrupts if necessary.
860 */
861 beq 13f
862 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
863 #endif
864 BEGIN_FW_FTR_SECTION
865 /*
866 * Here we have interrupts hard-disabled, so it is sufficient
867 * to restore paca->{soft,hard}_enable and get out.
868 */
869 beq fast_exc_return_irq /* Return from exception on success */
870 END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
872 /* For a hash failure, we don't bother re-enabling interrupts */
873 ble- 12f
875 /*
876 * hash_page couldn't handle it, set soft interrupt enable back
877 * to what it was before the trap. Note that .local_irq_restore
878 * handles any interrupts pending at this point.
879 */
880 ld r3,SOFTE(r1)
881 bl .local_irq_restore
882 b 11f
884 /* Here we have a page fault that hash_page can't handle. */
885 handle_page_fault:
886 ENABLE_INTS
887 11: ld r4,_DAR(r1)
888 ld r5,_DSISR(r1)
889 addi r3,r1,STACK_FRAME_OVERHEAD
890 bl .do_page_fault
891 cmpdi r3,0
892 beq+ 13f
893 bl .save_nvgprs
894 mr r5,r3
895 addi r3,r1,STACK_FRAME_OVERHEAD
896 lwz r4,_DAR(r1)
897 bl .bad_page_fault
898 b .ret_from_except
900 13: b .ret_from_except_lite
902 /* We have a page fault that hash_page could handle but HV refused
903 * the PTE insertion
904 */
905 12: bl .save_nvgprs
906 mr r5,r3
907 addi r3,r1,STACK_FRAME_OVERHEAD
908 ld r4,_DAR(r1)
909 bl .low_hash_fault
910 b .ret_from_except
912 /* here we have a segment miss */
913 do_ste_alloc:
914 bl .ste_allocate /* try to insert stab entry */
915 cmpdi r3,0
916 bne- handle_page_fault
917 b fast_exception_return
919 /*
920 * r13 points to the PACA, r9 contains the saved CR,
921 * r11 and r12 contain the saved SRR0 and SRR1.
922 * r9 - r13 are saved in paca->exslb.
923 * We assume we aren't going to take any exceptions during this procedure.
924 * We assume (DAR >> 60) == 0xc.
925 */
926 .align 7
927 _GLOBAL(do_stab_bolted)
928 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
929 std r11,PACA_EXSLB+EX_SRR0(r13) /* save SRR0 in exc. frame */
931 /* Hash to the primary group */
932 ld r10,PACASTABVIRT(r13)
933 mfspr r11,SPRN_DAR
934 srdi r11,r11,28
935 rldimi r10,r11,7,52 /* r10 = first ste of the group */
937 /* Calculate VSID */
938 /* This is a kernel address, so protovsid = ESID */
939 ASM_VSID_SCRAMBLE(r11, r9, 256M)
940 rldic r9,r11,12,16 /* r9 = vsid << 12 */
942 /* Search the primary group for a free entry */
943 1: ld r11,0(r10) /* Test valid bit of the current ste */
944 andi. r11,r11,0x80
945 beq 2f
946 addi r10,r10,16
947 andi. r11,r10,0x70
948 bne 1b
950 /* Stick for only searching the primary group for now. */
951 /* At least for now, we use a very simple random castout scheme */
952 /* Use the TB as a random number ; OR in 1 to avoid entry 0 */
953 mftb r11
954 rldic r11,r11,4,57 /* r11 = (r11 << 4) & 0x70 */
955 ori r11,r11,0x10
957 /* r10 currently points to an ste one past the group of interest */
958 /* make it point to the randomly selected entry */
959 subi r10,r10,128
960 or r10,r10,r11 /* r10 is the entry to invalidate */
962 isync /* mark the entry invalid */
963 ld r11,0(r10)
964 rldicl r11,r11,56,1 /* clear the valid bit */
965 rotldi r11,r11,8
966 std r11,0(r10)
967 sync
969 clrrdi r11,r11,28 /* Get the esid part of the ste */
970 slbie r11
972 2: std r9,8(r10) /* Store the vsid part of the ste */
973 eieio
975 mfspr r11,SPRN_DAR /* Get the new esid */
976 clrrdi r11,r11,28 /* Permits a full 32b of ESID */
977 ori r11,r11,0x90 /* Turn on valid and kp */
978 std r11,0(r10) /* Put new entry back into the stab */
980 sync
982 /* All done -- return from exception. */
983 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
984 ld r11,PACA_EXSLB+EX_SRR0(r13) /* get saved SRR0 */
986 andi. r10,r12,MSR_RI
987 beq- unrecov_slb
989 mtcrf 0x80,r9 /* restore CR */
991 mfmsr r10
992 clrrdi r10,r10,2
993 mtmsrd r10,1
995 mtspr SPRN_SRR0,r11
996 mtspr SPRN_SRR1,r12
997 ld r9,PACA_EXSLB+EX_R9(r13)
998 ld r10,PACA_EXSLB+EX_R10(r13)
999 ld r11,PACA_EXSLB+EX_R11(r13)
1000 ld r12,PACA_EXSLB+EX_R12(r13)
1001 ld r13,PACA_EXSLB+EX_R13(r13)
1002 rfid
1003 b . /* prevent speculative execution */
1005 /*
1006 * Space for CPU0's segment table.
1007 *
1008 * On iSeries, the hypervisor must fill in at least one entry before
1009 * we get control (with relocate on). The address is given to the hv
1010 * as a page number (see xLparMap below), so this must be at a
1011 * fixed address (the linker can't compute (u64)&initial_stab >>
1012 * PAGE_SHIFT).
1013 */
1014 . = STAB0_OFFSET /* 0x6000 */
1015 .globl initial_stab
1016 initial_stab:
1017 .space 4096
1019 #ifdef CONFIG_PPC_PSERIES
1020 /*
1021 * Data area reserved for FWNMI option.
1022 * This address (0x7000) is fixed by the RPA.
1023 */
1024 .= 0x7000
1025 .globl fwnmi_data_area
1026 fwnmi_data_area:
1027 #endif /* CONFIG_PPC_PSERIES */
1029 /* iSeries does not use the FWNMI stuff, so it is safe to put
1030 * this here, even if we later allow kernels that will boot on
1031 * both pSeries and iSeries */
1032 #ifdef CONFIG_PPC_ISERIES
1033 . = LPARMAP_PHYS
1034 .globl xLparMap
1035 xLparMap:
1036 .quad HvEsidsToMap /* xNumberEsids */
1037 .quad HvRangesToMap /* xNumberRanges */
1038 .quad STAB0_PAGE /* xSegmentTableOffs */
1039 .zero 40 /* xRsvd */
1040 /* xEsids (HvEsidsToMap entries of 2 quads) */
1041 .quad PAGE_OFFSET_ESID /* xKernelEsid */
1042 .quad PAGE_OFFSET_VSID /* xKernelVsid */
1043 .quad VMALLOC_START_ESID /* xKernelEsid */
1044 .quad VMALLOC_START_VSID /* xKernelVsid */
1045 /* xRanges (HvRangesToMap entries of 3 quads) */
1046 .quad HvPagesToMap /* xPages */
1047 .quad 0 /* xOffset */
1048 .quad PAGE_OFFSET_VSID << (SID_SHIFT - HW_PAGE_SHIFT) /* xVPN */
1050 #endif /* CONFIG_PPC_ISERIES */
1052 #ifdef CONFIG_PPC_PSERIES
1053 . = 0x8000
1054 #endif /* CONFIG_PPC_PSERIES */
1056 /*
1057 * On pSeries and most other platforms, secondary processors spin
1058 * in the following code.
1059 * At entry, r3 = this processor's number (physical cpu id)
1060 */
1061 _GLOBAL(generic_secondary_smp_init)
1062 mr r24,r3
1064 /* turn on 64-bit mode */
1065 bl .enable_64b_mode
1067 /* Set up a paca value for this processor. Since we have the
1068 * physical cpu id in r24, we need to search the pacas to find
1069 * which logical id maps to our physical one.
1070 */
1071 LOAD_REG_IMMEDIATE(r13, paca) /* Get base vaddr of paca array */
1072 li r5,0 /* logical cpu id */
1073 1: lhz r6,PACAHWCPUID(r13) /* Load HW procid from paca */
1074 cmpw r6,r24 /* Compare to our id */
1075 beq 2f
1076 addi r13,r13,PACA_SIZE /* Loop to next PACA on miss */
1077 addi r5,r5,1
1078 cmpwi r5,NR_CPUS
1079 blt 1b
1081 mr r3,r24 /* not found, copy phys to r3 */
1082 b .kexec_wait /* next kernel might do better */
1084 2: mtspr SPRN_SPRG3,r13 /* Save vaddr of paca in SPRG3 */
1085 /* From now on, r24 is expected to be logical cpuid */
1086 mr r24,r5
1087 3: HMT_LOW
1088 lbz r23,PACAPROCSTART(r13) /* Test if this processor should */
1089 /* start. */
1090 sync
1092 #ifndef CONFIG_SMP
1093 b 3b /* Never go on non-SMP */
1094 #else
1095 cmpwi 0,r23,0
1096 beq 3b /* Loop until told to go */
1098 /* See if we need to call a cpu state restore handler */
1099 LOAD_REG_IMMEDIATE(r23, cur_cpu_spec)
1100 ld r23,0(r23)
1101 ld r23,CPU_SPEC_RESTORE(r23)
1102 cmpdi 0,r23,0
1103 beq 4f
1104 ld r23,0(r23)
1105 mtctr r23
1106 bctrl
1108 4: /* Create a temp kernel stack for use before relocation is on. */
1109 ld r1,PACAEMERGSP(r13)
1110 subi r1,r1,STACK_FRAME_OVERHEAD
1112 b __secondary_start
1113 #endif
1115 _STATIC(__mmu_off)
1116 mfmsr r3
1117 andi. r0,r3,MSR_IR|MSR_DR
1118 beqlr
1119 andc r3,r3,r0
1120 mtspr SPRN_SRR0,r4
1121 mtspr SPRN_SRR1,r3
1122 sync
1123 rfid
1124 b . /* prevent speculative execution */
1127 /*
1128 * Here is our main kernel entry point. We support currently 2 kind of entries
1129 * depending on the value of r5.
1130 *
1131 * r5 != NULL -> OF entry, we go to prom_init, "legacy" parameter content
1132 * in r3...r7
1133 *
1134 * r5 == NULL -> kexec style entry. r3 is a physical pointer to the
1135 * DT block, r4 is a physical pointer to the kernel itself
1136 *
1137 */
1138 _GLOBAL(__start_initialization_multiplatform)
1139 /*
1140 * Are we booted from a PROM Of-type client-interface ?
1141 */
1142 cmpldi cr0,r5,0
1143 beq 1f
1144 b .__boot_from_prom /* yes -> prom */
1145 1:
1146 /* Save parameters */
1147 mr r31,r3
1148 mr r30,r4
1150 /* Make sure we are running in 64 bits mode */
1151 bl .enable_64b_mode
1153 /* Setup some critical 970 SPRs before switching MMU off */
1154 mfspr r0,SPRN_PVR
1155 srwi r0,r0,16
1156 cmpwi r0,0x39 /* 970 */
1157 beq 1f
1158 cmpwi r0,0x3c /* 970FX */
1159 beq 1f
1160 cmpwi r0,0x44 /* 970MP */
1161 beq 1f
1162 cmpwi r0,0x45 /* 970GX */
1163 bne 2f
1164 1: bl .__cpu_preinit_ppc970
1165 2:
1167 /* Switch off MMU if not already */
1168 LOAD_REG_IMMEDIATE(r4, .__after_prom_start - KERNELBASE)
1169 add r4,r4,r30
1170 bl .__mmu_off
1171 b .__after_prom_start
1173 _INIT_STATIC(__boot_from_prom)
1174 /* Save parameters */
1175 mr r31,r3
1176 mr r30,r4
1177 mr r29,r5
1178 mr r28,r6
1179 mr r27,r7
1181 /*
1182 * Align the stack to 16-byte boundary
1183 * Depending on the size and layout of the ELF sections in the initial
1184 * boot binary, the stack pointer will be unalignet on PowerMac
1185 */
1186 rldicr r1,r1,0,59
1188 /* Make sure we are running in 64 bits mode */
1189 bl .enable_64b_mode
1191 /* put a relocation offset into r3 */
1192 bl .reloc_offset
1194 LOAD_REG_IMMEDIATE(r2,__toc_start)
1195 addi r2,r2,0x4000
1196 addi r2,r2,0x4000
1198 /* Relocate the TOC from a virt addr to a real addr */
1199 add r2,r2,r3
1201 /* Restore parameters */
1202 mr r3,r31
1203 mr r4,r30
1204 mr r5,r29
1205 mr r6,r28
1206 mr r7,r27
1208 /* Do all of the interaction with OF client interface */
1209 bl .prom_init
1210 /* We never return */
1211 trap
1213 _STATIC(__after_prom_start)
1215 /*
1216 * We need to run with __start at physical address PHYSICAL_START.
1217 * This will leave some code in the first 256B of
1218 * real memory, which are reserved for software use.
1219 * The remainder of the first page is loaded with the fixed
1220 * interrupt vectors. The next two pages are filled with
1221 * unknown exception placeholders.
1222 *
1223 * Note: This process overwrites the OF exception vectors.
1224 * r26 == relocation offset
1225 * r27 == KERNELBASE
1226 */
1227 bl .reloc_offset
1228 mr r26,r3
1229 LOAD_REG_IMMEDIATE(r27, KERNELBASE)
1231 LOAD_REG_IMMEDIATE(r3, PHYSICAL_START) /* target addr */
1233 // XXX FIXME: Use phys returned by OF (r30)
1234 add r4,r27,r26 /* source addr */
1235 /* current address of _start */
1236 /* i.e. where we are running */
1237 /* the source addr */
1239 cmpdi r4,0 /* In some cases the loader may */
1240 bne 1f
1241 b .start_here_multiplatform /* have already put us at zero */
1242 /* so we can skip the copy. */
1243 1: LOAD_REG_IMMEDIATE(r5,copy_to_here) /* # bytes of memory to copy */
1244 sub r5,r5,r27
1246 li r6,0x100 /* Start offset, the first 0x100 */
1247 /* bytes were copied earlier. */
1249 bl .copy_and_flush /* copy the first n bytes */
1250 /* this includes the code being */
1251 /* executed here. */
1253 LOAD_REG_IMMEDIATE(r0, 4f) /* Jump to the copy of this code */
1254 mtctr r0 /* that we just made/relocated */
1255 bctr
1257 4: LOAD_REG_IMMEDIATE(r5,klimit)
1258 add r5,r5,r26
1259 ld r5,0(r5) /* get the value of klimit */
1260 sub r5,r5,r27
1261 bl .copy_and_flush /* copy the rest */
1262 b .start_here_multiplatform
1264 /*
1265 * Copy routine used to copy the kernel to start at physical address 0
1266 * and flush and invalidate the caches as needed.
1267 * r3 = dest addr, r4 = source addr, r5 = copy limit, r6 = start offset
1268 * on exit, r3, r4, r5 are unchanged, r6 is updated to be >= r5.
1269 *
1270 * Note: this routine *only* clobbers r0, r6 and lr
1271 */
1272 _GLOBAL(copy_and_flush)
1273 addi r5,r5,-8
1274 addi r6,r6,-8
1275 4: li r0,8 /* Use the smallest common */
1276 /* denominator cache line */
1277 /* size. This results in */
1278 /* extra cache line flushes */
1279 /* but operation is correct. */
1280 /* Can't get cache line size */
1281 /* from NACA as it is being */
1282 /* moved too. */
1284 mtctr r0 /* put # words/line in ctr */
1285 3: addi r6,r6,8 /* copy a cache line */
1286 ldx r0,r6,r4
1287 stdx r0,r6,r3
1288 bdnz 3b
1289 dcbst r6,r3 /* write it to memory */
1290 sync
1291 icbi r6,r3 /* flush the icache line */
1292 cmpld 0,r6,r5
1293 blt 4b
1294 sync
1295 addi r5,r5,8
1296 addi r6,r6,8
1297 blr
1299 .align 8
1300 copy_to_here:
1302 #ifdef CONFIG_SMP
1303 #ifdef CONFIG_PPC_PMAC
1304 /*
1305 * On PowerMac, secondary processors starts from the reset vector, which
1306 * is temporarily turned into a call to one of the functions below.
1307 */
1308 .section ".text";
1309 .align 2 ;
1311 .globl __secondary_start_pmac_0
1312 __secondary_start_pmac_0:
1313 /* NB the entries for cpus 0, 1, 2 must each occupy 8 bytes. */
1314 li r24,0
1315 b 1f
1316 li r24,1
1317 b 1f
1318 li r24,2
1319 b 1f
1320 li r24,3
1321 1:
1323 _GLOBAL(pmac_secondary_start)
1324 /* turn on 64-bit mode */
1325 bl .enable_64b_mode
1327 /* Copy some CPU settings from CPU 0 */
1328 bl .__restore_cpu_ppc970
1330 /* pSeries do that early though I don't think we really need it */
1331 mfmsr r3
1332 ori r3,r3,MSR_RI
1333 mtmsrd r3 /* RI on */
1335 /* Set up a paca value for this processor. */
1336 LOAD_REG_IMMEDIATE(r4, paca) /* Get base vaddr of paca array */
1337 mulli r13,r24,PACA_SIZE /* Calculate vaddr of right paca */
1338 add r13,r13,r4 /* for this processor. */
1339 mtspr SPRN_SPRG3,r13 /* Save vaddr of paca in SPRG3 */
1341 /* Create a temp kernel stack for use before relocation is on. */
1342 ld r1,PACAEMERGSP(r13)
1343 subi r1,r1,STACK_FRAME_OVERHEAD
1345 b __secondary_start
1347 #endif /* CONFIG_PPC_PMAC */
1349 /*
1350 * This function is called after the master CPU has released the
1351 * secondary processors. The execution environment is relocation off.
1352 * The paca for this processor has the following fields initialized at
1353 * this point:
1354 * 1. Processor number
1355 * 2. Segment table pointer (virtual address)
1356 * On entry the following are set:
1357 * r1 = stack pointer. vaddr for iSeries, raddr (temp stack) for pSeries
1358 * r24 = cpu# (in Linux terms)
1359 * r13 = paca virtual address
1360 * SPRG3 = paca virtual address
1361 */
1362 .globl __secondary_start
1363 __secondary_start:
1364 /* Set thread priority to MEDIUM */
1365 HMT_MEDIUM
1367 /* Load TOC */
1368 ld r2,PACATOC(r13)
1370 /* Do early setup for that CPU (stab, slb, hash table pointer) */
1371 bl .early_setup_secondary
1373 /* Initialize the kernel stack. Just a repeat for iSeries. */
1374 LOAD_REG_ADDR(r3, current_set)
1375 sldi r28,r24,3 /* get current_set[cpu#] */
1376 ldx r1,r3,r28
1377 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
1378 std r1,PACAKSAVE(r13)
1380 /* Clear backchain so we get nice backtraces */
1381 li r7,0
1382 mtlr r7
1384 /* enable MMU and jump to start_secondary */
1385 LOAD_REG_ADDR(r3, .start_secondary_prolog)
1386 LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
1387 #ifdef CONFIG_PPC_ISERIES
1388 BEGIN_FW_FTR_SECTION
1389 ori r4,r4,MSR_EE
1390 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
1391 #endif
1392 BEGIN_FW_FTR_SECTION
1393 stb r7,PACASOFTIRQEN(r13)
1394 stb r7,PACAHARDIRQEN(r13)
1395 END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
1397 mtspr SPRN_SRR0,r3
1398 mtspr SPRN_SRR1,r4
1399 rfid
1400 b . /* prevent speculative execution */
1402 /*
1403 * Running with relocation on at this point. All we want to do is
1404 * zero the stack back-chain pointer before going into C code.
1405 */
1406 _GLOBAL(start_secondary_prolog)
1407 li r3,0
1408 std r3,0(r1) /* Zero the stack frame pointer */
1409 bl .start_secondary
1410 b .
1411 #endif
1413 /*
1414 * This subroutine clobbers r11 and r12
1415 */
1416 _GLOBAL(enable_64b_mode)
1417 mfmsr r11 /* grab the current MSR */
1418 li r12,1
1419 rldicr r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
1420 or r11,r11,r12
1421 li r12,1
1422 rldicr r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
1423 or r11,r11,r12
1424 mtmsrd r11
1425 isync
1426 blr
1428 /*
1429 * This is where the main kernel code starts.
1430 */
1431 _INIT_STATIC(start_here_multiplatform)
1432 /* get a new offset, now that the kernel has moved. */
1433 bl .reloc_offset
1434 mr r26,r3
1436 /* Clear out the BSS. It may have been done in prom_init,
1437 * already but that's irrelevant since prom_init will soon
1438 * be detached from the kernel completely. Besides, we need
1439 * to clear it now for kexec-style entry.
1440 */
1441 LOAD_REG_IMMEDIATE(r11,__bss_stop)
1442 LOAD_REG_IMMEDIATE(r8,__bss_start)
1443 sub r11,r11,r8 /* bss size */
1444 addi r11,r11,7 /* round up to an even double word */
1445 rldicl. r11,r11,61,3 /* shift right by 3 */
1446 beq 4f
1447 addi r8,r8,-8
1448 li r0,0
1449 mtctr r11 /* zero this many doublewords */
1450 3: stdu r0,8(r8)
1451 bdnz 3b
1452 4:
1454 mfmsr r6
1455 ori r6,r6,MSR_RI
1456 mtmsrd r6 /* RI on */
1458 /* The following gets the stack and TOC set up with the regs */
1459 /* pointing to the real addr of the kernel stack. This is */
1460 /* all done to support the C function call below which sets */
1461 /* up the htab. This is done because we have relocated the */
1462 /* kernel but are still running in real mode. */
1464 LOAD_REG_IMMEDIATE(r3,init_thread_union)
1465 add r3,r3,r26
1467 /* set up a stack pointer (physical address) */
1468 addi r1,r3,THREAD_SIZE
1469 li r0,0
1470 stdu r0,-STACK_FRAME_OVERHEAD(r1)
1472 /* set up the TOC (physical address) */
1473 LOAD_REG_IMMEDIATE(r2,__toc_start)
1474 addi r2,r2,0x4000
1475 addi r2,r2,0x4000
1476 add r2,r2,r26
1478 /* Do very early kernel initializations, including initial hash table,
1479 * stab and slb setup before we turn on relocation. */
1481 /* Restore parameters passed from prom_init/kexec */
1482 mr r3,r31
1483 bl .early_setup
1485 LOAD_REG_IMMEDIATE(r3, .start_here_common)
1486 LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
1487 mtspr SPRN_SRR0,r3
1488 mtspr SPRN_SRR1,r4
1489 rfid
1490 b . /* prevent speculative execution */
1492 /* This is where all platforms converge execution */
1493 _INIT_GLOBAL(start_here_common)
1494 /* relocation is on at this point */
1496 /* The following code sets up the SP and TOC now that we are */
1497 /* running with translation enabled. */
1499 LOAD_REG_IMMEDIATE(r3,init_thread_union)
1501 /* set up the stack */
1502 addi r1,r3,THREAD_SIZE
1503 li r0,0
1504 stdu r0,-STACK_FRAME_OVERHEAD(r1)
1506 /* ptr to current */
1507 LOAD_REG_IMMEDIATE(r4, init_task)
1508 std r4,PACACURRENT(r13)
1510 /* Load the TOC */
1511 ld r2,PACATOC(r13)
1512 std r1,PACAKSAVE(r13)
1514 bl .setup_system
1516 /* Load up the kernel context */
1517 5:
1518 li r5,0
1519 stb r5,PACASOFTIRQEN(r13) /* Soft Disabled */
1520 #ifdef CONFIG_PPC_ISERIES
1521 BEGIN_FW_FTR_SECTION
1522 mfmsr r5
1523 ori r5,r5,MSR_EE /* Hard Enabled */
1524 mtmsrd r5
1525 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
1526 #endif
1527 BEGIN_FW_FTR_SECTION
1528 stb r5,PACAHARDIRQEN(r13)
1529 END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
1531 bl .start_kernel
1533 /* Not reached */
1534 BUG_OPCODE
1536 /*
1537 * We put a few things here that have to be page-aligned.
1538 * This stuff goes at the beginning of the bss, which is page-aligned.
1539 */
1540 .section ".bss"
1542 .align PAGE_SHIFT
1544 .globl empty_zero_page
1545 empty_zero_page:
1546 .space PAGE_SIZE
1548 .globl swapper_pg_dir
1549 swapper_pg_dir:
1550 .space PGD_TABLE_SIZE