| blob | 3065b472b95db7151912d8956933589fce4b736c |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 *
5 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
6 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
7 * Adapted for Power Macintosh by Paul Mackerras.
8 * Low-level exception handlers and MMU support
9 * rewritten by Paul Mackerras.
10 * Copyright (C) 1996 Paul Mackerras.
11 *
12 * Adapted for 64bit PowerPC by Dave Engebretsen, Peter Bergner, and
13 * Mike Corrigan {engebret|bergner|mikejc}@us.ibm.com
14 *
15 * This file contains the low-level support and setup for the
16 * PowerPC-64 platform, including trap and interrupt dispatch.
17 *
18 * This program is free software; you can redistribute it and/or
19 * modify it under the terms of the GNU General Public License
20 * as published by the Free Software Foundation; either version
21 * 2 of the License, or (at your option) any later version.
22 */
24 #include <linux/threads.h>
25 #include <asm/reg.h>
26 #include <asm/page.h>
27 #include <asm/mmu.h>
28 #include <asm/ppc_asm.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/bug.h>
31 #include <asm/cputable.h>
32 #include <asm/setup.h>
33 #include <asm/hvcall.h>
34 #include <asm/iseries/lpar_map.h>
35 #include <asm/thread_info.h>
37 #ifdef CONFIG_PPC_ISERIES
38 #define DO_SOFT_DISABLE
39 #endif
41 /*
42 * We layout physical memory as follows:
43 * 0x0000 - 0x00ff : Secondary processor spin code
44 * 0x0100 - 0x2fff : pSeries Interrupt prologs
45 * 0x3000 - 0x5fff : interrupt support, iSeries and common interrupt prologs
46 * 0x6000 - 0x6fff : Initial (CPU0) segment table
47 * 0x7000 - 0x7fff : FWNMI data area
48 * 0x8000 - : Early init and support code
49 */
51 /*
52 * SPRG Usage
53 *
54 * Register Definition
55 *
56 * SPRG0 reserved for hypervisor
57 * SPRG1 temp - used to save gpr
58 * SPRG2 temp - used to save gpr
59 * SPRG3 virt addr of paca
60 */
62 /*
63 * Entering into this code we make the following assumptions:
64 * For pSeries:
65 * 1. The MMU is off & open firmware is running in real mode.
66 * 2. The kernel is entered at __start
67 *
68 * For iSeries:
69 * 1. The MMU is on (as it always is for iSeries)
70 * 2. The kernel is entered at system_reset_iSeries
71 */
73 .text
74 .globl _stext
75 _stext:
76 #ifdef CONFIG_PPC_MULTIPLATFORM
77 _GLOBAL(__start)
78 /* NOP this out unconditionally */
79 BEGIN_FTR_SECTION
80 b .__start_initialization_multiplatform
81 END_FTR_SECTION(0, 1)
82 #endif /* CONFIG_PPC_MULTIPLATFORM */
84 /* Catch branch to 0 in real mode */
85 trap
87 /* Secondary processors spin on this value until it goes to 1. */
88 .globl __secondary_hold_spinloop
89 __secondary_hold_spinloop:
90 .llong 0x0
92 /* Secondary processors write this value with their cpu # */
93 /* after they enter the spin loop immediately below. */
94 .globl __secondary_hold_acknowledge
95 __secondary_hold_acknowledge:
96 .llong 0x0
98 #ifdef CONFIG_PPC_ISERIES
99 /*
100 * At offset 0x20, there is a pointer to iSeries LPAR data.
101 * This is required by the hypervisor
102 */
103 . = 0x20
104 .llong hvReleaseData-KERNELBASE
105 #endif /* CONFIG_PPC_ISERIES */
107 . = 0x60
108 /*
109 * The following code is used on pSeries to hold secondary processors
110 * in a spin loop after they have been freed from OpenFirmware, but
111 * before the bulk of the kernel has been relocated. This code
112 * is relocated to physical address 0x60 before prom_init is run.
113 * All of it must fit below the first exception vector at 0x100.
114 */
115 _GLOBAL(__secondary_hold)
116 mfmsr r24
117 ori r24,r24,MSR_RI
118 mtmsrd r24 /* RI on */
120 /* Grab our physical cpu number */
121 mr r24,r3
123 /* Tell the master cpu we're here */
124 /* Relocation is off & we are located at an address less */
125 /* than 0x100, so only need to grab low order offset. */
126 std r24,__secondary_hold_acknowledge@l(0)
127 sync
129 /* All secondary cpus wait here until told to start. */
130 100: ld r4,__secondary_hold_spinloop@l(0)
131 cmpdi 0,r4,1
132 bne 100b
134 #if defined(CONFIG_SMP) || defined(CONFIG_KEXEC)
135 LOAD_REG_IMMEDIATE(r4, .generic_secondary_smp_init)
136 mtctr r4
137 mr r3,r24
138 bctr
139 #else
140 BUG_OPCODE
141 #endif
143 /* This value is used to mark exception frames on the stack. */
144 .section ".toc","aw"
145 exception_marker:
146 .tc ID_72656773_68657265[TC],0x7265677368657265
147 .text
149 /*
150 * The following macros define the code that appears as
151 * the prologue to each of the exception handlers. They
152 * are split into two parts to allow a single kernel binary
153 * to be used for pSeries and iSeries.
154 * LOL. One day... - paulus
155 */
157 /*
158 * We make as much of the exception code common between native
159 * exception handlers (including pSeries LPAR) and iSeries LPAR
160 * implementations as possible.
161 */
163 /*
164 * This is the start of the interrupt handlers for pSeries
165 * This code runs with relocation off.
166 */
167 #define EX_R9 0
168 #define EX_R10 8
169 #define EX_R11 16
170 #define EX_R12 24
171 #define EX_R13 32
172 #define EX_SRR0 40
173 #define EX_DAR 48
174 #define EX_DSISR 56
175 #define EX_CCR 60
176 #define EX_R3 64
177 #define EX_LR 72
179 /*
180 * We're short on space and time in the exception prolog, so we can't
181 * use the normal SET_REG_IMMEDIATE macro. Normally we just need the
182 * low halfword of the address, but for Kdump we need the whole low
183 * word.
184 */
185 #ifdef CONFIG_CRASH_DUMP
186 #define LOAD_HANDLER(reg, label) \
187 oris reg,reg,(label)@h; /* virt addr of handler ... */ \
188 ori reg,reg,(label)@l; /* .. and the rest */
189 #else
190 #define LOAD_HANDLER(reg, label) \
191 ori reg,reg,(label)@l; /* virt addr of handler ... */
192 #endif
194 /*
195 * Equal to EXCEPTION_PROLOG_PSERIES, except that it forces 64bit mode.
196 * The firmware calls the registered system_reset_fwnmi and
197 * machine_check_fwnmi handlers in 32bit mode if the cpu happens to run
198 * a 32bit application at the time of the event.
199 * This firmware bug is present on POWER4 and JS20.
200 */
201 #define EXCEPTION_PROLOG_PSERIES_FORCE_64BIT(area, label) \
202 mfspr r13,SPRN_SPRG3; /* get paca address into r13 */ \
203 std r9,area+EX_R9(r13); /* save r9 - r12 */ \
204 std r10,area+EX_R10(r13); \
205 std r11,area+EX_R11(r13); \
206 std r12,area+EX_R12(r13); \
207 mfspr r9,SPRN_SPRG1; \
208 std r9,area+EX_R13(r13); \
209 mfcr r9; \
210 clrrdi r12,r13,32; /* get high part of &label */ \
211 mfmsr r10; \
212 /* force 64bit mode */ \
213 li r11,5; /* MSR_SF_LG|MSR_ISF_LG */ \
214 rldimi r10,r11,61,0; /* insert into top 3 bits */ \
215 /* done 64bit mode */ \
216 mfspr r11,SPRN_SRR0; /* save SRR0 */ \
217 LOAD_HANDLER(r12,label) \
218 ori r10,r10,MSR_IR|MSR_DR|MSR_RI; \
219 mtspr SPRN_SRR0,r12; \
220 mfspr r12,SPRN_SRR1; /* and SRR1 */ \
221 mtspr SPRN_SRR1,r10; \
222 rfid; \
223 b . /* prevent speculative execution */
225 #define EXCEPTION_PROLOG_PSERIES(area, label) \
226 mfspr r13,SPRN_SPRG3; /* get paca address into r13 */ \
227 std r9,area+EX_R9(r13); /* save r9 - r12 */ \
228 std r10,area+EX_R10(r13); \
229 std r11,area+EX_R11(r13); \
230 std r12,area+EX_R12(r13); \
231 mfspr r9,SPRN_SPRG1; \
232 std r9,area+EX_R13(r13); \
233 mfcr r9; \
234 clrrdi r12,r13,32; /* get high part of &label */ \
235 mfmsr r10; \
236 mfspr r11,SPRN_SRR0; /* save SRR0 */ \
237 LOAD_HANDLER(r12,label) \
238 ori r10,r10,MSR_IR|MSR_DR|MSR_RI; \
239 mtspr SPRN_SRR0,r12; \
240 mfspr r12,SPRN_SRR1; /* and SRR1 */ \
241 mtspr SPRN_SRR1,r10; \
242 rfid; \
243 b . /* prevent speculative execution */
245 /*
246 * This is the start of the interrupt handlers for iSeries
247 * This code runs with relocation on.
248 */
249 #define EXCEPTION_PROLOG_ISERIES_1(area) \
250 mfspr r13,SPRN_SPRG3; /* get paca address into r13 */ \
251 std r9,area+EX_R9(r13); /* save r9 - r12 */ \
252 std r10,area+EX_R10(r13); \
253 std r11,area+EX_R11(r13); \
254 std r12,area+EX_R12(r13); \
255 mfspr r9,SPRN_SPRG1; \
256 std r9,area+EX_R13(r13); \
257 mfcr r9
259 #define EXCEPTION_PROLOG_ISERIES_2 \
260 mfmsr r10; \
261 ld r12,PACALPPACAPTR(r13); \
262 ld r11,LPPACASRR0(r12); \
263 ld r12,LPPACASRR1(r12); \
264 ori r10,r10,MSR_RI; \
265 mtmsrd r10,1
267 /*
268 * The common exception prolog is used for all except a few exceptions
269 * such as a segment miss on a kernel address. We have to be prepared
270 * to take another exception from the point where we first touch the
271 * kernel stack onwards.
272 *
273 * On entry r13 points to the paca, r9-r13 are saved in the paca,
274 * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and
275 * SRR1, and relocation is on.
276 */
277 #define EXCEPTION_PROLOG_COMMON(n, area) \
278 andi. r10,r12,MSR_PR; /* See if coming from user */ \
279 mr r10,r1; /* Save r1 */ \
280 subi r1,r1,INT_FRAME_SIZE; /* alloc frame on kernel stack */ \
281 beq- 1f; \
282 ld r1,PACAKSAVE(r13); /* kernel stack to use */ \
283 1: cmpdi cr1,r1,0; /* check if r1 is in userspace */ \
284 bge- cr1,bad_stack; /* abort if it is */ \
285 std r9,_CCR(r1); /* save CR in stackframe */ \
286 std r11,_NIP(r1); /* save SRR0 in stackframe */ \
287 std r12,_MSR(r1); /* save SRR1 in stackframe */ \
288 std r10,0(r1); /* make stack chain pointer */ \
289 std r0,GPR0(r1); /* save r0 in stackframe */ \
290 std r10,GPR1(r1); /* save r1 in stackframe */ \
291 ACCOUNT_CPU_USER_ENTRY(r9, r10); \
292 std r2,GPR2(r1); /* save r2 in stackframe */ \
293 SAVE_4GPRS(3, r1); /* save r3 - r6 in stackframe */ \
294 SAVE_2GPRS(7, r1); /* save r7, r8 in stackframe */ \
295 ld r9,area+EX_R9(r13); /* move r9, r10 to stackframe */ \
296 ld r10,area+EX_R10(r13); \
297 std r9,GPR9(r1); \
298 std r10,GPR10(r1); \
299 ld r9,area+EX_R11(r13); /* move r11 - r13 to stackframe */ \
300 ld r10,area+EX_R12(r13); \
301 ld r11,area+EX_R13(r13); \
302 std r9,GPR11(r1); \
303 std r10,GPR12(r1); \
304 std r11,GPR13(r1); \
305 ld r2,PACATOC(r13); /* get kernel TOC into r2 */ \
306 mflr r9; /* save LR in stackframe */ \
307 std r9,_LINK(r1); \
308 mfctr r10; /* save CTR in stackframe */ \
309 std r10,_CTR(r1); \
310 mfspr r11,SPRN_XER; /* save XER in stackframe */ \
311 std r11,_XER(r1); \
312 li r9,(n)+1; \
313 std r9,_TRAP(r1); /* set trap number */ \
314 li r10,0; \
315 ld r11,exception_marker@toc(r2); \
316 std r10,RESULT(r1); /* clear regs->result */ \
317 std r11,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame */
319 /*
320 * Exception vectors.
321 */
322 #define STD_EXCEPTION_PSERIES(n, label) \
323 . = n; \
324 .globl label##_pSeries; \
325 label##_pSeries: \
326 HMT_MEDIUM; \
327 mtspr SPRN_SPRG1,r13; /* save r13 */ \
328 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common)
330 #define HSTD_EXCEPTION_PSERIES(n, label) \
331 . = n; \
332 .globl label##_pSeries; \
333 label##_pSeries: \
334 HMT_MEDIUM; \
335 mtspr SPRN_SPRG1,r20; /* save r20 */ \
336 mfspr r20,SPRN_HSRR0; /* copy HSRR0 to SRR0 */ \
337 mtspr SPRN_SRR0,r20; \
338 mfspr r20,SPRN_HSRR1; /* copy HSRR0 to SRR0 */ \
339 mtspr SPRN_SRR1,r20; \
340 mfspr r20,SPRN_SPRG1; /* restore r20 */ \
341 mtspr SPRN_SPRG1,r13; /* save r13 */ \
342 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common)
345 #define STD_EXCEPTION_ISERIES(n, label, area) \
346 .globl label##_iSeries; \
347 label##_iSeries: \
348 HMT_MEDIUM; \
349 mtspr SPRN_SPRG1,r13; /* save r13 */ \
350 EXCEPTION_PROLOG_ISERIES_1(area); \
351 EXCEPTION_PROLOG_ISERIES_2; \
352 b label##_common
354 #define MASKABLE_EXCEPTION_ISERIES(n, label) \
355 .globl label##_iSeries; \
356 label##_iSeries: \
357 HMT_MEDIUM; \
358 mtspr SPRN_SPRG1,r13; /* save r13 */ \
359 EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN); \
360 lbz r10,PACAPROCENABLED(r13); \
361 cmpwi 0,r10,0; \
362 beq- label##_iSeries_masked; \
363 EXCEPTION_PROLOG_ISERIES_2; \
364 b label##_common; \
366 #ifdef DO_SOFT_DISABLE
367 #define DISABLE_INTS \
368 lbz r10,PACAPROCENABLED(r13); \
369 li r11,0; \
370 std r10,SOFTE(r1); \
371 mfmsr r10; \
372 stb r11,PACAPROCENABLED(r13); \
373 ori r10,r10,MSR_EE; \
374 mtmsrd r10,1
376 #define ENABLE_INTS \
377 lbz r10,PACAPROCENABLED(r13); \
378 mfmsr r11; \
379 std r10,SOFTE(r1); \
380 ori r11,r11,MSR_EE; \
381 mtmsrd r11,1
383 #else /* hard enable/disable interrupts */
384 #define DISABLE_INTS
386 #define ENABLE_INTS \
387 ld r12,_MSR(r1); \
388 mfmsr r11; \
389 rlwimi r11,r12,0,MSR_EE; \
390 mtmsrd r11,1
392 #endif
394 #define STD_EXCEPTION_COMMON(trap, label, hdlr) \
395 .align 7; \
396 .globl label##_common; \
397 label##_common: \
398 EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN); \
399 DISABLE_INTS; \
400 bl .save_nvgprs; \
401 addi r3,r1,STACK_FRAME_OVERHEAD; \
402 bl hdlr; \
403 b .ret_from_except
405 /*
406 * Like STD_EXCEPTION_COMMON, but for exceptions that can occur
407 * in the idle task and therefore need the special idle handling.
408 */
409 #define STD_EXCEPTION_COMMON_IDLE(trap, label, hdlr) \
410 .align 7; \
411 .globl label##_common; \
412 label##_common: \
413 EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN); \
414 FINISH_NAP; \
415 DISABLE_INTS; \
416 bl .save_nvgprs; \
417 addi r3,r1,STACK_FRAME_OVERHEAD; \
418 bl hdlr; \
419 b .ret_from_except
421 #define STD_EXCEPTION_COMMON_LITE(trap, label, hdlr) \
422 .align 7; \
423 .globl label##_common; \
424 label##_common: \
425 EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN); \
426 FINISH_NAP; \
427 DISABLE_INTS; \
428 bl .ppc64_runlatch_on; \
429 addi r3,r1,STACK_FRAME_OVERHEAD; \
430 bl hdlr; \
431 b .ret_from_except_lite
433 /*
434 * When the idle code in power4_idle puts the CPU into NAP mode,
435 * it has to do so in a loop, and relies on the external interrupt
436 * and decrementer interrupt entry code to get it out of the loop.
437 * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags
438 * to signal that it is in the loop and needs help to get out.
439 */
440 #ifdef CONFIG_PPC_970_NAP
441 #define FINISH_NAP \
442 BEGIN_FTR_SECTION \
443 clrrdi r11,r1,THREAD_SHIFT; \
444 ld r9,TI_LOCAL_FLAGS(r11); \
445 andi. r10,r9,_TLF_NAPPING; \
446 bnel power4_fixup_nap; \
447 END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
448 #else
449 #define FINISH_NAP
450 #endif
452 /*
453 * Start of pSeries system interrupt routines
454 */
455 . = 0x100
456 .globl __start_interrupts
457 __start_interrupts:
459 STD_EXCEPTION_PSERIES(0x100, system_reset)
461 . = 0x200
462 _machine_check_pSeries:
463 HMT_MEDIUM
464 mtspr SPRN_SPRG1,r13 /* save r13 */
465 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
467 . = 0x300
468 .globl data_access_pSeries
469 data_access_pSeries:
470 HMT_MEDIUM
471 mtspr SPRN_SPRG1,r13
472 BEGIN_FTR_SECTION
473 mtspr SPRN_SPRG2,r12
474 mfspr r13,SPRN_DAR
475 mfspr r12,SPRN_DSISR
476 srdi r13,r13,60
477 rlwimi r13,r12,16,0x20
478 mfcr r12
479 cmpwi r13,0x2c
480 beq .do_stab_bolted_pSeries
481 mtcrf 0x80,r12
482 mfspr r12,SPRN_SPRG2
483 END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
484 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common)
486 . = 0x380
487 .globl data_access_slb_pSeries
488 data_access_slb_pSeries:
489 HMT_MEDIUM
490 mtspr SPRN_SPRG1,r13
491 mfspr r13,SPRN_SPRG3 /* get paca address into r13 */
492 std r3,PACA_EXSLB+EX_R3(r13)
493 mfspr r3,SPRN_DAR
494 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
495 mfcr r9
496 #ifdef __DISABLED__
497 /* Keep that around for when we re-implement dynamic VSIDs */
498 cmpdi r3,0
499 bge slb_miss_user_pseries
500 #endif /* __DISABLED__ */
501 std r10,PACA_EXSLB+EX_R10(r13)
502 std r11,PACA_EXSLB+EX_R11(r13)
503 std r12,PACA_EXSLB+EX_R12(r13)
504 mfspr r10,SPRN_SPRG1
505 std r10,PACA_EXSLB+EX_R13(r13)
506 mfspr r12,SPRN_SRR1 /* and SRR1 */
507 b .slb_miss_realmode /* Rel. branch works in real mode */
509 STD_EXCEPTION_PSERIES(0x400, instruction_access)
511 . = 0x480
512 .globl instruction_access_slb_pSeries
513 instruction_access_slb_pSeries:
514 HMT_MEDIUM
515 mtspr SPRN_SPRG1,r13
516 mfspr r13,SPRN_SPRG3 /* get paca address into r13 */
517 std r3,PACA_EXSLB+EX_R3(r13)
518 mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
519 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
520 mfcr r9
521 #ifdef __DISABLED__
522 /* Keep that around for when we re-implement dynamic VSIDs */
523 cmpdi r3,0
524 bge slb_miss_user_pseries
525 #endif /* __DISABLED__ */
526 std r10,PACA_EXSLB+EX_R10(r13)
527 std r11,PACA_EXSLB+EX_R11(r13)
528 std r12,PACA_EXSLB+EX_R12(r13)
529 mfspr r10,SPRN_SPRG1
530 std r10,PACA_EXSLB+EX_R13(r13)
531 mfspr r12,SPRN_SRR1 /* and SRR1 */
532 b .slb_miss_realmode /* Rel. branch works in real mode */
534 STD_EXCEPTION_PSERIES(0x500, hardware_interrupt)
535 STD_EXCEPTION_PSERIES(0x600, alignment)
536 STD_EXCEPTION_PSERIES(0x700, program_check)
537 STD_EXCEPTION_PSERIES(0x800, fp_unavailable)
538 STD_EXCEPTION_PSERIES(0x900, decrementer)
539 STD_EXCEPTION_PSERIES(0xa00, trap_0a)
540 STD_EXCEPTION_PSERIES(0xb00, trap_0b)
542 . = 0xc00
543 .globl system_call_pSeries
544 system_call_pSeries:
545 HMT_MEDIUM
546 mr r9,r13
547 mfmsr r10
548 mfspr r13,SPRN_SPRG3
549 mfspr r11,SPRN_SRR0
550 clrrdi r12,r13,32
551 oris r12,r12,system_call_common@h
552 ori r12,r12,system_call_common@l
553 mtspr SPRN_SRR0,r12
554 ori r10,r10,MSR_IR|MSR_DR|MSR_RI
555 mfspr r12,SPRN_SRR1
556 mtspr SPRN_SRR1,r10
557 rfid
558 b . /* prevent speculative execution */
560 STD_EXCEPTION_PSERIES(0xd00, single_step)
561 STD_EXCEPTION_PSERIES(0xe00, trap_0e)
563 /* We need to deal with the Altivec unavailable exception
564 * here which is at 0xf20, thus in the middle of the
565 * prolog code of the PerformanceMonitor one. A little
566 * trickery is thus necessary
567 */
568 . = 0xf00
569 b performance_monitor_pSeries
571 STD_EXCEPTION_PSERIES(0xf20, altivec_unavailable)
573 #ifdef CONFIG_CBE_RAS
574 HSTD_EXCEPTION_PSERIES(0x1200, cbe_system_error)
575 #endif /* CONFIG_CBE_RAS */
576 STD_EXCEPTION_PSERIES(0x1300, instruction_breakpoint)
577 #ifdef CONFIG_CBE_RAS
578 HSTD_EXCEPTION_PSERIES(0x1600, cbe_maintenance)
579 #endif /* CONFIG_CBE_RAS */
580 STD_EXCEPTION_PSERIES(0x1700, altivec_assist)
581 #ifdef CONFIG_CBE_RAS
582 HSTD_EXCEPTION_PSERIES(0x1800, cbe_thermal)
583 #endif /* CONFIG_CBE_RAS */
585 . = 0x3000
587 /*** pSeries interrupt support ***/
589 /* moved from 0xf00 */
590 STD_EXCEPTION_PSERIES(., performance_monitor)
592 .align 7
593 _GLOBAL(do_stab_bolted_pSeries)
594 mtcrf 0x80,r12
595 mfspr r12,SPRN_SPRG2
596 EXCEPTION_PROLOG_PSERIES(PACA_EXSLB, .do_stab_bolted)
598 /*
599 * We have some room here we use that to put
600 * the peries slb miss user trampoline code so it's reasonably
601 * away from slb_miss_user_common to avoid problems with rfid
602 *
603 * This is used for when the SLB miss handler has to go virtual,
604 * which doesn't happen for now anymore but will once we re-implement
605 * dynamic VSIDs for shared page tables
606 */
607 #ifdef __DISABLED__
608 slb_miss_user_pseries:
609 std r10,PACA_EXGEN+EX_R10(r13)
610 std r11,PACA_EXGEN+EX_R11(r13)
611 std r12,PACA_EXGEN+EX_R12(r13)
612 mfspr r10,SPRG1
613 ld r11,PACA_EXSLB+EX_R9(r13)
614 ld r12,PACA_EXSLB+EX_R3(r13)
615 std r10,PACA_EXGEN+EX_R13(r13)
616 std r11,PACA_EXGEN+EX_R9(r13)
617 std r12,PACA_EXGEN+EX_R3(r13)
618 clrrdi r12,r13,32
619 mfmsr r10
620 mfspr r11,SRR0 /* save SRR0 */
621 ori r12,r12,slb_miss_user_common@l /* virt addr of handler */
622 ori r10,r10,MSR_IR|MSR_DR|MSR_RI
623 mtspr SRR0,r12
624 mfspr r12,SRR1 /* and SRR1 */
625 mtspr SRR1,r10
626 rfid
627 b . /* prevent spec. execution */
628 #endif /* __DISABLED__ */
630 /*
631 * Vectors for the FWNMI option. Share common code.
632 */
633 .globl system_reset_fwnmi
634 .align 7
635 system_reset_fwnmi:
636 HMT_MEDIUM
637 mtspr SPRN_SPRG1,r13 /* save r13 */
638 EXCEPTION_PROLOG_PSERIES_FORCE_64BIT(PACA_EXGEN, system_reset_common)
640 .globl machine_check_fwnmi
641 .align 7
642 machine_check_fwnmi:
643 HMT_MEDIUM
644 mtspr SPRN_SPRG1,r13 /* save r13 */
645 EXCEPTION_PROLOG_PSERIES_FORCE_64BIT(PACA_EXMC, machine_check_common)
647 #ifdef CONFIG_PPC_ISERIES
648 /*** ISeries-LPAR interrupt handlers ***/
650 STD_EXCEPTION_ISERIES(0x200, machine_check, PACA_EXMC)
652 .globl data_access_iSeries
653 data_access_iSeries:
654 mtspr SPRN_SPRG1,r13
655 BEGIN_FTR_SECTION
656 mtspr SPRN_SPRG2,r12
657 mfspr r13,SPRN_DAR
658 mfspr r12,SPRN_DSISR
659 srdi r13,r13,60
660 rlwimi r13,r12,16,0x20
661 mfcr r12
662 cmpwi r13,0x2c
663 beq .do_stab_bolted_iSeries
664 mtcrf 0x80,r12
665 mfspr r12,SPRN_SPRG2
666 END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
667 EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN)
668 EXCEPTION_PROLOG_ISERIES_2
669 b data_access_common
671 .do_stab_bolted_iSeries:
672 mtcrf 0x80,r12
673 mfspr r12,SPRN_SPRG2
674 EXCEPTION_PROLOG_ISERIES_1(PACA_EXSLB)
675 EXCEPTION_PROLOG_ISERIES_2
676 b .do_stab_bolted
678 .globl data_access_slb_iSeries
679 data_access_slb_iSeries:
680 mtspr SPRN_SPRG1,r13 /* save r13 */
681 mfspr r13,SPRN_SPRG3 /* get paca address into r13 */
682 std r3,PACA_EXSLB+EX_R3(r13)
683 mfspr r3,SPRN_DAR
684 std r9,PACA_EXSLB+EX_R9(r13)
685 mfcr r9
686 #ifdef __DISABLED__
687 cmpdi r3,0
688 bge slb_miss_user_iseries
689 #endif
690 std r10,PACA_EXSLB+EX_R10(r13)
691 std r11,PACA_EXSLB+EX_R11(r13)
692 std r12,PACA_EXSLB+EX_R12(r13)
693 mfspr r10,SPRN_SPRG1
694 std r10,PACA_EXSLB+EX_R13(r13)
695 ld r12,PACALPPACAPTR(r13)
696 ld r12,LPPACASRR1(r12)
697 b .slb_miss_realmode
699 STD_EXCEPTION_ISERIES(0x400, instruction_access, PACA_EXGEN)
701 .globl instruction_access_slb_iSeries
702 instruction_access_slb_iSeries:
703 mtspr SPRN_SPRG1,r13 /* save r13 */
704 mfspr r13,SPRN_SPRG3 /* get paca address into r13 */
705 std r3,PACA_EXSLB+EX_R3(r13)
706 ld r3,PACALPPACAPTR(r13)
707 ld r3,LPPACASRR0(r3) /* get SRR0 value */
708 std r9,PACA_EXSLB+EX_R9(r13)
709 mfcr r9
710 #ifdef __DISABLED__
711 cmpdi r3,0
712 bge .slb_miss_user_iseries
713 #endif
714 std r10,PACA_EXSLB+EX_R10(r13)
715 std r11,PACA_EXSLB+EX_R11(r13)
716 std r12,PACA_EXSLB+EX_R12(r13)
717 mfspr r10,SPRN_SPRG1
718 std r10,PACA_EXSLB+EX_R13(r13)
719 ld r12,PACALPPACAPTR(r13)
720 ld r12,LPPACASRR1(r12)
721 b .slb_miss_realmode
723 #ifdef __DISABLED__
724 slb_miss_user_iseries:
725 std r10,PACA_EXGEN+EX_R10(r13)
726 std r11,PACA_EXGEN+EX_R11(r13)
727 std r12,PACA_EXGEN+EX_R12(r13)
728 mfspr r10,SPRG1
729 ld r11,PACA_EXSLB+EX_R9(r13)
730 ld r12,PACA_EXSLB+EX_R3(r13)
731 std r10,PACA_EXGEN+EX_R13(r13)
732 std r11,PACA_EXGEN+EX_R9(r13)
733 std r12,PACA_EXGEN+EX_R3(r13)
734 EXCEPTION_PROLOG_ISERIES_2
735 b slb_miss_user_common
736 #endif
738 MASKABLE_EXCEPTION_ISERIES(0x500, hardware_interrupt)
739 STD_EXCEPTION_ISERIES(0x600, alignment, PACA_EXGEN)
740 STD_EXCEPTION_ISERIES(0x700, program_check, PACA_EXGEN)
741 STD_EXCEPTION_ISERIES(0x800, fp_unavailable, PACA_EXGEN)
742 MASKABLE_EXCEPTION_ISERIES(0x900, decrementer)
743 STD_EXCEPTION_ISERIES(0xa00, trap_0a, PACA_EXGEN)
744 STD_EXCEPTION_ISERIES(0xb00, trap_0b, PACA_EXGEN)
746 .globl system_call_iSeries
747 system_call_iSeries:
748 mr r9,r13
749 mfspr r13,SPRN_SPRG3
750 EXCEPTION_PROLOG_ISERIES_2
751 b system_call_common
753 STD_EXCEPTION_ISERIES( 0xd00, single_step, PACA_EXGEN)
754 STD_EXCEPTION_ISERIES( 0xe00, trap_0e, PACA_EXGEN)
755 STD_EXCEPTION_ISERIES( 0xf00, performance_monitor, PACA_EXGEN)
757 .globl system_reset_iSeries
758 system_reset_iSeries:
759 mfspr r13,SPRN_SPRG3 /* Get paca address */
760 mfmsr r24
761 ori r24,r24,MSR_RI
762 mtmsrd r24 /* RI on */
763 lhz r24,PACAPACAINDEX(r13) /* Get processor # */
764 cmpwi 0,r24,0 /* Are we processor 0? */
765 beq .__start_initialization_iSeries /* Start up the first processor */
766 mfspr r4,SPRN_CTRLF
767 li r5,CTRL_RUNLATCH /* Turn off the run light */
768 andc r4,r4,r5
769 mtspr SPRN_CTRLT,r4
771 1:
772 HMT_LOW
773 #ifdef CONFIG_SMP
774 lbz r23,PACAPROCSTART(r13) /* Test if this processor
775 * should start */
776 sync
777 LOAD_REG_IMMEDIATE(r3,current_set)
778 sldi r28,r24,3 /* get current_set[cpu#] */
779 ldx r3,r3,r28
780 addi r1,r3,THREAD_SIZE
781 subi r1,r1,STACK_FRAME_OVERHEAD
783 cmpwi 0,r23,0
784 beq iSeries_secondary_smp_loop /* Loop until told to go */
785 bne .__secondary_start /* Loop until told to go */
786 iSeries_secondary_smp_loop:
787 /* Let the Hypervisor know we are alive */
788 /* 8002 is a call to HvCallCfg::getLps, a harmless Hypervisor function */
789 lis r3,0x8002
790 rldicr r3,r3,32,15 /* r0 = (r3 << 32) & 0xffff000000000000 */
791 #else /* CONFIG_SMP */
792 /* Yield the processor. This is required for non-SMP kernels
793 which are running on multi-threaded machines. */
794 lis r3,0x8000
795 rldicr r3,r3,32,15 /* r3 = (r3 << 32) & 0xffff000000000000 */
796 addi r3,r3,18 /* r3 = 0x8000000000000012 which is "yield" */
797 li r4,0 /* "yield timed" */
798 li r5,-1 /* "yield forever" */
799 #endif /* CONFIG_SMP */
800 li r0,-1 /* r0=-1 indicates a Hypervisor call */
801 sc /* Invoke the hypervisor via a system call */
802 mfspr r13,SPRN_SPRG3 /* Put r13 back ???? */
803 b 1b /* If SMP not configured, secondaries
804 * loop forever */
806 .globl decrementer_iSeries_masked
807 decrementer_iSeries_masked:
808 /* We may not have a valid TOC pointer in here. */
809 li r11,1
810 ld r12,PACALPPACAPTR(r13)
811 stb r11,LPPACADECRINT(r12)
812 LOAD_REG_IMMEDIATE(r12, tb_ticks_per_jiffy)
813 lwz r12,0(r12)
814 mtspr SPRN_DEC,r12
815 /* fall through */
817 .globl hardware_interrupt_iSeries_masked
818 hardware_interrupt_iSeries_masked:
819 mtcrf 0x80,r9 /* Restore regs */
820 ld r12,PACALPPACAPTR(r13)
821 ld r11,LPPACASRR0(r12)
822 ld r12,LPPACASRR1(r12)
823 mtspr SPRN_SRR0,r11
824 mtspr SPRN_SRR1,r12
825 ld r9,PACA_EXGEN+EX_R9(r13)
826 ld r10,PACA_EXGEN+EX_R10(r13)
827 ld r11,PACA_EXGEN+EX_R11(r13)
828 ld r12,PACA_EXGEN+EX_R12(r13)
829 ld r13,PACA_EXGEN+EX_R13(r13)
830 rfid
831 b . /* prevent speculative execution */
832 #endif /* CONFIG_PPC_ISERIES */
834 /*** Common interrupt handlers ***/
836 STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception)
838 /*
839 * Machine check is different because we use a different
840 * save area: PACA_EXMC instead of PACA_EXGEN.
841 */
842 .align 7
843 .globl machine_check_common
844 machine_check_common:
845 EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
846 FINISH_NAP
847 DISABLE_INTS
848 bl .save_nvgprs
849 addi r3,r1,STACK_FRAME_OVERHEAD
850 bl .machine_check_exception
851 b .ret_from_except
853 STD_EXCEPTION_COMMON_LITE(0x900, decrementer, .timer_interrupt)
854 STD_EXCEPTION_COMMON(0xa00, trap_0a, .unknown_exception)
855 STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception)
856 STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception)
857 STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception)
858 STD_EXCEPTION_COMMON_IDLE(0xf00, performance_monitor, .performance_monitor_exception)
859 STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception)
860 #ifdef CONFIG_ALTIVEC
861 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception)
862 #else
863 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception)
864 #endif
865 #ifdef CONFIG_CBE_RAS
866 STD_EXCEPTION_COMMON(0x1200, cbe_system_error, .cbe_system_error_exception)
867 STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, .cbe_maintenance_exception)
868 STD_EXCEPTION_COMMON(0x1800, cbe_thermal, .cbe_thermal_exception)
869 #endif /* CONFIG_CBE_RAS */
871 /*
872 * Here we have detected that the kernel stack pointer is bad.
873 * R9 contains the saved CR, r13 points to the paca,
874 * r10 contains the (bad) kernel stack pointer,
875 * r11 and r12 contain the saved SRR0 and SRR1.
876 * We switch to using an emergency stack, save the registers there,
877 * and call kernel_bad_stack(), which panics.
878 */
879 bad_stack:
880 ld r1,PACAEMERGSP(r13)
881 subi r1,r1,64+INT_FRAME_SIZE
882 std r9,_CCR(r1)
883 std r10,GPR1(r1)
884 std r11,_NIP(r1)
885 std r12,_MSR(r1)
886 mfspr r11,SPRN_DAR
887 mfspr r12,SPRN_DSISR
888 std r11,_DAR(r1)
889 std r12,_DSISR(r1)
890 mflr r10
891 mfctr r11
892 mfxer r12
893 std r10,_LINK(r1)
894 std r11,_CTR(r1)
895 std r12,_XER(r1)
896 SAVE_GPR(0,r1)
897 SAVE_GPR(2,r1)
898 SAVE_4GPRS(3,r1)
899 SAVE_2GPRS(7,r1)
900 SAVE_10GPRS(12,r1)
901 SAVE_10GPRS(22,r1)
902 addi r11,r1,INT_FRAME_SIZE
903 std r11,0(r1)
904 li r12,0
905 std r12,0(r11)
906 ld r2,PACATOC(r13)
907 1: addi r3,r1,STACK_FRAME_OVERHEAD
908 bl .kernel_bad_stack
909 b 1b
911 /*
912 * Return from an exception with minimal checks.
913 * The caller is assumed to have done EXCEPTION_PROLOG_COMMON.
914 * If interrupts have been enabled, or anything has been
915 * done that might have changed the scheduling status of
916 * any task or sent any task a signal, you should use
917 * ret_from_except or ret_from_except_lite instead of this.
918 */
919 .globl fast_exception_return
920 fast_exception_return:
921 ld r12,_MSR(r1)
922 ld r11,_NIP(r1)
923 andi. r3,r12,MSR_RI /* check if RI is set */
924 beq- unrecov_fer
926 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
927 andi. r3,r12,MSR_PR
928 beq 2f
929 ACCOUNT_CPU_USER_EXIT(r3, r4)
930 2:
931 #endif
933 ld r3,_CCR(r1)
934 ld r4,_LINK(r1)
935 ld r5,_CTR(r1)
936 ld r6,_XER(r1)
937 mtcr r3
938 mtlr r4
939 mtctr r5
940 mtxer r6
941 REST_GPR(0, r1)
942 REST_8GPRS(2, r1)
944 mfmsr r10
945 clrrdi r10,r10,2 /* clear RI (LE is 0 already) */
946 mtmsrd r10,1
948 mtspr SPRN_SRR1,r12
949 mtspr SPRN_SRR0,r11
950 REST_4GPRS(10, r1)
951 ld r1,GPR1(r1)
952 rfid
953 b . /* prevent speculative execution */
955 unrecov_fer:
956 bl .save_nvgprs
957 1: addi r3,r1,STACK_FRAME_OVERHEAD
958 bl .unrecoverable_exception
959 b 1b
961 /*
962 * Here r13 points to the paca, r9 contains the saved CR,
963 * SRR0 and SRR1 are saved in r11 and r12,
964 * r9 - r13 are saved in paca->exgen.
965 */
966 .align 7
967 .globl data_access_common
968 data_access_common:
969 mfspr r10,SPRN_DAR
970 std r10,PACA_EXGEN+EX_DAR(r13)
971 mfspr r10,SPRN_DSISR
972 stw r10,PACA_EXGEN+EX_DSISR(r13)
973 EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
974 ld r3,PACA_EXGEN+EX_DAR(r13)
975 lwz r4,PACA_EXGEN+EX_DSISR(r13)
976 li r5,0x300
977 b .do_hash_page /* Try to handle as hpte fault */
979 .align 7
980 .globl instruction_access_common
981 instruction_access_common:
982 EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
983 ld r3,_NIP(r1)
984 andis. r4,r12,0x5820
985 li r5,0x400
986 b .do_hash_page /* Try to handle as hpte fault */
988 /*
989 * Here is the common SLB miss user that is used when going to virtual
990 * mode for SLB misses, that is currently not used
991 */
992 #ifdef __DISABLED__
993 .align 7
994 .globl slb_miss_user_common
995 slb_miss_user_common:
996 mflr r10
997 std r3,PACA_EXGEN+EX_DAR(r13)
998 stw r9,PACA_EXGEN+EX_CCR(r13)
999 std r10,PACA_EXGEN+EX_LR(r13)
1000 std r11,PACA_EXGEN+EX_SRR0(r13)
1001 bl .slb_allocate_user
1003 ld r10,PACA_EXGEN+EX_LR(r13)
1004 ld r3,PACA_EXGEN+EX_R3(r13)
1005 lwz r9,PACA_EXGEN+EX_CCR(r13)
1006 ld r11,PACA_EXGEN+EX_SRR0(r13)
1007 mtlr r10
1008 beq- slb_miss_fault
1010 andi. r10,r12,MSR_RI /* check for unrecoverable exception */
1011 beq- unrecov_user_slb
1012 mfmsr r10
1014 .machine push
1015 .machine "power4"
1016 mtcrf 0x80,r9
1017 .machine pop
1019 clrrdi r10,r10,2 /* clear RI before setting SRR0/1 */
1020 mtmsrd r10,1
1022 mtspr SRR0,r11
1023 mtspr SRR1,r12
1025 ld r9,PACA_EXGEN+EX_R9(r13)
1026 ld r10,PACA_EXGEN+EX_R10(r13)
1027 ld r11,PACA_EXGEN+EX_R11(r13)
1028 ld r12,PACA_EXGEN+EX_R12(r13)
1029 ld r13,PACA_EXGEN+EX_R13(r13)
1030 rfid
1031 b .
1033 slb_miss_fault:
1034 EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN)
1035 ld r4,PACA_EXGEN+EX_DAR(r13)
1036 li r5,0
1037 std r4,_DAR(r1)
1038 std r5,_DSISR(r1)
1039 b .handle_page_fault
1041 unrecov_user_slb:
1042 EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN)
1043 DISABLE_INTS
1044 bl .save_nvgprs
1045 1: addi r3,r1,STACK_FRAME_OVERHEAD
1046 bl .unrecoverable_exception
1047 b 1b
1049 #endif /* __DISABLED__ */
1052 /*
1053 * r13 points to the PACA, r9 contains the saved CR,
1054 * r12 contain the saved SRR1, SRR0 is still ready for return
1055 * r3 has the faulting address
1056 * r9 - r13 are saved in paca->exslb.
1057 * r3 is saved in paca->slb_r3
1058 * We assume we aren't going to take any exceptions during this procedure.
1059 */
1060 _GLOBAL(slb_miss_realmode)
1061 mflr r10
1063 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
1064 std r10,PACA_EXSLB+EX_LR(r13) /* save LR */
1066 bl .slb_allocate_realmode
1068 /* All done -- return from exception. */
1070 ld r10,PACA_EXSLB+EX_LR(r13)
1071 ld r3,PACA_EXSLB+EX_R3(r13)
1072 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
1073 #ifdef CONFIG_PPC_ISERIES
1074 ld r11,PACALPPACAPTR(r13)
1075 ld r11,LPPACASRR0(r11) /* get SRR0 value */
1076 #endif /* CONFIG_PPC_ISERIES */
1078 mtlr r10
1080 andi. r10,r12,MSR_RI /* check for unrecoverable exception */
1081 beq- unrecov_slb
1083 .machine push
1084 .machine "power4"
1085 mtcrf 0x80,r9
1086 mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
1087 .machine pop
1089 #ifdef CONFIG_PPC_ISERIES
1090 mtspr SPRN_SRR0,r11
1091 mtspr SPRN_SRR1,r12
1092 #endif /* CONFIG_PPC_ISERIES */
1093 ld r9,PACA_EXSLB+EX_R9(r13)
1094 ld r10,PACA_EXSLB+EX_R10(r13)
1095 ld r11,PACA_EXSLB+EX_R11(r13)
1096 ld r12,PACA_EXSLB+EX_R12(r13)
1097 ld r13,PACA_EXSLB+EX_R13(r13)
1098 rfid
1099 b . /* prevent speculative execution */
1101 unrecov_slb:
1102 EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
1103 DISABLE_INTS
1104 bl .save_nvgprs
1105 1: addi r3,r1,STACK_FRAME_OVERHEAD
1106 bl .unrecoverable_exception
1107 b 1b
1109 .align 7
1110 .globl hardware_interrupt_common
1111 .globl hardware_interrupt_entry
1112 hardware_interrupt_common:
1113 EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN)
1114 FINISH_NAP
1115 hardware_interrupt_entry:
1116 DISABLE_INTS
1117 bl .ppc64_runlatch_on
1118 addi r3,r1,STACK_FRAME_OVERHEAD
1119 bl .do_IRQ
1120 b .ret_from_except_lite
1122 #ifdef CONFIG_PPC_970_NAP
1123 power4_fixup_nap:
1124 andc r9,r9,r10
1125 std r9,TI_LOCAL_FLAGS(r11)
1126 ld r10,_LINK(r1) /* make idle task do the */
1127 std r10,_NIP(r1) /* equivalent of a blr */
1128 blr
1129 #endif
1131 .align 7
1132 .globl alignment_common
1133 alignment_common:
1134 mfspr r10,SPRN_DAR
1135 std r10,PACA_EXGEN+EX_DAR(r13)
1136 mfspr r10,SPRN_DSISR
1137 stw r10,PACA_EXGEN+EX_DSISR(r13)
1138 EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
1139 ld r3,PACA_EXGEN+EX_DAR(r13)
1140 lwz r4,PACA_EXGEN+EX_DSISR(r13)
1141 std r3,_DAR(r1)
1142 std r4,_DSISR(r1)
1143 bl .save_nvgprs
1144 addi r3,r1,STACK_FRAME_OVERHEAD
1145 ENABLE_INTS
1146 bl .alignment_exception
1147 b .ret_from_except
1149 .align 7
1150 .globl program_check_common
1151 program_check_common:
1152 EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
1153 bl .save_nvgprs
1154 addi r3,r1,STACK_FRAME_OVERHEAD
1155 ENABLE_INTS
1156 bl .program_check_exception
1157 b .ret_from_except
1159 .align 7
1160 .globl fp_unavailable_common
1161 fp_unavailable_common:
1162 EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
1163 bne .load_up_fpu /* if from user, just load it up */
1164 bl .save_nvgprs
1165 addi r3,r1,STACK_FRAME_OVERHEAD
1166 ENABLE_INTS
1167 bl .kernel_fp_unavailable_exception
1168 BUG_OPCODE
1170 .align 7
1171 .globl altivec_unavailable_common
1172 altivec_unavailable_common:
1173 EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
1174 #ifdef CONFIG_ALTIVEC
1175 BEGIN_FTR_SECTION
1176 bne .load_up_altivec /* if from user, just load it up */
1177 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1178 #endif
1179 bl .save_nvgprs
1180 addi r3,r1,STACK_FRAME_OVERHEAD
1181 ENABLE_INTS
1182 bl .altivec_unavailable_exception
1183 b .ret_from_except
1185 #ifdef CONFIG_ALTIVEC
1186 /*
1187 * load_up_altivec(unused, unused, tsk)
1188 * Disable VMX for the task which had it previously,
1189 * and save its vector registers in its thread_struct.
1190 * Enables the VMX for use in the kernel on return.
1191 * On SMP we know the VMX is free, since we give it up every
1192 * switch (ie, no lazy save of the vector registers).
1193 * On entry: r13 == 'current' && last_task_used_altivec != 'current'
1194 */
1195 _STATIC(load_up_altivec)
1196 mfmsr r5 /* grab the current MSR */
1197 oris r5,r5,MSR_VEC@h
1198 mtmsrd r5 /* enable use of VMX now */
1199 isync
1201 /*
1202 * For SMP, we don't do lazy VMX switching because it just gets too
1203 * horrendously complex, especially when a task switches from one CPU
1204 * to another. Instead we call giveup_altvec in switch_to.
1205 * VRSAVE isn't dealt with here, that is done in the normal context
1206 * switch code. Note that we could rely on vrsave value to eventually
1207 * avoid saving all of the VREGs here...
1208 */
1209 #ifndef CONFIG_SMP
1210 ld r3,last_task_used_altivec@got(r2)
1211 ld r4,0(r3)
1212 cmpdi 0,r4,0
1213 beq 1f
1214 /* Save VMX state to last_task_used_altivec's THREAD struct */
1215 addi r4,r4,THREAD
1216 SAVE_32VRS(0,r5,r4)
1217 mfvscr vr0
1218 li r10,THREAD_VSCR
1219 stvx vr0,r10,r4
1220 /* Disable VMX for last_task_used_altivec */
1221 ld r5,PT_REGS(r4)
1222 ld r4,_MSR-STACK_FRAME_OVERHEAD(r5)
1223 lis r6,MSR_VEC@h
1224 andc r4,r4,r6
1225 std r4,_MSR-STACK_FRAME_OVERHEAD(r5)
1226 1:
1227 #endif /* CONFIG_SMP */
1228 /* Hack: if we get an altivec unavailable trap with VRSAVE
1229 * set to all zeros, we assume this is a broken application
1230 * that fails to set it properly, and thus we switch it to
1231 * all 1's
1232 */
1233 mfspr r4,SPRN_VRSAVE
1234 cmpdi 0,r4,0
1235 bne+ 1f
1236 li r4,-1
1237 mtspr SPRN_VRSAVE,r4
1238 1:
1239 /* enable use of VMX after return */
1240 ld r4,PACACURRENT(r13)
1241 addi r5,r4,THREAD /* Get THREAD */
1242 oris r12,r12,MSR_VEC@h
1243 std r12,_MSR(r1)
1244 li r4,1
1245 li r10,THREAD_VSCR
1246 stw r4,THREAD_USED_VR(r5)
1247 lvx vr0,r10,r5
1248 mtvscr vr0
1249 REST_32VRS(0,r4,r5)
1250 #ifndef CONFIG_SMP
1251 /* Update last_task_used_math to 'current' */
1252 subi r4,r5,THREAD /* Back to 'current' */
1253 std r4,0(r3)
1254 #endif /* CONFIG_SMP */
1255 /* restore registers and return */
1256 b fast_exception_return
1257 #endif /* CONFIG_ALTIVEC */
1259 /*
1260 * Hash table stuff
1261 */
1262 .align 7
1263 _GLOBAL(do_hash_page)
1264 std r3,_DAR(r1)
1265 std r4,_DSISR(r1)
1267 andis. r0,r4,0xa450 /* weird error? */
1268 bne- .handle_page_fault /* if not, try to insert a HPTE */
1269 BEGIN_FTR_SECTION
1270 andis. r0,r4,0x0020 /* Is it a segment table fault? */
1271 bne- .do_ste_alloc /* If so handle it */
1272 END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
1274 /*
1275 * We need to set the _PAGE_USER bit if MSR_PR is set or if we are
1276 * accessing a userspace segment (even from the kernel). We assume
1277 * kernel addresses always have the high bit set.
1278 */
1279 rlwinm r4,r4,32-25+9,31-9,31-9 /* DSISR_STORE -> _PAGE_RW */
1280 rotldi r0,r3,15 /* Move high bit into MSR_PR posn */
1281 orc r0,r12,r0 /* MSR_PR | ~high_bit */
1282 rlwimi r4,r0,32-13,30,30 /* becomes _PAGE_USER access bit */
1283 ori r4,r4,1 /* add _PAGE_PRESENT */
1284 rlwimi r4,r5,22+2,31-2,31-2 /* Set _PAGE_EXEC if trap is 0x400 */
1286 /*
1287 * On iSeries, we soft-disable interrupts here, then
1288 * hard-enable interrupts so that the hash_page code can spin on
1289 * the hash_table_lock without problems on a shared processor.
1290 */
1291 DISABLE_INTS
1293 /*
1294 * r3 contains the faulting address
1295 * r4 contains the required access permissions
1296 * r5 contains the trap number
1297 *
1298 * at return r3 = 0 for success
1299 */
1300 bl .hash_page /* build HPTE if possible */
1301 cmpdi r3,0 /* see if hash_page succeeded */
1303 #ifdef DO_SOFT_DISABLE
1304 /*
1305 * If we had interrupts soft-enabled at the point where the
1306 * DSI/ISI occurred, and an interrupt came in during hash_page,
1307 * handle it now.
1308 * We jump to ret_from_except_lite rather than fast_exception_return
1309 * because ret_from_except_lite will check for and handle pending
1310 * interrupts if necessary.
1311 */
1312 beq .ret_from_except_lite
1313 /* For a hash failure, we don't bother re-enabling interrupts */
1314 ble- 12f
1316 /*
1317 * hash_page couldn't handle it, set soft interrupt enable back
1318 * to what it was before the trap. Note that .local_irq_restore
1319 * handles any interrupts pending at this point.
1320 */
1321 ld r3,SOFTE(r1)
1322 bl .local_irq_restore
1323 b 11f
1324 #else
1325 beq fast_exception_return /* Return from exception on success */
1326 ble- 12f /* Failure return from hash_page */
1328 /* fall through */
1329 #endif
1331 /* Here we have a page fault that hash_page can't handle. */
1332 _GLOBAL(handle_page_fault)
1333 ENABLE_INTS
1334 11: ld r4,_DAR(r1)
1335 ld r5,_DSISR(r1)
1336 addi r3,r1,STACK_FRAME_OVERHEAD
1337 bl .do_page_fault
1338 cmpdi r3,0
1339 beq+ .ret_from_except_lite
1340 bl .save_nvgprs
1341 mr r5,r3
1342 addi r3,r1,STACK_FRAME_OVERHEAD
1343 lwz r4,_DAR(r1)
1344 bl .bad_page_fault
1345 b .ret_from_except
1347 /* We have a page fault that hash_page could handle but HV refused
1348 * the PTE insertion
1349 */
1350 12: bl .save_nvgprs
1351 addi r3,r1,STACK_FRAME_OVERHEAD
1352 lwz r4,_DAR(r1)
1353 bl .low_hash_fault
1354 b .ret_from_except
1356 /* here we have a segment miss */
1357 _GLOBAL(do_ste_alloc)
1358 bl .ste_allocate /* try to insert stab entry */
1359 cmpdi r3,0
1360 beq+ fast_exception_return
1361 b .handle_page_fault
1363 /*
1364 * r13 points to the PACA, r9 contains the saved CR,
1365 * r11 and r12 contain the saved SRR0 and SRR1.
1366 * r9 - r13 are saved in paca->exslb.
1367 * We assume we aren't going to take any exceptions during this procedure.
1368 * We assume (DAR >> 60) == 0xc.
1369 */
1370 .align 7
1371 _GLOBAL(do_stab_bolted)
1372 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
1373 std r11,PACA_EXSLB+EX_SRR0(r13) /* save SRR0 in exc. frame */
1375 /* Hash to the primary group */
1376 ld r10,PACASTABVIRT(r13)
1377 mfspr r11,SPRN_DAR
1378 srdi r11,r11,28
1379 rldimi r10,r11,7,52 /* r10 = first ste of the group */
1381 /* Calculate VSID */
1382 /* This is a kernel address, so protovsid = ESID */
1383 ASM_VSID_SCRAMBLE(r11, r9)
1384 rldic r9,r11,12,16 /* r9 = vsid << 12 */
1386 /* Search the primary group for a free entry */
1387 1: ld r11,0(r10) /* Test valid bit of the current ste */
1388 andi. r11,r11,0x80
1389 beq 2f
1390 addi r10,r10,16
1391 andi. r11,r10,0x70
1392 bne 1b
1394 /* Stick for only searching the primary group for now. */
1395 /* At least for now, we use a very simple random castout scheme */
1396 /* Use the TB as a random number ; OR in 1 to avoid entry 0 */
1397 mftb r11
1398 rldic r11,r11,4,57 /* r11 = (r11 << 4) & 0x70 */
1399 ori r11,r11,0x10
1401 /* r10 currently points to an ste one past the group of interest */
1402 /* make it point to the randomly selected entry */
1403 subi r10,r10,128
1404 or r10,r10,r11 /* r10 is the entry to invalidate */
1406 isync /* mark the entry invalid */
1407 ld r11,0(r10)
1408 rldicl r11,r11,56,1 /* clear the valid bit */
1409 rotldi r11,r11,8
1410 std r11,0(r10)
1411 sync
1413 clrrdi r11,r11,28 /* Get the esid part of the ste */
1414 slbie r11
1416 2: std r9,8(r10) /* Store the vsid part of the ste */
1417 eieio
1419 mfspr r11,SPRN_DAR /* Get the new esid */
1420 clrrdi r11,r11,28 /* Permits a full 32b of ESID */
1421 ori r11,r11,0x90 /* Turn on valid and kp */
1422 std r11,0(r10) /* Put new entry back into the stab */
1424 sync
1426 /* All done -- return from exception. */
1427 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
1428 ld r11,PACA_EXSLB+EX_SRR0(r13) /* get saved SRR0 */
1430 andi. r10,r12,MSR_RI
1431 beq- unrecov_slb
1433 mtcrf 0x80,r9 /* restore CR */
1435 mfmsr r10
1436 clrrdi r10,r10,2
1437 mtmsrd r10,1
1439 mtspr SPRN_SRR0,r11
1440 mtspr SPRN_SRR1,r12
1441 ld r9,PACA_EXSLB+EX_R9(r13)
1442 ld r10,PACA_EXSLB+EX_R10(r13)
1443 ld r11,PACA_EXSLB+EX_R11(r13)
1444 ld r12,PACA_EXSLB+EX_R12(r13)
1445 ld r13,PACA_EXSLB+EX_R13(r13)
1446 rfid
1447 b . /* prevent speculative execution */
1449 /*
1450 * Space for CPU0's segment table.
1451 *
1452 * On iSeries, the hypervisor must fill in at least one entry before
1453 * we get control (with relocate on). The address is give to the hv
1454 * as a page number (see xLparMap in lpardata.c), so this must be at a
1455 * fixed address (the linker can't compute (u64)&initial_stab >>
1456 * PAGE_SHIFT).
1457 */
1458 . = STAB0_OFFSET /* 0x6000 */
1459 .globl initial_stab
1460 initial_stab:
1461 .space 4096
1463 /*
1464 * Data area reserved for FWNMI option.
1465 * This address (0x7000) is fixed by the RPA.
1466 */
1467 .= 0x7000
1468 .globl fwnmi_data_area
1469 fwnmi_data_area:
1471 /* iSeries does not use the FWNMI stuff, so it is safe to put
1472 * this here, even if we later allow kernels that will boot on
1473 * both pSeries and iSeries */
1474 #ifdef CONFIG_PPC_ISERIES
1475 . = LPARMAP_PHYS
1476 #include "lparmap.s"
1477 /*
1478 * This ".text" is here for old compilers that generate a trailing
1479 * .note section when compiling .c files to .s
1480 */
1481 .text
1482 #endif /* CONFIG_PPC_ISERIES */
1484 . = 0x8000
1486 /*
1487 * On pSeries and most other platforms, secondary processors spin
1488 * in the following code.
1489 * At entry, r3 = this processor's number (physical cpu id)
1490 */
1491 _GLOBAL(generic_secondary_smp_init)
1492 mr r24,r3
1494 /* turn on 64-bit mode */
1495 bl .enable_64b_mode
1496 isync
1498 /* Set up a paca value for this processor. Since we have the
1499 * physical cpu id in r24, we need to search the pacas to find
1500 * which logical id maps to our physical one.
1501 */
1502 LOAD_REG_IMMEDIATE(r13, paca) /* Get base vaddr of paca array */
1503 li r5,0 /* logical cpu id */
1504 1: lhz r6,PACAHWCPUID(r13) /* Load HW procid from paca */
1505 cmpw r6,r24 /* Compare to our id */
1506 beq 2f
1507 addi r13,r13,PACA_SIZE /* Loop to next PACA on miss */
1508 addi r5,r5,1
1509 cmpwi r5,NR_CPUS
1510 blt 1b
1512 mr r3,r24 /* not found, copy phys to r3 */
1513 b .kexec_wait /* next kernel might do better */
1515 2: mtspr SPRN_SPRG3,r13 /* Save vaddr of paca in SPRG3 */
1516 /* From now on, r24 is expected to be logical cpuid */
1517 mr r24,r5
1518 3: HMT_LOW
1519 lbz r23,PACAPROCSTART(r13) /* Test if this processor should */
1520 /* start. */
1521 sync
1523 #ifndef CONFIG_SMP
1524 b 3b /* Never go on non-SMP */
1525 #else
1526 cmpwi 0,r23,0
1527 beq 3b /* Loop until told to go */
1529 /* See if we need to call a cpu state restore handler */
1530 LOAD_REG_IMMEDIATE(r23, cur_cpu_spec)
1531 ld r23,0(r23)
1532 ld r23,CPU_SPEC_RESTORE(r23)
1533 cmpdi 0,r23,0
1534 beq 4f
1535 ld r23,0(r23)
1536 mtctr r23
1537 bctrl
1539 4: /* Create a temp kernel stack for use before relocation is on. */
1540 ld r1,PACAEMERGSP(r13)
1541 subi r1,r1,STACK_FRAME_OVERHEAD
1543 b .__secondary_start
1544 #endif
1546 #ifdef CONFIG_PPC_ISERIES
1547 _STATIC(__start_initialization_iSeries)
1548 /* Clear out the BSS */
1549 LOAD_REG_IMMEDIATE(r11,__bss_stop)
1550 LOAD_REG_IMMEDIATE(r8,__bss_start)
1551 sub r11,r11,r8 /* bss size */
1552 addi r11,r11,7 /* round up to an even double word */
1553 rldicl. r11,r11,61,3 /* shift right by 3 */
1554 beq 4f
1555 addi r8,r8,-8
1556 li r0,0
1557 mtctr r11 /* zero this many doublewords */
1558 3: stdu r0,8(r8)
1559 bdnz 3b
1560 4:
1561 LOAD_REG_IMMEDIATE(r1,init_thread_union)
1562 addi r1,r1,THREAD_SIZE
1563 li r0,0
1564 stdu r0,-STACK_FRAME_OVERHEAD(r1)
1566 LOAD_REG_IMMEDIATE(r3,cpu_specs)
1567 LOAD_REG_IMMEDIATE(r4,cur_cpu_spec)
1568 li r5,0
1569 bl .identify_cpu
1571 LOAD_REG_IMMEDIATE(r2,__toc_start)
1572 addi r2,r2,0x4000
1573 addi r2,r2,0x4000
1575 bl .iSeries_early_setup
1576 bl .early_setup
1578 /* relocation is on at this point */
1580 b .start_here_common
1581 #endif /* CONFIG_PPC_ISERIES */
1583 #ifdef CONFIG_PPC_MULTIPLATFORM
1585 _STATIC(__mmu_off)
1586 mfmsr r3
1587 andi. r0,r3,MSR_IR|MSR_DR
1588 beqlr
1589 andc r3,r3,r0
1590 mtspr SPRN_SRR0,r4
1591 mtspr SPRN_SRR1,r3
1592 sync
1593 rfid
1594 b . /* prevent speculative execution */
1597 /*
1598 * Here is our main kernel entry point. We support currently 2 kind of entries
1599 * depending on the value of r5.
1600 *
1601 * r5 != NULL -> OF entry, we go to prom_init, "legacy" parameter content
1602 * in r3...r7
1603 *
1604 * r5 == NULL -> kexec style entry. r3 is a physical pointer to the
1605 * DT block, r4 is a physical pointer to the kernel itself
1606 *
1607 */
1608 _GLOBAL(__start_initialization_multiplatform)
1609 #ifdef CONFIG_PPC_MULTIPLATFORM
1610 /*
1611 * Are we booted from a PROM Of-type client-interface ?
1612 */
1613 cmpldi cr0,r5,0
1614 bne .__boot_from_prom /* yes -> prom */
1615 #endif
1617 /* Save parameters */
1618 mr r31,r3
1619 mr r30,r4
1621 /* Make sure we are running in 64 bits mode */
1622 bl .enable_64b_mode
1624 /* Setup some critical 970 SPRs before switching MMU off */
1625 mfspr r0,SPRN_PVR
1626 srwi r0,r0,16
1627 cmpwi r0,0x39 /* 970 */
1628 beq 1f
1629 cmpwi r0,0x3c /* 970FX */
1630 beq 1f
1631 cmpwi r0,0x44 /* 970MP */
1632 bne 2f
1633 1: bl .__cpu_preinit_ppc970
1634 2:
1636 /* Switch off MMU if not already */
1637 LOAD_REG_IMMEDIATE(r4, .__after_prom_start - KERNELBASE)
1638 add r4,r4,r30
1639 bl .__mmu_off
1640 b .__after_prom_start
1642 #ifdef CONFIG_PPC_MULTIPLATFORM
1643 _STATIC(__boot_from_prom)
1644 /* Save parameters */
1645 mr r31,r3
1646 mr r30,r4
1647 mr r29,r5
1648 mr r28,r6
1649 mr r27,r7
1651 /*
1652 * Align the stack to 16-byte boundary
1653 * Depending on the size and layout of the ELF sections in the initial
1654 * boot binary, the stack pointer will be unalignet on PowerMac
1655 */
1656 rldicr r1,r1,0,59
1658 /* Make sure we are running in 64 bits mode */
1659 bl .enable_64b_mode
1661 /* put a relocation offset into r3 */
1662 bl .reloc_offset
1664 LOAD_REG_IMMEDIATE(r2,__toc_start)
1665 addi r2,r2,0x4000
1666 addi r2,r2,0x4000
1668 /* Relocate the TOC from a virt addr to a real addr */
1669 add r2,r2,r3
1671 /* Restore parameters */
1672 mr r3,r31
1673 mr r4,r30
1674 mr r5,r29
1675 mr r6,r28
1676 mr r7,r27
1678 /* Do all of the interaction with OF client interface */
1679 bl .prom_init
1680 /* We never return */
1681 trap
1682 #endif
1684 /*
1685 * At this point, r3 contains the physical address we are running at,
1686 * returned by prom_init()
1687 */
1688 _STATIC(__after_prom_start)
1690 /*
1691 * We need to run with __start at physical address PHYSICAL_START.
1692 * This will leave some code in the first 256B of
1693 * real memory, which are reserved for software use.
1694 * The remainder of the first page is loaded with the fixed
1695 * interrupt vectors. The next two pages are filled with
1696 * unknown exception placeholders.
1697 *
1698 * Note: This process overwrites the OF exception vectors.
1699 * r26 == relocation offset
1700 * r27 == KERNELBASE
1701 */
1702 bl .reloc_offset
1703 mr r26,r3
1704 LOAD_REG_IMMEDIATE(r27, KERNELBASE)
1706 LOAD_REG_IMMEDIATE(r3, PHYSICAL_START) /* target addr */
1708 // XXX FIXME: Use phys returned by OF (r30)
1709 add r4,r27,r26 /* source addr */
1710 /* current address of _start */
1711 /* i.e. where we are running */
1712 /* the source addr */
1714 cmpdi r4,0 /* In some cases the loader may */
1715 beq .start_here_multiplatform /* have already put us at zero */
1716 /* so we can skip the copy. */
1717 LOAD_REG_IMMEDIATE(r5,copy_to_here) /* # bytes of memory to copy */
1718 sub r5,r5,r27
1720 li r6,0x100 /* Start offset, the first 0x100 */
1721 /* bytes were copied earlier. */
1723 bl .copy_and_flush /* copy the first n bytes */
1724 /* this includes the code being */
1725 /* executed here. */
1727 LOAD_REG_IMMEDIATE(r0, 4f) /* Jump to the copy of this code */
1728 mtctr r0 /* that we just made/relocated */
1729 bctr
1731 4: LOAD_REG_IMMEDIATE(r5,klimit)
1732 add r5,r5,r26
1733 ld r5,0(r5) /* get the value of klimit */
1734 sub r5,r5,r27
1735 bl .copy_and_flush /* copy the rest */
1736 b .start_here_multiplatform
1738 #endif /* CONFIG_PPC_MULTIPLATFORM */
1740 /*
1741 * Copy routine used to copy the kernel to start at physical address 0
1742 * and flush and invalidate the caches as needed.
1743 * r3 = dest addr, r4 = source addr, r5 = copy limit, r6 = start offset
1744 * on exit, r3, r4, r5 are unchanged, r6 is updated to be >= r5.
1745 *
1746 * Note: this routine *only* clobbers r0, r6 and lr
1747 */
1748 _GLOBAL(copy_and_flush)
1749 addi r5,r5,-8
1750 addi r6,r6,-8
1751 4: li r0,8 /* Use the smallest common */
1752 /* denominator cache line */
1753 /* size. This results in */
1754 /* extra cache line flushes */
1755 /* but operation is correct. */
1756 /* Can't get cache line size */
1757 /* from NACA as it is being */
1758 /* moved too. */
1760 mtctr r0 /* put # words/line in ctr */
1761 3: addi r6,r6,8 /* copy a cache line */
1762 ldx r0,r6,r4
1763 stdx r0,r6,r3
1764 bdnz 3b
1765 dcbst r6,r3 /* write it to memory */
1766 sync
1767 icbi r6,r3 /* flush the icache line */
1768 cmpld 0,r6,r5
1769 blt 4b
1770 sync
1771 addi r5,r5,8
1772 addi r6,r6,8
1773 blr
1775 .align 8
1776 copy_to_here:
1778 #ifdef CONFIG_SMP
1779 #ifdef CONFIG_PPC_PMAC
1780 /*
1781 * On PowerMac, secondary processors starts from the reset vector, which
1782 * is temporarily turned into a call to one of the functions below.
1783 */
1784 .section ".text";
1785 .align 2 ;
1787 .globl __secondary_start_pmac_0
1788 __secondary_start_pmac_0:
1789 /* NB the entries for cpus 0, 1, 2 must each occupy 8 bytes. */
1790 li r24,0
1791 b 1f
1792 li r24,1
1793 b 1f
1794 li r24,2
1795 b 1f
1796 li r24,3
1797 1:
1799 _GLOBAL(pmac_secondary_start)
1800 /* turn on 64-bit mode */
1801 bl .enable_64b_mode
1802 isync
1804 /* Copy some CPU settings from CPU 0 */
1805 bl .__restore_cpu_ppc970
1807 /* pSeries do that early though I don't think we really need it */
1808 mfmsr r3
1809 ori r3,r3,MSR_RI
1810 mtmsrd r3 /* RI on */
1812 /* Set up a paca value for this processor. */
1813 LOAD_REG_IMMEDIATE(r4, paca) /* Get base vaddr of paca array */
1814 mulli r13,r24,PACA_SIZE /* Calculate vaddr of right paca */
1815 add r13,r13,r4 /* for this processor. */
1816 mtspr SPRN_SPRG3,r13 /* Save vaddr of paca in SPRG3 */
1818 /* Create a temp kernel stack for use before relocation is on. */
1819 ld r1,PACAEMERGSP(r13)
1820 subi r1,r1,STACK_FRAME_OVERHEAD
1822 b .__secondary_start
1824 #endif /* CONFIG_PPC_PMAC */
1826 /*
1827 * This function is called after the master CPU has released the
1828 * secondary processors. The execution environment is relocation off.
1829 * The paca for this processor has the following fields initialized at
1830 * this point:
1831 * 1. Processor number
1832 * 2. Segment table pointer (virtual address)
1833 * On entry the following are set:
1834 * r1 = stack pointer. vaddr for iSeries, raddr (temp stack) for pSeries
1835 * r24 = cpu# (in Linux terms)
1836 * r13 = paca virtual address
1837 * SPRG3 = paca virtual address
1838 */
1839 _GLOBAL(__secondary_start)
1840 /* Set thread priority to MEDIUM */
1841 HMT_MEDIUM
1843 /* Load TOC */
1844 ld r2,PACATOC(r13)
1846 /* Do early setup for that CPU (stab, slb, hash table pointer) */
1847 bl .early_setup_secondary
1849 /* Initialize the kernel stack. Just a repeat for iSeries. */
1850 LOAD_REG_ADDR(r3, current_set)
1851 sldi r28,r24,3 /* get current_set[cpu#] */
1852 ldx r1,r3,r28
1853 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
1854 std r1,PACAKSAVE(r13)
1856 /* Clear backchain so we get nice backtraces */
1857 li r7,0
1858 mtlr r7
1860 /* enable MMU and jump to start_secondary */
1861 LOAD_REG_ADDR(r3, .start_secondary_prolog)
1862 LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
1863 #ifdef DO_SOFT_DISABLE
1864 ori r4,r4,MSR_EE
1865 #endif
1866 mtspr SPRN_SRR0,r3
1867 mtspr SPRN_SRR1,r4
1868 rfid
1869 b . /* prevent speculative execution */
1871 /*
1872 * Running with relocation on at this point. All we want to do is
1873 * zero the stack back-chain pointer before going into C code.
1874 */
1875 _GLOBAL(start_secondary_prolog)
1876 li r3,0
1877 std r3,0(r1) /* Zero the stack frame pointer */
1878 bl .start_secondary
1879 b .
1880 #endif
1882 /*
1883 * This subroutine clobbers r11 and r12
1884 */
1885 _GLOBAL(enable_64b_mode)
1886 mfmsr r11 /* grab the current MSR */
1887 li r12,1
1888 rldicr r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
1889 or r11,r11,r12
1890 li r12,1
1891 rldicr r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
1892 or r11,r11,r12
1893 mtmsrd r11
1894 isync
1895 blr
1897 #ifdef CONFIG_PPC_MULTIPLATFORM
1898 /*
1899 * This is where the main kernel code starts.
1900 */
1901 _STATIC(start_here_multiplatform)
1902 /* get a new offset, now that the kernel has moved. */
1903 bl .reloc_offset
1904 mr r26,r3
1906 /* Clear out the BSS. It may have been done in prom_init,
1907 * already but that's irrelevant since prom_init will soon
1908 * be detached from the kernel completely. Besides, we need
1909 * to clear it now for kexec-style entry.
1910 */
1911 LOAD_REG_IMMEDIATE(r11,__bss_stop)
1912 LOAD_REG_IMMEDIATE(r8,__bss_start)
1913 sub r11,r11,r8 /* bss size */
1914 addi r11,r11,7 /* round up to an even double word */
1915 rldicl. r11,r11,61,3 /* shift right by 3 */
1916 beq 4f
1917 addi r8,r8,-8
1918 li r0,0
1919 mtctr r11 /* zero this many doublewords */
1920 3: stdu r0,8(r8)
1921 bdnz 3b
1922 4:
1924 mfmsr r6
1925 ori r6,r6,MSR_RI
1926 mtmsrd r6 /* RI on */
1928 /* The following gets the stack and TOC set up with the regs */
1929 /* pointing to the real addr of the kernel stack. This is */
1930 /* all done to support the C function call below which sets */
1931 /* up the htab. This is done because we have relocated the */
1932 /* kernel but are still running in real mode. */
1934 LOAD_REG_IMMEDIATE(r3,init_thread_union)
1935 add r3,r3,r26
1937 /* set up a stack pointer (physical address) */
1938 addi r1,r3,THREAD_SIZE
1939 li r0,0
1940 stdu r0,-STACK_FRAME_OVERHEAD(r1)
1942 /* set up the TOC (physical address) */
1943 LOAD_REG_IMMEDIATE(r2,__toc_start)
1944 addi r2,r2,0x4000
1945 addi r2,r2,0x4000
1946 add r2,r2,r26
1948 LOAD_REG_IMMEDIATE(r3, cpu_specs)
1949 add r3,r3,r26
1950 LOAD_REG_IMMEDIATE(r4,cur_cpu_spec)
1951 add r4,r4,r26
1952 mr r5,r26
1953 bl .identify_cpu
1955 /* Do very early kernel initializations, including initial hash table,
1956 * stab and slb setup before we turn on relocation. */
1958 /* Restore parameters passed from prom_init/kexec */
1959 mr r3,r31
1960 bl .early_setup
1962 LOAD_REG_IMMEDIATE(r3, .start_here_common)
1963 LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
1964 mtspr SPRN_SRR0,r3
1965 mtspr SPRN_SRR1,r4
1966 rfid
1967 b . /* prevent speculative execution */
1968 #endif /* CONFIG_PPC_MULTIPLATFORM */
1970 /* This is where all platforms converge execution */
1971 _STATIC(start_here_common)
1972 /* relocation is on at this point */
1974 /* The following code sets up the SP and TOC now that we are */
1975 /* running with translation enabled. */
1977 LOAD_REG_IMMEDIATE(r3,init_thread_union)
1979 /* set up the stack */
1980 addi r1,r3,THREAD_SIZE
1981 li r0,0
1982 stdu r0,-STACK_FRAME_OVERHEAD(r1)
1984 /* Apply the CPUs-specific fixups (nop out sections not relevant
1985 * to this CPU
1986 */
1987 li r3,0
1988 bl .do_cpu_ftr_fixups
1990 /* ptr to current */
1991 LOAD_REG_IMMEDIATE(r4, init_task)
1992 std r4,PACACURRENT(r13)
1994 /* Load the TOC */
1995 ld r2,PACATOC(r13)
1996 std r1,PACAKSAVE(r13)
1998 bl .setup_system
2000 /* Load up the kernel context */
2001 5:
2002 #ifdef DO_SOFT_DISABLE
2003 li r5,0
2004 stb r5,PACAPROCENABLED(r13) /* Soft Disabled */
2005 mfmsr r5
2006 ori r5,r5,MSR_EE /* Hard Enabled */
2007 mtmsrd r5
2008 #endif
2010 bl .start_kernel
2012 /* Not reached */
2013 BUG_OPCODE
2015 /*
2016 * We put a few things here that have to be page-aligned.
2017 * This stuff goes at the beginning of the bss, which is page-aligned.
2018 */
2019 .section ".bss"
2021 .align PAGE_SHIFT
2023 .globl empty_zero_page
2024 empty_zero_page:
2025 .space PAGE_SIZE
2027 .globl swapper_pg_dir
2028 swapper_pg_dir:
2029 .space PAGE_SIZE
2031 /*
2032 * This space gets a copy of optional info passed to us by the bootstrap
2033 * Used to pass parameters into the kernel like root=/dev/sda1, etc.
2034 */
2035 .globl cmd_line
2036 cmd_line:
2037 .space COMMAND_LINE_SIZE