| blob | eb82d787d99a1140e660b4e5675a241ee2b92a5c |
1 /*
2 * Low level TLB miss handlers for Book3E
3 *
4 * Copyright (C) 2008-2009
5 * Ben. Herrenschmidt (benh@kernel.crashing.org), IBM Corp.
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 */
13 #include <asm/processor.h>
14 #include <asm/reg.h>
15 #include <asm/page.h>
16 #include <asm/mmu.h>
17 #include <asm/ppc_asm.h>
18 #include <asm/asm-offsets.h>
19 #include <asm/cputable.h>
20 #include <asm/pgtable.h>
21 #include <asm/exception-64e.h>
22 #include <asm/ppc-opcode.h>
23 #include <asm/kvm_asm.h>
24 #include <asm/kvm_booke_hv_asm.h>
26 #ifdef CONFIG_PPC_64K_PAGES
27 #define VPTE_PMD_SHIFT (PTE_INDEX_SIZE+1)
28 #else
29 #define VPTE_PMD_SHIFT (PTE_INDEX_SIZE)
30 #endif
31 #define VPTE_PUD_SHIFT (VPTE_PMD_SHIFT + PMD_INDEX_SIZE)
32 #define VPTE_PGD_SHIFT (VPTE_PUD_SHIFT + PUD_INDEX_SIZE)
33 #define VPTE_INDEX_SIZE (VPTE_PGD_SHIFT + PGD_INDEX_SIZE)
35 /**********************************************************************
36 * *
37 * TLB miss handling for Book3E with a bolted linear mapping *
38 * No virtual page table, no nested TLB misses *
39 * *
40 **********************************************************************/
42 /*
43 * Note that, unlike non-bolted handlers, TLB_EXFRAME is not
44 * modified by the TLB miss handlers themselves, since the TLB miss
45 * handler code will not itself cause a recursive TLB miss.
46 *
47 * TLB_EXFRAME will be modified when crit/mc/debug exceptions are
48 * entered/exited.
49 */
50 .macro tlb_prolog_bolted intnum addr
51 mtspr SPRN_SPRG_GEN_SCRATCH,r12
52 mfspr r12,SPRN_SPRG_TLB_EXFRAME
53 std r13,EX_TLB_R13(r12)
54 std r10,EX_TLB_R10(r12)
55 mfspr r13,SPRN_SPRG_PACA
57 mfcr r10
58 std r11,EX_TLB_R11(r12)
59 #ifdef CONFIG_KVM_BOOKE_HV
60 BEGIN_FTR_SECTION
61 mfspr r11, SPRN_SRR1
62 END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
63 #endif
64 DO_KVM \intnum, SPRN_SRR1
65 std r16,EX_TLB_R16(r12)
66 mfspr r16,\addr /* get faulting address */
67 std r14,EX_TLB_R14(r12)
68 ld r14,PACAPGD(r13)
69 std r15,EX_TLB_R15(r12)
70 std r10,EX_TLB_CR(r12)
71 #ifdef CONFIG_PPC_FSL_BOOK3E
72 std r7,EX_TLB_R7(r12)
73 #endif
74 TLB_MISS_PROLOG_STATS
75 .endm
77 .macro tlb_epilog_bolted
78 ld r14,EX_TLB_CR(r12)
79 #ifdef CONFIG_PPC_FSL_BOOK3E
80 ld r7,EX_TLB_R7(r12)
81 #endif
82 ld r10,EX_TLB_R10(r12)
83 ld r11,EX_TLB_R11(r12)
84 ld r13,EX_TLB_R13(r12)
85 mtcr r14
86 ld r14,EX_TLB_R14(r12)
87 ld r15,EX_TLB_R15(r12)
88 TLB_MISS_RESTORE_STATS
89 ld r16,EX_TLB_R16(r12)
90 mfspr r12,SPRN_SPRG_GEN_SCRATCH
91 .endm
93 /* Data TLB miss */
94 START_EXCEPTION(data_tlb_miss_bolted)
95 tlb_prolog_bolted BOOKE_INTERRUPT_DTLB_MISS SPRN_DEAR
97 /* We need _PAGE_PRESENT and _PAGE_ACCESSED set */
99 /* We do the user/kernel test for the PID here along with the RW test
100 */
101 /* We pre-test some combination of permissions to avoid double
102 * faults:
103 *
104 * We move the ESR:ST bit into the position of _PAGE_BAP_SW in the PTE
105 * ESR_ST is 0x00800000
106 * _PAGE_BAP_SW is 0x00000010
107 * So the shift is >> 19. This tests for supervisor writeability.
108 * If the page happens to be supervisor writeable and not user
109 * writeable, we will take a new fault later, but that should be
110 * a rare enough case.
111 *
112 * We also move ESR_ST in _PAGE_DIRTY position
113 * _PAGE_DIRTY is 0x00001000 so the shift is >> 11
114 *
115 * MAS1 is preset for all we need except for TID that needs to
116 * be cleared for kernel translations
117 */
119 mfspr r11,SPRN_ESR
121 srdi r15,r16,60 /* get region */
122 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
123 bne- dtlb_miss_fault_bolted /* Bail if fault addr is invalid */
125 rlwinm r10,r11,32-19,27,27
126 rlwimi r10,r11,32-16,19,19
127 cmpwi r15,0 /* user vs kernel check */
128 ori r10,r10,_PAGE_PRESENT
129 oris r11,r10,_PAGE_ACCESSED@h
131 TLB_MISS_STATS_SAVE_INFO_BOLTED
132 bne tlb_miss_kernel_bolted
134 tlb_miss_common_bolted:
135 /*
136 * This is the guts of the TLB miss handler for bolted-linear.
137 * We are entered with:
138 *
139 * r16 = faulting address
140 * r15 = crap (free to use)
141 * r14 = page table base
142 * r13 = PACA
143 * r11 = PTE permission mask
144 * r10 = crap (free to use)
145 */
146 rldicl r15,r16,64-PGDIR_SHIFT+3,64-PGD_INDEX_SIZE-3
147 cmpldi cr0,r14,0
148 clrrdi r15,r15,3
149 beq tlb_miss_fault_bolted /* No PGDIR, bail */
151 BEGIN_MMU_FTR_SECTION
152 /* Set the TLB reservation and search for existing entry. Then load
153 * the entry.
154 */
155 PPC_TLBSRX_DOT(0,R16)
156 ldx r14,r14,r15 /* grab pgd entry */
157 beq tlb_miss_done_bolted /* tlb exists already, bail */
158 MMU_FTR_SECTION_ELSE
159 ldx r14,r14,r15 /* grab pgd entry */
160 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBRSRV)
162 #ifndef CONFIG_PPC_64K_PAGES
163 rldicl r15,r16,64-PUD_SHIFT+3,64-PUD_INDEX_SIZE-3
164 clrrdi r15,r15,3
165 cmpdi cr0,r14,0
166 bge tlb_miss_fault_bolted /* Bad pgd entry or hugepage; bail */
167 ldx r14,r14,r15 /* grab pud entry */
168 #endif /* CONFIG_PPC_64K_PAGES */
170 rldicl r15,r16,64-PMD_SHIFT+3,64-PMD_INDEX_SIZE-3
171 clrrdi r15,r15,3
172 cmpdi cr0,r14,0
173 bge tlb_miss_fault_bolted
174 ldx r14,r14,r15 /* Grab pmd entry */
176 rldicl r15,r16,64-PAGE_SHIFT+3,64-PTE_INDEX_SIZE-3
177 clrrdi r15,r15,3
178 cmpdi cr0,r14,0
179 bge tlb_miss_fault_bolted
180 ldx r14,r14,r15 /* Grab PTE, normal (!huge) page */
182 /* Check if required permissions are met */
183 andc. r15,r11,r14
184 rldicr r15,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
185 bne- tlb_miss_fault_bolted
187 /* Now we build the MAS:
188 *
189 * MAS 0 : Fully setup with defaults in MAS4 and TLBnCFG
190 * MAS 1 : Almost fully setup
191 * - PID already updated by caller if necessary
192 * - TSIZE need change if !base page size, not
193 * yet implemented for now
194 * MAS 2 : Defaults not useful, need to be redone
195 * MAS 3+7 : Needs to be done
196 */
197 clrrdi r11,r16,12 /* Clear low crap in EA */
198 clrldi r15,r15,12 /* Clear crap at the top */
199 rlwimi r11,r14,32-19,27,31 /* Insert WIMGE */
200 rlwimi r15,r14,32-8,22,25 /* Move in U bits */
201 mtspr SPRN_MAS2,r11
202 andi. r11,r14,_PAGE_DIRTY
203 rlwimi r15,r14,32-2,26,31 /* Move in BAP bits */
205 /* Mask out SW and UW if !DIRTY (XXX optimize this !) */
206 bne 1f
207 li r11,MAS3_SW|MAS3_UW
208 andc r15,r15,r11
209 1:
210 mtspr SPRN_MAS7_MAS3,r15
211 tlbwe
213 tlb_miss_done_bolted:
214 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_NORM_OK)
215 tlb_epilog_bolted
216 rfi
218 itlb_miss_kernel_bolted:
219 li r11,_PAGE_PRESENT|_PAGE_BAP_SX /* Base perm */
220 oris r11,r11,_PAGE_ACCESSED@h
221 tlb_miss_kernel_bolted:
222 mfspr r10,SPRN_MAS1
223 ld r14,PACA_KERNELPGD(r13)
224 cmpldi cr0,r15,8 /* Check for vmalloc region */
225 rlwinm r10,r10,0,16,1 /* Clear TID */
226 mtspr SPRN_MAS1,r10
227 beq+ tlb_miss_common_bolted
229 tlb_miss_fault_bolted:
230 /* We need to check if it was an instruction miss */
231 andi. r10,r11,_PAGE_EXEC|_PAGE_BAP_SX
232 bne itlb_miss_fault_bolted
233 dtlb_miss_fault_bolted:
234 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
235 tlb_epilog_bolted
236 b exc_data_storage_book3e
237 itlb_miss_fault_bolted:
238 TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
239 tlb_epilog_bolted
240 b exc_instruction_storage_book3e
242 /* Instruction TLB miss */
243 START_EXCEPTION(instruction_tlb_miss_bolted)
244 tlb_prolog_bolted BOOKE_INTERRUPT_ITLB_MISS SPRN_SRR0
246 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
247 srdi r15,r16,60 /* get region */
248 TLB_MISS_STATS_SAVE_INFO_BOLTED
249 bne- itlb_miss_fault_bolted
251 li r11,_PAGE_PRESENT|_PAGE_EXEC /* Base perm */
253 /* We do the user/kernel test for the PID here along with the RW test
254 */
256 cmpldi cr0,r15,0 /* Check for user region */
257 oris r11,r11,_PAGE_ACCESSED@h
258 beq tlb_miss_common_bolted
259 b itlb_miss_kernel_bolted
261 #ifdef CONFIG_PPC_FSL_BOOK3E
262 /*
263 * TLB miss handling for e6500 and derivatives, using hardware tablewalk.
264 *
265 * Linear mapping is bolted: no virtual page table or nested TLB misses
266 * Indirect entries in TLB1, hardware loads resulting direct entries
267 * into TLB0
268 * No HES or NV hint on TLB1, so we need to do software round-robin
269 * No tlbsrx. so we need a spinlock, and we have to deal
270 * with MAS-damage caused by tlbsx
271 * 4K pages only
272 */
274 START_EXCEPTION(instruction_tlb_miss_e6500)
275 tlb_prolog_bolted BOOKE_INTERRUPT_ITLB_MISS SPRN_SRR0
277 ld r11,PACA_TCD_PTR(r13)
278 srdi. r15,r16,60 /* get region */
279 ori r16,r16,1
281 TLB_MISS_STATS_SAVE_INFO_BOLTED
282 bne tlb_miss_kernel_e6500 /* user/kernel test */
284 b tlb_miss_common_e6500
286 START_EXCEPTION(data_tlb_miss_e6500)
287 tlb_prolog_bolted BOOKE_INTERRUPT_DTLB_MISS SPRN_DEAR
289 ld r11,PACA_TCD_PTR(r13)
290 srdi. r15,r16,60 /* get region */
291 rldicr r16,r16,0,62
293 TLB_MISS_STATS_SAVE_INFO_BOLTED
294 bne tlb_miss_kernel_e6500 /* user vs kernel check */
296 /*
297 * This is the guts of the TLB miss handler for e6500 and derivatives.
298 * We are entered with:
299 *
300 * r16 = page of faulting address (low bit 0 if data, 1 if instruction)
301 * r15 = crap (free to use)
302 * r14 = page table base
303 * r13 = PACA
304 * r11 = tlb_per_core ptr
305 * r10 = crap (free to use)
306 * r7 = esel_next
307 */
308 tlb_miss_common_e6500:
309 crmove cr2*4+2,cr0*4+2 /* cr2.eq != 0 if kernel address */
311 BEGIN_FTR_SECTION /* CPU_FTR_SMT */
312 /*
313 * Search if we already have an indirect entry for that virtual
314 * address, and if we do, bail out.
315 *
316 * MAS6:IND should be already set based on MAS4
317 */
318 lhz r10,PACAPACAINDEX(r13)
319 addi r10,r10,1
320 crclr cr1*4+eq /* set cr1.eq = 0 for non-recursive */
321 1: lbarx r15,0,r11
322 cmpdi r15,0
323 bne 2f
324 stbcx. r10,0,r11
325 bne 1b
326 3:
327 .subsection 1
328 2: cmpd cr1,r15,r10 /* recursive lock due to mcheck/crit/etc? */
329 beq cr1,3b /* unlock will happen if cr1.eq = 0 */
330 10: lbz r15,0(r11)
331 cmpdi r15,0
332 bne 10b
333 b 1b
334 .previous
335 END_FTR_SECTION_IFSET(CPU_FTR_SMT)
337 lbz r7,TCD_ESEL_NEXT(r11)
339 BEGIN_FTR_SECTION /* CPU_FTR_SMT */
340 /*
341 * Erratum A-008139 says that we can't use tlbwe to change
342 * an indirect entry in any way (including replacing or
343 * invalidating) if the other thread could be in the process
344 * of a lookup. The workaround is to invalidate the entry
345 * with tlbilx before overwriting.
346 */
348 rlwinm r10,r7,16,0xff0000
349 oris r10,r10,MAS0_TLBSEL(1)@h
350 mtspr SPRN_MAS0,r10
351 isync
352 tlbre
353 mfspr r15,SPRN_MAS1
354 andis. r15,r15,MAS1_VALID@h
355 beq 5f
357 BEGIN_FTR_SECTION_NESTED(532)
358 mfspr r10,SPRN_MAS8
359 rlwinm r10,r10,0,0x80000fff /* tgs,tlpid -> sgs,slpid */
360 mtspr SPRN_MAS5,r10
361 END_FTR_SECTION_NESTED(CPU_FTR_EMB_HV,CPU_FTR_EMB_HV,532)
363 mfspr r10,SPRN_MAS1
364 rlwinm r15,r10,0,0x3fff0000 /* tid -> spid */
365 rlwimi r15,r10,20,0x00000003 /* ind,ts -> sind,sas */
366 mfspr r10,SPRN_MAS6
367 mtspr SPRN_MAS6,r15
369 mfspr r15,SPRN_MAS2
370 isync
371 tlbilxva 0,r15
372 isync
374 mtspr SPRN_MAS6,r10
376 5:
377 BEGIN_FTR_SECTION_NESTED(532)
378 li r10,0
379 mtspr SPRN_MAS8,r10
380 mtspr SPRN_MAS5,r10
381 END_FTR_SECTION_NESTED(CPU_FTR_EMB_HV,CPU_FTR_EMB_HV,532)
383 tlbsx 0,r16
384 mfspr r10,SPRN_MAS1
385 andis. r15,r10,MAS1_VALID@h
386 bne tlb_miss_done_e6500
387 FTR_SECTION_ELSE
388 mfspr r10,SPRN_MAS1
389 ALT_FTR_SECTION_END_IFSET(CPU_FTR_SMT)
391 oris r10,r10,MAS1_VALID@h
392 beq cr2,4f
393 rlwinm r10,r10,0,16,1 /* Clear TID */
394 4: mtspr SPRN_MAS1,r10
396 /* Now, we need to walk the page tables. First check if we are in
397 * range.
398 */
399 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
400 bne- tlb_miss_fault_e6500
402 rldicl r15,r16,64-PGDIR_SHIFT+3,64-PGD_INDEX_SIZE-3
403 cmpldi cr0,r14,0
404 clrrdi r15,r15,3
405 beq- tlb_miss_fault_e6500 /* No PGDIR, bail */
406 ldx r14,r14,r15 /* grab pgd entry */
408 rldicl r15,r16,64-PUD_SHIFT+3,64-PUD_INDEX_SIZE-3
409 clrrdi r15,r15,3
410 cmpdi cr0,r14,0
411 bge tlb_miss_huge_e6500 /* Bad pgd entry or hugepage; bail */
412 ldx r14,r14,r15 /* grab pud entry */
414 rldicl r15,r16,64-PMD_SHIFT+3,64-PMD_INDEX_SIZE-3
415 clrrdi r15,r15,3
416 cmpdi cr0,r14,0
417 bge tlb_miss_huge_e6500
418 ldx r14,r14,r15 /* Grab pmd entry */
420 mfspr r10,SPRN_MAS0
421 cmpdi cr0,r14,0
422 bge tlb_miss_huge_e6500
424 /* Now we build the MAS for a 2M indirect page:
425 *
426 * MAS 0 : ESEL needs to be filled by software round-robin
427 * MAS 1 : Fully set up
428 * - PID already updated by caller if necessary
429 * - TSIZE for now is base ind page size always
430 * - TID already cleared if necessary
431 * MAS 2 : Default not 2M-aligned, need to be redone
432 * MAS 3+7 : Needs to be done
433 */
435 ori r14,r14,(BOOK3E_PAGESZ_4K << MAS3_SPSIZE_SHIFT)
436 mtspr SPRN_MAS7_MAS3,r14
438 clrrdi r15,r16,21 /* make EA 2M-aligned */
439 mtspr SPRN_MAS2,r15
441 tlb_miss_huge_done_e6500:
442 lbz r16,TCD_ESEL_MAX(r11)
443 lbz r14,TCD_ESEL_FIRST(r11)
444 rlwimi r10,r7,16,0x00ff0000 /* insert esel_next into MAS0 */
445 addi r7,r7,1 /* increment esel_next */
446 mtspr SPRN_MAS0,r10
447 cmpw r7,r16
448 iseleq r7,r14,r7 /* if next == last use first */
449 stb r7,TCD_ESEL_NEXT(r11)
451 tlbwe
453 tlb_miss_done_e6500:
454 .macro tlb_unlock_e6500
455 BEGIN_FTR_SECTION
456 beq cr1,1f /* no unlock if lock was recursively grabbed */
457 li r15,0
458 isync
459 stb r15,0(r11)
460 1:
461 END_FTR_SECTION_IFSET(CPU_FTR_SMT)
462 .endm
464 tlb_unlock_e6500
465 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_NORM_OK)
466 tlb_epilog_bolted
467 rfi
469 tlb_miss_huge_e6500:
470 beq tlb_miss_fault_e6500
471 li r10,1
472 andi. r15,r14,HUGEPD_SHIFT_MASK@l /* r15 = psize */
473 rldimi r14,r10,63,0 /* Set PD_HUGE */
474 xor r14,r14,r15 /* Clear size bits */
475 ldx r14,0,r14
477 /*
478 * Now we build the MAS for a huge page.
479 *
480 * MAS 0 : ESEL needs to be filled by software round-robin
481 * - can be handled by indirect code
482 * MAS 1 : Need to clear IND and set TSIZE
483 * MAS 2,3+7: Needs to be redone similar to non-tablewalk handler
484 */
486 subi r15,r15,10 /* Convert psize to tsize */
487 mfspr r10,SPRN_MAS1
488 rlwinm r10,r10,0,~MAS1_IND
489 rlwimi r10,r15,MAS1_TSIZE_SHIFT,MAS1_TSIZE_MASK
490 mtspr SPRN_MAS1,r10
492 li r10,-0x400
493 sld r15,r10,r15 /* Generate mask based on size */
494 and r10,r16,r15
495 rldicr r15,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
496 rlwimi r10,r14,32-19,27,31 /* Insert WIMGE */
497 clrldi r15,r15,PAGE_SHIFT /* Clear crap at the top */
498 rlwimi r15,r14,32-8,22,25 /* Move in U bits */
499 mtspr SPRN_MAS2,r10
500 andi. r10,r14,_PAGE_DIRTY
501 rlwimi r15,r14,32-2,26,31 /* Move in BAP bits */
503 /* Mask out SW and UW if !DIRTY (XXX optimize this !) */
504 bne 1f
505 li r10,MAS3_SW|MAS3_UW
506 andc r15,r15,r10
507 1:
508 mtspr SPRN_MAS7_MAS3,r15
510 mfspr r10,SPRN_MAS0
511 b tlb_miss_huge_done_e6500
513 tlb_miss_kernel_e6500:
514 ld r14,PACA_KERNELPGD(r13)
515 cmpldi cr1,r15,8 /* Check for vmalloc region */
516 beq+ cr1,tlb_miss_common_e6500
518 tlb_miss_fault_e6500:
519 tlb_unlock_e6500
520 /* We need to check if it was an instruction miss */
521 andi. r16,r16,1
522 bne itlb_miss_fault_e6500
523 dtlb_miss_fault_e6500:
524 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
525 tlb_epilog_bolted
526 b exc_data_storage_book3e
527 itlb_miss_fault_e6500:
528 TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
529 tlb_epilog_bolted
530 b exc_instruction_storage_book3e
531 #endif /* CONFIG_PPC_FSL_BOOK3E */
533 /**********************************************************************
534 * *
535 * TLB miss handling for Book3E with TLB reservation and HES support *
536 * *
537 **********************************************************************/
540 /* Data TLB miss */
541 START_EXCEPTION(data_tlb_miss)
542 TLB_MISS_PROLOG
544 /* Now we handle the fault proper. We only save DEAR in normal
545 * fault case since that's the only interesting values here.
546 * We could probably also optimize by not saving SRR0/1 in the
547 * linear mapping case but I'll leave that for later
548 */
549 mfspr r14,SPRN_ESR
550 mfspr r16,SPRN_DEAR /* get faulting address */
551 srdi r15,r16,60 /* get region */
552 cmpldi cr0,r15,0xc /* linear mapping ? */
553 TLB_MISS_STATS_SAVE_INFO
554 beq tlb_load_linear /* yes -> go to linear map load */
556 /* The page tables are mapped virtually linear. At this point, though,
557 * we don't know whether we are trying to fault in a first level
558 * virtual address or a virtual page table address. We can get that
559 * from bit 0x1 of the region ID which we have set for a page table
560 */
561 andi. r10,r15,0x1
562 bne- virt_page_table_tlb_miss
564 std r14,EX_TLB_ESR(r12); /* save ESR */
565 std r16,EX_TLB_DEAR(r12); /* save DEAR */
567 /* We need _PAGE_PRESENT and _PAGE_ACCESSED set */
568 li r11,_PAGE_PRESENT
569 oris r11,r11,_PAGE_ACCESSED@h
571 /* We do the user/kernel test for the PID here along with the RW test
572 */
573 cmpldi cr0,r15,0 /* Check for user region */
575 /* We pre-test some combination of permissions to avoid double
576 * faults:
577 *
578 * We move the ESR:ST bit into the position of _PAGE_BAP_SW in the PTE
579 * ESR_ST is 0x00800000
580 * _PAGE_BAP_SW is 0x00000010
581 * So the shift is >> 19. This tests for supervisor writeability.
582 * If the page happens to be supervisor writeable and not user
583 * writeable, we will take a new fault later, but that should be
584 * a rare enough case.
585 *
586 * We also move ESR_ST in _PAGE_DIRTY position
587 * _PAGE_DIRTY is 0x00001000 so the shift is >> 11
588 *
589 * MAS1 is preset for all we need except for TID that needs to
590 * be cleared for kernel translations
591 */
592 rlwimi r11,r14,32-19,27,27
593 rlwimi r11,r14,32-16,19,19
594 beq normal_tlb_miss
595 /* XXX replace the RMW cycles with immediate loads + writes */
596 1: mfspr r10,SPRN_MAS1
597 cmpldi cr0,r15,8 /* Check for vmalloc region */
598 rlwinm r10,r10,0,16,1 /* Clear TID */
599 mtspr SPRN_MAS1,r10
600 beq+ normal_tlb_miss
602 /* We got a crappy address, just fault with whatever DEAR and ESR
603 * are here
604 */
605 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
606 TLB_MISS_EPILOG_ERROR
607 b exc_data_storage_book3e
609 /* Instruction TLB miss */
610 START_EXCEPTION(instruction_tlb_miss)
611 TLB_MISS_PROLOG
613 /* If we take a recursive fault, the second level handler may need
614 * to know whether we are handling a data or instruction fault in
615 * order to get to the right store fault handler. We provide that
616 * info by writing a crazy value in ESR in our exception frame
617 */
618 li r14,-1 /* store to exception frame is done later */
620 /* Now we handle the fault proper. We only save DEAR in the non
621 * linear mapping case since we know the linear mapping case will
622 * not re-enter. We could indeed optimize and also not save SRR0/1
623 * in the linear mapping case but I'll leave that for later
624 *
625 * Faulting address is SRR0 which is already in r16
626 */
627 srdi r15,r16,60 /* get region */
628 cmpldi cr0,r15,0xc /* linear mapping ? */
629 TLB_MISS_STATS_SAVE_INFO
630 beq tlb_load_linear /* yes -> go to linear map load */
632 /* We do the user/kernel test for the PID here along with the RW test
633 */
634 li r11,_PAGE_PRESENT|_PAGE_EXEC /* Base perm */
635 oris r11,r11,_PAGE_ACCESSED@h
637 cmpldi cr0,r15,0 /* Check for user region */
638 std r14,EX_TLB_ESR(r12) /* write crazy -1 to frame */
639 beq normal_tlb_miss
641 li r11,_PAGE_PRESENT|_PAGE_BAP_SX /* Base perm */
642 oris r11,r11,_PAGE_ACCESSED@h
643 /* XXX replace the RMW cycles with immediate loads + writes */
644 mfspr r10,SPRN_MAS1
645 cmpldi cr0,r15,8 /* Check for vmalloc region */
646 rlwinm r10,r10,0,16,1 /* Clear TID */
647 mtspr SPRN_MAS1,r10
648 beq+ normal_tlb_miss
650 /* We got a crappy address, just fault */
651 TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
652 TLB_MISS_EPILOG_ERROR
653 b exc_instruction_storage_book3e
655 /*
656 * This is the guts of the first-level TLB miss handler for direct
657 * misses. We are entered with:
658 *
659 * r16 = faulting address
660 * r15 = region ID
661 * r14 = crap (free to use)
662 * r13 = PACA
663 * r12 = TLB exception frame in PACA
664 * r11 = PTE permission mask
665 * r10 = crap (free to use)
666 */
667 normal_tlb_miss:
668 /* So we first construct the page table address. We do that by
669 * shifting the bottom of the address (not the region ID) by
670 * PAGE_SHIFT-3, clearing the bottom 3 bits (get a PTE ptr) and
671 * or'ing the fourth high bit.
672 *
673 * NOTE: For 64K pages, we do things slightly differently in
674 * order to handle the weird page table format used by linux
675 */
676 ori r10,r15,0x1
677 #ifdef CONFIG_PPC_64K_PAGES
678 /* For the top bits, 16 bytes per PTE */
679 rldicl r14,r16,64-(PAGE_SHIFT-4),PAGE_SHIFT-4+4
680 /* Now create the bottom bits as 0 in position 0x8000 and
681 * the rest calculated for 8 bytes per PTE
682 */
683 rldicl r15,r16,64-(PAGE_SHIFT-3),64-15
684 /* Insert the bottom bits in */
685 rlwimi r14,r15,0,16,31
686 #else
687 rldicl r14,r16,64-(PAGE_SHIFT-3),PAGE_SHIFT-3+4
688 #endif
689 sldi r15,r10,60
690 clrrdi r14,r14,3
691 or r10,r15,r14
693 BEGIN_MMU_FTR_SECTION
694 /* Set the TLB reservation and search for existing entry. Then load
695 * the entry.
696 */
697 PPC_TLBSRX_DOT(0,R16)
698 ld r14,0(r10)
699 beq normal_tlb_miss_done
700 MMU_FTR_SECTION_ELSE
701 ld r14,0(r10)
702 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBRSRV)
704 finish_normal_tlb_miss:
705 /* Check if required permissions are met */
706 andc. r15,r11,r14
707 bne- normal_tlb_miss_access_fault
709 /* Now we build the MAS:
710 *
711 * MAS 0 : Fully setup with defaults in MAS4 and TLBnCFG
712 * MAS 1 : Almost fully setup
713 * - PID already updated by caller if necessary
714 * - TSIZE need change if !base page size, not
715 * yet implemented for now
716 * MAS 2 : Defaults not useful, need to be redone
717 * MAS 3+7 : Needs to be done
718 *
719 * TODO: mix up code below for better scheduling
720 */
721 clrrdi r11,r16,12 /* Clear low crap in EA */
722 rlwimi r11,r14,32-19,27,31 /* Insert WIMGE */
723 mtspr SPRN_MAS2,r11
725 /* Check page size, if not standard, update MAS1 */
726 rldicl r11,r14,64-8,64-8
727 #ifdef CONFIG_PPC_64K_PAGES
728 cmpldi cr0,r11,BOOK3E_PAGESZ_64K
729 #else
730 cmpldi cr0,r11,BOOK3E_PAGESZ_4K
731 #endif
732 beq- 1f
733 mfspr r11,SPRN_MAS1
734 rlwimi r11,r14,31,21,24
735 rlwinm r11,r11,0,21,19
736 mtspr SPRN_MAS1,r11
737 1:
738 /* Move RPN in position */
739 rldicr r11,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
740 clrldi r15,r11,12 /* Clear crap at the top */
741 rlwimi r15,r14,32-8,22,25 /* Move in U bits */
742 rlwimi r15,r14,32-2,26,31 /* Move in BAP bits */
744 /* Mask out SW and UW if !DIRTY (XXX optimize this !) */
745 andi. r11,r14,_PAGE_DIRTY
746 bne 1f
747 li r11,MAS3_SW|MAS3_UW
748 andc r15,r15,r11
749 1:
750 BEGIN_MMU_FTR_SECTION
751 srdi r16,r15,32
752 mtspr SPRN_MAS3,r15
753 mtspr SPRN_MAS7,r16
754 MMU_FTR_SECTION_ELSE
755 mtspr SPRN_MAS7_MAS3,r15
756 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_PAIRED_MAS)
758 tlbwe
760 normal_tlb_miss_done:
761 /* We don't bother with restoring DEAR or ESR since we know we are
762 * level 0 and just going back to userland. They are only needed
763 * if you are going to take an access fault
764 */
765 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_NORM_OK)
766 TLB_MISS_EPILOG_SUCCESS
767 rfi
769 normal_tlb_miss_access_fault:
770 /* We need to check if it was an instruction miss */
771 andi. r10,r11,_PAGE_EXEC
772 bne 1f
773 ld r14,EX_TLB_DEAR(r12)
774 ld r15,EX_TLB_ESR(r12)
775 mtspr SPRN_DEAR,r14
776 mtspr SPRN_ESR,r15
777 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
778 TLB_MISS_EPILOG_ERROR
779 b exc_data_storage_book3e
780 1: TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
781 TLB_MISS_EPILOG_ERROR
782 b exc_instruction_storage_book3e
785 /*
786 * This is the guts of the second-level TLB miss handler for direct
787 * misses. We are entered with:
788 *
789 * r16 = virtual page table faulting address
790 * r15 = region (top 4 bits of address)
791 * r14 = crap (free to use)
792 * r13 = PACA
793 * r12 = TLB exception frame in PACA
794 * r11 = crap (free to use)
795 * r10 = crap (free to use)
796 *
797 * Note that this should only ever be called as a second level handler
798 * with the current scheme when using SW load.
799 * That means we can always get the original fault DEAR at
800 * EX_TLB_DEAR-EX_TLB_SIZE(r12)
801 *
802 * It can be re-entered by the linear mapping miss handler. However, to
803 * avoid too much complication, it will restart the whole fault at level
804 * 0 so we don't care too much about clobbers
805 *
806 * XXX That code was written back when we couldn't clobber r14. We can now,
807 * so we could probably optimize things a bit
808 */
809 virt_page_table_tlb_miss:
810 /* Are we hitting a kernel page table ? */
811 andi. r10,r15,0x8
813 /* The cool thing now is that r10 contains 0 for user and 8 for kernel,
814 * and we happen to have the swapper_pg_dir at offset 8 from the user
815 * pgdir in the PACA :-).
816 */
817 add r11,r10,r13
819 /* If kernel, we need to clear MAS1 TID */
820 beq 1f
821 /* XXX replace the RMW cycles with immediate loads + writes */
822 mfspr r10,SPRN_MAS1
823 rlwinm r10,r10,0,16,1 /* Clear TID */
824 mtspr SPRN_MAS1,r10
825 1:
826 BEGIN_MMU_FTR_SECTION
827 /* Search if we already have a TLB entry for that virtual address, and
828 * if we do, bail out.
829 */
830 PPC_TLBSRX_DOT(0,R16)
831 beq virt_page_table_tlb_miss_done
832 END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_TLBRSRV)
834 /* Now, we need to walk the page tables. First check if we are in
835 * range.
836 */
837 rldicl. r10,r16,64-(VPTE_INDEX_SIZE+3),VPTE_INDEX_SIZE+3+4
838 bne- virt_page_table_tlb_miss_fault
840 /* Get the PGD pointer */
841 ld r15,PACAPGD(r11)
842 cmpldi cr0,r15,0
843 beq- virt_page_table_tlb_miss_fault
845 /* Get to PGD entry */
846 rldicl r11,r16,64-VPTE_PGD_SHIFT,64-PGD_INDEX_SIZE-3
847 clrrdi r10,r11,3
848 ldx r15,r10,r15
849 cmpdi cr0,r15,0
850 bge virt_page_table_tlb_miss_fault
852 #ifndef CONFIG_PPC_64K_PAGES
853 /* Get to PUD entry */
854 rldicl r11,r16,64-VPTE_PUD_SHIFT,64-PUD_INDEX_SIZE-3
855 clrrdi r10,r11,3
856 ldx r15,r10,r15
857 cmpdi cr0,r15,0
858 bge virt_page_table_tlb_miss_fault
859 #endif /* CONFIG_PPC_64K_PAGES */
861 /* Get to PMD entry */
862 rldicl r11,r16,64-VPTE_PMD_SHIFT,64-PMD_INDEX_SIZE-3
863 clrrdi r10,r11,3
864 ldx r15,r10,r15
865 cmpdi cr0,r15,0
866 bge virt_page_table_tlb_miss_fault
868 /* Ok, we're all right, we can now create a kernel translation for
869 * a 4K or 64K page from r16 -> r15.
870 */
871 /* Now we build the MAS:
872 *
873 * MAS 0 : Fully setup with defaults in MAS4 and TLBnCFG
874 * MAS 1 : Almost fully setup
875 * - PID already updated by caller if necessary
876 * - TSIZE for now is base page size always
877 * MAS 2 : Use defaults
878 * MAS 3+7 : Needs to be done
879 *
880 * So we only do MAS 2 and 3 for now...
881 */
882 clrldi r11,r15,4 /* remove region ID from RPN */
883 ori r10,r11,1 /* Or-in SR */
885 BEGIN_MMU_FTR_SECTION
886 srdi r16,r10,32
887 mtspr SPRN_MAS3,r10
888 mtspr SPRN_MAS7,r16
889 MMU_FTR_SECTION_ELSE
890 mtspr SPRN_MAS7_MAS3,r10
891 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_PAIRED_MAS)
893 tlbwe
895 BEGIN_MMU_FTR_SECTION
896 virt_page_table_tlb_miss_done:
898 /* We have overridden MAS2:EPN but currently our primary TLB miss
899 * handler will always restore it so that should not be an issue,
900 * if we ever optimize the primary handler to not write MAS2 on
901 * some cases, we'll have to restore MAS2:EPN here based on the
902 * original fault's DEAR. If we do that we have to modify the
903 * ITLB miss handler to also store SRR0 in the exception frame
904 * as DEAR.
905 *
906 * However, one nasty thing we did is we cleared the reservation
907 * (well, potentially we did). We do a trick here thus if we
908 * are not a level 0 exception (we interrupted the TLB miss) we
909 * offset the return address by -4 in order to replay the tlbsrx
910 * instruction there
911 */
912 subf r10,r13,r12
913 cmpldi cr0,r10,PACA_EXTLB+EX_TLB_SIZE
914 bne- 1f
915 ld r11,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r13)
916 addi r10,r11,-4
917 std r10,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r13)
918 1:
919 END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_TLBRSRV)
920 /* Return to caller, normal case */
921 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_PT_OK);
922 TLB_MISS_EPILOG_SUCCESS
923 rfi
925 virt_page_table_tlb_miss_fault:
926 /* If we fault here, things are a little bit tricky. We need to call
927 * either data or instruction store fault, and we need to retrieve
928 * the original fault address and ESR (for data).
929 *
930 * The thing is, we know that in normal circumstances, this is
931 * always called as a second level tlb miss for SW load or as a first
932 * level TLB miss for HW load, so we should be able to peek at the
933 * relevant information in the first exception frame in the PACA.
934 *
935 * However, we do need to double check that, because we may just hit
936 * a stray kernel pointer or a userland attack trying to hit those
937 * areas. If that is the case, we do a data fault. (We can't get here
938 * from an instruction tlb miss anyway).
939 *
940 * Note also that when going to a fault, we must unwind the previous
941 * level as well. Since we are doing that, we don't need to clear or
942 * restore the TLB reservation neither.
943 */
944 subf r10,r13,r12
945 cmpldi cr0,r10,PACA_EXTLB+EX_TLB_SIZE
946 bne- virt_page_table_tlb_miss_whacko_fault
948 /* We dig the original DEAR and ESR from slot 0 */
949 ld r15,EX_TLB_DEAR+PACA_EXTLB(r13)
950 ld r16,EX_TLB_ESR+PACA_EXTLB(r13)
952 /* We check for the "special" ESR value for instruction faults */
953 cmpdi cr0,r16,-1
954 beq 1f
955 mtspr SPRN_DEAR,r15
956 mtspr SPRN_ESR,r16
957 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_PT_FAULT);
958 TLB_MISS_EPILOG_ERROR
959 b exc_data_storage_book3e
960 1: TLB_MISS_STATS_I(MMSTAT_TLB_MISS_PT_FAULT);
961 TLB_MISS_EPILOG_ERROR
962 b exc_instruction_storage_book3e
964 virt_page_table_tlb_miss_whacko_fault:
965 /* The linear fault will restart everything so ESR and DEAR will
966 * not have been clobbered, let's just fault with what we have
967 */
968 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_PT_FAULT);
969 TLB_MISS_EPILOG_ERROR
970 b exc_data_storage_book3e
973 /**************************************************************
974 * *
975 * TLB miss handling for Book3E with hw page table support *
976 * *
977 **************************************************************/
980 /* Data TLB miss */
981 START_EXCEPTION(data_tlb_miss_htw)
982 TLB_MISS_PROLOG
984 /* Now we handle the fault proper. We only save DEAR in normal
985 * fault case since that's the only interesting values here.
986 * We could probably also optimize by not saving SRR0/1 in the
987 * linear mapping case but I'll leave that for later
988 */
989 mfspr r14,SPRN_ESR
990 mfspr r16,SPRN_DEAR /* get faulting address */
991 srdi r11,r16,60 /* get region */
992 cmpldi cr0,r11,0xc /* linear mapping ? */
993 TLB_MISS_STATS_SAVE_INFO
994 beq tlb_load_linear /* yes -> go to linear map load */
996 /* We do the user/kernel test for the PID here along with the RW test
997 */
998 cmpldi cr0,r11,0 /* Check for user region */
999 ld r15,PACAPGD(r13) /* Load user pgdir */
1000 beq htw_tlb_miss
1002 /* XXX replace the RMW cycles with immediate loads + writes */
1003 1: mfspr r10,SPRN_MAS1
1004 cmpldi cr0,r11,8 /* Check for vmalloc region */
1005 rlwinm r10,r10,0,16,1 /* Clear TID */
1006 mtspr SPRN_MAS1,r10
1007 ld r15,PACA_KERNELPGD(r13) /* Load kernel pgdir */
1008 beq+ htw_tlb_miss
1010 /* We got a crappy address, just fault with whatever DEAR and ESR
1011 * are here
1012 */
1013 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
1014 TLB_MISS_EPILOG_ERROR
1015 b exc_data_storage_book3e
1017 /* Instruction TLB miss */
1018 START_EXCEPTION(instruction_tlb_miss_htw)
1019 TLB_MISS_PROLOG
1021 /* If we take a recursive fault, the second level handler may need
1022 * to know whether we are handling a data or instruction fault in
1023 * order to get to the right store fault handler. We provide that
1024 * info by keeping a crazy value for ESR in r14
1025 */
1026 li r14,-1 /* store to exception frame is done later */
1028 /* Now we handle the fault proper. We only save DEAR in the non
1029 * linear mapping case since we know the linear mapping case will
1030 * not re-enter. We could indeed optimize and also not save SRR0/1
1031 * in the linear mapping case but I'll leave that for later
1032 *
1033 * Faulting address is SRR0 which is already in r16
1034 */
1035 srdi r11,r16,60 /* get region */
1036 cmpldi cr0,r11,0xc /* linear mapping ? */
1037 TLB_MISS_STATS_SAVE_INFO
1038 beq tlb_load_linear /* yes -> go to linear map load */
1040 /* We do the user/kernel test for the PID here along with the RW test
1041 */
1042 cmpldi cr0,r11,0 /* Check for user region */
1043 ld r15,PACAPGD(r13) /* Load user pgdir */
1044 beq htw_tlb_miss
1046 /* XXX replace the RMW cycles with immediate loads + writes */
1047 1: mfspr r10,SPRN_MAS1
1048 cmpldi cr0,r11,8 /* Check for vmalloc region */
1049 rlwinm r10,r10,0,16,1 /* Clear TID */
1050 mtspr SPRN_MAS1,r10
1051 ld r15,PACA_KERNELPGD(r13) /* Load kernel pgdir */
1052 beq+ htw_tlb_miss
1054 /* We got a crappy address, just fault */
1055 TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
1056 TLB_MISS_EPILOG_ERROR
1057 b exc_instruction_storage_book3e
1060 /*
1061 * This is the guts of the second-level TLB miss handler for direct
1062 * misses. We are entered with:
1063 *
1064 * r16 = virtual page table faulting address
1065 * r15 = PGD pointer
1066 * r14 = ESR
1067 * r13 = PACA
1068 * r12 = TLB exception frame in PACA
1069 * r11 = crap (free to use)
1070 * r10 = crap (free to use)
1071 *
1072 * It can be re-entered by the linear mapping miss handler. However, to
1073 * avoid too much complication, it will save/restore things for us
1074 */
1075 htw_tlb_miss:
1076 /* Search if we already have a TLB entry for that virtual address, and
1077 * if we do, bail out.
1078 *
1079 * MAS1:IND should be already set based on MAS4
1080 */
1081 PPC_TLBSRX_DOT(0,R16)
1082 beq htw_tlb_miss_done
1084 /* Now, we need to walk the page tables. First check if we are in
1085 * range.
1086 */
1087 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
1088 bne- htw_tlb_miss_fault
1090 /* Get the PGD pointer */
1091 cmpldi cr0,r15,0
1092 beq- htw_tlb_miss_fault
1094 /* Get to PGD entry */
1095 rldicl r11,r16,64-(PGDIR_SHIFT-3),64-PGD_INDEX_SIZE-3
1096 clrrdi r10,r11,3
1097 ldx r15,r10,r15
1098 cmpdi cr0,r15,0
1099 bge htw_tlb_miss_fault
1101 #ifndef CONFIG_PPC_64K_PAGES
1102 /* Get to PUD entry */
1103 rldicl r11,r16,64-(PUD_SHIFT-3),64-PUD_INDEX_SIZE-3
1104 clrrdi r10,r11,3
1105 ldx r15,r10,r15
1106 cmpdi cr0,r15,0
1107 bge htw_tlb_miss_fault
1108 #endif /* CONFIG_PPC_64K_PAGES */
1110 /* Get to PMD entry */
1111 rldicl r11,r16,64-(PMD_SHIFT-3),64-PMD_INDEX_SIZE-3
1112 clrrdi r10,r11,3
1113 ldx r15,r10,r15
1114 cmpdi cr0,r15,0
1115 bge htw_tlb_miss_fault
1117 /* Ok, we're all right, we can now create an indirect entry for
1118 * a 1M or 256M page.
1119 *
1120 * The last trick is now that because we use "half" pages for
1121 * the HTW (1M IND is 2K and 256M IND is 32K) we need to account
1122 * for an added LSB bit to the RPN. For 64K pages, there is no
1123 * problem as we already use 32K arrays (half PTE pages), but for
1124 * 4K page we need to extract a bit from the virtual address and
1125 * insert it into the "PA52" bit of the RPN.
1126 */
1127 #ifndef CONFIG_PPC_64K_PAGES
1128 rlwimi r15,r16,32-9,20,20
1129 #endif
1130 /* Now we build the MAS:
1131 *
1132 * MAS 0 : Fully setup with defaults in MAS4 and TLBnCFG
1133 * MAS 1 : Almost fully setup
1134 * - PID already updated by caller if necessary
1135 * - TSIZE for now is base ind page size always
1136 * MAS 2 : Use defaults
1137 * MAS 3+7 : Needs to be done
1138 */
1139 #ifdef CONFIG_PPC_64K_PAGES
1140 ori r10,r15,(BOOK3E_PAGESZ_64K << MAS3_SPSIZE_SHIFT)
1141 #else
1142 ori r10,r15,(BOOK3E_PAGESZ_4K << MAS3_SPSIZE_SHIFT)
1143 #endif
1145 BEGIN_MMU_FTR_SECTION
1146 srdi r16,r10,32
1147 mtspr SPRN_MAS3,r10
1148 mtspr SPRN_MAS7,r16
1149 MMU_FTR_SECTION_ELSE
1150 mtspr SPRN_MAS7_MAS3,r10
1151 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_PAIRED_MAS)
1153 tlbwe
1155 htw_tlb_miss_done:
1156 /* We don't bother with restoring DEAR or ESR since we know we are
1157 * level 0 and just going back to userland. They are only needed
1158 * if you are going to take an access fault
1159 */
1160 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_PT_OK)
1161 TLB_MISS_EPILOG_SUCCESS
1162 rfi
1164 htw_tlb_miss_fault:
1165 /* We need to check if it was an instruction miss. We know this
1166 * though because r14 would contain -1
1167 */
1168 cmpdi cr0,r14,-1
1169 beq 1f
1170 mtspr SPRN_DEAR,r16
1171 mtspr SPRN_ESR,r14
1172 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_PT_FAULT)
1173 TLB_MISS_EPILOG_ERROR
1174 b exc_data_storage_book3e
1175 1: TLB_MISS_STATS_I(MMSTAT_TLB_MISS_PT_FAULT)
1176 TLB_MISS_EPILOG_ERROR
1177 b exc_instruction_storage_book3e
1179 /*
1180 * This is the guts of "any" level TLB miss handler for kernel linear
1181 * mapping misses. We are entered with:
1182 *
1183 *
1184 * r16 = faulting address
1185 * r15 = crap (free to use)
1186 * r14 = ESR (data) or -1 (instruction)
1187 * r13 = PACA
1188 * r12 = TLB exception frame in PACA
1189 * r11 = crap (free to use)
1190 * r10 = crap (free to use)
1191 *
1192 * In addition we know that we will not re-enter, so in theory, we could
1193 * use a simpler epilog not restoring SRR0/1 etc.. but we'll do that later.
1194 *
1195 * We also need to be careful about MAS registers here & TLB reservation,
1196 * as we know we'll have clobbered them if we interrupt the main TLB miss
1197 * handlers in which case we probably want to do a full restart at level
1198 * 0 rather than saving / restoring the MAS.
1199 *
1200 * Note: If we care about performance of that core, we can easily shuffle
1201 * a few things around
1202 */
1203 tlb_load_linear:
1204 /* For now, we assume the linear mapping is contiguous and stops at
1205 * linear_map_top. We also assume the size is a multiple of 1G, thus
1206 * we only use 1G pages for now. That might have to be changed in a
1207 * final implementation, especially when dealing with hypervisors
1208 */
1209 ld r11,PACATOC(r13)
1210 ld r11,linear_map_top@got(r11)
1211 ld r10,0(r11)
1212 tovirt(10,10)
1213 cmpld cr0,r16,r10
1214 bge tlb_load_linear_fault
1216 /* MAS1 need whole new setup. */
1217 li r15,(BOOK3E_PAGESZ_1GB<<MAS1_TSIZE_SHIFT)
1218 oris r15,r15,MAS1_VALID@h /* MAS1 needs V and TSIZE */
1219 mtspr SPRN_MAS1,r15
1221 /* Already somebody there ? */
1222 PPC_TLBSRX_DOT(0,R16)
1223 beq tlb_load_linear_done
1225 /* Now we build the remaining MAS. MAS0 and 2 should be fine
1226 * with their defaults, which leaves us with MAS 3 and 7. The
1227 * mapping is linear, so we just take the address, clear the
1228 * region bits, and or in the permission bits which are currently
1229 * hard wired
1230 */
1231 clrrdi r10,r16,30 /* 1G page index */
1232 clrldi r10,r10,4 /* clear region bits */
1233 ori r10,r10,MAS3_SR|MAS3_SW|MAS3_SX
1235 BEGIN_MMU_FTR_SECTION
1236 srdi r16,r10,32
1237 mtspr SPRN_MAS3,r10
1238 mtspr SPRN_MAS7,r16
1239 MMU_FTR_SECTION_ELSE
1240 mtspr SPRN_MAS7_MAS3,r10
1241 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_PAIRED_MAS)
1243 tlbwe
1245 tlb_load_linear_done:
1246 /* We use the "error" epilog for success as we do want to
1247 * restore to the initial faulting context, whatever it was.
1248 * We do that because we can't resume a fault within a TLB
1249 * miss handler, due to MAS and TLB reservation being clobbered.
1250 */
1251 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_LINEAR)
1252 TLB_MISS_EPILOG_ERROR
1253 rfi
1255 tlb_load_linear_fault:
1256 /* We keep the DEAR and ESR around, this shouldn't have happened */
1257 cmpdi cr0,r14,-1
1258 beq 1f
1259 TLB_MISS_EPILOG_ERROR_SPECIAL
1260 b exc_data_storage_book3e
1261 1: TLB_MISS_EPILOG_ERROR_SPECIAL
1262 b exc_instruction_storage_book3e
1265 #ifdef CONFIG_BOOK3E_MMU_TLB_STATS
1266 .tlb_stat_inc:
1267 1: ldarx r8,0,r9
1268 addi r8,r8,1
1269 stdcx. r8,0,r9
1270 bne- 1b
1271 blr
1272 #endif