| blob | 048b8e9f44928ebbe3d9892ae8f7154d56ff1954 |
1 /*
2 * This file contains low-level functions for performing various
3 * types of TLB invalidations on various processors with no hash
4 * table.
5 *
6 * This file implements the following functions for all no-hash
7 * processors. Some aren't implemented for some variants. Some
8 * are inline in tlbflush.h
9 *
10 * - tlbil_va
11 * - tlbil_pid
12 * - tlbil_all
13 * - tlbivax_bcast
14 *
15 * Code mostly moved over from misc_32.S
16 *
17 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
18 *
19 * Partially rewritten by Cort Dougan (cort@cs.nmt.edu)
20 * Paul Mackerras, Kumar Gala and Benjamin Herrenschmidt.
21 *
22 * This program is free software; you can redistribute it and/or
23 * modify it under the terms of the GNU General Public License
24 * as published by the Free Software Foundation; either version
25 * 2 of the License, or (at your option) any later version.
26 *
27 */
29 #include <asm/reg.h>
30 #include <asm/page.h>
31 #include <asm/cputable.h>
32 #include <asm/mmu.h>
33 #include <asm/ppc_asm.h>
34 #include <asm/asm-offsets.h>
35 #include <asm/processor.h>
36 #include <asm/bug.h>
38 #if defined(CONFIG_40x)
40 /*
41 * 40x implementation needs only tlbil_va
42 */
43 _GLOBAL(__tlbil_va)
44 /* We run the search with interrupts disabled because we have to change
45 * the PID and I don't want to preempt when that happens.
46 */
47 mfmsr r5
48 mfspr r6,SPRN_PID
49 wrteei 0
50 mtspr SPRN_PID,r4
51 tlbsx. r3, 0, r3
52 mtspr SPRN_PID,r6
53 wrtee r5
54 bne 1f
55 sync
56 /* There are only 64 TLB entries, so r3 < 64, which means bit 25 is
57 * clear. Since 25 is the V bit in the TLB_TAG, loading this value
58 * will invalidate the TLB entry. */
59 tlbwe r3, r3, TLB_TAG
60 isync
61 1: blr
63 #elif defined(CONFIG_PPC_8xx)
65 /*
66 * Nothing to do for 8xx, everything is inline
67 */
69 #elif defined(CONFIG_44x) /* Includes 47x */
71 /*
72 * 440 implementation uses tlbsx/we for tlbil_va and a full sweep
73 * of the TLB for everything else.
74 */
75 _GLOBAL(__tlbil_va)
76 mfspr r5,SPRN_MMUCR
77 mfmsr r10
79 /*
80 * We write 16 bits of STID since 47x supports that much, we
81 * will never be passed out of bounds values on 440 (hopefully)
82 */
83 rlwimi r5,r4,0,16,31
85 /* We have to run the search with interrupts disabled, otherwise
86 * an interrupt which causes a TLB miss can clobber the MMUCR
87 * between the mtspr and the tlbsx.
88 *
89 * Critical and Machine Check interrupts take care of saving
90 * and restoring MMUCR, so only normal interrupts have to be
91 * taken care of.
92 */
93 wrteei 0
94 mtspr SPRN_MMUCR,r5
95 tlbsx. r6,0,r3
96 bne 10f
97 sync
98 BEGIN_MMU_FTR_SECTION
99 b 2f
100 END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x)
101 /* On 440 There are only 64 TLB entries, so r3 < 64, which means bit
102 * 22, is clear. Since 22 is the V bit in the TLB_PAGEID, loading this
103 * value will invalidate the TLB entry.
104 */
105 tlbwe r6,r6,PPC44x_TLB_PAGEID
106 isync
107 10: wrtee r10
108 blr
109 2:
110 #ifdef CONFIG_PPC_47x
111 oris r7,r6,0x8000 /* specify way explicitly */
112 clrrwi r4,r3,12 /* get an EPN for the hashing with V = 0 */
113 ori r4,r4,PPC47x_TLBE_SIZE
114 tlbwe r4,r7,0 /* write it */
115 isync
116 wrtee r10
117 blr
118 #else /* CONFIG_PPC_47x */
119 1: trap
120 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0;
121 #endif /* !CONFIG_PPC_47x */
123 _GLOBAL(_tlbil_all)
124 _GLOBAL(_tlbil_pid)
125 BEGIN_MMU_FTR_SECTION
126 b 2f
127 END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x)
128 li r3,0
129 sync
131 /* Load high watermark */
132 lis r4,tlb_44x_hwater@ha
133 lwz r5,tlb_44x_hwater@l(r4)
135 1: tlbwe r3,r3,PPC44x_TLB_PAGEID
136 addi r3,r3,1
137 cmpw 0,r3,r5
138 ble 1b
140 isync
141 blr
142 2:
143 #ifdef CONFIG_PPC_47x
144 /* 476 variant. There's not simple way to do this, hopefully we'll
145 * try to limit the amount of such full invalidates
146 */
147 mfmsr r11 /* Interrupts off */
148 wrteei 0
149 li r3,-1 /* Current set */
150 lis r10,tlb_47x_boltmap@h
151 ori r10,r10,tlb_47x_boltmap@l
152 lis r7,0x8000 /* Specify way explicitly */
154 b 9f /* For each set */
156 1: li r9,4 /* Number of ways */
157 li r4,0 /* Current way */
158 li r6,0 /* Default entry value 0 */
159 andi. r0,r8,1 /* Check if way 0 is bolted */
160 mtctr r9 /* Load way counter */
161 bne- 3f /* Bolted, skip loading it */
163 2: /* For each way */
164 or r5,r3,r4 /* Make way|index for tlbre */
165 rlwimi r5,r5,16,8,15 /* Copy index into position */
166 tlbre r6,r5,0 /* Read entry */
167 3: addis r4,r4,0x2000 /* Next way */
168 andi. r0,r6,PPC47x_TLB0_VALID /* Valid entry ? */
169 beq 4f /* Nope, skip it */
170 rlwimi r7,r5,0,1,2 /* Insert way number */
171 rlwinm r6,r6,0,21,19 /* Clear V */
172 tlbwe r6,r7,0 /* Write it */
173 4: bdnz 2b /* Loop for each way */
174 srwi r8,r8,1 /* Next boltmap bit */
175 9: cmpwi cr1,r3,255 /* Last set done ? */
176 addi r3,r3,1 /* Next set */
177 beq cr1,1f /* End of loop */
178 andi. r0,r3,0x1f /* Need to load a new boltmap word ? */
179 bne 1b /* No, loop */
180 lwz r8,0(r10) /* Load boltmap entry */
181 addi r10,r10,4 /* Next word */
182 b 1b /* Then loop */
183 1: isync /* Sync shadows */
184 wrtee r11
185 #else /* CONFIG_PPC_47x */
186 1: trap
187 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0;
188 #endif /* !CONFIG_PPC_47x */
189 blr
191 #ifdef CONFIG_PPC_47x
193 /*
194 * _tlbivax_bcast is only on 47x. We don't bother doing a runtime
195 * check though, it will blow up soon enough if we mistakenly try
196 * to use it on a 440.
197 */
198 _GLOBAL(_tlbivax_bcast)
199 mfspr r5,SPRN_MMUCR
200 mfmsr r10
201 rlwimi r5,r4,0,16,31
202 wrteei 0
203 mtspr SPRN_MMUCR,r5
204 isync
205 PPC_TLBIVAX(0, R3)
206 isync
207 eieio
208 tlbsync
209 BEGIN_FTR_SECTION
210 b 1f
211 END_FTR_SECTION_IFSET(CPU_FTR_476_DD2)
212 sync
213 wrtee r10
214 blr
215 /*
216 * DD2 HW could hang if in instruction fetch happens before msync completes.
217 * Touch enough instruction cache lines to ensure cache hits
218 */
219 1: mflr r9
220 bl 2f
221 2: mflr r6
222 li r7,32
223 PPC_ICBT(0,R6,R7) /* touch next cache line */
224 add r6,r6,r7
225 PPC_ICBT(0,R6,R7) /* touch next cache line */
226 add r6,r6,r7
227 PPC_ICBT(0,R6,R7) /* touch next cache line */
228 sync
229 nop
230 nop
231 nop
232 nop
233 nop
234 nop
235 nop
236 nop
237 mtlr r9
238 wrtee r10
239 blr
240 #endif /* CONFIG_PPC_47x */
242 #elif defined(CONFIG_FSL_BOOKE)
243 /*
244 * FSL BookE implementations.
245 *
246 * Since feature sections are using _SECTION_ELSE we need
247 * to have the larger code path before the _SECTION_ELSE
248 */
250 /*
251 * Flush MMU TLB on the local processor
252 */
253 _GLOBAL(_tlbil_all)
254 BEGIN_MMU_FTR_SECTION
255 li r3,(MMUCSR0_TLBFI)@l
256 mtspr SPRN_MMUCSR0, r3
257 1:
258 mfspr r3,SPRN_MMUCSR0
259 andi. r3,r3,MMUCSR0_TLBFI@l
260 bne 1b
261 MMU_FTR_SECTION_ELSE
262 PPC_TLBILX_ALL(0,R0)
263 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX)
264 msync
265 isync
266 blr
268 _GLOBAL(_tlbil_pid)
269 BEGIN_MMU_FTR_SECTION
270 slwi r3,r3,16
271 mfmsr r10
272 wrteei 0
273 mfspr r4,SPRN_MAS6 /* save MAS6 */
274 mtspr SPRN_MAS6,r3
275 PPC_TLBILX_PID(0,R0)
276 mtspr SPRN_MAS6,r4 /* restore MAS6 */
277 wrtee r10
278 MMU_FTR_SECTION_ELSE
279 li r3,(MMUCSR0_TLBFI)@l
280 mtspr SPRN_MMUCSR0, r3
281 1:
282 mfspr r3,SPRN_MMUCSR0
283 andi. r3,r3,MMUCSR0_TLBFI@l
284 bne 1b
285 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBILX)
286 msync
287 isync
288 blr
290 /*
291 * Flush MMU TLB for a particular address, but only on the local processor
292 * (no broadcast)
293 */
294 _GLOBAL(__tlbil_va)
295 mfmsr r10
296 wrteei 0
297 slwi r4,r4,16
298 ori r4,r4,(MAS6_ISIZE(BOOK3E_PAGESZ_4K))@l
299 mtspr SPRN_MAS6,r4 /* assume AS=0 for now */
300 BEGIN_MMU_FTR_SECTION
301 tlbsx 0,r3
302 mfspr r4,SPRN_MAS1 /* check valid */
303 andis. r3,r4,MAS1_VALID@h
304 beq 1f
305 rlwinm r4,r4,0,1,31
306 mtspr SPRN_MAS1,r4
307 tlbwe
308 MMU_FTR_SECTION_ELSE
309 PPC_TLBILX_VA(0,R3)
310 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX)
311 msync
312 isync
313 1: wrtee r10
314 blr
315 #elif defined(CONFIG_PPC_BOOK3E)
316 /*
317 * New Book3E (>= 2.06) implementation
318 *
319 * Note: We may be able to get away without the interrupt masking stuff
320 * if we save/restore MAS6 on exceptions that might modify it
321 */
322 _GLOBAL(_tlbil_pid)
323 slwi r4,r3,MAS6_SPID_SHIFT
324 mfmsr r10
325 wrteei 0
326 mtspr SPRN_MAS6,r4
327 PPC_TLBILX_PID(0,R0)
328 wrtee r10
329 msync
330 isync
331 blr
333 _GLOBAL(_tlbil_pid_noind)
334 slwi r4,r3,MAS6_SPID_SHIFT
335 mfmsr r10
336 ori r4,r4,MAS6_SIND
337 wrteei 0
338 mtspr SPRN_MAS6,r4
339 PPC_TLBILX_PID(0,R0)
340 wrtee r10
341 msync
342 isync
343 blr
345 _GLOBAL(_tlbil_all)
346 PPC_TLBILX_ALL(0,R0)
347 msync
348 isync
349 blr
351 _GLOBAL(_tlbil_va)
352 mfmsr r10
353 wrteei 0
354 cmpwi cr0,r6,0
355 slwi r4,r4,MAS6_SPID_SHIFT
356 rlwimi r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK
357 beq 1f
358 rlwimi r4,r6,MAS6_SIND_SHIFT,MAS6_SIND
359 1: mtspr SPRN_MAS6,r4 /* assume AS=0 for now */
360 PPC_TLBILX_VA(0,R3)
361 msync
362 isync
363 wrtee r10
364 blr
366 _GLOBAL(_tlbivax_bcast)
367 mfmsr r10
368 wrteei 0
369 cmpwi cr0,r6,0
370 slwi r4,r4,MAS6_SPID_SHIFT
371 rlwimi r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK
372 beq 1f
373 rlwimi r4,r6,MAS6_SIND_SHIFT,MAS6_SIND
374 1: mtspr SPRN_MAS6,r4 /* assume AS=0 for now */
375 PPC_TLBIVAX(0,R3)
376 eieio
377 tlbsync
378 sync
379 wrtee r10
380 blr
382 _GLOBAL(set_context)
383 #ifdef CONFIG_BDI_SWITCH
384 /* Context switch the PTE pointer for the Abatron BDI2000.
385 * The PGDIR is the second parameter.
386 */
387 lis r5, abatron_pteptrs@h
388 ori r5, r5, abatron_pteptrs@l
389 stw r4, 0x4(r5)
390 #endif
391 mtspr SPRN_PID,r3
392 isync /* Force context change */
393 blr
394 #else
395 #error Unsupported processor type !
396 #endif
398 #if defined(CONFIG_PPC_FSL_BOOK3E)
399 /*
400 * extern void loadcam_entry(unsigned int index)
401 *
402 * Load TLBCAM[index] entry in to the L2 CAM MMU
403 * Must preserve r7, r8, r9, and r10
404 */
405 _GLOBAL(loadcam_entry)
406 mflr r5
407 LOAD_REG_ADDR_PIC(r4, TLBCAM)
408 mtlr r5
409 mulli r5,r3,TLBCAM_SIZE
410 add r3,r5,r4
411 lwz r4,TLBCAM_MAS0(r3)
412 mtspr SPRN_MAS0,r4
413 lwz r4,TLBCAM_MAS1(r3)
414 mtspr SPRN_MAS1,r4
415 PPC_LL r4,TLBCAM_MAS2(r3)
416 mtspr SPRN_MAS2,r4
417 lwz r4,TLBCAM_MAS3(r3)
418 mtspr SPRN_MAS3,r4
419 BEGIN_MMU_FTR_SECTION
420 lwz r4,TLBCAM_MAS7(r3)
421 mtspr SPRN_MAS7,r4
422 END_MMU_FTR_SECTION_IFSET(MMU_FTR_BIG_PHYS)
423 isync
424 tlbwe
425 isync
426 blr
428 /*
429 * Load multiple TLB entries at once, using an alternate-space
430 * trampoline so that we don't have to care about whether the same
431 * TLB entry maps us before and after.
432 *
433 * r3 = first entry to write
434 * r4 = number of entries to write
435 * r5 = temporary tlb entry
436 */
437 _GLOBAL(loadcam_multi)
438 mflr r8
440 /*
441 * Set up temporary TLB entry that is the same as what we're
442 * running from, but in AS=1.
443 */
444 bl 1f
445 1: mflr r6
446 tlbsx 0,r8
447 mfspr r6,SPRN_MAS1
448 ori r6,r6,MAS1_TS
449 mtspr SPRN_MAS1,r6
450 mfspr r6,SPRN_MAS0
451 rlwimi r6,r5,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK
452 mr r7,r5
453 mtspr SPRN_MAS0,r6
454 isync
455 tlbwe
456 isync
458 /* Switch to AS=1 */
459 mfmsr r6
460 ori r6,r6,MSR_IS|MSR_DS
461 mtmsr r6
462 isync
464 mr r9,r3
465 add r10,r3,r4
466 2: bl loadcam_entry
467 addi r9,r9,1
468 cmpw r9,r10
469 mr r3,r9
470 blt 2b
472 /* Return to AS=0 and clear the temporary entry */
473 mfmsr r6
474 rlwinm. r6,r6,0,~(MSR_IS|MSR_DS)
475 mtmsr r6
476 isync
478 li r6,0
479 mtspr SPRN_MAS1,r6
480 rlwinm r6,r7,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK
481 oris r6,r6,MAS0_TLBSEL(1)@h
482 mtspr SPRN_MAS0,r6
483 isync
484 tlbwe
485 isync
487 mtlr r8
488 blr
489 #endif