?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
kvm tools, setup: Create private directory
[linux-2.6/next.git] / arch / mips / mm / tlbex.c
blob424ed4b92e6d9435107a96f260329b3a40ac2992
1 /*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Synthesize TLB refill handlers at runtime.
7 *
8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
9 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki
10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
11 * Copyright (C) 2008, 2009 Cavium Networks, Inc.
12 *
13 * ... and the days got worse and worse and now you see
14 * I've gone completly out of my mind.
15 *
16 * They're coming to take me a away haha
17 * they're coming to take me a away hoho hihi haha
18 * to the funny farm where code is beautiful all the time ...
19 *
20 * (Condolences to Napoleon XIV)
21 */
22
23 #include <linux/bug.h>
24 #include <linux/kernel.h>
25 #include <linux/types.h>
26 #include <linux/smp.h>
27 #include <linux/string.h>
28 #include <linux/init.h>
29 #include <linux/cache.h>
30
31 #include <asm/cacheflush.h>
32 #include <asm/pgtable.h>
33 #include <asm/war.h>
34 #include <asm/uasm.h>
35
36 /*
37 * TLB load/store/modify handlers.
38 *
39 * Only the fastpath gets synthesized at runtime, the slowpath for
40 * do_page_fault remains normal asm.
41 */
42 extern void tlb_do_page_fault_0(void);
43 extern void tlb_do_page_fault_1(void);
44
45
46 static inline int r45k_bvahwbug(void)
47 {
48 /* XXX: We should probe for the presence of this bug, but we don't. */
49 return 0;
50 }
51
52 static inline int r4k_250MHZhwbug(void)
53 {
54 /* XXX: We should probe for the presence of this bug, but we don't. */
55 return 0;
56 }
57
58 static inline int __maybe_unused bcm1250_m3_war(void)
59 {
60 return BCM1250_M3_WAR;
61 }
62
63 static inline int __maybe_unused r10000_llsc_war(void)
64 {
65 return R10000_LLSC_WAR;
66 }
67
68 static int use_bbit_insns(void)
69 {
70 switch (current_cpu_type()) {
71 case CPU_CAVIUM_OCTEON:
72 case CPU_CAVIUM_OCTEON_PLUS:
73 case CPU_CAVIUM_OCTEON2:
74 return 1;
75 default:
76 return 0;
77 }
78 }
79
80 static int use_lwx_insns(void)
81 {
82 switch (current_cpu_type()) {
83 case CPU_CAVIUM_OCTEON2:
84 return 1;
85 default:
86 return 0;
87 }
88 }
89 #if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \
90 CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0
91 static bool scratchpad_available(void)
92 {
93 return true;
94 }
95 static int scratchpad_offset(int i)
96 {
97 /*
98 * CVMSEG starts at address -32768 and extends for
99 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines.
100 */
101 i += 1; /* Kernel use starts at the top and works down. */
102 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768;
103 }
104 #else
105 static bool scratchpad_available(void)
106 {
107 return false;
108 }
109 static int scratchpad_offset(int i)
110 {
111 BUG();
112 /* Really unreachable, but evidently some GCC want this. */
113 return 0;
114 }
115 #endif
116 /*
117 * Found by experiment: At least some revisions of the 4kc throw under
118 * some circumstances a machine check exception, triggered by invalid
119 * values in the index register. Delaying the tlbp instruction until
120 * after the next branch, plus adding an additional nop in front of
121 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows
122 * why; it's not an issue caused by the core RTL.
123 *
124 */
125 static int __cpuinit m4kc_tlbp_war(void)
126 {
127 return (current_cpu_data.processor_id & 0xffff00) ==
128 (PRID_COMP_MIPS | PRID_IMP_4KC);
129 }
130
131 /* Handle labels (which must be positive integers). */
132 enum label_id {
133 label_second_part = 1,
134 label_leave,
135 label_vmalloc,
136 label_vmalloc_done,
137 label_tlbw_hazard,
138 label_split,
139 label_tlbl_goaround1,
140 label_tlbl_goaround2,
141 label_nopage_tlbl,
142 label_nopage_tlbs,
143 label_nopage_tlbm,
144 label_smp_pgtable_change,
145 label_r3000_write_probe_fail,
146 label_large_segbits_fault,
147 #ifdef CONFIG_HUGETLB_PAGE
148 label_tlb_huge_update,
149 #endif
150 };
151
152 UASM_L_LA(_second_part)
153 UASM_L_LA(_leave)
154 UASM_L_LA(_vmalloc)
155 UASM_L_LA(_vmalloc_done)
156 UASM_L_LA(_tlbw_hazard)
157 UASM_L_LA(_split)
158 UASM_L_LA(_tlbl_goaround1)
159 UASM_L_LA(_tlbl_goaround2)
160 UASM_L_LA(_nopage_tlbl)
161 UASM_L_LA(_nopage_tlbs)
162 UASM_L_LA(_nopage_tlbm)
163 UASM_L_LA(_smp_pgtable_change)
164 UASM_L_LA(_r3000_write_probe_fail)
165 UASM_L_LA(_large_segbits_fault)
166 #ifdef CONFIG_HUGETLB_PAGE
167 UASM_L_LA(_tlb_huge_update)
168 #endif
169
170 /*
171 * For debug purposes.
172 */
173 static inline void dump_handler(const u32 *handler, int count)
174 {
175 int i;
176
177 pr_debug("\t.set push\n");
178 pr_debug("\t.set noreorder\n");
179
180 for (i = 0; i < count; i++)
181 pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]);
182
183 pr_debug("\t.set pop\n");
184 }
185
186 /* The only general purpose registers allowed in TLB handlers. */
187 #define K0 26
188 #define K1 27
189
190 /* Some CP0 registers */
191 #define C0_INDEX 0, 0
192 #define C0_ENTRYLO0 2, 0
193 #define C0_TCBIND 2, 2
194 #define C0_ENTRYLO1 3, 0
195 #define C0_CONTEXT 4, 0
196 #define C0_PAGEMASK 5, 0
197 #define C0_BADVADDR 8, 0
198 #define C0_ENTRYHI 10, 0
199 #define C0_EPC 14, 0
200 #define C0_XCONTEXT 20, 0
201
202 #ifdef CONFIG_64BIT
203 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT)
204 #else
205 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT)
206 #endif
207
208 /* The worst case length of the handler is around 18 instructions for
209 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
210 * Maximum space available is 32 instructions for R3000 and 64
211 * instructions for R4000.
212 *
213 * We deliberately chose a buffer size of 128, so we won't scribble
214 * over anything important on overflow before we panic.
215 */
216 static u32 tlb_handler[128] __cpuinitdata;
217
218 /* simply assume worst case size for labels and relocs */
219 static struct uasm_label labels[128] __cpuinitdata;
220 static struct uasm_reloc relocs[128] __cpuinitdata;
221
222 #ifdef CONFIG_64BIT
223 static int check_for_high_segbits __cpuinitdata;
224 #endif
225
226 static int check_for_high_segbits __cpuinitdata;
227
228 static unsigned int kscratch_used_mask __cpuinitdata;
229
230 static int __cpuinit allocate_kscratch(void)
231 {
232 int r;
233 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask;
234
235 r = ffs(a);
236
237 if (r == 0)
238 return -1;
239
240 r--; /* make it zero based */
241
242 kscratch_used_mask |= (1 << r);
243
244 return r;
245 }
246
247 static int scratch_reg __cpuinitdata;
248 static int pgd_reg __cpuinitdata;
249 enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch};
250
251 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
252
253 /*
254 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
255 * we cannot do r3000 under these circumstances.
256 *
257 * Declare pgd_current here instead of including mmu_context.h to avoid type
258 * conflicts for tlbmiss_handler_setup_pgd
259 */
260 extern unsigned long pgd_current[];
261
262 /*
263 * The R3000 TLB handler is simple.
264 */
265 static void __cpuinit build_r3000_tlb_refill_handler(void)
266 {
267 long pgdc = (long)pgd_current;
268 u32 *p;
269
270 memset(tlb_handler, 0, sizeof(tlb_handler));
271 p = tlb_handler;
272
273 uasm_i_mfc0(&p, K0, C0_BADVADDR);
274 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */
275 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1);
276 uasm_i_srl(&p, K0, K0, 22); /* load delay */
277 uasm_i_sll(&p, K0, K0, 2);
278 uasm_i_addu(&p, K1, K1, K0);
279 uasm_i_mfc0(&p, K0, C0_CONTEXT);
280 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */
281 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */
282 uasm_i_addu(&p, K1, K1, K0);
283 uasm_i_lw(&p, K0, 0, K1);
284 uasm_i_nop(&p); /* load delay */
285 uasm_i_mtc0(&p, K0, C0_ENTRYLO0);
286 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
287 uasm_i_tlbwr(&p); /* cp0 delay */
288 uasm_i_jr(&p, K1);
289 uasm_i_rfe(&p); /* branch delay */
290
291 if (p > tlb_handler + 32)
292 panic("TLB refill handler space exceeded");
293
294 pr_debug("Wrote TLB refill handler (%u instructions).\n",
295 (unsigned int)(p - tlb_handler));
296
297 memcpy((void *)ebase, tlb_handler, 0x80);
298
299 dump_handler((u32 *)ebase, 32);
300 }
301 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
302
303 /*
304 * The R4000 TLB handler is much more complicated. We have two
305 * consecutive handler areas with 32 instructions space each.
306 * Since they aren't used at the same time, we can overflow in the
307 * other one.To keep things simple, we first assume linear space,
308 * then we relocate it to the final handler layout as needed.
309 */
310 static u32 final_handler[64] __cpuinitdata;
311
312 /*
313 * Hazards
314 *
315 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0:
316 * 2. A timing hazard exists for the TLBP instruction.
317 *
318 * stalling_instruction
319 * TLBP
320 *
321 * The JTLB is being read for the TLBP throughout the stall generated by the
322 * previous instruction. This is not really correct as the stalling instruction
323 * can modify the address used to access the JTLB. The failure symptom is that
324 * the TLBP instruction will use an address created for the stalling instruction
325 * and not the address held in C0_ENHI and thus report the wrong results.
326 *
327 * The software work-around is to not allow the instruction preceding the TLBP
328 * to stall - make it an NOP or some other instruction guaranteed not to stall.
329 *
330 * Errata 2 will not be fixed. This errata is also on the R5000.
331 *
332 * As if we MIPS hackers wouldn't know how to nop pipelines happy ...
333 */
334 static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p)
335 {
336 switch (current_cpu_type()) {
337 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */
338 case CPU_R4600:
339 case CPU_R4700:
340 case CPU_R5000:
341 case CPU_R5000A:
342 case CPU_NEVADA:
343 uasm_i_nop(p);
344 uasm_i_tlbp(p);
345 break;
346
347 default:
348 uasm_i_tlbp(p);
349 break;
350 }
351 }
352
353 /*
354 * Write random or indexed TLB entry, and care about the hazards from
355 * the preceding mtc0 and for the following eret.
356 */
357 enum tlb_write_entry { tlb_random, tlb_indexed };
358
359 static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
360 struct uasm_reloc **r,
361 enum tlb_write_entry wmode)
362 {
363 void(*tlbw)(u32 **) = NULL;
364
365 switch (wmode) {
366 case tlb_random: tlbw = uasm_i_tlbwr; break;
367 case tlb_indexed: tlbw = uasm_i_tlbwi; break;
368 }
369
370 if (cpu_has_mips_r2) {
371 if (cpu_has_mips_r2_exec_hazard)
372 uasm_i_ehb(p);
373 tlbw(p);
374 return;
375 }
376
377 switch (current_cpu_type()) {
378 case CPU_R4000PC:
379 case CPU_R4000SC:
380 case CPU_R4000MC:
381 case CPU_R4400PC:
382 case CPU_R4400SC:
383 case CPU_R4400MC:
384 /*
385 * This branch uses up a mtc0 hazard nop slot and saves
386 * two nops after the tlbw instruction.
387 */
388 uasm_il_bgezl(p, r, 0, label_tlbw_hazard);
389 tlbw(p);
390 uasm_l_tlbw_hazard(l, *p);
391 uasm_i_nop(p);
392 break;
393
394 case CPU_R4600:
395 case CPU_R4700:
396 case CPU_R5000:
397 case CPU_R5000A:
398 uasm_i_nop(p);
399 tlbw(p);
400 uasm_i_nop(p);
401 break;
402
403 case CPU_R4300:
404 case CPU_5KC:
405 case CPU_TX49XX:
406 case CPU_PR4450:
407 case CPU_XLR:
408 uasm_i_nop(p);
409 tlbw(p);
410 break;
411
412 case CPU_R10000:
413 case CPU_R12000:
414 case CPU_R14000:
415 case CPU_4KC:
416 case CPU_4KEC:
417 case CPU_SB1:
418 case CPU_SB1A:
419 case CPU_4KSC:
420 case CPU_20KC:
421 case CPU_25KF:
422 case CPU_BMIPS32:
423 case CPU_BMIPS3300:
424 case CPU_BMIPS4350:
425 case CPU_BMIPS4380:
426 case CPU_BMIPS5000:
427 case CPU_LOONGSON2:
428 case CPU_R5500:
429 if (m4kc_tlbp_war())
430 uasm_i_nop(p);
431 case CPU_ALCHEMY:
432 tlbw(p);
433 break;
434
435 case CPU_NEVADA:
436 uasm_i_nop(p); /* QED specifies 2 nops hazard */
437 /*
438 * This branch uses up a mtc0 hazard nop slot and saves
439 * a nop after the tlbw instruction.
440 */
441 uasm_il_bgezl(p, r, 0, label_tlbw_hazard);
442 tlbw(p);
443 uasm_l_tlbw_hazard(l, *p);
444 break;
445
446 case CPU_RM7000:
447 uasm_i_nop(p);
448 uasm_i_nop(p);
449 uasm_i_nop(p);
450 uasm_i_nop(p);
451 tlbw(p);
452 break;
453
454 case CPU_RM9000:
455 /*
456 * When the JTLB is updated by tlbwi or tlbwr, a subsequent
457 * use of the JTLB for instructions should not occur for 4
458 * cpu cycles and use for data translations should not occur
459 * for 3 cpu cycles.
460 */
461 uasm_i_ssnop(p);
462 uasm_i_ssnop(p);
463 uasm_i_ssnop(p);
464 uasm_i_ssnop(p);
465 tlbw(p);
466 uasm_i_ssnop(p);
467 uasm_i_ssnop(p);
468 uasm_i_ssnop(p);
469 uasm_i_ssnop(p);
470 break;
471
472 case CPU_VR4111:
473 case CPU_VR4121:
474 case CPU_VR4122:
475 case CPU_VR4181:
476 case CPU_VR4181A:
477 uasm_i_nop(p);
478 uasm_i_nop(p);
479 tlbw(p);
480 uasm_i_nop(p);
481 uasm_i_nop(p);
482 break;
483
484 case CPU_VR4131:
485 case CPU_VR4133:
486 case CPU_R5432:
487 uasm_i_nop(p);
488 uasm_i_nop(p);
489 tlbw(p);
490 break;
491
492 case CPU_JZRISC:
493 tlbw(p);
494 uasm_i_nop(p);
495 break;
496
497 default:
498 panic("No TLB refill handler yet (CPU type: %d)",
499 current_cpu_data.cputype);
500 break;
501 }
502 }
503
504 static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
505 unsigned int reg)
506 {
507 if (kernel_uses_smartmips_rixi) {
508 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC));
509 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
510 } else {
511 #ifdef CONFIG_64BIT_PHYS_ADDR
512 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL));
513 #else
514 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL));
515 #endif
516 }
517 }
518
519 #ifdef CONFIG_HUGETLB_PAGE
520
521 static __cpuinit void build_restore_pagemask(u32 **p,
522 struct uasm_reloc **r,
523 unsigned int tmp,
524 enum label_id lid,
525 int restore_scratch)
526 {
527 if (restore_scratch) {
528 /* Reset default page size */
529 if (PM_DEFAULT_MASK >> 16) {
530 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
531 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
532 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
533 uasm_il_b(p, r, lid);
534 } else if (PM_DEFAULT_MASK) {
535 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
536 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
537 uasm_il_b(p, r, lid);
538 } else {
539 uasm_i_mtc0(p, 0, C0_PAGEMASK);
540 uasm_il_b(p, r, lid);
541 }
542 if (scratch_reg > 0)
543 UASM_i_MFC0(p, 1, 31, scratch_reg);
544 else
545 UASM_i_LW(p, 1, scratchpad_offset(0), 0);
546 } else {
547 /* Reset default page size */
548 if (PM_DEFAULT_MASK >> 16) {
549 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
550 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
551 uasm_il_b(p, r, lid);
552 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
553 } else if (PM_DEFAULT_MASK) {
554 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
555 uasm_il_b(p, r, lid);
556 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
557 } else {
558 uasm_il_b(p, r, lid);
559 uasm_i_mtc0(p, 0, C0_PAGEMASK);
560 }
561 }
562 }
563
564 static __cpuinit void build_huge_tlb_write_entry(u32 **p,
565 struct uasm_label **l,
566 struct uasm_reloc **r,
567 unsigned int tmp,
568 enum tlb_write_entry wmode,
569 int restore_scratch)
570 {
571 /* Set huge page tlb entry size */
572 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
573 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff);
574 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
575
576 build_tlb_write_entry(p, l, r, wmode);
577
578 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch);
579 }
580
581 /*
582 * Check if Huge PTE is present, if so then jump to LABEL.
583 */
584 static void __cpuinit
585 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
586 unsigned int pmd, int lid)
587 {
588 UASM_i_LW(p, tmp, 0, pmd);
589 if (use_bbit_insns()) {
590 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid);
591 } else {
592 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE);
593 uasm_il_bnez(p, r, tmp, lid);
594 }
595 }
596
597 static __cpuinit void build_huge_update_entries(u32 **p,
598 unsigned int pte,
599 unsigned int tmp)
600 {
601 int small_sequence;
602
603 /*
604 * A huge PTE describes an area the size of the
605 * configured huge page size. This is twice the
606 * of the large TLB entry size we intend to use.
607 * A TLB entry half the size of the configured
608 * huge page size is configured into entrylo0
609 * and entrylo1 to cover the contiguous huge PTE
610 * address space.
611 */
612 small_sequence = (HPAGE_SIZE >> 7) < 0x10000;
613
614 /* We can clobber tmp. It isn't used after this.*/
615 if (!small_sequence)
616 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16));
617
618 build_convert_pte_to_entrylo(p, pte);
619 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */
620 /* convert to entrylo1 */
621 if (small_sequence)
622 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7);
623 else
624 UASM_i_ADDU(p, pte, pte, tmp);
625
626 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */
627 }
628
629 static __cpuinit void build_huge_handler_tail(u32 **p,
630 struct uasm_reloc **r,
631 struct uasm_label **l,
632 unsigned int pte,
633 unsigned int ptr)
634 {
635 #ifdef CONFIG_SMP
636 UASM_i_SC(p, pte, 0, ptr);
637 uasm_il_beqz(p, r, pte, label_tlb_huge_update);
638 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */
639 #else
640 UASM_i_SW(p, pte, 0, ptr);
641 #endif
642 build_huge_update_entries(p, pte, ptr);
643 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0);
644 }
645 #endif /* CONFIG_HUGETLB_PAGE */
646
647 #ifdef CONFIG_64BIT
648 /*
649 * TMP and PTR are scratch.
650 * TMP will be clobbered, PTR will hold the pmd entry.
651 */
652 static void __cpuinit
653 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
654 unsigned int tmp, unsigned int ptr)
655 {
656 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
657 long pgdc = (long)pgd_current;
658 #endif
659 /*
660 * The vmalloc handling is not in the hotpath.
661 */
662 uasm_i_dmfc0(p, tmp, C0_BADVADDR);
663
664 if (check_for_high_segbits) {
665 /*
666 * The kernel currently implicitely assumes that the
667 * MIPS SEGBITS parameter for the processor is
668 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never
669 * allocate virtual addresses outside the maximum
670 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But
671 * that doesn't prevent user code from accessing the
672 * higher xuseg addresses. Here, we make sure that
673 * everything but the lower xuseg addresses goes down
674 * the module_alloc/vmalloc path.
675 */
676 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
677 uasm_il_bnez(p, r, ptr, label_vmalloc);
678 } else {
679 uasm_il_bltz(p, r, tmp, label_vmalloc);
680 }
681 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
682
683 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT
684 if (pgd_reg != -1) {
685 /* pgd is in pgd_reg */
686 UASM_i_MFC0(p, ptr, 31, pgd_reg);
687 } else {
688 /*
689 * &pgd << 11 stored in CONTEXT [23..63].
690 */
691 UASM_i_MFC0(p, ptr, C0_CONTEXT);
692
693 /* Clear lower 23 bits of context. */
694 uasm_i_dins(p, ptr, 0, 0, 23);
695
696 /* 1 0 1 0 1 << 6 xkphys cached */
697 uasm_i_ori(p, ptr, ptr, 0x540);
698 uasm_i_drotr(p, ptr, ptr, 11);
699 }
700 #elif defined(CONFIG_SMP)
701 # ifdef CONFIG_MIPS_MT_SMTC
702 /*
703 * SMTC uses TCBind value as "CPU" index
704 */
705 uasm_i_mfc0(p, ptr, C0_TCBIND);
706 uasm_i_dsrl_safe(p, ptr, ptr, 19);
707 # else
708 /*
709 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
710 * stored in CONTEXT.
711 */
712 uasm_i_dmfc0(p, ptr, C0_CONTEXT);
713 uasm_i_dsrl_safe(p, ptr, ptr, 23);
714 # endif
715 UASM_i_LA_mostly(p, tmp, pgdc);
716 uasm_i_daddu(p, ptr, ptr, tmp);
717 uasm_i_dmfc0(p, tmp, C0_BADVADDR);
718 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
719 #else
720 UASM_i_LA_mostly(p, ptr, pgdc);
721 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
722 #endif
723
724 uasm_l_vmalloc_done(l, *p);
725
726 /* get pgd offset in bytes */
727 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3);
728
729 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
730 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
731 #ifndef __PAGETABLE_PMD_FOLDED
732 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
733 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */
734 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
735 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
736 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
737 #endif
738 }
739
740 /*
741 * BVADDR is the faulting address, PTR is scratch.
742 * PTR will hold the pgd for vmalloc.
743 */
744 static void __cpuinit
745 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
746 unsigned int bvaddr, unsigned int ptr,
747 enum vmalloc64_mode mode)
748 {
749 long swpd = (long)swapper_pg_dir;
750 int single_insn_swpd;
751 int did_vmalloc_branch = 0;
752
753 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd);
754
755 uasm_l_vmalloc(l, *p);
756
757 if (mode != not_refill && check_for_high_segbits) {
758 if (single_insn_swpd) {
759 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done);
760 uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
761 did_vmalloc_branch = 1;
762 /* fall through */
763 } else {
764 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault);
765 }
766 }
767 if (!did_vmalloc_branch) {
768 if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
769 uasm_il_b(p, r, label_vmalloc_done);
770 uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
771 } else {
772 UASM_i_LA_mostly(p, ptr, swpd);
773 uasm_il_b(p, r, label_vmalloc_done);
774 if (uasm_in_compat_space_p(swpd))
775 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
776 else
777 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
778 }
779 }
780 if (mode != not_refill && check_for_high_segbits) {
781 uasm_l_large_segbits_fault(l, *p);
782 /*
783 * We get here if we are an xsseg address, or if we are
784 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary.
785 *
786 * Ignoring xsseg (assume disabled so would generate
787 * (address errors?), the only remaining possibility
788 * is the upper xuseg addresses. On processors with
789 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these
790 * addresses would have taken an address error. We try
791 * to mimic that here by taking a load/istream page
792 * fault.
793 */
794 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0);
795 uasm_i_jr(p, ptr);
796
797 if (mode == refill_scratch) {
798 if (scratch_reg > 0)
799 UASM_i_MFC0(p, 1, 31, scratch_reg);
800 else
801 UASM_i_LW(p, 1, scratchpad_offset(0), 0);
802 } else {
803 uasm_i_nop(p);
804 }
805 }
806 }
807
808 #else /* !CONFIG_64BIT */
809
810 /*
811 * TMP and PTR are scratch.
812 * TMP will be clobbered, PTR will hold the pgd entry.
813 */
814 static void __cpuinit __maybe_unused
815 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
816 {
817 long pgdc = (long)pgd_current;
818
819 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
820 #ifdef CONFIG_SMP
821 #ifdef CONFIG_MIPS_MT_SMTC
822 /*
823 * SMTC uses TCBind value as "CPU" index
824 */
825 uasm_i_mfc0(p, ptr, C0_TCBIND);
826 UASM_i_LA_mostly(p, tmp, pgdc);
827 uasm_i_srl(p, ptr, ptr, 19);
828 #else
829 /*
830 * smp_processor_id() << 3 is stored in CONTEXT.
831 */
832 uasm_i_mfc0(p, ptr, C0_CONTEXT);
833 UASM_i_LA_mostly(p, tmp, pgdc);
834 uasm_i_srl(p, ptr, ptr, 23);
835 #endif
836 uasm_i_addu(p, ptr, tmp, ptr);
837 #else
838 UASM_i_LA_mostly(p, ptr, pgdc);
839 #endif
840 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
841 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
842 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
843 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2);
844 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
845 }
846
847 #endif /* !CONFIG_64BIT */
848
849 static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx)
850 {
851 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
852 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
853
854 switch (current_cpu_type()) {
855 case CPU_VR41XX:
856 case CPU_VR4111:
857 case CPU_VR4121:
858 case CPU_VR4122:
859 case CPU_VR4131:
860 case CPU_VR4181:
861 case CPU_VR4181A:
862 case CPU_VR4133:
863 shift += 2;
864 break;
865
866 default:
867 break;
868 }
869
870 if (shift)
871 UASM_i_SRL(p, ctx, ctx, shift);
872 uasm_i_andi(p, ctx, ctx, mask);
873 }
874
875 static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
876 {
877 /*
878 * Bug workaround for the Nevada. It seems as if under certain
879 * circumstances the move from cp0_context might produce a
880 * bogus result when the mfc0 instruction and its consumer are
881 * in a different cacheline or a load instruction, probably any
882 * memory reference, is between them.
883 */
884 switch (current_cpu_type()) {
885 case CPU_NEVADA:
886 UASM_i_LW(p, ptr, 0, ptr);
887 GET_CONTEXT(p, tmp); /* get context reg */
888 break;
889
890 default:
891 GET_CONTEXT(p, tmp); /* get context reg */
892 UASM_i_LW(p, ptr, 0, ptr);
893 break;
894 }
895
896 build_adjust_context(p, tmp);
897 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
898 }
899
900 static void __cpuinit build_update_entries(u32 **p, unsigned int tmp,
901 unsigned int ptep)
902 {
903 /*
904 * 64bit address support (36bit on a 32bit CPU) in a 32bit
905 * Kernel is a special case. Only a few CPUs use it.
906 */
907 #ifdef CONFIG_64BIT_PHYS_ADDR
908 if (cpu_has_64bits) {
909 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */
910 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
911 if (kernel_uses_smartmips_rixi) {
912 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC));
913 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC));
914 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
915 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
916 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
917 } else {
918 uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */
919 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
920 uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */
921 }
922 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
923 } else {
924 int pte_off_even = sizeof(pte_t) / 2;
925 int pte_off_odd = pte_off_even + sizeof(pte_t);
926
927 /* The pte entries are pre-shifted */
928 uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
929 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
930 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
931 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
932 }
933 #else
934 UASM_i_LW(p, tmp, 0, ptep); /* get even pte */
935 UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
936 if (r45k_bvahwbug())
937 build_tlb_probe_entry(p);
938 if (kernel_uses_smartmips_rixi) {
939 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC));
940 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC));
941 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
942 if (r4k_250MHZhwbug())
943 UASM_i_MTC0(p, 0, C0_ENTRYLO0);
944 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
945 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
946 } else {
947 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */
948 if (r4k_250MHZhwbug())
949 UASM_i_MTC0(p, 0, C0_ENTRYLO0);
950 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
951 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */
952 if (r45k_bvahwbug())
953 uasm_i_mfc0(p, tmp, C0_INDEX);
954 }
955 if (r4k_250MHZhwbug())
956 UASM_i_MTC0(p, 0, C0_ENTRYLO1);
957 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
958 #endif
959 }
960
961 struct mips_huge_tlb_info {
962 int huge_pte;
963 int restore_scratch;
964 };
965
966 static struct mips_huge_tlb_info __cpuinit
967 build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
968 struct uasm_reloc **r, unsigned int tmp,
969 unsigned int ptr, int c0_scratch)
970 {
971 struct mips_huge_tlb_info rv;
972 unsigned int even, odd;
973 int vmalloc_branch_delay_filled = 0;
974 const int scratch = 1; /* Our extra working register */
975
976 rv.huge_pte = scratch;
977 rv.restore_scratch = 0;
978
979 if (check_for_high_segbits) {
980 UASM_i_MFC0(p, tmp, C0_BADVADDR);
981
982 if (pgd_reg != -1)
983 UASM_i_MFC0(p, ptr, 31, pgd_reg);
984 else
985 UASM_i_MFC0(p, ptr, C0_CONTEXT);
986
987 if (c0_scratch >= 0)
988 UASM_i_MTC0(p, scratch, 31, c0_scratch);
989 else
990 UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
991
992 uasm_i_dsrl_safe(p, scratch, tmp,
993 PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
994 uasm_il_bnez(p, r, scratch, label_vmalloc);
995
996 if (pgd_reg == -1) {
997 vmalloc_branch_delay_filled = 1;
998 /* Clear lower 23 bits of context. */
999 uasm_i_dins(p, ptr, 0, 0, 23);
1000 }
1001 } else {
1002 if (pgd_reg != -1)
1003 UASM_i_MFC0(p, ptr, 31, pgd_reg);
1004 else
1005 UASM_i_MFC0(p, ptr, C0_CONTEXT);
1006
1007 UASM_i_MFC0(p, tmp, C0_BADVADDR);
1008
1009 if (c0_scratch >= 0)
1010 UASM_i_MTC0(p, scratch, 31, c0_scratch);
1011 else
1012 UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
1013
1014 if (pgd_reg == -1)
1015 /* Clear lower 23 bits of context. */
1016 uasm_i_dins(p, ptr, 0, 0, 23);
1017
1018 uasm_il_bltz(p, r, tmp, label_vmalloc);
1019 }
1020
1021 if (pgd_reg == -1) {
1022 vmalloc_branch_delay_filled = 1;
1023 /* 1 0 1 0 1 << 6 xkphys cached */
1024 uasm_i_ori(p, ptr, ptr, 0x540);
1025 uasm_i_drotr(p, ptr, ptr, 11);
1026 }
1027
1028 #ifdef __PAGETABLE_PMD_FOLDED
1029 #define LOC_PTEP scratch
1030 #else
1031 #define LOC_PTEP ptr
1032 #endif
1033
1034 if (!vmalloc_branch_delay_filled)
1035 /* get pgd offset in bytes */
1036 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
1037
1038 uasm_l_vmalloc_done(l, *p);
1039
1040 /*
1041 * tmp ptr
1042 * fall-through case = badvaddr *pgd_current
1043 * vmalloc case = badvaddr swapper_pg_dir
1044 */
1045
1046 if (vmalloc_branch_delay_filled)
1047 /* get pgd offset in bytes */
1048 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
1049
1050 #ifdef __PAGETABLE_PMD_FOLDED
1051 GET_CONTEXT(p, tmp); /* get context reg */
1052 #endif
1053 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3);
1054
1055 if (use_lwx_insns()) {
1056 UASM_i_LWX(p, LOC_PTEP, scratch, ptr);
1057 } else {
1058 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */
1059 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */
1060 }
1061
1062 #ifndef __PAGETABLE_PMD_FOLDED
1063 /* get pmd offset in bytes */
1064 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3);
1065 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3);
1066 GET_CONTEXT(p, tmp); /* get context reg */
1067
1068 if (use_lwx_insns()) {
1069 UASM_i_LWX(p, scratch, scratch, ptr);
1070 } else {
1071 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */
1072 UASM_i_LW(p, scratch, 0, ptr);
1073 }
1074 #endif
1075 /* Adjust the context during the load latency. */
1076 build_adjust_context(p, tmp);
1077
1078 #ifdef CONFIG_HUGETLB_PAGE
1079 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update);
1080 /*
1081 * The in the LWX case we don't want to do the load in the
1082 * delay slot. It cannot issue in the same cycle and may be
1083 * speculative and unneeded.
1084 */
1085 if (use_lwx_insns())
1086 uasm_i_nop(p);
1087 #endif /* CONFIG_HUGETLB_PAGE */
1088
1089
1090 /* build_update_entries */
1091 if (use_lwx_insns()) {
1092 even = ptr;
1093 odd = tmp;
1094 UASM_i_LWX(p, even, scratch, tmp);
1095 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t));
1096 UASM_i_LWX(p, odd, scratch, tmp);
1097 } else {
1098 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */
1099 even = tmp;
1100 odd = ptr;
1101 UASM_i_LW(p, even, 0, ptr); /* get even pte */
1102 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */
1103 }
1104 if (kernel_uses_smartmips_rixi) {
1105 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_NO_EXEC));
1106 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_NO_EXEC));
1107 uasm_i_drotr(p, even, even,
1108 ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
1109 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
1110 uasm_i_drotr(p, odd, odd,
1111 ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
1112 } else {
1113 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL));
1114 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
1115 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL));
1116 }
1117 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */
1118
1119 if (c0_scratch >= 0) {
1120 UASM_i_MFC0(p, scratch, 31, c0_scratch);
1121 build_tlb_write_entry(p, l, r, tlb_random);
1122 uasm_l_leave(l, *p);
1123 rv.restore_scratch = 1;
1124 } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) {
1125 build_tlb_write_entry(p, l, r, tlb_random);
1126 uasm_l_leave(l, *p);
1127 UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
1128 } else {
1129 UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
1130 build_tlb_write_entry(p, l, r, tlb_random);
1131 uasm_l_leave(l, *p);
1132 rv.restore_scratch = 1;
1133 }
1134
1135 uasm_i_eret(p); /* return from trap */
1136
1137 return rv;
1138 }
1139
1140 /*
1141 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception
1142 * because EXL == 0. If we wrap, we can also use the 32 instruction
1143 * slots before the XTLB refill exception handler which belong to the
1144 * unused TLB refill exception.
1145 */
1146 #define MIPS64_REFILL_INSNS 32
1147
1148 static void __cpuinit build_r4000_tlb_refill_handler(void)
1149 {
1150 u32 *p = tlb_handler;
1151 struct uasm_label *l = labels;
1152 struct uasm_reloc *r = relocs;
1153 u32 *f;
1154 unsigned int final_len;
1155 struct mips_huge_tlb_info htlb_info __maybe_unused;
1156 enum vmalloc64_mode vmalloc_mode __maybe_unused;
1157
1158 memset(tlb_handler, 0, sizeof(tlb_handler));
1159 memset(labels, 0, sizeof(labels));
1160 memset(relocs, 0, sizeof(relocs));
1161 memset(final_handler, 0, sizeof(final_handler));
1162
1163 if (scratch_reg == 0)
1164 scratch_reg = allocate_kscratch();
1165
1166 if ((scratch_reg > 0 || scratchpad_available()) && use_bbit_insns()) {
1167 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1,
1168 scratch_reg);
1169 vmalloc_mode = refill_scratch;
1170 } else {
1171 htlb_info.huge_pte = K0;
1172 htlb_info.restore_scratch = 0;
1173 vmalloc_mode = refill_noscratch;
1174 /*
1175 * create the plain linear handler
1176 */
1177 if (bcm1250_m3_war()) {
1178 unsigned int segbits = 44;
1179
1180 uasm_i_dmfc0(&p, K0, C0_BADVADDR);
1181 uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
1182 uasm_i_xor(&p, K0, K0, K1);
1183 uasm_i_dsrl_safe(&p, K1, K0, 62);
1184 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
1185 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
1186 uasm_i_or(&p, K0, K0, K1);
1187 uasm_il_bnez(&p, &r, K0, label_leave);
1188 /* No need for uasm_i_nop */
1189 }
1190
1191 #ifdef CONFIG_64BIT
1192 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
1193 #else
1194 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
1195 #endif
1196
1197 #ifdef CONFIG_HUGETLB_PAGE
1198 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update);
1199 #endif
1200
1201 build_get_ptep(&p, K0, K1);
1202 build_update_entries(&p, K0, K1);
1203 build_tlb_write_entry(&p, &l, &r, tlb_random);
1204 uasm_l_leave(&l, p);
1205 uasm_i_eret(&p); /* return from trap */
1206 }
1207 #ifdef CONFIG_HUGETLB_PAGE
1208 uasm_l_tlb_huge_update(&l, p);
1209 build_huge_update_entries(&p, htlb_info.huge_pte, K1);
1210 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random,
1211 htlb_info.restore_scratch);
1212 #endif
1213
1214 #ifdef CONFIG_64BIT
1215 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode);
1216 #endif
1217
1218 /*
1219 * Overflow check: For the 64bit handler, we need at least one
1220 * free instruction slot for the wrap-around branch. In worst
1221 * case, if the intended insertion point is a delay slot, we
1222 * need three, with the second nop'ed and the third being
1223 * unused.
1224 */
1225 /* Loongson2 ebase is different than r4k, we have more space */
1226 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
1227 if ((p - tlb_handler) > 64)
1228 panic("TLB refill handler space exceeded");
1229 #else
1230 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)
1231 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)
1232 && uasm_insn_has_bdelay(relocs,
1233 tlb_handler + MIPS64_REFILL_INSNS - 3)))
1234 panic("TLB refill handler space exceeded");
1235 #endif
1236
1237 /*
1238 * Now fold the handler in the TLB refill handler space.
1239 */
1240 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
1241 f = final_handler;
1242 /* Simplest case, just copy the handler. */
1243 uasm_copy_handler(relocs, labels, tlb_handler, p, f);
1244 final_len = p - tlb_handler;
1245 #else /* CONFIG_64BIT */
1246 f = final_handler + MIPS64_REFILL_INSNS;
1247 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {
1248 /* Just copy the handler. */
1249 uasm_copy_handler(relocs, labels, tlb_handler, p, f);
1250 final_len = p - tlb_handler;
1251 } else {
1252 #if defined(CONFIG_HUGETLB_PAGE)
1253 const enum label_id ls = label_tlb_huge_update;
1254 #else
1255 const enum label_id ls = label_vmalloc;
1256 #endif
1257 u32 *split;
1258 int ov = 0;
1259 int i;
1260
1261 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++)
1262 ;
1263 BUG_ON(i == ARRAY_SIZE(labels));
1264 split = labels[i].addr;
1265
1266 /*
1267 * See if we have overflown one way or the other.
1268 */
1269 if (split > tlb_handler + MIPS64_REFILL_INSNS ||
1270 split < p - MIPS64_REFILL_INSNS)
1271 ov = 1;
1272
1273 if (ov) {
1274 /*
1275 * Split two instructions before the end. One
1276 * for the branch and one for the instruction
1277 * in the delay slot.
1278 */
1279 split = tlb_handler + MIPS64_REFILL_INSNS - 2;
1280
1281 /*
1282 * If the branch would fall in a delay slot,
1283 * we must back up an additional instruction
1284 * so that it is no longer in a delay slot.
1285 */
1286 if (uasm_insn_has_bdelay(relocs, split - 1))
1287 split--;
1288 }
1289 /* Copy first part of the handler. */
1290 uasm_copy_handler(relocs, labels, tlb_handler, split, f);
1291 f += split - tlb_handler;
1292
1293 if (ov) {
1294 /* Insert branch. */
1295 uasm_l_split(&l, final_handler);
1296 uasm_il_b(&f, &r, label_split);
1297 if (uasm_insn_has_bdelay(relocs, split))
1298 uasm_i_nop(&f);
1299 else {
1300 uasm_copy_handler(relocs, labels,
1301 split, split + 1, f);
1302 uasm_move_labels(labels, f, f + 1, -1);
1303 f++;
1304 split++;
1305 }
1306 }
1307
1308 /* Copy the rest of the handler. */
1309 uasm_copy_handler(relocs, labels, split, p, final_handler);
1310 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +
1311 (p - split);
1312 }
1313 #endif /* CONFIG_64BIT */
1314
1315 uasm_resolve_relocs(relocs, labels);
1316 pr_debug("Wrote TLB refill handler (%u instructions).\n",
1317 final_len);
1318
1319 memcpy((void *)ebase, final_handler, 0x100);
1320
1321 dump_handler((u32 *)ebase, 64);
1322 }
1323
1324 /*
1325 * 128 instructions for the fastpath handler is generous and should
1326 * never be exceeded.
1327 */
1328 #define FASTPATH_SIZE 128
1329
1330 u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
1331 u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
1332 u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
1333 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT
1334 u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned;
1335
1336 static void __cpuinit build_r4000_setup_pgd(void)
1337 {
1338 const int a0 = 4;
1339 const int a1 = 5;
1340 u32 *p = tlbmiss_handler_setup_pgd;
1341 struct uasm_label *l = labels;
1342 struct uasm_reloc *r = relocs;
1343
1344 memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd));
1345 memset(labels, 0, sizeof(labels));
1346 memset(relocs, 0, sizeof(relocs));
1347
1348 pgd_reg = allocate_kscratch();
1349
1350 if (pgd_reg == -1) {
1351 /* PGD << 11 in c0_Context */
1352 /*
1353 * If it is a ckseg0 address, convert to a physical
1354 * address. Shifting right by 29 and adding 4 will
1355 * result in zero for these addresses.
1356 *
1357 */
1358 UASM_i_SRA(&p, a1, a0, 29);
1359 UASM_i_ADDIU(&p, a1, a1, 4);
1360 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1);
1361 uasm_i_nop(&p);
1362 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29);
1363 uasm_l_tlbl_goaround1(&l, p);
1364 UASM_i_SLL(&p, a0, a0, 11);
1365 uasm_i_jr(&p, 31);
1366 UASM_i_MTC0(&p, a0, C0_CONTEXT);
1367 } else {
1368 /* PGD in c0_KScratch */
1369 uasm_i_jr(&p, 31);
1370 UASM_i_MTC0(&p, a0, 31, pgd_reg);
1371 }
1372 if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd))
1373 panic("tlbmiss_handler_setup_pgd space exceeded");
1374 uasm_resolve_relocs(relocs, labels);
1375 pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n",
1376 (unsigned int)(p - tlbmiss_handler_setup_pgd));
1377
1378 dump_handler(tlbmiss_handler_setup_pgd,
1379 ARRAY_SIZE(tlbmiss_handler_setup_pgd));
1380 }
1381 #endif
1382
1383 static void __cpuinit
1384 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
1385 {
1386 #ifdef CONFIG_SMP
1387 # ifdef CONFIG_64BIT_PHYS_ADDR
1388 if (cpu_has_64bits)
1389 uasm_i_lld(p, pte, 0, ptr);
1390 else
1391 # endif
1392 UASM_i_LL(p, pte, 0, ptr);
1393 #else
1394 # ifdef CONFIG_64BIT_PHYS_ADDR
1395 if (cpu_has_64bits)
1396 uasm_i_ld(p, pte, 0, ptr);
1397 else
1398 # endif
1399 UASM_i_LW(p, pte, 0, ptr);
1400 #endif
1401 }
1402
1403 static void __cpuinit
1404 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
1405 unsigned int mode)
1406 {
1407 #ifdef CONFIG_64BIT_PHYS_ADDR
1408 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
1409 #endif
1410
1411 uasm_i_ori(p, pte, pte, mode);
1412 #ifdef CONFIG_SMP
1413 # ifdef CONFIG_64BIT_PHYS_ADDR
1414 if (cpu_has_64bits)
1415 uasm_i_scd(p, pte, 0, ptr);
1416 else
1417 # endif
1418 UASM_i_SC(p, pte, 0, ptr);
1419
1420 if (r10000_llsc_war())
1421 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change);
1422 else
1423 uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
1424
1425 # ifdef CONFIG_64BIT_PHYS_ADDR
1426 if (!cpu_has_64bits) {
1427 /* no uasm_i_nop needed */
1428 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr);
1429 uasm_i_ori(p, pte, pte, hwmode);
1430 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr);
1431 uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
1432 /* no uasm_i_nop needed */
1433 uasm_i_lw(p, pte, 0, ptr);
1434 } else
1435 uasm_i_nop(p);
1436 # else
1437 uasm_i_nop(p);
1438 # endif
1439 #else
1440 # ifdef CONFIG_64BIT_PHYS_ADDR
1441 if (cpu_has_64bits)
1442 uasm_i_sd(p, pte, 0, ptr);
1443 else
1444 # endif
1445 UASM_i_SW(p, pte, 0, ptr);
1446
1447 # ifdef CONFIG_64BIT_PHYS_ADDR
1448 if (!cpu_has_64bits) {
1449 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr);
1450 uasm_i_ori(p, pte, pte, hwmode);
1451 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr);
1452 uasm_i_lw(p, pte, 0, ptr);
1453 }
1454 # endif
1455 #endif
1456 }
1457
1458 /*
1459 * Check if PTE is present, if not then jump to LABEL. PTR points to
1460 * the page table where this PTE is located, PTE will be re-loaded
1461 * with it's original value.
1462 */
1463 static void __cpuinit
1464 build_pte_present(u32 **p, struct uasm_reloc **r,
1465 unsigned int pte, unsigned int ptr, enum label_id lid)
1466 {
1467 if (kernel_uses_smartmips_rixi) {
1468 if (use_bbit_insns()) {
1469 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
1470 uasm_i_nop(p);
1471 } else {
1472 uasm_i_andi(p, pte, pte, _PAGE_PRESENT);
1473 uasm_il_beqz(p, r, pte, lid);
1474 iPTE_LW(p, pte, ptr);
1475 }
1476 } else {
1477 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
1478 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
1479 uasm_il_bnez(p, r, pte, lid);
1480 iPTE_LW(p, pte, ptr);
1481 }
1482 }
1483
1484 /* Make PTE valid, store result in PTR. */
1485 static void __cpuinit
1486 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
1487 unsigned int ptr)
1488 {
1489 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
1490
1491 iPTE_SW(p, r, pte, ptr, mode);
1492 }
1493
1494 /*
1495 * Check if PTE can be written to, if not branch to LABEL. Regardless
1496 * restore PTE with value from PTR when done.
1497 */
1498 static void __cpuinit
1499 build_pte_writable(u32 **p, struct uasm_reloc **r,
1500 unsigned int pte, unsigned int ptr, enum label_id lid)
1501 {
1502 if (use_bbit_insns()) {
1503 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
1504 uasm_i_nop(p);
1505 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
1506 uasm_i_nop(p);
1507 } else {
1508 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
1509 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
1510 uasm_il_bnez(p, r, pte, lid);
1511 iPTE_LW(p, pte, ptr);
1512 }
1513 }
1514
1515 /* Make PTE writable, update software status bits as well, then store
1516 * at PTR.
1517 */
1518 static void __cpuinit
1519 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
1520 unsigned int ptr)
1521 {
1522 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
1523 | _PAGE_DIRTY);
1524
1525 iPTE_SW(p, r, pte, ptr, mode);
1526 }
1527
1528 /*
1529 * Check if PTE can be modified, if not branch to LABEL. Regardless
1530 * restore PTE with value from PTR when done.
1531 */
1532 static void __cpuinit
1533 build_pte_modifiable(u32 **p, struct uasm_reloc **r,
1534 unsigned int pte, unsigned int ptr, enum label_id lid)
1535 {
1536 if (use_bbit_insns()) {
1537 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
1538 uasm_i_nop(p);
1539 } else {
1540 uasm_i_andi(p, pte, pte, _PAGE_WRITE);
1541 uasm_il_beqz(p, r, pte, lid);
1542 iPTE_LW(p, pte, ptr);
1543 }
1544 }
1545
1546 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
1547
1548
1549 /*
1550 * R3000 style TLB load/store/modify handlers.
1551 */
1552
1553 /*
1554 * This places the pte into ENTRYLO0 and writes it with tlbwi.
1555 * Then it returns.
1556 */
1557 static void __cpuinit
1558 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
1559 {
1560 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1561 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
1562 uasm_i_tlbwi(p);
1563 uasm_i_jr(p, tmp);
1564 uasm_i_rfe(p); /* branch delay */
1565 }
1566
1567 /*
1568 * This places the pte into ENTRYLO0 and writes it with tlbwi
1569 * or tlbwr as appropriate. This is because the index register
1570 * may have the probe fail bit set as a result of a trap on a
1571 * kseg2 access, i.e. without refill. Then it returns.
1572 */
1573 static void __cpuinit
1574 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
1575 struct uasm_reloc **r, unsigned int pte,
1576 unsigned int tmp)
1577 {
1578 uasm_i_mfc0(p, tmp, C0_INDEX);
1579 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1580 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
1581 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */
1582 uasm_i_tlbwi(p); /* cp0 delay */
1583 uasm_i_jr(p, tmp);
1584 uasm_i_rfe(p); /* branch delay */
1585 uasm_l_r3000_write_probe_fail(l, *p);
1586 uasm_i_tlbwr(p); /* cp0 delay */
1587 uasm_i_jr(p, tmp);
1588 uasm_i_rfe(p); /* branch delay */
1589 }
1590
1591 static void __cpuinit
1592 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
1593 unsigned int ptr)
1594 {
1595 long pgdc = (long)pgd_current;
1596
1597 uasm_i_mfc0(p, pte, C0_BADVADDR);
1598 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */
1599 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
1600 uasm_i_srl(p, pte, pte, 22); /* load delay */
1601 uasm_i_sll(p, pte, pte, 2);
1602 uasm_i_addu(p, ptr, ptr, pte);
1603 uasm_i_mfc0(p, pte, C0_CONTEXT);
1604 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */
1605 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */
1606 uasm_i_addu(p, ptr, ptr, pte);
1607 uasm_i_lw(p, pte, 0, ptr);
1608 uasm_i_tlbp(p); /* load delay */
1609 }
1610
1611 static void __cpuinit build_r3000_tlb_load_handler(void)
1612 {
1613 u32 *p = handle_tlbl;
1614 struct uasm_label *l = labels;
1615 struct uasm_reloc *r = relocs;
1616
1617 memset(handle_tlbl, 0, sizeof(handle_tlbl));
1618 memset(labels, 0, sizeof(labels));
1619 memset(relocs, 0, sizeof(relocs));
1620
1621 build_r3000_tlbchange_handler_head(&p, K0, K1);
1622 build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
1623 uasm_i_nop(&p); /* load delay */
1624 build_make_valid(&p, &r, K0, K1);
1625 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1626
1627 uasm_l_nopage_tlbl(&l, p);
1628 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1629 uasm_i_nop(&p);
1630
1631 if ((p - handle_tlbl) > FASTPATH_SIZE)
1632 panic("TLB load handler fastpath space exceeded");
1633
1634 uasm_resolve_relocs(relocs, labels);
1635 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
1636 (unsigned int)(p - handle_tlbl));
1637
1638 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
1639 }
1640
1641 static void __cpuinit build_r3000_tlb_store_handler(void)
1642 {
1643 u32 *p = handle_tlbs;
1644 struct uasm_label *l = labels;
1645 struct uasm_reloc *r = relocs;
1646
1647 memset(handle_tlbs, 0, sizeof(handle_tlbs));
1648 memset(labels, 0, sizeof(labels));
1649 memset(relocs, 0, sizeof(relocs));
1650
1651 build_r3000_tlbchange_handler_head(&p, K0, K1);
1652 build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs);
1653 uasm_i_nop(&p); /* load delay */
1654 build_make_write(&p, &r, K0, K1);
1655 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1656
1657 uasm_l_nopage_tlbs(&l, p);
1658 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1659 uasm_i_nop(&p);
1660
1661 if ((p - handle_tlbs) > FASTPATH_SIZE)
1662 panic("TLB store handler fastpath space exceeded");
1663
1664 uasm_resolve_relocs(relocs, labels);
1665 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
1666 (unsigned int)(p - handle_tlbs));
1667
1668 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
1669 }
1670
1671 static void __cpuinit build_r3000_tlb_modify_handler(void)
1672 {
1673 u32 *p = handle_tlbm;
1674 struct uasm_label *l = labels;
1675 struct uasm_reloc *r = relocs;
1676
1677 memset(handle_tlbm, 0, sizeof(handle_tlbm));
1678 memset(labels, 0, sizeof(labels));
1679 memset(relocs, 0, sizeof(relocs));
1680
1681 build_r3000_tlbchange_handler_head(&p, K0, K1);
1682 build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm);
1683 uasm_i_nop(&p); /* load delay */
1684 build_make_write(&p, &r, K0, K1);
1685 build_r3000_pte_reload_tlbwi(&p, K0, K1);
1686
1687 uasm_l_nopage_tlbm(&l, p);
1688 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1689 uasm_i_nop(&p);
1690
1691 if ((p - handle_tlbm) > FASTPATH_SIZE)
1692 panic("TLB modify handler fastpath space exceeded");
1693
1694 uasm_resolve_relocs(relocs, labels);
1695 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
1696 (unsigned int)(p - handle_tlbm));
1697
1698 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
1699 }
1700 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
1701
1702 /*
1703 * R4000 style TLB load/store/modify handlers.
1704 */
1705 static void __cpuinit
1706 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
1707 struct uasm_reloc **r, unsigned int pte,
1708 unsigned int ptr)
1709 {
1710 #ifdef CONFIG_64BIT
1711 build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */
1712 #else
1713 build_get_pgde32(p, pte, ptr); /* get pgd in ptr */
1714 #endif
1715
1716 #ifdef CONFIG_HUGETLB_PAGE
1717 /*
1718 * For huge tlb entries, pmd doesn't contain an address but
1719 * instead contains the tlb pte. Check the PAGE_HUGE bit and
1720 * see if we need to jump to huge tlb processing.
1721 */
1722 build_is_huge_pte(p, r, pte, ptr, label_tlb_huge_update);
1723 #endif
1724
1725 UASM_i_MFC0(p, pte, C0_BADVADDR);
1726 UASM_i_LW(p, ptr, 0, ptr);
1727 UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
1728 uasm_i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
1729 UASM_i_ADDU(p, ptr, ptr, pte);
1730
1731 #ifdef CONFIG_SMP
1732 uasm_l_smp_pgtable_change(l, *p);
1733 #endif
1734 iPTE_LW(p, pte, ptr); /* get even pte */
1735 if (!m4kc_tlbp_war())
1736 build_tlb_probe_entry(p);
1737 }
1738
1739 static void __cpuinit
1740 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
1741 struct uasm_reloc **r, unsigned int tmp,
1742 unsigned int ptr)
1743 {
1744 uasm_i_ori(p, ptr, ptr, sizeof(pte_t));
1745 uasm_i_xori(p, ptr, ptr, sizeof(pte_t));
1746 build_update_entries(p, tmp, ptr);
1747 build_tlb_write_entry(p, l, r, tlb_indexed);
1748 uasm_l_leave(l, *p);
1749 uasm_i_eret(p); /* return from trap */
1750
1751 #ifdef CONFIG_64BIT
1752 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill);
1753 #endif
1754 }
1755
1756 static void __cpuinit build_r4000_tlb_load_handler(void)
1757 {
1758 u32 *p = handle_tlbl;
1759 struct uasm_label *l = labels;
1760 struct uasm_reloc *r = relocs;
1761
1762 memset(handle_tlbl, 0, sizeof(handle_tlbl));
1763 memset(labels, 0, sizeof(labels));
1764 memset(relocs, 0, sizeof(relocs));
1765
1766 if (bcm1250_m3_war()) {
1767 unsigned int segbits = 44;
1768
1769 uasm_i_dmfc0(&p, K0, C0_BADVADDR);
1770 uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
1771 uasm_i_xor(&p, K0, K0, K1);
1772 uasm_i_dsrl_safe(&p, K1, K0, 62);
1773 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
1774 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
1775 uasm_i_or(&p, K0, K0, K1);
1776 uasm_il_bnez(&p, &r, K0, label_leave);
1777 /* No need for uasm_i_nop */
1778 }
1779
1780 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1781 build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
1782 if (m4kc_tlbp_war())
1783 build_tlb_probe_entry(&p);
1784
1785 if (kernel_uses_smartmips_rixi) {
1786 /*
1787 * If the page is not _PAGE_VALID, RI or XI could not
1788 * have triggered it. Skip the expensive test..
1789 */
1790 if (use_bbit_insns()) {
1791 uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID),
1792 label_tlbl_goaround1);
1793 } else {
1794 uasm_i_andi(&p, K0, K0, _PAGE_VALID);
1795 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1);
1796 }
1797 uasm_i_nop(&p);
1798
1799 uasm_i_tlbr(&p);
1800 /* Examine entrylo 0 or 1 based on ptr. */
1801 if (use_bbit_insns()) {
1802 uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8);
1803 } else {
1804 uasm_i_andi(&p, K0, K1, sizeof(pte_t));
1805 uasm_i_beqz(&p, K0, 8);
1806 }
1807
1808 UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
1809 UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
1810 /*
1811 * If the entryLo (now in K0) is valid (bit 1), RI or
1812 * XI must have triggered it.
1813 */
1814 if (use_bbit_insns()) {
1815 uasm_il_bbit1(&p, &r, K0, 1, label_nopage_tlbl);
1816 /* Reload the PTE value */
1817 iPTE_LW(&p, K0, K1);
1818 uasm_l_tlbl_goaround1(&l, p);
1819 } else {
1820 uasm_i_andi(&p, K0, K0, 2);
1821 uasm_il_bnez(&p, &r, K0, label_nopage_tlbl);
1822 uasm_l_tlbl_goaround1(&l, p);
1823 /* Reload the PTE value */
1824 iPTE_LW(&p, K0, K1);
1825 }
1826 }
1827 build_make_valid(&p, &r, K0, K1);
1828 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1829
1830 #ifdef CONFIG_HUGETLB_PAGE
1831 /*
1832 * This is the entry point when build_r4000_tlbchange_handler_head
1833 * spots a huge page.
1834 */
1835 uasm_l_tlb_huge_update(&l, p);
1836 iPTE_LW(&p, K0, K1);
1837 build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
1838 build_tlb_probe_entry(&p);
1839
1840 if (kernel_uses_smartmips_rixi) {
1841 /*
1842 * If the page is not _PAGE_VALID, RI or XI could not
1843 * have triggered it. Skip the expensive test..
1844 */
1845 if (use_bbit_insns()) {
1846 uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID),
1847 label_tlbl_goaround2);
1848 } else {
1849 uasm_i_andi(&p, K0, K0, _PAGE_VALID);
1850 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
1851 }
1852 uasm_i_nop(&p);
1853
1854 uasm_i_tlbr(&p);
1855 /* Examine entrylo 0 or 1 based on ptr. */
1856 if (use_bbit_insns()) {
1857 uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8);
1858 } else {
1859 uasm_i_andi(&p, K0, K1, sizeof(pte_t));
1860 uasm_i_beqz(&p, K0, 8);
1861 }
1862 UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
1863 UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
1864 /*
1865 * If the entryLo (now in K0) is valid (bit 1), RI or
1866 * XI must have triggered it.
1867 */
1868 if (use_bbit_insns()) {
1869 uasm_il_bbit0(&p, &r, K0, 1, label_tlbl_goaround2);
1870 } else {
1871 uasm_i_andi(&p, K0, K0, 2);
1872 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
1873 }
1874 /* Reload the PTE value */
1875 iPTE_LW(&p, K0, K1);
1876
1877 /*
1878 * We clobbered C0_PAGEMASK, restore it. On the other branch
1879 * it is restored in build_huge_tlb_write_entry.
1880 */
1881 build_restore_pagemask(&p, &r, K0, label_nopage_tlbl, 0);
1882
1883 uasm_l_tlbl_goaround2(&l, p);
1884 }
1885 uasm_i_ori(&p, K0, K0, (_PAGE_ACCESSED | _PAGE_VALID));
1886 build_huge_handler_tail(&p, &r, &l, K0, K1);
1887 #endif
1888
1889 uasm_l_nopage_tlbl(&l, p);
1890 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1891 uasm_i_nop(&p);
1892
1893 if ((p - handle_tlbl) > FASTPATH_SIZE)
1894 panic("TLB load handler fastpath space exceeded");
1895
1896 uasm_resolve_relocs(relocs, labels);
1897 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
1898 (unsigned int)(p - handle_tlbl));
1899
1900 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
1901 }
1902
1903 static void __cpuinit build_r4000_tlb_store_handler(void)
1904 {
1905 u32 *p = handle_tlbs;
1906 struct uasm_label *l = labels;
1907 struct uasm_reloc *r = relocs;
1908
1909 memset(handle_tlbs, 0, sizeof(handle_tlbs));
1910 memset(labels, 0, sizeof(labels));
1911 memset(relocs, 0, sizeof(relocs));
1912
1913 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1914 build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs);
1915 if (m4kc_tlbp_war())
1916 build_tlb_probe_entry(&p);
1917 build_make_write(&p, &r, K0, K1);
1918 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1919
1920 #ifdef CONFIG_HUGETLB_PAGE
1921 /*
1922 * This is the entry point when
1923 * build_r4000_tlbchange_handler_head spots a huge page.
1924 */
1925 uasm_l_tlb_huge_update(&l, p);
1926 iPTE_LW(&p, K0, K1);
1927 build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs);
1928 build_tlb_probe_entry(&p);
1929 uasm_i_ori(&p, K0, K0,
1930 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
1931 build_huge_handler_tail(&p, &r, &l, K0, K1);
1932 #endif
1933
1934 uasm_l_nopage_tlbs(&l, p);
1935 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1936 uasm_i_nop(&p);
1937
1938 if ((p - handle_tlbs) > FASTPATH_SIZE)
1939 panic("TLB store handler fastpath space exceeded");
1940
1941 uasm_resolve_relocs(relocs, labels);
1942 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
1943 (unsigned int)(p - handle_tlbs));
1944
1945 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
1946 }
1947
1948 static void __cpuinit build_r4000_tlb_modify_handler(void)
1949 {
1950 u32 *p = handle_tlbm;
1951 struct uasm_label *l = labels;
1952 struct uasm_reloc *r = relocs;
1953
1954 memset(handle_tlbm, 0, sizeof(handle_tlbm));
1955 memset(labels, 0, sizeof(labels));
1956 memset(relocs, 0, sizeof(relocs));
1957
1958 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1959 build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm);
1960 if (m4kc_tlbp_war())
1961 build_tlb_probe_entry(&p);
1962 /* Present and writable bits set, set accessed and dirty bits. */
1963 build_make_write(&p, &r, K0, K1);
1964 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1965
1966 #ifdef CONFIG_HUGETLB_PAGE
1967 /*
1968 * This is the entry point when
1969 * build_r4000_tlbchange_handler_head spots a huge page.
1970 */
1971 uasm_l_tlb_huge_update(&l, p);
1972 iPTE_LW(&p, K0, K1);
1973 build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm);
1974 build_tlb_probe_entry(&p);
1975 uasm_i_ori(&p, K0, K0,
1976 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
1977 build_huge_handler_tail(&p, &r, &l, K0, K1);
1978 #endif
1979
1980 uasm_l_nopage_tlbm(&l, p);
1981 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1982 uasm_i_nop(&p);
1983
1984 if ((p - handle_tlbm) > FASTPATH_SIZE)
1985 panic("TLB modify handler fastpath space exceeded");
1986
1987 uasm_resolve_relocs(relocs, labels);
1988 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
1989 (unsigned int)(p - handle_tlbm));
1990
1991 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
1992 }
1993
1994 void __cpuinit build_tlb_refill_handler(void)
1995 {
1996 /*
1997 * The refill handler is generated per-CPU, multi-node systems
1998 * may have local storage for it. The other handlers are only
1999 * needed once.
2000 */
2001 static int run_once = 0;
2002
2003 #ifdef CONFIG_64BIT
2004 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
2005 #endif
2006
2007 switch (current_cpu_type()) {
2008 case CPU_R2000:
2009 case CPU_R3000:
2010 case CPU_R3000A:
2011 case CPU_R3081E:
2012 case CPU_TX3912:
2013 case CPU_TX3922:
2014 case CPU_TX3927:
2015 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
2016 build_r3000_tlb_refill_handler();
2017 if (!run_once) {
2018 build_r3000_tlb_load_handler();
2019 build_r3000_tlb_store_handler();
2020 build_r3000_tlb_modify_handler();
2021 run_once++;
2022 }
2023 #else
2024 panic("No R3000 TLB refill handler");
2025 #endif
2026 break;
2027
2028 case CPU_R6000:
2029 case CPU_R6000A:
2030 panic("No R6000 TLB refill handler yet");
2031 break;
2032
2033 case CPU_R8000:
2034 panic("No R8000 TLB refill handler yet");
2035 break;
2036
2037 default:
2038 if (!run_once) {
2039 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT
2040 build_r4000_setup_pgd();
2041 #endif
2042 build_r4000_tlb_load_handler();
2043 build_r4000_tlb_store_handler();
2044 build_r4000_tlb_modify_handler();
2045 run_once++;
2046 }
2047 build_r4000_tlb_refill_handler();
2048 }
2049 }
2050
2051 void __cpuinit flush_tlb_handlers(void)
2052 {
2053 local_flush_icache_range((unsigned long)handle_tlbl,
2054 (unsigned long)handle_tlbl + sizeof(handle_tlbl));
2055 local_flush_icache_range((unsigned long)handle_tlbs,
2056 (unsigned long)handle_tlbs + sizeof(handle_tlbs));
2057 local_flush_icache_range((unsigned long)handle_tlbm,
2058 (unsigned long)handle_tlbm + sizeof(handle_tlbm));
2059 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT
2060 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
2061 (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm));
2062 #endif
2063 }
linux-next