2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Copyright (C) 1994 - 2000, 2001, 2003 Ralf Baechle
7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8 * Copyright (C) 2002, 2007 Maciej W. Rozycki
9 * Copyright (C) 2001, 2012 MIPS Technologies, Inc. All rights reserved.
11 #include <linux/init.h>
14 #include <asm/asmmacro.h>
15 #include <asm/cacheops.h>
16 #include <asm/irqflags.h>
17 #include <asm/regdef.h>
18 #include <asm/fpregdef.h>
19 #include <asm/mipsregs.h>
20 #include <asm/stackframe.h>
23 #include <asm/thread_info.h>
28 * General exception vector for all other CPUs.
30 * Be careful when changing this, it has to be at most 128 bytes
31 * to fit into space reserved for the exception handler.
33 NESTED(except_vec3_generic, 0, sp)
41 PTR_L k0, exception_handlers(k1)
44 END(except_vec3_generic)
47 * General exception handler for CPUs with virtual coherency exception.
49 * Be careful when changing this, it has to be at most 256 (as a special
50 * exception) bytes to fit into space reserved for the exception handler.
52 NESTED(except_vec3_r4000, 0, sp)
62 beq k1, k0, handle_vced
64 beq k1, k0, handle_vcei
69 PTR_L k0, exception_handlers(k1)
73 * Big shit, we now may have two dirty primary cache lines for the same
74 * physical address. We can safely invalidate the line pointed to by
75 * c0_badvaddr because after return from this exception handler the
76 * load / store will be re-executed.
80 li k1, -4 # Is this ...
81 and k0, k1 # ... really needed?
83 cache Index_Store_Tag_D, (k0)
84 cache Hit_Writeback_Inv_SD, (k0)
95 cache Hit_Writeback_Inv_SD, (k0) # also cleans pi
104 END(except_vec3_r4000)
108 .align 5 /* 32 byte rollback region */
112 /* start of rollback region */
113 LONG_L t0, TI_FLAGS($28)
115 andi t0, _TIF_NEED_RESCHED
120 #ifdef CONFIG_CPU_MICROMIPS
126 .set MIPS_ISA_ARCH_LEVEL_RAW
128 /* end of rollback region (the region size must be power of two) */
135 .macro BUILD_ROLLBACK_PROLOGUE handler
136 FEXPORT(rollback_\handler)
140 PTR_LA k1, __r4k_wait
141 ori k0, 0x1f /* 32 byte rollback region */
149 BUILD_ROLLBACK_PROLOGUE handle_int
150 NESTED(handle_int, PT_SIZE, sp)
152 #ifdef CONFIG_TRACE_IRQFLAGS
154 * Check to see if the interrupted code has just disabled
155 * interrupts and ignore this interrupt for now if so.
157 * local_irq_disable() disables interrupts and then calls
158 * trace_hardirqs_off() to track the state. If an interrupt is taken
159 * after interrupts are disabled but before the state is updated
160 * it will appear to restore_all that it is incorrectly returning with
161 * interrupts disabled
166 #if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX)
187 LONG_L s0, TI_REGS($28)
188 LONG_S sp, TI_REGS($28)
191 * SAVE_ALL ensures we are using a valid kernel stack for the thread.
192 * Check if we are already using the IRQ stack.
194 move s1, sp # Preserve the sp
196 /* Get IRQ stack for this CPU */
197 ASM_CPUID_MFC0 k0, ASM_SMP_CPUID_REG
198 #if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
199 lui k1, %hi(irq_stack)
201 lui k1, %highest(irq_stack)
202 daddiu k1, %higher(irq_stack)
204 daddiu k1, %hi(irq_stack)
207 LONG_SRL k0, SMP_CPUID_PTRSHIFT
209 LONG_L t0, %lo(irq_stack)(k1)
211 # Check if already on IRQ stack
212 PTR_LI t1, ~(_THREAD_SIZE-1)
216 /* Switch to IRQ stack */
217 li t1, _IRQ_STACK_START
220 /* Save task's sp on IRQ stack so that unwinding can follow it */
223 jal plat_irq_dispatch
229 #ifdef CONFIG_CPU_MICROMIPS
237 * Special interrupt vector for MIPS64 ISA & embedded MIPS processors.
238 * This is a dedicated interrupt exception vector which reduces the
239 * interrupt processing overhead. The jump instruction will be replaced
240 * at the initialization time.
242 * Be careful when changing this, it has to be at most 128 bytes
243 * to fit into space reserved for the exception handler.
245 NESTED(except_vec4, 0, sp)
246 1: j 1b /* Dummy, will be replaced */
250 * EJTAG debug exception handler.
251 * The EJTAG debug exception entry point is 0xbfc00480, which
252 * normally is in the boot PROM, so the boot PROM must do an
253 * unconditional jump to this vector.
255 NESTED(except_vec_ejtag_debug, 0, sp)
256 j ejtag_debug_handler
257 #ifdef CONFIG_CPU_MICROMIPS
260 END(except_vec_ejtag_debug)
265 * Vectored interrupt handler.
266 * This prototype is copied to ebase + n*IntCtl.VS and patched
267 * to invoke the handler
269 BUILD_ROLLBACK_PROLOGUE except_vec_vi
270 NESTED(except_vec_vi, 0, sp)
275 PTR_LA v1, except_vec_vi_handler
276 FEXPORT(except_vec_vi_lui)
277 lui v0, 0 /* Patched */
279 FEXPORT(except_vec_vi_ori)
280 ori v0, 0 /* Patched */
283 EXPORT(except_vec_vi_end)
286 * Common Vectored Interrupt code
287 * Complete the register saves and invoke the handler which is passed in $v0
289 NESTED(except_vec_vi_handler, 0, sp)
293 #ifdef CONFIG_TRACE_IRQFLAGS
299 LONG_L s0, TI_REGS($28)
300 LONG_S sp, TI_REGS($28)
303 * SAVE_ALL ensures we are using a valid kernel stack for the thread.
304 * Check if we are already using the IRQ stack.
306 move s1, sp # Preserve the sp
308 /* Get IRQ stack for this CPU */
309 ASM_CPUID_MFC0 k0, ASM_SMP_CPUID_REG
310 #if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
311 lui k1, %hi(irq_stack)
313 lui k1, %highest(irq_stack)
314 daddiu k1, %higher(irq_stack)
316 daddiu k1, %hi(irq_stack)
319 LONG_SRL k0, SMP_CPUID_PTRSHIFT
321 LONG_L t0, %lo(irq_stack)(k1)
323 # Check if already on IRQ stack
324 PTR_LI t1, ~(_THREAD_SIZE-1)
328 /* Switch to IRQ stack */
329 li t1, _IRQ_STACK_START
332 /* Save task's sp on IRQ stack so that unwinding can follow it */
341 END(except_vec_vi_handler)
344 * EJTAG debug exception handler.
346 NESTED(ejtag_debug_handler, PT_SIZE, sp)
352 sll k0, k0, 30 # Check for SDBBP.
353 bgez k0, ejtag_return
356 1: PTR_LA k0, ejtag_debug_buffer_spinlock
357 __SYNC(full, loongson3_war)
360 PTR_LA k0, ejtag_debug_buffer_spinlock
363 # ifdef CONFIG_WEAK_REORDERING_BEYOND_LLSC
367 PTR_LA k0, ejtag_debug_buffer
370 ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG
371 PTR_SRL k1, SMP_CPUID_PTRSHIFT
373 PTR_LA k0, ejtag_debug_buffer_per_cpu
376 PTR_LA k1, ejtag_debug_buffer
380 PTR_LA k0, ejtag_debug_buffer_spinlock
383 PTR_LA k0, ejtag_debug_buffer
389 jal ejtag_exception_handler
393 ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG
394 PTR_SRL k1, SMP_CPUID_PTRSHIFT
396 PTR_LA k0, ejtag_debug_buffer_per_cpu
400 PTR_LA k0, ejtag_debug_buffer
405 back_to_back_c0_hazard
410 END(ejtag_debug_handler)
413 * This buffer is reserved for the use of the EJTAG debug
417 EXPORT(ejtag_debug_buffer)
420 EXPORT(ejtag_debug_buffer_spinlock)
422 EXPORT(ejtag_debug_buffer_per_cpu)
423 .fill LONGSIZE * NR_CPUS
430 * NMI debug exception handler for MIPS reference boards.
431 * The NMI debug exception entry point is 0xbfc00000, which
432 * normally is in the boot PROM, so the boot PROM must do a
433 * unconditional jump to this vector.
435 NESTED(except_vec_nmi, 0, sp)
437 #ifdef CONFIG_CPU_MICROMIPS
444 NESTED(nmi_handler, PT_SIZE, sp)
449 * Clear ERL - restore segment mapping
450 * Clear BEV - required for page fault exception handler to work
454 li k1, ~(ST0_BEV | ST0_ERL)
460 jal nmi_exception_handler
461 /* nmi_exception_handler never returns */
465 .macro __build_clear_none
468 .macro __build_clear_sti
473 .macro __build_clear_cli
478 .macro __build_clear_fpe
482 /* gas fails to assemble cfc1 for some archs (octeon).*/ \
489 .macro __build_clear_msa_fpe
495 .macro __build_clear_ade
496 MFC0 t0, CP0_BADVADDR
497 PTR_S t0, PT_BVADDR(sp)
501 .macro __BUILD_silent exception
504 /* Gas tries to parse the PRINT argument as a string containing
505 string escapes and emits bogus warnings if it believes to
506 recognize an unknown escape code. So make the arguments
507 start with an n and gas will believe \n is ok ... */
508 .macro __BUILD_verbose nexception
509 LONG_L a1, PT_EPC(sp)
511 PRINT("Got \nexception at %08lx\012")
514 PRINT("Got \nexception at %016lx\012")
518 .macro __BUILD_count exception
519 LONG_L t0,exception_count_\exception
521 LONG_S t0,exception_count_\exception
522 .comm exception_count\exception, 8, 8
525 .macro __BUILD_HANDLER exception handler clear verbose ext
527 NESTED(handle_\exception, PT_SIZE, sp)
531 FEXPORT(handle_\exception\ext)
534 __BUILD_\verbose \exception
538 END(handle_\exception)
541 .macro BUILD_HANDLER exception handler clear verbose
542 __BUILD_HANDLER \exception \handler \clear \verbose _int
545 BUILD_HANDLER adel ade ade silent /* #4 */
546 BUILD_HANDLER ades ade ade silent /* #5 */
547 BUILD_HANDLER ibe be cli silent /* #6 */
548 BUILD_HANDLER dbe be cli silent /* #7 */
549 BUILD_HANDLER bp bp sti silent /* #9 */
550 BUILD_HANDLER ri ri sti silent /* #10 */
551 BUILD_HANDLER cpu cpu sti silent /* #11 */
552 BUILD_HANDLER ov ov sti silent /* #12 */
553 BUILD_HANDLER tr tr sti silent /* #13 */
554 BUILD_HANDLER msa_fpe msa_fpe msa_fpe silent /* #14 */
555 #ifdef CONFIG_MIPS_FP_SUPPORT
556 BUILD_HANDLER fpe fpe fpe silent /* #15 */
558 BUILD_HANDLER ftlb ftlb none silent /* #16 */
559 BUILD_HANDLER msa msa sti silent /* #21 */
560 BUILD_HANDLER mdmx mdmx sti silent /* #22 */
561 #ifdef CONFIG_HARDWARE_WATCHPOINTS
563 * For watch, interrupts will be enabled after the watch
564 * registers are read.
566 BUILD_HANDLER watch watch cli silent /* #23 */
568 BUILD_HANDLER watch watch sti verbose /* #23 */
570 BUILD_HANDLER mcheck mcheck cli verbose /* #24 */
571 BUILD_HANDLER mt mt sti silent /* #25 */
572 BUILD_HANDLER dsp dsp sti silent /* #26 */
573 BUILD_HANDLER reserved reserved sti verbose /* others */
576 LEAF(handle_ri_rdhwr_tlbp)
580 /* check if TLB contains a entry for EPC */
582 andi k1, MIPS_ENTRYHI_ASID | MIPS_ENTRYHI_ASIDX
584 PTR_SRL k0, _PAGE_SHIFT + 1
585 PTR_SLL k0, _PAGE_SHIFT + 1
593 bltz k1, handle_ri /* slow path */
595 END(handle_ri_rdhwr_tlbp)
597 LEAF(handle_ri_rdhwr)
601 /* MIPS32: 0x7c03e83b: rdhwr v1,$29 */
602 /* microMIPS: 0x007d6b3c: rdhwr v1,$29 */
604 #if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_MIPS32_R2) || defined(CONFIG_CPU_MIPS64_R2)
627 bne k0, k1, handle_ri /* if not ours */
630 /* The insn is rdhwr. No need to check CAUSE.BD here. */
631 get_saved_sp /* k1 := current_thread_info */
634 #if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX)
636 xori k1, _THREAD_MASK
637 LONG_L v1, TI_TP_VALUE(k1)
642 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
643 LONG_ADDIU k0, 4 /* stall on $k0 */
650 /* I hope three instructions between MTC0 and ERET are enough... */
652 xori k1, _THREAD_MASK
653 LONG_L v1, TI_TP_VALUE(k1)
662 #ifdef CONFIG_CPU_R4X00_BUGS64
663 /* A temporary overflow handler used by check_daddi(). */
667 BUILD_HANDLER daddi_ov daddi_ov none silent /* #12 */