| blob | 028a1b91e2b3a417f9a30a5ce6c7dd5980f4adfc |
1 /*
2 * Low-level exception handling code
3 *
4 * Copyright (C) 2012 ARM Ltd.
5 * Authors: Catalin Marinas <catalin.marinas@arm.com>
6 * Will Deacon <will.deacon@arm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
21 #include <linux/init.h>
22 #include <linux/linkage.h>
24 #include <asm/assembler.h>
25 #include <asm/asm-offsets.h>
26 #include <asm/errno.h>
27 #include <asm/esr.h>
28 #include <asm/thread_info.h>
29 #include <asm/unistd.h>
30 #include <asm/unistd32.h>
32 /*
33 * Bad Abort numbers
34 *-----------------
35 */
36 #define BAD_SYNC 0
37 #define BAD_IRQ 1
38 #define BAD_FIQ 2
39 #define BAD_ERROR 3
41 .macro kernel_entry, el, regsize = 64
42 sub sp, sp, #S_FRAME_SIZE - S_LR // room for LR, SP, SPSR, ELR
43 .if \regsize == 32
44 mov w0, w0 // zero upper 32 bits of x0
45 .endif
46 push x28, x29
47 push x26, x27
48 push x24, x25
49 push x22, x23
50 push x20, x21
51 push x18, x19
52 push x16, x17
53 push x14, x15
54 push x12, x13
55 push x10, x11
56 push x8, x9
57 push x6, x7
58 push x4, x5
59 push x2, x3
60 push x0, x1
61 .if \el == 0
62 mrs x21, sp_el0
63 .else
64 add x21, sp, #S_FRAME_SIZE
65 .endif
66 mrs x22, elr_el1
67 mrs x23, spsr_el1
68 stp lr, x21, [sp, #S_LR]
69 stp x22, x23, [sp, #S_PC]
71 /*
72 * Set syscallno to -1 by default (overridden later if real syscall).
73 */
74 .if \el == 0
75 mvn x21, xzr
76 str x21, [sp, #S_SYSCALLNO]
77 .endif
79 /*
80 * Registers that may be useful after this macro is invoked:
81 *
82 * x21 - aborted SP
83 * x22 - aborted PC
84 * x23 - aborted PSTATE
85 */
86 .endm
88 .macro kernel_exit, el, ret = 0
89 ldp x21, x22, [sp, #S_PC] // load ELR, SPSR
90 .if \el == 0
91 ldr x23, [sp, #S_SP] // load return stack pointer
92 .endif
93 .if \ret
94 ldr x1, [sp, #S_X1] // preserve x0 (syscall return)
95 add sp, sp, S_X2
96 .else
97 pop x0, x1
98 .endif
99 pop x2, x3 // load the rest of the registers
100 pop x4, x5
101 pop x6, x7
102 pop x8, x9
103 msr elr_el1, x21 // set up the return data
104 msr spsr_el1, x22
105 .if \el == 0
106 msr sp_el0, x23
107 .endif
108 pop x10, x11
109 pop x12, x13
110 pop x14, x15
111 pop x16, x17
112 pop x18, x19
113 pop x20, x21
114 pop x22, x23
115 pop x24, x25
116 pop x26, x27
117 pop x28, x29
118 ldr lr, [sp], #S_FRAME_SIZE - S_LR // load LR and restore SP
119 eret // return to kernel
120 .endm
122 .macro get_thread_info, rd
123 mov \rd, sp
124 and \rd, \rd, #~(THREAD_SIZE - 1) // top of stack
125 .endm
127 /*
128 * These are the registers used in the syscall handler, and allow us to
129 * have in theory up to 7 arguments to a function - x0 to x6.
130 *
131 * x7 is reserved for the system call number in 32-bit mode.
132 */
133 sc_nr .req x25 // number of system calls
134 scno .req x26 // syscall number
135 stbl .req x27 // syscall table pointer
136 tsk .req x28 // current thread_info
138 /*
139 * Interrupt handling.
140 */
141 .macro irq_handler
142 ldr x1, handle_arch_irq
143 mov x0, sp
144 blr x1
145 .endm
147 .text
149 /*
150 * Exception vectors.
151 */
153 .align 11
154 ENTRY(vectors)
155 ventry el1_sync_invalid // Synchronous EL1t
156 ventry el1_irq_invalid // IRQ EL1t
157 ventry el1_fiq_invalid // FIQ EL1t
158 ventry el1_error_invalid // Error EL1t
160 ventry el1_sync // Synchronous EL1h
161 ventry el1_irq // IRQ EL1h
162 ventry el1_fiq_invalid // FIQ EL1h
163 ventry el1_error_invalid // Error EL1h
165 ventry el0_sync // Synchronous 64-bit EL0
166 ventry el0_irq // IRQ 64-bit EL0
167 ventry el0_fiq_invalid // FIQ 64-bit EL0
168 ventry el0_error_invalid // Error 64-bit EL0
170 #ifdef CONFIG_COMPAT
171 ventry el0_sync_compat // Synchronous 32-bit EL0
172 ventry el0_irq_compat // IRQ 32-bit EL0
173 ventry el0_fiq_invalid_compat // FIQ 32-bit EL0
174 ventry el0_error_invalid_compat // Error 32-bit EL0
175 #else
176 ventry el0_sync_invalid // Synchronous 32-bit EL0
177 ventry el0_irq_invalid // IRQ 32-bit EL0
178 ventry el0_fiq_invalid // FIQ 32-bit EL0
179 ventry el0_error_invalid // Error 32-bit EL0
180 #endif
181 END(vectors)
183 /*
184 * Invalid mode handlers
185 */
186 .macro inv_entry, el, reason, regsize = 64
187 kernel_entry el, \regsize
188 mov x0, sp
189 mov x1, #\reason
190 mrs x2, esr_el1
191 b bad_mode
192 .endm
194 el0_sync_invalid:
195 inv_entry 0, BAD_SYNC
196 ENDPROC(el0_sync_invalid)
198 el0_irq_invalid:
199 inv_entry 0, BAD_IRQ
200 ENDPROC(el0_irq_invalid)
202 el0_fiq_invalid:
203 inv_entry 0, BAD_FIQ
204 ENDPROC(el0_fiq_invalid)
206 el0_error_invalid:
207 inv_entry 0, BAD_ERROR
208 ENDPROC(el0_error_invalid)
210 #ifdef CONFIG_COMPAT
211 el0_fiq_invalid_compat:
212 inv_entry 0, BAD_FIQ, 32
213 ENDPROC(el0_fiq_invalid_compat)
215 el0_error_invalid_compat:
216 inv_entry 0, BAD_ERROR, 32
217 ENDPROC(el0_error_invalid_compat)
218 #endif
220 el1_sync_invalid:
221 inv_entry 1, BAD_SYNC
222 ENDPROC(el1_sync_invalid)
224 el1_irq_invalid:
225 inv_entry 1, BAD_IRQ
226 ENDPROC(el1_irq_invalid)
228 el1_fiq_invalid:
229 inv_entry 1, BAD_FIQ
230 ENDPROC(el1_fiq_invalid)
232 el1_error_invalid:
233 inv_entry 1, BAD_ERROR
234 ENDPROC(el1_error_invalid)
236 /*
237 * EL1 mode handlers.
238 */
239 .align 6
240 el1_sync:
241 kernel_entry 1
242 mrs x1, esr_el1 // read the syndrome register
243 lsr x24, x1, #ESR_EL1_EC_SHIFT // exception class
244 cmp x24, #ESR_EL1_EC_DABT_EL1 // data abort in EL1
245 b.eq el1_da
246 cmp x24, #ESR_EL1_EC_SYS64 // configurable trap
247 b.eq el1_undef
248 cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception
249 b.eq el1_sp_pc
250 cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception
251 b.eq el1_sp_pc
252 cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL1
253 b.eq el1_undef
254 cmp x24, #ESR_EL1_EC_BREAKPT_EL1 // debug exception in EL1
255 b.ge el1_dbg
256 b el1_inv
257 el1_da:
258 /*
259 * Data abort handling
260 */
261 mrs x0, far_el1
262 enable_dbg_if_not_stepping x2
263 // re-enable interrupts if they were enabled in the aborted context
264 tbnz x23, #7, 1f // PSR_I_BIT
265 enable_irq
266 1:
267 mov x2, sp // struct pt_regs
268 bl do_mem_abort
270 // disable interrupts before pulling preserved data off the stack
271 disable_irq
272 kernel_exit 1
273 el1_sp_pc:
274 /*
275 * Stack or PC alignment exception handling
276 */
277 mrs x0, far_el1
278 mov x2, sp
279 b do_sp_pc_abort
280 el1_undef:
281 /*
282 * Undefined instruction
283 */
284 mov x0, sp
285 b do_undefinstr
286 el1_dbg:
287 /*
288 * Debug exception handling
289 */
290 tbz x24, #0, el1_inv // EL1 only
291 mrs x0, far_el1
292 mov x2, sp // struct pt_regs
293 bl do_debug_exception
295 kernel_exit 1
296 el1_inv:
297 // TODO: add support for undefined instructions in kernel mode
298 mov x0, sp
299 mov x1, #BAD_SYNC
300 mrs x2, esr_el1
301 b bad_mode
302 ENDPROC(el1_sync)
304 .align 6
305 el1_irq:
306 kernel_entry 1
307 enable_dbg_if_not_stepping x0
308 #ifdef CONFIG_TRACE_IRQFLAGS
309 bl trace_hardirqs_off
310 #endif
311 #ifdef CONFIG_PREEMPT
312 get_thread_info tsk
313 ldr x24, [tsk, #TI_PREEMPT] // get preempt count
314 add x0, x24, #1 // increment it
315 str x0, [tsk, #TI_PREEMPT]
316 #endif
317 irq_handler
318 #ifdef CONFIG_PREEMPT
319 str x24, [tsk, #TI_PREEMPT] // restore preempt count
320 cbnz x24, 1f // preempt count != 0
321 ldr x0, [tsk, #TI_FLAGS] // get flags
322 tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling?
323 bl el1_preempt
324 1:
325 #endif
326 #ifdef CONFIG_TRACE_IRQFLAGS
327 bl trace_hardirqs_on
328 #endif
329 kernel_exit 1
330 ENDPROC(el1_irq)
332 #ifdef CONFIG_PREEMPT
333 el1_preempt:
334 mov x24, lr
335 1: enable_dbg
336 bl preempt_schedule_irq // irq en/disable is done inside
337 ldr x0, [tsk, #TI_FLAGS] // get new tasks TI_FLAGS
338 tbnz x0, #TIF_NEED_RESCHED, 1b // needs rescheduling?
339 ret x24
340 #endif
342 /*
343 * EL0 mode handlers.
344 */
345 .align 6
346 el0_sync:
347 kernel_entry 0
348 mrs x25, esr_el1 // read the syndrome register
349 lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class
350 cmp x24, #ESR_EL1_EC_SVC64 // SVC in 64-bit state
351 b.eq el0_svc
352 adr lr, ret_from_exception
353 cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0
354 b.eq el0_da
355 cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0
356 b.eq el0_ia
357 cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access
358 b.eq el0_fpsimd_acc
359 cmp x24, #ESR_EL1_EC_FP_EXC64 // FP/ASIMD exception
360 b.eq el0_fpsimd_exc
361 cmp x24, #ESR_EL1_EC_SYS64 // configurable trap
362 b.eq el0_undef
363 cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception
364 b.eq el0_sp_pc
365 cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception
366 b.eq el0_sp_pc
367 cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0
368 b.eq el0_undef
369 cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0
370 b.ge el0_dbg
371 b el0_inv
373 #ifdef CONFIG_COMPAT
374 .align 6
375 el0_sync_compat:
376 kernel_entry 0, 32
377 mrs x25, esr_el1 // read the syndrome register
378 lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class
379 cmp x24, #ESR_EL1_EC_SVC32 // SVC in 32-bit state
380 b.eq el0_svc_compat
381 adr lr, ret_from_exception
382 cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0
383 b.eq el0_da
384 cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0
385 b.eq el0_ia
386 cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access
387 b.eq el0_fpsimd_acc
388 cmp x24, #ESR_EL1_EC_FP_EXC32 // FP/ASIMD exception
389 b.eq el0_fpsimd_exc
390 cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0
391 b.eq el0_undef
392 cmp x24, #ESR_EL1_EC_CP15_32 // CP15 MRC/MCR trap
393 b.eq el0_undef
394 cmp x24, #ESR_EL1_EC_CP15_64 // CP15 MRRC/MCRR trap
395 b.eq el0_undef
396 cmp x24, #ESR_EL1_EC_CP14_MR // CP14 MRC/MCR trap
397 b.eq el0_undef
398 cmp x24, #ESR_EL1_EC_CP14_LS // CP14 LDC/STC trap
399 b.eq el0_undef
400 cmp x24, #ESR_EL1_EC_CP14_64 // CP14 MRRC/MCRR trap
401 b.eq el0_undef
402 cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0
403 b.ge el0_dbg
404 b el0_inv
405 el0_svc_compat:
406 /*
407 * AArch32 syscall handling
408 */
409 adr stbl, compat_sys_call_table // load compat syscall table pointer
410 uxtw scno, w7 // syscall number in w7 (r7)
411 mov sc_nr, #__NR_compat_syscalls
412 b el0_svc_naked
414 .align 6
415 el0_irq_compat:
416 kernel_entry 0, 32
417 b el0_irq_naked
418 #endif
420 el0_da:
421 /*
422 * Data abort handling
423 */
424 mrs x0, far_el1
425 bic x0, x0, #(0xff << 56)
426 disable_step x1
427 isb
428 enable_dbg
429 // enable interrupts before calling the main handler
430 enable_irq
431 mov x1, x25
432 mov x2, sp
433 b do_mem_abort
434 el0_ia:
435 /*
436 * Instruction abort handling
437 */
438 mrs x0, far_el1
439 disable_step x1
440 isb
441 enable_dbg
442 // enable interrupts before calling the main handler
443 enable_irq
444 orr x1, x25, #1 << 24 // use reserved ISS bit for instruction aborts
445 mov x2, sp
446 b do_mem_abort
447 el0_fpsimd_acc:
448 /*
449 * Floating Point or Advanced SIMD access
450 */
451 mov x0, x25
452 mov x1, sp
453 b do_fpsimd_acc
454 el0_fpsimd_exc:
455 /*
456 * Floating Point or Advanced SIMD exception
457 */
458 mov x0, x25
459 mov x1, sp
460 b do_fpsimd_exc
461 el0_sp_pc:
462 /*
463 * Stack or PC alignment exception handling
464 */
465 mrs x0, far_el1
466 disable_step x1
467 isb
468 enable_dbg
469 // enable interrupts before calling the main handler
470 enable_irq
471 mov x1, x25
472 mov x2, sp
473 b do_sp_pc_abort
474 el0_undef:
475 /*
476 * Undefined instruction
477 */
478 mov x0, sp
479 // enable interrupts before calling the main handler
480 enable_irq
481 b do_undefinstr
482 el0_dbg:
483 /*
484 * Debug exception handling
485 */
486 tbnz x24, #0, el0_inv // EL0 only
487 mrs x0, far_el1
488 disable_step x1
489 mov x1, x25
490 mov x2, sp
491 b do_debug_exception
492 el0_inv:
493 mov x0, sp
494 mov x1, #BAD_SYNC
495 mrs x2, esr_el1
496 b bad_mode
497 ENDPROC(el0_sync)
499 .align 6
500 el0_irq:
501 kernel_entry 0
502 el0_irq_naked:
503 disable_step x1
504 isb
505 enable_dbg
506 #ifdef CONFIG_TRACE_IRQFLAGS
507 bl trace_hardirqs_off
508 #endif
509 get_thread_info tsk
510 #ifdef CONFIG_PREEMPT
511 ldr x24, [tsk, #TI_PREEMPT] // get preempt count
512 add x23, x24, #1 // increment it
513 str x23, [tsk, #TI_PREEMPT]
514 #endif
515 irq_handler
516 #ifdef CONFIG_PREEMPT
517 ldr x0, [tsk, #TI_PREEMPT]
518 str x24, [tsk, #TI_PREEMPT]
519 cmp x0, x23
520 b.eq 1f
521 mov x1, #0
522 str x1, [x1] // BUG
523 1:
524 #endif
525 #ifdef CONFIG_TRACE_IRQFLAGS
526 bl trace_hardirqs_on
527 #endif
528 b ret_to_user
529 ENDPROC(el0_irq)
531 /*
532 * This is the return code to user mode for abort handlers
533 */
534 ret_from_exception:
535 get_thread_info tsk
536 b ret_to_user
537 ENDPROC(ret_from_exception)
539 /*
540 * Register switch for AArch64. The callee-saved registers need to be saved
541 * and restored. On entry:
542 * x0 = previous task_struct (must be preserved across the switch)
543 * x1 = next task_struct
544 * Previous and next are guaranteed not to be the same.
545 *
546 */
547 ENTRY(cpu_switch_to)
548 add x8, x0, #THREAD_CPU_CONTEXT
549 mov x9, sp
550 stp x19, x20, [x8], #16 // store callee-saved registers
551 stp x21, x22, [x8], #16
552 stp x23, x24, [x8], #16
553 stp x25, x26, [x8], #16
554 stp x27, x28, [x8], #16
555 stp x29, x9, [x8], #16
556 str lr, [x8]
557 add x8, x1, #THREAD_CPU_CONTEXT
558 ldp x19, x20, [x8], #16 // restore callee-saved registers
559 ldp x21, x22, [x8], #16
560 ldp x23, x24, [x8], #16
561 ldp x25, x26, [x8], #16
562 ldp x27, x28, [x8], #16
563 ldp x29, x9, [x8], #16
564 ldr lr, [x8]
565 mov sp, x9
566 ret
567 ENDPROC(cpu_switch_to)
569 /*
570 * This is the fast syscall return path. We do as little as possible here,
571 * and this includes saving x0 back into the kernel stack.
572 */
573 ret_fast_syscall:
574 disable_irq // disable interrupts
575 ldr x1, [tsk, #TI_FLAGS]
576 and x2, x1, #_TIF_WORK_MASK
577 cbnz x2, fast_work_pending
578 tbz x1, #TIF_SINGLESTEP, fast_exit
579 disable_dbg
580 enable_step x2
581 fast_exit:
582 kernel_exit 0, ret = 1
584 /*
585 * Ok, we need to do extra processing, enter the slow path.
586 */
587 fast_work_pending:
588 str x0, [sp, #S_X0] // returned x0
589 work_pending:
590 tbnz x1, #TIF_NEED_RESCHED, work_resched
591 /* TIF_SIGPENDING or TIF_NOTIFY_RESUME case */
592 ldr x2, [sp, #S_PSTATE]
593 mov x0, sp // 'regs'
594 tst x2, #PSR_MODE_MASK // user mode regs?
595 b.ne no_work_pending // returning to kernel
596 enable_irq // enable interrupts for do_notify_resume()
597 bl do_notify_resume
598 b ret_to_user
599 work_resched:
600 enable_dbg
601 bl schedule
603 /*
604 * "slow" syscall return path.
605 */
606 ret_to_user:
607 disable_irq // disable interrupts
608 ldr x1, [tsk, #TI_FLAGS]
609 and x2, x1, #_TIF_WORK_MASK
610 cbnz x2, work_pending
611 tbz x1, #TIF_SINGLESTEP, no_work_pending
612 disable_dbg
613 enable_step x2
614 no_work_pending:
615 kernel_exit 0, ret = 0
616 ENDPROC(ret_to_user)
618 /*
619 * This is how we return from a fork.
620 */
621 ENTRY(ret_from_fork)
622 bl schedule_tail
623 cbz x19, 1f // not a kernel thread
624 mov x0, x20
625 blr x19
626 1: get_thread_info tsk
627 b ret_to_user
628 ENDPROC(ret_from_fork)
630 /*
631 * SVC handler.
632 */
633 .align 6
634 el0_svc:
635 adrp stbl, sys_call_table // load syscall table pointer
636 uxtw scno, w8 // syscall number in w8
637 mov sc_nr, #__NR_syscalls
638 el0_svc_naked: // compat entry point
639 stp x0, scno, [sp, #S_ORIG_X0] // save the original x0 and syscall number
640 disable_step x16
641 isb
642 enable_dbg
643 enable_irq
645 get_thread_info tsk
646 ldr x16, [tsk, #TI_FLAGS] // check for syscall tracing
647 tbnz x16, #TIF_SYSCALL_TRACE, __sys_trace // are we tracing syscalls?
648 adr lr, ret_fast_syscall // return address
649 cmp scno, sc_nr // check upper syscall limit
650 b.hs ni_sys
651 ldr x16, [stbl, scno, lsl #3] // address in the syscall table
652 br x16 // call sys_* routine
653 ni_sys:
654 mov x0, sp
655 b do_ni_syscall
656 ENDPROC(el0_svc)
658 /*
659 * This is the really slow path. We're going to be doing context
660 * switches, and waiting for our parent to respond.
661 */
662 __sys_trace:
663 mov x1, sp
664 mov w0, #0 // trace entry
665 bl syscall_trace
666 adr lr, __sys_trace_return // return address
667 uxtw scno, w0 // syscall number (possibly new)
668 mov x1, sp // pointer to regs
669 cmp scno, sc_nr // check upper syscall limit
670 b.hs ni_sys
671 ldp x0, x1, [sp] // restore the syscall args
672 ldp x2, x3, [sp, #S_X2]
673 ldp x4, x5, [sp, #S_X4]
674 ldp x6, x7, [sp, #S_X6]
675 ldr x16, [stbl, scno, lsl #3] // address in the syscall table
676 br x16 // call sys_* routine
678 __sys_trace_return:
679 str x0, [sp] // save returned x0
680 mov x1, sp
681 mov w0, #1 // trace exit
682 bl syscall_trace
683 b ret_to_user
685 /*
686 * Special system call wrappers.
687 */
688 ENTRY(sys_rt_sigreturn_wrapper)
689 mov x0, sp
690 b sys_rt_sigreturn
691 ENDPROC(sys_rt_sigreturn_wrapper)
693 ENTRY(handle_arch_irq)
694 .quad 0