| blob | 39ac630d83de77fcba77c61ca2e6817c2c8239b8 |
1 /*
2 * Low-level exception handling code
3 *
4 * Copyright (C) 2012 ARM Ltd.
5 * Authors: Catalin Marinas <catalin.marinas@arm.com>
6 * Will Deacon <will.deacon@arm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
21 #include <linux/init.h>
22 #include <linux/linkage.h>
24 #include <asm/assembler.h>
25 #include <asm/asm-offsets.h>
26 #include <asm/errno.h>
27 #include <asm/esr.h>
28 #include <asm/thread_info.h>
29 #include <asm/unistd.h>
30 #include <asm/unistd32.h>
32 /*
33 * Bad Abort numbers
34 *-----------------
35 */
36 #define BAD_SYNC 0
37 #define BAD_IRQ 1
38 #define BAD_FIQ 2
39 #define BAD_ERROR 3
41 .macro kernel_entry, el, regsize = 64
42 sub sp, sp, #S_FRAME_SIZE - S_LR // room for LR, SP, SPSR, ELR
43 .if \regsize == 32
44 mov w0, w0 // zero upper 32 bits of x0
45 .endif
46 push x28, x29
47 push x26, x27
48 push x24, x25
49 push x22, x23
50 push x20, x21
51 push x18, x19
52 push x16, x17
53 push x14, x15
54 push x12, x13
55 push x10, x11
56 push x8, x9
57 push x6, x7
58 push x4, x5
59 push x2, x3
60 push x0, x1
61 .if \el == 0
62 mrs x21, sp_el0
63 .else
64 add x21, sp, #S_FRAME_SIZE
65 .endif
66 mrs x22, elr_el1
67 mrs x23, spsr_el1
68 stp lr, x21, [sp, #S_LR]
69 stp x22, x23, [sp, #S_PC]
71 /*
72 * Set syscallno to -1 by default (overridden later if real syscall).
73 */
74 .if \el == 0
75 mvn x21, xzr
76 str x21, [sp, #S_SYSCALLNO]
77 .endif
79 /*
80 * Registers that may be useful after this macro is invoked:
81 *
82 * x21 - aborted SP
83 * x22 - aborted PC
84 * x23 - aborted PSTATE
85 */
86 .endm
88 .macro kernel_exit, el, ret = 0
89 ldp x21, x22, [sp, #S_PC] // load ELR, SPSR
90 .if \el == 0
91 ldr x23, [sp, #S_SP] // load return stack pointer
92 .endif
93 .if \ret
94 ldr x1, [sp, #S_X1] // preserve x0 (syscall return)
95 add sp, sp, S_X2
96 .else
97 pop x0, x1
98 .endif
99 pop x2, x3 // load the rest of the registers
100 pop x4, x5
101 pop x6, x7
102 pop x8, x9
103 msr elr_el1, x21 // set up the return data
104 msr spsr_el1, x22
105 .if \el == 0
106 msr sp_el0, x23
107 .endif
108 pop x10, x11
109 pop x12, x13
110 pop x14, x15
111 pop x16, x17
112 pop x18, x19
113 pop x20, x21
114 pop x22, x23
115 pop x24, x25
116 pop x26, x27
117 pop x28, x29
118 ldr lr, [sp], #S_FRAME_SIZE - S_LR // load LR and restore SP
119 eret // return to kernel
120 .endm
122 .macro get_thread_info, rd
123 mov \rd, sp
124 and \rd, \rd, #~(THREAD_SIZE - 1) // top of stack
125 .endm
127 /*
128 * These are the registers used in the syscall handler, and allow us to
129 * have in theory up to 7 arguments to a function - x0 to x6.
130 *
131 * x7 is reserved for the system call number in 32-bit mode.
132 */
133 sc_nr .req x25 // number of system calls
134 scno .req x26 // syscall number
135 stbl .req x27 // syscall table pointer
136 tsk .req x28 // current thread_info
138 /*
139 * Interrupt handling.
140 */
141 .macro irq_handler
142 ldr x1, handle_arch_irq
143 mov x0, sp
144 blr x1
145 .endm
147 .text
149 /*
150 * Exception vectors.
151 */
153 .align 11
154 ENTRY(vectors)
155 ventry el1_sync_invalid // Synchronous EL1t
156 ventry el1_irq_invalid // IRQ EL1t
157 ventry el1_fiq_invalid // FIQ EL1t
158 ventry el1_error_invalid // Error EL1t
160 ventry el1_sync // Synchronous EL1h
161 ventry el1_irq // IRQ EL1h
162 ventry el1_fiq_invalid // FIQ EL1h
163 ventry el1_error_invalid // Error EL1h
165 ventry el0_sync // Synchronous 64-bit EL0
166 ventry el0_irq // IRQ 64-bit EL0
167 ventry el0_fiq_invalid // FIQ 64-bit EL0
168 ventry el0_error_invalid // Error 64-bit EL0
170 #ifdef CONFIG_COMPAT
171 ventry el0_sync_compat // Synchronous 32-bit EL0
172 ventry el0_irq_compat // IRQ 32-bit EL0
173 ventry el0_fiq_invalid_compat // FIQ 32-bit EL0
174 ventry el0_error_invalid_compat // Error 32-bit EL0
175 #else
176 ventry el0_sync_invalid // Synchronous 32-bit EL0
177 ventry el0_irq_invalid // IRQ 32-bit EL0
178 ventry el0_fiq_invalid // FIQ 32-bit EL0
179 ventry el0_error_invalid // Error 32-bit EL0
180 #endif
181 END(vectors)
183 /*
184 * Invalid mode handlers
185 */
186 .macro inv_entry, el, reason, regsize = 64
187 kernel_entry el, \regsize
188 mov x0, sp
189 mov x1, #\reason
190 mrs x2, esr_el1
191 b bad_mode
192 .endm
194 el0_sync_invalid:
195 inv_entry 0, BAD_SYNC
196 ENDPROC(el0_sync_invalid)
198 el0_irq_invalid:
199 inv_entry 0, BAD_IRQ
200 ENDPROC(el0_irq_invalid)
202 el0_fiq_invalid:
203 inv_entry 0, BAD_FIQ
204 ENDPROC(el0_fiq_invalid)
206 el0_error_invalid:
207 inv_entry 0, BAD_ERROR
208 ENDPROC(el0_error_invalid)
210 #ifdef CONFIG_COMPAT
211 el0_fiq_invalid_compat:
212 inv_entry 0, BAD_FIQ, 32
213 ENDPROC(el0_fiq_invalid_compat)
215 el0_error_invalid_compat:
216 inv_entry 0, BAD_ERROR, 32
217 ENDPROC(el0_error_invalid_compat)
218 #endif
220 el1_sync_invalid:
221 inv_entry 1, BAD_SYNC
222 ENDPROC(el1_sync_invalid)
224 el1_irq_invalid:
225 inv_entry 1, BAD_IRQ
226 ENDPROC(el1_irq_invalid)
228 el1_fiq_invalid:
229 inv_entry 1, BAD_FIQ
230 ENDPROC(el1_fiq_invalid)
232 el1_error_invalid:
233 inv_entry 1, BAD_ERROR
234 ENDPROC(el1_error_invalid)
236 /*
237 * EL1 mode handlers.
238 */
239 .align 6
240 el1_sync:
241 kernel_entry 1
242 mrs x1, esr_el1 // read the syndrome register
243 lsr x24, x1, #ESR_EL1_EC_SHIFT // exception class
244 cmp x24, #ESR_EL1_EC_DABT_EL1 // data abort in EL1
245 b.eq el1_da
246 cmp x24, #ESR_EL1_EC_SYS64 // configurable trap
247 b.eq el1_undef
248 cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception
249 b.eq el1_sp_pc
250 cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception
251 b.eq el1_sp_pc
252 cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL1
253 b.eq el1_undef
254 cmp x24, #ESR_EL1_EC_BREAKPT_EL1 // debug exception in EL1
255 b.ge el1_dbg
256 b el1_inv
257 el1_da:
258 /*
259 * Data abort handling
260 */
261 mrs x0, far_el1
262 enable_dbg_if_not_stepping x2
263 // re-enable interrupts if they were enabled in the aborted context
264 tbnz x23, #7, 1f // PSR_I_BIT
265 enable_irq
266 1:
267 mov x2, sp // struct pt_regs
268 bl do_mem_abort
270 // disable interrupts before pulling preserved data off the stack
271 disable_irq
272 kernel_exit 1
273 el1_sp_pc:
274 /*
275 * Stack or PC alignment exception handling
276 */
277 mrs x0, far_el1
278 mov x1, x25
279 mov x2, sp
280 b do_sp_pc_abort
281 el1_undef:
282 /*
283 * Undefined instruction
284 */
285 mov x0, sp
286 b do_undefinstr
287 el1_dbg:
288 /*
289 * Debug exception handling
290 */
291 cmp x24, #ESR_EL1_EC_BRK64 // if BRK64
292 cinc x24, x24, eq // set bit '0'
293 tbz x24, #0, el1_inv // EL1 only
294 mrs x0, far_el1
295 mov x2, sp // struct pt_regs
296 bl do_debug_exception
298 kernel_exit 1
299 el1_inv:
300 // TODO: add support for undefined instructions in kernel mode
301 mov x0, sp
302 mov x1, #BAD_SYNC
303 mrs x2, esr_el1
304 b bad_mode
305 ENDPROC(el1_sync)
307 .align 6
308 el1_irq:
309 kernel_entry 1
310 enable_dbg_if_not_stepping x0
311 #ifdef CONFIG_TRACE_IRQFLAGS
312 bl trace_hardirqs_off
313 #endif
315 irq_handler
317 #ifdef CONFIG_PREEMPT
318 get_thread_info tsk
319 ldr w24, [tsk, #TI_PREEMPT] // get preempt count
320 cbnz w24, 1f // preempt count != 0
321 ldr x0, [tsk, #TI_FLAGS] // get flags
322 tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling?
323 bl el1_preempt
324 1:
325 #endif
326 #ifdef CONFIG_TRACE_IRQFLAGS
327 bl trace_hardirqs_on
328 #endif
329 kernel_exit 1
330 ENDPROC(el1_irq)
332 #ifdef CONFIG_PREEMPT
333 el1_preempt:
334 mov x24, lr
335 1: enable_dbg
336 bl preempt_schedule_irq // irq en/disable is done inside
337 ldr x0, [tsk, #TI_FLAGS] // get new tasks TI_FLAGS
338 tbnz x0, #TIF_NEED_RESCHED, 1b // needs rescheduling?
339 ret x24
340 #endif
342 /*
343 * EL0 mode handlers.
344 */
345 .align 6
346 el0_sync:
347 kernel_entry 0
348 mrs x25, esr_el1 // read the syndrome register
349 lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class
350 cmp x24, #ESR_EL1_EC_SVC64 // SVC in 64-bit state
351 b.eq el0_svc
352 adr lr, ret_from_exception
353 cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0
354 b.eq el0_da
355 cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0
356 b.eq el0_ia
357 cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access
358 b.eq el0_fpsimd_acc
359 cmp x24, #ESR_EL1_EC_FP_EXC64 // FP/ASIMD exception
360 b.eq el0_fpsimd_exc
361 cmp x24, #ESR_EL1_EC_SYS64 // configurable trap
362 b.eq el0_undef
363 cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception
364 b.eq el0_sp_pc
365 cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception
366 b.eq el0_sp_pc
367 cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0
368 b.eq el0_undef
369 cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0
370 b.ge el0_dbg
371 b el0_inv
373 #ifdef CONFIG_COMPAT
374 .align 6
375 el0_sync_compat:
376 kernel_entry 0, 32
377 mrs x25, esr_el1 // read the syndrome register
378 lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class
379 cmp x24, #ESR_EL1_EC_SVC32 // SVC in 32-bit state
380 b.eq el0_svc_compat
381 adr lr, ret_from_exception
382 cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0
383 b.eq el0_da
384 cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0
385 b.eq el0_ia
386 cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access
387 b.eq el0_fpsimd_acc
388 cmp x24, #ESR_EL1_EC_FP_EXC32 // FP/ASIMD exception
389 b.eq el0_fpsimd_exc
390 cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0
391 b.eq el0_undef
392 cmp x24, #ESR_EL1_EC_CP15_32 // CP15 MRC/MCR trap
393 b.eq el0_undef
394 cmp x24, #ESR_EL1_EC_CP15_64 // CP15 MRRC/MCRR trap
395 b.eq el0_undef
396 cmp x24, #ESR_EL1_EC_CP14_MR // CP14 MRC/MCR trap
397 b.eq el0_undef
398 cmp x24, #ESR_EL1_EC_CP14_LS // CP14 LDC/STC trap
399 b.eq el0_undef
400 cmp x24, #ESR_EL1_EC_CP14_64 // CP14 MRRC/MCRR trap
401 b.eq el0_undef
402 cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0
403 b.ge el0_dbg
404 b el0_inv
405 el0_svc_compat:
406 /*
407 * AArch32 syscall handling
408 */
409 adr stbl, compat_sys_call_table // load compat syscall table pointer
410 uxtw scno, w7 // syscall number in w7 (r7)
411 mov sc_nr, #__NR_compat_syscalls
412 b el0_svc_naked
414 .align 6
415 el0_irq_compat:
416 kernel_entry 0, 32
417 b el0_irq_naked
418 #endif
420 el0_da:
421 /*
422 * Data abort handling
423 */
424 mrs x0, far_el1
425 bic x0, x0, #(0xff << 56)
426 disable_step x1
427 isb
428 enable_dbg
429 // enable interrupts before calling the main handler
430 enable_irq
431 mov x1, x25
432 mov x2, sp
433 b do_mem_abort
434 el0_ia:
435 /*
436 * Instruction abort handling
437 */
438 mrs x0, far_el1
439 disable_step x1
440 isb
441 enable_dbg
442 // enable interrupts before calling the main handler
443 enable_irq
444 orr x1, x25, #1 << 24 // use reserved ISS bit for instruction aborts
445 mov x2, sp
446 b do_mem_abort
447 el0_fpsimd_acc:
448 /*
449 * Floating Point or Advanced SIMD access
450 */
451 mov x0, x25
452 mov x1, sp
453 b do_fpsimd_acc
454 el0_fpsimd_exc:
455 /*
456 * Floating Point or Advanced SIMD exception
457 */
458 mov x0, x25
459 mov x1, sp
460 b do_fpsimd_exc
461 el0_sp_pc:
462 /*
463 * Stack or PC alignment exception handling
464 */
465 mrs x0, far_el1
466 disable_step x1
467 isb
468 enable_dbg
469 // enable interrupts before calling the main handler
470 enable_irq
471 mov x1, x25
472 mov x2, sp
473 b do_sp_pc_abort
474 el0_undef:
475 /*
476 * Undefined instruction
477 */
478 mov x0, sp
479 // enable interrupts before calling the main handler
480 enable_irq
481 b do_undefinstr
482 el0_dbg:
483 /*
484 * Debug exception handling
485 */
486 tbnz x24, #0, el0_inv // EL0 only
487 mrs x0, far_el1
488 disable_step x1
489 mov x1, x25
490 mov x2, sp
491 b do_debug_exception
492 el0_inv:
493 mov x0, sp
494 mov x1, #BAD_SYNC
495 mrs x2, esr_el1
496 b bad_mode
497 ENDPROC(el0_sync)
499 .align 6
500 el0_irq:
501 kernel_entry 0
502 el0_irq_naked:
503 disable_step x1
504 isb
505 enable_dbg
506 #ifdef CONFIG_TRACE_IRQFLAGS
507 bl trace_hardirqs_off
508 #endif
510 irq_handler
511 get_thread_info tsk
513 #ifdef CONFIG_TRACE_IRQFLAGS
514 bl trace_hardirqs_on
515 #endif
516 b ret_to_user
517 ENDPROC(el0_irq)
519 /*
520 * This is the return code to user mode for abort handlers
521 */
522 ret_from_exception:
523 get_thread_info tsk
524 b ret_to_user
525 ENDPROC(ret_from_exception)
527 /*
528 * Register switch for AArch64. The callee-saved registers need to be saved
529 * and restored. On entry:
530 * x0 = previous task_struct (must be preserved across the switch)
531 * x1 = next task_struct
532 * Previous and next are guaranteed not to be the same.
533 *
534 */
535 ENTRY(cpu_switch_to)
536 add x8, x0, #THREAD_CPU_CONTEXT
537 mov x9, sp
538 stp x19, x20, [x8], #16 // store callee-saved registers
539 stp x21, x22, [x8], #16
540 stp x23, x24, [x8], #16
541 stp x25, x26, [x8], #16
542 stp x27, x28, [x8], #16
543 stp x29, x9, [x8], #16
544 str lr, [x8]
545 add x8, x1, #THREAD_CPU_CONTEXT
546 ldp x19, x20, [x8], #16 // restore callee-saved registers
547 ldp x21, x22, [x8], #16
548 ldp x23, x24, [x8], #16
549 ldp x25, x26, [x8], #16
550 ldp x27, x28, [x8], #16
551 ldp x29, x9, [x8], #16
552 ldr lr, [x8]
553 mov sp, x9
554 ret
555 ENDPROC(cpu_switch_to)
557 /*
558 * This is the fast syscall return path. We do as little as possible here,
559 * and this includes saving x0 back into the kernel stack.
560 */
561 ret_fast_syscall:
562 disable_irq // disable interrupts
563 ldr x1, [tsk, #TI_FLAGS]
564 and x2, x1, #_TIF_WORK_MASK
565 cbnz x2, fast_work_pending
566 tbz x1, #TIF_SINGLESTEP, fast_exit
567 disable_dbg
568 enable_step x2
569 fast_exit:
570 kernel_exit 0, ret = 1
572 /*
573 * Ok, we need to do extra processing, enter the slow path.
574 */
575 fast_work_pending:
576 str x0, [sp, #S_X0] // returned x0
577 work_pending:
578 tbnz x1, #TIF_NEED_RESCHED, work_resched
579 /* TIF_SIGPENDING or TIF_NOTIFY_RESUME case */
580 ldr x2, [sp, #S_PSTATE]
581 mov x0, sp // 'regs'
582 tst x2, #PSR_MODE_MASK // user mode regs?
583 b.ne no_work_pending // returning to kernel
584 enable_irq // enable interrupts for do_notify_resume()
585 bl do_notify_resume
586 b ret_to_user
587 work_resched:
588 enable_dbg
589 bl schedule
591 /*
592 * "slow" syscall return path.
593 */
594 ret_to_user:
595 disable_irq // disable interrupts
596 ldr x1, [tsk, #TI_FLAGS]
597 and x2, x1, #_TIF_WORK_MASK
598 cbnz x2, work_pending
599 tbz x1, #TIF_SINGLESTEP, no_work_pending
600 disable_dbg
601 enable_step x2
602 no_work_pending:
603 kernel_exit 0, ret = 0
604 ENDPROC(ret_to_user)
606 /*
607 * This is how we return from a fork.
608 */
609 ENTRY(ret_from_fork)
610 bl schedule_tail
611 cbz x19, 1f // not a kernel thread
612 mov x0, x20
613 blr x19
614 1: get_thread_info tsk
615 b ret_to_user
616 ENDPROC(ret_from_fork)
618 /*
619 * SVC handler.
620 */
621 .align 6
622 el0_svc:
623 adrp stbl, sys_call_table // load syscall table pointer
624 uxtw scno, w8 // syscall number in w8
625 mov sc_nr, #__NR_syscalls
626 el0_svc_naked: // compat entry point
627 stp x0, scno, [sp, #S_ORIG_X0] // save the original x0 and syscall number
628 disable_step x16
629 isb
630 enable_dbg
631 enable_irq
633 get_thread_info tsk
634 ldr x16, [tsk, #TI_FLAGS] // check for syscall tracing
635 tbnz x16, #TIF_SYSCALL_TRACE, __sys_trace // are we tracing syscalls?
636 adr lr, ret_fast_syscall // return address
637 cmp scno, sc_nr // check upper syscall limit
638 b.hs ni_sys
639 ldr x16, [stbl, scno, lsl #3] // address in the syscall table
640 br x16 // call sys_* routine
641 ni_sys:
642 mov x0, sp
643 b do_ni_syscall
644 ENDPROC(el0_svc)
646 /*
647 * This is the really slow path. We're going to be doing context
648 * switches, and waiting for our parent to respond.
649 */
650 __sys_trace:
651 mov x1, sp
652 mov w0, #0 // trace entry
653 bl syscall_trace
654 adr lr, __sys_trace_return // return address
655 uxtw scno, w0 // syscall number (possibly new)
656 mov x1, sp // pointer to regs
657 cmp scno, sc_nr // check upper syscall limit
658 b.hs ni_sys
659 ldp x0, x1, [sp] // restore the syscall args
660 ldp x2, x3, [sp, #S_X2]
661 ldp x4, x5, [sp, #S_X4]
662 ldp x6, x7, [sp, #S_X6]
663 ldr x16, [stbl, scno, lsl #3] // address in the syscall table
664 br x16 // call sys_* routine
666 __sys_trace_return:
667 str x0, [sp] // save returned x0
668 mov x1, sp
669 mov w0, #1 // trace exit
670 bl syscall_trace
671 b ret_to_user
673 /*
674 * Special system call wrappers.
675 */
676 ENTRY(sys_rt_sigreturn_wrapper)
677 mov x0, sp
678 b sys_rt_sigreturn
679 ENDPROC(sys_rt_sigreturn_wrapper)
681 ENTRY(handle_arch_irq)
682 .quad 0