| blob | ddcde093c433b83e0c86d28d7a5d5f81367d6c86 |
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Low-level exception handling code
4 *
5 * Copyright (C) 2012 ARM Ltd.
6 * Authors: Catalin Marinas <catalin.marinas@arm.com>
7 * Will Deacon <will.deacon@arm.com>
8 */
10 #include <linux/arm-smccc.h>
11 #include <linux/init.h>
12 #include <linux/linkage.h>
14 #include <asm/alternative.h>
15 #include <asm/assembler.h>
16 #include <asm/asm-offsets.h>
17 #include <asm/asm_pointer_auth.h>
18 #include <asm/cpufeature.h>
19 #include <asm/errno.h>
20 #include <asm/esr.h>
21 #include <asm/irq.h>
22 #include <asm/memory.h>
23 #include <asm/mmu.h>
24 #include <asm/processor.h>
25 #include <asm/ptrace.h>
26 #include <asm/thread_info.h>
27 #include <asm/asm-uaccess.h>
28 #include <asm/unistd.h>
30 /*
31 * Context tracking subsystem. Used to instrument transitions
32 * between user and kernel mode.
33 */
34 .macro ct_user_exit_irqoff
35 #ifdef CONFIG_CONTEXT_TRACKING
36 bl enter_from_user_mode
37 #endif
38 .endm
40 .macro ct_user_enter
41 #ifdef CONFIG_CONTEXT_TRACKING
42 bl context_tracking_user_enter
43 #endif
44 .endm
46 .macro clear_gp_regs
47 .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29
48 mov x\n, xzr
49 .endr
50 .endm
52 /*
53 * Bad Abort numbers
54 *-----------------
55 */
56 #define BAD_SYNC 0
57 #define BAD_IRQ 1
58 #define BAD_FIQ 2
59 #define BAD_ERROR 3
61 .macro kernel_ventry, el, label, regsize = 64
62 .align 7
63 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
64 .if \el == 0
65 alternative_if ARM64_UNMAP_KERNEL_AT_EL0
66 .if \regsize == 64
67 mrs x30, tpidrro_el0
68 msr tpidrro_el0, xzr
69 .else
70 mov x30, xzr
71 .endif
72 alternative_else_nop_endif
73 .endif
74 #endif
76 sub sp, sp, #S_FRAME_SIZE
77 #ifdef CONFIG_VMAP_STACK
78 /*
79 * Test whether the SP has overflowed, without corrupting a GPR.
80 * Task and IRQ stacks are aligned so that SP & (1 << THREAD_SHIFT)
81 * should always be zero.
82 */
83 add sp, sp, x0 // sp' = sp + x0
84 sub x0, sp, x0 // x0' = sp' - x0 = (sp + x0) - x0 = sp
85 tbnz x0, #THREAD_SHIFT, 0f
86 sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0
87 sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp
88 b el\()\el\()_\label
90 0:
91 /*
92 * Either we've just detected an overflow, or we've taken an exception
93 * while on the overflow stack. Either way, we won't return to
94 * userspace, and can clobber EL0 registers to free up GPRs.
95 */
97 /* Stash the original SP (minus S_FRAME_SIZE) in tpidr_el0. */
98 msr tpidr_el0, x0
100 /* Recover the original x0 value and stash it in tpidrro_el0 */
101 sub x0, sp, x0
102 msr tpidrro_el0, x0
104 /* Switch to the overflow stack */
105 adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0
107 /*
108 * Check whether we were already on the overflow stack. This may happen
109 * after panic() re-enables interrupts.
110 */
111 mrs x0, tpidr_el0 // sp of interrupted context
112 sub x0, sp, x0 // delta with top of overflow stack
113 tst x0, #~(OVERFLOW_STACK_SIZE - 1) // within range?
114 b.ne __bad_stack // no? -> bad stack pointer
116 /* We were already on the overflow stack. Restore sp/x0 and carry on. */
117 sub sp, sp, x0
118 mrs x0, tpidrro_el0
119 #endif
120 b el\()\el\()_\label
121 .endm
123 .macro tramp_alias, dst, sym
124 mov_q \dst, TRAMP_VALIAS
125 add \dst, \dst, #(\sym - .entry.tramp.text)
126 .endm
128 // This macro corrupts x0-x3. It is the caller's duty
129 // to save/restore them if required.
130 .macro apply_ssbd, state, tmp1, tmp2
131 #ifdef CONFIG_ARM64_SSBD
132 alternative_cb arm64_enable_wa2_handling
133 b .L__asm_ssbd_skip\@
134 alternative_cb_end
135 ldr_this_cpu \tmp2, arm64_ssbd_callback_required, \tmp1
136 cbz \tmp2, .L__asm_ssbd_skip\@
137 ldr \tmp2, [tsk, #TSK_TI_FLAGS]
138 tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@
139 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2
140 mov w1, #\state
141 alternative_cb arm64_update_smccc_conduit
142 nop // Patched to SMC/HVC #0
143 alternative_cb_end
144 .L__asm_ssbd_skip\@:
145 #endif
146 .endm
148 .macro kernel_entry, el, regsize = 64
149 .if \regsize == 32
150 mov w0, w0 // zero upper 32 bits of x0
151 .endif
152 stp x0, x1, [sp, #16 * 0]
153 stp x2, x3, [sp, #16 * 1]
154 stp x4, x5, [sp, #16 * 2]
155 stp x6, x7, [sp, #16 * 3]
156 stp x8, x9, [sp, #16 * 4]
157 stp x10, x11, [sp, #16 * 5]
158 stp x12, x13, [sp, #16 * 6]
159 stp x14, x15, [sp, #16 * 7]
160 stp x16, x17, [sp, #16 * 8]
161 stp x18, x19, [sp, #16 * 9]
162 stp x20, x21, [sp, #16 * 10]
163 stp x22, x23, [sp, #16 * 11]
164 stp x24, x25, [sp, #16 * 12]
165 stp x26, x27, [sp, #16 * 13]
166 stp x28, x29, [sp, #16 * 14]
168 .if \el == 0
169 clear_gp_regs
170 mrs x21, sp_el0
171 ldr_this_cpu tsk, __entry_task, x20
172 msr sp_el0, tsk
174 // Ensure MDSCR_EL1.SS is clear, since we can unmask debug exceptions
175 // when scheduling.
176 ldr x19, [tsk, #TSK_TI_FLAGS]
177 disable_step_tsk x19, x20
179 apply_ssbd 1, x22, x23
181 ptrauth_keys_install_kernel tsk, 1, x20, x22, x23
182 .else
183 add x21, sp, #S_FRAME_SIZE
184 get_current_task tsk
185 /* Save the task's original addr_limit and set USER_DS */
186 ldr x20, [tsk, #TSK_TI_ADDR_LIMIT]
187 str x20, [sp, #S_ORIG_ADDR_LIMIT]
188 mov x20, #USER_DS
189 str x20, [tsk, #TSK_TI_ADDR_LIMIT]
190 /* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */
191 .endif /* \el == 0 */
192 mrs x22, elr_el1
193 mrs x23, spsr_el1
194 stp lr, x21, [sp, #S_LR]
196 /*
197 * In order to be able to dump the contents of struct pt_regs at the
198 * time the exception was taken (in case we attempt to walk the call
199 * stack later), chain it together with the stack frames.
200 */
201 .if \el == 0
202 stp xzr, xzr, [sp, #S_STACKFRAME]
203 .else
204 stp x29, x22, [sp, #S_STACKFRAME]
205 .endif
206 add x29, sp, #S_STACKFRAME
208 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
209 /*
210 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from
211 * EL0, there is no need to check the state of TTBR0_EL1 since
212 * accesses are always enabled.
213 * Note that the meaning of this bit differs from the ARMv8.1 PAN
214 * feature as all TTBR0_EL1 accesses are disabled, not just those to
215 * user mappings.
216 */
217 alternative_if ARM64_HAS_PAN
218 b 1f // skip TTBR0 PAN
219 alternative_else_nop_endif
221 .if \el != 0
222 mrs x21, ttbr0_el1
223 tst x21, #TTBR_ASID_MASK // Check for the reserved ASID
224 orr x23, x23, #PSR_PAN_BIT // Set the emulated PAN in the saved SPSR
225 b.eq 1f // TTBR0 access already disabled
226 and x23, x23, #~PSR_PAN_BIT // Clear the emulated PAN in the saved SPSR
227 .endif
229 __uaccess_ttbr0_disable x21
230 1:
231 #endif
233 stp x22, x23, [sp, #S_PC]
235 /* Not in a syscall by default (el0_svc overwrites for real syscall) */
236 .if \el == 0
237 mov w21, #NO_SYSCALL
238 str w21, [sp, #S_SYSCALLNO]
239 .endif
241 /* Save pmr */
242 alternative_if ARM64_HAS_IRQ_PRIO_MASKING
243 mrs_s x20, SYS_ICC_PMR_EL1
244 str x20, [sp, #S_PMR_SAVE]
245 alternative_else_nop_endif
247 /*
248 * Registers that may be useful after this macro is invoked:
249 *
250 * x20 - ICC_PMR_EL1
251 * x21 - aborted SP
252 * x22 - aborted PC
253 * x23 - aborted PSTATE
254 */
255 .endm
257 .macro kernel_exit, el
258 .if \el != 0
259 disable_daif
261 /* Restore the task's original addr_limit. */
262 ldr x20, [sp, #S_ORIG_ADDR_LIMIT]
263 str x20, [tsk, #TSK_TI_ADDR_LIMIT]
265 /* No need to restore UAO, it will be restored from SPSR_EL1 */
266 .endif
268 /* Restore pmr */
269 alternative_if ARM64_HAS_IRQ_PRIO_MASKING
270 ldr x20, [sp, #S_PMR_SAVE]
271 msr_s SYS_ICC_PMR_EL1, x20
272 mrs_s x21, SYS_ICC_CTLR_EL1
273 tbz x21, #6, .L__skip_pmr_sync\@ // Check for ICC_CTLR_EL1.PMHE
274 dsb sy // Ensure priority change is seen by redistributor
275 .L__skip_pmr_sync\@:
276 alternative_else_nop_endif
278 ldp x21, x22, [sp, #S_PC] // load ELR, SPSR
279 .if \el == 0
280 ct_user_enter
281 .endif
283 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
284 /*
285 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
286 * PAN bit checking.
287 */
288 alternative_if ARM64_HAS_PAN
289 b 2f // skip TTBR0 PAN
290 alternative_else_nop_endif
292 .if \el != 0
293 tbnz x22, #22, 1f // Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
294 .endif
296 __uaccess_ttbr0_enable x0, x1
298 .if \el == 0
299 /*
300 * Enable errata workarounds only if returning to user. The only
301 * workaround currently required for TTBR0_EL1 changes are for the
302 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
303 * corruption).
304 */
305 bl post_ttbr_update_workaround
306 .endif
307 1:
308 .if \el != 0
309 and x22, x22, #~PSR_PAN_BIT // ARMv8.0 CPUs do not understand this bit
310 .endif
311 2:
312 #endif
314 .if \el == 0
315 ldr x23, [sp, #S_SP] // load return stack pointer
316 msr sp_el0, x23
317 tst x22, #PSR_MODE32_BIT // native task?
318 b.eq 3f
320 #ifdef CONFIG_ARM64_ERRATUM_845719
321 alternative_if ARM64_WORKAROUND_845719
322 #ifdef CONFIG_PID_IN_CONTEXTIDR
323 mrs x29, contextidr_el1
324 msr contextidr_el1, x29
325 #else
326 msr contextidr_el1, xzr
327 #endif
328 alternative_else_nop_endif
329 #endif
330 3:
331 #ifdef CONFIG_ARM64_ERRATUM_1418040
332 alternative_if_not ARM64_WORKAROUND_1418040
333 b 4f
334 alternative_else_nop_endif
335 /*
336 * if (x22.mode32 == cntkctl_el1.el0vcten)
337 * cntkctl_el1.el0vcten = ~cntkctl_el1.el0vcten
338 */
339 mrs x1, cntkctl_el1
340 eon x0, x1, x22, lsr #3
341 tbz x0, #1, 4f
342 eor x1, x1, #2 // ARCH_TIMER_USR_VCT_ACCESS_EN
343 msr cntkctl_el1, x1
344 4:
345 #endif
346 /* No kernel C function calls after this as user keys are set. */
347 ptrauth_keys_install_user tsk, x0, x1, x2
349 apply_ssbd 0, x0, x1
350 .endif
352 msr elr_el1, x21 // set up the return data
353 msr spsr_el1, x22
354 ldp x0, x1, [sp, #16 * 0]
355 ldp x2, x3, [sp, #16 * 1]
356 ldp x4, x5, [sp, #16 * 2]
357 ldp x6, x7, [sp, #16 * 3]
358 ldp x8, x9, [sp, #16 * 4]
359 ldp x10, x11, [sp, #16 * 5]
360 ldp x12, x13, [sp, #16 * 6]
361 ldp x14, x15, [sp, #16 * 7]
362 ldp x16, x17, [sp, #16 * 8]
363 ldp x18, x19, [sp, #16 * 9]
364 ldp x20, x21, [sp, #16 * 10]
365 ldp x22, x23, [sp, #16 * 11]
366 ldp x24, x25, [sp, #16 * 12]
367 ldp x26, x27, [sp, #16 * 13]
368 ldp x28, x29, [sp, #16 * 14]
369 ldr lr, [sp, #S_LR]
370 add sp, sp, #S_FRAME_SIZE // restore sp
372 .if \el == 0
373 alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
374 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
375 bne 5f
376 msr far_el1, x30
377 tramp_alias x30, tramp_exit_native
378 br x30
379 5:
380 tramp_alias x30, tramp_exit_compat
381 br x30
382 #endif
383 .else
384 eret
385 .endif
386 sb
387 .endm
389 .macro irq_stack_entry
390 mov x19, sp // preserve the original sp
392 /*
393 * Compare sp with the base of the task stack.
394 * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack,
395 * and should switch to the irq stack.
396 */
397 ldr x25, [tsk, TSK_STACK]
398 eor x25, x25, x19
399 and x25, x25, #~(THREAD_SIZE - 1)
400 cbnz x25, 9998f
402 ldr_this_cpu x25, irq_stack_ptr, x26
403 mov x26, #IRQ_STACK_SIZE
404 add x26, x25, x26
406 /* switch to the irq stack */
407 mov sp, x26
408 9998:
409 .endm
411 /*
412 * x19 should be preserved between irq_stack_entry and
413 * irq_stack_exit.
414 */
415 .macro irq_stack_exit
416 mov sp, x19
417 .endm
419 /* GPRs used by entry code */
420 tsk .req x28 // current thread_info
422 /*
423 * Interrupt handling.
424 */
425 .macro irq_handler
426 ldr_l x1, handle_arch_irq
427 mov x0, sp
428 irq_stack_entry
429 blr x1
430 irq_stack_exit
431 .endm
433 #ifdef CONFIG_ARM64_PSEUDO_NMI
434 /*
435 * Set res to 0 if irqs were unmasked in interrupted context.
436 * Otherwise set res to non-0 value.
437 */
438 .macro test_irqs_unmasked res:req, pmr:req
439 alternative_if ARM64_HAS_IRQ_PRIO_MASKING
440 sub \res, \pmr, #GIC_PRIO_IRQON
441 alternative_else
442 mov \res, xzr
443 alternative_endif
444 .endm
445 #endif
447 .macro gic_prio_kentry_setup, tmp:req
448 #ifdef CONFIG_ARM64_PSEUDO_NMI
449 alternative_if ARM64_HAS_IRQ_PRIO_MASKING
450 mov \tmp, #(GIC_PRIO_PSR_I_SET | GIC_PRIO_IRQON)
451 msr_s SYS_ICC_PMR_EL1, \tmp
452 alternative_else_nop_endif
453 #endif
454 .endm
456 .macro gic_prio_irq_setup, pmr:req, tmp:req
457 #ifdef CONFIG_ARM64_PSEUDO_NMI
458 alternative_if ARM64_HAS_IRQ_PRIO_MASKING
459 orr \tmp, \pmr, #GIC_PRIO_PSR_I_SET
460 msr_s SYS_ICC_PMR_EL1, \tmp
461 alternative_else_nop_endif
462 #endif
463 .endm
465 .text
467 /*
468 * Exception vectors.
469 */
470 .pushsection ".entry.text", "ax"
472 .align 11
473 SYM_CODE_START(vectors)
474 kernel_ventry 1, sync_invalid // Synchronous EL1t
475 kernel_ventry 1, irq_invalid // IRQ EL1t
476 kernel_ventry 1, fiq_invalid // FIQ EL1t
477 kernel_ventry 1, error_invalid // Error EL1t
479 kernel_ventry 1, sync // Synchronous EL1h
480 kernel_ventry 1, irq // IRQ EL1h
481 kernel_ventry 1, fiq_invalid // FIQ EL1h
482 kernel_ventry 1, error // Error EL1h
484 kernel_ventry 0, sync // Synchronous 64-bit EL0
485 kernel_ventry 0, irq // IRQ 64-bit EL0
486 kernel_ventry 0, fiq_invalid // FIQ 64-bit EL0
487 kernel_ventry 0, error // Error 64-bit EL0
489 #ifdef CONFIG_COMPAT
490 kernel_ventry 0, sync_compat, 32 // Synchronous 32-bit EL0
491 kernel_ventry 0, irq_compat, 32 // IRQ 32-bit EL0
492 kernel_ventry 0, fiq_invalid_compat, 32 // FIQ 32-bit EL0
493 kernel_ventry 0, error_compat, 32 // Error 32-bit EL0
494 #else
495 kernel_ventry 0, sync_invalid, 32 // Synchronous 32-bit EL0
496 kernel_ventry 0, irq_invalid, 32 // IRQ 32-bit EL0
497 kernel_ventry 0, fiq_invalid, 32 // FIQ 32-bit EL0
498 kernel_ventry 0, error_invalid, 32 // Error 32-bit EL0
499 #endif
500 SYM_CODE_END(vectors)
502 #ifdef CONFIG_VMAP_STACK
503 /*
504 * We detected an overflow in kernel_ventry, which switched to the
505 * overflow stack. Stash the exception regs, and head to our overflow
506 * handler.
507 */
508 __bad_stack:
509 /* Restore the original x0 value */
510 mrs x0, tpidrro_el0
512 /*
513 * Store the original GPRs to the new stack. The orginal SP (minus
514 * S_FRAME_SIZE) was stashed in tpidr_el0 by kernel_ventry.
515 */
516 sub sp, sp, #S_FRAME_SIZE
517 kernel_entry 1
518 mrs x0, tpidr_el0
519 add x0, x0, #S_FRAME_SIZE
520 str x0, [sp, #S_SP]
522 /* Stash the regs for handle_bad_stack */
523 mov x0, sp
525 /* Time to die */
526 bl handle_bad_stack
527 ASM_BUG()
528 #endif /* CONFIG_VMAP_STACK */
530 /*
531 * Invalid mode handlers
532 */
533 .macro inv_entry, el, reason, regsize = 64
534 kernel_entry \el, \regsize
535 mov x0, sp
536 mov x1, #\reason
537 mrs x2, esr_el1
538 bl bad_mode
539 ASM_BUG()
540 .endm
542 SYM_CODE_START_LOCAL(el0_sync_invalid)
543 inv_entry 0, BAD_SYNC
544 SYM_CODE_END(el0_sync_invalid)
546 SYM_CODE_START_LOCAL(el0_irq_invalid)
547 inv_entry 0, BAD_IRQ
548 SYM_CODE_END(el0_irq_invalid)
550 SYM_CODE_START_LOCAL(el0_fiq_invalid)
551 inv_entry 0, BAD_FIQ
552 SYM_CODE_END(el0_fiq_invalid)
554 SYM_CODE_START_LOCAL(el0_error_invalid)
555 inv_entry 0, BAD_ERROR
556 SYM_CODE_END(el0_error_invalid)
558 #ifdef CONFIG_COMPAT
559 SYM_CODE_START_LOCAL(el0_fiq_invalid_compat)
560 inv_entry 0, BAD_FIQ, 32
561 SYM_CODE_END(el0_fiq_invalid_compat)
562 #endif
564 SYM_CODE_START_LOCAL(el1_sync_invalid)
565 inv_entry 1, BAD_SYNC
566 SYM_CODE_END(el1_sync_invalid)
568 SYM_CODE_START_LOCAL(el1_irq_invalid)
569 inv_entry 1, BAD_IRQ
570 SYM_CODE_END(el1_irq_invalid)
572 SYM_CODE_START_LOCAL(el1_fiq_invalid)
573 inv_entry 1, BAD_FIQ
574 SYM_CODE_END(el1_fiq_invalid)
576 SYM_CODE_START_LOCAL(el1_error_invalid)
577 inv_entry 1, BAD_ERROR
578 SYM_CODE_END(el1_error_invalid)
580 /*
581 * EL1 mode handlers.
582 */
583 .align 6
584 SYM_CODE_START_LOCAL_NOALIGN(el1_sync)
585 kernel_entry 1
586 mov x0, sp
587 bl el1_sync_handler
588 kernel_exit 1
589 SYM_CODE_END(el1_sync)
591 .align 6
592 SYM_CODE_START_LOCAL_NOALIGN(el1_irq)
593 kernel_entry 1
594 gic_prio_irq_setup pmr=x20, tmp=x1
595 enable_da_f
597 #ifdef CONFIG_ARM64_PSEUDO_NMI
598 test_irqs_unmasked res=x0, pmr=x20
599 cbz x0, 1f
600 bl asm_nmi_enter
601 1:
602 #endif
604 #ifdef CONFIG_TRACE_IRQFLAGS
605 bl trace_hardirqs_off
606 #endif
608 irq_handler
610 #ifdef CONFIG_PREEMPTION
611 ldr x24, [tsk, #TSK_TI_PREEMPT] // get preempt count
612 alternative_if ARM64_HAS_IRQ_PRIO_MASKING
613 /*
614 * DA_F were cleared at start of handling. If anything is set in DAIF,
615 * we come back from an NMI, so skip preemption
616 */
617 mrs x0, daif
618 orr x24, x24, x0
619 alternative_else_nop_endif
620 cbnz x24, 1f // preempt count != 0 || NMI return path
621 bl arm64_preempt_schedule_irq // irq en/disable is done inside
622 1:
623 #endif
625 #ifdef CONFIG_ARM64_PSEUDO_NMI
626 /*
627 * When using IRQ priority masking, we can get spurious interrupts while
628 * PMR is set to GIC_PRIO_IRQOFF. An NMI might also have occurred in a
629 * section with interrupts disabled. Skip tracing in those cases.
630 */
631 test_irqs_unmasked res=x0, pmr=x20
632 cbz x0, 1f
633 bl asm_nmi_exit
634 1:
635 #endif
637 #ifdef CONFIG_TRACE_IRQFLAGS
638 #ifdef CONFIG_ARM64_PSEUDO_NMI
639 test_irqs_unmasked res=x0, pmr=x20
640 cbnz x0, 1f
641 #endif
642 bl trace_hardirqs_on
643 1:
644 #endif
646 kernel_exit 1
647 SYM_CODE_END(el1_irq)
649 /*
650 * EL0 mode handlers.
651 */
652 .align 6
653 SYM_CODE_START_LOCAL_NOALIGN(el0_sync)
654 kernel_entry 0
655 mov x0, sp
656 bl el0_sync_handler
657 b ret_to_user
658 SYM_CODE_END(el0_sync)
660 #ifdef CONFIG_COMPAT
661 .align 6
662 SYM_CODE_START_LOCAL_NOALIGN(el0_sync_compat)
663 kernel_entry 0, 32
664 mov x0, sp
665 bl el0_sync_compat_handler
666 b ret_to_user
667 SYM_CODE_END(el0_sync_compat)
669 .align 6
670 SYM_CODE_START_LOCAL_NOALIGN(el0_irq_compat)
671 kernel_entry 0, 32
672 b el0_irq_naked
673 SYM_CODE_END(el0_irq_compat)
675 SYM_CODE_START_LOCAL_NOALIGN(el0_error_compat)
676 kernel_entry 0, 32
677 b el0_error_naked
678 SYM_CODE_END(el0_error_compat)
679 #endif
681 .align 6
682 SYM_CODE_START_LOCAL_NOALIGN(el0_irq)
683 kernel_entry 0
684 el0_irq_naked:
685 gic_prio_irq_setup pmr=x20, tmp=x0
686 ct_user_exit_irqoff
687 enable_da_f
689 #ifdef CONFIG_TRACE_IRQFLAGS
690 bl trace_hardirqs_off
691 #endif
693 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
694 tbz x22, #55, 1f
695 bl do_el0_irq_bp_hardening
696 1:
697 #endif
698 irq_handler
700 #ifdef CONFIG_TRACE_IRQFLAGS
701 bl trace_hardirqs_on
702 #endif
703 b ret_to_user
704 SYM_CODE_END(el0_irq)
706 SYM_CODE_START_LOCAL(el1_error)
707 kernel_entry 1
708 mrs x1, esr_el1
709 gic_prio_kentry_setup tmp=x2
710 enable_dbg
711 mov x0, sp
712 bl do_serror
713 kernel_exit 1
714 SYM_CODE_END(el1_error)
716 SYM_CODE_START_LOCAL(el0_error)
717 kernel_entry 0
718 el0_error_naked:
719 mrs x25, esr_el1
720 gic_prio_kentry_setup tmp=x2
721 ct_user_exit_irqoff
722 enable_dbg
723 mov x0, sp
724 mov x1, x25
725 bl do_serror
726 enable_da_f
727 b ret_to_user
728 SYM_CODE_END(el0_error)
730 /*
731 * Ok, we need to do extra processing, enter the slow path.
732 */
733 work_pending:
734 mov x0, sp // 'regs'
735 bl do_notify_resume
736 #ifdef CONFIG_TRACE_IRQFLAGS
737 bl trace_hardirqs_on // enabled while in userspace
738 #endif
739 ldr x1, [tsk, #TSK_TI_FLAGS] // re-check for single-step
740 b finish_ret_to_user
741 /*
742 * "slow" syscall return path.
743 */
744 ret_to_user:
745 disable_daif
746 gic_prio_kentry_setup tmp=x3
747 ldr x1, [tsk, #TSK_TI_FLAGS]
748 and x2, x1, #_TIF_WORK_MASK
749 cbnz x2, work_pending
750 finish_ret_to_user:
751 enable_step_tsk x1, x2
752 #ifdef CONFIG_GCC_PLUGIN_STACKLEAK
753 bl stackleak_erase
754 #endif
755 kernel_exit 0
756 ENDPROC(ret_to_user)
758 .popsection // .entry.text
760 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
761 /*
762 * Exception vectors trampoline.
763 */
764 .pushsection ".entry.tramp.text", "ax"
766 .macro tramp_map_kernel, tmp
767 mrs \tmp, ttbr1_el1
768 add \tmp, \tmp, #(PAGE_SIZE + RESERVED_TTBR0_SIZE)
769 bic \tmp, \tmp, #USER_ASID_FLAG
770 msr ttbr1_el1, \tmp
771 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
772 alternative_if ARM64_WORKAROUND_QCOM_FALKOR_E1003
773 /* ASID already in \tmp[63:48] */
774 movk \tmp, #:abs_g2_nc:(TRAMP_VALIAS >> 12)
775 movk \tmp, #:abs_g1_nc:(TRAMP_VALIAS >> 12)
776 /* 2MB boundary containing the vectors, so we nobble the walk cache */
777 movk \tmp, #:abs_g0_nc:((TRAMP_VALIAS & ~(SZ_2M - 1)) >> 12)
778 isb
779 tlbi vae1, \tmp
780 dsb nsh
781 alternative_else_nop_endif
782 #endif /* CONFIG_QCOM_FALKOR_ERRATUM_1003 */
783 .endm
785 .macro tramp_unmap_kernel, tmp
786 mrs \tmp, ttbr1_el1
787 sub \tmp, \tmp, #(PAGE_SIZE + RESERVED_TTBR0_SIZE)
788 orr \tmp, \tmp, #USER_ASID_FLAG
789 msr ttbr1_el1, \tmp
790 /*
791 * We avoid running the post_ttbr_update_workaround here because
792 * it's only needed by Cavium ThunderX, which requires KPTI to be
793 * disabled.
794 */
795 .endm
797 .macro tramp_ventry, regsize = 64
798 .align 7
799 1:
800 .if \regsize == 64
801 msr tpidrro_el0, x30 // Restored in kernel_ventry
802 .endif
803 /*
804 * Defend against branch aliasing attacks by pushing a dummy
805 * entry onto the return stack and using a RET instruction to
806 * enter the full-fat kernel vectors.
807 */
808 bl 2f
809 b .
810 2:
811 tramp_map_kernel x30
812 #ifdef CONFIG_RANDOMIZE_BASE
813 adr x30, tramp_vectors + PAGE_SIZE
814 alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003
815 ldr x30, [x30]
816 #else
817 ldr x30, =vectors
818 #endif
819 alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM
820 prfm plil1strm, [x30, #(1b - tramp_vectors)]
821 alternative_else_nop_endif
822 msr vbar_el1, x30
823 add x30, x30, #(1b - tramp_vectors)
824 isb
825 ret
826 .endm
828 .macro tramp_exit, regsize = 64
829 adr x30, tramp_vectors
830 msr vbar_el1, x30
831 tramp_unmap_kernel x30
832 .if \regsize == 64
833 mrs x30, far_el1
834 .endif
835 eret
836 sb
837 .endm
839 .align 11
840 SYM_CODE_START_NOALIGN(tramp_vectors)
841 .space 0x400
843 tramp_ventry
844 tramp_ventry
845 tramp_ventry
846 tramp_ventry
848 tramp_ventry 32
849 tramp_ventry 32
850 tramp_ventry 32
851 tramp_ventry 32
852 SYM_CODE_END(tramp_vectors)
854 SYM_CODE_START(tramp_exit_native)
855 tramp_exit
856 SYM_CODE_END(tramp_exit_native)
858 SYM_CODE_START(tramp_exit_compat)
859 tramp_exit 32
860 SYM_CODE_END(tramp_exit_compat)
862 .ltorg
863 .popsection // .entry.tramp.text
864 #ifdef CONFIG_RANDOMIZE_BASE
865 .pushsection ".rodata", "a"
866 .align PAGE_SHIFT
867 SYM_DATA_START(__entry_tramp_data_start)
868 .quad vectors
869 SYM_DATA_END(__entry_tramp_data_start)
870 .popsection // .rodata
871 #endif /* CONFIG_RANDOMIZE_BASE */
872 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
874 /*
875 * Register switch for AArch64. The callee-saved registers need to be saved
876 * and restored. On entry:
877 * x0 = previous task_struct (must be preserved across the switch)
878 * x1 = next task_struct
879 * Previous and next are guaranteed not to be the same.
880 *
881 */
882 SYM_FUNC_START(cpu_switch_to)
883 mov x10, #THREAD_CPU_CONTEXT
884 add x8, x0, x10
885 mov x9, sp
886 stp x19, x20, [x8], #16 // store callee-saved registers
887 stp x21, x22, [x8], #16
888 stp x23, x24, [x8], #16
889 stp x25, x26, [x8], #16
890 stp x27, x28, [x8], #16
891 stp x29, x9, [x8], #16
892 str lr, [x8]
893 add x8, x1, x10
894 ldp x19, x20, [x8], #16 // restore callee-saved registers
895 ldp x21, x22, [x8], #16
896 ldp x23, x24, [x8], #16
897 ldp x25, x26, [x8], #16
898 ldp x27, x28, [x8], #16
899 ldp x29, x9, [x8], #16
900 ldr lr, [x8]
901 mov sp, x9
902 msr sp_el0, x1
903 ptrauth_keys_install_kernel x1, 1, x8, x9, x10
904 ret
905 SYM_FUNC_END(cpu_switch_to)
906 NOKPROBE(cpu_switch_to)
908 /*
909 * This is how we return from a fork.
910 */
911 SYM_CODE_START(ret_from_fork)
912 bl schedule_tail
913 cbz x19, 1f // not a kernel thread
914 mov x0, x20
915 blr x19
916 1: get_current_task tsk
917 b ret_to_user
918 SYM_CODE_END(ret_from_fork)
919 NOKPROBE(ret_from_fork)
921 #ifdef CONFIG_ARM_SDE_INTERFACE
923 #include <asm/sdei.h>
924 #include <uapi/linux/arm_sdei.h>
926 .macro sdei_handler_exit exit_mode
927 /* On success, this call never returns... */
928 cmp \exit_mode, #SDEI_EXIT_SMC
929 b.ne 99f
930 smc #0
931 b .
932 99: hvc #0
933 b .
934 .endm
936 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
937 /*
938 * The regular SDEI entry point may have been unmapped along with the rest of
939 * the kernel. This trampoline restores the kernel mapping to make the x1 memory
940 * argument accessible.
941 *
942 * This clobbers x4, __sdei_handler() will restore this from firmware's
943 * copy.
944 */
945 .ltorg
946 .pushsection ".entry.tramp.text", "ax"
947 SYM_CODE_START(__sdei_asm_entry_trampoline)
948 mrs x4, ttbr1_el1
949 tbz x4, #USER_ASID_BIT, 1f
951 tramp_map_kernel tmp=x4
952 isb
953 mov x4, xzr
955 /*
956 * Use reg->interrupted_regs.addr_limit to remember whether to unmap
957 * the kernel on exit.
958 */
959 1: str x4, [x1, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)]
961 #ifdef CONFIG_RANDOMIZE_BASE
962 adr x4, tramp_vectors + PAGE_SIZE
963 add x4, x4, #:lo12:__sdei_asm_trampoline_next_handler
964 ldr x4, [x4]
965 #else
966 ldr x4, =__sdei_asm_handler
967 #endif
968 br x4
969 SYM_CODE_END(__sdei_asm_entry_trampoline)
970 NOKPROBE(__sdei_asm_entry_trampoline)
972 /*
973 * Make the exit call and restore the original ttbr1_el1
974 *
975 * x0 & x1: setup for the exit API call
976 * x2: exit_mode
977 * x4: struct sdei_registered_event argument from registration time.
978 */
979 SYM_CODE_START(__sdei_asm_exit_trampoline)
980 ldr x4, [x4, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)]
981 cbnz x4, 1f
983 tramp_unmap_kernel tmp=x4
985 1: sdei_handler_exit exit_mode=x2
986 SYM_CODE_END(__sdei_asm_exit_trampoline)
987 NOKPROBE(__sdei_asm_exit_trampoline)
988 .ltorg
989 .popsection // .entry.tramp.text
990 #ifdef CONFIG_RANDOMIZE_BASE
991 .pushsection ".rodata", "a"
992 SYM_DATA_START(__sdei_asm_trampoline_next_handler)
993 .quad __sdei_asm_handler
994 SYM_DATA_END(__sdei_asm_trampoline_next_handler)
995 .popsection // .rodata
996 #endif /* CONFIG_RANDOMIZE_BASE */
997 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
999 /*
1000 * Software Delegated Exception entry point.
1001 *
1002 * x0: Event number
1003 * x1: struct sdei_registered_event argument from registration time.
1004 * x2: interrupted PC
1005 * x3: interrupted PSTATE
1006 * x4: maybe clobbered by the trampoline
1007 *
1008 * Firmware has preserved x0->x17 for us, we must save/restore the rest to
1009 * follow SMC-CC. We save (or retrieve) all the registers as the handler may
1010 * want them.
1011 */
1012 SYM_CODE_START(__sdei_asm_handler)
1013 stp x2, x3, [x1, #SDEI_EVENT_INTREGS + S_PC]
1014 stp x4, x5, [x1, #SDEI_EVENT_INTREGS + 16 * 2]
1015 stp x6, x7, [x1, #SDEI_EVENT_INTREGS + 16 * 3]
1016 stp x8, x9, [x1, #SDEI_EVENT_INTREGS + 16 * 4]
1017 stp x10, x11, [x1, #SDEI_EVENT_INTREGS + 16 * 5]
1018 stp x12, x13, [x1, #SDEI_EVENT_INTREGS + 16 * 6]
1019 stp x14, x15, [x1, #SDEI_EVENT_INTREGS + 16 * 7]
1020 stp x16, x17, [x1, #SDEI_EVENT_INTREGS + 16 * 8]
1021 stp x18, x19, [x1, #SDEI_EVENT_INTREGS + 16 * 9]
1022 stp x20, x21, [x1, #SDEI_EVENT_INTREGS + 16 * 10]
1023 stp x22, x23, [x1, #SDEI_EVENT_INTREGS + 16 * 11]
1024 stp x24, x25, [x1, #SDEI_EVENT_INTREGS + 16 * 12]
1025 stp x26, x27, [x1, #SDEI_EVENT_INTREGS + 16 * 13]
1026 stp x28, x29, [x1, #SDEI_EVENT_INTREGS + 16 * 14]
1027 mov x4, sp
1028 stp lr, x4, [x1, #SDEI_EVENT_INTREGS + S_LR]
1030 mov x19, x1
1032 #ifdef CONFIG_VMAP_STACK
1033 /*
1034 * entry.S may have been using sp as a scratch register, find whether
1035 * this is a normal or critical event and switch to the appropriate
1036 * stack for this CPU.
1037 */
1038 ldrb w4, [x19, #SDEI_EVENT_PRIORITY]
1039 cbnz w4, 1f
1040 ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
1041 b 2f
1042 1: ldr_this_cpu dst=x5, sym=sdei_stack_critical_ptr, tmp=x6
1043 2: mov x6, #SDEI_STACK_SIZE
1044 add x5, x5, x6
1045 mov sp, x5
1046 #endif
1048 /*
1049 * We may have interrupted userspace, or a guest, or exit-from or
1050 * return-to either of these. We can't trust sp_el0, restore it.
1051 */
1052 mrs x28, sp_el0
1053 ldr_this_cpu dst=x0, sym=__entry_task, tmp=x1
1054 msr sp_el0, x0
1056 /* If we interrupted the kernel point to the previous stack/frame. */
1057 and x0, x3, #0xc
1058 mrs x1, CurrentEL
1059 cmp x0, x1
1060 csel x29, x29, xzr, eq // fp, or zero
1061 csel x4, x2, xzr, eq // elr, or zero
1063 stp x29, x4, [sp, #-16]!
1064 mov x29, sp
1066 add x0, x19, #SDEI_EVENT_INTREGS
1067 mov x1, x19
1068 bl __sdei_handler
1070 msr sp_el0, x28
1071 /* restore regs >x17 that we clobbered */
1072 mov x4, x19 // keep x4 for __sdei_asm_exit_trampoline
1073 ldp x28, x29, [x4, #SDEI_EVENT_INTREGS + 16 * 14]
1074 ldp x18, x19, [x4, #SDEI_EVENT_INTREGS + 16 * 9]
1075 ldp lr, x1, [x4, #SDEI_EVENT_INTREGS + S_LR]
1076 mov sp, x1
1078 mov x1, x0 // address to complete_and_resume
1079 /* x0 = (x0 <= 1) ? EVENT_COMPLETE:EVENT_COMPLETE_AND_RESUME */
1080 cmp x0, #1
1081 mov_q x2, SDEI_1_0_FN_SDEI_EVENT_COMPLETE
1082 mov_q x3, SDEI_1_0_FN_SDEI_EVENT_COMPLETE_AND_RESUME
1083 csel x0, x2, x3, ls
1085 ldr_l x2, sdei_exit_mode
1087 alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
1088 sdei_handler_exit exit_mode=x2
1089 alternative_else_nop_endif
1091 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
1092 tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline
1093 br x5
1094 #endif
1095 SYM_CODE_END(__sdei_asm_handler)
1096 NOKPROBE(__sdei_asm_handler)
1097 #endif /* CONFIG_ARM_SDE_INTERFACE */