?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
WIP FPC-III support
[linux/fpc-iii.git] / arch / arm64 / kernel / traps.c
blob6895ce777e7f286ce82da4fb8c54a3ac58cd0b37
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Based on arch/arm/kernel/traps.c
4 *
5 * Copyright (C) 1995-2009 Russell King
6 * Copyright (C) 2012 ARM Ltd.
7 */
8
9 #include <linux/bug.h>
10 #include <linux/context_tracking.h>
11 #include <linux/signal.h>
12 #include <linux/personality.h>
13 #include <linux/kallsyms.h>
14 #include <linux/kprobes.h>
15 #include <linux/spinlock.h>
16 #include <linux/uaccess.h>
17 #include <linux/hardirq.h>
18 #include <linux/kdebug.h>
19 #include <linux/module.h>
20 #include <linux/kexec.h>
21 #include <linux/delay.h>
22 #include <linux/init.h>
23 #include <linux/sched/signal.h>
24 #include <linux/sched/debug.h>
25 #include <linux/sched/task_stack.h>
26 #include <linux/sizes.h>
27 #include <linux/syscalls.h>
28 #include <linux/mm_types.h>
29 #include <linux/kasan.h>
30
31 #include <asm/atomic.h>
32 #include <asm/bug.h>
33 #include <asm/cpufeature.h>
34 #include <asm/daifflags.h>
35 #include <asm/debug-monitors.h>
36 #include <asm/esr.h>
37 #include <asm/exception.h>
38 #include <asm/extable.h>
39 #include <asm/insn.h>
40 #include <asm/kprobes.h>
41 #include <asm/traps.h>
42 #include <asm/smp.h>
43 #include <asm/stack_pointer.h>
44 #include <asm/stacktrace.h>
45 #include <asm/system_misc.h>
46 #include <asm/sysreg.h>
47
48 static const char *handler[]= {
49 "Synchronous Abort",
50 "IRQ",
51 "FIQ",
52 "Error"
53 };
54
55 int show_unhandled_signals = 0;
56
57 static void dump_kernel_instr(const char *lvl, struct pt_regs *regs)
58 {
59 unsigned long addr = instruction_pointer(regs);
60 char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str;
61 int i;
62
63 if (user_mode(regs))
64 return;
65
66 for (i = -4; i < 1; i++) {
67 unsigned int val, bad;
68
69 bad = aarch64_insn_read(&((u32 *)addr)[i], &val);
70
71 if (!bad)
72 p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val);
73 else {
74 p += sprintf(p, "bad PC value");
75 break;
76 }
77 }
78
79 printk("%sCode: %s\n", lvl, str);
80 }
81
82 #ifdef CONFIG_PREEMPT
83 #define S_PREEMPT " PREEMPT"
84 #elif defined(CONFIG_PREEMPT_RT)
85 #define S_PREEMPT " PREEMPT_RT"
86 #else
87 #define S_PREEMPT ""
88 #endif
89
90 #define S_SMP " SMP"
91
92 static int __die(const char *str, int err, struct pt_regs *regs)
93 {
94 static int die_counter;
95 int ret;
96
97 pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n",
98 str, err, ++die_counter);
99
100 /* trap and error numbers are mostly meaningless on ARM */
101 ret = notify_die(DIE_OOPS, str, regs, err, 0, SIGSEGV);
102 if (ret == NOTIFY_STOP)
103 return ret;
104
105 print_modules();
106 show_regs(regs);
107
108 dump_kernel_instr(KERN_EMERG, regs);
109
110 return ret;
111 }
112
113 static DEFINE_RAW_SPINLOCK(die_lock);
114
115 /*
116 * This function is protected against re-entrancy.
117 */
118 void die(const char *str, struct pt_regs *regs, int err)
119 {
120 int ret;
121 unsigned long flags;
122
123 raw_spin_lock_irqsave(&die_lock, flags);
124
125 oops_enter();
126
127 console_verbose();
128 bust_spinlocks(1);
129 ret = __die(str, err, regs);
130
131 if (regs && kexec_should_crash(current))
132 crash_kexec(regs);
133
134 bust_spinlocks(0);
135 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
136 oops_exit();
137
138 if (in_interrupt())
139 panic("%s: Fatal exception in interrupt", str);
140 if (panic_on_oops)
141 panic("%s: Fatal exception", str);
142
143 raw_spin_unlock_irqrestore(&die_lock, flags);
144
145 if (ret != NOTIFY_STOP)
146 do_exit(SIGSEGV);
147 }
148
149 static void arm64_show_signal(int signo, const char *str)
150 {
151 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
152 DEFAULT_RATELIMIT_BURST);
153 struct task_struct *tsk = current;
154 unsigned int esr = tsk->thread.fault_code;
155 struct pt_regs *regs = task_pt_regs(tsk);
156
157 /* Leave if the signal won't be shown */
158 if (!show_unhandled_signals ||
159 !unhandled_signal(tsk, signo) ||
160 !__ratelimit(&rs))
161 return;
162
163 pr_info("%s[%d]: unhandled exception: ", tsk->comm, task_pid_nr(tsk));
164 if (esr)
165 pr_cont("%s, ESR 0x%08x, ", esr_get_class_string(esr), esr);
166
167 pr_cont("%s", str);
168 print_vma_addr(KERN_CONT " in ", regs->pc);
169 pr_cont("\n");
170 __show_regs(regs);
171 }
172
173 void arm64_force_sig_fault(int signo, int code, unsigned long far,
174 const char *str)
175 {
176 arm64_show_signal(signo, str);
177 if (signo == SIGKILL)
178 force_sig(SIGKILL);
179 else
180 force_sig_fault(signo, code, (void __user *)far);
181 }
182
183 void arm64_force_sig_mceerr(int code, unsigned long far, short lsb,
184 const char *str)
185 {
186 arm64_show_signal(SIGBUS, str);
187 force_sig_mceerr(code, (void __user *)far, lsb);
188 }
189
190 void arm64_force_sig_ptrace_errno_trap(int errno, unsigned long far,
191 const char *str)
192 {
193 arm64_show_signal(SIGTRAP, str);
194 force_sig_ptrace_errno_trap(errno, (void __user *)far);
195 }
196
197 void arm64_notify_die(const char *str, struct pt_regs *regs,
198 int signo, int sicode, unsigned long far,
199 int err)
200 {
201 if (user_mode(regs)) {
202 WARN_ON(regs != current_pt_regs());
203 current->thread.fault_address = 0;
204 current->thread.fault_code = err;
205
206 arm64_force_sig_fault(signo, sicode, far, str);
207 } else {
208 die(str, regs, err);
209 }
210 }
211
212 #ifdef CONFIG_COMPAT
213 #define PSTATE_IT_1_0_SHIFT 25
214 #define PSTATE_IT_1_0_MASK (0x3 << PSTATE_IT_1_0_SHIFT)
215 #define PSTATE_IT_7_2_SHIFT 10
216 #define PSTATE_IT_7_2_MASK (0x3f << PSTATE_IT_7_2_SHIFT)
217
218 static u32 compat_get_it_state(struct pt_regs *regs)
219 {
220 u32 it, pstate = regs->pstate;
221
222 it = (pstate & PSTATE_IT_1_0_MASK) >> PSTATE_IT_1_0_SHIFT;
223 it |= ((pstate & PSTATE_IT_7_2_MASK) >> PSTATE_IT_7_2_SHIFT) << 2;
224
225 return it;
226 }
227
228 static void compat_set_it_state(struct pt_regs *regs, u32 it)
229 {
230 u32 pstate_it;
231
232 pstate_it = (it << PSTATE_IT_1_0_SHIFT) & PSTATE_IT_1_0_MASK;
233 pstate_it |= ((it >> 2) << PSTATE_IT_7_2_SHIFT) & PSTATE_IT_7_2_MASK;
234
235 regs->pstate &= ~PSR_AA32_IT_MASK;
236 regs->pstate |= pstate_it;
237 }
238
239 static void advance_itstate(struct pt_regs *regs)
240 {
241 u32 it;
242
243 /* ARM mode */
244 if (!(regs->pstate & PSR_AA32_T_BIT) ||
245 !(regs->pstate & PSR_AA32_IT_MASK))
246 return;
247
248 it = compat_get_it_state(regs);
249
250 /*
251 * If this is the last instruction of the block, wipe the IT
252 * state. Otherwise advance it.
253 */
254 if (!(it & 7))
255 it = 0;
256 else
257 it = (it & 0xe0) | ((it << 1) & 0x1f);
258
259 compat_set_it_state(regs, it);
260 }
261 #else
262 static void advance_itstate(struct pt_regs *regs)
263 {
264 }
265 #endif
266
267 void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
268 {
269 regs->pc += size;
270
271 /*
272 * If we were single stepping, we want to get the step exception after
273 * we return from the trap.
274 */
275 if (user_mode(regs))
276 user_fastforward_single_step(current);
277
278 if (compat_user_mode(regs))
279 advance_itstate(regs);
280 else
281 regs->pstate &= ~PSR_BTYPE_MASK;
282 }
283
284 static LIST_HEAD(undef_hook);
285 static DEFINE_RAW_SPINLOCK(undef_lock);
286
287 void register_undef_hook(struct undef_hook *hook)
288 {
289 unsigned long flags;
290
291 raw_spin_lock_irqsave(&undef_lock, flags);
292 list_add(&hook->node, &undef_hook);
293 raw_spin_unlock_irqrestore(&undef_lock, flags);
294 }
295
296 void unregister_undef_hook(struct undef_hook *hook)
297 {
298 unsigned long flags;
299
300 raw_spin_lock_irqsave(&undef_lock, flags);
301 list_del(&hook->node);
302 raw_spin_unlock_irqrestore(&undef_lock, flags);
303 }
304
305 static int call_undef_hook(struct pt_regs *regs)
306 {
307 struct undef_hook *hook;
308 unsigned long flags;
309 u32 instr;
310 int (*fn)(struct pt_regs *regs, u32 instr) = NULL;
311 void __user *pc = (void __user *)instruction_pointer(regs);
312
313 if (!user_mode(regs)) {
314 __le32 instr_le;
315 if (get_kernel_nofault(instr_le, (__force __le32 *)pc))
316 goto exit;
317 instr = le32_to_cpu(instr_le);
318 } else if (compat_thumb_mode(regs)) {
319 /* 16-bit Thumb instruction */
320 __le16 instr_le;
321 if (get_user(instr_le, (__le16 __user *)pc))
322 goto exit;
323 instr = le16_to_cpu(instr_le);
324 if (aarch32_insn_is_wide(instr)) {
325 u32 instr2;
326
327 if (get_user(instr_le, (__le16 __user *)(pc + 2)))
328 goto exit;
329 instr2 = le16_to_cpu(instr_le);
330 instr = (instr << 16) | instr2;
331 }
332 } else {
333 /* 32-bit ARM instruction */
334 __le32 instr_le;
335 if (get_user(instr_le, (__le32 __user *)pc))
336 goto exit;
337 instr = le32_to_cpu(instr_le);
338 }
339
340 raw_spin_lock_irqsave(&undef_lock, flags);
341 list_for_each_entry(hook, &undef_hook, node)
342 if ((instr & hook->instr_mask) == hook->instr_val &&
343 (regs->pstate & hook->pstate_mask) == hook->pstate_val)
344 fn = hook->fn;
345
346 raw_spin_unlock_irqrestore(&undef_lock, flags);
347 exit:
348 return fn ? fn(regs, instr) : 1;
349 }
350
351 void force_signal_inject(int signal, int code, unsigned long address, unsigned int err)
352 {
353 const char *desc;
354 struct pt_regs *regs = current_pt_regs();
355
356 if (WARN_ON(!user_mode(regs)))
357 return;
358
359 switch (signal) {
360 case SIGILL:
361 desc = "undefined instruction";
362 break;
363 case SIGSEGV:
364 desc = "illegal memory access";
365 break;
366 default:
367 desc = "unknown or unrecoverable error";
368 break;
369 }
370
371 /* Force signals we don't understand to SIGKILL */
372 if (WARN_ON(signal != SIGKILL &&
373 siginfo_layout(signal, code) != SIL_FAULT)) {
374 signal = SIGKILL;
375 }
376
377 arm64_notify_die(desc, regs, signal, code, address, err);
378 }
379
380 /*
381 * Set up process info to signal segmentation fault - called on access error.
382 */
383 void arm64_notify_segfault(unsigned long addr)
384 {
385 int code;
386
387 mmap_read_lock(current->mm);
388 if (find_vma(current->mm, untagged_addr(addr)) == NULL)
389 code = SEGV_MAPERR;
390 else
391 code = SEGV_ACCERR;
392 mmap_read_unlock(current->mm);
393
394 force_signal_inject(SIGSEGV, code, addr, 0);
395 }
396
397 void do_undefinstr(struct pt_regs *regs)
398 {
399 /* check for AArch32 breakpoint instructions */
400 if (!aarch32_break_handler(regs))
401 return;
402
403 if (call_undef_hook(regs) == 0)
404 return;
405
406 BUG_ON(!user_mode(regs));
407 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
408 }
409 NOKPROBE_SYMBOL(do_undefinstr);
410
411 void do_bti(struct pt_regs *regs)
412 {
413 BUG_ON(!user_mode(regs));
414 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
415 }
416 NOKPROBE_SYMBOL(do_bti);
417
418 void do_ptrauth_fault(struct pt_regs *regs, unsigned int esr)
419 {
420 /*
421 * Unexpected FPAC exception or pointer authentication failure in
422 * the kernel: kill the task before it does any more harm.
423 */
424 BUG_ON(!user_mode(regs));
425 force_signal_inject(SIGILL, ILL_ILLOPN, regs->pc, esr);
426 }
427 NOKPROBE_SYMBOL(do_ptrauth_fault);
428
429 #define __user_cache_maint(insn, address, res) \
430 if (address >= user_addr_max()) { \
431 res = -EFAULT; \
432 } else { \
433 uaccess_ttbr0_enable(); \
434 asm volatile ( \
435 "1: " insn ", %1\n" \
436 " mov %w0, #0\n" \
437 "2:\n" \
438 " .pushsection .fixup,\"ax\"\n" \
439 " .align 2\n" \
440 "3: mov %w0, %w2\n" \
441 " b 2b\n" \
442 " .popsection\n" \
443 _ASM_EXTABLE(1b, 3b) \
444 : "=r" (res) \
445 : "r" (address), "i" (-EFAULT)); \
446 uaccess_ttbr0_disable(); \
447 }
448
449 static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs)
450 {
451 unsigned long tagged_address, address;
452 int rt = ESR_ELx_SYS64_ISS_RT(esr);
453 int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
454 int ret = 0;
455
456 tagged_address = pt_regs_read_reg(regs, rt);
457 address = untagged_addr(tagged_address);
458
459 switch (crm) {
460 case ESR_ELx_SYS64_ISS_CRM_DC_CVAU: /* DC CVAU, gets promoted */
461 __user_cache_maint("dc civac", address, ret);
462 break;
463 case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */
464 __user_cache_maint("dc civac", address, ret);
465 break;
466 case ESR_ELx_SYS64_ISS_CRM_DC_CVADP: /* DC CVADP */
467 __user_cache_maint("sys 3, c7, c13, 1", address, ret);
468 break;
469 case ESR_ELx_SYS64_ISS_CRM_DC_CVAP: /* DC CVAP */
470 __user_cache_maint("sys 3, c7, c12, 1", address, ret);
471 break;
472 case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */
473 __user_cache_maint("dc civac", address, ret);
474 break;
475 case ESR_ELx_SYS64_ISS_CRM_IC_IVAU: /* IC IVAU */
476 __user_cache_maint("ic ivau", address, ret);
477 break;
478 default:
479 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
480 return;
481 }
482
483 if (ret)
484 arm64_notify_segfault(tagged_address);
485 else
486 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
487 }
488
489 static void ctr_read_handler(unsigned int esr, struct pt_regs *regs)
490 {
491 int rt = ESR_ELx_SYS64_ISS_RT(esr);
492 unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0);
493
494 if (cpus_have_const_cap(ARM64_WORKAROUND_1542419)) {
495 /* Hide DIC so that we can trap the unnecessary maintenance...*/
496 val &= ~BIT(CTR_DIC_SHIFT);
497
498 /* ... and fake IminLine to reduce the number of traps. */
499 val &= ~CTR_IMINLINE_MASK;
500 val |= (PAGE_SHIFT - 2) & CTR_IMINLINE_MASK;
501 }
502
503 pt_regs_write_reg(regs, rt, val);
504
505 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
506 }
507
508 static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
509 {
510 int rt = ESR_ELx_SYS64_ISS_RT(esr);
511
512 pt_regs_write_reg(regs, rt, arch_timer_read_counter());
513 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
514 }
515
516 static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
517 {
518 int rt = ESR_ELx_SYS64_ISS_RT(esr);
519
520 pt_regs_write_reg(regs, rt, arch_timer_get_rate());
521 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
522 }
523
524 static void mrs_handler(unsigned int esr, struct pt_regs *regs)
525 {
526 u32 sysreg, rt;
527
528 rt = ESR_ELx_SYS64_ISS_RT(esr);
529 sysreg = esr_sys64_to_sysreg(esr);
530
531 if (do_emulate_mrs(regs, sysreg, rt) != 0)
532 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
533 }
534
535 static void wfi_handler(unsigned int esr, struct pt_regs *regs)
536 {
537 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
538 }
539
540 struct sys64_hook {
541 unsigned int esr_mask;
542 unsigned int esr_val;
543 void (*handler)(unsigned int esr, struct pt_regs *regs);
544 };
545
546 static const struct sys64_hook sys64_hooks[] = {
547 {
548 .esr_mask = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK,
549 .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL,
550 .handler = user_cache_maint_handler,
551 },
552 {
553 /* Trap read access to CTR_EL0 */
554 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
555 .esr_val = ESR_ELx_SYS64_ISS_SYS_CTR_READ,
556 .handler = ctr_read_handler,
557 },
558 {
559 /* Trap read access to CNTVCT_EL0 */
560 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
561 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT,
562 .handler = cntvct_read_handler,
563 },
564 {
565 /* Trap read access to CNTFRQ_EL0 */
566 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
567 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTFRQ,
568 .handler = cntfrq_read_handler,
569 },
570 {
571 /* Trap read access to CPUID registers */
572 .esr_mask = ESR_ELx_SYS64_ISS_SYS_MRS_OP_MASK,
573 .esr_val = ESR_ELx_SYS64_ISS_SYS_MRS_OP_VAL,
574 .handler = mrs_handler,
575 },
576 {
577 /* Trap WFI instructions executed in userspace */
578 .esr_mask = ESR_ELx_WFx_MASK,
579 .esr_val = ESR_ELx_WFx_WFI_VAL,
580 .handler = wfi_handler,
581 },
582 {},
583 };
584
585 #ifdef CONFIG_COMPAT
586 static bool cp15_cond_valid(unsigned int esr, struct pt_regs *regs)
587 {
588 int cond;
589
590 /* Only a T32 instruction can trap without CV being set */
591 if (!(esr & ESR_ELx_CV)) {
592 u32 it;
593
594 it = compat_get_it_state(regs);
595 if (!it)
596 return true;
597
598 cond = it >> 4;
599 } else {
600 cond = (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
601 }
602
603 return aarch32_opcode_cond_checks[cond](regs->pstate);
604 }
605
606 static void compat_cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
607 {
608 int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT;
609
610 pt_regs_write_reg(regs, reg, arch_timer_get_rate());
611 arm64_skip_faulting_instruction(regs, 4);
612 }
613
614 static const struct sys64_hook cp15_32_hooks[] = {
615 {
616 .esr_mask = ESR_ELx_CP15_32_ISS_SYS_MASK,
617 .esr_val = ESR_ELx_CP15_32_ISS_SYS_CNTFRQ,
618 .handler = compat_cntfrq_read_handler,
619 },
620 {},
621 };
622
623 static void compat_cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
624 {
625 int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT;
626 int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT;
627 u64 val = arch_timer_read_counter();
628
629 pt_regs_write_reg(regs, rt, lower_32_bits(val));
630 pt_regs_write_reg(regs, rt2, upper_32_bits(val));
631 arm64_skip_faulting_instruction(regs, 4);
632 }
633
634 static const struct sys64_hook cp15_64_hooks[] = {
635 {
636 .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK,
637 .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCT,
638 .handler = compat_cntvct_read_handler,
639 },
640 {},
641 };
642
643 void do_cp15instr(unsigned int esr, struct pt_regs *regs)
644 {
645 const struct sys64_hook *hook, *hook_base;
646
647 if (!cp15_cond_valid(esr, regs)) {
648 /*
649 * There is no T16 variant of a CP access, so we
650 * always advance PC by 4 bytes.
651 */
652 arm64_skip_faulting_instruction(regs, 4);
653 return;
654 }
655
656 switch (ESR_ELx_EC(esr)) {
657 case ESR_ELx_EC_CP15_32:
658 hook_base = cp15_32_hooks;
659 break;
660 case ESR_ELx_EC_CP15_64:
661 hook_base = cp15_64_hooks;
662 break;
663 default:
664 do_undefinstr(regs);
665 return;
666 }
667
668 for (hook = hook_base; hook->handler; hook++)
669 if ((hook->esr_mask & esr) == hook->esr_val) {
670 hook->handler(esr, regs);
671 return;
672 }
673
674 /*
675 * New cp15 instructions may previously have been undefined at
676 * EL0. Fall back to our usual undefined instruction handler
677 * so that we handle these consistently.
678 */
679 do_undefinstr(regs);
680 }
681 NOKPROBE_SYMBOL(do_cp15instr);
682 #endif
683
684 void do_sysinstr(unsigned int esr, struct pt_regs *regs)
685 {
686 const struct sys64_hook *hook;
687
688 for (hook = sys64_hooks; hook->handler; hook++)
689 if ((hook->esr_mask & esr) == hook->esr_val) {
690 hook->handler(esr, regs);
691 return;
692 }
693
694 /*
695 * New SYS instructions may previously have been undefined at EL0. Fall
696 * back to our usual undefined instruction handler so that we handle
697 * these consistently.
698 */
699 do_undefinstr(regs);
700 }
701 NOKPROBE_SYMBOL(do_sysinstr);
702
703 static const char *esr_class_str[] = {
704 [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC",
705 [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized",
706 [ESR_ELx_EC_WFx] = "WFI/WFE",
707 [ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC",
708 [ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC",
709 [ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC",
710 [ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC",
711 [ESR_ELx_EC_FP_ASIMD] = "ASIMD",
712 [ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS",
713 [ESR_ELx_EC_PAC] = "PAC",
714 [ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC",
715 [ESR_ELx_EC_BTI] = "BTI",
716 [ESR_ELx_EC_ILL] = "PSTATE.IL",
717 [ESR_ELx_EC_SVC32] = "SVC (AArch32)",
718 [ESR_ELx_EC_HVC32] = "HVC (AArch32)",
719 [ESR_ELx_EC_SMC32] = "SMC (AArch32)",
720 [ESR_ELx_EC_SVC64] = "SVC (AArch64)",
721 [ESR_ELx_EC_HVC64] = "HVC (AArch64)",
722 [ESR_ELx_EC_SMC64] = "SMC (AArch64)",
723 [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)",
724 [ESR_ELx_EC_SVE] = "SVE",
725 [ESR_ELx_EC_ERET] = "ERET/ERETAA/ERETAB",
726 [ESR_ELx_EC_FPAC] = "FPAC",
727 [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF",
728 [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)",
729 [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)",
730 [ESR_ELx_EC_PC_ALIGN] = "PC Alignment",
731 [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)",
732 [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)",
733 [ESR_ELx_EC_SP_ALIGN] = "SP Alignment",
734 [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)",
735 [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)",
736 [ESR_ELx_EC_SERROR] = "SError",
737 [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)",
738 [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)",
739 [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)",
740 [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)",
741 [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)",
742 [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)",
743 [ESR_ELx_EC_BKPT32] = "BKPT (AArch32)",
744 [ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)",
745 [ESR_ELx_EC_BRK64] = "BRK (AArch64)",
746 };
747
748 const char *esr_get_class_string(u32 esr)
749 {
750 return esr_class_str[ESR_ELx_EC(esr)];
751 }
752
753 /*
754 * bad_mode handles the impossible case in the exception vector. This is always
755 * fatal.
756 */
757 asmlinkage void notrace bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
758 {
759 arm64_enter_nmi(regs);
760
761 console_verbose();
762
763 pr_crit("Bad mode in %s handler detected on CPU%d, code 0x%08x -- %s\n",
764 handler[reason], smp_processor_id(), esr,
765 esr_get_class_string(esr));
766
767 __show_regs(regs);
768 local_daif_mask();
769 panic("bad mode");
770 }
771
772 /*
773 * bad_el0_sync handles unexpected, but potentially recoverable synchronous
774 * exceptions taken from EL0. Unlike bad_mode, this returns.
775 */
776 void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
777 {
778 unsigned long pc = instruction_pointer(regs);
779
780 current->thread.fault_address = 0;
781 current->thread.fault_code = esr;
782
783 arm64_force_sig_fault(SIGILL, ILL_ILLOPC, pc,
784 "Bad EL0 synchronous exception");
785 }
786
787 #ifdef CONFIG_VMAP_STACK
788
789 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
790 __aligned(16);
791
792 asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs)
793 {
794 unsigned long tsk_stk = (unsigned long)current->stack;
795 unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
796 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
797 unsigned int esr = read_sysreg(esr_el1);
798 unsigned long far = read_sysreg(far_el1);
799
800 arm64_enter_nmi(regs);
801
802 console_verbose();
803 pr_emerg("Insufficient stack space to handle exception!");
804
805 pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr));
806 pr_emerg("FAR: 0x%016lx\n", far);
807
808 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n",
809 tsk_stk, tsk_stk + THREAD_SIZE);
810 pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n",
811 irq_stk, irq_stk + IRQ_STACK_SIZE);
812 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
813 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
814
815 __show_regs(regs);
816
817 /*
818 * We use nmi_panic to limit the potential for recusive overflows, and
819 * to get a better stack trace.
820 */
821 nmi_panic(NULL, "kernel stack overflow");
822 cpu_park_loop();
823 }
824 #endif
825
826 void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr)
827 {
828 console_verbose();
829
830 pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n",
831 smp_processor_id(), esr, esr_get_class_string(esr));
832 if (regs)
833 __show_regs(regs);
834
835 nmi_panic(regs, "Asynchronous SError Interrupt");
836
837 cpu_park_loop();
838 unreachable();
839 }
840
841 bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr)
842 {
843 u32 aet = arm64_ras_serror_get_severity(esr);
844
845 switch (aet) {
846 case ESR_ELx_AET_CE: /* corrected error */
847 case ESR_ELx_AET_UEO: /* restartable, not yet consumed */
848 /*
849 * The CPU can make progress. We may take UEO again as
850 * a more severe error.
851 */
852 return false;
853
854 case ESR_ELx_AET_UEU: /* Uncorrected Unrecoverable */
855 case ESR_ELx_AET_UER: /* Uncorrected Recoverable */
856 /*
857 * The CPU can't make progress. The exception may have
858 * been imprecise.
859 *
860 * Neoverse-N1 #1349291 means a non-KVM SError reported as
861 * Unrecoverable should be treated as Uncontainable. We
862 * call arm64_serror_panic() in both cases.
863 */
864 return true;
865
866 case ESR_ELx_AET_UC: /* Uncontainable or Uncategorized error */
867 default:
868 /* Error has been silently propagated */
869 arm64_serror_panic(regs, esr);
870 }
871 }
872
873 asmlinkage void noinstr do_serror(struct pt_regs *regs, unsigned int esr)
874 {
875 arm64_enter_nmi(regs);
876
877 /* non-RAS errors are not containable */
878 if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr))
879 arm64_serror_panic(regs, esr);
880
881 arm64_exit_nmi(regs);
882 }
883
884 /* GENERIC_BUG traps */
885
886 int is_valid_bugaddr(unsigned long addr)
887 {
888 /*
889 * bug_handler() only called for BRK #BUG_BRK_IMM.
890 * So the answer is trivial -- any spurious instances with no
891 * bug table entry will be rejected by report_bug() and passed
892 * back to the debug-monitors code and handled as a fatal
893 * unexpected debug exception.
894 */
895 return 1;
896 }
897
898 static int bug_handler(struct pt_regs *regs, unsigned int esr)
899 {
900 switch (report_bug(regs->pc, regs)) {
901 case BUG_TRAP_TYPE_BUG:
902 die("Oops - BUG", regs, 0);
903 break;
904
905 case BUG_TRAP_TYPE_WARN:
906 break;
907
908 default:
909 /* unknown/unrecognised bug trap type */
910 return DBG_HOOK_ERROR;
911 }
912
913 /* If thread survives, skip over the BUG instruction and continue: */
914 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
915 return DBG_HOOK_HANDLED;
916 }
917
918 static struct break_hook bug_break_hook = {
919 .fn = bug_handler,
920 .imm = BUG_BRK_IMM,
921 };
922
923 static int reserved_fault_handler(struct pt_regs *regs, unsigned int esr)
924 {
925 pr_err("%s generated an invalid instruction at %pS!\n",
926 in_bpf_jit(regs) ? "BPF JIT" : "Kernel text patching",
927 (void *)instruction_pointer(regs));
928
929 /* We cannot handle this */
930 return DBG_HOOK_ERROR;
931 }
932
933 static struct break_hook fault_break_hook = {
934 .fn = reserved_fault_handler,
935 .imm = FAULT_BRK_IMM,
936 };
937
938 #ifdef CONFIG_KASAN_SW_TAGS
939
940 #define KASAN_ESR_RECOVER 0x20
941 #define KASAN_ESR_WRITE 0x10
942 #define KASAN_ESR_SIZE_MASK 0x0f
943 #define KASAN_ESR_SIZE(esr) (1 << ((esr) & KASAN_ESR_SIZE_MASK))
944
945 static int kasan_handler(struct pt_regs *regs, unsigned int esr)
946 {
947 bool recover = esr & KASAN_ESR_RECOVER;
948 bool write = esr & KASAN_ESR_WRITE;
949 size_t size = KASAN_ESR_SIZE(esr);
950 u64 addr = regs->regs[0];
951 u64 pc = regs->pc;
952
953 kasan_report(addr, size, write, pc);
954
955 /*
956 * The instrumentation allows to control whether we can proceed after
957 * a crash was detected. This is done by passing the -recover flag to
958 * the compiler. Disabling recovery allows to generate more compact
959 * code.
960 *
961 * Unfortunately disabling recovery doesn't work for the kernel right
962 * now. KASAN reporting is disabled in some contexts (for example when
963 * the allocator accesses slab object metadata; this is controlled by
964 * current->kasan_depth). All these accesses are detected by the tool,
965 * even though the reports for them are not printed.
966 *
967 * This is something that might be fixed at some point in the future.
968 */
969 if (!recover)
970 die("Oops - KASAN", regs, 0);
971
972 /* If thread survives, skip over the brk instruction and continue: */
973 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
974 return DBG_HOOK_HANDLED;
975 }
976
977 static struct break_hook kasan_break_hook = {
978 .fn = kasan_handler,
979 .imm = KASAN_BRK_IMM,
980 .mask = KASAN_BRK_MASK,
981 };
982 #endif
983
984 /*
985 * Initial handler for AArch64 BRK exceptions
986 * This handler only used until debug_traps_init().
987 */
988 int __init early_brk64(unsigned long addr, unsigned int esr,
989 struct pt_regs *regs)
990 {
991 #ifdef CONFIG_KASAN_SW_TAGS
992 unsigned int comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK;
993
994 if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
995 return kasan_handler(regs, esr) != DBG_HOOK_HANDLED;
996 #endif
997 return bug_handler(regs, esr) != DBG_HOOK_HANDLED;
998 }
999
1000 void __init trap_init(void)
1001 {
1002 register_kernel_break_hook(&bug_break_hook);
1003 register_kernel_break_hook(&fault_break_hook);
1004 #ifdef CONFIG_KASAN_SW_TAGS
1005 register_kernel_break_hook(&kasan_break_hook);
1006 #endif
1007 debug_traps_init();
1008 }
Linux with added FPC-III support