arch/arm64/kernel/entry-common.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Exception handling code
   4  *
   5  * Copyright (C) 2019 ARM Ltd.
   6  */
   7
   8 #include <linux/context_tracking.h>
   9 #include <linux/ptrace.h>
  10 #include <linux/thread_info.h>
  11
  12 #include <asm/cpufeature.h>
  13 #include <asm/daifflags.h>
  14 #include <asm/esr.h>
  15 #include <asm/exception.h>
  16 #include <asm/kprobes.h>
  17 #include <asm/mmu.h>
  18 #include <asm/sysreg.h>
  19
  20 /*
  21  * This is intended to match the logic in irqentry_enter(), handling the kernel
  22  * mode transitions only.
  23  */
  24 static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
  25 {
  26         regs->exit_rcu = false;
  27
  28         if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
  29                 lockdep_hardirqs_off(CALLER_ADDR0);
  30                 rcu_irq_enter();
  31                 trace_hardirqs_off_finish();
  32
  33                 regs->exit_rcu = true;
  34                 return;
  35         }
  36
  37         lockdep_hardirqs_off(CALLER_ADDR0);
  38         rcu_irq_enter_check_tick();
  39         trace_hardirqs_off_finish();
  40 }
  41
  42 /*
  43  * This is intended to match the logic in irqentry_exit(), handling the kernel
  44  * mode transitions only, and with preemption handled elsewhere.
  45  */
  46 static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
  47 {
  48         lockdep_assert_irqs_disabled();
  49
  50         if (interrupts_enabled(regs)) {
  51                 if (regs->exit_rcu) {
  52                         trace_hardirqs_on_prepare();
  53                         lockdep_hardirqs_on_prepare(CALLER_ADDR0);
  54                         rcu_irq_exit();
  55                         lockdep_hardirqs_on(CALLER_ADDR0);
  56                         return;
  57                 }
  58
  59                 trace_hardirqs_on();
  60         } else {
  61                 if (regs->exit_rcu)
  62                         rcu_irq_exit();
  63         }
  64 }
  65
  66 void noinstr arm64_enter_nmi(struct pt_regs *regs)
  67 {
  68         regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
  69
  70         __nmi_enter();
  71         lockdep_hardirqs_off(CALLER_ADDR0);
  72         lockdep_hardirq_enter();
  73         rcu_nmi_enter();
  74
  75         trace_hardirqs_off_finish();
  76         ftrace_nmi_enter();
  77 }
  78
  79 void noinstr arm64_exit_nmi(struct pt_regs *regs)
  80 {
  81         bool restore = regs->lockdep_hardirqs;
  82
  83         ftrace_nmi_exit();
  84         if (restore) {
  85                 trace_hardirqs_on_prepare();
  86                 lockdep_hardirqs_on_prepare(CALLER_ADDR0);
  87         }
  88
  89         rcu_nmi_exit();
  90         lockdep_hardirq_exit();
  91         if (restore)
  92                 lockdep_hardirqs_on(CALLER_ADDR0);
  93         __nmi_exit();
  94 }
  95
  96 asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)
  97 {
  98         if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
  99                 arm64_enter_nmi(regs);
 100         else
 101                 enter_from_kernel_mode(regs);
 102 }
 103
 104 asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)
 105 {
 106         if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
 107                 arm64_exit_nmi(regs);
 108         else
 109                 exit_to_kernel_mode(regs);
 110 }
 111
 112 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
 113 {
 114         unsigned long far = read_sysreg(far_el1);
 115
 116         enter_from_kernel_mode(regs);
 117         local_daif_inherit(regs);
 118         do_mem_abort(far, esr, regs);
 119         local_daif_mask();
 120         exit_to_kernel_mode(regs);
 121 }
 122
 123 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
 124 {
 125         unsigned long far = read_sysreg(far_el1);
 126
 127         enter_from_kernel_mode(regs);
 128         local_daif_inherit(regs);
 129         do_sp_pc_abort(far, esr, regs);
 130         local_daif_mask();
 131         exit_to_kernel_mode(regs);
 132 }
 133
 134 static void noinstr el1_undef(struct pt_regs *regs)
 135 {
 136         enter_from_kernel_mode(regs);
 137         local_daif_inherit(regs);
 138         do_undefinstr(regs);
 139         local_daif_mask();
 140         exit_to_kernel_mode(regs);
 141 }
 142
 143 static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr)
 144 {
 145         enter_from_kernel_mode(regs);
 146         local_daif_inherit(regs);
 147         bad_mode(regs, 0, esr);
 148         local_daif_mask();
 149         exit_to_kernel_mode(regs);
 150 }
 151
 152 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)
 153 {
 154         regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
 155
 156         lockdep_hardirqs_off(CALLER_ADDR0);
 157         rcu_nmi_enter();
 158
 159         trace_hardirqs_off_finish();
 160 }
 161
 162 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
 163 {
 164         bool restore = regs->lockdep_hardirqs;
 165
 166         if (restore) {
 167                 trace_hardirqs_on_prepare();
 168                 lockdep_hardirqs_on_prepare(CALLER_ADDR0);
 169         }
 170
 171         rcu_nmi_exit();
 172         if (restore)
 173                 lockdep_hardirqs_on(CALLER_ADDR0);
 174 }
 175
 176 static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
 177 {
 178         unsigned long far = read_sysreg(far_el1);
 179
 180         /*
 181          * The CPU masked interrupts, and we are leaving them masked during
 182          * do_debug_exception(). Update PMR as if we had called
 183          * local_daif_mask().
 184          */
 185         if (system_uses_irq_prio_masking())
 186                 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
 187
 188         arm64_enter_el1_dbg(regs);
 189         do_debug_exception(far, esr, regs);
 190         arm64_exit_el1_dbg(regs);
 191 }
 192
 193 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
 194 {
 195         enter_from_kernel_mode(regs);
 196         local_daif_inherit(regs);
 197         do_ptrauth_fault(regs, esr);
 198         local_daif_mask();
 199         exit_to_kernel_mode(regs);
 200 }
 201
 202 asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs)
 203 {
 204         unsigned long esr = read_sysreg(esr_el1);
 205
 206         switch (ESR_ELx_EC(esr)) {
 207         case ESR_ELx_EC_DABT_CUR:
 208         case ESR_ELx_EC_IABT_CUR:
 209                 el1_abort(regs, esr);
 210                 break;
 211         /*
 212          * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a
 213          * recursive exception when trying to push the initial pt_regs.
 214          */
 215         case ESR_ELx_EC_PC_ALIGN:
 216                 el1_pc(regs, esr);
 217                 break;
 218         case ESR_ELx_EC_SYS64:
 219         case ESR_ELx_EC_UNKNOWN:
 220                 el1_undef(regs);
 221                 break;
 222         case ESR_ELx_EC_BREAKPT_CUR:
 223         case ESR_ELx_EC_SOFTSTP_CUR:
 224         case ESR_ELx_EC_WATCHPT_CUR:
 225         case ESR_ELx_EC_BRK64:
 226                 el1_dbg(regs, esr);
 227                 break;
 228         case ESR_ELx_EC_FPAC:
 229                 el1_fpac(regs, esr);
 230                 break;
 231         default:
 232                 el1_inv(regs, esr);
 233         }
 234 }
 235
 236 asmlinkage void noinstr enter_from_user_mode(void)
 237 {
 238         lockdep_hardirqs_off(CALLER_ADDR0);
 239         CT_WARN_ON(ct_state() != CONTEXT_USER);
 240         user_exit_irqoff();
 241         trace_hardirqs_off_finish();
 242 }
 243
 244 asmlinkage void noinstr exit_to_user_mode(void)
 245 {
 246         trace_hardirqs_on_prepare();
 247         lockdep_hardirqs_on_prepare(CALLER_ADDR0);
 248         user_enter_irqoff();
 249         lockdep_hardirqs_on(CALLER_ADDR0);
 250 }
 251
 252 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
 253 {
 254         unsigned long far = read_sysreg(far_el1);
 255
 256         enter_from_user_mode();
 257         local_daif_restore(DAIF_PROCCTX);
 258         do_mem_abort(far, esr, regs);
 259 }
 260
 261 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
 262 {
 263         unsigned long far = read_sysreg(far_el1);
 264
 265         /*
 266          * We've taken an instruction abort from userspace and not yet
 267          * re-enabled IRQs. If the address is a kernel address, apply
 268          * BP hardening prior to enabling IRQs and pre-emption.
 269          */
 270         if (!is_ttbr0_addr(far))
 271                 arm64_apply_bp_hardening();
 272
 273         enter_from_user_mode();
 274         local_daif_restore(DAIF_PROCCTX);
 275         do_mem_abort(far, esr, regs);
 276 }
 277
 278 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
 279 {
 280         enter_from_user_mode();
 281         local_daif_restore(DAIF_PROCCTX);
 282         do_fpsimd_acc(esr, regs);
 283 }
 284
 285 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
 286 {
 287         enter_from_user_mode();
 288         local_daif_restore(DAIF_PROCCTX);
 289         do_sve_acc(esr, regs);
 290 }
 291
 292 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
 293 {
 294         enter_from_user_mode();
 295         local_daif_restore(DAIF_PROCCTX);
 296         do_fpsimd_exc(esr, regs);
 297 }
 298
 299 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
 300 {
 301         enter_from_user_mode();
 302         local_daif_restore(DAIF_PROCCTX);
 303         do_sysinstr(esr, regs);
 304 }
 305
 306 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
 307 {
 308         unsigned long far = read_sysreg(far_el1);
 309
 310         if (!is_ttbr0_addr(instruction_pointer(regs)))
 311                 arm64_apply_bp_hardening();
 312
 313         enter_from_user_mode();
 314         local_daif_restore(DAIF_PROCCTX);
 315         do_sp_pc_abort(far, esr, regs);
 316 }
 317
 318 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
 319 {
 320         enter_from_user_mode();
 321         local_daif_restore(DAIF_PROCCTX);
 322         do_sp_pc_abort(regs->sp, esr, regs);
 323 }
 324
 325 static void noinstr el0_undef(struct pt_regs *regs)
 326 {
 327         enter_from_user_mode();
 328         local_daif_restore(DAIF_PROCCTX);
 329         do_undefinstr(regs);
 330 }
 331
 332 static void noinstr el0_bti(struct pt_regs *regs)
 333 {
 334         enter_from_user_mode();
 335         local_daif_restore(DAIF_PROCCTX);
 336         do_bti(regs);
 337 }
 338
 339 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
 340 {
 341         enter_from_user_mode();
 342         local_daif_restore(DAIF_PROCCTX);
 343         bad_el0_sync(regs, 0, esr);
 344 }
 345
 346 static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
 347 {
 348         /* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */
 349         unsigned long far = read_sysreg(far_el1);
 350
 351         if (system_uses_irq_prio_masking())
 352                 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
 353
 354         enter_from_user_mode();
 355         do_debug_exception(far, esr, regs);
 356         local_daif_restore(DAIF_PROCCTX_NOIRQ);
 357 }
 358
 359 static void noinstr el0_svc(struct pt_regs *regs)
 360 {
 361         if (system_uses_irq_prio_masking())
 362                 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
 363
 364         enter_from_user_mode();
 365         do_el0_svc(regs);
 366 }
 367
 368 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
 369 {
 370         enter_from_user_mode();
 371         local_daif_restore(DAIF_PROCCTX);
 372         do_ptrauth_fault(regs, esr);
 373 }
 374
 375 asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs)
 376 {
 377         unsigned long esr = read_sysreg(esr_el1);
 378
 379         switch (ESR_ELx_EC(esr)) {
 380         case ESR_ELx_EC_SVC64:
 381                 el0_svc(regs);
 382                 break;
 383         case ESR_ELx_EC_DABT_LOW:
 384                 el0_da(regs, esr);
 385                 break;
 386         case ESR_ELx_EC_IABT_LOW:
 387                 el0_ia(regs, esr);
 388                 break;
 389         case ESR_ELx_EC_FP_ASIMD:
 390                 el0_fpsimd_acc(regs, esr);
 391                 break;
 392         case ESR_ELx_EC_SVE:
 393                 el0_sve_acc(regs, esr);
 394                 break;
 395         case ESR_ELx_EC_FP_EXC64:
 396                 el0_fpsimd_exc(regs, esr);
 397                 break;
 398         case ESR_ELx_EC_SYS64:
 399         case ESR_ELx_EC_WFx:
 400                 el0_sys(regs, esr);
 401                 break;
 402         case ESR_ELx_EC_SP_ALIGN:
 403                 el0_sp(regs, esr);
 404                 break;
 405         case ESR_ELx_EC_PC_ALIGN:
 406                 el0_pc(regs, esr);
 407                 break;
 408         case ESR_ELx_EC_UNKNOWN:
 409                 el0_undef(regs);
 410                 break;
 411         case ESR_ELx_EC_BTI:
 412                 el0_bti(regs);
 413                 break;
 414         case ESR_ELx_EC_BREAKPT_LOW:
 415         case ESR_ELx_EC_SOFTSTP_LOW:
 416         case ESR_ELx_EC_WATCHPT_LOW:
 417         case ESR_ELx_EC_BRK64:
 418                 el0_dbg(regs, esr);
 419                 break;
 420         case ESR_ELx_EC_FPAC:
 421                 el0_fpac(regs, esr);
 422                 break;
 423         default:
 424                 el0_inv(regs, esr);
 425         }
 426 }
 427
 428 #ifdef CONFIG_COMPAT
 429 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
 430 {
 431         enter_from_user_mode();
 432         local_daif_restore(DAIF_PROCCTX);
 433         do_cp15instr(esr, regs);
 434 }
 435
 436 static void noinstr el0_svc_compat(struct pt_regs *regs)
 437 {
 438         if (system_uses_irq_prio_masking())
 439                 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
 440
 441         enter_from_user_mode();
 442         do_el0_svc_compat(regs);
 443 }
 444
 445 asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)
 446 {
 447         unsigned long esr = read_sysreg(esr_el1);
 448
 449         switch (ESR_ELx_EC(esr)) {
 450         case ESR_ELx_EC_SVC32:
 451                 el0_svc_compat(regs);
 452                 break;
 453         case ESR_ELx_EC_DABT_LOW:
 454                 el0_da(regs, esr);
 455                 break;
 456         case ESR_ELx_EC_IABT_LOW:
 457                 el0_ia(regs, esr);
 458                 break;
 459         case ESR_ELx_EC_FP_ASIMD:
 460                 el0_fpsimd_acc(regs, esr);
 461                 break;
 462         case ESR_ELx_EC_FP_EXC32:
 463                 el0_fpsimd_exc(regs, esr);
 464                 break;
 465         case ESR_ELx_EC_PC_ALIGN:
 466                 el0_pc(regs, esr);
 467                 break;
 468         case ESR_ELx_EC_UNKNOWN:
 469         case ESR_ELx_EC_CP14_MR:
 470         case ESR_ELx_EC_CP14_LS:
 471         case ESR_ELx_EC_CP14_64:
 472                 el0_undef(regs);
 473                 break;
 474         case ESR_ELx_EC_CP15_32:
 475         case ESR_ELx_EC_CP15_64:
 476                 el0_cp15(regs, esr);
 477                 break;
 478         case ESR_ELx_EC_BREAKPT_LOW:
 479         case ESR_ELx_EC_SOFTSTP_LOW:
 480         case ESR_ELx_EC_WATCHPT_LOW:
 481         case ESR_ELx_EC_BKPT32:
 482                 el0_dbg(regs, esr);
 483                 break;
 484         default:
 485                 el0_inv(regs, esr);
 486         }
 487 }
 488 #endif /* CONFIG_COMPAT */