2 * Copyright (C) 2014 ARM Limited
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 #include <linux/init.h>
11 #include <linux/list.h>
12 #include <linux/perf_event.h>
13 #include <linux/sched.h>
14 #include <linux/slab.h>
15 #include <linux/sysctl.h>
17 #include <asm/alternative.h>
18 #include <asm/cpufeature.h>
20 #include <asm/opcodes.h>
21 #include <asm/sysreg.h>
22 #include <asm/system_misc.h>
23 #include <asm/traps.h>
24 #include <asm/uaccess.h>
25 #include <asm/cpufeature.h>
27 #define CREATE_TRACE_POINTS
28 #include "trace-events-emulation.h"
31 * The runtime support for deprecated instruction support can be in one of
32 * following three states -
35 * 1 = emulate (software emulation)
36 * 2 = hw (supported in hardware)
38 enum insn_emulation_mode
{
44 enum legacy_insn_status
{
49 struct insn_emulation_ops
{
51 enum legacy_insn_status status
;
52 struct undef_hook
*hooks
;
53 int (*set_hw_mode
)(bool enable
);
56 struct insn_emulation
{
57 struct list_head node
;
58 struct insn_emulation_ops
*ops
;
64 static LIST_HEAD(insn_emulation
);
65 static int nr_insn_emulated
;
66 static DEFINE_RAW_SPINLOCK(insn_emulation_lock
);
68 static void register_emulation_hooks(struct insn_emulation_ops
*ops
)
70 struct undef_hook
*hook
;
74 for (hook
= ops
->hooks
; hook
->instr_mask
; hook
++)
75 register_undef_hook(hook
);
77 pr_notice("Registered %s emulation handler\n", ops
->name
);
80 static void remove_emulation_hooks(struct insn_emulation_ops
*ops
)
82 struct undef_hook
*hook
;
86 for (hook
= ops
->hooks
; hook
->instr_mask
; hook
++)
87 unregister_undef_hook(hook
);
89 pr_notice("Removed %s emulation handler\n", ops
->name
);
92 static void enable_insn_hw_mode(void *data
)
94 struct insn_emulation
*insn
= (struct insn_emulation
*)data
;
95 if (insn
->ops
->set_hw_mode
)
96 insn
->ops
->set_hw_mode(true);
99 static void disable_insn_hw_mode(void *data
)
101 struct insn_emulation
*insn
= (struct insn_emulation
*)data
;
102 if (insn
->ops
->set_hw_mode
)
103 insn
->ops
->set_hw_mode(false);
106 /* Run set_hw_mode(mode) on all active CPUs */
107 static int run_all_cpu_set_hw_mode(struct insn_emulation
*insn
, bool enable
)
109 if (!insn
->ops
->set_hw_mode
)
112 on_each_cpu(enable_insn_hw_mode
, (void *)insn
, true);
114 on_each_cpu(disable_insn_hw_mode
, (void *)insn
, true);
119 * Run set_hw_mode for all insns on a starting CPU.
121 * 0 - If all the hooks ran successfully.
122 * -EINVAL - At least one hook is not supported by the CPU.
124 static int run_all_insn_set_hw_mode(unsigned long cpu
)
128 struct insn_emulation
*insn
;
130 raw_spin_lock_irqsave(&insn_emulation_lock
, flags
);
131 list_for_each_entry(insn
, &insn_emulation
, node
) {
132 bool enable
= (insn
->current_mode
== INSN_HW
);
133 if (insn
->ops
->set_hw_mode
&& insn
->ops
->set_hw_mode(enable
)) {
134 pr_warn("CPU[%ld] cannot support the emulation of %s",
135 cpu
, insn
->ops
->name
);
139 raw_spin_unlock_irqrestore(&insn_emulation_lock
, flags
);
143 static int update_insn_emulation_mode(struct insn_emulation
*insn
,
144 enum insn_emulation_mode prev
)
149 case INSN_UNDEF
: /* Nothing to be done */
152 remove_emulation_hooks(insn
->ops
);
155 if (!run_all_cpu_set_hw_mode(insn
, false))
156 pr_notice("Disabled %s support\n", insn
->ops
->name
);
160 switch (insn
->current_mode
) {
164 register_emulation_hooks(insn
->ops
);
167 ret
= run_all_cpu_set_hw_mode(insn
, true);
169 pr_notice("Enabled %s support\n", insn
->ops
->name
);
176 static void register_insn_emulation(struct insn_emulation_ops
*ops
)
179 struct insn_emulation
*insn
;
181 insn
= kzalloc(sizeof(*insn
), GFP_KERNEL
);
183 insn
->min
= INSN_UNDEF
;
185 switch (ops
->status
) {
186 case INSN_DEPRECATED
:
187 insn
->current_mode
= INSN_EMULATE
;
188 /* Disable the HW mode if it was turned on at early boot time */
189 run_all_cpu_set_hw_mode(insn
, false);
193 insn
->current_mode
= INSN_UNDEF
;
194 insn
->max
= INSN_EMULATE
;
198 raw_spin_lock_irqsave(&insn_emulation_lock
, flags
);
199 list_add(&insn
->node
, &insn_emulation
);
201 raw_spin_unlock_irqrestore(&insn_emulation_lock
, flags
);
203 /* Register any handlers if required */
204 update_insn_emulation_mode(insn
, INSN_UNDEF
);
207 static int emulation_proc_handler(struct ctl_table
*table
, int write
,
208 void __user
*buffer
, size_t *lenp
,
212 struct insn_emulation
*insn
= (struct insn_emulation
*) table
->data
;
213 enum insn_emulation_mode prev_mode
= insn
->current_mode
;
215 table
->data
= &insn
->current_mode
;
216 ret
= proc_dointvec_minmax(table
, write
, buffer
, lenp
, ppos
);
218 if (ret
|| !write
|| prev_mode
== insn
->current_mode
)
221 ret
= update_insn_emulation_mode(insn
, prev_mode
);
223 /* Mode change failed, revert to previous mode. */
224 insn
->current_mode
= prev_mode
;
225 update_insn_emulation_mode(insn
, INSN_UNDEF
);
232 static struct ctl_table ctl_abi
[] = {
240 static void register_insn_emulation_sysctl(struct ctl_table
*table
)
244 struct insn_emulation
*insn
;
245 struct ctl_table
*insns_sysctl
, *sysctl
;
247 insns_sysctl
= kzalloc(sizeof(*sysctl
) * (nr_insn_emulated
+ 1),
250 raw_spin_lock_irqsave(&insn_emulation_lock
, flags
);
251 list_for_each_entry(insn
, &insn_emulation
, node
) {
252 sysctl
= &insns_sysctl
[i
];
255 sysctl
->maxlen
= sizeof(int);
257 sysctl
->procname
= insn
->ops
->name
;
259 sysctl
->extra1
= &insn
->min
;
260 sysctl
->extra2
= &insn
->max
;
261 sysctl
->proc_handler
= emulation_proc_handler
;
264 raw_spin_unlock_irqrestore(&insn_emulation_lock
, flags
);
266 table
->child
= insns_sysctl
;
267 register_sysctl_table(table
);
271 * Implement emulation of the SWP/SWPB instructions using load-exclusive and
274 * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
275 * Where: Rt = destination
281 * Error-checking SWP macros implemented using ldxr{b}/stxr{b}
283 #define __user_swpX_asm(data, addr, res, temp, B) \
284 __asm__ __volatile__( \
285 ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \
288 "0: ldxr"B" %w1, [%3]\n" \
289 "1: stxr"B" %w0, %w2, [%3]\n" \
293 " .pushsection .fixup,\"ax\"\n" \
295 "3: mov %w0, %w5\n" \
298 " .pushsection __ex_table,\"a\"\n" \
303 ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \
305 : "=&r" (res), "+r" (data), "=&r" (temp) \
306 : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \
309 #define __user_swp_asm(data, addr, res, temp) \
310 __user_swpX_asm(data, addr, res, temp, "")
311 #define __user_swpb_asm(data, addr, res, temp) \
312 __user_swpX_asm(data, addr, res, temp, "b")
315 * Bit 22 of the instruction encoding distinguishes between
316 * the SWP and SWPB variants (bit set means SWPB).
318 #define TYPE_SWPB (1 << 22)
321 * Set up process info to signal segmentation fault - called on access error.
323 static void set_segfault(struct pt_regs
*regs
, unsigned long addr
)
327 down_read(¤t
->mm
->mmap_sem
);
328 if (find_vma(current
->mm
, addr
) == NULL
)
329 info
.si_code
= SEGV_MAPERR
;
331 info
.si_code
= SEGV_ACCERR
;
332 up_read(¤t
->mm
->mmap_sem
);
334 info
.si_signo
= SIGSEGV
;
336 info
.si_addr
= (void *) instruction_pointer(regs
);
338 pr_debug("SWP{B} emulation: access caused memory abort!\n");
339 arm64_notify_die("Illegal memory access", regs
, &info
, 0);
342 static int emulate_swpX(unsigned int address
, unsigned int *data
,
345 unsigned int res
= 0;
347 if ((type
!= TYPE_SWPB
) && (address
& 0x3)) {
348 /* SWP to unaligned address not permitted */
349 pr_debug("SWP instruction on unaligned pointer!\n");
356 if (type
== TYPE_SWPB
)
357 __user_swpb_asm(*data
, address
, res
, temp
);
359 __user_swp_asm(*data
, address
, res
, temp
);
361 if (likely(res
!= -EAGAIN
) || signal_pending(current
))
371 * swp_handler logs the id of calling process, dissects the instruction, sanity
372 * checks the memory location, calls emulate_swpX for the actual operation and
373 * deals with fixup/error handling before returning
375 static int swp_handler(struct pt_regs
*regs
, u32 instr
)
377 u32 destreg
, data
, type
, address
= 0;
378 int rn
, rt2
, res
= 0;
380 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, regs
->pc
);
382 type
= instr
& TYPE_SWPB
;
384 switch (arm_check_condition(instr
, regs
->pstate
)) {
385 case ARM_OPCODE_CONDTEST_PASS
:
387 case ARM_OPCODE_CONDTEST_FAIL
:
388 /* Condition failed - return to next instruction */
390 case ARM_OPCODE_CONDTEST_UNCOND
:
391 /* If unconditional encoding - not a SWP, undef */
397 rn
= aarch32_insn_extract_reg_num(instr
, A32_RN_OFFSET
);
398 rt2
= aarch32_insn_extract_reg_num(instr
, A32_RT2_OFFSET
);
400 address
= (u32
)regs
->user_regs
.regs
[rn
];
401 data
= (u32
)regs
->user_regs
.regs
[rt2
];
402 destreg
= aarch32_insn_extract_reg_num(instr
, A32_RT_OFFSET
);
404 pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
405 rn
, address
, destreg
,
406 aarch32_insn_extract_reg_num(instr
, A32_RT2_OFFSET
), data
);
408 /* Check access in reasonable access range for both SWP and SWPB */
409 if (!access_ok(VERIFY_WRITE
, (address
& ~3), 4)) {
410 pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
415 res
= emulate_swpX(address
, &data
, type
);
419 regs
->user_regs
.regs
[destreg
] = data
;
422 if (type
== TYPE_SWPB
)
423 trace_instruction_emulation("swpb", regs
->pc
);
425 trace_instruction_emulation("swp", regs
->pc
);
427 pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
428 current
->comm
, (unsigned long)current
->pid
, regs
->pc
);
434 set_segfault(regs
, address
);
440 * Only emulate SWP/SWPB executed in ARM state/User mode.
441 * The kernel must be SWP free and SWP{B} does not exist in Thumb.
443 static struct undef_hook swp_hooks
[] = {
445 .instr_mask
= 0x0fb00ff0,
446 .instr_val
= 0x01000090,
447 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
448 .pstate_val
= COMPAT_PSR_MODE_USR
,
454 static struct insn_emulation_ops swp_ops
= {
456 .status
= INSN_OBSOLETE
,
461 static int cp15barrier_handler(struct pt_regs
*regs
, u32 instr
)
463 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, regs
->pc
);
465 switch (arm_check_condition(instr
, regs
->pstate
)) {
466 case ARM_OPCODE_CONDTEST_PASS
:
468 case ARM_OPCODE_CONDTEST_FAIL
:
469 /* Condition failed - return to next instruction */
471 case ARM_OPCODE_CONDTEST_UNCOND
:
472 /* If unconditional encoding - not a barrier instruction */
478 switch (aarch32_insn_mcr_extract_crm(instr
)) {
481 * dmb - mcr p15, 0, Rt, c7, c10, 5
482 * dsb - mcr p15, 0, Rt, c7, c10, 4
484 if (aarch32_insn_mcr_extract_opc2(instr
) == 5) {
486 trace_instruction_emulation(
487 "mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs
->pc
);
490 trace_instruction_emulation(
491 "mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs
->pc
);
496 * isb - mcr p15, 0, Rt, c7, c5, 4
498 * Taking an exception or returning from one acts as an
499 * instruction barrier. So no explicit barrier needed here.
501 trace_instruction_emulation(
502 "mcr p15, 0, Rt, c7, c5, 4 ; isb", regs
->pc
);
507 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
508 current
->comm
, (unsigned long)current
->pid
, regs
->pc
);
514 static int cp15_barrier_set_hw_mode(bool enable
)
517 config_sctlr_el1(0, SCTLR_EL1_CP15BEN
);
519 config_sctlr_el1(SCTLR_EL1_CP15BEN
, 0);
523 static struct undef_hook cp15_barrier_hooks
[] = {
525 .instr_mask
= 0x0fff0fdf,
526 .instr_val
= 0x0e070f9a,
527 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
528 .pstate_val
= COMPAT_PSR_MODE_USR
,
529 .fn
= cp15barrier_handler
,
532 .instr_mask
= 0x0fff0fff,
533 .instr_val
= 0x0e070f95,
534 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
535 .pstate_val
= COMPAT_PSR_MODE_USR
,
536 .fn
= cp15barrier_handler
,
541 static struct insn_emulation_ops cp15_barrier_ops
= {
542 .name
= "cp15_barrier",
543 .status
= INSN_DEPRECATED
,
544 .hooks
= cp15_barrier_hooks
,
545 .set_hw_mode
= cp15_barrier_set_hw_mode
,
548 static int setend_set_hw_mode(bool enable
)
550 if (!cpu_supports_mixed_endian_el0())
554 config_sctlr_el1(SCTLR_EL1_SED
, 0);
556 config_sctlr_el1(0, SCTLR_EL1_SED
);
560 static int compat_setend_handler(struct pt_regs
*regs
, u32 big_endian
)
564 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, regs
->pc
);
568 regs
->pstate
|= COMPAT_PSR_E_BIT
;
571 regs
->pstate
&= ~COMPAT_PSR_E_BIT
;
574 trace_instruction_emulation(insn
, regs
->pc
);
575 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
576 current
->comm
, (unsigned long)current
->pid
, regs
->pc
);
581 static int a32_setend_handler(struct pt_regs
*regs
, u32 instr
)
583 int rc
= compat_setend_handler(regs
, (instr
>> 9) & 1);
588 static int t16_setend_handler(struct pt_regs
*regs
, u32 instr
)
590 int rc
= compat_setend_handler(regs
, (instr
>> 3) & 1);
595 static struct undef_hook setend_hooks
[] = {
597 .instr_mask
= 0xfffffdff,
598 .instr_val
= 0xf1010000,
599 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
600 .pstate_val
= COMPAT_PSR_MODE_USR
,
601 .fn
= a32_setend_handler
,
605 .instr_mask
= 0x0000fff7,
606 .instr_val
= 0x0000b650,
607 .pstate_mask
= (COMPAT_PSR_T_BIT
| COMPAT_PSR_MODE_MASK
),
608 .pstate_val
= (COMPAT_PSR_T_BIT
| COMPAT_PSR_MODE_USR
),
609 .fn
= t16_setend_handler
,
614 static struct insn_emulation_ops setend_ops
= {
616 .status
= INSN_DEPRECATED
,
617 .hooks
= setend_hooks
,
618 .set_hw_mode
= setend_set_hw_mode
,
621 static int insn_cpu_hotplug_notify(struct notifier_block
*b
,
622 unsigned long action
, void *hcpu
)
625 if ((action
& ~CPU_TASKS_FROZEN
) == CPU_STARTING
)
626 rc
= run_all_insn_set_hw_mode((unsigned long)hcpu
);
628 return notifier_from_errno(rc
);
631 static struct notifier_block insn_cpu_hotplug_notifier
= {
632 .notifier_call
= insn_cpu_hotplug_notify
,
636 * Invoked as late_initcall, since not needed before init spawned.
638 static int __init
armv8_deprecated_init(void)
640 if (IS_ENABLED(CONFIG_SWP_EMULATION
))
641 register_insn_emulation(&swp_ops
);
643 if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION
))
644 register_insn_emulation(&cp15_barrier_ops
);
646 if (IS_ENABLED(CONFIG_SETEND_EMULATION
)) {
647 if(system_supports_mixed_endian_el0())
648 register_insn_emulation(&setend_ops
);
650 pr_info("setend instruction emulation is not supported on the system");
653 register_cpu_notifier(&insn_cpu_hotplug_notifier
);
654 register_insn_emulation_sysctl(ctl_abi
);
659 late_initcall(armv8_deprecated_init
);