2 * Copyright (C) 2014 ARM Limited
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 #include <linux/init.h>
11 #include <linux/list.h>
12 #include <linux/perf_event.h>
13 #include <linux/sched.h>
14 #include <linux/slab.h>
15 #include <linux/sysctl.h>
17 #include <asm/alternative.h>
18 #include <asm/cpufeature.h>
20 #include <asm/opcodes.h>
21 #include <asm/sysreg.h>
22 #include <asm/system_misc.h>
23 #include <asm/traps.h>
24 #include <asm/uaccess.h>
25 #include <asm/cpufeature.h>
27 #define CREATE_TRACE_POINTS
28 #include "trace-events-emulation.h"
31 * The runtime support for deprecated instruction support can be in one of
32 * following three states -
35 * 1 = emulate (software emulation)
36 * 2 = hw (supported in hardware)
38 enum insn_emulation_mode
{
44 enum legacy_insn_status
{
49 struct insn_emulation_ops
{
51 enum legacy_insn_status status
;
52 struct undef_hook
*hooks
;
53 int (*set_hw_mode
)(bool enable
);
56 struct insn_emulation
{
57 struct list_head node
;
58 struct insn_emulation_ops
*ops
;
64 static LIST_HEAD(insn_emulation
);
65 static int nr_insn_emulated __initdata
;
66 static DEFINE_RAW_SPINLOCK(insn_emulation_lock
);
68 static void register_emulation_hooks(struct insn_emulation_ops
*ops
)
70 struct undef_hook
*hook
;
74 for (hook
= ops
->hooks
; hook
->instr_mask
; hook
++)
75 register_undef_hook(hook
);
77 pr_notice("Registered %s emulation handler\n", ops
->name
);
80 static void remove_emulation_hooks(struct insn_emulation_ops
*ops
)
82 struct undef_hook
*hook
;
86 for (hook
= ops
->hooks
; hook
->instr_mask
; hook
++)
87 unregister_undef_hook(hook
);
89 pr_notice("Removed %s emulation handler\n", ops
->name
);
92 static void enable_insn_hw_mode(void *data
)
94 struct insn_emulation
*insn
= (struct insn_emulation
*)data
;
95 if (insn
->ops
->set_hw_mode
)
96 insn
->ops
->set_hw_mode(true);
99 static void disable_insn_hw_mode(void *data
)
101 struct insn_emulation
*insn
= (struct insn_emulation
*)data
;
102 if (insn
->ops
->set_hw_mode
)
103 insn
->ops
->set_hw_mode(false);
106 /* Run set_hw_mode(mode) on all active CPUs */
107 static int run_all_cpu_set_hw_mode(struct insn_emulation
*insn
, bool enable
)
109 if (!insn
->ops
->set_hw_mode
)
112 on_each_cpu(enable_insn_hw_mode
, (void *)insn
, true);
114 on_each_cpu(disable_insn_hw_mode
, (void *)insn
, true);
119 * Run set_hw_mode for all insns on a starting CPU.
121 * 0 - If all the hooks ran successfully.
122 * -EINVAL - At least one hook is not supported by the CPU.
124 static int run_all_insn_set_hw_mode(unsigned long cpu
)
128 struct insn_emulation
*insn
;
130 raw_spin_lock_irqsave(&insn_emulation_lock
, flags
);
131 list_for_each_entry(insn
, &insn_emulation
, node
) {
132 bool enable
= (insn
->current_mode
== INSN_HW
);
133 if (insn
->ops
->set_hw_mode
&& insn
->ops
->set_hw_mode(enable
)) {
134 pr_warn("CPU[%ld] cannot support the emulation of %s",
135 cpu
, insn
->ops
->name
);
139 raw_spin_unlock_irqrestore(&insn_emulation_lock
, flags
);
143 static int update_insn_emulation_mode(struct insn_emulation
*insn
,
144 enum insn_emulation_mode prev
)
149 case INSN_UNDEF
: /* Nothing to be done */
152 remove_emulation_hooks(insn
->ops
);
155 if (!run_all_cpu_set_hw_mode(insn
, false))
156 pr_notice("Disabled %s support\n", insn
->ops
->name
);
160 switch (insn
->current_mode
) {
164 register_emulation_hooks(insn
->ops
);
167 ret
= run_all_cpu_set_hw_mode(insn
, true);
169 pr_notice("Enabled %s support\n", insn
->ops
->name
);
176 static void __init
register_insn_emulation(struct insn_emulation_ops
*ops
)
179 struct insn_emulation
*insn
;
181 insn
= kzalloc(sizeof(*insn
), GFP_KERNEL
);
183 insn
->min
= INSN_UNDEF
;
185 switch (ops
->status
) {
186 case INSN_DEPRECATED
:
187 insn
->current_mode
= INSN_EMULATE
;
188 /* Disable the HW mode if it was turned on at early boot time */
189 run_all_cpu_set_hw_mode(insn
, false);
193 insn
->current_mode
= INSN_UNDEF
;
194 insn
->max
= INSN_EMULATE
;
198 raw_spin_lock_irqsave(&insn_emulation_lock
, flags
);
199 list_add(&insn
->node
, &insn_emulation
);
201 raw_spin_unlock_irqrestore(&insn_emulation_lock
, flags
);
203 /* Register any handlers if required */
204 update_insn_emulation_mode(insn
, INSN_UNDEF
);
207 static int emulation_proc_handler(struct ctl_table
*table
, int write
,
208 void __user
*buffer
, size_t *lenp
,
212 struct insn_emulation
*insn
= (struct insn_emulation
*) table
->data
;
213 enum insn_emulation_mode prev_mode
= insn
->current_mode
;
215 table
->data
= &insn
->current_mode
;
216 ret
= proc_dointvec_minmax(table
, write
, buffer
, lenp
, ppos
);
218 if (ret
|| !write
|| prev_mode
== insn
->current_mode
)
221 ret
= update_insn_emulation_mode(insn
, prev_mode
);
223 /* Mode change failed, revert to previous mode. */
224 insn
->current_mode
= prev_mode
;
225 update_insn_emulation_mode(insn
, INSN_UNDEF
);
232 static struct ctl_table ctl_abi
[] = {
240 static void __init
register_insn_emulation_sysctl(struct ctl_table
*table
)
244 struct insn_emulation
*insn
;
245 struct ctl_table
*insns_sysctl
, *sysctl
;
247 insns_sysctl
= kzalloc(sizeof(*sysctl
) * (nr_insn_emulated
+ 1),
250 raw_spin_lock_irqsave(&insn_emulation_lock
, flags
);
251 list_for_each_entry(insn
, &insn_emulation
, node
) {
252 sysctl
= &insns_sysctl
[i
];
255 sysctl
->maxlen
= sizeof(int);
257 sysctl
->procname
= insn
->ops
->name
;
259 sysctl
->extra1
= &insn
->min
;
260 sysctl
->extra2
= &insn
->max
;
261 sysctl
->proc_handler
= emulation_proc_handler
;
264 raw_spin_unlock_irqrestore(&insn_emulation_lock
, flags
);
266 table
->child
= insns_sysctl
;
267 register_sysctl_table(table
);
271 * Implement emulation of the SWP/SWPB instructions using load-exclusive and
274 * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
275 * Where: Rt = destination
281 * Error-checking SWP macros implemented using ldxr{b}/stxr{b}
283 #define __user_swpX_asm(data, addr, res, temp, B) \
284 __asm__ __volatile__( \
285 ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \
287 "0: ldxr"B" %w2, [%3]\n" \
288 "1: stxr"B" %w0, %w1, [%3]\n" \
295 " .pushsection .fixup,\"ax\"\n" \
297 "4: mov %w0, %w5\n" \
300 " .pushsection __ex_table,\"a\"\n" \
305 ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \
307 : "=&r" (res), "+r" (data), "=&r" (temp) \
308 : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \
311 #define __user_swp_asm(data, addr, res, temp) \
312 __user_swpX_asm(data, addr, res, temp, "")
313 #define __user_swpb_asm(data, addr, res, temp) \
314 __user_swpX_asm(data, addr, res, temp, "b")
317 * Bit 22 of the instruction encoding distinguishes between
318 * the SWP and SWPB variants (bit set means SWPB).
320 #define TYPE_SWPB (1 << 22)
323 * Set up process info to signal segmentation fault - called on access error.
325 static void set_segfault(struct pt_regs
*regs
, unsigned long addr
)
329 down_read(¤t
->mm
->mmap_sem
);
330 if (find_vma(current
->mm
, addr
) == NULL
)
331 info
.si_code
= SEGV_MAPERR
;
333 info
.si_code
= SEGV_ACCERR
;
334 up_read(¤t
->mm
->mmap_sem
);
336 info
.si_signo
= SIGSEGV
;
338 info
.si_addr
= (void *) instruction_pointer(regs
);
340 pr_debug("SWP{B} emulation: access caused memory abort!\n");
341 arm64_notify_die("Illegal memory access", regs
, &info
, 0);
344 static int emulate_swpX(unsigned int address
, unsigned int *data
,
347 unsigned int res
= 0;
349 if ((type
!= TYPE_SWPB
) && (address
& 0x3)) {
350 /* SWP to unaligned address not permitted */
351 pr_debug("SWP instruction on unaligned pointer!\n");
358 if (type
== TYPE_SWPB
)
359 __user_swpb_asm(*data
, address
, res
, temp
);
361 __user_swp_asm(*data
, address
, res
, temp
);
363 if (likely(res
!= -EAGAIN
) || signal_pending(current
))
373 * swp_handler logs the id of calling process, dissects the instruction, sanity
374 * checks the memory location, calls emulate_swpX for the actual operation and
375 * deals with fixup/error handling before returning
377 static int swp_handler(struct pt_regs
*regs
, u32 instr
)
379 u32 destreg
, data
, type
, address
= 0;
380 int rn
, rt2
, res
= 0;
382 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, regs
->pc
);
384 type
= instr
& TYPE_SWPB
;
386 switch (arm_check_condition(instr
, regs
->pstate
)) {
387 case ARM_OPCODE_CONDTEST_PASS
:
389 case ARM_OPCODE_CONDTEST_FAIL
:
390 /* Condition failed - return to next instruction */
392 case ARM_OPCODE_CONDTEST_UNCOND
:
393 /* If unconditional encoding - not a SWP, undef */
399 rn
= aarch32_insn_extract_reg_num(instr
, A32_RN_OFFSET
);
400 rt2
= aarch32_insn_extract_reg_num(instr
, A32_RT2_OFFSET
);
402 address
= (u32
)regs
->user_regs
.regs
[rn
];
403 data
= (u32
)regs
->user_regs
.regs
[rt2
];
404 destreg
= aarch32_insn_extract_reg_num(instr
, A32_RT_OFFSET
);
406 pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
407 rn
, address
, destreg
,
408 aarch32_insn_extract_reg_num(instr
, A32_RT2_OFFSET
), data
);
410 /* Check access in reasonable access range for both SWP and SWPB */
411 if (!access_ok(VERIFY_WRITE
, (address
& ~3), 4)) {
412 pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
417 res
= emulate_swpX(address
, &data
, type
);
421 regs
->user_regs
.regs
[destreg
] = data
;
424 if (type
== TYPE_SWPB
)
425 trace_instruction_emulation("swpb", regs
->pc
);
427 trace_instruction_emulation("swp", regs
->pc
);
429 pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
430 current
->comm
, (unsigned long)current
->pid
, regs
->pc
);
436 set_segfault(regs
, address
);
442 * Only emulate SWP/SWPB executed in ARM state/User mode.
443 * The kernel must be SWP free and SWP{B} does not exist in Thumb.
445 static struct undef_hook swp_hooks
[] = {
447 .instr_mask
= 0x0fb00ff0,
448 .instr_val
= 0x01000090,
449 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
450 .pstate_val
= COMPAT_PSR_MODE_USR
,
456 static struct insn_emulation_ops swp_ops
= {
458 .status
= INSN_OBSOLETE
,
463 static int cp15barrier_handler(struct pt_regs
*regs
, u32 instr
)
465 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, regs
->pc
);
467 switch (arm_check_condition(instr
, regs
->pstate
)) {
468 case ARM_OPCODE_CONDTEST_PASS
:
470 case ARM_OPCODE_CONDTEST_FAIL
:
471 /* Condition failed - return to next instruction */
473 case ARM_OPCODE_CONDTEST_UNCOND
:
474 /* If unconditional encoding - not a barrier instruction */
480 switch (aarch32_insn_mcr_extract_crm(instr
)) {
483 * dmb - mcr p15, 0, Rt, c7, c10, 5
484 * dsb - mcr p15, 0, Rt, c7, c10, 4
486 if (aarch32_insn_mcr_extract_opc2(instr
) == 5) {
488 trace_instruction_emulation(
489 "mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs
->pc
);
492 trace_instruction_emulation(
493 "mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs
->pc
);
498 * isb - mcr p15, 0, Rt, c7, c5, 4
500 * Taking an exception or returning from one acts as an
501 * instruction barrier. So no explicit barrier needed here.
503 trace_instruction_emulation(
504 "mcr p15, 0, Rt, c7, c5, 4 ; isb", regs
->pc
);
509 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
510 current
->comm
, (unsigned long)current
->pid
, regs
->pc
);
516 static int cp15_barrier_set_hw_mode(bool enable
)
519 config_sctlr_el1(0, SCTLR_EL1_CP15BEN
);
521 config_sctlr_el1(SCTLR_EL1_CP15BEN
, 0);
525 static struct undef_hook cp15_barrier_hooks
[] = {
527 .instr_mask
= 0x0fff0fdf,
528 .instr_val
= 0x0e070f9a,
529 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
530 .pstate_val
= COMPAT_PSR_MODE_USR
,
531 .fn
= cp15barrier_handler
,
534 .instr_mask
= 0x0fff0fff,
535 .instr_val
= 0x0e070f95,
536 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
537 .pstate_val
= COMPAT_PSR_MODE_USR
,
538 .fn
= cp15barrier_handler
,
543 static struct insn_emulation_ops cp15_barrier_ops
= {
544 .name
= "cp15_barrier",
545 .status
= INSN_DEPRECATED
,
546 .hooks
= cp15_barrier_hooks
,
547 .set_hw_mode
= cp15_barrier_set_hw_mode
,
550 static int setend_set_hw_mode(bool enable
)
552 if (!cpu_supports_mixed_endian_el0())
556 config_sctlr_el1(SCTLR_EL1_SED
, 0);
558 config_sctlr_el1(0, SCTLR_EL1_SED
);
562 static int compat_setend_handler(struct pt_regs
*regs
, u32 big_endian
)
566 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, regs
->pc
);
570 regs
->pstate
|= COMPAT_PSR_E_BIT
;
573 regs
->pstate
&= ~COMPAT_PSR_E_BIT
;
576 trace_instruction_emulation(insn
, regs
->pc
);
577 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
578 current
->comm
, (unsigned long)current
->pid
, regs
->pc
);
583 static int a32_setend_handler(struct pt_regs
*regs
, u32 instr
)
585 int rc
= compat_setend_handler(regs
, (instr
>> 9) & 1);
590 static int t16_setend_handler(struct pt_regs
*regs
, u32 instr
)
592 int rc
= compat_setend_handler(regs
, (instr
>> 3) & 1);
597 static struct undef_hook setend_hooks
[] = {
599 .instr_mask
= 0xfffffdff,
600 .instr_val
= 0xf1010000,
601 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
602 .pstate_val
= COMPAT_PSR_MODE_USR
,
603 .fn
= a32_setend_handler
,
607 .instr_mask
= 0x0000fff7,
608 .instr_val
= 0x0000b650,
609 .pstate_mask
= (COMPAT_PSR_T_BIT
| COMPAT_PSR_MODE_MASK
),
610 .pstate_val
= (COMPAT_PSR_T_BIT
| COMPAT_PSR_MODE_USR
),
611 .fn
= t16_setend_handler
,
616 static struct insn_emulation_ops setend_ops
= {
618 .status
= INSN_DEPRECATED
,
619 .hooks
= setend_hooks
,
620 .set_hw_mode
= setend_set_hw_mode
,
623 static int insn_cpu_hotplug_notify(struct notifier_block
*b
,
624 unsigned long action
, void *hcpu
)
627 if ((action
& ~CPU_TASKS_FROZEN
) == CPU_STARTING
)
628 rc
= run_all_insn_set_hw_mode((unsigned long)hcpu
);
630 return notifier_from_errno(rc
);
633 static struct notifier_block insn_cpu_hotplug_notifier
= {
634 .notifier_call
= insn_cpu_hotplug_notify
,
638 * Invoked as late_initcall, since not needed before init spawned.
640 static int __init
armv8_deprecated_init(void)
642 if (IS_ENABLED(CONFIG_SWP_EMULATION
))
643 register_insn_emulation(&swp_ops
);
645 if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION
))
646 register_insn_emulation(&cp15_barrier_ops
);
648 if (IS_ENABLED(CONFIG_SETEND_EMULATION
)) {
649 if(system_supports_mixed_endian_el0())
650 register_insn_emulation(&setend_ops
);
652 pr_info("setend instruction emulation is not supported on the system");
655 register_cpu_notifier(&insn_cpu_hotplug_notifier
);
656 register_insn_emulation_sysctl(ctl_abi
);
661 late_initcall(armv8_deprecated_init
);