Merge tag 'pull-loongarch-20241016' of https://gitlab.com/gaosong/qemu into staging
[qemu/armbru.git] / target / arm / tcg / translate-a64.c
blobec0b1ee25239aeadb92ab9037244ad3a51ffbb7c
1 /*
2 * AArch64 translation
4 * Copyright (c) 2013 Alexander Graf <agraf@suse.de>
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
21 #include "exec/exec-all.h"
22 #include "translate.h"
23 #include "translate-a64.h"
24 #include "qemu/log.h"
25 #include "arm_ldst.h"
26 #include "semihosting/semihost.h"
27 #include "cpregs.h"
29 static TCGv_i64 cpu_X[32];
30 static TCGv_i64 cpu_pc;
32 /* Load/store exclusive handling */
33 static TCGv_i64 cpu_exclusive_high;
35 static const char *regnames[] = {
36 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
37 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
38 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
39 "x24", "x25", "x26", "x27", "x28", "x29", "lr", "sp"
42 enum a64_shift_type {
43 A64_SHIFT_TYPE_LSL = 0,
44 A64_SHIFT_TYPE_LSR = 1,
45 A64_SHIFT_TYPE_ASR = 2,
46 A64_SHIFT_TYPE_ROR = 3
50 * Helpers for extracting complex instruction fields
54 * For load/store with an unsigned 12 bit immediate scaled by the element
55 * size. The input has the immediate field in bits [14:3] and the element
56 * size in [2:0].
58 static int uimm_scaled(DisasContext *s, int x)
60 unsigned imm = x >> 3;
61 unsigned scale = extract32(x, 0, 3);
62 return imm << scale;
65 /* For load/store memory tags: scale offset by LOG2_TAG_GRANULE */
66 static int scale_by_log2_tag_granule(DisasContext *s, int x)
68 return x << LOG2_TAG_GRANULE;
72 * Include the generated decoders.
75 #include "decode-sme-fa64.c.inc"
76 #include "decode-a64.c.inc"
78 /* Table based decoder typedefs - used when the relevant bits for decode
79 * are too awkwardly scattered across the instruction (eg SIMD).
81 typedef void AArch64DecodeFn(DisasContext *s, uint32_t insn);
83 typedef struct AArch64DecodeTable {
84 uint32_t pattern;
85 uint32_t mask;
86 AArch64DecodeFn *disas_fn;
87 } AArch64DecodeTable;
89 /* initialize TCG globals. */
90 void a64_translate_init(void)
92 int i;
94 cpu_pc = tcg_global_mem_new_i64(tcg_env,
95 offsetof(CPUARMState, pc),
96 "pc");
97 for (i = 0; i < 32; i++) {
98 cpu_X[i] = tcg_global_mem_new_i64(tcg_env,
99 offsetof(CPUARMState, xregs[i]),
100 regnames[i]);
103 cpu_exclusive_high = tcg_global_mem_new_i64(tcg_env,
104 offsetof(CPUARMState, exclusive_high), "exclusive_high");
108 * Return the core mmu_idx to use for A64 load/store insns which
109 * have a "unprivileged load/store" variant. Those insns access
110 * EL0 if executed from an EL which has control over EL0 (usually
111 * EL1) but behave like normal loads and stores if executed from
112 * elsewhere (eg EL3).
114 * @unpriv : true for the unprivileged encoding; false for the
115 * normal encoding (in which case we will return the same
116 * thing as get_mem_index().
118 static int get_a64_user_mem_index(DisasContext *s, bool unpriv)
121 * If AccType_UNPRIV is not used, the insn uses AccType_NORMAL,
122 * which is the usual mmu_idx for this cpu state.
124 ARMMMUIdx useridx = s->mmu_idx;
126 if (unpriv && s->unpriv) {
128 * We have pre-computed the condition for AccType_UNPRIV.
129 * Therefore we should never get here with a mmu_idx for
130 * which we do not know the corresponding user mmu_idx.
132 switch (useridx) {
133 case ARMMMUIdx_E10_1:
134 case ARMMMUIdx_E10_1_PAN:
135 useridx = ARMMMUIdx_E10_0;
136 break;
137 case ARMMMUIdx_E20_2:
138 case ARMMMUIdx_E20_2_PAN:
139 useridx = ARMMMUIdx_E20_0;
140 break;
141 default:
142 g_assert_not_reached();
145 return arm_to_core_mmu_idx(useridx);
148 static void set_btype_raw(int val)
150 tcg_gen_st_i32(tcg_constant_i32(val), tcg_env,
151 offsetof(CPUARMState, btype));
154 static void set_btype(DisasContext *s, int val)
156 /* BTYPE is a 2-bit field, and 0 should be done with reset_btype. */
157 tcg_debug_assert(val >= 1 && val <= 3);
158 set_btype_raw(val);
159 s->btype = -1;
162 static void reset_btype(DisasContext *s)
164 if (s->btype != 0) {
165 set_btype_raw(0);
166 s->btype = 0;
170 static void gen_pc_plus_diff(DisasContext *s, TCGv_i64 dest, target_long diff)
172 assert(s->pc_save != -1);
173 if (tb_cflags(s->base.tb) & CF_PCREL) {
174 tcg_gen_addi_i64(dest, cpu_pc, (s->pc_curr - s->pc_save) + diff);
175 } else {
176 tcg_gen_movi_i64(dest, s->pc_curr + diff);
180 void gen_a64_update_pc(DisasContext *s, target_long diff)
182 gen_pc_plus_diff(s, cpu_pc, diff);
183 s->pc_save = s->pc_curr + diff;
187 * Handle Top Byte Ignore (TBI) bits.
189 * If address tagging is enabled via the TCR TBI bits:
190 * + for EL2 and EL3 there is only one TBI bit, and if it is set
191 * then the address is zero-extended, clearing bits [63:56]
192 * + for EL0 and EL1, TBI0 controls addresses with bit 55 == 0
193 * and TBI1 controls addresses with bit 55 == 1.
194 * If the appropriate TBI bit is set for the address then
195 * the address is sign-extended from bit 55 into bits [63:56]
197 * Here We have concatenated TBI{1,0} into tbi.
199 static void gen_top_byte_ignore(DisasContext *s, TCGv_i64 dst,
200 TCGv_i64 src, int tbi)
202 if (tbi == 0) {
203 /* Load unmodified address */
204 tcg_gen_mov_i64(dst, src);
205 } else if (!regime_has_2_ranges(s->mmu_idx)) {
206 /* Force tag byte to all zero */
207 tcg_gen_extract_i64(dst, src, 0, 56);
208 } else {
209 /* Sign-extend from bit 55. */
210 tcg_gen_sextract_i64(dst, src, 0, 56);
212 switch (tbi) {
213 case 1:
214 /* tbi0 but !tbi1: only use the extension if positive */
215 tcg_gen_and_i64(dst, dst, src);
216 break;
217 case 2:
218 /* !tbi0 but tbi1: only use the extension if negative */
219 tcg_gen_or_i64(dst, dst, src);
220 break;
221 case 3:
222 /* tbi0 and tbi1: always use the extension */
223 break;
224 default:
225 g_assert_not_reached();
230 static void gen_a64_set_pc(DisasContext *s, TCGv_i64 src)
233 * If address tagging is enabled for instructions via the TCR TBI bits,
234 * then loading an address into the PC will clear out any tag.
236 gen_top_byte_ignore(s, cpu_pc, src, s->tbii);
237 s->pc_save = -1;
241 * Handle MTE and/or TBI.
243 * For TBI, ideally, we would do nothing. Proper behaviour on fault is
244 * for the tag to be present in the FAR_ELx register. But for user-only
245 * mode we do not have a TLB with which to implement this, so we must
246 * remove the top byte now.
248 * Always return a fresh temporary that we can increment independently
249 * of the write-back address.
252 TCGv_i64 clean_data_tbi(DisasContext *s, TCGv_i64 addr)
254 TCGv_i64 clean = tcg_temp_new_i64();
255 #ifdef CONFIG_USER_ONLY
256 gen_top_byte_ignore(s, clean, addr, s->tbid);
257 #else
258 tcg_gen_mov_i64(clean, addr);
259 #endif
260 return clean;
263 /* Insert a zero tag into src, with the result at dst. */
264 static void gen_address_with_allocation_tag0(TCGv_i64 dst, TCGv_i64 src)
266 tcg_gen_andi_i64(dst, src, ~MAKE_64BIT_MASK(56, 4));
269 static void gen_probe_access(DisasContext *s, TCGv_i64 ptr,
270 MMUAccessType acc, int log2_size)
272 gen_helper_probe_access(tcg_env, ptr,
273 tcg_constant_i32(acc),
274 tcg_constant_i32(get_mem_index(s)),
275 tcg_constant_i32(1 << log2_size));
279 * For MTE, check a single logical or atomic access. This probes a single
280 * address, the exact one specified. The size and alignment of the access
281 * is not relevant to MTE, per se, but watchpoints do require the size,
282 * and we want to recognize those before making any other changes to state.
284 static TCGv_i64 gen_mte_check1_mmuidx(DisasContext *s, TCGv_i64 addr,
285 bool is_write, bool tag_checked,
286 MemOp memop, bool is_unpriv,
287 int core_idx)
289 if (tag_checked && s->mte_active[is_unpriv]) {
290 TCGv_i64 ret;
291 int desc = 0;
293 desc = FIELD_DP32(desc, MTEDESC, MIDX, core_idx);
294 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
295 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
296 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
297 desc = FIELD_DP32(desc, MTEDESC, ALIGN, memop_alignment_bits(memop));
298 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, memop_size(memop) - 1);
300 ret = tcg_temp_new_i64();
301 gen_helper_mte_check(ret, tcg_env, tcg_constant_i32(desc), addr);
303 return ret;
305 return clean_data_tbi(s, addr);
308 TCGv_i64 gen_mte_check1(DisasContext *s, TCGv_i64 addr, bool is_write,
309 bool tag_checked, MemOp memop)
311 return gen_mte_check1_mmuidx(s, addr, is_write, tag_checked, memop,
312 false, get_mem_index(s));
316 * For MTE, check multiple logical sequential accesses.
318 TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write,
319 bool tag_checked, int total_size, MemOp single_mop)
321 if (tag_checked && s->mte_active[0]) {
322 TCGv_i64 ret;
323 int desc = 0;
325 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
326 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
327 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
328 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
329 desc = FIELD_DP32(desc, MTEDESC, ALIGN, memop_alignment_bits(single_mop));
330 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, total_size - 1);
332 ret = tcg_temp_new_i64();
333 gen_helper_mte_check(ret, tcg_env, tcg_constant_i32(desc), addr);
335 return ret;
337 return clean_data_tbi(s, addr);
341 * Generate the special alignment check that applies to AccType_ATOMIC
342 * and AccType_ORDERED insns under FEAT_LSE2: the access need not be
343 * naturally aligned, but it must not cross a 16-byte boundary.
344 * See AArch64.CheckAlignment().
346 static void check_lse2_align(DisasContext *s, int rn, int imm,
347 bool is_write, MemOp mop)
349 TCGv_i32 tmp;
350 TCGv_i64 addr;
351 TCGLabel *over_label;
352 MMUAccessType type;
353 int mmu_idx;
355 tmp = tcg_temp_new_i32();
356 tcg_gen_extrl_i64_i32(tmp, cpu_reg_sp(s, rn));
357 tcg_gen_addi_i32(tmp, tmp, imm & 15);
358 tcg_gen_andi_i32(tmp, tmp, 15);
359 tcg_gen_addi_i32(tmp, tmp, memop_size(mop));
361 over_label = gen_new_label();
362 tcg_gen_brcondi_i32(TCG_COND_LEU, tmp, 16, over_label);
364 addr = tcg_temp_new_i64();
365 tcg_gen_addi_i64(addr, cpu_reg_sp(s, rn), imm);
367 type = is_write ? MMU_DATA_STORE : MMU_DATA_LOAD,
368 mmu_idx = get_mem_index(s);
369 gen_helper_unaligned_access(tcg_env, addr, tcg_constant_i32(type),
370 tcg_constant_i32(mmu_idx));
372 gen_set_label(over_label);
376 /* Handle the alignment check for AccType_ATOMIC instructions. */
377 static MemOp check_atomic_align(DisasContext *s, int rn, MemOp mop)
379 MemOp size = mop & MO_SIZE;
381 if (size == MO_8) {
382 return mop;
386 * If size == MO_128, this is a LDXP, and the operation is single-copy
387 * atomic for each doubleword, not the entire quadword; it still must
388 * be quadword aligned.
390 if (size == MO_128) {
391 return finalize_memop_atom(s, MO_128 | MO_ALIGN,
392 MO_ATOM_IFALIGN_PAIR);
394 if (dc_isar_feature(aa64_lse2, s)) {
395 check_lse2_align(s, rn, 0, true, mop);
396 } else {
397 mop |= MO_ALIGN;
399 return finalize_memop(s, mop);
402 /* Handle the alignment check for AccType_ORDERED instructions. */
403 static MemOp check_ordered_align(DisasContext *s, int rn, int imm,
404 bool is_write, MemOp mop)
406 MemOp size = mop & MO_SIZE;
408 if (size == MO_8) {
409 return mop;
411 if (size == MO_128) {
412 return finalize_memop_atom(s, MO_128 | MO_ALIGN,
413 MO_ATOM_IFALIGN_PAIR);
415 if (!dc_isar_feature(aa64_lse2, s)) {
416 mop |= MO_ALIGN;
417 } else if (!s->naa) {
418 check_lse2_align(s, rn, imm, is_write, mop);
420 return finalize_memop(s, mop);
423 typedef struct DisasCompare64 {
424 TCGCond cond;
425 TCGv_i64 value;
426 } DisasCompare64;
428 static void a64_test_cc(DisasCompare64 *c64, int cc)
430 DisasCompare c32;
432 arm_test_cc(&c32, cc);
435 * Sign-extend the 32-bit value so that the GE/LT comparisons work
436 * properly. The NE/EQ comparisons are also fine with this choice.
438 c64->cond = c32.cond;
439 c64->value = tcg_temp_new_i64();
440 tcg_gen_ext_i32_i64(c64->value, c32.value);
443 static void gen_rebuild_hflags(DisasContext *s)
445 gen_helper_rebuild_hflags_a64(tcg_env, tcg_constant_i32(s->current_el));
448 static void gen_exception_internal(int excp)
450 assert(excp_is_internal(excp));
451 gen_helper_exception_internal(tcg_env, tcg_constant_i32(excp));
454 static void gen_exception_internal_insn(DisasContext *s, int excp)
456 gen_a64_update_pc(s, 0);
457 gen_exception_internal(excp);
458 s->base.is_jmp = DISAS_NORETURN;
461 static void gen_exception_bkpt_insn(DisasContext *s, uint32_t syndrome)
463 gen_a64_update_pc(s, 0);
464 gen_helper_exception_bkpt_insn(tcg_env, tcg_constant_i32(syndrome));
465 s->base.is_jmp = DISAS_NORETURN;
468 static void gen_step_complete_exception(DisasContext *s)
470 /* We just completed step of an insn. Move from Active-not-pending
471 * to Active-pending, and then also take the swstep exception.
472 * This corresponds to making the (IMPDEF) choice to prioritize
473 * swstep exceptions over asynchronous exceptions taken to an exception
474 * level where debug is disabled. This choice has the advantage that
475 * we do not need to maintain internal state corresponding to the
476 * ISV/EX syndrome bits between completion of the step and generation
477 * of the exception, and our syndrome information is always correct.
479 gen_ss_advance(s);
480 gen_swstep_exception(s, 1, s->is_ldex);
481 s->base.is_jmp = DISAS_NORETURN;
484 static inline bool use_goto_tb(DisasContext *s, uint64_t dest)
486 if (s->ss_active) {
487 return false;
489 return translator_use_goto_tb(&s->base, dest);
492 static void gen_goto_tb(DisasContext *s, int n, int64_t diff)
494 if (use_goto_tb(s, s->pc_curr + diff)) {
496 * For pcrel, the pc must always be up-to-date on entry to
497 * the linked TB, so that it can use simple additions for all
498 * further adjustments. For !pcrel, the linked TB is compiled
499 * to know its full virtual address, so we can delay the
500 * update to pc to the unlinked path. A long chain of links
501 * can thus avoid many updates to the PC.
503 if (tb_cflags(s->base.tb) & CF_PCREL) {
504 gen_a64_update_pc(s, diff);
505 tcg_gen_goto_tb(n);
506 } else {
507 tcg_gen_goto_tb(n);
508 gen_a64_update_pc(s, diff);
510 tcg_gen_exit_tb(s->base.tb, n);
511 s->base.is_jmp = DISAS_NORETURN;
512 } else {
513 gen_a64_update_pc(s, diff);
514 if (s->ss_active) {
515 gen_step_complete_exception(s);
516 } else {
517 tcg_gen_lookup_and_goto_ptr();
518 s->base.is_jmp = DISAS_NORETURN;
524 * Register access functions
526 * These functions are used for directly accessing a register in where
527 * changes to the final register value are likely to be made. If you
528 * need to use a register for temporary calculation (e.g. index type
529 * operations) use the read_* form.
531 * B1.2.1 Register mappings
533 * In instruction register encoding 31 can refer to ZR (zero register) or
534 * the SP (stack pointer) depending on context. In QEMU's case we map SP
535 * to cpu_X[31] and ZR accesses to a temporary which can be discarded.
536 * This is the point of the _sp forms.
538 TCGv_i64 cpu_reg(DisasContext *s, int reg)
540 if (reg == 31) {
541 TCGv_i64 t = tcg_temp_new_i64();
542 tcg_gen_movi_i64(t, 0);
543 return t;
544 } else {
545 return cpu_X[reg];
549 /* register access for when 31 == SP */
550 TCGv_i64 cpu_reg_sp(DisasContext *s, int reg)
552 return cpu_X[reg];
555 /* read a cpu register in 32bit/64bit mode. Returns a TCGv_i64
556 * representing the register contents. This TCGv is an auto-freed
557 * temporary so it need not be explicitly freed, and may be modified.
559 TCGv_i64 read_cpu_reg(DisasContext *s, int reg, int sf)
561 TCGv_i64 v = tcg_temp_new_i64();
562 if (reg != 31) {
563 if (sf) {
564 tcg_gen_mov_i64(v, cpu_X[reg]);
565 } else {
566 tcg_gen_ext32u_i64(v, cpu_X[reg]);
568 } else {
569 tcg_gen_movi_i64(v, 0);
571 return v;
574 TCGv_i64 read_cpu_reg_sp(DisasContext *s, int reg, int sf)
576 TCGv_i64 v = tcg_temp_new_i64();
577 if (sf) {
578 tcg_gen_mov_i64(v, cpu_X[reg]);
579 } else {
580 tcg_gen_ext32u_i64(v, cpu_X[reg]);
582 return v;
585 /* Return the offset into CPUARMState of a slice (from
586 * the least significant end) of FP register Qn (ie
587 * Dn, Sn, Hn or Bn).
588 * (Note that this is not the same mapping as for A32; see cpu.h)
590 static inline int fp_reg_offset(DisasContext *s, int regno, MemOp size)
592 return vec_reg_offset(s, regno, 0, size);
595 /* Offset of the high half of the 128 bit vector Qn */
596 static inline int fp_reg_hi_offset(DisasContext *s, int regno)
598 return vec_reg_offset(s, regno, 1, MO_64);
601 /* Convenience accessors for reading and writing single and double
602 * FP registers. Writing clears the upper parts of the associated
603 * 128 bit vector register, as required by the architecture.
604 * Note that unlike the GP register accessors, the values returned
605 * by the read functions must be manually freed.
607 static TCGv_i64 read_fp_dreg(DisasContext *s, int reg)
609 TCGv_i64 v = tcg_temp_new_i64();
611 tcg_gen_ld_i64(v, tcg_env, fp_reg_offset(s, reg, MO_64));
612 return v;
615 static TCGv_i32 read_fp_sreg(DisasContext *s, int reg)
617 TCGv_i32 v = tcg_temp_new_i32();
619 tcg_gen_ld_i32(v, tcg_env, fp_reg_offset(s, reg, MO_32));
620 return v;
623 static TCGv_i32 read_fp_hreg(DisasContext *s, int reg)
625 TCGv_i32 v = tcg_temp_new_i32();
627 tcg_gen_ld16u_i32(v, tcg_env, fp_reg_offset(s, reg, MO_16));
628 return v;
631 /* Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64).
632 * If SVE is not enabled, then there are only 128 bits in the vector.
634 static void clear_vec_high(DisasContext *s, bool is_q, int rd)
636 unsigned ofs = fp_reg_offset(s, rd, MO_64);
637 unsigned vsz = vec_full_reg_size(s);
639 /* Nop move, with side effect of clearing the tail. */
640 tcg_gen_gvec_mov(MO_64, ofs, ofs, is_q ? 16 : 8, vsz);
643 void write_fp_dreg(DisasContext *s, int reg, TCGv_i64 v)
645 unsigned ofs = fp_reg_offset(s, reg, MO_64);
647 tcg_gen_st_i64(v, tcg_env, ofs);
648 clear_vec_high(s, false, reg);
651 static void write_fp_sreg(DisasContext *s, int reg, TCGv_i32 v)
653 TCGv_i64 tmp = tcg_temp_new_i64();
655 tcg_gen_extu_i32_i64(tmp, v);
656 write_fp_dreg(s, reg, tmp);
659 /* Expand a 2-operand AdvSIMD vector operation using an expander function. */
660 static void gen_gvec_fn2(DisasContext *s, bool is_q, int rd, int rn,
661 GVecGen2Fn *gvec_fn, int vece)
663 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
664 is_q ? 16 : 8, vec_full_reg_size(s));
667 /* Expand a 2-operand + immediate AdvSIMD vector operation using
668 * an expander function.
670 static void gen_gvec_fn2i(DisasContext *s, bool is_q, int rd, int rn,
671 int64_t imm, GVecGen2iFn *gvec_fn, int vece)
673 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
674 imm, is_q ? 16 : 8, vec_full_reg_size(s));
677 /* Expand a 3-operand AdvSIMD vector operation using an expander function. */
678 static void gen_gvec_fn3(DisasContext *s, bool is_q, int rd, int rn, int rm,
679 GVecGen3Fn *gvec_fn, int vece)
681 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
682 vec_full_reg_offset(s, rm), is_q ? 16 : 8, vec_full_reg_size(s));
685 /* Expand a 4-operand AdvSIMD vector operation using an expander function. */
686 static void gen_gvec_fn4(DisasContext *s, bool is_q, int rd, int rn, int rm,
687 int rx, GVecGen4Fn *gvec_fn, int vece)
689 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
690 vec_full_reg_offset(s, rm), vec_full_reg_offset(s, rx),
691 is_q ? 16 : 8, vec_full_reg_size(s));
694 /* Expand a 2-operand operation using an out-of-line helper. */
695 static void gen_gvec_op2_ool(DisasContext *s, bool is_q, int rd,
696 int rn, int data, gen_helper_gvec_2 *fn)
698 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd),
699 vec_full_reg_offset(s, rn),
700 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
703 /* Expand a 3-operand operation using an out-of-line helper. */
704 static void gen_gvec_op3_ool(DisasContext *s, bool is_q, int rd,
705 int rn, int rm, int data, gen_helper_gvec_3 *fn)
707 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
708 vec_full_reg_offset(s, rn),
709 vec_full_reg_offset(s, rm),
710 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
713 /* Expand a 3-operand + fpstatus pointer + simd data value operation using
714 * an out-of-line helper.
716 static void gen_gvec_op3_fpst(DisasContext *s, bool is_q, int rd, int rn,
717 int rm, bool is_fp16, int data,
718 gen_helper_gvec_3_ptr *fn)
720 TCGv_ptr fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR);
721 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
722 vec_full_reg_offset(s, rn),
723 vec_full_reg_offset(s, rm), fpst,
724 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
727 /* Expand a 4-operand operation using an out-of-line helper. */
728 static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn,
729 int rm, int ra, int data, gen_helper_gvec_4 *fn)
731 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
732 vec_full_reg_offset(s, rn),
733 vec_full_reg_offset(s, rm),
734 vec_full_reg_offset(s, ra),
735 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
739 * Expand a 4-operand operation using an out-of-line helper that takes
740 * a pointer to the CPU env.
742 static void gen_gvec_op4_env(DisasContext *s, bool is_q, int rd, int rn,
743 int rm, int ra, int data,
744 gen_helper_gvec_4_ptr *fn)
746 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
747 vec_full_reg_offset(s, rn),
748 vec_full_reg_offset(s, rm),
749 vec_full_reg_offset(s, ra),
750 tcg_env,
751 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
755 * Expand a 4-operand + fpstatus pointer + simd data value operation using
756 * an out-of-line helper.
758 static void gen_gvec_op4_fpst(DisasContext *s, bool is_q, int rd, int rn,
759 int rm, int ra, bool is_fp16, int data,
760 gen_helper_gvec_4_ptr *fn)
762 TCGv_ptr fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR);
763 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
764 vec_full_reg_offset(s, rn),
765 vec_full_reg_offset(s, rm),
766 vec_full_reg_offset(s, ra), fpst,
767 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
770 /* Set ZF and NF based on a 64 bit result. This is alas fiddlier
771 * than the 32 bit equivalent.
773 static inline void gen_set_NZ64(TCGv_i64 result)
775 tcg_gen_extr_i64_i32(cpu_ZF, cpu_NF, result);
776 tcg_gen_or_i32(cpu_ZF, cpu_ZF, cpu_NF);
779 /* Set NZCV as for a logical operation: NZ as per result, CV cleared. */
780 static inline void gen_logic_CC(int sf, TCGv_i64 result)
782 if (sf) {
783 gen_set_NZ64(result);
784 } else {
785 tcg_gen_extrl_i64_i32(cpu_ZF, result);
786 tcg_gen_mov_i32(cpu_NF, cpu_ZF);
788 tcg_gen_movi_i32(cpu_CF, 0);
789 tcg_gen_movi_i32(cpu_VF, 0);
792 /* dest = T0 + T1; compute C, N, V and Z flags */
793 static void gen_add64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
795 TCGv_i64 result, flag, tmp;
796 result = tcg_temp_new_i64();
797 flag = tcg_temp_new_i64();
798 tmp = tcg_temp_new_i64();
800 tcg_gen_movi_i64(tmp, 0);
801 tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp);
803 tcg_gen_extrl_i64_i32(cpu_CF, flag);
805 gen_set_NZ64(result);
807 tcg_gen_xor_i64(flag, result, t0);
808 tcg_gen_xor_i64(tmp, t0, t1);
809 tcg_gen_andc_i64(flag, flag, tmp);
810 tcg_gen_extrh_i64_i32(cpu_VF, flag);
812 tcg_gen_mov_i64(dest, result);
815 static void gen_add32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
817 TCGv_i32 t0_32 = tcg_temp_new_i32();
818 TCGv_i32 t1_32 = tcg_temp_new_i32();
819 TCGv_i32 tmp = tcg_temp_new_i32();
821 tcg_gen_movi_i32(tmp, 0);
822 tcg_gen_extrl_i64_i32(t0_32, t0);
823 tcg_gen_extrl_i64_i32(t1_32, t1);
824 tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp);
825 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
826 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
827 tcg_gen_xor_i32(tmp, t0_32, t1_32);
828 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
829 tcg_gen_extu_i32_i64(dest, cpu_NF);
832 static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
834 if (sf) {
835 gen_add64_CC(dest, t0, t1);
836 } else {
837 gen_add32_CC(dest, t0, t1);
841 /* dest = T0 - T1; compute C, N, V and Z flags */
842 static void gen_sub64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
844 /* 64 bit arithmetic */
845 TCGv_i64 result, flag, tmp;
847 result = tcg_temp_new_i64();
848 flag = tcg_temp_new_i64();
849 tcg_gen_sub_i64(result, t0, t1);
851 gen_set_NZ64(result);
853 tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1);
854 tcg_gen_extrl_i64_i32(cpu_CF, flag);
856 tcg_gen_xor_i64(flag, result, t0);
857 tmp = tcg_temp_new_i64();
858 tcg_gen_xor_i64(tmp, t0, t1);
859 tcg_gen_and_i64(flag, flag, tmp);
860 tcg_gen_extrh_i64_i32(cpu_VF, flag);
861 tcg_gen_mov_i64(dest, result);
864 static void gen_sub32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
866 /* 32 bit arithmetic */
867 TCGv_i32 t0_32 = tcg_temp_new_i32();
868 TCGv_i32 t1_32 = tcg_temp_new_i32();
869 TCGv_i32 tmp;
871 tcg_gen_extrl_i64_i32(t0_32, t0);
872 tcg_gen_extrl_i64_i32(t1_32, t1);
873 tcg_gen_sub_i32(cpu_NF, t0_32, t1_32);
874 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
875 tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32);
876 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
877 tmp = tcg_temp_new_i32();
878 tcg_gen_xor_i32(tmp, t0_32, t1_32);
879 tcg_gen_and_i32(cpu_VF, cpu_VF, tmp);
880 tcg_gen_extu_i32_i64(dest, cpu_NF);
883 static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
885 if (sf) {
886 gen_sub64_CC(dest, t0, t1);
887 } else {
888 gen_sub32_CC(dest, t0, t1);
892 /* dest = T0 + T1 + CF; do not compute flags. */
893 static void gen_adc(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
895 TCGv_i64 flag = tcg_temp_new_i64();
896 tcg_gen_extu_i32_i64(flag, cpu_CF);
897 tcg_gen_add_i64(dest, t0, t1);
898 tcg_gen_add_i64(dest, dest, flag);
900 if (!sf) {
901 tcg_gen_ext32u_i64(dest, dest);
905 /* dest = T0 + T1 + CF; compute C, N, V and Z flags. */
906 static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
908 if (sf) {
909 TCGv_i64 result = tcg_temp_new_i64();
910 TCGv_i64 cf_64 = tcg_temp_new_i64();
911 TCGv_i64 vf_64 = tcg_temp_new_i64();
912 TCGv_i64 tmp = tcg_temp_new_i64();
913 TCGv_i64 zero = tcg_constant_i64(0);
915 tcg_gen_extu_i32_i64(cf_64, cpu_CF);
916 tcg_gen_add2_i64(result, cf_64, t0, zero, cf_64, zero);
917 tcg_gen_add2_i64(result, cf_64, result, cf_64, t1, zero);
918 tcg_gen_extrl_i64_i32(cpu_CF, cf_64);
919 gen_set_NZ64(result);
921 tcg_gen_xor_i64(vf_64, result, t0);
922 tcg_gen_xor_i64(tmp, t0, t1);
923 tcg_gen_andc_i64(vf_64, vf_64, tmp);
924 tcg_gen_extrh_i64_i32(cpu_VF, vf_64);
926 tcg_gen_mov_i64(dest, result);
927 } else {
928 TCGv_i32 t0_32 = tcg_temp_new_i32();
929 TCGv_i32 t1_32 = tcg_temp_new_i32();
930 TCGv_i32 tmp = tcg_temp_new_i32();
931 TCGv_i32 zero = tcg_constant_i32(0);
933 tcg_gen_extrl_i64_i32(t0_32, t0);
934 tcg_gen_extrl_i64_i32(t1_32, t1);
935 tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, zero, cpu_CF, zero);
936 tcg_gen_add2_i32(cpu_NF, cpu_CF, cpu_NF, cpu_CF, t1_32, zero);
938 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
939 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
940 tcg_gen_xor_i32(tmp, t0_32, t1_32);
941 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
942 tcg_gen_extu_i32_i64(dest, cpu_NF);
947 * Load/Store generators
951 * Store from GPR register to memory.
953 static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source,
954 TCGv_i64 tcg_addr, MemOp memop, int memidx,
955 bool iss_valid,
956 unsigned int iss_srt,
957 bool iss_sf, bool iss_ar)
959 tcg_gen_qemu_st_i64(source, tcg_addr, memidx, memop);
961 if (iss_valid) {
962 uint32_t syn;
964 syn = syn_data_abort_with_iss(0,
965 (memop & MO_SIZE),
966 false,
967 iss_srt,
968 iss_sf,
969 iss_ar,
970 0, 0, 0, 0, 0, false);
971 disas_set_insn_syndrome(s, syn);
975 static void do_gpr_st(DisasContext *s, TCGv_i64 source,
976 TCGv_i64 tcg_addr, MemOp memop,
977 bool iss_valid,
978 unsigned int iss_srt,
979 bool iss_sf, bool iss_ar)
981 do_gpr_st_memidx(s, source, tcg_addr, memop, get_mem_index(s),
982 iss_valid, iss_srt, iss_sf, iss_ar);
986 * Load from memory to GPR register
988 static void do_gpr_ld_memidx(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,
989 MemOp memop, bool extend, int memidx,
990 bool iss_valid, unsigned int iss_srt,
991 bool iss_sf, bool iss_ar)
993 tcg_gen_qemu_ld_i64(dest, tcg_addr, memidx, memop);
995 if (extend && (memop & MO_SIGN)) {
996 g_assert((memop & MO_SIZE) <= MO_32);
997 tcg_gen_ext32u_i64(dest, dest);
1000 if (iss_valid) {
1001 uint32_t syn;
1003 syn = syn_data_abort_with_iss(0,
1004 (memop & MO_SIZE),
1005 (memop & MO_SIGN) != 0,
1006 iss_srt,
1007 iss_sf,
1008 iss_ar,
1009 0, 0, 0, 0, 0, false);
1010 disas_set_insn_syndrome(s, syn);
1014 static void do_gpr_ld(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,
1015 MemOp memop, bool extend,
1016 bool iss_valid, unsigned int iss_srt,
1017 bool iss_sf, bool iss_ar)
1019 do_gpr_ld_memidx(s, dest, tcg_addr, memop, extend, get_mem_index(s),
1020 iss_valid, iss_srt, iss_sf, iss_ar);
1024 * Store from FP register to memory
1026 static void do_fp_st(DisasContext *s, int srcidx, TCGv_i64 tcg_addr, MemOp mop)
1028 /* This writes the bottom N bits of a 128 bit wide vector to memory */
1029 TCGv_i64 tmplo = tcg_temp_new_i64();
1031 tcg_gen_ld_i64(tmplo, tcg_env, fp_reg_offset(s, srcidx, MO_64));
1033 if ((mop & MO_SIZE) < MO_128) {
1034 tcg_gen_qemu_st_i64(tmplo, tcg_addr, get_mem_index(s), mop);
1035 } else {
1036 TCGv_i64 tmphi = tcg_temp_new_i64();
1037 TCGv_i128 t16 = tcg_temp_new_i128();
1039 tcg_gen_ld_i64(tmphi, tcg_env, fp_reg_hi_offset(s, srcidx));
1040 tcg_gen_concat_i64_i128(t16, tmplo, tmphi);
1042 tcg_gen_qemu_st_i128(t16, tcg_addr, get_mem_index(s), mop);
1047 * Load from memory to FP register
1049 static void do_fp_ld(DisasContext *s, int destidx, TCGv_i64 tcg_addr, MemOp mop)
1051 /* This always zero-extends and writes to a full 128 bit wide vector */
1052 TCGv_i64 tmplo = tcg_temp_new_i64();
1053 TCGv_i64 tmphi = NULL;
1055 if ((mop & MO_SIZE) < MO_128) {
1056 tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), mop);
1057 } else {
1058 TCGv_i128 t16 = tcg_temp_new_i128();
1060 tcg_gen_qemu_ld_i128(t16, tcg_addr, get_mem_index(s), mop);
1062 tmphi = tcg_temp_new_i64();
1063 tcg_gen_extr_i128_i64(tmplo, tmphi, t16);
1066 tcg_gen_st_i64(tmplo, tcg_env, fp_reg_offset(s, destidx, MO_64));
1068 if (tmphi) {
1069 tcg_gen_st_i64(tmphi, tcg_env, fp_reg_hi_offset(s, destidx));
1071 clear_vec_high(s, tmphi != NULL, destidx);
1075 * Vector load/store helpers.
1077 * The principal difference between this and a FP load is that we don't
1078 * zero extend as we are filling a partial chunk of the vector register.
1079 * These functions don't support 128 bit loads/stores, which would be
1080 * normal load/store operations.
1082 * The _i32 versions are useful when operating on 32 bit quantities
1083 * (eg for floating point single or using Neon helper functions).
1086 /* Get value of an element within a vector register */
1087 static void read_vec_element(DisasContext *s, TCGv_i64 tcg_dest, int srcidx,
1088 int element, MemOp memop)
1090 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1091 switch ((unsigned)memop) {
1092 case MO_8:
1093 tcg_gen_ld8u_i64(tcg_dest, tcg_env, vect_off);
1094 break;
1095 case MO_16:
1096 tcg_gen_ld16u_i64(tcg_dest, tcg_env, vect_off);
1097 break;
1098 case MO_32:
1099 tcg_gen_ld32u_i64(tcg_dest, tcg_env, vect_off);
1100 break;
1101 case MO_8|MO_SIGN:
1102 tcg_gen_ld8s_i64(tcg_dest, tcg_env, vect_off);
1103 break;
1104 case MO_16|MO_SIGN:
1105 tcg_gen_ld16s_i64(tcg_dest, tcg_env, vect_off);
1106 break;
1107 case MO_32|MO_SIGN:
1108 tcg_gen_ld32s_i64(tcg_dest, tcg_env, vect_off);
1109 break;
1110 case MO_64:
1111 case MO_64|MO_SIGN:
1112 tcg_gen_ld_i64(tcg_dest, tcg_env, vect_off);
1113 break;
1114 default:
1115 g_assert_not_reached();
1119 static void read_vec_element_i32(DisasContext *s, TCGv_i32 tcg_dest, int srcidx,
1120 int element, MemOp memop)
1122 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1123 switch (memop) {
1124 case MO_8:
1125 tcg_gen_ld8u_i32(tcg_dest, tcg_env, vect_off);
1126 break;
1127 case MO_16:
1128 tcg_gen_ld16u_i32(tcg_dest, tcg_env, vect_off);
1129 break;
1130 case MO_8|MO_SIGN:
1131 tcg_gen_ld8s_i32(tcg_dest, tcg_env, vect_off);
1132 break;
1133 case MO_16|MO_SIGN:
1134 tcg_gen_ld16s_i32(tcg_dest, tcg_env, vect_off);
1135 break;
1136 case MO_32:
1137 case MO_32|MO_SIGN:
1138 tcg_gen_ld_i32(tcg_dest, tcg_env, vect_off);
1139 break;
1140 default:
1141 g_assert_not_reached();
1145 /* Set value of an element within a vector register */
1146 static void write_vec_element(DisasContext *s, TCGv_i64 tcg_src, int destidx,
1147 int element, MemOp memop)
1149 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1150 switch (memop) {
1151 case MO_8:
1152 tcg_gen_st8_i64(tcg_src, tcg_env, vect_off);
1153 break;
1154 case MO_16:
1155 tcg_gen_st16_i64(tcg_src, tcg_env, vect_off);
1156 break;
1157 case MO_32:
1158 tcg_gen_st32_i64(tcg_src, tcg_env, vect_off);
1159 break;
1160 case MO_64:
1161 tcg_gen_st_i64(tcg_src, tcg_env, vect_off);
1162 break;
1163 default:
1164 g_assert_not_reached();
1168 static void write_vec_element_i32(DisasContext *s, TCGv_i32 tcg_src,
1169 int destidx, int element, MemOp memop)
1171 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1172 switch (memop) {
1173 case MO_8:
1174 tcg_gen_st8_i32(tcg_src, tcg_env, vect_off);
1175 break;
1176 case MO_16:
1177 tcg_gen_st16_i32(tcg_src, tcg_env, vect_off);
1178 break;
1179 case MO_32:
1180 tcg_gen_st_i32(tcg_src, tcg_env, vect_off);
1181 break;
1182 default:
1183 g_assert_not_reached();
1187 /* Store from vector register to memory */
1188 static void do_vec_st(DisasContext *s, int srcidx, int element,
1189 TCGv_i64 tcg_addr, MemOp mop)
1191 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1193 read_vec_element(s, tcg_tmp, srcidx, element, mop & MO_SIZE);
1194 tcg_gen_qemu_st_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop);
1197 /* Load from memory to vector register */
1198 static void do_vec_ld(DisasContext *s, int destidx, int element,
1199 TCGv_i64 tcg_addr, MemOp mop)
1201 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1203 tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop);
1204 write_vec_element(s, tcg_tmp, destidx, element, mop & MO_SIZE);
1207 /* Check that FP/Neon access is enabled. If it is, return
1208 * true. If not, emit code to generate an appropriate exception,
1209 * and return false; the caller should not emit any code for
1210 * the instruction. Note that this check must happen after all
1211 * unallocated-encoding checks (otherwise the syndrome information
1212 * for the resulting exception will be incorrect).
1214 static bool fp_access_check_only(DisasContext *s)
1216 if (s->fp_excp_el) {
1217 assert(!s->fp_access_checked);
1218 s->fp_access_checked = true;
1220 gen_exception_insn_el(s, 0, EXCP_UDEF,
1221 syn_fp_access_trap(1, 0xe, false, 0),
1222 s->fp_excp_el);
1223 return false;
1225 s->fp_access_checked = true;
1226 return true;
1229 static bool fp_access_check(DisasContext *s)
1231 if (!fp_access_check_only(s)) {
1232 return false;
1234 if (s->sme_trap_nonstreaming && s->is_nonstreaming) {
1235 gen_exception_insn(s, 0, EXCP_UDEF,
1236 syn_smetrap(SME_ET_Streaming, false));
1237 return false;
1239 return true;
1243 * Check that SVE access is enabled. If it is, return true.
1244 * If not, emit code to generate an appropriate exception and return false.
1245 * This function corresponds to CheckSVEEnabled().
1247 bool sve_access_check(DisasContext *s)
1249 if (s->pstate_sm || !dc_isar_feature(aa64_sve, s)) {
1250 assert(dc_isar_feature(aa64_sme, s));
1251 if (!sme_sm_enabled_check(s)) {
1252 goto fail_exit;
1254 } else if (s->sve_excp_el) {
1255 gen_exception_insn_el(s, 0, EXCP_UDEF,
1256 syn_sve_access_trap(), s->sve_excp_el);
1257 goto fail_exit;
1259 s->sve_access_checked = true;
1260 return fp_access_check(s);
1262 fail_exit:
1263 /* Assert that we only raise one exception per instruction. */
1264 assert(!s->sve_access_checked);
1265 s->sve_access_checked = true;
1266 return false;
1270 * Check that SME access is enabled, raise an exception if not.
1271 * Note that this function corresponds to CheckSMEAccess and is
1272 * only used directly for cpregs.
1274 static bool sme_access_check(DisasContext *s)
1276 if (s->sme_excp_el) {
1277 gen_exception_insn_el(s, 0, EXCP_UDEF,
1278 syn_smetrap(SME_ET_AccessTrap, false),
1279 s->sme_excp_el);
1280 return false;
1282 return true;
1285 /* This function corresponds to CheckSMEEnabled. */
1286 bool sme_enabled_check(DisasContext *s)
1289 * Note that unlike sve_excp_el, we have not constrained sme_excp_el
1290 * to be zero when fp_excp_el has priority. This is because we need
1291 * sme_excp_el by itself for cpregs access checks.
1293 if (!s->fp_excp_el || s->sme_excp_el < s->fp_excp_el) {
1294 s->fp_access_checked = true;
1295 return sme_access_check(s);
1297 return fp_access_check_only(s);
1300 /* Common subroutine for CheckSMEAnd*Enabled. */
1301 bool sme_enabled_check_with_svcr(DisasContext *s, unsigned req)
1303 if (!sme_enabled_check(s)) {
1304 return false;
1306 if (FIELD_EX64(req, SVCR, SM) && !s->pstate_sm) {
1307 gen_exception_insn(s, 0, EXCP_UDEF,
1308 syn_smetrap(SME_ET_NotStreaming, false));
1309 return false;
1311 if (FIELD_EX64(req, SVCR, ZA) && !s->pstate_za) {
1312 gen_exception_insn(s, 0, EXCP_UDEF,
1313 syn_smetrap(SME_ET_InactiveZA, false));
1314 return false;
1316 return true;
1320 * Expanders for AdvSIMD translation functions.
1323 static bool do_gvec_op2_ool(DisasContext *s, arg_qrr_e *a, int data,
1324 gen_helper_gvec_2 *fn)
1326 if (!a->q && a->esz == MO_64) {
1327 return false;
1329 if (fp_access_check(s)) {
1330 gen_gvec_op2_ool(s, a->q, a->rd, a->rn, data, fn);
1332 return true;
1335 static bool do_gvec_op3_ool(DisasContext *s, arg_qrrr_e *a, int data,
1336 gen_helper_gvec_3 *fn)
1338 if (!a->q && a->esz == MO_64) {
1339 return false;
1341 if (fp_access_check(s)) {
1342 gen_gvec_op3_ool(s, a->q, a->rd, a->rn, a->rm, data, fn);
1344 return true;
1347 static bool do_gvec_fn3(DisasContext *s, arg_qrrr_e *a, GVecGen3Fn *fn)
1349 if (!a->q && a->esz == MO_64) {
1350 return false;
1352 if (fp_access_check(s)) {
1353 gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz);
1355 return true;
1358 static bool do_gvec_fn3_no64(DisasContext *s, arg_qrrr_e *a, GVecGen3Fn *fn)
1360 if (a->esz == MO_64) {
1361 return false;
1363 if (fp_access_check(s)) {
1364 gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz);
1366 return true;
1369 static bool do_gvec_fn3_no8_no64(DisasContext *s, arg_qrrr_e *a, GVecGen3Fn *fn)
1371 if (a->esz == MO_8) {
1372 return false;
1374 return do_gvec_fn3_no64(s, a, fn);
1377 static bool do_gvec_fn4(DisasContext *s, arg_qrrrr_e *a, GVecGen4Fn *fn)
1379 if (!a->q && a->esz == MO_64) {
1380 return false;
1382 if (fp_access_check(s)) {
1383 gen_gvec_fn4(s, a->q, a->rd, a->rn, a->rm, a->ra, fn, a->esz);
1385 return true;
1389 * This utility function is for doing register extension with an
1390 * optional shift. You will likely want to pass a temporary for the
1391 * destination register. See DecodeRegExtend() in the ARM ARM.
1393 static void ext_and_shift_reg(TCGv_i64 tcg_out, TCGv_i64 tcg_in,
1394 int option, unsigned int shift)
1396 int extsize = extract32(option, 0, 2);
1397 bool is_signed = extract32(option, 2, 1);
1399 tcg_gen_ext_i64(tcg_out, tcg_in, extsize | (is_signed ? MO_SIGN : 0));
1400 tcg_gen_shli_i64(tcg_out, tcg_out, shift);
1403 static inline void gen_check_sp_alignment(DisasContext *s)
1405 /* The AArch64 architecture mandates that (if enabled via PSTATE
1406 * or SCTLR bits) there is a check that SP is 16-aligned on every
1407 * SP-relative load or store (with an exception generated if it is not).
1408 * In line with general QEMU practice regarding misaligned accesses,
1409 * we omit these checks for the sake of guest program performance.
1410 * This function is provided as a hook so we can more easily add these
1411 * checks in future (possibly as a "favour catching guest program bugs
1412 * over speed" user selectable option).
1417 * This provides a simple table based table lookup decoder. It is
1418 * intended to be used when the relevant bits for decode are too
1419 * awkwardly placed and switch/if based logic would be confusing and
1420 * deeply nested. Since it's a linear search through the table, tables
1421 * should be kept small.
1423 * It returns the first handler where insn & mask == pattern, or
1424 * NULL if there is no match.
1425 * The table is terminated by an empty mask (i.e. 0)
1427 static inline AArch64DecodeFn *lookup_disas_fn(const AArch64DecodeTable *table,
1428 uint32_t insn)
1430 const AArch64DecodeTable *tptr = table;
1432 while (tptr->mask) {
1433 if ((insn & tptr->mask) == tptr->pattern) {
1434 return tptr->disas_fn;
1436 tptr++;
1438 return NULL;
1442 * The instruction disassembly implemented here matches
1443 * the instruction encoding classifications in chapter C4
1444 * of the ARM Architecture Reference Manual (DDI0487B_a);
1445 * classification names and decode diagrams here should generally
1446 * match up with those in the manual.
1449 static bool trans_B(DisasContext *s, arg_i *a)
1451 reset_btype(s);
1452 gen_goto_tb(s, 0, a->imm);
1453 return true;
1456 static bool trans_BL(DisasContext *s, arg_i *a)
1458 gen_pc_plus_diff(s, cpu_reg(s, 30), curr_insn_len(s));
1459 reset_btype(s);
1460 gen_goto_tb(s, 0, a->imm);
1461 return true;
1465 static bool trans_CBZ(DisasContext *s, arg_cbz *a)
1467 DisasLabel match;
1468 TCGv_i64 tcg_cmp;
1470 tcg_cmp = read_cpu_reg(s, a->rt, a->sf);
1471 reset_btype(s);
1473 match = gen_disas_label(s);
1474 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ,
1475 tcg_cmp, 0, match.label);
1476 gen_goto_tb(s, 0, 4);
1477 set_disas_label(s, match);
1478 gen_goto_tb(s, 1, a->imm);
1479 return true;
1482 static bool trans_TBZ(DisasContext *s, arg_tbz *a)
1484 DisasLabel match;
1485 TCGv_i64 tcg_cmp;
1487 tcg_cmp = tcg_temp_new_i64();
1488 tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, a->rt), 1ULL << a->bitpos);
1490 reset_btype(s);
1492 match = gen_disas_label(s);
1493 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ,
1494 tcg_cmp, 0, match.label);
1495 gen_goto_tb(s, 0, 4);
1496 set_disas_label(s, match);
1497 gen_goto_tb(s, 1, a->imm);
1498 return true;
1501 static bool trans_B_cond(DisasContext *s, arg_B_cond *a)
1503 /* BC.cond is only present with FEAT_HBC */
1504 if (a->c && !dc_isar_feature(aa64_hbc, s)) {
1505 return false;
1507 reset_btype(s);
1508 if (a->cond < 0x0e) {
1509 /* genuinely conditional branches */
1510 DisasLabel match = gen_disas_label(s);
1511 arm_gen_test_cc(a->cond, match.label);
1512 gen_goto_tb(s, 0, 4);
1513 set_disas_label(s, match);
1514 gen_goto_tb(s, 1, a->imm);
1515 } else {
1516 /* 0xe and 0xf are both "always" conditions */
1517 gen_goto_tb(s, 0, a->imm);
1519 return true;
1522 static void set_btype_for_br(DisasContext *s, int rn)
1524 if (dc_isar_feature(aa64_bti, s)) {
1525 /* BR to {x16,x17} or !guard -> 1, else 3. */
1526 if (rn == 16 || rn == 17) {
1527 set_btype(s, 1);
1528 } else {
1529 TCGv_i64 pc = tcg_temp_new_i64();
1530 gen_pc_plus_diff(s, pc, 0);
1531 gen_helper_guarded_page_br(tcg_env, pc);
1532 s->btype = -1;
1537 static void set_btype_for_blr(DisasContext *s)
1539 if (dc_isar_feature(aa64_bti, s)) {
1540 /* BLR sets BTYPE to 2, regardless of source guarded page. */
1541 set_btype(s, 2);
1545 static bool trans_BR(DisasContext *s, arg_r *a)
1547 set_btype_for_br(s, a->rn);
1548 gen_a64_set_pc(s, cpu_reg(s, a->rn));
1549 s->base.is_jmp = DISAS_JUMP;
1550 return true;
1553 static bool trans_BLR(DisasContext *s, arg_r *a)
1555 TCGv_i64 dst = cpu_reg(s, a->rn);
1556 TCGv_i64 lr = cpu_reg(s, 30);
1557 if (dst == lr) {
1558 TCGv_i64 tmp = tcg_temp_new_i64();
1559 tcg_gen_mov_i64(tmp, dst);
1560 dst = tmp;
1562 gen_pc_plus_diff(s, lr, curr_insn_len(s));
1563 gen_a64_set_pc(s, dst);
1564 set_btype_for_blr(s);
1565 s->base.is_jmp = DISAS_JUMP;
1566 return true;
1569 static bool trans_RET(DisasContext *s, arg_r *a)
1571 gen_a64_set_pc(s, cpu_reg(s, a->rn));
1572 s->base.is_jmp = DISAS_JUMP;
1573 return true;
1576 static TCGv_i64 auth_branch_target(DisasContext *s, TCGv_i64 dst,
1577 TCGv_i64 modifier, bool use_key_a)
1579 TCGv_i64 truedst;
1581 * Return the branch target for a BRAA/RETA/etc, which is either
1582 * just the destination dst, or that value with the pauth check
1583 * done and the code removed from the high bits.
1585 if (!s->pauth_active) {
1586 return dst;
1589 truedst = tcg_temp_new_i64();
1590 if (use_key_a) {
1591 gen_helper_autia_combined(truedst, tcg_env, dst, modifier);
1592 } else {
1593 gen_helper_autib_combined(truedst, tcg_env, dst, modifier);
1595 return truedst;
1598 static bool trans_BRAZ(DisasContext *s, arg_braz *a)
1600 TCGv_i64 dst;
1602 if (!dc_isar_feature(aa64_pauth, s)) {
1603 return false;
1606 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m);
1607 set_btype_for_br(s, a->rn);
1608 gen_a64_set_pc(s, dst);
1609 s->base.is_jmp = DISAS_JUMP;
1610 return true;
1613 static bool trans_BLRAZ(DisasContext *s, arg_braz *a)
1615 TCGv_i64 dst, lr;
1617 if (!dc_isar_feature(aa64_pauth, s)) {
1618 return false;
1621 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m);
1622 lr = cpu_reg(s, 30);
1623 if (dst == lr) {
1624 TCGv_i64 tmp = tcg_temp_new_i64();
1625 tcg_gen_mov_i64(tmp, dst);
1626 dst = tmp;
1628 gen_pc_plus_diff(s, lr, curr_insn_len(s));
1629 gen_a64_set_pc(s, dst);
1630 set_btype_for_blr(s);
1631 s->base.is_jmp = DISAS_JUMP;
1632 return true;
1635 static bool trans_RETA(DisasContext *s, arg_reta *a)
1637 TCGv_i64 dst;
1639 dst = auth_branch_target(s, cpu_reg(s, 30), cpu_X[31], !a->m);
1640 gen_a64_set_pc(s, dst);
1641 s->base.is_jmp = DISAS_JUMP;
1642 return true;
1645 static bool trans_BRA(DisasContext *s, arg_bra *a)
1647 TCGv_i64 dst;
1649 if (!dc_isar_feature(aa64_pauth, s)) {
1650 return false;
1652 dst = auth_branch_target(s, cpu_reg(s,a->rn), cpu_reg_sp(s, a->rm), !a->m);
1653 gen_a64_set_pc(s, dst);
1654 set_btype_for_br(s, a->rn);
1655 s->base.is_jmp = DISAS_JUMP;
1656 return true;
1659 static bool trans_BLRA(DisasContext *s, arg_bra *a)
1661 TCGv_i64 dst, lr;
1663 if (!dc_isar_feature(aa64_pauth, s)) {
1664 return false;
1666 dst = auth_branch_target(s, cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm), !a->m);
1667 lr = cpu_reg(s, 30);
1668 if (dst == lr) {
1669 TCGv_i64 tmp = tcg_temp_new_i64();
1670 tcg_gen_mov_i64(tmp, dst);
1671 dst = tmp;
1673 gen_pc_plus_diff(s, lr, curr_insn_len(s));
1674 gen_a64_set_pc(s, dst);
1675 set_btype_for_blr(s);
1676 s->base.is_jmp = DISAS_JUMP;
1677 return true;
1680 static bool trans_ERET(DisasContext *s, arg_ERET *a)
1682 TCGv_i64 dst;
1684 if (s->current_el == 0) {
1685 return false;
1687 if (s->trap_eret) {
1688 gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(0), 2);
1689 return true;
1691 dst = tcg_temp_new_i64();
1692 tcg_gen_ld_i64(dst, tcg_env,
1693 offsetof(CPUARMState, elr_el[s->current_el]));
1695 translator_io_start(&s->base);
1697 gen_helper_exception_return(tcg_env, dst);
1698 /* Must exit loop to check un-masked IRQs */
1699 s->base.is_jmp = DISAS_EXIT;
1700 return true;
1703 static bool trans_ERETA(DisasContext *s, arg_reta *a)
1705 TCGv_i64 dst;
1707 if (!dc_isar_feature(aa64_pauth, s)) {
1708 return false;
1710 if (s->current_el == 0) {
1711 return false;
1713 /* The FGT trap takes precedence over an auth trap. */
1714 if (s->trap_eret) {
1715 gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(a->m ? 3 : 2), 2);
1716 return true;
1718 dst = tcg_temp_new_i64();
1719 tcg_gen_ld_i64(dst, tcg_env,
1720 offsetof(CPUARMState, elr_el[s->current_el]));
1722 dst = auth_branch_target(s, dst, cpu_X[31], !a->m);
1724 translator_io_start(&s->base);
1726 gen_helper_exception_return(tcg_env, dst);
1727 /* Must exit loop to check un-masked IRQs */
1728 s->base.is_jmp = DISAS_EXIT;
1729 return true;
1732 static bool trans_NOP(DisasContext *s, arg_NOP *a)
1734 return true;
1737 static bool trans_YIELD(DisasContext *s, arg_YIELD *a)
1740 * When running in MTTCG we don't generate jumps to the yield and
1741 * WFE helpers as it won't affect the scheduling of other vCPUs.
1742 * If we wanted to more completely model WFE/SEV so we don't busy
1743 * spin unnecessarily we would need to do something more involved.
1745 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1746 s->base.is_jmp = DISAS_YIELD;
1748 return true;
1751 static bool trans_WFI(DisasContext *s, arg_WFI *a)
1753 s->base.is_jmp = DISAS_WFI;
1754 return true;
1757 static bool trans_WFE(DisasContext *s, arg_WFI *a)
1760 * When running in MTTCG we don't generate jumps to the yield and
1761 * WFE helpers as it won't affect the scheduling of other vCPUs.
1762 * If we wanted to more completely model WFE/SEV so we don't busy
1763 * spin unnecessarily we would need to do something more involved.
1765 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1766 s->base.is_jmp = DISAS_WFE;
1768 return true;
1771 static bool trans_WFIT(DisasContext *s, arg_WFIT *a)
1773 if (!dc_isar_feature(aa64_wfxt, s)) {
1774 return false;
1778 * Because we need to pass the register value to the helper,
1779 * it's easier to emit the code now, unlike trans_WFI which
1780 * defers it to aarch64_tr_tb_stop(). That means we need to
1781 * check ss_active so that single-stepping a WFIT doesn't halt.
1783 if (s->ss_active) {
1784 /* Act like a NOP under architectural singlestep */
1785 return true;
1788 gen_a64_update_pc(s, 4);
1789 gen_helper_wfit(tcg_env, cpu_reg(s, a->rd));
1790 /* Go back to the main loop to check for interrupts */
1791 s->base.is_jmp = DISAS_EXIT;
1792 return true;
1795 static bool trans_WFET(DisasContext *s, arg_WFET *a)
1797 if (!dc_isar_feature(aa64_wfxt, s)) {
1798 return false;
1802 * We rely here on our WFE implementation being a NOP, so we
1803 * don't need to do anything different to handle the WFET timeout
1804 * from what trans_WFE does.
1806 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1807 s->base.is_jmp = DISAS_WFE;
1809 return true;
1812 static bool trans_XPACLRI(DisasContext *s, arg_XPACLRI *a)
1814 if (s->pauth_active) {
1815 gen_helper_xpaci(cpu_X[30], tcg_env, cpu_X[30]);
1817 return true;
1820 static bool trans_PACIA1716(DisasContext *s, arg_PACIA1716 *a)
1822 if (s->pauth_active) {
1823 gen_helper_pacia(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
1825 return true;
1828 static bool trans_PACIB1716(DisasContext *s, arg_PACIB1716 *a)
1830 if (s->pauth_active) {
1831 gen_helper_pacib(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
1833 return true;
1836 static bool trans_AUTIA1716(DisasContext *s, arg_AUTIA1716 *a)
1838 if (s->pauth_active) {
1839 gen_helper_autia(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
1841 return true;
1844 static bool trans_AUTIB1716(DisasContext *s, arg_AUTIB1716 *a)
1846 if (s->pauth_active) {
1847 gen_helper_autib(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
1849 return true;
1852 static bool trans_ESB(DisasContext *s, arg_ESB *a)
1854 /* Without RAS, we must implement this as NOP. */
1855 if (dc_isar_feature(aa64_ras, s)) {
1857 * QEMU does not have a source of physical SErrors,
1858 * so we are only concerned with virtual SErrors.
1859 * The pseudocode in the ARM for this case is
1860 * if PSTATE.EL IN {EL0, EL1} && EL2Enabled() then
1861 * AArch64.vESBOperation();
1862 * Most of the condition can be evaluated at translation time.
1863 * Test for EL2 present, and defer test for SEL2 to runtime.
1865 if (s->current_el <= 1 && arm_dc_feature(s, ARM_FEATURE_EL2)) {
1866 gen_helper_vesb(tcg_env);
1869 return true;
1872 static bool trans_PACIAZ(DisasContext *s, arg_PACIAZ *a)
1874 if (s->pauth_active) {
1875 gen_helper_pacia(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
1877 return true;
1880 static bool trans_PACIASP(DisasContext *s, arg_PACIASP *a)
1882 if (s->pauth_active) {
1883 gen_helper_pacia(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
1885 return true;
1888 static bool trans_PACIBZ(DisasContext *s, arg_PACIBZ *a)
1890 if (s->pauth_active) {
1891 gen_helper_pacib(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
1893 return true;
1896 static bool trans_PACIBSP(DisasContext *s, arg_PACIBSP *a)
1898 if (s->pauth_active) {
1899 gen_helper_pacib(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
1901 return true;
1904 static bool trans_AUTIAZ(DisasContext *s, arg_AUTIAZ *a)
1906 if (s->pauth_active) {
1907 gen_helper_autia(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
1909 return true;
1912 static bool trans_AUTIASP(DisasContext *s, arg_AUTIASP *a)
1914 if (s->pauth_active) {
1915 gen_helper_autia(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
1917 return true;
1920 static bool trans_AUTIBZ(DisasContext *s, arg_AUTIBZ *a)
1922 if (s->pauth_active) {
1923 gen_helper_autib(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
1925 return true;
1928 static bool trans_AUTIBSP(DisasContext *s, arg_AUTIBSP *a)
1930 if (s->pauth_active) {
1931 gen_helper_autib(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
1933 return true;
1936 static bool trans_CLREX(DisasContext *s, arg_CLREX *a)
1938 tcg_gen_movi_i64(cpu_exclusive_addr, -1);
1939 return true;
1942 static bool trans_DSB_DMB(DisasContext *s, arg_DSB_DMB *a)
1944 /* We handle DSB and DMB the same way */
1945 TCGBar bar;
1947 switch (a->types) {
1948 case 1: /* MBReqTypes_Reads */
1949 bar = TCG_BAR_SC | TCG_MO_LD_LD | TCG_MO_LD_ST;
1950 break;
1951 case 2: /* MBReqTypes_Writes */
1952 bar = TCG_BAR_SC | TCG_MO_ST_ST;
1953 break;
1954 default: /* MBReqTypes_All */
1955 bar = TCG_BAR_SC | TCG_MO_ALL;
1956 break;
1958 tcg_gen_mb(bar);
1959 return true;
1962 static bool trans_ISB(DisasContext *s, arg_ISB *a)
1965 * We need to break the TB after this insn to execute
1966 * self-modifying code correctly and also to take
1967 * any pending interrupts immediately.
1969 reset_btype(s);
1970 gen_goto_tb(s, 0, 4);
1971 return true;
1974 static bool trans_SB(DisasContext *s, arg_SB *a)
1976 if (!dc_isar_feature(aa64_sb, s)) {
1977 return false;
1980 * TODO: There is no speculation barrier opcode for TCG;
1981 * MB and end the TB instead.
1983 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC);
1984 gen_goto_tb(s, 0, 4);
1985 return true;
1988 static bool trans_CFINV(DisasContext *s, arg_CFINV *a)
1990 if (!dc_isar_feature(aa64_condm_4, s)) {
1991 return false;
1993 tcg_gen_xori_i32(cpu_CF, cpu_CF, 1);
1994 return true;
1997 static bool trans_XAFLAG(DisasContext *s, arg_XAFLAG *a)
1999 TCGv_i32 z;
2001 if (!dc_isar_feature(aa64_condm_5, s)) {
2002 return false;
2005 z = tcg_temp_new_i32();
2007 tcg_gen_setcondi_i32(TCG_COND_EQ, z, cpu_ZF, 0);
2010 * (!C & !Z) << 31
2011 * (!(C | Z)) << 31
2012 * ~((C | Z) << 31)
2013 * ~-(C | Z)
2014 * (C | Z) - 1
2016 tcg_gen_or_i32(cpu_NF, cpu_CF, z);
2017 tcg_gen_subi_i32(cpu_NF, cpu_NF, 1);
2019 /* !(Z & C) */
2020 tcg_gen_and_i32(cpu_ZF, z, cpu_CF);
2021 tcg_gen_xori_i32(cpu_ZF, cpu_ZF, 1);
2023 /* (!C & Z) << 31 -> -(Z & ~C) */
2024 tcg_gen_andc_i32(cpu_VF, z, cpu_CF);
2025 tcg_gen_neg_i32(cpu_VF, cpu_VF);
2027 /* C | Z */
2028 tcg_gen_or_i32(cpu_CF, cpu_CF, z);
2030 return true;
2033 static bool trans_AXFLAG(DisasContext *s, arg_AXFLAG *a)
2035 if (!dc_isar_feature(aa64_condm_5, s)) {
2036 return false;
2039 tcg_gen_sari_i32(cpu_VF, cpu_VF, 31); /* V ? -1 : 0 */
2040 tcg_gen_andc_i32(cpu_CF, cpu_CF, cpu_VF); /* C & !V */
2042 /* !(Z | V) -> !(!ZF | V) -> ZF & !V -> ZF & ~VF */
2043 tcg_gen_andc_i32(cpu_ZF, cpu_ZF, cpu_VF);
2045 tcg_gen_movi_i32(cpu_NF, 0);
2046 tcg_gen_movi_i32(cpu_VF, 0);
2048 return true;
2051 static bool trans_MSR_i_UAO(DisasContext *s, arg_i *a)
2053 if (!dc_isar_feature(aa64_uao, s) || s->current_el == 0) {
2054 return false;
2056 if (a->imm & 1) {
2057 set_pstate_bits(PSTATE_UAO);
2058 } else {
2059 clear_pstate_bits(PSTATE_UAO);
2061 gen_rebuild_hflags(s);
2062 s->base.is_jmp = DISAS_TOO_MANY;
2063 return true;
2066 static bool trans_MSR_i_PAN(DisasContext *s, arg_i *a)
2068 if (!dc_isar_feature(aa64_pan, s) || s->current_el == 0) {
2069 return false;
2071 if (a->imm & 1) {
2072 set_pstate_bits(PSTATE_PAN);
2073 } else {
2074 clear_pstate_bits(PSTATE_PAN);
2076 gen_rebuild_hflags(s);
2077 s->base.is_jmp = DISAS_TOO_MANY;
2078 return true;
2081 static bool trans_MSR_i_SPSEL(DisasContext *s, arg_i *a)
2083 if (s->current_el == 0) {
2084 return false;
2086 gen_helper_msr_i_spsel(tcg_env, tcg_constant_i32(a->imm & PSTATE_SP));
2087 s->base.is_jmp = DISAS_TOO_MANY;
2088 return true;
2091 static bool trans_MSR_i_SBSS(DisasContext *s, arg_i *a)
2093 if (!dc_isar_feature(aa64_ssbs, s)) {
2094 return false;
2096 if (a->imm & 1) {
2097 set_pstate_bits(PSTATE_SSBS);
2098 } else {
2099 clear_pstate_bits(PSTATE_SSBS);
2101 /* Don't need to rebuild hflags since SSBS is a nop */
2102 s->base.is_jmp = DISAS_TOO_MANY;
2103 return true;
2106 static bool trans_MSR_i_DIT(DisasContext *s, arg_i *a)
2108 if (!dc_isar_feature(aa64_dit, s)) {
2109 return false;
2111 if (a->imm & 1) {
2112 set_pstate_bits(PSTATE_DIT);
2113 } else {
2114 clear_pstate_bits(PSTATE_DIT);
2116 /* There's no need to rebuild hflags because DIT is a nop */
2117 s->base.is_jmp = DISAS_TOO_MANY;
2118 return true;
2121 static bool trans_MSR_i_TCO(DisasContext *s, arg_i *a)
2123 if (dc_isar_feature(aa64_mte, s)) {
2124 /* Full MTE is enabled -- set the TCO bit as directed. */
2125 if (a->imm & 1) {
2126 set_pstate_bits(PSTATE_TCO);
2127 } else {
2128 clear_pstate_bits(PSTATE_TCO);
2130 gen_rebuild_hflags(s);
2131 /* Many factors, including TCO, go into MTE_ACTIVE. */
2132 s->base.is_jmp = DISAS_UPDATE_NOCHAIN;
2133 return true;
2134 } else if (dc_isar_feature(aa64_mte_insn_reg, s)) {
2135 /* Only "instructions accessible at EL0" -- PSTATE.TCO is WI. */
2136 return true;
2137 } else {
2138 /* Insn not present */
2139 return false;
2143 static bool trans_MSR_i_DAIFSET(DisasContext *s, arg_i *a)
2145 gen_helper_msr_i_daifset(tcg_env, tcg_constant_i32(a->imm));
2146 s->base.is_jmp = DISAS_TOO_MANY;
2147 return true;
2150 static bool trans_MSR_i_DAIFCLEAR(DisasContext *s, arg_i *a)
2152 gen_helper_msr_i_daifclear(tcg_env, tcg_constant_i32(a->imm));
2153 /* Exit the cpu loop to re-evaluate pending IRQs. */
2154 s->base.is_jmp = DISAS_UPDATE_EXIT;
2155 return true;
2158 static bool trans_MSR_i_ALLINT(DisasContext *s, arg_i *a)
2160 if (!dc_isar_feature(aa64_nmi, s) || s->current_el == 0) {
2161 return false;
2164 if (a->imm == 0) {
2165 clear_pstate_bits(PSTATE_ALLINT);
2166 } else if (s->current_el > 1) {
2167 set_pstate_bits(PSTATE_ALLINT);
2168 } else {
2169 gen_helper_msr_set_allint_el1(tcg_env);
2172 /* Exit the cpu loop to re-evaluate pending IRQs. */
2173 s->base.is_jmp = DISAS_UPDATE_EXIT;
2174 return true;
2177 static bool trans_MSR_i_SVCR(DisasContext *s, arg_MSR_i_SVCR *a)
2179 if (!dc_isar_feature(aa64_sme, s) || a->mask == 0) {
2180 return false;
2182 if (sme_access_check(s)) {
2183 int old = s->pstate_sm | (s->pstate_za << 1);
2184 int new = a->imm * 3;
2186 if ((old ^ new) & a->mask) {
2187 /* At least one bit changes. */
2188 gen_helper_set_svcr(tcg_env, tcg_constant_i32(new),
2189 tcg_constant_i32(a->mask));
2190 s->base.is_jmp = DISAS_TOO_MANY;
2193 return true;
2196 static void gen_get_nzcv(TCGv_i64 tcg_rt)
2198 TCGv_i32 tmp = tcg_temp_new_i32();
2199 TCGv_i32 nzcv = tcg_temp_new_i32();
2201 /* build bit 31, N */
2202 tcg_gen_andi_i32(nzcv, cpu_NF, (1U << 31));
2203 /* build bit 30, Z */
2204 tcg_gen_setcondi_i32(TCG_COND_EQ, tmp, cpu_ZF, 0);
2205 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 30, 1);
2206 /* build bit 29, C */
2207 tcg_gen_deposit_i32(nzcv, nzcv, cpu_CF, 29, 1);
2208 /* build bit 28, V */
2209 tcg_gen_shri_i32(tmp, cpu_VF, 31);
2210 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 28, 1);
2211 /* generate result */
2212 tcg_gen_extu_i32_i64(tcg_rt, nzcv);
2215 static void gen_set_nzcv(TCGv_i64 tcg_rt)
2217 TCGv_i32 nzcv = tcg_temp_new_i32();
2219 /* take NZCV from R[t] */
2220 tcg_gen_extrl_i64_i32(nzcv, tcg_rt);
2222 /* bit 31, N */
2223 tcg_gen_andi_i32(cpu_NF, nzcv, (1U << 31));
2224 /* bit 30, Z */
2225 tcg_gen_andi_i32(cpu_ZF, nzcv, (1 << 30));
2226 tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_ZF, cpu_ZF, 0);
2227 /* bit 29, C */
2228 tcg_gen_andi_i32(cpu_CF, nzcv, (1 << 29));
2229 tcg_gen_shri_i32(cpu_CF, cpu_CF, 29);
2230 /* bit 28, V */
2231 tcg_gen_andi_i32(cpu_VF, nzcv, (1 << 28));
2232 tcg_gen_shli_i32(cpu_VF, cpu_VF, 3);
2235 static void gen_sysreg_undef(DisasContext *s, bool isread,
2236 uint8_t op0, uint8_t op1, uint8_t op2,
2237 uint8_t crn, uint8_t crm, uint8_t rt)
2240 * Generate code to emit an UNDEF with correct syndrome
2241 * information for a failed system register access.
2242 * This is EC_UNCATEGORIZED (ie a standard UNDEF) in most cases,
2243 * but if FEAT_IDST is implemented then read accesses to registers
2244 * in the feature ID space are reported with the EC_SYSTEMREGISTERTRAP
2245 * syndrome.
2247 uint32_t syndrome;
2249 if (isread && dc_isar_feature(aa64_ids, s) &&
2250 arm_cpreg_encoding_in_idspace(op0, op1, op2, crn, crm)) {
2251 syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
2252 } else {
2253 syndrome = syn_uncategorized();
2255 gen_exception_insn(s, 0, EXCP_UDEF, syndrome);
2258 /* MRS - move from system register
2259 * MSR (register) - move to system register
2260 * SYS
2261 * SYSL
2262 * These are all essentially the same insn in 'read' and 'write'
2263 * versions, with varying op0 fields.
2265 static void handle_sys(DisasContext *s, bool isread,
2266 unsigned int op0, unsigned int op1, unsigned int op2,
2267 unsigned int crn, unsigned int crm, unsigned int rt)
2269 uint32_t key = ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP,
2270 crn, crm, op0, op1, op2);
2271 const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key);
2272 bool need_exit_tb = false;
2273 bool nv_trap_to_el2 = false;
2274 bool nv_redirect_reg = false;
2275 bool skip_fp_access_checks = false;
2276 bool nv2_mem_redirect = false;
2277 TCGv_ptr tcg_ri = NULL;
2278 TCGv_i64 tcg_rt;
2279 uint32_t syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
2281 if (crn == 11 || crn == 15) {
2283 * Check for TIDCP trap, which must take precedence over
2284 * the UNDEF for "no such register" etc.
2286 switch (s->current_el) {
2287 case 0:
2288 if (dc_isar_feature(aa64_tidcp1, s)) {
2289 gen_helper_tidcp_el0(tcg_env, tcg_constant_i32(syndrome));
2291 break;
2292 case 1:
2293 gen_helper_tidcp_el1(tcg_env, tcg_constant_i32(syndrome));
2294 break;
2298 if (!ri) {
2299 /* Unknown register; this might be a guest error or a QEMU
2300 * unimplemented feature.
2302 qemu_log_mask(LOG_UNIMP, "%s access to unsupported AArch64 "
2303 "system register op0:%d op1:%d crn:%d crm:%d op2:%d\n",
2304 isread ? "read" : "write", op0, op1, crn, crm, op2);
2305 gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt);
2306 return;
2309 if (s->nv2 && ri->nv2_redirect_offset) {
2311 * Some registers always redirect to memory; some only do so if
2312 * HCR_EL2.NV1 is 0, and some only if NV1 is 1 (these come in
2313 * pairs which share an offset; see the table in R_CSRPQ).
2315 if (ri->nv2_redirect_offset & NV2_REDIR_NV1) {
2316 nv2_mem_redirect = s->nv1;
2317 } else if (ri->nv2_redirect_offset & NV2_REDIR_NO_NV1) {
2318 nv2_mem_redirect = !s->nv1;
2319 } else {
2320 nv2_mem_redirect = true;
2324 /* Check access permissions */
2325 if (!cp_access_ok(s->current_el, ri, isread)) {
2327 * FEAT_NV/NV2 handling does not do the usual FP access checks
2328 * for registers only accessible at EL2 (though it *does* do them
2329 * for registers accessible at EL1).
2331 skip_fp_access_checks = true;
2332 if (s->nv2 && (ri->type & ARM_CP_NV2_REDIRECT)) {
2334 * This is one of the few EL2 registers which should redirect
2335 * to the equivalent EL1 register. We do that after running
2336 * the EL2 register's accessfn.
2338 nv_redirect_reg = true;
2339 assert(!nv2_mem_redirect);
2340 } else if (nv2_mem_redirect) {
2342 * NV2 redirect-to-memory takes precedence over trap to EL2 or
2343 * UNDEF to EL1.
2345 } else if (s->nv && arm_cpreg_traps_in_nv(ri)) {
2347 * This register / instruction exists and is an EL2 register, so
2348 * we must trap to EL2 if accessed in nested virtualization EL1
2349 * instead of UNDEFing. We'll do that after the usual access checks.
2350 * (This makes a difference only for a couple of registers like
2351 * VSTTBR_EL2 where the "UNDEF if NonSecure" should take priority
2352 * over the trap-to-EL2. Most trapped-by-FEAT_NV registers have
2353 * an accessfn which does nothing when called from EL1, because
2354 * the trap-to-EL3 controls which would apply to that register
2355 * at EL2 don't take priority over the FEAT_NV trap-to-EL2.)
2357 nv_trap_to_el2 = true;
2358 } else {
2359 gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt);
2360 return;
2364 if (ri->accessfn || (ri->fgt && s->fgt_active)) {
2365 /* Emit code to perform further access permissions checks at
2366 * runtime; this may result in an exception.
2368 gen_a64_update_pc(s, 0);
2369 tcg_ri = tcg_temp_new_ptr();
2370 gen_helper_access_check_cp_reg(tcg_ri, tcg_env,
2371 tcg_constant_i32(key),
2372 tcg_constant_i32(syndrome),
2373 tcg_constant_i32(isread));
2374 } else if (ri->type & ARM_CP_RAISES_EXC) {
2376 * The readfn or writefn might raise an exception;
2377 * synchronize the CPU state in case it does.
2379 gen_a64_update_pc(s, 0);
2382 if (!skip_fp_access_checks) {
2383 if ((ri->type & ARM_CP_FPU) && !fp_access_check_only(s)) {
2384 return;
2385 } else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) {
2386 return;
2387 } else if ((ri->type & ARM_CP_SME) && !sme_access_check(s)) {
2388 return;
2392 if (nv_trap_to_el2) {
2393 gen_exception_insn_el(s, 0, EXCP_UDEF, syndrome, 2);
2394 return;
2397 if (nv_redirect_reg) {
2399 * FEAT_NV2 redirection of an EL2 register to an EL1 register.
2400 * Conveniently in all cases the encoding of the EL1 register is
2401 * identical to the EL2 register except that opc1 is 0.
2402 * Get the reginfo for the EL1 register to use for the actual access.
2403 * We don't use the EL1 register's access function, and
2404 * fine-grained-traps on EL1 also do not apply here.
2406 key = ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP,
2407 crn, crm, op0, 0, op2);
2408 ri = get_arm_cp_reginfo(s->cp_regs, key);
2409 assert(ri);
2410 assert(cp_access_ok(s->current_el, ri, isread));
2412 * We might not have done an update_pc earlier, so check we don't
2413 * need it. We could support this in future if necessary.
2415 assert(!(ri->type & ARM_CP_RAISES_EXC));
2418 if (nv2_mem_redirect) {
2420 * This system register is being redirected into an EL2 memory access.
2421 * This means it is not an IO operation, doesn't change hflags,
2422 * and need not end the TB, because it has no side effects.
2424 * The access is 64-bit single copy atomic, guaranteed aligned because
2425 * of the definition of VCNR_EL2. Its endianness depends on
2426 * SCTLR_EL2.EE, not on the data endianness of EL1.
2427 * It is done under either the EL2 translation regime or the EL2&0
2428 * translation regime, depending on HCR_EL2.E2H. It behaves as if
2429 * PSTATE.PAN is 0.
2431 TCGv_i64 ptr = tcg_temp_new_i64();
2432 MemOp mop = MO_64 | MO_ALIGN | MO_ATOM_IFALIGN;
2433 ARMMMUIdx armmemidx = s->nv2_mem_e20 ? ARMMMUIdx_E20_2 : ARMMMUIdx_E2;
2434 int memidx = arm_to_core_mmu_idx(armmemidx);
2435 uint32_t syn;
2437 mop |= (s->nv2_mem_be ? MO_BE : MO_LE);
2439 tcg_gen_ld_i64(ptr, tcg_env, offsetof(CPUARMState, cp15.vncr_el2));
2440 tcg_gen_addi_i64(ptr, ptr,
2441 (ri->nv2_redirect_offset & ~NV2_REDIR_FLAG_MASK));
2442 tcg_rt = cpu_reg(s, rt);
2444 syn = syn_data_abort_vncr(0, !isread, 0);
2445 disas_set_insn_syndrome(s, syn);
2446 if (isread) {
2447 tcg_gen_qemu_ld_i64(tcg_rt, ptr, memidx, mop);
2448 } else {
2449 tcg_gen_qemu_st_i64(tcg_rt, ptr, memidx, mop);
2451 return;
2454 /* Handle special cases first */
2455 switch (ri->type & ARM_CP_SPECIAL_MASK) {
2456 case 0:
2457 break;
2458 case ARM_CP_NOP:
2459 return;
2460 case ARM_CP_NZCV:
2461 tcg_rt = cpu_reg(s, rt);
2462 if (isread) {
2463 gen_get_nzcv(tcg_rt);
2464 } else {
2465 gen_set_nzcv(tcg_rt);
2467 return;
2468 case ARM_CP_CURRENTEL:
2471 * Reads as current EL value from pstate, which is
2472 * guaranteed to be constant by the tb flags.
2473 * For nested virt we should report EL2.
2475 int el = s->nv ? 2 : s->current_el;
2476 tcg_rt = cpu_reg(s, rt);
2477 tcg_gen_movi_i64(tcg_rt, el << 2);
2478 return;
2480 case ARM_CP_DC_ZVA:
2481 /* Writes clear the aligned block of memory which rt points into. */
2482 if (s->mte_active[0]) {
2483 int desc = 0;
2485 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
2486 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
2487 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
2489 tcg_rt = tcg_temp_new_i64();
2490 gen_helper_mte_check_zva(tcg_rt, tcg_env,
2491 tcg_constant_i32(desc), cpu_reg(s, rt));
2492 } else {
2493 tcg_rt = clean_data_tbi(s, cpu_reg(s, rt));
2495 gen_helper_dc_zva(tcg_env, tcg_rt);
2496 return;
2497 case ARM_CP_DC_GVA:
2499 TCGv_i64 clean_addr, tag;
2502 * DC_GVA, like DC_ZVA, requires that we supply the original
2503 * pointer for an invalid page. Probe that address first.
2505 tcg_rt = cpu_reg(s, rt);
2506 clean_addr = clean_data_tbi(s, tcg_rt);
2507 gen_probe_access(s, clean_addr, MMU_DATA_STORE, MO_8);
2509 if (s->ata[0]) {
2510 /* Extract the tag from the register to match STZGM. */
2511 tag = tcg_temp_new_i64();
2512 tcg_gen_shri_i64(tag, tcg_rt, 56);
2513 gen_helper_stzgm_tags(tcg_env, clean_addr, tag);
2516 return;
2517 case ARM_CP_DC_GZVA:
2519 TCGv_i64 clean_addr, tag;
2521 /* For DC_GZVA, we can rely on DC_ZVA for the proper fault. */
2522 tcg_rt = cpu_reg(s, rt);
2523 clean_addr = clean_data_tbi(s, tcg_rt);
2524 gen_helper_dc_zva(tcg_env, clean_addr);
2526 if (s->ata[0]) {
2527 /* Extract the tag from the register to match STZGM. */
2528 tag = tcg_temp_new_i64();
2529 tcg_gen_shri_i64(tag, tcg_rt, 56);
2530 gen_helper_stzgm_tags(tcg_env, clean_addr, tag);
2533 return;
2534 default:
2535 g_assert_not_reached();
2538 if (ri->type & ARM_CP_IO) {
2539 /* I/O operations must end the TB here (whether read or write) */
2540 need_exit_tb = translator_io_start(&s->base);
2543 tcg_rt = cpu_reg(s, rt);
2545 if (isread) {
2546 if (ri->type & ARM_CP_CONST) {
2547 tcg_gen_movi_i64(tcg_rt, ri->resetvalue);
2548 } else if (ri->readfn) {
2549 if (!tcg_ri) {
2550 tcg_ri = gen_lookup_cp_reg(key);
2552 gen_helper_get_cp_reg64(tcg_rt, tcg_env, tcg_ri);
2553 } else {
2554 tcg_gen_ld_i64(tcg_rt, tcg_env, ri->fieldoffset);
2556 } else {
2557 if (ri->type & ARM_CP_CONST) {
2558 /* If not forbidden by access permissions, treat as WI */
2559 return;
2560 } else if (ri->writefn) {
2561 if (!tcg_ri) {
2562 tcg_ri = gen_lookup_cp_reg(key);
2564 gen_helper_set_cp_reg64(tcg_env, tcg_ri, tcg_rt);
2565 } else {
2566 tcg_gen_st_i64(tcg_rt, tcg_env, ri->fieldoffset);
2570 if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) {
2572 * A write to any coprocessor register that ends a TB
2573 * must rebuild the hflags for the next TB.
2575 gen_rebuild_hflags(s);
2577 * We default to ending the TB on a coprocessor register write,
2578 * but allow this to be suppressed by the register definition
2579 * (usually only necessary to work around guest bugs).
2581 need_exit_tb = true;
2583 if (need_exit_tb) {
2584 s->base.is_jmp = DISAS_UPDATE_EXIT;
2588 static bool trans_SYS(DisasContext *s, arg_SYS *a)
2590 handle_sys(s, a->l, a->op0, a->op1, a->op2, a->crn, a->crm, a->rt);
2591 return true;
2594 static bool trans_SVC(DisasContext *s, arg_i *a)
2597 * For SVC, HVC and SMC we advance the single-step state
2598 * machine before taking the exception. This is architecturally
2599 * mandated, to ensure that single-stepping a system call
2600 * instruction works properly.
2602 uint32_t syndrome = syn_aa64_svc(a->imm);
2603 if (s->fgt_svc) {
2604 gen_exception_insn_el(s, 0, EXCP_UDEF, syndrome, 2);
2605 return true;
2607 gen_ss_advance(s);
2608 gen_exception_insn(s, 4, EXCP_SWI, syndrome);
2609 return true;
2612 static bool trans_HVC(DisasContext *s, arg_i *a)
2614 int target_el = s->current_el == 3 ? 3 : 2;
2616 if (s->current_el == 0) {
2617 unallocated_encoding(s);
2618 return true;
2621 * The pre HVC helper handles cases when HVC gets trapped
2622 * as an undefined insn by runtime configuration.
2624 gen_a64_update_pc(s, 0);
2625 gen_helper_pre_hvc(tcg_env);
2626 /* Architecture requires ss advance before we do the actual work */
2627 gen_ss_advance(s);
2628 gen_exception_insn_el(s, 4, EXCP_HVC, syn_aa64_hvc(a->imm), target_el);
2629 return true;
2632 static bool trans_SMC(DisasContext *s, arg_i *a)
2634 if (s->current_el == 0) {
2635 unallocated_encoding(s);
2636 return true;
2638 gen_a64_update_pc(s, 0);
2639 gen_helper_pre_smc(tcg_env, tcg_constant_i32(syn_aa64_smc(a->imm)));
2640 /* Architecture requires ss advance before we do the actual work */
2641 gen_ss_advance(s);
2642 gen_exception_insn_el(s, 4, EXCP_SMC, syn_aa64_smc(a->imm), 3);
2643 return true;
2646 static bool trans_BRK(DisasContext *s, arg_i *a)
2648 gen_exception_bkpt_insn(s, syn_aa64_bkpt(a->imm));
2649 return true;
2652 static bool trans_HLT(DisasContext *s, arg_i *a)
2655 * HLT. This has two purposes.
2656 * Architecturally, it is an external halting debug instruction.
2657 * Since QEMU doesn't implement external debug, we treat this as
2658 * it is required for halting debug disabled: it will UNDEF.
2659 * Secondly, "HLT 0xf000" is the A64 semihosting syscall instruction.
2661 if (semihosting_enabled(s->current_el == 0) && a->imm == 0xf000) {
2662 gen_exception_internal_insn(s, EXCP_SEMIHOST);
2663 } else {
2664 unallocated_encoding(s);
2666 return true;
2670 * Load/Store exclusive instructions are implemented by remembering
2671 * the value/address loaded, and seeing if these are the same
2672 * when the store is performed. This is not actually the architecturally
2673 * mandated semantics, but it works for typical guest code sequences
2674 * and avoids having to monitor regular stores.
2676 * The store exclusive uses the atomic cmpxchg primitives to avoid
2677 * races in multi-threaded linux-user and when MTTCG softmmu is
2678 * enabled.
2680 static void gen_load_exclusive(DisasContext *s, int rt, int rt2, int rn,
2681 int size, bool is_pair)
2683 int idx = get_mem_index(s);
2684 TCGv_i64 dirty_addr, clean_addr;
2685 MemOp memop = check_atomic_align(s, rn, size + is_pair);
2687 s->is_ldex = true;
2688 dirty_addr = cpu_reg_sp(s, rn);
2689 clean_addr = gen_mte_check1(s, dirty_addr, false, rn != 31, memop);
2691 g_assert(size <= 3);
2692 if (is_pair) {
2693 g_assert(size >= 2);
2694 if (size == 2) {
2695 tcg_gen_qemu_ld_i64(cpu_exclusive_val, clean_addr, idx, memop);
2696 if (s->be_data == MO_LE) {
2697 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 0, 32);
2698 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 32, 32);
2699 } else {
2700 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 32, 32);
2701 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 0, 32);
2703 } else {
2704 TCGv_i128 t16 = tcg_temp_new_i128();
2706 tcg_gen_qemu_ld_i128(t16, clean_addr, idx, memop);
2708 if (s->be_data == MO_LE) {
2709 tcg_gen_extr_i128_i64(cpu_exclusive_val,
2710 cpu_exclusive_high, t16);
2711 } else {
2712 tcg_gen_extr_i128_i64(cpu_exclusive_high,
2713 cpu_exclusive_val, t16);
2715 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
2716 tcg_gen_mov_i64(cpu_reg(s, rt2), cpu_exclusive_high);
2718 } else {
2719 tcg_gen_qemu_ld_i64(cpu_exclusive_val, clean_addr, idx, memop);
2720 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
2722 tcg_gen_mov_i64(cpu_exclusive_addr, clean_addr);
2725 static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
2726 int rn, int size, int is_pair)
2728 /* if (env->exclusive_addr == addr && env->exclusive_val == [addr]
2729 * && (!is_pair || env->exclusive_high == [addr + datasize])) {
2730 * [addr] = {Rt};
2731 * if (is_pair) {
2732 * [addr + datasize] = {Rt2};
2734 * {Rd} = 0;
2735 * } else {
2736 * {Rd} = 1;
2738 * env->exclusive_addr = -1;
2740 TCGLabel *fail_label = gen_new_label();
2741 TCGLabel *done_label = gen_new_label();
2742 TCGv_i64 tmp, clean_addr;
2743 MemOp memop;
2746 * FIXME: We are out of spec here. We have recorded only the address
2747 * from load_exclusive, not the entire range, and we assume that the
2748 * size of the access on both sides match. The architecture allows the
2749 * store to be smaller than the load, so long as the stored bytes are
2750 * within the range recorded by the load.
2753 /* See AArch64.ExclusiveMonitorsPass() and AArch64.IsExclusiveVA(). */
2754 clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
2755 tcg_gen_brcond_i64(TCG_COND_NE, clean_addr, cpu_exclusive_addr, fail_label);
2758 * The write, and any associated faults, only happen if the virtual
2759 * and physical addresses pass the exclusive monitor check. These
2760 * faults are exceedingly unlikely, because normally the guest uses
2761 * the exact same address register for the load_exclusive, and we
2762 * would have recognized these faults there.
2764 * It is possible to trigger an alignment fault pre-LSE2, e.g. with an
2765 * unaligned 4-byte write within the range of an aligned 8-byte load.
2766 * With LSE2, the store would need to cross a 16-byte boundary when the
2767 * load did not, which would mean the store is outside the range
2768 * recorded for the monitor, which would have failed a corrected monitor
2769 * check above. For now, we assume no size change and retain the
2770 * MO_ALIGN to let tcg know what we checked in the load_exclusive.
2772 * It is possible to trigger an MTE fault, by performing the load with
2773 * a virtual address with a valid tag and performing the store with the
2774 * same virtual address and a different invalid tag.
2776 memop = size + is_pair;
2777 if (memop == MO_128 || !dc_isar_feature(aa64_lse2, s)) {
2778 memop |= MO_ALIGN;
2780 memop = finalize_memop(s, memop);
2781 gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
2783 tmp = tcg_temp_new_i64();
2784 if (is_pair) {
2785 if (size == 2) {
2786 if (s->be_data == MO_LE) {
2787 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt), cpu_reg(s, rt2));
2788 } else {
2789 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt2), cpu_reg(s, rt));
2791 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr,
2792 cpu_exclusive_val, tmp,
2793 get_mem_index(s), memop);
2794 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
2795 } else {
2796 TCGv_i128 t16 = tcg_temp_new_i128();
2797 TCGv_i128 c16 = tcg_temp_new_i128();
2798 TCGv_i64 a, b;
2800 if (s->be_data == MO_LE) {
2801 tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt), cpu_reg(s, rt2));
2802 tcg_gen_concat_i64_i128(c16, cpu_exclusive_val,
2803 cpu_exclusive_high);
2804 } else {
2805 tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt2), cpu_reg(s, rt));
2806 tcg_gen_concat_i64_i128(c16, cpu_exclusive_high,
2807 cpu_exclusive_val);
2810 tcg_gen_atomic_cmpxchg_i128(t16, cpu_exclusive_addr, c16, t16,
2811 get_mem_index(s), memop);
2813 a = tcg_temp_new_i64();
2814 b = tcg_temp_new_i64();
2815 if (s->be_data == MO_LE) {
2816 tcg_gen_extr_i128_i64(a, b, t16);
2817 } else {
2818 tcg_gen_extr_i128_i64(b, a, t16);
2821 tcg_gen_xor_i64(a, a, cpu_exclusive_val);
2822 tcg_gen_xor_i64(b, b, cpu_exclusive_high);
2823 tcg_gen_or_i64(tmp, a, b);
2825 tcg_gen_setcondi_i64(TCG_COND_NE, tmp, tmp, 0);
2827 } else {
2828 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val,
2829 cpu_reg(s, rt), get_mem_index(s), memop);
2830 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
2832 tcg_gen_mov_i64(cpu_reg(s, rd), tmp);
2833 tcg_gen_br(done_label);
2835 gen_set_label(fail_label);
2836 tcg_gen_movi_i64(cpu_reg(s, rd), 1);
2837 gen_set_label(done_label);
2838 tcg_gen_movi_i64(cpu_exclusive_addr, -1);
2841 static void gen_compare_and_swap(DisasContext *s, int rs, int rt,
2842 int rn, int size)
2844 TCGv_i64 tcg_rs = cpu_reg(s, rs);
2845 TCGv_i64 tcg_rt = cpu_reg(s, rt);
2846 int memidx = get_mem_index(s);
2847 TCGv_i64 clean_addr;
2848 MemOp memop;
2850 if (rn == 31) {
2851 gen_check_sp_alignment(s);
2853 memop = check_atomic_align(s, rn, size);
2854 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
2855 tcg_gen_atomic_cmpxchg_i64(tcg_rs, clean_addr, tcg_rs, tcg_rt,
2856 memidx, memop);
2859 static void gen_compare_and_swap_pair(DisasContext *s, int rs, int rt,
2860 int rn, int size)
2862 TCGv_i64 s1 = cpu_reg(s, rs);
2863 TCGv_i64 s2 = cpu_reg(s, rs + 1);
2864 TCGv_i64 t1 = cpu_reg(s, rt);
2865 TCGv_i64 t2 = cpu_reg(s, rt + 1);
2866 TCGv_i64 clean_addr;
2867 int memidx = get_mem_index(s);
2868 MemOp memop;
2870 if (rn == 31) {
2871 gen_check_sp_alignment(s);
2874 /* This is a single atomic access, despite the "pair". */
2875 memop = check_atomic_align(s, rn, size + 1);
2876 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
2878 if (size == 2) {
2879 TCGv_i64 cmp = tcg_temp_new_i64();
2880 TCGv_i64 val = tcg_temp_new_i64();
2882 if (s->be_data == MO_LE) {
2883 tcg_gen_concat32_i64(val, t1, t2);
2884 tcg_gen_concat32_i64(cmp, s1, s2);
2885 } else {
2886 tcg_gen_concat32_i64(val, t2, t1);
2887 tcg_gen_concat32_i64(cmp, s2, s1);
2890 tcg_gen_atomic_cmpxchg_i64(cmp, clean_addr, cmp, val, memidx, memop);
2892 if (s->be_data == MO_LE) {
2893 tcg_gen_extr32_i64(s1, s2, cmp);
2894 } else {
2895 tcg_gen_extr32_i64(s2, s1, cmp);
2897 } else {
2898 TCGv_i128 cmp = tcg_temp_new_i128();
2899 TCGv_i128 val = tcg_temp_new_i128();
2901 if (s->be_data == MO_LE) {
2902 tcg_gen_concat_i64_i128(val, t1, t2);
2903 tcg_gen_concat_i64_i128(cmp, s1, s2);
2904 } else {
2905 tcg_gen_concat_i64_i128(val, t2, t1);
2906 tcg_gen_concat_i64_i128(cmp, s2, s1);
2909 tcg_gen_atomic_cmpxchg_i128(cmp, clean_addr, cmp, val, memidx, memop);
2911 if (s->be_data == MO_LE) {
2912 tcg_gen_extr_i128_i64(s1, s2, cmp);
2913 } else {
2914 tcg_gen_extr_i128_i64(s2, s1, cmp);
2920 * Compute the ISS.SF bit for syndrome information if an exception
2921 * is taken on a load or store. This indicates whether the instruction
2922 * is accessing a 32-bit or 64-bit register. This logic is derived
2923 * from the ARMv8 specs for LDR (Shared decode for all encodings).
2925 static bool ldst_iss_sf(int size, bool sign, bool ext)
2928 if (sign) {
2930 * Signed loads are 64 bit results if we are not going to
2931 * do a zero-extend from 32 to 64 after the load.
2932 * (For a store, sign and ext are always false.)
2934 return !ext;
2935 } else {
2936 /* Unsigned loads/stores work at the specified size */
2937 return size == MO_64;
2941 static bool trans_STXR(DisasContext *s, arg_stxr *a)
2943 if (a->rn == 31) {
2944 gen_check_sp_alignment(s);
2946 if (a->lasr) {
2947 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2949 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, false);
2950 return true;
2953 static bool trans_LDXR(DisasContext *s, arg_stxr *a)
2955 if (a->rn == 31) {
2956 gen_check_sp_alignment(s);
2958 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, false);
2959 if (a->lasr) {
2960 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
2962 return true;
2965 static bool trans_STLR(DisasContext *s, arg_stlr *a)
2967 TCGv_i64 clean_addr;
2968 MemOp memop;
2969 bool iss_sf = ldst_iss_sf(a->sz, false, false);
2972 * StoreLORelease is the same as Store-Release for QEMU, but
2973 * needs the feature-test.
2975 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) {
2976 return false;
2978 /* Generate ISS for non-exclusive accesses including LASR. */
2979 if (a->rn == 31) {
2980 gen_check_sp_alignment(s);
2982 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2983 memop = check_ordered_align(s, a->rn, 0, true, a->sz);
2984 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn),
2985 true, a->rn != 31, memop);
2986 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, memop, true, a->rt,
2987 iss_sf, a->lasr);
2988 return true;
2991 static bool trans_LDAR(DisasContext *s, arg_stlr *a)
2993 TCGv_i64 clean_addr;
2994 MemOp memop;
2995 bool iss_sf = ldst_iss_sf(a->sz, false, false);
2997 /* LoadLOAcquire is the same as Load-Acquire for QEMU. */
2998 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) {
2999 return false;
3001 /* Generate ISS for non-exclusive accesses including LASR. */
3002 if (a->rn == 31) {
3003 gen_check_sp_alignment(s);
3005 memop = check_ordered_align(s, a->rn, 0, false, a->sz);
3006 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn),
3007 false, a->rn != 31, memop);
3008 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, memop, false, true,
3009 a->rt, iss_sf, a->lasr);
3010 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
3011 return true;
3014 static bool trans_STXP(DisasContext *s, arg_stxr *a)
3016 if (a->rn == 31) {
3017 gen_check_sp_alignment(s);
3019 if (a->lasr) {
3020 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
3022 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, true);
3023 return true;
3026 static bool trans_LDXP(DisasContext *s, arg_stxr *a)
3028 if (a->rn == 31) {
3029 gen_check_sp_alignment(s);
3031 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, true);
3032 if (a->lasr) {
3033 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
3035 return true;
3038 static bool trans_CASP(DisasContext *s, arg_CASP *a)
3040 if (!dc_isar_feature(aa64_atomics, s)) {
3041 return false;
3043 if (((a->rt | a->rs) & 1) != 0) {
3044 return false;
3047 gen_compare_and_swap_pair(s, a->rs, a->rt, a->rn, a->sz);
3048 return true;
3051 static bool trans_CAS(DisasContext *s, arg_CAS *a)
3053 if (!dc_isar_feature(aa64_atomics, s)) {
3054 return false;
3056 gen_compare_and_swap(s, a->rs, a->rt, a->rn, a->sz);
3057 return true;
3060 static bool trans_LD_lit(DisasContext *s, arg_ldlit *a)
3062 bool iss_sf = ldst_iss_sf(a->sz, a->sign, false);
3063 TCGv_i64 tcg_rt = cpu_reg(s, a->rt);
3064 TCGv_i64 clean_addr = tcg_temp_new_i64();
3065 MemOp memop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3067 gen_pc_plus_diff(s, clean_addr, a->imm);
3068 do_gpr_ld(s, tcg_rt, clean_addr, memop,
3069 false, true, a->rt, iss_sf, false);
3070 return true;
3073 static bool trans_LD_lit_v(DisasContext *s, arg_ldlit *a)
3075 /* Load register (literal), vector version */
3076 TCGv_i64 clean_addr;
3077 MemOp memop;
3079 if (!fp_access_check(s)) {
3080 return true;
3082 memop = finalize_memop_asimd(s, a->sz);
3083 clean_addr = tcg_temp_new_i64();
3084 gen_pc_plus_diff(s, clean_addr, a->imm);
3085 do_fp_ld(s, a->rt, clean_addr, memop);
3086 return true;
3089 static void op_addr_ldstpair_pre(DisasContext *s, arg_ldstpair *a,
3090 TCGv_i64 *clean_addr, TCGv_i64 *dirty_addr,
3091 uint64_t offset, bool is_store, MemOp mop)
3093 if (a->rn == 31) {
3094 gen_check_sp_alignment(s);
3097 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3098 if (!a->p) {
3099 tcg_gen_addi_i64(*dirty_addr, *dirty_addr, offset);
3102 *clean_addr = gen_mte_checkN(s, *dirty_addr, is_store,
3103 (a->w || a->rn != 31), 2 << a->sz, mop);
3106 static void op_addr_ldstpair_post(DisasContext *s, arg_ldstpair *a,
3107 TCGv_i64 dirty_addr, uint64_t offset)
3109 if (a->w) {
3110 if (a->p) {
3111 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3113 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
3117 static bool trans_STP(DisasContext *s, arg_ldstpair *a)
3119 uint64_t offset = a->imm << a->sz;
3120 TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
3121 MemOp mop = finalize_memop(s, a->sz);
3123 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, true, mop);
3124 tcg_rt = cpu_reg(s, a->rt);
3125 tcg_rt2 = cpu_reg(s, a->rt2);
3127 * We built mop above for the single logical access -- rebuild it
3128 * now for the paired operation.
3130 * With LSE2, non-sign-extending pairs are treated atomically if
3131 * aligned, and if unaligned one of the pair will be completely
3132 * within a 16-byte block and that element will be atomic.
3133 * Otherwise each element is separately atomic.
3134 * In all cases, issue one operation with the correct atomicity.
3136 mop = a->sz + 1;
3137 if (s->align_mem) {
3138 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8);
3140 mop = finalize_memop_pair(s, mop);
3141 if (a->sz == 2) {
3142 TCGv_i64 tmp = tcg_temp_new_i64();
3144 if (s->be_data == MO_LE) {
3145 tcg_gen_concat32_i64(tmp, tcg_rt, tcg_rt2);
3146 } else {
3147 tcg_gen_concat32_i64(tmp, tcg_rt2, tcg_rt);
3149 tcg_gen_qemu_st_i64(tmp, clean_addr, get_mem_index(s), mop);
3150 } else {
3151 TCGv_i128 tmp = tcg_temp_new_i128();
3153 if (s->be_data == MO_LE) {
3154 tcg_gen_concat_i64_i128(tmp, tcg_rt, tcg_rt2);
3155 } else {
3156 tcg_gen_concat_i64_i128(tmp, tcg_rt2, tcg_rt);
3158 tcg_gen_qemu_st_i128(tmp, clean_addr, get_mem_index(s), mop);
3160 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3161 return true;
3164 static bool trans_LDP(DisasContext *s, arg_ldstpair *a)
3166 uint64_t offset = a->imm << a->sz;
3167 TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
3168 MemOp mop = finalize_memop(s, a->sz);
3170 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, false, mop);
3171 tcg_rt = cpu_reg(s, a->rt);
3172 tcg_rt2 = cpu_reg(s, a->rt2);
3175 * We built mop above for the single logical access -- rebuild it
3176 * now for the paired operation.
3178 * With LSE2, non-sign-extending pairs are treated atomically if
3179 * aligned, and if unaligned one of the pair will be completely
3180 * within a 16-byte block and that element will be atomic.
3181 * Otherwise each element is separately atomic.
3182 * In all cases, issue one operation with the correct atomicity.
3184 * This treats sign-extending loads like zero-extending loads,
3185 * since that reuses the most code below.
3187 mop = a->sz + 1;
3188 if (s->align_mem) {
3189 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8);
3191 mop = finalize_memop_pair(s, mop);
3192 if (a->sz == 2) {
3193 int o2 = s->be_data == MO_LE ? 32 : 0;
3194 int o1 = o2 ^ 32;
3196 tcg_gen_qemu_ld_i64(tcg_rt, clean_addr, get_mem_index(s), mop);
3197 if (a->sign) {
3198 tcg_gen_sextract_i64(tcg_rt2, tcg_rt, o2, 32);
3199 tcg_gen_sextract_i64(tcg_rt, tcg_rt, o1, 32);
3200 } else {
3201 tcg_gen_extract_i64(tcg_rt2, tcg_rt, o2, 32);
3202 tcg_gen_extract_i64(tcg_rt, tcg_rt, o1, 32);
3204 } else {
3205 TCGv_i128 tmp = tcg_temp_new_i128();
3207 tcg_gen_qemu_ld_i128(tmp, clean_addr, get_mem_index(s), mop);
3208 if (s->be_data == MO_LE) {
3209 tcg_gen_extr_i128_i64(tcg_rt, tcg_rt2, tmp);
3210 } else {
3211 tcg_gen_extr_i128_i64(tcg_rt2, tcg_rt, tmp);
3214 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3215 return true;
3218 static bool trans_STP_v(DisasContext *s, arg_ldstpair *a)
3220 uint64_t offset = a->imm << a->sz;
3221 TCGv_i64 clean_addr, dirty_addr;
3222 MemOp mop;
3224 if (!fp_access_check(s)) {
3225 return true;
3228 /* LSE2 does not merge FP pairs; leave these as separate operations. */
3229 mop = finalize_memop_asimd(s, a->sz);
3230 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, true, mop);
3231 do_fp_st(s, a->rt, clean_addr, mop);
3232 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz);
3233 do_fp_st(s, a->rt2, clean_addr, mop);
3234 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3235 return true;
3238 static bool trans_LDP_v(DisasContext *s, arg_ldstpair *a)
3240 uint64_t offset = a->imm << a->sz;
3241 TCGv_i64 clean_addr, dirty_addr;
3242 MemOp mop;
3244 if (!fp_access_check(s)) {
3245 return true;
3248 /* LSE2 does not merge FP pairs; leave these as separate operations. */
3249 mop = finalize_memop_asimd(s, a->sz);
3250 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, false, mop);
3251 do_fp_ld(s, a->rt, clean_addr, mop);
3252 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz);
3253 do_fp_ld(s, a->rt2, clean_addr, mop);
3254 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3255 return true;
3258 static bool trans_STGP(DisasContext *s, arg_ldstpair *a)
3260 TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
3261 uint64_t offset = a->imm << LOG2_TAG_GRANULE;
3262 MemOp mop;
3263 TCGv_i128 tmp;
3265 /* STGP only comes in one size. */
3266 tcg_debug_assert(a->sz == MO_64);
3268 if (!dc_isar_feature(aa64_mte_insn_reg, s)) {
3269 return false;
3272 if (a->rn == 31) {
3273 gen_check_sp_alignment(s);
3276 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3277 if (!a->p) {
3278 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3281 clean_addr = clean_data_tbi(s, dirty_addr);
3282 tcg_rt = cpu_reg(s, a->rt);
3283 tcg_rt2 = cpu_reg(s, a->rt2);
3286 * STGP is defined as two 8-byte memory operations, aligned to TAG_GRANULE,
3287 * and one tag operation. We implement it as one single aligned 16-byte
3288 * memory operation for convenience. Note that the alignment ensures
3289 * MO_ATOM_IFALIGN_PAIR produces 8-byte atomicity for the memory store.
3291 mop = finalize_memop_atom(s, MO_128 | MO_ALIGN, MO_ATOM_IFALIGN_PAIR);
3293 tmp = tcg_temp_new_i128();
3294 if (s->be_data == MO_LE) {
3295 tcg_gen_concat_i64_i128(tmp, tcg_rt, tcg_rt2);
3296 } else {
3297 tcg_gen_concat_i64_i128(tmp, tcg_rt2, tcg_rt);
3299 tcg_gen_qemu_st_i128(tmp, clean_addr, get_mem_index(s), mop);
3301 /* Perform the tag store, if tag access enabled. */
3302 if (s->ata[0]) {
3303 if (tb_cflags(s->base.tb) & CF_PARALLEL) {
3304 gen_helper_stg_parallel(tcg_env, dirty_addr, dirty_addr);
3305 } else {
3306 gen_helper_stg(tcg_env, dirty_addr, dirty_addr);
3310 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3311 return true;
3314 static void op_addr_ldst_imm_pre(DisasContext *s, arg_ldst_imm *a,
3315 TCGv_i64 *clean_addr, TCGv_i64 *dirty_addr,
3316 uint64_t offset, bool is_store, MemOp mop)
3318 int memidx;
3320 if (a->rn == 31) {
3321 gen_check_sp_alignment(s);
3324 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3325 if (!a->p) {
3326 tcg_gen_addi_i64(*dirty_addr, *dirty_addr, offset);
3328 memidx = get_a64_user_mem_index(s, a->unpriv);
3329 *clean_addr = gen_mte_check1_mmuidx(s, *dirty_addr, is_store,
3330 a->w || a->rn != 31,
3331 mop, a->unpriv, memidx);
3334 static void op_addr_ldst_imm_post(DisasContext *s, arg_ldst_imm *a,
3335 TCGv_i64 dirty_addr, uint64_t offset)
3337 if (a->w) {
3338 if (a->p) {
3339 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3341 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
3345 static bool trans_STR_i(DisasContext *s, arg_ldst_imm *a)
3347 bool iss_sf, iss_valid = !a->w;
3348 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3349 int memidx = get_a64_user_mem_index(s, a->unpriv);
3350 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3352 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop);
3354 tcg_rt = cpu_reg(s, a->rt);
3355 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3357 do_gpr_st_memidx(s, tcg_rt, clean_addr, mop, memidx,
3358 iss_valid, a->rt, iss_sf, false);
3359 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3360 return true;
3363 static bool trans_LDR_i(DisasContext *s, arg_ldst_imm *a)
3365 bool iss_sf, iss_valid = !a->w;
3366 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3367 int memidx = get_a64_user_mem_index(s, a->unpriv);
3368 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3370 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop);
3372 tcg_rt = cpu_reg(s, a->rt);
3373 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3375 do_gpr_ld_memidx(s, tcg_rt, clean_addr, mop,
3376 a->ext, memidx, iss_valid, a->rt, iss_sf, false);
3377 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3378 return true;
3381 static bool trans_STR_v_i(DisasContext *s, arg_ldst_imm *a)
3383 TCGv_i64 clean_addr, dirty_addr;
3384 MemOp mop;
3386 if (!fp_access_check(s)) {
3387 return true;
3389 mop = finalize_memop_asimd(s, a->sz);
3390 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop);
3391 do_fp_st(s, a->rt, clean_addr, mop);
3392 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3393 return true;
3396 static bool trans_LDR_v_i(DisasContext *s, arg_ldst_imm *a)
3398 TCGv_i64 clean_addr, dirty_addr;
3399 MemOp mop;
3401 if (!fp_access_check(s)) {
3402 return true;
3404 mop = finalize_memop_asimd(s, a->sz);
3405 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop);
3406 do_fp_ld(s, a->rt, clean_addr, mop);
3407 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3408 return true;
3411 static void op_addr_ldst_pre(DisasContext *s, arg_ldst *a,
3412 TCGv_i64 *clean_addr, TCGv_i64 *dirty_addr,
3413 bool is_store, MemOp memop)
3415 TCGv_i64 tcg_rm;
3417 if (a->rn == 31) {
3418 gen_check_sp_alignment(s);
3420 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3422 tcg_rm = read_cpu_reg(s, a->rm, 1);
3423 ext_and_shift_reg(tcg_rm, tcg_rm, a->opt, a->s ? a->sz : 0);
3425 tcg_gen_add_i64(*dirty_addr, *dirty_addr, tcg_rm);
3426 *clean_addr = gen_mte_check1(s, *dirty_addr, is_store, true, memop);
3429 static bool trans_LDR(DisasContext *s, arg_ldst *a)
3431 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3432 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3433 MemOp memop;
3435 if (extract32(a->opt, 1, 1) == 0) {
3436 return false;
3439 memop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3440 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, false, memop);
3441 tcg_rt = cpu_reg(s, a->rt);
3442 do_gpr_ld(s, tcg_rt, clean_addr, memop,
3443 a->ext, true, a->rt, iss_sf, false);
3444 return true;
3447 static bool trans_STR(DisasContext *s, arg_ldst *a)
3449 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3450 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3451 MemOp memop;
3453 if (extract32(a->opt, 1, 1) == 0) {
3454 return false;
3457 memop = finalize_memop(s, a->sz);
3458 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, true, memop);
3459 tcg_rt = cpu_reg(s, a->rt);
3460 do_gpr_st(s, tcg_rt, clean_addr, memop, true, a->rt, iss_sf, false);
3461 return true;
3464 static bool trans_LDR_v(DisasContext *s, arg_ldst *a)
3466 TCGv_i64 clean_addr, dirty_addr;
3467 MemOp memop;
3469 if (extract32(a->opt, 1, 1) == 0) {
3470 return false;
3473 if (!fp_access_check(s)) {
3474 return true;
3477 memop = finalize_memop_asimd(s, a->sz);
3478 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, false, memop);
3479 do_fp_ld(s, a->rt, clean_addr, memop);
3480 return true;
3483 static bool trans_STR_v(DisasContext *s, arg_ldst *a)
3485 TCGv_i64 clean_addr, dirty_addr;
3486 MemOp memop;
3488 if (extract32(a->opt, 1, 1) == 0) {
3489 return false;
3492 if (!fp_access_check(s)) {
3493 return true;
3496 memop = finalize_memop_asimd(s, a->sz);
3497 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, true, memop);
3498 do_fp_st(s, a->rt, clean_addr, memop);
3499 return true;
3503 static bool do_atomic_ld(DisasContext *s, arg_atomic *a, AtomicThreeOpFn *fn,
3504 int sign, bool invert)
3506 MemOp mop = a->sz | sign;
3507 TCGv_i64 clean_addr, tcg_rs, tcg_rt;
3509 if (a->rn == 31) {
3510 gen_check_sp_alignment(s);
3512 mop = check_atomic_align(s, a->rn, mop);
3513 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false,
3514 a->rn != 31, mop);
3515 tcg_rs = read_cpu_reg(s, a->rs, true);
3516 tcg_rt = cpu_reg(s, a->rt);
3517 if (invert) {
3518 tcg_gen_not_i64(tcg_rs, tcg_rs);
3521 * The tcg atomic primitives are all full barriers. Therefore we
3522 * can ignore the Acquire and Release bits of this instruction.
3524 fn(tcg_rt, clean_addr, tcg_rs, get_mem_index(s), mop);
3526 if (mop & MO_SIGN) {
3527 switch (a->sz) {
3528 case MO_8:
3529 tcg_gen_ext8u_i64(tcg_rt, tcg_rt);
3530 break;
3531 case MO_16:
3532 tcg_gen_ext16u_i64(tcg_rt, tcg_rt);
3533 break;
3534 case MO_32:
3535 tcg_gen_ext32u_i64(tcg_rt, tcg_rt);
3536 break;
3537 case MO_64:
3538 break;
3539 default:
3540 g_assert_not_reached();
3543 return true;
3546 TRANS_FEAT(LDADD, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_add_i64, 0, false)
3547 TRANS_FEAT(LDCLR, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_and_i64, 0, true)
3548 TRANS_FEAT(LDEOR, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_xor_i64, 0, false)
3549 TRANS_FEAT(LDSET, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_or_i64, 0, false)
3550 TRANS_FEAT(LDSMAX, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_smax_i64, MO_SIGN, false)
3551 TRANS_FEAT(LDSMIN, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_smin_i64, MO_SIGN, false)
3552 TRANS_FEAT(LDUMAX, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_umax_i64, 0, false)
3553 TRANS_FEAT(LDUMIN, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_umin_i64, 0, false)
3554 TRANS_FEAT(SWP, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_xchg_i64, 0, false)
3556 static bool trans_LDAPR(DisasContext *s, arg_LDAPR *a)
3558 bool iss_sf = ldst_iss_sf(a->sz, false, false);
3559 TCGv_i64 clean_addr;
3560 MemOp mop;
3562 if (!dc_isar_feature(aa64_atomics, s) ||
3563 !dc_isar_feature(aa64_rcpc_8_3, s)) {
3564 return false;
3566 if (a->rn == 31) {
3567 gen_check_sp_alignment(s);
3569 mop = check_ordered_align(s, a->rn, 0, false, a->sz);
3570 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false,
3571 a->rn != 31, mop);
3573 * LDAPR* are a special case because they are a simple load, not a
3574 * fetch-and-do-something op.
3575 * The architectural consistency requirements here are weaker than
3576 * full load-acquire (we only need "load-acquire processor consistent"),
3577 * but we choose to implement them as full LDAQ.
3579 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, false,
3580 true, a->rt, iss_sf, true);
3581 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
3582 return true;
3585 static bool trans_LDRA(DisasContext *s, arg_LDRA *a)
3587 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3588 MemOp memop;
3590 /* Load with pointer authentication */
3591 if (!dc_isar_feature(aa64_pauth, s)) {
3592 return false;
3595 if (a->rn == 31) {
3596 gen_check_sp_alignment(s);
3598 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3600 if (s->pauth_active) {
3601 if (!a->m) {
3602 gen_helper_autda_combined(dirty_addr, tcg_env, dirty_addr,
3603 tcg_constant_i64(0));
3604 } else {
3605 gen_helper_autdb_combined(dirty_addr, tcg_env, dirty_addr,
3606 tcg_constant_i64(0));
3610 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
3612 memop = finalize_memop(s, MO_64);
3614 /* Note that "clean" and "dirty" here refer to TBI not PAC. */
3615 clean_addr = gen_mte_check1(s, dirty_addr, false,
3616 a->w || a->rn != 31, memop);
3618 tcg_rt = cpu_reg(s, a->rt);
3619 do_gpr_ld(s, tcg_rt, clean_addr, memop,
3620 /* extend */ false, /* iss_valid */ !a->w,
3621 /* iss_srt */ a->rt, /* iss_sf */ true, /* iss_ar */ false);
3623 if (a->w) {
3624 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
3626 return true;
3629 static bool trans_LDAPR_i(DisasContext *s, arg_ldapr_stlr_i *a)
3631 TCGv_i64 clean_addr, dirty_addr;
3632 MemOp mop = a->sz | (a->sign ? MO_SIGN : 0);
3633 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3635 if (!dc_isar_feature(aa64_rcpc_8_4, s)) {
3636 return false;
3639 if (a->rn == 31) {
3640 gen_check_sp_alignment(s);
3643 mop = check_ordered_align(s, a->rn, a->imm, false, mop);
3644 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3645 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
3646 clean_addr = clean_data_tbi(s, dirty_addr);
3649 * Load-AcquirePC semantics; we implement as the slightly more
3650 * restrictive Load-Acquire.
3652 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, a->ext, true,
3653 a->rt, iss_sf, true);
3654 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
3655 return true;
3658 static bool trans_STLR_i(DisasContext *s, arg_ldapr_stlr_i *a)
3660 TCGv_i64 clean_addr, dirty_addr;
3661 MemOp mop = a->sz;
3662 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3664 if (!dc_isar_feature(aa64_rcpc_8_4, s)) {
3665 return false;
3668 /* TODO: ARMv8.4-LSE SCTLR.nAA */
3670 if (a->rn == 31) {
3671 gen_check_sp_alignment(s);
3674 mop = check_ordered_align(s, a->rn, a->imm, true, mop);
3675 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3676 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
3677 clean_addr = clean_data_tbi(s, dirty_addr);
3679 /* Store-Release semantics */
3680 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
3681 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, mop, true, a->rt, iss_sf, true);
3682 return true;
3685 static bool trans_LD_mult(DisasContext *s, arg_ldst_mult *a)
3687 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3688 MemOp endian, align, mop;
3690 int total; /* total bytes */
3691 int elements; /* elements per vector */
3692 int r;
3693 int size = a->sz;
3695 if (!a->p && a->rm != 0) {
3696 /* For non-postindexed accesses the Rm field must be 0 */
3697 return false;
3699 if (size == 3 && !a->q && a->selem != 1) {
3700 return false;
3702 if (!fp_access_check(s)) {
3703 return true;
3706 if (a->rn == 31) {
3707 gen_check_sp_alignment(s);
3710 /* For our purposes, bytes are always little-endian. */
3711 endian = s->be_data;
3712 if (size == 0) {
3713 endian = MO_LE;
3716 total = a->rpt * a->selem * (a->q ? 16 : 8);
3717 tcg_rn = cpu_reg_sp(s, a->rn);
3720 * Issue the MTE check vs the logical repeat count, before we
3721 * promote consecutive little-endian elements below.
3723 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, total,
3724 finalize_memop_asimd(s, size));
3727 * Consecutive little-endian elements from a single register
3728 * can be promoted to a larger little-endian operation.
3730 align = MO_ALIGN;
3731 if (a->selem == 1 && endian == MO_LE) {
3732 align = pow2_align(size);
3733 size = 3;
3735 if (!s->align_mem) {
3736 align = 0;
3738 mop = endian | size | align;
3740 elements = (a->q ? 16 : 8) >> size;
3741 tcg_ebytes = tcg_constant_i64(1 << size);
3742 for (r = 0; r < a->rpt; r++) {
3743 int e;
3744 for (e = 0; e < elements; e++) {
3745 int xs;
3746 for (xs = 0; xs < a->selem; xs++) {
3747 int tt = (a->rt + r + xs) % 32;
3748 do_vec_ld(s, tt, e, clean_addr, mop);
3749 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3755 * For non-quad operations, setting a slice of the low 64 bits of
3756 * the register clears the high 64 bits (in the ARM ARM pseudocode
3757 * this is implicit in the fact that 'rval' is a 64 bit wide
3758 * variable). For quad operations, we might still need to zero
3759 * the high bits of SVE.
3761 for (r = 0; r < a->rpt * a->selem; r++) {
3762 int tt = (a->rt + r) % 32;
3763 clear_vec_high(s, a->q, tt);
3766 if (a->p) {
3767 if (a->rm == 31) {
3768 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3769 } else {
3770 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3773 return true;
3776 static bool trans_ST_mult(DisasContext *s, arg_ldst_mult *a)
3778 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3779 MemOp endian, align, mop;
3781 int total; /* total bytes */
3782 int elements; /* elements per vector */
3783 int r;
3784 int size = a->sz;
3786 if (!a->p && a->rm != 0) {
3787 /* For non-postindexed accesses the Rm field must be 0 */
3788 return false;
3790 if (size == 3 && !a->q && a->selem != 1) {
3791 return false;
3793 if (!fp_access_check(s)) {
3794 return true;
3797 if (a->rn == 31) {
3798 gen_check_sp_alignment(s);
3801 /* For our purposes, bytes are always little-endian. */
3802 endian = s->be_data;
3803 if (size == 0) {
3804 endian = MO_LE;
3807 total = a->rpt * a->selem * (a->q ? 16 : 8);
3808 tcg_rn = cpu_reg_sp(s, a->rn);
3811 * Issue the MTE check vs the logical repeat count, before we
3812 * promote consecutive little-endian elements below.
3814 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31, total,
3815 finalize_memop_asimd(s, size));
3818 * Consecutive little-endian elements from a single register
3819 * can be promoted to a larger little-endian operation.
3821 align = MO_ALIGN;
3822 if (a->selem == 1 && endian == MO_LE) {
3823 align = pow2_align(size);
3824 size = 3;
3826 if (!s->align_mem) {
3827 align = 0;
3829 mop = endian | size | align;
3831 elements = (a->q ? 16 : 8) >> size;
3832 tcg_ebytes = tcg_constant_i64(1 << size);
3833 for (r = 0; r < a->rpt; r++) {
3834 int e;
3835 for (e = 0; e < elements; e++) {
3836 int xs;
3837 for (xs = 0; xs < a->selem; xs++) {
3838 int tt = (a->rt + r + xs) % 32;
3839 do_vec_st(s, tt, e, clean_addr, mop);
3840 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3845 if (a->p) {
3846 if (a->rm == 31) {
3847 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3848 } else {
3849 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3852 return true;
3855 static bool trans_ST_single(DisasContext *s, arg_ldst_single *a)
3857 int xs, total, rt;
3858 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3859 MemOp mop;
3861 if (!a->p && a->rm != 0) {
3862 return false;
3864 if (!fp_access_check(s)) {
3865 return true;
3868 if (a->rn == 31) {
3869 gen_check_sp_alignment(s);
3872 total = a->selem << a->scale;
3873 tcg_rn = cpu_reg_sp(s, a->rn);
3875 mop = finalize_memop_asimd(s, a->scale);
3876 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31,
3877 total, mop);
3879 tcg_ebytes = tcg_constant_i64(1 << a->scale);
3880 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
3881 do_vec_st(s, rt, a->index, clean_addr, mop);
3882 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3885 if (a->p) {
3886 if (a->rm == 31) {
3887 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3888 } else {
3889 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3892 return true;
3895 static bool trans_LD_single(DisasContext *s, arg_ldst_single *a)
3897 int xs, total, rt;
3898 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3899 MemOp mop;
3901 if (!a->p && a->rm != 0) {
3902 return false;
3904 if (!fp_access_check(s)) {
3905 return true;
3908 if (a->rn == 31) {
3909 gen_check_sp_alignment(s);
3912 total = a->selem << a->scale;
3913 tcg_rn = cpu_reg_sp(s, a->rn);
3915 mop = finalize_memop_asimd(s, a->scale);
3916 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31,
3917 total, mop);
3919 tcg_ebytes = tcg_constant_i64(1 << a->scale);
3920 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
3921 do_vec_ld(s, rt, a->index, clean_addr, mop);
3922 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3925 if (a->p) {
3926 if (a->rm == 31) {
3927 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3928 } else {
3929 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3932 return true;
3935 static bool trans_LD_single_repl(DisasContext *s, arg_LD_single_repl *a)
3937 int xs, total, rt;
3938 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3939 MemOp mop;
3941 if (!a->p && a->rm != 0) {
3942 return false;
3944 if (!fp_access_check(s)) {
3945 return true;
3948 if (a->rn == 31) {
3949 gen_check_sp_alignment(s);
3952 total = a->selem << a->scale;
3953 tcg_rn = cpu_reg_sp(s, a->rn);
3955 mop = finalize_memop_asimd(s, a->scale);
3956 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31,
3957 total, mop);
3959 tcg_ebytes = tcg_constant_i64(1 << a->scale);
3960 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
3961 /* Load and replicate to all elements */
3962 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
3964 tcg_gen_qemu_ld_i64(tcg_tmp, clean_addr, get_mem_index(s), mop);
3965 tcg_gen_gvec_dup_i64(a->scale, vec_full_reg_offset(s, rt),
3966 (a->q + 1) * 8, vec_full_reg_size(s), tcg_tmp);
3967 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3970 if (a->p) {
3971 if (a->rm == 31) {
3972 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3973 } else {
3974 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3977 return true;
3980 static bool trans_STZGM(DisasContext *s, arg_ldst_tag *a)
3982 TCGv_i64 addr, clean_addr, tcg_rt;
3983 int size = 4 << s->dcz_blocksize;
3985 if (!dc_isar_feature(aa64_mte, s)) {
3986 return false;
3988 if (s->current_el == 0) {
3989 return false;
3992 if (a->rn == 31) {
3993 gen_check_sp_alignment(s);
3996 addr = read_cpu_reg_sp(s, a->rn, true);
3997 tcg_gen_addi_i64(addr, addr, a->imm);
3998 tcg_rt = cpu_reg(s, a->rt);
4000 if (s->ata[0]) {
4001 gen_helper_stzgm_tags(tcg_env, addr, tcg_rt);
4004 * The non-tags portion of STZGM is mostly like DC_ZVA,
4005 * except the alignment happens before the access.
4007 clean_addr = clean_data_tbi(s, addr);
4008 tcg_gen_andi_i64(clean_addr, clean_addr, -size);
4009 gen_helper_dc_zva(tcg_env, clean_addr);
4010 return true;
4013 static bool trans_STGM(DisasContext *s, arg_ldst_tag *a)
4015 TCGv_i64 addr, clean_addr, tcg_rt;
4017 if (!dc_isar_feature(aa64_mte, s)) {
4018 return false;
4020 if (s->current_el == 0) {
4021 return false;
4024 if (a->rn == 31) {
4025 gen_check_sp_alignment(s);
4028 addr = read_cpu_reg_sp(s, a->rn, true);
4029 tcg_gen_addi_i64(addr, addr, a->imm);
4030 tcg_rt = cpu_reg(s, a->rt);
4032 if (s->ata[0]) {
4033 gen_helper_stgm(tcg_env, addr, tcg_rt);
4034 } else {
4035 MMUAccessType acc = MMU_DATA_STORE;
4036 int size = 4 << s->gm_blocksize;
4038 clean_addr = clean_data_tbi(s, addr);
4039 tcg_gen_andi_i64(clean_addr, clean_addr, -size);
4040 gen_probe_access(s, clean_addr, acc, size);
4042 return true;
4045 static bool trans_LDGM(DisasContext *s, arg_ldst_tag *a)
4047 TCGv_i64 addr, clean_addr, tcg_rt;
4049 if (!dc_isar_feature(aa64_mte, s)) {
4050 return false;
4052 if (s->current_el == 0) {
4053 return false;
4056 if (a->rn == 31) {
4057 gen_check_sp_alignment(s);
4060 addr = read_cpu_reg_sp(s, a->rn, true);
4061 tcg_gen_addi_i64(addr, addr, a->imm);
4062 tcg_rt = cpu_reg(s, a->rt);
4064 if (s->ata[0]) {
4065 gen_helper_ldgm(tcg_rt, tcg_env, addr);
4066 } else {
4067 MMUAccessType acc = MMU_DATA_LOAD;
4068 int size = 4 << s->gm_blocksize;
4070 clean_addr = clean_data_tbi(s, addr);
4071 tcg_gen_andi_i64(clean_addr, clean_addr, -size);
4072 gen_probe_access(s, clean_addr, acc, size);
4073 /* The result tags are zeros. */
4074 tcg_gen_movi_i64(tcg_rt, 0);
4076 return true;
4079 static bool trans_LDG(DisasContext *s, arg_ldst_tag *a)
4081 TCGv_i64 addr, clean_addr, tcg_rt;
4083 if (!dc_isar_feature(aa64_mte_insn_reg, s)) {
4084 return false;
4087 if (a->rn == 31) {
4088 gen_check_sp_alignment(s);
4091 addr = read_cpu_reg_sp(s, a->rn, true);
4092 if (!a->p) {
4093 /* pre-index or signed offset */
4094 tcg_gen_addi_i64(addr, addr, a->imm);
4097 tcg_gen_andi_i64(addr, addr, -TAG_GRANULE);
4098 tcg_rt = cpu_reg(s, a->rt);
4099 if (s->ata[0]) {
4100 gen_helper_ldg(tcg_rt, tcg_env, addr, tcg_rt);
4101 } else {
4103 * Tag access disabled: we must check for aborts on the load
4104 * load from [rn+offset], and then insert a 0 tag into rt.
4106 clean_addr = clean_data_tbi(s, addr);
4107 gen_probe_access(s, clean_addr, MMU_DATA_LOAD, MO_8);
4108 gen_address_with_allocation_tag0(tcg_rt, tcg_rt);
4111 if (a->w) {
4112 /* pre-index or post-index */
4113 if (a->p) {
4114 /* post-index */
4115 tcg_gen_addi_i64(addr, addr, a->imm);
4117 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr);
4119 return true;
4122 static bool do_STG(DisasContext *s, arg_ldst_tag *a, bool is_zero, bool is_pair)
4124 TCGv_i64 addr, tcg_rt;
4126 if (a->rn == 31) {
4127 gen_check_sp_alignment(s);
4130 addr = read_cpu_reg_sp(s, a->rn, true);
4131 if (!a->p) {
4132 /* pre-index or signed offset */
4133 tcg_gen_addi_i64(addr, addr, a->imm);
4135 tcg_rt = cpu_reg_sp(s, a->rt);
4136 if (!s->ata[0]) {
4138 * For STG and ST2G, we need to check alignment and probe memory.
4139 * TODO: For STZG and STZ2G, we could rely on the stores below,
4140 * at least for system mode; user-only won't enforce alignment.
4142 if (is_pair) {
4143 gen_helper_st2g_stub(tcg_env, addr);
4144 } else {
4145 gen_helper_stg_stub(tcg_env, addr);
4147 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
4148 if (is_pair) {
4149 gen_helper_st2g_parallel(tcg_env, addr, tcg_rt);
4150 } else {
4151 gen_helper_stg_parallel(tcg_env, addr, tcg_rt);
4153 } else {
4154 if (is_pair) {
4155 gen_helper_st2g(tcg_env, addr, tcg_rt);
4156 } else {
4157 gen_helper_stg(tcg_env, addr, tcg_rt);
4161 if (is_zero) {
4162 TCGv_i64 clean_addr = clean_data_tbi(s, addr);
4163 TCGv_i64 zero64 = tcg_constant_i64(0);
4164 TCGv_i128 zero128 = tcg_temp_new_i128();
4165 int mem_index = get_mem_index(s);
4166 MemOp mop = finalize_memop(s, MO_128 | MO_ALIGN);
4168 tcg_gen_concat_i64_i128(zero128, zero64, zero64);
4170 /* This is 1 or 2 atomic 16-byte operations. */
4171 tcg_gen_qemu_st_i128(zero128, clean_addr, mem_index, mop);
4172 if (is_pair) {
4173 tcg_gen_addi_i64(clean_addr, clean_addr, 16);
4174 tcg_gen_qemu_st_i128(zero128, clean_addr, mem_index, mop);
4178 if (a->w) {
4179 /* pre-index or post-index */
4180 if (a->p) {
4181 /* post-index */
4182 tcg_gen_addi_i64(addr, addr, a->imm);
4184 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr);
4186 return true;
4189 TRANS_FEAT(STG, aa64_mte_insn_reg, do_STG, a, false, false)
4190 TRANS_FEAT(STZG, aa64_mte_insn_reg, do_STG, a, true, false)
4191 TRANS_FEAT(ST2G, aa64_mte_insn_reg, do_STG, a, false, true)
4192 TRANS_FEAT(STZ2G, aa64_mte_insn_reg, do_STG, a, true, true)
4194 typedef void SetFn(TCGv_env, TCGv_i32, TCGv_i32);
4196 static bool do_SET(DisasContext *s, arg_set *a, bool is_epilogue,
4197 bool is_setg, SetFn fn)
4199 int memidx;
4200 uint32_t syndrome, desc = 0;
4202 if (is_setg && !dc_isar_feature(aa64_mte, s)) {
4203 return false;
4207 * UNPREDICTABLE cases: we choose to UNDEF, which allows
4208 * us to pull this check before the CheckMOPSEnabled() test
4209 * (which we do in the helper function)
4211 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd ||
4212 a->rd == 31 || a->rn == 31) {
4213 return false;
4216 memidx = get_a64_user_mem_index(s, a->unpriv);
4219 * We pass option_a == true, matching our implementation;
4220 * we pass wrong_option == false: helper function may set that bit.
4222 syndrome = syn_mop(true, is_setg, (a->nontemp << 1) | a->unpriv,
4223 is_epilogue, false, true, a->rd, a->rs, a->rn);
4225 if (is_setg ? s->ata[a->unpriv] : s->mte_active[a->unpriv]) {
4226 /* We may need to do MTE tag checking, so assemble the descriptor */
4227 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
4228 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
4229 desc = FIELD_DP32(desc, MTEDESC, WRITE, true);
4230 /* SIZEM1 and ALIGN we leave 0 (byte write) */
4232 /* The helper function always needs the memidx even with MTE disabled */
4233 desc = FIELD_DP32(desc, MTEDESC, MIDX, memidx);
4236 * The helper needs the register numbers, but since they're in
4237 * the syndrome anyway, we let it extract them from there rather
4238 * than passing in an extra three integer arguments.
4240 fn(tcg_env, tcg_constant_i32(syndrome), tcg_constant_i32(desc));
4241 return true;
4244 TRANS_FEAT(SETP, aa64_mops, do_SET, a, false, false, gen_helper_setp)
4245 TRANS_FEAT(SETM, aa64_mops, do_SET, a, false, false, gen_helper_setm)
4246 TRANS_FEAT(SETE, aa64_mops, do_SET, a, true, false, gen_helper_sete)
4247 TRANS_FEAT(SETGP, aa64_mops, do_SET, a, false, true, gen_helper_setgp)
4248 TRANS_FEAT(SETGM, aa64_mops, do_SET, a, false, true, gen_helper_setgm)
4249 TRANS_FEAT(SETGE, aa64_mops, do_SET, a, true, true, gen_helper_setge)
4251 typedef void CpyFn(TCGv_env, TCGv_i32, TCGv_i32, TCGv_i32);
4253 static bool do_CPY(DisasContext *s, arg_cpy *a, bool is_epilogue, CpyFn fn)
4255 int rmemidx, wmemidx;
4256 uint32_t syndrome, rdesc = 0, wdesc = 0;
4257 bool wunpriv = extract32(a->options, 0, 1);
4258 bool runpriv = extract32(a->options, 1, 1);
4261 * UNPREDICTABLE cases: we choose to UNDEF, which allows
4262 * us to pull this check before the CheckMOPSEnabled() test
4263 * (which we do in the helper function)
4265 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd ||
4266 a->rd == 31 || a->rs == 31 || a->rn == 31) {
4267 return false;
4270 rmemidx = get_a64_user_mem_index(s, runpriv);
4271 wmemidx = get_a64_user_mem_index(s, wunpriv);
4274 * We pass option_a == true, matching our implementation;
4275 * we pass wrong_option == false: helper function may set that bit.
4277 syndrome = syn_mop(false, false, a->options, is_epilogue,
4278 false, true, a->rd, a->rs, a->rn);
4280 /* If we need to do MTE tag checking, assemble the descriptors */
4281 if (s->mte_active[runpriv]) {
4282 rdesc = FIELD_DP32(rdesc, MTEDESC, TBI, s->tbid);
4283 rdesc = FIELD_DP32(rdesc, MTEDESC, TCMA, s->tcma);
4285 if (s->mte_active[wunpriv]) {
4286 wdesc = FIELD_DP32(wdesc, MTEDESC, TBI, s->tbid);
4287 wdesc = FIELD_DP32(wdesc, MTEDESC, TCMA, s->tcma);
4288 wdesc = FIELD_DP32(wdesc, MTEDESC, WRITE, true);
4290 /* The helper function needs these parts of the descriptor regardless */
4291 rdesc = FIELD_DP32(rdesc, MTEDESC, MIDX, rmemidx);
4292 wdesc = FIELD_DP32(wdesc, MTEDESC, MIDX, wmemidx);
4295 * The helper needs the register numbers, but since they're in
4296 * the syndrome anyway, we let it extract them from there rather
4297 * than passing in an extra three integer arguments.
4299 fn(tcg_env, tcg_constant_i32(syndrome), tcg_constant_i32(wdesc),
4300 tcg_constant_i32(rdesc));
4301 return true;
4304 TRANS_FEAT(CPYP, aa64_mops, do_CPY, a, false, gen_helper_cpyp)
4305 TRANS_FEAT(CPYM, aa64_mops, do_CPY, a, false, gen_helper_cpym)
4306 TRANS_FEAT(CPYE, aa64_mops, do_CPY, a, true, gen_helper_cpye)
4307 TRANS_FEAT(CPYFP, aa64_mops, do_CPY, a, false, gen_helper_cpyfp)
4308 TRANS_FEAT(CPYFM, aa64_mops, do_CPY, a, false, gen_helper_cpyfm)
4309 TRANS_FEAT(CPYFE, aa64_mops, do_CPY, a, true, gen_helper_cpyfe)
4311 typedef void ArithTwoOp(TCGv_i64, TCGv_i64, TCGv_i64);
4313 static bool gen_rri(DisasContext *s, arg_rri_sf *a,
4314 bool rd_sp, bool rn_sp, ArithTwoOp *fn)
4316 TCGv_i64 tcg_rn = rn_sp ? cpu_reg_sp(s, a->rn) : cpu_reg(s, a->rn);
4317 TCGv_i64 tcg_rd = rd_sp ? cpu_reg_sp(s, a->rd) : cpu_reg(s, a->rd);
4318 TCGv_i64 tcg_imm = tcg_constant_i64(a->imm);
4320 fn(tcg_rd, tcg_rn, tcg_imm);
4321 if (!a->sf) {
4322 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4324 return true;
4328 * PC-rel. addressing
4331 static bool trans_ADR(DisasContext *s, arg_ri *a)
4333 gen_pc_plus_diff(s, cpu_reg(s, a->rd), a->imm);
4334 return true;
4337 static bool trans_ADRP(DisasContext *s, arg_ri *a)
4339 int64_t offset = (int64_t)a->imm << 12;
4341 /* The page offset is ok for CF_PCREL. */
4342 offset -= s->pc_curr & 0xfff;
4343 gen_pc_plus_diff(s, cpu_reg(s, a->rd), offset);
4344 return true;
4348 * Add/subtract (immediate)
4350 TRANS(ADD_i, gen_rri, a, 1, 1, tcg_gen_add_i64)
4351 TRANS(SUB_i, gen_rri, a, 1, 1, tcg_gen_sub_i64)
4352 TRANS(ADDS_i, gen_rri, a, 0, 1, a->sf ? gen_add64_CC : gen_add32_CC)
4353 TRANS(SUBS_i, gen_rri, a, 0, 1, a->sf ? gen_sub64_CC : gen_sub32_CC)
4356 * Add/subtract (immediate, with tags)
4359 static bool gen_add_sub_imm_with_tags(DisasContext *s, arg_rri_tag *a,
4360 bool sub_op)
4362 TCGv_i64 tcg_rn, tcg_rd;
4363 int imm;
4365 imm = a->uimm6 << LOG2_TAG_GRANULE;
4366 if (sub_op) {
4367 imm = -imm;
4370 tcg_rn = cpu_reg_sp(s, a->rn);
4371 tcg_rd = cpu_reg_sp(s, a->rd);
4373 if (s->ata[0]) {
4374 gen_helper_addsubg(tcg_rd, tcg_env, tcg_rn,
4375 tcg_constant_i32(imm),
4376 tcg_constant_i32(a->uimm4));
4377 } else {
4378 tcg_gen_addi_i64(tcg_rd, tcg_rn, imm);
4379 gen_address_with_allocation_tag0(tcg_rd, tcg_rd);
4381 return true;
4384 TRANS_FEAT(ADDG_i, aa64_mte_insn_reg, gen_add_sub_imm_with_tags, a, false)
4385 TRANS_FEAT(SUBG_i, aa64_mte_insn_reg, gen_add_sub_imm_with_tags, a, true)
4387 /* The input should be a value in the bottom e bits (with higher
4388 * bits zero); returns that value replicated into every element
4389 * of size e in a 64 bit integer.
4391 static uint64_t bitfield_replicate(uint64_t mask, unsigned int e)
4393 assert(e != 0);
4394 while (e < 64) {
4395 mask |= mask << e;
4396 e *= 2;
4398 return mask;
4402 * Logical (immediate)
4406 * Simplified variant of pseudocode DecodeBitMasks() for the case where we
4407 * only require the wmask. Returns false if the imms/immr/immn are a reserved
4408 * value (ie should cause a guest UNDEF exception), and true if they are
4409 * valid, in which case the decoded bit pattern is written to result.
4411 bool logic_imm_decode_wmask(uint64_t *result, unsigned int immn,
4412 unsigned int imms, unsigned int immr)
4414 uint64_t mask;
4415 unsigned e, levels, s, r;
4416 int len;
4418 assert(immn < 2 && imms < 64 && immr < 64);
4420 /* The bit patterns we create here are 64 bit patterns which
4421 * are vectors of identical elements of size e = 2, 4, 8, 16, 32 or
4422 * 64 bits each. Each element contains the same value: a run
4423 * of between 1 and e-1 non-zero bits, rotated within the
4424 * element by between 0 and e-1 bits.
4426 * The element size and run length are encoded into immn (1 bit)
4427 * and imms (6 bits) as follows:
4428 * 64 bit elements: immn = 1, imms = <length of run - 1>
4429 * 32 bit elements: immn = 0, imms = 0 : <length of run - 1>
4430 * 16 bit elements: immn = 0, imms = 10 : <length of run - 1>
4431 * 8 bit elements: immn = 0, imms = 110 : <length of run - 1>
4432 * 4 bit elements: immn = 0, imms = 1110 : <length of run - 1>
4433 * 2 bit elements: immn = 0, imms = 11110 : <length of run - 1>
4434 * Notice that immn = 0, imms = 11111x is the only combination
4435 * not covered by one of the above options; this is reserved.
4436 * Further, <length of run - 1> all-ones is a reserved pattern.
4438 * In all cases the rotation is by immr % e (and immr is 6 bits).
4441 /* First determine the element size */
4442 len = 31 - clz32((immn << 6) | (~imms & 0x3f));
4443 if (len < 1) {
4444 /* This is the immn == 0, imms == 0x11111x case */
4445 return false;
4447 e = 1 << len;
4449 levels = e - 1;
4450 s = imms & levels;
4451 r = immr & levels;
4453 if (s == levels) {
4454 /* <length of run - 1> mustn't be all-ones. */
4455 return false;
4458 /* Create the value of one element: s+1 set bits rotated
4459 * by r within the element (which is e bits wide)...
4461 mask = MAKE_64BIT_MASK(0, s + 1);
4462 if (r) {
4463 mask = (mask >> r) | (mask << (e - r));
4464 mask &= MAKE_64BIT_MASK(0, e);
4466 /* ...then replicate the element over the whole 64 bit value */
4467 mask = bitfield_replicate(mask, e);
4468 *result = mask;
4469 return true;
4472 static bool gen_rri_log(DisasContext *s, arg_rri_log *a, bool set_cc,
4473 void (*fn)(TCGv_i64, TCGv_i64, int64_t))
4475 TCGv_i64 tcg_rd, tcg_rn;
4476 uint64_t imm;
4478 /* Some immediate field values are reserved. */
4479 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1),
4480 extract32(a->dbm, 0, 6),
4481 extract32(a->dbm, 6, 6))) {
4482 return false;
4484 if (!a->sf) {
4485 imm &= 0xffffffffull;
4488 tcg_rd = set_cc ? cpu_reg(s, a->rd) : cpu_reg_sp(s, a->rd);
4489 tcg_rn = cpu_reg(s, a->rn);
4491 fn(tcg_rd, tcg_rn, imm);
4492 if (set_cc) {
4493 gen_logic_CC(a->sf, tcg_rd);
4495 if (!a->sf) {
4496 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4498 return true;
4501 TRANS(AND_i, gen_rri_log, a, false, tcg_gen_andi_i64)
4502 TRANS(ORR_i, gen_rri_log, a, false, tcg_gen_ori_i64)
4503 TRANS(EOR_i, gen_rri_log, a, false, tcg_gen_xori_i64)
4504 TRANS(ANDS_i, gen_rri_log, a, true, tcg_gen_andi_i64)
4507 * Move wide (immediate)
4510 static bool trans_MOVZ(DisasContext *s, arg_movw *a)
4512 int pos = a->hw << 4;
4513 tcg_gen_movi_i64(cpu_reg(s, a->rd), (uint64_t)a->imm << pos);
4514 return true;
4517 static bool trans_MOVN(DisasContext *s, arg_movw *a)
4519 int pos = a->hw << 4;
4520 uint64_t imm = a->imm;
4522 imm = ~(imm << pos);
4523 if (!a->sf) {
4524 imm = (uint32_t)imm;
4526 tcg_gen_movi_i64(cpu_reg(s, a->rd), imm);
4527 return true;
4530 static bool trans_MOVK(DisasContext *s, arg_movw *a)
4532 int pos = a->hw << 4;
4533 TCGv_i64 tcg_rd, tcg_im;
4535 tcg_rd = cpu_reg(s, a->rd);
4536 tcg_im = tcg_constant_i64(a->imm);
4537 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_im, pos, 16);
4538 if (!a->sf) {
4539 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4541 return true;
4545 * Bitfield
4548 static bool trans_SBFM(DisasContext *s, arg_SBFM *a)
4550 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
4551 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4552 unsigned int bitsize = a->sf ? 64 : 32;
4553 unsigned int ri = a->immr;
4554 unsigned int si = a->imms;
4555 unsigned int pos, len;
4557 if (si >= ri) {
4558 /* Wd<s-r:0> = Wn<s:r> */
4559 len = (si - ri) + 1;
4560 tcg_gen_sextract_i64(tcg_rd, tcg_tmp, ri, len);
4561 if (!a->sf) {
4562 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4564 } else {
4565 /* Wd<32+s-r,32-r> = Wn<s:0> */
4566 len = si + 1;
4567 pos = (bitsize - ri) & (bitsize - 1);
4569 if (len < ri) {
4571 * Sign extend the destination field from len to fill the
4572 * balance of the word. Let the deposit below insert all
4573 * of those sign bits.
4575 tcg_gen_sextract_i64(tcg_tmp, tcg_tmp, 0, len);
4576 len = ri;
4580 * We start with zero, and we haven't modified any bits outside
4581 * bitsize, therefore no final zero-extension is unneeded for !sf.
4583 tcg_gen_deposit_z_i64(tcg_rd, tcg_tmp, pos, len);
4585 return true;
4588 static bool trans_UBFM(DisasContext *s, arg_UBFM *a)
4590 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
4591 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4592 unsigned int bitsize = a->sf ? 64 : 32;
4593 unsigned int ri = a->immr;
4594 unsigned int si = a->imms;
4595 unsigned int pos, len;
4597 tcg_rd = cpu_reg(s, a->rd);
4598 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4600 if (si >= ri) {
4601 /* Wd<s-r:0> = Wn<s:r> */
4602 len = (si - ri) + 1;
4603 tcg_gen_extract_i64(tcg_rd, tcg_tmp, ri, len);
4604 } else {
4605 /* Wd<32+s-r,32-r> = Wn<s:0> */
4606 len = si + 1;
4607 pos = (bitsize - ri) & (bitsize - 1);
4608 tcg_gen_deposit_z_i64(tcg_rd, tcg_tmp, pos, len);
4610 return true;
4613 static bool trans_BFM(DisasContext *s, arg_BFM *a)
4615 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
4616 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4617 unsigned int bitsize = a->sf ? 64 : 32;
4618 unsigned int ri = a->immr;
4619 unsigned int si = a->imms;
4620 unsigned int pos, len;
4622 tcg_rd = cpu_reg(s, a->rd);
4623 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4625 if (si >= ri) {
4626 /* Wd<s-r:0> = Wn<s:r> */
4627 tcg_gen_shri_i64(tcg_tmp, tcg_tmp, ri);
4628 len = (si - ri) + 1;
4629 pos = 0;
4630 } else {
4631 /* Wd<32+s-r,32-r> = Wn<s:0> */
4632 len = si + 1;
4633 pos = (bitsize - ri) & (bitsize - 1);
4636 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_tmp, pos, len);
4637 if (!a->sf) {
4638 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4640 return true;
4643 static bool trans_EXTR(DisasContext *s, arg_extract *a)
4645 TCGv_i64 tcg_rd, tcg_rm, tcg_rn;
4647 tcg_rd = cpu_reg(s, a->rd);
4649 if (unlikely(a->imm == 0)) {
4651 * tcg shl_i32/shl_i64 is undefined for 32/64 bit shifts,
4652 * so an extract from bit 0 is a special case.
4654 if (a->sf) {
4655 tcg_gen_mov_i64(tcg_rd, cpu_reg(s, a->rm));
4656 } else {
4657 tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, a->rm));
4659 } else {
4660 tcg_rm = cpu_reg(s, a->rm);
4661 tcg_rn = cpu_reg(s, a->rn);
4663 if (a->sf) {
4664 /* Specialization to ROR happens in EXTRACT2. */
4665 tcg_gen_extract2_i64(tcg_rd, tcg_rm, tcg_rn, a->imm);
4666 } else {
4667 TCGv_i32 t0 = tcg_temp_new_i32();
4669 tcg_gen_extrl_i64_i32(t0, tcg_rm);
4670 if (a->rm == a->rn) {
4671 tcg_gen_rotri_i32(t0, t0, a->imm);
4672 } else {
4673 TCGv_i32 t1 = tcg_temp_new_i32();
4674 tcg_gen_extrl_i64_i32(t1, tcg_rn);
4675 tcg_gen_extract2_i32(t0, t0, t1, a->imm);
4677 tcg_gen_extu_i32_i64(tcg_rd, t0);
4680 return true;
4683 static bool trans_TBL_TBX(DisasContext *s, arg_TBL_TBX *a)
4685 if (fp_access_check(s)) {
4686 int len = (a->len + 1) * 16;
4688 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->rd),
4689 vec_full_reg_offset(s, a->rm), tcg_env,
4690 a->q ? 16 : 8, vec_full_reg_size(s),
4691 (len << 6) | (a->tbx << 5) | a->rn,
4692 gen_helper_simd_tblx);
4694 return true;
4697 typedef int simd_permute_idx_fn(int i, int part, int elements);
4699 static bool do_simd_permute(DisasContext *s, arg_qrrr_e *a,
4700 simd_permute_idx_fn *fn, int part)
4702 MemOp esz = a->esz;
4703 int datasize = a->q ? 16 : 8;
4704 int elements = datasize >> esz;
4705 TCGv_i64 tcg_res[2], tcg_ele;
4707 if (esz == MO_64 && !a->q) {
4708 return false;
4710 if (!fp_access_check(s)) {
4711 return true;
4714 tcg_res[0] = tcg_temp_new_i64();
4715 tcg_res[1] = a->q ? tcg_temp_new_i64() : NULL;
4716 tcg_ele = tcg_temp_new_i64();
4718 for (int i = 0; i < elements; i++) {
4719 int o, w, idx;
4721 idx = fn(i, part, elements);
4722 read_vec_element(s, tcg_ele, (idx & elements ? a->rm : a->rn),
4723 idx & (elements - 1), esz);
4725 w = (i << (esz + 3)) / 64;
4726 o = (i << (esz + 3)) % 64;
4727 if (o == 0) {
4728 tcg_gen_mov_i64(tcg_res[w], tcg_ele);
4729 } else {
4730 tcg_gen_deposit_i64(tcg_res[w], tcg_res[w], tcg_ele, o, 8 << esz);
4734 for (int i = a->q; i >= 0; --i) {
4735 write_vec_element(s, tcg_res[i], a->rd, i, MO_64);
4737 clear_vec_high(s, a->q, a->rd);
4738 return true;
4741 static int permute_load_uzp(int i, int part, int elements)
4743 return 2 * i + part;
4746 TRANS(UZP1, do_simd_permute, a, permute_load_uzp, 0)
4747 TRANS(UZP2, do_simd_permute, a, permute_load_uzp, 1)
4749 static int permute_load_trn(int i, int part, int elements)
4751 return (i & 1) * elements + (i & ~1) + part;
4754 TRANS(TRN1, do_simd_permute, a, permute_load_trn, 0)
4755 TRANS(TRN2, do_simd_permute, a, permute_load_trn, 1)
4757 static int permute_load_zip(int i, int part, int elements)
4759 return (i & 1) * elements + ((part * elements + i) >> 1);
4762 TRANS(ZIP1, do_simd_permute, a, permute_load_zip, 0)
4763 TRANS(ZIP2, do_simd_permute, a, permute_load_zip, 1)
4766 * Cryptographic AES, SHA, SHA512
4769 TRANS_FEAT(AESE, aa64_aes, do_gvec_op3_ool, a, 0, gen_helper_crypto_aese)
4770 TRANS_FEAT(AESD, aa64_aes, do_gvec_op3_ool, a, 0, gen_helper_crypto_aesd)
4771 TRANS_FEAT(AESMC, aa64_aes, do_gvec_op2_ool, a, 0, gen_helper_crypto_aesmc)
4772 TRANS_FEAT(AESIMC, aa64_aes, do_gvec_op2_ool, a, 0, gen_helper_crypto_aesimc)
4774 TRANS_FEAT(SHA1C, aa64_sha1, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha1c)
4775 TRANS_FEAT(SHA1P, aa64_sha1, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha1p)
4776 TRANS_FEAT(SHA1M, aa64_sha1, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha1m)
4777 TRANS_FEAT(SHA1SU0, aa64_sha1, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha1su0)
4779 TRANS_FEAT(SHA256H, aa64_sha256, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha256h)
4780 TRANS_FEAT(SHA256H2, aa64_sha256, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha256h2)
4781 TRANS_FEAT(SHA256SU1, aa64_sha256, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha256su1)
4783 TRANS_FEAT(SHA1H, aa64_sha1, do_gvec_op2_ool, a, 0, gen_helper_crypto_sha1h)
4784 TRANS_FEAT(SHA1SU1, aa64_sha1, do_gvec_op2_ool, a, 0, gen_helper_crypto_sha1su1)
4785 TRANS_FEAT(SHA256SU0, aa64_sha256, do_gvec_op2_ool, a, 0, gen_helper_crypto_sha256su0)
4787 TRANS_FEAT(SHA512H, aa64_sha512, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha512h)
4788 TRANS_FEAT(SHA512H2, aa64_sha512, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha512h2)
4789 TRANS_FEAT(SHA512SU1, aa64_sha512, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha512su1)
4790 TRANS_FEAT(RAX1, aa64_sha3, do_gvec_fn3, a, gen_gvec_rax1)
4791 TRANS_FEAT(SM3PARTW1, aa64_sm3, do_gvec_op3_ool, a, 0, gen_helper_crypto_sm3partw1)
4792 TRANS_FEAT(SM3PARTW2, aa64_sm3, do_gvec_op3_ool, a, 0, gen_helper_crypto_sm3partw2)
4793 TRANS_FEAT(SM4EKEY, aa64_sm4, do_gvec_op3_ool, a, 0, gen_helper_crypto_sm4ekey)
4795 TRANS_FEAT(SHA512SU0, aa64_sha512, do_gvec_op2_ool, a, 0, gen_helper_crypto_sha512su0)
4796 TRANS_FEAT(SM4E, aa64_sm4, do_gvec_op3_ool, a, 0, gen_helper_crypto_sm4e)
4798 TRANS_FEAT(EOR3, aa64_sha3, do_gvec_fn4, a, gen_gvec_eor3)
4799 TRANS_FEAT(BCAX, aa64_sha3, do_gvec_fn4, a, gen_gvec_bcax)
4801 static bool trans_SM3SS1(DisasContext *s, arg_SM3SS1 *a)
4803 if (!dc_isar_feature(aa64_sm3, s)) {
4804 return false;
4806 if (fp_access_check(s)) {
4807 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
4808 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
4809 TCGv_i32 tcg_op3 = tcg_temp_new_i32();
4810 TCGv_i32 tcg_res = tcg_temp_new_i32();
4811 unsigned vsz, dofs;
4813 read_vec_element_i32(s, tcg_op1, a->rn, 3, MO_32);
4814 read_vec_element_i32(s, tcg_op2, a->rm, 3, MO_32);
4815 read_vec_element_i32(s, tcg_op3, a->ra, 3, MO_32);
4817 tcg_gen_rotri_i32(tcg_res, tcg_op1, 20);
4818 tcg_gen_add_i32(tcg_res, tcg_res, tcg_op2);
4819 tcg_gen_add_i32(tcg_res, tcg_res, tcg_op3);
4820 tcg_gen_rotri_i32(tcg_res, tcg_res, 25);
4822 /* Clear the whole register first, then store bits [127:96]. */
4823 vsz = vec_full_reg_size(s);
4824 dofs = vec_full_reg_offset(s, a->rd);
4825 tcg_gen_gvec_dup_imm(MO_64, dofs, vsz, vsz, 0);
4826 write_vec_element_i32(s, tcg_res, a->rd, 3, MO_32);
4828 return true;
4831 static bool do_crypto3i(DisasContext *s, arg_crypto3i *a, gen_helper_gvec_3 *fn)
4833 if (fp_access_check(s)) {
4834 gen_gvec_op3_ool(s, true, a->rd, a->rn, a->rm, a->imm, fn);
4836 return true;
4838 TRANS_FEAT(SM3TT1A, aa64_sm3, do_crypto3i, a, gen_helper_crypto_sm3tt1a)
4839 TRANS_FEAT(SM3TT1B, aa64_sm3, do_crypto3i, a, gen_helper_crypto_sm3tt1b)
4840 TRANS_FEAT(SM3TT2A, aa64_sm3, do_crypto3i, a, gen_helper_crypto_sm3tt2a)
4841 TRANS_FEAT(SM3TT2B, aa64_sm3, do_crypto3i, a, gen_helper_crypto_sm3tt2b)
4843 static bool trans_XAR(DisasContext *s, arg_XAR *a)
4845 if (!dc_isar_feature(aa64_sha3, s)) {
4846 return false;
4848 if (fp_access_check(s)) {
4849 gen_gvec_xar(MO_64, vec_full_reg_offset(s, a->rd),
4850 vec_full_reg_offset(s, a->rn),
4851 vec_full_reg_offset(s, a->rm), a->imm, 16,
4852 vec_full_reg_size(s));
4854 return true;
4858 * Advanced SIMD copy
4861 static bool decode_esz_idx(int imm, MemOp *pesz, unsigned *pidx)
4863 unsigned esz = ctz32(imm);
4864 if (esz <= MO_64) {
4865 *pesz = esz;
4866 *pidx = imm >> (esz + 1);
4867 return true;
4869 return false;
4872 static bool trans_DUP_element_s(DisasContext *s, arg_DUP_element_s *a)
4874 MemOp esz;
4875 unsigned idx;
4877 if (!decode_esz_idx(a->imm, &esz, &idx)) {
4878 return false;
4880 if (fp_access_check(s)) {
4882 * This instruction just extracts the specified element and
4883 * zero-extends it into the bottom of the destination register.
4885 TCGv_i64 tmp = tcg_temp_new_i64();
4886 read_vec_element(s, tmp, a->rn, idx, esz);
4887 write_fp_dreg(s, a->rd, tmp);
4889 return true;
4892 static bool trans_DUP_element_v(DisasContext *s, arg_DUP_element_v *a)
4894 MemOp esz;
4895 unsigned idx;
4897 if (!decode_esz_idx(a->imm, &esz, &idx)) {
4898 return false;
4900 if (esz == MO_64 && !a->q) {
4901 return false;
4903 if (fp_access_check(s)) {
4904 tcg_gen_gvec_dup_mem(esz, vec_full_reg_offset(s, a->rd),
4905 vec_reg_offset(s, a->rn, idx, esz),
4906 a->q ? 16 : 8, vec_full_reg_size(s));
4908 return true;
4911 static bool trans_DUP_general(DisasContext *s, arg_DUP_general *a)
4913 MemOp esz;
4914 unsigned idx;
4916 if (!decode_esz_idx(a->imm, &esz, &idx)) {
4917 return false;
4919 if (esz == MO_64 && !a->q) {
4920 return false;
4922 if (fp_access_check(s)) {
4923 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd),
4924 a->q ? 16 : 8, vec_full_reg_size(s),
4925 cpu_reg(s, a->rn));
4927 return true;
4930 static bool do_smov_umov(DisasContext *s, arg_SMOV *a, MemOp is_signed)
4932 MemOp esz;
4933 unsigned idx;
4935 if (!decode_esz_idx(a->imm, &esz, &idx)) {
4936 return false;
4938 if (is_signed) {
4939 if (esz == MO_64 || (esz == MO_32 && !a->q)) {
4940 return false;
4942 } else {
4943 if (esz == MO_64 ? !a->q : a->q) {
4944 return false;
4947 if (fp_access_check(s)) {
4948 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
4949 read_vec_element(s, tcg_rd, a->rn, idx, esz | is_signed);
4950 if (is_signed && !a->q) {
4951 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4954 return true;
4957 TRANS(SMOV, do_smov_umov, a, MO_SIGN)
4958 TRANS(UMOV, do_smov_umov, a, 0)
4960 static bool trans_INS_general(DisasContext *s, arg_INS_general *a)
4962 MemOp esz;
4963 unsigned idx;
4965 if (!decode_esz_idx(a->imm, &esz, &idx)) {
4966 return false;
4968 if (fp_access_check(s)) {
4969 write_vec_element(s, cpu_reg(s, a->rn), a->rd, idx, esz);
4970 clear_vec_high(s, true, a->rd);
4972 return true;
4975 static bool trans_INS_element(DisasContext *s, arg_INS_element *a)
4977 MemOp esz;
4978 unsigned didx, sidx;
4980 if (!decode_esz_idx(a->di, &esz, &didx)) {
4981 return false;
4983 sidx = a->si >> esz;
4984 if (fp_access_check(s)) {
4985 TCGv_i64 tmp = tcg_temp_new_i64();
4987 read_vec_element(s, tmp, a->rn, sidx, esz);
4988 write_vec_element(s, tmp, a->rd, didx, esz);
4990 /* INS is considered a 128-bit write for SVE. */
4991 clear_vec_high(s, true, a->rd);
4993 return true;
4997 * Advanced SIMD three same
5000 typedef struct FPScalar {
5001 void (*gen_h)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_ptr);
5002 void (*gen_s)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_ptr);
5003 void (*gen_d)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_ptr);
5004 } FPScalar;
5006 static bool do_fp3_scalar(DisasContext *s, arg_rrr_e *a, const FPScalar *f)
5008 switch (a->esz) {
5009 case MO_64:
5010 if (fp_access_check(s)) {
5011 TCGv_i64 t0 = read_fp_dreg(s, a->rn);
5012 TCGv_i64 t1 = read_fp_dreg(s, a->rm);
5013 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_FPCR));
5014 write_fp_dreg(s, a->rd, t0);
5016 break;
5017 case MO_32:
5018 if (fp_access_check(s)) {
5019 TCGv_i32 t0 = read_fp_sreg(s, a->rn);
5020 TCGv_i32 t1 = read_fp_sreg(s, a->rm);
5021 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_FPCR));
5022 write_fp_sreg(s, a->rd, t0);
5024 break;
5025 case MO_16:
5026 if (!dc_isar_feature(aa64_fp16, s)) {
5027 return false;
5029 if (fp_access_check(s)) {
5030 TCGv_i32 t0 = read_fp_hreg(s, a->rn);
5031 TCGv_i32 t1 = read_fp_hreg(s, a->rm);
5032 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_FPCR_F16));
5033 write_fp_sreg(s, a->rd, t0);
5035 break;
5036 default:
5037 return false;
5039 return true;
5042 static const FPScalar f_scalar_fadd = {
5043 gen_helper_vfp_addh,
5044 gen_helper_vfp_adds,
5045 gen_helper_vfp_addd,
5047 TRANS(FADD_s, do_fp3_scalar, a, &f_scalar_fadd)
5049 static const FPScalar f_scalar_fsub = {
5050 gen_helper_vfp_subh,
5051 gen_helper_vfp_subs,
5052 gen_helper_vfp_subd,
5054 TRANS(FSUB_s, do_fp3_scalar, a, &f_scalar_fsub)
5056 static const FPScalar f_scalar_fdiv = {
5057 gen_helper_vfp_divh,
5058 gen_helper_vfp_divs,
5059 gen_helper_vfp_divd,
5061 TRANS(FDIV_s, do_fp3_scalar, a, &f_scalar_fdiv)
5063 static const FPScalar f_scalar_fmul = {
5064 gen_helper_vfp_mulh,
5065 gen_helper_vfp_muls,
5066 gen_helper_vfp_muld,
5068 TRANS(FMUL_s, do_fp3_scalar, a, &f_scalar_fmul)
5070 static const FPScalar f_scalar_fmax = {
5071 gen_helper_advsimd_maxh,
5072 gen_helper_vfp_maxs,
5073 gen_helper_vfp_maxd,
5075 TRANS(FMAX_s, do_fp3_scalar, a, &f_scalar_fmax)
5077 static const FPScalar f_scalar_fmin = {
5078 gen_helper_advsimd_minh,
5079 gen_helper_vfp_mins,
5080 gen_helper_vfp_mind,
5082 TRANS(FMIN_s, do_fp3_scalar, a, &f_scalar_fmin)
5084 static const FPScalar f_scalar_fmaxnm = {
5085 gen_helper_advsimd_maxnumh,
5086 gen_helper_vfp_maxnums,
5087 gen_helper_vfp_maxnumd,
5089 TRANS(FMAXNM_s, do_fp3_scalar, a, &f_scalar_fmaxnm)
5091 static const FPScalar f_scalar_fminnm = {
5092 gen_helper_advsimd_minnumh,
5093 gen_helper_vfp_minnums,
5094 gen_helper_vfp_minnumd,
5096 TRANS(FMINNM_s, do_fp3_scalar, a, &f_scalar_fminnm)
5098 static const FPScalar f_scalar_fmulx = {
5099 gen_helper_advsimd_mulxh,
5100 gen_helper_vfp_mulxs,
5101 gen_helper_vfp_mulxd,
5103 TRANS(FMULX_s, do_fp3_scalar, a, &f_scalar_fmulx)
5105 static void gen_fnmul_h(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_ptr s)
5107 gen_helper_vfp_mulh(d, n, m, s);
5108 gen_vfp_negh(d, d);
5111 static void gen_fnmul_s(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_ptr s)
5113 gen_helper_vfp_muls(d, n, m, s);
5114 gen_vfp_negs(d, d);
5117 static void gen_fnmul_d(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_ptr s)
5119 gen_helper_vfp_muld(d, n, m, s);
5120 gen_vfp_negd(d, d);
5123 static const FPScalar f_scalar_fnmul = {
5124 gen_fnmul_h,
5125 gen_fnmul_s,
5126 gen_fnmul_d,
5128 TRANS(FNMUL_s, do_fp3_scalar, a, &f_scalar_fnmul)
5130 static const FPScalar f_scalar_fcmeq = {
5131 gen_helper_advsimd_ceq_f16,
5132 gen_helper_neon_ceq_f32,
5133 gen_helper_neon_ceq_f64,
5135 TRANS(FCMEQ_s, do_fp3_scalar, a, &f_scalar_fcmeq)
5137 static const FPScalar f_scalar_fcmge = {
5138 gen_helper_advsimd_cge_f16,
5139 gen_helper_neon_cge_f32,
5140 gen_helper_neon_cge_f64,
5142 TRANS(FCMGE_s, do_fp3_scalar, a, &f_scalar_fcmge)
5144 static const FPScalar f_scalar_fcmgt = {
5145 gen_helper_advsimd_cgt_f16,
5146 gen_helper_neon_cgt_f32,
5147 gen_helper_neon_cgt_f64,
5149 TRANS(FCMGT_s, do_fp3_scalar, a, &f_scalar_fcmgt)
5151 static const FPScalar f_scalar_facge = {
5152 gen_helper_advsimd_acge_f16,
5153 gen_helper_neon_acge_f32,
5154 gen_helper_neon_acge_f64,
5156 TRANS(FACGE_s, do_fp3_scalar, a, &f_scalar_facge)
5158 static const FPScalar f_scalar_facgt = {
5159 gen_helper_advsimd_acgt_f16,
5160 gen_helper_neon_acgt_f32,
5161 gen_helper_neon_acgt_f64,
5163 TRANS(FACGT_s, do_fp3_scalar, a, &f_scalar_facgt)
5165 static void gen_fabd_h(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_ptr s)
5167 gen_helper_vfp_subh(d, n, m, s);
5168 gen_vfp_absh(d, d);
5171 static void gen_fabd_s(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_ptr s)
5173 gen_helper_vfp_subs(d, n, m, s);
5174 gen_vfp_abss(d, d);
5177 static void gen_fabd_d(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_ptr s)
5179 gen_helper_vfp_subd(d, n, m, s);
5180 gen_vfp_absd(d, d);
5183 static const FPScalar f_scalar_fabd = {
5184 gen_fabd_h,
5185 gen_fabd_s,
5186 gen_fabd_d,
5188 TRANS(FABD_s, do_fp3_scalar, a, &f_scalar_fabd)
5190 static const FPScalar f_scalar_frecps = {
5191 gen_helper_recpsf_f16,
5192 gen_helper_recpsf_f32,
5193 gen_helper_recpsf_f64,
5195 TRANS(FRECPS_s, do_fp3_scalar, a, &f_scalar_frecps)
5197 static const FPScalar f_scalar_frsqrts = {
5198 gen_helper_rsqrtsf_f16,
5199 gen_helper_rsqrtsf_f32,
5200 gen_helper_rsqrtsf_f64,
5202 TRANS(FRSQRTS_s, do_fp3_scalar, a, &f_scalar_frsqrts)
5204 static bool do_satacc_s(DisasContext *s, arg_rrr_e *a,
5205 MemOp sgn_n, MemOp sgn_m,
5206 void (*gen_bhs)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64, MemOp),
5207 void (*gen_d)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64))
5209 TCGv_i64 t0, t1, t2, qc;
5210 MemOp esz = a->esz;
5212 if (!fp_access_check(s)) {
5213 return true;
5216 t0 = tcg_temp_new_i64();
5217 t1 = tcg_temp_new_i64();
5218 t2 = tcg_temp_new_i64();
5219 qc = tcg_temp_new_i64();
5220 read_vec_element(s, t1, a->rn, 0, esz | sgn_n);
5221 read_vec_element(s, t2, a->rm, 0, esz | sgn_m);
5222 tcg_gen_ld_i64(qc, tcg_env, offsetof(CPUARMState, vfp.qc));
5224 if (esz == MO_64) {
5225 gen_d(t0, qc, t1, t2);
5226 } else {
5227 gen_bhs(t0, qc, t1, t2, esz);
5228 tcg_gen_ext_i64(t0, t0, esz);
5231 write_fp_dreg(s, a->rd, t0);
5232 tcg_gen_st_i64(qc, tcg_env, offsetof(CPUARMState, vfp.qc));
5233 return true;
5236 TRANS(SQADD_s, do_satacc_s, a, MO_SIGN, MO_SIGN, gen_sqadd_bhs, gen_sqadd_d)
5237 TRANS(SQSUB_s, do_satacc_s, a, MO_SIGN, MO_SIGN, gen_sqsub_bhs, gen_sqsub_d)
5238 TRANS(UQADD_s, do_satacc_s, a, 0, 0, gen_uqadd_bhs, gen_uqadd_d)
5239 TRANS(UQSUB_s, do_satacc_s, a, 0, 0, gen_uqsub_bhs, gen_uqsub_d)
5240 TRANS(SUQADD_s, do_satacc_s, a, MO_SIGN, 0, gen_suqadd_bhs, gen_suqadd_d)
5241 TRANS(USQADD_s, do_satacc_s, a, 0, MO_SIGN, gen_usqadd_bhs, gen_usqadd_d)
5243 static bool do_int3_scalar_d(DisasContext *s, arg_rrr_e *a,
5244 void (*fn)(TCGv_i64, TCGv_i64, TCGv_i64))
5246 if (fp_access_check(s)) {
5247 TCGv_i64 t0 = tcg_temp_new_i64();
5248 TCGv_i64 t1 = tcg_temp_new_i64();
5250 read_vec_element(s, t0, a->rn, 0, MO_64);
5251 read_vec_element(s, t1, a->rm, 0, MO_64);
5252 fn(t0, t0, t1);
5253 write_fp_dreg(s, a->rd, t0);
5255 return true;
5258 TRANS(SSHL_s, do_int3_scalar_d, a, gen_sshl_i64)
5259 TRANS(USHL_s, do_int3_scalar_d, a, gen_ushl_i64)
5260 TRANS(SRSHL_s, do_int3_scalar_d, a, gen_helper_neon_rshl_s64)
5261 TRANS(URSHL_s, do_int3_scalar_d, a, gen_helper_neon_rshl_u64)
5262 TRANS(ADD_s, do_int3_scalar_d, a, tcg_gen_add_i64)
5263 TRANS(SUB_s, do_int3_scalar_d, a, tcg_gen_sub_i64)
5265 typedef struct ENVScalar2 {
5266 NeonGenTwoOpEnvFn *gen_bhs[3];
5267 NeonGenTwo64OpEnvFn *gen_d;
5268 } ENVScalar2;
5270 static bool do_env_scalar2(DisasContext *s, arg_rrr_e *a, const ENVScalar2 *f)
5272 if (!fp_access_check(s)) {
5273 return true;
5275 if (a->esz == MO_64) {
5276 TCGv_i64 t0 = read_fp_dreg(s, a->rn);
5277 TCGv_i64 t1 = read_fp_dreg(s, a->rm);
5278 f->gen_d(t0, tcg_env, t0, t1);
5279 write_fp_dreg(s, a->rd, t0);
5280 } else {
5281 TCGv_i32 t0 = tcg_temp_new_i32();
5282 TCGv_i32 t1 = tcg_temp_new_i32();
5284 read_vec_element_i32(s, t0, a->rn, 0, a->esz);
5285 read_vec_element_i32(s, t1, a->rm, 0, a->esz);
5286 f->gen_bhs[a->esz](t0, tcg_env, t0, t1);
5287 write_fp_sreg(s, a->rd, t0);
5289 return true;
5292 static const ENVScalar2 f_scalar_sqshl = {
5293 { gen_helper_neon_qshl_s8,
5294 gen_helper_neon_qshl_s16,
5295 gen_helper_neon_qshl_s32 },
5296 gen_helper_neon_qshl_s64,
5298 TRANS(SQSHL_s, do_env_scalar2, a, &f_scalar_sqshl)
5300 static const ENVScalar2 f_scalar_uqshl = {
5301 { gen_helper_neon_qshl_u8,
5302 gen_helper_neon_qshl_u16,
5303 gen_helper_neon_qshl_u32 },
5304 gen_helper_neon_qshl_u64,
5306 TRANS(UQSHL_s, do_env_scalar2, a, &f_scalar_uqshl)
5308 static const ENVScalar2 f_scalar_sqrshl = {
5309 { gen_helper_neon_qrshl_s8,
5310 gen_helper_neon_qrshl_s16,
5311 gen_helper_neon_qrshl_s32 },
5312 gen_helper_neon_qrshl_s64,
5314 TRANS(SQRSHL_s, do_env_scalar2, a, &f_scalar_sqrshl)
5316 static const ENVScalar2 f_scalar_uqrshl = {
5317 { gen_helper_neon_qrshl_u8,
5318 gen_helper_neon_qrshl_u16,
5319 gen_helper_neon_qrshl_u32 },
5320 gen_helper_neon_qrshl_u64,
5322 TRANS(UQRSHL_s, do_env_scalar2, a, &f_scalar_uqrshl)
5324 static bool do_env_scalar2_hs(DisasContext *s, arg_rrr_e *a,
5325 const ENVScalar2 *f)
5327 if (a->esz == MO_16 || a->esz == MO_32) {
5328 return do_env_scalar2(s, a, f);
5330 return false;
5333 static const ENVScalar2 f_scalar_sqdmulh = {
5334 { NULL, gen_helper_neon_qdmulh_s16, gen_helper_neon_qdmulh_s32 }
5336 TRANS(SQDMULH_s, do_env_scalar2_hs, a, &f_scalar_sqdmulh)
5338 static const ENVScalar2 f_scalar_sqrdmulh = {
5339 { NULL, gen_helper_neon_qrdmulh_s16, gen_helper_neon_qrdmulh_s32 }
5341 TRANS(SQRDMULH_s, do_env_scalar2_hs, a, &f_scalar_sqrdmulh)
5343 typedef struct ENVScalar3 {
5344 NeonGenThreeOpEnvFn *gen_hs[2];
5345 } ENVScalar3;
5347 static bool do_env_scalar3_hs(DisasContext *s, arg_rrr_e *a,
5348 const ENVScalar3 *f)
5350 TCGv_i32 t0, t1, t2;
5352 if (a->esz != MO_16 && a->esz != MO_32) {
5353 return false;
5355 if (!fp_access_check(s)) {
5356 return true;
5359 t0 = tcg_temp_new_i32();
5360 t1 = tcg_temp_new_i32();
5361 t2 = tcg_temp_new_i32();
5362 read_vec_element_i32(s, t0, a->rn, 0, a->esz);
5363 read_vec_element_i32(s, t1, a->rm, 0, a->esz);
5364 read_vec_element_i32(s, t2, a->rd, 0, a->esz);
5365 f->gen_hs[a->esz - 1](t0, tcg_env, t0, t1, t2);
5366 write_fp_sreg(s, a->rd, t0);
5367 return true;
5370 static const ENVScalar3 f_scalar_sqrdmlah = {
5371 { gen_helper_neon_qrdmlah_s16, gen_helper_neon_qrdmlah_s32 }
5373 TRANS_FEAT(SQRDMLAH_s, aa64_rdm, do_env_scalar3_hs, a, &f_scalar_sqrdmlah)
5375 static const ENVScalar3 f_scalar_sqrdmlsh = {
5376 { gen_helper_neon_qrdmlsh_s16, gen_helper_neon_qrdmlsh_s32 }
5378 TRANS_FEAT(SQRDMLSH_s, aa64_rdm, do_env_scalar3_hs, a, &f_scalar_sqrdmlsh)
5380 static bool do_cmop_d(DisasContext *s, arg_rrr_e *a, TCGCond cond)
5382 if (fp_access_check(s)) {
5383 TCGv_i64 t0 = read_fp_dreg(s, a->rn);
5384 TCGv_i64 t1 = read_fp_dreg(s, a->rm);
5385 tcg_gen_negsetcond_i64(cond, t0, t0, t1);
5386 write_fp_dreg(s, a->rd, t0);
5388 return true;
5391 TRANS(CMGT_s, do_cmop_d, a, TCG_COND_GT)
5392 TRANS(CMHI_s, do_cmop_d, a, TCG_COND_GTU)
5393 TRANS(CMGE_s, do_cmop_d, a, TCG_COND_GE)
5394 TRANS(CMHS_s, do_cmop_d, a, TCG_COND_GEU)
5395 TRANS(CMEQ_s, do_cmop_d, a, TCG_COND_EQ)
5396 TRANS(CMTST_s, do_cmop_d, a, TCG_COND_TSTNE)
5398 static bool do_fp3_vector(DisasContext *s, arg_qrrr_e *a, int data,
5399 gen_helper_gvec_3_ptr * const fns[3])
5401 MemOp esz = a->esz;
5403 switch (esz) {
5404 case MO_64:
5405 if (!a->q) {
5406 return false;
5408 break;
5409 case MO_32:
5410 break;
5411 case MO_16:
5412 if (!dc_isar_feature(aa64_fp16, s)) {
5413 return false;
5415 break;
5416 default:
5417 return false;
5419 if (fp_access_check(s)) {
5420 gen_gvec_op3_fpst(s, a->q, a->rd, a->rn, a->rm,
5421 esz == MO_16, data, fns[esz - 1]);
5423 return true;
5426 static gen_helper_gvec_3_ptr * const f_vector_fadd[3] = {
5427 gen_helper_gvec_fadd_h,
5428 gen_helper_gvec_fadd_s,
5429 gen_helper_gvec_fadd_d,
5431 TRANS(FADD_v, do_fp3_vector, a, 0, f_vector_fadd)
5433 static gen_helper_gvec_3_ptr * const f_vector_fsub[3] = {
5434 gen_helper_gvec_fsub_h,
5435 gen_helper_gvec_fsub_s,
5436 gen_helper_gvec_fsub_d,
5438 TRANS(FSUB_v, do_fp3_vector, a, 0, f_vector_fsub)
5440 static gen_helper_gvec_3_ptr * const f_vector_fdiv[3] = {
5441 gen_helper_gvec_fdiv_h,
5442 gen_helper_gvec_fdiv_s,
5443 gen_helper_gvec_fdiv_d,
5445 TRANS(FDIV_v, do_fp3_vector, a, 0, f_vector_fdiv)
5447 static gen_helper_gvec_3_ptr * const f_vector_fmul[3] = {
5448 gen_helper_gvec_fmul_h,
5449 gen_helper_gvec_fmul_s,
5450 gen_helper_gvec_fmul_d,
5452 TRANS(FMUL_v, do_fp3_vector, a, 0, f_vector_fmul)
5454 static gen_helper_gvec_3_ptr * const f_vector_fmax[3] = {
5455 gen_helper_gvec_fmax_h,
5456 gen_helper_gvec_fmax_s,
5457 gen_helper_gvec_fmax_d,
5459 TRANS(FMAX_v, do_fp3_vector, a, 0, f_vector_fmax)
5461 static gen_helper_gvec_3_ptr * const f_vector_fmin[3] = {
5462 gen_helper_gvec_fmin_h,
5463 gen_helper_gvec_fmin_s,
5464 gen_helper_gvec_fmin_d,
5466 TRANS(FMIN_v, do_fp3_vector, a, 0, f_vector_fmin)
5468 static gen_helper_gvec_3_ptr * const f_vector_fmaxnm[3] = {
5469 gen_helper_gvec_fmaxnum_h,
5470 gen_helper_gvec_fmaxnum_s,
5471 gen_helper_gvec_fmaxnum_d,
5473 TRANS(FMAXNM_v, do_fp3_vector, a, 0, f_vector_fmaxnm)
5475 static gen_helper_gvec_3_ptr * const f_vector_fminnm[3] = {
5476 gen_helper_gvec_fminnum_h,
5477 gen_helper_gvec_fminnum_s,
5478 gen_helper_gvec_fminnum_d,
5480 TRANS(FMINNM_v, do_fp3_vector, a, 0, f_vector_fminnm)
5482 static gen_helper_gvec_3_ptr * const f_vector_fmulx[3] = {
5483 gen_helper_gvec_fmulx_h,
5484 gen_helper_gvec_fmulx_s,
5485 gen_helper_gvec_fmulx_d,
5487 TRANS(FMULX_v, do_fp3_vector, a, 0, f_vector_fmulx)
5489 static gen_helper_gvec_3_ptr * const f_vector_fmla[3] = {
5490 gen_helper_gvec_vfma_h,
5491 gen_helper_gvec_vfma_s,
5492 gen_helper_gvec_vfma_d,
5494 TRANS(FMLA_v, do_fp3_vector, a, 0, f_vector_fmla)
5496 static gen_helper_gvec_3_ptr * const f_vector_fmls[3] = {
5497 gen_helper_gvec_vfms_h,
5498 gen_helper_gvec_vfms_s,
5499 gen_helper_gvec_vfms_d,
5501 TRANS(FMLS_v, do_fp3_vector, a, 0, f_vector_fmls)
5503 static gen_helper_gvec_3_ptr * const f_vector_fcmeq[3] = {
5504 gen_helper_gvec_fceq_h,
5505 gen_helper_gvec_fceq_s,
5506 gen_helper_gvec_fceq_d,
5508 TRANS(FCMEQ_v, do_fp3_vector, a, 0, f_vector_fcmeq)
5510 static gen_helper_gvec_3_ptr * const f_vector_fcmge[3] = {
5511 gen_helper_gvec_fcge_h,
5512 gen_helper_gvec_fcge_s,
5513 gen_helper_gvec_fcge_d,
5515 TRANS(FCMGE_v, do_fp3_vector, a, 0, f_vector_fcmge)
5517 static gen_helper_gvec_3_ptr * const f_vector_fcmgt[3] = {
5518 gen_helper_gvec_fcgt_h,
5519 gen_helper_gvec_fcgt_s,
5520 gen_helper_gvec_fcgt_d,
5522 TRANS(FCMGT_v, do_fp3_vector, a, 0, f_vector_fcmgt)
5524 static gen_helper_gvec_3_ptr * const f_vector_facge[3] = {
5525 gen_helper_gvec_facge_h,
5526 gen_helper_gvec_facge_s,
5527 gen_helper_gvec_facge_d,
5529 TRANS(FACGE_v, do_fp3_vector, a, 0, f_vector_facge)
5531 static gen_helper_gvec_3_ptr * const f_vector_facgt[3] = {
5532 gen_helper_gvec_facgt_h,
5533 gen_helper_gvec_facgt_s,
5534 gen_helper_gvec_facgt_d,
5536 TRANS(FACGT_v, do_fp3_vector, a, 0, f_vector_facgt)
5538 static gen_helper_gvec_3_ptr * const f_vector_fabd[3] = {
5539 gen_helper_gvec_fabd_h,
5540 gen_helper_gvec_fabd_s,
5541 gen_helper_gvec_fabd_d,
5543 TRANS(FABD_v, do_fp3_vector, a, 0, f_vector_fabd)
5545 static gen_helper_gvec_3_ptr * const f_vector_frecps[3] = {
5546 gen_helper_gvec_recps_h,
5547 gen_helper_gvec_recps_s,
5548 gen_helper_gvec_recps_d,
5550 TRANS(FRECPS_v, do_fp3_vector, a, 0, f_vector_frecps)
5552 static gen_helper_gvec_3_ptr * const f_vector_frsqrts[3] = {
5553 gen_helper_gvec_rsqrts_h,
5554 gen_helper_gvec_rsqrts_s,
5555 gen_helper_gvec_rsqrts_d,
5557 TRANS(FRSQRTS_v, do_fp3_vector, a, 0, f_vector_frsqrts)
5559 static gen_helper_gvec_3_ptr * const f_vector_faddp[3] = {
5560 gen_helper_gvec_faddp_h,
5561 gen_helper_gvec_faddp_s,
5562 gen_helper_gvec_faddp_d,
5564 TRANS(FADDP_v, do_fp3_vector, a, 0, f_vector_faddp)
5566 static gen_helper_gvec_3_ptr * const f_vector_fmaxp[3] = {
5567 gen_helper_gvec_fmaxp_h,
5568 gen_helper_gvec_fmaxp_s,
5569 gen_helper_gvec_fmaxp_d,
5571 TRANS(FMAXP_v, do_fp3_vector, a, 0, f_vector_fmaxp)
5573 static gen_helper_gvec_3_ptr * const f_vector_fminp[3] = {
5574 gen_helper_gvec_fminp_h,
5575 gen_helper_gvec_fminp_s,
5576 gen_helper_gvec_fminp_d,
5578 TRANS(FMINP_v, do_fp3_vector, a, 0, f_vector_fminp)
5580 static gen_helper_gvec_3_ptr * const f_vector_fmaxnmp[3] = {
5581 gen_helper_gvec_fmaxnump_h,
5582 gen_helper_gvec_fmaxnump_s,
5583 gen_helper_gvec_fmaxnump_d,
5585 TRANS(FMAXNMP_v, do_fp3_vector, a, 0, f_vector_fmaxnmp)
5587 static gen_helper_gvec_3_ptr * const f_vector_fminnmp[3] = {
5588 gen_helper_gvec_fminnump_h,
5589 gen_helper_gvec_fminnump_s,
5590 gen_helper_gvec_fminnump_d,
5592 TRANS(FMINNMP_v, do_fp3_vector, a, 0, f_vector_fminnmp)
5594 static bool do_fmlal(DisasContext *s, arg_qrrr_e *a, bool is_s, bool is_2)
5596 if (fp_access_check(s)) {
5597 int data = (is_2 << 1) | is_s;
5598 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
5599 vec_full_reg_offset(s, a->rn),
5600 vec_full_reg_offset(s, a->rm), tcg_env,
5601 a->q ? 16 : 8, vec_full_reg_size(s),
5602 data, gen_helper_gvec_fmlal_a64);
5604 return true;
5607 TRANS_FEAT(FMLAL_v, aa64_fhm, do_fmlal, a, false, false)
5608 TRANS_FEAT(FMLSL_v, aa64_fhm, do_fmlal, a, true, false)
5609 TRANS_FEAT(FMLAL2_v, aa64_fhm, do_fmlal, a, false, true)
5610 TRANS_FEAT(FMLSL2_v, aa64_fhm, do_fmlal, a, true, true)
5612 TRANS(ADDP_v, do_gvec_fn3, a, gen_gvec_addp)
5613 TRANS(SMAXP_v, do_gvec_fn3_no64, a, gen_gvec_smaxp)
5614 TRANS(SMINP_v, do_gvec_fn3_no64, a, gen_gvec_sminp)
5615 TRANS(UMAXP_v, do_gvec_fn3_no64, a, gen_gvec_umaxp)
5616 TRANS(UMINP_v, do_gvec_fn3_no64, a, gen_gvec_uminp)
5618 TRANS(AND_v, do_gvec_fn3, a, tcg_gen_gvec_and)
5619 TRANS(BIC_v, do_gvec_fn3, a, tcg_gen_gvec_andc)
5620 TRANS(ORR_v, do_gvec_fn3, a, tcg_gen_gvec_or)
5621 TRANS(ORN_v, do_gvec_fn3, a, tcg_gen_gvec_orc)
5622 TRANS(EOR_v, do_gvec_fn3, a, tcg_gen_gvec_xor)
5624 static bool do_bitsel(DisasContext *s, bool is_q, int d, int a, int b, int c)
5626 if (fp_access_check(s)) {
5627 gen_gvec_fn4(s, is_q, d, a, b, c, tcg_gen_gvec_bitsel, 0);
5629 return true;
5632 TRANS(BSL_v, do_bitsel, a->q, a->rd, a->rd, a->rn, a->rm)
5633 TRANS(BIT_v, do_bitsel, a->q, a->rd, a->rm, a->rn, a->rd)
5634 TRANS(BIF_v, do_bitsel, a->q, a->rd, a->rm, a->rd, a->rn)
5636 TRANS(SQADD_v, do_gvec_fn3, a, gen_gvec_sqadd_qc)
5637 TRANS(UQADD_v, do_gvec_fn3, a, gen_gvec_uqadd_qc)
5638 TRANS(SQSUB_v, do_gvec_fn3, a, gen_gvec_sqsub_qc)
5639 TRANS(UQSUB_v, do_gvec_fn3, a, gen_gvec_uqsub_qc)
5640 TRANS(SUQADD_v, do_gvec_fn3, a, gen_gvec_suqadd_qc)
5641 TRANS(USQADD_v, do_gvec_fn3, a, gen_gvec_usqadd_qc)
5643 TRANS(SSHL_v, do_gvec_fn3, a, gen_gvec_sshl)
5644 TRANS(USHL_v, do_gvec_fn3, a, gen_gvec_ushl)
5645 TRANS(SRSHL_v, do_gvec_fn3, a, gen_gvec_srshl)
5646 TRANS(URSHL_v, do_gvec_fn3, a, gen_gvec_urshl)
5647 TRANS(SQSHL_v, do_gvec_fn3, a, gen_neon_sqshl)
5648 TRANS(UQSHL_v, do_gvec_fn3, a, gen_neon_uqshl)
5649 TRANS(SQRSHL_v, do_gvec_fn3, a, gen_neon_sqrshl)
5650 TRANS(UQRSHL_v, do_gvec_fn3, a, gen_neon_uqrshl)
5652 TRANS(ADD_v, do_gvec_fn3, a, tcg_gen_gvec_add)
5653 TRANS(SUB_v, do_gvec_fn3, a, tcg_gen_gvec_sub)
5654 TRANS(SHADD_v, do_gvec_fn3_no64, a, gen_gvec_shadd)
5655 TRANS(UHADD_v, do_gvec_fn3_no64, a, gen_gvec_uhadd)
5656 TRANS(SHSUB_v, do_gvec_fn3_no64, a, gen_gvec_shsub)
5657 TRANS(UHSUB_v, do_gvec_fn3_no64, a, gen_gvec_uhsub)
5658 TRANS(SRHADD_v, do_gvec_fn3_no64, a, gen_gvec_srhadd)
5659 TRANS(URHADD_v, do_gvec_fn3_no64, a, gen_gvec_urhadd)
5660 TRANS(SMAX_v, do_gvec_fn3_no64, a, tcg_gen_gvec_smax)
5661 TRANS(UMAX_v, do_gvec_fn3_no64, a, tcg_gen_gvec_umax)
5662 TRANS(SMIN_v, do_gvec_fn3_no64, a, tcg_gen_gvec_smin)
5663 TRANS(UMIN_v, do_gvec_fn3_no64, a, tcg_gen_gvec_umin)
5664 TRANS(SABA_v, do_gvec_fn3_no64, a, gen_gvec_saba)
5665 TRANS(UABA_v, do_gvec_fn3_no64, a, gen_gvec_uaba)
5666 TRANS(SABD_v, do_gvec_fn3_no64, a, gen_gvec_sabd)
5667 TRANS(UABD_v, do_gvec_fn3_no64, a, gen_gvec_uabd)
5668 TRANS(MUL_v, do_gvec_fn3_no64, a, tcg_gen_gvec_mul)
5669 TRANS(PMUL_v, do_gvec_op3_ool, a, 0, gen_helper_gvec_pmul_b)
5670 TRANS(MLA_v, do_gvec_fn3_no64, a, gen_gvec_mla)
5671 TRANS(MLS_v, do_gvec_fn3_no64, a, gen_gvec_mls)
5673 static bool do_cmop_v(DisasContext *s, arg_qrrr_e *a, TCGCond cond)
5675 if (a->esz == MO_64 && !a->q) {
5676 return false;
5678 if (fp_access_check(s)) {
5679 tcg_gen_gvec_cmp(cond, a->esz,
5680 vec_full_reg_offset(s, a->rd),
5681 vec_full_reg_offset(s, a->rn),
5682 vec_full_reg_offset(s, a->rm),
5683 a->q ? 16 : 8, vec_full_reg_size(s));
5685 return true;
5688 TRANS(CMGT_v, do_cmop_v, a, TCG_COND_GT)
5689 TRANS(CMHI_v, do_cmop_v, a, TCG_COND_GTU)
5690 TRANS(CMGE_v, do_cmop_v, a, TCG_COND_GE)
5691 TRANS(CMHS_v, do_cmop_v, a, TCG_COND_GEU)
5692 TRANS(CMEQ_v, do_cmop_v, a, TCG_COND_EQ)
5693 TRANS(CMTST_v, do_gvec_fn3, a, gen_gvec_cmtst)
5695 TRANS(SQDMULH_v, do_gvec_fn3_no8_no64, a, gen_gvec_sqdmulh_qc)
5696 TRANS(SQRDMULH_v, do_gvec_fn3_no8_no64, a, gen_gvec_sqrdmulh_qc)
5697 TRANS_FEAT(SQRDMLAH_v, aa64_rdm, do_gvec_fn3_no8_no64, a, gen_gvec_sqrdmlah_qc)
5698 TRANS_FEAT(SQRDMLSH_v, aa64_rdm, do_gvec_fn3_no8_no64, a, gen_gvec_sqrdmlsh_qc)
5700 static bool do_dot_vector(DisasContext *s, arg_qrrr_e *a,
5701 gen_helper_gvec_4 *fn)
5703 if (fp_access_check(s)) {
5704 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, 0, fn);
5706 return true;
5709 static bool do_dot_vector_env(DisasContext *s, arg_qrrr_e *a,
5710 gen_helper_gvec_4_ptr *fn)
5712 if (fp_access_check(s)) {
5713 gen_gvec_op4_env(s, a->q, a->rd, a->rn, a->rm, a->rd, 0, fn);
5715 return true;
5718 TRANS_FEAT(SDOT_v, aa64_dp, do_dot_vector, a, gen_helper_gvec_sdot_b)
5719 TRANS_FEAT(UDOT_v, aa64_dp, do_dot_vector, a, gen_helper_gvec_udot_b)
5720 TRANS_FEAT(USDOT_v, aa64_i8mm, do_dot_vector, a, gen_helper_gvec_usdot_b)
5721 TRANS_FEAT(BFDOT_v, aa64_bf16, do_dot_vector_env, a, gen_helper_gvec_bfdot)
5722 TRANS_FEAT(BFMMLA, aa64_bf16, do_dot_vector_env, a, gen_helper_gvec_bfmmla)
5723 TRANS_FEAT(SMMLA, aa64_i8mm, do_dot_vector, a, gen_helper_gvec_smmla_b)
5724 TRANS_FEAT(UMMLA, aa64_i8mm, do_dot_vector, a, gen_helper_gvec_ummla_b)
5725 TRANS_FEAT(USMMLA, aa64_i8mm, do_dot_vector, a, gen_helper_gvec_usmmla_b)
5727 static bool trans_BFMLAL_v(DisasContext *s, arg_qrrr_e *a)
5729 if (!dc_isar_feature(aa64_bf16, s)) {
5730 return false;
5732 if (fp_access_check(s)) {
5733 /* Q bit selects BFMLALB vs BFMLALT. */
5734 gen_gvec_op4_fpst(s, true, a->rd, a->rn, a->rm, a->rd, false, a->q,
5735 gen_helper_gvec_bfmlal);
5737 return true;
5740 static gen_helper_gvec_3_ptr * const f_vector_fcadd[3] = {
5741 gen_helper_gvec_fcaddh,
5742 gen_helper_gvec_fcadds,
5743 gen_helper_gvec_fcaddd,
5745 TRANS_FEAT(FCADD_90, aa64_fcma, do_fp3_vector, a, 0, f_vector_fcadd)
5746 TRANS_FEAT(FCADD_270, aa64_fcma, do_fp3_vector, a, 1, f_vector_fcadd)
5748 static bool trans_FCMLA_v(DisasContext *s, arg_FCMLA_v *a)
5750 gen_helper_gvec_4_ptr *fn;
5752 if (!dc_isar_feature(aa64_fcma, s)) {
5753 return false;
5755 switch (a->esz) {
5756 case MO_64:
5757 if (!a->q) {
5758 return false;
5760 fn = gen_helper_gvec_fcmlad;
5761 break;
5762 case MO_32:
5763 fn = gen_helper_gvec_fcmlas;
5764 break;
5765 case MO_16:
5766 if (!dc_isar_feature(aa64_fp16, s)) {
5767 return false;
5769 fn = gen_helper_gvec_fcmlah;
5770 break;
5771 default:
5772 return false;
5774 if (fp_access_check(s)) {
5775 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd,
5776 a->esz == MO_16, a->rot, fn);
5778 return true;
5782 * Widening vector x vector/indexed.
5784 * These read from the top or bottom half of a 128-bit vector.
5785 * After widening, optionally accumulate with a 128-bit vector.
5786 * Implement these inline, as the number of elements are limited
5787 * and the related SVE and SME operations on larger vectors use
5788 * even/odd elements instead of top/bottom half.
5790 * If idx >= 0, operand 2 is indexed, otherwise vector.
5791 * If acc, operand 0 is loaded with rd.
5794 /* For low half, iterating up. */
5795 static bool do_3op_widening(DisasContext *s, MemOp memop, int top,
5796 int rd, int rn, int rm, int idx,
5797 NeonGenTwo64OpFn *fn, bool acc)
5799 TCGv_i64 tcg_op0 = tcg_temp_new_i64();
5800 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
5801 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
5802 MemOp esz = memop & MO_SIZE;
5803 int half = 8 >> esz;
5804 int top_swap, top_half;
5806 /* There are no 64x64->128 bit operations. */
5807 if (esz >= MO_64) {
5808 return false;
5810 if (!fp_access_check(s)) {
5811 return true;
5814 if (idx >= 0) {
5815 read_vec_element(s, tcg_op2, rm, idx, memop);
5819 * For top half inputs, iterate forward; backward for bottom half.
5820 * This means the store to the destination will not occur until
5821 * overlapping input inputs are consumed.
5822 * Use top_swap to conditionally invert the forward iteration index.
5824 top_swap = top ? 0 : half - 1;
5825 top_half = top ? half : 0;
5827 for (int elt_fwd = 0; elt_fwd < half; ++elt_fwd) {
5828 int elt = elt_fwd ^ top_swap;
5830 read_vec_element(s, tcg_op1, rn, elt + top_half, memop);
5831 if (idx < 0) {
5832 read_vec_element(s, tcg_op2, rm, elt + top_half, memop);
5834 if (acc) {
5835 read_vec_element(s, tcg_op0, rd, elt, memop + 1);
5837 fn(tcg_op0, tcg_op1, tcg_op2);
5838 write_vec_element(s, tcg_op0, rd, elt, esz + 1);
5840 clear_vec_high(s, 1, rd);
5841 return true;
5844 static void gen_muladd_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
5846 TCGv_i64 t = tcg_temp_new_i64();
5847 tcg_gen_mul_i64(t, n, m);
5848 tcg_gen_add_i64(d, d, t);
5851 static void gen_mulsub_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
5853 TCGv_i64 t = tcg_temp_new_i64();
5854 tcg_gen_mul_i64(t, n, m);
5855 tcg_gen_sub_i64(d, d, t);
5858 TRANS(SMULL_v, do_3op_widening,
5859 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5860 tcg_gen_mul_i64, false)
5861 TRANS(UMULL_v, do_3op_widening,
5862 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5863 tcg_gen_mul_i64, false)
5864 TRANS(SMLAL_v, do_3op_widening,
5865 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5866 gen_muladd_i64, true)
5867 TRANS(UMLAL_v, do_3op_widening,
5868 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5869 gen_muladd_i64, true)
5870 TRANS(SMLSL_v, do_3op_widening,
5871 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5872 gen_mulsub_i64, true)
5873 TRANS(UMLSL_v, do_3op_widening,
5874 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5875 gen_mulsub_i64, true)
5877 TRANS(SMULL_vi, do_3op_widening,
5878 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
5879 tcg_gen_mul_i64, false)
5880 TRANS(UMULL_vi, do_3op_widening,
5881 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
5882 tcg_gen_mul_i64, false)
5883 TRANS(SMLAL_vi, do_3op_widening,
5884 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
5885 gen_muladd_i64, true)
5886 TRANS(UMLAL_vi, do_3op_widening,
5887 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
5888 gen_muladd_i64, true)
5889 TRANS(SMLSL_vi, do_3op_widening,
5890 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
5891 gen_mulsub_i64, true)
5892 TRANS(UMLSL_vi, do_3op_widening,
5893 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
5894 gen_mulsub_i64, true)
5896 static void gen_sabd_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
5898 TCGv_i64 t1 = tcg_temp_new_i64();
5899 TCGv_i64 t2 = tcg_temp_new_i64();
5901 tcg_gen_sub_i64(t1, n, m);
5902 tcg_gen_sub_i64(t2, m, n);
5903 tcg_gen_movcond_i64(TCG_COND_GE, d, n, m, t1, t2);
5906 static void gen_uabd_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
5908 TCGv_i64 t1 = tcg_temp_new_i64();
5909 TCGv_i64 t2 = tcg_temp_new_i64();
5911 tcg_gen_sub_i64(t1, n, m);
5912 tcg_gen_sub_i64(t2, m, n);
5913 tcg_gen_movcond_i64(TCG_COND_GEU, d, n, m, t1, t2);
5916 static void gen_saba_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
5918 TCGv_i64 t = tcg_temp_new_i64();
5919 gen_sabd_i64(t, n, m);
5920 tcg_gen_add_i64(d, d, t);
5923 static void gen_uaba_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
5925 TCGv_i64 t = tcg_temp_new_i64();
5926 gen_uabd_i64(t, n, m);
5927 tcg_gen_add_i64(d, d, t);
5930 TRANS(SADDL_v, do_3op_widening,
5931 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5932 tcg_gen_add_i64, false)
5933 TRANS(UADDL_v, do_3op_widening,
5934 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5935 tcg_gen_add_i64, false)
5936 TRANS(SSUBL_v, do_3op_widening,
5937 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5938 tcg_gen_sub_i64, false)
5939 TRANS(USUBL_v, do_3op_widening,
5940 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5941 tcg_gen_sub_i64, false)
5942 TRANS(SABDL_v, do_3op_widening,
5943 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5944 gen_sabd_i64, false)
5945 TRANS(UABDL_v, do_3op_widening,
5946 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5947 gen_uabd_i64, false)
5948 TRANS(SABAL_v, do_3op_widening,
5949 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5950 gen_saba_i64, true)
5951 TRANS(UABAL_v, do_3op_widening,
5952 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5953 gen_uaba_i64, true)
5955 static void gen_sqdmull_h(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
5957 tcg_gen_mul_i64(d, n, m);
5958 gen_helper_neon_addl_saturate_s32(d, tcg_env, d, d);
5961 static void gen_sqdmull_s(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
5963 tcg_gen_mul_i64(d, n, m);
5964 gen_helper_neon_addl_saturate_s64(d, tcg_env, d, d);
5967 static void gen_sqdmlal_h(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
5969 TCGv_i64 t = tcg_temp_new_i64();
5971 tcg_gen_mul_i64(t, n, m);
5972 gen_helper_neon_addl_saturate_s32(t, tcg_env, t, t);
5973 gen_helper_neon_addl_saturate_s32(d, tcg_env, d, t);
5976 static void gen_sqdmlal_s(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
5978 TCGv_i64 t = tcg_temp_new_i64();
5980 tcg_gen_mul_i64(t, n, m);
5981 gen_helper_neon_addl_saturate_s64(t, tcg_env, t, t);
5982 gen_helper_neon_addl_saturate_s64(d, tcg_env, d, t);
5985 static void gen_sqdmlsl_h(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
5987 TCGv_i64 t = tcg_temp_new_i64();
5989 tcg_gen_mul_i64(t, n, m);
5990 gen_helper_neon_addl_saturate_s32(t, tcg_env, t, t);
5991 tcg_gen_neg_i64(t, t);
5992 gen_helper_neon_addl_saturate_s32(d, tcg_env, d, t);
5995 static void gen_sqdmlsl_s(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
5997 TCGv_i64 t = tcg_temp_new_i64();
5999 tcg_gen_mul_i64(t, n, m);
6000 gen_helper_neon_addl_saturate_s64(t, tcg_env, t, t);
6001 tcg_gen_neg_i64(t, t);
6002 gen_helper_neon_addl_saturate_s64(d, tcg_env, d, t);
6005 TRANS(SQDMULL_v, do_3op_widening,
6006 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6007 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6008 TRANS(SQDMLAL_v, do_3op_widening,
6009 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6010 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6011 TRANS(SQDMLSL_v, do_3op_widening,
6012 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6013 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6015 TRANS(SQDMULL_vi, do_3op_widening,
6016 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6017 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6018 TRANS(SQDMLAL_vi, do_3op_widening,
6019 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6020 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6021 TRANS(SQDMLSL_vi, do_3op_widening,
6022 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6023 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6025 static bool do_addsub_wide(DisasContext *s, arg_qrrr_e *a,
6026 MemOp sign, bool sub)
6028 TCGv_i64 tcg_op0, tcg_op1;
6029 MemOp esz = a->esz;
6030 int half = 8 >> esz;
6031 bool top = a->q;
6032 int top_swap = top ? 0 : half - 1;
6033 int top_half = top ? half : 0;
6035 /* There are no 64x64->128 bit operations. */
6036 if (esz >= MO_64) {
6037 return false;
6039 if (!fp_access_check(s)) {
6040 return true;
6042 tcg_op0 = tcg_temp_new_i64();
6043 tcg_op1 = tcg_temp_new_i64();
6045 for (int elt_fwd = 0; elt_fwd < half; ++elt_fwd) {
6046 int elt = elt_fwd ^ top_swap;
6048 read_vec_element(s, tcg_op1, a->rm, elt + top_half, esz | sign);
6049 read_vec_element(s, tcg_op0, a->rn, elt, esz + 1);
6050 if (sub) {
6051 tcg_gen_sub_i64(tcg_op0, tcg_op0, tcg_op1);
6052 } else {
6053 tcg_gen_add_i64(tcg_op0, tcg_op0, tcg_op1);
6055 write_vec_element(s, tcg_op0, a->rd, elt, esz + 1);
6057 clear_vec_high(s, 1, a->rd);
6058 return true;
6061 TRANS(SADDW, do_addsub_wide, a, MO_SIGN, false)
6062 TRANS(UADDW, do_addsub_wide, a, 0, false)
6063 TRANS(SSUBW, do_addsub_wide, a, MO_SIGN, true)
6064 TRANS(USUBW, do_addsub_wide, a, 0, true)
6066 static bool do_addsub_highnarrow(DisasContext *s, arg_qrrr_e *a,
6067 bool sub, bool round)
6069 TCGv_i64 tcg_op0, tcg_op1;
6070 MemOp esz = a->esz;
6071 int half = 8 >> esz;
6072 bool top = a->q;
6073 int ebits = 8 << esz;
6074 uint64_t rbit = 1ull << (ebits - 1);
6075 int top_swap, top_half;
6077 /* There are no 128x128->64 bit operations. */
6078 if (esz >= MO_64) {
6079 return false;
6081 if (!fp_access_check(s)) {
6082 return true;
6084 tcg_op0 = tcg_temp_new_i64();
6085 tcg_op1 = tcg_temp_new_i64();
6088 * For top half inputs, iterate backward; forward for bottom half.
6089 * This means the store to the destination will not occur until
6090 * overlapping input inputs are consumed.
6092 top_swap = top ? half - 1 : 0;
6093 top_half = top ? half : 0;
6095 for (int elt_fwd = 0; elt_fwd < half; ++elt_fwd) {
6096 int elt = elt_fwd ^ top_swap;
6098 read_vec_element(s, tcg_op1, a->rm, elt, esz + 1);
6099 read_vec_element(s, tcg_op0, a->rn, elt, esz + 1);
6100 if (sub) {
6101 tcg_gen_sub_i64(tcg_op0, tcg_op0, tcg_op1);
6102 } else {
6103 tcg_gen_add_i64(tcg_op0, tcg_op0, tcg_op1);
6105 if (round) {
6106 tcg_gen_addi_i64(tcg_op0, tcg_op0, rbit);
6108 tcg_gen_shri_i64(tcg_op0, tcg_op0, ebits);
6109 write_vec_element(s, tcg_op0, a->rd, elt + top_half, esz);
6111 clear_vec_high(s, top, a->rd);
6112 return true;
6115 TRANS(ADDHN, do_addsub_highnarrow, a, false, false)
6116 TRANS(SUBHN, do_addsub_highnarrow, a, true, false)
6117 TRANS(RADDHN, do_addsub_highnarrow, a, false, true)
6118 TRANS(RSUBHN, do_addsub_highnarrow, a, true, true)
6120 static bool do_pmull(DisasContext *s, arg_qrrr_e *a, gen_helper_gvec_3 *fn)
6122 if (fp_access_check(s)) {
6123 /* The Q field specifies lo/hi half input for these insns. */
6124 gen_gvec_op3_ool(s, true, a->rd, a->rn, a->rm, a->q, fn);
6126 return true;
6129 TRANS(PMULL_p8, do_pmull, a, gen_helper_neon_pmull_h)
6130 TRANS_FEAT(PMULL_p64, aa64_pmull, do_pmull, a, gen_helper_gvec_pmull_q)
6133 * Advanced SIMD scalar/vector x indexed element
6136 static bool do_fp3_scalar_idx(DisasContext *s, arg_rrx_e *a, const FPScalar *f)
6138 switch (a->esz) {
6139 case MO_64:
6140 if (fp_access_check(s)) {
6141 TCGv_i64 t0 = read_fp_dreg(s, a->rn);
6142 TCGv_i64 t1 = tcg_temp_new_i64();
6144 read_vec_element(s, t1, a->rm, a->idx, MO_64);
6145 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_FPCR));
6146 write_fp_dreg(s, a->rd, t0);
6148 break;
6149 case MO_32:
6150 if (fp_access_check(s)) {
6151 TCGv_i32 t0 = read_fp_sreg(s, a->rn);
6152 TCGv_i32 t1 = tcg_temp_new_i32();
6154 read_vec_element_i32(s, t1, a->rm, a->idx, MO_32);
6155 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_FPCR));
6156 write_fp_sreg(s, a->rd, t0);
6158 break;
6159 case MO_16:
6160 if (!dc_isar_feature(aa64_fp16, s)) {
6161 return false;
6163 if (fp_access_check(s)) {
6164 TCGv_i32 t0 = read_fp_hreg(s, a->rn);
6165 TCGv_i32 t1 = tcg_temp_new_i32();
6167 read_vec_element_i32(s, t1, a->rm, a->idx, MO_16);
6168 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_FPCR_F16));
6169 write_fp_sreg(s, a->rd, t0);
6171 break;
6172 default:
6173 g_assert_not_reached();
6175 return true;
6178 TRANS(FMUL_si, do_fp3_scalar_idx, a, &f_scalar_fmul)
6179 TRANS(FMULX_si, do_fp3_scalar_idx, a, &f_scalar_fmulx)
6181 static bool do_fmla_scalar_idx(DisasContext *s, arg_rrx_e *a, bool neg)
6183 switch (a->esz) {
6184 case MO_64:
6185 if (fp_access_check(s)) {
6186 TCGv_i64 t0 = read_fp_dreg(s, a->rd);
6187 TCGv_i64 t1 = read_fp_dreg(s, a->rn);
6188 TCGv_i64 t2 = tcg_temp_new_i64();
6190 read_vec_element(s, t2, a->rm, a->idx, MO_64);
6191 if (neg) {
6192 gen_vfp_negd(t1, t1);
6194 gen_helper_vfp_muladdd(t0, t1, t2, t0, fpstatus_ptr(FPST_FPCR));
6195 write_fp_dreg(s, a->rd, t0);
6197 break;
6198 case MO_32:
6199 if (fp_access_check(s)) {
6200 TCGv_i32 t0 = read_fp_sreg(s, a->rd);
6201 TCGv_i32 t1 = read_fp_sreg(s, a->rn);
6202 TCGv_i32 t2 = tcg_temp_new_i32();
6204 read_vec_element_i32(s, t2, a->rm, a->idx, MO_32);
6205 if (neg) {
6206 gen_vfp_negs(t1, t1);
6208 gen_helper_vfp_muladds(t0, t1, t2, t0, fpstatus_ptr(FPST_FPCR));
6209 write_fp_sreg(s, a->rd, t0);
6211 break;
6212 case MO_16:
6213 if (!dc_isar_feature(aa64_fp16, s)) {
6214 return false;
6216 if (fp_access_check(s)) {
6217 TCGv_i32 t0 = read_fp_hreg(s, a->rd);
6218 TCGv_i32 t1 = read_fp_hreg(s, a->rn);
6219 TCGv_i32 t2 = tcg_temp_new_i32();
6221 read_vec_element_i32(s, t2, a->rm, a->idx, MO_16);
6222 if (neg) {
6223 gen_vfp_negh(t1, t1);
6225 gen_helper_advsimd_muladdh(t0, t1, t2, t0,
6226 fpstatus_ptr(FPST_FPCR_F16));
6227 write_fp_sreg(s, a->rd, t0);
6229 break;
6230 default:
6231 g_assert_not_reached();
6233 return true;
6236 TRANS(FMLA_si, do_fmla_scalar_idx, a, false)
6237 TRANS(FMLS_si, do_fmla_scalar_idx, a, true)
6239 static bool do_env_scalar2_idx_hs(DisasContext *s, arg_rrx_e *a,
6240 const ENVScalar2 *f)
6242 if (a->esz < MO_16 || a->esz > MO_32) {
6243 return false;
6245 if (fp_access_check(s)) {
6246 TCGv_i32 t0 = tcg_temp_new_i32();
6247 TCGv_i32 t1 = tcg_temp_new_i32();
6249 read_vec_element_i32(s, t0, a->rn, 0, a->esz);
6250 read_vec_element_i32(s, t1, a->rm, a->idx, a->esz);
6251 f->gen_bhs[a->esz](t0, tcg_env, t0, t1);
6252 write_fp_sreg(s, a->rd, t0);
6254 return true;
6257 TRANS(SQDMULH_si, do_env_scalar2_idx_hs, a, &f_scalar_sqdmulh)
6258 TRANS(SQRDMULH_si, do_env_scalar2_idx_hs, a, &f_scalar_sqrdmulh)
6260 static bool do_env_scalar3_idx_hs(DisasContext *s, arg_rrx_e *a,
6261 const ENVScalar3 *f)
6263 if (a->esz < MO_16 || a->esz > MO_32) {
6264 return false;
6266 if (fp_access_check(s)) {
6267 TCGv_i32 t0 = tcg_temp_new_i32();
6268 TCGv_i32 t1 = tcg_temp_new_i32();
6269 TCGv_i32 t2 = tcg_temp_new_i32();
6271 read_vec_element_i32(s, t0, a->rn, 0, a->esz);
6272 read_vec_element_i32(s, t1, a->rm, a->idx, a->esz);
6273 read_vec_element_i32(s, t2, a->rd, 0, a->esz);
6274 f->gen_hs[a->esz - 1](t0, tcg_env, t0, t1, t2);
6275 write_fp_sreg(s, a->rd, t0);
6277 return true;
6280 TRANS_FEAT(SQRDMLAH_si, aa64_rdm, do_env_scalar3_idx_hs, a, &f_scalar_sqrdmlah)
6281 TRANS_FEAT(SQRDMLSH_si, aa64_rdm, do_env_scalar3_idx_hs, a, &f_scalar_sqrdmlsh)
6283 static bool do_scalar_muladd_widening_idx(DisasContext *s, arg_rrx_e *a,
6284 NeonGenTwo64OpFn *fn, bool acc)
6286 if (fp_access_check(s)) {
6287 TCGv_i64 t0 = tcg_temp_new_i64();
6288 TCGv_i64 t1 = tcg_temp_new_i64();
6289 TCGv_i64 t2 = tcg_temp_new_i64();
6290 unsigned vsz, dofs;
6292 if (acc) {
6293 read_vec_element(s, t0, a->rd, 0, a->esz + 1);
6295 read_vec_element(s, t1, a->rn, 0, a->esz | MO_SIGN);
6296 read_vec_element(s, t2, a->rm, a->idx, a->esz | MO_SIGN);
6297 fn(t0, t1, t2);
6299 /* Clear the whole register first, then store scalar. */
6300 vsz = vec_full_reg_size(s);
6301 dofs = vec_full_reg_offset(s, a->rd);
6302 tcg_gen_gvec_dup_imm(MO_64, dofs, vsz, vsz, 0);
6303 write_vec_element(s, t0, a->rd, 0, a->esz + 1);
6305 return true;
6308 TRANS(SQDMULL_si, do_scalar_muladd_widening_idx, a,
6309 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6310 TRANS(SQDMLAL_si, do_scalar_muladd_widening_idx, a,
6311 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6312 TRANS(SQDMLSL_si, do_scalar_muladd_widening_idx, a,
6313 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6315 static bool do_fp3_vector_idx(DisasContext *s, arg_qrrx_e *a,
6316 gen_helper_gvec_3_ptr * const fns[3])
6318 MemOp esz = a->esz;
6320 switch (esz) {
6321 case MO_64:
6322 if (!a->q) {
6323 return false;
6325 break;
6326 case MO_32:
6327 break;
6328 case MO_16:
6329 if (!dc_isar_feature(aa64_fp16, s)) {
6330 return false;
6332 break;
6333 default:
6334 g_assert_not_reached();
6336 if (fp_access_check(s)) {
6337 gen_gvec_op3_fpst(s, a->q, a->rd, a->rn, a->rm,
6338 esz == MO_16, a->idx, fns[esz - 1]);
6340 return true;
6343 static gen_helper_gvec_3_ptr * const f_vector_idx_fmul[3] = {
6344 gen_helper_gvec_fmul_idx_h,
6345 gen_helper_gvec_fmul_idx_s,
6346 gen_helper_gvec_fmul_idx_d,
6348 TRANS(FMUL_vi, do_fp3_vector_idx, a, f_vector_idx_fmul)
6350 static gen_helper_gvec_3_ptr * const f_vector_idx_fmulx[3] = {
6351 gen_helper_gvec_fmulx_idx_h,
6352 gen_helper_gvec_fmulx_idx_s,
6353 gen_helper_gvec_fmulx_idx_d,
6355 TRANS(FMULX_vi, do_fp3_vector_idx, a, f_vector_idx_fmulx)
6357 static bool do_fmla_vector_idx(DisasContext *s, arg_qrrx_e *a, bool neg)
6359 static gen_helper_gvec_4_ptr * const fns[3] = {
6360 gen_helper_gvec_fmla_idx_h,
6361 gen_helper_gvec_fmla_idx_s,
6362 gen_helper_gvec_fmla_idx_d,
6364 MemOp esz = a->esz;
6366 switch (esz) {
6367 case MO_64:
6368 if (!a->q) {
6369 return false;
6371 break;
6372 case MO_32:
6373 break;
6374 case MO_16:
6375 if (!dc_isar_feature(aa64_fp16, s)) {
6376 return false;
6378 break;
6379 default:
6380 g_assert_not_reached();
6382 if (fp_access_check(s)) {
6383 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd,
6384 esz == MO_16, (a->idx << 1) | neg,
6385 fns[esz - 1]);
6387 return true;
6390 TRANS(FMLA_vi, do_fmla_vector_idx, a, false)
6391 TRANS(FMLS_vi, do_fmla_vector_idx, a, true)
6393 static bool do_fmlal_idx(DisasContext *s, arg_qrrx_e *a, bool is_s, bool is_2)
6395 if (fp_access_check(s)) {
6396 int data = (a->idx << 2) | (is_2 << 1) | is_s;
6397 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
6398 vec_full_reg_offset(s, a->rn),
6399 vec_full_reg_offset(s, a->rm), tcg_env,
6400 a->q ? 16 : 8, vec_full_reg_size(s),
6401 data, gen_helper_gvec_fmlal_idx_a64);
6403 return true;
6406 TRANS_FEAT(FMLAL_vi, aa64_fhm, do_fmlal_idx, a, false, false)
6407 TRANS_FEAT(FMLSL_vi, aa64_fhm, do_fmlal_idx, a, true, false)
6408 TRANS_FEAT(FMLAL2_vi, aa64_fhm, do_fmlal_idx, a, false, true)
6409 TRANS_FEAT(FMLSL2_vi, aa64_fhm, do_fmlal_idx, a, true, true)
6411 static bool do_int3_vector_idx(DisasContext *s, arg_qrrx_e *a,
6412 gen_helper_gvec_3 * const fns[2])
6414 assert(a->esz == MO_16 || a->esz == MO_32);
6415 if (fp_access_check(s)) {
6416 gen_gvec_op3_ool(s, a->q, a->rd, a->rn, a->rm, a->idx, fns[a->esz - 1]);
6418 return true;
6421 static gen_helper_gvec_3 * const f_vector_idx_mul[2] = {
6422 gen_helper_gvec_mul_idx_h,
6423 gen_helper_gvec_mul_idx_s,
6425 TRANS(MUL_vi, do_int3_vector_idx, a, f_vector_idx_mul)
6427 static bool do_mla_vector_idx(DisasContext *s, arg_qrrx_e *a, bool sub)
6429 static gen_helper_gvec_4 * const fns[2][2] = {
6430 { gen_helper_gvec_mla_idx_h, gen_helper_gvec_mls_idx_h },
6431 { gen_helper_gvec_mla_idx_s, gen_helper_gvec_mls_idx_s },
6434 assert(a->esz == MO_16 || a->esz == MO_32);
6435 if (fp_access_check(s)) {
6436 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd,
6437 a->idx, fns[a->esz - 1][sub]);
6439 return true;
6442 TRANS(MLA_vi, do_mla_vector_idx, a, false)
6443 TRANS(MLS_vi, do_mla_vector_idx, a, true)
6445 static bool do_int3_qc_vector_idx(DisasContext *s, arg_qrrx_e *a,
6446 gen_helper_gvec_4 * const fns[2])
6448 assert(a->esz == MO_16 || a->esz == MO_32);
6449 if (fp_access_check(s)) {
6450 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, a->rd),
6451 vec_full_reg_offset(s, a->rn),
6452 vec_full_reg_offset(s, a->rm),
6453 offsetof(CPUARMState, vfp.qc),
6454 a->q ? 16 : 8, vec_full_reg_size(s),
6455 a->idx, fns[a->esz - 1]);
6457 return true;
6460 static gen_helper_gvec_4 * const f_vector_idx_sqdmulh[2] = {
6461 gen_helper_neon_sqdmulh_idx_h,
6462 gen_helper_neon_sqdmulh_idx_s,
6464 TRANS(SQDMULH_vi, do_int3_qc_vector_idx, a, f_vector_idx_sqdmulh)
6466 static gen_helper_gvec_4 * const f_vector_idx_sqrdmulh[2] = {
6467 gen_helper_neon_sqrdmulh_idx_h,
6468 gen_helper_neon_sqrdmulh_idx_s,
6470 TRANS(SQRDMULH_vi, do_int3_qc_vector_idx, a, f_vector_idx_sqrdmulh)
6472 static gen_helper_gvec_4 * const f_vector_idx_sqrdmlah[2] = {
6473 gen_helper_neon_sqrdmlah_idx_h,
6474 gen_helper_neon_sqrdmlah_idx_s,
6476 TRANS_FEAT(SQRDMLAH_vi, aa64_rdm, do_int3_qc_vector_idx, a,
6477 f_vector_idx_sqrdmlah)
6479 static gen_helper_gvec_4 * const f_vector_idx_sqrdmlsh[2] = {
6480 gen_helper_neon_sqrdmlsh_idx_h,
6481 gen_helper_neon_sqrdmlsh_idx_s,
6483 TRANS_FEAT(SQRDMLSH_vi, aa64_rdm, do_int3_qc_vector_idx, a,
6484 f_vector_idx_sqrdmlsh)
6486 static bool do_dot_vector_idx(DisasContext *s, arg_qrrx_e *a,
6487 gen_helper_gvec_4 *fn)
6489 if (fp_access_check(s)) {
6490 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, a->idx, fn);
6492 return true;
6495 static bool do_dot_vector_idx_env(DisasContext *s, arg_qrrx_e *a,
6496 gen_helper_gvec_4_ptr *fn)
6498 if (fp_access_check(s)) {
6499 gen_gvec_op4_env(s, a->q, a->rd, a->rn, a->rm, a->rd, a->idx, fn);
6501 return true;
6504 TRANS_FEAT(SDOT_vi, aa64_dp, do_dot_vector_idx, a, gen_helper_gvec_sdot_idx_b)
6505 TRANS_FEAT(UDOT_vi, aa64_dp, do_dot_vector_idx, a, gen_helper_gvec_udot_idx_b)
6506 TRANS_FEAT(SUDOT_vi, aa64_i8mm, do_dot_vector_idx, a,
6507 gen_helper_gvec_sudot_idx_b)
6508 TRANS_FEAT(USDOT_vi, aa64_i8mm, do_dot_vector_idx, a,
6509 gen_helper_gvec_usdot_idx_b)
6510 TRANS_FEAT(BFDOT_vi, aa64_bf16, do_dot_vector_idx_env, a,
6511 gen_helper_gvec_bfdot_idx)
6513 static bool trans_BFMLAL_vi(DisasContext *s, arg_qrrx_e *a)
6515 if (!dc_isar_feature(aa64_bf16, s)) {
6516 return false;
6518 if (fp_access_check(s)) {
6519 /* Q bit selects BFMLALB vs BFMLALT. */
6520 gen_gvec_op4_fpst(s, true, a->rd, a->rn, a->rm, a->rd, 0,
6521 (a->idx << 1) | a->q,
6522 gen_helper_gvec_bfmlal_idx);
6524 return true;
6527 static bool trans_FCMLA_vi(DisasContext *s, arg_FCMLA_vi *a)
6529 gen_helper_gvec_4_ptr *fn;
6531 if (!dc_isar_feature(aa64_fcma, s)) {
6532 return false;
6534 switch (a->esz) {
6535 case MO_16:
6536 if (!dc_isar_feature(aa64_fp16, s)) {
6537 return false;
6539 fn = gen_helper_gvec_fcmlah_idx;
6540 break;
6541 case MO_32:
6542 fn = gen_helper_gvec_fcmlas_idx;
6543 break;
6544 default:
6545 g_assert_not_reached();
6547 if (fp_access_check(s)) {
6548 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd,
6549 a->esz == MO_16, (a->idx << 2) | a->rot, fn);
6551 return true;
6555 * Advanced SIMD scalar pairwise
6558 static bool do_fp3_scalar_pair(DisasContext *s, arg_rr_e *a, const FPScalar *f)
6560 switch (a->esz) {
6561 case MO_64:
6562 if (fp_access_check(s)) {
6563 TCGv_i64 t0 = tcg_temp_new_i64();
6564 TCGv_i64 t1 = tcg_temp_new_i64();
6566 read_vec_element(s, t0, a->rn, 0, MO_64);
6567 read_vec_element(s, t1, a->rn, 1, MO_64);
6568 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_FPCR));
6569 write_fp_dreg(s, a->rd, t0);
6571 break;
6572 case MO_32:
6573 if (fp_access_check(s)) {
6574 TCGv_i32 t0 = tcg_temp_new_i32();
6575 TCGv_i32 t1 = tcg_temp_new_i32();
6577 read_vec_element_i32(s, t0, a->rn, 0, MO_32);
6578 read_vec_element_i32(s, t1, a->rn, 1, MO_32);
6579 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_FPCR));
6580 write_fp_sreg(s, a->rd, t0);
6582 break;
6583 case MO_16:
6584 if (!dc_isar_feature(aa64_fp16, s)) {
6585 return false;
6587 if (fp_access_check(s)) {
6588 TCGv_i32 t0 = tcg_temp_new_i32();
6589 TCGv_i32 t1 = tcg_temp_new_i32();
6591 read_vec_element_i32(s, t0, a->rn, 0, MO_16);
6592 read_vec_element_i32(s, t1, a->rn, 1, MO_16);
6593 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_FPCR_F16));
6594 write_fp_sreg(s, a->rd, t0);
6596 break;
6597 default:
6598 g_assert_not_reached();
6600 return true;
6603 TRANS(FADDP_s, do_fp3_scalar_pair, a, &f_scalar_fadd)
6604 TRANS(FMAXP_s, do_fp3_scalar_pair, a, &f_scalar_fmax)
6605 TRANS(FMINP_s, do_fp3_scalar_pair, a, &f_scalar_fmin)
6606 TRANS(FMAXNMP_s, do_fp3_scalar_pair, a, &f_scalar_fmaxnm)
6607 TRANS(FMINNMP_s, do_fp3_scalar_pair, a, &f_scalar_fminnm)
6609 static bool trans_ADDP_s(DisasContext *s, arg_rr_e *a)
6611 if (fp_access_check(s)) {
6612 TCGv_i64 t0 = tcg_temp_new_i64();
6613 TCGv_i64 t1 = tcg_temp_new_i64();
6615 read_vec_element(s, t0, a->rn, 0, MO_64);
6616 read_vec_element(s, t1, a->rn, 1, MO_64);
6617 tcg_gen_add_i64(t0, t0, t1);
6618 write_fp_dreg(s, a->rd, t0);
6620 return true;
6624 * Floating-point conditional select
6627 static bool trans_FCSEL(DisasContext *s, arg_FCSEL *a)
6629 TCGv_i64 t_true, t_false;
6630 DisasCompare64 c;
6632 switch (a->esz) {
6633 case MO_32:
6634 case MO_64:
6635 break;
6636 case MO_16:
6637 if (!dc_isar_feature(aa64_fp16, s)) {
6638 return false;
6640 break;
6641 default:
6642 return false;
6645 if (!fp_access_check(s)) {
6646 return true;
6649 /* Zero extend sreg & hreg inputs to 64 bits now. */
6650 t_true = tcg_temp_new_i64();
6651 t_false = tcg_temp_new_i64();
6652 read_vec_element(s, t_true, a->rn, 0, a->esz);
6653 read_vec_element(s, t_false, a->rm, 0, a->esz);
6655 a64_test_cc(&c, a->cond);
6656 tcg_gen_movcond_i64(c.cond, t_true, c.value, tcg_constant_i64(0),
6657 t_true, t_false);
6660 * Note that sregs & hregs write back zeros to the high bits,
6661 * and we've already done the zero-extension.
6663 write_fp_dreg(s, a->rd, t_true);
6664 return true;
6668 * Advanced SIMD Extract
6671 static bool trans_EXT_d(DisasContext *s, arg_EXT_d *a)
6673 if (fp_access_check(s)) {
6674 TCGv_i64 lo = read_fp_dreg(s, a->rn);
6675 if (a->imm != 0) {
6676 TCGv_i64 hi = read_fp_dreg(s, a->rm);
6677 tcg_gen_extract2_i64(lo, lo, hi, a->imm * 8);
6679 write_fp_dreg(s, a->rd, lo);
6681 return true;
6684 static bool trans_EXT_q(DisasContext *s, arg_EXT_q *a)
6686 TCGv_i64 lo, hi;
6687 int pos = (a->imm & 7) * 8;
6688 int elt = a->imm >> 3;
6690 if (!fp_access_check(s)) {
6691 return true;
6694 lo = tcg_temp_new_i64();
6695 hi = tcg_temp_new_i64();
6697 read_vec_element(s, lo, a->rn, elt, MO_64);
6698 elt++;
6699 read_vec_element(s, hi, elt & 2 ? a->rm : a->rn, elt & 1, MO_64);
6700 elt++;
6702 if (pos != 0) {
6703 TCGv_i64 hh = tcg_temp_new_i64();
6704 tcg_gen_extract2_i64(lo, lo, hi, pos);
6705 read_vec_element(s, hh, a->rm, elt & 1, MO_64);
6706 tcg_gen_extract2_i64(hi, hi, hh, pos);
6709 write_vec_element(s, lo, a->rd, 0, MO_64);
6710 write_vec_element(s, hi, a->rd, 1, MO_64);
6711 clear_vec_high(s, true, a->rd);
6712 return true;
6716 * Floating-point data-processing (3 source)
6719 static bool do_fmadd(DisasContext *s, arg_rrrr_e *a, bool neg_a, bool neg_n)
6721 TCGv_ptr fpst;
6724 * These are fused multiply-add. Note that doing the negations here
6725 * as separate steps is correct: an input NaN should come out with
6726 * its sign bit flipped if it is a negated-input.
6728 switch (a->esz) {
6729 case MO_64:
6730 if (fp_access_check(s)) {
6731 TCGv_i64 tn = read_fp_dreg(s, a->rn);
6732 TCGv_i64 tm = read_fp_dreg(s, a->rm);
6733 TCGv_i64 ta = read_fp_dreg(s, a->ra);
6735 if (neg_a) {
6736 gen_vfp_negd(ta, ta);
6738 if (neg_n) {
6739 gen_vfp_negd(tn, tn);
6741 fpst = fpstatus_ptr(FPST_FPCR);
6742 gen_helper_vfp_muladdd(ta, tn, tm, ta, fpst);
6743 write_fp_dreg(s, a->rd, ta);
6745 break;
6747 case MO_32:
6748 if (fp_access_check(s)) {
6749 TCGv_i32 tn = read_fp_sreg(s, a->rn);
6750 TCGv_i32 tm = read_fp_sreg(s, a->rm);
6751 TCGv_i32 ta = read_fp_sreg(s, a->ra);
6753 if (neg_a) {
6754 gen_vfp_negs(ta, ta);
6756 if (neg_n) {
6757 gen_vfp_negs(tn, tn);
6759 fpst = fpstatus_ptr(FPST_FPCR);
6760 gen_helper_vfp_muladds(ta, tn, tm, ta, fpst);
6761 write_fp_sreg(s, a->rd, ta);
6763 break;
6765 case MO_16:
6766 if (!dc_isar_feature(aa64_fp16, s)) {
6767 return false;
6769 if (fp_access_check(s)) {
6770 TCGv_i32 tn = read_fp_hreg(s, a->rn);
6771 TCGv_i32 tm = read_fp_hreg(s, a->rm);
6772 TCGv_i32 ta = read_fp_hreg(s, a->ra);
6774 if (neg_a) {
6775 gen_vfp_negh(ta, ta);
6777 if (neg_n) {
6778 gen_vfp_negh(tn, tn);
6780 fpst = fpstatus_ptr(FPST_FPCR_F16);
6781 gen_helper_advsimd_muladdh(ta, tn, tm, ta, fpst);
6782 write_fp_sreg(s, a->rd, ta);
6784 break;
6786 default:
6787 return false;
6789 return true;
6792 TRANS(FMADD, do_fmadd, a, false, false)
6793 TRANS(FNMADD, do_fmadd, a, true, true)
6794 TRANS(FMSUB, do_fmadd, a, false, true)
6795 TRANS(FNMSUB, do_fmadd, a, true, false)
6798 * Advanced SIMD Across Lanes
6801 static bool do_int_reduction(DisasContext *s, arg_qrr_e *a, bool widen,
6802 MemOp src_sign, NeonGenTwo64OpFn *fn)
6804 TCGv_i64 tcg_res, tcg_elt;
6805 MemOp src_mop = a->esz | src_sign;
6806 int elements = (a->q ? 16 : 8) >> a->esz;
6808 /* Reject MO_64, and MO_32 without Q: a minimum of 4 elements. */
6809 if (elements < 4) {
6810 return false;
6812 if (!fp_access_check(s)) {
6813 return true;
6816 tcg_res = tcg_temp_new_i64();
6817 tcg_elt = tcg_temp_new_i64();
6819 read_vec_element(s, tcg_res, a->rn, 0, src_mop);
6820 for (int i = 1; i < elements; i++) {
6821 read_vec_element(s, tcg_elt, a->rn, i, src_mop);
6822 fn(tcg_res, tcg_res, tcg_elt);
6825 tcg_gen_ext_i64(tcg_res, tcg_res, a->esz + widen);
6826 write_fp_dreg(s, a->rd, tcg_res);
6827 return true;
6830 TRANS(ADDV, do_int_reduction, a, false, 0, tcg_gen_add_i64)
6831 TRANS(SADDLV, do_int_reduction, a, true, MO_SIGN, tcg_gen_add_i64)
6832 TRANS(UADDLV, do_int_reduction, a, true, 0, tcg_gen_add_i64)
6833 TRANS(SMAXV, do_int_reduction, a, false, MO_SIGN, tcg_gen_smax_i64)
6834 TRANS(UMAXV, do_int_reduction, a, false, 0, tcg_gen_umax_i64)
6835 TRANS(SMINV, do_int_reduction, a, false, MO_SIGN, tcg_gen_smin_i64)
6836 TRANS(UMINV, do_int_reduction, a, false, 0, tcg_gen_umin_i64)
6839 * do_fp_reduction helper
6841 * This mirrors the Reduce() pseudocode in the ARM ARM. It is
6842 * important for correct NaN propagation that we do these
6843 * operations in exactly the order specified by the pseudocode.
6845 * This is a recursive function.
6847 static TCGv_i32 do_reduction_op(DisasContext *s, int rn, MemOp esz,
6848 int ebase, int ecount, TCGv_ptr fpst,
6849 NeonGenTwoSingleOpFn *fn)
6851 if (ecount == 1) {
6852 TCGv_i32 tcg_elem = tcg_temp_new_i32();
6853 read_vec_element_i32(s, tcg_elem, rn, ebase, esz);
6854 return tcg_elem;
6855 } else {
6856 int half = ecount >> 1;
6857 TCGv_i32 tcg_hi, tcg_lo, tcg_res;
6859 tcg_hi = do_reduction_op(s, rn, esz, ebase + half, half, fpst, fn);
6860 tcg_lo = do_reduction_op(s, rn, esz, ebase, half, fpst, fn);
6861 tcg_res = tcg_temp_new_i32();
6863 fn(tcg_res, tcg_lo, tcg_hi, fpst);
6864 return tcg_res;
6868 static bool do_fp_reduction(DisasContext *s, arg_qrr_e *a,
6869 NeonGenTwoSingleOpFn *fn)
6871 if (fp_access_check(s)) {
6872 MemOp esz = a->esz;
6873 int elts = (a->q ? 16 : 8) >> esz;
6874 TCGv_ptr fpst = fpstatus_ptr(esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
6875 TCGv_i32 res = do_reduction_op(s, a->rn, esz, 0, elts, fpst, fn);
6876 write_fp_sreg(s, a->rd, res);
6878 return true;
6881 TRANS_FEAT(FMAXNMV_h, aa64_fp16, do_fp_reduction, a, gen_helper_advsimd_maxnumh)
6882 TRANS_FEAT(FMINNMV_h, aa64_fp16, do_fp_reduction, a, gen_helper_advsimd_minnumh)
6883 TRANS_FEAT(FMAXV_h, aa64_fp16, do_fp_reduction, a, gen_helper_advsimd_maxh)
6884 TRANS_FEAT(FMINV_h, aa64_fp16, do_fp_reduction, a, gen_helper_advsimd_minh)
6886 TRANS(FMAXNMV_s, do_fp_reduction, a, gen_helper_vfp_maxnums)
6887 TRANS(FMINNMV_s, do_fp_reduction, a, gen_helper_vfp_minnums)
6888 TRANS(FMAXV_s, do_fp_reduction, a, gen_helper_vfp_maxs)
6889 TRANS(FMINV_s, do_fp_reduction, a, gen_helper_vfp_mins)
6892 * Floating-point Immediate
6895 static bool trans_FMOVI_s(DisasContext *s, arg_FMOVI_s *a)
6897 switch (a->esz) {
6898 case MO_32:
6899 case MO_64:
6900 break;
6901 case MO_16:
6902 if (!dc_isar_feature(aa64_fp16, s)) {
6903 return false;
6905 break;
6906 default:
6907 return false;
6909 if (fp_access_check(s)) {
6910 uint64_t imm = vfp_expand_imm(a->esz, a->imm);
6911 write_fp_dreg(s, a->rd, tcg_constant_i64(imm));
6913 return true;
6917 * Advanced SIMD Modified Immediate
6920 static bool trans_FMOVI_v_h(DisasContext *s, arg_FMOVI_v_h *a)
6922 if (!dc_isar_feature(aa64_fp16, s)) {
6923 return false;
6925 if (fp_access_check(s)) {
6926 tcg_gen_gvec_dup_imm(MO_16, vec_full_reg_offset(s, a->rd),
6927 a->q ? 16 : 8, vec_full_reg_size(s),
6928 vfp_expand_imm(MO_16, a->abcdefgh));
6930 return true;
6933 static void gen_movi(unsigned vece, uint32_t dofs, uint32_t aofs,
6934 int64_t c, uint32_t oprsz, uint32_t maxsz)
6936 tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, c);
6939 static bool trans_Vimm(DisasContext *s, arg_Vimm *a)
6941 GVecGen2iFn *fn;
6943 /* Handle decode of cmode/op here between ORR/BIC/MOVI */
6944 if ((a->cmode & 1) && a->cmode < 12) {
6945 /* For op=1, the imm will be inverted, so BIC becomes AND. */
6946 fn = a->op ? tcg_gen_gvec_andi : tcg_gen_gvec_ori;
6947 } else {
6948 /* There is one unallocated cmode/op combination in this space */
6949 if (a->cmode == 15 && a->op == 1 && a->q == 0) {
6950 return false;
6952 fn = gen_movi;
6955 if (fp_access_check(s)) {
6956 uint64_t imm = asimd_imm_const(a->abcdefgh, a->cmode, a->op);
6957 gen_gvec_fn2i(s, a->q, a->rd, a->rd, imm, fn, MO_64);
6959 return true;
6963 * Advanced SIMD Shift by Immediate
6966 static bool do_vec_shift_imm(DisasContext *s, arg_qrri_e *a, GVecGen2iFn *fn)
6968 if (fp_access_check(s)) {
6969 gen_gvec_fn2i(s, a->q, a->rd, a->rn, a->imm, fn, a->esz);
6971 return true;
6974 TRANS(SSHR_v, do_vec_shift_imm, a, gen_gvec_sshr)
6975 TRANS(USHR_v, do_vec_shift_imm, a, gen_gvec_ushr)
6976 TRANS(SSRA_v, do_vec_shift_imm, a, gen_gvec_ssra)
6977 TRANS(USRA_v, do_vec_shift_imm, a, gen_gvec_usra)
6978 TRANS(SRSHR_v, do_vec_shift_imm, a, gen_gvec_srshr)
6979 TRANS(URSHR_v, do_vec_shift_imm, a, gen_gvec_urshr)
6980 TRANS(SRSRA_v, do_vec_shift_imm, a, gen_gvec_srsra)
6981 TRANS(URSRA_v, do_vec_shift_imm, a, gen_gvec_ursra)
6982 TRANS(SRI_v, do_vec_shift_imm, a, gen_gvec_sri)
6983 TRANS(SHL_v, do_vec_shift_imm, a, tcg_gen_gvec_shli)
6984 TRANS(SLI_v, do_vec_shift_imm, a, gen_gvec_sli);
6985 TRANS(SQSHL_vi, do_vec_shift_imm, a, gen_neon_sqshli)
6986 TRANS(UQSHL_vi, do_vec_shift_imm, a, gen_neon_uqshli)
6987 TRANS(SQSHLU_vi, do_vec_shift_imm, a, gen_neon_sqshlui)
6989 static bool do_vec_shift_imm_wide(DisasContext *s, arg_qrri_e *a, bool is_u)
6991 TCGv_i64 tcg_rn, tcg_rd;
6992 int esz = a->esz;
6993 int esize;
6995 if (!fp_access_check(s)) {
6996 return true;
7000 * For the LL variants the store is larger than the load,
7001 * so if rd == rn we would overwrite parts of our input.
7002 * So load everything right now and use shifts in the main loop.
7004 tcg_rd = tcg_temp_new_i64();
7005 tcg_rn = tcg_temp_new_i64();
7006 read_vec_element(s, tcg_rn, a->rn, a->q, MO_64);
7008 esize = 8 << esz;
7009 for (int i = 0, elements = 8 >> esz; i < elements; i++) {
7010 if (is_u) {
7011 tcg_gen_extract_i64(tcg_rd, tcg_rn, i * esize, esize);
7012 } else {
7013 tcg_gen_sextract_i64(tcg_rd, tcg_rn, i * esize, esize);
7015 tcg_gen_shli_i64(tcg_rd, tcg_rd, a->imm);
7016 write_vec_element(s, tcg_rd, a->rd, i, esz + 1);
7018 clear_vec_high(s, true, a->rd);
7019 return true;
7022 TRANS(SSHLL_v, do_vec_shift_imm_wide, a, false)
7023 TRANS(USHLL_v, do_vec_shift_imm_wide, a, true)
7025 static void gen_sshr_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7027 assert(shift >= 0 && shift <= 64);
7028 tcg_gen_sari_i64(dst, src, MIN(shift, 63));
7031 static void gen_ushr_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7033 assert(shift >= 0 && shift <= 64);
7034 if (shift == 64) {
7035 tcg_gen_movi_i64(dst, 0);
7036 } else {
7037 tcg_gen_shri_i64(dst, src, shift);
7041 static void gen_ssra_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7043 gen_sshr_d(src, src, shift);
7044 tcg_gen_add_i64(dst, dst, src);
7047 static void gen_usra_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7049 gen_ushr_d(src, src, shift);
7050 tcg_gen_add_i64(dst, dst, src);
7053 static void gen_srshr_bhs(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7055 assert(shift >= 0 && shift <= 32);
7056 if (shift) {
7057 TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1));
7058 tcg_gen_add_i64(dst, src, rnd);
7059 tcg_gen_sari_i64(dst, dst, shift);
7060 } else {
7061 tcg_gen_mov_i64(dst, src);
7065 static void gen_urshr_bhs(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7067 assert(shift >= 0 && shift <= 32);
7068 if (shift) {
7069 TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1));
7070 tcg_gen_add_i64(dst, src, rnd);
7071 tcg_gen_shri_i64(dst, dst, shift);
7072 } else {
7073 tcg_gen_mov_i64(dst, src);
7077 static void gen_srshr_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7079 assert(shift >= 0 && shift <= 64);
7080 if (shift == 0) {
7081 tcg_gen_mov_i64(dst, src);
7082 } else if (shift == 64) {
7083 /* Extension of sign bit (0,-1) plus sign bit (0,1) is zero. */
7084 tcg_gen_movi_i64(dst, 0);
7085 } else {
7086 TCGv_i64 rnd = tcg_temp_new_i64();
7087 tcg_gen_extract_i64(rnd, src, shift - 1, 1);
7088 tcg_gen_sari_i64(dst, src, shift);
7089 tcg_gen_add_i64(dst, dst, rnd);
7093 static void gen_urshr_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7095 assert(shift >= 0 && shift <= 64);
7096 if (shift == 0) {
7097 tcg_gen_mov_i64(dst, src);
7098 } else if (shift == 64) {
7099 /* Rounding will propagate bit 63 into bit 64. */
7100 tcg_gen_shri_i64(dst, src, 63);
7101 } else {
7102 TCGv_i64 rnd = tcg_temp_new_i64();
7103 tcg_gen_extract_i64(rnd, src, shift - 1, 1);
7104 tcg_gen_shri_i64(dst, src, shift);
7105 tcg_gen_add_i64(dst, dst, rnd);
7109 static void gen_srsra_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7111 gen_srshr_d(src, src, shift);
7112 tcg_gen_add_i64(dst, dst, src);
7115 static void gen_ursra_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7117 gen_urshr_d(src, src, shift);
7118 tcg_gen_add_i64(dst, dst, src);
7121 static void gen_sri_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7123 /* If shift is 64, dst is unchanged. */
7124 if (shift != 64) {
7125 tcg_gen_shri_i64(src, src, shift);
7126 tcg_gen_deposit_i64(dst, dst, src, 0, 64 - shift);
7130 static void gen_sli_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7132 tcg_gen_deposit_i64(dst, dst, src, shift, 64 - shift);
7135 static bool do_vec_shift_imm_narrow(DisasContext *s, arg_qrri_e *a,
7136 WideShiftImmFn * const fns[3], MemOp sign)
7138 TCGv_i64 tcg_rn, tcg_rd;
7139 int esz = a->esz;
7140 int esize;
7141 WideShiftImmFn *fn;
7143 tcg_debug_assert(esz >= MO_8 && esz <= MO_32);
7145 if (!fp_access_check(s)) {
7146 return true;
7149 tcg_rn = tcg_temp_new_i64();
7150 tcg_rd = tcg_temp_new_i64();
7151 tcg_gen_movi_i64(tcg_rd, 0);
7153 fn = fns[esz];
7154 esize = 8 << esz;
7155 for (int i = 0, elements = 8 >> esz; i < elements; i++) {
7156 read_vec_element(s, tcg_rn, a->rn, i, (esz + 1) | sign);
7157 fn(tcg_rn, tcg_rn, a->imm);
7158 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_rn, esize * i, esize);
7161 write_vec_element(s, tcg_rd, a->rd, a->q, MO_64);
7162 clear_vec_high(s, a->q, a->rd);
7163 return true;
7166 static void gen_sqshrn_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7168 tcg_gen_sari_i64(d, s, i);
7169 tcg_gen_ext16u_i64(d, d);
7170 gen_helper_neon_narrow_sat_s8(d, tcg_env, d);
7173 static void gen_sqshrn_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7175 tcg_gen_sari_i64(d, s, i);
7176 tcg_gen_ext32u_i64(d, d);
7177 gen_helper_neon_narrow_sat_s16(d, tcg_env, d);
7180 static void gen_sqshrn_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7182 gen_sshr_d(d, s, i);
7183 gen_helper_neon_narrow_sat_s32(d, tcg_env, d);
7186 static void gen_uqshrn_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7188 tcg_gen_shri_i64(d, s, i);
7189 gen_helper_neon_narrow_sat_u8(d, tcg_env, d);
7192 static void gen_uqshrn_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7194 tcg_gen_shri_i64(d, s, i);
7195 gen_helper_neon_narrow_sat_u16(d, tcg_env, d);
7198 static void gen_uqshrn_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7200 gen_ushr_d(d, s, i);
7201 gen_helper_neon_narrow_sat_u32(d, tcg_env, d);
7204 static void gen_sqshrun_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7206 tcg_gen_sari_i64(d, s, i);
7207 tcg_gen_ext16u_i64(d, d);
7208 gen_helper_neon_unarrow_sat8(d, tcg_env, d);
7211 static void gen_sqshrun_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7213 tcg_gen_sari_i64(d, s, i);
7214 tcg_gen_ext32u_i64(d, d);
7215 gen_helper_neon_unarrow_sat16(d, tcg_env, d);
7218 static void gen_sqshrun_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7220 gen_sshr_d(d, s, i);
7221 gen_helper_neon_unarrow_sat32(d, tcg_env, d);
7224 static void gen_sqrshrn_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7226 gen_srshr_bhs(d, s, i);
7227 tcg_gen_ext16u_i64(d, d);
7228 gen_helper_neon_narrow_sat_s8(d, tcg_env, d);
7231 static void gen_sqrshrn_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7233 gen_srshr_bhs(d, s, i);
7234 tcg_gen_ext32u_i64(d, d);
7235 gen_helper_neon_narrow_sat_s16(d, tcg_env, d);
7238 static void gen_sqrshrn_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7240 gen_srshr_d(d, s, i);
7241 gen_helper_neon_narrow_sat_s32(d, tcg_env, d);
7244 static void gen_uqrshrn_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7246 gen_urshr_bhs(d, s, i);
7247 gen_helper_neon_narrow_sat_u8(d, tcg_env, d);
7250 static void gen_uqrshrn_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7252 gen_urshr_bhs(d, s, i);
7253 gen_helper_neon_narrow_sat_u16(d, tcg_env, d);
7256 static void gen_uqrshrn_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7258 gen_urshr_d(d, s, i);
7259 gen_helper_neon_narrow_sat_u32(d, tcg_env, d);
7262 static void gen_sqrshrun_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7264 gen_srshr_bhs(d, s, i);
7265 tcg_gen_ext16u_i64(d, d);
7266 gen_helper_neon_unarrow_sat8(d, tcg_env, d);
7269 static void gen_sqrshrun_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7271 gen_srshr_bhs(d, s, i);
7272 tcg_gen_ext32u_i64(d, d);
7273 gen_helper_neon_unarrow_sat16(d, tcg_env, d);
7276 static void gen_sqrshrun_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7278 gen_srshr_d(d, s, i);
7279 gen_helper_neon_unarrow_sat32(d, tcg_env, d);
7282 static WideShiftImmFn * const shrn_fns[] = {
7283 tcg_gen_shri_i64,
7284 tcg_gen_shri_i64,
7285 gen_ushr_d,
7287 TRANS(SHRN_v, do_vec_shift_imm_narrow, a, shrn_fns, 0)
7289 static WideShiftImmFn * const rshrn_fns[] = {
7290 gen_urshr_bhs,
7291 gen_urshr_bhs,
7292 gen_urshr_d,
7294 TRANS(RSHRN_v, do_vec_shift_imm_narrow, a, rshrn_fns, 0)
7296 static WideShiftImmFn * const sqshrn_fns[] = {
7297 gen_sqshrn_b,
7298 gen_sqshrn_h,
7299 gen_sqshrn_s,
7301 TRANS(SQSHRN_v, do_vec_shift_imm_narrow, a, sqshrn_fns, MO_SIGN)
7303 static WideShiftImmFn * const uqshrn_fns[] = {
7304 gen_uqshrn_b,
7305 gen_uqshrn_h,
7306 gen_uqshrn_s,
7308 TRANS(UQSHRN_v, do_vec_shift_imm_narrow, a, uqshrn_fns, 0)
7310 static WideShiftImmFn * const sqshrun_fns[] = {
7311 gen_sqshrun_b,
7312 gen_sqshrun_h,
7313 gen_sqshrun_s,
7315 TRANS(SQSHRUN_v, do_vec_shift_imm_narrow, a, sqshrun_fns, MO_SIGN)
7317 static WideShiftImmFn * const sqrshrn_fns[] = {
7318 gen_sqrshrn_b,
7319 gen_sqrshrn_h,
7320 gen_sqrshrn_s,
7322 TRANS(SQRSHRN_v, do_vec_shift_imm_narrow, a, sqrshrn_fns, MO_SIGN)
7324 static WideShiftImmFn * const uqrshrn_fns[] = {
7325 gen_uqrshrn_b,
7326 gen_uqrshrn_h,
7327 gen_uqrshrn_s,
7329 TRANS(UQRSHRN_v, do_vec_shift_imm_narrow, a, uqrshrn_fns, 0)
7331 static WideShiftImmFn * const sqrshrun_fns[] = {
7332 gen_sqrshrun_b,
7333 gen_sqrshrun_h,
7334 gen_sqrshrun_s,
7336 TRANS(SQRSHRUN_v, do_vec_shift_imm_narrow, a, sqrshrun_fns, MO_SIGN)
7339 * Advanced SIMD Scalar Shift by Immediate
7342 static bool do_scalar_shift_imm(DisasContext *s, arg_rri_e *a,
7343 WideShiftImmFn *fn, bool accumulate,
7344 MemOp sign)
7346 if (fp_access_check(s)) {
7347 TCGv_i64 rd = tcg_temp_new_i64();
7348 TCGv_i64 rn = tcg_temp_new_i64();
7350 read_vec_element(s, rn, a->rn, 0, a->esz | sign);
7351 if (accumulate) {
7352 read_vec_element(s, rd, a->rd, 0, a->esz | sign);
7354 fn(rd, rn, a->imm);
7355 write_fp_dreg(s, a->rd, rd);
7357 return true;
7360 TRANS(SSHR_s, do_scalar_shift_imm, a, gen_sshr_d, false, 0)
7361 TRANS(USHR_s, do_scalar_shift_imm, a, gen_ushr_d, false, 0)
7362 TRANS(SSRA_s, do_scalar_shift_imm, a, gen_ssra_d, true, 0)
7363 TRANS(USRA_s, do_scalar_shift_imm, a, gen_usra_d, true, 0)
7364 TRANS(SRSHR_s, do_scalar_shift_imm, a, gen_srshr_d, false, 0)
7365 TRANS(URSHR_s, do_scalar_shift_imm, a, gen_urshr_d, false, 0)
7366 TRANS(SRSRA_s, do_scalar_shift_imm, a, gen_srsra_d, true, 0)
7367 TRANS(URSRA_s, do_scalar_shift_imm, a, gen_ursra_d, true, 0)
7368 TRANS(SRI_s, do_scalar_shift_imm, a, gen_sri_d, true, 0)
7370 TRANS(SHL_s, do_scalar_shift_imm, a, tcg_gen_shli_i64, false, 0)
7371 TRANS(SLI_s, do_scalar_shift_imm, a, gen_sli_d, true, 0)
7373 static void trunc_i64_env_imm(TCGv_i64 d, TCGv_i64 s, int64_t i,
7374 NeonGenTwoOpEnvFn *fn)
7376 TCGv_i32 t = tcg_temp_new_i32();
7377 tcg_gen_extrl_i64_i32(t, s);
7378 fn(t, tcg_env, t, tcg_constant_i32(i));
7379 tcg_gen_extu_i32_i64(d, t);
7382 static void gen_sqshli_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7384 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_s8);
7387 static void gen_sqshli_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7389 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_s16);
7392 static void gen_sqshli_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7394 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_s32);
7397 static void gen_sqshli_d(TCGv_i64 d, TCGv_i64 s, int64_t i)
7399 gen_helper_neon_qshl_s64(d, tcg_env, s, tcg_constant_i64(i));
7402 static void gen_uqshli_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7404 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_u8);
7407 static void gen_uqshli_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7409 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_u16);
7412 static void gen_uqshli_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7414 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_u32);
7417 static void gen_uqshli_d(TCGv_i64 d, TCGv_i64 s, int64_t i)
7419 gen_helper_neon_qshl_u64(d, tcg_env, s, tcg_constant_i64(i));
7422 static void gen_sqshlui_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7424 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshlu_s8);
7427 static void gen_sqshlui_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7429 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshlu_s16);
7432 static void gen_sqshlui_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7434 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshlu_s32);
7437 static void gen_sqshlui_d(TCGv_i64 d, TCGv_i64 s, int64_t i)
7439 gen_helper_neon_qshlu_s64(d, tcg_env, s, tcg_constant_i64(i));
7442 static WideShiftImmFn * const f_scalar_sqshli[] = {
7443 gen_sqshli_b, gen_sqshli_h, gen_sqshli_s, gen_sqshli_d
7446 static WideShiftImmFn * const f_scalar_uqshli[] = {
7447 gen_uqshli_b, gen_uqshli_h, gen_uqshli_s, gen_uqshli_d
7450 static WideShiftImmFn * const f_scalar_sqshlui[] = {
7451 gen_sqshlui_b, gen_sqshlui_h, gen_sqshlui_s, gen_sqshlui_d
7454 /* Note that the helpers sign-extend their inputs, so don't do it here. */
7455 TRANS(SQSHL_si, do_scalar_shift_imm, a, f_scalar_sqshli[a->esz], false, 0)
7456 TRANS(UQSHL_si, do_scalar_shift_imm, a, f_scalar_uqshli[a->esz], false, 0)
7457 TRANS(SQSHLU_si, do_scalar_shift_imm, a, f_scalar_sqshlui[a->esz], false, 0)
7459 static bool do_scalar_shift_imm_narrow(DisasContext *s, arg_rri_e *a,
7460 WideShiftImmFn * const fns[3],
7461 MemOp sign, bool zext)
7463 MemOp esz = a->esz;
7465 tcg_debug_assert(esz >= MO_8 && esz <= MO_32);
7467 if (fp_access_check(s)) {
7468 TCGv_i64 rd = tcg_temp_new_i64();
7469 TCGv_i64 rn = tcg_temp_new_i64();
7471 read_vec_element(s, rn, a->rn, 0, (esz + 1) | sign);
7472 fns[esz](rd, rn, a->imm);
7473 if (zext) {
7474 tcg_gen_ext_i64(rd, rd, esz);
7476 write_fp_dreg(s, a->rd, rd);
7478 return true;
7481 TRANS(SQSHRN_si, do_scalar_shift_imm_narrow, a, sqshrn_fns, MO_SIGN, true)
7482 TRANS(SQRSHRN_si, do_scalar_shift_imm_narrow, a, sqrshrn_fns, MO_SIGN, true)
7483 TRANS(UQSHRN_si, do_scalar_shift_imm_narrow, a, uqshrn_fns, 0, false)
7484 TRANS(UQRSHRN_si, do_scalar_shift_imm_narrow, a, uqrshrn_fns, 0, false)
7485 TRANS(SQSHRUN_si, do_scalar_shift_imm_narrow, a, sqshrun_fns, MO_SIGN, false)
7486 TRANS(SQRSHRUN_si, do_scalar_shift_imm_narrow, a, sqrshrun_fns, MO_SIGN, false)
7488 /* Shift a TCGv src by TCGv shift_amount, put result in dst.
7489 * Note that it is the caller's responsibility to ensure that the
7490 * shift amount is in range (ie 0..31 or 0..63) and provide the ARM
7491 * mandated semantics for out of range shifts.
7493 static void shift_reg(TCGv_i64 dst, TCGv_i64 src, int sf,
7494 enum a64_shift_type shift_type, TCGv_i64 shift_amount)
7496 switch (shift_type) {
7497 case A64_SHIFT_TYPE_LSL:
7498 tcg_gen_shl_i64(dst, src, shift_amount);
7499 break;
7500 case A64_SHIFT_TYPE_LSR:
7501 tcg_gen_shr_i64(dst, src, shift_amount);
7502 break;
7503 case A64_SHIFT_TYPE_ASR:
7504 if (!sf) {
7505 tcg_gen_ext32s_i64(dst, src);
7507 tcg_gen_sar_i64(dst, sf ? src : dst, shift_amount);
7508 break;
7509 case A64_SHIFT_TYPE_ROR:
7510 if (sf) {
7511 tcg_gen_rotr_i64(dst, src, shift_amount);
7512 } else {
7513 TCGv_i32 t0, t1;
7514 t0 = tcg_temp_new_i32();
7515 t1 = tcg_temp_new_i32();
7516 tcg_gen_extrl_i64_i32(t0, src);
7517 tcg_gen_extrl_i64_i32(t1, shift_amount);
7518 tcg_gen_rotr_i32(t0, t0, t1);
7519 tcg_gen_extu_i32_i64(dst, t0);
7521 break;
7522 default:
7523 assert(FALSE); /* all shift types should be handled */
7524 break;
7527 if (!sf) { /* zero extend final result */
7528 tcg_gen_ext32u_i64(dst, dst);
7532 /* Shift a TCGv src by immediate, put result in dst.
7533 * The shift amount must be in range (this should always be true as the
7534 * relevant instructions will UNDEF on bad shift immediates).
7536 static void shift_reg_imm(TCGv_i64 dst, TCGv_i64 src, int sf,
7537 enum a64_shift_type shift_type, unsigned int shift_i)
7539 assert(shift_i < (sf ? 64 : 32));
7541 if (shift_i == 0) {
7542 tcg_gen_mov_i64(dst, src);
7543 } else {
7544 shift_reg(dst, src, sf, shift_type, tcg_constant_i64(shift_i));
7548 /* Logical (shifted register)
7549 * 31 30 29 28 24 23 22 21 20 16 15 10 9 5 4 0
7550 * +----+-----+-----------+-------+---+------+--------+------+------+
7551 * | sf | opc | 0 1 0 1 0 | shift | N | Rm | imm6 | Rn | Rd |
7552 * +----+-----+-----------+-------+---+------+--------+------+------+
7554 static void disas_logic_reg(DisasContext *s, uint32_t insn)
7556 TCGv_i64 tcg_rd, tcg_rn, tcg_rm;
7557 unsigned int sf, opc, shift_type, invert, rm, shift_amount, rn, rd;
7559 sf = extract32(insn, 31, 1);
7560 opc = extract32(insn, 29, 2);
7561 shift_type = extract32(insn, 22, 2);
7562 invert = extract32(insn, 21, 1);
7563 rm = extract32(insn, 16, 5);
7564 shift_amount = extract32(insn, 10, 6);
7565 rn = extract32(insn, 5, 5);
7566 rd = extract32(insn, 0, 5);
7568 if (!sf && (shift_amount & (1 << 5))) {
7569 unallocated_encoding(s);
7570 return;
7573 tcg_rd = cpu_reg(s, rd);
7575 if (opc == 1 && shift_amount == 0 && shift_type == 0 && rn == 31) {
7576 /* Unshifted ORR and ORN with WZR/XZR is the standard encoding for
7577 * register-register MOV and MVN, so it is worth special casing.
7579 tcg_rm = cpu_reg(s, rm);
7580 if (invert) {
7581 tcg_gen_not_i64(tcg_rd, tcg_rm);
7582 if (!sf) {
7583 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
7585 } else {
7586 if (sf) {
7587 tcg_gen_mov_i64(tcg_rd, tcg_rm);
7588 } else {
7589 tcg_gen_ext32u_i64(tcg_rd, tcg_rm);
7592 return;
7595 tcg_rm = read_cpu_reg(s, rm, sf);
7597 if (shift_amount) {
7598 shift_reg_imm(tcg_rm, tcg_rm, sf, shift_type, shift_amount);
7601 tcg_rn = cpu_reg(s, rn);
7603 switch (opc | (invert << 2)) {
7604 case 0: /* AND */
7605 case 3: /* ANDS */
7606 tcg_gen_and_i64(tcg_rd, tcg_rn, tcg_rm);
7607 break;
7608 case 1: /* ORR */
7609 tcg_gen_or_i64(tcg_rd, tcg_rn, tcg_rm);
7610 break;
7611 case 2: /* EOR */
7612 tcg_gen_xor_i64(tcg_rd, tcg_rn, tcg_rm);
7613 break;
7614 case 4: /* BIC */
7615 case 7: /* BICS */
7616 tcg_gen_andc_i64(tcg_rd, tcg_rn, tcg_rm);
7617 break;
7618 case 5: /* ORN */
7619 tcg_gen_orc_i64(tcg_rd, tcg_rn, tcg_rm);
7620 break;
7621 case 6: /* EON */
7622 tcg_gen_eqv_i64(tcg_rd, tcg_rn, tcg_rm);
7623 break;
7624 default:
7625 assert(FALSE);
7626 break;
7629 if (!sf) {
7630 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
7633 if (opc == 3) {
7634 gen_logic_CC(sf, tcg_rd);
7639 * Add/subtract (extended register)
7641 * 31|30|29|28 24|23 22|21|20 16|15 13|12 10|9 5|4 0|
7642 * +--+--+--+-----------+-----+--+-------+------+------+----+----+
7643 * |sf|op| S| 0 1 0 1 1 | opt | 1| Rm |option| imm3 | Rn | Rd |
7644 * +--+--+--+-----------+-----+--+-------+------+------+----+----+
7646 * sf: 0 -> 32bit, 1 -> 64bit
7647 * op: 0 -> add , 1 -> sub
7648 * S: 1 -> set flags
7649 * opt: 00
7650 * option: extension type (see DecodeRegExtend)
7651 * imm3: optional shift to Rm
7653 * Rd = Rn + LSL(extend(Rm), amount)
7655 static void disas_add_sub_ext_reg(DisasContext *s, uint32_t insn)
7657 int rd = extract32(insn, 0, 5);
7658 int rn = extract32(insn, 5, 5);
7659 int imm3 = extract32(insn, 10, 3);
7660 int option = extract32(insn, 13, 3);
7661 int rm = extract32(insn, 16, 5);
7662 int opt = extract32(insn, 22, 2);
7663 bool setflags = extract32(insn, 29, 1);
7664 bool sub_op = extract32(insn, 30, 1);
7665 bool sf = extract32(insn, 31, 1);
7667 TCGv_i64 tcg_rm, tcg_rn; /* temps */
7668 TCGv_i64 tcg_rd;
7669 TCGv_i64 tcg_result;
7671 if (imm3 > 4 || opt != 0) {
7672 unallocated_encoding(s);
7673 return;
7676 /* non-flag setting ops may use SP */
7677 if (!setflags) {
7678 tcg_rd = cpu_reg_sp(s, rd);
7679 } else {
7680 tcg_rd = cpu_reg(s, rd);
7682 tcg_rn = read_cpu_reg_sp(s, rn, sf);
7684 tcg_rm = read_cpu_reg(s, rm, sf);
7685 ext_and_shift_reg(tcg_rm, tcg_rm, option, imm3);
7687 tcg_result = tcg_temp_new_i64();
7689 if (!setflags) {
7690 if (sub_op) {
7691 tcg_gen_sub_i64(tcg_result, tcg_rn, tcg_rm);
7692 } else {
7693 tcg_gen_add_i64(tcg_result, tcg_rn, tcg_rm);
7695 } else {
7696 if (sub_op) {
7697 gen_sub_CC(sf, tcg_result, tcg_rn, tcg_rm);
7698 } else {
7699 gen_add_CC(sf, tcg_result, tcg_rn, tcg_rm);
7703 if (sf) {
7704 tcg_gen_mov_i64(tcg_rd, tcg_result);
7705 } else {
7706 tcg_gen_ext32u_i64(tcg_rd, tcg_result);
7711 * Add/subtract (shifted register)
7713 * 31 30 29 28 24 23 22 21 20 16 15 10 9 5 4 0
7714 * +--+--+--+-----------+-----+--+-------+---------+------+------+
7715 * |sf|op| S| 0 1 0 1 1 |shift| 0| Rm | imm6 | Rn | Rd |
7716 * +--+--+--+-----------+-----+--+-------+---------+------+------+
7718 * sf: 0 -> 32bit, 1 -> 64bit
7719 * op: 0 -> add , 1 -> sub
7720 * S: 1 -> set flags
7721 * shift: 00 -> LSL, 01 -> LSR, 10 -> ASR, 11 -> RESERVED
7722 * imm6: Shift amount to apply to Rm before the add/sub
7724 static void disas_add_sub_reg(DisasContext *s, uint32_t insn)
7726 int rd = extract32(insn, 0, 5);
7727 int rn = extract32(insn, 5, 5);
7728 int imm6 = extract32(insn, 10, 6);
7729 int rm = extract32(insn, 16, 5);
7730 int shift_type = extract32(insn, 22, 2);
7731 bool setflags = extract32(insn, 29, 1);
7732 bool sub_op = extract32(insn, 30, 1);
7733 bool sf = extract32(insn, 31, 1);
7735 TCGv_i64 tcg_rd = cpu_reg(s, rd);
7736 TCGv_i64 tcg_rn, tcg_rm;
7737 TCGv_i64 tcg_result;
7739 if ((shift_type == 3) || (!sf && (imm6 > 31))) {
7740 unallocated_encoding(s);
7741 return;
7744 tcg_rn = read_cpu_reg(s, rn, sf);
7745 tcg_rm = read_cpu_reg(s, rm, sf);
7747 shift_reg_imm(tcg_rm, tcg_rm, sf, shift_type, imm6);
7749 tcg_result = tcg_temp_new_i64();
7751 if (!setflags) {
7752 if (sub_op) {
7753 tcg_gen_sub_i64(tcg_result, tcg_rn, tcg_rm);
7754 } else {
7755 tcg_gen_add_i64(tcg_result, tcg_rn, tcg_rm);
7757 } else {
7758 if (sub_op) {
7759 gen_sub_CC(sf, tcg_result, tcg_rn, tcg_rm);
7760 } else {
7761 gen_add_CC(sf, tcg_result, tcg_rn, tcg_rm);
7765 if (sf) {
7766 tcg_gen_mov_i64(tcg_rd, tcg_result);
7767 } else {
7768 tcg_gen_ext32u_i64(tcg_rd, tcg_result);
7772 /* Data-processing (3 source)
7774 * 31 30 29 28 24 23 21 20 16 15 14 10 9 5 4 0
7775 * +--+------+-----------+------+------+----+------+------+------+
7776 * |sf| op54 | 1 1 0 1 1 | op31 | Rm | o0 | Ra | Rn | Rd |
7777 * +--+------+-----------+------+------+----+------+------+------+
7779 static void disas_data_proc_3src(DisasContext *s, uint32_t insn)
7781 int rd = extract32(insn, 0, 5);
7782 int rn = extract32(insn, 5, 5);
7783 int ra = extract32(insn, 10, 5);
7784 int rm = extract32(insn, 16, 5);
7785 int op_id = (extract32(insn, 29, 3) << 4) |
7786 (extract32(insn, 21, 3) << 1) |
7787 extract32(insn, 15, 1);
7788 bool sf = extract32(insn, 31, 1);
7789 bool is_sub = extract32(op_id, 0, 1);
7790 bool is_high = extract32(op_id, 2, 1);
7791 bool is_signed = false;
7792 TCGv_i64 tcg_op1;
7793 TCGv_i64 tcg_op2;
7794 TCGv_i64 tcg_tmp;
7796 /* Note that op_id is sf:op54:op31:o0 so it includes the 32/64 size flag */
7797 switch (op_id) {
7798 case 0x42: /* SMADDL */
7799 case 0x43: /* SMSUBL */
7800 case 0x44: /* SMULH */
7801 is_signed = true;
7802 break;
7803 case 0x0: /* MADD (32bit) */
7804 case 0x1: /* MSUB (32bit) */
7805 case 0x40: /* MADD (64bit) */
7806 case 0x41: /* MSUB (64bit) */
7807 case 0x4a: /* UMADDL */
7808 case 0x4b: /* UMSUBL */
7809 case 0x4c: /* UMULH */
7810 break;
7811 default:
7812 unallocated_encoding(s);
7813 return;
7816 if (is_high) {
7817 TCGv_i64 low_bits = tcg_temp_new_i64(); /* low bits discarded */
7818 TCGv_i64 tcg_rd = cpu_reg(s, rd);
7819 TCGv_i64 tcg_rn = cpu_reg(s, rn);
7820 TCGv_i64 tcg_rm = cpu_reg(s, rm);
7822 if (is_signed) {
7823 tcg_gen_muls2_i64(low_bits, tcg_rd, tcg_rn, tcg_rm);
7824 } else {
7825 tcg_gen_mulu2_i64(low_bits, tcg_rd, tcg_rn, tcg_rm);
7827 return;
7830 tcg_op1 = tcg_temp_new_i64();
7831 tcg_op2 = tcg_temp_new_i64();
7832 tcg_tmp = tcg_temp_new_i64();
7834 if (op_id < 0x42) {
7835 tcg_gen_mov_i64(tcg_op1, cpu_reg(s, rn));
7836 tcg_gen_mov_i64(tcg_op2, cpu_reg(s, rm));
7837 } else {
7838 if (is_signed) {
7839 tcg_gen_ext32s_i64(tcg_op1, cpu_reg(s, rn));
7840 tcg_gen_ext32s_i64(tcg_op2, cpu_reg(s, rm));
7841 } else {
7842 tcg_gen_ext32u_i64(tcg_op1, cpu_reg(s, rn));
7843 tcg_gen_ext32u_i64(tcg_op2, cpu_reg(s, rm));
7847 if (ra == 31 && !is_sub) {
7848 /* Special-case MADD with rA == XZR; it is the standard MUL alias */
7849 tcg_gen_mul_i64(cpu_reg(s, rd), tcg_op1, tcg_op2);
7850 } else {
7851 tcg_gen_mul_i64(tcg_tmp, tcg_op1, tcg_op2);
7852 if (is_sub) {
7853 tcg_gen_sub_i64(cpu_reg(s, rd), cpu_reg(s, ra), tcg_tmp);
7854 } else {
7855 tcg_gen_add_i64(cpu_reg(s, rd), cpu_reg(s, ra), tcg_tmp);
7859 if (!sf) {
7860 tcg_gen_ext32u_i64(cpu_reg(s, rd), cpu_reg(s, rd));
7864 /* Add/subtract (with carry)
7865 * 31 30 29 28 27 26 25 24 23 22 21 20 16 15 10 9 5 4 0
7866 * +--+--+--+------------------------+------+-------------+------+-----+
7867 * |sf|op| S| 1 1 0 1 0 0 0 0 | rm | 0 0 0 0 0 0 | Rn | Rd |
7868 * +--+--+--+------------------------+------+-------------+------+-----+
7871 static void disas_adc_sbc(DisasContext *s, uint32_t insn)
7873 unsigned int sf, op, setflags, rm, rn, rd;
7874 TCGv_i64 tcg_y, tcg_rn, tcg_rd;
7876 sf = extract32(insn, 31, 1);
7877 op = extract32(insn, 30, 1);
7878 setflags = extract32(insn, 29, 1);
7879 rm = extract32(insn, 16, 5);
7880 rn = extract32(insn, 5, 5);
7881 rd = extract32(insn, 0, 5);
7883 tcg_rd = cpu_reg(s, rd);
7884 tcg_rn = cpu_reg(s, rn);
7886 if (op) {
7887 tcg_y = tcg_temp_new_i64();
7888 tcg_gen_not_i64(tcg_y, cpu_reg(s, rm));
7889 } else {
7890 tcg_y = cpu_reg(s, rm);
7893 if (setflags) {
7894 gen_adc_CC(sf, tcg_rd, tcg_rn, tcg_y);
7895 } else {
7896 gen_adc(sf, tcg_rd, tcg_rn, tcg_y);
7901 * Rotate right into flags
7902 * 31 30 29 21 15 10 5 4 0
7903 * +--+--+--+-----------------+--------+-----------+------+--+------+
7904 * |sf|op| S| 1 1 0 1 0 0 0 0 | imm6 | 0 0 0 0 1 | Rn |o2| mask |
7905 * +--+--+--+-----------------+--------+-----------+------+--+------+
7907 static void disas_rotate_right_into_flags(DisasContext *s, uint32_t insn)
7909 int mask = extract32(insn, 0, 4);
7910 int o2 = extract32(insn, 4, 1);
7911 int rn = extract32(insn, 5, 5);
7912 int imm6 = extract32(insn, 15, 6);
7913 int sf_op_s = extract32(insn, 29, 3);
7914 TCGv_i64 tcg_rn;
7915 TCGv_i32 nzcv;
7917 if (sf_op_s != 5 || o2 != 0 || !dc_isar_feature(aa64_condm_4, s)) {
7918 unallocated_encoding(s);
7919 return;
7922 tcg_rn = read_cpu_reg(s, rn, 1);
7923 tcg_gen_rotri_i64(tcg_rn, tcg_rn, imm6);
7925 nzcv = tcg_temp_new_i32();
7926 tcg_gen_extrl_i64_i32(nzcv, tcg_rn);
7928 if (mask & 8) { /* N */
7929 tcg_gen_shli_i32(cpu_NF, nzcv, 31 - 3);
7931 if (mask & 4) { /* Z */
7932 tcg_gen_not_i32(cpu_ZF, nzcv);
7933 tcg_gen_andi_i32(cpu_ZF, cpu_ZF, 4);
7935 if (mask & 2) { /* C */
7936 tcg_gen_extract_i32(cpu_CF, nzcv, 1, 1);
7938 if (mask & 1) { /* V */
7939 tcg_gen_shli_i32(cpu_VF, nzcv, 31 - 0);
7944 * Evaluate into flags
7945 * 31 30 29 21 15 14 10 5 4 0
7946 * +--+--+--+-----------------+---------+----+---------+------+--+------+
7947 * |sf|op| S| 1 1 0 1 0 0 0 0 | opcode2 | sz | 0 0 1 0 | Rn |o3| mask |
7948 * +--+--+--+-----------------+---------+----+---------+------+--+------+
7950 static void disas_evaluate_into_flags(DisasContext *s, uint32_t insn)
7952 int o3_mask = extract32(insn, 0, 5);
7953 int rn = extract32(insn, 5, 5);
7954 int o2 = extract32(insn, 15, 6);
7955 int sz = extract32(insn, 14, 1);
7956 int sf_op_s = extract32(insn, 29, 3);
7957 TCGv_i32 tmp;
7958 int shift;
7960 if (sf_op_s != 1 || o2 != 0 || o3_mask != 0xd ||
7961 !dc_isar_feature(aa64_condm_4, s)) {
7962 unallocated_encoding(s);
7963 return;
7965 shift = sz ? 16 : 24; /* SETF16 or SETF8 */
7967 tmp = tcg_temp_new_i32();
7968 tcg_gen_extrl_i64_i32(tmp, cpu_reg(s, rn));
7969 tcg_gen_shli_i32(cpu_NF, tmp, shift);
7970 tcg_gen_shli_i32(cpu_VF, tmp, shift - 1);
7971 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
7972 tcg_gen_xor_i32(cpu_VF, cpu_VF, cpu_NF);
7975 /* Conditional compare (immediate / register)
7976 * 31 30 29 28 27 26 25 24 23 22 21 20 16 15 12 11 10 9 5 4 3 0
7977 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+
7978 * |sf|op| S| 1 1 0 1 0 0 1 0 |imm5/rm | cond |i/r |o2| Rn |o3|nzcv |
7979 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+
7980 * [1] y [0] [0]
7982 static void disas_cc(DisasContext *s, uint32_t insn)
7984 unsigned int sf, op, y, cond, rn, nzcv, is_imm;
7985 TCGv_i32 tcg_t0, tcg_t1, tcg_t2;
7986 TCGv_i64 tcg_tmp, tcg_y, tcg_rn;
7987 DisasCompare c;
7989 if (!extract32(insn, 29, 1)) {
7990 unallocated_encoding(s);
7991 return;
7993 if (insn & (1 << 10 | 1 << 4)) {
7994 unallocated_encoding(s);
7995 return;
7997 sf = extract32(insn, 31, 1);
7998 op = extract32(insn, 30, 1);
7999 is_imm = extract32(insn, 11, 1);
8000 y = extract32(insn, 16, 5); /* y = rm (reg) or imm5 (imm) */
8001 cond = extract32(insn, 12, 4);
8002 rn = extract32(insn, 5, 5);
8003 nzcv = extract32(insn, 0, 4);
8005 /* Set T0 = !COND. */
8006 tcg_t0 = tcg_temp_new_i32();
8007 arm_test_cc(&c, cond);
8008 tcg_gen_setcondi_i32(tcg_invert_cond(c.cond), tcg_t0, c.value, 0);
8010 /* Load the arguments for the new comparison. */
8011 if (is_imm) {
8012 tcg_y = tcg_temp_new_i64();
8013 tcg_gen_movi_i64(tcg_y, y);
8014 } else {
8015 tcg_y = cpu_reg(s, y);
8017 tcg_rn = cpu_reg(s, rn);
8019 /* Set the flags for the new comparison. */
8020 tcg_tmp = tcg_temp_new_i64();
8021 if (op) {
8022 gen_sub_CC(sf, tcg_tmp, tcg_rn, tcg_y);
8023 } else {
8024 gen_add_CC(sf, tcg_tmp, tcg_rn, tcg_y);
8027 /* If COND was false, force the flags to #nzcv. Compute two masks
8028 * to help with this: T1 = (COND ? 0 : -1), T2 = (COND ? -1 : 0).
8029 * For tcg hosts that support ANDC, we can make do with just T1.
8030 * In either case, allow the tcg optimizer to delete any unused mask.
8032 tcg_t1 = tcg_temp_new_i32();
8033 tcg_t2 = tcg_temp_new_i32();
8034 tcg_gen_neg_i32(tcg_t1, tcg_t0);
8035 tcg_gen_subi_i32(tcg_t2, tcg_t0, 1);
8037 if (nzcv & 8) { /* N */
8038 tcg_gen_or_i32(cpu_NF, cpu_NF, tcg_t1);
8039 } else {
8040 if (TCG_TARGET_HAS_andc_i32) {
8041 tcg_gen_andc_i32(cpu_NF, cpu_NF, tcg_t1);
8042 } else {
8043 tcg_gen_and_i32(cpu_NF, cpu_NF, tcg_t2);
8046 if (nzcv & 4) { /* Z */
8047 if (TCG_TARGET_HAS_andc_i32) {
8048 tcg_gen_andc_i32(cpu_ZF, cpu_ZF, tcg_t1);
8049 } else {
8050 tcg_gen_and_i32(cpu_ZF, cpu_ZF, tcg_t2);
8052 } else {
8053 tcg_gen_or_i32(cpu_ZF, cpu_ZF, tcg_t0);
8055 if (nzcv & 2) { /* C */
8056 tcg_gen_or_i32(cpu_CF, cpu_CF, tcg_t0);
8057 } else {
8058 if (TCG_TARGET_HAS_andc_i32) {
8059 tcg_gen_andc_i32(cpu_CF, cpu_CF, tcg_t1);
8060 } else {
8061 tcg_gen_and_i32(cpu_CF, cpu_CF, tcg_t2);
8064 if (nzcv & 1) { /* V */
8065 tcg_gen_or_i32(cpu_VF, cpu_VF, tcg_t1);
8066 } else {
8067 if (TCG_TARGET_HAS_andc_i32) {
8068 tcg_gen_andc_i32(cpu_VF, cpu_VF, tcg_t1);
8069 } else {
8070 tcg_gen_and_i32(cpu_VF, cpu_VF, tcg_t2);
8075 /* Conditional select
8076 * 31 30 29 28 21 20 16 15 12 11 10 9 5 4 0
8077 * +----+----+---+-----------------+------+------+-----+------+------+
8078 * | sf | op | S | 1 1 0 1 0 1 0 0 | Rm | cond | op2 | Rn | Rd |
8079 * +----+----+---+-----------------+------+------+-----+------+------+
8081 static void disas_cond_select(DisasContext *s, uint32_t insn)
8083 unsigned int sf, else_inv, rm, cond, else_inc, rn, rd;
8084 TCGv_i64 tcg_rd, zero;
8085 DisasCompare64 c;
8087 if (extract32(insn, 29, 1) || extract32(insn, 11, 1)) {
8088 /* S == 1 or op2<1> == 1 */
8089 unallocated_encoding(s);
8090 return;
8092 sf = extract32(insn, 31, 1);
8093 else_inv = extract32(insn, 30, 1);
8094 rm = extract32(insn, 16, 5);
8095 cond = extract32(insn, 12, 4);
8096 else_inc = extract32(insn, 10, 1);
8097 rn = extract32(insn, 5, 5);
8098 rd = extract32(insn, 0, 5);
8100 tcg_rd = cpu_reg(s, rd);
8102 a64_test_cc(&c, cond);
8103 zero = tcg_constant_i64(0);
8105 if (rn == 31 && rm == 31 && (else_inc ^ else_inv)) {
8106 /* CSET & CSETM. */
8107 if (else_inv) {
8108 tcg_gen_negsetcond_i64(tcg_invert_cond(c.cond),
8109 tcg_rd, c.value, zero);
8110 } else {
8111 tcg_gen_setcond_i64(tcg_invert_cond(c.cond),
8112 tcg_rd, c.value, zero);
8114 } else {
8115 TCGv_i64 t_true = cpu_reg(s, rn);
8116 TCGv_i64 t_false = read_cpu_reg(s, rm, 1);
8117 if (else_inv && else_inc) {
8118 tcg_gen_neg_i64(t_false, t_false);
8119 } else if (else_inv) {
8120 tcg_gen_not_i64(t_false, t_false);
8121 } else if (else_inc) {
8122 tcg_gen_addi_i64(t_false, t_false, 1);
8124 tcg_gen_movcond_i64(c.cond, tcg_rd, c.value, zero, t_true, t_false);
8127 if (!sf) {
8128 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
8132 static void handle_clz(DisasContext *s, unsigned int sf,
8133 unsigned int rn, unsigned int rd)
8135 TCGv_i64 tcg_rd, tcg_rn;
8136 tcg_rd = cpu_reg(s, rd);
8137 tcg_rn = cpu_reg(s, rn);
8139 if (sf) {
8140 tcg_gen_clzi_i64(tcg_rd, tcg_rn, 64);
8141 } else {
8142 TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
8143 tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
8144 tcg_gen_clzi_i32(tcg_tmp32, tcg_tmp32, 32);
8145 tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
8149 static void handle_cls(DisasContext *s, unsigned int sf,
8150 unsigned int rn, unsigned int rd)
8152 TCGv_i64 tcg_rd, tcg_rn;
8153 tcg_rd = cpu_reg(s, rd);
8154 tcg_rn = cpu_reg(s, rn);
8156 if (sf) {
8157 tcg_gen_clrsb_i64(tcg_rd, tcg_rn);
8158 } else {
8159 TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
8160 tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
8161 tcg_gen_clrsb_i32(tcg_tmp32, tcg_tmp32);
8162 tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
8166 static void handle_rbit(DisasContext *s, unsigned int sf,
8167 unsigned int rn, unsigned int rd)
8169 TCGv_i64 tcg_rd, tcg_rn;
8170 tcg_rd = cpu_reg(s, rd);
8171 tcg_rn = cpu_reg(s, rn);
8173 if (sf) {
8174 gen_helper_rbit64(tcg_rd, tcg_rn);
8175 } else {
8176 TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
8177 tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
8178 gen_helper_rbit(tcg_tmp32, tcg_tmp32);
8179 tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
8183 /* REV with sf==1, opcode==3 ("REV64") */
8184 static void handle_rev64(DisasContext *s, unsigned int sf,
8185 unsigned int rn, unsigned int rd)
8187 if (!sf) {
8188 unallocated_encoding(s);
8189 return;
8191 tcg_gen_bswap64_i64(cpu_reg(s, rd), cpu_reg(s, rn));
8194 /* REV with sf==0, opcode==2
8195 * REV32 (sf==1, opcode==2)
8197 static void handle_rev32(DisasContext *s, unsigned int sf,
8198 unsigned int rn, unsigned int rd)
8200 TCGv_i64 tcg_rd = cpu_reg(s, rd);
8201 TCGv_i64 tcg_rn = cpu_reg(s, rn);
8203 if (sf) {
8204 tcg_gen_bswap64_i64(tcg_rd, tcg_rn);
8205 tcg_gen_rotri_i64(tcg_rd, tcg_rd, 32);
8206 } else {
8207 tcg_gen_bswap32_i64(tcg_rd, tcg_rn, TCG_BSWAP_OZ);
8211 /* REV16 (opcode==1) */
8212 static void handle_rev16(DisasContext *s, unsigned int sf,
8213 unsigned int rn, unsigned int rd)
8215 TCGv_i64 tcg_rd = cpu_reg(s, rd);
8216 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
8217 TCGv_i64 tcg_rn = read_cpu_reg(s, rn, sf);
8218 TCGv_i64 mask = tcg_constant_i64(sf ? 0x00ff00ff00ff00ffull : 0x00ff00ff);
8220 tcg_gen_shri_i64(tcg_tmp, tcg_rn, 8);
8221 tcg_gen_and_i64(tcg_rd, tcg_rn, mask);
8222 tcg_gen_and_i64(tcg_tmp, tcg_tmp, mask);
8223 tcg_gen_shli_i64(tcg_rd, tcg_rd, 8);
8224 tcg_gen_or_i64(tcg_rd, tcg_rd, tcg_tmp);
8227 /* Data-processing (1 source)
8228 * 31 30 29 28 21 20 16 15 10 9 5 4 0
8229 * +----+---+---+-----------------+---------+--------+------+------+
8230 * | sf | 1 | S | 1 1 0 1 0 1 1 0 | opcode2 | opcode | Rn | Rd |
8231 * +----+---+---+-----------------+---------+--------+------+------+
8233 static void disas_data_proc_1src(DisasContext *s, uint32_t insn)
8235 unsigned int sf, opcode, opcode2, rn, rd;
8236 TCGv_i64 tcg_rd;
8238 if (extract32(insn, 29, 1)) {
8239 unallocated_encoding(s);
8240 return;
8243 sf = extract32(insn, 31, 1);
8244 opcode = extract32(insn, 10, 6);
8245 opcode2 = extract32(insn, 16, 5);
8246 rn = extract32(insn, 5, 5);
8247 rd = extract32(insn, 0, 5);
8249 #define MAP(SF, O2, O1) ((SF) | (O1 << 1) | (O2 << 7))
8251 switch (MAP(sf, opcode2, opcode)) {
8252 case MAP(0, 0x00, 0x00): /* RBIT */
8253 case MAP(1, 0x00, 0x00):
8254 handle_rbit(s, sf, rn, rd);
8255 break;
8256 case MAP(0, 0x00, 0x01): /* REV16 */
8257 case MAP(1, 0x00, 0x01):
8258 handle_rev16(s, sf, rn, rd);
8259 break;
8260 case MAP(0, 0x00, 0x02): /* REV/REV32 */
8261 case MAP(1, 0x00, 0x02):
8262 handle_rev32(s, sf, rn, rd);
8263 break;
8264 case MAP(1, 0x00, 0x03): /* REV64 */
8265 handle_rev64(s, sf, rn, rd);
8266 break;
8267 case MAP(0, 0x00, 0x04): /* CLZ */
8268 case MAP(1, 0x00, 0x04):
8269 handle_clz(s, sf, rn, rd);
8270 break;
8271 case MAP(0, 0x00, 0x05): /* CLS */
8272 case MAP(1, 0x00, 0x05):
8273 handle_cls(s, sf, rn, rd);
8274 break;
8275 case MAP(1, 0x01, 0x00): /* PACIA */
8276 if (s->pauth_active) {
8277 tcg_rd = cpu_reg(s, rd);
8278 gen_helper_pacia(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
8279 } else if (!dc_isar_feature(aa64_pauth, s)) {
8280 goto do_unallocated;
8282 break;
8283 case MAP(1, 0x01, 0x01): /* PACIB */
8284 if (s->pauth_active) {
8285 tcg_rd = cpu_reg(s, rd);
8286 gen_helper_pacib(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
8287 } else if (!dc_isar_feature(aa64_pauth, s)) {
8288 goto do_unallocated;
8290 break;
8291 case MAP(1, 0x01, 0x02): /* PACDA */
8292 if (s->pauth_active) {
8293 tcg_rd = cpu_reg(s, rd);
8294 gen_helper_pacda(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
8295 } else if (!dc_isar_feature(aa64_pauth, s)) {
8296 goto do_unallocated;
8298 break;
8299 case MAP(1, 0x01, 0x03): /* PACDB */
8300 if (s->pauth_active) {
8301 tcg_rd = cpu_reg(s, rd);
8302 gen_helper_pacdb(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
8303 } else if (!dc_isar_feature(aa64_pauth, s)) {
8304 goto do_unallocated;
8306 break;
8307 case MAP(1, 0x01, 0x04): /* AUTIA */
8308 if (s->pauth_active) {
8309 tcg_rd = cpu_reg(s, rd);
8310 gen_helper_autia(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
8311 } else if (!dc_isar_feature(aa64_pauth, s)) {
8312 goto do_unallocated;
8314 break;
8315 case MAP(1, 0x01, 0x05): /* AUTIB */
8316 if (s->pauth_active) {
8317 tcg_rd = cpu_reg(s, rd);
8318 gen_helper_autib(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
8319 } else if (!dc_isar_feature(aa64_pauth, s)) {
8320 goto do_unallocated;
8322 break;
8323 case MAP(1, 0x01, 0x06): /* AUTDA */
8324 if (s->pauth_active) {
8325 tcg_rd = cpu_reg(s, rd);
8326 gen_helper_autda(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
8327 } else if (!dc_isar_feature(aa64_pauth, s)) {
8328 goto do_unallocated;
8330 break;
8331 case MAP(1, 0x01, 0x07): /* AUTDB */
8332 if (s->pauth_active) {
8333 tcg_rd = cpu_reg(s, rd);
8334 gen_helper_autdb(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
8335 } else if (!dc_isar_feature(aa64_pauth, s)) {
8336 goto do_unallocated;
8338 break;
8339 case MAP(1, 0x01, 0x08): /* PACIZA */
8340 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
8341 goto do_unallocated;
8342 } else if (s->pauth_active) {
8343 tcg_rd = cpu_reg(s, rd);
8344 gen_helper_pacia(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
8346 break;
8347 case MAP(1, 0x01, 0x09): /* PACIZB */
8348 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
8349 goto do_unallocated;
8350 } else if (s->pauth_active) {
8351 tcg_rd = cpu_reg(s, rd);
8352 gen_helper_pacib(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
8354 break;
8355 case MAP(1, 0x01, 0x0a): /* PACDZA */
8356 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
8357 goto do_unallocated;
8358 } else if (s->pauth_active) {
8359 tcg_rd = cpu_reg(s, rd);
8360 gen_helper_pacda(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
8362 break;
8363 case MAP(1, 0x01, 0x0b): /* PACDZB */
8364 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
8365 goto do_unallocated;
8366 } else if (s->pauth_active) {
8367 tcg_rd = cpu_reg(s, rd);
8368 gen_helper_pacdb(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
8370 break;
8371 case MAP(1, 0x01, 0x0c): /* AUTIZA */
8372 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
8373 goto do_unallocated;
8374 } else if (s->pauth_active) {
8375 tcg_rd = cpu_reg(s, rd);
8376 gen_helper_autia(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
8378 break;
8379 case MAP(1, 0x01, 0x0d): /* AUTIZB */
8380 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
8381 goto do_unallocated;
8382 } else if (s->pauth_active) {
8383 tcg_rd = cpu_reg(s, rd);
8384 gen_helper_autib(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
8386 break;
8387 case MAP(1, 0x01, 0x0e): /* AUTDZA */
8388 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
8389 goto do_unallocated;
8390 } else if (s->pauth_active) {
8391 tcg_rd = cpu_reg(s, rd);
8392 gen_helper_autda(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
8394 break;
8395 case MAP(1, 0x01, 0x0f): /* AUTDZB */
8396 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
8397 goto do_unallocated;
8398 } else if (s->pauth_active) {
8399 tcg_rd = cpu_reg(s, rd);
8400 gen_helper_autdb(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
8402 break;
8403 case MAP(1, 0x01, 0x10): /* XPACI */
8404 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
8405 goto do_unallocated;
8406 } else if (s->pauth_active) {
8407 tcg_rd = cpu_reg(s, rd);
8408 gen_helper_xpaci(tcg_rd, tcg_env, tcg_rd);
8410 break;
8411 case MAP(1, 0x01, 0x11): /* XPACD */
8412 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
8413 goto do_unallocated;
8414 } else if (s->pauth_active) {
8415 tcg_rd = cpu_reg(s, rd);
8416 gen_helper_xpacd(tcg_rd, tcg_env, tcg_rd);
8418 break;
8419 default:
8420 do_unallocated:
8421 unallocated_encoding(s);
8422 break;
8425 #undef MAP
8428 static void handle_div(DisasContext *s, bool is_signed, unsigned int sf,
8429 unsigned int rm, unsigned int rn, unsigned int rd)
8431 TCGv_i64 tcg_n, tcg_m, tcg_rd;
8432 tcg_rd = cpu_reg(s, rd);
8434 if (!sf && is_signed) {
8435 tcg_n = tcg_temp_new_i64();
8436 tcg_m = tcg_temp_new_i64();
8437 tcg_gen_ext32s_i64(tcg_n, cpu_reg(s, rn));
8438 tcg_gen_ext32s_i64(tcg_m, cpu_reg(s, rm));
8439 } else {
8440 tcg_n = read_cpu_reg(s, rn, sf);
8441 tcg_m = read_cpu_reg(s, rm, sf);
8444 if (is_signed) {
8445 gen_helper_sdiv64(tcg_rd, tcg_n, tcg_m);
8446 } else {
8447 gen_helper_udiv64(tcg_rd, tcg_n, tcg_m);
8450 if (!sf) { /* zero extend final result */
8451 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
8455 /* LSLV, LSRV, ASRV, RORV */
8456 static void handle_shift_reg(DisasContext *s,
8457 enum a64_shift_type shift_type, unsigned int sf,
8458 unsigned int rm, unsigned int rn, unsigned int rd)
8460 TCGv_i64 tcg_shift = tcg_temp_new_i64();
8461 TCGv_i64 tcg_rd = cpu_reg(s, rd);
8462 TCGv_i64 tcg_rn = read_cpu_reg(s, rn, sf);
8464 tcg_gen_andi_i64(tcg_shift, cpu_reg(s, rm), sf ? 63 : 31);
8465 shift_reg(tcg_rd, tcg_rn, sf, shift_type, tcg_shift);
8468 /* CRC32[BHWX], CRC32C[BHWX] */
8469 static void handle_crc32(DisasContext *s,
8470 unsigned int sf, unsigned int sz, bool crc32c,
8471 unsigned int rm, unsigned int rn, unsigned int rd)
8473 TCGv_i64 tcg_acc, tcg_val;
8474 TCGv_i32 tcg_bytes;
8476 if (!dc_isar_feature(aa64_crc32, s)
8477 || (sf == 1 && sz != 3)
8478 || (sf == 0 && sz == 3)) {
8479 unallocated_encoding(s);
8480 return;
8483 if (sz == 3) {
8484 tcg_val = cpu_reg(s, rm);
8485 } else {
8486 uint64_t mask;
8487 switch (sz) {
8488 case 0:
8489 mask = 0xFF;
8490 break;
8491 case 1:
8492 mask = 0xFFFF;
8493 break;
8494 case 2:
8495 mask = 0xFFFFFFFF;
8496 break;
8497 default:
8498 g_assert_not_reached();
8500 tcg_val = tcg_temp_new_i64();
8501 tcg_gen_andi_i64(tcg_val, cpu_reg(s, rm), mask);
8504 tcg_acc = cpu_reg(s, rn);
8505 tcg_bytes = tcg_constant_i32(1 << sz);
8507 if (crc32c) {
8508 gen_helper_crc32c_64(cpu_reg(s, rd), tcg_acc, tcg_val, tcg_bytes);
8509 } else {
8510 gen_helper_crc32_64(cpu_reg(s, rd), tcg_acc, tcg_val, tcg_bytes);
8514 /* Data-processing (2 source)
8515 * 31 30 29 28 21 20 16 15 10 9 5 4 0
8516 * +----+---+---+-----------------+------+--------+------+------+
8517 * | sf | 0 | S | 1 1 0 1 0 1 1 0 | Rm | opcode | Rn | Rd |
8518 * +----+---+---+-----------------+------+--------+------+------+
8520 static void disas_data_proc_2src(DisasContext *s, uint32_t insn)
8522 unsigned int sf, rm, opcode, rn, rd, setflag;
8523 sf = extract32(insn, 31, 1);
8524 setflag = extract32(insn, 29, 1);
8525 rm = extract32(insn, 16, 5);
8526 opcode = extract32(insn, 10, 6);
8527 rn = extract32(insn, 5, 5);
8528 rd = extract32(insn, 0, 5);
8530 if (setflag && opcode != 0) {
8531 unallocated_encoding(s);
8532 return;
8535 switch (opcode) {
8536 case 0: /* SUBP(S) */
8537 if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) {
8538 goto do_unallocated;
8539 } else {
8540 TCGv_i64 tcg_n, tcg_m, tcg_d;
8542 tcg_n = read_cpu_reg_sp(s, rn, true);
8543 tcg_m = read_cpu_reg_sp(s, rm, true);
8544 tcg_gen_sextract_i64(tcg_n, tcg_n, 0, 56);
8545 tcg_gen_sextract_i64(tcg_m, tcg_m, 0, 56);
8546 tcg_d = cpu_reg(s, rd);
8548 if (setflag) {
8549 gen_sub_CC(true, tcg_d, tcg_n, tcg_m);
8550 } else {
8551 tcg_gen_sub_i64(tcg_d, tcg_n, tcg_m);
8554 break;
8555 case 2: /* UDIV */
8556 handle_div(s, false, sf, rm, rn, rd);
8557 break;
8558 case 3: /* SDIV */
8559 handle_div(s, true, sf, rm, rn, rd);
8560 break;
8561 case 4: /* IRG */
8562 if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) {
8563 goto do_unallocated;
8565 if (s->ata[0]) {
8566 gen_helper_irg(cpu_reg_sp(s, rd), tcg_env,
8567 cpu_reg_sp(s, rn), cpu_reg(s, rm));
8568 } else {
8569 gen_address_with_allocation_tag0(cpu_reg_sp(s, rd),
8570 cpu_reg_sp(s, rn));
8572 break;
8573 case 5: /* GMI */
8574 if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) {
8575 goto do_unallocated;
8576 } else {
8577 TCGv_i64 t = tcg_temp_new_i64();
8579 tcg_gen_extract_i64(t, cpu_reg_sp(s, rn), 56, 4);
8580 tcg_gen_shl_i64(t, tcg_constant_i64(1), t);
8581 tcg_gen_or_i64(cpu_reg(s, rd), cpu_reg(s, rm), t);
8583 break;
8584 case 8: /* LSLV */
8585 handle_shift_reg(s, A64_SHIFT_TYPE_LSL, sf, rm, rn, rd);
8586 break;
8587 case 9: /* LSRV */
8588 handle_shift_reg(s, A64_SHIFT_TYPE_LSR, sf, rm, rn, rd);
8589 break;
8590 case 10: /* ASRV */
8591 handle_shift_reg(s, A64_SHIFT_TYPE_ASR, sf, rm, rn, rd);
8592 break;
8593 case 11: /* RORV */
8594 handle_shift_reg(s, A64_SHIFT_TYPE_ROR, sf, rm, rn, rd);
8595 break;
8596 case 12: /* PACGA */
8597 if (sf == 0 || !dc_isar_feature(aa64_pauth, s)) {
8598 goto do_unallocated;
8600 gen_helper_pacga(cpu_reg(s, rd), tcg_env,
8601 cpu_reg(s, rn), cpu_reg_sp(s, rm));
8602 break;
8603 case 16:
8604 case 17:
8605 case 18:
8606 case 19:
8607 case 20:
8608 case 21:
8609 case 22:
8610 case 23: /* CRC32 */
8612 int sz = extract32(opcode, 0, 2);
8613 bool crc32c = extract32(opcode, 2, 1);
8614 handle_crc32(s, sf, sz, crc32c, rm, rn, rd);
8615 break;
8617 default:
8618 do_unallocated:
8619 unallocated_encoding(s);
8620 break;
8625 * Data processing - register
8626 * 31 30 29 28 25 21 20 16 10 0
8627 * +--+---+--+---+-------+-----+-------+-------+---------+
8628 * | |op0| |op1| 1 0 1 | op2 | | op3 | |
8629 * +--+---+--+---+-------+-----+-------+-------+---------+
8631 static void disas_data_proc_reg(DisasContext *s, uint32_t insn)
8633 int op0 = extract32(insn, 30, 1);
8634 int op1 = extract32(insn, 28, 1);
8635 int op2 = extract32(insn, 21, 4);
8636 int op3 = extract32(insn, 10, 6);
8638 if (!op1) {
8639 if (op2 & 8) {
8640 if (op2 & 1) {
8641 /* Add/sub (extended register) */
8642 disas_add_sub_ext_reg(s, insn);
8643 } else {
8644 /* Add/sub (shifted register) */
8645 disas_add_sub_reg(s, insn);
8647 } else {
8648 /* Logical (shifted register) */
8649 disas_logic_reg(s, insn);
8651 return;
8654 switch (op2) {
8655 case 0x0:
8656 switch (op3) {
8657 case 0x00: /* Add/subtract (with carry) */
8658 disas_adc_sbc(s, insn);
8659 break;
8661 case 0x01: /* Rotate right into flags */
8662 case 0x21:
8663 disas_rotate_right_into_flags(s, insn);
8664 break;
8666 case 0x02: /* Evaluate into flags */
8667 case 0x12:
8668 case 0x22:
8669 case 0x32:
8670 disas_evaluate_into_flags(s, insn);
8671 break;
8673 default:
8674 goto do_unallocated;
8676 break;
8678 case 0x2: /* Conditional compare */
8679 disas_cc(s, insn); /* both imm and reg forms */
8680 break;
8682 case 0x4: /* Conditional select */
8683 disas_cond_select(s, insn);
8684 break;
8686 case 0x6: /* Data-processing */
8687 if (op0) { /* (1 source) */
8688 disas_data_proc_1src(s, insn);
8689 } else { /* (2 source) */
8690 disas_data_proc_2src(s, insn);
8692 break;
8693 case 0x8 ... 0xf: /* (3 source) */
8694 disas_data_proc_3src(s, insn);
8695 break;
8697 default:
8698 do_unallocated:
8699 unallocated_encoding(s);
8700 break;
8704 static void handle_fp_compare(DisasContext *s, int size,
8705 unsigned int rn, unsigned int rm,
8706 bool cmp_with_zero, bool signal_all_nans)
8708 TCGv_i64 tcg_flags = tcg_temp_new_i64();
8709 TCGv_ptr fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
8711 if (size == MO_64) {
8712 TCGv_i64 tcg_vn, tcg_vm;
8714 tcg_vn = read_fp_dreg(s, rn);
8715 if (cmp_with_zero) {
8716 tcg_vm = tcg_constant_i64(0);
8717 } else {
8718 tcg_vm = read_fp_dreg(s, rm);
8720 if (signal_all_nans) {
8721 gen_helper_vfp_cmped_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
8722 } else {
8723 gen_helper_vfp_cmpd_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
8725 } else {
8726 TCGv_i32 tcg_vn = tcg_temp_new_i32();
8727 TCGv_i32 tcg_vm = tcg_temp_new_i32();
8729 read_vec_element_i32(s, tcg_vn, rn, 0, size);
8730 if (cmp_with_zero) {
8731 tcg_gen_movi_i32(tcg_vm, 0);
8732 } else {
8733 read_vec_element_i32(s, tcg_vm, rm, 0, size);
8736 switch (size) {
8737 case MO_32:
8738 if (signal_all_nans) {
8739 gen_helper_vfp_cmpes_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
8740 } else {
8741 gen_helper_vfp_cmps_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
8743 break;
8744 case MO_16:
8745 if (signal_all_nans) {
8746 gen_helper_vfp_cmpeh_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
8747 } else {
8748 gen_helper_vfp_cmph_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
8750 break;
8751 default:
8752 g_assert_not_reached();
8756 gen_set_nzcv(tcg_flags);
8759 /* Floating point compare
8760 * 31 30 29 28 24 23 22 21 20 16 15 14 13 10 9 5 4 0
8761 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+
8762 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | op | 1 0 0 0 | Rn | op2 |
8763 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+
8765 static void disas_fp_compare(DisasContext *s, uint32_t insn)
8767 unsigned int mos, type, rm, op, rn, opc, op2r;
8768 int size;
8770 mos = extract32(insn, 29, 3);
8771 type = extract32(insn, 22, 2);
8772 rm = extract32(insn, 16, 5);
8773 op = extract32(insn, 14, 2);
8774 rn = extract32(insn, 5, 5);
8775 opc = extract32(insn, 3, 2);
8776 op2r = extract32(insn, 0, 3);
8778 if (mos || op || op2r) {
8779 unallocated_encoding(s);
8780 return;
8783 switch (type) {
8784 case 0:
8785 size = MO_32;
8786 break;
8787 case 1:
8788 size = MO_64;
8789 break;
8790 case 3:
8791 size = MO_16;
8792 if (dc_isar_feature(aa64_fp16, s)) {
8793 break;
8795 /* fallthru */
8796 default:
8797 unallocated_encoding(s);
8798 return;
8801 if (!fp_access_check(s)) {
8802 return;
8805 handle_fp_compare(s, size, rn, rm, opc & 1, opc & 2);
8808 /* Floating point conditional compare
8809 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 3 0
8810 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+
8811 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | cond | 0 1 | Rn | op | nzcv |
8812 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+
8814 static void disas_fp_ccomp(DisasContext *s, uint32_t insn)
8816 unsigned int mos, type, rm, cond, rn, op, nzcv;
8817 TCGLabel *label_continue = NULL;
8818 int size;
8820 mos = extract32(insn, 29, 3);
8821 type = extract32(insn, 22, 2);
8822 rm = extract32(insn, 16, 5);
8823 cond = extract32(insn, 12, 4);
8824 rn = extract32(insn, 5, 5);
8825 op = extract32(insn, 4, 1);
8826 nzcv = extract32(insn, 0, 4);
8828 if (mos) {
8829 unallocated_encoding(s);
8830 return;
8833 switch (type) {
8834 case 0:
8835 size = MO_32;
8836 break;
8837 case 1:
8838 size = MO_64;
8839 break;
8840 case 3:
8841 size = MO_16;
8842 if (dc_isar_feature(aa64_fp16, s)) {
8843 break;
8845 /* fallthru */
8846 default:
8847 unallocated_encoding(s);
8848 return;
8851 if (!fp_access_check(s)) {
8852 return;
8855 if (cond < 0x0e) { /* not always */
8856 TCGLabel *label_match = gen_new_label();
8857 label_continue = gen_new_label();
8858 arm_gen_test_cc(cond, label_match);
8859 /* nomatch: */
8860 gen_set_nzcv(tcg_constant_i64(nzcv << 28));
8861 tcg_gen_br(label_continue);
8862 gen_set_label(label_match);
8865 handle_fp_compare(s, size, rn, rm, false, op);
8867 if (cond < 0x0e) {
8868 gen_set_label(label_continue);
8872 /* Floating-point data-processing (1 source) - half precision */
8873 static void handle_fp_1src_half(DisasContext *s, int opcode, int rd, int rn)
8875 TCGv_ptr fpst = NULL;
8876 TCGv_i32 tcg_op = read_fp_hreg(s, rn);
8877 TCGv_i32 tcg_res = tcg_temp_new_i32();
8879 switch (opcode) {
8880 case 0x0: /* FMOV */
8881 tcg_gen_mov_i32(tcg_res, tcg_op);
8882 break;
8883 case 0x1: /* FABS */
8884 gen_vfp_absh(tcg_res, tcg_op);
8885 break;
8886 case 0x2: /* FNEG */
8887 gen_vfp_negh(tcg_res, tcg_op);
8888 break;
8889 case 0x3: /* FSQRT */
8890 fpst = fpstatus_ptr(FPST_FPCR_F16);
8891 gen_helper_sqrt_f16(tcg_res, tcg_op, fpst);
8892 break;
8893 case 0x8: /* FRINTN */
8894 case 0x9: /* FRINTP */
8895 case 0xa: /* FRINTM */
8896 case 0xb: /* FRINTZ */
8897 case 0xc: /* FRINTA */
8899 TCGv_i32 tcg_rmode;
8901 fpst = fpstatus_ptr(FPST_FPCR_F16);
8902 tcg_rmode = gen_set_rmode(opcode & 7, fpst);
8903 gen_helper_advsimd_rinth(tcg_res, tcg_op, fpst);
8904 gen_restore_rmode(tcg_rmode, fpst);
8905 break;
8907 case 0xe: /* FRINTX */
8908 fpst = fpstatus_ptr(FPST_FPCR_F16);
8909 gen_helper_advsimd_rinth_exact(tcg_res, tcg_op, fpst);
8910 break;
8911 case 0xf: /* FRINTI */
8912 fpst = fpstatus_ptr(FPST_FPCR_F16);
8913 gen_helper_advsimd_rinth(tcg_res, tcg_op, fpst);
8914 break;
8915 default:
8916 g_assert_not_reached();
8919 write_fp_sreg(s, rd, tcg_res);
8922 /* Floating-point data-processing (1 source) - single precision */
8923 static void handle_fp_1src_single(DisasContext *s, int opcode, int rd, int rn)
8925 void (*gen_fpst)(TCGv_i32, TCGv_i32, TCGv_ptr);
8926 TCGv_i32 tcg_op, tcg_res;
8927 TCGv_ptr fpst;
8928 int rmode = -1;
8930 tcg_op = read_fp_sreg(s, rn);
8931 tcg_res = tcg_temp_new_i32();
8933 switch (opcode) {
8934 case 0x0: /* FMOV */
8935 tcg_gen_mov_i32(tcg_res, tcg_op);
8936 goto done;
8937 case 0x1: /* FABS */
8938 gen_vfp_abss(tcg_res, tcg_op);
8939 goto done;
8940 case 0x2: /* FNEG */
8941 gen_vfp_negs(tcg_res, tcg_op);
8942 goto done;
8943 case 0x3: /* FSQRT */
8944 gen_helper_vfp_sqrts(tcg_res, tcg_op, tcg_env);
8945 goto done;
8946 case 0x6: /* BFCVT */
8947 gen_fpst = gen_helper_bfcvt;
8948 break;
8949 case 0x8: /* FRINTN */
8950 case 0x9: /* FRINTP */
8951 case 0xa: /* FRINTM */
8952 case 0xb: /* FRINTZ */
8953 case 0xc: /* FRINTA */
8954 rmode = opcode & 7;
8955 gen_fpst = gen_helper_rints;
8956 break;
8957 case 0xe: /* FRINTX */
8958 gen_fpst = gen_helper_rints_exact;
8959 break;
8960 case 0xf: /* FRINTI */
8961 gen_fpst = gen_helper_rints;
8962 break;
8963 case 0x10: /* FRINT32Z */
8964 rmode = FPROUNDING_ZERO;
8965 gen_fpst = gen_helper_frint32_s;
8966 break;
8967 case 0x11: /* FRINT32X */
8968 gen_fpst = gen_helper_frint32_s;
8969 break;
8970 case 0x12: /* FRINT64Z */
8971 rmode = FPROUNDING_ZERO;
8972 gen_fpst = gen_helper_frint64_s;
8973 break;
8974 case 0x13: /* FRINT64X */
8975 gen_fpst = gen_helper_frint64_s;
8976 break;
8977 default:
8978 g_assert_not_reached();
8981 fpst = fpstatus_ptr(FPST_FPCR);
8982 if (rmode >= 0) {
8983 TCGv_i32 tcg_rmode = gen_set_rmode(rmode, fpst);
8984 gen_fpst(tcg_res, tcg_op, fpst);
8985 gen_restore_rmode(tcg_rmode, fpst);
8986 } else {
8987 gen_fpst(tcg_res, tcg_op, fpst);
8990 done:
8991 write_fp_sreg(s, rd, tcg_res);
8994 /* Floating-point data-processing (1 source) - double precision */
8995 static void handle_fp_1src_double(DisasContext *s, int opcode, int rd, int rn)
8997 void (*gen_fpst)(TCGv_i64, TCGv_i64, TCGv_ptr);
8998 TCGv_i64 tcg_op, tcg_res;
8999 TCGv_ptr fpst;
9000 int rmode = -1;
9002 switch (opcode) {
9003 case 0x0: /* FMOV */
9004 gen_gvec_fn2(s, false, rd, rn, tcg_gen_gvec_mov, 0);
9005 return;
9008 tcg_op = read_fp_dreg(s, rn);
9009 tcg_res = tcg_temp_new_i64();
9011 switch (opcode) {
9012 case 0x1: /* FABS */
9013 gen_vfp_absd(tcg_res, tcg_op);
9014 goto done;
9015 case 0x2: /* FNEG */
9016 gen_vfp_negd(tcg_res, tcg_op);
9017 goto done;
9018 case 0x3: /* FSQRT */
9019 gen_helper_vfp_sqrtd(tcg_res, tcg_op, tcg_env);
9020 goto done;
9021 case 0x8: /* FRINTN */
9022 case 0x9: /* FRINTP */
9023 case 0xa: /* FRINTM */
9024 case 0xb: /* FRINTZ */
9025 case 0xc: /* FRINTA */
9026 rmode = opcode & 7;
9027 gen_fpst = gen_helper_rintd;
9028 break;
9029 case 0xe: /* FRINTX */
9030 gen_fpst = gen_helper_rintd_exact;
9031 break;
9032 case 0xf: /* FRINTI */
9033 gen_fpst = gen_helper_rintd;
9034 break;
9035 case 0x10: /* FRINT32Z */
9036 rmode = FPROUNDING_ZERO;
9037 gen_fpst = gen_helper_frint32_d;
9038 break;
9039 case 0x11: /* FRINT32X */
9040 gen_fpst = gen_helper_frint32_d;
9041 break;
9042 case 0x12: /* FRINT64Z */
9043 rmode = FPROUNDING_ZERO;
9044 gen_fpst = gen_helper_frint64_d;
9045 break;
9046 case 0x13: /* FRINT64X */
9047 gen_fpst = gen_helper_frint64_d;
9048 break;
9049 default:
9050 g_assert_not_reached();
9053 fpst = fpstatus_ptr(FPST_FPCR);
9054 if (rmode >= 0) {
9055 TCGv_i32 tcg_rmode = gen_set_rmode(rmode, fpst);
9056 gen_fpst(tcg_res, tcg_op, fpst);
9057 gen_restore_rmode(tcg_rmode, fpst);
9058 } else {
9059 gen_fpst(tcg_res, tcg_op, fpst);
9062 done:
9063 write_fp_dreg(s, rd, tcg_res);
9066 static void handle_fp_fcvt(DisasContext *s, int opcode,
9067 int rd, int rn, int dtype, int ntype)
9069 switch (ntype) {
9070 case 0x0:
9072 TCGv_i32 tcg_rn = read_fp_sreg(s, rn);
9073 if (dtype == 1) {
9074 /* Single to double */
9075 TCGv_i64 tcg_rd = tcg_temp_new_i64();
9076 gen_helper_vfp_fcvtds(tcg_rd, tcg_rn, tcg_env);
9077 write_fp_dreg(s, rd, tcg_rd);
9078 } else {
9079 /* Single to half */
9080 TCGv_i32 tcg_rd = tcg_temp_new_i32();
9081 TCGv_i32 ahp = get_ahp_flag();
9082 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
9084 gen_helper_vfp_fcvt_f32_to_f16(tcg_rd, tcg_rn, fpst, ahp);
9085 /* write_fp_sreg is OK here because top half of tcg_rd is zero */
9086 write_fp_sreg(s, rd, tcg_rd);
9088 break;
9090 case 0x1:
9092 TCGv_i64 tcg_rn = read_fp_dreg(s, rn);
9093 TCGv_i32 tcg_rd = tcg_temp_new_i32();
9094 if (dtype == 0) {
9095 /* Double to single */
9096 gen_helper_vfp_fcvtsd(tcg_rd, tcg_rn, tcg_env);
9097 } else {
9098 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
9099 TCGv_i32 ahp = get_ahp_flag();
9100 /* Double to half */
9101 gen_helper_vfp_fcvt_f64_to_f16(tcg_rd, tcg_rn, fpst, ahp);
9102 /* write_fp_sreg is OK here because top half of tcg_rd is zero */
9104 write_fp_sreg(s, rd, tcg_rd);
9105 break;
9107 case 0x3:
9109 TCGv_i32 tcg_rn = read_fp_sreg(s, rn);
9110 TCGv_ptr tcg_fpst = fpstatus_ptr(FPST_FPCR);
9111 TCGv_i32 tcg_ahp = get_ahp_flag();
9112 tcg_gen_ext16u_i32(tcg_rn, tcg_rn);
9113 if (dtype == 0) {
9114 /* Half to single */
9115 TCGv_i32 tcg_rd = tcg_temp_new_i32();
9116 gen_helper_vfp_fcvt_f16_to_f32(tcg_rd, tcg_rn, tcg_fpst, tcg_ahp);
9117 write_fp_sreg(s, rd, tcg_rd);
9118 } else {
9119 /* Half to double */
9120 TCGv_i64 tcg_rd = tcg_temp_new_i64();
9121 gen_helper_vfp_fcvt_f16_to_f64(tcg_rd, tcg_rn, tcg_fpst, tcg_ahp);
9122 write_fp_dreg(s, rd, tcg_rd);
9124 break;
9126 default:
9127 g_assert_not_reached();
9131 /* Floating point data-processing (1 source)
9132 * 31 30 29 28 24 23 22 21 20 15 14 10 9 5 4 0
9133 * +---+---+---+-----------+------+---+--------+-----------+------+------+
9134 * | M | 0 | S | 1 1 1 1 0 | type | 1 | opcode | 1 0 0 0 0 | Rn | Rd |
9135 * +---+---+---+-----------+------+---+--------+-----------+------+------+
9137 static void disas_fp_1src(DisasContext *s, uint32_t insn)
9139 int mos = extract32(insn, 29, 3);
9140 int type = extract32(insn, 22, 2);
9141 int opcode = extract32(insn, 15, 6);
9142 int rn = extract32(insn, 5, 5);
9143 int rd = extract32(insn, 0, 5);
9145 if (mos) {
9146 goto do_unallocated;
9149 switch (opcode) {
9150 case 0x4: case 0x5: case 0x7:
9152 /* FCVT between half, single and double precision */
9153 int dtype = extract32(opcode, 0, 2);
9154 if (type == 2 || dtype == type) {
9155 goto do_unallocated;
9157 if (!fp_access_check(s)) {
9158 return;
9161 handle_fp_fcvt(s, opcode, rd, rn, dtype, type);
9162 break;
9165 case 0x10 ... 0x13: /* FRINT{32,64}{X,Z} */
9166 if (type > 1 || !dc_isar_feature(aa64_frint, s)) {
9167 goto do_unallocated;
9169 /* fall through */
9170 case 0x0 ... 0x3:
9171 case 0x8 ... 0xc:
9172 case 0xe ... 0xf:
9173 /* 32-to-32 and 64-to-64 ops */
9174 switch (type) {
9175 case 0:
9176 if (!fp_access_check(s)) {
9177 return;
9179 handle_fp_1src_single(s, opcode, rd, rn);
9180 break;
9181 case 1:
9182 if (!fp_access_check(s)) {
9183 return;
9185 handle_fp_1src_double(s, opcode, rd, rn);
9186 break;
9187 case 3:
9188 if (!dc_isar_feature(aa64_fp16, s)) {
9189 goto do_unallocated;
9192 if (!fp_access_check(s)) {
9193 return;
9195 handle_fp_1src_half(s, opcode, rd, rn);
9196 break;
9197 default:
9198 goto do_unallocated;
9200 break;
9202 case 0x6:
9203 switch (type) {
9204 case 1: /* BFCVT */
9205 if (!dc_isar_feature(aa64_bf16, s)) {
9206 goto do_unallocated;
9208 if (!fp_access_check(s)) {
9209 return;
9211 handle_fp_1src_single(s, opcode, rd, rn);
9212 break;
9213 default:
9214 goto do_unallocated;
9216 break;
9218 default:
9219 do_unallocated:
9220 unallocated_encoding(s);
9221 break;
9225 /* Handle floating point <=> fixed point conversions. Note that we can
9226 * also deal with fp <=> integer conversions as a special case (scale == 64)
9227 * OPTME: consider handling that special case specially or at least skipping
9228 * the call to scalbn in the helpers for zero shifts.
9230 static void handle_fpfpcvt(DisasContext *s, int rd, int rn, int opcode,
9231 bool itof, int rmode, int scale, int sf, int type)
9233 bool is_signed = !(opcode & 1);
9234 TCGv_ptr tcg_fpstatus;
9235 TCGv_i32 tcg_shift, tcg_single;
9236 TCGv_i64 tcg_double;
9238 tcg_fpstatus = fpstatus_ptr(type == 3 ? FPST_FPCR_F16 : FPST_FPCR);
9240 tcg_shift = tcg_constant_i32(64 - scale);
9242 if (itof) {
9243 TCGv_i64 tcg_int = cpu_reg(s, rn);
9244 if (!sf) {
9245 TCGv_i64 tcg_extend = tcg_temp_new_i64();
9247 if (is_signed) {
9248 tcg_gen_ext32s_i64(tcg_extend, tcg_int);
9249 } else {
9250 tcg_gen_ext32u_i64(tcg_extend, tcg_int);
9253 tcg_int = tcg_extend;
9256 switch (type) {
9257 case 1: /* float64 */
9258 tcg_double = tcg_temp_new_i64();
9259 if (is_signed) {
9260 gen_helper_vfp_sqtod(tcg_double, tcg_int,
9261 tcg_shift, tcg_fpstatus);
9262 } else {
9263 gen_helper_vfp_uqtod(tcg_double, tcg_int,
9264 tcg_shift, tcg_fpstatus);
9266 write_fp_dreg(s, rd, tcg_double);
9267 break;
9269 case 0: /* float32 */
9270 tcg_single = tcg_temp_new_i32();
9271 if (is_signed) {
9272 gen_helper_vfp_sqtos(tcg_single, tcg_int,
9273 tcg_shift, tcg_fpstatus);
9274 } else {
9275 gen_helper_vfp_uqtos(tcg_single, tcg_int,
9276 tcg_shift, tcg_fpstatus);
9278 write_fp_sreg(s, rd, tcg_single);
9279 break;
9281 case 3: /* float16 */
9282 tcg_single = tcg_temp_new_i32();
9283 if (is_signed) {
9284 gen_helper_vfp_sqtoh(tcg_single, tcg_int,
9285 tcg_shift, tcg_fpstatus);
9286 } else {
9287 gen_helper_vfp_uqtoh(tcg_single, tcg_int,
9288 tcg_shift, tcg_fpstatus);
9290 write_fp_sreg(s, rd, tcg_single);
9291 break;
9293 default:
9294 g_assert_not_reached();
9296 } else {
9297 TCGv_i64 tcg_int = cpu_reg(s, rd);
9298 TCGv_i32 tcg_rmode;
9300 if (extract32(opcode, 2, 1)) {
9301 /* There are too many rounding modes to all fit into rmode,
9302 * so FCVTA[US] is a special case.
9304 rmode = FPROUNDING_TIEAWAY;
9307 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus);
9309 switch (type) {
9310 case 1: /* float64 */
9311 tcg_double = read_fp_dreg(s, rn);
9312 if (is_signed) {
9313 if (!sf) {
9314 gen_helper_vfp_tosld(tcg_int, tcg_double,
9315 tcg_shift, tcg_fpstatus);
9316 } else {
9317 gen_helper_vfp_tosqd(tcg_int, tcg_double,
9318 tcg_shift, tcg_fpstatus);
9320 } else {
9321 if (!sf) {
9322 gen_helper_vfp_tould(tcg_int, tcg_double,
9323 tcg_shift, tcg_fpstatus);
9324 } else {
9325 gen_helper_vfp_touqd(tcg_int, tcg_double,
9326 tcg_shift, tcg_fpstatus);
9329 if (!sf) {
9330 tcg_gen_ext32u_i64(tcg_int, tcg_int);
9332 break;
9334 case 0: /* float32 */
9335 tcg_single = read_fp_sreg(s, rn);
9336 if (sf) {
9337 if (is_signed) {
9338 gen_helper_vfp_tosqs(tcg_int, tcg_single,
9339 tcg_shift, tcg_fpstatus);
9340 } else {
9341 gen_helper_vfp_touqs(tcg_int, tcg_single,
9342 tcg_shift, tcg_fpstatus);
9344 } else {
9345 TCGv_i32 tcg_dest = tcg_temp_new_i32();
9346 if (is_signed) {
9347 gen_helper_vfp_tosls(tcg_dest, tcg_single,
9348 tcg_shift, tcg_fpstatus);
9349 } else {
9350 gen_helper_vfp_touls(tcg_dest, tcg_single,
9351 tcg_shift, tcg_fpstatus);
9353 tcg_gen_extu_i32_i64(tcg_int, tcg_dest);
9355 break;
9357 case 3: /* float16 */
9358 tcg_single = read_fp_sreg(s, rn);
9359 if (sf) {
9360 if (is_signed) {
9361 gen_helper_vfp_tosqh(tcg_int, tcg_single,
9362 tcg_shift, tcg_fpstatus);
9363 } else {
9364 gen_helper_vfp_touqh(tcg_int, tcg_single,
9365 tcg_shift, tcg_fpstatus);
9367 } else {
9368 TCGv_i32 tcg_dest = tcg_temp_new_i32();
9369 if (is_signed) {
9370 gen_helper_vfp_toslh(tcg_dest, tcg_single,
9371 tcg_shift, tcg_fpstatus);
9372 } else {
9373 gen_helper_vfp_toulh(tcg_dest, tcg_single,
9374 tcg_shift, tcg_fpstatus);
9376 tcg_gen_extu_i32_i64(tcg_int, tcg_dest);
9378 break;
9380 default:
9381 g_assert_not_reached();
9384 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
9388 /* Floating point <-> fixed point conversions
9389 * 31 30 29 28 24 23 22 21 20 19 18 16 15 10 9 5 4 0
9390 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+
9391 * | sf | 0 | S | 1 1 1 1 0 | type | 0 | rmode | opcode | scale | Rn | Rd |
9392 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+
9394 static void disas_fp_fixed_conv(DisasContext *s, uint32_t insn)
9396 int rd = extract32(insn, 0, 5);
9397 int rn = extract32(insn, 5, 5);
9398 int scale = extract32(insn, 10, 6);
9399 int opcode = extract32(insn, 16, 3);
9400 int rmode = extract32(insn, 19, 2);
9401 int type = extract32(insn, 22, 2);
9402 bool sbit = extract32(insn, 29, 1);
9403 bool sf = extract32(insn, 31, 1);
9404 bool itof;
9406 if (sbit || (!sf && scale < 32)) {
9407 unallocated_encoding(s);
9408 return;
9411 switch (type) {
9412 case 0: /* float32 */
9413 case 1: /* float64 */
9414 break;
9415 case 3: /* float16 */
9416 if (dc_isar_feature(aa64_fp16, s)) {
9417 break;
9419 /* fallthru */
9420 default:
9421 unallocated_encoding(s);
9422 return;
9425 switch ((rmode << 3) | opcode) {
9426 case 0x2: /* SCVTF */
9427 case 0x3: /* UCVTF */
9428 itof = true;
9429 break;
9430 case 0x18: /* FCVTZS */
9431 case 0x19: /* FCVTZU */
9432 itof = false;
9433 break;
9434 default:
9435 unallocated_encoding(s);
9436 return;
9439 if (!fp_access_check(s)) {
9440 return;
9443 handle_fpfpcvt(s, rd, rn, opcode, itof, FPROUNDING_ZERO, scale, sf, type);
9446 static void handle_fmov(DisasContext *s, int rd, int rn, int type, bool itof)
9448 /* FMOV: gpr to or from float, double, or top half of quad fp reg,
9449 * without conversion.
9452 if (itof) {
9453 TCGv_i64 tcg_rn = cpu_reg(s, rn);
9454 TCGv_i64 tmp;
9456 switch (type) {
9457 case 0:
9458 /* 32 bit */
9459 tmp = tcg_temp_new_i64();
9460 tcg_gen_ext32u_i64(tmp, tcg_rn);
9461 write_fp_dreg(s, rd, tmp);
9462 break;
9463 case 1:
9464 /* 64 bit */
9465 write_fp_dreg(s, rd, tcg_rn);
9466 break;
9467 case 2:
9468 /* 64 bit to top half. */
9469 tcg_gen_st_i64(tcg_rn, tcg_env, fp_reg_hi_offset(s, rd));
9470 clear_vec_high(s, true, rd);
9471 break;
9472 case 3:
9473 /* 16 bit */
9474 tmp = tcg_temp_new_i64();
9475 tcg_gen_ext16u_i64(tmp, tcg_rn);
9476 write_fp_dreg(s, rd, tmp);
9477 break;
9478 default:
9479 g_assert_not_reached();
9481 } else {
9482 TCGv_i64 tcg_rd = cpu_reg(s, rd);
9484 switch (type) {
9485 case 0:
9486 /* 32 bit */
9487 tcg_gen_ld32u_i64(tcg_rd, tcg_env, fp_reg_offset(s, rn, MO_32));
9488 break;
9489 case 1:
9490 /* 64 bit */
9491 tcg_gen_ld_i64(tcg_rd, tcg_env, fp_reg_offset(s, rn, MO_64));
9492 break;
9493 case 2:
9494 /* 64 bits from top half */
9495 tcg_gen_ld_i64(tcg_rd, tcg_env, fp_reg_hi_offset(s, rn));
9496 break;
9497 case 3:
9498 /* 16 bit */
9499 tcg_gen_ld16u_i64(tcg_rd, tcg_env, fp_reg_offset(s, rn, MO_16));
9500 break;
9501 default:
9502 g_assert_not_reached();
9507 static void handle_fjcvtzs(DisasContext *s, int rd, int rn)
9509 TCGv_i64 t = read_fp_dreg(s, rn);
9510 TCGv_ptr fpstatus = fpstatus_ptr(FPST_FPCR);
9512 gen_helper_fjcvtzs(t, t, fpstatus);
9514 tcg_gen_ext32u_i64(cpu_reg(s, rd), t);
9515 tcg_gen_extrh_i64_i32(cpu_ZF, t);
9516 tcg_gen_movi_i32(cpu_CF, 0);
9517 tcg_gen_movi_i32(cpu_NF, 0);
9518 tcg_gen_movi_i32(cpu_VF, 0);
9521 /* Floating point <-> integer conversions
9522 * 31 30 29 28 24 23 22 21 20 19 18 16 15 10 9 5 4 0
9523 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+
9524 * | sf | 0 | S | 1 1 1 1 0 | type | 1 | rmode | opc | 0 0 0 0 0 0 | Rn | Rd |
9525 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+
9527 static void disas_fp_int_conv(DisasContext *s, uint32_t insn)
9529 int rd = extract32(insn, 0, 5);
9530 int rn = extract32(insn, 5, 5);
9531 int opcode = extract32(insn, 16, 3);
9532 int rmode = extract32(insn, 19, 2);
9533 int type = extract32(insn, 22, 2);
9534 bool sbit = extract32(insn, 29, 1);
9535 bool sf = extract32(insn, 31, 1);
9536 bool itof = false;
9538 if (sbit) {
9539 goto do_unallocated;
9542 switch (opcode) {
9543 case 2: /* SCVTF */
9544 case 3: /* UCVTF */
9545 itof = true;
9546 /* fallthru */
9547 case 4: /* FCVTAS */
9548 case 5: /* FCVTAU */
9549 if (rmode != 0) {
9550 goto do_unallocated;
9552 /* fallthru */
9553 case 0: /* FCVT[NPMZ]S */
9554 case 1: /* FCVT[NPMZ]U */
9555 switch (type) {
9556 case 0: /* float32 */
9557 case 1: /* float64 */
9558 break;
9559 case 3: /* float16 */
9560 if (!dc_isar_feature(aa64_fp16, s)) {
9561 goto do_unallocated;
9563 break;
9564 default:
9565 goto do_unallocated;
9567 if (!fp_access_check(s)) {
9568 return;
9570 handle_fpfpcvt(s, rd, rn, opcode, itof, rmode, 64, sf, type);
9571 break;
9573 default:
9574 switch (sf << 7 | type << 5 | rmode << 3 | opcode) {
9575 case 0b01100110: /* FMOV half <-> 32-bit int */
9576 case 0b01100111:
9577 case 0b11100110: /* FMOV half <-> 64-bit int */
9578 case 0b11100111:
9579 if (!dc_isar_feature(aa64_fp16, s)) {
9580 goto do_unallocated;
9582 /* fallthru */
9583 case 0b00000110: /* FMOV 32-bit */
9584 case 0b00000111:
9585 case 0b10100110: /* FMOV 64-bit */
9586 case 0b10100111:
9587 case 0b11001110: /* FMOV top half of 128-bit */
9588 case 0b11001111:
9589 if (!fp_access_check(s)) {
9590 return;
9592 itof = opcode & 1;
9593 handle_fmov(s, rd, rn, type, itof);
9594 break;
9596 case 0b00111110: /* FJCVTZS */
9597 if (!dc_isar_feature(aa64_jscvt, s)) {
9598 goto do_unallocated;
9599 } else if (fp_access_check(s)) {
9600 handle_fjcvtzs(s, rd, rn);
9602 break;
9604 default:
9605 do_unallocated:
9606 unallocated_encoding(s);
9607 return;
9609 break;
9613 /* FP-specific subcases of table C3-6 (SIMD and FP data processing)
9614 * 31 30 29 28 25 24 0
9615 * +---+---+---+---------+-----------------------------+
9616 * | | 0 | | 1 1 1 1 | |
9617 * +---+---+---+---------+-----------------------------+
9619 static void disas_data_proc_fp(DisasContext *s, uint32_t insn)
9621 if (extract32(insn, 24, 1)) {
9622 unallocated_encoding(s); /* in decodetree */
9623 } else if (extract32(insn, 21, 1) == 0) {
9624 /* Floating point to fixed point conversions */
9625 disas_fp_fixed_conv(s, insn);
9626 } else {
9627 switch (extract32(insn, 10, 2)) {
9628 case 1:
9629 /* Floating point conditional compare */
9630 disas_fp_ccomp(s, insn);
9631 break;
9632 case 2:
9633 /* Floating point data-processing (2 source) */
9634 unallocated_encoding(s); /* in decodetree */
9635 break;
9636 case 3:
9637 /* Floating point conditional select */
9638 unallocated_encoding(s); /* in decodetree */
9639 break;
9640 case 0:
9641 switch (ctz32(extract32(insn, 12, 4))) {
9642 case 0: /* [15:12] == xxx1 */
9643 /* Floating point immediate */
9644 unallocated_encoding(s); /* in decodetree */
9645 break;
9646 case 1: /* [15:12] == xx10 */
9647 /* Floating point compare */
9648 disas_fp_compare(s, insn);
9649 break;
9650 case 2: /* [15:12] == x100 */
9651 /* Floating point data-processing (1 source) */
9652 disas_fp_1src(s, insn);
9653 break;
9654 case 3: /* [15:12] == 1000 */
9655 unallocated_encoding(s);
9656 break;
9657 default: /* [15:12] == 0000 */
9658 /* Floating point <-> integer conversions */
9659 disas_fp_int_conv(s, insn);
9660 break;
9662 break;
9667 /* Common vector code for handling integer to FP conversion */
9668 static void handle_simd_intfp_conv(DisasContext *s, int rd, int rn,
9669 int elements, int is_signed,
9670 int fracbits, int size)
9672 TCGv_ptr tcg_fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
9673 TCGv_i32 tcg_shift = NULL;
9675 MemOp mop = size | (is_signed ? MO_SIGN : 0);
9676 int pass;
9678 if (fracbits || size == MO_64) {
9679 tcg_shift = tcg_constant_i32(fracbits);
9682 if (size == MO_64) {
9683 TCGv_i64 tcg_int64 = tcg_temp_new_i64();
9684 TCGv_i64 tcg_double = tcg_temp_new_i64();
9686 for (pass = 0; pass < elements; pass++) {
9687 read_vec_element(s, tcg_int64, rn, pass, mop);
9689 if (is_signed) {
9690 gen_helper_vfp_sqtod(tcg_double, tcg_int64,
9691 tcg_shift, tcg_fpst);
9692 } else {
9693 gen_helper_vfp_uqtod(tcg_double, tcg_int64,
9694 tcg_shift, tcg_fpst);
9696 if (elements == 1) {
9697 write_fp_dreg(s, rd, tcg_double);
9698 } else {
9699 write_vec_element(s, tcg_double, rd, pass, MO_64);
9702 } else {
9703 TCGv_i32 tcg_int32 = tcg_temp_new_i32();
9704 TCGv_i32 tcg_float = tcg_temp_new_i32();
9706 for (pass = 0; pass < elements; pass++) {
9707 read_vec_element_i32(s, tcg_int32, rn, pass, mop);
9709 switch (size) {
9710 case MO_32:
9711 if (fracbits) {
9712 if (is_signed) {
9713 gen_helper_vfp_sltos(tcg_float, tcg_int32,
9714 tcg_shift, tcg_fpst);
9715 } else {
9716 gen_helper_vfp_ultos(tcg_float, tcg_int32,
9717 tcg_shift, tcg_fpst);
9719 } else {
9720 if (is_signed) {
9721 gen_helper_vfp_sitos(tcg_float, tcg_int32, tcg_fpst);
9722 } else {
9723 gen_helper_vfp_uitos(tcg_float, tcg_int32, tcg_fpst);
9726 break;
9727 case MO_16:
9728 if (fracbits) {
9729 if (is_signed) {
9730 gen_helper_vfp_sltoh(tcg_float, tcg_int32,
9731 tcg_shift, tcg_fpst);
9732 } else {
9733 gen_helper_vfp_ultoh(tcg_float, tcg_int32,
9734 tcg_shift, tcg_fpst);
9736 } else {
9737 if (is_signed) {
9738 gen_helper_vfp_sitoh(tcg_float, tcg_int32, tcg_fpst);
9739 } else {
9740 gen_helper_vfp_uitoh(tcg_float, tcg_int32, tcg_fpst);
9743 break;
9744 default:
9745 g_assert_not_reached();
9748 if (elements == 1) {
9749 write_fp_sreg(s, rd, tcg_float);
9750 } else {
9751 write_vec_element_i32(s, tcg_float, rd, pass, size);
9756 clear_vec_high(s, elements << size == 16, rd);
9759 /* UCVTF/SCVTF - Integer to FP conversion */
9760 static void handle_simd_shift_intfp_conv(DisasContext *s, bool is_scalar,
9761 bool is_q, bool is_u,
9762 int immh, int immb, int opcode,
9763 int rn, int rd)
9765 int size, elements, fracbits;
9766 int immhb = immh << 3 | immb;
9768 if (immh & 8) {
9769 size = MO_64;
9770 if (!is_scalar && !is_q) {
9771 unallocated_encoding(s);
9772 return;
9774 } else if (immh & 4) {
9775 size = MO_32;
9776 } else if (immh & 2) {
9777 size = MO_16;
9778 if (!dc_isar_feature(aa64_fp16, s)) {
9779 unallocated_encoding(s);
9780 return;
9782 } else {
9783 /* immh == 0 would be a failure of the decode logic */
9784 g_assert(immh == 1);
9785 unallocated_encoding(s);
9786 return;
9789 if (is_scalar) {
9790 elements = 1;
9791 } else {
9792 elements = (8 << is_q) >> size;
9794 fracbits = (16 << size) - immhb;
9796 if (!fp_access_check(s)) {
9797 return;
9800 handle_simd_intfp_conv(s, rd, rn, elements, !is_u, fracbits, size);
9803 /* FCVTZS, FVCVTZU - FP to fixedpoint conversion */
9804 static void handle_simd_shift_fpint_conv(DisasContext *s, bool is_scalar,
9805 bool is_q, bool is_u,
9806 int immh, int immb, int rn, int rd)
9808 int immhb = immh << 3 | immb;
9809 int pass, size, fracbits;
9810 TCGv_ptr tcg_fpstatus;
9811 TCGv_i32 tcg_rmode, tcg_shift;
9813 if (immh & 0x8) {
9814 size = MO_64;
9815 if (!is_scalar && !is_q) {
9816 unallocated_encoding(s);
9817 return;
9819 } else if (immh & 0x4) {
9820 size = MO_32;
9821 } else if (immh & 0x2) {
9822 size = MO_16;
9823 if (!dc_isar_feature(aa64_fp16, s)) {
9824 unallocated_encoding(s);
9825 return;
9827 } else {
9828 /* Should have split out AdvSIMD modified immediate earlier. */
9829 assert(immh == 1);
9830 unallocated_encoding(s);
9831 return;
9834 if (!fp_access_check(s)) {
9835 return;
9838 assert(!(is_scalar && is_q));
9840 tcg_fpstatus = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
9841 tcg_rmode = gen_set_rmode(FPROUNDING_ZERO, tcg_fpstatus);
9842 fracbits = (16 << size) - immhb;
9843 tcg_shift = tcg_constant_i32(fracbits);
9845 if (size == MO_64) {
9846 int maxpass = is_scalar ? 1 : 2;
9848 for (pass = 0; pass < maxpass; pass++) {
9849 TCGv_i64 tcg_op = tcg_temp_new_i64();
9851 read_vec_element(s, tcg_op, rn, pass, MO_64);
9852 if (is_u) {
9853 gen_helper_vfp_touqd(tcg_op, tcg_op, tcg_shift, tcg_fpstatus);
9854 } else {
9855 gen_helper_vfp_tosqd(tcg_op, tcg_op, tcg_shift, tcg_fpstatus);
9857 write_vec_element(s, tcg_op, rd, pass, MO_64);
9859 clear_vec_high(s, is_q, rd);
9860 } else {
9861 void (*fn)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_ptr);
9862 int maxpass = is_scalar ? 1 : ((8 << is_q) >> size);
9864 switch (size) {
9865 case MO_16:
9866 if (is_u) {
9867 fn = gen_helper_vfp_touhh;
9868 } else {
9869 fn = gen_helper_vfp_toshh;
9871 break;
9872 case MO_32:
9873 if (is_u) {
9874 fn = gen_helper_vfp_touls;
9875 } else {
9876 fn = gen_helper_vfp_tosls;
9878 break;
9879 default:
9880 g_assert_not_reached();
9883 for (pass = 0; pass < maxpass; pass++) {
9884 TCGv_i32 tcg_op = tcg_temp_new_i32();
9886 read_vec_element_i32(s, tcg_op, rn, pass, size);
9887 fn(tcg_op, tcg_op, tcg_shift, tcg_fpstatus);
9888 if (is_scalar) {
9889 if (size == MO_16 && !is_u) {
9890 tcg_gen_ext16u_i32(tcg_op, tcg_op);
9892 write_fp_sreg(s, rd, tcg_op);
9893 } else {
9894 write_vec_element_i32(s, tcg_op, rd, pass, size);
9897 if (!is_scalar) {
9898 clear_vec_high(s, is_q, rd);
9902 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
9905 /* AdvSIMD scalar shift by immediate
9906 * 31 30 29 28 23 22 19 18 16 15 11 10 9 5 4 0
9907 * +-----+---+-------------+------+------+--------+---+------+------+
9908 * | 0 1 | U | 1 1 1 1 1 0 | immh | immb | opcode | 1 | Rn | Rd |
9909 * +-----+---+-------------+------+------+--------+---+------+------+
9911 * This is the scalar version so it works on a fixed sized registers
9913 static void disas_simd_scalar_shift_imm(DisasContext *s, uint32_t insn)
9915 int rd = extract32(insn, 0, 5);
9916 int rn = extract32(insn, 5, 5);
9917 int opcode = extract32(insn, 11, 5);
9918 int immb = extract32(insn, 16, 3);
9919 int immh = extract32(insn, 19, 4);
9920 bool is_u = extract32(insn, 29, 1);
9922 if (immh == 0) {
9923 unallocated_encoding(s);
9924 return;
9927 switch (opcode) {
9928 case 0x1c: /* SCVTF, UCVTF */
9929 handle_simd_shift_intfp_conv(s, true, false, is_u, immh, immb,
9930 opcode, rn, rd);
9931 break;
9932 case 0x1f: /* FCVTZS, FCVTZU */
9933 handle_simd_shift_fpint_conv(s, true, false, is_u, immh, immb, rn, rd);
9934 break;
9935 default:
9936 case 0x00: /* SSHR / USHR */
9937 case 0x02: /* SSRA / USRA */
9938 case 0x04: /* SRSHR / URSHR */
9939 case 0x06: /* SRSRA / URSRA */
9940 case 0x08: /* SRI */
9941 case 0x0a: /* SHL / SLI */
9942 case 0x0c: /* SQSHLU */
9943 case 0x0e: /* SQSHL, UQSHL */
9944 case 0x10: /* SQSHRUN */
9945 case 0x11: /* SQRSHRUN */
9946 case 0x12: /* SQSHRN, UQSHRN */
9947 case 0x13: /* SQRSHRN, UQRSHRN */
9948 unallocated_encoding(s);
9949 break;
9953 static void handle_2misc_64(DisasContext *s, int opcode, bool u,
9954 TCGv_i64 tcg_rd, TCGv_i64 tcg_rn,
9955 TCGv_i32 tcg_rmode, TCGv_ptr tcg_fpstatus)
9957 /* Handle 64->64 opcodes which are shared between the scalar and
9958 * vector 2-reg-misc groups. We cover every integer opcode where size == 3
9959 * is valid in either group and also the double-precision fp ops.
9960 * The caller only need provide tcg_rmode and tcg_fpstatus if the op
9961 * requires them.
9963 TCGCond cond;
9965 switch (opcode) {
9966 case 0x4: /* CLS, CLZ */
9967 if (u) {
9968 tcg_gen_clzi_i64(tcg_rd, tcg_rn, 64);
9969 } else {
9970 tcg_gen_clrsb_i64(tcg_rd, tcg_rn);
9972 break;
9973 case 0x5: /* NOT */
9974 /* This opcode is shared with CNT and RBIT but we have earlier
9975 * enforced that size == 3 if and only if this is the NOT insn.
9977 tcg_gen_not_i64(tcg_rd, tcg_rn);
9978 break;
9979 case 0x7: /* SQABS, SQNEG */
9980 if (u) {
9981 gen_helper_neon_qneg_s64(tcg_rd, tcg_env, tcg_rn);
9982 } else {
9983 gen_helper_neon_qabs_s64(tcg_rd, tcg_env, tcg_rn);
9985 break;
9986 case 0xa: /* CMLT */
9987 cond = TCG_COND_LT;
9988 do_cmop:
9989 /* 64 bit integer comparison against zero, result is test ? -1 : 0. */
9990 tcg_gen_negsetcond_i64(cond, tcg_rd, tcg_rn, tcg_constant_i64(0));
9991 break;
9992 case 0x8: /* CMGT, CMGE */
9993 cond = u ? TCG_COND_GE : TCG_COND_GT;
9994 goto do_cmop;
9995 case 0x9: /* CMEQ, CMLE */
9996 cond = u ? TCG_COND_LE : TCG_COND_EQ;
9997 goto do_cmop;
9998 case 0xb: /* ABS, NEG */
9999 if (u) {
10000 tcg_gen_neg_i64(tcg_rd, tcg_rn);
10001 } else {
10002 tcg_gen_abs_i64(tcg_rd, tcg_rn);
10004 break;
10005 case 0x2f: /* FABS */
10006 gen_vfp_absd(tcg_rd, tcg_rn);
10007 break;
10008 case 0x6f: /* FNEG */
10009 gen_vfp_negd(tcg_rd, tcg_rn);
10010 break;
10011 case 0x7f: /* FSQRT */
10012 gen_helper_vfp_sqrtd(tcg_rd, tcg_rn, tcg_env);
10013 break;
10014 case 0x1a: /* FCVTNS */
10015 case 0x1b: /* FCVTMS */
10016 case 0x1c: /* FCVTAS */
10017 case 0x3a: /* FCVTPS */
10018 case 0x3b: /* FCVTZS */
10019 gen_helper_vfp_tosqd(tcg_rd, tcg_rn, tcg_constant_i32(0), tcg_fpstatus);
10020 break;
10021 case 0x5a: /* FCVTNU */
10022 case 0x5b: /* FCVTMU */
10023 case 0x5c: /* FCVTAU */
10024 case 0x7a: /* FCVTPU */
10025 case 0x7b: /* FCVTZU */
10026 gen_helper_vfp_touqd(tcg_rd, tcg_rn, tcg_constant_i32(0), tcg_fpstatus);
10027 break;
10028 case 0x18: /* FRINTN */
10029 case 0x19: /* FRINTM */
10030 case 0x38: /* FRINTP */
10031 case 0x39: /* FRINTZ */
10032 case 0x58: /* FRINTA */
10033 case 0x79: /* FRINTI */
10034 gen_helper_rintd(tcg_rd, tcg_rn, tcg_fpstatus);
10035 break;
10036 case 0x59: /* FRINTX */
10037 gen_helper_rintd_exact(tcg_rd, tcg_rn, tcg_fpstatus);
10038 break;
10039 case 0x1e: /* FRINT32Z */
10040 case 0x5e: /* FRINT32X */
10041 gen_helper_frint32_d(tcg_rd, tcg_rn, tcg_fpstatus);
10042 break;
10043 case 0x1f: /* FRINT64Z */
10044 case 0x5f: /* FRINT64X */
10045 gen_helper_frint64_d(tcg_rd, tcg_rn, tcg_fpstatus);
10046 break;
10047 default:
10048 g_assert_not_reached();
10052 static void handle_2misc_fcmp_zero(DisasContext *s, int opcode,
10053 bool is_scalar, bool is_u, bool is_q,
10054 int size, int rn, int rd)
10056 bool is_double = (size == MO_64);
10057 TCGv_ptr fpst;
10059 if (!fp_access_check(s)) {
10060 return;
10063 fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
10065 if (is_double) {
10066 TCGv_i64 tcg_op = tcg_temp_new_i64();
10067 TCGv_i64 tcg_zero = tcg_constant_i64(0);
10068 TCGv_i64 tcg_res = tcg_temp_new_i64();
10069 NeonGenTwoDoubleOpFn *genfn;
10070 bool swap = false;
10071 int pass;
10073 switch (opcode) {
10074 case 0x2e: /* FCMLT (zero) */
10075 swap = true;
10076 /* fallthrough */
10077 case 0x2c: /* FCMGT (zero) */
10078 genfn = gen_helper_neon_cgt_f64;
10079 break;
10080 case 0x2d: /* FCMEQ (zero) */
10081 genfn = gen_helper_neon_ceq_f64;
10082 break;
10083 case 0x6d: /* FCMLE (zero) */
10084 swap = true;
10085 /* fall through */
10086 case 0x6c: /* FCMGE (zero) */
10087 genfn = gen_helper_neon_cge_f64;
10088 break;
10089 default:
10090 g_assert_not_reached();
10093 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
10094 read_vec_element(s, tcg_op, rn, pass, MO_64);
10095 if (swap) {
10096 genfn(tcg_res, tcg_zero, tcg_op, fpst);
10097 } else {
10098 genfn(tcg_res, tcg_op, tcg_zero, fpst);
10100 write_vec_element(s, tcg_res, rd, pass, MO_64);
10103 clear_vec_high(s, !is_scalar, rd);
10104 } else {
10105 TCGv_i32 tcg_op = tcg_temp_new_i32();
10106 TCGv_i32 tcg_zero = tcg_constant_i32(0);
10107 TCGv_i32 tcg_res = tcg_temp_new_i32();
10108 NeonGenTwoSingleOpFn *genfn;
10109 bool swap = false;
10110 int pass, maxpasses;
10112 if (size == MO_16) {
10113 switch (opcode) {
10114 case 0x2e: /* FCMLT (zero) */
10115 swap = true;
10116 /* fall through */
10117 case 0x2c: /* FCMGT (zero) */
10118 genfn = gen_helper_advsimd_cgt_f16;
10119 break;
10120 case 0x2d: /* FCMEQ (zero) */
10121 genfn = gen_helper_advsimd_ceq_f16;
10122 break;
10123 case 0x6d: /* FCMLE (zero) */
10124 swap = true;
10125 /* fall through */
10126 case 0x6c: /* FCMGE (zero) */
10127 genfn = gen_helper_advsimd_cge_f16;
10128 break;
10129 default:
10130 g_assert_not_reached();
10132 } else {
10133 switch (opcode) {
10134 case 0x2e: /* FCMLT (zero) */
10135 swap = true;
10136 /* fall through */
10137 case 0x2c: /* FCMGT (zero) */
10138 genfn = gen_helper_neon_cgt_f32;
10139 break;
10140 case 0x2d: /* FCMEQ (zero) */
10141 genfn = gen_helper_neon_ceq_f32;
10142 break;
10143 case 0x6d: /* FCMLE (zero) */
10144 swap = true;
10145 /* fall through */
10146 case 0x6c: /* FCMGE (zero) */
10147 genfn = gen_helper_neon_cge_f32;
10148 break;
10149 default:
10150 g_assert_not_reached();
10154 if (is_scalar) {
10155 maxpasses = 1;
10156 } else {
10157 int vector_size = 8 << is_q;
10158 maxpasses = vector_size >> size;
10161 for (pass = 0; pass < maxpasses; pass++) {
10162 read_vec_element_i32(s, tcg_op, rn, pass, size);
10163 if (swap) {
10164 genfn(tcg_res, tcg_zero, tcg_op, fpst);
10165 } else {
10166 genfn(tcg_res, tcg_op, tcg_zero, fpst);
10168 if (is_scalar) {
10169 write_fp_sreg(s, rd, tcg_res);
10170 } else {
10171 write_vec_element_i32(s, tcg_res, rd, pass, size);
10175 if (!is_scalar) {
10176 clear_vec_high(s, is_q, rd);
10181 static void handle_2misc_reciprocal(DisasContext *s, int opcode,
10182 bool is_scalar, bool is_u, bool is_q,
10183 int size, int rn, int rd)
10185 bool is_double = (size == 3);
10186 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
10188 if (is_double) {
10189 TCGv_i64 tcg_op = tcg_temp_new_i64();
10190 TCGv_i64 tcg_res = tcg_temp_new_i64();
10191 int pass;
10193 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
10194 read_vec_element(s, tcg_op, rn, pass, MO_64);
10195 switch (opcode) {
10196 case 0x3d: /* FRECPE */
10197 gen_helper_recpe_f64(tcg_res, tcg_op, fpst);
10198 break;
10199 case 0x3f: /* FRECPX */
10200 gen_helper_frecpx_f64(tcg_res, tcg_op, fpst);
10201 break;
10202 case 0x7d: /* FRSQRTE */
10203 gen_helper_rsqrte_f64(tcg_res, tcg_op, fpst);
10204 break;
10205 default:
10206 g_assert_not_reached();
10208 write_vec_element(s, tcg_res, rd, pass, MO_64);
10210 clear_vec_high(s, !is_scalar, rd);
10211 } else {
10212 TCGv_i32 tcg_op = tcg_temp_new_i32();
10213 TCGv_i32 tcg_res = tcg_temp_new_i32();
10214 int pass, maxpasses;
10216 if (is_scalar) {
10217 maxpasses = 1;
10218 } else {
10219 maxpasses = is_q ? 4 : 2;
10222 for (pass = 0; pass < maxpasses; pass++) {
10223 read_vec_element_i32(s, tcg_op, rn, pass, MO_32);
10225 switch (opcode) {
10226 case 0x3c: /* URECPE */
10227 gen_helper_recpe_u32(tcg_res, tcg_op);
10228 break;
10229 case 0x3d: /* FRECPE */
10230 gen_helper_recpe_f32(tcg_res, tcg_op, fpst);
10231 break;
10232 case 0x3f: /* FRECPX */
10233 gen_helper_frecpx_f32(tcg_res, tcg_op, fpst);
10234 break;
10235 case 0x7d: /* FRSQRTE */
10236 gen_helper_rsqrte_f32(tcg_res, tcg_op, fpst);
10237 break;
10238 default:
10239 g_assert_not_reached();
10242 if (is_scalar) {
10243 write_fp_sreg(s, rd, tcg_res);
10244 } else {
10245 write_vec_element_i32(s, tcg_res, rd, pass, MO_32);
10248 if (!is_scalar) {
10249 clear_vec_high(s, is_q, rd);
10254 static void handle_2misc_narrow(DisasContext *s, bool scalar,
10255 int opcode, bool u, bool is_q,
10256 int size, int rn, int rd)
10258 /* Handle 2-reg-misc ops which are narrowing (so each 2*size element
10259 * in the source becomes a size element in the destination).
10261 int pass;
10262 TCGv_i64 tcg_res[2];
10263 int destelt = is_q ? 2 : 0;
10264 int passes = scalar ? 1 : 2;
10266 if (scalar) {
10267 tcg_res[1] = tcg_constant_i64(0);
10270 for (pass = 0; pass < passes; pass++) {
10271 TCGv_i64 tcg_op = tcg_temp_new_i64();
10272 NeonGenOne64OpFn *genfn = NULL;
10273 NeonGenOne64OpEnvFn *genenvfn = NULL;
10275 if (scalar) {
10276 read_vec_element(s, tcg_op, rn, pass, size + 1);
10277 } else {
10278 read_vec_element(s, tcg_op, rn, pass, MO_64);
10280 tcg_res[pass] = tcg_temp_new_i64();
10282 switch (opcode) {
10283 case 0x12: /* XTN, SQXTUN */
10285 static NeonGenOne64OpFn * const xtnfns[3] = {
10286 gen_helper_neon_narrow_u8,
10287 gen_helper_neon_narrow_u16,
10288 tcg_gen_ext32u_i64,
10290 static NeonGenOne64OpEnvFn * const sqxtunfns[3] = {
10291 gen_helper_neon_unarrow_sat8,
10292 gen_helper_neon_unarrow_sat16,
10293 gen_helper_neon_unarrow_sat32,
10295 if (u) {
10296 genenvfn = sqxtunfns[size];
10297 } else {
10298 genfn = xtnfns[size];
10300 break;
10302 case 0x14: /* SQXTN, UQXTN */
10304 static NeonGenOne64OpEnvFn * const fns[3][2] = {
10305 { gen_helper_neon_narrow_sat_s8,
10306 gen_helper_neon_narrow_sat_u8 },
10307 { gen_helper_neon_narrow_sat_s16,
10308 gen_helper_neon_narrow_sat_u16 },
10309 { gen_helper_neon_narrow_sat_s32,
10310 gen_helper_neon_narrow_sat_u32 },
10312 genenvfn = fns[size][u];
10313 break;
10315 case 0x16: /* FCVTN, FCVTN2 */
10316 /* 32 bit to 16 bit or 64 bit to 32 bit float conversion */
10317 if (size == 2) {
10318 TCGv_i32 tmp = tcg_temp_new_i32();
10319 gen_helper_vfp_fcvtsd(tmp, tcg_op, tcg_env);
10320 tcg_gen_extu_i32_i64(tcg_res[pass], tmp);
10321 } else {
10322 TCGv_i32 tcg_lo = tcg_temp_new_i32();
10323 TCGv_i32 tcg_hi = tcg_temp_new_i32();
10324 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
10325 TCGv_i32 ahp = get_ahp_flag();
10327 tcg_gen_extr_i64_i32(tcg_lo, tcg_hi, tcg_op);
10328 gen_helper_vfp_fcvt_f32_to_f16(tcg_lo, tcg_lo, fpst, ahp);
10329 gen_helper_vfp_fcvt_f32_to_f16(tcg_hi, tcg_hi, fpst, ahp);
10330 tcg_gen_deposit_i32(tcg_lo, tcg_lo, tcg_hi, 16, 16);
10331 tcg_gen_extu_i32_i64(tcg_res[pass], tcg_lo);
10333 break;
10334 case 0x36: /* BFCVTN, BFCVTN2 */
10336 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
10337 TCGv_i32 tmp = tcg_temp_new_i32();
10338 gen_helper_bfcvt_pair(tmp, tcg_op, fpst);
10339 tcg_gen_extu_i32_i64(tcg_res[pass], tmp);
10341 break;
10342 case 0x56: /* FCVTXN, FCVTXN2 */
10345 * 64 bit to 32 bit float conversion
10346 * with von Neumann rounding (round to odd)
10348 TCGv_i32 tmp = tcg_temp_new_i32();
10349 assert(size == 2);
10350 gen_helper_fcvtx_f64_to_f32(tmp, tcg_op, tcg_env);
10351 tcg_gen_extu_i32_i64(tcg_res[pass], tmp);
10353 break;
10354 default:
10355 g_assert_not_reached();
10358 if (genfn) {
10359 genfn(tcg_res[pass], tcg_op);
10360 } else if (genenvfn) {
10361 genenvfn(tcg_res[pass], tcg_env, tcg_op);
10365 for (pass = 0; pass < 2; pass++) {
10366 write_vec_element(s, tcg_res[pass], rd, destelt + pass, MO_32);
10368 clear_vec_high(s, is_q, rd);
10371 /* AdvSIMD scalar two reg misc
10372 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0
10373 * +-----+---+-----------+------+-----------+--------+-----+------+------+
10374 * | 0 1 | U | 1 1 1 1 0 | size | 1 0 0 0 0 | opcode | 1 0 | Rn | Rd |
10375 * +-----+---+-----------+------+-----------+--------+-----+------+------+
10377 static void disas_simd_scalar_two_reg_misc(DisasContext *s, uint32_t insn)
10379 int rd = extract32(insn, 0, 5);
10380 int rn = extract32(insn, 5, 5);
10381 int opcode = extract32(insn, 12, 5);
10382 int size = extract32(insn, 22, 2);
10383 bool u = extract32(insn, 29, 1);
10384 bool is_fcvt = false;
10385 int rmode;
10386 TCGv_i32 tcg_rmode;
10387 TCGv_ptr tcg_fpstatus;
10389 switch (opcode) {
10390 case 0x7: /* SQABS / SQNEG */
10391 break;
10392 case 0xa: /* CMLT */
10393 if (u) {
10394 unallocated_encoding(s);
10395 return;
10397 /* fall through */
10398 case 0x8: /* CMGT, CMGE */
10399 case 0x9: /* CMEQ, CMLE */
10400 case 0xb: /* ABS, NEG */
10401 if (size != 3) {
10402 unallocated_encoding(s);
10403 return;
10405 break;
10406 case 0x12: /* SQXTUN */
10407 if (!u) {
10408 unallocated_encoding(s);
10409 return;
10411 /* fall through */
10412 case 0x14: /* SQXTN, UQXTN */
10413 if (size == 3) {
10414 unallocated_encoding(s);
10415 return;
10417 if (!fp_access_check(s)) {
10418 return;
10420 handle_2misc_narrow(s, true, opcode, u, false, size, rn, rd);
10421 return;
10422 case 0xc ... 0xf:
10423 case 0x16 ... 0x1d:
10424 case 0x1f:
10425 /* Floating point: U, size[1] and opcode indicate operation;
10426 * size[0] indicates single or double precision.
10428 opcode |= (extract32(size, 1, 1) << 5) | (u << 6);
10429 size = extract32(size, 0, 1) ? 3 : 2;
10430 switch (opcode) {
10431 case 0x2c: /* FCMGT (zero) */
10432 case 0x2d: /* FCMEQ (zero) */
10433 case 0x2e: /* FCMLT (zero) */
10434 case 0x6c: /* FCMGE (zero) */
10435 case 0x6d: /* FCMLE (zero) */
10436 handle_2misc_fcmp_zero(s, opcode, true, u, true, size, rn, rd);
10437 return;
10438 case 0x1d: /* SCVTF */
10439 case 0x5d: /* UCVTF */
10441 bool is_signed = (opcode == 0x1d);
10442 if (!fp_access_check(s)) {
10443 return;
10445 handle_simd_intfp_conv(s, rd, rn, 1, is_signed, 0, size);
10446 return;
10448 case 0x3d: /* FRECPE */
10449 case 0x3f: /* FRECPX */
10450 case 0x7d: /* FRSQRTE */
10451 if (!fp_access_check(s)) {
10452 return;
10454 handle_2misc_reciprocal(s, opcode, true, u, true, size, rn, rd);
10455 return;
10456 case 0x1a: /* FCVTNS */
10457 case 0x1b: /* FCVTMS */
10458 case 0x3a: /* FCVTPS */
10459 case 0x3b: /* FCVTZS */
10460 case 0x5a: /* FCVTNU */
10461 case 0x5b: /* FCVTMU */
10462 case 0x7a: /* FCVTPU */
10463 case 0x7b: /* FCVTZU */
10464 is_fcvt = true;
10465 rmode = extract32(opcode, 5, 1) | (extract32(opcode, 0, 1) << 1);
10466 break;
10467 case 0x1c: /* FCVTAS */
10468 case 0x5c: /* FCVTAU */
10469 /* TIEAWAY doesn't fit in the usual rounding mode encoding */
10470 is_fcvt = true;
10471 rmode = FPROUNDING_TIEAWAY;
10472 break;
10473 case 0x56: /* FCVTXN, FCVTXN2 */
10474 if (size == 2) {
10475 unallocated_encoding(s);
10476 return;
10478 if (!fp_access_check(s)) {
10479 return;
10481 handle_2misc_narrow(s, true, opcode, u, false, size - 1, rn, rd);
10482 return;
10483 default:
10484 unallocated_encoding(s);
10485 return;
10487 break;
10488 default:
10489 case 0x3: /* USQADD / SUQADD */
10490 unallocated_encoding(s);
10491 return;
10494 if (!fp_access_check(s)) {
10495 return;
10498 if (is_fcvt) {
10499 tcg_fpstatus = fpstatus_ptr(FPST_FPCR);
10500 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus);
10501 } else {
10502 tcg_fpstatus = NULL;
10503 tcg_rmode = NULL;
10506 if (size == 3) {
10507 TCGv_i64 tcg_rn = read_fp_dreg(s, rn);
10508 TCGv_i64 tcg_rd = tcg_temp_new_i64();
10510 handle_2misc_64(s, opcode, u, tcg_rd, tcg_rn, tcg_rmode, tcg_fpstatus);
10511 write_fp_dreg(s, rd, tcg_rd);
10512 } else {
10513 TCGv_i32 tcg_rn = tcg_temp_new_i32();
10514 TCGv_i32 tcg_rd = tcg_temp_new_i32();
10516 read_vec_element_i32(s, tcg_rn, rn, 0, size);
10518 switch (opcode) {
10519 case 0x7: /* SQABS, SQNEG */
10521 NeonGenOneOpEnvFn *genfn;
10522 static NeonGenOneOpEnvFn * const fns[3][2] = {
10523 { gen_helper_neon_qabs_s8, gen_helper_neon_qneg_s8 },
10524 { gen_helper_neon_qabs_s16, gen_helper_neon_qneg_s16 },
10525 { gen_helper_neon_qabs_s32, gen_helper_neon_qneg_s32 },
10527 genfn = fns[size][u];
10528 genfn(tcg_rd, tcg_env, tcg_rn);
10529 break;
10531 case 0x1a: /* FCVTNS */
10532 case 0x1b: /* FCVTMS */
10533 case 0x1c: /* FCVTAS */
10534 case 0x3a: /* FCVTPS */
10535 case 0x3b: /* FCVTZS */
10536 gen_helper_vfp_tosls(tcg_rd, tcg_rn, tcg_constant_i32(0),
10537 tcg_fpstatus);
10538 break;
10539 case 0x5a: /* FCVTNU */
10540 case 0x5b: /* FCVTMU */
10541 case 0x5c: /* FCVTAU */
10542 case 0x7a: /* FCVTPU */
10543 case 0x7b: /* FCVTZU */
10544 gen_helper_vfp_touls(tcg_rd, tcg_rn, tcg_constant_i32(0),
10545 tcg_fpstatus);
10546 break;
10547 default:
10548 g_assert_not_reached();
10551 write_fp_sreg(s, rd, tcg_rd);
10554 if (is_fcvt) {
10555 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
10559 /* AdvSIMD shift by immediate
10560 * 31 30 29 28 23 22 19 18 16 15 11 10 9 5 4 0
10561 * +---+---+---+-------------+------+------+--------+---+------+------+
10562 * | 0 | Q | U | 0 1 1 1 1 0 | immh | immb | opcode | 1 | Rn | Rd |
10563 * +---+---+---+-------------+------+------+--------+---+------+------+
10565 static void disas_simd_shift_imm(DisasContext *s, uint32_t insn)
10567 int rd = extract32(insn, 0, 5);
10568 int rn = extract32(insn, 5, 5);
10569 int opcode = extract32(insn, 11, 5);
10570 int immb = extract32(insn, 16, 3);
10571 int immh = extract32(insn, 19, 4);
10572 bool is_u = extract32(insn, 29, 1);
10573 bool is_q = extract32(insn, 30, 1);
10575 if (immh == 0) {
10576 unallocated_encoding(s);
10577 return;
10580 switch (opcode) {
10581 case 0x1c: /* SCVTF / UCVTF */
10582 handle_simd_shift_intfp_conv(s, false, is_q, is_u, immh, immb,
10583 opcode, rn, rd);
10584 break;
10585 case 0x1f: /* FCVTZS/ FCVTZU */
10586 handle_simd_shift_fpint_conv(s, false, is_q, is_u, immh, immb, rn, rd);
10587 return;
10588 default:
10589 case 0x00: /* SSHR / USHR */
10590 case 0x02: /* SSRA / USRA (accumulate) */
10591 case 0x04: /* SRSHR / URSHR (rounding) */
10592 case 0x06: /* SRSRA / URSRA (accum + rounding) */
10593 case 0x08: /* SRI */
10594 case 0x0a: /* SHL / SLI */
10595 case 0x0c: /* SQSHLU */
10596 case 0x0e: /* SQSHL, UQSHL */
10597 case 0x10: /* SHRN / SQSHRUN */
10598 case 0x11: /* RSHRN / SQRSHRUN */
10599 case 0x12: /* SQSHRN / UQSHRN */
10600 case 0x13: /* SQRSHRN / UQRSHRN */
10601 case 0x14: /* SSHLL / USHLL */
10602 unallocated_encoding(s);
10603 return;
10607 static void handle_2misc_widening(DisasContext *s, int opcode, bool is_q,
10608 int size, int rn, int rd)
10610 /* Handle 2-reg-misc ops which are widening (so each size element
10611 * in the source becomes a 2*size element in the destination.
10612 * The only instruction like this is FCVTL.
10614 int pass;
10616 if (size == 3) {
10617 /* 32 -> 64 bit fp conversion */
10618 TCGv_i64 tcg_res[2];
10619 int srcelt = is_q ? 2 : 0;
10621 for (pass = 0; pass < 2; pass++) {
10622 TCGv_i32 tcg_op = tcg_temp_new_i32();
10623 tcg_res[pass] = tcg_temp_new_i64();
10625 read_vec_element_i32(s, tcg_op, rn, srcelt + pass, MO_32);
10626 gen_helper_vfp_fcvtds(tcg_res[pass], tcg_op, tcg_env);
10628 for (pass = 0; pass < 2; pass++) {
10629 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
10631 } else {
10632 /* 16 -> 32 bit fp conversion */
10633 int srcelt = is_q ? 4 : 0;
10634 TCGv_i32 tcg_res[4];
10635 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
10636 TCGv_i32 ahp = get_ahp_flag();
10638 for (pass = 0; pass < 4; pass++) {
10639 tcg_res[pass] = tcg_temp_new_i32();
10641 read_vec_element_i32(s, tcg_res[pass], rn, srcelt + pass, MO_16);
10642 gen_helper_vfp_fcvt_f16_to_f32(tcg_res[pass], tcg_res[pass],
10643 fpst, ahp);
10645 for (pass = 0; pass < 4; pass++) {
10646 write_vec_element_i32(s, tcg_res[pass], rd, pass, MO_32);
10651 static void handle_rev(DisasContext *s, int opcode, bool u,
10652 bool is_q, int size, int rn, int rd)
10654 int op = (opcode << 1) | u;
10655 int opsz = op + size;
10656 int grp_size = 3 - opsz;
10657 int dsize = is_q ? 128 : 64;
10658 int i;
10660 if (opsz >= 3) {
10661 unallocated_encoding(s);
10662 return;
10665 if (!fp_access_check(s)) {
10666 return;
10669 if (size == 0) {
10670 /* Special case bytes, use bswap op on each group of elements */
10671 int groups = dsize / (8 << grp_size);
10673 for (i = 0; i < groups; i++) {
10674 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
10676 read_vec_element(s, tcg_tmp, rn, i, grp_size);
10677 switch (grp_size) {
10678 case MO_16:
10679 tcg_gen_bswap16_i64(tcg_tmp, tcg_tmp, TCG_BSWAP_IZ);
10680 break;
10681 case MO_32:
10682 tcg_gen_bswap32_i64(tcg_tmp, tcg_tmp, TCG_BSWAP_IZ);
10683 break;
10684 case MO_64:
10685 tcg_gen_bswap64_i64(tcg_tmp, tcg_tmp);
10686 break;
10687 default:
10688 g_assert_not_reached();
10690 write_vec_element(s, tcg_tmp, rd, i, grp_size);
10692 clear_vec_high(s, is_q, rd);
10693 } else {
10694 int revmask = (1 << grp_size) - 1;
10695 int esize = 8 << size;
10696 int elements = dsize / esize;
10697 TCGv_i64 tcg_rn = tcg_temp_new_i64();
10698 TCGv_i64 tcg_rd[2];
10700 for (i = 0; i < 2; i++) {
10701 tcg_rd[i] = tcg_temp_new_i64();
10702 tcg_gen_movi_i64(tcg_rd[i], 0);
10705 for (i = 0; i < elements; i++) {
10706 int e_rev = (i & 0xf) ^ revmask;
10707 int w = (e_rev * esize) / 64;
10708 int o = (e_rev * esize) % 64;
10710 read_vec_element(s, tcg_rn, rn, i, size);
10711 tcg_gen_deposit_i64(tcg_rd[w], tcg_rd[w], tcg_rn, o, esize);
10714 for (i = 0; i < 2; i++) {
10715 write_vec_element(s, tcg_rd[i], rd, i, MO_64);
10717 clear_vec_high(s, true, rd);
10721 static void handle_2misc_pairwise(DisasContext *s, int opcode, bool u,
10722 bool is_q, int size, int rn, int rd)
10724 /* Implement the pairwise operations from 2-misc:
10725 * SADDLP, UADDLP, SADALP, UADALP.
10726 * These all add pairs of elements in the input to produce a
10727 * double-width result element in the output (possibly accumulating).
10729 bool accum = (opcode == 0x6);
10730 int maxpass = is_q ? 2 : 1;
10731 int pass;
10732 TCGv_i64 tcg_res[2];
10734 if (size == 2) {
10735 /* 32 + 32 -> 64 op */
10736 MemOp memop = size + (u ? 0 : MO_SIGN);
10738 for (pass = 0; pass < maxpass; pass++) {
10739 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
10740 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
10742 tcg_res[pass] = tcg_temp_new_i64();
10744 read_vec_element(s, tcg_op1, rn, pass * 2, memop);
10745 read_vec_element(s, tcg_op2, rn, pass * 2 + 1, memop);
10746 tcg_gen_add_i64(tcg_res[pass], tcg_op1, tcg_op2);
10747 if (accum) {
10748 read_vec_element(s, tcg_op1, rd, pass, MO_64);
10749 tcg_gen_add_i64(tcg_res[pass], tcg_res[pass], tcg_op1);
10752 } else {
10753 for (pass = 0; pass < maxpass; pass++) {
10754 TCGv_i64 tcg_op = tcg_temp_new_i64();
10755 NeonGenOne64OpFn *genfn;
10756 static NeonGenOne64OpFn * const fns[2][2] = {
10757 { gen_helper_neon_addlp_s8, gen_helper_neon_addlp_u8 },
10758 { gen_helper_neon_addlp_s16, gen_helper_neon_addlp_u16 },
10761 genfn = fns[size][u];
10763 tcg_res[pass] = tcg_temp_new_i64();
10765 read_vec_element(s, tcg_op, rn, pass, MO_64);
10766 genfn(tcg_res[pass], tcg_op);
10768 if (accum) {
10769 read_vec_element(s, tcg_op, rd, pass, MO_64);
10770 if (size == 0) {
10771 gen_helper_neon_addl_u16(tcg_res[pass],
10772 tcg_res[pass], tcg_op);
10773 } else {
10774 gen_helper_neon_addl_u32(tcg_res[pass],
10775 tcg_res[pass], tcg_op);
10780 if (!is_q) {
10781 tcg_res[1] = tcg_constant_i64(0);
10783 for (pass = 0; pass < 2; pass++) {
10784 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
10788 static void handle_shll(DisasContext *s, bool is_q, int size, int rn, int rd)
10790 /* Implement SHLL and SHLL2 */
10791 int pass;
10792 int part = is_q ? 2 : 0;
10793 TCGv_i64 tcg_res[2];
10795 for (pass = 0; pass < 2; pass++) {
10796 static NeonGenWidenFn * const widenfns[3] = {
10797 gen_helper_neon_widen_u8,
10798 gen_helper_neon_widen_u16,
10799 tcg_gen_extu_i32_i64,
10801 NeonGenWidenFn *widenfn = widenfns[size];
10802 TCGv_i32 tcg_op = tcg_temp_new_i32();
10804 read_vec_element_i32(s, tcg_op, rn, part + pass, MO_32);
10805 tcg_res[pass] = tcg_temp_new_i64();
10806 widenfn(tcg_res[pass], tcg_op);
10807 tcg_gen_shli_i64(tcg_res[pass], tcg_res[pass], 8 << size);
10810 for (pass = 0; pass < 2; pass++) {
10811 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
10815 /* AdvSIMD two reg misc
10816 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0
10817 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
10818 * | 0 | Q | U | 0 1 1 1 0 | size | 1 0 0 0 0 | opcode | 1 0 | Rn | Rd |
10819 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
10821 static void disas_simd_two_reg_misc(DisasContext *s, uint32_t insn)
10823 int size = extract32(insn, 22, 2);
10824 int opcode = extract32(insn, 12, 5);
10825 bool u = extract32(insn, 29, 1);
10826 bool is_q = extract32(insn, 30, 1);
10827 int rn = extract32(insn, 5, 5);
10828 int rd = extract32(insn, 0, 5);
10829 bool need_fpstatus = false;
10830 int rmode = -1;
10831 TCGv_i32 tcg_rmode;
10832 TCGv_ptr tcg_fpstatus;
10834 switch (opcode) {
10835 case 0x0: /* REV64, REV32 */
10836 case 0x1: /* REV16 */
10837 handle_rev(s, opcode, u, is_q, size, rn, rd);
10838 return;
10839 case 0x5: /* CNT, NOT, RBIT */
10840 if (u && size == 0) {
10841 /* NOT */
10842 break;
10843 } else if (u && size == 1) {
10844 /* RBIT */
10845 break;
10846 } else if (!u && size == 0) {
10847 /* CNT */
10848 break;
10850 unallocated_encoding(s);
10851 return;
10852 case 0x12: /* XTN, XTN2, SQXTUN, SQXTUN2 */
10853 case 0x14: /* SQXTN, SQXTN2, UQXTN, UQXTN2 */
10854 if (size == 3) {
10855 unallocated_encoding(s);
10856 return;
10858 if (!fp_access_check(s)) {
10859 return;
10862 handle_2misc_narrow(s, false, opcode, u, is_q, size, rn, rd);
10863 return;
10864 case 0x4: /* CLS, CLZ */
10865 if (size == 3) {
10866 unallocated_encoding(s);
10867 return;
10869 break;
10870 case 0x2: /* SADDLP, UADDLP */
10871 case 0x6: /* SADALP, UADALP */
10872 if (size == 3) {
10873 unallocated_encoding(s);
10874 return;
10876 if (!fp_access_check(s)) {
10877 return;
10879 handle_2misc_pairwise(s, opcode, u, is_q, size, rn, rd);
10880 return;
10881 case 0x13: /* SHLL, SHLL2 */
10882 if (u == 0 || size == 3) {
10883 unallocated_encoding(s);
10884 return;
10886 if (!fp_access_check(s)) {
10887 return;
10889 handle_shll(s, is_q, size, rn, rd);
10890 return;
10891 case 0xa: /* CMLT */
10892 if (u == 1) {
10893 unallocated_encoding(s);
10894 return;
10896 /* fall through */
10897 case 0x8: /* CMGT, CMGE */
10898 case 0x9: /* CMEQ, CMLE */
10899 case 0xb: /* ABS, NEG */
10900 if (size == 3 && !is_q) {
10901 unallocated_encoding(s);
10902 return;
10904 break;
10905 case 0x7: /* SQABS, SQNEG */
10906 if (size == 3 && !is_q) {
10907 unallocated_encoding(s);
10908 return;
10910 break;
10911 case 0xc ... 0xf:
10912 case 0x16 ... 0x1f:
10914 /* Floating point: U, size[1] and opcode indicate operation;
10915 * size[0] indicates single or double precision.
10917 int is_double = extract32(size, 0, 1);
10918 opcode |= (extract32(size, 1, 1) << 5) | (u << 6);
10919 size = is_double ? 3 : 2;
10920 switch (opcode) {
10921 case 0x2f: /* FABS */
10922 case 0x6f: /* FNEG */
10923 if (size == 3 && !is_q) {
10924 unallocated_encoding(s);
10925 return;
10927 break;
10928 case 0x1d: /* SCVTF */
10929 case 0x5d: /* UCVTF */
10931 bool is_signed = (opcode == 0x1d) ? true : false;
10932 int elements = is_double ? 2 : is_q ? 4 : 2;
10933 if (is_double && !is_q) {
10934 unallocated_encoding(s);
10935 return;
10937 if (!fp_access_check(s)) {
10938 return;
10940 handle_simd_intfp_conv(s, rd, rn, elements, is_signed, 0, size);
10941 return;
10943 case 0x2c: /* FCMGT (zero) */
10944 case 0x2d: /* FCMEQ (zero) */
10945 case 0x2e: /* FCMLT (zero) */
10946 case 0x6c: /* FCMGE (zero) */
10947 case 0x6d: /* FCMLE (zero) */
10948 if (size == 3 && !is_q) {
10949 unallocated_encoding(s);
10950 return;
10952 handle_2misc_fcmp_zero(s, opcode, false, u, is_q, size, rn, rd);
10953 return;
10954 case 0x7f: /* FSQRT */
10955 if (size == 3 && !is_q) {
10956 unallocated_encoding(s);
10957 return;
10959 break;
10960 case 0x1a: /* FCVTNS */
10961 case 0x1b: /* FCVTMS */
10962 case 0x3a: /* FCVTPS */
10963 case 0x3b: /* FCVTZS */
10964 case 0x5a: /* FCVTNU */
10965 case 0x5b: /* FCVTMU */
10966 case 0x7a: /* FCVTPU */
10967 case 0x7b: /* FCVTZU */
10968 need_fpstatus = true;
10969 rmode = extract32(opcode, 5, 1) | (extract32(opcode, 0, 1) << 1);
10970 if (size == 3 && !is_q) {
10971 unallocated_encoding(s);
10972 return;
10974 break;
10975 case 0x5c: /* FCVTAU */
10976 case 0x1c: /* FCVTAS */
10977 need_fpstatus = true;
10978 rmode = FPROUNDING_TIEAWAY;
10979 if (size == 3 && !is_q) {
10980 unallocated_encoding(s);
10981 return;
10983 break;
10984 case 0x3c: /* URECPE */
10985 if (size == 3) {
10986 unallocated_encoding(s);
10987 return;
10989 /* fall through */
10990 case 0x3d: /* FRECPE */
10991 case 0x7d: /* FRSQRTE */
10992 if (size == 3 && !is_q) {
10993 unallocated_encoding(s);
10994 return;
10996 if (!fp_access_check(s)) {
10997 return;
10999 handle_2misc_reciprocal(s, opcode, false, u, is_q, size, rn, rd);
11000 return;
11001 case 0x56: /* FCVTXN, FCVTXN2 */
11002 if (size == 2) {
11003 unallocated_encoding(s);
11004 return;
11006 /* fall through */
11007 case 0x16: /* FCVTN, FCVTN2 */
11008 /* handle_2misc_narrow does a 2*size -> size operation, but these
11009 * instructions encode the source size rather than dest size.
11011 if (!fp_access_check(s)) {
11012 return;
11014 handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd);
11015 return;
11016 case 0x36: /* BFCVTN, BFCVTN2 */
11017 if (!dc_isar_feature(aa64_bf16, s) || size != 2) {
11018 unallocated_encoding(s);
11019 return;
11021 if (!fp_access_check(s)) {
11022 return;
11024 handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd);
11025 return;
11026 case 0x17: /* FCVTL, FCVTL2 */
11027 if (!fp_access_check(s)) {
11028 return;
11030 handle_2misc_widening(s, opcode, is_q, size, rn, rd);
11031 return;
11032 case 0x18: /* FRINTN */
11033 case 0x19: /* FRINTM */
11034 case 0x38: /* FRINTP */
11035 case 0x39: /* FRINTZ */
11036 rmode = extract32(opcode, 5, 1) | (extract32(opcode, 0, 1) << 1);
11037 /* fall through */
11038 case 0x59: /* FRINTX */
11039 case 0x79: /* FRINTI */
11040 need_fpstatus = true;
11041 if (size == 3 && !is_q) {
11042 unallocated_encoding(s);
11043 return;
11045 break;
11046 case 0x58: /* FRINTA */
11047 rmode = FPROUNDING_TIEAWAY;
11048 need_fpstatus = true;
11049 if (size == 3 && !is_q) {
11050 unallocated_encoding(s);
11051 return;
11053 break;
11054 case 0x7c: /* URSQRTE */
11055 if (size == 3) {
11056 unallocated_encoding(s);
11057 return;
11059 break;
11060 case 0x1e: /* FRINT32Z */
11061 case 0x1f: /* FRINT64Z */
11062 rmode = FPROUNDING_ZERO;
11063 /* fall through */
11064 case 0x5e: /* FRINT32X */
11065 case 0x5f: /* FRINT64X */
11066 need_fpstatus = true;
11067 if ((size == 3 && !is_q) || !dc_isar_feature(aa64_frint, s)) {
11068 unallocated_encoding(s);
11069 return;
11071 break;
11072 default:
11073 unallocated_encoding(s);
11074 return;
11076 break;
11078 default:
11079 case 0x3: /* SUQADD, USQADD */
11080 unallocated_encoding(s);
11081 return;
11084 if (!fp_access_check(s)) {
11085 return;
11088 if (need_fpstatus || rmode >= 0) {
11089 tcg_fpstatus = fpstatus_ptr(FPST_FPCR);
11090 } else {
11091 tcg_fpstatus = NULL;
11093 if (rmode >= 0) {
11094 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus);
11095 } else {
11096 tcg_rmode = NULL;
11099 switch (opcode) {
11100 case 0x5:
11101 if (u && size == 0) { /* NOT */
11102 gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_not, 0);
11103 return;
11105 break;
11106 case 0x8: /* CMGT, CMGE */
11107 if (u) {
11108 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_cge0, size);
11109 } else {
11110 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_cgt0, size);
11112 return;
11113 case 0x9: /* CMEQ, CMLE */
11114 if (u) {
11115 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_cle0, size);
11116 } else {
11117 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_ceq0, size);
11119 return;
11120 case 0xa: /* CMLT */
11121 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_clt0, size);
11122 return;
11123 case 0xb:
11124 if (u) { /* ABS, NEG */
11125 gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_neg, size);
11126 } else {
11127 gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_abs, size);
11129 return;
11132 if (size == 3) {
11133 /* All 64-bit element operations can be shared with scalar 2misc */
11134 int pass;
11136 /* Coverity claims (size == 3 && !is_q) has been eliminated
11137 * from all paths leading to here.
11139 tcg_debug_assert(is_q);
11140 for (pass = 0; pass < 2; pass++) {
11141 TCGv_i64 tcg_op = tcg_temp_new_i64();
11142 TCGv_i64 tcg_res = tcg_temp_new_i64();
11144 read_vec_element(s, tcg_op, rn, pass, MO_64);
11146 handle_2misc_64(s, opcode, u, tcg_res, tcg_op,
11147 tcg_rmode, tcg_fpstatus);
11149 write_vec_element(s, tcg_res, rd, pass, MO_64);
11151 } else {
11152 int pass;
11154 for (pass = 0; pass < (is_q ? 4 : 2); pass++) {
11155 TCGv_i32 tcg_op = tcg_temp_new_i32();
11156 TCGv_i32 tcg_res = tcg_temp_new_i32();
11158 read_vec_element_i32(s, tcg_op, rn, pass, MO_32);
11160 if (size == 2) {
11161 /* Special cases for 32 bit elements */
11162 switch (opcode) {
11163 case 0x4: /* CLS */
11164 if (u) {
11165 tcg_gen_clzi_i32(tcg_res, tcg_op, 32);
11166 } else {
11167 tcg_gen_clrsb_i32(tcg_res, tcg_op);
11169 break;
11170 case 0x7: /* SQABS, SQNEG */
11171 if (u) {
11172 gen_helper_neon_qneg_s32(tcg_res, tcg_env, tcg_op);
11173 } else {
11174 gen_helper_neon_qabs_s32(tcg_res, tcg_env, tcg_op);
11176 break;
11177 case 0x2f: /* FABS */
11178 gen_vfp_abss(tcg_res, tcg_op);
11179 break;
11180 case 0x6f: /* FNEG */
11181 gen_vfp_negs(tcg_res, tcg_op);
11182 break;
11183 case 0x7f: /* FSQRT */
11184 gen_helper_vfp_sqrts(tcg_res, tcg_op, tcg_env);
11185 break;
11186 case 0x1a: /* FCVTNS */
11187 case 0x1b: /* FCVTMS */
11188 case 0x1c: /* FCVTAS */
11189 case 0x3a: /* FCVTPS */
11190 case 0x3b: /* FCVTZS */
11191 gen_helper_vfp_tosls(tcg_res, tcg_op,
11192 tcg_constant_i32(0), tcg_fpstatus);
11193 break;
11194 case 0x5a: /* FCVTNU */
11195 case 0x5b: /* FCVTMU */
11196 case 0x5c: /* FCVTAU */
11197 case 0x7a: /* FCVTPU */
11198 case 0x7b: /* FCVTZU */
11199 gen_helper_vfp_touls(tcg_res, tcg_op,
11200 tcg_constant_i32(0), tcg_fpstatus);
11201 break;
11202 case 0x18: /* FRINTN */
11203 case 0x19: /* FRINTM */
11204 case 0x38: /* FRINTP */
11205 case 0x39: /* FRINTZ */
11206 case 0x58: /* FRINTA */
11207 case 0x79: /* FRINTI */
11208 gen_helper_rints(tcg_res, tcg_op, tcg_fpstatus);
11209 break;
11210 case 0x59: /* FRINTX */
11211 gen_helper_rints_exact(tcg_res, tcg_op, tcg_fpstatus);
11212 break;
11213 case 0x7c: /* URSQRTE */
11214 gen_helper_rsqrte_u32(tcg_res, tcg_op);
11215 break;
11216 case 0x1e: /* FRINT32Z */
11217 case 0x5e: /* FRINT32X */
11218 gen_helper_frint32_s(tcg_res, tcg_op, tcg_fpstatus);
11219 break;
11220 case 0x1f: /* FRINT64Z */
11221 case 0x5f: /* FRINT64X */
11222 gen_helper_frint64_s(tcg_res, tcg_op, tcg_fpstatus);
11223 break;
11224 default:
11225 g_assert_not_reached();
11227 } else {
11228 /* Use helpers for 8 and 16 bit elements */
11229 switch (opcode) {
11230 case 0x5: /* CNT, RBIT */
11231 /* For these two insns size is part of the opcode specifier
11232 * (handled earlier); they always operate on byte elements.
11234 if (u) {
11235 gen_helper_neon_rbit_u8(tcg_res, tcg_op);
11236 } else {
11237 gen_helper_neon_cnt_u8(tcg_res, tcg_op);
11239 break;
11240 case 0x7: /* SQABS, SQNEG */
11242 NeonGenOneOpEnvFn *genfn;
11243 static NeonGenOneOpEnvFn * const fns[2][2] = {
11244 { gen_helper_neon_qabs_s8, gen_helper_neon_qneg_s8 },
11245 { gen_helper_neon_qabs_s16, gen_helper_neon_qneg_s16 },
11247 genfn = fns[size][u];
11248 genfn(tcg_res, tcg_env, tcg_op);
11249 break;
11251 case 0x4: /* CLS, CLZ */
11252 if (u) {
11253 if (size == 0) {
11254 gen_helper_neon_clz_u8(tcg_res, tcg_op);
11255 } else {
11256 gen_helper_neon_clz_u16(tcg_res, tcg_op);
11258 } else {
11259 if (size == 0) {
11260 gen_helper_neon_cls_s8(tcg_res, tcg_op);
11261 } else {
11262 gen_helper_neon_cls_s16(tcg_res, tcg_op);
11265 break;
11266 default:
11267 g_assert_not_reached();
11271 write_vec_element_i32(s, tcg_res, rd, pass, MO_32);
11274 clear_vec_high(s, is_q, rd);
11276 if (tcg_rmode) {
11277 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
11281 /* AdvSIMD [scalar] two register miscellaneous (FP16)
11283 * 31 30 29 28 27 24 23 22 21 17 16 12 11 10 9 5 4 0
11284 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+
11285 * | 0 | Q | U | S | 1 1 1 0 | a | 1 1 1 1 0 0 | opcode | 1 0 | Rn | Rd |
11286 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+
11287 * mask: 1000 1111 0111 1110 0000 1100 0000 0000 0x8f7e 0c00
11288 * val: 0000 1110 0111 1000 0000 1000 0000 0000 0x0e78 0800
11290 * This actually covers two groups where scalar access is governed by
11291 * bit 28. A bunch of the instructions (float to integral) only exist
11292 * in the vector form and are un-allocated for the scalar decode. Also
11293 * in the scalar decode Q is always 1.
11295 static void disas_simd_two_reg_misc_fp16(DisasContext *s, uint32_t insn)
11297 int fpop, opcode, a, u;
11298 int rn, rd;
11299 bool is_q;
11300 bool is_scalar;
11301 bool only_in_vector = false;
11303 int pass;
11304 TCGv_i32 tcg_rmode = NULL;
11305 TCGv_ptr tcg_fpstatus = NULL;
11306 bool need_fpst = true;
11307 int rmode = -1;
11309 if (!dc_isar_feature(aa64_fp16, s)) {
11310 unallocated_encoding(s);
11311 return;
11314 rd = extract32(insn, 0, 5);
11315 rn = extract32(insn, 5, 5);
11317 a = extract32(insn, 23, 1);
11318 u = extract32(insn, 29, 1);
11319 is_scalar = extract32(insn, 28, 1);
11320 is_q = extract32(insn, 30, 1);
11322 opcode = extract32(insn, 12, 5);
11323 fpop = deposit32(opcode, 5, 1, a);
11324 fpop = deposit32(fpop, 6, 1, u);
11326 switch (fpop) {
11327 case 0x1d: /* SCVTF */
11328 case 0x5d: /* UCVTF */
11330 int elements;
11332 if (is_scalar) {
11333 elements = 1;
11334 } else {
11335 elements = (is_q ? 8 : 4);
11338 if (!fp_access_check(s)) {
11339 return;
11341 handle_simd_intfp_conv(s, rd, rn, elements, !u, 0, MO_16);
11342 return;
11344 break;
11345 case 0x2c: /* FCMGT (zero) */
11346 case 0x2d: /* FCMEQ (zero) */
11347 case 0x2e: /* FCMLT (zero) */
11348 case 0x6c: /* FCMGE (zero) */
11349 case 0x6d: /* FCMLE (zero) */
11350 handle_2misc_fcmp_zero(s, fpop, is_scalar, 0, is_q, MO_16, rn, rd);
11351 return;
11352 case 0x3d: /* FRECPE */
11353 case 0x3f: /* FRECPX */
11354 break;
11355 case 0x18: /* FRINTN */
11356 only_in_vector = true;
11357 rmode = FPROUNDING_TIEEVEN;
11358 break;
11359 case 0x19: /* FRINTM */
11360 only_in_vector = true;
11361 rmode = FPROUNDING_NEGINF;
11362 break;
11363 case 0x38: /* FRINTP */
11364 only_in_vector = true;
11365 rmode = FPROUNDING_POSINF;
11366 break;
11367 case 0x39: /* FRINTZ */
11368 only_in_vector = true;
11369 rmode = FPROUNDING_ZERO;
11370 break;
11371 case 0x58: /* FRINTA */
11372 only_in_vector = true;
11373 rmode = FPROUNDING_TIEAWAY;
11374 break;
11375 case 0x59: /* FRINTX */
11376 case 0x79: /* FRINTI */
11377 only_in_vector = true;
11378 /* current rounding mode */
11379 break;
11380 case 0x1a: /* FCVTNS */
11381 rmode = FPROUNDING_TIEEVEN;
11382 break;
11383 case 0x1b: /* FCVTMS */
11384 rmode = FPROUNDING_NEGINF;
11385 break;
11386 case 0x1c: /* FCVTAS */
11387 rmode = FPROUNDING_TIEAWAY;
11388 break;
11389 case 0x3a: /* FCVTPS */
11390 rmode = FPROUNDING_POSINF;
11391 break;
11392 case 0x3b: /* FCVTZS */
11393 rmode = FPROUNDING_ZERO;
11394 break;
11395 case 0x5a: /* FCVTNU */
11396 rmode = FPROUNDING_TIEEVEN;
11397 break;
11398 case 0x5b: /* FCVTMU */
11399 rmode = FPROUNDING_NEGINF;
11400 break;
11401 case 0x5c: /* FCVTAU */
11402 rmode = FPROUNDING_TIEAWAY;
11403 break;
11404 case 0x7a: /* FCVTPU */
11405 rmode = FPROUNDING_POSINF;
11406 break;
11407 case 0x7b: /* FCVTZU */
11408 rmode = FPROUNDING_ZERO;
11409 break;
11410 case 0x2f: /* FABS */
11411 case 0x6f: /* FNEG */
11412 need_fpst = false;
11413 break;
11414 case 0x7d: /* FRSQRTE */
11415 case 0x7f: /* FSQRT (vector) */
11416 break;
11417 default:
11418 unallocated_encoding(s);
11419 return;
11423 /* Check additional constraints for the scalar encoding */
11424 if (is_scalar) {
11425 if (!is_q) {
11426 unallocated_encoding(s);
11427 return;
11429 /* FRINTxx is only in the vector form */
11430 if (only_in_vector) {
11431 unallocated_encoding(s);
11432 return;
11436 if (!fp_access_check(s)) {
11437 return;
11440 if (rmode >= 0 || need_fpst) {
11441 tcg_fpstatus = fpstatus_ptr(FPST_FPCR_F16);
11444 if (rmode >= 0) {
11445 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus);
11448 if (is_scalar) {
11449 TCGv_i32 tcg_op = read_fp_hreg(s, rn);
11450 TCGv_i32 tcg_res = tcg_temp_new_i32();
11452 switch (fpop) {
11453 case 0x1a: /* FCVTNS */
11454 case 0x1b: /* FCVTMS */
11455 case 0x1c: /* FCVTAS */
11456 case 0x3a: /* FCVTPS */
11457 case 0x3b: /* FCVTZS */
11458 gen_helper_advsimd_f16tosinth(tcg_res, tcg_op, tcg_fpstatus);
11459 break;
11460 case 0x3d: /* FRECPE */
11461 gen_helper_recpe_f16(tcg_res, tcg_op, tcg_fpstatus);
11462 break;
11463 case 0x3f: /* FRECPX */
11464 gen_helper_frecpx_f16(tcg_res, tcg_op, tcg_fpstatus);
11465 break;
11466 case 0x5a: /* FCVTNU */
11467 case 0x5b: /* FCVTMU */
11468 case 0x5c: /* FCVTAU */
11469 case 0x7a: /* FCVTPU */
11470 case 0x7b: /* FCVTZU */
11471 gen_helper_advsimd_f16touinth(tcg_res, tcg_op, tcg_fpstatus);
11472 break;
11473 case 0x6f: /* FNEG */
11474 tcg_gen_xori_i32(tcg_res, tcg_op, 0x8000);
11475 break;
11476 case 0x7d: /* FRSQRTE */
11477 gen_helper_rsqrte_f16(tcg_res, tcg_op, tcg_fpstatus);
11478 break;
11479 default:
11480 g_assert_not_reached();
11483 /* limit any sign extension going on */
11484 tcg_gen_andi_i32(tcg_res, tcg_res, 0xffff);
11485 write_fp_sreg(s, rd, tcg_res);
11486 } else {
11487 for (pass = 0; pass < (is_q ? 8 : 4); pass++) {
11488 TCGv_i32 tcg_op = tcg_temp_new_i32();
11489 TCGv_i32 tcg_res = tcg_temp_new_i32();
11491 read_vec_element_i32(s, tcg_op, rn, pass, MO_16);
11493 switch (fpop) {
11494 case 0x1a: /* FCVTNS */
11495 case 0x1b: /* FCVTMS */
11496 case 0x1c: /* FCVTAS */
11497 case 0x3a: /* FCVTPS */
11498 case 0x3b: /* FCVTZS */
11499 gen_helper_advsimd_f16tosinth(tcg_res, tcg_op, tcg_fpstatus);
11500 break;
11501 case 0x3d: /* FRECPE */
11502 gen_helper_recpe_f16(tcg_res, tcg_op, tcg_fpstatus);
11503 break;
11504 case 0x5a: /* FCVTNU */
11505 case 0x5b: /* FCVTMU */
11506 case 0x5c: /* FCVTAU */
11507 case 0x7a: /* FCVTPU */
11508 case 0x7b: /* FCVTZU */
11509 gen_helper_advsimd_f16touinth(tcg_res, tcg_op, tcg_fpstatus);
11510 break;
11511 case 0x18: /* FRINTN */
11512 case 0x19: /* FRINTM */
11513 case 0x38: /* FRINTP */
11514 case 0x39: /* FRINTZ */
11515 case 0x58: /* FRINTA */
11516 case 0x79: /* FRINTI */
11517 gen_helper_advsimd_rinth(tcg_res, tcg_op, tcg_fpstatus);
11518 break;
11519 case 0x59: /* FRINTX */
11520 gen_helper_advsimd_rinth_exact(tcg_res, tcg_op, tcg_fpstatus);
11521 break;
11522 case 0x2f: /* FABS */
11523 tcg_gen_andi_i32(tcg_res, tcg_op, 0x7fff);
11524 break;
11525 case 0x6f: /* FNEG */
11526 tcg_gen_xori_i32(tcg_res, tcg_op, 0x8000);
11527 break;
11528 case 0x7d: /* FRSQRTE */
11529 gen_helper_rsqrte_f16(tcg_res, tcg_op, tcg_fpstatus);
11530 break;
11531 case 0x7f: /* FSQRT */
11532 gen_helper_sqrt_f16(tcg_res, tcg_op, tcg_fpstatus);
11533 break;
11534 default:
11535 g_assert_not_reached();
11538 write_vec_element_i32(s, tcg_res, rd, pass, MO_16);
11541 clear_vec_high(s, is_q, rd);
11544 if (tcg_rmode) {
11545 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
11549 /* C3.6 Data processing - SIMD, inc Crypto
11551 * As the decode gets a little complex we are using a table based
11552 * approach for this part of the decode.
11554 static const AArch64DecodeTable data_proc_simd[] = {
11555 /* pattern , mask , fn */
11556 { 0x0e200800, 0x9f3e0c00, disas_simd_two_reg_misc },
11557 { 0x0f000400, 0x9f800400, disas_simd_shift_imm },
11558 { 0x5e200800, 0xdf3e0c00, disas_simd_scalar_two_reg_misc },
11559 { 0x5f000400, 0xdf800400, disas_simd_scalar_shift_imm },
11560 { 0x0e780800, 0x8f7e0c00, disas_simd_two_reg_misc_fp16 },
11561 { 0x00000000, 0x00000000, NULL }
11564 static void disas_data_proc_simd(DisasContext *s, uint32_t insn)
11566 /* Note that this is called with all non-FP cases from
11567 * table C3-6 so it must UNDEF for entries not specifically
11568 * allocated to instructions in that table.
11570 AArch64DecodeFn *fn = lookup_disas_fn(&data_proc_simd[0], insn);
11571 if (fn) {
11572 fn(s, insn);
11573 } else {
11574 unallocated_encoding(s);
11578 /* C3.6 Data processing - SIMD and floating point */
11579 static void disas_data_proc_simd_fp(DisasContext *s, uint32_t insn)
11581 if (extract32(insn, 28, 1) == 1 && extract32(insn, 30, 1) == 0) {
11582 disas_data_proc_fp(s, insn);
11583 } else {
11584 /* SIMD, including crypto */
11585 disas_data_proc_simd(s, insn);
11589 static bool trans_OK(DisasContext *s, arg_OK *a)
11591 return true;
11594 static bool trans_FAIL(DisasContext *s, arg_OK *a)
11596 s->is_nonstreaming = true;
11597 return true;
11601 * btype_destination_ok:
11602 * @insn: The instruction at the branch destination
11603 * @bt: SCTLR_ELx.BT
11604 * @btype: PSTATE.BTYPE, and is non-zero
11606 * On a guarded page, there are a limited number of insns
11607 * that may be present at the branch target:
11608 * - branch target identifiers,
11609 * - paciasp, pacibsp,
11610 * - BRK insn
11611 * - HLT insn
11612 * Anything else causes a Branch Target Exception.
11614 * Return true if the branch is compatible, false to raise BTITRAP.
11616 static bool btype_destination_ok(uint32_t insn, bool bt, int btype)
11618 if ((insn & 0xfffff01fu) == 0xd503201fu) {
11619 /* HINT space */
11620 switch (extract32(insn, 5, 7)) {
11621 case 0b011001: /* PACIASP */
11622 case 0b011011: /* PACIBSP */
11624 * If SCTLR_ELx.BT, then PACI*SP are not compatible
11625 * with btype == 3. Otherwise all btype are ok.
11627 return !bt || btype != 3;
11628 case 0b100000: /* BTI */
11629 /* Not compatible with any btype. */
11630 return false;
11631 case 0b100010: /* BTI c */
11632 /* Not compatible with btype == 3 */
11633 return btype != 3;
11634 case 0b100100: /* BTI j */
11635 /* Not compatible with btype == 2 */
11636 return btype != 2;
11637 case 0b100110: /* BTI jc */
11638 /* Compatible with any btype. */
11639 return true;
11641 } else {
11642 switch (insn & 0xffe0001fu) {
11643 case 0xd4200000u: /* BRK */
11644 case 0xd4400000u: /* HLT */
11645 /* Give priority to the breakpoint exception. */
11646 return true;
11649 return false;
11652 /* C3.1 A64 instruction index by encoding */
11653 static void disas_a64_legacy(DisasContext *s, uint32_t insn)
11655 switch (extract32(insn, 25, 4)) {
11656 case 0x5:
11657 case 0xd: /* Data processing - register */
11658 disas_data_proc_reg(s, insn);
11659 break;
11660 case 0x7:
11661 case 0xf: /* Data processing - SIMD and floating point */
11662 disas_data_proc_simd_fp(s, insn);
11663 break;
11664 default:
11665 unallocated_encoding(s);
11666 break;
11670 static void aarch64_tr_init_disas_context(DisasContextBase *dcbase,
11671 CPUState *cpu)
11673 DisasContext *dc = container_of(dcbase, DisasContext, base);
11674 CPUARMState *env = cpu_env(cpu);
11675 ARMCPU *arm_cpu = env_archcpu(env);
11676 CPUARMTBFlags tb_flags = arm_tbflags_from_tb(dc->base.tb);
11677 int bound, core_mmu_idx;
11679 dc->isar = &arm_cpu->isar;
11680 dc->condjmp = 0;
11681 dc->pc_save = dc->base.pc_first;
11682 dc->aarch64 = true;
11683 dc->thumb = false;
11684 dc->sctlr_b = 0;
11685 dc->be_data = EX_TBFLAG_ANY(tb_flags, BE_DATA) ? MO_BE : MO_LE;
11686 dc->condexec_mask = 0;
11687 dc->condexec_cond = 0;
11688 core_mmu_idx = EX_TBFLAG_ANY(tb_flags, MMUIDX);
11689 dc->mmu_idx = core_to_aa64_mmu_idx(core_mmu_idx);
11690 dc->tbii = EX_TBFLAG_A64(tb_flags, TBII);
11691 dc->tbid = EX_TBFLAG_A64(tb_flags, TBID);
11692 dc->tcma = EX_TBFLAG_A64(tb_flags, TCMA);
11693 dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx, false);
11694 #if !defined(CONFIG_USER_ONLY)
11695 dc->user = (dc->current_el == 0);
11696 #endif
11697 dc->fp_excp_el = EX_TBFLAG_ANY(tb_flags, FPEXC_EL);
11698 dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM);
11699 dc->pstate_il = EX_TBFLAG_ANY(tb_flags, PSTATE__IL);
11700 dc->fgt_active = EX_TBFLAG_ANY(tb_flags, FGT_ACTIVE);
11701 dc->fgt_svc = EX_TBFLAG_ANY(tb_flags, FGT_SVC);
11702 dc->trap_eret = EX_TBFLAG_A64(tb_flags, TRAP_ERET);
11703 dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL);
11704 dc->sme_excp_el = EX_TBFLAG_A64(tb_flags, SMEEXC_EL);
11705 dc->vl = (EX_TBFLAG_A64(tb_flags, VL) + 1) * 16;
11706 dc->svl = (EX_TBFLAG_A64(tb_flags, SVL) + 1) * 16;
11707 dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE);
11708 dc->bt = EX_TBFLAG_A64(tb_flags, BT);
11709 dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE);
11710 dc->unpriv = EX_TBFLAG_A64(tb_flags, UNPRIV);
11711 dc->ata[0] = EX_TBFLAG_A64(tb_flags, ATA);
11712 dc->ata[1] = EX_TBFLAG_A64(tb_flags, ATA0);
11713 dc->mte_active[0] = EX_TBFLAG_A64(tb_flags, MTE_ACTIVE);
11714 dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE);
11715 dc->pstate_sm = EX_TBFLAG_A64(tb_flags, PSTATE_SM);
11716 dc->pstate_za = EX_TBFLAG_A64(tb_flags, PSTATE_ZA);
11717 dc->sme_trap_nonstreaming = EX_TBFLAG_A64(tb_flags, SME_TRAP_NONSTREAMING);
11718 dc->naa = EX_TBFLAG_A64(tb_flags, NAA);
11719 dc->nv = EX_TBFLAG_A64(tb_flags, NV);
11720 dc->nv1 = EX_TBFLAG_A64(tb_flags, NV1);
11721 dc->nv2 = EX_TBFLAG_A64(tb_flags, NV2);
11722 dc->nv2_mem_e20 = EX_TBFLAG_A64(tb_flags, NV2_MEM_E20);
11723 dc->nv2_mem_be = EX_TBFLAG_A64(tb_flags, NV2_MEM_BE);
11724 dc->vec_len = 0;
11725 dc->vec_stride = 0;
11726 dc->cp_regs = arm_cpu->cp_regs;
11727 dc->features = env->features;
11728 dc->dcz_blocksize = arm_cpu->dcz_blocksize;
11729 dc->gm_blocksize = arm_cpu->gm_blocksize;
11731 #ifdef CONFIG_USER_ONLY
11732 /* In sve_probe_page, we assume TBI is enabled. */
11733 tcg_debug_assert(dc->tbid & 1);
11734 #endif
11736 dc->lse2 = dc_isar_feature(aa64_lse2, dc);
11738 /* Single step state. The code-generation logic here is:
11739 * SS_ACTIVE == 0:
11740 * generate code with no special handling for single-stepping (except
11741 * that anything that can make us go to SS_ACTIVE == 1 must end the TB;
11742 * this happens anyway because those changes are all system register or
11743 * PSTATE writes).
11744 * SS_ACTIVE == 1, PSTATE.SS == 1: (active-not-pending)
11745 * emit code for one insn
11746 * emit code to clear PSTATE.SS
11747 * emit code to generate software step exception for completed step
11748 * end TB (as usual for having generated an exception)
11749 * SS_ACTIVE == 1, PSTATE.SS == 0: (active-pending)
11750 * emit code to generate a software step exception
11751 * end the TB
11753 dc->ss_active = EX_TBFLAG_ANY(tb_flags, SS_ACTIVE);
11754 dc->pstate_ss = EX_TBFLAG_ANY(tb_flags, PSTATE__SS);
11755 dc->is_ldex = false;
11757 /* Bound the number of insns to execute to those left on the page. */
11758 bound = -(dc->base.pc_first | TARGET_PAGE_MASK) / 4;
11760 /* If architectural single step active, limit to 1. */
11761 if (dc->ss_active) {
11762 bound = 1;
11764 dc->base.max_insns = MIN(dc->base.max_insns, bound);
11767 static void aarch64_tr_tb_start(DisasContextBase *db, CPUState *cpu)
11771 static void aarch64_tr_insn_start(DisasContextBase *dcbase, CPUState *cpu)
11773 DisasContext *dc = container_of(dcbase, DisasContext, base);
11774 target_ulong pc_arg = dc->base.pc_next;
11776 if (tb_cflags(dcbase->tb) & CF_PCREL) {
11777 pc_arg &= ~TARGET_PAGE_MASK;
11779 tcg_gen_insn_start(pc_arg, 0, 0);
11780 dc->insn_start_updated = false;
11783 static void aarch64_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu)
11785 DisasContext *s = container_of(dcbase, DisasContext, base);
11786 CPUARMState *env = cpu_env(cpu);
11787 uint64_t pc = s->base.pc_next;
11788 uint32_t insn;
11790 /* Singlestep exceptions have the highest priority. */
11791 if (s->ss_active && !s->pstate_ss) {
11792 /* Singlestep state is Active-pending.
11793 * If we're in this state at the start of a TB then either
11794 * a) we just took an exception to an EL which is being debugged
11795 * and this is the first insn in the exception handler
11796 * b) debug exceptions were masked and we just unmasked them
11797 * without changing EL (eg by clearing PSTATE.D)
11798 * In either case we're going to take a swstep exception in the
11799 * "did not step an insn" case, and so the syndrome ISV and EX
11800 * bits should be zero.
11802 assert(s->base.num_insns == 1);
11803 gen_swstep_exception(s, 0, 0);
11804 s->base.is_jmp = DISAS_NORETURN;
11805 s->base.pc_next = pc + 4;
11806 return;
11809 if (pc & 3) {
11811 * PC alignment fault. This has priority over the instruction abort
11812 * that we would receive from a translation fault via arm_ldl_code.
11813 * This should only be possible after an indirect branch, at the
11814 * start of the TB.
11816 assert(s->base.num_insns == 1);
11817 gen_helper_exception_pc_alignment(tcg_env, tcg_constant_tl(pc));
11818 s->base.is_jmp = DISAS_NORETURN;
11819 s->base.pc_next = QEMU_ALIGN_UP(pc, 4);
11820 return;
11823 s->pc_curr = pc;
11824 insn = arm_ldl_code(env, &s->base, pc, s->sctlr_b);
11825 s->insn = insn;
11826 s->base.pc_next = pc + 4;
11828 s->fp_access_checked = false;
11829 s->sve_access_checked = false;
11831 if (s->pstate_il) {
11833 * Illegal execution state. This has priority over BTI
11834 * exceptions, but comes after instruction abort exceptions.
11836 gen_exception_insn(s, 0, EXCP_UDEF, syn_illegalstate());
11837 return;
11840 if (dc_isar_feature(aa64_bti, s)) {
11841 if (s->base.num_insns == 1) {
11842 /* First insn can have btype set to non-zero. */
11843 tcg_debug_assert(s->btype >= 0);
11846 * Note that the Branch Target Exception has fairly high
11847 * priority -- below debugging exceptions but above most
11848 * everything else. This allows us to handle this now
11849 * instead of waiting until the insn is otherwise decoded.
11851 * We can check all but the guarded page check here;
11852 * defer the latter to a helper.
11854 if (s->btype != 0
11855 && !btype_destination_ok(insn, s->bt, s->btype)) {
11856 gen_helper_guarded_page_check(tcg_env);
11858 } else {
11859 /* Not the first insn: btype must be 0. */
11860 tcg_debug_assert(s->btype == 0);
11864 s->is_nonstreaming = false;
11865 if (s->sme_trap_nonstreaming) {
11866 disas_sme_fa64(s, insn);
11869 if (!disas_a64(s, insn) &&
11870 !disas_sme(s, insn) &&
11871 !disas_sve(s, insn)) {
11872 disas_a64_legacy(s, insn);
11876 * After execution of most insns, btype is reset to 0.
11877 * Note that we set btype == -1 when the insn sets btype.
11879 if (s->btype > 0 && s->base.is_jmp != DISAS_NORETURN) {
11880 reset_btype(s);
11884 static void aarch64_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu)
11886 DisasContext *dc = container_of(dcbase, DisasContext, base);
11888 if (unlikely(dc->ss_active)) {
11889 /* Note that this means single stepping WFI doesn't halt the CPU.
11890 * For conditional branch insns this is harmless unreachable code as
11891 * gen_goto_tb() has already handled emitting the debug exception
11892 * (and thus a tb-jump is not possible when singlestepping).
11894 switch (dc->base.is_jmp) {
11895 default:
11896 gen_a64_update_pc(dc, 4);
11897 /* fall through */
11898 case DISAS_EXIT:
11899 case DISAS_JUMP:
11900 gen_step_complete_exception(dc);
11901 break;
11902 case DISAS_NORETURN:
11903 break;
11905 } else {
11906 switch (dc->base.is_jmp) {
11907 case DISAS_NEXT:
11908 case DISAS_TOO_MANY:
11909 gen_goto_tb(dc, 1, 4);
11910 break;
11911 default:
11912 case DISAS_UPDATE_EXIT:
11913 gen_a64_update_pc(dc, 4);
11914 /* fall through */
11915 case DISAS_EXIT:
11916 tcg_gen_exit_tb(NULL, 0);
11917 break;
11918 case DISAS_UPDATE_NOCHAIN:
11919 gen_a64_update_pc(dc, 4);
11920 /* fall through */
11921 case DISAS_JUMP:
11922 tcg_gen_lookup_and_goto_ptr();
11923 break;
11924 case DISAS_NORETURN:
11925 case DISAS_SWI:
11926 break;
11927 case DISAS_WFE:
11928 gen_a64_update_pc(dc, 4);
11929 gen_helper_wfe(tcg_env);
11930 break;
11931 case DISAS_YIELD:
11932 gen_a64_update_pc(dc, 4);
11933 gen_helper_yield(tcg_env);
11934 break;
11935 case DISAS_WFI:
11937 * This is a special case because we don't want to just halt
11938 * the CPU if trying to debug across a WFI.
11940 gen_a64_update_pc(dc, 4);
11941 gen_helper_wfi(tcg_env, tcg_constant_i32(4));
11943 * The helper doesn't necessarily throw an exception, but we
11944 * must go back to the main loop to check for interrupts anyway.
11946 tcg_gen_exit_tb(NULL, 0);
11947 break;
11952 const TranslatorOps aarch64_translator_ops = {
11953 .init_disas_context = aarch64_tr_init_disas_context,
11954 .tb_start = aarch64_tr_tb_start,
11955 .insn_start = aarch64_tr_insn_start,
11956 .translate_insn = aarch64_tr_translate_insn,
11957 .tb_stop = aarch64_tr_tb_stop,