| blob | 071058dbeb33297e94f862e029253f0202799978 |
1 ;; Machine description for AArch64 architecture.
2 ;; Copyright (C) 2009-2025 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
21 ;; Register numbers
22 (define_constants
23 [
24 (R0_REGNUM 0)
25 (R1_REGNUM 1)
26 (R2_REGNUM 2)
27 (R3_REGNUM 3)
28 (R4_REGNUM 4)
29 (R5_REGNUM 5)
30 (R6_REGNUM 6)
31 (R7_REGNUM 7)
32 (R8_REGNUM 8)
33 (R9_REGNUM 9)
34 (R10_REGNUM 10)
35 (R11_REGNUM 11)
36 (R12_REGNUM 12)
37 (R13_REGNUM 13)
38 (R14_REGNUM 14)
39 (R15_REGNUM 15)
40 (R16_REGNUM 16)
41 (R17_REGNUM 17)
42 (R18_REGNUM 18)
43 (R19_REGNUM 19)
44 (R20_REGNUM 20)
45 (R21_REGNUM 21)
46 (R22_REGNUM 22)
47 (R23_REGNUM 23)
48 (R24_REGNUM 24)
49 (R25_REGNUM 25)
50 (R26_REGNUM 26)
51 (R27_REGNUM 27)
52 (R28_REGNUM 28)
53 (R29_REGNUM 29)
54 (R30_REGNUM 30)
55 (SP_REGNUM 31)
56 (V0_REGNUM 32)
57 (V1_REGNUM 33)
58 (V2_REGNUM 34)
59 (V3_REGNUM 35)
60 (V4_REGNUM 36)
61 (V5_REGNUM 37)
62 (V6_REGNUM 38)
63 (V7_REGNUM 39)
64 (V8_REGNUM 40)
65 (V9_REGNUM 41)
66 (V10_REGNUM 42)
67 (V11_REGNUM 43)
68 (V12_REGNUM 44)
69 (V13_REGNUM 45)
70 (V14_REGNUM 46)
71 (V15_REGNUM 47)
72 (V16_REGNUM 48)
73 (V17_REGNUM 49)
74 (V18_REGNUM 50)
75 (V19_REGNUM 51)
76 (V20_REGNUM 52)
77 (V21_REGNUM 53)
78 (V22_REGNUM 54)
79 (V23_REGNUM 55)
80 (V24_REGNUM 56)
81 (V25_REGNUM 57)
82 (V26_REGNUM 58)
83 (V27_REGNUM 59)
84 (V28_REGNUM 60)
85 (V29_REGNUM 61)
86 (V30_REGNUM 62)
87 (V31_REGNUM 63)
88 (SFP_REGNUM 64)
89 (AP_REGNUM 65)
90 (CC_REGNUM 66)
91 ;; Defined only to make the DWARF description simpler.
92 (VG_REGNUM 67)
93 (P0_REGNUM 68)
94 (P1_REGNUM 69)
95 (P2_REGNUM 70)
96 (P3_REGNUM 71)
97 (P4_REGNUM 72)
98 (P5_REGNUM 73)
99 (P6_REGNUM 74)
100 (P7_REGNUM 75)
101 (P8_REGNUM 76)
102 (P9_REGNUM 77)
103 (P10_REGNUM 78)
104 (P11_REGNUM 79)
105 (P12_REGNUM 80)
106 (P13_REGNUM 81)
107 (P14_REGNUM 82)
108 (P15_REGNUM 83)
109 (LAST_SAVED_REGNUM 83)
111 ;; Floating Point Mode Register, used in FP8 insns.
112 (FPM_REGNUM 84)
114 (FFR_REGNUM 85)
115 ;; "FFR token": a fake register used for representing the scheduling
116 ;; restrictions on FFR-related operations.
117 (FFRT_REGNUM 86)
119 ;; ----------------------------------------------------------------
120 ;; Fake registers
121 ;; ----------------------------------------------------------------
122 ;; These registers represent abstract things, rather than real
123 ;; architected registers.
125 ;; Sometimes we use placeholder instructions to mark where later
126 ;; ABI-related lowering is needed. These placeholders read and
127 ;; write this register. Instructions that depend on the lowering
128 ;; read the register.
129 (LOWERING_REGNUM 87)
131 ;; Represents the contents of the current function's TPIDR2 block,
132 ;; in abstract form.
133 (TPIDR2_BLOCK_REGNUM 88)
135 ;; Holds the value that the current function wants PSTATE.ZA to be.
136 ;; The actual value can sometimes vary, because it does not track
137 ;; changes to PSTATE.ZA that happen during a lazy save and restore.
138 ;; Those effects are instead tracked by ZA_SAVED_REGNUM.
139 (SME_STATE_REGNUM 89)
141 ;; Instructions write to this register if they set TPIDR2_EL0 to a
142 ;; well-defined value. Instructions read from the register if they
143 ;; depend on the result of such writes.
144 ;;
145 ;; The register does not model the architected TPIDR2_ELO, just the
146 ;; current function's management of it.
147 (TPIDR2_SETUP_REGNUM 90)
149 ;; Represents the property "has an incoming lazy save been committed?".
150 (ZA_FREE_REGNUM 91)
152 ;; Represents the property "are the current function's ZA contents
153 ;; stored in the lazy save buffer, rather than in ZA itself?".
154 (ZA_SAVED_REGNUM 92)
156 ;; Represents the contents of the current function's ZA state in
157 ;; abstract form. At various times in the function, these contents
158 ;; might be stored in ZA itself, or in the function's lazy save buffer.
159 ;;
160 ;; The contents persist even when the architected ZA is off. Private-ZA
161 ;; functions have no effect on its contents.
162 (ZA_REGNUM 93)
164 ;; Similarly represents the contents of the current function's ZT0 state.
165 (ZT0_REGNUM 94)
167 (FIRST_FAKE_REGNUM LOWERING_REGNUM)
168 (LAST_FAKE_REGNUM ZT0_REGNUM)
169 ;; ----------------------------------------------------------------
171 ;; The pair of scratch registers used for stack probing with -fstack-check.
172 ;; Leave R9 alone as a possible choice for the static chain.
173 ;; Note that the use of these registers is mutually exclusive with the use
174 ;; of STACK_CLASH_SVE_CFA_REGNUM, which is for -fstack-clash-protection
175 ;; rather than -fstack-check.
176 (PROBE_STACK_FIRST_REGNUM 10)
177 (PROBE_STACK_SECOND_REGNUM 11)
178 ;; Scratch register used by stack clash protection to calculate
179 ;; SVE CFA offsets during probing.
180 (STACK_CLASH_SVE_CFA_REGNUM 11)
181 ;; Scratch registers for prologue/epilogue use.
182 (EP0_REGNUM 12)
183 (EP1_REGNUM 13)
184 ;; A couple of call-clobbered registers that we need to reserve when
185 ;; tracking speculation this is not ABI, so is subject to change.
186 (SPECULATION_SCRATCH_REGNUM 14)
187 (SPECULATION_TRACKER_REGNUM 15)
188 ;; Scratch registers used in frame layout.
189 (IP0_REGNUM 16)
190 (IP1_REGNUM 17)
191 (FP_REGNUM 29)
192 (LR_REGNUM 30)
193 ]
194 )
196 (define_c_enum "unspec" [
197 UNSPEC_AUTIA1716
198 UNSPEC_AUTIB1716
199 UNSPEC_AUTIASP
200 UNSPEC_AUTIBSP
201 UNSPEC_BSL
202 UNSPEC_CALLEE_ABI
203 UNSPEC_CASESI
204 UNSPEC_COMBINE
205 UNSPEC_CPYMEM
206 UNSPEC_CRC32B
207 UNSPEC_CRC32CB
208 UNSPEC_CRC32CH
209 UNSPEC_CRC32CW
210 UNSPEC_CRC32CX
211 UNSPEC_CRC32H
212 UNSPEC_CRC32W
213 UNSPEC_CRC32X
214 UNSPEC_DUP
215 UNSPEC_DUP_LANE
216 UNSPEC_FCVTZS
217 UNSPEC_FCVTZU
218 UNSPEC_FJCVTZS
219 UNSPEC_FRINT32Z
220 UNSPEC_FRINT32X
221 UNSPEC_FRINT64Z
222 UNSPEC_FRINT64X
223 UNSPEC_URECPE
224 UNSPEC_FRECPE
225 UNSPEC_FRECPS
226 UNSPEC_FRECPX
227 UNSPEC_FRINTA
228 UNSPEC_FRINTI
229 UNSPEC_FRINTM
230 UNSPEC_FRINTN
231 UNSPEC_FRINTP
232 UNSPEC_FRINTX
233 UNSPEC_FRINTZ
234 UNSPEC_GET_LANE
235 UNSPEC_GOTSMALLPIC
236 UNSPEC_GOTSMALLPIC28K
237 UNSPEC_GOTSMALLTLS
238 UNSPEC_GOTTINYPIC
239 UNSPEC_GOTTINYTLS
240 UNSPEC_STP
241 UNSPEC_LDP_FST
242 UNSPEC_LDP_SND
243 UNSPEC_LD1
244 UNSPEC_LD1_DUP
245 UNSPEC_LD1x2
246 UNSPEC_LD1x3
247 UNSPEC_LD1x4
248 UNSPEC_LD2
249 UNSPEC_LD2_DREG
250 UNSPEC_LD2_DUP
251 UNSPEC_LD3
252 UNSPEC_LD3_DREG
253 UNSPEC_LD3_DUP
254 UNSPEC_LD4
255 UNSPEC_LD4_DREG
256 UNSPEC_LD4_DUP
257 UNSPEC_LD2_LANE
258 UNSPEC_LD3_LANE
259 UNSPEC_LD4_LANE
260 UNSPEC_LD64B
261 UNSPEC_ST64B
262 UNSPEC_ST64BV
263 UNSPEC_ST64BV_RET
264 UNSPEC_ST64BV0
265 UNSPEC_ST64BV0_RET
266 UNSPEC_MB
267 UNSPEC_MOVMEM
268 UNSPEC_NOP
269 UNSPEC_PACIA1716
270 UNSPEC_PACIB1716
271 UNSPEC_PACIASP
272 UNSPEC_PACIBSP
273 UNSPEC_PRLG_STK
274 UNSPEC_REV
275 UNSPEC_SADALP
276 UNSPEC_SCVTF
277 UNSPEC_SET_LANE
278 UNSPEC_SETMEM
279 UNSPEC_SISD_NEG
280 UNSPEC_SISD_SSHL
281 UNSPEC_SISD_USHL
282 UNSPEC_SSHL_2S
283 UNSPEC_ST1
284 UNSPEC_ST1_LANE
285 UNSPEC_ST1x2
286 UNSPEC_ST1x3
287 UNSPEC_ST1x4
288 UNSPEC_ST2
289 UNSPEC_ST3
290 UNSPEC_ST4
291 UNSPEC_ST2_LANE
292 UNSPEC_ST3_LANE
293 UNSPEC_ST4_LANE
294 UNSPEC_TLS
295 UNSPEC_TLSDESC
296 UNSPEC_TLSLE12
297 UNSPEC_TLSLE24
298 UNSPEC_TLSLE32
299 UNSPEC_TLSLE48
300 UNSPEC_UADALP
301 UNSPEC_UCVTF
302 UNSPEC_USHL_2S
303 UNSPEC_VSTRUCTDUMMY
304 UNSPEC_SSP_SYSREG
305 UNSPEC_SP_SET
306 UNSPEC_SP_TEST
307 UNSPEC_RSQRT
308 UNSPEC_RSQRTE
309 UNSPEC_RSQRTS
310 UNSPEC_NZCV
311 UNSPEC_XPACLRI
312 UNSPEC_LD1_SVE
313 UNSPEC_ST1_SVE
314 UNSPEC_LDNT1_SVE
315 UNSPEC_STNT1_SVE
316 UNSPEC_LD1RQ
317 UNSPEC_LD1_GATHER
318 UNSPEC_LDFF1_GATHER
319 UNSPEC_LDNT1_GATHER
320 UNSPEC_ST1_SCATTER
321 UNSPEC_STNT1_SCATTER
322 UNSPEC_PRED_X
323 UNSPEC_PRED_Z
324 UNSPEC_PTEST
325 UNSPEC_PTRUE
326 UNSPEC_UNPACKSHI
327 UNSPEC_UNPACKUHI
328 UNSPEC_UNPACKSLO
329 UNSPEC_UNPACKULO
330 UNSPEC_PACK
331 UNSPEC_VCREATE
332 UNSPEC_VEC_COPY
333 UNSPEC_WHILEGE
334 UNSPEC_WHILEGT
335 UNSPEC_WHILEHI
336 UNSPEC_WHILEHS
337 UNSPEC_WHILELE
338 UNSPEC_WHILELO
339 UNSPEC_WHILELS
340 UNSPEC_WHILELT
341 UNSPEC_WHILERW
342 UNSPEC_WHILEWR
343 UNSPEC_LDN
344 UNSPEC_STN
345 UNSPEC_INSR
346 UNSPEC_CLASTA
347 UNSPEC_CLASTB
348 UNSPEC_FADDA
349 UNSPEC_REV_SUBREG
350 UNSPEC_REINTERPRET
351 UNSPEC_SPECULATION_TRACKER
352 UNSPEC_SPECULATION_TRACKER_REV
353 UNSPEC_COPYSIGN
354 UNSPEC_TTEST ; Represent transaction test.
355 UNSPEC_UPDATE_FFR
356 UNSPEC_UPDATE_FFRT
357 UNSPEC_RDFFR
358 UNSPEC_WRFFR
359 UNSPEC_WRITE_FPMR
360 UNSPEC_SYSREG_RDI
361 UNSPEC_SYSREG_RTI
362 UNSPEC_SYSREG_WDI
363 UNSPEC_SYSREG_WTI
364 UNSPEC_PLDX
365 ;; Represents an SVE-style lane index, in which the indexing applies
366 ;; within the containing 128-bit block.
367 UNSPEC_SVE_LANE_SELECT
368 UNSPEC_SVE_CNT_PAT
369 UNSPEC_SVE_PREFETCH
370 UNSPEC_SVE_PREFETCH_GATHER
371 UNSPEC_SVE_COMPACT
372 UNSPEC_SVE_SPLICE
373 UNSPEC_GEN_TAG ; Generate a 4-bit MTE tag.
374 UNSPEC_GEN_TAG_RND ; Generate a random 4-bit MTE tag.
375 UNSPEC_TAG_SPACE ; Translate address to MTE tag address space.
376 UNSPEC_LD1RO
377 UNSPEC_SALT_ADDR
378 UNSPEC_SAVE_NZCV
379 UNSPEC_RESTORE_NZCV
380 UNSPECV_PATCHABLE_AREA
381 UNSPEC_LDAP1_LANE
382 UNSPEC_STL1_LANE
383 ;; Wraps a constant integer that should be multiplied by the number
384 ;; of quadwords in an SME vector.
385 UNSPEC_SME_VQ
386 ])
388 (define_c_enum "unspecv" [
389 UNSPECV_EH_RETURN ; Represent EH_RETURN
390 UNSPECV_GET_FPCR ; Represent fetch of FPCR content.
391 UNSPECV_SET_FPCR ; Represent assign of FPCR content.
392 UNSPECV_GET_FPSR ; Represent fetch of FPSR content.
393 UNSPECV_SET_FPSR ; Represent assign of FPSR content.
394 UNSPECV_BLOCKAGE ; Represent a blockage
395 UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing.
396 UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
397 UNSPECV_BTI_NOARG ; Represent BTI.
398 UNSPECV_BTI_C ; Represent BTI c.
399 UNSPECV_BTI_J ; Represent BTI j.
400 UNSPECV_BTI_JC ; Represent BTI jc.
401 UNSPECV_CHKFEAT ; Represent CHKFEAT X16.
402 UNSPECV_GCSPR ; Represent MRS Xn, GCSPR_EL0
403 UNSPECV_GCSPOPM ; Represent GCSPOPM.
404 UNSPECV_GCSSS1 ; Represent GCSSS1 Xt.
405 UNSPECV_GCSSS2 ; Represent GCSSS2 Xt.
406 UNSPECV_TSTART ; Represent transaction start.
407 UNSPECV_TCOMMIT ; Represent transaction commit.
408 UNSPECV_TCANCEL ; Represent transaction cancel.
409 UNSPEC_RNDR ; Represent RNDR
410 UNSPEC_RNDRRS ; Represent RNDRRS
411 ]
412 )
414 ;; These constants are used as a const_int in various SVE unspecs
415 ;; to indicate whether the governing predicate is known to be a PTRUE.
416 (define_constants
417 [; Indicates that the predicate might not be a PTRUE.
418 (SVE_MAYBE_NOT_PTRUE 0)
420 ; Indicates that the predicate is known to be a PTRUE.
421 (SVE_KNOWN_PTRUE 1)])
423 ;; These constants are used as a const_int in predicated SVE FP arithmetic
424 ;; to indicate whether the operation is allowed to make additional lanes
425 ;; active without worrying about the effect on faulting behavior.
426 (define_constants
427 [; Indicates either that all lanes are active or that the instruction may
428 ; operate on inactive inputs even if doing so could induce a fault.
429 (SVE_RELAXED_GP 0)
431 ; Indicates that some lanes might be inactive and that the instruction
432 ; must not operate on inactive inputs if doing so could induce a fault.
433 (SVE_STRICT_GP 1)])
435 (include "constraints.md")
436 (include "predicates.md")
437 (include "iterators.md")
439 ;; -------------------------------------------------------------------
440 ;; Instruction types and attributes
441 ;; -------------------------------------------------------------------
443 ; The "type" attribute is included here from AArch32 backend to be able
444 ; to share pipeline descriptions.
445 (include "../arm/types.md")
447 ;; It is important to set the fp or simd attributes to yes when a pattern
448 ;; alternative uses the FP or SIMD register files, usually signified by use of
449 ;; the 'w' constraint. This will ensure that the alternative will be
450 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
451 ;; architecture extensions. If all the alternatives in a pattern use the
452 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
453 ;; or TARGET_SIMD.
455 ;; Attributes of the architecture required to support the instruction (or
456 ;; alternative). This attribute is used to compute attribute "enabled", use type
457 ;; "any" to enable an alternative in all cases.
458 ;;
459 ;; As a convenience, "fp_q" means "fp" + the ability to move between
460 ;; Q registers and is equivalent to "simd".
462 (define_enum "arches" [any rcpc8_4 fp fp_q base_simd nobase_simd
463 simd nosimd sve fp16 sme])
465 (define_enum_attr "arch" "arches" (const_string "any"))
467 ;; Whether a normal INSN in fact contains a call. Sometimes we represent
468 ;; calls to functions that use an ad-hoc ABI as normal insns, both for
469 ;; optimization reasons and to avoid the need to describe the ABI to
470 ;; target-independent code.
471 (define_attr "is_call" "no,yes" (const_string "no"))
473 ;; Indicates whether we want to enable the pattern with an optional early
474 ;; clobber for SVE predicates.
475 (define_attr "pred_clobber" "any,no,yes" (const_string "any"))
477 ;; [For compatibility with Arm in pipeline models]
478 ;; Attribute that specifies whether or not the instruction touches fp
479 ;; registers.
480 ;; Note that this attribute is not used anywhere in either the arm or aarch64
481 ;; backends except in the scheduling description for xgene1. In that
482 ;; scheduling description this attribute is used to subclass the load_4 and
483 ;; load_8 types.
484 (define_attr "fp" "no,yes"
485 (if_then_else
486 (eq_attr "arch" "fp")
487 (const_string "yes")
488 (const_string "no")))
490 (define_attr "arch_enabled" "no,yes"
491 (if_then_else
492 (and
493 (ior
494 (and
495 (eq_attr "pred_clobber" "no")
496 (match_test "!TARGET_SVE_PRED_CLOBBER"))
497 (and
498 (eq_attr "pred_clobber" "yes")
499 (match_test "TARGET_SVE_PRED_CLOBBER"))
500 (eq_attr "pred_clobber" "any"))
502 (ior
503 (eq_attr "arch" "any")
505 (and (eq_attr "arch" "rcpc8_4")
506 (match_test "TARGET_RCPC2"))
508 (and (eq_attr "arch" "fp")
509 (match_test "TARGET_FLOAT"))
511 (and (eq_attr "arch" "base_simd")
512 (match_test "TARGET_BASE_SIMD"))
514 (and (eq_attr "arch" "nobase_simd")
515 (match_test "!TARGET_BASE_SIMD"))
517 (and (eq_attr "arch" "fp_q, simd")
518 (match_test "TARGET_SIMD"))
520 (and (eq_attr "arch" "nosimd")
521 (match_test "!TARGET_SIMD"))
523 (and (eq_attr "arch" "fp16")
524 (match_test "TARGET_FP_F16INST"))
526 (and (eq_attr "arch" "sve")
527 (match_test "TARGET_SVE"))
529 (and (eq_attr "arch" "sme")
530 (match_test "TARGET_SME"))))
531 (const_string "yes")
532 (const_string "no")))
534 ;; True if this a bfloat16 operation. Only used for certain instructions.
535 (define_attr "is_bf16" "false,true" (const_string "false"))
537 ;; True if this alternative uses an SVE instruction in which the operands
538 ;; are reversed. This can happen for naturally commutative operations
539 ;; such as FADD, or when using things like FSUBR in preference to FSUB,
540 ;; or similarly when using things like FMAD in preference to FMLA.
541 (define_attr "is_rev" "false,true" (const_string "false"))
543 ;; True if this operation supports is_rev-style instructions for bfloat16.
544 (define_attr "supports_bf16_rev" "false,true" (const_string "false"))
546 ;; Selectively enable alternatives based on the mode of the operation.
547 (define_attr "mode_enabled" "false,true"
548 (cond [(and (eq_attr "is_bf16" "true")
549 (eq_attr "is_rev" "true")
550 (eq_attr "supports_bf16_rev" "false"))
551 (const_string "false")]
552 (const_string "true")))
554 ;; Attribute that controls whether an alternative is enabled or not.
555 (define_attr "enabled" "no,yes"
556 (if_then_else (and (eq_attr "arch_enabled" "yes")
557 (eq_attr "mode_enabled" "true"))
558 (const_string "yes")
559 (const_string "no")))
561 ;; Attribute that specifies whether we are dealing with a branch to a
562 ;; label that is far away, i.e. further away than the maximum/minimum
563 ;; representable in a signed 21-bits number.
564 ;; 0 :=: no
565 ;; 1 :=: yes
566 (define_attr "far_branch" "" (const_int 0))
568 ;; Attribute that specifies whether the alternative uses MOVPRFX.
569 (define_attr "movprfx" "no,yes" (const_string "no"))
571 ;; Attribute to specify that an alternative has the length of a single
572 ;; instruction plus a speculation barrier.
573 (define_attr "sls_length" "none,retbr,casesi" (const_string "none"))
575 (define_attr "length" ""
576 (cond [(eq_attr "movprfx" "yes")
577 (const_int 8)
579 (eq_attr "sls_length" "retbr")
580 (cond [(match_test "!aarch64_harden_sls_retbr_p ()") (const_int 4)
581 (match_test "TARGET_SB") (const_int 8)]
582 (const_int 12))
584 (eq_attr "sls_length" "casesi")
585 (cond [(match_test "!aarch64_harden_sls_retbr_p ()") (const_int 16)
586 (match_test "TARGET_SB") (const_int 20)]
587 (const_int 24))
588 ]
589 (const_int 4)))
591 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
592 ;; no predicated insns.
593 (define_attr "predicated" "yes,no" (const_string "no"))
595 ;; Set to true on an insn that requires the speculation tracking state to be
596 ;; in the tracking register before the insn issues. Otherwise the compiler
597 ;; may chose to hold the tracking state encoded in SP.
598 (define_attr "speculation_barrier" "true,false" (const_string "false"))
600 ;; This attribute is attached to multi-register instructions that have
601 ;; two forms: one in which the registers are consecutive and one in
602 ;; which they are strided. The consecutive and strided forms have
603 ;; different define_insns, with different operands. The mapping between
604 ;; the RTL of the consecutive form and the RTL of the strided form varies
605 ;; from one type of instruction to another.
606 ;;
607 ;; The attribute gives two pieces of information:
608 ;; - does the current instruction have consecutive or strided registers?
609 ;; - what kind of RTL rewrite is needed to move between forms?
610 ;;
611 ;; For example, all consecutive LD*1 instructions have the same basic
612 ;; RTL structure. The same applies to all strided LD*1 instructions.
613 ;; The RTL mapping therefore applies at LD1 granularity, rather than
614 ;; being broken down into individual types of load.
615 (define_attr "stride_type"
616 "none,ld1_consecutive,ld1_strided,st1_consecutive,st1_strided"
617 (const_string "none"))
619 ;; Attribute used to identify load pair and store pair instructions.
620 ;; Currently the attribute is only applied to the non-writeback ldp/stp
621 ;; patterns.
622 (define_attr "ldpstp" "ldp,stp,none" (const_string "none"))
624 ;; -------------------------------------------------------------------
625 ;; Pipeline descriptions and scheduling
626 ;; -------------------------------------------------------------------
628 ;; Processor types.
629 (include "aarch64-tune.md")
631 ;; Scheduling
632 (include "../arm/cortex-a53.md")
633 (include "../arm/cortex-a57.md")
634 (include "../arm/exynos-m1.md")
635 (include "thunderx.md")
636 (include "../arm/xgene1.md")
637 (include "thunderx2t99.md")
638 (include "tsv110.md")
639 (include "thunderx3t110.md")
641 ;; -------------------------------------------------------------------
642 ;; Jumps and other miscellaneous insns
643 ;; -------------------------------------------------------------------
645 (define_insn "aarch64_read_sysregdi"
646 [(set (match_operand:DI 0 "register_operand" "=r")
647 (unspec_volatile:DI [(match_operand 1 "aarch64_sysreg_string" "")]
648 UNSPEC_SYSREG_RDI))]
649 ""
650 "mrs\t%x0, %1"
651 )
653 (define_insn "aarch64_read_sysregti"
654 [(set (match_operand:TI 0 "register_operand" "=r")
655 (unspec_volatile:TI [(match_operand 1 "aarch64_sysreg_string" "")]
656 UNSPEC_SYSREG_RTI))]
657 "TARGET_D128"
658 "mrrs\t%x0, %H0, %x1"
659 )
661 (define_insn "aarch64_write_sysregdi"
662 [(unspec_volatile:DI [(match_operand 0 "aarch64_sysreg_string" "")
663 (match_operand:DI 1 "aarch64_reg_or_zero" "rZ")]
664 UNSPEC_SYSREG_WDI)]
665 ""
666 "msr\t%0, %x1"
667 )
669 (define_insn "aarch64_write_sysregti"
670 [(unspec_volatile:TI [(match_operand 0 "aarch64_sysreg_string" "")
671 (match_operand:TI 1 "register_operand" "r")]
672 UNSPEC_SYSREG_WTI)]
673 "TARGET_D128"
674 "msrr\t%x0, %x1, %H1"
675 )
677 (define_insn "indirect_jump"
678 [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
679 ""
680 {
681 output_asm_insn ("br\\t%0", operands);
682 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
683 }
684 [(set_attr "type" "branch")
685 (set_attr "sls_length" "retbr")]
686 )
688 (define_insn "jump"
689 [(set (pc) (label_ref (match_operand 0 "" "")))]
690 ""
691 "b\\t%l0"
692 [(set_attr "type" "branch")]
693 )
695 (define_expand "cbranch<mode>4"
696 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
697 [(match_operand:GPI 1 "register_operand")
698 (match_operand:GPI 2 "aarch64_plus_operand")])
699 (label_ref (match_operand 3 "" ""))
700 (pc)))]
701 ""
702 "
703 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
704 operands[2]);
705 operands[2] = const0_rtx;
706 "
707 )
709 (define_expand "cbranch<mode>4"
710 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
711 [(match_operand:GPF 1 "register_operand")
712 (match_operand:GPF 2 "aarch64_fp_compare_operand")])
713 (label_ref (match_operand 3 "" ""))
714 (pc)))]
715 ""
716 "
717 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
718 operands[2]);
719 operands[2] = const0_rtx;
720 "
721 )
723 (define_expand "cbranchcc4"
724 [(set (pc) (if_then_else
725 (match_operator 0 "aarch64_comparison_operator"
726 [(match_operand 1 "cc_register")
727 (match_operand 2 "const0_operand")])
728 (label_ref (match_operand 3 "" ""))
729 (pc)))]
730 ""
731 "")
733 (define_insn "@ccmp<CC_ONLY:mode><GPI:mode>"
734 [(set (match_operand:CC_ONLY 1 "cc_register")
735 (if_then_else:CC_ONLY
736 (match_operator 4 "aarch64_comparison_operator"
737 [(match_operand 0 "cc_register")
738 (const_int 0)])
739 (compare:CC_ONLY
740 (match_operand:GPI 2 "register_operand")
741 (match_operand:GPI 3 "aarch64_ccmp_operand"))
742 (unspec:CC_ONLY
743 [(match_operand 5 "immediate_operand")]
744 UNSPEC_NZCV)))]
745 ""
746 {@ [ cons: 2 , 3 ; attrs: type ]
747 [ r , r ; alus_sreg ] ccmp\t%<w>2, %<w>3, %k5, %m4
748 [ r , Uss ; alus_imm ] ccmp\t%<w>2, %3, %k5, %m4
749 [ r , Usn ; alus_imm ] ccmn\t%<w>2, #%n3, %k5, %m4
750 }
751 )
753 (define_insn "@ccmp<CCFP_CCFPE:mode><GPF:mode>"
754 [(set (match_operand:CCFP_CCFPE 1 "cc_register" "")
755 (if_then_else:CCFP_CCFPE
756 (match_operator 4 "aarch64_comparison_operator"
757 [(match_operand 0 "cc_register" "")
758 (const_int 0)])
759 (compare:CCFP_CCFPE
760 (match_operand:GPF 2 "register_operand" "w")
761 (match_operand:GPF 3 "register_operand" "w"))
762 (unspec:CCFP_CCFPE
763 [(match_operand 5 "immediate_operand")]
764 UNSPEC_NZCV)))]
765 "TARGET_FLOAT"
766 "fccmp<e>\\t%<s>2, %<s>3, %k5, %m4"
767 [(set_attr "type" "fccmp<s>")]
768 )
770 (define_insn "@ccmp<CC_ONLY:mode><GPI:mode>_rev"
771 [(set (match_operand:CC_ONLY 1 "cc_register")
772 (if_then_else:CC_ONLY
773 (match_operator 4 "aarch64_comparison_operator"
774 [(match_operand 0 "cc_register")
775 (const_int 0)])
776 (unspec:CC_ONLY
777 [(match_operand 5 "immediate_operand")]
778 UNSPEC_NZCV)
779 (compare:CC_ONLY
780 (match_operand:GPI 2 "register_operand")
781 (match_operand:GPI 3 "aarch64_ccmp_operand"))))]
782 ""
783 {@ [ cons: 2 , 3 ; attrs: type ]
784 [ r , r ; alus_sreg ] ccmp\t%<w>2, %<w>3, %k5, %M4
785 [ r , Uss ; alus_imm ] ccmp\t%<w>2, %3, %k5, %M4
786 [ r , Usn ; alus_imm ] ccmn\t%<w>2, #%n3, %k5, %M4
787 }
788 )
790 (define_insn "@ccmp<CCFP_CCFPE:mode><GPF:mode>_rev"
791 [(set (match_operand:CCFP_CCFPE 1 "cc_register" "")
792 (if_then_else:CCFP_CCFPE
793 (match_operator 4 "aarch64_comparison_operator"
794 [(match_operand 0 "cc_register" "")
795 (const_int 0)])
796 (unspec:CCFP_CCFPE
797 [(match_operand 5 "immediate_operand")]
798 UNSPEC_NZCV)
799 (compare:CCFP_CCFPE
800 (match_operand:GPF 2 "register_operand" "w")
801 (match_operand:GPF 3 "register_operand" "w"))))]
802 "TARGET_FLOAT"
803 "fccmp<e>\\t%<s>2, %<s>3, %k5, %M4"
804 [(set_attr "type" "fccmp<s>")]
805 )
807 ;; Expansion of signed mod by a power of 2 using CSNEG.
808 ;; For x0 % n where n is a power of 2 produce:
809 ;; negs x1, x0
810 ;; and x0, x0, #(n - 1)
811 ;; and x1, x1, #(n - 1)
812 ;; csneg x0, x0, x1, mi
814 (define_expand "mod<mode>3"
815 [(match_operand:GPI 0 "register_operand")
816 (match_operand:GPI 1 "register_operand")
817 (match_operand:GPI 2 "const_int_operand")]
818 ""
819 {
820 HOST_WIDE_INT val = INTVAL (operands[2]);
822 if (val <= 0
823 || exact_log2 (val) <= 0
824 || !aarch64_bitmask_imm (val - 1, <MODE>mode))
825 FAIL;
827 rtx mask = GEN_INT (val - 1);
829 /* In the special case of x0 % 2 we can do the even shorter:
830 cmp x0, xzr
831 and x0, x0, 1
832 cneg x0, x0, lt. */
833 if (val == 2)
834 {
835 rtx masked = gen_reg_rtx (<MODE>mode);
836 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
837 emit_insn (gen_and<mode>3 (masked, operands[1], mask));
838 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
839 emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
840 DONE;
841 }
843 rtx neg_op = gen_reg_rtx (<MODE>mode);
844 rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
846 /* Extract the condition register and mode. */
847 rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
848 rtx cc_reg = SET_DEST (cmp);
849 rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
851 rtx masked_pos = gen_reg_rtx (<MODE>mode);
852 emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
854 rtx masked_neg = gen_reg_rtx (<MODE>mode);
855 emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
857 emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
858 masked_neg, masked_pos));
859 DONE;
860 }
861 )
863 (define_insn "condjump"
864 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
865 [(match_operand 1 "cc_register" "") (const_int 0)])
866 (label_ref (match_operand 2 "" ""))
867 (pc)))]
868 ""
869 {
870 /* GCC's traditional style has been to use "beq" instead of "b.eq", etc.,
871 but the "." is required for SVE conditions. */
872 bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode;
873 if (get_attr_length (insn) == 8)
874 return aarch64_gen_far_branch (operands, 2, "Lbcond",
875 use_dot_p ? "b.%M0\\t" : "b%M0\\t");
876 else
877 return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2";
878 }
879 [(set_attr "type" "branch")
880 (set (attr "length")
881 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
882 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
883 (const_int 4)
884 (const_int 8)))
885 (set (attr "far_branch")
886 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
887 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
888 (const_int 0)
889 (const_int 1)))]
890 )
892 ;; For a 24-bit immediate CST we can optimize the compare for equality
893 ;; and branch sequence from:
894 ;; mov x0, #imm1
895 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
896 ;; cmp x1, x0
897 ;; b<ne,eq> .Label
898 ;; into the shorter:
899 ;; sub x0, x1, #(CST & 0xfff000)
900 ;; subs x0, x0, #(CST & 0x000fff)
901 ;; b<ne,eq> .Label
902 (define_insn_and_split "*compare_condjump<GPI:mode>"
903 [(set (pc) (if_then_else (EQL
904 (match_operand:GPI 0 "register_operand" "r")
905 (match_operand:GPI 1 "aarch64_imm24" "n"))
906 (label_ref:P (match_operand 2 "" ""))
907 (pc)))]
908 "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode)
909 && !aarch64_plus_operand (operands[1], <GPI:MODE>mode)
910 && !reload_completed"
911 "#"
912 "&& true"
913 [(const_int 0)]
914 {
915 HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
916 HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
917 rtx tmp = gen_reg_rtx (<GPI:MODE>mode);
918 emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
919 emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
920 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
921 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode,
922 cc_reg, const0_rtx);
923 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
924 DONE;
925 }
926 )
928 (define_expand "casesi"
929 [(match_operand:SI 0 "register_operand") ; Index
930 (match_operand:SI 1 "const_int_operand") ; Lower bound
931 (match_operand:SI 2 "const_int_operand") ; Total range
932 (match_operand:DI 3 "" "") ; Table label
933 (match_operand:DI 4 "" "")] ; Out of range label
934 ""
935 {
936 if (operands[1] != const0_rtx)
937 {
938 rtx reg = gen_reg_rtx (SImode);
940 /* Canonical RTL says that if you have:
942 (minus (X) (CONST))
944 then this should be emitted as:
946 (plus (X) (-CONST))
948 The use of trunc_int_for_mode ensures that the resulting
949 constant can be represented in SImode, this is important
950 for the corner case where operand[1] is INT_MIN. */
952 operands[1]
953 = GEN_INT (trunc_int_for_mode (-UINTVAL (operands[1]), SImode));
955 if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
956 (operands[1], SImode))
957 operands[1] = force_reg (SImode, operands[1]);
958 emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
959 operands[0] = reg;
960 }
962 if (!aarch64_plus_operand (operands[2], SImode))
963 operands[2] = force_reg (SImode, operands[2]);
964 emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
965 const0_rtx),
966 operands[0], operands[2], operands[4]));
968 operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
969 operands[2]
970 = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]),
971 UNSPEC_CASESI);
972 operands[2] = gen_rtx_MEM (DImode, operands[2]);
973 MEM_READONLY_P (operands[2]) = 1;
974 MEM_NOTRAP_P (operands[2]) = 1;
975 emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3]));
976 DONE;
977 }
978 )
980 (define_expand "casesi_dispatch"
981 [(parallel
982 [(set (pc) (match_operand:DI 0 ""))
983 (clobber (reg:CC CC_REGNUM))
984 (clobber (match_scratch:DI 2))
985 (clobber (match_scratch:DI 3))
986 (use (label_ref:DI (match_operand 1 "")))])]
987 "")
989 (define_insn "*casesi_dispatch"
990 [(parallel
991 [(set (pc)
992 (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
993 (match_operand:SI 1 "register_operand" "r")]
994 UNSPEC_CASESI)))
995 (clobber (reg:CC CC_REGNUM))
996 (clobber (match_scratch:DI 3 "=r"))
997 (clobber (match_scratch:DI 4 "=r"))
998 (use (label_ref:DI (match_operand 2 "" "")))])]
999 ""
1000 "*
1001 return aarch64_output_casesi (operands);
1002 "
1003 [(set_attr "sls_length" "casesi")
1004 (set_attr "type" "branch")]
1005 )
1007 (define_insn "nop"
1008 [(unspec[(const_int 0)] UNSPEC_NOP)]
1009 ""
1010 "nop"
1011 [(set_attr "type" "no_insn")]
1012 )
1014 (define_insn "prefetch"
1015 [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
1016 (match_operand:QI 1 "const_int_operand" "")
1017 (match_operand:QI 2 "const_int_operand" ""))]
1018 ""
1019 {
1020 const char * pftype[2][4] =
1021 {
1022 {"prfm\\tPLDL1STRM, %0",
1023 "prfm\\tPLDL3KEEP, %0",
1024 "prfm\\tPLDL2KEEP, %0",
1025 "prfm\\tPLDL1KEEP, %0"},
1026 {"prfm\\tPSTL1STRM, %0",
1027 "prfm\\tPSTL3KEEP, %0",
1028 "prfm\\tPSTL2KEEP, %0",
1029 "prfm\\tPSTL1KEEP, %0"},
1030 };
1032 int locality = INTVAL (operands[2]);
1034 gcc_assert (IN_RANGE (locality, 0, 3));
1036 /* PRFM accepts the same addresses as a 64-bit LDR so wrap
1037 the address into a DImode MEM so that aarch64_print_operand knows
1038 how to print it. */
1039 operands[0] = gen_rtx_MEM (DImode, operands[0]);
1040 return pftype[INTVAL (operands[1]) & 1][locality];
1041 }
1042 [(set_attr "type" "load_4")]
1043 )
1045 (define_insn "aarch64_pldx"
1046 [(unspec [(match_operand 0 "" "")
1047 (match_operand:DI 1 "aarch64_prefetch_operand" "Dp")] UNSPEC_PLDX)]
1048 ""
1049 {
1050 operands[1] = gen_rtx_MEM (DImode, operands[1]);
1051 return "prfm\\t%0, %1";
1052 }
1053 [(set_attr "type" "load_4")]
1054 )
1056 (define_insn "trap"
1057 [(trap_if (const_int 1) (const_int 8))]
1058 ""
1059 "brk #1000"
1060 [(set_attr "type" "trap")])
1062 (define_expand "prologue"
1063 [(clobber (const_int 0))]
1064 ""
1065 "
1066 aarch64_expand_prologue ();
1067 DONE;
1068 "
1069 )
1071 (define_expand "epilogue"
1072 [(clobber (const_int 0))]
1073 ""
1074 "
1075 aarch64_expand_epilogue (nullptr);
1076 DONE;
1077 "
1078 )
1080 (define_insn "*do_return"
1081 [(return)]
1082 ""
1083 {
1084 const char *ret = NULL;
1085 if (aarch64_return_address_signing_enabled ()
1086 && (TARGET_PAUTH))
1087 {
1088 if (aarch64_ra_sign_key == AARCH64_KEY_B)
1089 ret = "retab";
1090 else
1091 ret = "retaa";
1092 }
1093 else
1094 ret = "ret";
1095 output_asm_insn (ret, operands);
1096 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
1097 }
1098 [(set_attr "type" "branch")
1099 (set_attr "sls_length" "retbr")]
1100 )
1102 (define_expand "return"
1103 [(simple_return)]
1104 "aarch64_use_return_insn_p ()"
1105 ""
1106 )
1108 (define_insn "simple_return"
1109 [(simple_return)]
1110 ""
1111 {
1112 output_asm_insn ("ret", operands);
1113 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
1114 }
1115 [(set_attr "type" "branch")
1116 (set_attr "sls_length" "retbr")]
1117 )
1119 (define_insn "aarch64_cb<optab><mode>1"
1120 [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
1121 (const_int 0))
1122 (label_ref (match_operand 1 "" ""))
1123 (pc)))]
1124 "!aarch64_track_speculation"
1125 {
1126 if (get_attr_length (insn) == 8)
1127 return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
1128 else
1129 return "<cbz>\\t%<w>0, %l1";
1130 }
1131 [(set_attr "type" "branch")
1132 (set (attr "length")
1133 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
1134 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
1135 (const_int 4)
1136 (const_int 8)))
1137 (set (attr "far_branch")
1138 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
1139 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
1140 (const_int 0)
1141 (const_int 1)))]
1142 )
1144 (define_expand "tbranch_<code><mode>3"
1145 [(set (pc) (if_then_else
1146 (EQL (match_operand:SHORT 0 "register_operand")
1147 (match_operand 1 "const0_operand"))
1148 (label_ref (match_operand 2 ""))
1149 (pc)))]
1150 ""
1151 {
1152 rtx bitvalue = gen_reg_rtx (<ZEROM>mode);
1153 rtx reg = gen_lowpart (<ZEROM>mode, operands[0]);
1154 rtx val = gen_int_mode (HOST_WIDE_INT_1U << UINTVAL (operands[1]), <MODE>mode);
1155 emit_insn (gen_and<zerom>3 (bitvalue, reg, val));
1156 operands[1] = const0_rtx;
1157 operands[0] = aarch64_gen_compare_reg (<CODE>, bitvalue,
1158 operands[1]);
1159 })
1161 (define_insn "@aarch64_tb<optab><ALLI:mode><GPI:mode>"
1162 [(set (pc) (if_then_else
1163 (EQL (zero_extract:GPI (match_operand:ALLI 0 "register_operand" "r")
1164 (const_int 1)
1165 (match_operand 1
1166 "aarch64_simd_shift_imm_<ALLI:mode>" "n"))
1167 (const_int 0))
1168 (label_ref (match_operand 2 "" ""))
1169 (pc)))
1170 (clobber (reg:CC CC_REGNUM))]
1171 "!aarch64_track_speculation"
1172 {
1173 if (get_attr_length (insn) == 8)
1174 {
1175 if (get_attr_far_branch (insn) == 1)
1176 return aarch64_gen_far_branch (operands, 2, "Ltb",
1177 "<inv_tb>\\t%<ALLI:w>0, %1, ");
1178 else
1179 {
1180 operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
1181 return "tst\t%<ALLI:w>0, %1\;<bcond>\t%l2";
1182 }
1183 }
1184 else
1185 return "<tbz>\t%<ALLI:w>0, %1, %l2";
1186 }
1187 [(set_attr "type" "branch")
1188 (set (attr "length")
1189 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
1190 (lt (minus (match_dup 2) (pc)) (const_int 32764)))
1191 (const_int 4)
1192 (const_int 8)))
1193 (set (attr "far_branch")
1194 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
1195 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
1196 (const_int 0)
1197 (const_int 1)))]
1199 )
1201 (define_insn "*cb<optab><mode>1"
1202 [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
1203 (const_int 0))
1204 (label_ref (match_operand 1 "" ""))
1205 (pc)))
1206 (clobber (reg:CC CC_REGNUM))]
1207 "!aarch64_track_speculation"
1208 {
1209 if (get_attr_length (insn) == 8)
1210 {
1211 if (get_attr_far_branch (insn) == 1)
1212 return aarch64_gen_far_branch (operands, 1, "Ltb",
1213 "<inv_tb>\\t%<w>0, <sizem1>, ");
1214 else
1215 {
1216 char buf[64];
1217 uint64_t val = ((uint64_t) 1)
1218 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
1219 sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
1220 output_asm_insn (buf, operands);
1221 return "<bcond>\t%l1";
1222 }
1223 }
1224 else
1225 return "<tbz>\t%<w>0, <sizem1>, %l1";
1226 }
1227 [(set_attr "type" "branch")
1228 (set (attr "length")
1229 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
1230 (lt (minus (match_dup 1) (pc)) (const_int 32764)))
1231 (const_int 4)
1232 (const_int 8)))
1233 (set (attr "far_branch")
1234 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
1235 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
1236 (const_int 0)
1237 (const_int 1)))]
1238 )
1240 (define_expand "save_stack_nonlocal"
1241 [(set (match_operand 0 "memory_operand")
1242 (match_operand 1 "register_operand"))]
1243 ""
1244 {
1245 rtx stack_slot = adjust_address (operands[0], Pmode, 0);
1246 emit_move_insn (stack_slot, operands[1]);
1248 if (aarch64_gcs_enabled ())
1249 {
1250 /* Save GCS with code like
1251 mov x16, 1
1252 chkfeat x16
1253 tbnz x16, 0, .L_done
1254 mrs tmp, gcspr_el0
1255 str tmp, [%0, 8]
1256 .L_done: */
1258 rtx done_label = gen_label_rtx ();
1259 rtx r16 = gen_rtx_REG (DImode, R16_REGNUM);
1260 emit_move_insn (r16, const1_rtx);
1261 emit_insn (gen_aarch64_chkfeat ());
1262 emit_insn (gen_tbranch_neqi3 (r16, const0_rtx, done_label));
1263 rtx gcs_slot = adjust_address (operands[0], Pmode, GET_MODE_SIZE (Pmode));
1264 rtx gcs = gen_reg_rtx (Pmode);
1265 emit_insn (gen_aarch64_load_gcspr (gcs));
1266 emit_move_insn (gcs_slot, gcs);
1267 emit_label (done_label);
1268 }
1269 DONE;
1270 })
1272 (define_expand "restore_stack_nonlocal"
1273 [(set (match_operand 0 "register_operand" "")
1274 (match_operand 1 "memory_operand" ""))]
1275 ""
1276 {
1277 rtx stack_slot = adjust_address (operands[1], Pmode, 0);
1278 emit_move_insn (operands[0], stack_slot);
1280 if (aarch64_gcs_enabled ())
1281 {
1282 /* Restore GCS with code like
1283 mov x16, 1
1284 chkfeat x16
1285 tbnz x16, 0, .L_done
1286 ldr tmp1, [%1, 8]
1287 mrs tmp2, gcspr_el0
1288 subs tmp2, tmp1, tmp2
1289 b.eq .L_done
1290 .L_loop:
1291 gcspopm
1292 subs tmp2, tmp2, 8
1293 b.ne .L_loop
1294 .L_done: */
1296 rtx loop_label = gen_label_rtx ();
1297 rtx done_label = gen_label_rtx ();
1298 rtx r16 = gen_rtx_REG (DImode, R16_REGNUM);
1299 emit_move_insn (r16, const1_rtx);
1300 emit_insn (gen_aarch64_chkfeat ());
1301 emit_insn (gen_tbranch_neqi3 (r16, const0_rtx, done_label));
1302 rtx gcs_slot = adjust_address (operands[1], Pmode, GET_MODE_SIZE (Pmode));
1303 rtx gcs_old = gen_reg_rtx (Pmode);
1304 emit_move_insn (gcs_old, gcs_slot);
1305 rtx gcs_now = gen_reg_rtx (Pmode);
1306 emit_insn (gen_aarch64_load_gcspr (gcs_now));
1307 emit_insn (gen_subdi3_compare1 (gcs_now, gcs_old, gcs_now));
1308 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
1309 rtx cmp_rtx = gen_rtx_fmt_ee (EQ, DImode, cc_reg, const0_rtx);
1310 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, done_label));
1311 emit_label (loop_label);
1312 emit_insn (gen_aarch64_gcspopm_xzr ());
1313 emit_insn (gen_adddi3_compare0 (gcs_now, gcs_now, GEN_INT (-8)));
1314 cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
1315 cmp_rtx = gen_rtx_fmt_ee (NE, DImode, cc_reg, const0_rtx);
1316 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, loop_label));
1317 emit_label (done_label);
1318 }
1319 DONE;
1320 })
1322 ;; -------------------------------------------------------------------
1323 ;; Subroutine calls and sibcalls
1324 ;; -------------------------------------------------------------------
1326 (define_expand "call"
1327 [(parallel
1328 [(call (match_operand 0 "memory_operand")
1329 (match_operand 1 "general_operand"))
1330 (unspec:DI [(match_operand 2)] UNSPEC_CALLEE_ABI)
1331 (clobber (reg:DI LR_REGNUM))])]
1332 ""
1333 "
1334 {
1335 aarch64_expand_call (NULL_RTX, operands[0], operands[2], false);
1336 DONE;
1337 }"
1338 )
1340 (define_insn "*call_insn"
1341 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand"))
1342 (match_operand 1 "" ""))
1343 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1344 (clobber (reg:DI LR_REGNUM))]
1345 ""
1346 {@ [ cons: 0 ; attrs: type ]
1347 [ Ucr ; call ] << aarch64_indirect_call_asm (operands[0]);
1348 [ Usf ; call ] bl\t%c0
1349 }
1350 )
1352 (define_expand "call_value"
1353 [(parallel
1354 [(set (match_operand 0 "")
1355 (call (match_operand 1 "memory_operand")
1356 (match_operand 2 "general_operand")))
1357 (unspec:DI [(match_operand 3)] UNSPEC_CALLEE_ABI)
1358 (clobber (reg:DI LR_REGNUM))])]
1359 ""
1360 "
1361 {
1362 aarch64_expand_call (operands[0], operands[1], operands[3], false);
1363 DONE;
1364 }"
1365 )
1367 (define_insn "*call_value_insn"
1368 [(set (match_operand 0 "" "")
1369 (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand"))
1370 (match_operand 2 "" "")))
1371 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1372 (clobber (reg:DI LR_REGNUM))]
1373 ""
1374 {@ [ cons: 1 ; attrs: type ]
1375 [ Ucr ; call ] << aarch64_indirect_call_asm (operands[1]);
1376 [ Usf ; call ] bl\t%c1
1377 }
1378 )
1380 (define_expand "sibcall"
1381 [(parallel
1382 [(call (match_operand 0 "memory_operand")
1383 (match_operand 1 "general_operand"))
1384 (unspec:DI [(match_operand 2)] UNSPEC_CALLEE_ABI)
1385 (return)])]
1386 ""
1387 {
1388 aarch64_expand_call (NULL_RTX, operands[0], operands[2], true);
1389 DONE;
1390 }
1391 )
1393 (define_expand "sibcall_value"
1394 [(parallel
1395 [(set (match_operand 0 "")
1396 (call (match_operand 1 "memory_operand")
1397 (match_operand 2 "general_operand")))
1398 (unspec:DI [(match_operand 3)] UNSPEC_CALLEE_ABI)
1399 (return)])]
1400 ""
1401 {
1402 aarch64_expand_call (operands[0], operands[1], operands[3], true);
1403 DONE;
1404 }
1405 )
1407 (define_insn "*sibcall_insn"
1408 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
1409 (match_operand 1 ""))
1410 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1411 (return)]
1412 "SIBLING_CALL_P (insn)"
1413 {
1414 if (which_alternative == 0)
1415 {
1416 output_asm_insn ("br\\t%0", operands);
1417 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
1418 }
1419 return "b\\t%c0";
1420 }
1421 [(set_attr "type" "branch, branch")
1422 (set_attr "sls_length" "retbr,none")]
1423 )
1425 (define_insn "*sibcall_value_insn"
1426 [(set (match_operand 0 "")
1427 (call (mem:DI
1428 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
1429 (match_operand 2 "")))
1430 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1431 (return)]
1432 "SIBLING_CALL_P (insn)"
1433 {
1434 if (which_alternative == 0)
1435 {
1436 output_asm_insn ("br\\t%1", operands);
1437 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ());
1438 }
1439 return "b\\t%c1";
1440 }
1441 [(set_attr "type" "branch, branch")
1442 (set_attr "sls_length" "retbr,none")]
1443 )
1445 ;; Call subroutine returning any type.
1447 (define_expand "untyped_call"
1448 [(parallel [(call (match_operand 0 "")
1449 (const_int 0))
1450 (match_operand 1 "")
1451 (match_operand 2 "")])]
1452 ""
1453 {
1454 int i;
1456 /* Generate a PARALLEL that contains all of the register results.
1457 The offsets are somewhat arbitrary, since we don't know the
1458 actual return type. The main thing we need to avoid is having
1459 overlapping byte ranges, since those might give the impression
1460 that two registers are known to have data in common. */
1461 rtvec rets = rtvec_alloc (XVECLEN (operands[2], 0));
1462 poly_int64 offset = 0;
1463 for (i = 0; i < XVECLEN (operands[2], 0); i++)
1464 {
1465 rtx reg = SET_SRC (XVECEXP (operands[2], 0, i));
1466 gcc_assert (REG_P (reg));
1467 rtx offset_rtx = gen_int_mode (offset, Pmode);
1468 rtx piece = gen_rtx_EXPR_LIST (VOIDmode, reg, offset_rtx);
1469 RTVEC_ELT (rets, i) = piece;
1470 offset += GET_MODE_SIZE (GET_MODE (reg));
1471 }
1472 rtx ret = gen_rtx_PARALLEL (VOIDmode, rets);
1474 /* Untyped calls always use the default ABI. It's only possible to use
1475 ABI variants if we know the type of the target function. */
1476 emit_call_insn (gen_call_value (ret, operands[0], const0_rtx, const0_rtx));
1478 for (i = 0; i < XVECLEN (operands[2], 0); i++)
1479 {
1480 rtx set = XVECEXP (operands[2], 0, i);
1481 emit_move_insn (SET_DEST (set), SET_SRC (set));
1482 }
1484 /* The optimizer does not know that the call sets the function value
1485 registers we stored in the result block. We avoid problems by
1486 claiming that all hard registers are used and clobbered at this
1487 point. */
1488 emit_insn (gen_blockage ());
1489 DONE;
1490 })
1492 ;; -------------------------------------------------------------------
1493 ;; Moves
1494 ;; -------------------------------------------------------------------
1496 (define_expand "mov<mode>"
1497 [(set (match_operand:SHORT 0 "nonimmediate_operand")
1498 (match_operand:SHORT 1 "general_operand"))]
1499 ""
1500 "
1501 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1502 operands[1] = force_reg (<MODE>mode, operands[1]);
1504 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1505 {
1506 aarch64_expand_mov_immediate (operands[0], operands[1]);
1507 DONE;
1508 }
1509 "
1510 )
1512 (define_insn "*mov<mode>_aarch64"
1513 [(set (match_operand:SHORT 0 "nonimmediate_operand")
1514 (match_operand:SHORT 1 "aarch64_mov_operand"))]
1515 "(register_operand (operands[0], <MODE>mode)
1516 || aarch64_reg_or_zero (operands[1], <MODE>mode))"
1517 {@ [cons: =0, 1; attrs: type, arch]
1518 [w, Z ; neon_move , simd ] movi\t%0.<Vbtype>, #0
1519 [r, r ; mov_reg , * ] mov\t%w0, %w1
1520 [r, M ; mov_imm , * ] mov\t%w0, %1
1521 [w, D<hq>; neon_move , simd ] << aarch64_output_scalar_simd_mov_immediate (operands[1], <MODE>mode);
1522 /* The "mov_imm" type for CNT is just a placeholder. */
1523 [r, Usv ; mov_imm , sve ] << aarch64_output_sve_cnt_immediate ("cnt", "%x0", operands[1]);
1524 [r, Usr ; mov_imm , sve ] << aarch64_output_sve_rdvl (operands[1]);
1525 [r, m ; load_4 , * ] ldr<size>\t%w0, %1
1526 [w, m ; load_4 , * ] ldr\t%<size>0, %1
1527 [m, r Z ; store_4 , * ] str<size>\\t%w1, %0
1528 [m, w ; store_4 , * ] str\t%<size>1, %0
1529 [r, w ; neon_to_gp<q> , base_simd ] umov\t%w0, %1.<v>[0]
1530 [r, w ; neon_to_gp<q> , nobase_simd] fmov\t%w0, %s1
1531 [w, r Z ; neon_from_gp<q>, simd ] dup\t%0.<Vallxd>, %w1
1532 [w, r Z ; neon_from_gp<q>, nosimd ] fmov\t%s0, %w1
1533 [w, w ; neon_dup , simd ] dup\t%<Vetype>0, %1.<v>[0]
1534 [w, w ; neon_dup , nosimd ] fmov\t%s0, %s1
1535 [Umv, rZ ; mrs , * ] msr\t%0, %x1
1536 [r, Umv ; mrs , * ] mrs\t%x0, %1
1537 }
1538 )
1540 (define_expand "mov<mode>"
1541 [(set (match_operand:GPI 0 "nonimmediate_operand")
1542 (match_operand:GPI 1 "general_operand"))]
1543 ""
1544 "
1545 if (MEM_P (operands[0]) && !MEM_VOLATILE_P (operands[0])
1546 && CONST_INT_P (operands[1]) && <MODE>mode == DImode
1547 && aarch64_split_dimode_const_store (operands[0], operands[1]))
1548 DONE;
1550 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1551 operands[1] = force_reg (<MODE>mode, operands[1]);
1553 /* Lower moves of symbolic constants into individual instructions.
1554 Doing this now is sometimes necessary for correctness, since some
1555 sequences require temporary pseudo registers. Lowering now is also
1556 often better for optimization, since more RTL passes get the
1557 chance to optimize the individual instructions.
1559 When called after RA, also split multi-instruction moves into
1560 smaller pieces now, since we can't be sure that sure that there
1561 will be a following split pass. */
1562 if (CONST_INT_P (operands[1])
1563 ? (reload_completed
1564 && !aarch64_mov_imm_operand (operands[1], <MODE>mode))
1565 : CONSTANT_P (operands[1]))
1566 {
1567 aarch64_expand_mov_immediate (operands[0], operands[1]);
1568 DONE;
1569 }
1570 "
1571 )
1573 (define_insn_and_split "*movsi_aarch64"
1574 [(set (match_operand:SI 0 "nonimmediate_operand")
1575 (match_operand:SI 1 "aarch64_mov_operand"))]
1576 "(register_operand (operands[0], SImode)
1577 || aarch64_reg_or_zero (operands[1], SImode))"
1578 {@ [cons: =0, 1; attrs: type, arch, length]
1579 [w , Z ; neon_move, simd, 4] movi\t%0.2d, #0
1580 [r k, r ; mov_reg , * , 4] mov\t%w0, %w1
1581 [r , k ; mov_reg , * , 4] ^
1582 [r , M ; mov_imm , * , 4] mov\t%w0, %1
1583 [r , n ; mov_imm , * ,16] #
1584 /* The "mov_imm" type for CNT is just a placeholder. */
1585 [r , Usv; mov_imm , sve , 4] << aarch64_output_sve_cnt_immediate ("cnt", "%x0", operands[1]);
1586 [r , Usr; mov_imm , sve, 4] << aarch64_output_sve_rdvl (operands[1]);
1587 [r , UsR; mov_imm , sme, 4] << aarch64_output_rdsvl (operands[1]);
1588 [r , m ; load_4 , * , 4] ldr\t%w0, %1
1589 [w , m ; load_4 , fp , 4] ldr\t%s0, %1
1590 [m , r Z; store_4 , * , 4] str\t%w1, %0
1591 [m , w ; store_4 , fp , 4] str\t%s1, %0
1592 [r , Usw; load_4 , * , 8] adrp\t%x0, %A1\;ldr\t%w0, [%x0, %L1]
1593 [r , Usa; adr , * , 4] adr\t%x0, %c1
1594 [r , Ush; adr , * , 4] adrp\t%x0, %A1
1595 [w , r Z; f_mcr , fp , 4] fmov\t%s0, %w1
1596 [r , w ; f_mrc , fp , 4] fmov\t%w0, %s1
1597 [w , w ; fmov , fp , 4] fmov\t%s0, %s1
1598 [w , Ds ; neon_move, simd, 4] << aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1599 [Umv, rZ ; mrs , * , 4] msr\t%0, %x1
1600 [r, Umv ; mrs , * , 4] mrs\t%x0, %1
1601 }
1602 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1603 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1604 [(const_int 0)]
1605 {
1606 aarch64_expand_mov_immediate (operands[0], operands[1]);
1607 DONE;
1608 }
1609 )
1611 (define_insn_and_split "*movdi_aarch64"
1612 [(set (match_operand:DI 0 "nonimmediate_operand")
1613 (match_operand:DI 1 "aarch64_mov_operand"))]
1614 "(register_operand (operands[0], DImode)
1615 || aarch64_reg_or_zero (operands[1], DImode))"
1616 {@ [cons: =0, 1; attrs: type, arch, length]
1617 [w, Z ; neon_move, simd, 4] movi\t%0.2d, #0
1618 [r, r ; mov_reg , * , 4] mov\t%x0, %x1
1619 [k, r ; mov_reg , * , 4] mov\t%0, %x1
1620 [r, k ; mov_reg , * , 4] mov\t%x0, %1
1621 [r, O ; mov_imm , * , 4] << aarch64_is_mov_xn_imm (INTVAL (operands[1])) ? "mov\t%x0, %1" : "mov\t%w0, %1";
1622 [r, n ; mov_imm , * ,16] #
1623 /* The "mov_imm" type for CNT is just a placeholder. */
1624 [r, Usv ; mov_imm , sve , 4] << aarch64_output_sve_cnt_immediate ("cnt", "%x0", operands[1]);
1625 [r, Usr ; mov_imm , sve, 4] << aarch64_output_sve_rdvl (operands[1]);
1626 [r, UsR ; mov_imm , sme, 4] << aarch64_output_rdsvl (operands[1]);
1627 [r, m ; load_8 , * , 4] ldr\t%x0, %1
1628 [w, m ; load_8 , fp , 4] ldr\t%d0, %1
1629 [m, r Z ; store_8 , * , 4] str\t%x1, %0
1630 [m, w ; store_8 , fp , 4] str\t%d1, %0
1631 [r, Usw ; load_8 , * , 8] << TARGET_ILP32 ? "adrp\t%0, %A1\;ldr\t%w0, [%0, %L1]" : "adrp\t%0, %A1\;ldr\t%0, [%0, %L1]";
1632 [r, Usa ; adr , * , 4] adr\t%x0, %c1
1633 [r, Ush ; adr , * , 4] adrp\t%x0, %A1
1634 [w, r Z ; f_mcr , fp , 4] fmov\t%d0, %x1
1635 [r, w ; f_mrc , fp , 4] fmov\t%x0, %d1
1636 [w, w ; fmov , fp , 4] fmov\t%d0, %d1
1637 [w, Dd ; neon_move, simd, 4] << aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1638 [w, Dx ; neon_move, simd, 8] #
1639 [Umv, rZ; mrs , * , 4] msr\t%0, %x1
1640 [r, Umv ; mrs , * , 4] mrs\t%0, %1
1641 }
1642 "CONST_INT_P (operands[1])
1643 && REG_P (operands[0])
1644 && ((!aarch64_move_imm (INTVAL (operands[1]), DImode)
1645 && GP_REGNUM_P (REGNO (operands[0])))
1646 || (aarch64_simd_special_constant_p (operands[1], DImode)
1647 && FP_REGNUM_P (REGNO (operands[0]))))"
1648 [(const_int 0)]
1649 {
1650 if (GP_REGNUM_P (REGNO (operands[0])))
1651 aarch64_expand_mov_immediate (operands[0], operands[1]);
1652 else
1653 aarch64_maybe_generate_simd_constant (operands[0], operands[1], DImode);
1654 DONE;
1655 }
1656 )
1658 (define_insn "insv_imm<mode>"
1659 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1660 (const_int 16)
1661 (match_operand:GPI 1 "const_int_operand" "n"))
1662 (match_operand:GPI 2 "const_int_operand" "n"))]
1663 "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1664 && UINTVAL (operands[1]) % 16 == 0"
1665 "movk\\t%<w>0, %X2, lsl %1"
1666 [(set_attr "type" "mov_imm")]
1667 )
1669 ;; Match MOVK as a normal AND and IOR operation.
1670 (define_insn "aarch64_movk<mode>"
1671 [(set (match_operand:GPI 0 "register_operand" "=r")
1672 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
1673 (match_operand:GPI 2 "const_int_operand"))
1674 (match_operand:GPI 3 "const_int_operand")))]
1675 "aarch64_movk_shift (rtx_mode_t (operands[2], <MODE>mode),
1676 rtx_mode_t (operands[3], <MODE>mode)) >= 0"
1677 {
1678 int shift = aarch64_movk_shift (rtx_mode_t (operands[2], <MODE>mode),
1679 rtx_mode_t (operands[3], <MODE>mode));
1680 operands[2] = gen_int_mode (UINTVAL (operands[3]) >> shift, SImode);
1681 operands[3] = gen_int_mode (shift, SImode);
1682 return "movk\\t%<w>0, #%X2, lsl %3";
1683 }
1684 [(set_attr "type" "mov_imm")]
1685 )
1687 (define_expand "movti"
1688 [(set (match_operand:TI 0 "nonimmediate_operand")
1689 (match_operand:TI 1 "general_operand"))]
1690 ""
1691 "
1692 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1693 operands[1] = force_reg (TImode, operands[1]);
1695 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1696 {
1697 emit_move_insn (gen_lowpart (DImode, operands[0]),
1698 gen_lowpart (DImode, operands[1]));
1699 emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1700 DONE;
1701 }
1702 "
1703 )
1705 (define_insn "*movti_aarch64"
1706 [(set (match_operand:TI 0
1707 "nonimmediate_operand" "= r,w,w,w, r,w,w,r,m,m,w,m")
1708 (match_operand:TI 1
1709 "aarch64_movti_operand" " rUti,Z,Z,r, w,w,w,m,r,Z,m,w"))]
1710 "(register_operand (operands[0], TImode)
1711 || aarch64_reg_or_zero (operands[1], TImode))"
1712 "@
1713 #
1714 movi\\t%0.2d, #0
1715 fmov\t%d0, xzr
1716 #
1717 #
1718 mov\\t%0.16b, %1.16b
1719 mov\\t%Z0.d, %Z1.d
1720 ldp\\t%0, %H0, %1
1721 stp\\t%1, %H1, %0
1722 stp\\txzr, xzr, %0
1723 ldr\\t%q0, %1
1724 str\\t%q1, %0"
1725 [(set_attr "type" "multiple,neon_move,f_mcr,f_mcr,f_mrc,neon_logic_q,*,\
1726 load_16,store_16,store_16,\
1727 load_16,store_16")
1728 (set_attr "length" "8,4,4,8,8,4,4,4,4,4,4,4")
1729 (set_attr "arch" "*,simd,*,*,*,simd,sve,*,*,*,fp,fp")]
1730 )
1732 ;; Split a TImode register-register or register-immediate move into
1733 ;; its component DImode pieces, taking care to handle overlapping
1734 ;; source and dest registers.
1735 (define_split
1736 [(set (match_operand:TI 0 "register_operand" "")
1737 (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1738 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1739 [(const_int 0)]
1740 {
1741 aarch64_split_128bit_move (operands[0], operands[1]);
1742 DONE;
1743 })
1745 (define_expand "mov<mode>"
1746 [(set (match_operand:GPF_TF_F16_MOV 0 "nonimmediate_operand")
1747 (match_operand:GPF_TF_F16_MOV 1 "general_operand"))]
1748 ""
1749 {
1750 if (!TARGET_FLOAT)
1751 {
1752 aarch64_err_no_fpadvsimd (<MODE>mode);
1753 machine_mode intmode
1754 = int_mode_for_size (GET_MODE_BITSIZE (<MODE>mode), 0).require ();
1755 emit_move_insn (gen_lowpart (intmode, operands[0]),
1756 gen_lowpart (intmode, operands[1]));
1757 DONE;
1758 }
1760 if (GET_CODE (operands[0]) == MEM
1761 && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1762 && aarch64_float_const_zero_rtx_p (operands[1])))
1763 operands[1] = force_reg (<MODE>mode, operands[1]);
1765 if (!DECIMAL_FLOAT_MODE_P (<MODE>mode)
1766 && GET_CODE (operands[1]) == CONST_DOUBLE
1767 && can_create_pseudo_p ()
1768 && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1769 && !aarch64_float_const_representable_p (operands[1])
1770 && !aarch64_float_const_zero_rtx_p (operands[1])
1771 && aarch64_float_const_rtx_p (operands[1]))
1772 {
1773 unsigned HOST_WIDE_INT ival;
1774 bool res = aarch64_reinterpret_float_as_int (operands[1], &ival);
1775 gcc_assert (res);
1777 machine_mode intmode
1778 = int_mode_for_size (GET_MODE_BITSIZE (<MODE>mode), 0).require ();
1779 rtx tmp = gen_reg_rtx (intmode);
1780 emit_move_insn (tmp, gen_int_mode (ival, intmode));
1781 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1782 DONE;
1783 }
1784 }
1785 )
1787 (define_insn "*mov<mode>_aarch64"
1788 [(set (match_operand:HFBF 0 "nonimmediate_operand")
1789 (match_operand:HFBF 1 "general_operand"))]
1790 "aarch64_valid_fp_move (operands[0], operands[1], <MODE>mode)"
1791 {@ [ cons: =0 , 1 ; attrs: type , arch ]
1792 [ w , Y ; neon_move , simd ] movi\t%0.4h, #0
1793 [ w , ?rY ; f_mcr , fp16 ] fmov\t%h0, %w1
1794 [ w , ?r ; neon_move , simd ] dup\t%w0.4h, %w1
1795 [ w , ?rY ; f_mcr , * ] fmov\t%s0, %w1
1796 [ ?r , w ; neon_to_gp , simd ] umov\t%w0, %1.h[0]
1797 [ ?r , w ; f_mrc , * ] fmov\t%w0, %s1
1798 [ w , w ; neon_move , simd ] mov\t%0.h[0], %1.h[0]
1799 [ w , w ; fmov , * ] fmov\t%s0, %s1
1800 [ w , Ufc ; fconsts , fp16 ] fmov\t%h0, %1
1801 [ w , Uvi ; neon_move , simd ] << aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1802 [ w , m ; f_loads , * ] ldr\t%h0, %1
1803 [ m , w ; f_stores , * ] str\t%h1, %0
1804 [ r , m ; load_4 , * ] ldrh\t%w0, %1
1805 [ m , rY ; store_4 , * ] strh\t%w1, %0
1806 [ r , r ; mov_reg , * ] mov\t%w0, %w1
1807 }
1808 )
1810 (define_insn "*mov<mode>_aarch64"
1811 [(set (match_operand:SFD 0 "nonimmediate_operand")
1812 (match_operand:SFD 1 "general_operand"))]
1813 "aarch64_valid_fp_move (operands[0], operands[1], <MODE>mode)"
1814 {@ [ cons: =0 , 1 ; attrs: type , arch ]
1815 [ w , Y ; neon_move , simd ] movi\t%0.2s, #0
1816 [ w , ?rY ; f_mcr , * ] fmov\t%s0, %w1
1817 [ ?r , w ; f_mrc , * ] fmov\t%w0, %s1
1818 [ w , w ; fmov , * ] fmov\t%s0, %s1
1819 [ w , Ufc ; fconsts , * ] fmov\t%s0, %1
1820 [ w , Uvi ; neon_move , simd ] << aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1821 [ w , m ; f_loads , * ] ldr\t%s0, %1
1822 [ m , w ; f_stores , * ] str\t%s1, %0
1823 [ r , m ; load_4 , * ] ldr\t%w0, %1
1824 [ m , rY ; store_4 , * ] str\t%w1, %0
1825 [ r , r ; mov_reg , * ] mov\t%w0, %w1
1826 [ r , M ; fconsts , * ] mov\t%w0, %1
1827 }
1828 )
1830 (define_insn "*mov<mode>_aarch64"
1831 [(set (match_operand:DFD 0 "nonimmediate_operand")
1832 (match_operand:DFD 1 "general_operand"))]
1833 "aarch64_valid_fp_move (operands[0], operands[1], <MODE>mode)"
1834 {@ [ cons: =0 , 1 ; attrs: type , arch ]
1835 [ w , Y ; neon_move , simd ] movi\t%d0, #0
1836 [ w , ?rY ; f_mcr , * ] fmov\t%d0, %x1
1837 [ ?r , w ; f_mrc , * ] fmov\t%x0, %d1
1838 [ w , w ; fmov , * ] fmov\t%d0, %d1
1839 [ w , Ufc ; fconstd , * ] fmov\t%d0, %1
1840 [ w , Uvi ; neon_move , simd ] << aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1841 [ w , m ; f_loadd , * ] ldr\t%d0, %1
1842 [ m , w ; f_stored , * ] str\t%d1, %0
1843 [ r , m ; load_8 , * ] ldr\t%x0, %1
1844 [ m , rY ; store_8 , * ] str\t%x1, %0
1845 [ r , r ; mov_reg , * ] mov\t%x0, %x1
1846 [ r , O ; fconstd , * ] << aarch64_is_mov_xn_imm (INTVAL (operands[1])) ? "mov\t%x0, %1" : "mov\t%w0, %1";
1847 }
1848 )
1850 (define_insn "*mov<mode>_aarch64"
1851 [(set (match_operand:TFD 0
1852 "nonimmediate_operand" "=w,w,?r ,w ,?r,w,?w,w,m,?r,m ,m")
1853 (match_operand:TFD 1
1854 "general_operand" " w,w,?rY,?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1855 "TARGET_FLOAT && (register_operand (operands[0], <MODE>mode)
1856 || aarch64_reg_or_fp_zero (operands[1], <MODE>mode))"
1857 "@
1858 mov\\t%0.16b, %1.16b
1859 mov\\t%Z0.d, %Z1.d
1860 #
1861 #
1862 #
1863 movi\\t%0.2d, #0
1864 fmov\\t%s0, wzr
1865 ldr\\t%q0, %1
1866 str\\t%q1, %0
1867 ldp\\t%0, %H0, %1
1868 stp\\t%1, %H1, %0
1869 stp\\txzr, xzr, %0"
1870 [(set_attr "type" "logic_reg,*,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1871 f_loadd,f_stored,load_16,store_16,store_16")
1872 (set_attr "length" "4,4,8,8,8,4,4,4,4,4,4,4")
1873 (set_attr "arch" "simd,sve,*,*,*,simd,*,*,*,*,*,*")]
1874 )
1876 (define_split
1877 [(set (match_operand:TFD 0 "register_operand" "")
1878 (match_operand:TFD 1 "nonmemory_operand" ""))]
1879 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1880 [(const_int 0)]
1881 {
1882 aarch64_split_128bit_move (operands[0], operands[1]);
1883 DONE;
1884 }
1885 )
1887 ;; The preferred way of writing to the FPMR is to test whether it already
1888 ;; has the desired value and branch around the write if so. This reduces
1889 ;; the number of redundant FPMR writes caused by ABI boundaries, such as in:
1890 ;;
1891 ;; for (...)
1892 ;; fp8_kernel (..., fpmr_value);
1893 ;;
1894 ;; Without this optimization, fp8_kernel would set FPMR to fpmr_value each
1895 ;; time that it is called.
1896 ;;
1897 ;; We do this as a split so that hardreg_pre can optimize the moves first.
1898 (define_split
1899 [(set (reg:DI FPM_REGNUM)
1900 (match_operand:DI 0 "aarch64_reg_or_zero"))]
1901 "TARGET_FP8 && !TARGET_CHEAP_FPMR_WRITE && can_create_pseudo_p ()"
1902 [(const_int 0)]
1903 {
1904 auto label = gen_label_rtx ();
1905 rtx current = copy_to_reg (gen_rtx_REG (DImode, FPM_REGNUM));
1906 rtx cond = gen_rtx_EQ (VOIDmode, current, operands[0]);
1907 emit_jump_insn (gen_cbranchdi4 (cond, current, operands[0], label));
1908 emit_insn (gen_aarch64_write_fpmr (operands[0]));
1909 emit_label (label);
1910 DONE;
1911 }
1912 )
1914 ;; A write to the FPMR that is already protected by a conditional branch.
1915 ;; Since this instruction is introduced late, it shouldn't matter too much
1916 ;; that we're using an unspec for a move.
1917 (define_insn "aarch64_write_fpmr"
1918 [(set (reg:DI FPM_REGNUM)
1919 (unspec:DI [(match_operand:DI 0 "aarch64_reg_or_zero" "rZ")]
1920 UNSPEC_WRITE_FPMR))]
1921 "TARGET_FP8"
1922 "msr\tfpmr, %x0"
1923 )
1925 (define_expand "aarch64_cpymemdi"
1926 [(parallel
1927 [(set (match_operand 2) (const_int 0))
1928 (clobber (match_dup 3))
1929 (clobber (match_dup 4))
1930 (clobber (reg:CC CC_REGNUM))
1931 (set (match_operand 0)
1932 (unspec:BLK [(match_operand 1) (match_dup 2)] UNSPEC_CPYMEM))])]
1933 "TARGET_MOPS"
1934 {
1935 operands[3] = XEXP (operands[0], 0);
1936 operands[4] = XEXP (operands[1], 0);
1937 }
1938 )
1940 (define_insn "*aarch64_cpymemdi"
1941 [(set (match_operand:DI 2 "register_operand" "+&r") (const_int 0))
1942 (clobber (match_operand:DI 0 "register_operand" "+&r"))
1943 (clobber (match_operand:DI 1 "register_operand" "+&r"))
1944 (clobber (reg:CC CC_REGNUM))
1945 (set (mem:BLK (match_dup 0))
1946 (unspec:BLK [(mem:BLK (match_dup 1)) (match_dup 2)] UNSPEC_CPYMEM))]
1947 "TARGET_MOPS"
1948 "cpyfp\t[%x0]!, [%x1]!, %x2!\;cpyfm\t[%x0]!, [%x1]!, %x2!\;cpyfe\t[%x0]!, [%x1]!, %x2!"
1949 [(set_attr "length" "12")]
1950 )
1952 ;; 0 is dst
1953 ;; 1 is src
1954 ;; 2 is size of copy in bytes
1955 ;; 3 is alignment
1957 (define_expand "cpymemdi"
1958 [(match_operand:BLK 0 "memory_operand")
1959 (match_operand:BLK 1 "memory_operand")
1960 (match_operand:DI 2 "general_operand")
1961 (match_operand:DI 3 "immediate_operand")]
1962 ""
1963 {
1964 if (aarch64_expand_cpymem (operands, false))
1965 DONE;
1966 FAIL;
1967 }
1968 )
1970 (define_expand "aarch64_movmemdi"
1971 [(parallel
1972 [(set (match_operand 2) (const_int 0))
1973 (clobber (match_dup 3))
1974 (clobber (match_dup 4))
1975 (clobber (reg:CC CC_REGNUM))
1976 (set (match_operand 0)
1977 (unspec:BLK [(match_operand 1) (match_dup 2)] UNSPEC_MOVMEM))])]
1978 "TARGET_MOPS"
1979 {
1980 operands[3] = XEXP (operands[0], 0);
1981 operands[4] = XEXP (operands[1], 0);
1982 }
1983 )
1985 (define_insn "*aarch64_movmemdi"
1986 [(parallel [
1987 (set (match_operand:DI 2 "register_operand" "+&r") (const_int 0))
1988 (clobber (match_operand:DI 0 "register_operand" "+&r"))
1989 (clobber (match_operand:DI 1 "register_operand" "+&r"))
1990 (clobber (reg:CC CC_REGNUM))
1991 (set (mem:BLK (match_dup 0))
1992 (unspec:BLK [(mem:BLK (match_dup 1)) (match_dup 2)] UNSPEC_MOVMEM))])]
1993 "TARGET_MOPS"
1994 "cpyp\t[%x0]!, [%x1]!, %x2!\;cpym\t[%x0]!, [%x1]!, %x2!\;cpye\t[%x0]!, [%x1]!, %x2!"
1995 [(set_attr "length" "12")]
1996 )
1998 ;; 0 is dst
1999 ;; 1 is src
2000 ;; 2 is size of copy in bytes
2001 ;; 3 is alignment
2003 (define_expand "movmemdi"
2004 [(match_operand:BLK 0 "memory_operand")
2005 (match_operand:BLK 1 "memory_operand")
2006 (match_operand:DI 2 "general_operand")
2007 (match_operand:DI 3 "immediate_operand")]
2008 ""
2009 {
2010 if (aarch64_expand_cpymem (operands, true))
2011 DONE;
2012 FAIL;
2013 }
2014 )
2016 (define_expand "aarch64_setmemdi"
2017 [(parallel
2018 [(set (match_operand 2) (const_int 0))
2019 (clobber (match_dup 3))
2020 (clobber (reg:CC CC_REGNUM))
2021 (set (match_operand 0)
2022 (unspec:BLK [(match_operand 1)
2023 (match_dup 2)] UNSPEC_SETMEM))])]
2024 "TARGET_MOPS"
2025 {
2026 operands[3] = XEXP (operands[0], 0);
2027 }
2028 )
2030 (define_insn "*aarch64_setmemdi"
2031 [(set (match_operand:DI 2 "register_operand" "+&r") (const_int 0))
2032 (clobber (match_operand:DI 0 "register_operand" "+&r"))
2033 (clobber (reg:CC CC_REGNUM))
2034 (set (mem:BLK (match_dup 0))
2035 (unspec:BLK [(match_operand:QI 1 "aarch64_reg_or_zero" "rZ")
2036 (match_dup 2)] UNSPEC_SETMEM))]
2037 "TARGET_MOPS"
2038 "setp\t[%x0]!, %x2!, %x1\;setm\t[%x0]!, %x2!, %x1\;sete\t[%x0]!, %x2!, %x1"
2039 [(set_attr "length" "12")]
2040 )
2042 ;; 0 is dst
2043 ;; 1 is val
2044 ;; 2 is size of copy in bytes
2045 ;; 3 is alignment
2046 (define_expand "setmemdi"
2047 [(set (match_operand:BLK 0 "memory_operand") ;; Dest
2048 (match_operand:QI 2 "nonmemory_operand")) ;; Value
2049 (use (match_operand:DI 1 "general_operand")) ;; Length
2050 (match_operand 3 "immediate_operand")] ;; Align
2051 ""
2052 {
2053 if (aarch64_expand_setmem (operands))
2054 DONE;
2056 FAIL;
2057 })
2059 (define_insn "*load_pair_<ldst_sz>"
2060 [(set (match_operand:GPI 0 "aarch64_ldp_reg_operand")
2061 (unspec [
2062 (match_operand:<VPAIR> 1 "aarch64_mem_pair_lanes_operand")
2063 ] UNSPEC_LDP_FST))
2064 (set (match_operand:GPI 2 "aarch64_ldp_reg_operand")
2065 (unspec [
2066 (match_dup 1)
2067 ] UNSPEC_LDP_SND))]
2068 ""
2069 {@ [cons: =0, 1, =2; attrs: type, arch]
2070 [ r, Umn, r; load_<ldpstp_sz>, * ] ldp\t%<w>0, %<w>2, %y1
2071 [ w, Umn, w; neon_load1_2reg, fp ] ldp\t%<v>0, %<v>2, %y1
2072 }
2073 [(set_attr "ldpstp" "ldp")]
2074 )
2076 (define_insn "*load_pair_16"
2077 [(set (match_operand:TI 0 "aarch64_ldp_reg_operand" "=w")
2078 (unspec [
2079 (match_operand:V2x16QI 1 "aarch64_mem_pair_lanes_operand" "Umn")
2080 ] UNSPEC_LDP_FST))
2081 (set (match_operand:TI 2 "aarch64_ldp_reg_operand" "=w")
2082 (unspec [
2083 (match_dup 1)
2084 ] UNSPEC_LDP_SND))]
2085 "TARGET_FLOAT"
2086 "ldp\\t%q0, %q2, %y1"
2087 [(set_attr "type" "neon_ldp_q")
2088 (set_attr "fp" "yes")
2089 (set_attr "ldpstp" "ldp")]
2090 )
2092 (define_insn "*store_pair_<ldst_sz>"
2093 [(set (match_operand:<VPAIR> 0 "aarch64_mem_pair_lanes_operand")
2094 (unspec:<VPAIR>
2095 [(match_operand:GPI 1 "aarch64_stp_reg_operand")
2096 (match_operand:GPI 2 "aarch64_stp_reg_operand")] UNSPEC_STP))]
2097 ""
2098 {@ [cons: =0, 1, 2; attrs: type , arch]
2099 [ Umn, rYZ, rYZ; store_<ldpstp_sz>, * ] stp\t%<w>1, %<w>2, %y0
2100 [ Umn, w, w; neon_store1_2reg , fp ] stp\t%<v>1, %<v>2, %y0
2101 }
2102 [(set_attr "ldpstp" "stp")]
2103 )
2105 (define_insn "*store_pair_16"
2106 [(set (match_operand:V2x16QI 0 "aarch64_mem_pair_lanes_operand" "=Umn")
2107 (unspec:V2x16QI
2108 [(match_operand:TI 1 "aarch64_ldp_reg_operand" "w")
2109 (match_operand:TI 2 "aarch64_ldp_reg_operand" "w")] UNSPEC_STP))]
2110 "TARGET_FLOAT"
2111 "stp\t%q1, %q2, %y0"
2112 [(set_attr "type" "neon_stp_q")
2113 (set_attr "fp" "yes")
2114 (set_attr "ldpstp" "stp")]
2115 )
2117 ;; Writeback load/store pair patterns.
2118 ;;
2119 ;; Note that modes in the patterns [SI DI TI] are used only as a proxy for their
2120 ;; size; aarch64_ldp_reg_operand and aarch64_mem_pair_operator are special
2121 ;; predicates which accept a wide range of operand modes, with the requirement
2122 ;; that the contextual (pattern) mode is of the same size as the operand mode.
2124 ;; Load pair with post-index writeback. This is primarily used in function
2125 ;; epilogues.
2126 (define_insn "*loadwb_post_pair_<ldst_sz>"
2127 [(set (match_operand 0 "pmode_register_operand")
2128 (match_operator 7 "pmode_plus_operator" [
2129 (match_operand 1 "pmode_register_operand")
2130 (match_operand 4 "const_int_operand")]))
2131 (set (match_operand:GPI 2 "aarch64_ldp_reg_operand")
2132 (match_operator 5 "memory_operand" [(match_dup 1)]))
2133 (set (match_operand:GPI 3 "aarch64_ldp_reg_operand")
2134 (match_operator 6 "memory_operand" [
2135 (match_operator 8 "pmode_plus_operator" [
2136 (match_dup 1)
2137 (const_int <ldst_sz>)])]))]
2138 "aarch64_mem_pair_offset (operands[4], <MODE>mode)"
2139 {@ [cons: =0, 1, =2, =3; attrs: type]
2140 [ rk, 0, r, r; load_<ldpstp_sz>] ldp\t%<w>2, %<w>3, [%1], %4
2141 [ rk, 0, w, w; neon_load1_2reg ] ldp\t%<v>2, %<v>3, [%1], %4
2142 }
2143 )
2145 ;; q-register variant of the above
2146 (define_insn "*loadwb_post_pair_16"
2147 [(set (match_operand 0 "pmode_register_operand" "=rk")
2148 (match_operator 7 "pmode_plus_operator" [
2149 (match_operand 1 "pmode_register_operand" "0")
2150 (match_operand 4 "const_int_operand")]))
2151 (set (match_operand:TI 2 "aarch64_ldp_reg_operand" "=w")
2152 (match_operator 5 "memory_operand" [(match_dup 1)]))
2153 (set (match_operand:TI 3 "aarch64_ldp_reg_operand" "=w")
2154 (match_operator 6 "memory_operand"
2155 [(match_operator 8 "pmode_plus_operator" [
2156 (match_dup 1)
2157 (const_int 16)])]))]
2158 "TARGET_FLOAT
2159 && aarch64_mem_pair_offset (operands[4], TImode)"
2160 "ldp\t%q2, %q3, [%1], %4"
2161 [(set_attr "type" "neon_ldp_q")]
2162 )
2164 ;; Load pair with pre-index writeback.
2165 (define_insn "*loadwb_pre_pair_<ldst_sz>"
2166 [(set (match_operand 0 "pmode_register_operand")
2167 (match_operator 8 "pmode_plus_operator" [
2168 (match_operand 1 "pmode_register_operand")
2169 (match_operand 4 "const_int_operand")]))
2170 (set (match_operand:GPI 2 "aarch64_ldp_reg_operand")
2171 (match_operator 6 "memory_operand" [
2172 (match_operator 9 "pmode_plus_operator" [
2173 (match_dup 1)
2174 (match_dup 4)
2175 ])]))
2176 (set (match_operand:GPI 3 "aarch64_ldp_reg_operand")
2177 (match_operator 7 "memory_operand" [
2178 (match_operator 10 "pmode_plus_operator" [
2179 (match_dup 1)
2180 (match_operand 5 "const_int_operand")
2181 ])]))]
2182 "aarch64_mem_pair_offset (operands[4], <MODE>mode)
2183 && known_eq (INTVAL (operands[5]),
2184 INTVAL (operands[4]) + GET_MODE_SIZE (<MODE>mode))"
2185 {@ [cons: =&0, 1, =2, =3; attrs: type ]
2186 [ rk, 0, r, r; load_<ldpstp_sz>] ldp\t%<w>2, %<w>3, [%0, %4]!
2187 [ rk, 0, w, w; neon_load1_2reg ] ldp\t%<v>2, %<v>3, [%0, %4]!
2188 }
2189 )
2191 ;; q-register variant of the above
2192 (define_insn "*loadwb_pre_pair_16"
2193 [(set (match_operand 0 "pmode_register_operand" "=&rk")
2194 (match_operator 8 "pmode_plus_operator" [
2195 (match_operand 1 "pmode_register_operand" "0")
2196 (match_operand 4 "const_int_operand")]))
2197 (set (match_operand:TI 2 "aarch64_ldp_reg_operand" "=w")
2198 (match_operator 6 "memory_operand" [
2199 (match_operator 9 "pmode_plus_operator" [
2200 (match_dup 1)
2201 (match_dup 4)
2202 ])]))
2203 (set (match_operand:TI 3 "aarch64_ldp_reg_operand" "=w")
2204 (match_operator 7 "memory_operand" [
2205 (match_operator 10 "pmode_plus_operator" [
2206 (match_dup 1)
2207 (match_operand 5 "const_int_operand")
2208 ])]))]
2209 "TARGET_FLOAT
2210 && aarch64_mem_pair_offset (operands[4], TImode)
2211 && known_eq (INTVAL (operands[5]), INTVAL (operands[4]) + 16)"
2212 "ldp\t%q2, %q3, [%0, %4]!"
2213 [(set_attr "type" "neon_ldp_q")]
2214 )
2216 ;; Store pair with pre-index writeback. This is primarily used in function
2217 ;; prologues.
2218 (define_insn "*storewb_pre_pair_<ldst_sz>"
2219 [(set (match_operand 0 "pmode_register_operand")
2220 (match_operator 6 "pmode_plus_operator" [
2221 (match_operand 1 "pmode_register_operand")
2222 (match_operand 4 "const_int_operand")
2223 ]))
2224 (set (match_operator:GPI 7 "aarch64_mem_pair_operator" [
2225 (match_operator 8 "pmode_plus_operator" [
2226 (match_dup 0)
2227 (match_dup 4)
2228 ])])
2229 (match_operand:GPI 2 "aarch64_stp_reg_operand"))
2230 (set (match_operator:GPI 9 "aarch64_mem_pair_operator" [
2231 (match_operator 10 "pmode_plus_operator" [
2232 (match_dup 0)
2233 (match_operand 5 "const_int_operand")
2234 ])])
2235 (match_operand:GPI 3 "aarch64_stp_reg_operand"))]
2236 "aarch64_mem_pair_offset (operands[4], <MODE>mode)
2237 && known_eq (INTVAL (operands[5]),
2238 INTVAL (operands[4]) + GET_MODE_SIZE (<MODE>mode))
2239 && !reg_overlap_mentioned_p (operands[0], operands[2])
2240 && !reg_overlap_mentioned_p (operands[0], operands[3])"
2241 {@ [cons: =&0, 1, 2, 3; attrs: type ]
2242 [ rk, 0, rYZ, rYZ; store_<ldpstp_sz>] stp\t%<w>2, %<w>3, [%0, %4]!
2243 [ rk, 0, w, w; neon_store1_2reg ] stp\t%<v>2, %<v>3, [%0, %4]!
2244 }
2245 )
2247 ;; q-register variant of the above.
2248 (define_insn "*storewb_pre_pair_16"
2249 [(set (match_operand 0 "pmode_register_operand" "=&rk")
2250 (match_operator 6 "pmode_plus_operator" [
2251 (match_operand 1 "pmode_register_operand" "0")
2252 (match_operand 4 "const_int_operand")
2253 ]))
2254 (set (match_operator:TI 7 "aarch64_mem_pair_operator" [
2255 (match_operator 8 "pmode_plus_operator" [
2256 (match_dup 0)
2257 (match_dup 4)
2258 ])])
2259 (match_operand:TI 2 "aarch64_ldp_reg_operand" "w"))
2260 (set (match_operator:TI 9 "aarch64_mem_pair_operator" [
2261 (match_operator 10 "pmode_plus_operator" [
2262 (match_dup 0)
2263 (match_operand 5 "const_int_operand")
2264 ])])
2265 (match_operand:TI 3 "aarch64_ldp_reg_operand" "w"))]
2266 "TARGET_FLOAT
2267 && aarch64_mem_pair_offset (operands[4], TImode)
2268 && known_eq (INTVAL (operands[5]), INTVAL (operands[4]) + 16)
2269 && !reg_overlap_mentioned_p (operands[0], operands[2])
2270 && !reg_overlap_mentioned_p (operands[0], operands[3])"
2271 "stp\\t%q2, %q3, [%0, %4]!"
2272 [(set_attr "type" "neon_stp_q")]
2273 )
2275 ;; Store pair with post-index writeback.
2276 (define_insn "*storewb_post_pair_<ldst_sz>"
2277 [(set (match_operand 0 "pmode_register_operand")
2278 (match_operator 5 "pmode_plus_operator" [
2279 (match_operand 1 "pmode_register_operand")
2280 (match_operand 4 "const_int_operand")
2281 ]))
2282 (set (match_operator:GPI 6 "aarch64_mem_pair_operator" [(match_dup 1)])
2283 (match_operand 2 "aarch64_stp_reg_operand"))
2284 (set (match_operator:GPI 7 "aarch64_mem_pair_operator" [
2285 (match_operator 8 "pmode_plus_operator" [
2286 (match_dup 0)
2287 (const_int <ldst_sz>)
2288 ])])
2289 (match_operand 3 "aarch64_stp_reg_operand"))]
2290 "aarch64_mem_pair_offset (operands[4], <MODE>mode)
2291 && !reg_overlap_mentioned_p (operands[0], operands[2])
2292 && !reg_overlap_mentioned_p (operands[0], operands[3])"
2293 {@ [cons: =0, 1, 2, 3; attrs: type ]
2294 [ rk, 0, rYZ, rYZ; store_<ldpstp_sz>] stp\t%<w>2, %<w>3, [%0], %4
2295 [ rk, 0, w, w; neon_store1_2reg ] stp\t%<v>2, %<v>3, [%0], %4
2296 }
2297 )
2299 ;; Store pair with post-index writeback.
2300 (define_insn "*storewb_post_pair_16"
2301 [(set (match_operand 0 "pmode_register_operand" "=rk")
2302 (match_operator 5 "pmode_plus_operator" [
2303 (match_operand 1 "pmode_register_operand" "0")
2304 (match_operand 4 "const_int_operand")
2305 ]))
2306 (set (match_operator:TI 6 "aarch64_mem_pair_operator" [(match_dup 1)])
2307 (match_operand:TI 2 "aarch64_ldp_reg_operand" "w"))
2308 (set (match_operator:TI 7 "aarch64_mem_pair_operator" [
2309 (match_operator 8 "pmode_plus_operator" [
2310 (match_dup 0)
2311 (const_int 16)
2312 ])])
2313 (match_operand:TI 3 "aarch64_ldp_reg_operand" "w"))]
2314 "TARGET_FLOAT
2315 && aarch64_mem_pair_offset (operands[4], TImode)
2316 && !reg_overlap_mentioned_p (operands[0], operands[2])
2317 && !reg_overlap_mentioned_p (operands[0], operands[3])"
2318 "stp\t%q2, %q3, [%0], %4"
2319 [(set_attr "type" "neon_stp_q")]
2320 )
2322 ;; -------------------------------------------------------------------
2323 ;; Sign/Zero extension
2324 ;; -------------------------------------------------------------------
2326 (define_expand "<optab>sidi2"
2327 [(set (match_operand:DI 0 "register_operand")
2328 (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
2329 ""
2330 )
2332 (define_insn "*extendsidi2_aarch64"
2333 [(set (match_operand:DI 0 "register_operand")
2334 (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand")))]
2335 ""
2336 {@ [ cons: =0 , 1 ; attrs: type ]
2337 [ r , r ; extend ] sxtw\t%0, %w1
2338 [ r , m ; load_4 ] ldrsw\t%0, %1
2339 }
2340 )
2342 (define_insn "*load_pair_extendsidi2_aarch64"
2343 [(set (match_operand:DI 0 "register_operand" "=r")
2344 (sign_extend:DI (unspec:SI [
2345 (match_operand:V2x4QI 1 "aarch64_mem_pair_lanes_operand" "Umn")
2346 ] UNSPEC_LDP_FST)))
2347 (set (match_operand:DI 2 "register_operand" "=r")
2348 (sign_extend:DI (unspec:SI [
2349 (match_dup 1)
2350 ] UNSPEC_LDP_SND)))]
2351 ""
2352 "ldpsw\\t%0, %2, %y1"
2353 [(set_attr "type" "load_8")]
2354 )
2356 (define_insn "*zero_extendsidi2_aarch64"
2357 [(set (match_operand:DI 0 "register_operand")
2358 (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand")))]
2359 ""
2360 {@ [ cons: =0 , 1 ; attrs: type , arch ]
2361 [ r , r ; mov_reg , * ] uxtw\t%0, %w1
2362 [ r , m ; load_4 , * ] ldr\t%w0, %1
2363 [ w , r ; f_mcr , fp ] fmov\t%s0, %w1
2364 [ w , m ; f_loads , fp ] ldr\t%s0, %1
2365 [ r , w ; f_mrc , fp ] fmov\t%w0, %s1
2366 [ w , w ; fmov , fp ] fmov\t%s0, %s1
2367 }
2368 )
2370 (define_insn "*load_pair_zero_extendsidi2_aarch64"
2371 [(set (match_operand:DI 0 "register_operand")
2372 (zero_extend:DI (unspec:SI [
2373 (match_operand:V2x4QI 1 "aarch64_mem_pair_lanes_operand")
2374 ] UNSPEC_LDP_FST)))
2375 (set (match_operand:DI 2 "register_operand")
2376 (zero_extend:DI (unspec:SI [
2377 (match_dup 1)
2378 ] UNSPEC_LDP_SND)))]
2379 ""
2380 {@ [ cons: =0 , 1 , =2; attrs: type , arch]
2381 [ r , Umn , r ; load_8 , * ] ldp\t%w0, %w2, %y1
2382 [ w , Umn , w ; neon_load1_2reg, fp ] ldp\t%s0, %s2, %y1
2383 }
2384 )
2386 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
2387 [(set (match_operand:GPI 0 "register_operand")
2388 (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
2389 ""
2390 )
2392 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
2393 [(set (match_operand:GPI 0 "register_operand")
2394 (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
2395 ""
2396 {@ [ cons: =0 , 1 ; attrs: type , arch ]
2397 [ r , r ; extend , * ] sxt<SHORT:size>\t%<GPI:w>0, %w1
2398 [ r , m ; load_4 , * ] ldrs<SHORT:size>\t%<GPI:w>0, %1
2399 [ r , w ; neon_to_gp , fp ] smov\t%<GPI:w>0, %1.<SHORT:size>[0]
2400 }
2401 )
2403 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
2404 [(set (match_operand:GPI 0 "register_operand")
2405 (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
2406 ""
2407 {@ [ cons: =0 , 1 ; attrs: type , arch ]
2408 [ r , r ; logic_imm , * ] and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
2409 [ r , m ; load_4 , * ] ldr<SHORT:size>\t%w0, %1
2410 [ w , m ; f_loads , fp ] ldr\t%<SHORT:size>0, %1
2411 [ r , w ; neon_to_gp , fp ] umov\t%w0, %1.<SHORT:size>[0]
2412 }
2413 )
2415 (define_expand "<optab>qihi2"
2416 [(set (match_operand:HI 0 "register_operand")
2417 (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
2418 ""
2419 )
2421 (define_insn "*extendqihi2_aarch64"
2422 [(set (match_operand:HI 0 "register_operand")
2423 (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand")))]
2424 ""
2425 {@ [ cons: =0 , 1 ; attrs: type ]
2426 [ r , r ; extend ] sxtb\t%w0, %w1
2427 [ r , m ; load_4 ] ldrsb\t%w0, %1
2428 }
2429 )
2431 (define_insn "*zero_extendqihi2_aarch64"
2432 [(set (match_operand:HI 0 "register_operand")
2433 (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand")))]
2434 ""
2435 {@ [ cons: =0 , 1 ; attrs: type ]
2436 [ r , r ; logic_imm ] and\t%w0, %w1, 255
2437 [ r , m ; load_4 ] ldrb\t%w0, %1
2438 }
2439 )
2441 ;; -------------------------------------------------------------------
2442 ;; Simple arithmetic
2443 ;; -------------------------------------------------------------------
2445 (define_expand "add<mode>3"
2446 [(set
2447 (match_operand:GPI 0 "register_operand")
2448 (plus:GPI (match_operand:GPI 1 "register_operand")
2449 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))]
2450 ""
2451 {
2452 /* If operands[1] is a subreg extract the inner RTX. */
2453 rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
2455 /* If the constant is too large for a single instruction and isn't frame
2456 based, split off the immediate so it is available for CSE. */
2457 if (!aarch64_plus_immediate (operands[2], <MODE>mode)
2458 && !(TARGET_SVE && aarch64_sve_plus_immediate (operands[2], <MODE>mode))
2459 && can_create_pseudo_p ()
2460 && (!REG_P (op1)
2461 || !REGNO_PTR_FRAME_P (REGNO (op1))))
2462 operands[2] = force_reg (<MODE>mode, operands[2]);
2463 /* Some tunings prefer to avoid VL-based operations.
2464 Split off the poly immediate here. The rtx costs hook will reject attempts
2465 to combine them back. */
2466 else if (GET_CODE (operands[2]) == CONST_POLY_INT
2467 && can_create_pseudo_p ()
2468 && (aarch64_tune_params.extra_tuning_flags
2469 & AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS))
2470 operands[2] = force_reg (<MODE>mode, operands[2]);
2471 /* Expand polynomial additions now if the destination is the stack
2472 pointer, since we don't want to use that as a temporary. */
2473 else if (operands[0] == stack_pointer_rtx
2474 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
2475 {
2476 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
2477 operands[2], NULL_RTX, NULL_RTX);
2478 DONE;
2479 }
2480 })
2482 (define_insn "*add<mode>3_aarch64"
2483 [(set
2484 (match_operand:GPI 0 "register_operand")
2485 (plus:GPI
2486 (match_operand:GPI 1 "register_operand")
2487 (match_operand:GPI 2 "aarch64_pluslong_operand")))]
2488 ""
2489 {@ [ cons: =0 , 1 , 2 ; attrs: type , arch ]
2490 [ rk , %rk , I ; alu_imm , * ] add\t%<w>0, %<w>1, %2
2491 [ rk , rk , r ; alu_sreg , * ] add\t%<w>0, %<w>1, %<w>2
2492 [ w , w , w ; neon_add , simd ] add\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
2493 [ rk , rk , J ; alu_imm , * ] sub\t%<w>0, %<w>1, #%n2
2494 [ r , rk , Uaa ; multiple , * ] #
2495 [ r , 0 , Uai ; alu_imm , sve ] << aarch64_output_sve_scalar_inc_dec (operands[2]);
2496 [ rk , rk , Uav ; alu_imm , sve ] << aarch64_output_sve_addvl_addpl (operands[2]);
2497 [ rk , rk , UaV ; alu_imm , sme ] << aarch64_output_addsvl_addspl (operands[2]);
2498 }
2499 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
2500 )
2502 ;; zero_extend version of above
2503 (define_insn "*addsi3_aarch64_uxtw"
2504 [(set
2505 (match_operand:DI 0 "register_operand")
2506 (zero_extend:DI
2507 (plus:SI (match_operand:SI 1 "register_operand")
2508 (match_operand:SI 2 "aarch64_pluslong_operand"))))]
2509 ""
2510 {@ [ cons: =0 , 1 , 2 ; attrs: type ]
2511 [ rk , %rk , I ; alu_imm ] add\t%w0, %w1, %2
2512 [ rk , rk , r ; alu_sreg ] add\t%w0, %w1, %w2
2513 [ rk , rk , J ; alu_imm ] sub\t%w0, %w1, #%n2
2514 [ r , rk , Uaa ; multiple ] #
2515 }
2516 )
2518 ;; If there's a free register, and we can load the constant with a
2519 ;; single instruction, do so. This has a chance to improve scheduling.
2520 (define_peephole2
2521 [(match_scratch:GPI 3 "r")
2522 (set (match_operand:GPI 0 "register_operand")
2523 (plus:GPI
2524 (match_operand:GPI 1 "register_operand")
2525 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
2526 "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
2527 [(set (match_dup 3) (match_dup 2))
2528 (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
2529 )
2531 (define_peephole2
2532 [(match_scratch:SI 3 "r")
2533 (set (match_operand:DI 0 "register_operand")
2534 (zero_extend:DI
2535 (plus:SI
2536 (match_operand:SI 1 "register_operand")
2537 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
2538 "aarch64_move_imm (INTVAL (operands[2]), SImode)"
2539 [(set (match_dup 3) (match_dup 2))
2540 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
2541 )
2543 ;; After peephole2 has had a chance to run, split any remaining long
2544 ;; additions into two add immediates.
2545 (define_split
2546 [(set (match_operand:GPI 0 "register_operand")
2547 (plus:GPI
2548 (match_operand:GPI 1 "register_operand")
2549 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
2550 "epilogue_completed"
2551 [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
2552 (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
2553 {
2554 HOST_WIDE_INT i = INTVAL (operands[2]);
2555 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
2556 operands[3] = GEN_INT (i - s);
2557 operands[4] = GEN_INT (s);
2558 }
2559 )
2561 ;; Match addition of polynomial offsets that require one temporary, for which
2562 ;; we can use the early-clobbered destination register. This is a separate
2563 ;; pattern so that the early clobber doesn't affect register allocation
2564 ;; for other forms of addition. However, we still need to provide an
2565 ;; all-register alternative, in case the offset goes out of range after
2566 ;; elimination. For completeness we might as well provide all GPR-based
2567 ;; alternatives from the main pattern.
2568 ;;
2569 ;; We don't have a pattern for additions requiring two temporaries since at
2570 ;; present LRA doesn't allow new scratches to be added during elimination.
2571 ;; Such offsets should be rare anyway.
2572 ;;
2573 ;; ??? But if we added LRA support for new scratches, much of the ugliness
2574 ;; here would go away. We could just handle all polynomial constants in
2575 ;; this pattern.
2576 (define_insn_and_split "*add<mode>3_poly_1"
2577 [(set
2578 (match_operand:GPI 0 "register_operand")
2579 (plus:GPI
2580 (match_operand:GPI 1 "register_operand")
2581 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))]
2582 "TARGET_SVE && operands[0] != stack_pointer_rtx"
2583 {@ [ cons: =0 , 1 , 2 ; attrs: type ]
2584 [ r , %rk , I ; alu_imm ] add\t%<w>0, %<w>1, %2
2585 [ r , rk , r ; alu_sreg ] add\t%<w>0, %<w>1, %<w>2
2586 [ r , rk , J ; alu_imm ] sub\t%<w>0, %<w>1, #%n2
2587 [ r , rk , Uaa ; multiple ] #
2588 [ r , 0 , Uai ; alu_imm ] << aarch64_output_sve_scalar_inc_dec (operands[2]);
2589 [ r , rk , Uav ; alu_imm ] << aarch64_output_sve_addvl_addpl (operands[2]);
2590 [ &r , rk , Uat ; multiple ] #
2591 }
2592 "&& epilogue_completed
2593 && !reg_overlap_mentioned_p (operands[0], operands[1])
2594 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
2595 [(const_int 0)]
2596 {
2597 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
2598 operands[2], operands[0], NULL_RTX);
2599 DONE;
2600 }
2601 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
2602 )
2604 (define_split
2605 [(set (match_operand:DI 0 "register_operand")
2606 (zero_extend:DI
2607 (plus:SI
2608 (match_operand:SI 1 "register_operand")
2609 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
2610 "epilogue_completed"
2611 [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
2612 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
2613 {
2614 HOST_WIDE_INT i = INTVAL (operands[2]);
2615 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
2616 operands[3] = GEN_INT (i - s);
2617 operands[4] = GEN_INT (s);
2618 operands[5] = gen_lowpart (SImode, operands[0]);
2619 }
2620 )
2622 (define_expand "addv<mode>4"
2623 [(match_operand:GPI 0 "register_operand")
2624 (match_operand:GPI 1 "register_operand")
2625 (match_operand:GPI 2 "aarch64_plus_operand")
2626 (label_ref (match_operand 3 "" ""))]
2627 ""
2628 {
2629 if (CONST_INT_P (operands[2]))
2630 emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
2631 operands[2]));
2632 else
2633 emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
2634 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2636 DONE;
2637 })
2639 (define_expand "uaddv<mode>4"
2640 [(match_operand:GPI 0 "register_operand")
2641 (match_operand:GPI 1 "register_operand")
2642 (match_operand:GPI 2 "register_operand")
2643 (label_ref (match_operand 3 "" ""))]
2644 ""
2645 {
2646 emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
2647 aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]);
2649 DONE;
2650 })
2652 (define_expand "addti3"
2653 [(set (match_operand:TI 0 "register_operand")
2654 (plus:TI (match_operand:TI 1 "register_operand")
2655 (match_operand:TI 2 "aarch64_reg_or_imm")))]
2656 ""
2657 {
2658 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2660 aarch64_addti_scratch_regs (operands[1], operands[2],
2661 &low_dest, &op1_low, &op2_low,
2662 &high_dest, &op1_high, &op2_high);
2664 if (op2_low == const0_rtx)
2665 {
2666 low_dest = op1_low;
2667 if (!aarch64_pluslong_operand (op2_high, DImode))
2668 op2_high = force_reg (DImode, op2_high);
2669 emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
2670 }
2671 else
2672 {
2673 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2674 force_reg (DImode, op2_low)));
2675 emit_insn (gen_adddi3_carryin (high_dest, op1_high,
2676 force_reg (DImode, op2_high)));
2677 }
2679 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2680 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2682 DONE;
2683 })
2685 (define_expand "addvti4"
2686 [(match_operand:TI 0 "register_operand")
2687 (match_operand:TI 1 "register_operand")
2688 (match_operand:TI 2 "aarch64_reg_or_imm")
2689 (label_ref (match_operand 3 "" ""))]
2690 ""
2691 {
2692 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2694 aarch64_addti_scratch_regs (operands[1], operands[2],
2695 &low_dest, &op1_low, &op2_low,
2696 &high_dest, &op1_high, &op2_high);
2698 if (op2_low == const0_rtx)
2699 {
2700 low_dest = op1_low;
2701 emit_insn (gen_adddi3_compareV (high_dest, op1_high,
2702 force_reg (DImode, op2_high)));
2703 }
2704 else
2705 {
2706 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2707 force_reg (DImode, op2_low)));
2708 emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
2709 force_reg (DImode, op2_high)));
2710 }
2712 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2713 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2715 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2716 DONE;
2717 })
2719 (define_expand "uaddvti4"
2720 [(match_operand:TI 0 "register_operand")
2721 (match_operand:TI 1 "register_operand")
2722 (match_operand:TI 2 "aarch64_reg_or_imm")
2723 (label_ref (match_operand 3 "" ""))]
2724 ""
2725 {
2726 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2728 aarch64_addti_scratch_regs (operands[1], operands[2],
2729 &low_dest, &op1_low, &op2_low,
2730 &high_dest, &op1_high, &op2_high);
2732 if (op2_low == const0_rtx)
2733 {
2734 low_dest = op1_low;
2735 emit_insn (gen_adddi3_compareC (high_dest, op1_high,
2736 force_reg (DImode, op2_high)));
2737 }
2738 else
2739 {
2740 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2741 force_reg (DImode, op2_low)));
2742 emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
2743 force_reg (DImode, op2_high)));
2744 }
2746 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2747 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2749 aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]);
2750 DONE;
2751 })
2753 (define_insn "add<mode>3_compare0"
2754 [(set (reg:CC_NZ CC_REGNUM)
2755 (compare:CC_NZ
2756 (plus:GPI (match_operand:GPI 1 "register_operand")
2757 (match_operand:GPI 2 "aarch64_plus_operand"))
2758 (const_int 0)))
2759 (set (match_operand:GPI 0 "register_operand")
2760 (plus:GPI (match_dup 1) (match_dup 2)))]
2761 ""
2762 {@ [ cons: =0 , 1 , 2 ; attrs: type ]
2763 [ r , %rk , r ; alus_sreg ] adds\t%<w>0, %<w>1, %<w>2
2764 [ r , rk , I ; alus_imm ] adds\t%<w>0, %<w>1, %2
2765 [ r , rk , J ; alus_imm ] subs\t%<w>0, %<w>1, #%n2
2766 }
2767 )
2769 ;; zero_extend version of above
2770 (define_insn "*addsi3_compare0_uxtw"
2771 [(set (reg:CC_NZ CC_REGNUM)
2772 (compare:CC_NZ
2773 (plus:SI (match_operand:SI 1 "register_operand")
2774 (match_operand:SI 2 "aarch64_plus_operand"))
2775 (const_int 0)))
2776 (set (match_operand:DI 0 "register_operand")
2777 (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2778 ""
2779 {@ [ cons: =0 , 1 , 2 ; attrs: type ]
2780 [ r , %rk , r ; alus_sreg ] adds\t%w0, %w1, %w2
2781 [ r , rk , I ; alus_imm ] adds\t%w0, %w1, %2
2782 [ r , rk , J ; alus_imm ] subs\t%w0, %w1, #%n2
2783 }
2784 )
2786 (define_insn "*add<mode>3_compareC_cconly"
2787 [(set (reg:CC_C CC_REGNUM)
2788 (compare:CC_C
2789 (plus:GPI
2790 (match_operand:GPI 0 "register_operand")
2791 (match_operand:GPI 1 "aarch64_plus_operand"))
2792 (match_dup 0)))]
2793 ""
2794 {@ [ cons: 0 , 1 ; attrs: type ]
2795 [ r , r ; alus_sreg ] cmn\t%<w>0, %<w>1
2796 [ r , I ; alus_imm ] cmn\t%<w>0, %1
2797 [ r , J ; alus_imm ] cmp\t%<w>0, #%n1
2798 }
2799 )
2801 (define_insn "add<mode>3_compareC"
2802 [(set (reg:CC_C CC_REGNUM)
2803 (compare:CC_C
2804 (plus:GPI
2805 (match_operand:GPI 1 "register_operand")
2806 (match_operand:GPI 2 "aarch64_plus_operand"))
2807 (match_dup 1)))
2808 (set (match_operand:GPI 0 "register_operand")
2809 (plus:GPI (match_dup 1) (match_dup 2)))]
2810 ""
2811 {@ [ cons: =0 , 1 , 2 ; attrs: type ]
2812 [ r , rk , r ; alus_sreg ] adds\t%<w>0, %<w>1, %<w>2
2813 [ r , rk , I ; alus_imm ] adds\t%<w>0, %<w>1, %2
2814 [ r , rk , J ; alus_imm ] subs\t%<w>0, %<w>1, #%n2
2815 }
2816 )
2818 (define_insn "*add<mode>3_compareV_cconly_imm"
2819 [(set (reg:CC_V CC_REGNUM)
2820 (compare:CC_V
2821 (plus:<DWI>
2822 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand"))
2823 (match_operand:<DWI> 1 "const_scalar_int_operand"))
2824 (sign_extend:<DWI>
2825 (plus:GPI
2826 (match_dup 0)
2827 (match_operand:GPI 2 "aarch64_plus_immediate")))))]
2828 "INTVAL (operands[1]) == INTVAL (operands[2])"
2829 {@ [ cons: 0 , 2 ]
2830 [ r , I ] cmn\t%<w>0, %<w>1
2831 [ r , J ] cmp\t%<w>0, #%n1
2832 }
2833 [(set_attr "type" "alus_imm")]
2834 )
2836 (define_insn "*add<mode>3_compareV_cconly"
2837 [(set (reg:CC_V CC_REGNUM)
2838 (compare:CC_V
2839 (plus:<DWI>
2840 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2841 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2842 (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2843 ""
2844 "cmn\\t%<w>0, %<w>1"
2845 [(set_attr "type" "alus_sreg")]
2846 )
2848 (define_insn "add<mode>3_compareV_imm"
2849 [(set (reg:CC_V CC_REGNUM)
2850 (compare:CC_V
2851 (plus:<DWI>
2852 (sign_extend:<DWI>
2853 (match_operand:GPI 1 "register_operand"))
2854 (match_operand:GPI 2 "aarch64_plus_immediate"))
2855 (sign_extend:<DWI>
2856 (plus:GPI (match_dup 1) (match_dup 2)))))
2857 (set (match_operand:GPI 0 "register_operand")
2858 (plus:GPI (match_dup 1) (match_dup 2)))]
2859 ""
2860 {@ [ cons: =0 , 1 , 2 ; attrs: type ]
2861 [ r , rk , I ; alus_imm ] adds\t%<w>0, %<w>1, %<w>2
2862 [ r , rk , J ; alus_imm ] subs\t%<w>0, %<w>1, #%n2
2863 }
2864 )
2866 (define_insn "add<mode>3_compareV"
2867 [(set (reg:CC_V CC_REGNUM)
2868 (compare:CC_V
2869 (plus:<DWI>
2870 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk"))
2871 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2872 (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2873 (set (match_operand:GPI 0 "register_operand" "=r")
2874 (plus:GPI (match_dup 1) (match_dup 2)))]
2875 ""
2876 "adds\\t%<w>0, %<w>1, %<w>2"
2877 [(set_attr "type" "alus_sreg")]
2878 )
2880 (define_insn "*adds_shift_imm_<mode>"
2881 [(set (reg:CC_NZ CC_REGNUM)
2882 (compare:CC_NZ
2883 (plus:GPI (ASHIFT:GPI
2884 (match_operand:GPI 1 "register_operand" "r")
2885 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2886 (match_operand:GPI 3 "register_operand" "r"))
2887 (const_int 0)))
2888 (set (match_operand:GPI 0 "register_operand" "=r")
2889 (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2890 (match_dup 3)))]
2891 ""
2892 "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2893 [(set_attr "type" "alus_shift_imm")]
2894 )
2896 (define_insn "*subs_shift_imm_<mode>"
2897 [(set (reg:CC_NZ CC_REGNUM)
2898 (compare:CC_NZ
2899 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2900 (ASHIFT:GPI
2901 (match_operand:GPI 2 "register_operand" "r")
2902 (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2903 (const_int 0)))
2904 (set (match_operand:GPI 0 "register_operand" "=r")
2905 (minus:GPI (match_dup 1)
2906 (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2907 ""
2908 "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2909 [(set_attr "type" "alus_shift_imm")]
2910 )
2912 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2913 [(set (reg:CC_NZ CC_REGNUM)
2914 (compare:CC_NZ
2915 (plus:GPI
2916 (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2917 (match_operand:GPI 2 "register_operand" "rk"))
2918 (const_int 0)))
2919 (set (match_operand:GPI 0 "register_operand" "=r")
2920 (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2921 ""
2922 "adds\\t%<GPI:w>0, %<GPI:w>2, %w1, <su>xt<ALLX:size>"
2923 [(set_attr "type" "alus_ext")]
2924 )
2926 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2927 [(set (reg:CC_NZ CC_REGNUM)
2928 (compare:CC_NZ
2929 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2930 (ANY_EXTEND:GPI
2931 (match_operand:ALLX 2 "register_operand" "r")))
2932 (const_int 0)))
2933 (set (match_operand:GPI 0 "register_operand" "=r")
2934 (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2935 ""
2936 "subs\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size>"
2937 [(set_attr "type" "alus_ext")]
2938 )
2940 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2941 [(set (reg:CC_NZ CC_REGNUM)
2942 (compare:CC_NZ
2943 (plus:GPI (ashift:GPI
2944 (ANY_EXTEND:GPI
2945 (match_operand:ALLX 1 "register_operand" "r"))
2946 (match_operand 2 "aarch64_imm3" "Ui3"))
2947 (match_operand:GPI 3 "register_operand" "rk"))
2948 (const_int 0)))
2949 (set (match_operand:GPI 0 "register_operand" "=rk")
2950 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2951 (match_dup 2))
2952 (match_dup 3)))]
2953 ""
2954 "adds\\t%<GPI:w>0, %<GPI:w>3, %w1, <su>xt<ALLX:size> %2"
2955 [(set_attr "type" "alus_ext")]
2956 )
2958 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2959 [(set (reg:CC_NZ CC_REGNUM)
2960 (compare:CC_NZ
2961 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2962 (ashift:GPI
2963 (ANY_EXTEND:GPI
2964 (match_operand:ALLX 2 "register_operand" "r"))
2965 (match_operand 3 "aarch64_imm3" "Ui3")))
2966 (const_int 0)))
2967 (set (match_operand:GPI 0 "register_operand" "=rk")
2968 (minus:GPI (match_dup 1)
2969 (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2970 (match_dup 3))))]
2971 ""
2972 "subs\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size> %3"
2973 [(set_attr "type" "alus_ext")]
2974 )
2976 (define_insn "*add<mode>3nr_compare0"
2977 [(set (reg:CC_NZ CC_REGNUM)
2978 (compare:CC_NZ
2979 (plus:GPI (match_operand:GPI 0 "register_operand")
2980 (match_operand:GPI 1 "aarch64_plus_operand"))
2981 (const_int 0)))]
2982 ""
2983 {@ [ cons: 0 , 1 ; attrs: type ]
2984 [ %r , r ; alus_sreg ] cmn\t%<w>0, %<w>1
2985 [ r , I ; alus_imm ] cmn\t%<w>0, %1
2986 [ r , J ; alus_imm ] cmp\t%<w>0, #%n1
2987 }
2988 )
2990 (define_insn "aarch64_sub<mode>_compare0"
2991 [(set (reg:CC_NZ CC_REGNUM)
2992 (compare:CC_NZ
2993 (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2994 (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2995 (const_int 0)))]
2996 ""
2997 "cmp\\t%<w>0, %<w>1"
2998 [(set_attr "type" "alus_sreg")]
2999 )
3001 (define_insn "*compare_neg<mode>"
3002 [(set (reg:CC_Z CC_REGNUM)
3003 (compare:CC_Z
3004 (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
3005 (match_operand:GPI 1 "register_operand" "r")))]
3006 ""
3007 "cmn\\t%<w>1, %<w>0"
3008 [(set_attr "type" "alus_sreg")]
3009 )
3011 (define_insn "*add_<shift>_<mode>"
3012 [(set (match_operand:GPI 0 "register_operand" "=r")
3013 (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
3014 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3015 (match_operand:GPI 3 "register_operand" "r")))]
3016 ""
3017 "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
3018 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
3019 )
3021 ;; zero_extend version of above
3022 (define_insn "*add_<shift>_si_uxtw"
3023 [(set (match_operand:DI 0 "register_operand" "=r")
3024 (zero_extend:DI
3025 (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
3026 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3027 (match_operand:SI 3 "register_operand" "r"))))]
3028 ""
3029 "add\\t%w0, %w3, %w1, <shift> %2"
3030 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
3031 )
3033 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
3034 [(set (match_operand:GPI 0 "register_operand" "=rk")
3035 (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
3036 (match_operand:GPI 2 "register_operand" "r")))]
3037 ""
3038 "add\\t%<GPI:w>0, %<GPI:w>2, %w1, <su>xt<ALLX:size>"
3039 [(set_attr "type" "alu_ext")]
3040 )
3042 ;; zero_extend version of above
3043 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
3044 [(set (match_operand:DI 0 "register_operand" "=rk")
3045 (zero_extend:DI
3046 (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
3047 (match_operand:GPI 2 "register_operand" "r"))))]
3048 ""
3049 "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
3050 [(set_attr "type" "alu_ext")]
3051 )
3053 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
3054 [(set (match_operand:GPI 0 "register_operand" "=rk")
3055 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
3056 (match_operand:ALLX 1 "register_operand" "r"))
3057 (match_operand 2 "aarch64_imm3" "Ui3"))
3058 (match_operand:GPI 3 "register_operand" "r")))]
3059 ""
3060 "add\\t%<GPI:w>0, %<GPI:w>3, %w1, <su>xt<ALLX:size> %2"
3061 [(set_attr "type" "alu_ext")]
3062 )
3064 ;; zero_extend version of above
3065 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
3066 [(set (match_operand:DI 0 "register_operand" "=rk")
3067 (zero_extend:DI
3068 (plus:SI (ashift:SI (ANY_EXTEND:SI
3069 (match_operand:SHORT 1 "register_operand" "r"))
3070 (match_operand 2 "aarch64_imm3" "Ui3"))
3071 (match_operand:SI 3 "register_operand" "r"))))]
3072 ""
3073 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
3074 [(set_attr "type" "alu_ext")]
3075 )
3077 (define_expand "add<mode>3_carryin"
3078 [(set (match_operand:GPI 0 "register_operand")
3079 (plus:GPI
3080 (plus:GPI
3081 (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0))
3082 (match_operand:GPI 1 "aarch64_reg_or_zero"))
3083 (match_operand:GPI 2 "aarch64_reg_or_zero")))]
3084 ""
3085 ""
3086 )
3088 ;; Note that add with carry with two zero inputs is matched by cset,
3089 ;; and that add with carry with one zero input is matched by cinc.
3091 (define_insn "*add<mode>3_carryin"
3092 [(set (match_operand:GPI 0 "register_operand" "=r")
3093 (plus:GPI
3094 (plus:GPI
3095 (match_operand:GPI 3 "aarch64_carry_operation" "")
3096 (match_operand:GPI 1 "register_operand" "r"))
3097 (match_operand:GPI 2 "register_operand" "r")))]
3098 ""
3099 "adc\\t%<w>0, %<w>1, %<w>2"
3100 [(set_attr "type" "adc_reg")]
3101 )
3103 ;; zero_extend version of above
3104 (define_insn "*addsi3_carryin_uxtw"
3105 [(set (match_operand:DI 0 "register_operand" "=r")
3106 (zero_extend:DI
3107 (plus:SI
3108 (plus:SI
3109 (match_operand:SI 3 "aarch64_carry_operation" "")
3110 (match_operand:SI 1 "register_operand" "r"))
3111 (match_operand:SI 2 "register_operand" "r"))))]
3112 ""
3113 "adc\\t%w0, %w1, %w2"
3114 [(set_attr "type" "adc_reg")]
3115 )
3117 (define_expand "add<mode>3_carryinC"
3118 [(parallel
3119 [(set (match_dup 3)
3120 (compare:CC_ADC
3121 (plus:<DWI>
3122 (plus:<DWI>
3123 (match_dup 4)
3124 (zero_extend:<DWI>
3125 (match_operand:GPI 1 "register_operand")))
3126 (zero_extend:<DWI>
3127 (match_operand:GPI 2 "register_operand")))
3128 (match_dup 6)))
3129 (set (match_operand:GPI 0 "register_operand")
3130 (plus:GPI
3131 (plus:GPI (match_dup 5) (match_dup 1))
3132 (match_dup 2)))])]
3133 ""
3134 {
3135 operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM);
3136 rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM);
3137 operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx);
3138 operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
3139 operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode)
3140 << GET_MODE_BITSIZE (<MODE>mode),
3141 TImode);
3142 })
3144 (define_insn "*add<mode>3_carryinC_zero"
3145 [(set (reg:CC_ADC CC_REGNUM)
3146 (compare:CC_ADC
3147 (plus:<DWI>
3148 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
3149 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
3150 (match_operand 4 "const_scalar_int_operand" "")))
3151 (set (match_operand:GPI 0 "register_operand" "=r")
3152 (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "")
3153 (match_dup 1)))]
3154 "rtx_mode_t (operands[4], <DWI>mode)
3155 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
3156 "adcs\\t%<w>0, %<w>1, <w>zr"
3157 [(set_attr "type" "adc_reg")]
3158 )
3160 (define_insn "*add<mode>3_carryinC"
3161 [(set (reg:CC_ADC CC_REGNUM)
3162 (compare:CC_ADC
3163 (plus:<DWI>
3164 (plus:<DWI>
3165 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
3166 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
3167 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
3168 (match_operand 5 "const_scalar_int_operand" "")))
3169 (set (match_operand:GPI 0 "register_operand" "=r")
3170 (plus:GPI
3171 (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "")
3172 (match_dup 1))
3173 (match_dup 2)))]
3174 "rtx_mode_t (operands[5], <DWI>mode)
3175 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
3176 "adcs\\t%<w>0, %<w>1, %<w>2"
3177 [(set_attr "type" "adc_reg")]
3178 )
3180 (define_expand "add<mode>3_carryinV"
3181 [(parallel
3182 [(set (reg:CC_V CC_REGNUM)
3183 (compare:CC_V
3184 (plus:<DWI>
3185 (plus:<DWI>
3186 (match_dup 3)
3187 (sign_extend:<DWI>
3188 (match_operand:GPI 1 "register_operand")))
3189 (sign_extend:<DWI>
3190 (match_operand:GPI 2 "register_operand")))
3191 (sign_extend:<DWI>
3192 (plus:GPI
3193 (plus:GPI (match_dup 4) (match_dup 1))
3194 (match_dup 2)))))
3195 (set (match_operand:GPI 0 "register_operand")
3196 (plus:GPI
3197 (plus:GPI (match_dup 4) (match_dup 1))
3198 (match_dup 2)))])]
3199 ""
3200 {
3201 rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
3202 operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx);
3203 operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx);
3204 })
3206 (define_insn "*add<mode>3_carryinV_zero"
3207 [(set (reg:CC_V CC_REGNUM)
3208 (compare:CC_V
3209 (plus:<DWI>
3210 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
3211 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
3212 (sign_extend:<DWI>
3213 (plus:GPI
3214 (match_operand:GPI 3 "aarch64_carry_operation" "")
3215 (match_dup 1)))))
3216 (set (match_operand:GPI 0 "register_operand" "=r")
3217 (plus:GPI (match_dup 3) (match_dup 1)))]
3218 ""
3219 "adcs\\t%<w>0, %<w>1, <w>zr"
3220 [(set_attr "type" "adc_reg")]
3221 )
3223 (define_insn "*add<mode>3_carryinV"
3224 [(set (reg:CC_V CC_REGNUM)
3225 (compare:CC_V
3226 (plus:<DWI>
3227 (plus:<DWI>
3228 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
3229 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
3230 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
3231 (sign_extend:<DWI>
3232 (plus:GPI
3233 (plus:GPI
3234 (match_operand:GPI 4 "aarch64_carry_operation" "")
3235 (match_dup 1))
3236 (match_dup 2)))))
3237 (set (match_operand:GPI 0 "register_operand" "=r")
3238 (plus:GPI
3239 (plus:GPI (match_dup 4) (match_dup 1))
3240 (match_dup 2)))]
3241 ""
3242 "adcs\\t%<w>0, %<w>1, %<w>2"
3243 [(set_attr "type" "adc_reg")]
3244 )
3246 (define_insn "*add_uxt<mode>_shift2"
3247 [(set (match_operand:GPI 0 "register_operand" "=rk")
3248 (plus:GPI (and:GPI
3249 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3250 (match_operand 2 "aarch64_imm3" "Ui3"))
3251 (match_operand 3 "const_int_operand" "n"))
3252 (match_operand:GPI 4 "register_operand" "r")))]
3253 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
3254 "*
3255 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
3256 INTVAL (operands[3])));
3257 return \"add\t%<w>0, %<w>4, %w1, uxt%e3 %2\";"
3258 [(set_attr "type" "alu_ext")]
3259 )
3261 ;; zero_extend version of above
3262 (define_insn "*add_uxtsi_shift2_uxtw"
3263 [(set (match_operand:DI 0 "register_operand" "=rk")
3264 (zero_extend:DI
3265 (plus:SI (and:SI
3266 (ashift:SI (match_operand:SI 1 "register_operand" "r")
3267 (match_operand 2 "aarch64_imm3" "Ui3"))
3268 (match_operand 3 "const_int_operand" "n"))
3269 (match_operand:SI 4 "register_operand" "r"))))]
3270 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
3271 "*
3272 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3273 INTVAL (operands[3])));
3274 return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
3275 [(set_attr "type" "alu_ext")]
3276 )
3278 (define_insn "subsi3"
3279 [(set (match_operand:SI 0 "register_operand" "=rk")
3280 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3281 (match_operand:SI 2 "register_operand" "r")))]
3282 ""
3283 "sub\\t%w0, %w1, %w2"
3284 [(set_attr "type" "alu_sreg")]
3285 )
3287 ;; zero_extend version of above
3288 (define_insn "*subsi3_uxtw"
3289 [(set (match_operand:DI 0 "register_operand" "=rk")
3290 (zero_extend:DI
3291 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3292 (match_operand:SI 2 "register_operand" "r"))))]
3293 ""
3294 "sub\\t%w0, %w1, %w2"
3295 [(set_attr "type" "alu_sreg")]
3296 )
3298 (define_insn "subdi3"
3299 [(set (match_operand:DI 0 "register_operand")
3300 (minus:DI (match_operand:DI 1 "register_operand")
3301 (match_operand:DI 2 "register_operand")))]
3302 ""
3303 {@ [ cons: =0 , 1 , 2 ; attrs: type , arch ]
3304 [ rk , rk , r ; alu_sreg , * ] sub\t%x0, %x1, %x2
3305 [ w , w , w ; neon_sub , simd ] sub\t%d0, %d1, %d2
3306 }
3307 )
3309 (define_expand "subv<GPI:mode>4"
3310 [(match_operand:GPI 0 "register_operand")
3311 (match_operand:GPI 1 "register_operand")
3312 (match_operand:GPI 2 "aarch64_plus_operand")
3313 (label_ref (match_operand 3 "" ""))]
3314 ""
3315 {
3316 if (CONST_INT_P (operands[2]))
3317 emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
3318 else
3319 emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
3320 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
3322 DONE;
3323 })
3325 (define_insn "subv<GPI:mode>_insn"
3326 [(set (reg:CC_V CC_REGNUM)
3327 (compare:CC_V
3328 (sign_extend:<DWI>
3329 (minus:GPI
3330 (match_operand:GPI 1 "register_operand" "rk")
3331 (match_operand:GPI 2 "register_operand" "r")))
3332 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
3333 (sign_extend:<DWI> (match_dup 2)))))
3334 (set (match_operand:GPI 0 "register_operand" "=r")
3335 (minus:GPI (match_dup 1) (match_dup 2)))]
3336 ""
3337 "subs\\t%<w>0, %<w>1, %<w>2"
3338 [(set_attr "type" "alus_sreg")]
3339 )
3341 (define_insn "subv<GPI:mode>_imm"
3342 [(set (reg:CC_V CC_REGNUM)
3343 (compare:CC_V
3344 (sign_extend:<DWI>
3345 (minus:GPI
3346 (match_operand:GPI 1 "register_operand")
3347 (match_operand:GPI 2 "aarch64_plus_immediate")))
3348 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
3349 (match_dup 2))))
3350 (set (match_operand:GPI 0 "register_operand")
3351 (minus:GPI (match_dup 1) (match_dup 2)))]
3352 ""
3353 {@ [ cons: =0 , 1 , 2 ]
3354 [ r , rk , I ] subs\t%<w>0, %<w>1, %2
3355 [ r , rk , J ] adds\t%<w>0, %<w>1, #%n2
3356 }
3357 [(set_attr "type" "alus_sreg")]
3358 )
3360 (define_expand "negv<GPI:mode>3"
3361 [(match_operand:GPI 0 "register_operand")
3362 (match_operand:GPI 1 "register_operand")
3363 (label_ref (match_operand 2 "" ""))]
3364 ""
3365 {
3366 emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
3367 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
3369 DONE;
3370 }
3371 )
3373 (define_insn "negv<GPI:mode>_insn"
3374 [(set (reg:CC_V CC_REGNUM)
3375 (compare:CC_V
3376 (sign_extend:<DWI>
3377 (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
3378 (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
3379 (set (match_operand:GPI 0 "register_operand" "=r")
3380 (neg:GPI (match_dup 1)))]
3381 ""
3382 "negs\\t%<w>0, %<w>1"
3383 [(set_attr "type" "alus_sreg")]
3384 )
3386 (define_insn "negv<GPI:mode>_cmp_only"
3387 [(set (reg:CC_V CC_REGNUM)
3388 (compare:CC_V
3389 (sign_extend:<DWI>
3390 (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
3391 (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
3392 ""
3393 "negs\\t%<w>zr, %<w>0"
3394 [(set_attr "type" "alus_sreg")]
3395 )
3397 (define_insn "*cmpv<GPI:mode>_insn"
3398 [(set (reg:CC_V CC_REGNUM)
3399 (compare:CC_V
3400 (sign_extend:<DWI>
3401 (minus:GPI (match_operand:GPI 0 "register_operand")
3402 (match_operand:GPI 1 "aarch64_plus_operand")))
3403 (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
3404 (sign_extend:<DWI> (match_dup 1)))))]
3405 ""
3406 {@ [ cons: 0 , 1 ]
3407 [ r , r ] cmp\t%<w>0, %<w>1
3408 [ r , I ] cmp\t%<w>0, %1
3409 [ r , J ] cmp\t%<w>0, #%n1
3410 }
3411 [(set_attr "type" "alus_sreg")]
3412 )
3414 (define_expand "usubv<mode>4"
3415 [(match_operand:GPI 0 "register_operand")
3416 (match_operand:GPI 1 "aarch64_reg_or_zero")
3417 (match_operand:GPI 2 "aarch64_reg_or_zero")
3418 (label_ref (match_operand 3 "" ""))]
3419 ""
3420 {
3421 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
3422 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
3424 DONE;
3425 })
3427 (define_expand "subti3"
3428 [(set (match_operand:TI 0 "register_operand")
3429 (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero")
3430 (match_operand:TI 2 "register_operand")))]
3431 ""
3432 {
3433 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
3435 aarch64_subvti_scratch_regs (operands[1], operands[2],
3436 &low_dest, &op1_low, &op2_low,
3437 &high_dest, &op1_high, &op2_high);
3439 emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
3440 emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
3442 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
3443 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
3444 DONE;
3445 })
3447 (define_expand "subvti4"
3448 [(match_operand:TI 0 "register_operand")
3449 (match_operand:TI 1 "register_operand")
3450 (match_operand:TI 2 "aarch64_reg_or_imm")
3451 (label_ref (match_operand 3 "" ""))]
3452 ""
3453 {
3454 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
3456 aarch64_subvti_scratch_regs (operands[1], operands[2],
3457 &low_dest, &op1_low, &op2_low,
3458 &high_dest, &op1_high, &op2_high);
3459 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
3460 high_dest, op1_high, op2_high, false);
3462 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
3463 DONE;
3464 })
3466 (define_expand "usubvti4"
3467 [(match_operand:TI 0 "register_operand")
3468 (match_operand:TI 1 "register_operand")
3469 (match_operand:TI 2 "aarch64_reg_or_imm")
3470 (label_ref (match_operand 3 "" ""))]
3471 ""
3472 {
3473 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
3475 aarch64_subvti_scratch_regs (operands[1], operands[2],
3476 &low_dest, &op1_low, &op2_low,
3477 &high_dest, &op1_high, &op2_high);
3478 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
3479 high_dest, op1_high, op2_high, true);
3481 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
3482 DONE;
3483 })
3485 (define_expand "negvti3"
3486 [(match_operand:TI 0 "register_operand")
3487 (match_operand:TI 1 "register_operand")
3488 (label_ref (match_operand 2 "" ""))]
3489 ""
3490 {
3491 emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
3492 gen_lowpart (DImode, operands[1])));
3493 emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
3494 gen_highpart (DImode, operands[1])));
3495 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
3497 DONE;
3498 }
3499 )
3501 (define_insn "negdi_carryout"
3502 [(set (reg:CC CC_REGNUM)
3503 (compare:CC
3504 (const_int 0) (match_operand:DI 1 "register_operand" "r")))
3505 (set (match_operand:DI 0 "register_operand" "=r")
3506 (neg:DI (match_dup 1)))]
3507 ""
3508 "negs\\t%0, %1"
3509 [(set_attr "type" "alus_sreg")]
3510 )
3512 (define_insn "negvdi_carryinV"
3513 [(set (reg:CC_V CC_REGNUM)
3514 (compare:CC_V
3515 (neg:TI (plus:TI
3516 (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
3517 (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
3518 (sign_extend:TI
3519 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
3520 (match_dup 1))))))
3521 (set (match_operand:DI 0 "register_operand" "=r")
3522 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
3523 (match_dup 1))))]
3524 ""
3525 "ngcs\\t%0, %1"
3526 [(set_attr "type" "alus_sreg")]
3527 )
3529 (define_insn "*sub<mode>3_compare0"
3530 [(set (reg:CC_NZ CC_REGNUM)
3531 (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3532 (match_operand:GPI 2 "register_operand" "r"))
3533 (const_int 0)))
3534 (set (match_operand:GPI 0 "register_operand" "=r")
3535 (minus:GPI (match_dup 1) (match_dup 2)))]
3536 ""
3537 "subs\\t%<w>0, %<w>1, %<w>2"
3538 [(set_attr "type" "alus_sreg")]
3539 )
3541 ;; zero_extend version of above
3542 (define_insn "*subsi3_compare0_uxtw"
3543 [(set (reg:CC_NZ CC_REGNUM)
3544 (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk")
3545 (match_operand:SI 2 "register_operand" "r"))
3546 (const_int 0)))
3547 (set (match_operand:DI 0 "register_operand" "=r")
3548 (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
3549 ""
3550 "subs\\t%w0, %w1, %w2"
3551 [(set_attr "type" "alus_sreg")]
3552 )
3554 (define_insn "sub<mode>3_compare1_imm"
3555 [(set (reg:CC CC_REGNUM)
3556 (compare:CC
3557 (match_operand:GPI 1 "register_operand")
3558 (match_operand:GPI 2 "aarch64_plus_immediate")))
3559 (set (match_operand:GPI 0 "register_operand")
3560 (plus:GPI
3561 (match_dup 1)
3562 (match_operand:GPI 3 "aarch64_plus_immediate")))]
3563 "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
3564 {@ [ cons: =0 , 1 , 2 , 3 ]
3565 [ r , rk , I , J ] subs\t%<w>0, %<w>1, %2
3566 [ r , rk , J , I ] adds\t%<w>0, %<w>1, #%n2
3567 }
3568 [(set_attr "type" "alus_imm")]
3569 )
3571 (define_insn "sub<mode>3_compare1"
3572 [(set (reg:CC CC_REGNUM)
3573 (compare:CC
3574 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
3575 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
3576 (set (match_operand:GPI 0 "register_operand" "=r")
3577 (minus:GPI (match_dup 1) (match_dup 2)))]
3578 ""
3579 "subs\\t%<w>0, %<w>1, %<w>2"
3580 [(set_attr "type" "alus_sreg")]
3581 )
3583 (define_peephole2
3584 [(set (match_operand:GPI 0 "aarch64_general_reg")
3585 (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3586 (match_operand:GPI 2 "aarch64_reg_or_zero")))
3587 (set (reg:CC CC_REGNUM)
3588 (compare:CC
3589 (match_dup 1)
3590 (match_dup 2)))]
3591 "!reg_overlap_mentioned_p (operands[0], operands[1])
3592 && !reg_overlap_mentioned_p (operands[0], operands[2])"
3593 [(const_int 0)]
3594 {
3595 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3596 operands[2]));
3597 DONE;
3598 }
3599 )
3601 ;; Same as the above peephole but with the compare and minus in
3602 ;; swapped order. The restriction on overlap between operand 0
3603 ;; and operands 1 and 2 doesn't apply here.
3604 (define_peephole2
3605 [(set (reg:CC CC_REGNUM)
3606 (compare:CC
3607 (match_operand:GPI 1 "aarch64_reg_or_zero")
3608 (match_operand:GPI 2 "aarch64_reg_or_zero")))
3609 (set (match_operand:GPI 0 "aarch64_general_reg")
3610 (minus:GPI (match_dup 1)
3611 (match_dup 2)))]
3612 ""
3613 [(const_int 0)]
3614 {
3615 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3616 operands[2]));
3617 DONE;
3618 }
3619 )
3621 (define_peephole2
3622 [(set (match_operand:GPI 0 "aarch64_general_reg")
3623 (plus:GPI (match_operand:GPI 1 "register_operand")
3624 (match_operand:GPI 2 "aarch64_plus_immediate")))
3625 (set (reg:CC CC_REGNUM)
3626 (compare:CC
3627 (match_dup 1)
3628 (match_operand:GPI 3 "const_int_operand")))]
3629 "!reg_overlap_mentioned_p (operands[0], operands[1])
3630 && INTVAL (operands[3]) == -INTVAL (operands[2])"
3631 [(const_int 0)]
3632 {
3633 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3634 operands[3], operands[2]));
3635 DONE;
3636 }
3637 )
3639 ;; Same as the above peephole but with the compare and minus in
3640 ;; swapped order. The restriction on overlap between operand 0
3641 ;; and operands 1 doesn't apply here.
3642 (define_peephole2
3643 [(set (reg:CC CC_REGNUM)
3644 (compare:CC
3645 (match_operand:GPI 1 "register_operand")
3646 (match_operand:GPI 3 "const_int_operand")))
3647 (set (match_operand:GPI 0 "aarch64_general_reg")
3648 (plus:GPI (match_dup 1)
3649 (match_operand:GPI 2 "aarch64_plus_immediate")))]
3650 "INTVAL (operands[3]) == -INTVAL (operands[2])"
3651 [(const_int 0)]
3652 {
3653 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3654 operands[3], operands[2]));
3655 DONE;
3656 }
3657 )
3659 (define_insn "*sub_<shift>_<mode>"
3660 [(set (match_operand:GPI 0 "register_operand" "=r")
3661 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3662 (ASHIFT:GPI
3663 (match_operand:GPI 1 "register_operand" "r")
3664 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3665 ""
3666 "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
3667 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
3668 )
3670 ;; zero_extend version of above
3671 (define_insn "*sub_<shift>_si_uxtw"
3672 [(set (match_operand:DI 0 "register_operand" "=r")
3673 (zero_extend:DI
3674 (minus:SI (match_operand:SI 3 "register_operand" "r")
3675 (ASHIFT:SI
3676 (match_operand:SI 1 "register_operand" "r")
3677 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3678 ""
3679 "sub\\t%w0, %w3, %w1, <shift> %2"
3680 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
3681 )
3683 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3684 [(set (match_operand:GPI 0 "register_operand" "=rk")
3685 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3686 (ANY_EXTEND:GPI
3687 (match_operand:ALLX 2 "register_operand" "r"))))]
3688 ""
3689 "sub\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size>"
3690 [(set_attr "type" "alu_ext")]
3691 )
3693 ;; zero_extend version of above
3694 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3695 [(set (match_operand:DI 0 "register_operand" "=rk")
3696 (zero_extend:DI
3697 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3698 (ANY_EXTEND:SI
3699 (match_operand:SHORT 2 "register_operand" "r")))))]
3700 ""
3701 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
3702 [(set_attr "type" "alu_ext")]
3703 )
3705 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3706 [(set (match_operand:GPI 0 "register_operand" "=rk")
3707 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3708 (ashift:GPI (ANY_EXTEND:GPI
3709 (match_operand:ALLX 2 "register_operand" "r"))
3710 (match_operand 3 "aarch64_imm3" "Ui3"))))]
3711 ""
3712 "sub\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size> %3"
3713 [(set_attr "type" "alu_ext")]
3714 )
3716 ;; zero_extend version of above
3717 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3718 [(set (match_operand:DI 0 "register_operand" "=rk")
3719 (zero_extend:DI
3720 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3721 (ashift:SI (ANY_EXTEND:SI
3722 (match_operand:SHORT 2 "register_operand" "r"))
3723 (match_operand 3 "aarch64_imm3" "Ui3")))))]
3724 ""
3725 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
3726 [(set_attr "type" "alu_ext")]
3727 )
3729 ;; The hardware description is op1 + ~op2 + C.
3730 ;; = op1 + (-op2 + 1) + (1 - !C)
3731 ;; = op1 - op2 - 1 + 1 - !C
3732 ;; = op1 - op2 - !C.
3733 ;; We describe the latter.
3735 (define_insn "*sub<mode>3_carryin0"
3736 [(set (match_operand:GPI 0 "register_operand" "=r")
3737 (minus:GPI
3738 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3739 (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3740 ""
3741 "sbc\\t%<w>0, %<w>1, <w>zr"
3742 [(set_attr "type" "adc_reg")]
3743 )
3745 ;; zero_extend version of the above
3746 (define_insn "*subsi3_carryin_uxtw"
3747 [(set (match_operand:DI 0 "register_operand" "=r")
3748 (zero_extend:DI
3749 (minus:SI
3750 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3751 (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3752 ""
3753 "sbc\\t%w0, %w1, wzr"
3754 [(set_attr "type" "adc_reg")]
3755 )
3757 (define_expand "sub<mode>3_carryin"
3758 [(set (match_operand:GPI 0 "register_operand")
3759 (minus:GPI
3760 (minus:GPI
3761 (match_operand:GPI 1 "aarch64_reg_or_zero")
3762 (match_operand:GPI 2 "register_operand"))
3763 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3764 ""
3765 ""
3766 )
3768 (define_insn "*sub<mode>3_carryin"
3769 [(set (match_operand:GPI 0 "register_operand" "=r")
3770 (minus:GPI
3771 (minus:GPI
3772 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3773 (match_operand:GPI 2 "register_operand" "r"))
3774 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3776 ""
3777 "sbc\\t%<w>0, %<w>1, %<w>2"
3778 [(set_attr "type" "adc_reg")]
3779 )
3781 ;; zero_extend version of the above
3782 (define_insn "*subsi3_carryin_uxtw"
3783 [(set (match_operand:DI 0 "register_operand" "=r")
3784 (zero_extend:DI
3785 (minus:SI
3786 (minus:SI
3787 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3788 (match_operand:SI 2 "register_operand" "r"))
3789 (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3791 ""
3792 "sbc\\t%w0, %w1, %w2"
3793 [(set_attr "type" "adc_reg")]
3794 )
3796 (define_insn "*sub<mode>3_carryin_alt"
3797 [(set (match_operand:GPI 0 "register_operand" "=r")
3798 (minus:GPI
3799 (minus:GPI
3800 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3801 (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3802 (match_operand:GPI 2 "register_operand" "r")))]
3803 ""
3804 "sbc\\t%<w>0, %<w>1, %<w>2"
3805 [(set_attr "type" "adc_reg")]
3806 )
3808 ;; zero_extend version of the above
3809 (define_insn "*subsi3_carryin_alt_uxtw"
3810 [(set (match_operand:DI 0 "register_operand" "=r")
3811 (zero_extend:DI
3812 (minus:SI
3813 (minus:SI
3814 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3815 (match_operand:SI 3 "aarch64_borrow_operation" ""))
3816 (match_operand:SI 2 "register_operand" "r"))))]
3817 ""
3818 "sbc\\t%w0, %w1, %w2"
3819 [(set_attr "type" "adc_reg")]
3820 )
3822 (define_expand "usub<GPI:mode>3_carryinC"
3823 [(parallel
3824 [(set (reg:CC CC_REGNUM)
3825 (compare:CC
3826 (zero_extend:<DWI>
3827 (match_operand:GPI 1 "aarch64_reg_or_zero"))
3828 (plus:<DWI>
3829 (zero_extend:<DWI>
3830 (match_operand:GPI 2 "register_operand"))
3831 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3832 (set (match_operand:GPI 0 "register_operand")
3833 (minus:GPI
3834 (minus:GPI (match_dup 1) (match_dup 2))
3835 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3836 ""
3837 )
3839 (define_insn "*usub<GPI:mode>3_carryinC_z1"
3840 [(set (reg:CC CC_REGNUM)
3841 (compare:CC
3842 (const_int 0)
3843 (plus:<DWI>
3844 (zero_extend:<DWI>
3845 (match_operand:GPI 1 "register_operand" "r"))
3846 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3847 (set (match_operand:GPI 0 "register_operand" "=r")
3848 (minus:GPI
3849 (neg:GPI (match_dup 1))
3850 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3851 ""
3852 "sbcs\\t%<w>0, <w>zr, %<w>1"
3853 [(set_attr "type" "adc_reg")]
3854 )
3856 (define_insn "*usub<GPI:mode>3_carryinC_z2"
3857 [(set (reg:CC CC_REGNUM)
3858 (compare:CC
3859 (zero_extend:<DWI>
3860 (match_operand:GPI 1 "register_operand" "r"))
3861 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3862 (set (match_operand:GPI 0 "register_operand" "=r")
3863 (minus:GPI
3864 (match_dup 1)
3865 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3866 ""
3867 "sbcs\\t%<w>0, %<w>1, <w>zr"
3868 [(set_attr "type" "adc_reg")]
3869 )
3871 (define_insn "*usub<GPI:mode>3_carryinC"
3872 [(set (reg:CC CC_REGNUM)
3873 (compare:CC
3874 (zero_extend:<DWI>
3875 (match_operand:GPI 1 "register_operand" "r"))
3876 (plus:<DWI>
3877 (zero_extend:<DWI>
3878 (match_operand:GPI 2 "register_operand" "r"))
3879 (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3880 (set (match_operand:GPI 0 "register_operand" "=r")
3881 (minus:GPI
3882 (minus:GPI (match_dup 1) (match_dup 2))
3883 (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3884 ""
3885 "sbcs\\t%<w>0, %<w>1, %<w>2"
3886 [(set_attr "type" "adc_reg")]
3887 )
3889 (define_expand "sub<GPI:mode>3_carryinV"
3890 [(parallel
3891 [(set (reg:CC_V CC_REGNUM)
3892 (compare:CC_V
3893 (minus:<DWI>
3894 (sign_extend:<DWI>
3895 (match_operand:GPI 1 "aarch64_reg_or_zero"))
3896 (plus:<DWI>
3897 (sign_extend:<DWI>
3898 (match_operand:GPI 2 "register_operand"))
3899 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3900 (sign_extend:<DWI>
3901 (minus:GPI (match_dup 1)
3902 (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3903 (match_dup 2))))))
3904 (set (match_operand:GPI 0 "register_operand")
3905 (minus:GPI
3906 (minus:GPI (match_dup 1) (match_dup 2))
3907 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3908 ""
3909 )
3911 (define_insn "*sub<mode>3_carryinV_z2"
3912 [(set (reg:CC_V CC_REGNUM)
3913 (compare:CC_V
3914 (minus:<DWI>
3915 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3916 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3917 (sign_extend:<DWI>
3918 (minus:GPI (match_dup 1)
3919 (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3920 (set (match_operand:GPI 0 "register_operand" "=r")
3921 (minus:GPI
3922 (match_dup 1) (match_dup 3)))]
3923 ""
3924 "sbcs\\t%<w>0, %<w>1, <w>zr"
3925 [(set_attr "type" "adc_reg")]
3926 )
3928 (define_insn "*sub<mode>3_carryinV"
3929 [(set (reg:CC_V CC_REGNUM)
3930 (compare:CC_V
3931 (minus:<DWI>
3932 (sign_extend:<DWI>
3933 (match_operand:GPI 1 "register_operand" "r"))
3934 (plus:<DWI>
3935 (sign_extend:<DWI>
3936 (match_operand:GPI 2 "register_operand" "r"))
3937 (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3938 (sign_extend:<DWI>
3939 (minus:GPI
3940 (match_dup 1)
3941 (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3942 (match_dup 2))))))
3943 (set (match_operand:GPI 0 "register_operand" "=r")
3944 (minus:GPI
3945 (minus:GPI (match_dup 1) (match_dup 2))
3946 (match_dup 4)))]
3947 ""
3948 "sbcs\\t%<w>0, %<w>1, %<w>2"
3949 [(set_attr "type" "adc_reg")]
3950 )
3952 (define_insn "*sub_uxt<mode>_shift2"
3953 [(set (match_operand:GPI 0 "register_operand" "=rk")
3954 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3955 (and:GPI
3956 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3957 (match_operand 2 "aarch64_imm3" "Ui3"))
3958 (match_operand 3 "const_int_operand" "n"))))]
3959 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3960 "*
3961 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3962 INTVAL (operands[3])));
3963 return \"sub\t%<w>0, %<w>4, %w1, uxt%e3 %2\";"
3964 [(set_attr "type" "alu_ext")]
3965 )
3967 ;; zero_extend version of above
3968 (define_insn "*sub_uxtsi_shift2_uxtw"
3969 [(set (match_operand:DI 0 "register_operand" "=rk")
3970 (zero_extend:DI
3971 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3972 (and:SI
3973 (ashift:SI (match_operand:SI 1 "register_operand" "r")
3974 (match_operand 2 "aarch64_imm3" "Ui3"))
3975 (match_operand 3 "const_int_operand" "n")))))]
3976 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3977 "*
3978 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3979 INTVAL (operands[3])));
3980 return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3981 [(set_attr "type" "alu_ext")]
3982 )
3984 (define_insn "*aarch64_abs<mode>2_cssc_insn"
3985 [(set (match_operand:GPI 0 "register_operand" "=r")
3986 (abs:GPI (match_operand:GPI 1 "register_operand" "r")))]
3987 "TARGET_CSSC"
3988 "abs\\t%<w>0, %<w>1"
3989 [(set_attr "type" "alu_sreg")]
3990 )
3992 (define_expand "abs<mode>2"
3993 [(set (match_operand:GPI 0 "register_operand")
3994 (abs:GPI (match_operand:GPI 1 "register_operand")))]
3995 ""
3996 {
3997 if (!TARGET_CSSC)
3998 {
3999 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
4000 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
4001 emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
4002 DONE;
4003 }
4004 }
4005 )
4007 (define_insn "neg<mode>2"
4008 [(set (match_operand:GPI 0 "register_operand")
4009 (neg:GPI (match_operand:GPI 1 "register_operand")))]
4010 ""
4011 {@ [ cons: =0 , 1 ; attrs: type , arch ]
4012 [ r , r ; alu_sreg , * ] neg\t%<w>0, %<w>1
4013 [ w , w ; neon_neg<q> , simd ] neg\t%<rtn>0<vas>, %<rtn>1<vas>
4014 }
4015 )
4017 ;; zero_extend version of above
4018 (define_insn "*negsi2_uxtw"
4019 [(set (match_operand:DI 0 "register_operand" "=r")
4020 (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
4021 ""
4022 "neg\\t%w0, %w1"
4023 [(set_attr "type" "alu_sreg")]
4024 )
4026 (define_insn "*ngc<mode>"
4027 [(set (match_operand:GPI 0 "register_operand" "=r")
4028 (minus:GPI
4029 (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
4030 (match_operand:GPI 1 "register_operand" "r")))]
4031 ""
4032 "ngc\\t%<w>0, %<w>1"
4033 [(set_attr "type" "adc_reg")]
4034 )
4036 (define_insn "*ngcsi_uxtw"
4037 [(set (match_operand:DI 0 "register_operand" "=r")
4038 (zero_extend:DI
4039 (minus:SI
4040 (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
4041 (match_operand:SI 1 "register_operand" "r"))))]
4042 ""
4043 "ngc\\t%w0, %w1"
4044 [(set_attr "type" "adc_reg")]
4045 )
4047 (define_insn "neg<mode>2_compare0"
4048 [(set (reg:CC_NZ CC_REGNUM)
4049 (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
4050 (const_int 0)))
4051 (set (match_operand:GPI 0 "register_operand" "=r")
4052 (neg:GPI (match_dup 1)))]
4053 ""
4054 "negs\\t%<w>0, %<w>1"
4055 [(set_attr "type" "alus_sreg")]
4056 )
4058 ;; zero_extend version of above
4059 (define_insn "*negsi2_compare0_uxtw"
4060 [(set (reg:CC_NZ CC_REGNUM)
4061 (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
4062 (const_int 0)))
4063 (set (match_operand:DI 0 "register_operand" "=r")
4064 (zero_extend:DI (neg:SI (match_dup 1))))]
4065 ""
4066 "negs\\t%w0, %w1"
4067 [(set_attr "type" "alus_sreg")]
4068 )
4070 (define_insn "*neg_<shift><mode>3_compare0"
4071 [(set (reg:CC_NZ CC_REGNUM)
4072 (compare:CC_NZ
4073 (neg:GPI (ASHIFT:GPI
4074 (match_operand:GPI 1 "register_operand" "r")
4075 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4076 (const_int 0)))
4077 (set (match_operand:GPI 0 "register_operand" "=r")
4078 (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
4079 ""
4080 "negs\\t%<w>0, %<w>1, <shift> %2"
4081 [(set_attr "type" "alus_shift_imm")]
4082 )
4084 (define_insn "*neg_<shift>_<mode>2"
4085 [(set (match_operand:GPI 0 "register_operand" "=r")
4086 (neg:GPI (ASHIFT:GPI
4087 (match_operand:GPI 1 "register_operand" "r")
4088 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4089 ""
4090 "neg\\t%<w>0, %<w>1, <shift> %2"
4091 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
4092 )
4094 ;; zero_extend version of above
4095 (define_insn "*neg_<shift>_si2_uxtw"
4096 [(set (match_operand:DI 0 "register_operand" "=r")
4097 (zero_extend:DI
4098 (neg:SI (ASHIFT:SI
4099 (match_operand:SI 1 "register_operand" "r")
4100 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
4101 ""
4102 "neg\\t%w0, %w1, <shift> %2"
4103 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")]
4104 )
4106 (define_insn "*neg_asr_si2_extr"
4107 [(set (match_operand:SI 0 "register_operand" "=r")
4108 (neg:SI (match_operator:SI 4 "subreg_lowpart_operator"
4109 [(sign_extract:DI
4110 (match_operand:DI 1 "register_operand" "r")
4111 (match_operand 3 "aarch64_simd_shift_imm_offset_si" "n")
4112 (match_operand 2 "aarch64_simd_shift_imm_offset_si" "n"))])))]
4113 "INTVAL (operands[2]) + INTVAL (operands[3]) == 32"
4114 "neg\\t%w0, %w1, asr %2"
4115 [(set_attr "autodetect_type" "alu_shift_asr_op2")]
4116 )
4118 (define_insn "mul<mode>3"
4119 [(set (match_operand:GPI 0 "register_operand" "=r")
4120 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
4121 (match_operand:GPI 2 "register_operand" "r")))]
4122 ""
4123 "mul\\t%<w>0, %<w>1, %<w>2"
4124 [(set_attr "type" "mul")]
4125 )
4127 ;; zero_extend version of above
4128 (define_insn "*mulsi3_uxtw"
4129 [(set (match_operand:DI 0 "register_operand" "=r")
4130 (zero_extend:DI
4131 (mult:SI (match_operand:SI 1 "register_operand" "r")
4132 (match_operand:SI 2 "register_operand" "r"))))]
4133 ""
4134 "mul\\t%w0, %w1, %w2"
4135 [(set_attr "type" "mul")]
4136 )
4138 (define_insn "madd<mode>"
4139 [(set (match_operand:GPI 0 "register_operand" "=r")
4140 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
4141 (match_operand:GPI 2 "register_operand" "r"))
4142 (match_operand:GPI 3 "register_operand" "r")))]
4143 ""
4144 "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
4145 [(set_attr "type" "mla")]
4146 )
4148 ;; zero_extend version of above
4149 (define_insn "*maddsi_uxtw"
4150 [(set (match_operand:DI 0 "register_operand" "=r")
4151 (zero_extend:DI
4152 (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
4153 (match_operand:SI 2 "register_operand" "r"))
4154 (match_operand:SI 3 "register_operand" "r"))))]
4155 ""
4156 "madd\\t%w0, %w1, %w2, %w3"
4157 [(set_attr "type" "mla")]
4158 )
4160 (define_insn "*msub<mode>"
4161 [(set (match_operand:GPI 0 "register_operand" "=r")
4162 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
4163 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
4164 (match_operand:GPI 2 "register_operand" "r"))))]
4166 ""
4167 "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
4168 [(set_attr "type" "mla")]
4169 )
4171 ;; zero_extend version of above
4172 (define_insn "*msubsi_uxtw"
4173 [(set (match_operand:DI 0 "register_operand" "=r")
4174 (zero_extend:DI
4175 (minus:SI (match_operand:SI 3 "register_operand" "r")
4176 (mult:SI (match_operand:SI 1 "register_operand" "r")
4177 (match_operand:SI 2 "register_operand" "r")))))]
4179 ""
4180 "msub\\t%w0, %w1, %w2, %w3"
4181 [(set_attr "type" "mla")]
4182 )
4184 (define_insn "*mul<mode>_neg"
4185 [(set (match_operand:GPI 0 "register_operand" "=r")
4186 (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
4187 (match_operand:GPI 2 "register_operand" "r")))]
4189 ""
4190 "mneg\\t%<w>0, %<w>1, %<w>2"
4191 [(set_attr "type" "mul")]
4192 )
4194 ;; zero_extend version of above
4195 (define_insn "*mulsi_neg_uxtw"
4196 [(set (match_operand:DI 0 "register_operand" "=r")
4197 (zero_extend:DI
4198 (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
4199 (match_operand:SI 2 "register_operand" "r"))))]
4201 ""
4202 "mneg\\t%w0, %w1, %w2"
4203 [(set_attr "type" "mul")]
4204 )
4206 (define_insn "<su_optab>mulsidi3"
4207 [(set (match_operand:DI 0 "register_operand" "=r")
4208 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
4209 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
4210 ""
4211 "<su>mull\\t%0, %w1, %w2"
4212 [(set_attr "type" "<su>mull")]
4213 )
4215 (define_insn "<su_optab>maddsidi4"
4216 [(set (match_operand:DI 0 "register_operand" "=r")
4217 (plus:DI (mult:DI
4218 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
4219 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
4220 (match_operand:DI 3 "register_operand" "r")))]
4221 ""
4222 "<su>maddl\\t%0, %w1, %w2, %3"
4223 [(set_attr "type" "<su>mlal")]
4224 )
4226 (define_insn "<su_optab>msubsidi4"
4227 [(set (match_operand:DI 0 "register_operand" "=r")
4228 (minus:DI
4229 (match_operand:DI 3 "register_operand" "r")
4230 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
4231 (ANY_EXTEND:DI
4232 (match_operand:SI 2 "register_operand" "r")))))]
4233 ""
4234 "<su>msubl\\t%0, %w1, %w2, %3"
4235 [(set_attr "type" "<su>mlal")]
4236 )
4238 (define_insn "*<su_optab>mulsidi_neg"
4239 [(set (match_operand:DI 0 "register_operand" "=r")
4240 (mult:DI (neg:DI
4241 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
4242 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
4243 ""
4244 "<su>mnegl\\t%0, %w1, %w2"
4245 [(set_attr "type" "<su>mull")]
4246 )
4248 (define_expand "<su_optab>mulditi3"
4249 [(set (match_operand:TI 0 "register_operand")
4250 (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
4251 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
4252 ""
4253 {
4254 rtx low = gen_reg_rtx (DImode);
4255 emit_insn (gen_muldi3 (low, operands[1], operands[2]));
4257 rtx high = gen_reg_rtx (DImode);
4258 emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
4260 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
4261 emit_move_insn (gen_highpart (DImode, operands[0]), high);
4262 DONE;
4263 })
4265 ;; The default expansion of multi3 using umuldi3_highpart will perform
4266 ;; the additions in an order that fails to combine into two madd insns.
4267 (define_expand "multi3"
4268 [(set (match_operand:TI 0 "register_operand")
4269 (mult:TI (match_operand:TI 1 "register_operand")
4270 (match_operand:TI 2 "register_operand")))]
4271 ""
4272 {
4273 rtx l0 = gen_reg_rtx (DImode);
4274 rtx l1 = gen_lowpart (DImode, operands[1]);
4275 rtx l2 = gen_lowpart (DImode, operands[2]);
4276 rtx h0 = gen_reg_rtx (DImode);
4277 rtx h1 = gen_highpart (DImode, operands[1]);
4278 rtx h2 = gen_highpart (DImode, operands[2]);
4280 emit_insn (gen_muldi3 (l0, l1, l2));
4281 emit_insn (gen_umuldi3_highpart (h0, l1, l2));
4282 emit_insn (gen_madddi (h0, h1, l2, h0));
4283 emit_insn (gen_madddi (h0, l1, h2, h0));
4285 emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
4286 emit_move_insn (gen_highpart (DImode, operands[0]), h0);
4287 DONE;
4288 })
4290 (define_insn "<su>muldi3_highpart"
4291 [(set (match_operand:DI 0 "register_operand" "=r")
4292 (truncate:DI
4293 (lshiftrt:TI
4294 (mult:TI
4295 (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
4296 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
4297 (const_int 64))))]
4298 ""
4299 "<su>mulh\\t%0, %1, %2"
4300 [(set_attr "type" "<su>mull")]
4301 )
4303 (define_insn "<su_optab>div<mode>3"
4304 [(set (match_operand:GPI 0 "register_operand" "=r")
4305 (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
4306 (match_operand:GPI 2 "register_operand" "r")))]
4307 ""
4308 "<su>div\\t%<w>0, %<w>1, %<w>2"
4309 [(set_attr "type" "<su>div")]
4310 )
4312 ;; zero_extend version of above
4313 (define_insn "*<su_optab>divsi3_uxtw"
4314 [(set (match_operand:DI 0 "register_operand" "=r")
4315 (zero_extend:DI
4316 (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
4317 (match_operand:SI 2 "register_operand" "r"))))]
4318 ""
4319 "<su>div\\t%w0, %w1, %w2"
4320 [(set_attr "type" "<su>div")]
4321 )
4323 ;; -------------------------------------------------------------------
4324 ;; Comparison insns
4325 ;; -------------------------------------------------------------------
4327 (define_insn "cmp<mode>"
4328 [(set (reg:CC CC_REGNUM)
4329 (compare:CC (match_operand:GPI 0 "register_operand")
4330 (match_operand:GPI 1 "aarch64_plus_operand")))]
4331 ""
4332 {@ [ cons: 0 , 1 ; attrs: type ]
4333 [ rk , r ; alus_sreg ] cmp\t%<w>0, %<w>1
4334 [ rk , I ; alus_imm ] cmp\t%<w>0, %1
4335 [ rk , J ; alus_imm ] cmn\t%<w>0, #%n1
4336 }
4337 )
4339 (define_insn "fcmp<mode>"
4340 [(set (reg:CCFP CC_REGNUM)
4341 (compare:CCFP (match_operand:GPF 0 "register_operand")
4342 (match_operand:GPF 1 "aarch64_fp_compare_operand")))]
4343 "TARGET_FLOAT"
4344 {@ [ cons: 0 , 1 ]
4345 [ w , Y ] fcmp\t%<s>0, #0.0
4346 [ w , w ] fcmp\t%<s>0, %<s>1
4347 }
4348 [(set_attr "type" "fcmp<s>")]
4349 )
4351 (define_insn "fcmpe<mode>"
4352 [(set (reg:CCFPE CC_REGNUM)
4353 (compare:CCFPE (match_operand:GPF 0 "register_operand")
4354 (match_operand:GPF 1 "aarch64_fp_compare_operand")))]
4355 "TARGET_FLOAT"
4356 {@ [ cons: 0 , 1 ]
4357 [ w , Y ] fcmpe\t%<s>0, #0.0
4358 [ w , w ] fcmpe\t%<s>0, %<s>1
4359 }
4360 [(set_attr "type" "fcmp<s>")]
4361 )
4363 (define_insn "*cmp_swp_<shift>_reg<mode>"
4364 [(set (reg:CC_SWP CC_REGNUM)
4365 (compare:CC_SWP (ASHIFT:GPI
4366 (match_operand:GPI 0 "register_operand" "r")
4367 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4368 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
4369 ""
4370 "cmp\\t%<w>2, %<w>0, <shift> %1"
4371 [(set_attr "type" "alus_shift_imm")]
4372 )
4374 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
4375 [(set (reg:CC_SWP CC_REGNUM)
4376 (compare:CC_SWP (ANY_EXTEND:GPI
4377 (match_operand:ALLX 0 "register_operand" "r"))
4378 (match_operand:GPI 1 "register_operand" "r")))]
4379 ""
4380 "cmp\\t%<GPI:w>1, %w0, <su>xt<ALLX:size>"
4381 [(set_attr "type" "alus_ext")]
4382 )
4384 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
4385 [(set (reg:CC_SWP CC_REGNUM)
4386 (compare:CC_SWP (ashift:GPI
4387 (ANY_EXTEND:GPI
4388 (match_operand:ALLX 0 "register_operand" "r"))
4389 (match_operand 1 "aarch64_imm3" "Ui3"))
4390 (match_operand:GPI 2 "register_operand" "r")))]
4391 ""
4392 "cmp\\t%<GPI:w>2, %w0, <su>xt<ALLX:size> %1"
4393 [(set_attr "type" "alus_ext")]
4394 )
4396 ;; -------------------------------------------------------------------
4397 ;; Store-flag and conditional select insns
4398 ;; -------------------------------------------------------------------
4400 (define_expand "cstore<mode>4"
4401 [(set (match_operand:SI 0 "register_operand")
4402 (match_operator:SI 1 "aarch64_comparison_operator"
4403 [(match_operand:GPI 2 "register_operand")
4404 (match_operand:GPI 3 "aarch64_plus_operand")]))]
4405 ""
4406 "
4407 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4408 operands[3]);
4409 operands[3] = const0_rtx;
4410 "
4411 )
4413 (define_expand "cstorecc4"
4414 [(set (match_operand:SI 0 "register_operand")
4415 (match_operator 1 "aarch64_comparison_operator_mode"
4416 [(match_operand 2 "cc_register")
4417 (match_operand 3 "const0_operand")]))]
4418 ""
4419 "{
4420 emit_insn (gen_rtx_SET (operands[0], operands[1]));
4421 DONE;
4422 }")
4425 (define_expand "cstore<mode>4"
4426 [(set (match_operand:SI 0 "register_operand")
4427 (match_operator:SI 1 "aarch64_comparison_operator_mode"
4428 [(match_operand:GPF 2 "register_operand")
4429 (match_operand:GPF 3 "aarch64_fp_compare_operand")]))]
4430 ""
4431 "
4432 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4433 operands[3]);
4434 operands[3] = const0_rtx;
4435 "
4436 )
4438 (define_insn "aarch64_cstore<mode>"
4439 [(set (match_operand:ALLI 0 "register_operand" "=r")
4440 (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
4441 [(match_operand 2 "cc_register" "") (const_int 0)]))]
4442 ""
4443 "cset\\t%<w>0, %m1"
4444 [(set_attr "type" "csel")]
4445 )
4447 ;; For a 24-bit immediate CST we can optimize the compare for equality
4448 ;; and branch sequence from:
4449 ;; mov x0, #imm1
4450 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
4451 ;; cmp x1, x0
4452 ;; cset x2, <ne,eq>
4453 ;; into the shorter:
4454 ;; sub x0, x1, #(CST & 0xfff000)
4455 ;; subs x0, x0, #(CST & 0x000fff)
4456 ;; cset x2, <ne, eq>.
4457 (define_insn_and_split "*compare_cstore<mode>_insn"
4458 [(set (match_operand:GPI 0 "register_operand" "=r")
4459 (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
4460 (match_operand:GPI 2 "aarch64_imm24" "n")))
4461 (clobber (reg:CC CC_REGNUM))]
4462 "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
4463 && !aarch64_plus_operand (operands[2], <MODE>mode)
4464 && !reload_completed"
4465 "#"
4466 "&& true"
4467 [(const_int 0)]
4468 {
4469 HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
4470 HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
4471 rtx tmp = gen_reg_rtx (<MODE>mode);
4472 emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
4473 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
4474 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
4475 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
4476 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
4477 DONE;
4478 }
4479 [(set_attr "type" "csel")]
4480 )
4482 ;; zero_extend version of the above
4483 (define_insn "*cstoresi_insn_uxtw"
4484 [(set (match_operand:DI 0 "register_operand" "=r")
4485 (zero_extend:DI
4486 (match_operator:SI 1 "aarch64_comparison_operator_mode"
4487 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4488 ""
4489 "cset\\t%w0, %m1"
4490 [(set_attr "type" "csel")]
4491 )
4493 (define_insn "cstore<mode>_neg"
4494 [(set (match_operand:ALLI 0 "register_operand" "=r")
4495 (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
4496 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4497 ""
4498 "csetm\\t%<w>0, %m1"
4499 [(set_attr "type" "csel")]
4500 )
4502 ;; zero_extend version of the above
4503 (define_insn "*cstoresi_neg_uxtw"
4504 [(set (match_operand:DI 0 "register_operand" "=r")
4505 (zero_extend:DI
4506 (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
4507 [(match_operand 2 "cc_register" "") (const_int 0)]))))]
4508 ""
4509 "csetm\\t%w0, %m1"
4510 [(set_attr "type" "csel")]
4511 )
4513 (define_expand "cmov<mode>6"
4514 [(set (match_operand:GPI 0 "register_operand")
4515 (if_then_else:GPI
4516 (match_operator 1 "aarch64_comparison_operator"
4517 [(match_operand:GPI 2 "register_operand")
4518 (match_operand:GPI 3 "aarch64_plus_operand")])
4519 (match_operand:GPI 4 "register_operand")
4520 (match_operand:GPI 5 "register_operand")))]
4521 ""
4522 "
4523 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4524 operands[3]);
4525 operands[3] = const0_rtx;
4526 "
4527 )
4529 (define_expand "cmov<mode>6"
4530 [(set (match_operand:GPF 0 "register_operand")
4531 (if_then_else:GPF
4532 (match_operator 1 "aarch64_comparison_operator"
4533 [(match_operand:GPF 2 "register_operand")
4534 (match_operand:GPF 3 "aarch64_fp_compare_operand")])
4535 (match_operand:GPF 4 "register_operand")
4536 (match_operand:GPF 5 "register_operand")))]
4537 ""
4538 "
4539 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4540 operands[3]);
4541 operands[3] = const0_rtx;
4542 "
4543 )
4545 (define_insn "*cmov<mode>_insn"
4546 [(set (match_operand:ALLI 0 "register_operand")
4547 (if_then_else:ALLI
4548 (match_operator 1 "aarch64_comparison_operator"
4549 [(match_operand 2 "cc_register") (const_int 0)])
4550 (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1")
4551 (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1")))]
4552 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4553 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4554 ;; Final two alternatives should be unreachable, but included for completeness
4555 {@ [ cons: =0 , 3 , 4 ; attrs: type ]
4556 [ r , rZ , rZ ; csel ] csel\t%<w>0, %<w>3, %<w>4, %m1
4557 [ r , rZ , UsM ; csel ] csinv\t%<w>0, %<w>3, <w>zr, %m1
4558 [ r , UsM , rZ ; csel ] csinv\t%<w>0, %<w>4, <w>zr, %M1
4559 [ r , rZ , Ui1 ; csel ] csinc\t%<w>0, %<w>3, <w>zr, %m1
4560 [ r , Ui1 , rZ ; csel ] csinc\t%<w>0, %<w>4, <w>zr, %M1
4561 [ r , UsM , UsM ; mov_imm ] mov\t%<w>0, -1
4562 [ r , Ui1 , Ui1 ; mov_imm ] mov\t%<w>0, 1
4563 }
4564 )
4566 ;; zero_extend version of above
4567 (define_insn "*cmovsi_insn_uxtw"
4568 [(set (match_operand:DI 0 "register_operand")
4569 (zero_extend:DI
4570 (if_then_else:SI
4571 (match_operator 1 "aarch64_comparison_operator"
4572 [(match_operand 2 "cc_register") (const_int 0)])
4573 (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1")
4574 (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1"))))]
4575 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4576 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4577 ;; Final two alternatives should be unreachable, but included for completeness
4578 {@ [ cons: =0 , 3 , 4 ; attrs: type ]
4579 [ r , rZ , rZ ; csel ] csel\t%w0, %w3, %w4, %m1
4580 [ r , rZ , UsM ; csel ] csinv\t%w0, %w3, wzr, %m1
4581 [ r , UsM , rZ ; csel ] csinv\t%w0, %w4, wzr, %M1
4582 [ r , rZ , Ui1 ; csel ] csinc\t%w0, %w3, wzr, %m1
4583 [ r , Ui1 , rZ ; csel ] csinc\t%w0, %w4, wzr, %M1
4584 [ r , UsM , UsM ; mov_imm ] mov\t%w0, -1
4585 [ r , Ui1 , Ui1 ; mov_imm ] mov\t%w0, 1
4586 }
4587 )
4589 ;; There are two canonical forms for `cmp ? -1 : a`.
4590 ;; This is the second form and is here to help combine.
4591 ;; Support `-(cmp) | a` into `cmp ? -1 : a` to be canonical in the backend.
4592 (define_insn_and_split "*cmov<mode>_insn_m1"
4593 [(set (match_operand:GPI 0 "register_operand" "=r")
4594 (ior:GPI
4595 (neg:GPI
4596 (match_operator:GPI 1 "aarch64_comparison_operator"
4597 [(match_operand 2 "cc_register" "") (const_int 0)]))
4598 (match_operand 3 "register_operand" "r")))]
4599 ""
4600 "#"
4601 "&& true"
4602 [(set (match_dup 0)
4603 (if_then_else:GPI (match_dup 1)
4604 (const_int -1)
4605 (match_dup 3)))]
4606 {}
4607 [(set_attr "type" "csel")]
4608 )
4610 (define_insn "*cmovdi_insn_uxtw"
4611 [(set (match_operand:DI 0 "register_operand" "=r")
4612 (if_then_else:DI
4613 (match_operator 1 "aarch64_comparison_operator"
4614 [(match_operand 2 "cc_register" "") (const_int 0)])
4615 (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4616 (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4617 ""
4618 "csel\\t%w0, %w3, %w4, %m1"
4619 [(set_attr "type" "csel")]
4620 )
4622 (define_insn "*cmov<mode>_insn"
4623 [(set (match_operand:GPF 0 "register_operand" "=w")
4624 (if_then_else:GPF
4625 (match_operator 1 "aarch64_comparison_operator"
4626 [(match_operand 2 "cc_register" "") (const_int 0)])
4627 (match_operand:GPF 3 "register_operand" "w")
4628 (match_operand:GPF 4 "register_operand" "w")))]
4629 "TARGET_FLOAT"
4630 "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
4631 [(set_attr "type" "fcsel")]
4632 )
4634 (define_expand "mov<mode>cc"
4635 [(set (match_operand:ALLI 0 "register_operand")
4636 (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator")
4637 (match_operand:ALLI 2 "register_operand")
4638 (match_operand:ALLI 3 "register_operand")))]
4639 ""
4640 {
4641 rtx ccreg;
4642 enum rtx_code code = GET_CODE (operands[1]);
4644 if (code == UNEQ || code == LTGT)
4645 FAIL;
4647 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4648 XEXP (operands[1], 1));
4649 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4650 }
4651 )
4653 (define_expand "mov<GPF:mode><GPI:mode>cc"
4654 [(set (match_operand:GPI 0 "register_operand")
4655 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4656 (match_operand:GPF 2 "register_operand")
4657 (match_operand:GPF 3 "register_operand")))]
4658 ""
4659 {
4660 rtx ccreg;
4661 enum rtx_code code = GET_CODE (operands[1]);
4663 if (code == UNEQ || code == LTGT)
4664 FAIL;
4666 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4667 XEXP (operands[1], 1));
4668 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4669 }
4670 )
4672 (define_expand "mov<mode>cc"
4673 [(set (match_operand:GPF 0 "register_operand")
4674 (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator")
4675 (match_operand:GPF 2 "register_operand")
4676 (match_operand:GPF 3 "register_operand")))]
4677 ""
4678 {
4679 rtx ccreg;
4680 enum rtx_code code = GET_CODE (operands[1]);
4682 if (code == UNEQ || code == LTGT)
4683 FAIL;
4685 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4686 XEXP (operands[1], 1));
4687 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4688 }
4689 )
4691 (define_expand "<neg_not_op><mode>cc"
4692 [(set (match_operand:GPI 0 "register_operand")
4693 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4694 (NEG_NOT:GPI (match_operand:GPI 2 "register_operand"))
4695 (match_operand:GPI 3 "register_operand")))]
4696 ""
4697 {
4698 rtx ccreg;
4699 enum rtx_code code = GET_CODE (operands[1]);
4701 if (code == UNEQ || code == LTGT)
4702 FAIL;
4704 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4705 XEXP (operands[1], 1));
4706 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4707 }
4708 )
4710 ;; CRC32 instructions.
4711 (define_insn "aarch64_<crc_variant>"
4712 [(set (match_operand:SI 0 "register_operand" "=r")
4713 (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4714 (match_operand:<crc_mode> 2 "register_operand" "r")]
4715 CRC))]
4716 "TARGET_CRC32"
4717 {
4718 if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
4719 return "<crc_variant>\\t%w0, %w1, %x2";
4720 else
4721 return "<crc_variant>\\t%w0, %w1, %w2";
4722 }
4723 [(set_attr "type" "crc")]
4724 )
4726 ;; Reversed CRC
4727 (define_expand "crc_rev<ALLI:mode><ALLX:mode>4"
4728 [;; return value (calculated CRC)
4729 (match_operand:ALLX 0 "register_operand" "=r")
4730 ;; initial CRC
4731 (match_operand:ALLX 1 "register_operand" "r")
4732 ;; data
4733 (match_operand:ALLI 2 "register_operand" "r")
4734 ;; polynomial without leading 1
4735 (match_operand:ALLX 3)]
4736 ""
4737 {
4738 /* If the polynomial is the same as the polynomial of crc32c* instruction,
4739 put that instruction. crc32c uses iSCSI polynomial. */
4740 if (TARGET_CRC32 && INTVAL (operands[3]) == 0x1EDC6F41
4741 && <ALLX:MODE>mode == SImode)
4742 emit_insn (gen_aarch64_crc32c<ALLI:crc_data_type> (operands[0],
4743 operands[1],
4744 operands[2]));
4745 /* If the polynomial is the same as the polynomial of crc32* instruction,
4746 put that instruction. crc32 uses HDLC etc. polynomial. */
4747 else if (TARGET_CRC32 && INTVAL (operands[3]) == 0x04C11DB7
4748 && <ALLX:MODE>mode == SImode)
4749 emit_insn (gen_aarch64_crc32<ALLI:crc_data_type> (operands[0],
4750 operands[1],
4751 operands[2]));
4752 else if (TARGET_AES && <ALLI:sizen> <= <ALLX:sizen>)
4753 aarch64_expand_reversed_crc_using_pmull (<ALLX:MODE>mode,
4754 <ALLI:MODE>mode,
4755 operands);
4756 else
4757 /* Otherwise, generate table-based CRC. */
4758 expand_reversed_crc_table_based (operands[0], operands[1], operands[2],
4759 operands[3], <ALLI:MODE>mode,
4760 generate_reflecting_code_standard);
4761 DONE;
4762 }
4763 )
4765 ;; Bit-forward CRC
4766 (define_expand "crc<ALLI:mode><ALLX:mode>4"
4767 [;; return value (calculated CRC)
4768 (match_operand:ALLX 0 "register_operand" "=r")
4769 ;; initial CRC
4770 (match_operand:ALLX 1 "register_operand" "r")
4771 ;; data
4772 (match_operand:ALLI 2 "register_operand" "r")
4773 ;; polynomial without leading 1
4774 (match_operand:ALLX 3)]
4775 "TARGET_AES && <ALLI:sizen> <= <ALLX:sizen>"
4776 {
4777 aarch64_expand_crc_using_pmull (<ALLX:MODE>mode, <ALLI:MODE>mode,
4778 operands);
4779 DONE;
4780 }
4781 )
4783 (define_insn "*csinc2<mode>_insn"
4784 [(set (match_operand:GPI 0 "register_operand" "=r")
4785 (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4786 (match_operand:GPI 1 "register_operand" "r")))]
4787 ""
4788 "cinc\\t%<w>0, %<w>1, %m2"
4789 [(set_attr "type" "csel")]
4790 )
4792 (define_insn "csinc3<mode>_insn"
4793 [(set (match_operand:GPI 0 "register_operand" "=r")
4794 (if_then_else:GPI
4795 (match_operand 1 "aarch64_comparison_operation" "")
4796 (plus:GPI (match_operand:GPI 2 "register_operand" "r")
4797 (const_int 1))
4798 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4799 ""
4800 "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
4801 [(set_attr "type" "csel")]
4802 )
4804 (define_insn "*csinv3<mode>_insn"
4805 [(set (match_operand:GPI 0 "register_operand" "=r")
4806 (if_then_else:GPI
4807 (match_operand 1 "aarch64_comparison_operation" "")
4808 (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4809 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4810 ""
4811 "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
4812 [(set_attr "type" "csel")]
4813 )
4815 (define_insn "*cs<neg_not_cs>3_uxtw_insn4"
4816 [(set (match_operand:DI 0 "register_operand" "=r")
4817 (zero_extend:DI
4818 (if_then_else:SI
4819 (match_operand 1 "aarch64_comparison_operation" "")
4820 (NEG_NOT:SI (match_operand:SI 2 "register_operand" "r"))
4821 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4822 ""
4823 "cs<neg_not_cs>\\t%w0, %w3, %w2, %M1"
4824 [(set_attr "type" "csel")]
4825 )
4827 (define_insn "csneg3<mode>_insn"
4828 [(set (match_operand:GPI 0 "register_operand" "=r")
4829 (if_then_else:GPI
4830 (match_operand 1 "aarch64_comparison_operation" "")
4831 (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4832 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4833 ""
4834 "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
4835 [(set_attr "type" "csel")]
4836 )
4838 (define_insn "*csinv3_uxtw_insn1"
4839 [(set (match_operand:DI 0 "register_operand" "=r")
4840 (if_then_else:DI
4841 (match_operand 1 "aarch64_comparison_operation" "")
4842 (zero_extend:DI
4843 (match_operand:SI 2 "register_operand" "r"))
4844 (zero_extend:DI
4845 (NEG_NOT:SI (match_operand:SI 3 "register_operand" "r")))))]
4846 ""
4847 "cs<neg_not_cs>\\t%w0, %w2, %w3, %m1"
4848 [(set_attr "type" "csel")]
4849 )
4851 (define_insn "*csinv3_uxtw_insn2"
4852 [(set (match_operand:DI 0 "register_operand" "=r")
4853 (if_then_else:DI
4854 (match_operand 1 "aarch64_comparison_operation" "")
4855 (zero_extend:DI
4856 (NEG_NOT:SI (match_operand:SI 2 "register_operand" "r")))
4857 (zero_extend:DI
4858 (match_operand:SI 3 "register_operand" "r"))))]
4859 ""
4860 "cs<neg_not_cs>\\t%w0, %w3, %w2, %M1"
4861 [(set_attr "type" "csel")]
4862 )
4864 (define_insn "*csinv3_uxtw_insn3"
4865 [(set (match_operand:DI 0 "register_operand" "=r")
4866 (if_then_else:DI
4867 (match_operand 1 "aarch64_comparison_operation" "")
4868 (zero_extend:DI
4869 (NEG_NOT:SI (match_operand:SI 2 "register_operand" "r")))
4870 (const_int 0)))]
4871 ""
4872 "cs<neg_not_cs>\\t%w0, wzr, %w2, %M1"
4873 [(set_attr "type" "csel")]
4874 )
4876 ;; There are two canonical forms for `cmp ? ~a : a`.
4877 ;; This is the second form and is here to help combine.
4878 ;; Support `-(cmp) ^ a` into `cmp ? ~a : a`
4879 ;; The second pattern is to support the zero extend'ed version.
4881 (define_insn_and_split "*cmov<mode>_insn_insv"
4882 [(set (match_operand:GPI 0 "register_operand" "=r")
4883 (xor:GPI
4884 (neg:GPI
4885 (match_operator:GPI 1 "aarch64_comparison_operator"
4886 [(match_operand 2 "cc_register" "") (const_int 0)]))
4887 (match_operand:GPI 3 "general_operand" "r")))]
4888 ""
4889 "#"
4890 "&& true"
4891 [(set (match_dup 0)
4892 (if_then_else:GPI (match_dup 1)
4893 (not:GPI (match_dup 3))
4894 (match_dup 3)))]
4895 {
4896 /* After reload this will be a nop due to the constraint. */
4897 operands[3] = force_reg (<MODE>mode, operands[3]);
4898 }
4899 [(set_attr "type" "csel")]
4900 )
4902 (define_insn_and_split "*cmov_uxtw_insn_insv"
4903 [(set (match_operand:DI 0 "register_operand" "=r")
4904 (zero_extend:DI
4905 (xor:SI
4906 (neg:SI
4907 (match_operator:SI 1 "aarch64_comparison_operator"
4908 [(match_operand 2 "cc_register" "") (const_int 0)]))
4909 (match_operand:SI 3 "general_operand" "r"))))]
4910 ""
4911 "#"
4912 "&& true"
4913 [(set (match_dup 0)
4914 (if_then_else:DI (match_dup 1)
4915 (zero_extend:DI (not:SI (match_dup 3)))
4916 (zero_extend:DI (match_dup 3))))]
4917 {
4918 operands[3] = force_reg (SImode, operands[3]);
4919 }
4920 [(set_attr "type" "csel")]
4921 )
4923 ;; If X can be loaded by a single CNT[BHWD] instruction,
4924 ;;
4925 ;; A = UMAX (B, X)
4926 ;;
4927 ;; is equivalent to:
4928 ;;
4929 ;; TMP = UQDEC[BHWD] (B, X)
4930 ;; A = TMP + X
4931 ;;
4932 ;; Defining the pattern this way means that:
4933 ;;
4934 ;; A = UMAX (B, X) - X
4935 ;;
4936 ;; becomes:
4937 ;;
4938 ;; TMP1 = UQDEC[BHWD] (B, X)
4939 ;; TMP2 = TMP1 + X
4940 ;; A = TMP2 - X
4941 ;;
4942 ;; which combine can optimize to:
4943 ;;
4944 ;; A = UQDEC[BHWD] (B, X)
4945 ;;
4946 ;; We don't use match_operand predicates because the order of the operands
4947 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4948 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4949 (define_expand "umax<mode>3"
4950 [(set (match_operand:GPI 0 "register_operand")
4951 (umax:GPI (match_operand:GPI 1 "")
4952 (match_operand:GPI 2 "")))]
4953 "TARGET_SVE || TARGET_CSSC"
4954 {
4955 if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4956 std::swap (operands[1], operands[2]);
4957 else if (aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4958 ;
4959 else if (TARGET_CSSC)
4960 {
4961 if (aarch64_uminmax_immediate (operands[1], <MODE>mode))
4962 std::swap (operands[1], operands[2]);
4963 operands[1] = force_reg (<MODE>mode, operands[1]);
4964 if (!aarch64_uminmax_operand (operands[2], <MODE>mode))
4965 operands[2] = force_reg (<MODE>mode, operands[2]);
4966 emit_move_insn (operands[0], gen_rtx_UMAX (<MODE>mode, operands[1],
4967 operands[2]));
4968 DONE;
4969 }
4970 else
4971 FAIL;
4972 rtx temp = gen_reg_rtx (<MODE>mode);
4973 operands[1] = force_reg (<MODE>mode, operands[1]);
4974 emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4975 emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4976 DONE;
4977 }
4978 )
4980 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4981 (define_insn "aarch64_uqdec<mode>"
4982 [(set (match_operand:GPI 0 "register_operand" "=r")
4983 (minus:GPI
4984 (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4985 (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4986 (match_dup 2)))]
4987 "TARGET_SVE"
4988 {
4989 return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4990 }
4991 )
4993 ;; Implement MAX/MIN (A, B) - C using SUBS/ADDS followed by CSEL/CSINV/CSINC.
4994 ;; See aarch64_maxmin_plus_const for details about the supported cases.
4995 (define_insn_and_split "*aarch64_minmax_plus"
4996 [(set (match_operand:GPI 0 "register_operand" "=r")
4997 (plus:GPI
4998 (MAXMIN:GPI
4999 (match_operand:GPI 1 "register_operand" "r")
5000 (match_operand:GPI 2 "const_int_operand"))
5001 (match_operand:GPI 3 "aarch64_plus_immediate")))
5002 (clobber (reg:CC CC_REGNUM))]
5003 "aarch64_maxmin_plus_const (<CODE>, operands, false)"
5004 "#"
5005 "&& 1"
5006 [(parallel
5007 [(set (reg:CC CC_REGNUM)
5008 (compare:CC (match_dup 1) (match_dup 4)))
5009 (set (match_dup 6)
5010 (plus:GPI (match_dup 1) (match_dup 3)))])
5011 (set (match_dup 0)
5012 (if_then_else:GPI (match_dup 5) (match_dup 6) (match_dup 7)))]
5013 {
5014 if (!aarch64_maxmin_plus_const (<CODE>, operands, true))
5015 gcc_unreachable ();
5016 }
5017 [(set_attr "length" "8")]
5018 )
5020 ;; -------------------------------------------------------------------
5021 ;; Logical operations
5022 ;; -------------------------------------------------------------------
5025 (define_insn_and_split "*aarch64_and<mode>_imm2"
5026 [(set (match_operand:GPI 0 "register_operand" "=rk")
5027 (and:GPI (match_operand:GPI 1 "register_operand" "%r")
5028 (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
5029 ""
5030 "#"
5031 "true"
5032 [(const_int 0)]
5033 {
5034 HOST_WIDE_INT val = INTVAL (operands[2]);
5035 rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
5036 rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
5038 emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
5039 emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
5040 DONE;
5041 }
5042 )
5044 (define_insn "<optab><mode>3"
5045 [(set (match_operand:GPI 0 "register_operand")
5046 (LOGICAL:GPI (match_operand:GPI 1 "register_operand")
5047 (match_operand:GPI 2 "aarch64_logical_operand")))]
5048 ""
5049 {@ [ cons: =0 , 1 , 2 ; attrs: type , arch ]
5050 [ r , %r , r ; logic_reg , * ] <logical>\t%<w>0, %<w>1, %<w>2
5051 [ rk , r , <lconst> ; logic_imm , * ] <logical>\t%<w>0, %<w>1, %2
5052 [ w , 0 , <lconst> ; * , sve ] <logical>\t%Z0.<s>, %Z0.<s>, #%2
5053 [ w , w , w ; neon_logic , simd ] <logical>\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>
5054 }
5055 )
5057 ;; zero_extend version of above
5058 (define_insn "*<optab>si3_uxtw"
5059 [(set (match_operand:DI 0 "register_operand")
5060 (zero_extend:DI
5061 (LOGICAL:SI (match_operand:SI 1 "register_operand")
5062 (match_operand:SI 2 "aarch64_logical_operand"))))]
5063 ""
5064 {@ [ cons: =0 , 1 , 2 ; attrs: type ]
5065 [ r , %r , r ; logic_reg ] <logical>\t%w0, %w1, %w2
5066 [ rk , r , K ; logic_imm ] <logical>\t%w0, %w1, %2
5067 }
5068 )
5070 (define_insn "*and<mode>3_compare0"
5071 [(set (reg:CC_NZV CC_REGNUM)
5072 (compare:CC_NZV
5073 (and:GPI (match_operand:GPI 1 "register_operand")
5074 (match_operand:GPI 2 "aarch64_logical_operand"))
5075 (const_int 0)))
5076 (set (match_operand:GPI 0 "register_operand")
5077 (and:GPI (match_dup 1) (match_dup 2)))]
5078 ""
5079 {@ [ cons: =0 , 1 , 2 ; attrs: type ]
5080 [ r , %r , r ; logics_reg ] ands\t%<w>0, %<w>1, %<w>2
5081 [ r , r , <lconst> ; logics_imm ] ands\t%<w>0, %<w>1, %2
5082 }
5083 )
5085 ;; zero_extend version of above
5086 (define_insn "*andsi3_compare0_uxtw"
5087 [(set (reg:CC_NZV CC_REGNUM)
5088 (compare:CC_NZV
5089 (and:SI (match_operand:SI 1 "register_operand")
5090 (match_operand:SI 2 "aarch64_logical_operand"))
5091 (const_int 0)))
5092 (set (match_operand:DI 0 "register_operand")
5093 (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
5094 ""
5095 {@ [ cons: =0 , 1 , 2 ; attrs: type ]
5096 [ r , %r , r ; logics_reg ] ands\t%w0, %w1, %w2
5097 [ r , r , K ; logics_imm ] ands\t%w0, %w1, %2
5098 }
5099 )
5101 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
5102 [(set (reg:CC_NZV CC_REGNUM)
5103 (compare:CC_NZV
5104 (and:GPI (SHIFT:GPI
5105 (match_operand:GPI 1 "register_operand" "r")
5106 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
5107 (match_operand:GPI 3 "register_operand" "r"))
5108 (const_int 0)))
5109 (set (match_operand:GPI 0 "register_operand" "=r")
5110 (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
5111 ""
5112 {
5113 if (<SHIFT:is_rotl>)
5114 operands[2] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5115 return "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2";
5116 }
5117 [(set_attr "type" "logics_shift_imm")]
5118 )
5120 ;; zero_extend version of above
5121 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
5122 [(set (reg:CC_NZV CC_REGNUM)
5123 (compare:CC_NZV
5124 (and:SI (SHIFT:SI
5125 (match_operand:SI 1 "register_operand" "r")
5126 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
5127 (match_operand:SI 3 "register_operand" "r"))
5128 (const_int 0)))
5129 (set (match_operand:DI 0 "register_operand" "=r")
5130 (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
5131 (match_dup 3))))]
5132 ""
5133 {
5134 if (<SHIFT:is_rotl>)
5135 operands[2] = GEN_INT (32 - UINTVAL (operands[2]));
5136 return "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2";
5137 }
5138 [(set_attr "type" "logics_shift_imm")]
5139 )
5141 (define_insn "<LOGICAL:optab>_<SHIFT:optab><mode>3"
5142 [(set (match_operand:GPI 0 "register_operand" "=r")
5143 (LOGICAL:GPI (SHIFT:GPI
5144 (match_operand:GPI 1 "register_operand" "r")
5145 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
5146 (match_operand:GPI 3 "register_operand" "r")))]
5147 ""
5148 {
5149 if (<SHIFT:is_rotl>)
5150 operands[2] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5151 return "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2";
5152 }
5153 [(set_attr "type" "logic_shift_imm")]
5154 )
5156 (define_split
5157 [(set (match_operand:GPI 0 "register_operand")
5158 (LOGICAL_OR_PLUS:GPI
5159 (and:GPI
5160 (lshiftrt:GPI (match_operand:GPI 1 "register_operand")
5161 (match_operand:QI 2 "aarch64_shift_imm_<mode>"))
5162 (match_operand:GPI 3 "aarch64_logical_immediate"))
5163 (match_operand:GPI 4 "register_operand")))]
5164 "can_create_pseudo_p ()
5165 && aarch64_bitmask_imm (UINTVAL (operands[3]) << UINTVAL (operands[2]),
5166 <MODE>mode)"
5167 [(set (match_dup 5) (and:GPI (match_dup 1) (match_dup 6)))
5168 (set (match_dup 0) (LOGICAL_OR_PLUS:GPI
5169 (lshiftrt:GPI (match_dup 5) (match_dup 2))
5170 (match_dup 4)))]
5171 {
5172 operands[5] = gen_reg_rtx (<MODE>mode);
5173 operands[6]
5174 = gen_int_mode (UINTVAL (operands[3]) << UINTVAL (operands[2]),
5175 <MODE>mode);
5176 }
5177 )
5179 (define_split
5180 [(set (match_operand:GPI 0 "register_operand")
5181 (LOGICAL_OR_PLUS:GPI
5182 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand")
5183 (match_operand:QI 2 "aarch64_shift_imm_<mode>"))
5184 (match_operand:GPI 3 "const_int_operand"))
5185 (zero_extend:GPI (match_operand 4 "register_operand"))))]
5186 "can_create_pseudo_p ()
5187 && ((paradoxical_subreg_p (operands[1])
5188 && rtx_equal_p (SUBREG_REG (operands[1]), operands[4]))
5189 || (REG_P (operands[1])
5190 && REG_P (operands[4])
5191 && REGNO (operands[1]) == REGNO (operands[4])))
5192 && (trunc_int_for_mode (GET_MODE_MASK (GET_MODE (operands[4]))
5193 << INTVAL (operands[2]), <MODE>mode)
5194 == INTVAL (operands[3]))
5195 && (<CODE> != PLUS
5196 || (GET_MODE_MASK (GET_MODE (operands[4]))
5197 & INTVAL (operands[3])) == 0)"
5198 [(set (match_dup 5) (zero_extend:GPI (match_dup 4)))
5199 (set (match_dup 0) (match_dup 6))]
5200 {
5201 operands[5] = gen_reg_rtx (<MODE>mode);
5202 rtx shift = gen_rtx_ASHIFT (<MODE>mode, operands[5], operands[2]);
5203 rtx_code new_code = (<CODE> == PLUS ? IOR : <CODE>);
5204 operands[6] = gen_rtx_fmt_ee (new_code, <MODE>mode, shift, operands[5]);
5205 }
5206 )
5208 (define_split
5209 [(set (match_operand:GPI 0 "register_operand")
5210 (LOGICAL_OR_PLUS:GPI
5211 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand")
5212 (match_operand:QI 2 "aarch64_shift_imm_<mode>"))
5213 (match_operand:GPI 4 "const_int_operand"))
5214 (and:GPI (match_dup 1) (match_operand:GPI 3 "const_int_operand"))))]
5215 "can_create_pseudo_p ()
5216 && pow2_or_zerop (UINTVAL (operands[3]) + 1)
5217 && (trunc_int_for_mode (UINTVAL (operands[3])
5218 << INTVAL (operands[2]), <MODE>mode)
5219 == INTVAL (operands[4]))
5220 && (<CODE> != PLUS
5221 || (INTVAL (operands[4]) & INTVAL (operands[3])) == 0)"
5222 [(set (match_dup 5) (and:GPI (match_dup 1) (match_dup 3)))
5223 (set (match_dup 0) (match_dup 6))]
5224 {
5225 operands[5] = gen_reg_rtx (<MODE>mode);
5226 rtx shift = gen_rtx_ASHIFT (<MODE>mode, operands[5], operands[2]);
5227 rtx_code new_code = (<CODE> == PLUS ? IOR : <CODE>);
5228 operands[6] = gen_rtx_fmt_ee (new_code, <MODE>mode, shift, operands[5]);
5229 }
5230 )
5232 (define_split
5233 [(set (match_operand:GPI 0 "register_operand")
5234 (LOGICAL:GPI
5235 (ashift:GPI (sign_extend:GPI (match_operand 1 "register_operand"))
5236 (match_operand:QI 2 "aarch64_shift_imm_<mode>"))
5237 (sign_extend:GPI (match_dup 1))))]
5238 "can_create_pseudo_p ()"
5239 [(set (match_dup 3) (sign_extend:GPI (match_dup 1)))
5240 (set (match_dup 0) (LOGICAL:GPI (ashift:GPI (match_dup 3) (match_dup 2))
5241 (match_dup 3)))]
5242 "operands[3] = gen_reg_rtx (<MODE>mode);"
5243 )
5245 ;; zero_extend versions of above
5246 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
5247 [(set (match_operand:DI 0 "register_operand" "=r")
5248 (zero_extend:DI
5249 (LOGICAL:SI (SHIFT:SI
5250 (match_operand:SI 1 "register_operand" "r")
5251 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
5252 (match_operand:SI 3 "register_operand" "r"))))]
5253 ""
5254 {
5255 if (<SHIFT:is_rotl>)
5256 operands[2] = GEN_INT (32 - UINTVAL (operands[2]));
5257 return "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2";
5258 }
5259 [(set_attr "type" "logic_shift_imm")]
5260 )
5262 (define_insn "one_cmpl<mode>2"
5263 [(set (match_operand:GPI 0 "register_operand")
5264 (not:GPI (match_operand:GPI 1 "register_operand")))]
5265 ""
5266 {@ [ cons: =0 , 1 ; attrs: type , arch ]
5267 [ r , r ; logic_reg , * ] mvn\t%<w>0, %<w>1
5268 [ w , w ; neon_logic , simd ] mvn\t%0.8b, %1.8b
5269 }
5270 )
5272 (define_insn "*one_cmpl_zero_extend"
5273 [(set (match_operand:DI 0 "register_operand" "=r")
5274 (zero_extend:DI
5275 (not:SI (match_operand:SI 1 "register_operand" "r"))))]
5276 ""
5277 "mvn\\t%w0, %w1"
5278 [(set_attr "type" "logic_reg")]
5279 )
5281 (define_insn "*one_cmpl_<optab><mode>2"
5282 [(set (match_operand:GPI 0 "register_operand" "=r")
5283 (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
5284 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
5285 ""
5286 {
5287 if (<SHIFT:is_rotl>)
5288 operands[2] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5289 return "mvn\\t%<w>0, %<w>1, <shift> %2";
5290 }
5291 [(set_attr "type" "logic_shift_imm")]
5292 )
5294 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
5296 (define_insn "<NLOGICAL:optab>n<mode>3"
5297 [(set (match_operand:GPI 0 "register_operand")
5298 (NLOGICAL:GPI (not:GPI (match_operand:GPI 2 "register_operand"))
5299 (match_operand:GPI 1 "register_operand")))]
5300 ""
5301 {@ [ cons: =0 , 1 , 2 ; attrs: type , arch ]
5302 [ r , r , r ; logic_reg , * ] <NLOGICAL:nlogical>\t%<w>0, %<w>1, %<w>2
5303 [ w , w , w ; neon_logic , simd ] <NLOGICAL:nlogical>\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>
5304 }
5305 )
5307 (define_insn "*<NLOGICAL:optab>nsidi3_ze"
5308 [(set (match_operand:DI 0 "register_operand" "=r")
5309 (zero_extend:DI
5310 (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
5311 (match_operand:SI 2 "register_operand" "r"))))]
5312 ""
5313 "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
5314 [(set_attr "type" "logic_reg")]
5315 )
5317 (define_insn "*xor_one_cmplsidi3_ze"
5318 [(set (match_operand:DI 0 "register_operand" "=r")
5319 (zero_extend:DI
5320 (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
5321 (match_operand:SI 2 "register_operand" "r")))))]
5322 ""
5323 "eon\\t%w0, %w1, %w2"
5324 [(set_attr "type" "logic_reg")]
5325 )
5327 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
5328 ;; eon does not operate on SIMD registers so the vector variant must be split.
5329 (define_insn_and_split "*xor_one_cmpl<mode>3"
5330 [(set (match_operand:GPI 0 "register_operand" "=r,w")
5331 (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
5332 (match_operand:GPI 2 "register_operand" "r,w"))))]
5333 ""
5334 "@
5335 eon\\t%<w>0, %<w>1, %<w>2
5336 #"
5337 "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
5338 [(set (match_operand:GPI 0 "register_operand" "=w")
5339 (xor:GPI (match_operand:GPI 1 "register_operand" "w")
5340 (match_operand:GPI 2 "register_operand" "w")))
5341 (set (match_dup 0) (not:GPI (match_dup 0)))]
5342 ""
5343 [(set_attr "type" "logic_reg,multiple")
5344 (set_attr "arch" "*,simd")]
5345 )
5347 (define_insn "*and_one_cmpl<mode>3_compare0"
5348 [(set (reg:CC_NZV CC_REGNUM)
5349 (compare:CC_NZV
5350 (and:GPI (not:GPI
5351 (match_operand:GPI 1 "register_operand" "r"))
5352 (match_operand:GPI 2 "register_operand" "r"))
5353 (const_int 0)))
5354 (set (match_operand:GPI 0 "register_operand" "=r")
5355 (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
5356 ""
5357 "bics\\t%<w>0, %<w>2, %<w>1"
5358 [(set_attr "type" "logics_reg")]
5359 )
5361 ;; zero_extend version of above
5362 (define_insn "*and_one_cmplsi3_compare0_uxtw"
5363 [(set (reg:CC_NZV CC_REGNUM)
5364 (compare:CC_NZV
5365 (and:SI (not:SI
5366 (match_operand:SI 1 "register_operand" "r"))
5367 (match_operand:SI 2 "register_operand" "r"))
5368 (const_int 0)))
5369 (set (match_operand:DI 0 "register_operand" "=r")
5370 (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
5371 ""
5372 "bics\\t%w0, %w2, %w1"
5373 [(set_attr "type" "logics_reg")]
5374 )
5376 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
5377 [(set (reg:CC_NZV CC_REGNUM)
5378 (compare:CC_NZV
5379 (and:GPI (not:GPI
5380 (match_operand:GPI 0 "register_operand" "r"))
5381 (match_operand:GPI 1 "register_operand" "r"))
5382 (const_int 0)))]
5383 ""
5384 "bics\\t<w>zr, %<w>1, %<w>0"
5385 [(set_attr "type" "logics_reg")]
5386 )
5388 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
5389 [(set (match_operand:GPI 0 "register_operand" "=r")
5390 (LOGICAL:GPI (not:GPI
5391 (SHIFT:GPI
5392 (match_operand:GPI 1 "register_operand" "r")
5393 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
5394 (match_operand:GPI 3 "register_operand" "r")))]
5395 ""
5396 {
5397 if (<SHIFT:is_rotl>)
5398 operands[2] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5399 return "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2";
5400 }
5401 [(set_attr "type" "logic_shift_imm")]
5402 )
5404 ;; Zero-extend version of the above.
5405 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab>sidi_uxtw"
5406 [(set (match_operand:DI 0 "register_operand" "=r")
5407 (zero_extend:DI (LOGICAL:SI (not:SI
5408 (SHIFT:SI
5409 (match_operand:SI 1 "register_operand" "r")
5410 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
5411 (match_operand:SI 3 "register_operand" "r"))))]
5412 ""
5413 {
5414 if (<SHIFT:is_rotl>)
5415 operands[2] = GEN_INT (32 - UINTVAL (operands[2]));
5416 return "<LOGICAL:nlogical>\\t%w0, %w3, %w1, <SHIFT:shift> %2";
5417 }
5418 [(set_attr "type" "logic_shift_imm")]
5419 )
5421 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
5422 [(set (match_operand:GPI 0 "register_operand" "=r")
5423 (not:GPI (xor:GPI
5424 (SHIFT:GPI
5425 (match_operand:GPI 1 "register_operand" "r")
5426 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
5427 (match_operand:GPI 3 "register_operand" "r"))))]
5428 ""
5429 {
5430 if (<SHIFT:is_rotl>)
5431 operands[2] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5432 return "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2";
5433 }
5434 [(set_attr "type" "logic_shift_imm")]
5435 )
5437 ;; Zero-extend version of the above.
5438 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
5439 [(set (match_operand:DI 0 "register_operand" "=r")
5440 (zero_extend:DI
5441 (not:SI (xor:SI
5442 (SHIFT:SI
5443 (match_operand:SI 1 "register_operand" "r")
5444 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
5445 (match_operand:SI 3 "register_operand" "r")))))]
5446 ""
5447 {
5448 if (<SHIFT:is_rotl>)
5449 operands[2] = GEN_INT (32 - UINTVAL (operands[2]));
5450 return "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2";
5451 }
5452 [(set_attr "type" "logic_shift_imm")]
5453 )
5455 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
5456 [(set (reg:CC_NZV CC_REGNUM)
5457 (compare:CC_NZV
5458 (and:GPI (not:GPI
5459 (SHIFT:GPI
5460 (match_operand:GPI 1 "register_operand" "r")
5461 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
5462 (match_operand:GPI 3 "register_operand" "r"))
5463 (const_int 0)))
5464 (set (match_operand:GPI 0 "register_operand" "=r")
5465 (and:GPI (not:GPI
5466 (SHIFT:GPI
5467 (match_dup 1) (match_dup 2))) (match_dup 3)))]
5468 ""
5469 {
5470 if (<SHIFT:is_rotl>)
5471 operands[2] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5472 return "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2";
5473 }
5474 [(set_attr "type" "logics_shift_imm")]
5475 )
5477 ;; zero_extend version of above
5478 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
5479 [(set (reg:CC_NZV CC_REGNUM)
5480 (compare:CC_NZV
5481 (and:SI (not:SI
5482 (SHIFT:SI
5483 (match_operand:SI 1 "register_operand" "r")
5484 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
5485 (match_operand:SI 3 "register_operand" "r"))
5486 (const_int 0)))
5487 (set (match_operand:DI 0 "register_operand" "=r")
5488 (zero_extend:DI (and:SI
5489 (not:SI
5490 (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
5491 ""
5492 {
5493 if (<SHIFT:is_rotl>)
5494 operands[2] = GEN_INT (32 - UINTVAL (operands[2]));
5495 return "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2";
5496 }
5497 [(set_attr "type" "logics_shift_imm")]
5498 )
5500 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
5501 [(set (reg:CC_NZV CC_REGNUM)
5502 (compare:CC_NZV
5503 (and:GPI (not:GPI
5504 (SHIFT:GPI
5505 (match_operand:GPI 0 "register_operand" "r")
5506 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
5507 (match_operand:GPI 2 "register_operand" "r"))
5508 (const_int 0)))]
5509 ""
5510 {
5511 if (<SHIFT:is_rotl>)
5512 operands[1] = GEN_INT (<sizen> - UINTVAL (operands[1]));
5513 return "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1";
5514 }
5515 [(set_attr "type" "logics_shift_imm")]
5516 )
5518 (define_insn "clz<mode>2"
5519 [(set (match_operand:GPI 0 "register_operand" "=r")
5520 (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
5521 ""
5522 "clz\\t%<w>0, %<w>1"
5523 [(set_attr "type" "clz")]
5524 )
5526 (define_expand "ffs<mode>2"
5527 [(match_operand:GPI 0 "register_operand")
5528 (match_operand:GPI 1 "register_operand")]
5529 ""
5530 {
5531 rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
5532 rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
5534 emit_insn (gen_aarch64_rbit (<MODE>mode, operands[0], operands[1]));
5535 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
5536 emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
5537 DONE;
5538 }
5539 )
5541 (define_insn "*aarch64_popcount<mode>2_cssc_insn"
5542 [(set (match_operand:GPI 0 "register_operand" "=r")
5543 (popcount:GPI (match_operand:GPI 1 "register_operand" "r")))]
5544 "TARGET_CSSC"
5545 "cnt\\t%<w>0, %<w>1"
5546 [(set_attr "type" "clz")]
5547 )
5549 ;; The CSSC instructions can do popcount in the GP registers directly through
5550 ;; CNT. If it is not available then we can use CNT on the Advanced SIMD side
5551 ;; through:
5552 ;; MOV v.1d, x0
5553 ;; CNT v1.8b, v.8b
5554 ;; ADDV b2, v1.8b
5555 ;; MOV w0, v2.b[0]
5557 (define_expand "popcount<mode>2"
5558 [(set (match_operand:ALLI 0 "register_operand")
5559 (popcount:ALLI (match_operand:ALLI 1 "register_operand")))]
5560 "TARGET_CSSC ? GET_MODE_BITSIZE (<MODE>mode) >= 32 : TARGET_SIMD"
5561 {
5562 if (!TARGET_CSSC && TARGET_SVE && <MODE>mode != QImode)
5563 {
5564 rtx tmp = gen_reg_rtx (<VEC_POP_MODE>mode);
5565 rtx op1 = gen_lowpart (<VEC_POP_MODE>mode, operands[1]);
5566 emit_insn (gen_popcount<vec_pop_mode>2 (tmp, op1));
5567 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
5568 DONE;
5569 }
5571 if (!TARGET_CSSC)
5572 {
5573 rtx v = gen_reg_rtx (V8QImode);
5574 rtx v1 = gen_reg_rtx (V8QImode);
5575 rtx in = operands[1];
5576 rtx out = operands[0];
5577 /* SImode and HImode should be zero extended to DImode.
5578 popcount does not change if we have extra zeros. */
5579 if (<MODE>mode == SImode || <MODE>mode == HImode)
5580 in = convert_to_mode (DImode, in, true);
5582 emit_move_insn (v, gen_lowpart (V8QImode, in));
5583 emit_insn (gen_popcountv8qi2 (v1, v));
5584 /* QImode, just extract from the v8qi vector. */
5585 if (<MODE>mode == QImode)
5586 emit_move_insn (out, gen_lowpart (QImode, v1));
5587 /* HI and SI, reduction is zero extended to SImode. */
5588 else if (<MODE>mode == SImode || <MODE>mode == HImode)
5589 {
5590 rtx out1 = gen_reg_rtx (SImode);
5591 emit_insn (gen_aarch64_zero_extendsi_reduc_plus_v8qi (out1, v1));
5592 emit_move_insn (out, gen_lowpart (<MODE>mode, out1));
5593 }
5594 /* DImode, reduction is zero extended to DImode. */
5595 else
5596 {
5597 gcc_assert (<MODE>mode == DImode);
5598 emit_insn (gen_aarch64_zero_extenddi_reduc_plus_v8qi (out, v1));
5599 }
5600 DONE;
5601 }
5602 })
5604 (define_expand "popcountti2"
5605 [(match_operand:DI 0 "register_operand")
5606 (match_operand:TI 1 "register_operand")]
5607 "TARGET_SIMD && !TARGET_CSSC"
5608 {
5609 rtx v = gen_reg_rtx (V16QImode);
5610 rtx v1 = gen_reg_rtx (V16QImode);
5611 emit_move_insn (v, gen_lowpart (V16QImode, operands[1]));
5612 emit_insn (gen_popcountv16qi2 (v1, v));
5613 emit_insn (gen_aarch64_zero_extenddi_reduc_plus_v16qi (operands[0], v1));
5614 DONE;
5615 })
5617 (define_insn "clrsb<mode>2"
5618 [(set (match_operand:GPI 0 "register_operand" "=r")
5619 (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
5620 ""
5621 "cls\\t%<w>0, %<w>1"
5622 [(set_attr "type" "clz")]
5623 )
5625 (define_insn "@aarch64_rbit<mode>"
5626 [(set (match_operand:GPI 0 "register_operand" "=r")
5627 (bitreverse:GPI (match_operand:GPI 1 "register_operand" "r")))]
5628 ""
5629 "rbit\\t%<w>0, %<w>1"
5630 [(set_attr "type" "rbit")]
5631 )
5633 ;; Split after reload into RBIT + CLZ. Since RBIT is represented as an UNSPEC
5634 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
5635 ;; expression and split after reload to enable scheduling them apart if
5636 ;; needed. For TARGET_CSSC we have a single CTZ instruction that can do this.
5638 (define_insn_and_split "ctz<mode>2"
5639 [(set (match_operand:GPI 0 "register_operand" "=r")
5640 (ctz:GPI (match_operand:GPI 1 "register_operand" "r")))]
5641 ""
5642 { return TARGET_CSSC ? "ctz\\t%<w>0, %<w>1" : "#"; }
5643 "reload_completed && !TARGET_CSSC"
5644 [(const_int 0)]
5645 "
5646 emit_insn (gen_aarch64_rbit (<MODE>mode, operands[0], operands[1]));
5647 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
5648 DONE;
5649 ")
5651 (define_insn "*and<mode>_compare0"
5652 [(set (reg:CC_Z CC_REGNUM)
5653 (compare:CC_Z
5654 (match_operand:SHORT 0 "register_operand" "r")
5655 (const_int 0)))]
5656 ""
5657 "tst\\t%<w>0, <short_mask>"
5658 [(set_attr "type" "logics_imm")]
5659 )
5661 (define_insn "*ands<GPI:mode>_compare0"
5662 [(set (reg:CC_Z CC_REGNUM)
5663 (compare:CC_Z
5664 (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
5665 (const_int 0)))
5666 (set (match_operand:GPI 0 "register_operand" "=r")
5667 (zero_extend:GPI (match_dup 1)))]
5668 ""
5669 "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
5670 [(set_attr "type" "alus_imm")]
5671 )
5673 (define_insn "@aarch64_and<mode>3nr_compare0"
5674 [(set (reg:CC_NZV CC_REGNUM)
5675 (compare:CC_NZV
5676 (and:GPI (match_operand:GPI 0 "register_operand")
5677 (match_operand:GPI 1 "aarch64_logical_operand"))
5678 (const_int 0)))]
5679 ""
5680 {@ [ cons: 0 , 1 ; attrs: type ]
5681 [ %r , r ; logics_reg ] tst\t%<w>0, %<w>1
5682 [ r , <lconst> ; logics_imm ] tst\t%<w>0, %1
5683 }
5684 )
5686 (define_split
5687 [(set (reg:CC_NZV CC_REGNUM)
5688 (compare:CC_NZV
5689 (and:GPI (match_operand:GPI 0 "register_operand")
5690 (match_operand:GPI 1 "aarch64_mov_imm_operand"))
5691 (const_int 0)))
5692 (clobber (match_operand:SI 2 "register_operand"))]
5693 ""
5694 [(set (match_dup 2) (match_dup 1))
5695 (set (reg:CC_NZV CC_REGNUM)
5696 (compare:CC_NZV
5697 (and:GPI (match_dup 0)
5698 (match_dup 2))
5699 (const_int 0)))]
5700 )
5702 (define_insn "*and<mode>3nr_compare0_zextract"
5703 [(set (reg:CC_Z CC_REGNUM)
5704 (compare:CC_Z
5705 (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
5706 (match_operand:GPI 1 "const_int_operand" "n")
5707 (match_operand:GPI 2 "const_int_operand" "n"))
5708 (const_int 0)))]
5709 "INTVAL (operands[1]) > 0
5710 && ((INTVAL (operands[1]) + INTVAL (operands[2]))
5711 <= GET_MODE_BITSIZE (<MODE>mode))
5712 && aarch64_bitmask_imm (
5713 UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
5714 operands[2])),
5715 <MODE>mode)"
5716 {
5717 operands[1]
5718 = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
5719 return "tst\\t%<w>0, %1";
5720 }
5721 [(set_attr "type" "logics_shift_imm")]
5722 )
5724 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
5725 [(set (reg:CC_NZV CC_REGNUM)
5726 (compare:CC_NZV
5727 (and:GPI (SHIFT:GPI
5728 (match_operand:GPI 0 "register_operand" "r")
5729 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
5730 (match_operand:GPI 2 "register_operand" "r"))
5731 (const_int 0)))]
5732 ""
5733 {
5734 if (<SHIFT:is_rotl>)
5735 operands[1] = GEN_INT (<sizen> - UINTVAL (operands[1]));
5736 return "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1";
5737 }
5738 [(set_attr "type" "logics_shift_imm")]
5739 )
5741 (define_split
5742 [(set (reg:CC_NZV CC_REGNUM)
5743 (compare:CC_NZV
5744 (and:GPI (SHIFT:GPI
5745 (match_operand:GPI 0 "register_operand")
5746 (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
5747 (match_operand:GPI 2 "aarch64_mov_imm_operand"))
5748 (const_int 0)))
5749 (clobber (match_operand:SI 3 "register_operand"))]
5750 ""
5751 [(set (match_dup 3) (match_dup 2))
5752 (set (reg:CC_NZV CC_REGNUM)
5753 (compare:CC_NZV
5754 (and:GPI (SHIFT:GPI
5755 (match_dup 0)
5756 (match_dup 1))
5757 (match_dup 3))
5758 (const_int 0)))]
5759 )
5761 ;; -------------------------------------------------------------------
5762 ;; Shifts
5763 ;; -------------------------------------------------------------------
5765 (define_expand "<optab><mode>3"
5766 [(set (match_operand:GPI 0 "register_operand")
5767 (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
5768 (match_operand:QI 2 "aarch64_reg_or_imm")))]
5769 ""
5770 {
5771 if (CONST_INT_P (operands[2]))
5772 {
5773 operands[2] = GEN_INT (INTVAL (operands[2])
5774 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5776 if (operands[2] == const0_rtx)
5777 {
5778 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5779 DONE;
5780 }
5781 }
5782 }
5783 )
5785 (define_expand "ashl<mode>3"
5786 [(set (match_operand:SHORT 0 "register_operand")
5787 (ashift:SHORT (match_operand:SHORT 1 "register_operand")
5788 (match_operand:QI 2 "const_int_operand")))]
5789 ""
5790 {
5791 operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
5793 if (operands[2] == const0_rtx)
5794 {
5795 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5796 DONE;
5797 }
5798 }
5799 )
5801 (define_expand "rotr<mode>3"
5802 [(set (match_operand:GPI 0 "register_operand")
5803 (rotatert:GPI (match_operand:GPI 1 "register_operand")
5804 (match_operand:QI 2 "aarch64_reg_or_imm")))]
5805 ""
5806 {
5807 if (CONST_INT_P (operands[2]))
5808 {
5809 operands[2] = GEN_INT (INTVAL (operands[2])
5810 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5812 if (operands[2] == const0_rtx)
5813 {
5814 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5815 DONE;
5816 }
5817 }
5818 }
5819 )
5821 (define_expand "rotl<mode>3"
5822 [(set (match_operand:GPI 0 "register_operand")
5823 (rotatert:GPI (match_operand:GPI 1 "register_operand")
5824 (match_operand:QI 2 "aarch64_reg_or_imm")))]
5825 ""
5826 {
5827 /* (SZ - cnt) % SZ == -cnt % SZ */
5828 if (CONST_INT_P (operands[2]))
5829 {
5830 operands[2] = GEN_INT ((-UINTVAL (operands[2]))
5831 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
5832 if (operands[2] == const0_rtx)
5833 {
5834 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5835 DONE;
5836 }
5837 }
5838 else
5839 operands[2] = expand_simple_unop (QImode, NEG, operands[2],
5840 NULL_RTX, 1);
5841 }
5842 )
5844 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
5845 ;; they truncate the shift/rotate amount by the size of the registers they
5846 ;; operate on: 32 for W-regs, 64 for X-regs. This allows us to optimise away
5847 ;; such redundant masking instructions. GCC can do that automatically when
5848 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
5849 ;; because some of the SISD shift alternatives don't perform this truncations.
5850 ;; So this pattern exists to catch such cases.
5852 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
5853 [(set (match_operand:GPI 0 "register_operand" "=r")
5854 (SHIFT:GPI
5855 (match_operand:GPI 1 "register_operand" "r")
5856 (match_operator 4 "subreg_lowpart_operator"
5857 [(and:GPI (match_operand:GPI 2 "register_operand" "r")
5858 (match_operand 3 "const_int_operand" "n"))])))]
5859 "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
5860 "<shift>\t%<w>0, %<w>1, %<w>2"
5861 [(set_attr "type" "shift_reg")]
5862 )
5864 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
5865 [(set (match_operand:GPI 0 "register_operand" "=&r")
5866 (SHIFT:GPI
5867 (match_operand:GPI 1 "register_operand" "r")
5868 (match_operator 4 "subreg_lowpart_operator"
5869 [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
5870 (match_operand 3 "const_int_operand" "n")))])))]
5871 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
5872 "#"
5873 "&& true"
5874 [(const_int 0)]
5875 {
5876 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5877 : lowpart_subreg (SImode, operands[0], <MODE>mode));
5878 emit_insn (gen_negsi2 (tmp, operands[2]));
5880 rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
5881 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
5882 SUBREG_BYTE (operands[4]));
5883 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
5884 DONE;
5885 }
5886 )
5888 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
5889 [(set (match_operand:GPI 0 "register_operand" "=&r")
5890 (ashift:GPI
5891 (match_operand:GPI 1 "register_operand" "r")
5892 (minus:QI (match_operand 2 "const_int_operand" "n")
5893 (match_operator 5 "subreg_lowpart_operator"
5894 [(and:SI (match_operand:SI 3 "register_operand" "r")
5895 (match_operand 4 "const_int_operand" "n"))]))))]
5896 "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
5897 && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5898 "#"
5899 "&& true"
5900 [(const_int 0)]
5901 {
5902 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5903 : operands[0]);
5905 emit_insn (gen_negsi2 (tmp, operands[3]));
5907 rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5908 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5909 SUBREG_BYTE (operands[5]));
5911 emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5912 DONE;
5913 }
5914 )
5916 (define_insn "*aarch64_<optab>_reg_di3_mask2"
5917 [(set (match_operand:DI 0 "register_operand" "=r")
5918 (SHIFT:DI
5919 (match_operand:DI 1 "register_operand" "r")
5920 (match_operator 4 "subreg_lowpart_operator"
5921 [(and:SI (match_operand:SI 2 "register_operand" "r")
5922 (match_operand 3 "const_int_operand" "n"))])))]
5923 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
5924 {
5925 rtx xop[3];
5926 xop[0] = operands[0];
5927 xop[1] = operands[1];
5928 xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5929 output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5930 return "";
5931 }
5932 [(set_attr "type" "shift_reg")]
5933 )
5935 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
5936 [(set (match_operand:GPI 0 "register_operand" "=&r")
5937 (ASHIFT:GPI
5938 (match_operand:GPI 1 "register_operand" "r")
5939 (minus:QI (match_operand 2 "const_int_operand" "n")
5940 (match_operand:QI 3 "register_operand" "r"))))]
5941 "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5942 "#"
5943 "&& true"
5944 [(const_int 0)]
5945 {
5946 rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5948 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5949 : gen_lowpart (SImode, operands[0]));
5951 emit_insn (gen_negsi2 (tmp, subreg_tmp));
5953 rtx and_op = gen_rtx_AND (SImode, tmp,
5954 GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5956 rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5958 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5959 DONE;
5960 }
5961 [(set_attr "length" "8")]
5962 )
5964 ;; Logical left shift using SISD or Integer instruction
5965 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
5966 [(set (match_operand:GPI 0 "register_operand")
5967 (ashift:GPI
5968 (match_operand:GPI 1 "register_operand")
5969 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>")))]
5970 ""
5971 {@ [ cons: =0 , 1 , 2 ; attrs: type , arch ]
5972 [ r , r , Us<cmode> ; bfx , * ] lsl\t%<w>0, %<w>1, %2
5973 [ r , r , r ; shift_reg , * ] lsl\t%<w>0, %<w>1, %<w>2
5974 [ w , w , Us<cmode> ; neon_shift_imm<q> , simd ] shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5975 [ w , w , w ; neon_shift_reg<q> , simd ] ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
5976 }
5977 )
5979 ;; Logical right shift using SISD or Integer instruction
5980 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
5981 [(set (match_operand:GPI 0 "register_operand")
5982 (lshiftrt:GPI
5983 (match_operand:GPI 1 "register_operand")
5984 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>")))]
5985 ""
5986 {@ [ cons: =0 , 1 , 2 ; attrs: type , arch ]
5987 [ r , r , Us<cmode> ; bfx , * ] lsr\t%<w>0, %<w>1, %2
5988 [ r , r , r ; shift_reg , * ] lsr\t%<w>0, %<w>1, %<w>2
5989 [ w , w , Us<cmode_simd> ; neon_shift_imm<q> , simd ] ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5990 [ &w , w , w ; neon_shift_reg<q> , simd ] #
5991 [ &w , w , 0 ; neon_shift_reg<q> , simd ] #
5992 }
5993 )
5995 (define_split
5996 [(set (match_operand:DI 0 "aarch64_simd_register")
5997 (lshiftrt:DI
5998 (match_operand:DI 1 "aarch64_simd_register")
5999 (match_operand:QI 2 "aarch64_simd_register")))]
6000 "TARGET_SIMD && reload_completed"
6001 [(set (match_dup 3)
6002 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
6003 (set (match_dup 0)
6004 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
6005 {
6006 operands[3] = gen_lowpart (QImode, operands[0]);
6007 }
6008 )
6010 (define_split
6011 [(set (match_operand:SI 0 "aarch64_simd_register")
6012 (lshiftrt:SI
6013 (match_operand:SI 1 "aarch64_simd_register")
6014 (match_operand:QI 2 "aarch64_simd_register")))]
6015 "TARGET_SIMD && reload_completed"
6016 [(set (match_dup 3)
6017 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
6018 (set (match_dup 0)
6019 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
6020 {
6021 operands[3] = gen_lowpart (QImode, operands[0]);
6022 }
6023 )
6025 ;; Arithmetic right shift using SISD or Integer instruction
6026 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
6027 [(set (match_operand:GPI 0 "register_operand")
6028 (ashiftrt:GPI
6029 (match_operand:GPI 1 "register_operand")
6030 (match_operand:QI 2 "aarch64_reg_or_shift_imm_di")))]
6031 ""
6032 {@ [ cons: =0 , 1 , 2 ; attrs: type , arch ]
6033 [ r , r , Us<cmode> ; bfx , * ] asr\t%<w>0, %<w>1, %2
6034 [ r , r , r ; shift_reg , * ] asr\t%<w>0, %<w>1, %<w>2
6035 [ w , w , Us<cmode_simd> ; neon_shift_imm<q> , simd ] sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
6036 [ &w , w , w ; neon_shift_reg<q> , simd ] #
6037 [ &w , w , 0 ; neon_shift_reg<q> , simd ] #
6038 }
6039 )
6041 (define_split
6042 [(set (match_operand:DI 0 "aarch64_simd_register")
6043 (ashiftrt:DI
6044 (match_operand:DI 1 "aarch64_simd_register")
6045 (match_operand:QI 2 "aarch64_simd_register")))]
6046 "TARGET_SIMD && reload_completed"
6047 [(set (match_dup 3)
6048 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
6049 (set (match_dup 0)
6050 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
6051 {
6052 operands[3] = gen_lowpart (QImode, operands[0]);
6053 }
6054 )
6056 (define_split
6057 [(set (match_operand:SI 0 "aarch64_simd_register")
6058 (ashiftrt:SI
6059 (match_operand:SI 1 "aarch64_simd_register")
6060 (match_operand:QI 2 "aarch64_simd_register")))]
6061 "TARGET_SIMD && reload_completed"
6062 [(set (match_dup 3)
6063 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
6064 (set (match_dup 0)
6065 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
6066 {
6067 operands[3] = gen_lowpart (QImode, operands[0]);
6068 }
6069 )
6071 (define_insn "*aarch64_sisd_ushl"
6072 [(set (match_operand:DI 0 "register_operand" "=w")
6073 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
6074 (match_operand:QI 2 "register_operand" "w")]
6075 UNSPEC_SISD_USHL))]
6076 "TARGET_SIMD"
6077 "ushl\t%d0, %d1, %d2"
6078 [(set_attr "type" "neon_shift_reg")]
6079 )
6081 (define_insn "*aarch64_ushl_2s"
6082 [(set (match_operand:SI 0 "register_operand" "=w")
6083 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
6084 (match_operand:QI 2 "register_operand" "w")]
6085 UNSPEC_USHL_2S))]
6086 "TARGET_SIMD"
6087 "ushl\t%0.2s, %1.2s, %2.2s"
6088 [(set_attr "type" "neon_shift_reg")]
6089 )
6091 (define_insn "*aarch64_sisd_sshl"
6092 [(set (match_operand:DI 0 "register_operand" "=w")
6093 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
6094 (match_operand:QI 2 "register_operand" "w")]
6095 UNSPEC_SISD_SSHL))]
6096 "TARGET_SIMD"
6097 "sshl\t%d0, %d1, %d2"
6098 [(set_attr "type" "neon_shift_reg")]
6099 )
6101 (define_insn "*aarch64_sshl_2s"
6102 [(set (match_operand:SI 0 "register_operand" "=w")
6103 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
6104 (match_operand:QI 2 "register_operand" "w")]
6105 UNSPEC_SSHL_2S))]
6106 "TARGET_SIMD"
6107 "sshl\t%0.2s, %1.2s, %2.2s"
6108 [(set_attr "type" "neon_shift_reg")]
6109 )
6111 (define_insn "*aarch64_sisd_neg_qi"
6112 [(set (match_operand:QI 0 "register_operand" "=w")
6113 (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
6114 UNSPEC_SISD_NEG))]
6115 "TARGET_SIMD"
6116 "neg\t%d0, %d1"
6117 [(set_attr "type" "neon_neg")]
6118 )
6120 ;; Rotate right
6121 (define_insn "*ror<mode>3_insn"
6122 [(set (match_operand:GPI 0 "register_operand")
6123 (rotatert:GPI
6124 (match_operand:GPI 1 "register_operand")
6125 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>")))]
6126 ""
6127 {@ [ cons: =0 , 1 , 2 ; attrs: type ]
6128 [ r , r , Us<cmode> ; rotate_imm ] ror\t%<w>0, %<w>1, %2
6129 [ r , r , r ; shift_reg ] ror\t%<w>0, %<w>1, %<w>2
6130 }
6131 )
6133 (define_insn "*rol<mode>3_insn"
6134 [(set (match_operand:GPI 0 "register_operand" "=r")
6135 (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
6136 (match_operand 2 "const_int_operand" "n")))]
6137 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
6138 {
6139 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
6140 return "ror\\t%<w>0, %<w>1, %3";
6141 }
6142 [(set_attr "type" "rotate_imm")]
6143 )
6145 ;; zero_extend version of shifts
6146 (define_insn "*<optab>si3_insn_uxtw"
6147 [(set (match_operand:DI 0 "register_operand")
6148 (zero_extend:DI (SHIFT_no_rotate:SI
6149 (match_operand:SI 1 "register_operand")
6150 (match_operand:QI 2 "aarch64_reg_or_shift_imm_si"))))]
6151 ""
6152 {@ [ cons: =0 , 1 , 2 ; attrs: type ]
6153 [ r , r , Uss ; bfx ] <shift>\t%w0, %w1, %2
6154 [ r , r , r ; shift_reg ] <shift>\t%w0, %w1, %w2
6155 }
6156 )
6158 ;; zero_extend version of rotate right
6159 (define_insn "*rorsi3_insn_uxtw"
6160 [(set (match_operand:DI 0 "register_operand" "=r")
6161 (zero_extend:DI
6162 (rotatert:SI (match_operand:SI 1 "register_operand" "r")
6163 (match_operand 2 "const_int_operand" "n"))))]
6164 "UINTVAL (operands[2]) < 32"
6165 "ror\\t%w0, %w1, %2"
6166 [(set_attr "type" "rotate_imm")]
6167 )
6169 ;; zero_extend version of rotate left
6170 (define_insn "*rolsi3_insn_uxtw"
6171 [(set (match_operand:DI 0 "register_operand" "=r")
6172 (zero_extend:DI
6173 (rotate:SI (match_operand:SI 1 "register_operand" "r")
6174 (match_operand 2 "const_int_operand" "n"))))]
6175 "UINTVAL (operands[2]) < 32"
6176 {
6177 operands[2] = GEN_INT (32 - UINTVAL (operands[2]));
6178 return "ror\\t%w0, %w1, %2";
6179 }
6180 [(set_attr "type" "rotate_imm")]
6181 )
6183 (define_insn "*<optab><mode>3_insn"
6184 [(set (match_operand:SHORT 0 "register_operand" "=r")
6185 (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
6186 (match_operand 2 "const_int_operand" "n")))]
6187 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
6188 {
6189 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
6190 return "<bfshift>\t%w0, %w1, %2, %3";
6191 }
6192 [(set_attr "type" "bfx")]
6193 )
6195 (define_insn "*extr<mode>5_insn"
6196 [(set (match_operand:GPI 0 "register_operand" "=r")
6197 (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
6198 (match_operand 3 "const_int_operand" "n"))
6199 (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
6200 (match_operand 4 "const_int_operand" "n"))))]
6201 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
6202 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
6203 "extr\\t%<w>0, %<w>1, %<w>2, %4"
6204 [(set_attr "type" "rotate_imm")]
6205 )
6207 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
6208 ;; so we have to match both orderings.
6209 (define_insn "*extr<mode>5_insn_alt"
6210 [(set (match_operand:GPI 0 "register_operand" "=r")
6211 (ior:GPI (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
6212 (match_operand 4 "const_int_operand" "n"))
6213 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
6214 (match_operand 3 "const_int_operand" "n"))))]
6215 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
6216 && (UINTVAL (operands[3]) + UINTVAL (operands[4])
6217 == GET_MODE_BITSIZE (<MODE>mode))"
6218 "extr\\t%<w>0, %<w>1, %<w>2, %4"
6219 [(set_attr "type" "rotate_imm")]
6220 )
6222 ;; zero_extend version of the above
6223 (define_insn "*extrsi5_insn_uxtw"
6224 [(set (match_operand:DI 0 "register_operand" "=r")
6225 (zero_extend:DI
6226 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
6227 (match_operand 3 "const_int_operand" "n"))
6228 (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
6229 (match_operand 4 "const_int_operand" "n")))))]
6230 "UINTVAL (operands[3]) < 32 &&
6231 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
6232 "extr\\t%w0, %w1, %w2, %4"
6233 [(set_attr "type" "rotate_imm")]
6234 )
6236 (define_insn "*extrsi5_insn_uxtw_alt"
6237 [(set (match_operand:DI 0 "register_operand" "=r")
6238 (zero_extend:DI
6239 (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
6240 (match_operand 4 "const_int_operand" "n"))
6241 (ashift:SI (match_operand:SI 1 "register_operand" "r")
6242 (match_operand 3 "const_int_operand" "n")))))]
6243 "UINTVAL (operands[3]) < 32 &&
6244 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
6245 "extr\\t%w0, %w1, %w2, %4"
6246 [(set_attr "type" "rotate_imm")]
6247 )
6249 (define_insn "*extrsi5_insn_di"
6250 [(set (match_operand:SI 0 "register_operand" "=r")
6251 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
6252 (match_operand 3 "const_int_operand" "n"))
6253 (match_operator:SI 6 "subreg_lowpart_operator"
6254 [(zero_extract:DI
6255 (match_operand:DI 2 "register_operand" "r")
6256 (match_operand 5 "const_int_operand" "n")
6257 (match_operand 4 "const_int_operand" "n"))])))]
6258 "UINTVAL (operands[3]) < 32
6259 && UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32
6260 && INTVAL (operands[3]) == INTVAL (operands[5])"
6261 "extr\\t%w0, %w1, %w2, %4"
6262 [(set_attr "type" "rotate_imm")]
6263 )
6265 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
6266 [(set (match_operand:GPI 0 "register_operand" "=r")
6267 (ANY_EXTEND:GPI
6268 (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
6269 (match_operand 2 "const_int_operand" "n"))))]
6270 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
6271 {
6272 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
6273 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
6274 }
6275 [(set_attr "type" "bfx")]
6276 )
6278 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
6279 [(set (match_operand:GPI 0 "register_operand" "=r")
6280 (zero_extend:GPI
6281 (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
6282 (match_operand 2 "const_int_operand" "n"))))]
6283 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
6284 {
6285 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
6286 return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
6287 }
6288 [(set_attr "type" "bfx")]
6289 )
6291 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
6292 [(set (match_operand:GPI 0 "register_operand" "=r")
6293 (sign_extend:GPI
6294 (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
6295 (match_operand 2 "const_int_operand" "n"))))]
6296 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
6297 {
6298 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
6299 return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
6300 }
6301 [(set_attr "type" "bfx")]
6302 )
6304 ;; -------------------------------------------------------------------
6305 ;; Bitfields
6306 ;; -------------------------------------------------------------------
6308 (define_expand "<optab>"
6309 [(set (match_operand:DI 0 "register_operand")
6310 (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
6311 (match_operand 2
6312 "aarch64_simd_shift_imm_offset_di")
6313 (match_operand 3 "aarch64_simd_shift_imm_di")))]
6314 ""
6315 {
6316 if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
6317 1, GET_MODE_BITSIZE (DImode) - 1))
6318 FAIL;
6319 }
6320 )
6323 (define_insn "*<optab><mode>"
6324 [(set (match_operand:GPI 0 "register_operand" "=r")
6325 (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
6326 (match_operand 2
6327 "aarch64_simd_shift_imm_offset_<mode>" "n")
6328 (match_operand 3
6329 "aarch64_simd_shift_imm_<mode>" "n")))]
6330 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
6331 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
6332 "<su>bfx\\t%<w>0, %<w>1, %3, %2"
6333 [(set_attr "type" "bfx")]
6334 )
6336 ;; When the bit position and width add up to 32 we can use a W-reg LSR
6337 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
6338 (define_split
6339 [(set (match_operand:DI 0 "register_operand")
6340 (zero_extract:DI (match_operand:DI 1 "register_operand")
6341 (match_operand 2
6342 "aarch64_simd_shift_imm_offset_di")
6343 (match_operand 3
6344 "aarch64_simd_shift_imm_di")))]
6345 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
6346 GET_MODE_BITSIZE (DImode) - 1)
6347 && (INTVAL (operands[2]) + INTVAL (operands[3]))
6348 == GET_MODE_BITSIZE (SImode)"
6349 [(set (match_dup 0)
6350 (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
6351 {
6352 operands[4] = gen_lowpart (SImode, operands[1]);
6353 }
6354 )
6356 ;; Bitfield Insert (insv)
6357 (define_expand "insv<mode>"
6358 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
6359 (match_operand 1 "const_int_operand")
6360 (match_operand 2 "const_int_operand"))
6361 (match_operand:GPI 3 "general_operand"))]
6362 ""
6363 {
6364 unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
6365 unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
6366 rtx value = operands[3];
6368 if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
6369 FAIL;
6371 if (CONST_INT_P (value))
6372 {
6373 unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
6375 /* Prefer AND/OR for inserting all zeros or all ones. */
6376 if ((UINTVAL (value) & mask) == 0
6377 || (UINTVAL (value) & mask) == mask)
6378 FAIL;
6380 /* 16-bit aligned 16-bit wide insert is handled by insv_imm. */
6381 if (width == 16 && (pos % 16) == 0)
6382 DONE;
6383 }
6384 operands[3] = force_reg (<MODE>mode, value);
6385 })
6387 (define_insn "*insv_reg<mode>_<SUBDI_BITS>"
6388 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r,w,?w")
6389 (const_int SUBDI_BITS)
6390 (match_operand 1 "const_int_operand"))
6391 (match_operand:GPI 2 "register_operand" "r,w,r"))]
6392 "multiple_p (UINTVAL (operands[1]), <SUBDI_BITS>)
6393 && UINTVAL (operands[1]) + <SUBDI_BITS> <= <GPI:sizen>"
6394 {
6395 if (which_alternative == 0)
6396 return "bfi\t%<w>0, %<w>2, %1, <SUBDI_BITS>";
6398 operands[1] = gen_int_mode (UINTVAL (operands[1]) / <SUBDI_BITS>, SImode);
6399 if (which_alternative == 1)
6400 return "ins\t%0.<bits_etype>[%1], %2.<bits_etype>[0]";
6401 return "ins\t%0.<bits_etype>[%1], %w2";
6402 }
6403 [(set_attr "type" "bfm,neon_ins_q,neon_ins_q")
6404 (set_attr "arch" "*,simd,simd")]
6405 )
6407 (define_insn "*insv_reg<mode>"
6408 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
6409 (match_operand 1 "const_int_operand" "n")
6410 (match_operand 2 "const_int_operand" "n"))
6411 (match_operand:GPI 3 "register_operand" "r"))]
6412 "!(UINTVAL (operands[1]) == 0
6413 || (UINTVAL (operands[2]) + UINTVAL (operands[1])
6414 > GET_MODE_BITSIZE (<MODE>mode)))"
6415 "bfi\\t%<w>0, %<w>3, %2, %1"
6416 [(set_attr "type" "bfm")]
6417 )
6419 (define_insn_and_split "*aarch64_bfi<GPI:mode><ALLX:mode>_<SUBDI_BITS>"
6420 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r,w,?w")
6421 (const_int SUBDI_BITS)
6422 (match_operand 1 "const_int_operand"))
6423 (zero_extend:GPI (match_operand:ALLX 2 "register_operand" "r,w,r")))]
6424 "<SUBDI_BITS> <= <ALLX:sizen>
6425 && multiple_p (UINTVAL (operands[1]), <SUBDI_BITS>)
6426 && UINTVAL (operands[1]) + <SUBDI_BITS> <= <GPI:sizen>"
6427 "#"
6428 "&& 1"
6429 [(set (zero_extract:GPI (match_dup 0)
6430 (const_int SUBDI_BITS)
6431 (match_dup 1))
6432 (match_dup 2))]
6433 {
6434 operands[2] = lowpart_subreg (<GPI:MODE>mode, operands[2],
6435 <ALLX:MODE>mode);
6436 }
6437 [(set_attr "type" "bfm,neon_ins_q,neon_ins_q")
6438 (set_attr "arch" "*,simd,simd")]
6439 )
6441 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
6442 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
6443 (match_operand 1 "const_int_operand" "n")
6444 (match_operand 2 "const_int_operand" "n"))
6445 (zero_extend:GPI (match_operand:ALLX 3 "register_operand" "r")))]
6446 "UINTVAL (operands[1]) <= <ALLX:sizen>"
6447 "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
6448 [(set_attr "type" "bfm")]
6449 )
6451 (define_insn_and_split "*aarch64_bfidi<ALLX:mode>_subreg_<SUBDI_BITS>"
6452 [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+r,w,?w")
6453 (const_int SUBDI_BITS)
6454 (match_operand 1 "const_int_operand"))
6455 (match_operator:DI 2 "subreg_lowpart_operator"
6456 [(zero_extend:SI
6457 (match_operand:ALLX 3 "register_operand" "r,w,r"))]))]
6458 "<SUBDI_BITS> <= <ALLX:sizen>
6459 && multiple_p (UINTVAL (operands[1]), <SUBDI_BITS>)
6460 && UINTVAL (operands[1]) + <SUBDI_BITS> <= 64"
6461 "#"
6462 "&& 1"
6463 [(set (zero_extract:DI (match_dup 0)
6464 (const_int SUBDI_BITS)
6465 (match_dup 1))
6466 (match_dup 2))]
6467 {
6468 operands[2] = lowpart_subreg (DImode, operands[3], <ALLX:MODE>mode);
6469 }
6470 [(set_attr "type" "bfm,neon_ins_q,neon_ins_q")
6471 (set_attr "arch" "*,simd,simd")]
6472 )
6474 ;; Match a bfi instruction where the shift of OP3 means that we are
6475 ;; actually copying the least significant bits of OP3 into OP0 by way
6476 ;; of the AND masks and the IOR instruction. A similar instruction
6477 ;; with the two parts of the IOR swapped around was never triggered
6478 ;; in a bootstrap build and test of GCC so it was not included.
6480 (define_insn "*aarch64_bfi<GPI:mode>5_shift"
6481 [(set (match_operand:GPI 0 "register_operand" "=r")
6482 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
6483 (match_operand:GPI 2 "const_int_operand" "n"))
6484 (and:GPI (ashift:GPI
6485 (match_operand:GPI 3 "register_operand" "r")
6486 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))
6487 (match_operand:GPI 5 "const_int_operand" "n"))))]
6488 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
6489 UINTVAL (operands[4]),
6490 UINTVAL(operands[5]))"
6491 "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5"
6492 [(set_attr "type" "bfm")]
6493 )
6495 (define_insn "*aarch64_bfi<GPI:mode>5_shift_alt"
6496 [(set (match_operand:GPI 0 "register_operand" "=r")
6497 (ior:GPI (and:GPI (ashift:GPI
6498 (match_operand:GPI 1 "register_operand" "r")
6499 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
6500 (match_operand:GPI 3 "const_int_operand" "n"))
6501 (and:GPI (match_operand:GPI 4 "register_operand" "0")
6502 (match_operand:GPI 5 "const_int_operand" "n"))))]
6503 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]),
6504 UINTVAL (operands[2]),
6505 UINTVAL(operands[3]))"
6506 "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3"
6507 [(set_attr "type" "bfm")]
6508 )
6510 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because
6511 ;; the shift is large enough to remove the need for an AND instruction.
6513 (define_insn "*aarch64_bfi<GPI:mode>4_noand"
6514 [(set (match_operand:GPI 0 "register_operand" "=r")
6515 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
6516 (match_operand:GPI 2 "const_int_operand" "n"))
6517 (ashift:GPI
6518 (match_operand:GPI 3 "register_operand" "r")
6519 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))]
6520 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
6521 UINTVAL (operands[4]),
6522 HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )"
6523 {
6524 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4]));
6525 return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5";
6526 }
6527 [(set_attr "type" "bfm")]
6528 )
6530 (define_insn "*aarch64_bfi<GPI:mode>4_noand_alt"
6531 [(set (match_operand:GPI 0 "register_operand" "=r")
6532 (ior:GPI (ashift:GPI
6533 (match_operand:GPI 1 "register_operand" "r")
6534 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
6535 (and:GPI (match_operand:GPI 3 "register_operand" "0")
6536 (match_operand:GPI 4 "const_int_operand" "n"))))]
6537 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]),
6538 UINTVAL (operands[2]),
6539 HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )"
6540 {
6541 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2]));
6542 return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5";
6543 }
6544 [(set_attr "type" "bfm")]
6545 )
6547 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just
6548 ;; copying the least significant bits of OP3 to OP0. We need two versions
6549 ;; of the instruction to handle different checks on the constant values.
6551 (define_insn "*aarch64_bfi<GPI:mode>4_noshift"
6552 [(set (match_operand:GPI 0 "register_operand" "=r")
6553 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
6554 (match_operand:GPI 2 "const_int_operand" "n"))
6555 (and:GPI (match_operand:GPI 3 "register_operand" "r")
6556 (match_operand:GPI 4 "const_int_operand" "n"))))]
6557 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
6558 UINTVAL (operands[4]))"
6559 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
6560 [(set_attr "type" "bfm")]
6561 )
6563 (define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt"
6564 [(set (match_operand:GPI 0 "register_operand" "=r")
6565 (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r")
6566 (match_operand:GPI 4 "const_int_operand" "n"))
6567 (and:GPI (match_operand:GPI 1 "register_operand" "0")
6568 (match_operand:GPI 2 "const_int_operand" "n"))))]
6569 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
6570 UINTVAL (operands[4]))"
6571 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
6572 [(set_attr "type" "bfm")]
6573 )
6575 (define_insn "*aarch64_bfxil<mode>_extr"
6576 [(set (match_operand:GPI 0 "register_operand" "=r")
6577 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
6578 (match_operand:GPI 2 "const_int_operand" "n"))
6579 (zero_extract:GPI
6580 (match_operand:GPI 3 "register_operand" "r")
6581 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n")
6582 (match_operand:GPI 5 "aarch64_simd_shift_imm_<mode>" "n"))))]
6583 "UINTVAL (operands[2]) == HOST_WIDE_INT_M1U << INTVAL (operands[4])
6584 && INTVAL (operands[4])
6585 && (UINTVAL (operands[4]) + UINTVAL (operands[5])
6586 <= GET_MODE_BITSIZE (<MODE>mode))"
6587 "bfxil\t%<GPI:w>0, %<GPI:w>3, %5, %4"
6588 [(set_attr "type" "bfm")]
6589 )
6591 (define_insn "*aarch64_bfxilsi_extrdi"
6592 [(set (match_operand:SI 0 "register_operand" "=r")
6593 (ior:SI (and:SI (match_operand:SI 1 "register_operand" "0")
6594 (match_operand:SI 2 "const_int_operand" "n"))
6595 (match_operator:SI 6 "subreg_lowpart_operator"
6596 [(zero_extract:DI
6597 (match_operand:DI 3 "register_operand" "r")
6598 (match_operand:SI 4 "aarch64_simd_shift_imm_si" "n")
6599 (match_operand:SI 5 "aarch64_simd_shift_imm_si" "n"))])))]
6600 "UINTVAL (operands[2]) == HOST_WIDE_INT_M1U << INTVAL (operands[4])
6601 && INTVAL (operands[4])
6602 && UINTVAL (operands[4]) + UINTVAL (operands[5]) <= 32"
6603 "bfxil\t%w0, %w3, %5, %4"
6604 [(set_attr "type" "bfm")]
6605 )
6607 (define_insn "*extr_insv_lower_reg<mode>"
6608 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
6609 (match_operand 1 "const_int_operand" "n")
6610 (const_int 0))
6611 (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
6612 (match_dup 1)
6613 (match_operand 3 "const_int_operand" "n")))]
6614 "!(UINTVAL (operands[1]) == 0
6615 || (UINTVAL (operands[3]) + UINTVAL (operands[1])
6616 > GET_MODE_BITSIZE (<MODE>mode)))"
6617 "bfxil\\t%<w>0, %<w>2, %3, %1"
6618 [(set_attr "type" "bfm")]
6619 )
6621 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
6622 [(set (match_operand:GPI 0 "register_operand" "=r")
6623 (ashift:GPI (ANY_EXTEND:GPI
6624 (match_operand:ALLX 1 "register_operand" "r"))
6625 (match_operand 2 "const_int_operand" "n")))]
6626 "UINTVAL (operands[2]) < <GPI:sizen>"
6627 {
6628 operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
6629 ? GEN_INT (<ALLX:sizen>)
6630 : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
6631 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
6632 }
6633 [(set_attr "type" "bfx")]
6634 )
6636 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
6638 (define_insn "*andim_ashift<mode>_bfiz"
6639 [(set (match_operand:GPI 0 "register_operand" "=r")
6640 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
6641 (match_operand 2 "const_int_operand" "n"))
6642 (match_operand 3 "const_int_operand" "n")))]
6643 "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
6644 "ubfiz\\t%<w>0, %<w>1, %2, %P3"
6645 [(set_attr "type" "bfx")]
6646 )
6648 ;; Match sbfiz pattern in a shift left + shift right operation.
6650 (define_insn "*ashift<mode>_extv_bfiz"
6651 [(set (match_operand:GPI 0 "register_operand" "=r")
6652 (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
6653 (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
6654 (const_int 0))
6655 (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
6656 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
6657 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
6658 "sbfiz\\t%<w>0, %<w>1, %3, %2"
6659 [(set_attr "type" "bfx")]
6660 )
6662 (define_insn "*ashiftsi_extvdi_bfiz"
6663 [(set (match_operand:SI 0 "register_operand" "=r")
6664 (ashift:SI
6665 (match_operator:SI 4 "subreg_lowpart_operator"
6666 [(sign_extract:DI
6667 (match_operand:DI 1 "register_operand" "r")
6668 (match_operand 2 "aarch64_simd_shift_imm_offset_si")
6669 (const_int 0))])
6670 (match_operand 3 "aarch64_simd_shift_imm_si")))]
6671 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
6672 1, GET_MODE_BITSIZE (SImode) - 1)"
6673 "sbfiz\\t%w0, %w1, %3, %2"
6674 [(set_attr "type" "bfx")]
6675 )
6677 ;; When the bit position and width of the equivalent extraction add up to 32
6678 ;; we can use a W-reg LSL instruction taking advantage of the implicit
6679 ;; zero-extension of the X-reg.
6680 (define_split
6681 [(set (match_operand:DI 0 "register_operand")
6682 (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
6683 (match_operand 2 "const_int_operand"))
6684 (match_operand 3 "const_int_operand")))]
6685 "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
6686 && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
6687 == GET_MODE_BITSIZE (SImode)"
6688 [(set (match_dup 0)
6689 (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
6690 {
6691 operands[4] = gen_lowpart (SImode, operands[1]);
6692 }
6693 )
6695 (define_insn "bswap<mode>2"
6696 [(set (match_operand:GPI 0 "register_operand" "=r")
6697 (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
6698 ""
6699 "rev\\t%<w>0, %<w>1"
6700 [(set_attr "type" "rev")]
6701 )
6703 (define_insn "bswaphi2"
6704 [(set (match_operand:HI 0 "register_operand" "=r")
6705 (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
6706 ""
6707 "rev16\\t%w0, %w1"
6708 [(set_attr "type" "rev")]
6709 )
6711 (define_insn "*aarch64_bfxil<mode>"
6712 [(set (match_operand:GPI 0 "register_operand" "=r,r")
6713 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
6714 (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
6715 (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
6716 (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
6717 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
6718 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
6719 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
6720 {
6721 switch (which_alternative)
6722 {
6723 case 0:
6724 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
6725 return "bfxil\\t%<w>0, %<w>1, 0, %3";
6726 case 1:
6727 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
6728 return "bfxil\\t%<w>0, %<w>2, 0, %3";
6729 default:
6730 gcc_unreachable ();
6731 }
6732 }
6733 [(set_attr "type" "bfm")]
6734 )
6736 ; Zero-extended version of above (aarch64_bfxil)
6737 (define_insn "*aarch64_bfxilsi_uxtw"
6738 [(set (match_operand:DI 0 "register_operand" "=r,r")
6739 (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
6740 "r,0")
6741 (match_operand:SI 3 "const_int_operand" "n, Ulc"))
6742 (and:SI (match_operand:SI 2 "register_operand" "0,r")
6743 (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
6744 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
6745 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
6746 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
6747 {
6748 switch (which_alternative)
6749 {
6750 case 0:
6751 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
6752 return "bfxil\\t%w0, %w1, 0, %3";
6753 case 1:
6754 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
6755 return "bfxil\\t%w0, %w2, 0, %3";
6756 default:
6757 gcc_unreachable ();
6758 }
6759 }
6760 [(set_attr "type" "bfm")]
6761 )
6763 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
6764 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
6765 ;; each valid permutation.
6767 (define_insn "aarch64_rev16<mode>2_alt1"
6768 [(set (match_operand:GPI 0 "register_operand" "=r")
6769 (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
6770 (const_int 8))
6771 (match_operand:GPI 3 "const_int_operand" "n"))
6772 (and:GPI (lshiftrt:GPI (match_dup 1)
6773 (const_int 8))
6774 (match_operand:GPI 2 "const_int_operand" "n"))))]
6775 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
6776 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
6777 "rev16\\t%<w>0, %<w>1"
6778 [(set_attr "type" "rev")]
6779 )
6781 (define_insn "*aarch64_rev16<mode>2_alt2"
6782 [(set (match_operand:GPI 0 "register_operand" "=r")
6783 (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
6784 (const_int 8))
6785 (match_operand:GPI 2 "const_int_operand" "n"))
6786 (and:GPI (ashift:GPI (match_dup 1)
6787 (const_int 8))
6788 (match_operand:GPI 3 "const_int_operand" "n"))))]
6789 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
6790 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
6791 "rev16\\t%<w>0, %<w>1"
6792 [(set_attr "type" "rev")]
6793 )
6795 ;; Similar pattern to match (rotate (bswap) 16)
6796 (define_insn "aarch64_rev16si2_alt3"
6797 [(set (match_operand:SI 0 "register_operand" "=r")
6798 (rotate:SI (bswap:SI (match_operand:SI 1 "register_operand" "r"))
6799 (const_int 16)))]
6800 ""
6801 "rev16\\t%w0, %w1"
6802 [(set_attr "type" "rev")]
6803 )
6805 ;; zero_extend version of above
6806 (define_insn "*bswapsi2_uxtw"
6807 [(set (match_operand:DI 0 "register_operand" "=r")
6808 (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
6809 ""
6810 "rev\\t%w0, %w1"
6811 [(set_attr "type" "rev")]
6812 )
6814 ;; Expander for __rev16 intrinsics. We have organic RTL patterns for rev16 above.
6815 ;; Use this expander to just create the shift constants needed.
6816 (define_expand "@aarch64_rev16<mode>"
6817 [(match_operand:GPI 0 "register_operand")
6818 (match_operand:GPI 1 "register_operand")]
6819 ""
6820 {
6821 rtx left = gen_int_mode (HOST_WIDE_INT_C (0xff00ff00ff00ff00), <MODE>mode);
6822 rtx right = gen_int_mode (HOST_WIDE_INT_C (0xff00ff00ff00ff), <MODE>mode);
6823 emit_insn (gen_aarch64_rev16<mode>2_alt1 (operands[0], operands[1],
6824 right, left));
6825 DONE;
6826 }
6827 )
6829 ;; -------------------------------------------------------------------
6830 ;; Floating-point intrinsics
6831 ;; -------------------------------------------------------------------
6833 ;; frint floating-point round to integral standard patterns.
6834 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, roundeven.
6836 (define_insn "<frint_pattern><mode>2"
6837 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6838 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
6839 FRINT))]
6840 "TARGET_FLOAT"
6841 "frint<frint_suffix>\\t%<s>0, %<s>1"
6842 [(set_attr "type" "f_rint<stype>")]
6843 )
6845 ;; frcvt floating-point round to integer and convert standard patterns.
6846 ;; Expands to lbtrunc, lceil, lfloor, lround.
6847 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
6848 [(set (match_operand:GPI 0 "register_operand" "=r")
6849 (FIXUORS:GPI
6850 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
6851 FCVT)))]
6852 "TARGET_FLOAT"
6853 "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
6854 [(set_attr "type" "f_cvtf2i")]
6855 )
6857 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
6858 [(set (match_operand:GPI 0 "register_operand" "=r")
6859 (FIXUORS:GPI
6860 (mult:GPF
6861 (match_operand:GPF 1 "register_operand" "w")
6862 (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
6863 "TARGET_FLOAT
6864 && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
6865 GET_MODE_BITSIZE (<GPI:MODE>mode))"
6866 {
6867 int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
6868 char buf[64];
6869 snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
6870 output_asm_insn (buf, operands);
6871 return "";
6872 }
6873 [(set_attr "type" "f_cvtf2i")]
6874 )
6876 ;; fma - expand fma into patterns with the accumulator operand first since
6877 ;; reusing the accumulator results in better register allocation.
6878 ;; The register allocator considers copy preferences in operand order,
6879 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
6881 (define_expand "fma<mode>4"
6882 [(set (match_operand:GPF_F16 0 "register_operand")
6883 (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
6884 (match_operand:GPF_F16 2 "register_operand")
6885 (match_operand:GPF_F16 3 "register_operand")))]
6886 "TARGET_FLOAT"
6887 )
6889 (define_insn "*aarch64_fma<mode>4"
6890 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6891 (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
6892 (match_operand:GPF_F16 3 "register_operand" "w")
6893 (match_operand:GPF_F16 1 "register_operand" "w")))]
6894 "TARGET_FLOAT"
6895 "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
6896 [(set_attr "type" "fmac<stype>")]
6897 )
6899 (define_expand "fnma<mode>4"
6900 [(set (match_operand:GPF_F16 0 "register_operand")
6901 (fma:GPF_F16
6902 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
6903 (match_operand:GPF_F16 2 "register_operand")
6904 (match_operand:GPF_F16 3 "register_operand")))]
6905 "TARGET_FLOAT"
6906 )
6908 (define_insn "*aarch64_fnma<mode>4"
6909 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6910 (fma:GPF_F16
6911 (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
6912 (match_operand:GPF_F16 3 "register_operand" "w")
6913 (match_operand:GPF_F16 1 "register_operand" "w")))]
6914 "TARGET_FLOAT"
6915 "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
6916 [(set_attr "type" "fmac<stype>")]
6917 )
6920 (define_expand "fms<mode>4"
6921 [(set (match_operand:GPF 0 "register_operand")
6922 (fma:GPF (match_operand:GPF 1 "register_operand")
6923 (match_operand:GPF 2 "register_operand")
6924 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
6925 "TARGET_FLOAT"
6926 )
6928 (define_insn "*aarch64_fms<mode>4"
6929 [(set (match_operand:GPF 0 "register_operand" "=w")
6930 (fma:GPF (match_operand:GPF 2 "register_operand" "w")
6931 (match_operand:GPF 3 "register_operand" "w")
6932 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
6933 "TARGET_FLOAT"
6934 "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
6935 [(set_attr "type" "fmac<s>")]
6936 )
6938 (define_expand "fnms<mode>4"
6939 [(set (match_operand:GPF 0 "register_operand")
6940 (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
6941 (match_operand:GPF 2 "register_operand")
6942 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
6943 "TARGET_FLOAT"
6944 )
6946 (define_insn "*aarch64_fnms<mode>4"
6947 [(set (match_operand:GPF 0 "register_operand" "=w")
6948 (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
6949 (match_operand:GPF 3 "register_operand" "w")
6950 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
6951 "TARGET_FLOAT"
6952 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
6953 [(set_attr "type" "fmac<s>")]
6954 )
6956 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
6957 (define_insn "*aarch64_fnmadd<mode>4"
6958 [(set (match_operand:GPF 0 "register_operand" "=w")
6959 (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
6960 (match_operand:GPF 3 "register_operand" "w")
6961 (match_operand:GPF 1 "register_operand" "w"))))]
6962 "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
6963 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
6964 [(set_attr "type" "fmac<s>")]
6965 )
6967 ;; -------------------------------------------------------------------
6968 ;; Floating-point conversions
6969 ;; -------------------------------------------------------------------
6971 (define_insn "extendsfdf2"
6972 [(set (match_operand:DF 0 "register_operand" "=w")
6973 (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
6974 "TARGET_FLOAT"
6975 "fcvt\\t%d0, %s1"
6976 [(set_attr "type" "f_cvt")]
6977 )
6979 (define_insn "extendhfsf2"
6980 [(set (match_operand:SF 0 "register_operand" "=w")
6981 (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
6982 "TARGET_FLOAT"
6983 "fcvt\\t%s0, %h1"
6984 [(set_attr "type" "f_cvt")]
6985 )
6987 (define_insn "extendhfdf2"
6988 [(set (match_operand:DF 0 "register_operand" "=w")
6989 (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
6990 "TARGET_FLOAT"
6991 "fcvt\\t%d0, %h1"
6992 [(set_attr "type" "f_cvt")]
6993 )
6995 (define_insn "truncdfsf2"
6996 [(set (match_operand:SF 0 "register_operand" "=w")
6997 (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
6998 "TARGET_FLOAT"
6999 "fcvt\\t%s0, %d1"
7000 [(set_attr "type" "f_cvt")]
7001 )
7003 (define_insn "truncsfhf2"
7004 [(set (match_operand:HF 0 "register_operand" "=w")
7005 (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
7006 "TARGET_FLOAT"
7007 "fcvt\\t%h0, %s1"
7008 [(set_attr "type" "f_cvt")]
7009 )
7011 (define_insn "truncdfhf2"
7012 [(set (match_operand:HF 0 "register_operand" "=w")
7013 (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
7014 "TARGET_FLOAT"
7015 "fcvt\\t%h0, %d1"
7016 [(set_attr "type" "f_cvt")]
7017 )
7019 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
7020 ;; and making r = w more expensive
7022 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
7023 [(set (match_operand:GPI 0 "register_operand")
7024 (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand")))]
7025 "TARGET_FLOAT"
7026 {@ [ cons: =0 , 1 ; attrs: type , arch ]
7027 [ w , w ; neon_fp_to_int_s , simd ] fcvtz<su>\t%<s>0, %<s>1
7028 [ ?r , w ; f_cvtf2i , fp ] fcvtz<su>\t%<w>0, %<s>1
7029 }
7030 )
7032 ;; Convert HF -> SI or DI
7034 (define_insn "<optab>_trunchf<GPI:mode>2"
7035 [(set (match_operand:GPI 0 "register_operand" "=r")
7036 (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
7037 "TARGET_FP_F16INST"
7038 "fcvtz<su>\t%<w>0, %h1"
7039 [(set_attr "type" "f_cvtf2i")]
7040 )
7042 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
7043 ;; input in a fp register and output in a integer register
7045 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
7046 [(set (match_operand:GPI 0 "register_operand" "=r")
7047 (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
7048 "TARGET_FLOAT"
7049 "fcvtz<su>\t%<w>0, %<fpw>1"
7050 [(set_attr "type" "f_cvtf2i")]
7051 )
7053 (define_insn "*fix_to_zero_extend<mode>di2"
7054 [(set (match_operand:DI 0 "register_operand" "=r")
7055 (zero_extend:DI
7056 (unsigned_fix:SI
7057 (match_operand:GPF 1 "register_operand" "w"))))]
7058 "TARGET_FLOAT"
7059 "fcvtzu\t%w0, %<s>1"
7060 [(set_attr "type" "f_cvtf2i")]
7061 )
7063 ;; Equal width integer to fp and multiply combine.
7064 (define_insn "*aarch64_<su_optab>cvtf<fcvt_target><GPF:mode>2_mult"
7065 [(set (match_operand:GPF 0 "register_operand" "=w,w")
7066 (mult:GPF (FLOATUORS:GPF
7067 (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r"))
7068 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt,Dt")))]
7069 "TARGET_FLOAT"
7070 {
7071 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
7072 switch (which_alternative)
7073 {
7074 case 0:
7075 return "<su_optab>cvtf\t%<GPF:s>0, %<s>1, #%2";
7076 case 1:
7077 return "<su_optab>cvtf\t%<GPF:s>0, %<w1>1, #%2";
7078 default:
7079 gcc_unreachable ();
7080 }
7081 }
7082 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
7083 (set_attr "arch" "simd,fp")]
7084 )
7086 ;; Unequal width integer to fp and multiply combine.
7087 (define_insn "*aarch64_<su_optab>cvtf<fcvt_iesize><GPF:mode>2_mult"
7088 [(set (match_operand:GPF 0 "register_operand" "=w")
7089 (mult:GPF (FLOATUORS:GPF
7090 (match_operand:<FCVT_IESIZE> 1 "register_operand" "r"))
7091 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt")))]
7092 "TARGET_FLOAT"
7093 {
7094 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
7095 return "<su_optab>cvtf\t%<GPF:s>0, %<w2>1, #%2";
7096 }
7097 [(set_attr "type" "f_cvti2f")]
7098 )
7100 ;; Equal width integer to fp conversion.
7101 (define_insn "<optab><fcvt_target><GPF:mode>2"
7102 [(set (match_operand:GPF 0 "register_operand")
7103 (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand")))]
7104 "TARGET_FLOAT"
7105 {@ [ cons: =0 , 1 ; attrs: type , arch ]
7106 [ w , w ; neon_int_to_fp_<Vetype> , simd ] <su_optab>cvtf\t%<GPF:s>0, %<s>1
7107 [ w , ?r ; f_cvti2f , fp ] <su_optab>cvtf\t%<GPF:s>0, %<w1>1
7108 }
7109 )
7111 ;; Unequal width integer to fp conversions.
7112 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
7113 [(set (match_operand:GPF 0 "register_operand" "=w")
7114 (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
7115 "TARGET_FLOAT"
7116 "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
7117 [(set_attr "type" "f_cvti2f")]
7118 )
7120 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
7121 ;; midend will arrange for an SImode conversion to HFmode to first go
7122 ;; through DFmode, then to HFmode. But first it will try converting
7123 ;; to DImode then down, which would match our DImode pattern below and
7124 ;; give very poor code-generation. So, we must provide our own emulation
7125 ;; of the mid-end logic.
7127 (define_insn "aarch64_fp16_<optab><mode>hf2"
7128 [(set (match_operand:HF 0 "register_operand" "=w")
7129 (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
7130 "TARGET_FP_F16INST"
7131 "<su_optab>cvtf\t%h0, %<w>1"
7132 [(set_attr "type" "f_cvti2f")]
7133 )
7135 (define_expand "<optab>sihf2"
7136 [(set (match_operand:HF 0 "register_operand")
7137 (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
7138 "TARGET_FLOAT"
7139 {
7140 if (TARGET_FP_F16INST)
7141 emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
7142 else
7143 {
7144 rtx convert_target = gen_reg_rtx (DFmode);
7145 emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
7146 emit_insn (gen_truncdfhf2 (operands[0], convert_target));
7147 }
7148 DONE;
7149 }
7150 )
7152 ;; For DImode there is no wide enough floating-point mode that we
7153 ;; can convert through natively (TFmode would work, but requires a library
7154 ;; call). However, we know that any value >= 65504 will be rounded
7155 ;; to infinity on conversion. This is well within the range of SImode, so
7156 ;; we can:
7157 ;; Saturate to SImode.
7158 ;; Convert from that to DFmode
7159 ;; Convert from that to HFmode (phew!).
7160 ;; Note that the saturation to SImode requires the SIMD extensions. If
7161 ;; we ever need to provide this pattern where the SIMD extensions are not
7162 ;; available, we would need a different approach.
7164 (define_expand "<optab>dihf2"
7165 [(set (match_operand:HF 0 "register_operand")
7166 (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
7167 "TARGET_FP_F16INST || TARGET_SIMD"
7168 {
7169 if (TARGET_FP_F16INST)
7170 emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
7171 else
7172 {
7173 rtx sat_target = gen_reg_rtx (SImode);
7174 emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
7175 emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
7176 }
7178 DONE;
7179 }
7180 )
7182 ;; Convert between fixed-point and floating-point (scalar modes)
7184 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
7185 [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand")
7186 (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand")
7187 (match_operand:SI 2 "immediate_operand")]
7188 FCVT_F2FIXED))]
7189 ""
7190 {@ [ cons: =0 , 1 , 2 ; attrs: type , arch ]
7191 [ r , w , i ; f_cvtf2i , fp ] <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
7192 [ w , w , i ; neon_fp_to_int_<GPF:Vetype> , simd ] <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2
7193 }
7194 )
7196 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
7197 [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand")
7198 (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand")
7199 (match_operand:SI 2 "immediate_operand")]
7200 FCVT_FIXED2F))]
7201 ""
7202 {@ [ cons: =0 , 1 , 2 ; attrs: type , arch ]
7203 [ w , r , i ; f_cvti2f , fp ] <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
7204 [ w , w , i ; neon_int_to_fp_<GPI:Vetype> , simd ] <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2
7205 }
7206 )
7208 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
7209 [(set (match_operand:GPI 0 "register_operand" "=r")
7210 (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
7211 (match_operand:SI 2 "immediate_operand" "i")]
7212 FCVT_F2FIXED))]
7213 "TARGET_FP_F16INST"
7214 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
7215 [(set_attr "type" "f_cvtf2i")]
7216 )
7218 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
7219 [(set (match_operand:HF 0 "register_operand" "=w")
7220 (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
7221 (match_operand:SI 2 "immediate_operand" "i")]
7222 FCVT_FIXED2F))]
7223 "TARGET_FP_F16INST"
7224 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
7225 [(set_attr "type" "f_cvti2f")]
7226 )
7228 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
7229 [(set (match_operand:HI 0 "register_operand" "=w")
7230 (unspec:HI [(match_operand:HF 1 "register_operand" "w")
7231 (match_operand:SI 2 "immediate_operand" "i")]
7232 FCVT_F2FIXED))]
7233 "TARGET_SIMD"
7234 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
7235 [(set_attr "type" "neon_fp_to_int_s")]
7236 )
7238 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
7239 [(set (match_operand:HF 0 "register_operand" "=w")
7240 (unspec:HF [(match_operand:HI 1 "register_operand" "w")
7241 (match_operand:SI 2 "immediate_operand" "i")]
7242 FCVT_FIXED2F))]
7243 "TARGET_SIMD"
7244 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
7245 [(set_attr "type" "neon_int_to_fp_s")]
7246 )
7248 ;; -------------------------------------------------------------------
7249 ;; Floating-point arithmetic
7250 ;; -------------------------------------------------------------------
7252 (define_insn "add<mode>3"
7253 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
7254 (plus:GPF_F16
7255 (match_operand:GPF_F16 1 "register_operand" "w")
7256 (match_operand:GPF_F16 2 "register_operand" "w")))]
7257 "TARGET_FLOAT"
7258 "fadd\\t%<s>0, %<s>1, %<s>2"
7259 [(set_attr "type" "fadd<stype>")]
7260 )
7262 (define_insn "sub<mode>3"
7263 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
7264 (minus:GPF_F16
7265 (match_operand:GPF_F16 1 "register_operand" "w")
7266 (match_operand:GPF_F16 2 "register_operand" "w")))]
7267 "TARGET_FLOAT"
7268 "fsub\\t%<s>0, %<s>1, %<s>2"
7269 [(set_attr "type" "fadd<stype>")]
7270 )
7272 (define_insn "mul<mode>3"
7273 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
7274 (mult:GPF_F16
7275 (match_operand:GPF_F16 1 "register_operand" "w")
7276 (match_operand:GPF_F16 2 "register_operand" "w")))]
7277 "TARGET_FLOAT"
7278 "fmul\\t%<s>0, %<s>1, %<s>2"
7279 [(set_attr "type" "fmul<stype>")]
7280 )
7282 (define_insn "*fnmul<mode>3"
7283 [(set (match_operand:GPF 0 "register_operand" "=w")
7284 (mult:GPF
7285 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
7286 (match_operand:GPF 2 "register_operand" "w")))]
7287 "TARGET_FLOAT && !flag_rounding_math"
7288 "fnmul\\t%<s>0, %<s>1, %<s>2"
7289 [(set_attr "type" "fmul<s>")]
7290 )
7292 (define_insn "*fnmul<mode>3"
7293 [(set (match_operand:GPF 0 "register_operand" "=w")
7294 (neg:GPF (mult:GPF
7295 (match_operand:GPF 1 "register_operand" "w")
7296 (match_operand:GPF 2 "register_operand" "w"))))]
7297 "TARGET_FLOAT"
7298 "fnmul\\t%<s>0, %<s>1, %<s>2"
7299 [(set_attr "type" "fmul<s>")]
7300 )
7302 (define_expand "div<mode>3"
7303 [(set (match_operand:GPF_F16 0 "register_operand")
7304 (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
7305 (match_operand:GPF_F16 2 "register_operand")))]
7306 "TARGET_FLOAT"
7307 {
7308 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
7309 DONE;
7311 operands[1] = force_reg (<MODE>mode, operands[1]);
7312 })
7314 (define_insn "*div<mode>3"
7315 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
7316 (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
7317 (match_operand:GPF_F16 2 "register_operand" "w")))]
7318 "TARGET_FLOAT"
7319 "fdiv\\t%<s>0, %<s>1, %<s>2"
7320 [(set_attr "type" "fdiv<stype>")]
7321 )
7323 (define_insn "neg<mode>2"
7324 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
7325 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
7326 "TARGET_FLOAT"
7327 "fneg\\t%<s>0, %<s>1"
7328 [(set_attr "type" "ffarith<stype>")]
7329 )
7331 (define_expand "sqrt<mode>2"
7332 [(set (match_operand:GPF_F16 0 "register_operand")
7333 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))]
7334 "TARGET_FLOAT"
7335 {
7336 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
7337 DONE;
7338 })
7340 (define_insn "*sqrt<mode>2"
7341 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
7342 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
7343 "TARGET_FLOAT"
7344 "fsqrt\\t%<s>0, %<s>1"
7345 [(set_attr "type" "fsqrt<stype>")]
7346 )
7348 (define_insn "abs<mode>2"
7349 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
7350 (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
7351 "TARGET_FLOAT"
7352 "fabs\\t%<s>0, %<s>1"
7353 [(set_attr "type" "ffarith<stype>")]
7354 )
7356 ;; Expander for integer smin, smax, umin. Mainly used to generate
7357 ;; straightforward RTL for TARGET_CSSC. When that is not available
7358 ;; FAIL and let the generic expanders generate the CMP + CSEL sequences,
7359 ;; except for the SMIN and SMAX with zero cases, for which we have a
7360 ;; single instruction even for the base architecture.
7361 (define_expand "<optab><mode>3"
7362 [(set (match_operand:GPI 0 "register_operand")
7363 (MAXMIN_NOUMAX:GPI
7364 (match_operand:GPI 1 "register_operand")
7365 (match_operand:GPI 2 "aarch64_<su>minmax_operand")))]
7366 ""
7367 {
7368 if (!TARGET_CSSC)
7369 {
7370 if (operands[2] != CONST0_RTX (<MODE>mode)
7371 || !(<CODE> == SMAX || <CODE> == SMIN))
7372 FAIL;
7373 }
7374 }
7375 )
7377 (define_insn "*aarch64_<optab><mode>3_cssc"
7378 [(set (match_operand:GPI 0 "register_operand")
7379 (MAXMIN:GPI (match_operand:GPI 1 "register_operand")
7380 (match_operand:GPI 2 "aarch64_<su>minmax_operand")))]
7381 "TARGET_CSSC"
7382 {@ [ cons: =0 , 1 , 2 ; attrs: type ]
7383 [ r , r , r ; alu_sreg ] <optab>\t%<w>0, %<w>1, %<w>2
7384 [ r , r , U<su>m ; alu_imm ] <optab>\t%<w>0, %<w>1, %2
7385 }
7386 )
7388 (define_insn "*aarch64_<optab><mode>3_zero"
7389 [(set (match_operand:GPI 0 "register_operand" "=r")
7390 (FMAXMIN:GPI
7391 (match_operand:GPI 1 "register_operand" "r")
7392 (const_int 0)))]
7393 ""
7394 "<maxminand>\\t%<w>0, %<w>1, %<w>1, asr <sizem1>";
7395 [(set_attr "type" "logic_shift_imm")]
7396 )
7398 ;; Given that smax/smin do not specify the result when either input is NaN,
7399 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
7400 ;; for smin.
7402 (define_insn "smax<mode>3"
7403 [(set (match_operand:GPF 0 "register_operand" "=w")
7404 (smax:GPF (match_operand:GPF 1 "register_operand" "w")
7405 (match_operand:GPF 2 "register_operand" "w")))]
7406 "TARGET_FLOAT"
7407 "fmaxnm\\t%<s>0, %<s>1, %<s>2"
7408 [(set_attr "type" "f_minmax<s>")]
7409 )
7411 (define_insn "smin<mode>3"
7412 [(set (match_operand:GPF 0 "register_operand" "=w")
7413 (smin:GPF (match_operand:GPF 1 "register_operand" "w")
7414 (match_operand:GPF 2 "register_operand" "w")))]
7415 "TARGET_FLOAT"
7416 "fminnm\\t%<s>0, %<s>1, %<s>2"
7417 [(set_attr "type" "f_minmax<s>")]
7418 )
7420 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
7421 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
7422 ;; which implement the IEEE fmax ()/fmin () functions.
7423 (define_insn "<fmaxmin><mode>3"
7424 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
7425 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
7426 (match_operand:GPF_F16 2 "register_operand" "w")]
7427 FMAXMIN_UNS))]
7428 "TARGET_FLOAT"
7429 "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
7430 [(set_attr "type" "f_minmax<stype>")]
7431 )
7433 (define_expand "lrint<GPF:mode><GPI:mode>2"
7434 [(match_operand:GPI 0 "register_operand")
7435 (match_operand:GPF 1 "register_operand")]
7436 "TARGET_FLOAT
7437 && ((GET_MODE_BITSIZE (<GPF:MODE>mode) <= LONG_TYPE_SIZE)
7438 || !flag_trapping_math || flag_fp_int_builtin_inexact)"
7439 {
7440 rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
7441 emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
7442 emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
7443 DONE;
7444 }
7445 )
7447 ;; For copysignf (x, y), we want to generate:
7448 ;;
7449 ;; movi v31.4s, 0x80, lsl 24
7450 ;; bit v0.16b, v1.16b, v31.16b
7451 ;;
7452 ;; Because we expect these operations to nearly always operate on
7453 ;; floating-point values, we do not want the operation to be
7454 ;; simplified into a bit-field insert operation that operates on the
7455 ;; integer side, since typically that would involve three inter-bank
7456 ;; register copies. As we do not expect copysign to be followed by
7457 ;; other logical operations on the result, it seems preferable to keep
7458 ;; this as an unspec operation, rather than exposing the underlying
7459 ;; logic to the compiler.
7461 (define_expand "copysign<GPF:mode>3"
7462 [(match_operand:GPF 0 "register_operand")
7463 (match_operand:GPF 1 "register_operand")
7464 (match_operand:GPF 2 "nonmemory_operand")]
7465 "TARGET_SIMD"
7466 {
7467 rtx sign = GEN_INT (HOST_WIDE_INT_M1U << (GET_MODE_BITSIZE (<MODE>mode) - 1));
7468 rtx v_bitmask = gen_const_vec_duplicate (<VQ_INT_EQUIV>mode, sign);
7469 v_bitmask = force_reg (<VQ_INT_EQUIV>mode, v_bitmask);
7471 /* copysign (x, -1) should instead be expanded as orr with the signbit. */
7472 rtx op2_elt = unwrap_const_vec_duplicate (operands[2]);
7474 if (GET_CODE (op2_elt) == CONST_DOUBLE
7475 && real_isneg (CONST_DOUBLE_REAL_VALUE (op2_elt)))
7476 {
7477 emit_insn (gen_ior<vq_int_equiv>3 (
7478 lowpart_subreg (<VQ_INT_EQUIV>mode, operands[0], <MODE>mode),
7479 lowpart_subreg (<VQ_INT_EQUIV>mode, operands[1], <MODE>mode),
7480 v_bitmask));
7481 DONE;
7482 }
7483 operands[2] = force_reg (<MODE>mode, operands[2]);
7484 emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
7485 v_bitmask));
7486 DONE;
7487 }
7488 )
7490 (define_insn "copysign<GPF:mode>3_insn"
7491 [(set (match_operand:GPF 0 "register_operand")
7492 (unspec:GPF [(match_operand:GPF 1 "register_operand")
7493 (match_operand:GPF 2 "register_operand")
7494 (match_operand:<VQ_INT_EQUIV> 3 "register_operand")]
7495 UNSPEC_COPYSIGN))]
7496 "TARGET_SIMD"
7497 {@ [ cons: =0 , 1 , 2 , 3 ; attrs: type ]
7498 [ w , w , w , 0 ; neon_bsl<q> ] bsl\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
7499 [ w , 0 , w , w ; neon_bsl<q> ] bit\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
7500 [ w , w , 0 , w ; neon_bsl<q> ] bif\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
7501 }
7502 )
7504 ;; For xorsignf (x, y), we want to generate:
7505 ;;
7506 ;; movi v31.4s, 0x80, lsl 24
7507 ;; and v31.16b, v31.16b, v1.16b
7508 ;; eor v0.16b, v31.16b, v0.16b
7509 ;;
7511 (define_expand "@xorsign<mode>3"
7512 [(match_operand:GPF 0 "register_operand")
7513 (match_operand:GPF 1 "register_operand")
7514 (match_operand:GPF 2 "register_operand")]
7515 "TARGET_SIMD"
7516 {
7517 rtx tmp = gen_reg_rtx (<VCONQ>mode);
7518 rtx op1 = force_lowpart_subreg (<VCONQ>mode, operands[1], <MODE>mode);
7519 rtx op2 = force_lowpart_subreg (<VCONQ>mode, operands[2], <MODE>mode);
7520 emit_insn (gen_xorsign3 (<VCONQ>mode, tmp, op1, op2));
7521 emit_move_insn (operands[0],
7522 lowpart_subreg (<MODE>mode, tmp, <VCONQ>mode));
7523 DONE;
7524 }
7525 )
7527 ;; -------------------------------------------------------------------
7528 ;; Reload support
7529 ;; -------------------------------------------------------------------
7530 ;; Reload Scalar Floating point modes from constant pool.
7531 ;; The AArch64 port doesn't have __int128 constant move support.
7532 ;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook.
7533 (define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
7534 [(set (match_operand:GPF_TF 0 "register_operand" "=w")
7535 (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
7536 (clobber (match_operand:P 2 "register_operand" "=&r"))]
7537 "TARGET_FLOAT"
7538 {
7539 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
7540 emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
7541 DONE;
7542 }
7543 )
7545 ;; Reload Vector modes from constant pool.
7546 (define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
7547 [(set (match_operand:VALL 0 "register_operand" "=w")
7548 (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
7549 (clobber (match_operand:P 2 "register_operand" "=&r"))]
7550 "TARGET_FLOAT"
7551 {
7552 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
7553 emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
7554 DONE;
7555 }
7556 )
7558 (define_expand "@aarch64_reload_mov<mode>"
7559 [(set (match_operand:VTX 0 "register_operand" "=w")
7560 (match_operand:VTX 1 "register_operand" "w"))
7561 (clobber (match_operand:DI 2 "register_operand" "=&r"))
7562 ]
7563 "TARGET_FLOAT"
7564 {
7565 rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
7566 rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
7567 gen_aarch64_movtilow_tilow (op0, op1);
7568 gen_aarch64_movdi_tihigh (operands[2], op1);
7569 gen_aarch64_movtihigh_di (op0, operands[2]);
7570 DONE;
7571 }
7572 )
7574 ;; The following secondary reload helpers patterns are invoked
7575 ;; after or during reload as we don't want these patterns to start
7576 ;; kicking in during the combiner.
7578 (define_insn "@aarch64_movdi_<mode>low"
7579 [(set (match_operand:DI 0 "register_operand" "=r")
7580 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
7581 (const_int 64) (const_int 0)))]
7582 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
7583 "fmov\\t%x0, %d1"
7584 [(set_attr "type" "f_mrc")
7585 (set_attr "length" "4")
7586 ])
7588 (define_insn "@aarch64_movdi_<mode>high"
7589 [(set (match_operand:DI 0 "register_operand" "=r")
7590 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
7591 (const_int 64) (const_int 64)))]
7592 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
7593 "fmov\\t%x0, %1.d[1]"
7594 [(set_attr "type" "f_mrc")
7595 (set_attr "length" "4")
7596 ])
7598 (define_insn "@aarch64_mov<mode>high_di"
7599 [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
7600 (const_int 64) (const_int 64))
7601 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
7602 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
7603 "fmov\\t%0.d[1], %x1"
7604 [(set_attr "type" "f_mcr")
7605 (set_attr "length" "4")
7606 ])
7608 (define_insn "@aarch64_mov<mode>low_di"
7609 [(set (match_operand:TX 0 "register_operand" "=w")
7610 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
7611 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
7612 "fmov\\t%d0, %x1"
7613 [(set_attr "type" "f_mcr")
7614 (set_attr "length" "4")
7615 ])
7617 (define_insn "aarch64_movtilow_tilow"
7618 [(set (match_operand:TI 0 "register_operand" "=w")
7619 (zero_extend:TI
7620 (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
7621 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
7622 "fmov\\t%d0, %d1"
7623 [(set_attr "type" "fmov")
7624 (set_attr "length" "4")
7625 ])
7627 ;; There is a deliberate reason why the parameters of high and lo_sum's
7628 ;; don't have modes for ADRP and ADD instructions. This is to allow high
7629 ;; and lo_sum's to be used with the labels defining the jump tables in
7630 ;; rodata section.
7632 (define_expand "add_losym"
7633 [(set (match_operand 0 "register_operand")
7634 (lo_sum (match_operand 1 "register_operand")
7635 (match_operand 2 "aarch64_valid_symref")))]
7636 ""
7637 {
7638 machine_mode mode = GET_MODE (operands[0]);
7640 emit_insn ((mode == DImode
7641 ? gen_add_losym_di
7642 : gen_add_losym_si) (operands[0],
7643 operands[1],
7644 operands[2]));
7645 DONE;
7646 })
7648 (define_insn "add_losym_<mode>"
7649 [(set (match_operand:P 0 "register_operand" "=r")
7650 (lo_sum:P (match_operand:P 1 "register_operand" "r")
7651 (match_operand 2 "aarch64_valid_symref" "S")))]
7652 ""
7653 "add\\t%<w>0, %<w>1, :lo12:%c2"
7654 [(set_attr "type" "alu_imm")]
7655 )
7657 (define_insn "ldr_got_small_28k_<mode>"
7658 [(set (match_operand:PTR 0 "register_operand" "=r")
7659 (unspec:PTR [(mem:PTR (lo_sum:PTR
7660 (match_operand:PTR 1 "register_operand" "r")
7661 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
7662 UNSPEC_GOTSMALLPIC28K))]
7663 ""
7664 "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
7665 [(set_attr "type" "load_<ldst_sz>")]
7666 )
7668 (define_insn "ldr_got_small_28k_sidi"
7669 [(set (match_operand:DI 0 "register_operand" "=r")
7670 (zero_extend:DI
7671 (unspec:SI [(mem:SI (lo_sum:DI
7672 (match_operand:DI 1 "register_operand" "r")
7673 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
7674 UNSPEC_GOTSMALLPIC28K)))]
7675 "TARGET_ILP32"
7676 "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
7677 [(set_attr "type" "load_4")]
7678 )
7680 (define_insn "@ldr_got_tiny_<mode>"
7681 [(set (match_operand:PTR 0 "register_operand" "=r")
7682 (unspec:PTR [(match_operand:PTR 1 "aarch64_valid_symref" "S")]
7683 UNSPEC_GOTTINYPIC))]
7684 ""
7685 "ldr\t%<w>0, %L1"
7686 [(set_attr "type" "load_<ldst_sz>")]
7687 )
7689 (define_insn "ldr_got_tiny_sidi"
7690 [(set (match_operand:DI 0 "register_operand" "=r")
7691 (zero_extend:DI
7692 (unspec:SI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
7693 UNSPEC_GOTTINYPIC)))]
7694 "TARGET_ILP32"
7695 "ldr\t%w0, %L1"
7696 [(set_attr "type" "load_4")]
7697 )
7699 (define_insn "aarch64_load_tp_hard"
7700 [(set (match_operand:DI 0 "register_operand" "=r")
7701 (unspec:DI [(const_int 0)] UNSPEC_TLS))]
7702 ""
7703 "* return aarch64_output_load_tp (operands[0]);"
7704 [(set_attr "type" "mrs")]
7705 )
7707 ;; The TLS ABI specifically requires that the compiler does not schedule
7708 ;; instructions in the TLS stubs, in order to enable linker relaxation.
7709 ;; Therefore we treat the stubs as an atomic sequence.
7710 (define_expand "tlsgd_small_<mode>"
7711 [(parallel [(set (match_operand:PTR 0 "register_operand")
7712 (call (mem:DI (match_dup 2)) (const_int 1)))
7713 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
7714 (unspec:DI [(match_operand 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS)
7715 (clobber (reg:DI LR_REGNUM))])]
7716 ""
7717 {
7718 operands[2] = aarch64_tls_get_addr ();
7719 })
7721 (define_insn "*tlsgd_small_<mode>"
7722 [(set (match_operand:PTR 0 "register_operand" "")
7723 (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
7724 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
7725 (unspec:DI [(match_operand 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
7726 (clobber (reg:DI LR_REGNUM))
7727 ]
7728 ""
7729 "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
7730 [(set_attr "type" "call")
7731 (set_attr "length" "16")])
7733 (define_insn "tlsie_small_<mode>"
7734 [(set (match_operand:PTR 0 "register_operand" "=r")
7735 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
7736 UNSPEC_GOTSMALLTLS))]
7737 ""
7738 "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
7739 [(set_attr "type" "load_4")
7740 (set_attr "length" "8")]
7741 )
7743 (define_insn "tlsie_small_sidi"
7744 [(set (match_operand:DI 0 "register_operand" "=r")
7745 (zero_extend:DI
7746 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
7747 UNSPEC_GOTSMALLTLS)))]
7748 ""
7749 "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
7750 [(set_attr "type" "load_4")
7751 (set_attr "length" "8")]
7752 )
7754 (define_insn "tlsie_tiny_<mode>"
7755 [(set (match_operand:PTR 0 "register_operand" "=&r")
7756 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
7757 (match_operand:PTR 2 "register_operand" "r")]
7758 UNSPEC_GOTTINYTLS))]
7759 ""
7760 "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
7761 [(set_attr "type" "multiple")
7762 (set_attr "length" "8")]
7763 )
7765 (define_insn "tlsie_tiny_sidi"
7766 [(set (match_operand:DI 0 "register_operand" "=&r")
7767 (zero_extend:DI
7768 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
7769 (match_operand:DI 2 "register_operand" "r")
7770 ]
7771 UNSPEC_GOTTINYTLS)))]
7772 ""
7773 "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
7774 [(set_attr "type" "multiple")
7775 (set_attr "length" "8")]
7776 )
7778 (define_insn "tlsle12_<mode>"
7779 [(set (match_operand:P 0 "register_operand" "=r")
7780 (unspec:P [(match_operand:P 1 "register_operand" "r")
7781 (match_operand 2 "aarch64_tls_le_symref" "S")]
7782 UNSPEC_TLSLE12))]
7783 ""
7784 "add\\t%<w>0, %<w>1, #%L2";
7785 [(set_attr "type" "alu_sreg")
7786 (set_attr "length" "4")]
7787 )
7789 (define_insn "tlsle24_<mode>"
7790 [(set (match_operand:P 0 "register_operand" "=r")
7791 (unspec:P [(match_operand:P 1 "register_operand" "r")
7792 (match_operand 2 "aarch64_tls_le_symref" "S")]
7793 UNSPEC_TLSLE24))]
7794 ""
7795 "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
7796 [(set_attr "type" "multiple")
7797 (set_attr "length" "8")]
7798 )
7800 (define_insn "tlsle32_<mode>"
7801 [(set (match_operand:P 0 "register_operand" "=r")
7802 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
7803 UNSPEC_TLSLE32))]
7804 ""
7805 "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
7806 [(set_attr "type" "multiple")
7807 (set_attr "length" "8")]
7808 )
7810 (define_insn "tlsle48_<mode>"
7811 [(set (match_operand:P 0 "register_operand" "=r")
7812 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
7813 UNSPEC_TLSLE48))]
7814 ""
7815 "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
7816 [(set_attr "type" "multiple")
7817 (set_attr "length" "12")]
7818 )
7820 (define_expand "tlsdesc_small_<mode>"
7821 [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
7822 "TARGET_TLS_DESC"
7823 {
7824 if (TARGET_SVE)
7825 {
7826 rtx abi = aarch64_gen_callee_cookie (AARCH64_ISA_MODE,
7827 aarch64_tlsdesc_abi_id (),
7828 false);
7829 rtx_insn *call
7830 = emit_call_insn (gen_tlsdesc_small_sve_<mode> (operands[0], abi));
7831 RTL_CONST_CALL_P (call) = 1;
7832 }
7833 else
7834 emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
7835 DONE;
7836 }
7837 )
7839 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
7840 ;; R0 and LR.
7841 (define_insn "tlsdesc_small_advsimd_<mode>"
7842 [(set (reg:PTR R0_REGNUM)
7843 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
7844 UNSPEC_TLSDESC))
7845 (clobber (reg:DI LR_REGNUM))
7846 (clobber (reg:CC CC_REGNUM))
7847 (clobber (match_scratch:DI 1 "=r"))
7848 (use (reg:DI FP_REGNUM))]
7849 "TARGET_TLS_DESC && !TARGET_SVE"
7850 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
7851 [(set_attr "type" "call")
7852 (set_attr "length" "16")])
7854 ;; For SVE, model tlsdesc calls as normal calls, with the callee ABI
7855 ;; describing the extra call-preserved guarantees. This would work
7856 ;; for non-SVE too, but avoiding a call is probably better if we can.
7857 (define_insn "tlsdesc_small_sve_<mode>"
7858 [(set (reg:PTR R0_REGNUM)
7859 (call (mem:DI (unspec:PTR
7860 [(match_operand 0 "aarch64_valid_symref")]
7861 UNSPEC_TLSDESC))
7862 (const_int 0)))
7863 (unspec:DI [(match_operand:DI 1 "const_int_operand")] UNSPEC_CALLEE_ABI)
7864 (clobber (reg:DI LR_REGNUM))]
7865 "TARGET_TLS_DESC && TARGET_SVE"
7866 "adrp\\tx0, %A0\;ldr\\t<w>30, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\tx30"
7867 [(set_attr "type" "call")
7868 (set_attr "length" "16")])
7870 (define_insn "stack_tie"
7871 [(set (mem:BLK (scratch))
7872 (unspec:BLK [(reg:DI SP_REGNUM)
7873 (match_operand:DI 0 "register_operand" "rk")
7874 (match_operand:DI 1 "const_int_operand")]
7875 UNSPEC_PRLG_STK))]
7876 "REGNO (operands[0]) == INTVAL (operands[1])"
7877 ""
7878 [(set_attr "length" "0")]
7879 )
7881 (define_insn "aarch64_fjcvtzs"
7882 [(set (match_operand:SI 0 "register_operand" "=r")
7883 (unspec:SI [(match_operand:DF 1 "register_operand" "w")]
7884 UNSPEC_FJCVTZS))
7885 (clobber (reg:CC CC_REGNUM))]
7886 "TARGET_JSCVT"
7887 "fjcvtzs\\t%w0, %d1"
7888 [(set_attr "type" "f_cvtf2i")]
7889 )
7891 ;; Pointer authentication patterns are always provided. On targets that
7892 ;; don't implement FEAT_PAuth these HINT instructions operate as NOPs.
7893 ;; This lets the user write portable software which authenticates pointers
7894 ;; when run on something which implements FEAT_PAuth, and which runs
7895 ;; correctly, but does not authenticate pointers, where FEAT_PAuth is not
7896 ;; implemented.
7898 ;; Signing/Authenticating R30 using SP as the salt.
7900 (define_insn "<pauth_mnem_prefix>sp"
7901 [(set (reg:DI R30_REGNUM)
7902 (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
7903 ""
7904 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp";
7905 )
7907 ;; Signing/Authenticating X17 using X16 as the salt.
7909 (define_insn "<pauth_mnem_prefix>1716"
7910 [(set (reg:DI R17_REGNUM)
7911 (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
7912 ""
7913 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716";
7914 )
7916 ;; Stripping the signature in R30.
7918 (define_insn "xpaclri"
7919 [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
7920 ""
7921 "hint\t7 // xpaclri"
7922 )
7924 ;; Save X30 in the X18-based POST_INC stack (consistent with clang).
7925 (define_expand "scs_push"
7926 [(set (mem:DI (post_inc:DI (reg:DI R18_REGNUM)))
7927 (reg:DI R30_REGNUM))])
7929 ;; Load X30 form the X18-based PRE_DEC stack (consistent with clang).
7930 (define_expand "scs_pop"
7931 [(set (reg:DI R30_REGNUM)
7932 (mem:DI (pre_dec:DI (reg:DI R18_REGNUM))))])
7934 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
7935 ;; all of memory. This blocks insns from being moved across this point.
7937 (define_insn "blockage"
7938 [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
7939 ""
7940 ""
7941 [(set_attr "length" "0")
7942 (set_attr "type" "block")]
7943 )
7945 (define_insn "probe_stack_range"
7946 [(set (match_operand:DI 0 "register_operand" "=rk")
7947 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
7948 (match_operand:DI 2 "register_operand" "r")]
7949 UNSPECV_PROBE_STACK_RANGE))]
7950 ""
7951 {
7952 return aarch64_output_probe_stack_range (operands[0], operands[2]);
7953 }
7954 [(set_attr "length" "32")]
7955 )
7957 ;; This instruction is used to generate the stack clash stack adjustment and
7958 ;; probing loop. We can't change the control flow during prologue and epilogue
7959 ;; code generation. So we must emit a volatile unspec and expand it later on.
7961 (define_insn "@probe_sve_stack_clash_<mode>"
7962 [(set (match_operand:P 0 "register_operand" "=rk")
7963 (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
7964 (match_operand:P 2 "register_operand" "r")
7965 (match_operand:P 3 "const_int_operand" "n")
7966 (match_operand:P 4 "aarch64_plus_immediate" "L")]
7967 UNSPECV_PROBE_STACK_RANGE))]
7968 "TARGET_SVE"
7969 {
7970 return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
7971 operands[3], operands[4]);
7972 }
7973 [(set_attr "length" "28")]
7974 )
7976 ;; Named pattern for expanding thread pointer reference.
7977 (define_expand "get_thread_pointerdi"
7978 [(match_operand:DI 0 "register_operand")]
7979 ""
7980 {
7981 rtx tmp = aarch64_load_tp (operands[0]);
7982 if (tmp != operands[0])
7983 emit_move_insn (operands[0], tmp);
7984 DONE;
7985 })
7987 ;; Defined for -mstack-protector-guard=sysreg, which goes through this
7988 ;; pattern rather than stack_protect_combined_set. Our implementation
7989 ;; of the latter can handle both.
7990 (define_expand "stack_protect_set"
7991 [(match_operand 0 "memory_operand")
7992 (match_operand 1 "")]
7993 ""
7994 {
7995 emit_insn (gen_stack_protect_combined_set (operands[0], operands[1]));
7996 DONE;
7997 })
7999 (define_expand "stack_protect_combined_set"
8000 [(match_operand 0 "memory_operand")
8001 (match_operand 1 "")]
8002 ""
8003 {
8004 machine_mode mode = GET_MODE (operands[0]);
8005 operands[1] = aarch64_stack_protect_canary_mem (mode, operands[1],
8006 AARCH64_SALT_SSP_SET);
8007 emit_insn ((mode == DImode
8008 ? gen_stack_protect_set_di
8009 : gen_stack_protect_set_si) (operands[0], operands[1]));
8010 DONE;
8011 })
8013 ;; Operand 1 is either AARCH64_SALT_SSP_SET or AARCH64_SALT_SSP_TEST.
8014 (define_insn "reg_stack_protect_address_<mode>"
8015 [(set (match_operand:PTR 0 "register_operand" "=r")
8016 (unspec:PTR [(match_operand 1 "const_int_operand")]
8017 UNSPEC_SSP_SYSREG))]
8018 "aarch64_stack_protector_guard != SSP_GLOBAL"
8019 {
8020 char buf[150];
8021 snprintf (buf, 150, "mrs\\t%%<w>0, %s",
8022 aarch64_stack_protector_guard_reg_str);
8023 output_asm_insn (buf, operands);
8024 return "";
8025 }
8026 [(set_attr "type" "mrs")])
8028 ;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the
8029 ;; canary value does not live beyond the life of this sequence.
8030 (define_insn "stack_protect_set_<mode>"
8031 [(set (match_operand:PTR 0 "memory_operand" "=m")
8032 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
8033 UNSPEC_SP_SET))
8034 (set (match_scratch:PTR 2 "=&r") (const_int 0))]
8035 ""
8036 "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2, 0"
8037 [(set_attr "length" "12")
8038 (set_attr "type" "multiple")])
8040 ;; Defined for -mstack-protector-guard=sysreg, which goes through this
8041 ;; pattern rather than stack_protect_combined_test. Our implementation
8042 ;; of the latter can handle both.
8043 (define_expand "stack_protect_test"
8044 [(match_operand 0 "memory_operand")
8045 (match_operand 1 "")
8046 (match_operand 2)]
8047 ""
8048 {
8049 emit_insn (gen_stack_protect_combined_test (operands[0], operands[1],
8050 operands[2]));
8051 DONE;
8052 })
8054 (define_expand "stack_protect_combined_test"
8055 [(match_operand 0 "memory_operand")
8056 (match_operand 1 "")
8057 (match_operand 2)]
8058 ""
8059 {
8060 machine_mode mode = GET_MODE (operands[0]);
8061 operands[1] = aarch64_stack_protect_canary_mem (mode, operands[1],
8062 AARCH64_SALT_SSP_TEST);
8063 emit_insn ((mode == DImode
8064 ? gen_stack_protect_test_di
8065 : gen_stack_protect_test_si) (operands[0], operands[1]));
8067 rtx cc_reg = gen_rtx_REG (CCmode, CC_REGNUM);
8068 emit_jump_insn (gen_condjump (gen_rtx_EQ (VOIDmode, cc_reg, const0_rtx),
8069 cc_reg, operands[2]));
8070 DONE;
8071 })
8073 ;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the
8074 ;; canary value does not live beyond the end of this sequence.
8075 (define_insn "stack_protect_test_<mode>"
8076 [(set (reg:CC CC_REGNUM)
8077 (unspec:CC [(match_operand:PTR 0 "memory_operand" "m")
8078 (match_operand:PTR 1 "memory_operand" "m")]
8079 UNSPEC_SP_TEST))
8080 (clobber (match_scratch:PTR 2 "=&r"))
8081 (clobber (match_scratch:PTR 3 "=&r"))]
8082 ""
8083 "ldr\t%<w>2, %0\;ldr\t%<w>3, %1\;subs\t%<w>2, %<w>2, %<w>3\;mov\t%3, 0"
8084 [(set_attr "length" "16")
8085 (set_attr "type" "multiple")])
8087 ;; Write into the Floating-point Status or Control Register.
8088 (define_insn "@aarch64_set_<fpscr_name><GPI:mode>"
8089 [(unspec_volatile [(match_operand:GPI 0 "register_operand" "r")] SET_FPSCR)]
8090 ""
8091 "msr\\t<fpscr_name>, %0"
8092 [(set_attr "type" "mrs")])
8094 ;; Read into the Floating-point Status or Control Register.
8095 (define_insn "@aarch64_get_<fpscr_name><GPI:mode>"
8096 [(set (match_operand:GPI 0 "register_operand" "=r")
8097 (unspec_volatile:GPI [(const_int 0)] GET_FPSCR))]
8098 ""
8099 "mrs\\t%0, <fpscr_name>"
8100 [(set_attr "type" "mrs")])
8102 ;; Define the subtract-one-and-jump insns so loop.c
8103 ;; knows what to generate.
8104 (define_expand "doloop_end"
8105 [(use (match_operand 0 "" "")) ; loop pseudo
8106 (use (match_operand 1 "" ""))] ; label
8107 "optimize > 0 && flag_modulo_sched"
8108 {
8109 rtx s0;
8110 rtx bcomp;
8111 rtx loc_ref;
8112 rtx cc_reg;
8113 rtx insn;
8114 rtx cmp;
8116 /* Currently SMS relies on the do-loop pattern to recognize loops
8117 where (1) the control part consists of all insns defining and/or
8118 using a certain 'count' register and (2) the loop count can be
8119 adjusted by modifying this register prior to the loop.
8120 ??? The possible introduction of a new block to initialize the
8121 new IV can potentially affect branch optimizations. */
8123 if (GET_MODE (operands[0]) != DImode)
8124 FAIL;
8126 s0 = operands [0];
8127 insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
8129 cmp = XVECEXP (PATTERN (insn), 0, 0);
8130 cc_reg = SET_DEST (cmp);
8131 bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
8132 loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
8133 emit_jump_insn (gen_rtx_SET (pc_rtx,
8134 gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
8135 loc_ref, pc_rtx)));
8136 DONE;
8137 })
8139 ;; Track speculation through conditional branches. We assume that
8140 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
8141 (define_insn "speculation_tracker"
8142 [(set (reg:DI SPECULATION_TRACKER_REGNUM)
8143 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
8144 UNSPEC_SPECULATION_TRACKER))]
8145 ""
8146 {
8147 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
8148 output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
8149 return "";
8150 }
8151 [(set_attr "type" "csel")]
8152 )
8154 ;; Like speculation_tracker, but track the inverse condition.
8155 (define_insn "speculation_tracker_rev"
8156 [(set (reg:DI SPECULATION_TRACKER_REGNUM)
8157 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
8158 UNSPEC_SPECULATION_TRACKER_REV))]
8159 ""
8160 {
8161 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
8162 output_asm_insn ("csel\\t%1, %1, xzr, %M0", operands);
8163 return "";
8164 }
8165 [(set_attr "type" "csel")]
8166 )
8168 ;; BTI <target> instructions
8169 (define_insn "bti_noarg"
8170 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)]
8171 ""
8172 "hint\t32 // bti"
8173 [(set_attr "type" "no_insn")]
8174 )
8176 (define_insn "bti_c"
8177 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)]
8178 ""
8179 "hint\t34 // bti c"
8180 [(set_attr "type" "no_insn")]
8181 )
8183 (define_insn "bti_j"
8184 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)]
8185 ""
8186 "hint\t36 // bti j"
8187 [(set_attr "type" "no_insn")]
8188 )
8190 (define_insn "bti_jc"
8191 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)]
8192 ""
8193 "hint\t38 // bti jc"
8194 [(set_attr "type" "no_insn")]
8195 )
8197 ;; Hard speculation barrier.
8198 (define_insn "speculation_barrier"
8199 [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
8200 ""
8201 "isb\;dsb\\tsy"
8202 [(set_attr "length" "8")
8203 (set_attr "type" "block")
8204 (set_attr "speculation_barrier" "true")]
8205 )
8207 ;; Support for __builtin_speculation_safe_value when we have speculation
8208 ;; tracking enabled. Use the speculation tracker to decide whether to
8209 ;; copy operand 1 to the target, or to copy the fail value (operand 2).
8210 (define_expand "@despeculate_copy<ALLI_TI:mode>"
8211 [(set (match_operand:ALLI_TI 0 "register_operand")
8212 (unspec_volatile:ALLI_TI
8213 [(match_operand:ALLI_TI 1 "register_operand")
8214 (match_operand:ALLI_TI 2 "aarch64_reg_or_zero")
8215 (use (reg:DI SPECULATION_TRACKER_REGNUM))
8216 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
8217 ""
8218 "
8219 {
8220 if (operands[2] == const0_rtx)
8221 {
8222 rtx tracker;
8223 if (<MODE>mode == TImode)
8224 tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
8225 else
8226 tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
8228 emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
8229 tracker));
8230 DONE;
8231 }
8232 }
8233 "
8234 )
8236 ;; Patterns to match despeculate_copy<mode>. Note that "hint 0x14" is the
8237 ;; encoding for CSDB, but will work in older versions of the assembler.
8238 (define_insn "*despeculate_copy<ALLI:mode>_insn"
8239 [(set (match_operand:ALLI 0 "register_operand" "=r")
8240 (unspec_volatile:ALLI
8241 [(match_operand:ALLI 1 "register_operand" "r")
8242 (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
8243 (use (reg:DI SPECULATION_TRACKER_REGNUM))
8244 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
8245 ""
8246 {
8247 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
8248 output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
8249 operands);
8250 return "";
8251 }
8252 [(set_attr "length" "12")
8253 (set_attr "type" "block")
8254 (set_attr "speculation_barrier" "true")]
8255 )
8257 ;; Pattern to match despeculate_copyti
8258 (define_insn "*despeculate_copyti_insn"
8259 [(set (match_operand:TI 0 "register_operand" "=r")
8260 (unspec_volatile:TI
8261 [(match_operand:TI 1 "register_operand" "r")
8262 (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
8263 (use (reg:DI SPECULATION_TRACKER_REGNUM))
8264 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
8265 ""
8266 {
8267 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
8268 output_asm_insn
8269 ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
8270 operands);
8271 return "";
8272 }
8273 [(set_attr "length" "16")
8274 (set_attr "type" "block")
8275 (set_attr "speculation_barrier" "true")]
8276 )
8278 (define_insn "despeculate_simple<ALLI:mode>"
8279 [(set (match_operand:ALLI 0 "register_operand" "=r")
8280 (unspec_volatile:ALLI
8281 [(match_operand:ALLI 1 "register_operand" "r")
8282 (use (match_operand:ALLI 2 "register_operand" ""))]
8283 UNSPECV_SPECULATION_BARRIER))]
8284 ""
8285 "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
8286 [(set_attr "type" "block")
8287 (set_attr "length" "8")
8288 (set_attr "speculation_barrier" "true")]
8289 )
8291 (define_insn "despeculate_simpleti"
8292 [(set (match_operand:TI 0 "register_operand" "=r")
8293 (unspec_volatile:TI
8294 [(match_operand:TI 1 "register_operand" "r")
8295 (use (match_operand:DI 2 "register_operand" ""))]
8296 UNSPECV_SPECULATION_BARRIER))]
8297 ""
8298 "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
8299 [(set_attr "type" "block")
8300 (set_attr "length" "12")
8301 (set_attr "speculation_barrier" "true")]
8302 )
8304 (define_insn "aarch64_<frintnzs_op><mode>"
8305 [(set (match_operand:VSFDF 0 "register_operand" "=w")
8306 (unspec:VSFDF [(match_operand:VSFDF 1 "register_operand" "w")]
8307 FRINTNZX))]
8308 "TARGET_FRINT && TARGET_FLOAT
8309 && !(VECTOR_MODE_P (<MODE>mode) && !TARGET_SIMD)"
8310 "<frintnzs_op>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
8311 [(set_attr "type" "f_rint<stype>")]
8312 )
8314 ;; Transactional Memory Extension (TME) instructions.
8316 (define_insn "tstart"
8317 [(set (match_operand:DI 0 "register_operand" "=r")
8318 (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART))
8319 (clobber (mem:BLK (scratch)))]
8320 "TARGET_TME"
8321 "tstart\\t%0"
8322 [(set_attr "type" "tme")]
8323 )
8325 (define_insn "ttest"
8326 [(set (match_operand:DI 0 "register_operand" "=r")
8327 (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST))
8328 (clobber (mem:BLK (scratch)))]
8329 "TARGET_TME"
8330 "ttest\\t%0"
8331 [(set_attr "type" "tme")]
8332 )
8334 (define_insn "tcommit"
8335 [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT)
8336 (clobber (mem:BLK (scratch)))]
8337 "TARGET_TME"
8338 "tcommit"
8339 [(set_attr "type" "tme")]
8340 )
8342 (define_insn "tcancel"
8343 [(unspec_volatile:BLK
8344 [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL)
8345 (clobber (mem:BLK (scratch)))]
8346 "TARGET_TME && (UINTVAL (operands[0]) <= 65535)"
8347 "tcancel\\t#%0"
8348 [(set_attr "type" "tme")]
8349 )
8351 (define_insn "aarch64_rndr"
8352 [(set (match_operand:DI 0 "register_operand" "=r")
8353 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDR))
8354 (set (reg:CC_Z CC_REGNUM)
8355 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDR))]
8356 "TARGET_RNG"
8357 "mrs\t%0, RNDR"
8358 [(set_attr "type" "mrs")]
8359 )
8361 (define_insn "aarch64_rndrrs"
8362 [(set (match_operand:DI 0 "register_operand" "=r")
8363 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDRRS))
8364 (set (reg:CC_Z CC_REGNUM)
8365 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDRRS))]
8366 "TARGET_RNG"
8367 "mrs\t%0, RNDRRS"
8368 [(set_attr "type" "mrs")]
8369 )
8371 ;; Memory Tagging Extension (MTE) instructions.
8373 (define_insn "irg"
8374 [(set (match_operand:DI 0 "register_operand" "=rk")
8375 (ior:DI
8376 (and:DI (match_operand:DI 1 "register_operand" "rk")
8377 (const_int -1080863910568919041)) ;; 0xf0ff...
8378 (ashift:DI (unspec:QI [(match_operand:DI 2 "register_operand" "r")]
8379 UNSPEC_GEN_TAG_RND)
8380 (const_int 56))))]
8381 "TARGET_MEMTAG"
8382 "irg\\t%0, %1, %2"
8383 [(set_attr "type" "memtag")]
8384 )
8386 (define_insn "gmi"
8387 [(set (match_operand:DI 0 "register_operand" "=r")
8388 (ior:DI (ashift:DI
8389 (const_int 1)
8390 (and:QI (lshiftrt:DI
8391 (match_operand:DI 1 "register_operand" "rk")
8392 (const_int 56)) (const_int 15)))
8393 (match_operand:DI 2 "register_operand" "r")))]
8394 "TARGET_MEMTAG"
8395 "gmi\\t%0, %1, %2"
8396 [(set_attr "type" "memtag")]
8397 )
8399 (define_insn "addg"
8400 [(set (match_operand:DI 0 "register_operand" "=rk")
8401 (ior:DI
8402 (and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk")
8403 (match_operand:DI 2 "aarch64_granule16_uimm6" "i"))
8404 (const_int -1080863910568919041)) ;; 0xf0ff...
8405 (ashift:DI
8406 (unspec:QI
8407 [(and:QI (lshiftrt:DI (match_dup 1) (const_int 56)) (const_int 15))
8408 (match_operand:QI 3 "aarch64_memtag_tag_offset" "i")]
8409 UNSPEC_GEN_TAG)
8410 (const_int 56))))]
8411 "TARGET_MEMTAG"
8412 "addg\\t%0, %1, #%2, #%3"
8413 [(set_attr "type" "memtag")]
8414 )
8416 (define_insn "subp"
8417 [(set (match_operand:DI 0 "register_operand" "=r")
8418 (minus:DI
8419 (and:DI (match_operand:DI 1 "register_operand" "rk")
8420 (const_int 72057594037927935)) ;; 0x00ff...
8421 (and:DI (match_operand:DI 2 "register_operand" "rk")
8422 (const_int 72057594037927935))))] ;; 0x00ff...
8423 "TARGET_MEMTAG"
8424 "subp\\t%0, %1, %2"
8425 [(set_attr "type" "memtag")]
8426 )
8428 ;; LDG will use the 16-byte aligned value of the address.
8429 (define_insn "ldg"
8430 [(set (match_operand:DI 0 "register_operand" "+r")
8431 (ior:DI
8432 (and:DI (match_dup 0) (const_int -1080863910568919041)) ;; 0xf0ff...
8433 (ashift:DI
8434 (mem:QI (unspec:DI
8435 [(and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk")
8436 (match_operand:DI 2 "aarch64_granule16_simm9" "i"))
8437 (const_int -16))] UNSPEC_TAG_SPACE))
8438 (const_int 56))))]
8439 "TARGET_MEMTAG"
8440 "ldg\\t%0, [%1, #%2]"
8441 [(set_attr "type" "memtag")]
8442 )
8444 ;; STG doesn't align the address but aborts with alignment fault
8445 ;; when the address is not 16-byte aligned.
8446 (define_insn "stg"
8447 [(set (mem:QI (unspec:DI
8448 [(plus:DI (match_operand:DI 1 "register_operand" "rk")
8449 (match_operand:DI 2 "aarch64_granule16_simm9" "i"))]
8450 UNSPEC_TAG_SPACE))
8451 (and:QI (lshiftrt:DI (match_operand:DI 0 "register_operand" "rk")
8452 (const_int 56)) (const_int 15)))]
8453 "TARGET_MEMTAG"
8454 "stg\\t%0, [%1, #%2]"
8455 [(set_attr "type" "memtag")]
8456 )
8458 ;; Load/Store 64-bit (LS64) instructions.
8459 (define_insn "ld64b"
8460 [(set (match_operand:V8DI 0 "register_operand" "=r")
8461 (unspec_volatile:V8DI
8462 [(mem:V8DI (match_operand:DI 1 "register_operand" "r"))]
8463 UNSPEC_LD64B)
8464 )]
8465 "TARGET_LS64"
8466 "ld64b\\t%0, [%1]"
8467 [(set_attr "type" "ls64")]
8468 )
8470 (define_insn "st64b"
8471 [(set (mem:V8DI (match_operand:DI 0 "register_operand" "r"))
8472 (unspec_volatile:V8DI [(match_operand:V8DI 1 "register_operand" "r")]
8473 UNSPEC_ST64B)
8474 )]
8475 "TARGET_LS64"
8476 "st64b\\t%1, [%0]"
8477 [(set_attr "type" "ls64")]
8478 )
8480 (define_insn "st64bv"
8481 [(set (match_operand:DI 0 "register_operand" "=r")
8482 (unspec_volatile:DI [(const_int 0)] UNSPEC_ST64BV_RET))
8483 (set (mem:V8DI (match_operand:DI 1 "register_operand" "r"))
8484 (unspec_volatile:V8DI [(match_operand:V8DI 2 "register_operand" "r")]
8485 UNSPEC_ST64BV)
8486 )]
8487 "TARGET_LS64"
8488 "st64bv\\t%0, %2, [%1]"
8489 [(set_attr "type" "ls64")]
8490 )
8492 (define_insn "st64bv0"
8493 [(set (match_operand:DI 0 "register_operand" "=r")
8494 (unspec_volatile:DI [(const_int 0)] UNSPEC_ST64BV0_RET))
8495 (set (mem:V8DI (match_operand:DI 1 "register_operand" "r"))
8496 (unspec_volatile:V8DI [(match_operand:V8DI 2 "register_operand" "r")]
8497 UNSPEC_ST64BV0)
8498 )]
8499 "TARGET_LS64"
8500 "st64bv0\\t%0, %2, [%1]"
8501 [(set_attr "type" "ls64")]
8502 )
8504 (define_insn "patchable_area"
8505 [(unspec_volatile [(match_operand 0 "const_int_operand")
8506 (match_operand 1 "const_int_operand")]
8507 UNSPECV_PATCHABLE_AREA)]
8508 ""
8509 {
8510 aarch64_output_patchable_area (INTVAL (operands[0]),
8511 INTVAL (operands[1]) != 0);
8512 return "";
8513 }
8514 [(set (attr "length") (symbol_ref "INTVAL (operands[0])"))]
8515 )
8517 (define_insn "aarch64_save_nzcv"
8518 [(set (match_operand:DI 0 "register_operand" "=r")
8519 (unspec:DI [(reg:CC CC_REGNUM)] UNSPEC_SAVE_NZCV))]
8520 ""
8521 "mrs\t%0, nzcv"
8522 )
8524 (define_insn "aarch64_restore_nzcv"
8525 [(set (reg:CC CC_REGNUM)
8526 (unspec:CC [(match_operand:DI 0 "register_operand" "r")]
8527 UNSPEC_RESTORE_NZCV))]
8528 ""
8529 "msr\tnzcv, %0"
8530 )
8532 ;; CHKFEAT instruction
8533 (define_insn "aarch64_chkfeat"
8534 [(set (reg:DI R16_REGNUM)
8535 (unspec_volatile:DI [(reg:DI R16_REGNUM)] UNSPECV_CHKFEAT))]
8536 ""
8537 "hint\\t40 // chkfeat x16"
8538 )
8540 ;; Guarded Control Stack (GCS) instructions
8541 (define_insn "aarch64_load_gcspr"
8542 [(set (match_operand:DI 0 "register_operand" "=r")
8543 (unspec_volatile:DI [(const_int 0)] UNSPECV_GCSPR))]
8544 ""
8545 "mrs\\t%0, s3_3_c2_c5_1 // gcspr_el0"
8546 [(set_attr "type" "mrs")]
8547 )
8549 (define_insn "aarch64_gcspopm"
8550 [(set (match_operand:DI 0 "register_operand" "=r")
8551 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")] UNSPECV_GCSPOPM))]
8552 ""
8553 "sysl\\t%0, #3, c7, c7, #1 // gcspopm"
8554 )
8556 (define_insn "aarch64_gcspopm_xzr"
8557 [(unspec_volatile [(const_int 0)] UNSPECV_GCSPOPM)]
8558 ""
8559 "sysl\\txzr, #3, c7, c7, #1 // gcspopm"
8560 )
8562 (define_insn "aarch64_gcsss1"
8563 [(unspec_volatile [(match_operand:DI 0 "register_operand" "r")] UNSPECV_GCSSS1)]
8564 ""
8565 "sys\\t#3, c7, c7, #2, %0 // gcsss1"
8566 )
8568 (define_insn "aarch64_gcsss2"
8569 [(set (match_operand:DI 0 "register_operand" "=r")
8570 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")] UNSPECV_GCSSS2))]
8571 ""
8572 "sysl\\t%0, #3, c7, c7, #3 // gcsss2"
8573 )
8575 ;; AdvSIMD Stuff
8576 (include "aarch64-simd.md")
8578 ;; Atomic Operations
8579 (include "atomics.md")
8581 ;; ldp/stp peephole patterns
8582 (include "aarch64-ldpstp.md")
8584 ;; SVE.
8585 (include "aarch64-sve.md")
8587 ;; SVE2.
8588 (include "aarch64-sve2.md")
8590 ;; SME and extensions
8591 (include "aarch64-sme.md")