| blob | c49affd0dd39709428127f8a6fdf0424f31b5072 |
1 ;; Machine description for AArch64 SME.
2 ;; Copyright (C) 2023-2025 Free Software Foundation, Inc.
3 ;;
4 ;; This file is part of GCC.
5 ;;
6 ;; GCC is free software; you can redistribute it and/or modify it
7 ;; under the terms of the GNU General Public License as published by
8 ;; the Free Software Foundation; either version 3, or (at your option)
9 ;; any later version.
10 ;;
11 ;; GCC is distributed in the hope that it will be useful, but
12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 ;; General Public License for more details.
15 ;;
16 ;; You should have received a copy of the GNU General Public License
17 ;; along with GCC; see the file COPYING3. If not see
18 ;; <http://www.gnu.org/licenses/>.
20 ;; The file is organised into the following sections (search for the full
21 ;; line):
22 ;;
23 ;; == State management
24 ;; ---- Test current state
25 ;; ---- PSTATE.SM management
26 ;; ---- PSTATE.ZA management
27 ;;
28 ;; == Loads, stores and moves
29 ;; ---- Single-vector loads
30 ;; ---- Table loads
31 ;; ---- Single-vector stores
32 ;; ---- Table stores
33 ;; ---- Single-vector moves
34 ;; ---- Multi-vector moves
35 ;; ---- Zeroing
36 ;;
37 ;; == Binary arithmetic
38 ;; ---- Binary arithmetic on ZA tile
39 ;; ---- Binary arithmetic on ZA slice
40 ;; ---- Binary arithmetic, writing to ZA slice
41 ;;
42 ;; == Ternary arithmetic
43 ;; ---- [INT] Dot product
44 ;; ---- [INT] Ternary widening arithmetic on ZA slice
45 ;; ---- [INT] Sum of outer products
46 ;; ---- [FP] Dot product
47 ;; ---- [FP] Ternary arithmetic on ZA slice
48 ;; ---- [FP] Ternary widening arithmetic on ZA slice
49 ;; ---- [FP] Sum of outer products
50 ;;
51 ;; == Table lookup
52 ;; ---- Table lookup
54 ;; =========================================================================
55 ;; == State management
56 ;; =========================================================================
57 ;;
58 ;; Many of the instructions in this section are only valid when SME is
59 ;; present. However, they don't have a TARGET_SME condition since
60 ;; (a) they are only emitted under direct control of aarch64 code and
61 ;; (b) they are sometimes used conditionally, particularly in streaming-
62 ;; compatible code.
63 ;;
64 ;; =========================================================================
66 ;; -------------------------------------------------------------------------
67 ;; ---- Test current state
68 ;; -------------------------------------------------------------------------
70 (define_c_enum "unspec" [
71 UNSPEC_OLD_VG_SAVED
72 UNSPEC_UPDATE_VG
73 UNSPEC_GET_SME_STATE
74 UNSPEC_READ_SVCR
75 ])
77 ;; A marker instruction to say that the old value of the DWARF VG register
78 ;; has been saved to the stack, for CFI purposes. Operand 0 is the old
79 ;; value of the register and operand 1 is the save slot.
80 (define_insn "aarch64_old_vg_saved"
81 [(set (reg:DI VG_REGNUM)
82 (unspec:DI [(match_operand 0)
83 (match_operand 1)] UNSPEC_OLD_VG_SAVED))]
84 ""
85 ""
86 [(set_attr "type" "no_insn")]
87 )
89 ;; A marker to indicate places where a call temporarily changes VG.
90 (define_insn "aarch64_update_vg"
91 [(set (reg:DI VG_REGNUM)
92 (unspec:DI [(reg:DI VG_REGNUM)] UNSPEC_UPDATE_VG))]
93 ""
94 ""
95 [(set_attr "type" "no_insn")]
96 )
98 (define_insn "aarch64_get_sme_state"
99 [(set (reg:TI R0_REGNUM)
100 (unspec_volatile:TI [(const_int 0)] UNSPEC_GET_SME_STATE))
101 (clobber (reg:DI R16_REGNUM))
102 (clobber (reg:DI R17_REGNUM))
103 (clobber (reg:DI R18_REGNUM))
104 (clobber (reg:DI R30_REGNUM))
105 (clobber (reg:CC CC_REGNUM))]
106 ""
107 "bl\t__arm_sme_state"
108 [(set_attr "is_call" "yes")]
109 )
111 (define_insn "aarch64_read_svcr"
112 [(set (match_operand:DI 0 "register_operand" "=r")
113 (unspec_volatile:DI [(const_int 0)] UNSPEC_READ_SVCR))]
114 ""
115 "mrs\t%0, svcr"
116 )
118 ;; -------------------------------------------------------------------------
119 ;; ---- PSTATE.SM management
120 ;; -------------------------------------------------------------------------
121 ;; Includes:
122 ;; - SMSTART SM
123 ;; - SMSTOP SM
124 ;; -------------------------------------------------------------------------
126 (define_c_enum "unspec" [
127 UNSPEC_SMSTART_SM
128 UNSPEC_SMSTOP_SM
129 ])
131 ;; Turn on streaming mode. This clobbers all SVE state.
132 ;;
133 ;; Depend on VG_REGNUM to ensure that the VG save slot has already been
134 ;; initialized.
135 (define_insn "aarch64_smstart_sm"
136 [(unspec_volatile [(const_int 0)] UNSPEC_SMSTART_SM)
137 (use (reg:DI VG_REGNUM))
138 (clobber (reg:V4x16QI V0_REGNUM))
139 (clobber (reg:V4x16QI V4_REGNUM))
140 (clobber (reg:V4x16QI V8_REGNUM))
141 (clobber (reg:V4x16QI V12_REGNUM))
142 (clobber (reg:V4x16QI V16_REGNUM))
143 (clobber (reg:V4x16QI V20_REGNUM))
144 (clobber (reg:V4x16QI V24_REGNUM))
145 (clobber (reg:V4x16QI V28_REGNUM))
146 (clobber (reg:VNx16BI P0_REGNUM))
147 (clobber (reg:VNx16BI P1_REGNUM))
148 (clobber (reg:VNx16BI P2_REGNUM))
149 (clobber (reg:VNx16BI P3_REGNUM))
150 (clobber (reg:VNx16BI P4_REGNUM))
151 (clobber (reg:VNx16BI P5_REGNUM))
152 (clobber (reg:VNx16BI P6_REGNUM))
153 (clobber (reg:VNx16BI P7_REGNUM))
154 (clobber (reg:VNx16BI P8_REGNUM))
155 (clobber (reg:VNx16BI P9_REGNUM))
156 (clobber (reg:VNx16BI P10_REGNUM))
157 (clobber (reg:VNx16BI P11_REGNUM))
158 (clobber (reg:VNx16BI P12_REGNUM))
159 (clobber (reg:VNx16BI P13_REGNUM))
160 (clobber (reg:VNx16BI P14_REGNUM))
161 (clobber (reg:VNx16BI P15_REGNUM))]
162 ""
163 "smstart\tsm"
164 )
166 ;; Turn off streaming mode. This clobbers all SVE state.
167 ;;
168 ;; Depend on VG_REGNUM to ensure that the VG save slot has already been
169 ;; initialized.
170 (define_insn "aarch64_smstop_sm"
171 [(unspec_volatile [(const_int 0)] UNSPEC_SMSTOP_SM)
172 (use (reg:DI VG_REGNUM))
173 (clobber (reg:V4x16QI V0_REGNUM))
174 (clobber (reg:V4x16QI V4_REGNUM))
175 (clobber (reg:V4x16QI V8_REGNUM))
176 (clobber (reg:V4x16QI V12_REGNUM))
177 (clobber (reg:V4x16QI V16_REGNUM))
178 (clobber (reg:V4x16QI V20_REGNUM))
179 (clobber (reg:V4x16QI V24_REGNUM))
180 (clobber (reg:V4x16QI V28_REGNUM))
181 (clobber (reg:VNx16BI P0_REGNUM))
182 (clobber (reg:VNx16BI P1_REGNUM))
183 (clobber (reg:VNx16BI P2_REGNUM))
184 (clobber (reg:VNx16BI P3_REGNUM))
185 (clobber (reg:VNx16BI P4_REGNUM))
186 (clobber (reg:VNx16BI P5_REGNUM))
187 (clobber (reg:VNx16BI P6_REGNUM))
188 (clobber (reg:VNx16BI P7_REGNUM))
189 (clobber (reg:VNx16BI P8_REGNUM))
190 (clobber (reg:VNx16BI P9_REGNUM))
191 (clobber (reg:VNx16BI P10_REGNUM))
192 (clobber (reg:VNx16BI P11_REGNUM))
193 (clobber (reg:VNx16BI P12_REGNUM))
194 (clobber (reg:VNx16BI P13_REGNUM))
195 (clobber (reg:VNx16BI P14_REGNUM))
196 (clobber (reg:VNx16BI P15_REGNUM))]
197 ""
198 "smstop\tsm"
199 )
201 ;; -------------------------------------------------------------------------
202 ;; ---- PSTATE.ZA management
203 ;; -------------------------------------------------------------------------
204 ;; Includes:
205 ;; - SMSTART ZA
206 ;; - SMSTOP ZA
207 ;; plus calls to support routines.
208 ;; -------------------------------------------------------------------------
210 (define_c_enum "unspec" [
211 UNSPEC_SMSTOP_ZA
212 UNSPEC_INITIAL_ZERO_ZA
213 UNSPEC_TPIDR2_SAVE
214 UNSPEC_TPIDR2_RESTORE
215 UNSPEC_READ_TPIDR2
216 UNSPEC_WRITE_TPIDR2
217 UNSPEC_SETUP_LOCAL_TPIDR2
218 UNSPEC_RESTORE_ZA
219 UNSPEC_START_PRIVATE_ZA_CALL
220 UNSPEC_END_PRIVATE_ZA_CALL
221 UNSPEC_COMMIT_LAZY_SAVE
222 ])
224 (define_c_enum "unspecv" [
225 UNSPECV_ASM_UPDATE_ZA
226 UNSPECV_ASM_UPDATE_ZT0
227 ])
229 ;; Use the ABI-defined routine to commit an uncommitted lazy save.
230 ;; This relies on the current PSTATE.ZA, so depends on SME_STATE_REGNUM.
231 ;; The fake TPIDR2_SETUP_REGNUM register initially holds the incoming
232 ;; value of the architected TPIDR2_EL0.
233 (define_insn "aarch64_tpidr2_save"
234 [(set (reg:DI ZA_FREE_REGNUM)
235 (unspec:DI [(reg:DI SME_STATE_REGNUM)
236 (reg:DI TPIDR2_SETUP_REGNUM)] UNSPEC_TPIDR2_SAVE))
237 (clobber (reg:DI R14_REGNUM))
238 (clobber (reg:DI R15_REGNUM))
239 (clobber (reg:DI R16_REGNUM))
240 (clobber (reg:DI R17_REGNUM))
241 (clobber (reg:DI R18_REGNUM))
242 (clobber (reg:DI R30_REGNUM))
243 (clobber (reg:CC CC_REGNUM))]
244 ""
245 "bl\t__arm_tpidr2_save"
246 [(set_attr "is_call" "yes")]
247 )
249 ;; Set PSTATE.ZA to 1. If ZA was previously dormant or active,
250 ;; it remains in the same state afterwards, with the same contents.
251 ;; Otherwise, it goes from off to on with zeroed contents.
252 ;;
253 ;; Later writes of TPIDR2_EL0 to a nonzero value must not be moved
254 ;; up past this instruction, since that could create an invalid
255 ;; combination of having an active lazy save while ZA is off.
256 ;; Create an anti-dependence by reading the current contents
257 ;; of TPIDR2_SETUP_REGNUM.
258 ;;
259 ;; Making this depend on ZA_FREE_REGNUM ensures that contents belonging
260 ;; to the caller have already been saved. That isn't necessary for this
261 ;; instruction itself, since PSTATE.ZA is already 1 if it contains data.
262 ;; But doing this here means that other uses of ZA can just depend on
263 ;; SME_STATE_REGNUM, rather than both SME_STATE_REGNUM and ZA_FREE_REGNUM.
264 (define_insn "aarch64_smstart_za"
265 [(set (reg:DI SME_STATE_REGNUM)
266 (const_int 1))
267 (use (reg:DI TPIDR2_SETUP_REGNUM))
268 (use (reg:DI ZA_FREE_REGNUM))]
269 ""
270 "smstart\tza"
271 )
273 ;; Disable ZA and discard its current contents.
274 ;;
275 ;; The ABI says that the ZA save buffer must be null whenever PSTATE.ZA
276 ;; is zero, so earlier writes to TPIDR2_EL0 must not be moved down past
277 ;; this instruction. Depend on TPIDR2_SETUP_REGNUM to ensure this.
278 ;;
279 ;; We can only turn ZA off once we know that it is free (i.e. doesn't
280 ;; contain data belonging to the caller). Depend on ZA_FREE_REGNUM
281 ;; to ensure this.
282 ;;
283 ;; We only turn ZA off when the current function's ZA state is dead,
284 ;; or perhaps if we're sure that the contents are saved. Either way,
285 ;; we know whether ZA is saved or not.
286 (define_insn "aarch64_smstop_za"
287 [(set (reg:DI SME_STATE_REGNUM)
288 (const_int 0))
289 (set (reg:DI ZA_SAVED_REGNUM)
290 (unspec:DI [(reg:DI TPIDR2_SETUP_REGNUM)
291 (reg:DI ZA_FREE_REGNUM)] UNSPEC_SMSTOP_ZA))]
292 ""
293 "smstop\tza"
294 )
296 ;; Zero ZA after committing a lazy save. The sequencing is enforced
297 ;; by reading ZA_FREE_REGNUM.
298 (define_insn "aarch64_initial_zero_za"
299 [(set (reg:DI ZA_REGNUM)
300 (unspec:DI [(reg:DI SME_STATE_REGNUM)
301 (reg:DI ZA_FREE_REGNUM)] UNSPEC_INITIAL_ZERO_ZA))]
302 ""
303 "zero\t{ za }"
304 )
306 ;; Initialize the abstract TPIDR2_BLOCK_REGNUM from the contents of
307 ;; the current function's TPIDR2 block. Other instructions can then
308 ;; depend on TPIDR2_BLOCK_REGNUM rather than on the memory block.
309 (define_insn "aarch64_setup_local_tpidr2"
310 [(set (reg:DI TPIDR2_BLOCK_REGNUM)
311 (unspec:DI [(match_operand:V16QI 0 "memory_operand" "m")]
312 UNSPEC_SETUP_LOCAL_TPIDR2))]
313 ""
314 ""
315 [(set_attr "type" "no_insn")]
316 )
318 ;; Clear TPIDR2_EL0, cancelling any uncommitted lazy save.
319 (define_insn "aarch64_clear_tpidr2"
320 [(set (reg:DI TPIDR2_SETUP_REGNUM)
321 (const_int 0))]
322 ""
323 "msr\ttpidr2_el0, xzr"
324 )
326 ;; Point TPIDR2_EL0 to the current function's TPIDR2 block, whose address
327 ;; is given by operand 0. TPIDR2_BLOCK_REGNUM represents the contents of the
328 ;; pointed-to block.
329 (define_insn "aarch64_write_tpidr2"
330 [(set (reg:DI TPIDR2_SETUP_REGNUM)
331 (unspec:DI [(match_operand 0 "pmode_register_operand" "r")
332 (reg:DI TPIDR2_BLOCK_REGNUM)] UNSPEC_WRITE_TPIDR2))]
333 ""
334 "msr\ttpidr2_el0, %0"
335 )
337 ;; Check whether ZA has been saved. The system depends on the value that
338 ;; we wrote to TPIDR2_EL0 previously, so it depends on TPDIR2_SETUP_REGNUM.
339 (define_insn "aarch64_read_tpidr2"
340 [(set (match_operand:DI 0 "register_operand" "=r")
341 (unspec:DI [(reg:DI TPIDR2_SETUP_REGNUM)
342 (reg:DI ZA_SAVED_REGNUM)] UNSPEC_READ_TPIDR2))]
343 ""
344 "mrs\t%0, tpidr2_el0"
345 )
347 ;; Use the ABI-defined routine to restore lazy-saved ZA contents
348 ;; from the TPIDR2 block pointed to by X0. ZA must already be active.
349 (define_insn "aarch64_tpidr2_restore"
350 [(set (reg:DI ZA_SAVED_REGNUM)
351 (unspec:DI [(reg:DI R0_REGNUM)] UNSPEC_TPIDR2_RESTORE))
352 (set (reg:DI SME_STATE_REGNUM)
353 (unspec:DI [(reg:DI SME_STATE_REGNUM)] UNSPEC_TPIDR2_RESTORE))
354 (clobber (reg:DI R14_REGNUM))
355 (clobber (reg:DI R15_REGNUM))
356 (clobber (reg:DI R16_REGNUM))
357 (clobber (reg:DI R17_REGNUM))
358 (clobber (reg:DI R18_REGNUM))
359 (clobber (reg:DI R30_REGNUM))
360 (clobber (reg:CC CC_REGNUM))]
361 ""
362 "bl\t__arm_tpidr2_restore"
363 [(set_attr "is_call" "yes")]
364 )
366 ;; Check whether a lazy save set up by aarch64_save_za was committed
367 ;; and restore the saved contents if so.
368 ;;
369 ;; Operand 0 is the address of the current function's TPIDR2 block.
370 (define_insn_and_split "aarch64_restore_za"
371 [(set (reg:DI ZA_SAVED_REGNUM)
372 (unspec:DI [(match_operand 0 "pmode_register_operand" "r")
373 (reg:DI SME_STATE_REGNUM)
374 (reg:DI TPIDR2_SETUP_REGNUM)
375 (reg:DI ZA_SAVED_REGNUM)] UNSPEC_RESTORE_ZA))
376 (clobber (reg:DI R0_REGNUM))
377 (clobber (reg:DI R14_REGNUM))
378 (clobber (reg:DI R15_REGNUM))
379 (clobber (reg:DI R16_REGNUM))
380 (clobber (reg:DI R17_REGNUM))
381 (clobber (reg:DI R18_REGNUM))
382 (clobber (reg:DI R30_REGNUM))
383 (clobber (reg:CC CC_REGNUM))]
384 ""
385 "#"
386 "&& epilogue_completed"
387 [(const_int 0)]
388 {
389 auto label = gen_label_rtx ();
390 auto tpidr2 = gen_rtx_REG (DImode, R16_REGNUM);
391 emit_insn (gen_aarch64_read_tpidr2 (tpidr2));
392 auto jump = emit_likely_jump_insn (gen_aarch64_cbnedi1 (tpidr2, label));
393 JUMP_LABEL (jump) = label;
395 aarch64_restore_za (operands[0]);
396 emit_label (label);
397 DONE;
398 }
399 )
401 ;; This instruction is emitted after asms that alter ZA, in order to model
402 ;; the effect on dataflow. The asm itself can't have ZA as an input or
403 ;; an output, since there is no associated data type. Instead it retains
404 ;; the original "za" clobber, which on its own would indicate that ZA
405 ;; is dead.
406 ;;
407 ;; The operand is a unique identifier.
408 (define_insn "aarch64_asm_update_za"
409 [(set (reg:VNx16QI ZA_REGNUM)
410 (unspec_volatile:VNx16QI
411 [(reg:VNx16QI ZA_REGNUM)
412 (reg:DI SME_STATE_REGNUM)
413 (match_operand 0 "const_int_operand")]
414 UNSPECV_ASM_UPDATE_ZA))]
415 ""
416 ""
417 [(set_attr "type" "no_insn")]
418 )
420 ;; A similar pattern for ZT0.
421 (define_insn "aarch64_asm_update_zt0"
422 [(set (reg:V8DI ZT0_REGNUM)
423 (unspec_volatile:V8DI
424 [(reg:V8DI ZT0_REGNUM)
425 (reg:DI SME_STATE_REGNUM)
426 (match_operand 0 "const_int_operand")]
427 UNSPECV_ASM_UPDATE_ZT0))]
428 ""
429 ""
430 [(set_attr "type" "no_insn")]
431 )
433 ;; This pseudo-instruction is emitted as part of a call to a private-ZA
434 ;; function from a function with ZA state. It marks a natural place to set
435 ;; up a lazy save, if that turns out to be necessary. The save itself
436 ;; is managed by the mode-switching pass.
437 (define_insn "aarch64_start_private_za_call"
438 [(set (reg:DI LOWERING_REGNUM)
439 (unspec:DI [(reg:DI LOWERING_REGNUM)] UNSPEC_START_PRIVATE_ZA_CALL))]
440 ""
441 ""
442 [(set_attr "type" "no_insn")]
443 )
445 ;; This pseudo-instruction is emitted as part of a call to a private-ZA
446 ;; function from a function with ZA state. It marks a natural place to restore
447 ;; the current function's ZA contents from the lazy save buffer, if that
448 ;; turns out to be necessary. The save itself is managed by the
449 ;; mode-switching pass.
450 (define_insn "aarch64_end_private_za_call"
451 [(set (reg:DI LOWERING_REGNUM)
452 (unspec:DI [(reg:DI LOWERING_REGNUM)] UNSPEC_END_PRIVATE_ZA_CALL))]
453 ""
454 ""
455 [(set_attr "type" "no_insn")]
456 )
458 ;; =========================================================================
459 ;; == Loads, stores and moves
460 ;; =========================================================================
462 ;; -------------------------------------------------------------------------
463 ;; ---- Single-vector loads
464 ;; -------------------------------------------------------------------------
465 ;; Includes:
466 ;; - LD1
467 ;; - LDR
468 ;; -------------------------------------------------------------------------
470 (define_c_enum "unspec" [
471 UNSPEC_SME_LDR
472 ])
474 (define_insn "@aarch64_sme_<optab><mode>"
475 [(set (reg:SME_ZA_I ZA_REGNUM)
476 (unspec:SME_ZA_I
477 [(reg:SME_ZA_I ZA_REGNUM)
478 (reg:DI SME_STATE_REGNUM)
479 (match_operand:DI 0 "const_int_operand")
480 (match_operand:SI 1 "register_operand" "Ucj")
481 (match_operand:<VPRED> 2 "register_operand" "Upl")
482 (match_operand:SME_ZA_I 3 "aarch64_sve_ldff1_operand" "Utf")]
483 SME_LD1))]
484 "TARGET_STREAMING"
485 "ld1<Vesize>\t{ za%0<hv>.<Vetype>[%w1, 0] }, %2/z, %3"
486 )
488 (define_insn "@aarch64_sme_<optab><mode>_plus"
489 [(set (reg:SME_ZA_I ZA_REGNUM)
490 (unspec:SME_ZA_I
491 [(reg:SME_ZA_I ZA_REGNUM)
492 (reg:DI SME_STATE_REGNUM)
493 (match_operand:DI 0 "const_int_operand")
494 (plus:SI (match_operand:SI 1 "register_operand" "Ucj")
495 (match_operand:SI 2 "const_int_operand"))
496 (match_operand:<VPRED> 3 "register_operand" "Upl")
497 (match_operand:SME_ZA_I 4 "aarch64_sve_ldff1_operand" "Utf")]
498 SME_LD1))]
499 "TARGET_STREAMING
500 && UINTVAL (operands[2]) < 128 / <elem_bits>"
501 "ld1<Vesize>\t{ za%0<hv>.<Vetype>[%w1, %2] }, %3/z, %4"
502 )
504 (define_insn "aarch64_sme_ldr0"
505 [(set (reg:VNx16QI ZA_REGNUM)
506 (unspec:VNx16QI
507 [(reg:VNx16QI ZA_REGNUM)
508 (reg:DI SME_STATE_REGNUM)
509 (match_operand:SI 0 "register_operand" "Ucj")
510 (mem:VNx16QI (match_operand 1 "pmode_register_operand" "rk"))]
511 UNSPEC_SME_LDR))]
512 "TARGET_SME"
513 "ldr\tza[%w0, 0], [%1, #0, mul vl]"
514 )
516 (define_insn "@aarch64_sme_ldrn<mode>"
517 [(set (reg:VNx16QI ZA_REGNUM)
518 (unspec:VNx16QI
519 [(reg:VNx16QI ZA_REGNUM)
520 (reg:DI SME_STATE_REGNUM)
521 (plus:SI (match_operand:SI 0 "register_operand" "Ucj")
522 (match_operand:SI 1 "const_int_operand"))
523 (mem:VNx16QI
524 (plus:P (match_operand:P 2 "register_operand" "rk")
525 (match_operand:P 3 "aarch64_mov_operand")))]
526 UNSPEC_SME_LDR))]
527 "TARGET_SME
528 && aarch64_sme_ldr_vnum_offset_p (operands[1], operands[3])"
529 "ldr\tza[%w0, %1], [%2, #%1, mul vl]"
530 )
532 ;; -------------------------------------------------------------------------
533 ;; ---- Table loads
534 ;; -------------------------------------------------------------------------
535 ;; Includes:
536 ;; - LDR (SME2)
537 ;; -------------------------------------------------------------------------
539 (define_c_enum "unspec" [
540 UNSPEC_RESTORE_ZT0
541 ])
543 (define_insn "aarch64_sme_ldr_zt0"
544 [(set (reg:V8DI ZT0_REGNUM)
545 (match_operand:V8DI 0 "aarch64_sync_memory_operand" "Q"))
546 (use (reg:DI SME_STATE_REGNUM))]
547 "TARGET_SME2"
548 "ldr\tzt0, %0"
549 )
551 ;; This version is used after calls to private-ZA functions. Since ZT0_REGNUM
552 ;; represents the current function's state, it isn't clobbered by private-ZA
553 ;; functions, so we need to make it depend on the ZA reinitialization code.
554 (define_insn "aarch64_restore_zt0"
555 [(set (reg:V8DI ZT0_REGNUM)
556 (unspec:V8DI
557 [(reg:DI SME_STATE_REGNUM)
558 (match_operand:V8DI 0 "aarch64_sync_memory_operand" "Q")]
559 UNSPEC_RESTORE_ZT0))]
560 "TARGET_SME2"
561 "ldr\tzt0, %0"
562 )
564 ;; -------------------------------------------------------------------------
565 ;; ---- Single-vector stores
566 ;; -------------------------------------------------------------------------
567 ;; Includes:
568 ;; - ST1
569 ;; - STR
570 ;; -------------------------------------------------------------------------
572 (define_c_enum "unspec" [
573 UNSPEC_SME_STR
574 ])
576 (define_insn "@aarch64_sme_<optab><mode>"
577 [(set (match_operand:SME_ZA_I 0 "aarch64_sve_ldff1_operand" "+Utf")
578 (unspec:SME_ZA_I
579 [(reg:SME_ZA_I ZA_REGNUM)
580 (reg:DI SME_STATE_REGNUM)
581 (match_dup 0)
582 (match_operand:DI 1 "const_int_operand")
583 (match_operand:SI 2 "register_operand" "Ucj")
584 (match_operand:<VPRED> 3 "register_operand" "Upl")]
585 SME_ST1))]
586 "TARGET_STREAMING"
587 "st1<Vesize>\t{ za%1<hv>.<Vetype>[%w2, 0] }, %3, %0"
588 )
590 (define_insn "@aarch64_sme_<optab><mode>_plus"
591 [(set (match_operand:SME_ZA_I 0 "aarch64_sve_ldff1_operand" "+Utf")
592 (unspec:SME_ZA_I
593 [(reg:SME_ZA_I ZA_REGNUM)
594 (reg:DI SME_STATE_REGNUM)
595 (match_dup 0)
596 (match_operand:DI 1 "const_int_operand")
597 (plus:SI (match_operand:SI 2 "register_operand" "Ucj")
598 (match_operand:SI 3 "const_int_operand"))
599 (match_operand:<VPRED> 4 "register_operand" "Upl")]
600 SME_ST1))]
601 "TARGET_STREAMING
602 && UINTVAL (operands[3]) < 128 / <elem_bits>"
603 "st1<Vesize>\t{ za%1<hv>.<Vetype>[%w2, %3] }, %4, %0"
604 )
606 (define_insn "aarch64_sme_str0"
607 [(set (mem:VNx16QI (match_operand 1 "pmode_register_operand" "rk"))
608 (unspec:VNx16QI
609 [(reg:VNx16QI ZA_REGNUM)
610 (reg:DI SME_STATE_REGNUM)
611 (mem:VNx16QI (match_dup 1))
612 (match_operand:SI 0 "register_operand" "Ucj")]
613 UNSPEC_SME_STR))]
614 "TARGET_SME"
615 "str\tza[%w0, 0], [%1, #0, mul vl]"
616 )
618 (define_insn "@aarch64_sme_strn<mode>"
619 [(set (mem:VNx16QI
620 (plus:P (match_operand:P 2 "register_operand" "rk")
621 (match_operand:P 3 "aarch64_mov_operand")))
622 (unspec:VNx16QI
623 [(reg:VNx16QI ZA_REGNUM)
624 (reg:DI SME_STATE_REGNUM)
625 (mem:VNx16QI (plus:P (match_dup 2) (match_dup 3)))
626 (plus:SI (match_operand:SI 0 "register_operand" "Ucj")
627 (match_operand:SI 1 "const_int_operand"))]
628 UNSPEC_SME_STR))]
629 "TARGET_SME
630 && aarch64_sme_ldr_vnum_offset_p (operands[1], operands[3])"
631 "str\tza[%w0, %1], [%2, #%1, mul vl]"
632 )
634 ;; -------------------------------------------------------------------------
635 ;; ---- Table stores
636 ;; -------------------------------------------------------------------------
637 ;; Includes:
638 ;; - STR (SME2)
639 ;; -------------------------------------------------------------------------
641 (define_insn "aarch64_sme_str_zt0"
642 [(set (match_operand:V8DI 0 "aarch64_sync_memory_operand" "=Q")
643 (reg:V8DI ZT0_REGNUM))
644 (use (reg:DI SME_STATE_REGNUM))]
645 "TARGET_SME2"
646 "str\tzt0, %0"
647 )
649 ;; -------------------------------------------------------------------------
650 ;; ---- Single-vector moves
651 ;; -------------------------------------------------------------------------
652 ;; Includes:
653 ;; - MOVA
654 ;; - MOVAZ (SME2p1)
655 ;; -------------------------------------------------------------------------
657 (define_insn "@aarch64_sme_<optab><v_int_container><mode>"
658 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
659 (unspec:SVE_FULL
660 [(reg:<V_INT_CONTAINER> ZA_REGNUM)
661 (reg:DI SME_STATE_REGNUM)
662 (match_operand:SVE_FULL 1 "register_operand" "0")
663 (match_operand:<VPRED> 2 "register_operand" "Upl")
664 (match_operand:DI 3 "const_int_operand")
665 (match_operand:SI 4 "register_operand" "Ucj")]
666 SME_READ_HV))]
667 "TARGET_STREAMING"
668 "mova\t%0.<Vetype>, %2/m, za%3<hv>.<Vetype>[%w4, 0]"
669 )
671 (define_insn "*aarch64_sme_<optab><v_int_container><mode>_plus"
672 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
673 (unspec:SVE_FULL
674 [(reg:<V_INT_CONTAINER> ZA_REGNUM)
675 (reg:DI SME_STATE_REGNUM)
676 (match_operand:SVE_FULL 1 "register_operand" "0")
677 (match_operand:<VPRED> 2 "register_operand" "Upl")
678 (match_operand:DI 3 "const_int_operand")
679 (plus:SI (match_operand:SI 4 "register_operand" "Ucj")
680 (match_operand:SI 5 "const_int_operand"))]
681 SME_READ_HV))]
682 "TARGET_STREAMING
683 && UINTVAL (operands[5]) < 128 / <elem_bits>"
684 "mova\t%0.<Vetype>, %2/m, za%3<hv>.<Vetype>[%w4, %5]"
685 )
687 (define_insn "@aarch64_sme_<optab><VNx1TI_ONLY:mode><SVE_FULL:mode>"
688 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
689 (unspec:SVE_FULL
690 [(reg:VNx1TI_ONLY ZA_REGNUM)
691 (reg:DI SME_STATE_REGNUM)
692 (match_operand:SVE_FULL 1 "register_operand" "0")
693 (match_operand:VNx2BI 2 "register_operand" "Upl")
694 (match_operand:DI 3 "const_int_operand")
695 (match_operand:SI 4 "register_operand" "Ucj")]
696 SME_READ_HV))]
697 "TARGET_STREAMING"
698 "mova\t%0.q, %2/m, za%3<hv>.q[%w4, 0]"
699 )
701 (define_insn "@aarch64_sme_<optab><v_int_container><mode>"
702 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
703 (unspec:SVE_FULL
704 [(reg:<V_INT_CONTAINER> ZA_REGNUM)
705 (reg:DI SME_STATE_REGNUM)
706 (match_operand:DI 1 "const_int_operand")
707 (match_operand:SI 2 "register_operand" "Ucj")
708 (const_int 0)]
709 SME_READZ_HV))
710 (set (reg:<V_INT_CONTAINER> ZA_REGNUM)
711 (unspec:<V_INT_CONTAINER>
712 [(reg:<V_INT_CONTAINER> ZA_REGNUM)
713 (reg:DI SME_STATE_REGNUM)
714 (match_dup 1)
715 (match_dup 2)
716 (const_int 1)]
717 SME_READZ_HV))]
718 "TARGET_STREAMING_SME2p1"
719 "movaz\t%0.<Vetype>, za%1<hv>.<Vetype>[%w2, 0]"
720 )
722 (define_insn "*aarch64_sme_<optab><v_int_container><mode>_plus"
723 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
724 (unspec:SVE_FULL
725 [(reg:<V_INT_CONTAINER> ZA_REGNUM)
726 (reg:DI SME_STATE_REGNUM)
727 (match_operand:DI 1 "const_int_operand")
728 (plus:SI (match_operand:SI 2 "register_operand" "Ucj")
729 (match_operand:SI 3 "const_int_operand"))
730 (const_int 0)]
731 SME_READZ_HV))
732 (set (reg:<V_INT_CONTAINER> ZA_REGNUM)
733 (unspec:<V_INT_CONTAINER>
734 [(reg:<V_INT_CONTAINER> ZA_REGNUM)
735 (reg:DI SME_STATE_REGNUM)
736 (match_dup 1)
737 (plus:SI (match_dup 2)
738 (match_dup 3))
739 (const_int 1)]
740 SME_READZ_HV))]
741 "TARGET_STREAMING_SME2p1
742 && UINTVAL (operands[3]) < 128 / <elem_bits>"
743 "movaz\t%0.<Vetype>, za%1<hv>.<Vetype>[%w2, %3]"
744 )
746 (define_insn "@aarch64_sme_<optab><VNx1TI_ONLY:mode><SVE_FULL:mode>"
747 [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
748 (unspec:SVE_FULL
749 [(reg:VNx1TI_ONLY ZA_REGNUM)
750 (reg:DI SME_STATE_REGNUM)
751 (match_operand:DI 1 "const_int_operand")
752 (match_operand:SI 2 "register_operand" "Ucj")
753 (const_int 0)]
754 SME_READZ_HV))
755 (set (reg:VNx1TI_ONLY ZA_REGNUM)
756 (unspec:VNx1TI_ONLY
757 [(reg:VNx1TI_ONLY ZA_REGNUM)
758 (reg:DI SME_STATE_REGNUM)
759 (match_dup 1)
760 (match_dup 2)
761 (const_int 0)]
762 SME_READZ_HV))]
763 "TARGET_STREAMING_SME2p1"
764 "movaz\t%0.q, za%1<hv>.q[%w2, 0]"
765 )
767 (define_insn "@aarch64_sme_<optab><v_int_container><mode>"
768 [(set (reg:<V_INT_CONTAINER> ZA_REGNUM)
769 (unspec:<V_INT_CONTAINER>
770 [(reg:SVE_FULL ZA_REGNUM)
771 (reg:DI SME_STATE_REGNUM)
772 (match_operand:DI 0 "const_int_operand")
773 (match_operand:SI 1 "register_operand" "Ucj")
774 (match_operand:<VPRED> 2 "register_operand" "Upl")
775 (match_operand:SVE_FULL 3 "register_operand" "w")]
776 SME_WRITE_HV))]
777 "TARGET_STREAMING"
778 "mova\tza%0<hv>.<Vetype>[%w1, 0], %2/m, %3.<Vetype>"
779 )
781 (define_insn "*aarch64_sme_<optab><v_int_container><mode>_plus"
782 [(set (reg:<V_INT_CONTAINER> ZA_REGNUM)
783 (unspec:<V_INT_CONTAINER>
784 [(reg:SVE_FULL ZA_REGNUM)
785 (reg:DI SME_STATE_REGNUM)
786 (match_operand:DI 0 "const_int_operand")
787 (plus:SI (match_operand:SI 1 "register_operand" "Ucj")
788 (match_operand:SI 2 "const_int_operand"))
789 (match_operand:<VPRED> 3 "register_operand" "Upl")
790 (match_operand:SVE_FULL 4 "register_operand" "w")]
791 SME_WRITE_HV))]
792 "TARGET_STREAMING
793 && UINTVAL (operands[2]) < 128 / <elem_bits>"
794 "mova\tza%0<hv>.<Vetype>[%w1, %2], %3/m, %4.<Vetype>"
795 )
797 (define_insn "@aarch64_sme_<optab><VNx1TI_ONLY:mode><SVE_FULL:mode>"
798 [(set (reg:VNx1TI_ONLY ZA_REGNUM)
799 (unspec:VNx1TI_ONLY
800 [(reg:VNx1TI_ONLY ZA_REGNUM)
801 (reg:DI SME_STATE_REGNUM)
802 (match_operand:DI 0 "const_int_operand")
803 (match_operand:SI 1 "register_operand" "Ucj")
804 (match_operand:VNx2BI 2 "register_operand" "Upl")
805 (match_operand:SVE_FULL 3 "register_operand" "w")]
806 SME_WRITE_HV))]
807 "TARGET_STREAMING"
808 "mova\tza%0<hv>.q[%w1, 0], %2/m, %3.q"
809 )
811 ;; -------------------------------------------------------------------------
812 ;; ---- Multi-vector moves
813 ;; -------------------------------------------------------------------------
814 ;; Includes:
815 ;; - MOVA
816 ;; - MOVAZ (SME2p1)
817 ;; -------------------------------------------------------------------------
819 (define_insn "@aarch64_sme_<optab><mode><mode>"
820 [(set (match_operand:SVE_FULLx24 0 "aligned_register_operand" "=Uw<vector_count>")
821 (unspec:SVE_FULLx24
822 [(reg:SVE_FULLx24 ZA_REGNUM)
823 (reg:DI SME_STATE_REGNUM)
824 (match_operand:DI 1 "const_int_operand")
825 (match_operand:SI 2 "register_operand" "Ucj")]
826 SME_READ_HV))]
827 "TARGET_STREAMING_SME2"
828 {
829 operands[3] = GEN_INT (<vector_count> - 1);
830 return "mova\t%0, za%1<hv>.<Vetype>[%w2, 0:%3]";
831 }
832 )
834 (define_insn "*aarch64_sme_<optab><mode><mode>_plus"
835 [(set (match_operand:SVE_FULLx24 0 "aligned_register_operand" "=Uw<vector_count>")
836 (unspec:SVE_FULLx24
837 [(reg:SVE_FULLx24 ZA_REGNUM)
838 (reg:DI SME_STATE_REGNUM)
839 (match_operand:DI 1 "const_int_operand")
840 (plus:SI
841 (match_operand:SI 2 "register_operand" "Ucj")
842 (match_operand:SI 3 "const_int_operand"))]
843 SME_READ_HV))]
844 "TARGET_STREAMING_SME2
845 && UINTVAL (operands[3]) % <vector_count> == 0
846 && UINTVAL (operands[3]) < 128 / <elem_bits>"
847 {
848 operands[4] = GEN_INT (INTVAL (operands[3]) + <vector_count> - 1);
849 return "mova\t%0, za%1<hv>.<Vetype>[%w2, %3:%4]";
850 }
851 )
853 (define_insn "@aarch64_sme_<optab><mode><mode>"
854 [(set (match_operand:SVE_FULLx24 0 "aligned_register_operand" "=Uw<vector_count>")
855 (unspec:SVE_FULLx24
856 [(reg:SVE_FULLx24 ZA_REGNUM)
857 (reg:DI SME_STATE_REGNUM)
858 (match_operand:DI 1 "const_int_operand")
859 (match_operand:SI 2 "register_operand" "Ucj")
860 (const_int 0)]
861 SME_READZ_HV))
862 (set (reg:SVE_FULLx24 ZA_REGNUM)
863 (unspec:SVE_FULLx24
864 [(reg:SVE_FULLx24 ZA_REGNUM)
865 (reg:DI SME_STATE_REGNUM)
866 (match_dup 1)
867 (match_dup 2)
868 (const_int 1)]
869 SME_READZ_HV))]
870 "TARGET_STREAMING_SME2p1"
871 {
872 operands[3] = GEN_INT (<vector_count> - 1);
873 return "movaz\t%0, za%1<hv>.<Vetype>[%w2, 0:%3]";
874 }
875 )
877 (define_insn "*aarch64_sme_<optab><mode><mode>_plus"
878 [(set (match_operand:SVE_FULLx24 0 "aligned_register_operand" "=Uw<vector_count>")
879 (unspec:SVE_FULLx24
880 [(reg:SVE_FULLx24 ZA_REGNUM)
881 (reg:DI SME_STATE_REGNUM)
882 (match_operand:DI 1 "const_int_operand")
883 (plus:SI
884 (match_operand:SI 2 "register_operand" "Ucj")
885 (match_operand:SI 3 "const_int_operand"))
886 (const_int 0)]
887 SME_READZ_HV))
888 (set (reg:SVE_FULLx24 ZA_REGNUM)
889 (unspec:SVE_FULLx24
890 [(reg:SVE_FULLx24 ZA_REGNUM)
891 (reg:DI SME_STATE_REGNUM)
892 (match_dup 1)
893 (plus:SI
894 (match_dup 2)
895 (match_dup 3))
896 (const_int 1)]
897 SME_READZ_HV))]
898 "TARGET_STREAMING_SME2p1
899 && UINTVAL (operands[3]) % <vector_count> == 0
900 && UINTVAL (operands[3]) < 128 / <elem_bits>"
901 {
902 operands[4] = GEN_INT (INTVAL (operands[3]) + <vector_count> - 1);
903 return "movaz\t%0, za%1<hv>.<Vetype>[%w2, %3:%4]";
904 }
905 )
907 (define_insn "@aarch64_sme_read<mode>"
908 [(set (match_operand:SVE_DIx24 0 "aligned_register_operand" "=Uw<vector_count>")
909 (unspec:SVE_DIx24
910 [(reg:SVE_DIx24 ZA_REGNUM)
911 (reg:DI SME_STATE_REGNUM)
912 (match_operand:SI 1 "register_operand" "Uci")]
913 UNSPEC_SME_READ))]
914 "TARGET_STREAMING_SME2"
915 "mova\t%0, za.d[%w1, 0, vgx<vector_count>]"
916 )
918 (define_insn "*aarch64_sme_read<mode>_plus"
919 [(set (match_operand:SVE_DIx24 0 "aligned_register_operand" "=Uw<vector_count>")
920 (unspec:SVE_DIx24
921 [(reg:SVE_DIx24 ZA_REGNUM)
922 (reg:DI SME_STATE_REGNUM)
923 (plus:SI (match_operand:SI 1 "register_operand" "Uci")
924 (match_operand:SI 2 "const_0_to_7_operand"))]
925 UNSPEC_SME_READ))]
926 "TARGET_STREAMING_SME2"
927 "mova\t%0, za.d[%w1, %2, vgx<vector_count>]"
928 )
930 (define_insn "@aarch64_sme_readz<mode>"
931 [(set (match_operand:SVE_DIx24 0 "aligned_register_operand" "=Uw<vector_count>")
932 (unspec:SVE_DIx24
933 [(reg:SVE_DIx24 ZA_REGNUM)
934 (reg:DI SME_STATE_REGNUM)
935 (match_operand:SI 1 "register_operand" "Uci")
936 (const_int 0)]
937 UNSPEC_SME_READZ))
938 (set (reg:SVE_DIx24 ZA_REGNUM)
939 (unspec:SVE_DIx24
940 [(reg:SVE_DIx24 ZA_REGNUM)
941 (reg:DI SME_STATE_REGNUM)
942 (match_dup 1)
943 (const_int 1)]
944 UNSPEC_SME_READZ))]
945 "TARGET_STREAMING_SME2p1"
946 "movaz\t%0, za.d[%w1, 0, vgx<vector_count>]"
947 )
949 (define_insn "*aarch64_sme_readz<mode>_plus"
950 [(set (match_operand:SVE_DIx24 0 "aligned_register_operand" "=Uw<vector_count>")
951 (unspec:SVE_DIx24
952 [(reg:SVE_DIx24 ZA_REGNUM)
953 (reg:DI SME_STATE_REGNUM)
954 (plus:SI (match_operand:SI 1 "register_operand" "Uci")
955 (match_operand:SI 2 "const_0_to_7_operand"))
956 (const_int 0)]
957 UNSPEC_SME_READZ))
958 (set (reg:SVE_DIx24 ZA_REGNUM)
959 (unspec:SVE_DIx24
960 [(reg:SVE_DIx24 ZA_REGNUM)
961 (reg:DI SME_STATE_REGNUM)
962 (plus:SI (match_dup 1)
963 (match_dup 2))
964 (const_int 1)]
965 UNSPEC_SME_READZ))]
966 "TARGET_STREAMING_SME2p1"
967 "movaz\t%0, za.d[%w1, %2, vgx<vector_count>]"
968 )
970 (define_insn "@aarch64_sme_<optab><mode><mode>"
971 [(set (reg:SVE_FULLx24 ZA_REGNUM)
972 (unspec:SVE_FULLx24
973 [(reg:SVE_FULLx24 ZA_REGNUM)
974 (reg:DI SME_STATE_REGNUM)
975 (match_operand:DI 0 "const_int_operand")
976 (match_operand:SI 1 "register_operand" "Ucj")
977 (match_operand:SVE_FULLx24 2 "aligned_register_operand" "Uw<vector_count>")]
978 SME_WRITE_HV))]
979 "TARGET_STREAMING_SME2"
980 {
981 operands[3] = GEN_INT (<vector_count> - 1);
982 return "mova\tza%0<hv>.<Vetype>[%w1, 0:%3], %2";
983 }
984 )
986 (define_insn "*aarch64_sme_<optab><mode><mode>_plus"
987 [(set (reg:SVE_FULLx24 ZA_REGNUM)
988 (unspec:SVE_FULLx24
989 [(reg:SVE_FULLx24 ZA_REGNUM)
990 (reg:DI SME_STATE_REGNUM)
991 (match_operand:DI 0 "const_int_operand")
992 (plus:SI
993 (match_operand:SI 1 "register_operand" "Ucj")
994 (match_operand:SI 2 "const_int_operand"))
995 (match_operand:SVE_FULLx24 3 "aligned_register_operand" "Uw<vector_count>")]
996 SME_WRITE_HV))]
997 "TARGET_STREAMING_SME2
998 && UINTVAL (operands[2]) % <vector_count> == 0
999 && UINTVAL (operands[2]) < 128 / <elem_bits>"
1000 {
1001 operands[4] = GEN_INT (INTVAL (operands[2]) + <vector_count> - 1);
1002 return "mova\tza%0<hv>.<Vetype>[%w1, %2:%4], %3";
1003 }
1004 )
1006 (define_insn "@aarch64_sme_write<mode>"
1007 [(set (reg:SVE_DIx24 ZA_REGNUM)
1008 (unspec:SVE_DIx24
1009 [(reg:SVE_DIx24 ZA_REGNUM)
1010 (reg:DI SME_STATE_REGNUM)
1011 (match_operand:SI 0 "register_operand" "Uci")
1012 (match_operand:SVE_DIx24 1 "aligned_register_operand" "Uw<vector_count>")]
1013 UNSPEC_SME_WRITE))]
1014 "TARGET_STREAMING_SME2"
1015 "mova\tza.d[%w0, 0, vgx<vector_count>], %1"
1016 )
1018 (define_insn "*aarch64_sme_write<mode>_plus"
1019 [(set (reg:SVE_DIx24 ZA_REGNUM)
1020 (unspec:SVE_DIx24
1021 [(reg:SVE_DIx24 ZA_REGNUM)
1022 (reg:DI SME_STATE_REGNUM)
1023 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1024 (match_operand:SI 1 "const_0_to_7_operand"))
1025 (match_operand:SVE_DIx24 2 "aligned_register_operand" "Uw<vector_count>")]
1026 UNSPEC_SME_WRITE))]
1027 "TARGET_STREAMING_SME2"
1028 "mova\tza.d[%w0, %1, vgx<vector_count>], %2"
1029 )
1031 ;; -------------------------------------------------------------------------
1032 ;; ---- Zeroing
1033 ;; -------------------------------------------------------------------------
1034 ;; Includes:
1035 ;; - ZERO
1036 ;; -------------------------------------------------------------------------
1038 (define_c_enum "unspec" [UNSPEC_SME_ZERO UNSPEC_SME_ZERO_SLICES])
1040 (define_insn "aarch64_sme_zero_za"
1041 [(set (reg:VNx16QI ZA_REGNUM)
1042 (unspec:VNx16QI [(reg:VNx16QI ZA_REGNUM)
1043 (reg:DI SME_STATE_REGNUM)
1044 (match_operand:DI 0 "const_int_operand")]
1045 UNSPEC_SME_ZERO))]
1046 "TARGET_SME"
1047 {
1048 return aarch64_output_sme_zero_za (operands[0]);
1049 }
1050 )
1052 (define_insn "@aarch64_sme_zero_za_slices<mode>"
1053 [(set (reg:VNx16QI ZA_REGNUM)
1054 (unspec:VNx16QI
1055 [(reg:VNx16QI ZA_REGNUM)
1056 (reg:DI SME_STATE_REGNUM)
1057 (scratch:SME_ZA_SDIx24)
1058 (match_operand:SI 0 "register_operand" "Uci")]
1059 UNSPEC_SME_ZERO_SLICES))]
1060 "TARGET_STREAMING_SME2p1"
1061 "zero\tza.d[%w0, 0, vgx<vector_count>]"
1062 )
1064 (define_insn "*aarch64_sme_zero_za_slices<mode>_plus"
1065 [(set (reg:VNx16QI ZA_REGNUM)
1066 (unspec:VNx16QI
1067 [(reg:VNx16QI ZA_REGNUM)
1068 (reg:DI SME_STATE_REGNUM)
1069 (scratch:SME_ZA_SDIx24)
1070 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1071 (match_operand:SI 1 "const_0_to_7_operand"))]
1072 UNSPEC_SME_ZERO_SLICES))]
1073 "TARGET_STREAMING_SME2p1"
1074 "zero\tza.d[%w0, %1, vgx<vector_count>]"
1075 )
1077 (define_insn "@aarch64_sme_zero_za_slices<mode>"
1078 [(set (reg:VNx16QI ZA_REGNUM)
1079 (unspec:VNx16QI
1080 [(reg:VNx16QI ZA_REGNUM)
1081 (reg:DI SME_STATE_REGNUM)
1082 (scratch:SME_ZA_BHIx124)
1083 (match_operand:SI 0 "register_operand" "Uci")]
1084 UNSPEC_SME_ZERO_SLICES))]
1085 "TARGET_STREAMING_SME2p1"
1086 "zero\tza.d[%w0, 0:<za32_last_offset><vg_modifier>]"
1087 )
1089 (define_insn "*aarch64_sme_zero_za_slices<mode>_plus"
1090 [(set (reg:VNx16QI ZA_REGNUM)
1091 (unspec:VNx16QI
1092 [(reg:VNx16QI ZA_REGNUM)
1093 (reg:DI SME_STATE_REGNUM)
1094 (scratch:SME_ZA_BHIx124)
1095 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1096 (match_operand:SI 1 "const_<za32_offset_range>_operand"))]
1097 UNSPEC_SME_ZERO_SLICES))]
1098 "TARGET_STREAMING_SME2p1"
1099 {
1100 operands[2] = GEN_INT (INTVAL (operands[1]) + <za32_last_offset>);
1101 return "zero\tza.d[%w0, %1:%2<vg_modifier>]";
1102 }
1103 )
1105 (define_insn "aarch64_sme_zero_zt0"
1106 [(set (reg:V8DI ZT0_REGNUM)
1107 (const_int 0))
1108 (use (reg:DI SME_STATE_REGNUM))]
1109 "TARGET_SME2"
1110 "zero\t{ zt0 }"
1111 )
1113 ;; =========================================================================
1114 ;; == Binary arithmetic
1115 ;; =========================================================================
1117 ;; -------------------------------------------------------------------------
1118 ;; ---- Binary arithmetic on ZA tile
1119 ;; -------------------------------------------------------------------------
1120 ;; Includes:
1121 ;; - ADDHA
1122 ;; - ADDVA
1123 ;; -------------------------------------------------------------------------
1125 (define_insn "@aarch64_sme_<optab><mode>"
1126 [(set (reg:SME_ZA_SDI ZA_REGNUM)
1127 (unspec:SME_ZA_SDI
1128 [(reg:SME_ZA_SDI ZA_REGNUM)
1129 (reg:DI SME_STATE_REGNUM)
1130 (match_operand:DI 0 "const_int_operand")
1131 (match_operand:<VPRED> 1 "register_operand" "Upl")
1132 (match_operand:<VPRED> 2 "register_operand" "Upl")
1133 (match_operand:SME_ZA_SDI 3 "register_operand" "w")]
1134 SME_BINARY_SDI))]
1135 "TARGET_STREAMING"
1136 "<optab>\tza%0.<Vetype>, %1/m, %2/m, %3.<Vetype>"
1137 )
1139 ;; -------------------------------------------------------------------------
1140 ;; ---- Binary arithmetic on ZA slice
1141 ;; -------------------------------------------------------------------------
1142 ;; Includes:
1143 ;; - ADD (SME2)
1144 ;; - BFADD (SME_B16B16)
1145 ;; - BFSUB (SME_B16B16)
1146 ;; - FADD (SME2)
1147 ;; - FSUB (SME2)
1148 ;; - SUB (SME2)
1149 ;; -------------------------------------------------------------------------
1151 (define_insn "@aarch64_sme_<optab><mode>"
1152 [(set (reg:SME_ZA_SDIx24 ZA_REGNUM)
1153 (unspec:SME_ZA_SDIx24
1154 [(reg:SME_ZA_SDIx24 ZA_REGNUM)
1155 (reg:DI SME_STATE_REGNUM)
1156 (match_operand:SI 0 "register_operand" "Uci")
1157 (match_operand:SME_ZA_SDIx24 1 "aligned_register_operand" "Uw<vector_count>")]
1158 SME_BINARY_SLICE_SDI))]
1159 "TARGET_STREAMING_SME2"
1160 "<optab>\tza.<Vetype>[%w0, 0, vgx<vector_count>], %1"
1161 )
1163 (define_insn "*aarch64_sme_<optab><mode>_plus"
1164 [(set (reg:SME_ZA_SDIx24 ZA_REGNUM)
1165 (unspec:SME_ZA_SDIx24
1166 [(reg:SME_ZA_SDIx24 ZA_REGNUM)
1167 (reg:DI SME_STATE_REGNUM)
1168 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1169 (match_operand:SI 1 "const_0_to_7_operand"))
1170 (match_operand:SME_ZA_SDIx24 2 "aligned_register_operand" "Uw<vector_count>")]
1171 SME_BINARY_SLICE_SDI))]
1172 "TARGET_STREAMING_SME2"
1173 "<optab>\tza.<Vetype>[%w0, %1, vgx<vector_count>], %2"
1174 )
1176 (define_insn "@aarch64_sme_<optab><mode>"
1177 [(set (reg:SME_ZA_HSDFx24 ZA_REGNUM)
1178 (unspec:SME_ZA_HSDFx24
1179 [(reg:SME_ZA_HSDFx24 ZA_REGNUM)
1180 (reg:DI SME_STATE_REGNUM)
1181 (match_operand:SI 0 "register_operand" "Uci")
1182 (match_operand:SME_ZA_HSDFx24 1 "aligned_register_operand" "Uw<vector_count>")]
1183 SME_BINARY_SLICE_HSDF))]
1184 "TARGET_STREAMING_SME2"
1185 "<b><optab>\tza.<Vetype>[%w0, 0, vgx<vector_count>], %1"
1186 )
1188 (define_insn "*aarch64_sme_<optab><mode>_plus"
1189 [(set (reg:SME_ZA_HSDFx24 ZA_REGNUM)
1190 (unspec:SME_ZA_HSDFx24
1191 [(reg:SME_ZA_HSDFx24 ZA_REGNUM)
1192 (reg:DI SME_STATE_REGNUM)
1193 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1194 (match_operand:SI 1 "const_0_to_7_operand"))
1195 (match_operand:SME_ZA_HSDFx24 2 "aligned_register_operand" "Uw<vector_count>")]
1196 SME_BINARY_SLICE_HSDF))]
1197 "TARGET_STREAMING_SME2"
1198 "<b><optab>\tza.<Vetype>[%w0, %1, vgx<vector_count>], %2"
1199 )
1201 ;; -------------------------------------------------------------------------
1202 ;; ---- Binary arithmetic, writing to ZA slice
1203 ;; -------------------------------------------------------------------------
1204 ;; Includes:
1205 ;; - ADD (SME2)
1206 ;; - SUB (SME2)
1207 ;; -------------------------------------------------------------------------
1209 (define_insn "@aarch64_sme_<optab><mode>"
1210 [(set (reg:SME_ZA_SDIx24 ZA_REGNUM)
1211 (unspec:SME_ZA_SDIx24
1212 [(reg:SME_ZA_SDIx24 ZA_REGNUM)
1213 (reg:DI SME_STATE_REGNUM)
1214 (match_operand:SI 0 "register_operand" "Uci")
1215 (match_operand:SME_ZA_SDIx24 1 "aligned_register_operand" "Uw<vector_count>")
1216 (match_operand:SME_ZA_SDIx24 2 "aligned_register_operand" "Uw<vector_count>")]
1217 SME_BINARY_WRITE_SLICE_SDI))]
1218 "TARGET_STREAMING_SME2"
1219 "<sme_int_op>\tza.<Vetype>[%w0, 0, vgx<vector_count>], %1, %2"
1220 )
1222 (define_insn "*aarch64_sme_<optab><mode>_plus"
1223 [(set (reg:SME_ZA_SDIx24 ZA_REGNUM)
1224 (unspec:SME_ZA_SDIx24
1225 [(reg:SME_ZA_SDIx24 ZA_REGNUM)
1226 (reg:DI SME_STATE_REGNUM)
1227 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1228 (match_operand:SI 1 "const_0_to_7_operand"))
1229 (match_operand:SME_ZA_SDIx24 2 "aligned_register_operand" "Uw<vector_count>")
1230 (match_operand:SME_ZA_SDIx24 3 "aligned_register_operand" "Uw<vector_count>")]
1231 SME_BINARY_WRITE_SLICE_SDI))]
1232 "TARGET_STREAMING_SME2"
1233 "<sme_int_op>\tza.<Vetype>[%w0, %1, vgx<vector_count>], %2, %3"
1234 )
1236 (define_insn "@aarch64_sme_single_<optab><mode>"
1237 [(set (reg:SME_ZA_SDIx24 ZA_REGNUM)
1238 (unspec:SME_ZA_SDIx24
1239 [(reg:SME_ZA_SDIx24 ZA_REGNUM)
1240 (reg:DI SME_STATE_REGNUM)
1241 (match_operand:SI 0 "register_operand" "Uci")
1242 (match_operand:SME_ZA_SDIx24 1 "register_operand" "w")
1243 (vec_duplicate:SME_ZA_SDIx24
1244 (match_operand:<VSINGLE> 2 "register_operand" "x"))]
1245 SME_BINARY_WRITE_SLICE_SDI))]
1246 "TARGET_STREAMING_SME2"
1247 "<sme_int_op>\tza.<Vetype>[%w0, 0, vgx<vector_count>], %1, %2.<Vetype>"
1248 )
1250 (define_insn "*aarch64_sme_single_<optab><mode>_plus"
1251 [(set (reg:SME_ZA_SDIx24 ZA_REGNUM)
1252 (unspec:SME_ZA_SDIx24
1253 [(reg:SME_ZA_SDIx24 ZA_REGNUM)
1254 (reg:DI SME_STATE_REGNUM)
1255 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1256 (match_operand:SI 1 "const_0_to_7_operand"))
1257 (match_operand:SME_ZA_SDIx24 2 "register_operand" "w")
1258 (vec_duplicate:SME_ZA_SDIx24
1259 (match_operand:<VSINGLE> 3 "register_operand" "x"))]
1260 SME_BINARY_WRITE_SLICE_SDI))]
1261 "TARGET_STREAMING_SME2"
1262 "<sme_int_op>\tza.<Vetype>[%w0, %1, vgx<vector_count>], %2, %3.<Vetype>"
1263 )
1265 ;; =========================================================================
1266 ;; == Ternary arithmetic
1267 ;; =========================================================================
1269 ;; -------------------------------------------------------------------------
1270 ;; ---- [INT] Dot product
1271 ;; -------------------------------------------------------------------------
1272 ;; Includes:
1273 ;; - SDOT (SME2)
1274 ;; - SUDOT (SME2)
1275 ;; - UDOT (SME2)
1276 ;; - USDOT (SME2)
1277 ;; -------------------------------------------------------------------------
1279 (define_insn "@aarch64_sme_<optab><SME_ZA_SDI:mode><SME_ZA_BHIx24:mode>"
1280 [(set (reg:SME_ZA_SDI ZA_REGNUM)
1281 (unspec:SME_ZA_SDI
1282 [(reg:SME_ZA_SDI ZA_REGNUM)
1283 (reg:DI SME_STATE_REGNUM)
1284 (match_operand:SI 0 "register_operand" "Uci")
1285 (match_operand:SME_ZA_BHIx24 1 "aligned_register_operand" "Uw<vector_count>")
1286 (match_operand:SME_ZA_BHIx24 2 "aligned_register_operand" "Uw<vector_count>")]
1287 SME_INT_DOTPROD))]
1288 "TARGET_STREAMING_SME2
1289 && (<SME_ZA_SDI:elem_bits> == 32 || <SME_ZA_BHIx24:elem_bits> == 16)
1290 && (<SME_ZA_BHIx24:elem_bits> == 8 || <has_16bit_form>)"
1291 "<optab>\tza.<SME_ZA_SDI:Vetype>[%w0, 0, vgx<vector_count>], %1, %2"
1292 )
1294 (define_insn "*aarch64_sme_<optab><SME_ZA_SDI:mode><SME_ZA_BHIx24:mode>_plus"
1295 [(set (reg:SME_ZA_SDI ZA_REGNUM)
1296 (unspec:SME_ZA_SDI
1297 [(reg:SME_ZA_SDI ZA_REGNUM)
1298 (reg:DI SME_STATE_REGNUM)
1299 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1300 (match_operand:SI 1 "const_0_to_7_operand"))
1301 (match_operand:SME_ZA_BHIx24 2 "aligned_register_operand" "Uw<vector_count>")
1302 (match_operand:SME_ZA_BHIx24 3 "aligned_register_operand" "Uw<vector_count>")]
1303 SME_INT_DOTPROD))]
1304 "TARGET_STREAMING_SME2
1305 && (<SME_ZA_SDI:elem_bits> == 32 || <SME_ZA_BHIx24:elem_bits> == 16)
1306 && (<SME_ZA_BHIx24:elem_bits> == 8 || <has_16bit_form>)"
1307 "<optab>\tza.<SME_ZA_SDI:Vetype>[%w0, %1, vgx<vector_count>], %2, %3"
1308 )
1310 (define_insn "@aarch64_sme_single_<optab><SME_ZA_SDI:mode><SME_ZA_BHIx24:mode>"
1311 [(set (reg:SME_ZA_SDI ZA_REGNUM)
1312 (unspec:SME_ZA_SDI
1313 [(reg:SME_ZA_SDI ZA_REGNUM)
1314 (reg:DI SME_STATE_REGNUM)
1315 (match_operand:SI 0 "register_operand" "Uci")
1316 (match_operand:SME_ZA_BHIx24 1 "register_operand" "w")
1317 (vec_duplicate:SME_ZA_BHIx24
1318 (match_operand:<VSINGLE> 2 "register_operand" "x"))]
1319 SME_INT_DOTPROD))]
1320 "TARGET_STREAMING_SME2
1321 && (<SME_ZA_SDI:elem_bits> == 32 || <SME_ZA_BHIx24:elem_bits> == 16)
1322 && (<SME_ZA_BHIx24:elem_bits> == 8 || <has_16bit_form>)"
1323 "<optab>\tza.<SME_ZA_SDI:Vetype>[%w0, 0, vgx<vector_count>], %1, %2.<SME_ZA_BHIx24:Vetype>"
1324 )
1326 (define_insn "*aarch64_sme_single_<optab><SME_ZA_SDI:mode><SME_ZA_BHIx24:mode>_plus"
1327 [(set (reg:SME_ZA_SDI ZA_REGNUM)
1328 (unspec:SME_ZA_SDI
1329 [(reg:SME_ZA_SDI ZA_REGNUM)
1330 (reg:DI SME_STATE_REGNUM)
1331 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1332 (match_operand:SI 1 "const_0_to_7_operand"))
1333 (match_operand:SME_ZA_BHIx24 2 "register_operand" "w")
1334 (vec_duplicate:SME_ZA_BHIx24
1335 (match_operand:<VSINGLE> 3 "register_operand" "x"))]
1336 SME_INT_DOTPROD))]
1337 "TARGET_STREAMING_SME2
1338 && (<SME_ZA_SDI:elem_bits> == 32 || <SME_ZA_BHIx24:elem_bits> == 16)
1339 && (<SME_ZA_BHIx24:elem_bits> == 8 || <has_16bit_form>)"
1340 "<optab>\tza.<SME_ZA_SDI:Vetype>[%w0, %1, vgx<vector_count>], %2, %3.<SME_ZA_BHIx24:Vetype>"
1341 )
1343 ;; SUDOT is USDOT with the operands swapped.
1344 (define_insn "@aarch64_sme_single_sudot<VNx4SI_ONLY:mode><SME_ZA_BIx24:mode>"
1345 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1346 (unspec:VNx4SI_ONLY
1347 [(reg:VNx4SI_ONLY ZA_REGNUM)
1348 (reg:DI SME_STATE_REGNUM)
1349 (match_operand:SI 0 "register_operand" "Uci")
1350 (vec_duplicate:SME_ZA_BIx24
1351 (match_operand:<VSINGLE> 2 "register_operand" "x"))
1352 (match_operand:SME_ZA_BIx24 1 "register_operand" "w")]
1353 UNSPEC_SME_USDOT))]
1354 "TARGET_STREAMING_SME2"
1355 "sudot\tza.s[%w0, 0, vgx<vector_count>], %1, %2.b"
1356 )
1358 (define_insn "*aarch64_sme_single_sudot<VNx4SI_ONLY:mode><SME_ZA_BIx24:mode>_plus"
1359 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1360 (unspec:VNx4SI_ONLY
1361 [(reg:VNx4SI_ONLY ZA_REGNUM)
1362 (reg:DI SME_STATE_REGNUM)
1363 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1364 (match_operand:SI 1 "const_0_to_7_operand"))
1365 (vec_duplicate:SME_ZA_BIx24
1366 (match_operand:<VSINGLE> 3 "register_operand" "x"))
1367 (match_operand:SME_ZA_BIx24 2 "register_operand" "w")]
1368 UNSPEC_SME_USDOT))]
1369 "TARGET_STREAMING_SME2"
1370 "sudot\tza.s[%w0, %1, vgx<vector_count>], %2, %3.b"
1371 )
1373 (define_insn "@aarch64_sme_lane_<optab><SME_ZA_SDI:mode><SME_ZA_BHIx24:mode>"
1374 [(set (reg:SME_ZA_SDI ZA_REGNUM)
1375 (unspec:SME_ZA_SDI
1376 [(reg:SME_ZA_SDI ZA_REGNUM)
1377 (reg:DI SME_STATE_REGNUM)
1378 (match_operand:SI 0 "register_operand" "Uci")
1379 (match_operand:SME_ZA_BHIx24 1 "aligned_register_operand" "Uw<vector_count>")
1380 (unspec:SME_ZA_BHIx24
1381 [(match_operand:<VSINGLE> 2 "register_operand" "x")
1382 (match_operand:SI 3 "const_int_operand")]
1383 UNSPEC_SVE_LANE_SELECT)]
1384 SME_INT_DOTPROD_LANE))]
1385 "TARGET_STREAMING_SME2
1386 && (<SME_ZA_SDI:elem_bits> == 32 || <SME_ZA_BHIx24:elem_bits> == 16)
1387 && (<SME_ZA_BHIx24:elem_bits> == 8 || <has_16bit_form>)"
1388 "<optab>\tza.<SME_ZA_SDI:Vetype>[%w0, 0, vgx<vector_count>], %1, %2.<SME_ZA_BHIx24:Vetype>[%3]"
1389 )
1391 (define_insn "*aarch64_sme_lane_<optab><SME_ZA_SDI:mode><SME_ZA_BHIx24:mode>_plus"
1392 [(set (reg:SME_ZA_SDI ZA_REGNUM)
1393 (unspec:SME_ZA_SDI
1394 [(reg:SME_ZA_SDI ZA_REGNUM)
1395 (reg:DI SME_STATE_REGNUM)
1396 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1397 (match_operand:SI 1 "const_0_to_7_operand"))
1398 (match_operand:SME_ZA_BHIx24 2 "aligned_register_operand" "Uw<vector_count>")
1399 (unspec:SME_ZA_BHIx24
1400 [(match_operand:<VSINGLE> 3 "register_operand" "x")
1401 (match_operand:SI 4 "const_int_operand")]
1402 UNSPEC_SVE_LANE_SELECT)]
1403 SME_INT_DOTPROD_LANE))]
1404 "TARGET_STREAMING_SME2
1405 && (<SME_ZA_SDI:elem_bits> == 32 || <SME_ZA_BHIx24:elem_bits> == 16)
1406 && (<SME_ZA_BHIx24:elem_bits> == 8 || <has_16bit_form>)"
1407 "<optab>\tza.<SME_ZA_SDI:Vetype>[%w0, %1, vgx<vector_count>], %2, %3.<SME_ZA_BHIx24:Vetype>[%4]"
1408 )
1410 ;; -------------------------------------------------------------------------
1411 ;; ---- [INT] Ternary widening arithmetic on ZA slice
1412 ;; -------------------------------------------------------------------------
1413 ;; Includes:
1414 ;; - SMLA (SME2)
1415 ;; - SMLS (SME2)
1416 ;; - UMLA (SME2)
1417 ;; - UMLS (SME2)
1418 ;; -------------------------------------------------------------------------
1420 (define_insn "@aarch64_sme_<optab><VNx4SI_ONLY:mode><SVE_FULL_BHI:mode>"
1421 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1422 (unspec:VNx4SI_ONLY
1423 [(reg:VNx4SI_ONLY ZA_REGNUM)
1424 (reg:DI SME_STATE_REGNUM)
1425 (match_operand:SI 0 "register_operand" "Uci")
1426 (match_operand:SVE_FULL_BHI 1 "register_operand" "w")
1427 (match_operand:SVE_FULL_BHI 2 "register_operand" "x")]
1428 SME_INT_TERNARY_SLICE))]
1429 "TARGET_STREAMING_SME2"
1430 "<optab><za32_long>\tza.s[%w0, 0:<za32_last_offset>], %1.<SVE_FULL_BHI:Vetype>, %2.<SVE_FULL_BHI:Vetype>"
1431 )
1433 (define_insn "*aarch64_sme_<optab><VNx4SI_ONLY:mode><SVE_FULL_BHI:mode>_plus"
1434 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1435 (unspec:VNx4SI_ONLY
1436 [(reg:VNx4SI_ONLY ZA_REGNUM)
1437 (reg:DI SME_STATE_REGNUM)
1438 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1439 (match_operand:SI 1 "const_<za32_offset_range>_operand"))
1440 (match_operand:SVE_FULL_BHI 2 "register_operand" "w")
1441 (match_operand:SVE_FULL_BHI 3 "register_operand" "x")]
1442 SME_INT_TERNARY_SLICE))]
1443 "TARGET_STREAMING_SME2"
1444 {
1445 operands[4] = GEN_INT (INTVAL (operands[1]) + <za32_last_offset>);
1446 return "<optab><za32_long>\tza.s[%w0, %1:%4], %2.<SVE_FULL_BHI:Vetype>, %3.<SVE_FULL_BHI:Vetype>";
1447 }
1448 )
1450 (define_insn "@aarch64_sme_<optab><VNx4SI_ONLY:mode><SME_ZA_BHIx24:mode>"
1451 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1452 (unspec:VNx4SI_ONLY
1453 [(reg:VNx4SI_ONLY ZA_REGNUM)
1454 (reg:DI SME_STATE_REGNUM)
1455 (match_operand:SI 0 "register_operand" "Uci")
1456 (match_operand:SME_ZA_BHIx24 1 "aligned_register_operand" "Uw<vector_count>")
1457 (match_operand:SME_ZA_BHIx24 2 "aligned_register_operand" "Uw<vector_count>")]
1458 SME_INT_TERNARY_SLICE))]
1459 "TARGET_STREAMING_SME2"
1460 "<optab><za32_long>\tza.s[%w0, 0:<za32_last_offset>, vgx<vector_count>], %1, %2"
1461 )
1463 (define_insn "*aarch64_sme_<optab><VNx4SI_ONLY:mode><SME_ZA_BHIx24:mode>_plus"
1464 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1465 (unspec:VNx4SI_ONLY
1466 [(reg:VNx4SI_ONLY ZA_REGNUM)
1467 (reg:DI SME_STATE_REGNUM)
1468 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1469 (match_operand:SI 1 "const_<za32_offset_range>_operand"))
1470 (match_operand:SME_ZA_BHIx24 2 "aligned_register_operand" "Uw<vector_count>")
1471 (match_operand:SME_ZA_BHIx24 3 "aligned_register_operand" "Uw<vector_count>")]
1472 SME_INT_TERNARY_SLICE))]
1473 "TARGET_STREAMING_SME2"
1474 {
1475 operands[4] = GEN_INT (INTVAL (operands[1]) + <za32_last_offset>);
1476 return "<optab><za32_long>\tza.s[%w0, %1:%4, vgx<vector_count>], %2, %3";
1477 }
1478 )
1480 (define_insn "@aarch64_sme_single_<optab><VNx4SI_ONLY:mode><SME_ZA_BHIx24:mode>"
1481 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1482 (unspec:VNx4SI_ONLY
1483 [(reg:VNx4SI_ONLY ZA_REGNUM)
1484 (reg:DI SME_STATE_REGNUM)
1485 (match_operand:SI 0 "register_operand" "Uci")
1486 (match_operand:SME_ZA_BHIx24 1 "register_operand" "w")
1487 (vec_duplicate:SME_ZA_BHIx24
1488 (match_operand:<SME_ZA_BHIx24:VSINGLE> 2 "register_operand" "x"))]
1489 SME_INT_TERNARY_SLICE))]
1490 "TARGET_STREAMING_SME2"
1491 "<optab><za32_long>\tza.s[%w0, 0:<za32_last_offset>, vgx<vector_count>], %1, %2.<SME_ZA_BHIx24:Vetype>"
1492 )
1494 (define_insn "*aarch64_sme_single_<optab><VNx4SI_ONLY:mode><SME_ZA_BHIx24:mode>_plus"
1495 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1496 (unspec:VNx4SI_ONLY
1497 [(reg:VNx4SI_ONLY ZA_REGNUM)
1498 (reg:DI SME_STATE_REGNUM)
1499 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1500 (match_operand:SI 1 "const_<za32_offset_range>_operand"))
1501 (match_operand:SME_ZA_BHIx24 2 "register_operand" "w")
1502 (vec_duplicate:SME_ZA_BHIx24
1503 (match_operand:<SME_ZA_BHIx24:VSINGLE> 3 "register_operand" "x"))]
1504 SME_INT_TERNARY_SLICE))]
1505 "TARGET_STREAMING_SME2"
1506 {
1507 operands[4] = GEN_INT (INTVAL (operands[1]) + <za32_last_offset>);
1508 return "<optab><za32_long>\tza.s[%w0, %1:%4, vgx<vector_count>], %2, %3.<SME_ZA_BHIx24:Vetype>";
1509 }
1510 )
1512 (define_insn "@aarch64_sme_lane_<optab><VNx4SI_ONLY:mode><SME_ZA_BHIx124:mode>"
1513 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1514 (unspec:VNx4SI_ONLY
1515 [(reg:VNx4SI_ONLY ZA_REGNUM)
1516 (reg:DI SME_STATE_REGNUM)
1517 (match_operand:SI 0 "register_operand" "Uci")
1518 (match_operand:SME_ZA_BHIx124 1 "<aligned_operand>" "<aligned_fpr>")
1519 (unspec:SME_ZA_BHIx124
1520 [(match_operand:<VSINGLE> 2 "register_operand" "x")
1521 (match_operand:SI 3 "const_int_operand")]
1522 UNSPEC_SVE_LANE_SELECT)]
1523 SME_INT_TERNARY_SLICE))]
1524 "TARGET_STREAMING_SME2"
1525 "<optab><za32_long>\tza.s[%w0, 0:<za32_last_offset><vg_modifier>], %1<z_suffix>, %2.<SME_ZA_BHIx124:Vetype>[%3]"
1526 )
1528 (define_insn "*aarch64_sme_lane_<optab><VNx4SI_ONLY:mode><SME_ZA_BHIx124:mode>"
1529 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1530 (unspec:VNx4SI_ONLY
1531 [(reg:VNx4SI_ONLY ZA_REGNUM)
1532 (reg:DI SME_STATE_REGNUM)
1533 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1534 (match_operand:SI 1 "const_<za32_offset_range>_operand"))
1535 (match_operand:SME_ZA_BHIx124 2 "<aligned_operand>" "<aligned_fpr>")
1536 (unspec:SME_ZA_BHIx124
1537 [(match_operand:<VSINGLE> 3 "register_operand" "x")
1538 (match_operand:SI 4 "const_int_operand")]
1539 UNSPEC_SVE_LANE_SELECT)]
1540 SME_INT_TERNARY_SLICE))]
1541 "TARGET_STREAMING_SME2"
1542 {
1543 operands[5] = GEN_INT (INTVAL (operands[1]) + <za32_last_offset>);
1544 return "<optab><za32_long>\tza.s[%w0, %1:%5<vg_modifier>], %2<z_suffix>, %3.<SME_ZA_BHIx124:Vetype>[%4]";
1545 }
1546 )
1548 (define_insn "@aarch64_sme_<optab><VNx2DI_ONLY:mode><VNx8HI_ONLY:mode>"
1549 [(set (reg:VNx2DI_ONLY ZA_REGNUM)
1550 (unspec:VNx2DI_ONLY
1551 [(reg:VNx2DI_ONLY ZA_REGNUM)
1552 (reg:DI SME_STATE_REGNUM)
1553 (match_operand:SI 0 "register_operand" "Uci")
1554 (match_operand:VNx8HI_ONLY 1 "register_operand" "w")
1555 (match_operand:VNx8HI_ONLY 2 "register_operand" "x")]
1556 SME_INT_TERNARY_SLICE))]
1557 "TARGET_STREAMING_SME2 && TARGET_SME_I16I64"
1558 "<optab>ll\tza.d[%w0, 0:3], %1.h, %2.h"
1559 )
1561 (define_insn "*aarch64_sme_<optab><VNx2DI_ONLY:mode><VNx8HI_ONLY:mode>_plus"
1562 [(set (reg:VNx2DI_ONLY ZA_REGNUM)
1563 (unspec:VNx2DI_ONLY
1564 [(reg:VNx2DI_ONLY ZA_REGNUM)
1565 (reg:DI SME_STATE_REGNUM)
1566 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1567 (match_operand:SI 1 "const_<za64_offset_range>_operand"))
1568 (match_operand:VNx8HI_ONLY 2 "register_operand" "w")
1569 (match_operand:VNx8HI_ONLY 3 "register_operand" "x")]
1570 SME_INT_TERNARY_SLICE))]
1571 "TARGET_STREAMING_SME2 && TARGET_SME_I16I64"
1572 {
1573 operands[4] = GEN_INT (INTVAL (operands[1]) + 3);
1574 return "<optab>ll\tza.d[%w0, %1:%4], %2.h, %3.h";
1575 }
1576 )
1578 (define_insn "@aarch64_sme_<optab><VNx2DI_ONLY:mode><SME_ZA_HIx24:mode>"
1579 [(set (reg:VNx2DI_ONLY ZA_REGNUM)
1580 (unspec:VNx2DI_ONLY
1581 [(reg:VNx2DI_ONLY ZA_REGNUM)
1582 (reg:DI SME_STATE_REGNUM)
1583 (match_operand:SI 0 "register_operand" "Uci")
1584 (match_operand:SME_ZA_HIx24 1 "aligned_register_operand" "Uw<vector_count>")
1585 (match_operand:SME_ZA_HIx24 2 "aligned_register_operand" "Uw<vector_count>")]
1586 SME_INT_TERNARY_SLICE))]
1587 "TARGET_STREAMING_SME2 && TARGET_SME_I16I64"
1588 "<optab>ll\tza.d[%w0, 0:3, vgx<vector_count>], %1, %2"
1589 )
1591 (define_insn "*aarch64_sme_<optab><VNx2DI_ONLY:mode><SME_ZA_HIx24:mode>_plus"
1592 [(set (reg:VNx2DI_ONLY ZA_REGNUM)
1593 (unspec:VNx2DI_ONLY
1594 [(reg:VNx2DI_ONLY ZA_REGNUM)
1595 (reg:DI SME_STATE_REGNUM)
1596 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1597 (match_operand:SI 1 "const_<za64_offset_range>_operand"))
1598 (match_operand:SME_ZA_HIx24 2 "aligned_register_operand" "Uw<vector_count>")
1599 (match_operand:SME_ZA_HIx24 3 "aligned_register_operand" "Uw<vector_count>")]
1600 SME_INT_TERNARY_SLICE))]
1601 "TARGET_STREAMING_SME2 && TARGET_SME_I16I64"
1602 {
1603 operands[4] = GEN_INT (INTVAL (operands[1]) + 3);
1604 return "<optab>ll\tza.d[%w0, %1:%4, vgx<vector_count>], %2, %3";
1605 }
1606 )
1608 (define_insn "@aarch64_sme_single_<optab><VNx2DI_ONLY:mode><SME_ZA_HIx24:mode>"
1609 [(set (reg:VNx2DI_ONLY ZA_REGNUM)
1610 (unspec:VNx2DI_ONLY
1611 [(reg:VNx2DI_ONLY ZA_REGNUM)
1612 (reg:DI SME_STATE_REGNUM)
1613 (match_operand:SI 0 "register_operand" "Uci")
1614 (match_operand:SME_ZA_HIx24 1 "register_operand" "w")
1615 (vec_duplicate:SME_ZA_HIx24
1616 (match_operand:<SME_ZA_HIx24:VSINGLE> 2 "register_operand" "x"))]
1617 SME_INT_TERNARY_SLICE))]
1618 "TARGET_STREAMING_SME2 && TARGET_SME_I16I64"
1619 "<optab>ll\tza.d[%w0, 0:3, vgx<vector_count>], %1, %2.h"
1620 )
1622 (define_insn "*aarch64_sme_single_<optab><VNx2DI_ONLY:mode><SME_ZA_HIx24:mode>_plus"
1623 [(set (reg:VNx2DI_ONLY ZA_REGNUM)
1624 (unspec:VNx2DI_ONLY
1625 [(reg:VNx2DI_ONLY ZA_REGNUM)
1626 (reg:DI SME_STATE_REGNUM)
1627 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1628 (match_operand:SI 1 "const_<za64_offset_range>_operand"))
1629 (match_operand:SME_ZA_HIx24 2 "register_operand" "w")
1630 (vec_duplicate:SME_ZA_HIx24
1631 (match_operand:<SME_ZA_HIx24:VSINGLE> 3 "register_operand" "x"))]
1632 SME_INT_TERNARY_SLICE))]
1633 "TARGET_STREAMING_SME2 && TARGET_SME_I16I64"
1634 {
1635 operands[4] = GEN_INT (INTVAL (operands[1]) + 3);
1636 return "<optab>ll\tza.d[%w0, %1:%4, vgx<vector_count>], %2, %3.h";
1637 }
1638 )
1640 (define_insn "@aarch64_sme_lane_<optab><VNx2DI_ONLY:mode><SME_ZA_HIx124:mode>"
1641 [(set (reg:VNx2DI_ONLY ZA_REGNUM)
1642 (unspec:VNx2DI_ONLY
1643 [(reg:VNx2DI_ONLY ZA_REGNUM)
1644 (reg:DI SME_STATE_REGNUM)
1645 (match_operand:SI 0 "register_operand" "Uci")
1646 (match_operand:SME_ZA_HIx124 1 "<aligned_operand>" "<aligned_fpr>")
1647 (unspec:SME_ZA_HIx124
1648 [(match_operand:<VSINGLE> 2 "register_operand" "x")
1649 (match_operand:SI 3 "const_int_operand")]
1650 UNSPEC_SVE_LANE_SELECT)]
1651 SME_INT_TERNARY_SLICE))]
1652 "TARGET_STREAMING_SME2 && TARGET_SME_I16I64"
1653 "<optab>ll\tza.d[%w0, 0:3<vg_modifier>], %1<z_suffix>, %2.h[%3]"
1654 )
1656 (define_insn "*aarch64_sme_lane_<optab><VNx2DI_ONLY:mode><SME_ZA_HIx124:mode>"
1657 [(set (reg:VNx2DI_ONLY ZA_REGNUM)
1658 (unspec:VNx2DI_ONLY
1659 [(reg:VNx2DI_ONLY ZA_REGNUM)
1660 (reg:DI SME_STATE_REGNUM)
1661 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1662 (match_operand:SI 1 "const_<za64_offset_range>_operand"))
1663 (match_operand:SME_ZA_HIx124 2 "<aligned_operand>" "<aligned_fpr>")
1664 (unspec:SME_ZA_HIx124
1665 [(match_operand:<VSINGLE> 3 "register_operand" "x")
1666 (match_operand:SI 4 "const_int_operand")]
1667 UNSPEC_SVE_LANE_SELECT)]
1668 SME_INT_TERNARY_SLICE))]
1669 "TARGET_STREAMING_SME2 && TARGET_SME_I16I64"
1670 {
1671 operands[5] = GEN_INT (INTVAL (operands[1]) + 3);
1672 return "<optab>ll\tza.d[%w0, %1:%5<vg_modifier>], %2<z_suffix>, %3.h[%4]";
1673 }
1674 )
1676 ;; -------------------------------------------------------------------------
1677 ;; ---- [INT] Sum of outer products
1678 ;; -------------------------------------------------------------------------
1679 ;; - BMOPA (SME2)
1680 ;; - BMOPS (SME2)
1681 ;; - SMOPA
1682 ;; - SMOPS
1683 ;; - SUMOPA
1684 ;; - SUMOPS
1685 ;; - UMOPA
1686 ;; - UMOPS
1687 ;; - USMOPA
1688 ;; - USMOPS
1689 ;; -------------------------------------------------------------------------
1691 (define_insn "@aarch64_sme_<optab><VNx4SI_ONLY:mode><VNx16QI_ONLY:mode>"
1692 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1693 (unspec:VNx4SI_ONLY
1694 [(reg:VNx4SI_ONLY ZA_REGNUM)
1695 (reg:DI SME_STATE_REGNUM)
1696 (match_operand:DI 0 "const_int_operand")
1697 (match_operand:<VNx4SI_ONLY:VPRED> 1 "register_operand" "Upl")
1698 (match_operand:<VNx4SI_ONLY:VPRED> 2 "register_operand" "Upl")
1699 (match_operand:VNx16QI_ONLY 3 "register_operand" "w")
1700 (match_operand:VNx16QI_ONLY 4 "register_operand" "w")]
1701 SME_INT_MOP))]
1702 "TARGET_STREAMING"
1703 "<optab>\tza%0.s, %1/m, %2/m, %3.b, %4.b"
1704 )
1706 (define_insn "@aarch64_sme_<optab><VNx2DI_ONLY:mode><VNx8HI_ONLY:mode>"
1707 [(set (reg:VNx2DI_ONLY ZA_REGNUM)
1708 (unspec:VNx2DI_ONLY
1709 [(reg:VNx2DI_ONLY ZA_REGNUM)
1710 (reg:DI SME_STATE_REGNUM)
1711 (match_operand:DI 0 "const_int_operand")
1712 (match_operand:<VNx2DI_ONLY:VPRED> 1 "register_operand" "Upl")
1713 (match_operand:<VNx2DI_ONLY:VPRED> 2 "register_operand" "Upl")
1714 (match_operand:VNx8HI_ONLY 3 "register_operand" "w")
1715 (match_operand:VNx8HI_ONLY 4 "register_operand" "w")]
1716 SME_INT_MOP))]
1717 "TARGET_STREAMING && TARGET_SME_I16I64"
1718 "<optab>\tza%0.d, %1/m, %2/m, %3.h, %4.h"
1719 )
1721 (define_insn "@aarch64_sme_<optab><VNx4SI_ONLY:mode><VNx8HI_ONLY:mode>"
1722 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1723 (unspec:VNx4SI_ONLY
1724 [(reg:VNx4SI_ONLY ZA_REGNUM)
1725 (reg:DI SME_STATE_REGNUM)
1726 (match_operand:DI 0 "const_int_operand")
1727 (match_operand:<VNx4SI_ONLY:VPRED> 1 "register_operand" "Upl")
1728 (match_operand:<VNx4SI_ONLY:VPRED> 2 "register_operand" "Upl")
1729 (match_operand:VNx8HI_ONLY 3 "register_operand" "w")
1730 (match_operand:VNx8HI_ONLY 4 "register_operand" "w")]
1731 SME2_INT_MOP))]
1732 "TARGET_STREAMING_SME2"
1733 "<optab>\tza%0.s, %1/m, %2/m, %3.h, %4.h"
1734 )
1736 (define_insn "@aarch64_sme_<optab><VNx4SI_ONLY:mode><VNx4SI_ONLY:mode>"
1737 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1738 (unspec:VNx4SI_ONLY
1739 [(reg:VNx4SI_ONLY ZA_REGNUM)
1740 (reg:DI SME_STATE_REGNUM)
1741 (match_operand:DI 0 "const_int_operand")
1742 (match_operand:<VNx4SI_ONLY:VPRED> 1 "register_operand" "Upl")
1743 (match_operand:<VNx4SI_ONLY:VPRED> 2 "register_operand" "Upl")
1744 (match_operand:VNx4SI_ONLY 3 "register_operand" "w")
1745 (match_operand:VNx4SI_ONLY 4 "register_operand" "w")]
1746 SME2_BMOP))]
1747 "TARGET_STREAMING_SME2"
1748 "<optab>\tza%0.s, %1/m, %2/m, %3.s, %4.s"
1749 )
1751 ;; -------------------------------------------------------------------------
1752 ;; ---- [FP] Dot product
1753 ;; -------------------------------------------------------------------------
1754 ;; Includes:
1755 ;; - BFDOT (SME2)
1756 ;; - FDOT (SME2)
1757 ;; -------------------------------------------------------------------------
1759 (define_insn "@aarch64_sme_<optab><VNx4SI_ONLY:mode><SME_ZA_HFx24:mode>"
1760 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1761 (unspec:VNx4SI_ONLY
1762 [(reg:VNx4SI_ONLY ZA_REGNUM)
1763 (reg:DI SME_STATE_REGNUM)
1764 (match_operand:SI 0 "register_operand" "Uci")
1765 (match_operand:SME_ZA_HFx24 1 "aligned_register_operand" "Uw<vector_count>")
1766 (match_operand:SME_ZA_HFx24 2 "aligned_register_operand" "Uw<vector_count>")]
1767 SME_FP_DOTPROD))]
1768 "TARGET_STREAMING_SME2"
1769 "<b><optab>\tza.s[%w0, 0, vgx<vector_count>], %1, %2"
1770 )
1772 (define_insn "*aarch64_sme_<optab><VNx4SI_ONLY:mode><SME_ZA_HFx24:mode>_plus"
1773 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1774 (unspec:VNx4SI_ONLY
1775 [(reg:VNx4SI_ONLY ZA_REGNUM)
1776 (reg:DI SME_STATE_REGNUM)
1777 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1778 (match_operand:SI 1 "const_0_to_7_operand"))
1779 (match_operand:SME_ZA_HFx24 2 "aligned_register_operand" "Uw<vector_count>")
1780 (match_operand:SME_ZA_HFx24 3 "aligned_register_operand" "Uw<vector_count>")]
1781 SME_FP_DOTPROD))]
1782 "TARGET_STREAMING_SME2"
1783 "<b><optab>\tza.s[%w0, %1, vgx<vector_count>], %2, %3"
1784 )
1786 (define_insn "@aarch64_sme_single_<optab><VNx4SI_ONLY:mode><SME_ZA_HFx24:mode>"
1787 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1788 (unspec:VNx4SI_ONLY
1789 [(reg:VNx4SI_ONLY ZA_REGNUM)
1790 (reg:DI SME_STATE_REGNUM)
1791 (match_operand:SI 0 "register_operand" "Uci")
1792 (match_operand:SME_ZA_HFx24 1 "register_operand" "w")
1793 (vec_duplicate:SME_ZA_HFx24
1794 (match_operand:<VSINGLE> 2 "register_operand" "x"))]
1795 SME_FP_DOTPROD))]
1796 "TARGET_STREAMING_SME2"
1797 "<b><optab>\tza.s[%w0, 0, vgx<vector_count>], %1, %2.h"
1798 )
1800 (define_insn "*aarch64_sme_single_<optab><VNx4SI_ONLY:mode><SME_ZA_HFx24:mode>_plus"
1801 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1802 (unspec:VNx4SI_ONLY
1803 [(reg:VNx4SI_ONLY ZA_REGNUM)
1804 (reg:DI SME_STATE_REGNUM)
1805 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1806 (match_operand:SI 1 "const_0_to_7_operand"))
1807 (match_operand:SME_ZA_HFx24 2 "register_operand" "w")
1808 (vec_duplicate:SME_ZA_HFx24
1809 (match_operand:<VSINGLE> 3 "register_operand" "x"))]
1810 SME_FP_DOTPROD))]
1811 "TARGET_STREAMING_SME2"
1812 "<b><optab>\tza.s[%w0, %1, vgx<vector_count>], %2, %3.h"
1813 )
1815 (define_insn "@aarch64_sme_lane_<optab><VNx4SI_ONLY:mode><SME_ZA_HFx24:mode>"
1816 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1817 (unspec:VNx4SI_ONLY
1818 [(reg:VNx4SI_ONLY ZA_REGNUM)
1819 (reg:DI SME_STATE_REGNUM)
1820 (match_operand:SI 0 "register_operand" "Uci")
1821 (match_operand:SME_ZA_HFx24 1 "aligned_register_operand" "Uw<vector_count>")
1822 (unspec:SME_ZA_HFx24
1823 [(match_operand:<VSINGLE> 2 "register_operand" "x")
1824 (match_operand:SI 3 "const_int_operand")]
1825 UNSPEC_SVE_LANE_SELECT)]
1826 SME_FP_DOTPROD_LANE))]
1827 "TARGET_STREAMING_SME2"
1828 "<b><optab>\tza.s[%w0, 0, vgx<vector_count>], %1, %2.h[%3]"
1829 )
1831 (define_insn "*aarch64_sme_lane_<optab><VNx4SI_ONLY:mode><SME_ZA_HFx24:mode>_plus"
1832 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1833 (unspec:VNx4SI_ONLY
1834 [(reg:VNx4SI_ONLY ZA_REGNUM)
1835 (reg:DI SME_STATE_REGNUM)
1836 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1837 (match_operand:SI 1 "const_0_to_7_operand"))
1838 (match_operand:SME_ZA_HFx24 2 "aligned_register_operand" "Uw<vector_count>")
1839 (unspec:SME_ZA_HFx24
1840 [(match_operand:<VSINGLE> 3 "register_operand" "x")
1841 (match_operand:SI 4 "const_int_operand")]
1842 UNSPEC_SVE_LANE_SELECT)]
1843 SME_FP_DOTPROD_LANE))]
1844 "TARGET_STREAMING_SME2"
1845 "<b><optab>\tza.s[%w0, %1, vgx<vector_count>], %2, %3.h[%4]"
1846 )
1848 ;; -------------------------------------------------------------------------
1849 ;; ---- [FP] Ternary arithmetic on ZA slice
1850 ;; -------------------------------------------------------------------------
1851 ;; Includes:
1852 ;; - BFMLA (SME_B16B16)
1853 ;; - BFMLS (SME_B16B16)
1854 ;; - FMLA (SME2)
1855 ;; - FMLS (SME2)
1856 ;; -------------------------------------------------------------------------
1858 (define_insn "@aarch64_sme_<optab><mode><mode>"
1859 [(set (reg:SME_ZA_HSDFx24 ZA_REGNUM)
1860 (unspec:SME_ZA_HSDFx24
1861 [(reg:SME_ZA_HSDFx24 ZA_REGNUM)
1862 (reg:DI SME_STATE_REGNUM)
1863 (match_operand:SI 0 "register_operand" "Uci")
1864 (match_operand:SME_ZA_HSDFx24 1 "aligned_register_operand" "Uw<vector_count>")
1865 (match_operand:SME_ZA_HSDFx24 2 "aligned_register_operand" "Uw<vector_count>")]
1866 SME_FP_TERNARY_SLICE))]
1867 "TARGET_STREAMING_SME2"
1868 "<b><optab>\tza.<Vetype>[%w0, 0, vgx<vector_count>], %1, %2"
1869 )
1871 (define_insn "*aarch64_sme_<optab><mode><mode>_plus"
1872 [(set (reg:SME_ZA_HSDFx24 ZA_REGNUM)
1873 (unspec:SME_ZA_HSDFx24
1874 [(reg:SME_ZA_HSDFx24 ZA_REGNUM)
1875 (reg:DI SME_STATE_REGNUM)
1876 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1877 (match_operand:SI 1 "const_0_to_7_operand"))
1878 (match_operand:SME_ZA_HSDFx24 2 "aligned_register_operand" "Uw<vector_count>")
1879 (match_operand:SME_ZA_HSDFx24 3 "aligned_register_operand" "Uw<vector_count>")]
1880 SME_FP_TERNARY_SLICE))]
1881 "TARGET_STREAMING_SME2"
1882 "<b><optab>\tza.<Vetype>[%w0, %1, vgx<vector_count>], %2, %3"
1883 )
1885 (define_insn "@aarch64_sme_single_<optab><mode><mode>"
1886 [(set (reg:SME_ZA_HSDFx24 ZA_REGNUM)
1887 (unspec:SME_ZA_HSDFx24
1888 [(reg:SME_ZA_HSDFx24 ZA_REGNUM)
1889 (reg:DI SME_STATE_REGNUM)
1890 (match_operand:SI 0 "register_operand" "Uci")
1891 (match_operand:SME_ZA_HSDFx24 1 "register_operand" "w")
1892 (vec_duplicate:SME_ZA_HSDFx24
1893 (match_operand:<SME_ZA_HSDFx24:VSINGLE> 2 "register_operand" "x"))]
1894 SME_FP_TERNARY_SLICE))]
1895 "TARGET_STREAMING_SME2"
1896 "<b><optab>\tza.<Vetype>[%w0, 0, vgx<vector_count>], %1, %2.<Vetype>"
1897 )
1899 (define_insn "*aarch64_sme_single_<optab><mode><mode>_plus"
1900 [(set (reg:SME_ZA_HSDFx24 ZA_REGNUM)
1901 (unspec:SME_ZA_HSDFx24
1902 [(reg:SME_ZA_HSDFx24 ZA_REGNUM)
1903 (reg:DI SME_STATE_REGNUM)
1904 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1905 (match_operand:SI 1 "const_0_to_7_operand"))
1906 (match_operand:SME_ZA_HSDFx24 2 "register_operand" "w")
1907 (vec_duplicate:SME_ZA_HSDFx24
1908 (match_operand:<SME_ZA_HSDFx24:VSINGLE> 3 "register_operand" "x"))]
1909 SME_FP_TERNARY_SLICE))]
1910 "TARGET_STREAMING_SME2"
1911 "<b><optab>\tza.<Vetype>[%w0, %1, vgx<vector_count>], %2, %3.<Vetype>"
1912 )
1914 (define_insn "@aarch64_sme_lane_<optab><mode><mode>"
1915 [(set (reg:SME_ZA_HSDFx24 ZA_REGNUM)
1916 (unspec:SME_ZA_HSDFx24
1917 [(reg:SME_ZA_HSDFx24 ZA_REGNUM)
1918 (reg:DI SME_STATE_REGNUM)
1919 (match_operand:SI 0 "register_operand" "Uci")
1920 (match_operand:SME_ZA_HSDFx24 1 "aligned_register_operand" "Uw<vector_count>")
1921 (unspec:SME_ZA_HSDFx24
1922 [(match_operand:<SME_ZA_HSDFx24:VSINGLE> 2 "register_operand" "x")
1923 (match_operand:SI 3 "const_int_operand")]
1924 UNSPEC_SVE_LANE_SELECT)]
1925 SME_FP_TERNARY_SLICE))]
1926 "TARGET_STREAMING_SME2"
1927 "<b><optab>\tza.<Vetype>[%w0, 0, vgx<vector_count>], %1, %2.<Vetype>[%3]"
1928 )
1930 (define_insn "*aarch64_sme_lane_<optab><mode><mode>"
1931 [(set (reg:SME_ZA_HSDFx24 ZA_REGNUM)
1932 (unspec:SME_ZA_HSDFx24
1933 [(reg:SME_ZA_HSDFx24 ZA_REGNUM)
1934 (reg:DI SME_STATE_REGNUM)
1935 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1936 (match_operand:SI 1 "const_0_to_7_operand"))
1937 (match_operand:SME_ZA_HSDFx24 2 "aligned_register_operand" "Uw<vector_count>")
1938 (unspec:SME_ZA_HSDFx24
1939 [(match_operand:<SME_ZA_HSDFx24:VSINGLE> 3 "register_operand" "x")
1940 (match_operand:SI 4 "const_int_operand")]
1941 UNSPEC_SVE_LANE_SELECT)]
1942 SME_FP_TERNARY_SLICE))]
1943 "TARGET_STREAMING_SME2"
1944 "<b><optab>\tza.<Vetype>[%w0, %1, vgx<vector_count>], %2, %3.<Vetype>[%4]"
1945 )
1947 ;; -------------------------------------------------------------------------
1948 ;; ---- [FP] Ternary widening arithmetic on ZA slice
1949 ;; -------------------------------------------------------------------------
1950 ;; Includes:
1951 ;; - BFMLAL (SME2)
1952 ;; - BFMLSL (SME2)
1953 ;; - FMLAL (SME2)
1954 ;; - FMLSL (SME2)
1955 ;; -------------------------------------------------------------------------
1957 (define_insn "@aarch64_sme_<optab><VNx4SI_ONLY:mode><SVE_FULL_HF:mode>"
1958 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1959 (unspec:VNx4SI_ONLY
1960 [(reg:VNx4SI_ONLY ZA_REGNUM)
1961 (reg:DI SME_STATE_REGNUM)
1962 (match_operand:SI 0 "register_operand" "Uci")
1963 (match_operand:SVE_FULL_HF 1 "register_operand" "w")
1964 (match_operand:SVE_FULL_HF 2 "register_operand" "x")]
1965 SME_FP_TERNARY_SLICE))]
1966 "TARGET_STREAMING_SME2"
1967 "<b><optab>l\tza.s[%w0, 0:1], %1.h, %2.h"
1968 )
1970 (define_insn "*aarch64_sme_<optab><VNx4SI_ONLY:mode><SVE_FULL_HF:mode>_plus"
1971 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1972 (unspec:VNx4SI_ONLY
1973 [(reg:VNx4SI_ONLY ZA_REGNUM)
1974 (reg:DI SME_STATE_REGNUM)
1975 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
1976 (match_operand:SI 1 "const_<za32_offset_range>_operand"))
1977 (match_operand:SVE_FULL_HF 2 "register_operand" "w")
1978 (match_operand:SVE_FULL_HF 3 "register_operand" "x")]
1979 SME_FP_TERNARY_SLICE))]
1980 "TARGET_STREAMING_SME2"
1981 {
1982 operands[4] = GEN_INT (INTVAL (operands[1]) + 1);
1983 return "<b><optab>l\tza.s[%w0, %1:%4], %2.h, %3.h";
1984 }
1985 )
1987 (define_insn "@aarch64_sme_<optab><VNx4SI_ONLY:mode><SME_ZA_HFx24:mode>"
1988 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
1989 (unspec:VNx4SI_ONLY
1990 [(reg:VNx4SI_ONLY ZA_REGNUM)
1991 (reg:DI SME_STATE_REGNUM)
1992 (match_operand:SI 0 "register_operand" "Uci")
1993 (match_operand:SME_ZA_HFx24 1 "aligned_register_operand" "Uw<vector_count>")
1994 (match_operand:SME_ZA_HFx24 2 "aligned_register_operand" "Uw<vector_count>")]
1995 SME_FP_TERNARY_SLICE))]
1996 "TARGET_STREAMING_SME2"
1997 "<b><optab>l\tza.s[%w0, 0:1, vgx<vector_count>], %1, %2"
1998 )
2000 (define_insn "*aarch64_sme_<optab><VNx4SI_ONLY:mode><SME_ZA_HFx24:mode>_plus"
2001 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
2002 (unspec:VNx4SI_ONLY
2003 [(reg:VNx4SI_ONLY ZA_REGNUM)
2004 (reg:DI SME_STATE_REGNUM)
2005 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
2006 (match_operand:SI 1 "const_<za32_offset_range>_operand"))
2007 (match_operand:SME_ZA_HFx24 2 "aligned_register_operand" "Uw<vector_count>")
2008 (match_operand:SME_ZA_HFx24 3 "aligned_register_operand" "Uw<vector_count>")]
2009 SME_FP_TERNARY_SLICE))]
2010 "TARGET_STREAMING_SME2"
2011 {
2012 operands[4] = GEN_INT (INTVAL (operands[1]) + 1);
2013 return "<b><optab>l\tza.s[%w0, %1:%4, vgx<vector_count>], %2, %3";
2014 }
2015 )
2017 (define_insn "@aarch64_sme_single_<optab><VNx4SI_ONLY:mode><SME_ZA_HFx24:mode>"
2018 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
2019 (unspec:VNx4SI_ONLY
2020 [(reg:VNx4SI_ONLY ZA_REGNUM)
2021 (reg:DI SME_STATE_REGNUM)
2022 (match_operand:SI 0 "register_operand" "Uci")
2023 (match_operand:SME_ZA_HFx24 1 "register_operand" "w")
2024 (vec_duplicate:SME_ZA_HFx24
2025 (match_operand:<SME_ZA_HFx24:VSINGLE> 2 "register_operand" "x"))]
2026 SME_FP_TERNARY_SLICE))]
2027 "TARGET_STREAMING_SME2"
2028 "<b><optab>l\tza.s[%w0, 0:1, vgx<vector_count>], %1, %2.h"
2029 )
2031 (define_insn "*aarch64_sme_single_<optab><VNx4SI_ONLY:mode><SME_ZA_HFx24:mode>_plus"
2032 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
2033 (unspec:VNx4SI_ONLY
2034 [(reg:VNx4SI_ONLY ZA_REGNUM)
2035 (reg:DI SME_STATE_REGNUM)
2036 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
2037 (match_operand:SI 1 "const_<za32_offset_range>_operand"))
2038 (match_operand:SME_ZA_HFx24 2 "register_operand" "w")
2039 (vec_duplicate:SME_ZA_HFx24
2040 (match_operand:<SME_ZA_HFx24:VSINGLE> 3 "register_operand" "x"))]
2041 SME_FP_TERNARY_SLICE))]
2042 "TARGET_STREAMING_SME2"
2043 {
2044 operands[4] = GEN_INT (INTVAL (operands[1]) + 1);
2045 return "<b><optab>l\tza.s[%w0, %1:%4, vgx<vector_count>], %2, %3.h";
2046 }
2047 )
2049 (define_insn "@aarch64_sme_lane_<optab><VNx4SI_ONLY:mode><SME_ZA_HFx124:mode>"
2050 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
2051 (unspec:VNx4SI_ONLY
2052 [(reg:VNx4SI_ONLY ZA_REGNUM)
2053 (reg:DI SME_STATE_REGNUM)
2054 (match_operand:SI 0 "register_operand" "Uci")
2055 (match_operand:SME_ZA_HFx124 1 "<aligned_operand>" "<aligned_fpr>")
2056 (unspec:SME_ZA_HFx124
2057 [(match_operand:<VSINGLE> 2 "register_operand" "x")
2058 (match_operand:SI 3 "const_int_operand")]
2059 UNSPEC_SVE_LANE_SELECT)]
2060 SME_FP_TERNARY_SLICE))]
2061 "TARGET_STREAMING_SME2"
2062 "<b><optab>l\tza.s[%w0, 0:1<vg_modifier>], %1<z_suffix>, %2.h[%3]"
2063 )
2065 (define_insn "*aarch64_sme_lane_<optab><VNx4SI_ONLY:mode><SME_ZA_HFx124:mode>"
2066 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
2067 (unspec:VNx4SI_ONLY
2068 [(reg:VNx4SI_ONLY ZA_REGNUM)
2069 (reg:DI SME_STATE_REGNUM)
2070 (plus:SI (match_operand:SI 0 "register_operand" "Uci")
2071 (match_operand:SI 1 "const_<za32_offset_range>_operand"))
2072 (match_operand:SME_ZA_HFx124 2 "<aligned_operand>" "<aligned_fpr>")
2073 (unspec:SME_ZA_HFx124
2074 [(match_operand:<VSINGLE> 3 "register_operand" "x")
2075 (match_operand:SI 4 "const_int_operand")]
2076 UNSPEC_SVE_LANE_SELECT)]
2077 SME_FP_TERNARY_SLICE))]
2078 "TARGET_STREAMING_SME2"
2079 {
2080 operands[5] = GEN_INT (INTVAL (operands[1]) + 1);
2081 return "<b><optab>l\tza.s[%w0, %1:%5<vg_modifier>], %2<z_suffix>, %3.h[%4]";
2082 }
2083 )
2085 ;; -------------------------------------------------------------------------
2086 ;; ---- [FP] Sum of outer products
2087 ;; -------------------------------------------------------------------------
2088 ;; Includes:
2089 ;; - BFMOPA (SME_B16B16)
2090 ;; - BFMOPS (SME_B16B16)
2091 ;; - FMOPA
2092 ;; - FMOPS
2093 ;; -------------------------------------------------------------------------
2095 (define_insn "@aarch64_sme_<optab><mode><mode>"
2096 [(set (reg:SME_MOP_HSDF ZA_REGNUM)
2097 (unspec:SME_MOP_HSDF
2098 [(reg:SME_MOP_HSDF ZA_REGNUM)
2099 (reg:DI SME_STATE_REGNUM)
2100 (match_operand:DI 0 "const_int_operand")
2101 (match_operand:<VPRED> 1 "register_operand" "Upl")
2102 (match_operand:<VPRED> 2 "register_operand" "Upl")
2103 (match_operand:SME_MOP_HSDF 3 "register_operand" "w")
2104 (match_operand:SME_MOP_HSDF 4 "register_operand" "w")]
2105 SME_FP_MOP))]
2106 "TARGET_STREAMING"
2107 "<b><optab>\tza%0.<Vetype>, %1/m, %2/m, %3.<Vetype>, %4.<Vetype>"
2108 )
2110 (define_insn "@aarch64_sme_<optab><VNx4SI_ONLY:mode><SVE_FULL_HF:mode>"
2111 [(set (reg:VNx4SI_ONLY ZA_REGNUM)
2112 (unspec:VNx4SI_ONLY
2113 [(reg:VNx4SI_ONLY ZA_REGNUM)
2114 (reg:DI SME_STATE_REGNUM)
2115 (match_operand:DI 0 "const_int_operand")
2116 (match_operand:<VNx4SI_ONLY:VPRED> 1 "register_operand" "Upl")
2117 (match_operand:<VNx4SI_ONLY:VPRED> 2 "register_operand" "Upl")
2118 (match_operand:SVE_FULL_HF 3 "register_operand" "w")
2119 (match_operand:SVE_FULL_HF 4 "register_operand" "w")]
2120 SME_FP_MOP))]
2121 "TARGET_STREAMING"
2122 "<b><optab>\tza%0.<VNx4SI_ONLY:Vetype>, %1/m, %2/m, %3.<SVE_FULL_HF:Vetype>, %4.<SVE_FULL_HF:Vetype>"
2123 )
2125 ;; =========================================================================
2126 ;; == Table lookup
2127 ;; =========================================================================
2129 ;; -------------------------------------------------------------------------
2130 ;; ---- Table lookup
2131 ;; -------------------------------------------------------------------------
2132 ;; Includes:
2133 ;; - LUTI2 (SME2)
2134 ;; - LUTI4 (SME2)
2135 ;; -------------------------------------------------------------------------
2137 (define_c_enum "unspec" [
2138 UNSPEC_SME_LUTI
2139 ])
2141 (define_insn "@aarch64_sme_lut<LUTI_BITS><mode>"
2142 [(set (match_operand:SVE_FULL_BHS 0 "register_operand" "=w")
2143 (unspec:SVE_FULL_BHS
2144 [(reg:V8DI ZT0_REGNUM)
2145 (reg:DI SME_STATE_REGNUM)
2146 (match_operand:VNx16QI 1 "register_operand" "w")
2147 (match_operand:DI 2 "const_int_operand")
2148 (const_int LUTI_BITS)]
2149 UNSPEC_SME_LUTI))]
2150 "TARGET_STREAMING_SME2"
2151 "luti<LUTI_BITS>\t%0.<Vetype>, zt0, %1[%2]"
2152 )
2154 (define_insn "@aarch64_sme_lut<LUTI_BITS><mode>"
2155 [(set (match_operand:SVE_BHSx24 0 "aligned_register_operand" "=Uw<vector_count>")
2156 (unspec:SVE_BHSx24
2157 [(reg:V8DI ZT0_REGNUM)
2158 (reg:DI SME_STATE_REGNUM)
2159 (match_operand:VNx16QI 1 "register_operand" "w")
2160 (match_operand:DI 2 "const_int_operand")
2161 (const_int LUTI_BITS)]
2162 UNSPEC_SME_LUTI))]
2163 "TARGET_STREAMING_SME2
2164 && !(<LUTI_BITS> == 4 && <vector_count> == 4 && <elem_bits> == 8)"
2165 "luti<LUTI_BITS>\t%0, zt0, %1[%2]"
2166 )