| blob | c345c1e50bbbb7f34fa4d6bc5ab46a28b583536e |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
3 ; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod -global-isel -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
5 ; CHECK-GI: warning: Instruction selection used fallback path for test_udot_v5i8
6 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v5i8_nomla
7 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8
8 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8_double
9 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8_double_nomla
10 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v25i8
11 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v25i8_nomla
12 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8
13 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8_double
14 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8_double_nomla
15 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v33i8
16 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v33i8_nomla
17 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8
18 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8_double
19 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8_double_nomla
21 declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
22 declare i32 @llvm.vector.reduce.add.v5i32(<5 x i32>)
23 declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
24 declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
25 declare i32 @llvm.vector.reduce.add.v24i32(<24 x i32>)
26 declare i32 @llvm.vector.reduce.add.v25i32(<25 x i32>)
27 declare i32 @llvm.vector.reduce.add.v32i32(<32 x i32>)
28 declare i32 @llvm.vector.reduce.add.v33i32(<33 x i32>)
29 declare i32 @llvm.vector.reduce.add.v48i32(<48 x i32>)
30 declare i32 @llvm.vector.reduce.add.v64i32(<64 x i32>)
32 define i32 @test_udot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
33 ; CHECK-SD-LABEL: test_udot_v4i8:
34 ; CHECK-SD: // %bb.0: // %entry
35 ; CHECK-SD-NEXT: ldr s0, [x0]
36 ; CHECK-SD-NEXT: ldr s1, [x1]
37 ; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0
38 ; CHECK-SD-NEXT: ushll v1.8h, v1.8b, #0
39 ; CHECK-SD-NEXT: umull v0.4s, v1.4h, v0.4h
40 ; CHECK-SD-NEXT: addv s0, v0.4s
41 ; CHECK-SD-NEXT: fmov w8, s0
42 ; CHECK-SD-NEXT: add w0, w8, w2
43 ; CHECK-SD-NEXT: ret
44 ;
45 ; CHECK-GI-LABEL: test_udot_v4i8:
46 ; CHECK-GI: // %bb.0: // %entry
47 ; CHECK-GI-NEXT: ldr w8, [x0]
48 ; CHECK-GI-NEXT: ldr w9, [x1]
49 ; CHECK-GI-NEXT: fmov s0, w8
50 ; CHECK-GI-NEXT: fmov s2, w9
51 ; CHECK-GI-NEXT: uxtb w8, w8
52 ; CHECK-GI-NEXT: uxtb w9, w9
53 ; CHECK-GI-NEXT: mov b1, v0.b[1]
54 ; CHECK-GI-NEXT: mov b3, v0.b[2]
55 ; CHECK-GI-NEXT: mov b5, v2.b[2]
56 ; CHECK-GI-NEXT: mov b4, v0.b[3]
57 ; CHECK-GI-NEXT: mov b0, v2.b[1]
58 ; CHECK-GI-NEXT: mov b6, v2.b[3]
59 ; CHECK-GI-NEXT: fmov s2, w9
60 ; CHECK-GI-NEXT: fmov w10, s1
61 ; CHECK-GI-NEXT: fmov w11, s3
62 ; CHECK-GI-NEXT: fmov s1, w8
63 ; CHECK-GI-NEXT: fmov w13, s5
64 ; CHECK-GI-NEXT: fmov w8, s4
65 ; CHECK-GI-NEXT: fmov w12, s0
66 ; CHECK-GI-NEXT: uxtb w10, w10
67 ; CHECK-GI-NEXT: uxtb w11, w11
68 ; CHECK-GI-NEXT: uxtb w13, w13
69 ; CHECK-GI-NEXT: uxtb w8, w8
70 ; CHECK-GI-NEXT: uxtb w12, w12
71 ; CHECK-GI-NEXT: mov v1.h[1], w10
72 ; CHECK-GI-NEXT: fmov w10, s6
73 ; CHECK-GI-NEXT: fmov s0, w11
74 ; CHECK-GI-NEXT: fmov s3, w13
75 ; CHECK-GI-NEXT: mov v2.h[1], w12
76 ; CHECK-GI-NEXT: uxtb w10, w10
77 ; CHECK-GI-NEXT: mov v0.h[1], w8
78 ; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0
79 ; CHECK-GI-NEXT: mov v3.h[1], w10
80 ; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0
81 ; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
82 ; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0
83 ; CHECK-GI-NEXT: mov v1.d[1], v0.d[0]
84 ; CHECK-GI-NEXT: mov v2.d[1], v3.d[0]
85 ; CHECK-GI-NEXT: mul v0.4s, v2.4s, v1.4s
86 ; CHECK-GI-NEXT: addv s0, v0.4s
87 ; CHECK-GI-NEXT: fmov w8, s0
88 ; CHECK-GI-NEXT: add w0, w8, w2
89 ; CHECK-GI-NEXT: ret
90 entry:
91 %0 = load <4 x i8>, ptr %a
92 %1 = zext <4 x i8> %0 to <4 x i32>
93 %2 = load <4 x i8>, ptr %b
94 %3 = zext <4 x i8> %2 to <4 x i32>
95 %4 = mul nuw nsw <4 x i32> %3, %1
96 %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)
97 %op.extra = add i32 %5, %sum
98 ret i32 %op.extra
99 }
101 define i32 @test_udot_v4i8_nomla(ptr nocapture readonly %a1) {
102 ; CHECK-SD-LABEL: test_udot_v4i8_nomla:
103 ; CHECK-SD: // %bb.0: // %entry
104 ; CHECK-SD-NEXT: ldr s0, [x0]
105 ; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0
106 ; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
107 ; CHECK-SD-NEXT: addv s0, v0.4s
108 ; CHECK-SD-NEXT: fmov w0, s0
109 ; CHECK-SD-NEXT: ret
110 ;
111 ; CHECK-GI-LABEL: test_udot_v4i8_nomla:
112 ; CHECK-GI: // %bb.0: // %entry
113 ; CHECK-GI-NEXT: ldr w8, [x0]
114 ; CHECK-GI-NEXT: fmov s0, w8
115 ; CHECK-GI-NEXT: mov b1, v0.b[1]
116 ; CHECK-GI-NEXT: mov v2.b[0], v0.b[0]
117 ; CHECK-GI-NEXT: mov b3, v0.b[2]
118 ; CHECK-GI-NEXT: mov b0, v0.b[3]
119 ; CHECK-GI-NEXT: mov v2.b[1], v1.b[0]
120 ; CHECK-GI-NEXT: mov v2.b[2], v3.b[0]
121 ; CHECK-GI-NEXT: mov v2.b[3], v0.b[0]
122 ; CHECK-GI-NEXT: ushll v0.8h, v2.8b, #0
123 ; CHECK-GI-NEXT: uaddlv s0, v0.4h
124 ; CHECK-GI-NEXT: fmov w8, s0
125 ; CHECK-GI-NEXT: and w0, w8, #0xffff
126 ; CHECK-GI-NEXT: ret
127 entry:
128 %0 = load <4 x i8>, ptr %a1
129 %1 = zext <4 x i8> %0 to <4 x i32>
130 %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)
131 ret i32 %2
132 }
133 define i32 @test_sdot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
134 ; CHECK-SD-LABEL: test_sdot_v4i8:
135 ; CHECK-SD: // %bb.0: // %entry
136 ; CHECK-SD-NEXT: ldr s0, [x0]
137 ; CHECK-SD-NEXT: ldr s1, [x1]
138 ; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0
139 ; CHECK-SD-NEXT: sshll v1.8h, v1.8b, #0
140 ; CHECK-SD-NEXT: smull v0.4s, v1.4h, v0.4h
141 ; CHECK-SD-NEXT: addv s0, v0.4s
142 ; CHECK-SD-NEXT: fmov w8, s0
143 ; CHECK-SD-NEXT: add w0, w8, w2
144 ; CHECK-SD-NEXT: ret
145 ;
146 ; CHECK-GI-LABEL: test_sdot_v4i8:
147 ; CHECK-GI: // %bb.0: // %entry
148 ; CHECK-GI-NEXT: ldr w8, [x0]
149 ; CHECK-GI-NEXT: ldr w9, [x1]
150 ; CHECK-GI-NEXT: fmov s0, w8
151 ; CHECK-GI-NEXT: fmov s2, w9
152 ; CHECK-GI-NEXT: sxtb w8, w8
153 ; CHECK-GI-NEXT: sxtb w9, w9
154 ; CHECK-GI-NEXT: mov b1, v0.b[1]
155 ; CHECK-GI-NEXT: mov b3, v0.b[2]
156 ; CHECK-GI-NEXT: mov b5, v2.b[2]
157 ; CHECK-GI-NEXT: mov b4, v0.b[3]
158 ; CHECK-GI-NEXT: mov b0, v2.b[1]
159 ; CHECK-GI-NEXT: mov b6, v2.b[3]
160 ; CHECK-GI-NEXT: fmov s2, w9
161 ; CHECK-GI-NEXT: fmov w10, s1
162 ; CHECK-GI-NEXT: fmov w11, s3
163 ; CHECK-GI-NEXT: fmov s1, w8
164 ; CHECK-GI-NEXT: fmov w13, s5
165 ; CHECK-GI-NEXT: fmov w8, s4
166 ; CHECK-GI-NEXT: fmov w12, s0
167 ; CHECK-GI-NEXT: sxtb w10, w10
168 ; CHECK-GI-NEXT: sxtb w11, w11
169 ; CHECK-GI-NEXT: sxtb w13, w13
170 ; CHECK-GI-NEXT: sxtb w8, w8
171 ; CHECK-GI-NEXT: sxtb w12, w12
172 ; CHECK-GI-NEXT: mov v1.h[1], w10
173 ; CHECK-GI-NEXT: fmov w10, s6
174 ; CHECK-GI-NEXT: fmov s0, w11
175 ; CHECK-GI-NEXT: fmov s3, w13
176 ; CHECK-GI-NEXT: mov v2.h[1], w12
177 ; CHECK-GI-NEXT: sxtb w10, w10
178 ; CHECK-GI-NEXT: mov v0.h[1], w8
179 ; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0
180 ; CHECK-GI-NEXT: mov v3.h[1], w10
181 ; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0
182 ; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0
183 ; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #0
184 ; CHECK-GI-NEXT: mov v1.d[1], v0.d[0]
185 ; CHECK-GI-NEXT: mov v2.d[1], v3.d[0]
186 ; CHECK-GI-NEXT: mul v0.4s, v2.4s, v1.4s
187 ; CHECK-GI-NEXT: addv s0, v0.4s
188 ; CHECK-GI-NEXT: fmov w8, s0
189 ; CHECK-GI-NEXT: add w0, w8, w2
190 ; CHECK-GI-NEXT: ret
191 entry:
192 %0 = load <4 x i8>, ptr %a
193 %1 = sext <4 x i8> %0 to <4 x i32>
194 %2 = load <4 x i8>, ptr %b
195 %3 = sext <4 x i8> %2 to <4 x i32>
196 %4 = mul nsw <4 x i32> %3, %1
197 %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)
198 %op.extra = add nsw i32 %5, %sum
199 ret i32 %op.extra
200 }
202 define i32 @test_sdot_v4i8_double(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
203 ; CHECK-SD-LABEL: test_sdot_v4i8_double:
204 ; CHECK-SD: // %bb.0: // %entry
205 ; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
206 ; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0
207 ; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
208 ; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
209 ; CHECK-SD-NEXT: shl v2.4s, v2.4s, #24
210 ; CHECK-SD-NEXT: shl v3.4s, v3.4s, #24
211 ; CHECK-SD-NEXT: shl v1.4s, v1.4s, #24
212 ; CHECK-SD-NEXT: shl v0.4s, v0.4s, #24
213 ; CHECK-SD-NEXT: sshr v2.4s, v2.4s, #24
214 ; CHECK-SD-NEXT: sshr v3.4s, v3.4s, #24
215 ; CHECK-SD-NEXT: sshr v1.4s, v1.4s, #24
216 ; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #24
217 ; CHECK-SD-NEXT: mul v2.4s, v2.4s, v3.4s
218 ; CHECK-SD-NEXT: mla v2.4s, v0.4s, v1.4s
219 ; CHECK-SD-NEXT: addv s0, v2.4s
220 ; CHECK-SD-NEXT: fmov w0, s0
221 ; CHECK-SD-NEXT: ret
222 ;
223 ; CHECK-GI-LABEL: test_sdot_v4i8_double:
224 ; CHECK-GI: // %bb.0: // %entry
225 ; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
226 ; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0
227 ; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0
228 ; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0
229 ; CHECK-GI-NEXT: shl v0.4s, v0.4s, #24
230 ; CHECK-GI-NEXT: shl v1.4s, v1.4s, #24
231 ; CHECK-GI-NEXT: shl v2.4s, v2.4s, #24
232 ; CHECK-GI-NEXT: shl v3.4s, v3.4s, #24
233 ; CHECK-GI-NEXT: sshr v0.4s, v0.4s, #24
234 ; CHECK-GI-NEXT: sshr v1.4s, v1.4s, #24
235 ; CHECK-GI-NEXT: sshr v2.4s, v2.4s, #24
236 ; CHECK-GI-NEXT: sshr v3.4s, v3.4s, #24
237 ; CHECK-GI-NEXT: mul v0.4s, v0.4s, v1.4s
238 ; CHECK-GI-NEXT: mul v1.4s, v2.4s, v3.4s
239 ; CHECK-GI-NEXT: addv s0, v0.4s
240 ; CHECK-GI-NEXT: addv s1, v1.4s
241 ; CHECK-GI-NEXT: fmov w8, s0
242 ; CHECK-GI-NEXT: fmov w9, s1
243 ; CHECK-GI-NEXT: add w0, w8, w9
244 ; CHECK-GI-NEXT: ret
245 entry:
246 %az = sext <4 x i8> %a to <4 x i32>
247 %bz = sext <4 x i8> %b to <4 x i32>
248 %m1 = mul nuw nsw <4 x i32> %az, %bz
249 %r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m1)
250 %cz = sext <4 x i8> %c to <4 x i32>
251 %dz = sext <4 x i8> %d to <4 x i32>
252 %m2 = mul nuw nsw <4 x i32> %cz, %dz
253 %r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m2)
254 %x = add i32 %r1, %r2
255 ret i32 %x
256 }
258 define i32 @test_sdot_v4i8_double_nomla(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
259 ; CHECK-SD-LABEL: test_sdot_v4i8_double_nomla:
260 ; CHECK-SD: // %bb.0: // %entry
261 ; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
262 ; CHECK-SD-NEXT: ushll v1.4s, v2.4h, #0
263 ; CHECK-SD-NEXT: shl v0.4s, v0.4s, #24
264 ; CHECK-SD-NEXT: shl v1.4s, v1.4s, #24
265 ; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #24
266 ; CHECK-SD-NEXT: ssra v0.4s, v1.4s, #24
267 ; CHECK-SD-NEXT: addv s0, v0.4s
268 ; CHECK-SD-NEXT: fmov w0, s0
269 ; CHECK-SD-NEXT: ret
270 ;
271 ; CHECK-GI-LABEL: test_sdot_v4i8_double_nomla:
272 ; CHECK-GI: // %bb.0: // %entry
273 ; CHECK-GI-NEXT: shl v1.4h, v2.4h, #8
274 ; CHECK-GI-NEXT: shl v0.4h, v0.4h, #8
275 ; CHECK-GI-NEXT: sshr v1.4h, v1.4h, #8
276 ; CHECK-GI-NEXT: sshr v0.4h, v0.4h, #8
277 ; CHECK-GI-NEXT: saddlv s1, v1.4h
278 ; CHECK-GI-NEXT: saddlv s0, v0.4h
279 ; CHECK-GI-NEXT: fmov w8, s1
280 ; CHECK-GI-NEXT: fmov w9, s0
281 ; CHECK-GI-NEXT: sxth w8, w8
282 ; CHECK-GI-NEXT: add w0, w8, w9, sxth
283 ; CHECK-GI-NEXT: ret
284 entry:
285 %az = sext <4 x i8> %a to <4 x i32>
286 %r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %az)
287 %cz = sext <4 x i8> %c to <4 x i32>
288 %r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %cz)
289 %x = add i32 %r1, %r2
290 ret i32 %x
291 }
293 define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
294 ; CHECK-LABEL: test_udot_v5i8:
295 ; CHECK: // %bb.0: // %entry
296 ; CHECK-NEXT: ldr d0, [x0]
297 ; CHECK-NEXT: ldr d1, [x1]
298 ; CHECK-NEXT: umull v0.8h, v1.8b, v0.8b
299 ; CHECK-NEXT: movi v1.2d, #0000000000000000
300 ; CHECK-NEXT: ushll2 v2.4s, v0.8h, #0
301 ; CHECK-NEXT: mov v1.s[0], v2.s[0]
302 ; CHECK-NEXT: uaddw v0.4s, v1.4s, v0.4h
303 ; CHECK-NEXT: addv s0, v0.4s
304 ; CHECK-NEXT: fmov w8, s0
305 ; CHECK-NEXT: add w0, w8, w2
306 ; CHECK-NEXT: ret
307 entry:
308 %0 = load <5 x i8>, ptr %a
309 %1 = zext <5 x i8> %0 to <5 x i32>
310 %2 = load <5 x i8>, ptr %b
311 %3 = zext <5 x i8> %2 to <5 x i32>
312 %4 = mul nuw nsw <5 x i32> %3, %1
313 %5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4)
314 %op.extra = add i32 %5, %sum
315 ret i32 %op.extra
316 }
318 define i32 @test_udot_v5i8_nomla(ptr nocapture readonly %a1) {
319 ; CHECK-LABEL: test_udot_v5i8_nomla:
320 ; CHECK: // %bb.0: // %entry
321 ; CHECK-NEXT: ldr d0, [x0]
322 ; CHECK-NEXT: movi v1.2d, #0000000000000000
323 ; CHECK-NEXT: ushll v0.8h, v0.8b, #0
324 ; CHECK-NEXT: ushll2 v2.4s, v0.8h, #0
325 ; CHECK-NEXT: mov v1.s[0], v2.s[0]
326 ; CHECK-NEXT: uaddw v0.4s, v1.4s, v0.4h
327 ; CHECK-NEXT: addv s0, v0.4s
328 ; CHECK-NEXT: fmov w0, s0
329 ; CHECK-NEXT: ret
330 entry:
331 %0 = load <5 x i8>, ptr %a1
332 %1 = zext <5 x i8> %0 to <5 x i32>
333 %2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %1)
334 ret i32 %2
335 }
336 define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
337 ; CHECK-LABEL: test_sdot_v5i8:
338 ; CHECK: // %bb.0: // %entry
339 ; CHECK-NEXT: ldr d0, [x0]
340 ; CHECK-NEXT: ldr d1, [x1]
341 ; CHECK-NEXT: smull v0.8h, v1.8b, v0.8b
342 ; CHECK-NEXT: movi v1.2d, #0000000000000000
343 ; CHECK-NEXT: sshll2 v2.4s, v0.8h, #0
344 ; CHECK-NEXT: mov v1.s[0], v2.s[0]
345 ; CHECK-NEXT: saddw v0.4s, v1.4s, v0.4h
346 ; CHECK-NEXT: addv s0, v0.4s
347 ; CHECK-NEXT: fmov w8, s0
348 ; CHECK-NEXT: add w0, w8, w2
349 ; CHECK-NEXT: ret
350 entry:
351 %0 = load <5 x i8>, ptr %a
352 %1 = sext <5 x i8> %0 to <5 x i32>
353 %2 = load <5 x i8>, ptr %b
354 %3 = sext <5 x i8> %2 to <5 x i32>
355 %4 = mul nsw <5 x i32> %3, %1
356 %5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4)
357 %op.extra = add nsw i32 %5, %sum
358 ret i32 %op.extra
359 }
361 define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {
362 ; CHECK-LABEL: test_sdot_v5i8_double:
363 ; CHECK: // %bb.0: // %entry
364 ; CHECK-NEXT: smull v2.8h, v2.8b, v3.8b
365 ; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b
366 ; CHECK-NEXT: movi v1.2d, #0000000000000000
367 ; CHECK-NEXT: movi v3.2d, #0000000000000000
368 ; CHECK-NEXT: sshll2 v4.4s, v0.8h, #0
369 ; CHECK-NEXT: sshll2 v5.4s, v2.8h, #0
370 ; CHECK-NEXT: mov v3.s[0], v4.s[0]
371 ; CHECK-NEXT: mov v1.s[0], v5.s[0]
372 ; CHECK-NEXT: saddw v0.4s, v3.4s, v0.4h
373 ; CHECK-NEXT: saddw v1.4s, v1.4s, v2.4h
374 ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
375 ; CHECK-NEXT: addv s0, v0.4s
376 ; CHECK-NEXT: fmov w0, s0
377 ; CHECK-NEXT: ret
378 entry:
379 %az = sext <5 x i8> %a to <5 x i32>
380 %bz = sext <5 x i8> %b to <5 x i32>
381 %m1 = mul nuw nsw <5 x i32> %az, %bz
382 %r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m1)
383 %cz = sext <5 x i8> %c to <5 x i32>
384 %dz = sext <5 x i8> %d to <5 x i32>
385 %m2 = mul nuw nsw <5 x i32> %cz, %dz
386 %r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m2)
387 %x = add i32 %r1, %r2
388 ret i32 %x
389 }
391 define i32 @test_sdot_v5i8_double_nomla(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {
392 ; CHECK-LABEL: test_sdot_v5i8_double_nomla:
393 ; CHECK: // %bb.0: // %entry
394 ; CHECK-NEXT: sshll v0.8h, v0.8b, #0
395 ; CHECK-NEXT: sshll v1.8h, v2.8b, #0
396 ; CHECK-NEXT: movi v2.2d, #0000000000000000
397 ; CHECK-NEXT: movi v3.2d, #0000000000000000
398 ; CHECK-NEXT: sshll2 v4.4s, v0.8h, #0
399 ; CHECK-NEXT: sshll2 v5.4s, v1.8h, #0
400 ; CHECK-NEXT: mov v3.s[0], v4.s[0]
401 ; CHECK-NEXT: mov v2.s[0], v5.s[0]
402 ; CHECK-NEXT: saddw v0.4s, v3.4s, v0.4h
403 ; CHECK-NEXT: saddw v1.4s, v2.4s, v1.4h
404 ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
405 ; CHECK-NEXT: addv s0, v0.4s
406 ; CHECK-NEXT: fmov w0, s0
407 ; CHECK-NEXT: ret
408 entry:
409 %az = sext <5 x i8> %a to <5 x i32>
410 %r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %az)
411 %cz = sext <5 x i8> %c to <5 x i32>
412 %r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %cz)
413 %x = add i32 %r1, %r2
414 ret i32 %x
415 }
417 define i32 @test_udot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {
418 ; CHECK-LABEL: test_udot_v8i8:
419 ; CHECK: // %bb.0: // %entry
420 ; CHECK-NEXT: movi v0.2d, #0000000000000000
421 ; CHECK-NEXT: ldr d1, [x0]
422 ; CHECK-NEXT: ldr d2, [x1]
423 ; CHECK-NEXT: udot v0.2s, v2.8b, v1.8b
424 ; CHECK-NEXT: addp v0.2s, v0.2s, v0.2s
425 ; CHECK-NEXT: fmov w0, s0
426 ; CHECK-NEXT: ret
427 entry:
428 %0 = load <8 x i8>, ptr %a
429 %1 = zext <8 x i8> %0 to <8 x i32>
430 %2 = load <8 x i8>, ptr %b
431 %3 = zext <8 x i8> %2 to <8 x i32>
432 %4 = mul nuw nsw <8 x i32> %3, %1
433 %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)
434 ret i32 %5
435 }
437 define i32 @test_udot_v8i8_nomla(ptr nocapture readonly %a1) {
438 ; CHECK-SD-LABEL: test_udot_v8i8_nomla:
439 ; CHECK-SD: // %bb.0: // %entry
440 ; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
441 ; CHECK-SD-NEXT: movi v1.8b, #1
442 ; CHECK-SD-NEXT: ldr d2, [x0]
443 ; CHECK-SD-NEXT: udot v0.2s, v2.8b, v1.8b
444 ; CHECK-SD-NEXT: addp v0.2s, v0.2s, v0.2s
445 ; CHECK-SD-NEXT: fmov w0, s0
446 ; CHECK-SD-NEXT: ret
447 ;
448 ; CHECK-GI-LABEL: test_udot_v8i8_nomla:
449 ; CHECK-GI: // %bb.0: // %entry
450 ; CHECK-GI-NEXT: movi v0.8b, #1
451 ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
452 ; CHECK-GI-NEXT: ldr d2, [x0]
453 ; CHECK-GI-NEXT: udot v1.2s, v2.8b, v0.8b
454 ; CHECK-GI-NEXT: addp v0.2s, v1.2s, v1.2s
455 ; CHECK-GI-NEXT: fmov w0, s0
456 ; CHECK-GI-NEXT: ret
457 entry:
458 %0 = load <8 x i8>, ptr %a1
459 %1 = zext <8 x i8> %0 to <8 x i32>
460 %2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
461 ret i32 %2
462 }
464 define i32 @test_sdot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {
465 ; CHECK-LABEL: test_sdot_v8i8:
466 ; CHECK: // %bb.0: // %entry
467 ; CHECK-NEXT: movi v0.2d, #0000000000000000
468 ; CHECK-NEXT: ldr d1, [x0]
469 ; CHECK-NEXT: ldr d2, [x1]
470 ; CHECK-NEXT: sdot v0.2s, v2.8b, v1.8b
471 ; CHECK-NEXT: addp v0.2s, v0.2s, v0.2s
472 ; CHECK-NEXT: fmov w0, s0
473 ; CHECK-NEXT: ret
474 entry:
475 %0 = load <8 x i8>, ptr %a
476 %1 = sext <8 x i8> %0 to <8 x i32>
477 %2 = load <8 x i8>, ptr %b
478 %3 = sext <8 x i8> %2 to <8 x i32>
479 %4 = mul nsw <8 x i32> %3, %1
480 %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)
481 ret i32 %5
482 }
484 define i32 @test_sdot_v8i8_nomla(ptr nocapture readonly %a1) {
485 ; CHECK-SD-LABEL: test_sdot_v8i8_nomla:
486 ; CHECK-SD: // %bb.0: // %entry
487 ; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
488 ; CHECK-SD-NEXT: movi v1.8b, #1
489 ; CHECK-SD-NEXT: ldr d2, [x0]
490 ; CHECK-SD-NEXT: sdot v0.2s, v2.8b, v1.8b
491 ; CHECK-SD-NEXT: addp v0.2s, v0.2s, v0.2s
492 ; CHECK-SD-NEXT: fmov w0, s0
493 ; CHECK-SD-NEXT: ret
494 ;
495 ; CHECK-GI-LABEL: test_sdot_v8i8_nomla:
496 ; CHECK-GI: // %bb.0: // %entry
497 ; CHECK-GI-NEXT: movi v0.8b, #1
498 ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
499 ; CHECK-GI-NEXT: ldr d2, [x0]
500 ; CHECK-GI-NEXT: sdot v1.2s, v2.8b, v0.8b
501 ; CHECK-GI-NEXT: addp v0.2s, v1.2s, v1.2s
502 ; CHECK-GI-NEXT: fmov w0, s0
503 ; CHECK-GI-NEXT: ret
504 entry:
505 %0 = load <8 x i8>, ptr %a1
506 %1 = sext <8 x i8> %0 to <8 x i32>
507 %2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
508 ret i32 %2
509 }
512 define i32 @test_udot_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
513 ; CHECK-LABEL: test_udot_v16i8:
514 ; CHECK: // %bb.0: // %entry
515 ; CHECK-NEXT: movi v0.2d, #0000000000000000
516 ; CHECK-NEXT: ldr q1, [x0]
517 ; CHECK-NEXT: ldr q2, [x1]
518 ; CHECK-NEXT: udot v0.4s, v2.16b, v1.16b
519 ; CHECK-NEXT: addv s0, v0.4s
520 ; CHECK-NEXT: fmov w8, s0
521 ; CHECK-NEXT: add w0, w8, w2
522 ; CHECK-NEXT: ret
523 entry:
524 %0 = load <16 x i8>, ptr %a
525 %1 = zext <16 x i8> %0 to <16 x i32>
526 %2 = load <16 x i8>, ptr %b
527 %3 = zext <16 x i8> %2 to <16 x i32>
528 %4 = mul nuw nsw <16 x i32> %3, %1
529 %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)
530 %op.extra = add i32 %5, %sum
531 ret i32 %op.extra
532 }
534 define i32 @test_udot_v16i8_nomla(ptr nocapture readonly %a1) {
535 ; CHECK-LABEL: test_udot_v16i8_nomla:
536 ; CHECK: // %bb.0: // %entry
537 ; CHECK-NEXT: movi v0.16b, #1
538 ; CHECK-NEXT: movi v1.2d, #0000000000000000
539 ; CHECK-NEXT: ldr q2, [x0]
540 ; CHECK-NEXT: udot v1.4s, v2.16b, v0.16b
541 ; CHECK-NEXT: addv s0, v1.4s
542 ; CHECK-NEXT: fmov w0, s0
543 ; CHECK-NEXT: ret
544 entry:
545 %0 = load <16 x i8>, ptr %a1
546 %1 = zext <16 x i8> %0 to <16 x i32>
547 %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)
548 ret i32 %2
549 }
551 define i32 @test_sdot_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
552 ; CHECK-LABEL: test_sdot_v16i8:
553 ; CHECK: // %bb.0: // %entry
554 ; CHECK-NEXT: movi v0.2d, #0000000000000000
555 ; CHECK-NEXT: ldr q1, [x0]
556 ; CHECK-NEXT: ldr q2, [x1]
557 ; CHECK-NEXT: sdot v0.4s, v2.16b, v1.16b
558 ; CHECK-NEXT: addv s0, v0.4s
559 ; CHECK-NEXT: fmov w8, s0
560 ; CHECK-NEXT: add w0, w8, w2
561 ; CHECK-NEXT: ret
562 entry:
563 %0 = load <16 x i8>, ptr %a
564 %1 = sext <16 x i8> %0 to <16 x i32>
565 %2 = load <16 x i8>, ptr %b
566 %3 = sext <16 x i8> %2 to <16 x i32>
567 %4 = mul nsw <16 x i32> %3, %1
568 %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)
569 %op.extra = add nsw i32 %5, %sum
570 ret i32 %op.extra
571 }
573 define i32 @test_sdot_v16i8_nomla(ptr nocapture readonly %a1) {
574 ; CHECK-LABEL: test_sdot_v16i8_nomla:
575 ; CHECK: // %bb.0: // %entry
576 ; CHECK-NEXT: movi v0.16b, #1
577 ; CHECK-NEXT: movi v1.2d, #0000000000000000
578 ; CHECK-NEXT: ldr q2, [x0]
579 ; CHECK-NEXT: sdot v1.4s, v2.16b, v0.16b
580 ; CHECK-NEXT: addv s0, v1.4s
581 ; CHECK-NEXT: fmov w0, s0
582 ; CHECK-NEXT: ret
583 entry:
584 %0 = load <16 x i8>, ptr %a1
585 %1 = sext <16 x i8> %0 to <16 x i32>
586 %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)
587 ret i32 %2
588 }
591 define i32 @test_udot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
592 ; CHECK-SD-LABEL: test_udot_v8i8_double:
593 ; CHECK-SD: // %bb.0: // %entry
594 ; CHECK-SD-NEXT: movi v4.2d, #0000000000000000
595 ; CHECK-SD-NEXT: udot v4.2s, v2.8b, v3.8b
596 ; CHECK-SD-NEXT: udot v4.2s, v0.8b, v1.8b
597 ; CHECK-SD-NEXT: addp v0.2s, v4.2s, v4.2s
598 ; CHECK-SD-NEXT: fmov w0, s0
599 ; CHECK-SD-NEXT: ret
600 ;
601 ; CHECK-GI-LABEL: test_udot_v8i8_double:
602 ; CHECK-GI: // %bb.0: // %entry
603 ; CHECK-GI-NEXT: movi v4.2d, #0000000000000000
604 ; CHECK-GI-NEXT: movi v5.2d, #0000000000000000
605 ; CHECK-GI-NEXT: udot v5.2s, v0.8b, v1.8b
606 ; CHECK-GI-NEXT: udot v4.2s, v2.8b, v3.8b
607 ; CHECK-GI-NEXT: addp v0.2s, v5.2s, v5.2s
608 ; CHECK-GI-NEXT: addp v1.2s, v4.2s, v4.2s
609 ; CHECK-GI-NEXT: fmov w8, s0
610 ; CHECK-GI-NEXT: fmov w9, s1
611 ; CHECK-GI-NEXT: add w0, w8, w9
612 ; CHECK-GI-NEXT: ret
613 entry:
614 %az = zext <8 x i8> %a to <8 x i32>
615 %bz = zext <8 x i8> %b to <8 x i32>
616 %m1 = mul nuw nsw <8 x i32> %az, %bz
617 %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)
618 %cz = zext <8 x i8> %c to <8 x i32>
619 %dz = zext <8 x i8> %d to <8 x i32>
620 %m2 = mul nuw nsw <8 x i32> %cz, %dz
621 %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)
622 %x = add i32 %r1, %r2
623 ret i32 %x
624 }
626 define i32 @test_udot_v8i8_double_nomla(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
627 ; CHECK-SD-LABEL: test_udot_v8i8_double_nomla:
628 ; CHECK-SD: // %bb.0: // %entry
629 ; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
630 ; CHECK-SD-NEXT: movi v3.8b, #1
631 ; CHECK-SD-NEXT: udot v1.2s, v2.8b, v3.8b
632 ; CHECK-SD-NEXT: udot v1.2s, v0.8b, v3.8b
633 ; CHECK-SD-NEXT: addp v0.2s, v1.2s, v1.2s
634 ; CHECK-SD-NEXT: fmov w0, s0
635 ; CHECK-SD-NEXT: ret
636 ;
637 ; CHECK-GI-LABEL: test_udot_v8i8_double_nomla:
638 ; CHECK-GI: // %bb.0: // %entry
639 ; CHECK-GI-NEXT: movi v1.8b, #1
640 ; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
641 ; CHECK-GI-NEXT: movi v4.2d, #0000000000000000
642 ; CHECK-GI-NEXT: udot v4.2s, v0.8b, v1.8b
643 ; CHECK-GI-NEXT: udot v3.2s, v2.8b, v1.8b
644 ; CHECK-GI-NEXT: addp v0.2s, v4.2s, v4.2s
645 ; CHECK-GI-NEXT: addp v1.2s, v3.2s, v3.2s
646 ; CHECK-GI-NEXT: fmov w8, s0
647 ; CHECK-GI-NEXT: fmov w9, s1
648 ; CHECK-GI-NEXT: add w0, w8, w9
649 ; CHECK-GI-NEXT: ret
650 entry:
651 %az = zext <8 x i8> %a to <8 x i32>
652 %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %az)
653 %cz = zext <8 x i8> %c to <8 x i32>
654 %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %cz)
655 %x = add i32 %r1, %r2
656 ret i32 %x
657 }
659 define i32 @test_udot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
660 ; CHECK-SD-LABEL: test_udot_v16i8_double:
661 ; CHECK-SD: // %bb.0: // %entry
662 ; CHECK-SD-NEXT: movi v4.2d, #0000000000000000
663 ; CHECK-SD-NEXT: udot v4.4s, v2.16b, v3.16b
664 ; CHECK-SD-NEXT: udot v4.4s, v0.16b, v1.16b
665 ; CHECK-SD-NEXT: addv s0, v4.4s
666 ; CHECK-SD-NEXT: fmov w0, s0
667 ; CHECK-SD-NEXT: ret
668 ;
669 ; CHECK-GI-LABEL: test_udot_v16i8_double:
670 ; CHECK-GI: // %bb.0: // %entry
671 ; CHECK-GI-NEXT: movi v4.2d, #0000000000000000
672 ; CHECK-GI-NEXT: movi v5.2d, #0000000000000000
673 ; CHECK-GI-NEXT: udot v5.4s, v0.16b, v1.16b
674 ; CHECK-GI-NEXT: udot v4.4s, v2.16b, v3.16b
675 ; CHECK-GI-NEXT: addv s0, v5.4s
676 ; CHECK-GI-NEXT: addv s1, v4.4s
677 ; CHECK-GI-NEXT: fmov w8, s0
678 ; CHECK-GI-NEXT: fmov w9, s1
679 ; CHECK-GI-NEXT: add w0, w8, w9
680 ; CHECK-GI-NEXT: ret
681 entry:
682 %az = zext <16 x i8> %a to <16 x i32>
683 %bz = zext <16 x i8> %b to <16 x i32>
684 %m1 = mul nuw nsw <16 x i32> %az, %bz
685 %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)
686 %cz = zext <16 x i8> %c to <16 x i32>
687 %dz = zext <16 x i8> %d to <16 x i32>
688 %m2 = mul nuw nsw <16 x i32> %cz, %dz
689 %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)
690 %x = add i32 %r1, %r2
691 ret i32 %x
692 }
694 define i32 @test_udot_v16i8_double_nomla(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
695 ; CHECK-SD-LABEL: test_udot_v16i8_double_nomla:
696 ; CHECK-SD: // %bb.0: // %entry
697 ; CHECK-SD-NEXT: movi v1.16b, #1
698 ; CHECK-SD-NEXT: movi v3.2d, #0000000000000000
699 ; CHECK-SD-NEXT: udot v3.4s, v2.16b, v1.16b
700 ; CHECK-SD-NEXT: udot v3.4s, v0.16b, v1.16b
701 ; CHECK-SD-NEXT: addv s0, v3.4s
702 ; CHECK-SD-NEXT: fmov w0, s0
703 ; CHECK-SD-NEXT: ret
704 ;
705 ; CHECK-GI-LABEL: test_udot_v16i8_double_nomla:
706 ; CHECK-GI: // %bb.0: // %entry
707 ; CHECK-GI-NEXT: movi v1.16b, #1
708 ; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
709 ; CHECK-GI-NEXT: movi v4.2d, #0000000000000000
710 ; CHECK-GI-NEXT: udot v4.4s, v0.16b, v1.16b
711 ; CHECK-GI-NEXT: udot v3.4s, v2.16b, v1.16b
712 ; CHECK-GI-NEXT: addv s0, v4.4s
713 ; CHECK-GI-NEXT: addv s1, v3.4s
714 ; CHECK-GI-NEXT: fmov w8, s0
715 ; CHECK-GI-NEXT: fmov w9, s1
716 ; CHECK-GI-NEXT: add w0, w8, w9
717 ; CHECK-GI-NEXT: ret
718 entry:
719 %az = zext <16 x i8> %a to <16 x i32>
720 %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %az)
721 %cz = zext <16 x i8> %c to <16 x i32>
722 %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %cz)
723 %x = add i32 %r1, %r2
724 ret i32 %x
725 }
727 define i32 @test_sdot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
728 ; CHECK-SD-LABEL: test_sdot_v8i8_double:
729 ; CHECK-SD: // %bb.0: // %entry
730 ; CHECK-SD-NEXT: movi v4.2d, #0000000000000000
731 ; CHECK-SD-NEXT: sdot v4.2s, v2.8b, v3.8b
732 ; CHECK-SD-NEXT: sdot v4.2s, v0.8b, v1.8b
733 ; CHECK-SD-NEXT: addp v0.2s, v4.2s, v4.2s
734 ; CHECK-SD-NEXT: fmov w0, s0
735 ; CHECK-SD-NEXT: ret
736 ;
737 ; CHECK-GI-LABEL: test_sdot_v8i8_double:
738 ; CHECK-GI: // %bb.0: // %entry
739 ; CHECK-GI-NEXT: movi v4.2d, #0000000000000000
740 ; CHECK-GI-NEXT: movi v5.2d, #0000000000000000
741 ; CHECK-GI-NEXT: sdot v5.2s, v0.8b, v1.8b
742 ; CHECK-GI-NEXT: sdot v4.2s, v2.8b, v3.8b
743 ; CHECK-GI-NEXT: addp v0.2s, v5.2s, v5.2s
744 ; CHECK-GI-NEXT: addp v1.2s, v4.2s, v4.2s
745 ; CHECK-GI-NEXT: fmov w8, s0
746 ; CHECK-GI-NEXT: fmov w9, s1
747 ; CHECK-GI-NEXT: add w0, w8, w9
748 ; CHECK-GI-NEXT: ret
749 entry:
750 %az = sext <8 x i8> %a to <8 x i32>
751 %bz = sext <8 x i8> %b to <8 x i32>
752 %m1 = mul nuw nsw <8 x i32> %az, %bz
753 %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)
754 %cz = sext <8 x i8> %c to <8 x i32>
755 %dz = sext <8 x i8> %d to <8 x i32>
756 %m2 = mul nuw nsw <8 x i32> %cz, %dz
757 %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)
758 %x = add i32 %r1, %r2
759 ret i32 %x
760 }
762 define i32 @test_sdot_v8i8_double_nomla(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
763 ; CHECK-SD-LABEL: test_sdot_v8i8_double_nomla:
764 ; CHECK-SD: // %bb.0: // %entry
765 ; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
766 ; CHECK-SD-NEXT: movi v3.8b, #1
767 ; CHECK-SD-NEXT: sdot v1.2s, v2.8b, v3.8b
768 ; CHECK-SD-NEXT: sdot v1.2s, v0.8b, v3.8b
769 ; CHECK-SD-NEXT: addp v0.2s, v1.2s, v1.2s
770 ; CHECK-SD-NEXT: fmov w0, s0
771 ; CHECK-SD-NEXT: ret
772 ;
773 ; CHECK-GI-LABEL: test_sdot_v8i8_double_nomla:
774 ; CHECK-GI: // %bb.0: // %entry
775 ; CHECK-GI-NEXT: movi v1.8b, #1
776 ; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
777 ; CHECK-GI-NEXT: movi v4.2d, #0000000000000000
778 ; CHECK-GI-NEXT: sdot v4.2s, v0.8b, v1.8b
779 ; CHECK-GI-NEXT: sdot v3.2s, v2.8b, v1.8b
780 ; CHECK-GI-NEXT: addp v0.2s, v4.2s, v4.2s
781 ; CHECK-GI-NEXT: addp v1.2s, v3.2s, v3.2s
782 ; CHECK-GI-NEXT: fmov w8, s0
783 ; CHECK-GI-NEXT: fmov w9, s1
784 ; CHECK-GI-NEXT: add w0, w8, w9
785 ; CHECK-GI-NEXT: ret
786 entry:
787 %az = sext <8 x i8> %a to <8 x i32>
788 %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %az)
789 %cz = sext <8 x i8> %c to <8 x i32>
790 %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %cz)
791 %x = add i32 %r1, %r2
792 ret i32 %x
793 }
795 define i32 @test_sdot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
796 ; CHECK-SD-LABEL: test_sdot_v16i8_double:
797 ; CHECK-SD: // %bb.0: // %entry
798 ; CHECK-SD-NEXT: movi v4.2d, #0000000000000000
799 ; CHECK-SD-NEXT: sdot v4.4s, v2.16b, v3.16b
800 ; CHECK-SD-NEXT: sdot v4.4s, v0.16b, v1.16b
801 ; CHECK-SD-NEXT: addv s0, v4.4s
802 ; CHECK-SD-NEXT: fmov w0, s0
803 ; CHECK-SD-NEXT: ret
804 ;
805 ; CHECK-GI-LABEL: test_sdot_v16i8_double:
806 ; CHECK-GI: // %bb.0: // %entry
807 ; CHECK-GI-NEXT: movi v4.2d, #0000000000000000
808 ; CHECK-GI-NEXT: movi v5.2d, #0000000000000000
809 ; CHECK-GI-NEXT: sdot v5.4s, v0.16b, v1.16b
810 ; CHECK-GI-NEXT: sdot v4.4s, v2.16b, v3.16b
811 ; CHECK-GI-NEXT: addv s0, v5.4s
812 ; CHECK-GI-NEXT: addv s1, v4.4s
813 ; CHECK-GI-NEXT: fmov w8, s0
814 ; CHECK-GI-NEXT: fmov w9, s1
815 ; CHECK-GI-NEXT: add w0, w8, w9
816 ; CHECK-GI-NEXT: ret
817 entry:
818 %az = sext <16 x i8> %a to <16 x i32>
819 %bz = sext <16 x i8> %b to <16 x i32>
820 %m1 = mul nuw nsw <16 x i32> %az, %bz
821 %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)
822 %cz = sext <16 x i8> %c to <16 x i32>
823 %dz = sext <16 x i8> %d to <16 x i32>
824 %m2 = mul nuw nsw <16 x i32> %cz, %dz
825 %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)
826 %x = add i32 %r1, %r2
827 ret i32 %x
828 }
830 define i32 @test_sdot_v16i8_double_nomla(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
831 ; CHECK-SD-LABEL: test_sdot_v16i8_double_nomla:
832 ; CHECK-SD: // %bb.0: // %entry
833 ; CHECK-SD-NEXT: movi v1.16b, #1
834 ; CHECK-SD-NEXT: movi v3.2d, #0000000000000000
835 ; CHECK-SD-NEXT: sdot v3.4s, v2.16b, v1.16b
836 ; CHECK-SD-NEXT: sdot v3.4s, v0.16b, v1.16b
837 ; CHECK-SD-NEXT: addv s0, v3.4s
838 ; CHECK-SD-NEXT: fmov w0, s0
839 ; CHECK-SD-NEXT: ret
840 ;
841 ; CHECK-GI-LABEL: test_sdot_v16i8_double_nomla:
842 ; CHECK-GI: // %bb.0: // %entry
843 ; CHECK-GI-NEXT: movi v1.16b, #1
844 ; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
845 ; CHECK-GI-NEXT: movi v4.2d, #0000000000000000
846 ; CHECK-GI-NEXT: sdot v4.4s, v0.16b, v1.16b
847 ; CHECK-GI-NEXT: sdot v3.4s, v2.16b, v1.16b
848 ; CHECK-GI-NEXT: addv s0, v4.4s
849 ; CHECK-GI-NEXT: addv s1, v3.4s
850 ; CHECK-GI-NEXT: fmov w8, s0
851 ; CHECK-GI-NEXT: fmov w9, s1
852 ; CHECK-GI-NEXT: add w0, w8, w9
853 ; CHECK-GI-NEXT: ret
854 entry:
855 %az = sext <16 x i8> %a to <16 x i32>
856 %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %az)
857 %cz = sext <16 x i8> %c to <16 x i32>
858 %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %cz)
859 %x = add i32 %r1, %r2
860 ret i32 %x
861 }
863 define i32 @test_udot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
864 ; CHECK-SD-LABEL: test_udot_v24i8:
865 ; CHECK-SD: // %bb.0: // %entry
866 ; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
867 ; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
868 ; CHECK-SD-NEXT: ldr q2, [x0]
869 ; CHECK-SD-NEXT: ldr q3, [x1]
870 ; CHECK-SD-NEXT: ldr d4, [x0, #16]
871 ; CHECK-SD-NEXT: ldr d5, [x1, #16]
872 ; CHECK-SD-NEXT: udot v1.2s, v5.8b, v4.8b
873 ; CHECK-SD-NEXT: udot v0.4s, v3.16b, v2.16b
874 ; CHECK-SD-NEXT: addp v1.2s, v1.2s, v1.2s
875 ; CHECK-SD-NEXT: addv s0, v0.4s
876 ; CHECK-SD-NEXT: fmov w8, s1
877 ; CHECK-SD-NEXT: fmov w9, s0
878 ; CHECK-SD-NEXT: add w8, w9, w8
879 ; CHECK-SD-NEXT: add w0, w8, w2
880 ; CHECK-SD-NEXT: ret
881 ;
882 ; CHECK-GI-LABEL: test_udot_v24i8:
883 ; CHECK-GI: // %bb.0: // %entry
884 ; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
885 ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
886 ; CHECK-GI-NEXT: ldr q2, [x0]
887 ; CHECK-GI-NEXT: ldr d3, [x0, #16]
888 ; CHECK-GI-NEXT: ldr q4, [x1]
889 ; CHECK-GI-NEXT: ldr d5, [x1, #16]
890 ; CHECK-GI-NEXT: udot v1.4s, v4.16b, v2.16b
891 ; CHECK-GI-NEXT: udot v0.4s, v5.16b, v3.16b
892 ; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s
893 ; CHECK-GI-NEXT: addv s0, v0.4s
894 ; CHECK-GI-NEXT: fmov w8, s0
895 ; CHECK-GI-NEXT: add w0, w8, w2
896 ; CHECK-GI-NEXT: ret
897 entry:
898 %0 = load <24 x i8>, ptr %a
899 %1 = zext <24 x i8> %0 to <24 x i32>
900 %2 = load <24 x i8>, ptr %b
901 %3 = zext <24 x i8> %2 to <24 x i32>
902 %4 = mul nuw nsw <24 x i32> %3, %1
903 %5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4)
904 %op.extra = add i32 %5, %sum
905 ret i32 %op.extra
906 }
908 define i32 @test_udot_v24i8_nomla(ptr nocapture readonly %a1) {
909 ; CHECK-SD-LABEL: test_udot_v24i8_nomla:
910 ; CHECK-SD: // %bb.0: // %entry
911 ; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
912 ; CHECK-SD-NEXT: movi v1.8b, #1
913 ; CHECK-SD-NEXT: ldr q4, [x0]
914 ; CHECK-SD-NEXT: movi v2.2d, #0000000000000000
915 ; CHECK-SD-NEXT: movi v3.16b, #1
916 ; CHECK-SD-NEXT: ldr d5, [x0, #16]
917 ; CHECK-SD-NEXT: udot v2.2s, v5.8b, v1.8b
918 ; CHECK-SD-NEXT: udot v0.4s, v4.16b, v3.16b
919 ; CHECK-SD-NEXT: addp v1.2s, v2.2s, v2.2s
920 ; CHECK-SD-NEXT: addv s0, v0.4s
921 ; CHECK-SD-NEXT: fmov w8, s1
922 ; CHECK-SD-NEXT: fmov w9, s0
923 ; CHECK-SD-NEXT: add w0, w9, w8
924 ; CHECK-SD-NEXT: ret
925 ;
926 ; CHECK-GI-LABEL: test_udot_v24i8_nomla:
927 ; CHECK-GI: // %bb.0: // %entry
928 ; CHECK-GI-NEXT: movi v0.8b, #1
929 ; CHECK-GI-NEXT: movi v1.8b, #1
930 ; CHECK-GI-NEXT: ldr q4, [x0]
931 ; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
932 ; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
933 ; CHECK-GI-NEXT: ldr d5, [x0, #16]
934 ; CHECK-GI-NEXT: mov v1.d[1], v0.d[0]
935 ; CHECK-GI-NEXT: udot v2.4s, v5.16b, v0.16b
936 ; CHECK-GI-NEXT: udot v3.4s, v4.16b, v1.16b
937 ; CHECK-GI-NEXT: add v0.4s, v3.4s, v2.4s
938 ; CHECK-GI-NEXT: addv s0, v0.4s
939 ; CHECK-GI-NEXT: fmov w0, s0
940 ; CHECK-GI-NEXT: ret
941 entry:
942 %0 = load <24 x i8>, ptr %a1
943 %1 = zext <24 x i8> %0 to <24 x i32>
944 %2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %1)
945 ret i32 %2
946 }
947 define i32 @test_sdot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
948 ; CHECK-SD-LABEL: test_sdot_v24i8:
949 ; CHECK-SD: // %bb.0: // %entry
950 ; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
951 ; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
952 ; CHECK-SD-NEXT: ldr q2, [x0]
953 ; CHECK-SD-NEXT: ldr q3, [x1]
954 ; CHECK-SD-NEXT: ldr d4, [x0, #16]
955 ; CHECK-SD-NEXT: ldr d5, [x1, #16]
956 ; CHECK-SD-NEXT: sdot v1.2s, v5.8b, v4.8b
957 ; CHECK-SD-NEXT: sdot v0.4s, v3.16b, v2.16b
958 ; CHECK-SD-NEXT: addp v1.2s, v1.2s, v1.2s
959 ; CHECK-SD-NEXT: addv s0, v0.4s
960 ; CHECK-SD-NEXT: fmov w8, s1
961 ; CHECK-SD-NEXT: fmov w9, s0
962 ; CHECK-SD-NEXT: add w8, w9, w8
963 ; CHECK-SD-NEXT: add w0, w8, w2
964 ; CHECK-SD-NEXT: ret
965 ;
966 ; CHECK-GI-LABEL: test_sdot_v24i8:
967 ; CHECK-GI: // %bb.0: // %entry
968 ; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
969 ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
970 ; CHECK-GI-NEXT: ldr q2, [x0]
971 ; CHECK-GI-NEXT: ldr d3, [x0, #16]
972 ; CHECK-GI-NEXT: ldr q4, [x1]
973 ; CHECK-GI-NEXT: ldr d5, [x1, #16]
974 ; CHECK-GI-NEXT: sdot v1.4s, v4.16b, v2.16b
975 ; CHECK-GI-NEXT: sdot v0.4s, v5.16b, v3.16b
976 ; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s
977 ; CHECK-GI-NEXT: addv s0, v0.4s
978 ; CHECK-GI-NEXT: fmov w8, s0
979 ; CHECK-GI-NEXT: add w0, w8, w2
980 ; CHECK-GI-NEXT: ret
981 entry:
982 %0 = load <24 x i8>, ptr %a
983 %1 = sext <24 x i8> %0 to <24 x i32>
984 %2 = load <24 x i8>, ptr %b
985 %3 = sext <24 x i8> %2 to <24 x i32>
986 %4 = mul nsw <24 x i32> %3, %1
987 %5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4)
988 %op.extra = add nsw i32 %5, %sum
989 ret i32 %op.extra
990 }
992 define i32 @test_sdot_v24i8_double(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
993 ; CHECK-SD-LABEL: test_sdot_v24i8_double:
994 ; CHECK-SD: // %bb.0: // %entry
995 ; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
996 ; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
997 ; CHECK-SD-NEXT: .cfi_offset w29, -16
998 ; CHECK-SD-NEXT: fmov s0, w0
999 ; CHECK-SD-NEXT: ldr b1, [sp, #144]
1000 ; CHECK-SD-NEXT: add x10, sp, #152
1001 ; CHECK-SD-NEXT: add x9, sp, #160
1002 ; CHECK-SD-NEXT: add x8, sp, #168
1003 ; CHECK-SD-NEXT: ldr b2, [sp, #272]
1004 ; CHECK-SD-NEXT: ld1 { v1.b }[1], [x10]
1005 ; CHECK-SD-NEXT: add x11, sp, #280
1006 ; CHECK-SD-NEXT: ldr b3, [sp, #80]
1007 ; CHECK-SD-NEXT: mov v0.b[1], w1
1008 ; CHECK-SD-NEXT: ldr b4, [sp, #528]
1009 ; CHECK-SD-NEXT: add x10, sp, #88
1010 ; CHECK-SD-NEXT: ld1 { v2.b }[1], [x11]
1011 ; CHECK-SD-NEXT: add x11, sp, #536
1012 ; CHECK-SD-NEXT: ldr b5, [sp, #336]
1013 ; CHECK-SD-NEXT: ld1 { v1.b }[2], [x9]
1014 ; CHECK-SD-NEXT: ld1 { v3.b }[1], [x10]
1015 ; CHECK-SD-NEXT: add x10, sp, #344
1016 ; CHECK-SD-NEXT: ld1 { v4.b }[1], [x11]
1017 ; CHECK-SD-NEXT: add x11, sp, #176
1018 ; CHECK-SD-NEXT: ldr b6, [sp, #656]
1019 ; CHECK-SD-NEXT: mov v0.b[2], w2
1020 ; CHECK-SD-NEXT: ld1 { v5.b }[1], [x10]
1021 ; CHECK-SD-NEXT: ldr b7, [sp, #464]
1022 ; CHECK-SD-NEXT: ld1 { v1.b }[3], [x8]
1023 ; CHECK-SD-NEXT: add x12, sp, #664
1024 ; CHECK-SD-NEXT: add x9, sp, #472
1025 ; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12]
1026 ; CHECK-SD-NEXT: add x8, sp, #96
1027 ; CHECK-SD-NEXT: add x10, sp, #184
1028 ; CHECK-SD-NEXT: add x12, sp, #288
1029 ; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9]
1030 ; CHECK-SD-NEXT: ld1 { v3.b }[2], [x8]
1031 ; CHECK-SD-NEXT: mov v0.b[3], w3
1032 ; CHECK-SD-NEXT: ld1 { v1.b }[4], [x11]
1033 ; CHECK-SD-NEXT: add x8, sp, #352
1034 ; CHECK-SD-NEXT: ld1 { v2.b }[2], [x12]
1035 ; CHECK-SD-NEXT: add x13, sp, #544
1036 ; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8]
1037 ; CHECK-SD-NEXT: add x8, sp, #672
1038 ; CHECK-SD-NEXT: ld1 { v4.b }[2], [x13]
1039 ; CHECK-SD-NEXT: add x9, sp, #192
1040 ; CHECK-SD-NEXT: ld1 { v1.b }[5], [x10]
1041 ; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8]
1042 ; CHECK-SD-NEXT: add x8, sp, #480
1043 ; CHECK-SD-NEXT: mov v0.b[4], w4
1044 ; CHECK-SD-NEXT: ld1 { v7.b }[2], [x8]
1045 ; CHECK-SD-NEXT: add x8, sp, #296
1046 ; CHECK-SD-NEXT: ld1 { v2.b }[3], [x8]
1047 ; CHECK-SD-NEXT: add x8, sp, #552
1048 ; CHECK-SD-NEXT: add x12, sp, #200
1049 ; CHECK-SD-NEXT: ld1 { v1.b }[6], [x9]
1050 ; CHECK-SD-NEXT: ld1 { v4.b }[3], [x8]
1051 ; CHECK-SD-NEXT: add x8, sp, #360
1052 ; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8]
1053 ; CHECK-SD-NEXT: add x8, sp, #104
1054 ; CHECK-SD-NEXT: add x9, sp, #560
1055 ; CHECK-SD-NEXT: mov v0.b[5], w5
1056 ; CHECK-SD-NEXT: ld1 { v3.b }[3], [x8]
1057 ; CHECK-SD-NEXT: add x8, sp, #368
1058 ; CHECK-SD-NEXT: ld1 { v1.b }[7], [x12]
1059 ; CHECK-SD-NEXT: ld1 { v4.b }[4], [x9]
1060 ; CHECK-SD-NEXT: add x13, sp, #208
1061 ; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8]
1062 ; CHECK-SD-NEXT: add x12, sp, #304
1063 ; CHECK-SD-NEXT: add x8, sp, #568
1064 ; CHECK-SD-NEXT: ld1 { v2.b }[4], [x12]
1065 ; CHECK-SD-NEXT: add x12, sp, #16
1066 ; CHECK-SD-NEXT: add x17, sp, #376
1067 ; CHECK-SD-NEXT: mov v0.b[6], w6
1068 ; CHECK-SD-NEXT: ld1 { v1.b }[8], [x13]
1069 ; CHECK-SD-NEXT: ld1 { v4.b }[5], [x8]
1070 ; CHECK-SD-NEXT: add x14, sp, #216
1071 ; CHECK-SD-NEXT: ld1 { v5.b }[5], [x17]
1072 ; CHECK-SD-NEXT: add x13, sp, #576
1073 ; CHECK-SD-NEXT: add x11, sp, #224
1074 ; CHECK-SD-NEXT: add x10, sp, #232
1075 ; CHECK-SD-NEXT: add x15, sp, #240
1076 ; CHECK-SD-NEXT: ld1 { v1.b }[9], [x14]
1077 ; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13]
1078 ; CHECK-SD-NEXT: add x13, sp, #384
1079 ; CHECK-SD-NEXT: mov v0.b[7], w7
1080 ; CHECK-SD-NEXT: ld1 { v5.b }[6], [x13]
1081 ; CHECK-SD-NEXT: add x13, sp, #112
1082 ; CHECK-SD-NEXT: ld1 { v3.b }[4], [x13]
1083 ; CHECK-SD-NEXT: add x13, sp, #32
1084 ; CHECK-SD-NEXT: add x14, sp, #584
1085 ; CHECK-SD-NEXT: ld1 { v1.b }[10], [x11]
1086 ; CHECK-SD-NEXT: ld1 { v4.b }[7], [x14]
1087 ; CHECK-SD-NEXT: add x11, sp, #312
1088 ; CHECK-SD-NEXT: add x14, sp, #40
1089 ; CHECK-SD-NEXT: ld1 { v2.b }[5], [x11]
1090 ; CHECK-SD-NEXT: add x11, sp, #592
1091 ; CHECK-SD-NEXT: ld1 { v0.b }[8], [x12]
1092 ; CHECK-SD-NEXT: add x12, sp, #24
1093 ; CHECK-SD-NEXT: add x16, sp, #248
1094 ; CHECK-SD-NEXT: ld1 { v1.b }[11], [x10]
1095 ; CHECK-SD-NEXT: ld1 { v4.b }[8], [x11]
1096 ; CHECK-SD-NEXT: add x11, sp, #400
1097 ; CHECK-SD-NEXT: add x9, sp, #256
1098 ; CHECK-SD-NEXT: add x8, sp, #264
1099 ; CHECK-SD-NEXT: add x10, sp, #72
1100 ; CHECK-SD-NEXT: ld1 { v0.b }[9], [x12]
1101 ; CHECK-SD-NEXT: add x12, sp, #392
1102 ; CHECK-SD-NEXT: movi v16.2d, #0000000000000000
1103 ; CHECK-SD-NEXT: ld1 { v5.b }[7], [x12]
1104 ; CHECK-SD-NEXT: add x12, sp, #48
1105 ; CHECK-SD-NEXT: ld1 { v1.b }[12], [x15]
1106 ; CHECK-SD-NEXT: add x15, sp, #120
1107 ; CHECK-SD-NEXT: movi v17.2d, #0000000000000000
1108 ; CHECK-SD-NEXT: movi v18.2d, #0000000000000000
1109 ; CHECK-SD-NEXT: ld1 { v0.b }[10], [x13]
1110 ; CHECK-SD-NEXT: ld1 { v3.b }[5], [x15]
1111 ; CHECK-SD-NEXT: add x15, sp, #408
1112 ; CHECK-SD-NEXT: ld1 { v5.b }[8], [x11]
1113 ; CHECK-SD-NEXT: add x13, sp, #56
1114 ; CHECK-SD-NEXT: ld1 { v1.b }[13], [x16]
1115 ; CHECK-SD-NEXT: add x11, sp, #64
1116 ; CHECK-SD-NEXT: add x16, sp, #616
1117 ; CHECK-SD-NEXT: movi v19.2d, #0000000000000000
1118 ; CHECK-SD-NEXT: ld1 { v0.b }[11], [x14]
1119 ; CHECK-SD-NEXT: add x14, sp, #600
1120 ; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14]
1121 ; CHECK-SD-NEXT: ld1 { v5.b }[9], [x15]
1122 ; CHECK-SD-NEXT: add x15, sp, #608
1123 ; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9]
1124 ; CHECK-SD-NEXT: add x9, sp, #488
1125 ; CHECK-SD-NEXT: add x14, sp, #320
1126 ; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12]
1127 ; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
1128 ; CHECK-SD-NEXT: ld1 { v2.b }[6], [x14]
1129 ; CHECK-SD-NEXT: ld1 { v4.b }[10], [x15]
1130 ; CHECK-SD-NEXT: add x14, sp, #624
1131 ; CHECK-SD-NEXT: add x9, sp, #688
1132 ; CHECK-SD-NEXT: ld1 { v1.b }[15], [x8]
1133 ; CHECK-SD-NEXT: add x8, sp, #432
1134 ; CHECK-SD-NEXT: add x12, sp, #328
1135 ; CHECK-SD-NEXT: ld1 { v0.b }[13], [x13]
1136 ; CHECK-SD-NEXT: add x13, sp, #416
1137 ; CHECK-SD-NEXT: ld1 { v2.b }[7], [x12]
1138 ; CHECK-SD-NEXT: ld1 { v5.b }[10], [x13]
1139 ; CHECK-SD-NEXT: ld1 { v4.b }[11], [x16]
1140 ; CHECK-SD-NEXT: add x16, sp, #680
1141 ; CHECK-SD-NEXT: ld1 { v6.b }[3], [x16]
1142 ; CHECK-SD-NEXT: add x13, sp, #632
1143 ; CHECK-SD-NEXT: add x12, sp, #504
1144 ; CHECK-SD-NEXT: ld1 { v0.b }[14], [x11]
1145 ; CHECK-SD-NEXT: add x11, sp, #424
1146 ; CHECK-SD-NEXT: add x15, sp, #128
1147 ; CHECK-SD-NEXT: ld1 { v5.b }[11], [x11]
1148 ; CHECK-SD-NEXT: ld1 { v4.b }[12], [x14]
1149 ; CHECK-SD-NEXT: add x11, sp, #696
1150 ; CHECK-SD-NEXT: ld1 { v6.b }[4], [x9]
1151 ; CHECK-SD-NEXT: ld1 { v3.b }[6], [x15]
1152 ; CHECK-SD-NEXT: add x9, sp, #640
1153 ; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10]
1154 ; CHECK-SD-NEXT: add x10, sp, #496
1155 ; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8]
1156 ; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10]
1157 ; CHECK-SD-NEXT: ld1 { v4.b }[13], [x13]
1158 ; CHECK-SD-NEXT: add x10, sp, #440
1159 ; CHECK-SD-NEXT: ld1 { v6.b }[5], [x11]
1160 ; CHECK-SD-NEXT: add x11, sp, #512
1161 ; CHECK-SD-NEXT: add x8, sp, #136
1162 ; CHECK-SD-NEXT: sdot v17.4s, v0.16b, v1.16b
1163 ; CHECK-SD-NEXT: ld1 { v5.b }[13], [x10]
1164 ; CHECK-SD-NEXT: ld1 { v7.b }[5], [x12]
1165 ; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9]
1166 ; CHECK-SD-NEXT: add x9, sp, #448
1167 ; CHECK-SD-NEXT: add x10, sp, #704
1168 ; CHECK-SD-NEXT: ld1 { v3.b }[7], [x8]
1169 ; CHECK-SD-NEXT: ld1 { v6.b }[6], [x10]
1170 ; CHECK-SD-NEXT: add x8, sp, #648
1171 ; CHECK-SD-NEXT: add x10, sp, #520
1172 ; CHECK-SD-NEXT: ld1 { v5.b }[14], [x9]
1173 ; CHECK-SD-NEXT: ld1 { v7.b }[6], [x11]
1174 ; CHECK-SD-NEXT: ld1 { v4.b }[15], [x8]
1175 ; CHECK-SD-NEXT: add x8, sp, #456
1176 ; CHECK-SD-NEXT: add x9, sp, #712
1177 ; CHECK-SD-NEXT: sdot v19.2s, v3.8b, v2.8b
1178 ; CHECK-SD-NEXT: ld1 { v6.b }[7], [x9]
1179 ; CHECK-SD-NEXT: addv s0, v17.4s
1180 ; CHECK-SD-NEXT: ld1 { v5.b }[15], [x8]
1181 ; CHECK-SD-NEXT: ld1 { v7.b }[7], [x10]
1182 ; CHECK-SD-NEXT: addp v1.2s, v19.2s, v19.2s
1183 ; CHECK-SD-NEXT: fmov w8, s0
1184 ; CHECK-SD-NEXT: sdot v16.4s, v5.16b, v4.16b
1185 ; CHECK-SD-NEXT: sdot v18.2s, v7.8b, v6.8b
1186 ; CHECK-SD-NEXT: fmov w9, s1
1187 ; CHECK-SD-NEXT: addv s2, v16.4s
1188 ; CHECK-SD-NEXT: addp v3.2s, v18.2s, v18.2s
1189 ; CHECK-SD-NEXT: add w8, w8, w9
1190 ; CHECK-SD-NEXT: fmov w10, s2
1191 ; CHECK-SD-NEXT: fmov w11, s3
1192 ; CHECK-SD-NEXT: add w9, w10, w11
1193 ; CHECK-SD-NEXT: add w0, w8, w9
1194 ; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
1195 ; CHECK-SD-NEXT: ret
1196 ;
1197 ; CHECK-GI-LABEL: test_sdot_v24i8_double:
1198 ; CHECK-GI: // %bb.0: // %entry
1199 ; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
1200 ; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
1201 ; CHECK-GI-NEXT: .cfi_offset w29, -16
1202 ; CHECK-GI-NEXT: ldr w8, [sp, #80]
1203 ; CHECK-GI-NEXT: ldr w9, [sp, #88]
1204 ; CHECK-GI-NEXT: fmov s1, w0
1205 ; CHECK-GI-NEXT: ldr w11, [sp, #336]
1206 ; CHECK-GI-NEXT: ldr w10, [sp, #280]
1207 ; CHECK-GI-NEXT: movi v16.2d, #0000000000000000
1208 ; CHECK-GI-NEXT: fmov s0, w8
1209 ; CHECK-GI-NEXT: ldr w8, [sp, #96]
1210 ; CHECK-GI-NEXT: ldr w12, [sp, #152]
1211 ; CHECK-GI-NEXT: mov v1.b[1], w1
1212 ; CHECK-GI-NEXT: fmov s4, w11
1213 ; CHECK-GI-NEXT: ldr w11, [sp, #584]
1214 ; CHECK-GI-NEXT: movi v17.2d, #0000000000000000
1215 ; CHECK-GI-NEXT: movi v18.2d, #0000000000000000
1216 ; CHECK-GI-NEXT: movi v19.2d, #0000000000000000
1217 ; CHECK-GI-NEXT: mov v0.b[1], w9
1218 ; CHECK-GI-NEXT: ldr w9, [sp, #272]
1219 ; CHECK-GI-NEXT: fmov s2, w9
1220 ; CHECK-GI-NEXT: ldr w9, [sp, #144]
1221 ; CHECK-GI-NEXT: mov v1.b[2], w2
1222 ; CHECK-GI-NEXT: mov v0.b[2], w8
1223 ; CHECK-GI-NEXT: ldr w8, [sp, #528]
1224 ; CHECK-GI-NEXT: fmov s3, w9
1225 ; CHECK-GI-NEXT: mov v2.b[1], w10
1226 ; CHECK-GI-NEXT: ldr w9, [sp, #344]
1227 ; CHECK-GI-NEXT: ldr w10, [sp, #536]
1228 ; CHECK-GI-NEXT: fmov s5, w8
1229 ; CHECK-GI-NEXT: ldr w8, [sp, #288]
1230 ; CHECK-GI-NEXT: mov v1.b[3], w3
1231 ; CHECK-GI-NEXT: mov v3.b[1], w12
1232 ; CHECK-GI-NEXT: mov v4.b[1], w9
1233 ; CHECK-GI-NEXT: ldr w9, [sp, #160]
1234 ; CHECK-GI-NEXT: mov v5.b[1], w10
1235 ; CHECK-GI-NEXT: mov v2.b[2], w8
1236 ; CHECK-GI-NEXT: ldr w8, [sp, #104]
1237 ; CHECK-GI-NEXT: ldr w10, [sp, #352]
1238 ; CHECK-GI-NEXT: mov v1.b[4], w4
1239 ; CHECK-GI-NEXT: mov v3.b[2], w9
1240 ; CHECK-GI-NEXT: ldr w9, [sp, #544]
1241 ; CHECK-GI-NEXT: mov v0.b[3], w8
1242 ; CHECK-GI-NEXT: ldr w8, [sp, #296]
1243 ; CHECK-GI-NEXT: mov v4.b[2], w10
1244 ; CHECK-GI-NEXT: ldr w10, [sp, #360]
1245 ; CHECK-GI-NEXT: mov v5.b[2], w9
1246 ; CHECK-GI-NEXT: ldr w9, [sp, #168]
1247 ; CHECK-GI-NEXT: mov v2.b[3], w8
1248 ; CHECK-GI-NEXT: ldr w8, [sp, #112]
1249 ; CHECK-GI-NEXT: mov v1.b[5], w5
1250 ; CHECK-GI-NEXT: mov v3.b[3], w9
1251 ; CHECK-GI-NEXT: ldr w9, [sp, #552]
1252 ; CHECK-GI-NEXT: mov v0.b[4], w8
1253 ; CHECK-GI-NEXT: ldr w8, [sp, #304]
1254 ; CHECK-GI-NEXT: mov v4.b[3], w10
1255 ; CHECK-GI-NEXT: mov v5.b[3], w9
1256 ; CHECK-GI-NEXT: ldr w9, [sp, #176]
1257 ; CHECK-GI-NEXT: ldr w10, [sp, #368]
1258 ; CHECK-GI-NEXT: mov v2.b[4], w8
1259 ; CHECK-GI-NEXT: ldr w8, [sp, #120]
1260 ; CHECK-GI-NEXT: mov v1.b[6], w6
1261 ; CHECK-GI-NEXT: mov v3.b[4], w9
1262 ; CHECK-GI-NEXT: ldr w9, [sp, #560]
1263 ; CHECK-GI-NEXT: mov v0.b[5], w8
1264 ; CHECK-GI-NEXT: ldr w8, [sp, #312]
1265 ; CHECK-GI-NEXT: mov v4.b[4], w10
1266 ; CHECK-GI-NEXT: mov v5.b[4], w9
1267 ; CHECK-GI-NEXT: ldr w9, [sp, #184]
1268 ; CHECK-GI-NEXT: ldr w10, [sp, #376]
1269 ; CHECK-GI-NEXT: mov v2.b[5], w8
1270 ; CHECK-GI-NEXT: ldr w8, [sp, #128]
1271 ; CHECK-GI-NEXT: mov v1.b[7], w7
1272 ; CHECK-GI-NEXT: mov v3.b[5], w9
1273 ; CHECK-GI-NEXT: ldr w9, [sp, #568]
1274 ; CHECK-GI-NEXT: mov v0.b[6], w8
1275 ; CHECK-GI-NEXT: ldr w8, [sp, #320]
1276 ; CHECK-GI-NEXT: mov v4.b[5], w10
1277 ; CHECK-GI-NEXT: mov v5.b[5], w9
1278 ; CHECK-GI-NEXT: ldr w9, [sp, #192]
1279 ; CHECK-GI-NEXT: ldr w10, [sp, #384]
1280 ; CHECK-GI-NEXT: mov v2.b[6], w8
1281 ; CHECK-GI-NEXT: ldr w8, [sp, #136]
1282 ; CHECK-GI-NEXT: mov v3.b[6], w9
1283 ; CHECK-GI-NEXT: ldr w9, [sp, #576]
1284 ; CHECK-GI-NEXT: mov v0.b[7], w8
1285 ; CHECK-GI-NEXT: ldr w8, [sp, #328]
1286 ; CHECK-GI-NEXT: mov v4.b[6], w10
1287 ; CHECK-GI-NEXT: ldr w10, [sp, #200]
1288 ; CHECK-GI-NEXT: mov v5.b[6], w9
1289 ; CHECK-GI-NEXT: ldr w9, [sp, #392]
1290 ; CHECK-GI-NEXT: mov v2.b[7], w8
1291 ; CHECK-GI-NEXT: ldr w8, [sp, #464]
1292 ; CHECK-GI-NEXT: mov v3.b[7], w10
1293 ; CHECK-GI-NEXT: ldr w10, [sp, #16]
1294 ; CHECK-GI-NEXT: fmov s6, w8
1295 ; CHECK-GI-NEXT: ldr w8, [sp, #208]
1296 ; CHECK-GI-NEXT: mov v4.b[7], w9
1297 ; CHECK-GI-NEXT: mov v1.b[8], w10
1298 ; CHECK-GI-NEXT: ldr w10, [sp, #656]
1299 ; CHECK-GI-NEXT: ldr w9, [sp, #472]
1300 ; CHECK-GI-NEXT: mov v5.b[7], w11
1301 ; CHECK-GI-NEXT: ldr w11, [sp, #400]
1302 ; CHECK-GI-NEXT: fmov d0, d0
1303 ; CHECK-GI-NEXT: fmov s7, w10
1304 ; CHECK-GI-NEXT: mov v6.b[1], w9
1305 ; CHECK-GI-NEXT: ldr w9, [sp, #592]
1306 ; CHECK-GI-NEXT: mov v3.b[8], w8
1307 ; CHECK-GI-NEXT: ldr w10, [sp, #664]
1308 ; CHECK-GI-NEXT: ldr w8, [sp, #24]
1309 ; CHECK-GI-NEXT: mov v4.b[8], w11
1310 ; CHECK-GI-NEXT: ldr w11, [sp, #216]
1311 ; CHECK-GI-NEXT: fmov d2, d2
1312 ; CHECK-GI-NEXT: mov v5.b[8], w9
1313 ; CHECK-GI-NEXT: ldr w9, [sp, #480]
1314 ; CHECK-GI-NEXT: mov v7.b[1], w10
1315 ; CHECK-GI-NEXT: mov v1.b[9], w8
1316 ; CHECK-GI-NEXT: ldr w8, [sp, #408]
1317 ; CHECK-GI-NEXT: ldr w10, [sp, #600]
1318 ; CHECK-GI-NEXT: mov v3.b[9], w11
1319 ; CHECK-GI-NEXT: mov v6.b[2], w9
1320 ; CHECK-GI-NEXT: ldr w9, [sp, #672]
1321 ; CHECK-GI-NEXT: ldr w11, [sp, #32]
1322 ; CHECK-GI-NEXT: mov v4.b[9], w8
1323 ; CHECK-GI-NEXT: ldr w8, [sp, #224]
1324 ; CHECK-GI-NEXT: mov v5.b[9], w10
1325 ; CHECK-GI-NEXT: ldr w10, [sp, #488]
1326 ; CHECK-GI-NEXT: mov v7.b[2], w9
1327 ; CHECK-GI-NEXT: mov v1.b[10], w11
1328 ; CHECK-GI-NEXT: ldr w9, [sp, #416]
1329 ; CHECK-GI-NEXT: ldr w11, [sp, #608]
1330 ; CHECK-GI-NEXT: mov v3.b[10], w8
1331 ; CHECK-GI-NEXT: mov v6.b[3], w10
1332 ; CHECK-GI-NEXT: ldr w10, [sp, #680]
1333 ; CHECK-GI-NEXT: ldr w8, [sp, #40]
1334 ; CHECK-GI-NEXT: mov v4.b[10], w9
1335 ; CHECK-GI-NEXT: ldr w9, [sp, #232]
1336 ; CHECK-GI-NEXT: mov v5.b[10], w11
1337 ; CHECK-GI-NEXT: ldr w11, [sp, #496]
1338 ; CHECK-GI-NEXT: mov v7.b[3], w10
1339 ; CHECK-GI-NEXT: mov v1.b[11], w8
1340 ; CHECK-GI-NEXT: ldr w8, [sp, #424]
1341 ; CHECK-GI-NEXT: ldr w10, [sp, #616]
1342 ; CHECK-GI-NEXT: mov v3.b[11], w9
1343 ; CHECK-GI-NEXT: mov v6.b[4], w11
1344 ; CHECK-GI-NEXT: ldr w11, [sp, #688]
1345 ; CHECK-GI-NEXT: ldr w9, [sp, #48]
1346 ; CHECK-GI-NEXT: mov v4.b[11], w8
1347 ; CHECK-GI-NEXT: ldr w8, [sp, #240]
1348 ; CHECK-GI-NEXT: mov v5.b[11], w10
1349 ; CHECK-GI-NEXT: ldr w10, [sp, #504]
1350 ; CHECK-GI-NEXT: mov v7.b[4], w11
1351 ; CHECK-GI-NEXT: mov v1.b[12], w9
1352 ; CHECK-GI-NEXT: ldr w9, [sp, #432]
1353 ; CHECK-GI-NEXT: ldr w11, [sp, #624]
1354 ; CHECK-GI-NEXT: mov v3.b[12], w8
1355 ; CHECK-GI-NEXT: mov v6.b[5], w10
1356 ; CHECK-GI-NEXT: ldr w10, [sp, #696]
1357 ; CHECK-GI-NEXT: ldr w8, [sp, #56]
1358 ; CHECK-GI-NEXT: mov v4.b[12], w9
1359 ; CHECK-GI-NEXT: ldr w9, [sp, #248]
1360 ; CHECK-GI-NEXT: mov v5.b[12], w11
1361 ; CHECK-GI-NEXT: ldr w11, [sp, #512]
1362 ; CHECK-GI-NEXT: mov v7.b[5], w10
1363 ; CHECK-GI-NEXT: mov v1.b[13], w8
1364 ; CHECK-GI-NEXT: ldr w8, [sp, #440]
1365 ; CHECK-GI-NEXT: ldr w10, [sp, #632]
1366 ; CHECK-GI-NEXT: mov v3.b[13], w9
1367 ; CHECK-GI-NEXT: mov v6.b[6], w11
1368 ; CHECK-GI-NEXT: ldr w11, [sp, #704]
1369 ; CHECK-GI-NEXT: ldr w9, [sp, #64]
1370 ; CHECK-GI-NEXT: mov v4.b[13], w8
1371 ; CHECK-GI-NEXT: ldr w8, [sp, #256]
1372 ; CHECK-GI-NEXT: mov v5.b[13], w10
1373 ; CHECK-GI-NEXT: ldr w10, [sp, #520]
1374 ; CHECK-GI-NEXT: mov v7.b[6], w11
1375 ; CHECK-GI-NEXT: mov v1.b[14], w9
1376 ; CHECK-GI-NEXT: ldr w9, [sp, #448]
1377 ; CHECK-GI-NEXT: ldr w11, [sp, #640]
1378 ; CHECK-GI-NEXT: mov v3.b[14], w8
1379 ; CHECK-GI-NEXT: mov v6.b[7], w10
1380 ; CHECK-GI-NEXT: ldr w10, [sp, #712]
1381 ; CHECK-GI-NEXT: ldr w8, [sp, #72]
1382 ; CHECK-GI-NEXT: mov v4.b[14], w9
1383 ; CHECK-GI-NEXT: ldr w9, [sp, #264]
1384 ; CHECK-GI-NEXT: mov v5.b[14], w11
1385 ; CHECK-GI-NEXT: mov v7.b[7], w10
1386 ; CHECK-GI-NEXT: sdot v18.4s, v0.16b, v2.16b
1387 ; CHECK-GI-NEXT: mov v1.b[15], w8
1388 ; CHECK-GI-NEXT: ldr w8, [sp, #456]
1389 ; CHECK-GI-NEXT: mov v3.b[15], w9
1390 ; CHECK-GI-NEXT: ldr w9, [sp, #648]
1391 ; CHECK-GI-NEXT: fmov d6, d6
1392 ; CHECK-GI-NEXT: mov v4.b[15], w8
1393 ; CHECK-GI-NEXT: mov v5.b[15], w9
1394 ; CHECK-GI-NEXT: fmov d7, d7
1395 ; CHECK-GI-NEXT: sdot v17.4s, v1.16b, v3.16b
1396 ; CHECK-GI-NEXT: sdot v19.4s, v4.16b, v5.16b
1397 ; CHECK-GI-NEXT: sdot v16.4s, v6.16b, v7.16b
1398 ; CHECK-GI-NEXT: add v0.4s, v17.4s, v18.4s
1399 ; CHECK-GI-NEXT: add v1.4s, v19.4s, v16.4s
1400 ; CHECK-GI-NEXT: addv s0, v0.4s
1401 ; CHECK-GI-NEXT: addv s1, v1.4s
1402 ; CHECK-GI-NEXT: fmov w8, s0
1403 ; CHECK-GI-NEXT: fmov w9, s1
1404 ; CHECK-GI-NEXT: add w0, w8, w9
1405 ; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
1406 ; CHECK-GI-NEXT: ret
1407 entry:
1408 %az = sext <24 x i8> %a to <24 x i32>
1409 %bz = sext <24 x i8> %b to <24 x i32>
1410 %m1 = mul nuw nsw <24 x i32> %az, %bz
1411 %r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m1)
1412 %cz = sext <24 x i8> %c to <24 x i32>
1413 %dz = sext <24 x i8> %d to <24 x i32>
1414 %m2 = mul nuw nsw <24 x i32> %cz, %dz
1415 %r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m2)
1416 %x = add i32 %r1, %r2
1417 ret i32 %x
1418 }
1420 define i32 @test_sdot_v24i8_double_nomla(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
1421 ; CHECK-SD-LABEL: test_sdot_v24i8_double_nomla:
1422 ; CHECK-SD: // %bb.0: // %entry
1423 ; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
1424 ; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
1425 ; CHECK-SD-NEXT: .cfi_offset w29, -16
1426 ; CHECK-SD-NEXT: fmov s0, w0
1427 ; CHECK-SD-NEXT: ldr b1, [sp, #336]
1428 ; CHECK-SD-NEXT: add x8, sp, #344
1429 ; CHECK-SD-NEXT: add x9, sp, #400
1430 ; CHECK-SD-NEXT: ldr b2, [sp, #80]
1431 ; CHECK-SD-NEXT: ldr b3, [sp, #464]
1432 ; CHECK-SD-NEXT: ld1 { v1.b }[1], [x8]
1433 ; CHECK-SD-NEXT: add x8, sp, #352
1434 ; CHECK-SD-NEXT: add x10, sp, #408
1435 ; CHECK-SD-NEXT: mov v0.b[1], w1
1436 ; CHECK-SD-NEXT: add x11, sp, #472
1437 ; CHECK-SD-NEXT: add x12, sp, #480
1438 ; CHECK-SD-NEXT: ld1 { v3.b }[1], [x11]
1439 ; CHECK-SD-NEXT: add x11, sp, #416
1440 ; CHECK-SD-NEXT: add x13, sp, #488
1441 ; CHECK-SD-NEXT: ld1 { v1.b }[2], [x8]
1442 ; CHECK-SD-NEXT: add x8, sp, #360
1443 ; CHECK-SD-NEXT: add x14, sp, #496
1444 ; CHECK-SD-NEXT: movi v4.16b, #1
1445 ; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
1446 ; CHECK-SD-NEXT: movi v6.2d, #0000000000000000
1447 ; CHECK-SD-NEXT: mov v0.b[2], w2
1448 ; CHECK-SD-NEXT: ld1 { v3.b }[2], [x12]
1449 ; CHECK-SD-NEXT: add x12, sp, #424
1450 ; CHECK-SD-NEXT: ld1 { v1.b }[3], [x8]
1451 ; CHECK-SD-NEXT: add x8, sp, #368
1452 ; CHECK-SD-NEXT: movi v7.2d, #0000000000000000
1453 ; CHECK-SD-NEXT: movi v16.8b, #1
1454 ; CHECK-SD-NEXT: movi v17.2d, #0000000000000000
1455 ; CHECK-SD-NEXT: ld1 { v3.b }[3], [x13]
1456 ; CHECK-SD-NEXT: add x13, sp, #432
1457 ; CHECK-SD-NEXT: mov v0.b[3], w3
1458 ; CHECK-SD-NEXT: ld1 { v1.b }[4], [x8]
1459 ; CHECK-SD-NEXT: add x8, sp, #376
1460 ; CHECK-SD-NEXT: ld1 { v3.b }[4], [x14]
1461 ; CHECK-SD-NEXT: ld1 { v1.b }[5], [x8]
1462 ; CHECK-SD-NEXT: add x8, sp, #384
1463 ; CHECK-SD-NEXT: mov v0.b[4], w4
1464 ; CHECK-SD-NEXT: ld1 { v1.b }[6], [x8]
1465 ; CHECK-SD-NEXT: add x8, sp, #392
1466 ; CHECK-SD-NEXT: mov v0.b[5], w5
1467 ; CHECK-SD-NEXT: ld1 { v1.b }[7], [x8]
1468 ; CHECK-SD-NEXT: add x8, sp, #16
1469 ; CHECK-SD-NEXT: mov v0.b[6], w6
1470 ; CHECK-SD-NEXT: ld1 { v1.b }[8], [x9]
1471 ; CHECK-SD-NEXT: add x9, sp, #88
1472 ; CHECK-SD-NEXT: ld1 { v2.b }[1], [x9]
1473 ; CHECK-SD-NEXT: add x9, sp, #40
1474 ; CHECK-SD-NEXT: ld1 { v1.b }[9], [x10]
1475 ; CHECK-SD-NEXT: add x10, sp, #96
1476 ; CHECK-SD-NEXT: mov v0.b[7], w7
1477 ; CHECK-SD-NEXT: ld1 { v2.b }[2], [x10]
1478 ; CHECK-SD-NEXT: add x10, sp, #56
1479 ; CHECK-SD-NEXT: ld1 { v1.b }[10], [x11]
1480 ; CHECK-SD-NEXT: add x11, sp, #104
1481 ; CHECK-SD-NEXT: ld1 { v2.b }[3], [x11]
1482 ; CHECK-SD-NEXT: add x11, sp, #72
1483 ; CHECK-SD-NEXT: ld1 { v0.b }[8], [x8]
1484 ; CHECK-SD-NEXT: add x8, sp, #24
1485 ; CHECK-SD-NEXT: ld1 { v1.b }[11], [x12]
1486 ; CHECK-SD-NEXT: add x12, sp, #112
1487 ; CHECK-SD-NEXT: ld1 { v2.b }[4], [x12]
1488 ; CHECK-SD-NEXT: add x12, sp, #440
1489 ; CHECK-SD-NEXT: ld1 { v0.b }[9], [x8]
1490 ; CHECK-SD-NEXT: add x8, sp, #32
1491 ; CHECK-SD-NEXT: ld1 { v1.b }[12], [x13]
1492 ; CHECK-SD-NEXT: add x13, sp, #504
1493 ; CHECK-SD-NEXT: ld1 { v3.b }[5], [x13]
1494 ; CHECK-SD-NEXT: add x13, sp, #512
1495 ; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8]
1496 ; CHECK-SD-NEXT: add x8, sp, #48
1497 ; CHECK-SD-NEXT: ld1 { v1.b }[13], [x12]
1498 ; CHECK-SD-NEXT: add x12, sp, #448
1499 ; CHECK-SD-NEXT: ld1 { v3.b }[6], [x13]
1500 ; CHECK-SD-NEXT: ld1 { v0.b }[11], [x9]
1501 ; CHECK-SD-NEXT: add x9, sp, #64
1502 ; CHECK-SD-NEXT: ld1 { v1.b }[14], [x12]
1503 ; CHECK-SD-NEXT: ld1 { v0.b }[12], [x8]
1504 ; CHECK-SD-NEXT: add x8, sp, #120
1505 ; CHECK-SD-NEXT: ld1 { v2.b }[5], [x8]
1506 ; CHECK-SD-NEXT: add x8, sp, #128
1507 ; CHECK-SD-NEXT: ld1 { v0.b }[13], [x10]
1508 ; CHECK-SD-NEXT: add x10, sp, #136
1509 ; CHECK-SD-NEXT: ld1 { v2.b }[6], [x8]
1510 ; CHECK-SD-NEXT: add x8, sp, #456
1511 ; CHECK-SD-NEXT: ld1 { v1.b }[15], [x8]
1512 ; CHECK-SD-NEXT: ld1 { v0.b }[14], [x9]
1513 ; CHECK-SD-NEXT: add x9, sp, #520
1514 ; CHECK-SD-NEXT: ld1 { v2.b }[7], [x10]
1515 ; CHECK-SD-NEXT: ld1 { v3.b }[7], [x9]
1516 ; CHECK-SD-NEXT: sdot v5.4s, v1.16b, v4.16b
1517 ; CHECK-SD-NEXT: ld1 { v0.b }[15], [x11]
1518 ; CHECK-SD-NEXT: sdot v17.2s, v2.8b, v16.8b
1519 ; CHECK-SD-NEXT: sdot v7.2s, v3.8b, v16.8b
1520 ; CHECK-SD-NEXT: sdot v6.4s, v0.16b, v4.16b
1521 ; CHECK-SD-NEXT: addv s3, v5.4s
1522 ; CHECK-SD-NEXT: addp v1.2s, v17.2s, v17.2s
1523 ; CHECK-SD-NEXT: addp v2.2s, v7.2s, v7.2s
1524 ; CHECK-SD-NEXT: fmov w10, s3
1525 ; CHECK-SD-NEXT: addv s0, v6.4s
1526 ; CHECK-SD-NEXT: fmov w9, s1
1527 ; CHECK-SD-NEXT: fmov w11, s2
1528 ; CHECK-SD-NEXT: fmov w8, s0
1529 ; CHECK-SD-NEXT: add w8, w8, w9
1530 ; CHECK-SD-NEXT: add w9, w10, w11
1531 ; CHECK-SD-NEXT: add w0, w8, w9
1532 ; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
1533 ; CHECK-SD-NEXT: ret
1534 ;
1535 ; CHECK-GI-LABEL: test_sdot_v24i8_double_nomla:
1536 ; CHECK-GI: // %bb.0: // %entry
1537 ; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
1538 ; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
1539 ; CHECK-GI-NEXT: .cfi_offset w29, -16
1540 ; CHECK-GI-NEXT: ldr w9, [sp, #336]
1541 ; CHECK-GI-NEXT: ldr w8, [sp, #344]
1542 ; CHECK-GI-NEXT: fmov s0, w0
1543 ; CHECK-GI-NEXT: ldr w10, [sp, #16]
1544 ; CHECK-GI-NEXT: ldr w11, [sp, #88]
1545 ; CHECK-GI-NEXT: movi v4.8b, #1
1546 ; CHECK-GI-NEXT: fmov s1, w9
1547 ; CHECK-GI-NEXT: ldr w9, [sp, #464]
1548 ; CHECK-GI-NEXT: ldr w12, [sp, #400]
1549 ; CHECK-GI-NEXT: mov v0.b[1], w1
1550 ; CHECK-GI-NEXT: movi v5.8b, #1
1551 ; CHECK-GI-NEXT: movi v6.8b, #1
1552 ; CHECK-GI-NEXT: fmov s2, w9
1553 ; CHECK-GI-NEXT: ldr w9, [sp, #96]
1554 ; CHECK-GI-NEXT: movi v7.2d, #0000000000000000
1555 ; CHECK-GI-NEXT: mov v1.b[1], w8
1556 ; CHECK-GI-NEXT: ldr w8, [sp, #352]
1557 ; CHECK-GI-NEXT: movi v16.2d, #0000000000000000
1558 ; CHECK-GI-NEXT: movi v17.2d, #0000000000000000
1559 ; CHECK-GI-NEXT: movi v18.2d, #0000000000000000
1560 ; CHECK-GI-NEXT: mov v0.b[2], w2
1561 ; CHECK-GI-NEXT: mov v5.d[1], v4.d[0]
1562 ; CHECK-GI-NEXT: mov v6.d[1], v4.d[0]
1563 ; CHECK-GI-NEXT: mov v1.b[2], w8
1564 ; CHECK-GI-NEXT: ldr w8, [sp, #360]
1565 ; CHECK-GI-NEXT: mov v0.b[3], w3
1566 ; CHECK-GI-NEXT: mov v1.b[3], w8
1567 ; CHECK-GI-NEXT: ldr w8, [sp, #368]
1568 ; CHECK-GI-NEXT: mov v0.b[4], w4
1569 ; CHECK-GI-NEXT: mov v1.b[4], w8
1570 ; CHECK-GI-NEXT: ldr w8, [sp, #376]
1571 ; CHECK-GI-NEXT: mov v0.b[5], w5
1572 ; CHECK-GI-NEXT: mov v1.b[5], w8
1573 ; CHECK-GI-NEXT: ldr w8, [sp, #384]
1574 ; CHECK-GI-NEXT: mov v0.b[6], w6
1575 ; CHECK-GI-NEXT: mov v1.b[6], w8
1576 ; CHECK-GI-NEXT: ldr w8, [sp, #392]
1577 ; CHECK-GI-NEXT: mov v0.b[7], w7
1578 ; CHECK-GI-NEXT: mov v1.b[7], w8
1579 ; CHECK-GI-NEXT: ldr w8, [sp, #80]
1580 ; CHECK-GI-NEXT: fmov s3, w8
1581 ; CHECK-GI-NEXT: ldr w8, [sp, #472]
1582 ; CHECK-GI-NEXT: mov v0.b[8], w10
1583 ; CHECK-GI-NEXT: ldr w10, [sp, #408]
1584 ; CHECK-GI-NEXT: mov v1.b[8], w12
1585 ; CHECK-GI-NEXT: mov v2.b[1], w8
1586 ; CHECK-GI-NEXT: ldr w8, [sp, #24]
1587 ; CHECK-GI-NEXT: mov v3.b[1], w11
1588 ; CHECK-GI-NEXT: ldr w11, [sp, #480]
1589 ; CHECK-GI-NEXT: mov v0.b[9], w8
1590 ; CHECK-GI-NEXT: ldr w8, [sp, #32]
1591 ; CHECK-GI-NEXT: mov v1.b[9], w10
1592 ; CHECK-GI-NEXT: mov v2.b[2], w11
1593 ; CHECK-GI-NEXT: ldr w10, [sp, #416]
1594 ; CHECK-GI-NEXT: mov v3.b[2], w9
1595 ; CHECK-GI-NEXT: ldr w9, [sp, #104]
1596 ; CHECK-GI-NEXT: ldr w11, [sp, #488]
1597 ; CHECK-GI-NEXT: mov v0.b[10], w8
1598 ; CHECK-GI-NEXT: ldr w8, [sp, #40]
1599 ; CHECK-GI-NEXT: mov v1.b[10], w10
1600 ; CHECK-GI-NEXT: mov v2.b[3], w11
1601 ; CHECK-GI-NEXT: ldr w10, [sp, #424]
1602 ; CHECK-GI-NEXT: mov v3.b[3], w9
1603 ; CHECK-GI-NEXT: ldr w9, [sp, #112]
1604 ; CHECK-GI-NEXT: ldr w11, [sp, #496]
1605 ; CHECK-GI-NEXT: mov v0.b[11], w8
1606 ; CHECK-GI-NEXT: ldr w8, [sp, #48]
1607 ; CHECK-GI-NEXT: mov v1.b[11], w10
1608 ; CHECK-GI-NEXT: mov v2.b[4], w11
1609 ; CHECK-GI-NEXT: ldr w10, [sp, #432]
1610 ; CHECK-GI-NEXT: mov v3.b[4], w9
1611 ; CHECK-GI-NEXT: ldr w9, [sp, #120]
1612 ; CHECK-GI-NEXT: ldr w11, [sp, #504]
1613 ; CHECK-GI-NEXT: mov v0.b[12], w8
1614 ; CHECK-GI-NEXT: ldr w8, [sp, #56]
1615 ; CHECK-GI-NEXT: mov v1.b[12], w10
1616 ; CHECK-GI-NEXT: mov v2.b[5], w11
1617 ; CHECK-GI-NEXT: ldr w10, [sp, #440]
1618 ; CHECK-GI-NEXT: mov v3.b[5], w9
1619 ; CHECK-GI-NEXT: ldr w9, [sp, #128]
1620 ; CHECK-GI-NEXT: ldr w11, [sp, #512]
1621 ; CHECK-GI-NEXT: mov v0.b[13], w8
1622 ; CHECK-GI-NEXT: ldr w8, [sp, #64]
1623 ; CHECK-GI-NEXT: mov v1.b[13], w10
1624 ; CHECK-GI-NEXT: mov v2.b[6], w11
1625 ; CHECK-GI-NEXT: ldr w10, [sp, #448]
1626 ; CHECK-GI-NEXT: mov v3.b[6], w9
1627 ; CHECK-GI-NEXT: ldr w9, [sp, #136]
1628 ; CHECK-GI-NEXT: ldr w11, [sp, #520]
1629 ; CHECK-GI-NEXT: mov v0.b[14], w8
1630 ; CHECK-GI-NEXT: ldr w8, [sp, #72]
1631 ; CHECK-GI-NEXT: mov v1.b[14], w10
1632 ; CHECK-GI-NEXT: mov v2.b[7], w11
1633 ; CHECK-GI-NEXT: mov v3.b[7], w9
1634 ; CHECK-GI-NEXT: ldr w9, [sp, #456]
1635 ; CHECK-GI-NEXT: mov v0.b[15], w8
1636 ; CHECK-GI-NEXT: mov v1.b[15], w9
1637 ; CHECK-GI-NEXT: fmov d2, d2
1638 ; CHECK-GI-NEXT: fmov d3, d3
1639 ; CHECK-GI-NEXT: sdot v16.4s, v0.16b, v5.16b
1640 ; CHECK-GI-NEXT: sdot v18.4s, v1.16b, v6.16b
1641 ; CHECK-GI-NEXT: sdot v7.4s, v2.16b, v4.16b
1642 ; CHECK-GI-NEXT: sdot v17.4s, v3.16b, v4.16b
1643 ; CHECK-GI-NEXT: add v1.4s, v18.4s, v7.4s
1644 ; CHECK-GI-NEXT: add v0.4s, v16.4s, v17.4s
1645 ; CHECK-GI-NEXT: addv s1, v1.4s
1646 ; CHECK-GI-NEXT: addv s0, v0.4s
1647 ; CHECK-GI-NEXT: fmov w9, s1
1648 ; CHECK-GI-NEXT: fmov w8, s0
1649 ; CHECK-GI-NEXT: add w0, w8, w9
1650 ; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
1651 ; CHECK-GI-NEXT: ret
1652 entry:
1653 %az = sext <24 x i8> %a to <24 x i32>
1654 %r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %az)
1655 %cz = sext <24 x i8> %c to <24 x i32>
1656 %r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %cz)
1657 %x = add i32 %r1, %r2
1658 ret i32 %x
1659 }
1662 define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
1663 ; CHECK-LABEL: test_udot_v25i8:
1664 ; CHECK: // %bb.0: // %entry
1665 ; CHECK-NEXT: ldp q3, q0, [x1]
1666 ; CHECK-NEXT: movi v5.2d, #0000000000000000
1667 ; CHECK-NEXT: ldp q2, q1, [x0]
1668 ; CHECK-NEXT: umull2 v4.8h, v0.16b, v1.16b
1669 ; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b
1670 ; CHECK-NEXT: umull v1.8h, v3.8b, v2.8b
1671 ; CHECK-NEXT: umull2 v2.8h, v3.16b, v2.16b
1672 ; CHECK-NEXT: ushll v3.4s, v4.4h, #0
1673 ; CHECK-NEXT: uaddl2 v4.4s, v1.8h, v0.8h
1674 ; CHECK-NEXT: uaddl v0.4s, v1.4h, v0.4h
1675 ; CHECK-NEXT: mov v5.s[0], v3.s[0]
1676 ; CHECK-NEXT: uaddw2 v1.4s, v4.4s, v2.8h
1677 ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
1678 ; CHECK-NEXT: uaddw v2.4s, v5.4s, v2.4h
1679 ; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
1680 ; CHECK-NEXT: addv s0, v0.4s
1681 ; CHECK-NEXT: fmov w8, s0
1682 ; CHECK-NEXT: add w0, w8, w2
1683 ; CHECK-NEXT: ret
1684 entry:
1685 %0 = load <25 x i8>, ptr %a
1686 %1 = zext <25 x i8> %0 to <25 x i32>
1687 %2 = load <25 x i8>, ptr %b
1688 %3 = zext <25 x i8> %2 to <25 x i32>
1689 %4 = mul nuw nsw <25 x i32> %3, %1
1690 %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
1691 %op.extra = add i32 %5, %sum
1692 ret i32 %op.extra
1693 }
1695 define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
1696 ; CHECK-LABEL: test_udot_v25i8_nomla:
1697 ; CHECK: // %bb.0: // %entry
1698 ; CHECK-NEXT: ldp q2, q1, [x0]
1699 ; CHECK-NEXT: movi v0.2d, #0000000000000000
1700 ; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0
1701 ; CHECK-NEXT: ushll v1.8h, v1.8b, #0
1702 ; CHECK-NEXT: ushll v4.8h, v2.8b, #0
1703 ; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
1704 ; CHECK-NEXT: ushll v3.4s, v3.4h, #0
1705 ; CHECK-NEXT: uaddl2 v5.4s, v4.8h, v1.8h
1706 ; CHECK-NEXT: uaddl v1.4s, v4.4h, v1.4h
1707 ; CHECK-NEXT: mov v0.s[0], v3.s[0]
1708 ; CHECK-NEXT: uaddw2 v3.4s, v5.4s, v2.8h
1709 ; CHECK-NEXT: add v1.4s, v1.4s, v3.4s
1710 ; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h
1711 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s
1712 ; CHECK-NEXT: addv s0, v0.4s
1713 ; CHECK-NEXT: fmov w0, s0
1714 ; CHECK-NEXT: ret
1715 entry:
1716 %0 = load <25 x i8>, ptr %a1
1717 %1 = zext <25 x i8> %0 to <25 x i32>
1718 %2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %1)
1719 ret i32 %2
1720 }
1721 define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
1722 ; CHECK-LABEL: test_sdot_v25i8:
1723 ; CHECK: // %bb.0: // %entry
1724 ; CHECK-NEXT: ldp q3, q0, [x1]
1725 ; CHECK-NEXT: movi v5.2d, #0000000000000000
1726 ; CHECK-NEXT: ldp q2, q1, [x0]
1727 ; CHECK-NEXT: smull2 v4.8h, v0.16b, v1.16b
1728 ; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b
1729 ; CHECK-NEXT: smull v1.8h, v3.8b, v2.8b
1730 ; CHECK-NEXT: smull2 v2.8h, v3.16b, v2.16b
1731 ; CHECK-NEXT: sshll v3.4s, v4.4h, #0
1732 ; CHECK-NEXT: saddl2 v4.4s, v1.8h, v0.8h
1733 ; CHECK-NEXT: saddl v0.4s, v1.4h, v0.4h
1734 ; CHECK-NEXT: mov v5.s[0], v3.s[0]
1735 ; CHECK-NEXT: saddw2 v1.4s, v4.4s, v2.8h
1736 ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
1737 ; CHECK-NEXT: saddw v2.4s, v5.4s, v2.4h
1738 ; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
1739 ; CHECK-NEXT: addv s0, v0.4s
1740 ; CHECK-NEXT: fmov w8, s0
1741 ; CHECK-NEXT: add w0, w8, w2
1742 ; CHECK-NEXT: ret
1743 entry:
1744 %0 = load <25 x i8>, ptr %a
1745 %1 = sext <25 x i8> %0 to <25 x i32>
1746 %2 = load <25 x i8>, ptr %b
1747 %3 = sext <25 x i8> %2 to <25 x i32>
1748 %4 = mul nsw <25 x i32> %3, %1
1749 %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
1750 %op.extra = add nsw i32 %5, %sum
1751 ret i32 %op.extra
1752 }
1754 define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
1755 ; CHECK-LABEL: test_sdot_v25i8_double:
1756 ; CHECK: // %bb.0: // %entry
1757 ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
1758 ; CHECK-NEXT: .cfi_def_cfa_offset 16
1759 ; CHECK-NEXT: .cfi_offset w29, -16
1760 ; CHECK-NEXT: ldr b0, [sp, #216]
1761 ; CHECK-NEXT: add x8, sp, #224
1762 ; CHECK-NEXT: ldr b1, [sp, #16]
1763 ; CHECK-NEXT: ldr b2, [sp, #280]
1764 ; CHECK-NEXT: add x9, sp, #240
1765 ; CHECK-NEXT: ldr b4, [sp, #80]
1766 ; CHECK-NEXT: ld1 { v0.b }[1], [x8]
1767 ; CHECK-NEXT: add x8, sp, #24
1768 ; CHECK-NEXT: add x10, sp, #48
1769 ; CHECK-NEXT: ld1 { v1.b }[1], [x8]
1770 ; CHECK-NEXT: add x8, sp, #232
1771 ; CHECK-NEXT: add x11, sp, #96
1772 ; CHECK-NEXT: ldr b5, [sp, #152]
1773 ; CHECK-NEXT: add x12, sp, #168
1774 ; CHECK-NEXT: ldr b6, [sp, #616]
1775 ; CHECK-NEXT: ld1 { v0.b }[2], [x8]
1776 ; CHECK-NEXT: add x8, sp, #32
1777 ; CHECK-NEXT: fmov s3, w0
1778 ; CHECK-NEXT: ld1 { v1.b }[2], [x8]
1779 ; CHECK-NEXT: add x8, sp, #288
1780 ; CHECK-NEXT: ldr b7, [sp, #416]
1781 ; CHECK-NEXT: ld1 { v2.b }[1], [x8]
1782 ; CHECK-NEXT: add x8, sp, #40
1783 ; CHECK-NEXT: ldr b22, [sp, #744]
1784 ; CHECK-NEXT: ld1 { v0.b }[3], [x9]
1785 ; CHECK-NEXT: add x9, sp, #248
1786 ; CHECK-NEXT: mov v3.b[1], w1
1787 ; CHECK-NEXT: ld1 { v1.b }[3], [x8]
1788 ; CHECK-NEXT: add x8, sp, #88
1789 ; CHECK-NEXT: ldr b23, [sp, #544]
1790 ; CHECK-NEXT: ld1 { v4.b }[1], [x8]
1791 ; CHECK-NEXT: add x8, sp, #256
1792 ; CHECK-NEXT: ldr b19, [sp, #680]
1793 ; CHECK-NEXT: ld1 { v0.b }[4], [x9]
1794 ; CHECK-NEXT: add x9, sp, #296
1795 ; CHECK-NEXT: ldr b20, [sp, #480]
1796 ; CHECK-NEXT: ld1 { v1.b }[4], [x10]
1797 ; CHECK-NEXT: ld1 { v2.b }[2], [x9]
1798 ; CHECK-NEXT: add x10, sp, #160
1799 ; CHECK-NEXT: ld1 { v4.b }[2], [x11]
1800 ; CHECK-NEXT: add x11, sp, #304
1801 ; CHECK-NEXT: ld1 { v5.b }[1], [x10]
1802 ; CHECK-NEXT: ld1 { v0.b }[5], [x8]
1803 ; CHECK-NEXT: add x8, sp, #56
1804 ; CHECK-NEXT: add x10, sp, #264
1805 ; CHECK-NEXT: ld1 { v1.b }[5], [x8]
1806 ; CHECK-NEXT: add x8, sp, #64
1807 ; CHECK-NEXT: ld1 { v2.b }[3], [x11]
1808 ; CHECK-NEXT: add x9, sp, #272
1809 ; CHECK-NEXT: ld1 { v5.b }[2], [x12]
1810 ; CHECK-NEXT: add x11, sp, #72
1811 ; CHECK-NEXT: ld1 { v0.b }[6], [x10]
1812 ; CHECK-NEXT: add x10, sp, #312
1813 ; CHECK-NEXT: mov v3.b[2], w2
1814 ; CHECK-NEXT: ld1 { v1.b }[6], [x8]
1815 ; CHECK-NEXT: add x8, sp, #104
1816 ; CHECK-NEXT: ld1 { v2.b }[4], [x10]
1817 ; CHECK-NEXT: ld1 { v4.b }[3], [x8]
1818 ; CHECK-NEXT: add x8, sp, #112
1819 ; CHECK-NEXT: add x10, sp, #128
1820 ; CHECK-NEXT: ld1 { v0.b }[7], [x9]
1821 ; CHECK-NEXT: add x9, sp, #320
1822 ; CHECK-NEXT: ldr b21, [sp, #552]
1823 ; CHECK-NEXT: ld1 { v2.b }[5], [x9]
1824 ; CHECK-NEXT: add x9, sp, #176
1825 ; CHECK-NEXT: ld1 { v1.b }[7], [x11]
1826 ; CHECK-NEXT: ld1 { v4.b }[4], [x8]
1827 ; CHECK-NEXT: add x8, sp, #624
1828 ; CHECK-NEXT: ld1 { v5.b }[3], [x9]
1829 ; CHECK-NEXT: ld1 { v6.b }[1], [x8]
1830 ; CHECK-NEXT: add x8, sp, #120
1831 ; CHECK-NEXT: add x9, sp, #328
1832 ; CHECK-NEXT: ld1 { v2.b }[6], [x9]
1833 ; CHECK-NEXT: add x9, sp, #184
1834 ; CHECK-NEXT: add x11, sp, #192
1835 ; CHECK-NEXT: ld1 { v4.b }[5], [x8]
1836 ; CHECK-NEXT: add x8, sp, #632
1837 ; CHECK-NEXT: ld1 { v5.b }[4], [x9]
1838 ; CHECK-NEXT: ld1 { v6.b }[2], [x8]
1839 ; CHECK-NEXT: add x9, sp, #640
1840 ; CHECK-NEXT: add x8, sp, #336
1841 ; CHECK-NEXT: ld1 { v2.b }[7], [x8]
1842 ; CHECK-NEXT: add x8, sp, #656
1843 ; CHECK-NEXT: smull v23.8h, v23.8b, v22.8b
1844 ; CHECK-NEXT: ld1 { v5.b }[5], [x11]
1845 ; CHECK-NEXT: add x11, sp, #648
1846 ; CHECK-NEXT: ld1 { v4.b }[6], [x10]
1847 ; CHECK-NEXT: ld1 { v6.b }[3], [x9]
1848 ; CHECK-NEXT: add x9, sp, #200
1849 ; CHECK-NEXT: add x10, sp, #136
1850 ; CHECK-NEXT: ldr b22, [sp, #352]
1851 ; CHECK-NEXT: add x12, sp, #360
1852 ; CHECK-NEXT: mov v3.b[3], w3
1853 ; CHECK-NEXT: ld1 { v5.b }[6], [x9]
1854 ; CHECK-NEXT: add x9, sp, #208
1855 ; CHECK-NEXT: ld1 { v4.b }[7], [x10]
1856 ; CHECK-NEXT: ld1 { v6.b }[4], [x11]
1857 ; CHECK-NEXT: add x11, sp, #424
1858 ; CHECK-NEXT: add x10, sp, #488
1859 ; CHECK-NEXT: ld1 { v7.b }[1], [x11]
1860 ; CHECK-NEXT: add x11, sp, #560
1861 ; CHECK-NEXT: ld1 { v20.b }[1], [x10]
1862 ; CHECK-NEXT: ld1 { v5.b }[7], [x9]
1863 ; CHECK-NEXT: add x9, sp, #440
1864 ; CHECK-NEXT: ld1 { v21.b }[1], [x11]
1865 ; CHECK-NEXT: ld1 { v6.b }[5], [x8]
1866 ; CHECK-NEXT: add x8, sp, #432
1867 ; CHECK-NEXT: ld1 { v22.b }[1], [x12]
1868 ; CHECK-NEXT: ld1 { v7.b }[2], [x8]
1869 ; CHECK-NEXT: add x11, sp, #496
1870 ; CHECK-NEXT: add x12, sp, #568
1871 ; CHECK-NEXT: add x13, sp, #368
1872 ; CHECK-NEXT: ld1 { v20.b }[2], [x11]
1873 ; CHECK-NEXT: ld1 { v21.b }[2], [x12]
1874 ; CHECK-NEXT: ld1 { v22.b }[2], [x13]
1875 ; CHECK-NEXT: add x10, sp, #448
1876 ; CHECK-NEXT: mov v3.b[4], w4
1877 ; CHECK-NEXT: ld1 { v7.b }[3], [x9]
1878 ; CHECK-NEXT: add x9, sp, #688
1879 ; CHECK-NEXT: add x11, sp, #576
1880 ; CHECK-NEXT: ld1 { v19.b }[1], [x9]
1881 ; CHECK-NEXT: add x9, sp, #696
1882 ; CHECK-NEXT: add x12, sp, #376
1883 ; CHECK-NEXT: ld1 { v21.b }[3], [x11]
1884 ; CHECK-NEXT: ld1 { v22.b }[3], [x12]
1885 ; CHECK-NEXT: add x11, sp, #512
1886 ; CHECK-NEXT: ld1 { v7.b }[4], [x10]
1887 ; CHECK-NEXT: add x10, sp, #504
1888 ; CHECK-NEXT: add x12, sp, #584
1889 ; CHECK-NEXT: ld1 { v19.b }[2], [x9]
1890 ; CHECK-NEXT: add x9, sp, #704
1891 ; CHECK-NEXT: ld1 { v20.b }[3], [x10]
1892 ; CHECK-NEXT: add x13, sp, #384
1893 ; CHECK-NEXT: mov v3.b[5], w5
1894 ; CHECK-NEXT: ld1 { v21.b }[4], [x12]
1895 ; CHECK-NEXT: ld1 { v22.b }[4], [x13]
1896 ; CHECK-NEXT: add x10, sp, #456
1897 ; CHECK-NEXT: ldr b16, [sp, #344]
1898 ; CHECK-NEXT: ld1 { v19.b }[3], [x9]
1899 ; CHECK-NEXT: add x9, sp, #712
1900 ; CHECK-NEXT: ld1 { v20.b }[4], [x11]
1901 ; CHECK-NEXT: ldr b17, [sp, #144]
1902 ; CHECK-NEXT: ld1 { v7.b }[5], [x10]
1903 ; CHECK-NEXT: add x10, sp, #520
1904 ; CHECK-NEXT: add x11, sp, #592
1905 ; CHECK-NEXT: add x12, sp, #392
1906 ; CHECK-NEXT: mov v3.b[6], w6
1907 ; CHECK-NEXT: ld1 { v19.b }[4], [x9]
1908 ; CHECK-NEXT: add x9, sp, #720
1909 ; CHECK-NEXT: ld1 { v20.b }[5], [x10]
1910 ; CHECK-NEXT: ld1 { v21.b }[5], [x11]
1911 ; CHECK-NEXT: ld1 { v22.b }[5], [x12]
1912 ; CHECK-NEXT: smull v16.8h, v17.8b, v16.8b
1913 ; CHECK-NEXT: add x8, sp, #664
1914 ; CHECK-NEXT: add x10, sp, #464
1915 ; CHECK-NEXT: add x11, sp, #528
1916 ; CHECK-NEXT: ld1 { v19.b }[5], [x9]
1917 ; CHECK-NEXT: add x9, sp, #728
1918 ; CHECK-NEXT: add x12, sp, #600
1919 ; CHECK-NEXT: add x13, sp, #400
1920 ; CHECK-NEXT: ld1 { v6.b }[6], [x8]
1921 ; CHECK-NEXT: ld1 { v20.b }[6], [x11]
1922 ; CHECK-NEXT: ld1 { v21.b }[6], [x12]
1923 ; CHECK-NEXT: ld1 { v22.b }[6], [x13]
1924 ; CHECK-NEXT: ld1 { v7.b }[6], [x10]
1925 ; CHECK-NEXT: ld1 { v19.b }[6], [x9]
1926 ; CHECK-NEXT: add x9, sp, #736
1927 ; CHECK-NEXT: mov v3.b[7], w7
1928 ; CHECK-NEXT: sshll v18.4s, v16.4h, #0
1929 ; CHECK-NEXT: movi v16.2d, #0000000000000000
1930 ; CHECK-NEXT: movi v17.2d, #0000000000000000
1931 ; CHECK-NEXT: add x8, sp, #672
1932 ; CHECK-NEXT: add x10, sp, #472
1933 ; CHECK-NEXT: add x11, sp, #608
1934 ; CHECK-NEXT: ld1 { v19.b }[7], [x9]
1935 ; CHECK-NEXT: add x9, sp, #536
1936 ; CHECK-NEXT: add x12, sp, #408
1937 ; CHECK-NEXT: ld1 { v20.b }[7], [x9]
1938 ; CHECK-NEXT: ld1 { v21.b }[7], [x11]
1939 ; CHECK-NEXT: ld1 { v22.b }[7], [x12]
1940 ; CHECK-NEXT: ld1 { v6.b }[7], [x8]
1941 ; CHECK-NEXT: ld1 { v7.b }[7], [x10]
1942 ; CHECK-NEXT: sshll v23.4s, v23.4h, #0
1943 ; CHECK-NEXT: smull v0.8h, v1.8b, v0.8b
1944 ; CHECK-NEXT: smull v1.8h, v4.8b, v2.8b
1945 ; CHECK-NEXT: smull v2.8h, v3.8b, v5.8b
1946 ; CHECK-NEXT: smull v3.8h, v20.8b, v19.8b
1947 ; CHECK-NEXT: smull v4.8h, v22.8b, v21.8b
1948 ; CHECK-NEXT: mov v17.s[0], v18.s[0]
1949 ; CHECK-NEXT: smull v5.8h, v7.8b, v6.8b
1950 ; CHECK-NEXT: mov v16.s[0], v23.s[0]
1951 ; CHECK-NEXT: saddl2 v6.4s, v2.8h, v1.8h
1952 ; CHECK-NEXT: saddl v1.4s, v2.4h, v1.4h
1953 ; CHECK-NEXT: saddl2 v2.4s, v4.8h, v3.8h
1954 ; CHECK-NEXT: saddl v3.4s, v4.4h, v3.4h
1955 ; CHECK-NEXT: saddw v4.4s, v17.4s, v0.4h
1956 ; CHECK-NEXT: saddw v7.4s, v16.4s, v5.4h
1957 ; CHECK-NEXT: saddw2 v0.4s, v6.4s, v0.8h
1958 ; CHECK-NEXT: add v1.4s, v1.4s, v4.4s
1959 ; CHECK-NEXT: saddw2 v2.4s, v2.4s, v5.8h
1960 ; CHECK-NEXT: add v3.4s, v3.4s, v7.4s
1961 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s
1962 ; CHECK-NEXT: add v1.4s, v3.4s, v2.4s
1963 ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
1964 ; CHECK-NEXT: addv s0, v0.4s
1965 ; CHECK-NEXT: fmov w0, s0
1966 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
1967 ; CHECK-NEXT: ret
1968 entry:
1969 %az = sext <25 x i8> %a to <25 x i32>
1970 %bz = sext <25 x i8> %b to <25 x i32>
1971 %m1 = mul nuw nsw <25 x i32> %az, %bz
1972 %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m1)
1973 %cz = sext <25 x i8> %c to <25 x i32>
1974 %dz = sext <25 x i8> %d to <25 x i32>
1975 %m2 = mul nuw nsw <25 x i32> %cz, %dz
1976 %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m2)
1977 %x = add i32 %r1, %r2
1978 ret i32 %x
1979 }
1981 define i32 @test_sdot_v25i8_double_nomla(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
1982 ; CHECK-LABEL: test_sdot_v25i8_double_nomla:
1983 ; CHECK: // %bb.0: // %entry
1984 ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
1985 ; CHECK-NEXT: .cfi_def_cfa_offset 16
1986 ; CHECK-NEXT: .cfi_offset w29, -16
1987 ; CHECK-NEXT: fmov s0, w0
1988 ; CHECK-NEXT: ldr b1, [sp, #80]
1989 ; CHECK-NEXT: add x10, sp, #88
1990 ; CHECK-NEXT: ldr b2, [sp, #16]
1991 ; CHECK-NEXT: add x9, sp, #96
1992 ; CHECK-NEXT: ldr b3, [sp, #480]
1993 ; CHECK-NEXT: ld1 { v1.b }[1], [x10]
1994 ; CHECK-NEXT: add x10, sp, #24
1995 ; CHECK-NEXT: ldr b4, [sp, #352]
1996 ; CHECK-NEXT: mov v0.b[1], w1
1997 ; CHECK-NEXT: ld1 { v2.b }[1], [x10]
1998 ; CHECK-NEXT: add x11, sp, #488
1999 ; CHECK-NEXT: add x10, sp, #360
2000 ; CHECK-NEXT: ldr b5, [sp, #416]
2001 ; CHECK-NEXT: add x8, sp, #104
2002 ; CHECK-NEXT: ld1 { v1.b }[2], [x9]
2003 ; CHECK-NEXT: add x9, sp, #32
2004 ; CHECK-NEXT: ld1 { v3.b }[1], [x11]
2005 ; CHECK-NEXT: ld1 { v2.b }[2], [x9]
2006 ; CHECK-NEXT: add x11, sp, #424
2007 ; CHECK-NEXT: ld1 { v4.b }[1], [x10]
2008 ; CHECK-NEXT: mov v0.b[2], w2
2009 ; CHECK-NEXT: ld1 { v5.b }[1], [x11]
2010 ; CHECK-NEXT: add x9, sp, #368
2011 ; CHECK-NEXT: ld1 { v1.b }[3], [x8]
2012 ; CHECK-NEXT: add x8, sp, #40
2013 ; CHECK-NEXT: add x12, sp, #496
2014 ; CHECK-NEXT: ld1 { v2.b }[3], [x8]
2015 ; CHECK-NEXT: ld1 { v4.b }[2], [x9]
2016 ; CHECK-NEXT: add x8, sp, #432
2017 ; CHECK-NEXT: ld1 { v3.b }[2], [x12]
2018 ; CHECK-NEXT: add x13, sp, #48
2019 ; CHECK-NEXT: ld1 { v5.b }[2], [x8]
2020 ; CHECK-NEXT: mov v0.b[3], w3
2021 ; CHECK-NEXT: add x10, sp, #112
2022 ; CHECK-NEXT: add x8, sp, #504
2023 ; CHECK-NEXT: ld1 { v2.b }[4], [x13]
2024 ; CHECK-NEXT: add x13, sp, #376
2025 ; CHECK-NEXT: ld1 { v1.b }[4], [x10]
2026 ; CHECK-NEXT: ld1 { v4.b }[3], [x13]
2027 ; CHECK-NEXT: add x13, sp, #440
2028 ; CHECK-NEXT: ld1 { v3.b }[3], [x8]
2029 ; CHECK-NEXT: ld1 { v5.b }[3], [x13]
2030 ; CHECK-NEXT: add x11, sp, #120
2031 ; CHECK-NEXT: add x8, sp, #56
2032 ; CHECK-NEXT: mov v0.b[4], w4
2033 ; CHECK-NEXT: add x13, sp, #512
2034 ; CHECK-NEXT: ld1 { v1.b }[5], [x11]
2035 ; CHECK-NEXT: ld1 { v2.b }[5], [x8]
2036 ; CHECK-NEXT: add x8, sp, #384
2037 ; CHECK-NEXT: add x11, sp, #448
2038 ; CHECK-NEXT: ld1 { v3.b }[4], [x13]
2039 ; CHECK-NEXT: ld1 { v4.b }[4], [x8]
2040 ; CHECK-NEXT: ld1 { v5.b }[4], [x11]
2041 ; CHECK-NEXT: add x12, sp, #128
2042 ; CHECK-NEXT: add x10, sp, #64
2043 ; CHECK-NEXT: add x8, sp, #520
2044 ; CHECK-NEXT: mov v0.b[5], w5
2045 ; CHECK-NEXT: ld1 { v1.b }[6], [x12]
2046 ; CHECK-NEXT: ld1 { v2.b }[6], [x10]
2047 ; CHECK-NEXT: add x10, sp, #392
2048 ; CHECK-NEXT: add x11, sp, #456
2049 ; CHECK-NEXT: ldr b6, [sp, #144]
2050 ; CHECK-NEXT: ldr b7, [sp, #544]
2051 ; CHECK-NEXT: ld1 { v3.b }[5], [x8]
2052 ; CHECK-NEXT: ld1 { v4.b }[5], [x10]
2053 ; CHECK-NEXT: ld1 { v5.b }[5], [x11]
2054 ; CHECK-NEXT: add x9, sp, #136
2055 ; CHECK-NEXT: sshll v6.8h, v6.8b, #0
2056 ; CHECK-NEXT: mov v0.b[6], w6
2057 ; CHECK-NEXT: ld1 { v1.b }[7], [x9]
2058 ; CHECK-NEXT: add x8, sp, #528
2059 ; CHECK-NEXT: add x9, sp, #400
2060 ; CHECK-NEXT: add x10, sp, #464
2061 ; CHECK-NEXT: sshll v7.8h, v7.8b, #0
2062 ; CHECK-NEXT: ld1 { v3.b }[6], [x8]
2063 ; CHECK-NEXT: ld1 { v4.b }[6], [x9]
2064 ; CHECK-NEXT: ld1 { v5.b }[6], [x10]
2065 ; CHECK-NEXT: movi v16.2d, #0000000000000000
2066 ; CHECK-NEXT: movi v17.2d, #0000000000000000
2067 ; CHECK-NEXT: add x14, sp, #72
2068 ; CHECK-NEXT: mov v0.b[7], w7
2069 ; CHECK-NEXT: sshll v6.4s, v6.4h, #0
2070 ; CHECK-NEXT: add x8, sp, #536
2071 ; CHECK-NEXT: add x9, sp, #408
2072 ; CHECK-NEXT: add x10, sp, #472
2073 ; CHECK-NEXT: sshll v7.4s, v7.4h, #0
2074 ; CHECK-NEXT: ld1 { v2.b }[7], [x14]
2075 ; CHECK-NEXT: ld1 { v3.b }[7], [x8]
2076 ; CHECK-NEXT: ld1 { v4.b }[7], [x9]
2077 ; CHECK-NEXT: ld1 { v5.b }[7], [x10]
2078 ; CHECK-NEXT: mov v16.s[0], v6.s[0]
2079 ; CHECK-NEXT: sshll v1.8h, v1.8b, #0
2080 ; CHECK-NEXT: mov v17.s[0], v7.s[0]
2081 ; CHECK-NEXT: sshll v0.8h, v0.8b, #0
2082 ; CHECK-NEXT: sshll v2.8h, v2.8b, #0
2083 ; CHECK-NEXT: sshll v3.8h, v3.8b, #0
2084 ; CHECK-NEXT: sshll v4.8h, v4.8b, #0
2085 ; CHECK-NEXT: sshll v5.8h, v5.8b, #0
2086 ; CHECK-NEXT: saddl v7.4s, v0.4h, v1.4h
2087 ; CHECK-NEXT: saddl2 v0.4s, v0.8h, v1.8h
2088 ; CHECK-NEXT: saddw v6.4s, v16.4s, v2.4h
2089 ; CHECK-NEXT: saddl v1.4s, v4.4h, v3.4h
2090 ; CHECK-NEXT: saddl2 v3.4s, v4.8h, v3.8h
2091 ; CHECK-NEXT: saddw v4.4s, v17.4s, v5.4h
2092 ; CHECK-NEXT: saddw2 v0.4s, v0.4s, v2.8h
2093 ; CHECK-NEXT: add v6.4s, v7.4s, v6.4s
2094 ; CHECK-NEXT: saddw2 v2.4s, v3.4s, v5.8h
2095 ; CHECK-NEXT: add v1.4s, v1.4s, v4.4s
2096 ; CHECK-NEXT: add v0.4s, v6.4s, v0.4s
2097 ; CHECK-NEXT: add v1.4s, v1.4s, v2.4s
2098 ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
2099 ; CHECK-NEXT: addv s0, v0.4s
2100 ; CHECK-NEXT: fmov w0, s0
2101 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
2102 ; CHECK-NEXT: ret
2103 entry:
2104 %az = sext <25 x i8> %a to <25 x i32>
2105 %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %az)
2106 %cz = sext <25 x i8> %c to <25 x i32>
2107 %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %cz)
2108 %x = add i32 %r1, %r2
2109 ret i32 %x
2110 }
2112 define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
2113 ; CHECK-SD-LABEL: test_udot_v32i8:
2114 ; CHECK-SD: // %bb.0: // %entry
2115 ; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
2116 ; CHECK-SD-NEXT: ldp q1, q3, [x0]
2117 ; CHECK-SD-NEXT: ldp q2, q4, [x1]
2118 ; CHECK-SD-NEXT: udot v0.4s, v4.16b, v3.16b
2119 ; CHECK-SD-NEXT: udot v0.4s, v2.16b, v1.16b
2120 ; CHECK-SD-NEXT: addv s0, v0.4s
2121 ; CHECK-SD-NEXT: fmov w8, s0
2122 ; CHECK-SD-NEXT: add w0, w8, w2
2123 ; CHECK-SD-NEXT: ret
2124 ;
2125 ; CHECK-GI-LABEL: test_udot_v32i8:
2126 ; CHECK-GI: // %bb.0: // %entry
2127 ; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
2128 ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
2129 ; CHECK-GI-NEXT: ldp q2, q3, [x0]
2130 ; CHECK-GI-NEXT: ldp q4, q5, [x1]
2131 ; CHECK-GI-NEXT: udot v1.4s, v4.16b, v2.16b
2132 ; CHECK-GI-NEXT: udot v0.4s, v5.16b, v3.16b
2133 ; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s
2134 ; CHECK-GI-NEXT: addv s0, v0.4s
2135 ; CHECK-GI-NEXT: fmov w8, s0
2136 ; CHECK-GI-NEXT: add w0, w8, w2
2137 ; CHECK-GI-NEXT: ret
2138 entry:
2139 %0 = load <32 x i8>, ptr %a
2140 %1 = zext <32 x i8> %0 to <32 x i32>
2141 %2 = load <32 x i8>, ptr %b
2142 %3 = zext <32 x i8> %2 to <32 x i32>
2143 %4 = mul nuw nsw <32 x i32> %3, %1
2144 %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)
2145 %op.extra = add i32 %5, %sum
2146 ret i32 %op.extra
2147 }
2149 define i32 @test_udot_v32i8_nomla(ptr nocapture readonly %a1) {
2150 ; CHECK-SD-LABEL: test_udot_v32i8_nomla:
2151 ; CHECK-SD: // %bb.0: // %entry
2152 ; CHECK-SD-NEXT: movi v0.16b, #1
2153 ; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
2154 ; CHECK-SD-NEXT: ldp q2, q3, [x0]
2155 ; CHECK-SD-NEXT: udot v1.4s, v3.16b, v0.16b
2156 ; CHECK-SD-NEXT: udot v1.4s, v2.16b, v0.16b
2157 ; CHECK-SD-NEXT: addv s0, v1.4s
2158 ; CHECK-SD-NEXT: fmov w0, s0
2159 ; CHECK-SD-NEXT: ret
2160 ;
2161 ; CHECK-GI-LABEL: test_udot_v32i8_nomla:
2162 ; CHECK-GI: // %bb.0: // %entry
2163 ; CHECK-GI-NEXT: movi v0.16b, #1
2164 ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
2165 ; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
2166 ; CHECK-GI-NEXT: ldp q3, q4, [x0]
2167 ; CHECK-GI-NEXT: udot v2.4s, v3.16b, v0.16b
2168 ; CHECK-GI-NEXT: udot v1.4s, v4.16b, v0.16b
2169 ; CHECK-GI-NEXT: add v0.4s, v2.4s, v1.4s
2170 ; CHECK-GI-NEXT: addv s0, v0.4s
2171 ; CHECK-GI-NEXT: fmov w0, s0
2172 ; CHECK-GI-NEXT: ret
2173 entry:
2174 %0 = load <32 x i8>, ptr %a1
2175 %1 = zext <32 x i8> %0 to <32 x i32>
2176 %2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)
2177 ret i32 %2
2178 }
2179 define i32 @test_sdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
2180 ; CHECK-SD-LABEL: test_sdot_v32i8:
2181 ; CHECK-SD: // %bb.0: // %entry
2182 ; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
2183 ; CHECK-SD-NEXT: ldp q1, q3, [x0]
2184 ; CHECK-SD-NEXT: ldp q2, q4, [x1]
2185 ; CHECK-SD-NEXT: sdot v0.4s, v4.16b, v3.16b
2186 ; CHECK-SD-NEXT: sdot v0.4s, v2.16b, v1.16b
2187 ; CHECK-SD-NEXT: addv s0, v0.4s
2188 ; CHECK-SD-NEXT: fmov w8, s0
2189 ; CHECK-SD-NEXT: add w0, w8, w2
2190 ; CHECK-SD-NEXT: ret
2191 ;
2192 ; CHECK-GI-LABEL: test_sdot_v32i8:
2193 ; CHECK-GI: // %bb.0: // %entry
2194 ; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
2195 ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
2196 ; CHECK-GI-NEXT: ldp q2, q3, [x0]
2197 ; CHECK-GI-NEXT: ldp q4, q5, [x1]
2198 ; CHECK-GI-NEXT: sdot v1.4s, v4.16b, v2.16b
2199 ; CHECK-GI-NEXT: sdot v0.4s, v5.16b, v3.16b
2200 ; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s
2201 ; CHECK-GI-NEXT: addv s0, v0.4s
2202 ; CHECK-GI-NEXT: fmov w8, s0
2203 ; CHECK-GI-NEXT: add w0, w8, w2
2204 ; CHECK-GI-NEXT: ret
2205 entry:
2206 %0 = load <32 x i8>, ptr %a
2207 %1 = sext <32 x i8> %0 to <32 x i32>
2208 %2 = load <32 x i8>, ptr %b
2209 %3 = sext <32 x i8> %2 to <32 x i32>
2210 %4 = mul nsw <32 x i32> %3, %1
2211 %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)
2212 %op.extra = add nsw i32 %5, %sum
2213 ret i32 %op.extra
2214 }
2216 define i32 @test_sdot_v32i8_double(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
2217 ; CHECK-SD-LABEL: test_sdot_v32i8_double:
2218 ; CHECK-SD: // %bb.0: // %entry
2219 ; CHECK-SD-NEXT: movi v16.2d, #0000000000000000
2220 ; CHECK-SD-NEXT: movi v17.2d, #0000000000000000
2221 ; CHECK-SD-NEXT: sdot v17.4s, v1.16b, v3.16b
2222 ; CHECK-SD-NEXT: sdot v16.4s, v5.16b, v7.16b
2223 ; CHECK-SD-NEXT: sdot v17.4s, v0.16b, v2.16b
2224 ; CHECK-SD-NEXT: sdot v16.4s, v4.16b, v6.16b
2225 ; CHECK-SD-NEXT: add v0.4s, v17.4s, v16.4s
2226 ; CHECK-SD-NEXT: addv s0, v0.4s
2227 ; CHECK-SD-NEXT: fmov w0, s0
2228 ; CHECK-SD-NEXT: ret
2229 ;
2230 ; CHECK-GI-LABEL: test_sdot_v32i8_double:
2231 ; CHECK-GI: // %bb.0: // %entry
2232 ; CHECK-GI-NEXT: movi v16.2d, #0000000000000000
2233 ; CHECK-GI-NEXT: movi v17.2d, #0000000000000000
2234 ; CHECK-GI-NEXT: movi v18.2d, #0000000000000000
2235 ; CHECK-GI-NEXT: movi v19.2d, #0000000000000000
2236 ; CHECK-GI-NEXT: sdot v16.4s, v0.16b, v2.16b
2237 ; CHECK-GI-NEXT: sdot v18.4s, v1.16b, v3.16b
2238 ; CHECK-GI-NEXT: sdot v17.4s, v5.16b, v7.16b
2239 ; CHECK-GI-NEXT: sdot v19.4s, v4.16b, v6.16b
2240 ; CHECK-GI-NEXT: add v0.4s, v16.4s, v18.4s
2241 ; CHECK-GI-NEXT: add v1.4s, v19.4s, v17.4s
2242 ; CHECK-GI-NEXT: addv s0, v0.4s
2243 ; CHECK-GI-NEXT: addv s1, v1.4s
2244 ; CHECK-GI-NEXT: fmov w8, s0
2245 ; CHECK-GI-NEXT: fmov w9, s1
2246 ; CHECK-GI-NEXT: add w0, w8, w9
2247 ; CHECK-GI-NEXT: ret
2248 entry:
2249 %az = sext <32 x i8> %a to <32 x i32>
2250 %bz = sext <32 x i8> %b to <32 x i32>
2251 %m1 = mul nuw nsw <32 x i32> %az, %bz
2252 %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m1)
2253 %cz = sext <32 x i8> %c to <32 x i32>
2254 %dz = sext <32 x i8> %d to <32 x i32>
2255 %m2 = mul nuw nsw <32 x i32> %cz, %dz
2256 %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m2)
2257 %x = add i32 %r1, %r2
2258 ret i32 %x
2259 }
2261 define i32 @test_sdot_v32i8_double_nomla(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
2262 ; CHECK-SD-LABEL: test_sdot_v32i8_double_nomla:
2263 ; CHECK-SD: // %bb.0: // %entry
2264 ; CHECK-SD-NEXT: movi v2.16b, #1
2265 ; CHECK-SD-NEXT: movi v3.2d, #0000000000000000
2266 ; CHECK-SD-NEXT: movi v6.2d, #0000000000000000
2267 ; CHECK-SD-NEXT: sdot v6.4s, v1.16b, v2.16b
2268 ; CHECK-SD-NEXT: sdot v3.4s, v5.16b, v2.16b
2269 ; CHECK-SD-NEXT: sdot v6.4s, v0.16b, v2.16b
2270 ; CHECK-SD-NEXT: sdot v3.4s, v4.16b, v2.16b
2271 ; CHECK-SD-NEXT: add v0.4s, v6.4s, v3.4s
2272 ; CHECK-SD-NEXT: addv s0, v0.4s
2273 ; CHECK-SD-NEXT: fmov w0, s0
2274 ; CHECK-SD-NEXT: ret
2275 ;
2276 ; CHECK-GI-LABEL: test_sdot_v32i8_double_nomla:
2277 ; CHECK-GI: // %bb.0: // %entry
2278 ; CHECK-GI-NEXT: movi v2.16b, #1
2279 ; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
2280 ; CHECK-GI-NEXT: movi v6.2d, #0000000000000000
2281 ; CHECK-GI-NEXT: movi v7.2d, #0000000000000000
2282 ; CHECK-GI-NEXT: movi v16.2d, #0000000000000000
2283 ; CHECK-GI-NEXT: sdot v3.4s, v0.16b, v2.16b
2284 ; CHECK-GI-NEXT: sdot v6.4s, v5.16b, v2.16b
2285 ; CHECK-GI-NEXT: sdot v7.4s, v1.16b, v2.16b
2286 ; CHECK-GI-NEXT: sdot v16.4s, v4.16b, v2.16b
2287 ; CHECK-GI-NEXT: add v0.4s, v3.4s, v7.4s
2288 ; CHECK-GI-NEXT: add v1.4s, v16.4s, v6.4s
2289 ; CHECK-GI-NEXT: addv s0, v0.4s
2290 ; CHECK-GI-NEXT: addv s1, v1.4s
2291 ; CHECK-GI-NEXT: fmov w8, s0
2292 ; CHECK-GI-NEXT: fmov w9, s1
2293 ; CHECK-GI-NEXT: add w0, w8, w9
2294 ; CHECK-GI-NEXT: ret
2295 entry:
2296 %az = sext <32 x i8> %a to <32 x i32>
2297 %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %az)
2298 %cz = sext <32 x i8> %c to <32 x i32>
2299 %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %cz)
2300 %x = add i32 %r1, %r2
2301 ret i32 %x
2302 }
2304 define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
2305 ; CHECK-LABEL: test_udot_v33i8:
2306 ; CHECK: // %bb.0: // %entry
2307 ; CHECK-NEXT: ldr b0, [x0, #32]
2308 ; CHECK-NEXT: ldr b1, [x1, #32]
2309 ; CHECK-NEXT: movi v7.2d, #0000000000000000
2310 ; CHECK-NEXT: ldp q3, q4, [x1]
2311 ; CHECK-NEXT: umull v0.8h, v1.8b, v0.8b
2312 ; CHECK-NEXT: ldp q1, q2, [x0]
2313 ; CHECK-NEXT: umull v5.8h, v4.8b, v2.8b
2314 ; CHECK-NEXT: umull v6.8h, v3.8b, v1.8b
2315 ; CHECK-NEXT: umull2 v2.8h, v4.16b, v2.16b
2316 ; CHECK-NEXT: ushll v0.4s, v0.4h, #0
2317 ; CHECK-NEXT: umull2 v1.8h, v3.16b, v1.16b
2318 ; CHECK-NEXT: mov v7.s[0], v0.s[0]
2319 ; CHECK-NEXT: uaddl2 v3.4s, v6.8h, v5.8h
2320 ; CHECK-NEXT: uaddl2 v0.4s, v1.8h, v2.8h
2321 ; CHECK-NEXT: uaddl v1.4s, v1.4h, v2.4h
2322 ; CHECK-NEXT: add v0.4s, v3.4s, v0.4s
2323 ; CHECK-NEXT: uaddw v2.4s, v7.4s, v6.4h
2324 ; CHECK-NEXT: uaddw v2.4s, v2.4s, v5.4h
2325 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s
2326 ; CHECK-NEXT: add v0.4s, v2.4s, v0.4s
2327 ; CHECK-NEXT: addv s0, v0.4s
2328 ; CHECK-NEXT: fmov w8, s0
2329 ; CHECK-NEXT: add w0, w8, w2
2330 ; CHECK-NEXT: ret
2331 entry:
2332 %0 = load <33 x i8>, ptr %a
2333 %1 = zext <33 x i8> %0 to <33 x i32>
2334 %2 = load <33 x i8>, ptr %b
2335 %3 = zext <33 x i8> %2 to <33 x i32>
2336 %4 = mul nuw nsw <33 x i32> %3, %1
2337 %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4)
2338 %op.extra = add i32 %5, %sum
2339 ret i32 %op.extra
2340 }
2342 define i32 @test_udot_v33i8_nomla(ptr nocapture readonly %a1) {
2343 ; CHECK-LABEL: test_udot_v33i8_nomla:
2344 ; CHECK: // %bb.0: // %entry
2345 ; CHECK-NEXT: ldr b1, [x0, #32]
2346 ; CHECK-NEXT: ldp q3, q2, [x0]
2347 ; CHECK-NEXT: movi v0.2d, #0000000000000000
2348 ; CHECK-NEXT: ushll v1.8h, v1.8b, #0
2349 ; CHECK-NEXT: ushll v4.8h, v2.8b, #0
2350 ; CHECK-NEXT: ushll v5.8h, v3.8b, #0
2351 ; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
2352 ; CHECK-NEXT: ushll2 v3.8h, v3.16b, #0
2353 ; CHECK-NEXT: ushll v1.4s, v1.4h, #0
2354 ; CHECK-NEXT: uaddl2 v6.4s, v5.8h, v4.8h
2355 ; CHECK-NEXT: mov v0.s[0], v1.s[0]
2356 ; CHECK-NEXT: uaddl2 v1.4s, v3.8h, v2.8h
2357 ; CHECK-NEXT: uaddl v2.4s, v3.4h, v2.4h
2358 ; CHECK-NEXT: add v1.4s, v6.4s, v1.4s
2359 ; CHECK-NEXT: uaddw v0.4s, v0.4s, v5.4h
2360 ; CHECK-NEXT: add v1.4s, v2.4s, v1.4s
2361 ; CHECK-NEXT: uaddw v0.4s, v0.4s, v4.4h
2362 ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
2363 ; CHECK-NEXT: addv s0, v0.4s
2364 ; CHECK-NEXT: fmov w0, s0
2365 ; CHECK-NEXT: ret
2366 entry:
2367 %0 = load <33 x i8>, ptr %a1
2368 %1 = zext <33 x i8> %0 to <33 x i32>
2369 %2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %1)
2370 ret i32 %2
2371 }
2372 define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
2373 ; CHECK-LABEL: test_sdot_v33i8:
2374 ; CHECK: // %bb.0: // %entry
2375 ; CHECK-NEXT: ldr b0, [x0, #32]
2376 ; CHECK-NEXT: ldr b1, [x1, #32]
2377 ; CHECK-NEXT: movi v7.2d, #0000000000000000
2378 ; CHECK-NEXT: ldp q3, q4, [x1]
2379 ; CHECK-NEXT: smull v0.8h, v1.8b, v0.8b
2380 ; CHECK-NEXT: ldp q1, q2, [x0]
2381 ; CHECK-NEXT: smull v5.8h, v4.8b, v2.8b
2382 ; CHECK-NEXT: smull v6.8h, v3.8b, v1.8b
2383 ; CHECK-NEXT: smull2 v2.8h, v4.16b, v2.16b
2384 ; CHECK-NEXT: sshll v0.4s, v0.4h, #0
2385 ; CHECK-NEXT: smull2 v1.8h, v3.16b, v1.16b
2386 ; CHECK-NEXT: mov v7.s[0], v0.s[0]
2387 ; CHECK-NEXT: saddl2 v3.4s, v6.8h, v5.8h
2388 ; CHECK-NEXT: saddl2 v0.4s, v1.8h, v2.8h
2389 ; CHECK-NEXT: saddl v1.4s, v1.4h, v2.4h
2390 ; CHECK-NEXT: add v0.4s, v3.4s, v0.4s
2391 ; CHECK-NEXT: saddw v2.4s, v7.4s, v6.4h
2392 ; CHECK-NEXT: saddw v2.4s, v2.4s, v5.4h
2393 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s
2394 ; CHECK-NEXT: add v0.4s, v2.4s, v0.4s
2395 ; CHECK-NEXT: addv s0, v0.4s
2396 ; CHECK-NEXT: fmov w8, s0
2397 ; CHECK-NEXT: add w0, w8, w2
2398 ; CHECK-NEXT: ret
2399 entry:
2400 %0 = load <33 x i8>, ptr %a
2401 %1 = sext <33 x i8> %0 to <33 x i32>
2402 %2 = load <33 x i8>, ptr %b
2403 %3 = sext <33 x i8> %2 to <33 x i32>
2404 %4 = mul nsw <33 x i32> %3, %1
2405 %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4)
2406 %op.extra = add nsw i32 %5, %sum
2407 ret i32 %op.extra
2408 }
2410 define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 x i8> %d) {
2411 ; CHECK-LABEL: test_sdot_v33i8_double:
2412 ; CHECK: // %bb.0: // %entry
2413 ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
2414 ; CHECK-NEXT: .cfi_def_cfa_offset 16
2415 ; CHECK-NEXT: .cfi_offset w29, -16
2416 ; CHECK-NEXT: ldr b0, [sp, #344]
2417 ; CHECK-NEXT: add x8, sp, #352
2418 ; CHECK-NEXT: ldr b1, [sp, #80]
2419 ; CHECK-NEXT: ldr b2, [sp, #216]
2420 ; CHECK-NEXT: add x9, sp, #96
2421 ; CHECK-NEXT: add x10, sp, #104
2422 ; CHECK-NEXT: ld1 { v0.b }[1], [x8]
2423 ; CHECK-NEXT: add x8, sp, #88
2424 ; CHECK-NEXT: ldr b4, [sp, #408]
2425 ; CHECK-NEXT: ld1 { v1.b }[1], [x8]
2426 ; CHECK-NEXT: add x8, sp, #360
2427 ; CHECK-NEXT: add x12, sp, #248
2428 ; CHECK-NEXT: add x13, sp, #432
2429 ; CHECK-NEXT: add x11, sp, #384
2430 ; CHECK-NEXT: ldr b5, [sp, #144]
2431 ; CHECK-NEXT: ld1 { v0.b }[2], [x8]
2432 ; CHECK-NEXT: add x8, sp, #224
2433 ; CHECK-NEXT: ldr b6, [sp, #280]
2434 ; CHECK-NEXT: ld1 { v2.b }[1], [x8]
2435 ; CHECK-NEXT: ld1 { v1.b }[2], [x9]
2436 ; CHECK-NEXT: add x8, sp, #368
2437 ; CHECK-NEXT: add x9, sp, #232
2438 ; CHECK-NEXT: ldr b16, [sp, #744]
2439 ; CHECK-NEXT: ldr b17, [sp, #480]
2440 ; CHECK-NEXT: ld1 { v0.b }[3], [x8]
2441 ; CHECK-NEXT: add x8, sp, #376
2442 ; CHECK-NEXT: ldr b18, [sp, #936]
2443 ; CHECK-NEXT: ld1 { v2.b }[2], [x9]
2444 ; CHECK-NEXT: ld1 { v1.b }[3], [x10]
2445 ; CHECK-NEXT: add x9, sp, #240
2446 ; CHECK-NEXT: add x10, sp, #392
2447 ; CHECK-NEXT: ldr b19, [sp, #672]
2448 ; CHECK-NEXT: ldr b7, [sp, #16]
2449 ; CHECK-NEXT: ld1 { v0.b }[4], [x8]
2450 ; CHECK-NEXT: add x8, sp, #112
2451 ; CHECK-NEXT: ldr b21, [sp, #1000]
2452 ; CHECK-NEXT: ld1 { v2.b }[3], [x9]
2453 ; CHECK-NEXT: ld1 { v1.b }[4], [x8]
2454 ; CHECK-NEXT: add x8, sp, #416
2455 ; CHECK-NEXT: ld1 { v4.b }[1], [x8]
2456 ; CHECK-NEXT: add x8, sp, #120
2457 ; CHECK-NEXT: add x9, sp, #400
2458 ; CHECK-NEXT: ld1 { v0.b }[5], [x11]
2459 ; CHECK-NEXT: add x11, sp, #128
2460 ; CHECK-NEXT: ldr b22, [sp, #736]
2461 ; CHECK-NEXT: ld1 { v2.b }[4], [x12]
2462 ; CHECK-NEXT: add x12, sp, #424
2463 ; CHECK-NEXT: ld1 { v1.b }[5], [x8]
2464 ; CHECK-NEXT: ld1 { v4.b }[2], [x12]
2465 ; CHECK-NEXT: add x12, sp, #152
2466 ; CHECK-NEXT: add x8, sp, #136
2467 ; CHECK-NEXT: ld1 { v5.b }[1], [x12]
2468 ; CHECK-NEXT: add x12, sp, #440
2469 ; CHECK-NEXT: ld1 { v0.b }[6], [x10]
2470 ; CHECK-NEXT: ld1 { v1.b }[6], [x11]
2471 ; CHECK-NEXT: add x11, sp, #288
2472 ; CHECK-NEXT: add x10, sp, #256
2473 ; CHECK-NEXT: ld1 { v4.b }[3], [x13]
2474 ; CHECK-NEXT: ld1 { v6.b }[1], [x11]
2475 ; CHECK-NEXT: add x11, sp, #296
2476 ; CHECK-NEXT: ld1 { v0.b }[7], [x9]
2477 ; CHECK-NEXT: add x9, sp, #160
2478 ; CHECK-NEXT: ld1 { v2.b }[5], [x10]
2479 ; CHECK-NEXT: ld1 { v5.b }[2], [x9]
2480 ; CHECK-NEXT: add x10, sp, #168
2481 ; CHECK-NEXT: ld1 { v1.b }[7], [x8]
2482 ; CHECK-NEXT: ld1 { v4.b }[4], [x12]
2483 ; CHECK-NEXT: add x12, sp, #448
2484 ; CHECK-NEXT: ld1 { v6.b }[2], [x11]
2485 ; CHECK-NEXT: add x11, sp, #304
2486 ; CHECK-NEXT: add x8, sp, #464
2487 ; CHECK-NEXT: add x13, sp, #768
2488 ; CHECK-NEXT: ld1 { v5.b }[3], [x10]
2489 ; CHECK-NEXT: add x10, sp, #176
2490 ; CHECK-NEXT: add x9, sp, #264
2491 ; CHECK-NEXT: ld1 { v4.b }[5], [x12]
2492 ; CHECK-NEXT: add x12, sp, #456
2493 ; CHECK-NEXT: ld1 { v6.b }[3], [x11]
2494 ; CHECK-NEXT: add x11, sp, #760
2495 ; CHECK-NEXT: ld1 { v2.b }[6], [x9]
2496 ; CHECK-NEXT: add x9, sp, #272
2497 ; CHECK-NEXT: ld1 { v5.b }[4], [x10]
2498 ; CHECK-NEXT: add x10, sp, #312
2499 ; CHECK-NEXT: fmov s3, w0
2500 ; CHECK-NEXT: ld1 { v4.b }[6], [x12]
2501 ; CHECK-NEXT: ld1 { v6.b }[4], [x10]
2502 ; CHECK-NEXT: add x10, sp, #320
2503 ; CHECK-NEXT: add x12, sp, #680
2504 ; CHECK-NEXT: ld1 { v2.b }[7], [x9]
2505 ; CHECK-NEXT: add x9, sp, #184
2506 ; CHECK-NEXT: ld1 { v19.b }[1], [x12]
2507 ; CHECK-NEXT: add x12, sp, #776
2508 ; CHECK-NEXT: ld1 { v5.b }[5], [x9]
2509 ; CHECK-NEXT: ld1 { v4.b }[7], [x8]
2510 ; CHECK-NEXT: add x8, sp, #752
2511 ; CHECK-NEXT: ld1 { v6.b }[5], [x10]
2512 ; CHECK-NEXT: ld1 { v16.b }[1], [x8]
2513 ; CHECK-NEXT: add x10, sp, #24
2514 ; CHECK-NEXT: smull v22.8h, v22.8b, v21.8b
2515 ; CHECK-NEXT: ld1 { v7.b }[1], [x10]
2516 ; CHECK-NEXT: add x10, sp, #496
2517 ; CHECK-NEXT: mov v3.b[1], w1
2518 ; CHECK-NEXT: add x9, sp, #192
2519 ; CHECK-NEXT: ldr b20, [sp, #472]
2520 ; CHECK-NEXT: ldr b23, [sp, #208]
2521 ; CHECK-NEXT: ld1 { v16.b }[2], [x11]
2522 ; CHECK-NEXT: add x11, sp, #488
2523 ; CHECK-NEXT: ld1 { v5.b }[6], [x9]
2524 ; CHECK-NEXT: ld1 { v17.b }[1], [x11]
2525 ; CHECK-NEXT: add x11, sp, #944
2526 ; CHECK-NEXT: add x9, sp, #328
2527 ; CHECK-NEXT: ld1 { v18.b }[1], [x11]
2528 ; CHECK-NEXT: add x11, sp, #688
2529 ; CHECK-NEXT: ld1 { v6.b }[6], [x9]
2530 ; CHECK-NEXT: ld1 { v16.b }[3], [x13]
2531 ; CHECK-NEXT: ld1 { v19.b }[2], [x11]
2532 ; CHECK-NEXT: add x11, sp, #504
2533 ; CHECK-NEXT: ld1 { v17.b }[2], [x10]
2534 ; CHECK-NEXT: add x10, sp, #952
2535 ; CHECK-NEXT: add x13, sp, #784
2536 ; CHECK-NEXT: ld1 { v18.b }[2], [x10]
2537 ; CHECK-NEXT: add x10, sp, #32
2538 ; CHECK-NEXT: add x9, sp, #40
2539 ; CHECK-NEXT: ld1 { v16.b }[4], [x12]
2540 ; CHECK-NEXT: add x12, sp, #696
2541 ; CHECK-NEXT: ld1 { v7.b }[2], [x10]
2542 ; CHECK-NEXT: ld1 { v17.b }[3], [x11]
2543 ; CHECK-NEXT: add x11, sp, #960
2544 ; CHECK-NEXT: ld1 { v19.b }[3], [x12]
2545 ; CHECK-NEXT: ld1 { v18.b }[3], [x11]
2546 ; CHECK-NEXT: add x10, sp, #512
2547 ; CHECK-NEXT: add x11, sp, #704
2548 ; CHECK-NEXT: ld1 { v16.b }[5], [x13]
2549 ; CHECK-NEXT: add x12, sp, #792
2550 ; CHECK-NEXT: sshll v24.4s, v22.4h, #0
2551 ; CHECK-NEXT: ld1 { v17.b }[4], [x10]
2552 ; CHECK-NEXT: add x10, sp, #968
2553 ; CHECK-NEXT: ld1 { v19.b }[4], [x11]
2554 ; CHECK-NEXT: ld1 { v18.b }[4], [x10]
2555 ; CHECK-NEXT: add x10, sp, #520
2556 ; CHECK-NEXT: add x11, sp, #976
2557 ; CHECK-NEXT: ld1 { v16.b }[6], [x12]
2558 ; CHECK-NEXT: add x12, sp, #712
2559 ; CHECK-NEXT: smull v20.8h, v23.8b, v20.8b
2560 ; CHECK-NEXT: ld1 { v17.b }[5], [x10]
2561 ; CHECK-NEXT: ld1 { v19.b }[5], [x12]
2562 ; CHECK-NEXT: add x12, sp, #720
2563 ; CHECK-NEXT: ld1 { v18.b }[5], [x11]
2564 ; CHECK-NEXT: add x11, sp, #528
2565 ; CHECK-NEXT: add x10, sp, #800
2566 ; CHECK-NEXT: ld1 { v16.b }[7], [x10]
2567 ; CHECK-NEXT: add x10, sp, #536
2568 ; CHECK-NEXT: ldr b22, [sp, #872]
2569 ; CHECK-NEXT: ld1 { v17.b }[6], [x11]
2570 ; CHECK-NEXT: add x11, sp, #984
2571 ; CHECK-NEXT: ld1 { v19.b }[6], [x12]
2572 ; CHECK-NEXT: ld1 { v18.b }[6], [x11]
2573 ; CHECK-NEXT: add x11, sp, #992
2574 ; CHECK-NEXT: add x12, sp, #728
2575 ; CHECK-NEXT: ldr b23, [sp, #608]
2576 ; CHECK-NEXT: ld1 { v7.b }[3], [x9]
2577 ; CHECK-NEXT: add x9, sp, #880
2578 ; CHECK-NEXT: ld1 { v17.b }[7], [x10]
2579 ; CHECK-NEXT: ld1 { v19.b }[7], [x12]
2580 ; CHECK-NEXT: add x10, sp, #816
2581 ; CHECK-NEXT: ld1 { v18.b }[7], [x11]
2582 ; CHECK-NEXT: add x11, sp, #552
2583 ; CHECK-NEXT: add x12, sp, #616
2584 ; CHECK-NEXT: mov v3.b[2], w2
2585 ; CHECK-NEXT: ld1 { v22.b }[1], [x9]
2586 ; CHECK-NEXT: ld1 { v23.b }[1], [x12]
2587 ; CHECK-NEXT: smull v16.8h, v17.8b, v16.8b
2588 ; CHECK-NEXT: add x12, sp, #560
2589 ; CHECK-NEXT: add x9, sp, #888
2590 ; CHECK-NEXT: smull v17.8h, v19.8b, v18.8b
2591 ; CHECK-NEXT: ldr b18, [sp, #808]
2592 ; CHECK-NEXT: ldr b19, [sp, #544]
2593 ; CHECK-NEXT: add x13, sp, #624
2594 ; CHECK-NEXT: ld1 { v22.b }[2], [x9]
2595 ; CHECK-NEXT: add x9, sp, #896
2596 ; CHECK-NEXT: ld1 { v18.b }[1], [x10]
2597 ; CHECK-NEXT: ld1 { v19.b }[1], [x11]
2598 ; CHECK-NEXT: add x11, sp, #824
2599 ; CHECK-NEXT: add x10, sp, #48
2600 ; CHECK-NEXT: ld1 { v23.b }[2], [x13]
2601 ; CHECK-NEXT: mov v3.b[3], w3
2602 ; CHECK-NEXT: ld1 { v7.b }[4], [x10]
2603 ; CHECK-NEXT: add x10, sp, #832
2604 ; CHECK-NEXT: ld1 { v22.b }[3], [x9]
2605 ; CHECK-NEXT: ld1 { v18.b }[2], [x11]
2606 ; CHECK-NEXT: ld1 { v19.b }[2], [x12]
2607 ; CHECK-NEXT: add x11, sp, #568
2608 ; CHECK-NEXT: add x12, sp, #632
2609 ; CHECK-NEXT: add x9, sp, #904
2610 ; CHECK-NEXT: add x13, sp, #640
2611 ; CHECK-NEXT: ld1 { v23.b }[3], [x12]
2612 ; CHECK-NEXT: add x12, sp, #576
2613 ; CHECK-NEXT: mov v3.b[4], w4
2614 ; CHECK-NEXT: ld1 { v18.b }[3], [x10]
2615 ; CHECK-NEXT: ld1 { v19.b }[3], [x11]
2616 ; CHECK-NEXT: add x11, sp, #840
2617 ; CHECK-NEXT: add x10, sp, #56
2618 ; CHECK-NEXT: ld1 { v22.b }[4], [x9]
2619 ; CHECK-NEXT: add x9, sp, #912
2620 ; CHECK-NEXT: ld1 { v23.b }[4], [x13]
2621 ; CHECK-NEXT: ld1 { v7.b }[5], [x10]
2622 ; CHECK-NEXT: add x10, sp, #848
2623 ; CHECK-NEXT: ld1 { v18.b }[4], [x11]
2624 ; CHECK-NEXT: ld1 { v19.b }[4], [x12]
2625 ; CHECK-NEXT: add x11, sp, #584
2626 ; CHECK-NEXT: add x12, sp, #648
2627 ; CHECK-NEXT: mov v3.b[5], w5
2628 ; CHECK-NEXT: ld1 { v22.b }[5], [x9]
2629 ; CHECK-NEXT: ld1 { v23.b }[5], [x12]
2630 ; CHECK-NEXT: add x12, sp, #592
2631 ; CHECK-NEXT: movi v21.2d, #0000000000000000
2632 ; CHECK-NEXT: ld1 { v18.b }[5], [x10]
2633 ; CHECK-NEXT: ld1 { v19.b }[5], [x11]
2634 ; CHECK-NEXT: add x11, sp, #856
2635 ; CHECK-NEXT: add x9, sp, #920
2636 ; CHECK-NEXT: add x13, sp, #656
2637 ; CHECK-NEXT: add x10, sp, #64
2638 ; CHECK-NEXT: ld1 { v22.b }[6], [x9]
2639 ; CHECK-NEXT: ld1 { v23.b }[6], [x13]
2640 ; CHECK-NEXT: mov v3.b[6], w6
2641 ; CHECK-NEXT: ld1 { v18.b }[6], [x11]
2642 ; CHECK-NEXT: ld1 { v19.b }[6], [x12]
2643 ; CHECK-NEXT: ld1 { v7.b }[6], [x10]
2644 ; CHECK-NEXT: add x10, sp, #864
2645 ; CHECK-NEXT: add x11, sp, #600
2646 ; CHECK-NEXT: add x9, sp, #928
2647 ; CHECK-NEXT: add x12, sp, #664
2648 ; CHECK-NEXT: mov v21.s[0], v24.s[0]
2649 ; CHECK-NEXT: ld1 { v22.b }[7], [x9]
2650 ; CHECK-NEXT: ld1 { v18.b }[7], [x10]
2651 ; CHECK-NEXT: ld1 { v19.b }[7], [x11]
2652 ; CHECK-NEXT: ld1 { v23.b }[7], [x12]
2653 ; CHECK-NEXT: add x8, sp, #200
2654 ; CHECK-NEXT: mov v3.b[7], w7
2655 ; CHECK-NEXT: add x10, sp, #336
2656 ; CHECK-NEXT: ld1 { v5.b }[7], [x8]
2657 ; CHECK-NEXT: add x8, sp, #72
2658 ; CHECK-NEXT: ld1 { v6.b }[7], [x10]
2659 ; CHECK-NEXT: smull v18.8h, v19.8b, v18.8b
2660 ; CHECK-NEXT: movi v19.2d, #0000000000000000
2661 ; CHECK-NEXT: ld1 { v7.b }[7], [x8]
2662 ; CHECK-NEXT: smull v22.8h, v23.8b, v22.8b
2663 ; CHECK-NEXT: sshll v20.4s, v20.4h, #0
2664 ; CHECK-NEXT: smull v0.8h, v1.8b, v0.8b
2665 ; CHECK-NEXT: saddw v1.4s, v21.4s, v16.4h
2666 ; CHECK-NEXT: smull v2.8h, v3.8b, v2.8b
2667 ; CHECK-NEXT: smull v3.8h, v5.8b, v4.8b
2668 ; CHECK-NEXT: smull v4.8h, v7.8b, v6.8b
2669 ; CHECK-NEXT: mov v19.s[0], v20.s[0]
2670 ; CHECK-NEXT: saddl2 v5.4s, v18.8h, v17.8h
2671 ; CHECK-NEXT: saddl v7.4s, v18.4h, v17.4h
2672 ; CHECK-NEXT: saddl2 v6.4s, v16.8h, v22.8h
2673 ; CHECK-NEXT: saddw v1.4s, v1.4s, v22.4h
2674 ; CHECK-NEXT: saddl2 v17.4s, v2.8h, v0.8h
2675 ; CHECK-NEXT: saddl2 v16.4s, v4.8h, v3.8h
2676 ; CHECK-NEXT: saddl v3.4s, v4.4h, v3.4h
2677 ; CHECK-NEXT: saddw v2.4s, v19.4s, v2.4h
2678 ; CHECK-NEXT: add v5.4s, v6.4s, v5.4s
2679 ; CHECK-NEXT: add v1.4s, v1.4s, v7.4s
2680 ; CHECK-NEXT: add v6.4s, v17.4s, v16.4s
2681 ; CHECK-NEXT: saddw v0.4s, v2.4s, v0.4h
2682 ; CHECK-NEXT: add v1.4s, v1.4s, v5.4s
2683 ; CHECK-NEXT: add v0.4s, v0.4s, v3.4s
2684 ; CHECK-NEXT: add v1.4s, v6.4s, v1.4s
2685 ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
2686 ; CHECK-NEXT: addv s0, v0.4s
2687 ; CHECK-NEXT: fmov w0, s0
2688 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
2689 ; CHECK-NEXT: ret
2690 entry:
2691 %az = sext <33 x i8> %a to <33 x i32>
2692 %bz = sext <33 x i8> %b to <33 x i32>
2693 %m1 = mul nuw nsw <33 x i32> %az, %bz
2694 %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m1)
2695 %cz = sext <33 x i8> %c to <33 x i32>
2696 %dz = sext <33 x i8> %d to <33 x i32>
2697 %m2 = mul nuw nsw <33 x i32> %cz, %dz
2698 %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m2)
2699 %x = add i32 %r1, %r2
2700 ret i32 %x
2701 }
2703 define i32 @test_sdot_v33i8_double_nomla(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 x i8> %d) {
2704 ; CHECK-LABEL: test_sdot_v33i8_double_nomla:
2705 ; CHECK: // %bb.0: // %entry
2706 ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
2707 ; CHECK-NEXT: .cfi_def_cfa_offset 16
2708 ; CHECK-NEXT: .cfi_offset w29, -16
2709 ; CHECK-NEXT: ldr b0, [sp, #80]
2710 ; CHECK-NEXT: add x8, sp, #88
2711 ; CHECK-NEXT: ldr b2, [sp, #144]
2712 ; CHECK-NEXT: add x9, sp, #152
2713 ; CHECK-NEXT: ldr b3, [sp, #16]
2714 ; CHECK-NEXT: add x11, sp, #104
2715 ; CHECK-NEXT: ld1 { v0.b }[1], [x8]
2716 ; CHECK-NEXT: ld1 { v2.b }[1], [x9]
2717 ; CHECK-NEXT: add x9, sp, #24
2718 ; CHECK-NEXT: add x8, sp, #96
2719 ; CHECK-NEXT: ld1 { v3.b }[1], [x9]
2720 ; CHECK-NEXT: ldr b5, [sp, #480]
2721 ; CHECK-NEXT: fmov s1, w0
2722 ; CHECK-NEXT: add x10, sp, #112
2723 ; CHECK-NEXT: add x12, sp, #168
2724 ; CHECK-NEXT: ld1 { v0.b }[2], [x8]
2725 ; CHECK-NEXT: add x8, sp, #160
2726 ; CHECK-NEXT: ldr b4, [sp, #608]
2727 ; CHECK-NEXT: ld1 { v2.b }[2], [x8]
2728 ; CHECK-NEXT: add x8, sp, #32
2729 ; CHECK-NEXT: add x13, sp, #496
2730 ; CHECK-NEXT: ld1 { v3.b }[2], [x8]
2731 ; CHECK-NEXT: mov v1.b[1], w1
2732 ; CHECK-NEXT: ldr b6, [sp, #672]
2733 ; CHECK-NEXT: ld1 { v0.b }[3], [x11]
2734 ; CHECK-NEXT: add x11, sp, #488
2735 ; CHECK-NEXT: add x9, sp, #120
2736 ; CHECK-NEXT: ld1 { v5.b }[1], [x11]
2737 ; CHECK-NEXT: add x11, sp, #40
2738 ; CHECK-NEXT: ld1 { v2.b }[3], [x12]
2739 ; CHECK-NEXT: ld1 { v3.b }[3], [x11]
2740 ; CHECK-NEXT: add x12, sp, #616
2741 ; CHECK-NEXT: ldr b16, [sp, #544]
2742 ; CHECK-NEXT: ld1 { v0.b }[4], [x10]
2743 ; CHECK-NEXT: add x10, sp, #48
2744 ; CHECK-NEXT: ld1 { v4.b }[1], [x12]
2745 ; CHECK-NEXT: add x12, sp, #176
2746 ; CHECK-NEXT: ld1 { v5.b }[2], [x13]
2747 ; CHECK-NEXT: add x13, sp, #680
2748 ; CHECK-NEXT: ld1 { v3.b }[4], [x10]
2749 ; CHECK-NEXT: ld1 { v2.b }[4], [x12]
2750 ; CHECK-NEXT: ld1 { v6.b }[1], [x13]
2751 ; CHECK-NEXT: add x13, sp, #56
2752 ; CHECK-NEXT: ld1 { v0.b }[5], [x9]
2753 ; CHECK-NEXT: mov v1.b[2], w2
2754 ; CHECK-NEXT: add x8, sp, #128
2755 ; CHECK-NEXT: add x14, sp, #184
2756 ; CHECK-NEXT: add x11, sp, #136
2757 ; CHECK-NEXT: ld1 { v3.b }[5], [x13]
2758 ; CHECK-NEXT: add x13, sp, #552
2759 ; CHECK-NEXT: ld1 { v2.b }[5], [x14]
2760 ; CHECK-NEXT: ld1 { v16.b }[1], [x13]
2761 ; CHECK-NEXT: add x14, sp, #624
2762 ; CHECK-NEXT: ld1 { v0.b }[6], [x8]
2763 ; CHECK-NEXT: add x8, sp, #688
2764 ; CHECK-NEXT: add x13, sp, #504
2765 ; CHECK-NEXT: ld1 { v4.b }[2], [x14]
2766 ; CHECK-NEXT: ld1 { v6.b }[2], [x8]
2767 ; CHECK-NEXT: add x8, sp, #560
2768 ; CHECK-NEXT: ld1 { v5.b }[3], [x13]
2769 ; CHECK-NEXT: ld1 { v16.b }[2], [x8]
2770 ; CHECK-NEXT: mov v1.b[3], w3
2771 ; CHECK-NEXT: add x9, sp, #64
2772 ; CHECK-NEXT: add x15, sp, #632
2773 ; CHECK-NEXT: ld1 { v3.b }[6], [x9]
2774 ; CHECK-NEXT: ld1 { v0.b }[7], [x11]
2775 ; CHECK-NEXT: ld1 { v4.b }[3], [x15]
2776 ; CHECK-NEXT: add x8, sp, #696
2777 ; CHECK-NEXT: add x9, sp, #568
2778 ; CHECK-NEXT: add x11, sp, #512
2779 ; CHECK-NEXT: ld1 { v6.b }[3], [x8]
2780 ; CHECK-NEXT: ld1 { v16.b }[3], [x9]
2781 ; CHECK-NEXT: ld1 { v5.b }[4], [x11]
2782 ; CHECK-NEXT: add x8, sp, #640
2783 ; CHECK-NEXT: mov v1.b[4], w4
2784 ; CHECK-NEXT: ld1 { v4.b }[4], [x8]
2785 ; CHECK-NEXT: add x8, sp, #704
2786 ; CHECK-NEXT: add x9, sp, #576
2787 ; CHECK-NEXT: add x11, sp, #520
2788 ; CHECK-NEXT: ld1 { v6.b }[4], [x8]
2789 ; CHECK-NEXT: ld1 { v16.b }[4], [x9]
2790 ; CHECK-NEXT: ld1 { v5.b }[5], [x11]
2791 ; CHECK-NEXT: ldr b18, [sp, #736]
2792 ; CHECK-NEXT: add x12, sp, #192
2793 ; CHECK-NEXT: ld1 { v2.b }[6], [x12]
2794 ; CHECK-NEXT: add x8, sp, #648
2795 ; CHECK-NEXT: add x9, sp, #528
2796 ; CHECK-NEXT: add x11, sp, #712
2797 ; CHECK-NEXT: add x12, sp, #584
2798 ; CHECK-NEXT: sshll v18.8h, v18.8b, #0
2799 ; CHECK-NEXT: mov v1.b[5], w5
2800 ; CHECK-NEXT: ld1 { v6.b }[5], [x11]
2801 ; CHECK-NEXT: ld1 { v16.b }[5], [x12]
2802 ; CHECK-NEXT: ld1 { v4.b }[5], [x8]
2803 ; CHECK-NEXT: ld1 { v5.b }[6], [x9]
2804 ; CHECK-NEXT: movi v17.2d, #0000000000000000
2805 ; CHECK-NEXT: add x8, sp, #656
2806 ; CHECK-NEXT: add x9, sp, #536
2807 ; CHECK-NEXT: add x11, sp, #720
2808 ; CHECK-NEXT: add x12, sp, #592
2809 ; CHECK-NEXT: sshll v18.4s, v18.4h, #0
2810 ; CHECK-NEXT: ldr b7, [sp, #208]
2811 ; CHECK-NEXT: ld1 { v6.b }[6], [x11]
2812 ; CHECK-NEXT: ld1 { v16.b }[6], [x12]
2813 ; CHECK-NEXT: ld1 { v4.b }[6], [x8]
2814 ; CHECK-NEXT: ld1 { v5.b }[7], [x9]
2815 ; CHECK-NEXT: mov v1.b[6], w6
2816 ; CHECK-NEXT: sshll v7.8h, v7.8b, #0
2817 ; CHECK-NEXT: add x8, sp, #664
2818 ; CHECK-NEXT: add x9, sp, #728
2819 ; CHECK-NEXT: add x11, sp, #600
2820 ; CHECK-NEXT: mov v17.s[0], v18.s[0]
2821 ; CHECK-NEXT: ld1 { v6.b }[7], [x9]
2822 ; CHECK-NEXT: ld1 { v16.b }[7], [x11]
2823 ; CHECK-NEXT: ld1 { v4.b }[7], [x8]
2824 ; CHECK-NEXT: sshll v5.8h, v5.8b, #0
2825 ; CHECK-NEXT: movi v18.2d, #0000000000000000
2826 ; CHECK-NEXT: add x10, sp, #200
2827 ; CHECK-NEXT: mov v1.b[7], w7
2828 ; CHECK-NEXT: add x9, sp, #72
2829 ; CHECK-NEXT: sshll v7.4s, v7.4h, #0
2830 ; CHECK-NEXT: ld1 { v2.b }[7], [x10]
2831 ; CHECK-NEXT: ld1 { v3.b }[7], [x9]
2832 ; CHECK-NEXT: sshll v6.8h, v6.8b, #0
2833 ; CHECK-NEXT: sshll v16.8h, v16.8b, #0
2834 ; CHECK-NEXT: sshll v4.8h, v4.8b, #0
2835 ; CHECK-NEXT: saddw v17.4s, v17.4s, v5.4h
2836 ; CHECK-NEXT: sshll v0.8h, v0.8b, #0
2837 ; CHECK-NEXT: mov v18.s[0], v7.s[0]
2838 ; CHECK-NEXT: sshll v1.8h, v1.8b, #0
2839 ; CHECK-NEXT: sshll v2.8h, v2.8b, #0
2840 ; CHECK-NEXT: sshll v3.8h, v3.8b, #0
2841 ; CHECK-NEXT: saddl2 v7.4s, v16.8h, v6.8h
2842 ; CHECK-NEXT: saddl2 v5.4s, v5.8h, v4.8h
2843 ; CHECK-NEXT: saddl v6.4s, v16.4h, v6.4h
2844 ; CHECK-NEXT: saddw v4.4s, v17.4s, v4.4h
2845 ; CHECK-NEXT: saddl2 v17.4s, v1.8h, v0.8h
2846 ; CHECK-NEXT: saddl2 v16.4s, v3.8h, v2.8h
2847 ; CHECK-NEXT: saddw v1.4s, v18.4s, v1.4h
2848 ; CHECK-NEXT: add v5.4s, v5.4s, v7.4s
2849 ; CHECK-NEXT: add v4.4s, v4.4s, v6.4s
2850 ; CHECK-NEXT: saddl v2.4s, v3.4h, v2.4h
2851 ; CHECK-NEXT: add v6.4s, v17.4s, v16.4s
2852 ; CHECK-NEXT: saddw v0.4s, v1.4s, v0.4h
2853 ; CHECK-NEXT: add v1.4s, v4.4s, v5.4s
2854 ; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
2855 ; CHECK-NEXT: add v1.4s, v6.4s, v1.4s
2856 ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
2857 ; CHECK-NEXT: addv s0, v0.4s
2858 ; CHECK-NEXT: fmov w0, s0
2859 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
2860 ; CHECK-NEXT: ret
2861 entry:
2862 %az = sext <33 x i8> %a to <33 x i32>
2863 %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %az)
2864 %cz = sext <33 x i8> %c to <33 x i32>
2865 %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %cz)
2866 %x = add i32 %r1, %r2
2867 ret i32 %x
2868 }
2869 define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
2870 ; CHECK-SD-LABEL: test_udot_v48i8:
2871 ; CHECK-SD: // %bb.0: // %entry
2872 ; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
2873 ; CHECK-SD-NEXT: ldr q1, [x0, #32]
2874 ; CHECK-SD-NEXT: ldr q2, [x1, #32]
2875 ; CHECK-SD-NEXT: udot v0.4s, v2.16b, v1.16b
2876 ; CHECK-SD-NEXT: ldp q3, q1, [x0]
2877 ; CHECK-SD-NEXT: ldp q4, q2, [x1]
2878 ; CHECK-SD-NEXT: udot v0.4s, v4.16b, v3.16b
2879 ; CHECK-SD-NEXT: udot v0.4s, v2.16b, v1.16b
2880 ; CHECK-SD-NEXT: addv s0, v0.4s
2881 ; CHECK-SD-NEXT: fmov w8, s0
2882 ; CHECK-SD-NEXT: add w0, w8, w2
2883 ; CHECK-SD-NEXT: ret
2884 ;
2885 ; CHECK-GI-LABEL: test_udot_v48i8:
2886 ; CHECK-GI: // %bb.0: // %entry
2887 ; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
2888 ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
2889 ; CHECK-GI-NEXT: ldr q7, [x0, #32]
2890 ; CHECK-GI-NEXT: ldp q3, q4, [x0]
2891 ; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
2892 ; CHECK-GI-NEXT: ldp q5, q6, [x1]
2893 ; CHECK-GI-NEXT: ldr q16, [x1, #32]
2894 ; CHECK-GI-NEXT: udot v0.4s, v5.16b, v3.16b
2895 ; CHECK-GI-NEXT: udot v1.4s, v6.16b, v4.16b
2896 ; CHECK-GI-NEXT: udot v2.4s, v16.16b, v7.16b
2897 ; CHECK-GI-NEXT: addv s0, v0.4s
2898 ; CHECK-GI-NEXT: addv s1, v1.4s
2899 ; CHECK-GI-NEXT: addv s2, v2.4s
2900 ; CHECK-GI-NEXT: fmov w8, s0
2901 ; CHECK-GI-NEXT: fmov w9, s1
2902 ; CHECK-GI-NEXT: add w8, w8, w9
2903 ; CHECK-GI-NEXT: fmov w9, s2
2904 ; CHECK-GI-NEXT: add w8, w8, w9
2905 ; CHECK-GI-NEXT: add w0, w8, w2
2906 ; CHECK-GI-NEXT: ret
2907 entry:
2908 %0 = load <48 x i8>, ptr %a
2909 %1 = zext <48 x i8> %0 to <48 x i32>
2910 %2 = load <48 x i8>, ptr %b
2911 %3 = zext <48 x i8> %2 to <48 x i32>
2912 %4 = mul nuw nsw <48 x i32> %3, %1
2913 %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4)
2914 %op.extra = add i32 %5, %sum
2915 ret i32 %op.extra
2916 }
2918 define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) {
2919 ; CHECK-SD-LABEL: test_udot_v48i8_nomla:
2920 ; CHECK-SD: // %bb.0: // %entry
2921 ; CHECK-SD-NEXT: movi v0.16b, #1
2922 ; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
2923 ; CHECK-SD-NEXT: ldr q2, [x0, #32]
2924 ; CHECK-SD-NEXT: udot v1.4s, v2.16b, v0.16b
2925 ; CHECK-SD-NEXT: ldp q3, q2, [x0]
2926 ; CHECK-SD-NEXT: udot v1.4s, v3.16b, v0.16b
2927 ; CHECK-SD-NEXT: udot v1.4s, v2.16b, v0.16b
2928 ; CHECK-SD-NEXT: addv s0, v1.4s
2929 ; CHECK-SD-NEXT: fmov w0, s0
2930 ; CHECK-SD-NEXT: ret
2931 ;
2932 ; CHECK-GI-LABEL: test_udot_v48i8_nomla:
2933 ; CHECK-GI: // %bb.0: // %entry
2934 ; CHECK-GI-NEXT: movi v0.16b, #1
2935 ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
2936 ; CHECK-GI-NEXT: ldr q6, [x0, #32]
2937 ; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
2938 ; CHECK-GI-NEXT: ldp q4, q5, [x0]
2939 ; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
2940 ; CHECK-GI-NEXT: udot v1.4s, v4.16b, v0.16b
2941 ; CHECK-GI-NEXT: udot v2.4s, v5.16b, v0.16b
2942 ; CHECK-GI-NEXT: udot v3.4s, v6.16b, v0.16b
2943 ; CHECK-GI-NEXT: addv s0, v1.4s
2944 ; CHECK-GI-NEXT: addv s1, v2.4s
2945 ; CHECK-GI-NEXT: addv s2, v3.4s
2946 ; CHECK-GI-NEXT: fmov w8, s0
2947 ; CHECK-GI-NEXT: fmov w9, s1
2948 ; CHECK-GI-NEXT: add w8, w8, w9
2949 ; CHECK-GI-NEXT: fmov w9, s2
2950 ; CHECK-GI-NEXT: add w0, w8, w9
2951 ; CHECK-GI-NEXT: ret
2952 entry:
2953 %0 = load <48 x i8>, ptr %a1
2954 %1 = zext <48 x i8> %0 to <48 x i32>
2955 %2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %1)
2956 ret i32 %2
2957 }
2958 define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
2959 ; CHECK-SD-LABEL: test_sdot_v48i8:
2960 ; CHECK-SD: // %bb.0: // %entry
2961 ; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
2962 ; CHECK-SD-NEXT: ldr q1, [x0, #32]
2963 ; CHECK-SD-NEXT: ldr q2, [x1, #32]
2964 ; CHECK-SD-NEXT: sdot v0.4s, v2.16b, v1.16b
2965 ; CHECK-SD-NEXT: ldp q3, q1, [x0]
2966 ; CHECK-SD-NEXT: ldp q4, q2, [x1]
2967 ; CHECK-SD-NEXT: sdot v0.4s, v4.16b, v3.16b
2968 ; CHECK-SD-NEXT: sdot v0.4s, v2.16b, v1.16b
2969 ; CHECK-SD-NEXT: addv s0, v0.4s
2970 ; CHECK-SD-NEXT: fmov w8, s0
2971 ; CHECK-SD-NEXT: add w0, w8, w2
2972 ; CHECK-SD-NEXT: ret
2973 ;
2974 ; CHECK-GI-LABEL: test_sdot_v48i8:
2975 ; CHECK-GI: // %bb.0: // %entry
2976 ; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
2977 ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
2978 ; CHECK-GI-NEXT: ldr q7, [x0, #32]
2979 ; CHECK-GI-NEXT: ldp q3, q4, [x0]
2980 ; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
2981 ; CHECK-GI-NEXT: ldp q5, q6, [x1]
2982 ; CHECK-GI-NEXT: ldr q16, [x1, #32]
2983 ; CHECK-GI-NEXT: sdot v0.4s, v5.16b, v3.16b
2984 ; CHECK-GI-NEXT: sdot v1.4s, v6.16b, v4.16b
2985 ; CHECK-GI-NEXT: sdot v2.4s, v16.16b, v7.16b
2986 ; CHECK-GI-NEXT: addv s0, v0.4s
2987 ; CHECK-GI-NEXT: addv s1, v1.4s
2988 ; CHECK-GI-NEXT: addv s2, v2.4s
2989 ; CHECK-GI-NEXT: fmov w8, s0
2990 ; CHECK-GI-NEXT: fmov w9, s1
2991 ; CHECK-GI-NEXT: add w8, w8, w9
2992 ; CHECK-GI-NEXT: fmov w9, s2
2993 ; CHECK-GI-NEXT: add w8, w8, w9
2994 ; CHECK-GI-NEXT: add w0, w8, w2
2995 ; CHECK-GI-NEXT: ret
2996 entry:
2997 %0 = load <48 x i8>, ptr %a
2998 %1 = sext <48 x i8> %0 to <48 x i32>
2999 %2 = load <48 x i8>, ptr %b
3000 %3 = sext <48 x i8> %2 to <48 x i32>
3001 %4 = mul nsw <48 x i32> %3, %1
3002 %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4)
3003 %op.extra = add nsw i32 %5, %sum
3004 ret i32 %op.extra
3005 }
3007 define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
3008 ; CHECK-SD-LABEL: test_sdot_v48i8_double:
3009 ; CHECK-SD: // %bb.0: // %entry
3010 ; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
3011 ; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
3012 ; CHECK-SD-NEXT: .cfi_offset w29, -16
3013 ; CHECK-SD-NEXT: ldr b3, [sp, #592]
3014 ; CHECK-SD-NEXT: add x8, sp, #600
3015 ; CHECK-SD-NEXT: ldr b6, [sp, #208]
3016 ; CHECK-SD-NEXT: ldr b0, [sp, #336]
3017 ; CHECK-SD-NEXT: add x9, sp, #344
3018 ; CHECK-SD-NEXT: ldr b2, [sp, #464]
3019 ; CHECK-SD-NEXT: ld1 { v3.b }[1], [x8]
3020 ; CHECK-SD-NEXT: add x8, sp, #216
3021 ; CHECK-SD-NEXT: add x10, sp, #624
3022 ; CHECK-SD-NEXT: ld1 { v6.b }[1], [x8]
3023 ; CHECK-SD-NEXT: add x8, sp, #608
3024 ; CHECK-SD-NEXT: ld1 { v0.b }[1], [x9]
3025 ; CHECK-SD-NEXT: add x9, sp, #232
3026 ; CHECK-SD-NEXT: fmov s1, w0
3027 ; CHECK-SD-NEXT: ldr b7, [sp, #1360]
3028 ; CHECK-SD-NEXT: ld1 { v3.b }[2], [x8]
3029 ; CHECK-SD-NEXT: add x8, sp, #224
3030 ; CHECK-SD-NEXT: add x11, sp, #648
3031 ; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8]
3032 ; CHECK-SD-NEXT: add x8, sp, #616
3033 ; CHECK-SD-NEXT: add x12, sp, #376
3034 ; CHECK-SD-NEXT: mov v1.b[1], w1
3035 ; CHECK-SD-NEXT: ldr b16, [sp, #976]
3036 ; CHECK-SD-NEXT: add x14, sp, #288
3037 ; CHECK-SD-NEXT: ld1 { v3.b }[3], [x8]
3038 ; CHECK-SD-NEXT: add x8, sp, #632
3039 ; CHECK-SD-NEXT: add x15, sp, #408
3040 ; CHECK-SD-NEXT: ld1 { v6.b }[3], [x9]
3041 ; CHECK-SD-NEXT: add x9, sp, #472
3042 ; CHECK-SD-NEXT: add x13, sp, #696
3043 ; CHECK-SD-NEXT: ld1 { v2.b }[1], [x9]
3044 ; CHECK-SD-NEXT: add x9, sp, #240
3045 ; CHECK-SD-NEXT: add x16, sp, #448
3046 ; CHECK-SD-NEXT: ld1 { v3.b }[4], [x10]
3047 ; CHECK-SD-NEXT: add x10, sp, #352
3048 ; CHECK-SD-NEXT: mov v1.b[2], w2
3049 ; CHECK-SD-NEXT: ld1 { v6.b }[4], [x9]
3050 ; CHECK-SD-NEXT: ld1 { v0.b }[2], [x10]
3051 ; CHECK-SD-NEXT: add x10, sp, #1368
3052 ; CHECK-SD-NEXT: ld1 { v7.b }[1], [x10]
3053 ; CHECK-SD-NEXT: add x10, sp, #248
3054 ; CHECK-SD-NEXT: add x9, sp, #640
3055 ; CHECK-SD-NEXT: ld1 { v3.b }[5], [x8]
3056 ; CHECK-SD-NEXT: add x8, sp, #656
3057 ; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
3058 ; CHECK-SD-NEXT: ld1 { v6.b }[5], [x10]
3059 ; CHECK-SD-NEXT: add x10, sp, #360
3060 ; CHECK-SD-NEXT: mov v1.b[3], w3
3061 ; CHECK-SD-NEXT: ld1 { v0.b }[3], [x10]
3062 ; CHECK-SD-NEXT: add x10, sp, #256
3063 ; CHECK-SD-NEXT: movi v4.2d, #0000000000000000
3064 ; CHECK-SD-NEXT: ld1 { v3.b }[6], [x9]
3065 ; CHECK-SD-NEXT: add x9, sp, #368
3066 ; CHECK-SD-NEXT: ldr b17, [sp, #720]
3067 ; CHECK-SD-NEXT: ld1 { v6.b }[6], [x10]
3068 ; CHECK-SD-NEXT: add x10, sp, #984
3069 ; CHECK-SD-NEXT: ld1 { v0.b }[4], [x9]
3070 ; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10]
3071 ; CHECK-SD-NEXT: add x10, sp, #664
3072 ; CHECK-SD-NEXT: ld1 { v3.b }[7], [x11]
3073 ; CHECK-SD-NEXT: add x11, sp, #264
3074 ; CHECK-SD-NEXT: mov v1.b[4], w4
3075 ; CHECK-SD-NEXT: ld1 { v6.b }[7], [x11]
3076 ; CHECK-SD-NEXT: add x9, sp, #672
3077 ; CHECK-SD-NEXT: add x11, sp, #680
3078 ; CHECK-SD-NEXT: ld1 { v0.b }[5], [x12]
3079 ; CHECK-SD-NEXT: add x12, sp, #480
3080 ; CHECK-SD-NEXT: ld1 { v2.b }[2], [x12]
3081 ; CHECK-SD-NEXT: add x12, sp, #272
3082 ; CHECK-SD-NEXT: ld1 { v3.b }[8], [x8]
3083 ; CHECK-SD-NEXT: ld1 { v6.b }[8], [x12]
3084 ; CHECK-SD-NEXT: add x12, sp, #384
3085 ; CHECK-SD-NEXT: mov v1.b[5], w5
3086 ; CHECK-SD-NEXT: ld1 { v0.b }[6], [x12]
3087 ; CHECK-SD-NEXT: add x12, sp, #280
3088 ; CHECK-SD-NEXT: add x8, sp, #688
3089 ; CHECK-SD-NEXT: ld1 { v3.b }[9], [x10]
3090 ; CHECK-SD-NEXT: add x10, sp, #1376
3091 ; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10]
3092 ; CHECK-SD-NEXT: add x10, sp, #392
3093 ; CHECK-SD-NEXT: ld1 { v6.b }[9], [x12]
3094 ; CHECK-SD-NEXT: ld1 { v0.b }[7], [x10]
3095 ; CHECK-SD-NEXT: mov v1.b[6], w6
3096 ; CHECK-SD-NEXT: add x12, sp, #704
3097 ; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9]
3098 ; CHECK-SD-NEXT: add x9, sp, #400
3099 ; CHECK-SD-NEXT: add x10, sp, #712
3100 ; CHECK-SD-NEXT: ld1 { v6.b }[10], [x14]
3101 ; CHECK-SD-NEXT: add x14, sp, #992
3102 ; CHECK-SD-NEXT: ld1 { v0.b }[8], [x9]
3103 ; CHECK-SD-NEXT: ld1 { v16.b }[2], [x14]
3104 ; CHECK-SD-NEXT: add x14, sp, #296
3105 ; CHECK-SD-NEXT: ld1 { v3.b }[11], [x11]
3106 ; CHECK-SD-NEXT: add x9, sp, #304
3107 ; CHECK-SD-NEXT: add x11, sp, #312
3108 ; CHECK-SD-NEXT: ld1 { v6.b }[11], [x14]
3109 ; CHECK-SD-NEXT: mov v1.b[7], w7
3110 ; CHECK-SD-NEXT: add x14, sp, #320
3111 ; CHECK-SD-NEXT: ld1 { v0.b }[9], [x15]
3112 ; CHECK-SD-NEXT: add x15, sp, #328
3113 ; CHECK-SD-NEXT: ld1 { v3.b }[12], [x8]
3114 ; CHECK-SD-NEXT: add x8, sp, #416
3115 ; CHECK-SD-NEXT: ld1 { v6.b }[12], [x9]
3116 ; CHECK-SD-NEXT: add x9, sp, #1384
3117 ; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8]
3118 ; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
3119 ; CHECK-SD-NEXT: add x9, sp, #424
3120 ; CHECK-SD-NEXT: ld1 { v3.b }[13], [x13]
3121 ; CHECK-SD-NEXT: add x8, sp, #432
3122 ; CHECK-SD-NEXT: add x13, sp, #440
3123 ; CHECK-SD-NEXT: ld1 { v6.b }[13], [x11]
3124 ; CHECK-SD-NEXT: add x11, sp, #16
3125 ; CHECK-SD-NEXT: ld1 { v0.b }[11], [x9]
3126 ; CHECK-SD-NEXT: add x9, sp, #1000
3127 ; CHECK-SD-NEXT: ld1 { v1.b }[8], [x11]
3128 ; CHECK-SD-NEXT: ld1 { v16.b }[3], [x9]
3129 ; CHECK-SD-NEXT: ld1 { v3.b }[14], [x12]
3130 ; CHECK-SD-NEXT: add x12, sp, #488
3131 ; CHECK-SD-NEXT: ld1 { v6.b }[14], [x14]
3132 ; CHECK-SD-NEXT: add x14, sp, #1392
3133 ; CHECK-SD-NEXT: ld1 { v2.b }[3], [x12]
3134 ; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14]
3135 ; CHECK-SD-NEXT: add x11, sp, #1008
3136 ; CHECK-SD-NEXT: ld1 { v0.b }[12], [x8]
3137 ; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11]
3138 ; CHECK-SD-NEXT: add x8, sp, #1400
3139 ; CHECK-SD-NEXT: ld1 { v3.b }[15], [x10]
3140 ; CHECK-SD-NEXT: add x10, sp, #496
3141 ; CHECK-SD-NEXT: add x9, sp, #24
3142 ; CHECK-SD-NEXT: ld1 { v6.b }[15], [x15]
3143 ; CHECK-SD-NEXT: ld1 { v7.b }[5], [x8]
3144 ; CHECK-SD-NEXT: ld1 { v2.b }[4], [x10]
3145 ; CHECK-SD-NEXT: add x10, sp, #1016
3146 ; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10]
3147 ; CHECK-SD-NEXT: ld1 { v0.b }[13], [x13]
3148 ; CHECK-SD-NEXT: add x8, sp, #1408
3149 ; CHECK-SD-NEXT: ld1 { v1.b }[9], [x9]
3150 ; CHECK-SD-NEXT: add x9, sp, #504
3151 ; CHECK-SD-NEXT: add x10, sp, #512
3152 ; CHECK-SD-NEXT: ld1 { v7.b }[6], [x8]
3153 ; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9]
3154 ; CHECK-SD-NEXT: add x9, sp, #1024
3155 ; CHECK-SD-NEXT: add x8, sp, #32
3156 ; CHECK-SD-NEXT: ld1 { v16.b }[6], [x9]
3157 ; CHECK-SD-NEXT: ld1 { v0.b }[14], [x16]
3158 ; CHECK-SD-NEXT: ld1 { v1.b }[10], [x8]
3159 ; CHECK-SD-NEXT: add x8, sp, #1416
3160 ; CHECK-SD-NEXT: add x9, sp, #456
3161 ; CHECK-SD-NEXT: ld1 { v7.b }[7], [x8]
3162 ; CHECK-SD-NEXT: ld1 { v2.b }[6], [x10]
3163 ; CHECK-SD-NEXT: add x10, sp, #1032
3164 ; CHECK-SD-NEXT: add x8, sp, #40
3165 ; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10]
3166 ; CHECK-SD-NEXT: ld1 { v0.b }[15], [x9]
3167 ; CHECK-SD-NEXT: ld1 { v1.b }[11], [x8]
3168 ; CHECK-SD-NEXT: add x8, sp, #1424
3169 ; CHECK-SD-NEXT: add x9, sp, #520
3170 ; CHECK-SD-NEXT: ld1 { v7.b }[8], [x8]
3171 ; CHECK-SD-NEXT: ld1 { v2.b }[7], [x9]
3172 ; CHECK-SD-NEXT: add x9, sp, #1040
3173 ; CHECK-SD-NEXT: add x8, sp, #48
3174 ; CHECK-SD-NEXT: ld1 { v16.b }[8], [x9]
3175 ; CHECK-SD-NEXT: add x10, sp, #528
3176 ; CHECK-SD-NEXT: ld1 { v1.b }[12], [x8]
3177 ; CHECK-SD-NEXT: add x8, sp, #1432
3178 ; CHECK-SD-NEXT: sdot v5.4s, v6.16b, v3.16b
3179 ; CHECK-SD-NEXT: ld1 { v7.b }[9], [x8]
3180 ; CHECK-SD-NEXT: ld1 { v2.b }[8], [x10]
3181 ; CHECK-SD-NEXT: add x8, sp, #1048
3182 ; CHECK-SD-NEXT: ldr b3, [sp, #80]
3183 ; CHECK-SD-NEXT: ld1 { v16.b }[9], [x8]
3184 ; CHECK-SD-NEXT: add x10, sp, #88
3185 ; CHECK-SD-NEXT: add x8, sp, #536
3186 ; CHECK-SD-NEXT: add x11, sp, #1440
3187 ; CHECK-SD-NEXT: add x9, sp, #56
3188 ; CHECK-SD-NEXT: ld1 { v3.b }[1], [x10]
3189 ; CHECK-SD-NEXT: ld1 { v2.b }[9], [x8]
3190 ; CHECK-SD-NEXT: add x8, sp, #1056
3191 ; CHECK-SD-NEXT: ld1 { v7.b }[10], [x11]
3192 ; CHECK-SD-NEXT: ld1 { v16.b }[10], [x8]
3193 ; CHECK-SD-NEXT: ld1 { v1.b }[13], [x9]
3194 ; CHECK-SD-NEXT: add x9, sp, #96
3195 ; CHECK-SD-NEXT: add x8, sp, #544
3196 ; CHECK-SD-NEXT: add x10, sp, #1448
3197 ; CHECK-SD-NEXT: ld1 { v3.b }[2], [x9]
3198 ; CHECK-SD-NEXT: ld1 { v2.b }[10], [x8]
3199 ; CHECK-SD-NEXT: add x8, sp, #1064
3200 ; CHECK-SD-NEXT: ld1 { v7.b }[11], [x10]
3201 ; CHECK-SD-NEXT: ld1 { v16.b }[11], [x8]
3202 ; CHECK-SD-NEXT: add x10, sp, #104
3203 ; CHECK-SD-NEXT: add x8, sp, #552
3204 ; CHECK-SD-NEXT: add x11, sp, #1456
3205 ; CHECK-SD-NEXT: add x9, sp, #64
3206 ; CHECK-SD-NEXT: ld1 { v3.b }[3], [x10]
3207 ; CHECK-SD-NEXT: ld1 { v2.b }[11], [x8]
3208 ; CHECK-SD-NEXT: add x8, sp, #1072
3209 ; CHECK-SD-NEXT: ld1 { v7.b }[12], [x11]
3210 ; CHECK-SD-NEXT: ld1 { v16.b }[12], [x8]
3211 ; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9]
3212 ; CHECK-SD-NEXT: add x9, sp, #112
3213 ; CHECK-SD-NEXT: add x8, sp, #560
3214 ; CHECK-SD-NEXT: add x10, sp, #1464
3215 ; CHECK-SD-NEXT: ld1 { v3.b }[4], [x9]
3216 ; CHECK-SD-NEXT: ld1 { v2.b }[12], [x8]
3217 ; CHECK-SD-NEXT: add x8, sp, #1080
3218 ; CHECK-SD-NEXT: ld1 { v7.b }[13], [x10]
3219 ; CHECK-SD-NEXT: ld1 { v16.b }[13], [x8]
3220 ; CHECK-SD-NEXT: add x10, sp, #120
3221 ; CHECK-SD-NEXT: add x8, sp, #568
3222 ; CHECK-SD-NEXT: add x11, sp, #1472
3223 ; CHECK-SD-NEXT: add x9, sp, #72
3224 ; CHECK-SD-NEXT: ld1 { v3.b }[5], [x10]
3225 ; CHECK-SD-NEXT: ld1 { v2.b }[13], [x8]
3226 ; CHECK-SD-NEXT: add x8, sp, #1088
3227 ; CHECK-SD-NEXT: ld1 { v7.b }[14], [x11]
3228 ; CHECK-SD-NEXT: ld1 { v16.b }[14], [x8]
3229 ; CHECK-SD-NEXT: ld1 { v1.b }[15], [x9]
3230 ; CHECK-SD-NEXT: add x9, sp, #128
3231 ; CHECK-SD-NEXT: ldr b6, [sp, #1104]
3232 ; CHECK-SD-NEXT: add x10, sp, #1480
3233 ; CHECK-SD-NEXT: ld1 { v3.b }[6], [x9]
3234 ; CHECK-SD-NEXT: add x8, sp, #1096
3235 ; CHECK-SD-NEXT: add x9, sp, #1112
3236 ; CHECK-SD-NEXT: ld1 { v7.b }[15], [x10]
3237 ; CHECK-SD-NEXT: ld1 { v16.b }[15], [x8]
3238 ; CHECK-SD-NEXT: ld1 { v6.b }[1], [x9]
3239 ; CHECK-SD-NEXT: add x8, sp, #728
3240 ; CHECK-SD-NEXT: add x9, sp, #576
3241 ; CHECK-SD-NEXT: add x10, sp, #136
3242 ; CHECK-SD-NEXT: ld1 { v17.b }[1], [x8]
3243 ; CHECK-SD-NEXT: add x8, sp, #1120
3244 ; CHECK-SD-NEXT: ld1 { v2.b }[14], [x9]
3245 ; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b
3246 ; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8]
3247 ; CHECK-SD-NEXT: add x8, sp, #736
3248 ; CHECK-SD-NEXT: ldr b7, [sp, #1232]
3249 ; CHECK-SD-NEXT: ldr b16, [sp, #848]
3250 ; CHECK-SD-NEXT: ld1 { v3.b }[7], [x10]
3251 ; CHECK-SD-NEXT: ld1 { v17.b }[2], [x8]
3252 ; CHECK-SD-NEXT: add x9, sp, #1240
3253 ; CHECK-SD-NEXT: add x10, sp, #856
3254 ; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9]
3255 ; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10]
3256 ; CHECK-SD-NEXT: add x8, sp, #1128
3257 ; CHECK-SD-NEXT: add x11, sp, #744
3258 ; CHECK-SD-NEXT: ld1 { v6.b }[3], [x8]
3259 ; CHECK-SD-NEXT: add x10, sp, #1248
3260 ; CHECK-SD-NEXT: ld1 { v17.b }[3], [x11]
3261 ; CHECK-SD-NEXT: add x11, sp, #864
3262 ; CHECK-SD-NEXT: add x9, sp, #144
3263 ; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10]
3264 ; CHECK-SD-NEXT: ld1 { v16.b }[2], [x11]
3265 ; CHECK-SD-NEXT: add x8, sp, #1136
3266 ; CHECK-SD-NEXT: add x12, sp, #752
3267 ; CHECK-SD-NEXT: ld1 { v3.b }[8], [x9]
3268 ; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8]
3269 ; CHECK-SD-NEXT: ld1 { v17.b }[4], [x12]
3270 ; CHECK-SD-NEXT: add x9, sp, #1256
3271 ; CHECK-SD-NEXT: add x10, sp, #872
3272 ; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
3273 ; CHECK-SD-NEXT: ld1 { v16.b }[3], [x10]
3274 ; CHECK-SD-NEXT: add x8, sp, #1144
3275 ; CHECK-SD-NEXT: add x11, sp, #760
3276 ; CHECK-SD-NEXT: ld1 { v6.b }[5], [x8]
3277 ; CHECK-SD-NEXT: add x10, sp, #1264
3278 ; CHECK-SD-NEXT: ld1 { v17.b }[5], [x11]
3279 ; CHECK-SD-NEXT: add x11, sp, #880
3280 ; CHECK-SD-NEXT: add x9, sp, #152
3281 ; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10]
3282 ; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11]
3283 ; CHECK-SD-NEXT: add x8, sp, #1152
3284 ; CHECK-SD-NEXT: add x12, sp, #768
3285 ; CHECK-SD-NEXT: ld1 { v3.b }[9], [x9]
3286 ; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8]
3287 ; CHECK-SD-NEXT: ld1 { v17.b }[6], [x12]
3288 ; CHECK-SD-NEXT: add x9, sp, #1272
3289 ; CHECK-SD-NEXT: add x10, sp, #888
3290 ; CHECK-SD-NEXT: ld1 { v7.b }[5], [x9]
3291 ; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10]
3292 ; CHECK-SD-NEXT: add x8, sp, #1160
3293 ; CHECK-SD-NEXT: add x11, sp, #776
3294 ; CHECK-SD-NEXT: ld1 { v6.b }[7], [x8]
3295 ; CHECK-SD-NEXT: add x10, sp, #1280
3296 ; CHECK-SD-NEXT: ld1 { v17.b }[7], [x11]
3297 ; CHECK-SD-NEXT: add x11, sp, #896
3298 ; CHECK-SD-NEXT: add x9, sp, #160
3299 ; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10]
3300 ; CHECK-SD-NEXT: ld1 { v16.b }[6], [x11]
3301 ; CHECK-SD-NEXT: add x8, sp, #1168
3302 ; CHECK-SD-NEXT: add x12, sp, #784
3303 ; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9]
3304 ; CHECK-SD-NEXT: ld1 { v6.b }[8], [x8]
3305 ; CHECK-SD-NEXT: ld1 { v17.b }[8], [x12]
3306 ; CHECK-SD-NEXT: add x9, sp, #1288
3307 ; CHECK-SD-NEXT: add x10, sp, #904
3308 ; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9]
3309 ; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10]
3310 ; CHECK-SD-NEXT: add x8, sp, #1176
3311 ; CHECK-SD-NEXT: add x11, sp, #792
3312 ; CHECK-SD-NEXT: ld1 { v6.b }[9], [x8]
3313 ; CHECK-SD-NEXT: add x10, sp, #1296
3314 ; CHECK-SD-NEXT: ld1 { v17.b }[9], [x11]
3315 ; CHECK-SD-NEXT: add x11, sp, #912
3316 ; CHECK-SD-NEXT: add x9, sp, #168
3317 ; CHECK-SD-NEXT: ld1 { v7.b }[8], [x10]
3318 ; CHECK-SD-NEXT: ld1 { v16.b }[8], [x11]
3319 ; CHECK-SD-NEXT: add x8, sp, #1184
3320 ; CHECK-SD-NEXT: add x12, sp, #800
3321 ; CHECK-SD-NEXT: ld1 { v3.b }[11], [x9]
3322 ; CHECK-SD-NEXT: ld1 { v6.b }[10], [x8]
3323 ; CHECK-SD-NEXT: ld1 { v17.b }[10], [x12]
3324 ; CHECK-SD-NEXT: add x9, sp, #1304
3325 ; CHECK-SD-NEXT: add x10, sp, #920
3326 ; CHECK-SD-NEXT: ld1 { v7.b }[9], [x9]
3327 ; CHECK-SD-NEXT: ld1 { v16.b }[9], [x10]
3328 ; CHECK-SD-NEXT: add x8, sp, #1192
3329 ; CHECK-SD-NEXT: add x11, sp, #808
3330 ; CHECK-SD-NEXT: ld1 { v6.b }[11], [x8]
3331 ; CHECK-SD-NEXT: add x10, sp, #1312
3332 ; CHECK-SD-NEXT: ld1 { v17.b }[11], [x11]
3333 ; CHECK-SD-NEXT: add x11, sp, #928
3334 ; CHECK-SD-NEXT: add x9, sp, #176
3335 ; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10]
3336 ; CHECK-SD-NEXT: ld1 { v16.b }[10], [x11]
3337 ; CHECK-SD-NEXT: add x8, sp, #1200
3338 ; CHECK-SD-NEXT: add x12, sp, #816
3339 ; CHECK-SD-NEXT: ld1 { v3.b }[12], [x9]
3340 ; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8]
3341 ; CHECK-SD-NEXT: ld1 { v17.b }[12], [x12]
3342 ; CHECK-SD-NEXT: add x9, sp, #1320
3343 ; CHECK-SD-NEXT: add x10, sp, #936
3344 ; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9]
3345 ; CHECK-SD-NEXT: ld1 { v16.b }[11], [x10]
3346 ; CHECK-SD-NEXT: add x8, sp, #1208
3347 ; CHECK-SD-NEXT: add x11, sp, #824
3348 ; CHECK-SD-NEXT: ld1 { v6.b }[13], [x8]
3349 ; CHECK-SD-NEXT: add x10, sp, #1328
3350 ; CHECK-SD-NEXT: ld1 { v17.b }[13], [x11]
3351 ; CHECK-SD-NEXT: add x11, sp, #944
3352 ; CHECK-SD-NEXT: add x9, sp, #184
3353 ; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10]
3354 ; CHECK-SD-NEXT: ld1 { v16.b }[12], [x11]
3355 ; CHECK-SD-NEXT: add x8, sp, #1216
3356 ; CHECK-SD-NEXT: add x12, sp, #832
3357 ; CHECK-SD-NEXT: ld1 { v3.b }[13], [x9]
3358 ; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8]
3359 ; CHECK-SD-NEXT: ld1 { v17.b }[14], [x12]
3360 ; CHECK-SD-NEXT: add x9, sp, #1336
3361 ; CHECK-SD-NEXT: add x10, sp, #952
3362 ; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9]
3363 ; CHECK-SD-NEXT: ld1 { v16.b }[13], [x10]
3364 ; CHECK-SD-NEXT: add x8, sp, #1224
3365 ; CHECK-SD-NEXT: add x11, sp, #840
3366 ; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8]
3367 ; CHECK-SD-NEXT: add x8, sp, #192
3368 ; CHECK-SD-NEXT: ld1 { v17.b }[15], [x11]
3369 ; CHECK-SD-NEXT: add x10, sp, #1344
3370 ; CHECK-SD-NEXT: add x11, sp, #960
3371 ; CHECK-SD-NEXT: ld1 { v3.b }[14], [x8]
3372 ; CHECK-SD-NEXT: ld1 { v7.b }[14], [x10]
3373 ; CHECK-SD-NEXT: ld1 { v16.b }[14], [x11]
3374 ; CHECK-SD-NEXT: add x9, sp, #584
3375 ; CHECK-SD-NEXT: sdot v5.4s, v1.16b, v0.16b
3376 ; CHECK-SD-NEXT: add x8, sp, #200
3377 ; CHECK-SD-NEXT: sdot v4.4s, v17.16b, v6.16b
3378 ; CHECK-SD-NEXT: ld1 { v2.b }[15], [x9]
3379 ; CHECK-SD-NEXT: add x9, sp, #1352
3380 ; CHECK-SD-NEXT: add x10, sp, #968
3381 ; CHECK-SD-NEXT: ld1 { v3.b }[15], [x8]
3382 ; CHECK-SD-NEXT: ld1 { v7.b }[15], [x9]
3383 ; CHECK-SD-NEXT: ld1 { v16.b }[15], [x10]
3384 ; CHECK-SD-NEXT: sdot v5.4s, v3.16b, v2.16b
3385 ; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b
3386 ; CHECK-SD-NEXT: add v0.4s, v5.4s, v4.4s
3387 ; CHECK-SD-NEXT: addv s0, v0.4s
3388 ; CHECK-SD-NEXT: fmov w0, s0
3389 ; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
3390 ; CHECK-SD-NEXT: ret
3391 ;
3392 ; CHECK-GI-LABEL: test_sdot_v48i8_double:
3393 ; CHECK-GI: // %bb.0: // %entry
3394 ; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
3395 ; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
3396 ; CHECK-GI-NEXT: .cfi_offset w29, -16
3397 ; CHECK-GI-NEXT: ldr w11, [sp, #80]
3398 ; CHECK-GI-NEXT: ldr w10, [sp, #208]
3399 ; CHECK-GI-NEXT: fmov s0, w0
3400 ; CHECK-GI-NEXT: ldr w8, [sp, #88]
3401 ; CHECK-GI-NEXT: ldr w12, [sp, #344]
3402 ; CHECK-GI-NEXT: movi v20.2d, #0000000000000000
3403 ; CHECK-GI-NEXT: fmov s1, w11
3404 ; CHECK-GI-NEXT: ldr w11, [sp, #336]
3405 ; CHECK-GI-NEXT: fmov s2, w10
3406 ; CHECK-GI-NEXT: ldr w10, [sp, #464]
3407 ; CHECK-GI-NEXT: ldr w9, [sp, #216]
3408 ; CHECK-GI-NEXT: mov v0.b[1], w1
3409 ; CHECK-GI-NEXT: fmov s3, w11
3410 ; CHECK-GI-NEXT: ldr w11, [sp, #600]
3411 ; CHECK-GI-NEXT: movi v21.2d, #0000000000000000
3412 ; CHECK-GI-NEXT: mov v1.b[1], w8
3413 ; CHECK-GI-NEXT: ldr w8, [sp, #592]
3414 ; CHECK-GI-NEXT: fmov s4, w10
3415 ; CHECK-GI-NEXT: mov v2.b[1], w9
3416 ; CHECK-GI-NEXT: ldr w9, [sp, #472]
3417 ; CHECK-GI-NEXT: ldr w10, [sp, #608]
3418 ; CHECK-GI-NEXT: mov v3.b[1], w12
3419 ; CHECK-GI-NEXT: fmov s5, w8
3420 ; CHECK-GI-NEXT: ldr w8, [sp, #96]
3421 ; CHECK-GI-NEXT: mov v4.b[1], w9
3422 ; CHECK-GI-NEXT: ldr w9, [sp, #224]
3423 ; CHECK-GI-NEXT: mov v0.b[2], w2
3424 ; CHECK-GI-NEXT: mov v1.b[2], w8
3425 ; CHECK-GI-NEXT: ldr w8, [sp, #352]
3426 ; CHECK-GI-NEXT: ldr w12, [sp, #848]
3427 ; CHECK-GI-NEXT: mov v2.b[2], w9
3428 ; CHECK-GI-NEXT: ldr w9, [sp, #480]
3429 ; CHECK-GI-NEXT: mov v5.b[1], w11
3430 ; CHECK-GI-NEXT: mov v3.b[2], w8
3431 ; CHECK-GI-NEXT: ldr w8, [sp, #104]
3432 ; CHECK-GI-NEXT: ldr w11, [sp, #16]
3433 ; CHECK-GI-NEXT: mov v4.b[2], w9
3434 ; CHECK-GI-NEXT: ldr w9, [sp, #232]
3435 ; CHECK-GI-NEXT: mov v0.b[3], w3
3436 ; CHECK-GI-NEXT: mov v1.b[3], w8
3437 ; CHECK-GI-NEXT: ldr w8, [sp, #360]
3438 ; CHECK-GI-NEXT: fmov s7, w12
3439 ; CHECK-GI-NEXT: mov v2.b[3], w9
3440 ; CHECK-GI-NEXT: ldr w9, [sp, #488]
3441 ; CHECK-GI-NEXT: mov v5.b[2], w10
3442 ; CHECK-GI-NEXT: mov v3.b[3], w8
3443 ; CHECK-GI-NEXT: ldr w8, [sp, #112]
3444 ; CHECK-GI-NEXT: ldr w10, [sp, #616]
3445 ; CHECK-GI-NEXT: mov v4.b[3], w9
3446 ; CHECK-GI-NEXT: ldr w9, [sp, #240]
3447 ; CHECK-GI-NEXT: mov v0.b[4], w4
3448 ; CHECK-GI-NEXT: mov v1.b[4], w8
3449 ; CHECK-GI-NEXT: ldr w8, [sp, #368]
3450 ; CHECK-GI-NEXT: ldr w12, [sp, #1112]
3451 ; CHECK-GI-NEXT: mov v2.b[4], w9
3452 ; CHECK-GI-NEXT: ldr w9, [sp, #496]
3453 ; CHECK-GI-NEXT: mov v5.b[3], w10
3454 ; CHECK-GI-NEXT: mov v3.b[4], w8
3455 ; CHECK-GI-NEXT: ldr w8, [sp, #120]
3456 ; CHECK-GI-NEXT: ldr w10, [sp, #624]
3457 ; CHECK-GI-NEXT: mov v4.b[4], w9
3458 ; CHECK-GI-NEXT: ldr w9, [sp, #248]
3459 ; CHECK-GI-NEXT: mov v0.b[5], w5
3460 ; CHECK-GI-NEXT: mov v1.b[5], w8
3461 ; CHECK-GI-NEXT: ldr w8, [sp, #376]
3462 ; CHECK-GI-NEXT: movi v22.2d, #0000000000000000
3463 ; CHECK-GI-NEXT: mov v2.b[5], w9
3464 ; CHECK-GI-NEXT: ldr w9, [sp, #504]
3465 ; CHECK-GI-NEXT: mov v5.b[4], w10
3466 ; CHECK-GI-NEXT: mov v3.b[5], w8
3467 ; CHECK-GI-NEXT: ldr w8, [sp, #128]
3468 ; CHECK-GI-NEXT: ldr w10, [sp, #632]
3469 ; CHECK-GI-NEXT: mov v4.b[5], w9
3470 ; CHECK-GI-NEXT: ldr w9, [sp, #256]
3471 ; CHECK-GI-NEXT: mov v0.b[6], w6
3472 ; CHECK-GI-NEXT: mov v1.b[6], w8
3473 ; CHECK-GI-NEXT: ldr w8, [sp, #384]
3474 ; CHECK-GI-NEXT: movi v23.2d, #0000000000000000
3475 ; CHECK-GI-NEXT: mov v2.b[6], w9
3476 ; CHECK-GI-NEXT: ldr w9, [sp, #512]
3477 ; CHECK-GI-NEXT: mov v5.b[5], w10
3478 ; CHECK-GI-NEXT: mov v3.b[6], w8
3479 ; CHECK-GI-NEXT: ldr w8, [sp, #136]
3480 ; CHECK-GI-NEXT: ldr w10, [sp, #640]
3481 ; CHECK-GI-NEXT: mov v4.b[6], w9
3482 ; CHECK-GI-NEXT: ldr w9, [sp, #264]
3483 ; CHECK-GI-NEXT: mov v0.b[7], w7
3484 ; CHECK-GI-NEXT: mov v1.b[7], w8
3485 ; CHECK-GI-NEXT: ldr w8, [sp, #392]
3486 ; CHECK-GI-NEXT: movi v24.2d, #0000000000000000
3487 ; CHECK-GI-NEXT: mov v2.b[7], w9
3488 ; CHECK-GI-NEXT: ldr w9, [sp, #520]
3489 ; CHECK-GI-NEXT: mov v5.b[6], w10
3490 ; CHECK-GI-NEXT: mov v3.b[7], w8
3491 ; CHECK-GI-NEXT: ldr w8, [sp, #144]
3492 ; CHECK-GI-NEXT: ldr w10, [sp, #648]
3493 ; CHECK-GI-NEXT: mov v4.b[7], w9
3494 ; CHECK-GI-NEXT: ldr w9, [sp, #272]
3495 ; CHECK-GI-NEXT: mov v0.b[8], w11
3496 ; CHECK-GI-NEXT: mov v1.b[8], w8
3497 ; CHECK-GI-NEXT: ldr w8, [sp, #400]
3498 ; CHECK-GI-NEXT: ldr w11, [sp, #24]
3499 ; CHECK-GI-NEXT: mov v2.b[8], w9
3500 ; CHECK-GI-NEXT: ldr w9, [sp, #528]
3501 ; CHECK-GI-NEXT: mov v5.b[7], w10
3502 ; CHECK-GI-NEXT: mov v3.b[8], w8
3503 ; CHECK-GI-NEXT: ldr w8, [sp, #152]
3504 ; CHECK-GI-NEXT: ldr w10, [sp, #656]
3505 ; CHECK-GI-NEXT: mov v4.b[8], w9
3506 ; CHECK-GI-NEXT: ldr w9, [sp, #280]
3507 ; CHECK-GI-NEXT: mov v0.b[9], w11
3508 ; CHECK-GI-NEXT: mov v1.b[9], w8
3509 ; CHECK-GI-NEXT: ldr w8, [sp, #408]
3510 ; CHECK-GI-NEXT: ldr w11, [sp, #32]
3511 ; CHECK-GI-NEXT: mov v2.b[9], w9
3512 ; CHECK-GI-NEXT: ldr w9, [sp, #536]
3513 ; CHECK-GI-NEXT: mov v5.b[8], w10
3514 ; CHECK-GI-NEXT: mov v3.b[9], w8
3515 ; CHECK-GI-NEXT: ldr w8, [sp, #160]
3516 ; CHECK-GI-NEXT: ldr w10, [sp, #664]
3517 ; CHECK-GI-NEXT: mov v4.b[9], w9
3518 ; CHECK-GI-NEXT: ldr w9, [sp, #288]
3519 ; CHECK-GI-NEXT: mov v0.b[10], w11
3520 ; CHECK-GI-NEXT: mov v1.b[10], w8
3521 ; CHECK-GI-NEXT: ldr w8, [sp, #416]
3522 ; CHECK-GI-NEXT: ldr w11, [sp, #40]
3523 ; CHECK-GI-NEXT: mov v2.b[10], w9
3524 ; CHECK-GI-NEXT: ldr w9, [sp, #544]
3525 ; CHECK-GI-NEXT: mov v5.b[9], w10
3526 ; CHECK-GI-NEXT: mov v3.b[10], w8
3527 ; CHECK-GI-NEXT: ldr w8, [sp, #168]
3528 ; CHECK-GI-NEXT: ldr w10, [sp, #672]
3529 ; CHECK-GI-NEXT: mov v4.b[10], w9
3530 ; CHECK-GI-NEXT: ldr w9, [sp, #296]
3531 ; CHECK-GI-NEXT: mov v0.b[11], w11
3532 ; CHECK-GI-NEXT: mov v1.b[11], w8
3533 ; CHECK-GI-NEXT: ldr w8, [sp, #424]
3534 ; CHECK-GI-NEXT: ldr w11, [sp, #48]
3535 ; CHECK-GI-NEXT: mov v2.b[11], w9
3536 ; CHECK-GI-NEXT: ldr w9, [sp, #552]
3537 ; CHECK-GI-NEXT: mov v5.b[10], w10
3538 ; CHECK-GI-NEXT: mov v3.b[11], w8
3539 ; CHECK-GI-NEXT: ldr w8, [sp, #176]
3540 ; CHECK-GI-NEXT: ldr w10, [sp, #680]
3541 ; CHECK-GI-NEXT: mov v4.b[11], w9
3542 ; CHECK-GI-NEXT: ldr w9, [sp, #304]
3543 ; CHECK-GI-NEXT: mov v0.b[12], w11
3544 ; CHECK-GI-NEXT: mov v1.b[12], w8
3545 ; CHECK-GI-NEXT: ldr w8, [sp, #432]
3546 ; CHECK-GI-NEXT: ldr w11, [sp, #56]
3547 ; CHECK-GI-NEXT: mov v2.b[12], w9
3548 ; CHECK-GI-NEXT: ldr w9, [sp, #560]
3549 ; CHECK-GI-NEXT: mov v5.b[11], w10
3550 ; CHECK-GI-NEXT: mov v3.b[12], w8
3551 ; CHECK-GI-NEXT: ldr w8, [sp, #184]
3552 ; CHECK-GI-NEXT: ldr w10, [sp, #688]
3553 ; CHECK-GI-NEXT: mov v4.b[12], w9
3554 ; CHECK-GI-NEXT: ldr w9, [sp, #312]
3555 ; CHECK-GI-NEXT: mov v0.b[13], w11
3556 ; CHECK-GI-NEXT: mov v1.b[13], w8
3557 ; CHECK-GI-NEXT: ldr w8, [sp, #440]
3558 ; CHECK-GI-NEXT: ldr w11, [sp, #64]
3559 ; CHECK-GI-NEXT: mov v2.b[13], w9
3560 ; CHECK-GI-NEXT: ldr w9, [sp, #568]
3561 ; CHECK-GI-NEXT: mov v5.b[12], w10
3562 ; CHECK-GI-NEXT: mov v3.b[13], w8
3563 ; CHECK-GI-NEXT: ldr w8, [sp, #192]
3564 ; CHECK-GI-NEXT: ldr w10, [sp, #696]
3565 ; CHECK-GI-NEXT: mov v4.b[13], w9
3566 ; CHECK-GI-NEXT: ldr w9, [sp, #320]
3567 ; CHECK-GI-NEXT: mov v0.b[14], w11
3568 ; CHECK-GI-NEXT: mov v1.b[14], w8
3569 ; CHECK-GI-NEXT: ldr w8, [sp, #448]
3570 ; CHECK-GI-NEXT: ldr w11, [sp, #72]
3571 ; CHECK-GI-NEXT: mov v2.b[14], w9
3572 ; CHECK-GI-NEXT: ldr w9, [sp, #576]
3573 ; CHECK-GI-NEXT: mov v5.b[13], w10
3574 ; CHECK-GI-NEXT: mov v3.b[14], w8
3575 ; CHECK-GI-NEXT: ldr w8, [sp, #720]
3576 ; CHECK-GI-NEXT: ldr w10, [sp, #704]
3577 ; CHECK-GI-NEXT: mov v4.b[14], w9
3578 ; CHECK-GI-NEXT: ldr w9, [sp, #728]
3579 ; CHECK-GI-NEXT: mov v0.b[15], w11
3580 ; CHECK-GI-NEXT: fmov s6, w8
3581 ; CHECK-GI-NEXT: ldr w8, [sp, #328]
3582 ; CHECK-GI-NEXT: ldr w11, [sp, #456]
3583 ; CHECK-GI-NEXT: mov v5.b[14], w10
3584 ; CHECK-GI-NEXT: ldr w10, [sp, #200]
3585 ; CHECK-GI-NEXT: movi v25.2d, #0000000000000000
3586 ; CHECK-GI-NEXT: mov v2.b[15], w8
3587 ; CHECK-GI-NEXT: mov v3.b[15], w11
3588 ; CHECK-GI-NEXT: ldr w11, [sp, #736]
3589 ; CHECK-GI-NEXT: mov v6.b[1], w9
3590 ; CHECK-GI-NEXT: ldr w9, [sp, #584]
3591 ; CHECK-GI-NEXT: ldr w8, [sp, #856]
3592 ; CHECK-GI-NEXT: mov v1.b[15], w10
3593 ; CHECK-GI-NEXT: ldr w10, [sp, #712]
3594 ; CHECK-GI-NEXT: mov v4.b[15], w9
3595 ; CHECK-GI-NEXT: ldr w9, [sp, #976]
3596 ; CHECK-GI-NEXT: mov v7.b[1], w8
3597 ; CHECK-GI-NEXT: ldr w8, [sp, #1232]
3598 ; CHECK-GI-NEXT: mov v5.b[15], w10
3599 ; CHECK-GI-NEXT: ldr w10, [sp, #984]
3600 ; CHECK-GI-NEXT: mov v6.b[2], w11
3601 ; CHECK-GI-NEXT: ldr w11, [sp, #1104]
3602 ; CHECK-GI-NEXT: fmov s16, w9
3603 ; CHECK-GI-NEXT: ldr w9, [sp, #1360]
3604 ; CHECK-GI-NEXT: fmov s18, w8
3605 ; CHECK-GI-NEXT: ldr w8, [sp, #1368]
3606 ; CHECK-GI-NEXT: fmov s17, w11
3607 ; CHECK-GI-NEXT: ldr w11, [sp, #1240]
3608 ; CHECK-GI-NEXT: sdot v20.4s, v0.16b, v3.16b
3609 ; CHECK-GI-NEXT: mov v16.b[1], w10
3610 ; CHECK-GI-NEXT: fmov s19, w9
3611 ; CHECK-GI-NEXT: ldr w10, [sp, #864]
3612 ; CHECK-GI-NEXT: mov v18.b[1], w11
3613 ; CHECK-GI-NEXT: ldr w11, [sp, #992]
3614 ; CHECK-GI-NEXT: ldr w9, [sp, #1120]
3615 ; CHECK-GI-NEXT: mov v17.b[1], w12
3616 ; CHECK-GI-NEXT: mov v7.b[2], w10
3617 ; CHECK-GI-NEXT: ldr w10, [sp, #1248]
3618 ; CHECK-GI-NEXT: mov v19.b[1], w8
3619 ; CHECK-GI-NEXT: ldr w8, [sp, #744]
3620 ; CHECK-GI-NEXT: sdot v21.4s, v1.16b, v4.16b
3621 ; CHECK-GI-NEXT: mov v16.b[2], w11
3622 ; CHECK-GI-NEXT: ldr w11, [sp, #872]
3623 ; CHECK-GI-NEXT: addv s0, v20.4s
3624 ; CHECK-GI-NEXT: mov v6.b[3], w8
3625 ; CHECK-GI-NEXT: ldr w8, [sp, #1000]
3626 ; CHECK-GI-NEXT: mov v18.b[2], w10
3627 ; CHECK-GI-NEXT: mov v17.b[2], w9
3628 ; CHECK-GI-NEXT: ldr w9, [sp, #1376]
3629 ; CHECK-GI-NEXT: ldr w10, [sp, #1128]
3630 ; CHECK-GI-NEXT: mov v7.b[3], w11
3631 ; CHECK-GI-NEXT: ldr w11, [sp, #880]
3632 ; CHECK-GI-NEXT: addv s1, v21.4s
3633 ; CHECK-GI-NEXT: mov v19.b[2], w9
3634 ; CHECK-GI-NEXT: ldr w9, [sp, #752]
3635 ; CHECK-GI-NEXT: mov v16.b[3], w8
3636 ; CHECK-GI-NEXT: ldr w8, [sp, #1256]
3637 ; CHECK-GI-NEXT: sdot v25.4s, v2.16b, v5.16b
3638 ; CHECK-GI-NEXT: mov v17.b[3], w10
3639 ; CHECK-GI-NEXT: ldr w10, [sp, #1384]
3640 ; CHECK-GI-NEXT: mov v6.b[4], w9
3641 ; CHECK-GI-NEXT: ldr w9, [sp, #1008]
3642 ; CHECK-GI-NEXT: mov v18.b[3], w8
3643 ; CHECK-GI-NEXT: ldr w8, [sp, #1136]
3644 ; CHECK-GI-NEXT: mov v19.b[3], w10
3645 ; CHECK-GI-NEXT: ldr w10, [sp, #760]
3646 ; CHECK-GI-NEXT: mov v7.b[4], w11
3647 ; CHECK-GI-NEXT: mov v16.b[4], w9
3648 ; CHECK-GI-NEXT: ldr w9, [sp, #1264]
3649 ; CHECK-GI-NEXT: ldr w11, [sp, #888]
3650 ; CHECK-GI-NEXT: mov v17.b[4], w8
3651 ; CHECK-GI-NEXT: ldr w8, [sp, #1392]
3652 ; CHECK-GI-NEXT: mov v6.b[5], w10
3653 ; CHECK-GI-NEXT: ldr w10, [sp, #1016]
3654 ; CHECK-GI-NEXT: mov v18.b[4], w9
3655 ; CHECK-GI-NEXT: ldr w9, [sp, #1144]
3656 ; CHECK-GI-NEXT: mov v19.b[4], w8
3657 ; CHECK-GI-NEXT: ldr w8, [sp, #768]
3658 ; CHECK-GI-NEXT: mov v7.b[5], w11
3659 ; CHECK-GI-NEXT: mov v16.b[5], w10
3660 ; CHECK-GI-NEXT: ldr w10, [sp, #1272]
3661 ; CHECK-GI-NEXT: ldr w11, [sp, #896]
3662 ; CHECK-GI-NEXT: mov v17.b[5], w9
3663 ; CHECK-GI-NEXT: ldr w9, [sp, #1400]
3664 ; CHECK-GI-NEXT: mov v6.b[6], w8
3665 ; CHECK-GI-NEXT: ldr w8, [sp, #1024]
3666 ; CHECK-GI-NEXT: mov v18.b[5], w10
3667 ; CHECK-GI-NEXT: ldr w10, [sp, #1152]
3668 ; CHECK-GI-NEXT: mov v19.b[5], w9
3669 ; CHECK-GI-NEXT: ldr w9, [sp, #776]
3670 ; CHECK-GI-NEXT: mov v7.b[6], w11
3671 ; CHECK-GI-NEXT: mov v16.b[6], w8
3672 ; CHECK-GI-NEXT: ldr w8, [sp, #1280]
3673 ; CHECK-GI-NEXT: ldr w11, [sp, #904]
3674 ; CHECK-GI-NEXT: mov v17.b[6], w10
3675 ; CHECK-GI-NEXT: ldr w10, [sp, #1408]
3676 ; CHECK-GI-NEXT: mov v6.b[7], w9
3677 ; CHECK-GI-NEXT: ldr w9, [sp, #1032]
3678 ; CHECK-GI-NEXT: mov v18.b[6], w8
3679 ; CHECK-GI-NEXT: ldr w8, [sp, #1160]
3680 ; CHECK-GI-NEXT: mov v19.b[6], w10
3681 ; CHECK-GI-NEXT: ldr w10, [sp, #784]
3682 ; CHECK-GI-NEXT: mov v7.b[7], w11
3683 ; CHECK-GI-NEXT: mov v16.b[7], w9
3684 ; CHECK-GI-NEXT: ldr w9, [sp, #1288]
3685 ; CHECK-GI-NEXT: ldr w11, [sp, #912]
3686 ; CHECK-GI-NEXT: mov v17.b[7], w8
3687 ; CHECK-GI-NEXT: ldr w8, [sp, #1416]
3688 ; CHECK-GI-NEXT: mov v6.b[8], w10
3689 ; CHECK-GI-NEXT: ldr w10, [sp, #1040]
3690 ; CHECK-GI-NEXT: mov v18.b[7], w9
3691 ; CHECK-GI-NEXT: ldr w9, [sp, #1168]
3692 ; CHECK-GI-NEXT: mov v19.b[7], w8
3693 ; CHECK-GI-NEXT: ldr w8, [sp, #792]
3694 ; CHECK-GI-NEXT: mov v7.b[8], w11
3695 ; CHECK-GI-NEXT: mov v16.b[8], w10
3696 ; CHECK-GI-NEXT: ldr w10, [sp, #1296]
3697 ; CHECK-GI-NEXT: ldr w11, [sp, #920]
3698 ; CHECK-GI-NEXT: mov v17.b[8], w9
3699 ; CHECK-GI-NEXT: ldr w9, [sp, #1424]
3700 ; CHECK-GI-NEXT: mov v6.b[9], w8
3701 ; CHECK-GI-NEXT: ldr w8, [sp, #1048]
3702 ; CHECK-GI-NEXT: mov v18.b[8], w10
3703 ; CHECK-GI-NEXT: ldr w10, [sp, #1176]
3704 ; CHECK-GI-NEXT: mov v19.b[8], w9
3705 ; CHECK-GI-NEXT: ldr w9, [sp, #800]
3706 ; CHECK-GI-NEXT: mov v7.b[9], w11
3707 ; CHECK-GI-NEXT: mov v16.b[9], w8
3708 ; CHECK-GI-NEXT: ldr w8, [sp, #1304]
3709 ; CHECK-GI-NEXT: ldr w11, [sp, #928]
3710 ; CHECK-GI-NEXT: mov v17.b[9], w10
3711 ; CHECK-GI-NEXT: ldr w10, [sp, #1432]
3712 ; CHECK-GI-NEXT: mov v6.b[10], w9
3713 ; CHECK-GI-NEXT: ldr w9, [sp, #1056]
3714 ; CHECK-GI-NEXT: mov v18.b[9], w8
3715 ; CHECK-GI-NEXT: ldr w8, [sp, #1184]
3716 ; CHECK-GI-NEXT: mov v19.b[9], w10
3717 ; CHECK-GI-NEXT: ldr w10, [sp, #808]
3718 ; CHECK-GI-NEXT: mov v7.b[10], w11
3719 ; CHECK-GI-NEXT: mov v16.b[10], w9
3720 ; CHECK-GI-NEXT: ldr w9, [sp, #1312]
3721 ; CHECK-GI-NEXT: ldr w11, [sp, #936]
3722 ; CHECK-GI-NEXT: mov v17.b[10], w8
3723 ; CHECK-GI-NEXT: ldr w8, [sp, #1440]
3724 ; CHECK-GI-NEXT: mov v6.b[11], w10
3725 ; CHECK-GI-NEXT: ldr w10, [sp, #1064]
3726 ; CHECK-GI-NEXT: mov v18.b[10], w9
3727 ; CHECK-GI-NEXT: ldr w9, [sp, #1192]
3728 ; CHECK-GI-NEXT: mov v19.b[10], w8
3729 ; CHECK-GI-NEXT: ldr w8, [sp, #816]
3730 ; CHECK-GI-NEXT: mov v7.b[11], w11
3731 ; CHECK-GI-NEXT: mov v16.b[11], w10
3732 ; CHECK-GI-NEXT: ldr w10, [sp, #1320]
3733 ; CHECK-GI-NEXT: ldr w11, [sp, #944]
3734 ; CHECK-GI-NEXT: mov v17.b[11], w9
3735 ; CHECK-GI-NEXT: ldr w9, [sp, #1448]
3736 ; CHECK-GI-NEXT: mov v6.b[12], w8
3737 ; CHECK-GI-NEXT: ldr w8, [sp, #1072]
3738 ; CHECK-GI-NEXT: mov v18.b[11], w10
3739 ; CHECK-GI-NEXT: ldr w10, [sp, #1200]
3740 ; CHECK-GI-NEXT: mov v19.b[11], w9
3741 ; CHECK-GI-NEXT: ldr w9, [sp, #824]
3742 ; CHECK-GI-NEXT: mov v7.b[12], w11
3743 ; CHECK-GI-NEXT: mov v16.b[12], w8
3744 ; CHECK-GI-NEXT: ldr w8, [sp, #1328]
3745 ; CHECK-GI-NEXT: ldr w11, [sp, #952]
3746 ; CHECK-GI-NEXT: mov v17.b[12], w10
3747 ; CHECK-GI-NEXT: ldr w10, [sp, #1456]
3748 ; CHECK-GI-NEXT: mov v6.b[13], w9
3749 ; CHECK-GI-NEXT: ldr w9, [sp, #1080]
3750 ; CHECK-GI-NEXT: mov v18.b[12], w8
3751 ; CHECK-GI-NEXT: ldr w8, [sp, #1208]
3752 ; CHECK-GI-NEXT: mov v19.b[12], w10
3753 ; CHECK-GI-NEXT: ldr w10, [sp, #832]
3754 ; CHECK-GI-NEXT: mov v7.b[13], w11
3755 ; CHECK-GI-NEXT: mov v16.b[13], w9
3756 ; CHECK-GI-NEXT: ldr w9, [sp, #1336]
3757 ; CHECK-GI-NEXT: ldr w11, [sp, #960]
3758 ; CHECK-GI-NEXT: mov v17.b[13], w8
3759 ; CHECK-GI-NEXT: ldr w8, [sp, #1464]
3760 ; CHECK-GI-NEXT: mov v6.b[14], w10
3761 ; CHECK-GI-NEXT: ldr w10, [sp, #1088]
3762 ; CHECK-GI-NEXT: mov v18.b[13], w9
3763 ; CHECK-GI-NEXT: ldr w9, [sp, #1216]
3764 ; CHECK-GI-NEXT: mov v19.b[13], w8
3765 ; CHECK-GI-NEXT: ldr w8, [sp, #840]
3766 ; CHECK-GI-NEXT: mov v7.b[14], w11
3767 ; CHECK-GI-NEXT: mov v16.b[14], w10
3768 ; CHECK-GI-NEXT: ldr w10, [sp, #1344]
3769 ; CHECK-GI-NEXT: ldr w11, [sp, #968]
3770 ; CHECK-GI-NEXT: mov v17.b[14], w9
3771 ; CHECK-GI-NEXT: mov v6.b[15], w8
3772 ; CHECK-GI-NEXT: ldr w8, [sp, #1096]
3773 ; CHECK-GI-NEXT: mov v18.b[14], w10
3774 ; CHECK-GI-NEXT: ldr w9, [sp, #1472]
3775 ; CHECK-GI-NEXT: ldr w10, [sp, #1224]
3776 ; CHECK-GI-NEXT: mov v7.b[15], w11
3777 ; CHECK-GI-NEXT: addv s4, v25.4s
3778 ; CHECK-GI-NEXT: mov v16.b[15], w8
3779 ; CHECK-GI-NEXT: ldr w8, [sp, #1352]
3780 ; CHECK-GI-NEXT: mov v19.b[14], w9
3781 ; CHECK-GI-NEXT: mov v17.b[15], w10
3782 ; CHECK-GI-NEXT: ldr w9, [sp, #1480]
3783 ; CHECK-GI-NEXT: mov v18.b[15], w8
3784 ; CHECK-GI-NEXT: fmov w8, s0
3785 ; CHECK-GI-NEXT: fmov w11, s4
3786 ; CHECK-GI-NEXT: mov v19.b[15], w9
3787 ; CHECK-GI-NEXT: fmov w9, s1
3788 ; CHECK-GI-NEXT: sdot v22.4s, v6.16b, v17.16b
3789 ; CHECK-GI-NEXT: sdot v23.4s, v7.16b, v18.16b
3790 ; CHECK-GI-NEXT: add w8, w8, w9
3791 ; CHECK-GI-NEXT: sdot v24.4s, v16.16b, v19.16b
3792 ; CHECK-GI-NEXT: add w8, w8, w11
3793 ; CHECK-GI-NEXT: addv s2, v22.4s
3794 ; CHECK-GI-NEXT: addv s3, v23.4s
3795 ; CHECK-GI-NEXT: addv s5, v24.4s
3796 ; CHECK-GI-NEXT: fmov w9, s2
3797 ; CHECK-GI-NEXT: fmov w10, s3
3798 ; CHECK-GI-NEXT: add w9, w9, w10
3799 ; CHECK-GI-NEXT: fmov w10, s5
3800 ; CHECK-GI-NEXT: add w9, w9, w10
3801 ; CHECK-GI-NEXT: add w0, w8, w9
3802 ; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
3803 ; CHECK-GI-NEXT: ret
3804 entry:
3805 %az = sext <48 x i8> %a to <48 x i32>
3806 %bz = sext <48 x i8> %b to <48 x i32>
3807 %m1 = mul nuw nsw <48 x i32> %az, %bz
3808 %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1)
3809 %cz = sext <48 x i8> %c to <48 x i32>
3810 %dz = sext <48 x i8> %d to <48 x i32>
3811 %m2 = mul nuw nsw <48 x i32> %cz, %dz
3812 %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2)
3813 %x = add i32 %r1, %r2
3814 ret i32 %x
3815 }
3817 define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
3818 ; CHECK-SD-LABEL: test_sdot_v48i8_double_nomla:
3819 ; CHECK-SD: // %bb.0: // %entry
3820 ; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
3821 ; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
3822 ; CHECK-SD-NEXT: .cfi_offset w29, -16
3823 ; CHECK-SD-NEXT: ldr b5, [sp, #208]
3824 ; CHECK-SD-NEXT: add x8, sp, #216
3825 ; CHECK-SD-NEXT: fmov s0, w0
3826 ; CHECK-SD-NEXT: ldr b4, [sp, #976]
3827 ; CHECK-SD-NEXT: add x9, sp, #984
3828 ; CHECK-SD-NEXT: add x12, sp, #328
3829 ; CHECK-SD-NEXT: ld1 { v5.b }[1], [x8]
3830 ; CHECK-SD-NEXT: add x8, sp, #224
3831 ; CHECK-SD-NEXT: movi v1.16b, #1
3832 ; CHECK-SD-NEXT: mov v0.b[1], w1
3833 ; CHECK-SD-NEXT: ld1 { v4.b }[1], [x9]
3834 ; CHECK-SD-NEXT: movi v3.2d, #0000000000000000
3835 ; CHECK-SD-NEXT: add x11, sp, #992
3836 ; CHECK-SD-NEXT: ldr b6, [sp, #720]
3837 ; CHECK-SD-NEXT: ldr b7, [sp, #80]
3838 ; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8]
3839 ; CHECK-SD-NEXT: add x8, sp, #232
3840 ; CHECK-SD-NEXT: add x13, sp, #88
3841 ; CHECK-SD-NEXT: ld1 { v4.b }[2], [x11]
3842 ; CHECK-SD-NEXT: ld1 { v7.b }[1], [x13]
3843 ; CHECK-SD-NEXT: add x13, sp, #856
3844 ; CHECK-SD-NEXT: mov v0.b[2], w2
3845 ; CHECK-SD-NEXT: add x14, sp, #1008
3846 ; CHECK-SD-NEXT: add x15, sp, #872
3847 ; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8]
3848 ; CHECK-SD-NEXT: add x8, sp, #240
3849 ; CHECK-SD-NEXT: add x16, sp, #888
3850 ; CHECK-SD-NEXT: add x10, sp, #16
3851 ; CHECK-SD-NEXT: add x9, sp, #24
3852 ; CHECK-SD-NEXT: add x11, sp, #40
3853 ; CHECK-SD-NEXT: movi v2.2d, #0000000000000000
3854 ; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8]
3855 ; CHECK-SD-NEXT: add x8, sp, #248
3856 ; CHECK-SD-NEXT: mov v0.b[3], w3
3857 ; CHECK-SD-NEXT: ld1 { v5.b }[5], [x8]
3858 ; CHECK-SD-NEXT: add x8, sp, #256
3859 ; CHECK-SD-NEXT: mov v0.b[4], w4
3860 ; CHECK-SD-NEXT: ld1 { v5.b }[6], [x8]
3861 ; CHECK-SD-NEXT: add x8, sp, #264
3862 ; CHECK-SD-NEXT: mov v0.b[5], w5
3863 ; CHECK-SD-NEXT: ld1 { v5.b }[7], [x8]
3864 ; CHECK-SD-NEXT: add x8, sp, #272
3865 ; CHECK-SD-NEXT: ld1 { v5.b }[8], [x8]
3866 ; CHECK-SD-NEXT: add x8, sp, #280
3867 ; CHECK-SD-NEXT: mov v0.b[6], w6
3868 ; CHECK-SD-NEXT: ld1 { v5.b }[9], [x8]
3869 ; CHECK-SD-NEXT: add x8, sp, #288
3870 ; CHECK-SD-NEXT: mov v0.b[7], w7
3871 ; CHECK-SD-NEXT: ld1 { v5.b }[10], [x8]
3872 ; CHECK-SD-NEXT: add x8, sp, #296
3873 ; CHECK-SD-NEXT: ld1 { v0.b }[8], [x10]
3874 ; CHECK-SD-NEXT: add x10, sp, #128
3875 ; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8]
3876 ; CHECK-SD-NEXT: add x8, sp, #304
3877 ; CHECK-SD-NEXT: ld1 { v0.b }[9], [x9]
3878 ; CHECK-SD-NEXT: add x9, sp, #136
3879 ; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8]
3880 ; CHECK-SD-NEXT: add x8, sp, #312
3881 ; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8]
3882 ; CHECK-SD-NEXT: add x8, sp, #320
3883 ; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8]
3884 ; CHECK-SD-NEXT: add x8, sp, #32
3885 ; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8]
3886 ; CHECK-SD-NEXT: add x8, sp, #144
3887 ; CHECK-SD-NEXT: ld1 { v5.b }[15], [x12]
3888 ; CHECK-SD-NEXT: add x12, sp, #728
3889 ; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12]
3890 ; CHECK-SD-NEXT: add x12, sp, #1000
3891 ; CHECK-SD-NEXT: ld1 { v0.b }[11], [x11]
3892 ; CHECK-SD-NEXT: ld1 { v4.b }[3], [x12]
3893 ; CHECK-SD-NEXT: add x12, sp, #736
3894 ; CHECK-SD-NEXT: add x11, sp, #920
3895 ; CHECK-SD-NEXT: sdot v3.4s, v5.16b, v1.16b
3896 ; CHECK-SD-NEXT: ldr b5, [sp, #848]
3897 ; CHECK-SD-NEXT: ld1 { v6.b }[2], [x12]
3898 ; CHECK-SD-NEXT: add x12, sp, #48
3899 ; CHECK-SD-NEXT: ld1 { v5.b }[1], [x13]
3900 ; CHECK-SD-NEXT: add x13, sp, #744
3901 ; CHECK-SD-NEXT: ld1 { v4.b }[4], [x14]
3902 ; CHECK-SD-NEXT: add x14, sp, #96
3903 ; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12]
3904 ; CHECK-SD-NEXT: ld1 { v6.b }[3], [x13]
3905 ; CHECK-SD-NEXT: add x13, sp, #864
3906 ; CHECK-SD-NEXT: ld1 { v7.b }[2], [x14]
3907 ; CHECK-SD-NEXT: add x14, sp, #1016
3908 ; CHECK-SD-NEXT: ld1 { v5.b }[2], [x13]
3909 ; CHECK-SD-NEXT: add x13, sp, #752
3910 ; CHECK-SD-NEXT: ld1 { v4.b }[5], [x14]
3911 ; CHECK-SD-NEXT: add x14, sp, #104
3912 ; CHECK-SD-NEXT: ld1 { v6.b }[4], [x13]
3913 ; CHECK-SD-NEXT: add x13, sp, #1024
3914 ; CHECK-SD-NEXT: ld1 { v7.b }[3], [x14]
3915 ; CHECK-SD-NEXT: ld1 { v5.b }[3], [x15]
3916 ; CHECK-SD-NEXT: add x15, sp, #760
3917 ; CHECK-SD-NEXT: add x14, sp, #112
3918 ; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13]
3919 ; CHECK-SD-NEXT: add x13, sp, #880
3920 ; CHECK-SD-NEXT: ld1 { v6.b }[5], [x15]
3921 ; CHECK-SD-NEXT: add x15, sp, #1032
3922 ; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14]
3923 ; CHECK-SD-NEXT: ld1 { v5.b }[4], [x13]
3924 ; CHECK-SD-NEXT: add x14, sp, #768
3925 ; CHECK-SD-NEXT: add x13, sp, #120
3926 ; CHECK-SD-NEXT: ld1 { v4.b }[7], [x15]
3927 ; CHECK-SD-NEXT: add x15, sp, #1040
3928 ; CHECK-SD-NEXT: ld1 { v6.b }[6], [x14]
3929 ; CHECK-SD-NEXT: ld1 { v7.b }[5], [x13]
3930 ; CHECK-SD-NEXT: add x13, sp, #776
3931 ; CHECK-SD-NEXT: ld1 { v5.b }[5], [x16]
3932 ; CHECK-SD-NEXT: add x14, sp, #1048
3933 ; CHECK-SD-NEXT: ld1 { v4.b }[8], [x15]
3934 ; CHECK-SD-NEXT: add x15, sp, #896
3935 ; CHECK-SD-NEXT: ld1 { v6.b }[7], [x13]
3936 ; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10]
3937 ; CHECK-SD-NEXT: add x10, sp, #784
3938 ; CHECK-SD-NEXT: ld1 { v5.b }[6], [x15]
3939 ; CHECK-SD-NEXT: add x13, sp, #1056
3940 ; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14]
3941 ; CHECK-SD-NEXT: add x14, sp, #904
3942 ; CHECK-SD-NEXT: ld1 { v6.b }[8], [x10]
3943 ; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9]
3944 ; CHECK-SD-NEXT: add x9, sp, #792
3945 ; CHECK-SD-NEXT: ld1 { v5.b }[7], [x14]
3946 ; CHECK-SD-NEXT: add x10, sp, #1064
3947 ; CHECK-SD-NEXT: ld1 { v4.b }[10], [x13]
3948 ; CHECK-SD-NEXT: add x13, sp, #912
3949 ; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9]
3950 ; CHECK-SD-NEXT: ld1 { v7.b }[8], [x8]
3951 ; CHECK-SD-NEXT: add x9, sp, #800
3952 ; CHECK-SD-NEXT: ld1 { v5.b }[8], [x13]
3953 ; CHECK-SD-NEXT: add x8, sp, #152
3954 ; CHECK-SD-NEXT: ld1 { v4.b }[11], [x10]
3955 ; CHECK-SD-NEXT: add x10, sp, #1072
3956 ; CHECK-SD-NEXT: ld1 { v6.b }[10], [x9]
3957 ; CHECK-SD-NEXT: ld1 { v7.b }[9], [x8]
3958 ; CHECK-SD-NEXT: add x9, sp, #808
3959 ; CHECK-SD-NEXT: ld1 { v5.b }[9], [x11]
3960 ; CHECK-SD-NEXT: add x8, sp, #56
3961 ; CHECK-SD-NEXT: ld1 { v4.b }[12], [x10]
3962 ; CHECK-SD-NEXT: add x10, sp, #160
3963 ; CHECK-SD-NEXT: ld1 { v0.b }[13], [x8]
3964 ; CHECK-SD-NEXT: ld1 { v6.b }[11], [x9]
3965 ; CHECK-SD-NEXT: add x9, sp, #928
3966 ; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10]
3967 ; CHECK-SD-NEXT: add x10, sp, #1080
3968 ; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9]
3969 ; CHECK-SD-NEXT: add x8, sp, #816
3970 ; CHECK-SD-NEXT: ld1 { v4.b }[13], [x10]
3971 ; CHECK-SD-NEXT: add x9, sp, #168
3972 ; CHECK-SD-NEXT: add x10, sp, #176
3973 ; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8]
3974 ; CHECK-SD-NEXT: add x8, sp, #936
3975 ; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9]
3976 ; CHECK-SD-NEXT: add x9, sp, #1088
3977 ; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8]
3978 ; CHECK-SD-NEXT: add x8, sp, #64
3979 ; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9]
3980 ; CHECK-SD-NEXT: add x9, sp, #824
3981 ; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8]
3982 ; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9]
3983 ; CHECK-SD-NEXT: add x9, sp, #944
3984 ; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10]
3985 ; CHECK-SD-NEXT: add x10, sp, #1096
3986 ; CHECK-SD-NEXT: ld1 { v5.b }[12], [x9]
3987 ; CHECK-SD-NEXT: add x8, sp, #832
3988 ; CHECK-SD-NEXT: ld1 { v4.b }[15], [x10]
3989 ; CHECK-SD-NEXT: add x9, sp, #184
3990 ; CHECK-SD-NEXT: add x10, sp, #72
3991 ; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8]
3992 ; CHECK-SD-NEXT: add x8, sp, #952
3993 ; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9]
3994 ; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8]
3995 ; CHECK-SD-NEXT: add x8, sp, #840
3996 ; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10]
3997 ; CHECK-SD-NEXT: sdot v2.4s, v4.16b, v1.16b
3998 ; CHECK-SD-NEXT: add x9, sp, #192
3999 ; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8]
4000 ; CHECK-SD-NEXT: add x8, sp, #960
4001 ; CHECK-SD-NEXT: ld1 { v7.b }[14], [x9]
4002 ; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8]
4003 ; CHECK-SD-NEXT: sdot v3.4s, v0.16b, v1.16b
4004 ; CHECK-SD-NEXT: add x8, sp, #200
4005 ; CHECK-SD-NEXT: add x9, sp, #968
4006 ; CHECK-SD-NEXT: sdot v2.4s, v6.16b, v1.16b
4007 ; CHECK-SD-NEXT: ld1 { v7.b }[15], [x8]
4008 ; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9]
4009 ; CHECK-SD-NEXT: sdot v3.4s, v7.16b, v1.16b
4010 ; CHECK-SD-NEXT: sdot v2.4s, v5.16b, v1.16b
4011 ; CHECK-SD-NEXT: add v0.4s, v3.4s, v2.4s
4012 ; CHECK-SD-NEXT: addv s0, v0.4s
4013 ; CHECK-SD-NEXT: fmov w0, s0
4014 ; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
4015 ; CHECK-SD-NEXT: ret
4016 ;
4017 ; CHECK-GI-LABEL: test_sdot_v48i8_double_nomla:
4018 ; CHECK-GI: // %bb.0: // %entry
4019 ; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
4020 ; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
4021 ; CHECK-GI-NEXT: .cfi_offset w29, -16
4022 ; CHECK-GI-NEXT: ldr w10, [sp, #80]
4023 ; CHECK-GI-NEXT: ldr w11, [sp, #208]
4024 ; CHECK-GI-NEXT: fmov s0, w0
4025 ; CHECK-GI-NEXT: ldr w9, [sp, #88]
4026 ; CHECK-GI-NEXT: ldr w12, [sp, #728]
4027 ; CHECK-GI-NEXT: movi v6.16b, #1
4028 ; CHECK-GI-NEXT: fmov s1, w10
4029 ; CHECK-GI-NEXT: fmov s2, w11
4030 ; CHECK-GI-NEXT: ldr w11, [sp, #720]
4031 ; CHECK-GI-NEXT: ldr w10, [sp, #216]
4032 ; CHECK-GI-NEXT: mov v0.b[1], w1
4033 ; CHECK-GI-NEXT: ldr w13, [sp, #856]
4034 ; CHECK-GI-NEXT: fmov s3, w11
4035 ; CHECK-GI-NEXT: ldr w8, [sp, #96]
4036 ; CHECK-GI-NEXT: ldr w11, [sp, #224]
4037 ; CHECK-GI-NEXT: mov v1.b[1], w9
4038 ; CHECK-GI-NEXT: ldr w9, [sp, #848]
4039 ; CHECK-GI-NEXT: mov v2.b[1], w10
4040 ; CHECK-GI-NEXT: ldr w10, [sp, #976]
4041 ; CHECK-GI-NEXT: movi v7.2d, #0000000000000000
4042 ; CHECK-GI-NEXT: movi v16.2d, #0000000000000000
4043 ; CHECK-GI-NEXT: fmov s4, w9
4044 ; CHECK-GI-NEXT: mov v3.b[1], w12
4045 ; CHECK-GI-NEXT: ldr w9, [sp, #984]
4046 ; CHECK-GI-NEXT: fmov s5, w10
4047 ; CHECK-GI-NEXT: mov v0.b[2], w2
4048 ; CHECK-GI-NEXT: ldr w10, [sp, #736]
4049 ; CHECK-GI-NEXT: mov v1.b[2], w8
4050 ; CHECK-GI-NEXT: ldr w8, [sp, #864]
4051 ; CHECK-GI-NEXT: mov v2.b[2], w11
4052 ; CHECK-GI-NEXT: mov v4.b[1], w13
4053 ; CHECK-GI-NEXT: ldr w11, [sp, #992]
4054 ; CHECK-GI-NEXT: ldr w12, [sp, #776]
4055 ; CHECK-GI-NEXT: mov v5.b[1], w9
4056 ; CHECK-GI-NEXT: mov v3.b[2], w10
4057 ; CHECK-GI-NEXT: ldr w9, [sp, #104]
4058 ; CHECK-GI-NEXT: ldr w10, [sp, #232]
4059 ; CHECK-GI-NEXT: mov v0.b[3], w3
4060 ; CHECK-GI-NEXT: movi v17.2d, #0000000000000000
4061 ; CHECK-GI-NEXT: mov v1.b[3], w9
4062 ; CHECK-GI-NEXT: ldr w9, [sp, #872]
4063 ; CHECK-GI-NEXT: movi v18.2d, #0000000000000000
4064 ; CHECK-GI-NEXT: mov v4.b[2], w8
4065 ; CHECK-GI-NEXT: ldr w8, [sp, #744]
4066 ; CHECK-GI-NEXT: mov v2.b[3], w10
4067 ; CHECK-GI-NEXT: mov v5.b[2], w11
4068 ; CHECK-GI-NEXT: ldr w11, [sp, #1000]
4069 ; CHECK-GI-NEXT: ldr w10, [sp, #240]
4070 ; CHECK-GI-NEXT: mov v3.b[3], w8
4071 ; CHECK-GI-NEXT: ldr w8, [sp, #112]
4072 ; CHECK-GI-NEXT: mov v0.b[4], w4
4073 ; CHECK-GI-NEXT: movi v19.2d, #0000000000000000
4074 ; CHECK-GI-NEXT: movi v20.2d, #0000000000000000
4075 ; CHECK-GI-NEXT: mov v4.b[3], w9
4076 ; CHECK-GI-NEXT: ldr w9, [sp, #752]
4077 ; CHECK-GI-NEXT: mov v1.b[4], w8
4078 ; CHECK-GI-NEXT: ldr w8, [sp, #880]
4079 ; CHECK-GI-NEXT: mov v5.b[3], w11
4080 ; CHECK-GI-NEXT: mov v2.b[4], w10
4081 ; CHECK-GI-NEXT: mov v3.b[4], w9
4082 ; CHECK-GI-NEXT: ldr w9, [sp, #120]
4083 ; CHECK-GI-NEXT: ldr w11, [sp, #1008]
4084 ; CHECK-GI-NEXT: ldr w10, [sp, #248]
4085 ; CHECK-GI-NEXT: mov v0.b[5], w5
4086 ; CHECK-GI-NEXT: mov v4.b[4], w8
4087 ; CHECK-GI-NEXT: ldr w8, [sp, #760]
4088 ; CHECK-GI-NEXT: mov v1.b[5], w9
4089 ; CHECK-GI-NEXT: ldr w9, [sp, #888]
4090 ; CHECK-GI-NEXT: mov v5.b[4], w11
4091 ; CHECK-GI-NEXT: mov v2.b[5], w10
4092 ; CHECK-GI-NEXT: mov v3.b[5], w8
4093 ; CHECK-GI-NEXT: ldr w8, [sp, #128]
4094 ; CHECK-GI-NEXT: ldr w11, [sp, #1016]
4095 ; CHECK-GI-NEXT: ldr w10, [sp, #256]
4096 ; CHECK-GI-NEXT: mov v0.b[6], w6
4097 ; CHECK-GI-NEXT: mov v4.b[5], w9
4098 ; CHECK-GI-NEXT: ldr w9, [sp, #768]
4099 ; CHECK-GI-NEXT: mov v1.b[6], w8
4100 ; CHECK-GI-NEXT: ldr w8, [sp, #896]
4101 ; CHECK-GI-NEXT: mov v5.b[5], w11
4102 ; CHECK-GI-NEXT: mov v2.b[6], w10
4103 ; CHECK-GI-NEXT: mov v3.b[6], w9
4104 ; CHECK-GI-NEXT: ldr w9, [sp, #136]
4105 ; CHECK-GI-NEXT: ldr w11, [sp, #1024]
4106 ; CHECK-GI-NEXT: ldr w10, [sp, #264]
4107 ; CHECK-GI-NEXT: mov v0.b[7], w7
4108 ; CHECK-GI-NEXT: mov v4.b[6], w8
4109 ; CHECK-GI-NEXT: mov v1.b[7], w9
4110 ; CHECK-GI-NEXT: ldr w9, [sp, #904]
4111 ; CHECK-GI-NEXT: mov v5.b[6], w11
4112 ; CHECK-GI-NEXT: mov v2.b[7], w10
4113 ; CHECK-GI-NEXT: ldr w8, [sp, #16]
4114 ; CHECK-GI-NEXT: mov v3.b[7], w12
4115 ; CHECK-GI-NEXT: ldr w10, [sp, #144]
4116 ; CHECK-GI-NEXT: ldr w12, [sp, #1032]
4117 ; CHECK-GI-NEXT: mov v0.b[8], w8
4118 ; CHECK-GI-NEXT: ldr w8, [sp, #784]
4119 ; CHECK-GI-NEXT: ldr w11, [sp, #272]
4120 ; CHECK-GI-NEXT: mov v4.b[7], w9
4121 ; CHECK-GI-NEXT: mov v1.b[8], w10
4122 ; CHECK-GI-NEXT: ldr w10, [sp, #912]
4123 ; CHECK-GI-NEXT: mov v5.b[7], w12
4124 ; CHECK-GI-NEXT: ldr w9, [sp, #24]
4125 ; CHECK-GI-NEXT: ldr w12, [sp, #1040]
4126 ; CHECK-GI-NEXT: mov v3.b[8], w8
4127 ; CHECK-GI-NEXT: ldr w8, [sp, #152]
4128 ; CHECK-GI-NEXT: mov v2.b[8], w11
4129 ; CHECK-GI-NEXT: mov v0.b[9], w9
4130 ; CHECK-GI-NEXT: ldr w9, [sp, #792]
4131 ; CHECK-GI-NEXT: ldr w11, [sp, #280]
4132 ; CHECK-GI-NEXT: mov v4.b[8], w10
4133 ; CHECK-GI-NEXT: mov v1.b[9], w8
4134 ; CHECK-GI-NEXT: ldr w10, [sp, #920]
4135 ; CHECK-GI-NEXT: mov v5.b[8], w12
4136 ; CHECK-GI-NEXT: ldr w8, [sp, #32]
4137 ; CHECK-GI-NEXT: ldr w12, [sp, #1048]
4138 ; CHECK-GI-NEXT: mov v3.b[9], w9
4139 ; CHECK-GI-NEXT: ldr w9, [sp, #160]
4140 ; CHECK-GI-NEXT: mov v2.b[9], w11
4141 ; CHECK-GI-NEXT: mov v0.b[10], w8
4142 ; CHECK-GI-NEXT: ldr w8, [sp, #800]
4143 ; CHECK-GI-NEXT: ldr w11, [sp, #288]
4144 ; CHECK-GI-NEXT: mov v4.b[9], w10
4145 ; CHECK-GI-NEXT: mov v1.b[10], w9
4146 ; CHECK-GI-NEXT: ldr w10, [sp, #928]
4147 ; CHECK-GI-NEXT: mov v5.b[9], w12
4148 ; CHECK-GI-NEXT: ldr w9, [sp, #40]
4149 ; CHECK-GI-NEXT: ldr w12, [sp, #1056]
4150 ; CHECK-GI-NEXT: mov v3.b[10], w8
4151 ; CHECK-GI-NEXT: ldr w8, [sp, #168]
4152 ; CHECK-GI-NEXT: mov v2.b[10], w11
4153 ; CHECK-GI-NEXT: mov v0.b[11], w9
4154 ; CHECK-GI-NEXT: ldr w9, [sp, #808]
4155 ; CHECK-GI-NEXT: ldr w11, [sp, #296]
4156 ; CHECK-GI-NEXT: mov v4.b[10], w10
4157 ; CHECK-GI-NEXT: mov v1.b[11], w8
4158 ; CHECK-GI-NEXT: ldr w10, [sp, #936]
4159 ; CHECK-GI-NEXT: mov v5.b[10], w12
4160 ; CHECK-GI-NEXT: ldr w8, [sp, #48]
4161 ; CHECK-GI-NEXT: ldr w12, [sp, #1064]
4162 ; CHECK-GI-NEXT: mov v3.b[11], w9
4163 ; CHECK-GI-NEXT: ldr w9, [sp, #176]
4164 ; CHECK-GI-NEXT: mov v2.b[11], w11
4165 ; CHECK-GI-NEXT: mov v0.b[12], w8
4166 ; CHECK-GI-NEXT: ldr w8, [sp, #816]
4167 ; CHECK-GI-NEXT: ldr w11, [sp, #304]
4168 ; CHECK-GI-NEXT: mov v4.b[11], w10
4169 ; CHECK-GI-NEXT: mov v1.b[12], w9
4170 ; CHECK-GI-NEXT: ldr w10, [sp, #944]
4171 ; CHECK-GI-NEXT: mov v5.b[11], w12
4172 ; CHECK-GI-NEXT: ldr w9, [sp, #56]
4173 ; CHECK-GI-NEXT: ldr w12, [sp, #1072]
4174 ; CHECK-GI-NEXT: mov v3.b[12], w8
4175 ; CHECK-GI-NEXT: ldr w8, [sp, #184]
4176 ; CHECK-GI-NEXT: mov v2.b[12], w11
4177 ; CHECK-GI-NEXT: mov v0.b[13], w9
4178 ; CHECK-GI-NEXT: ldr w9, [sp, #824]
4179 ; CHECK-GI-NEXT: ldr w11, [sp, #312]
4180 ; CHECK-GI-NEXT: mov v4.b[12], w10
4181 ; CHECK-GI-NEXT: mov v1.b[13], w8
4182 ; CHECK-GI-NEXT: ldr w10, [sp, #952]
4183 ; CHECK-GI-NEXT: mov v5.b[12], w12
4184 ; CHECK-GI-NEXT: ldr w8, [sp, #64]
4185 ; CHECK-GI-NEXT: ldr w12, [sp, #1080]
4186 ; CHECK-GI-NEXT: mov v3.b[13], w9
4187 ; CHECK-GI-NEXT: ldr w9, [sp, #192]
4188 ; CHECK-GI-NEXT: mov v2.b[13], w11
4189 ; CHECK-GI-NEXT: mov v0.b[14], w8
4190 ; CHECK-GI-NEXT: ldr w8, [sp, #832]
4191 ; CHECK-GI-NEXT: ldr w11, [sp, #320]
4192 ; CHECK-GI-NEXT: mov v4.b[13], w10
4193 ; CHECK-GI-NEXT: mov v1.b[14], w9
4194 ; CHECK-GI-NEXT: ldr w10, [sp, #960]
4195 ; CHECK-GI-NEXT: mov v5.b[13], w12
4196 ; CHECK-GI-NEXT: ldr w9, [sp, #72]
4197 ; CHECK-GI-NEXT: ldr w12, [sp, #1088]
4198 ; CHECK-GI-NEXT: mov v3.b[14], w8
4199 ; CHECK-GI-NEXT: ldr w8, [sp, #200]
4200 ; CHECK-GI-NEXT: mov v2.b[14], w11
4201 ; CHECK-GI-NEXT: mov v0.b[15], w9
4202 ; CHECK-GI-NEXT: ldr w9, [sp, #840]
4203 ; CHECK-GI-NEXT: ldr w11, [sp, #328]
4204 ; CHECK-GI-NEXT: mov v4.b[14], w10
4205 ; CHECK-GI-NEXT: mov v1.b[15], w8
4206 ; CHECK-GI-NEXT: ldr w8, [sp, #968]
4207 ; CHECK-GI-NEXT: mov v5.b[14], w12
4208 ; CHECK-GI-NEXT: ldr w10, [sp, #1096]
4209 ; CHECK-GI-NEXT: mov v3.b[15], w9
4210 ; CHECK-GI-NEXT: mov v2.b[15], w11
4211 ; CHECK-GI-NEXT: sdot v7.4s, v0.16b, v6.16b
4212 ; CHECK-GI-NEXT: mov v4.b[15], w8
4213 ; CHECK-GI-NEXT: sdot v16.4s, v1.16b, v6.16b
4214 ; CHECK-GI-NEXT: mov v5.b[15], w10
4215 ; CHECK-GI-NEXT: sdot v17.4s, v3.16b, v6.16b
4216 ; CHECK-GI-NEXT: sdot v20.4s, v2.16b, v6.16b
4217 ; CHECK-GI-NEXT: addv s0, v7.4s
4218 ; CHECK-GI-NEXT: sdot v18.4s, v4.16b, v6.16b
4219 ; CHECK-GI-NEXT: addv s1, v16.4s
4220 ; CHECK-GI-NEXT: sdot v19.4s, v5.16b, v6.16b
4221 ; CHECK-GI-NEXT: addv s2, v17.4s
4222 ; CHECK-GI-NEXT: addv s4, v20.4s
4223 ; CHECK-GI-NEXT: fmov w8, s0
4224 ; CHECK-GI-NEXT: fmov w9, s1
4225 ; CHECK-GI-NEXT: addv s3, v18.4s
4226 ; CHECK-GI-NEXT: addv s5, v19.4s
4227 ; CHECK-GI-NEXT: fmov w10, s2
4228 ; CHECK-GI-NEXT: add w8, w8, w9
4229 ; CHECK-GI-NEXT: fmov w9, s4
4230 ; CHECK-GI-NEXT: fmov w11, s3
4231 ; CHECK-GI-NEXT: add w8, w8, w9
4232 ; CHECK-GI-NEXT: add w10, w10, w11
4233 ; CHECK-GI-NEXT: fmov w11, s5
4234 ; CHECK-GI-NEXT: add w9, w10, w11
4235 ; CHECK-GI-NEXT: add w0, w8, w9
4236 ; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
4237 ; CHECK-GI-NEXT: ret
4238 entry:
4239 %az = sext <48 x i8> %a to <48 x i32>
4240 %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %az)
4241 %cz = sext <48 x i8> %c to <48 x i32>
4242 %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %cz)
4243 %x = add i32 %r1, %r2
4244 ret i32 %x
4245 }
4247 define i32 @test_udot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
4248 ; CHECK-SD-LABEL: test_udot_v64i8:
4249 ; CHECK-SD: // %bb.0: // %entry
4250 ; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
4251 ; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
4252 ; CHECK-SD-NEXT: ldp q2, q3, [x0, #32]
4253 ; CHECK-SD-NEXT: ldp q4, q5, [x1, #32]
4254 ; CHECK-SD-NEXT: udot v1.4s, v5.16b, v3.16b
4255 ; CHECK-SD-NEXT: udot v0.4s, v4.16b, v2.16b
4256 ; CHECK-SD-NEXT: ldp q2, q3, [x0]
4257 ; CHECK-SD-NEXT: ldp q4, q5, [x1]
4258 ; CHECK-SD-NEXT: udot v1.4s, v5.16b, v3.16b
4259 ; CHECK-SD-NEXT: udot v0.4s, v4.16b, v2.16b
4260 ; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
4261 ; CHECK-SD-NEXT: addv s0, v0.4s
4262 ; CHECK-SD-NEXT: fmov w8, s0
4263 ; CHECK-SD-NEXT: add w0, w8, w2
4264 ; CHECK-SD-NEXT: ret
4265 ;
4266 ; CHECK-GI-LABEL: test_udot_v64i8:
4267 ; CHECK-GI: // %bb.0: // %entry
4268 ; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
4269 ; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
4270 ; CHECK-GI-NEXT: movi v4.2d, #0000000000000000
4271 ; CHECK-GI-NEXT: movi v5.2d, #0000000000000000
4272 ; CHECK-GI-NEXT: ldp q1, q2, [x0]
4273 ; CHECK-GI-NEXT: ldp q6, q7, [x0, #32]
4274 ; CHECK-GI-NEXT: ldp q16, q17, [x1]
4275 ; CHECK-GI-NEXT: ldp q18, q19, [x1, #32]
4276 ; CHECK-GI-NEXT: udot v0.4s, v16.16b, v1.16b
4277 ; CHECK-GI-NEXT: udot v4.4s, v17.16b, v2.16b
4278 ; CHECK-GI-NEXT: udot v5.4s, v18.16b, v6.16b
4279 ; CHECK-GI-NEXT: udot v3.4s, v19.16b, v7.16b
4280 ; CHECK-GI-NEXT: add v0.4s, v0.4s, v4.4s
4281 ; CHECK-GI-NEXT: add v1.4s, v5.4s, v3.4s
4282 ; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
4283 ; CHECK-GI-NEXT: addv s0, v0.4s
4284 ; CHECK-GI-NEXT: fmov w8, s0
4285 ; CHECK-GI-NEXT: add w0, w8, w2
4286 ; CHECK-GI-NEXT: ret
4287 entry:
4288 %0 = load <64 x i8>, ptr %a
4289 %1 = zext <64 x i8> %0 to <64 x i32>
4290 %2 = load <64 x i8>, ptr %b
4291 %3 = zext <64 x i8> %2 to <64 x i32>
4292 %4 = mul nuw nsw <64 x i32> %3, %1
4293 %5 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %4)
4294 %op.extra = add i32 %5, %sum
4295 ret i32 %op.extra
4296 }
4298 define i32 @test_udot_v64i8_nomla(ptr nocapture readonly %a1) {
4299 ; CHECK-SD-LABEL: test_udot_v64i8_nomla:
4300 ; CHECK-SD: // %bb.0: // %entry
4301 ; CHECK-SD-NEXT: movi v0.16b, #1
4302 ; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
4303 ; CHECK-SD-NEXT: movi v2.2d, #0000000000000000
4304 ; CHECK-SD-NEXT: ldp q3, q4, [x0, #32]
4305 ; CHECK-SD-NEXT: udot v2.4s, v4.16b, v0.16b
4306 ; CHECK-SD-NEXT: udot v1.4s, v3.16b, v0.16b
4307 ; CHECK-SD-NEXT: ldp q3, q4, [x0]
4308 ; CHECK-SD-NEXT: udot v2.4s, v4.16b, v0.16b
4309 ; CHECK-SD-NEXT: udot v1.4s, v3.16b, v0.16b
4310 ; CHECK-SD-NEXT: add v0.4s, v1.4s, v2.4s
4311 ; CHECK-SD-NEXT: addv s0, v0.4s
4312 ; CHECK-SD-NEXT: fmov w0, s0
4313 ; CHECK-SD-NEXT: ret
4314 ;
4315 ; CHECK-GI-LABEL: test_udot_v64i8_nomla:
4316 ; CHECK-GI: // %bb.0: // %entry
4317 ; CHECK-GI-NEXT: movi v0.16b, #1
4318 ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
4319 ; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
4320 ; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
4321 ; CHECK-GI-NEXT: ldp q5, q6, [x0]
4322 ; CHECK-GI-NEXT: movi v4.2d, #0000000000000000
4323 ; CHECK-GI-NEXT: ldp q7, q16, [x0, #32]
4324 ; CHECK-GI-NEXT: udot v1.4s, v5.16b, v0.16b
4325 ; CHECK-GI-NEXT: udot v3.4s, v6.16b, v0.16b
4326 ; CHECK-GI-NEXT: udot v2.4s, v16.16b, v0.16b
4327 ; CHECK-GI-NEXT: udot v4.4s, v7.16b, v0.16b
4328 ; CHECK-GI-NEXT: add v0.4s, v1.4s, v3.4s
4329 ; CHECK-GI-NEXT: add v1.4s, v4.4s, v2.4s
4330 ; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
4331 ; CHECK-GI-NEXT: addv s0, v0.4s
4332 ; CHECK-GI-NEXT: fmov w0, s0
4333 ; CHECK-GI-NEXT: ret
4334 entry:
4335 %0 = load <64 x i8>, ptr %a1
4336 %1 = zext <64 x i8> %0 to <64 x i32>
4337 %2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %1)
4338 ret i32 %2
4339 }
4340 define i32 @test_sdot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
4341 ; CHECK-SD-LABEL: test_sdot_v64i8:
4342 ; CHECK-SD: // %bb.0: // %entry
4343 ; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
4344 ; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
4345 ; CHECK-SD-NEXT: ldp q2, q3, [x0, #32]
4346 ; CHECK-SD-NEXT: ldp q4, q5, [x1, #32]
4347 ; CHECK-SD-NEXT: sdot v1.4s, v5.16b, v3.16b
4348 ; CHECK-SD-NEXT: sdot v0.4s, v4.16b, v2.16b
4349 ; CHECK-SD-NEXT: ldp q2, q3, [x0]
4350 ; CHECK-SD-NEXT: ldp q4, q5, [x1]
4351 ; CHECK-SD-NEXT: sdot v1.4s, v5.16b, v3.16b
4352 ; CHECK-SD-NEXT: sdot v0.4s, v4.16b, v2.16b
4353 ; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
4354 ; CHECK-SD-NEXT: addv s0, v0.4s
4355 ; CHECK-SD-NEXT: fmov w8, s0
4356 ; CHECK-SD-NEXT: add w0, w8, w2
4357 ; CHECK-SD-NEXT: ret
4358 ;
4359 ; CHECK-GI-LABEL: test_sdot_v64i8:
4360 ; CHECK-GI: // %bb.0: // %entry
4361 ; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
4362 ; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
4363 ; CHECK-GI-NEXT: movi v4.2d, #0000000000000000
4364 ; CHECK-GI-NEXT: movi v5.2d, #0000000000000000
4365 ; CHECK-GI-NEXT: ldp q1, q2, [x0]
4366 ; CHECK-GI-NEXT: ldp q6, q7, [x0, #32]
4367 ; CHECK-GI-NEXT: ldp q16, q17, [x1]
4368 ; CHECK-GI-NEXT: ldp q18, q19, [x1, #32]
4369 ; CHECK-GI-NEXT: sdot v0.4s, v16.16b, v1.16b
4370 ; CHECK-GI-NEXT: sdot v4.4s, v17.16b, v2.16b
4371 ; CHECK-GI-NEXT: sdot v5.4s, v18.16b, v6.16b
4372 ; CHECK-GI-NEXT: sdot v3.4s, v19.16b, v7.16b
4373 ; CHECK-GI-NEXT: add v0.4s, v0.4s, v4.4s
4374 ; CHECK-GI-NEXT: add v1.4s, v5.4s, v3.4s
4375 ; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
4376 ; CHECK-GI-NEXT: addv s0, v0.4s
4377 ; CHECK-GI-NEXT: fmov w8, s0
4378 ; CHECK-GI-NEXT: add w0, w8, w2
4379 ; CHECK-GI-NEXT: ret
4380 entry:
4381 %0 = load <64 x i8>, ptr %a
4382 %1 = sext <64 x i8> %0 to <64 x i32>
4383 %2 = load <64 x i8>, ptr %b
4384 %3 = sext <64 x i8> %2 to <64 x i32>
4385 %4 = mul nsw <64 x i32> %3, %1
4386 %5 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %4)
4387 %op.extra = add nsw i32 %5, %sum
4388 ret i32 %op.extra
4389 }
4391 define i32 @test_sdot_v64i8_double(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
4392 ; CHECK-SD-LABEL: test_sdot_v64i8_double:
4393 ; CHECK-SD: // %bb.0: // %entry
4394 ; CHECK-SD-NEXT: movi v16.2d, #0000000000000000
4395 ; CHECK-SD-NEXT: movi v17.2d, #0000000000000000
4396 ; CHECK-SD-NEXT: movi v18.2d, #0000000000000000
4397 ; CHECK-SD-NEXT: movi v19.2d, #0000000000000000
4398 ; CHECK-SD-NEXT: ldp q20, q21, [sp, #96]
4399 ; CHECK-SD-NEXT: ldp q22, q23, [sp, #32]
4400 ; CHECK-SD-NEXT: sdot v16.4s, v3.16b, v7.16b
4401 ; CHECK-SD-NEXT: sdot v18.4s, v2.16b, v6.16b
4402 ; CHECK-SD-NEXT: sdot v19.4s, v23.16b, v21.16b
4403 ; CHECK-SD-NEXT: sdot v17.4s, v22.16b, v20.16b
4404 ; CHECK-SD-NEXT: ldp q2, q3, [sp, #64]
4405 ; CHECK-SD-NEXT: ldp q6, q7, [sp]
4406 ; CHECK-SD-NEXT: sdot v16.4s, v1.16b, v5.16b
4407 ; CHECK-SD-NEXT: sdot v18.4s, v0.16b, v4.16b
4408 ; CHECK-SD-NEXT: sdot v19.4s, v7.16b, v3.16b
4409 ; CHECK-SD-NEXT: sdot v17.4s, v6.16b, v2.16b
4410 ; CHECK-SD-NEXT: add v0.4s, v18.4s, v16.4s
4411 ; CHECK-SD-NEXT: add v1.4s, v17.4s, v19.4s
4412 ; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
4413 ; CHECK-SD-NEXT: addv s0, v0.4s
4414 ; CHECK-SD-NEXT: fmov w0, s0
4415 ; CHECK-SD-NEXT: ret
4416 ;
4417 ; CHECK-GI-LABEL: test_sdot_v64i8_double:
4418 ; CHECK-GI: // %bb.0: // %entry
4419 ; CHECK-GI-NEXT: movi v18.2d, #0000000000000000
4420 ; CHECK-GI-NEXT: movi v21.2d, #0000000000000000
4421 ; CHECK-GI-NEXT: movi v22.2d, #0000000000000000
4422 ; CHECK-GI-NEXT: movi v23.2d, #0000000000000000
4423 ; CHECK-GI-NEXT: ldp q16, q17, [sp]
4424 ; CHECK-GI-NEXT: movi v24.2d, #0000000000000000
4425 ; CHECK-GI-NEXT: movi v25.2d, #0000000000000000
4426 ; CHECK-GI-NEXT: movi v26.2d, #0000000000000000
4427 ; CHECK-GI-NEXT: movi v27.2d, #0000000000000000
4428 ; CHECK-GI-NEXT: ldp q19, q20, [sp, #32]
4429 ; CHECK-GI-NEXT: sdot v18.4s, v0.16b, v4.16b
4430 ; CHECK-GI-NEXT: ldp q0, q4, [sp, #64]
4431 ; CHECK-GI-NEXT: sdot v21.4s, v1.16b, v5.16b
4432 ; CHECK-GI-NEXT: ldp q1, q5, [sp, #96]
4433 ; CHECK-GI-NEXT: sdot v22.4s, v2.16b, v6.16b
4434 ; CHECK-GI-NEXT: sdot v23.4s, v3.16b, v7.16b
4435 ; CHECK-GI-NEXT: sdot v24.4s, v16.16b, v0.16b
4436 ; CHECK-GI-NEXT: sdot v26.4s, v17.16b, v4.16b
4437 ; CHECK-GI-NEXT: sdot v27.4s, v19.16b, v1.16b
4438 ; CHECK-GI-NEXT: sdot v25.4s, v20.16b, v5.16b
4439 ; CHECK-GI-NEXT: add v0.4s, v18.4s, v21.4s
4440 ; CHECK-GI-NEXT: add v1.4s, v22.4s, v23.4s
4441 ; CHECK-GI-NEXT: add v2.4s, v24.4s, v26.4s
4442 ; CHECK-GI-NEXT: add v3.4s, v27.4s, v25.4s
4443 ; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
4444 ; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
4445 ; CHECK-GI-NEXT: addv s0, v0.4s
4446 ; CHECK-GI-NEXT: addv s1, v1.4s
4447 ; CHECK-GI-NEXT: fmov w8, s0
4448 ; CHECK-GI-NEXT: fmov w9, s1
4449 ; CHECK-GI-NEXT: add w0, w8, w9
4450 ; CHECK-GI-NEXT: ret
4451 entry:
4452 %az = sext <64 x i8> %a to <64 x i32>
4453 %bz = sext <64 x i8> %b to <64 x i32>
4454 %m1 = mul nuw nsw <64 x i32> %az, %bz
4455 %r1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m1)
4456 %cz = sext <64 x i8> %c to <64 x i32>
4457 %dz = sext <64 x i8> %d to <64 x i32>
4458 %m2 = mul nuw nsw <64 x i32> %cz, %dz
4459 %r2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m2)
4460 %x = add i32 %r1, %r2
4461 ret i32 %x
4462 }
4464 define i32 @test_sdot_v64i8_double_nomla(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
4465 ; CHECK-SD-LABEL: test_sdot_v64i8_double_nomla:
4466 ; CHECK-SD: // %bb.0: // %entry
4467 ; CHECK-SD-NEXT: movi v4.16b, #1
4468 ; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
4469 ; CHECK-SD-NEXT: movi v6.2d, #0000000000000000
4470 ; CHECK-SD-NEXT: movi v7.2d, #0000000000000000
4471 ; CHECK-SD-NEXT: ldp q17, q18, [sp, #32]
4472 ; CHECK-SD-NEXT: movi v16.2d, #0000000000000000
4473 ; CHECK-SD-NEXT: sdot v5.4s, v3.16b, v4.16b
4474 ; CHECK-SD-NEXT: sdot v6.4s, v17.16b, v4.16b
4475 ; CHECK-SD-NEXT: sdot v7.4s, v2.16b, v4.16b
4476 ; CHECK-SD-NEXT: ldp q2, q3, [sp]
4477 ; CHECK-SD-NEXT: sdot v16.4s, v18.16b, v4.16b
4478 ; CHECK-SD-NEXT: sdot v5.4s, v1.16b, v4.16b
4479 ; CHECK-SD-NEXT: sdot v6.4s, v2.16b, v4.16b
4480 ; CHECK-SD-NEXT: sdot v7.4s, v0.16b, v4.16b
4481 ; CHECK-SD-NEXT: sdot v16.4s, v3.16b, v4.16b
4482 ; CHECK-SD-NEXT: add v0.4s, v7.4s, v5.4s
4483 ; CHECK-SD-NEXT: add v1.4s, v6.4s, v16.4s
4484 ; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
4485 ; CHECK-SD-NEXT: addv s0, v0.4s
4486 ; CHECK-SD-NEXT: fmov w0, s0
4487 ; CHECK-SD-NEXT: ret
4488 ;
4489 ; CHECK-GI-LABEL: test_sdot_v64i8_double_nomla:
4490 ; CHECK-GI: // %bb.0: // %entry
4491 ; CHECK-GI-NEXT: movi v4.16b, #1
4492 ; CHECK-GI-NEXT: movi v5.2d, #0000000000000000
4493 ; CHECK-GI-NEXT: movi v6.2d, #0000000000000000
4494 ; CHECK-GI-NEXT: movi v7.2d, #0000000000000000
4495 ; CHECK-GI-NEXT: ldp q21, q22, [sp]
4496 ; CHECK-GI-NEXT: movi v16.2d, #0000000000000000
4497 ; CHECK-GI-NEXT: movi v17.2d, #0000000000000000
4498 ; CHECK-GI-NEXT: movi v18.2d, #0000000000000000
4499 ; CHECK-GI-NEXT: movi v19.2d, #0000000000000000
4500 ; CHECK-GI-NEXT: movi v20.2d, #0000000000000000
4501 ; CHECK-GI-NEXT: sdot v5.4s, v0.16b, v4.16b
4502 ; CHECK-GI-NEXT: sdot v6.4s, v1.16b, v4.16b
4503 ; CHECK-GI-NEXT: ldp q0, q1, [sp, #32]
4504 ; CHECK-GI-NEXT: sdot v7.4s, v2.16b, v4.16b
4505 ; CHECK-GI-NEXT: sdot v16.4s, v3.16b, v4.16b
4506 ; CHECK-GI-NEXT: sdot v17.4s, v21.16b, v4.16b
4507 ; CHECK-GI-NEXT: sdot v19.4s, v22.16b, v4.16b
4508 ; CHECK-GI-NEXT: sdot v20.4s, v0.16b, v4.16b
4509 ; CHECK-GI-NEXT: sdot v18.4s, v1.16b, v4.16b
4510 ; CHECK-GI-NEXT: add v0.4s, v5.4s, v6.4s
4511 ; CHECK-GI-NEXT: add v1.4s, v7.4s, v16.4s
4512 ; CHECK-GI-NEXT: add v2.4s, v17.4s, v19.4s
4513 ; CHECK-GI-NEXT: add v3.4s, v20.4s, v18.4s
4514 ; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
4515 ; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
4516 ; CHECK-GI-NEXT: addv s0, v0.4s
4517 ; CHECK-GI-NEXT: addv s1, v1.4s
4518 ; CHECK-GI-NEXT: fmov w8, s0
4519 ; CHECK-GI-NEXT: fmov w9, s1
4520 ; CHECK-GI-NEXT: add w0, w8, w9
4521 ; CHECK-GI-NEXT: ret
4522 entry:
4523 %az = sext <64 x i8> %a to <64 x i32>
4524 %r1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %az)
4525 %cz = sext <64 x i8> %c to <64 x i32>
4526 %r2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %cz)
4527 %x = add i32 %r1, %r2
4528 ret i32 %x
4529 }