1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK
4 declare {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32>, <1 x i32>)
5 declare {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32>, <2 x i32>)
6 declare {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32>, <3 x i32>)
7 declare {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32>, <4 x i32>)
8 declare {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32>, <6 x i32>)
9 declare {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>)
11 declare {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8>, <16 x i8>)
12 declare {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16>, <8 x i16>)
13 declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
15 declare {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24>, <4 x i24>)
16 declare {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1>, <4 x i1>)
17 declare {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128>, <2 x i128>)
19 define <1 x i32> @umulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
20 ; CHECK-LABEL: umulo_v1i32:
22 ; CHECK-NEXT: umull v1.2d, v0.2s, v1.2s
23 ; CHECK-NEXT: shrn v0.2s, v1.2d, #32
24 ; CHECK-NEXT: cmtst v0.2s, v0.2s, v0.2s
25 ; CHECK-NEXT: xtn v1.2s, v1.2d
26 ; CHECK-NEXT: str s1, [x0]
28 %t = call {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
29 %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
30 %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
31 %res = sext <1 x i1> %obit to <1 x i32>
32 store <1 x i32> %val, <1 x i32>* %p2
36 define <2 x i32> @umulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
37 ; CHECK-LABEL: umulo_v2i32:
39 ; CHECK-NEXT: umull v1.2d, v0.2s, v1.2s
40 ; CHECK-NEXT: shrn v0.2s, v1.2d, #32
41 ; CHECK-NEXT: cmtst v0.2s, v0.2s, v0.2s
42 ; CHECK-NEXT: xtn v1.2s, v1.2d
43 ; CHECK-NEXT: str d1, [x0]
45 %t = call {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
46 %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
47 %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
48 %res = sext <2 x i1> %obit to <2 x i32>
49 store <2 x i32> %val, <2 x i32>* %p2
53 define <3 x i32> @umulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
54 ; CHECK-LABEL: umulo_v3i32:
56 ; CHECK-NEXT: umull2 v2.2d, v0.4s, v1.4s
57 ; CHECK-NEXT: umull v3.2d, v0.2s, v1.2s
58 ; CHECK-NEXT: mul v1.4s, v0.4s, v1.4s
59 ; CHECK-NEXT: uzp2 v0.4s, v3.4s, v2.4s
60 ; CHECK-NEXT: add x8, x0, #8
61 ; CHECK-NEXT: cmtst v0.4s, v0.4s, v0.4s
62 ; CHECK-NEXT: st1 { v1.s }[2], [x8]
63 ; CHECK-NEXT: str d1, [x0]
65 %t = call {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
66 %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
67 %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
68 %res = sext <3 x i1> %obit to <3 x i32>
69 store <3 x i32> %val, <3 x i32>* %p2
73 define <4 x i32> @umulo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
74 ; CHECK-LABEL: umulo_v4i32:
76 ; CHECK-NEXT: umull2 v2.2d, v0.4s, v1.4s
77 ; CHECK-NEXT: umull v3.2d, v0.2s, v1.2s
78 ; CHECK-NEXT: uzp2 v2.4s, v3.4s, v2.4s
79 ; CHECK-NEXT: cmtst v2.4s, v2.4s, v2.4s
80 ; CHECK-NEXT: mul v0.4s, v0.4s, v1.4s
81 ; CHECK-NEXT: str q0, [x0]
82 ; CHECK-NEXT: mov v0.16b, v2.16b
84 %t = call {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
85 %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
86 %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
87 %res = sext <4 x i1> %obit to <4 x i32>
88 store <4 x i32> %val, <4 x i32>* %p2
92 define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
93 ; CHECK-LABEL: umulo_v6i32:
95 ; CHECK-NEXT: fmov s2, w6
96 ; CHECK-NEXT: ldr s0, [sp, #16]
97 ; CHECK-NEXT: mov x9, sp
98 ; CHECK-NEXT: mov v2.s[1], w7
99 ; CHECK-NEXT: ld1 { v2.s }[2], [x9]
100 ; CHECK-NEXT: add x8, sp, #24
101 ; CHECK-NEXT: add x10, sp, #8
102 ; CHECK-NEXT: ld1 { v0.s }[1], [x8]
103 ; CHECK-NEXT: fmov s3, w0
104 ; CHECK-NEXT: ldr x11, [sp, #32]
105 ; CHECK-NEXT: ld1 { v2.s }[3], [x10]
106 ; CHECK-NEXT: fmov s1, w4
107 ; CHECK-NEXT: mov v3.s[1], w1
108 ; CHECK-NEXT: mov v1.s[1], w5
109 ; CHECK-NEXT: mov v3.s[2], w2
110 ; CHECK-NEXT: mov v3.s[3], w3
111 ; CHECK-NEXT: umull2 v4.2d, v1.4s, v0.4s
112 ; CHECK-NEXT: umull v5.2d, v1.2s, v0.2s
113 ; CHECK-NEXT: mul v0.4s, v1.4s, v0.4s
114 ; CHECK-NEXT: uzp2 v1.4s, v5.4s, v4.4s
115 ; CHECK-NEXT: str d0, [x11, #16]
116 ; CHECK-NEXT: umull2 v0.2d, v3.4s, v2.4s
117 ; CHECK-NEXT: umull v4.2d, v3.2s, v2.2s
118 ; CHECK-NEXT: uzp2 v0.4s, v4.4s, v0.4s
119 ; CHECK-NEXT: cmtst v1.4s, v1.4s, v1.4s
120 ; CHECK-NEXT: cmtst v0.4s, v0.4s, v0.4s
121 ; CHECK-NEXT: mul v2.4s, v3.4s, v2.4s
122 ; CHECK-NEXT: mov w5, v1.s[1]
123 ; CHECK-NEXT: mov w1, v0.s[1]
124 ; CHECK-NEXT: mov w2, v0.s[2]
125 ; CHECK-NEXT: mov w3, v0.s[3]
126 ; CHECK-NEXT: fmov w4, s1
127 ; CHECK-NEXT: fmov w0, s0
128 ; CHECK-NEXT: str q2, [x11]
130 %t = call {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
131 %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
132 %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
133 %res = sext <6 x i1> %obit to <6 x i32>
134 store <6 x i32> %val, <6 x i32>* %p2
138 define <8 x i32> @umulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
139 ; CHECK-LABEL: umulo_v8i32:
141 ; CHECK-NEXT: umull2 v4.2d, v0.4s, v2.4s
142 ; CHECK-NEXT: umull v5.2d, v0.2s, v2.2s
143 ; CHECK-NEXT: umull2 v6.2d, v1.4s, v3.4s
144 ; CHECK-NEXT: mul v2.4s, v0.4s, v2.4s
145 ; CHECK-NEXT: umull v0.2d, v1.2s, v3.2s
146 ; CHECK-NEXT: mul v3.4s, v1.4s, v3.4s
147 ; CHECK-NEXT: uzp2 v1.4s, v5.4s, v4.4s
148 ; CHECK-NEXT: uzp2 v4.4s, v0.4s, v6.4s
149 ; CHECK-NEXT: cmtst v0.4s, v1.4s, v1.4s
150 ; CHECK-NEXT: cmtst v1.4s, v4.4s, v4.4s
151 ; CHECK-NEXT: stp q2, q3, [x0]
153 %t = call {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
154 %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
155 %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
156 %res = sext <8 x i1> %obit to <8 x i32>
157 store <8 x i32> %val, <8 x i32>* %p2
161 define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
162 ; CHECK-LABEL: umulo_v16i8:
164 ; CHECK-NEXT: umull2 v2.8h, v0.16b, v1.16b
165 ; CHECK-NEXT: umull v3.8h, v0.8b, v1.8b
166 ; CHECK-NEXT: mul v4.16b, v0.16b, v1.16b
167 ; CHECK-NEXT: uzp2 v0.16b, v3.16b, v2.16b
168 ; CHECK-NEXT: cmtst v0.16b, v0.16b, v0.16b
169 ; CHECK-NEXT: zip1 v1.8b, v0.8b, v0.8b
170 ; CHECK-NEXT: zip2 v2.8b, v0.8b, v0.8b
171 ; CHECK-NEXT: ext v0.16b, v0.16b, v0.16b, #8
172 ; CHECK-NEXT: ushll v1.4s, v1.4h, #0
173 ; CHECK-NEXT: ushll v2.4s, v2.4h, #0
174 ; CHECK-NEXT: zip1 v3.8b, v0.8b, v0.8b
175 ; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
176 ; CHECK-NEXT: shl v1.4s, v1.4s, #31
177 ; CHECK-NEXT: shl v2.4s, v2.4s, #31
178 ; CHECK-NEXT: ushll v3.4s, v3.4h, #0
179 ; CHECK-NEXT: ushll v5.4s, v0.4h, #0
180 ; CHECK-NEXT: sshr v0.4s, v1.4s, #31
181 ; CHECK-NEXT: sshr v1.4s, v2.4s, #31
182 ; CHECK-NEXT: shl v2.4s, v3.4s, #31
183 ; CHECK-NEXT: shl v3.4s, v5.4s, #31
184 ; CHECK-NEXT: sshr v2.4s, v2.4s, #31
185 ; CHECK-NEXT: sshr v3.4s, v3.4s, #31
186 ; CHECK-NEXT: str q4, [x0]
188 %t = call {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
189 %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
190 %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
191 %res = sext <16 x i1> %obit to <16 x i32>
192 store <16 x i8> %val, <16 x i8>* %p2
196 define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
197 ; CHECK-LABEL: umulo_v8i16:
199 ; CHECK-NEXT: umull2 v2.4s, v0.8h, v1.8h
200 ; CHECK-NEXT: umull v3.4s, v0.4h, v1.4h
201 ; CHECK-NEXT: mul v4.8h, v0.8h, v1.8h
202 ; CHECK-NEXT: uzp2 v0.8h, v3.8h, v2.8h
203 ; CHECK-NEXT: cmtst v0.8h, v0.8h, v0.8h
204 ; CHECK-NEXT: xtn v0.8b, v0.8h
205 ; CHECK-NEXT: zip1 v1.8b, v0.8b, v0.8b
206 ; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
207 ; CHECK-NEXT: ushll v1.4s, v1.4h, #0
208 ; CHECK-NEXT: ushll v0.4s, v0.4h, #0
209 ; CHECK-NEXT: shl v1.4s, v1.4s, #31
210 ; CHECK-NEXT: shl v2.4s, v0.4s, #31
211 ; CHECK-NEXT: sshr v0.4s, v1.4s, #31
212 ; CHECK-NEXT: sshr v1.4s, v2.4s, #31
213 ; CHECK-NEXT: str q4, [x0]
215 %t = call {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
216 %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
217 %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
218 %res = sext <8 x i1> %obit to <8 x i32>
219 store <8 x i16> %val, <8 x i16>* %p2
223 define <2 x i32> @umulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
224 ; CHECK-LABEL: umulo_v2i64:
226 ; CHECK-NEXT: mov x8, v1.d[1]
227 ; CHECK-NEXT: fmov x9, d1
228 ; CHECK-NEXT: mov x10, v0.d[1]
229 ; CHECK-NEXT: fmov x11, d0
230 ; CHECK-NEXT: umulh x12, x11, x9
231 ; CHECK-NEXT: mul x9, x11, x9
232 ; CHECK-NEXT: umulh x11, x10, x8
233 ; CHECK-NEXT: cmp xzr, x11
234 ; CHECK-NEXT: csetm x11, ne
235 ; CHECK-NEXT: cmp xzr, x12
236 ; CHECK-NEXT: csetm x12, ne
237 ; CHECK-NEXT: fmov d0, x12
238 ; CHECK-NEXT: mul x8, x10, x8
239 ; CHECK-NEXT: fmov d1, x9
240 ; CHECK-NEXT: mov v0.d[1], x11
241 ; CHECK-NEXT: xtn v0.2s, v0.2d
242 ; CHECK-NEXT: mov v1.d[1], x8
243 ; CHECK-NEXT: str q1, [x0]
245 %t = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
246 %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
247 %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
248 %res = sext <2 x i1> %obit to <2 x i32>
249 store <2 x i64> %val, <2 x i64>* %p2
253 define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
254 ; CHECK-LABEL: umulo_v4i24:
256 ; CHECK-NEXT: bic v1.4s, #255, lsl #24
257 ; CHECK-NEXT: bic v0.4s, #255, lsl #24
258 ; CHECK-NEXT: umull2 v2.2d, v0.4s, v1.4s
259 ; CHECK-NEXT: umull v3.2d, v0.2s, v1.2s
260 ; CHECK-NEXT: mul v0.4s, v0.4s, v1.4s
261 ; CHECK-NEXT: uzp2 v1.4s, v3.4s, v2.4s
262 ; CHECK-NEXT: ushr v2.4s, v0.4s, #24
263 ; CHECK-NEXT: mov w8, v0.s[3]
264 ; CHECK-NEXT: mov w9, v0.s[2]
265 ; CHECK-NEXT: mov w10, v0.s[1]
266 ; CHECK-NEXT: fmov w11, s0
267 ; CHECK-NEXT: cmeq v0.4s, v1.4s, #0
268 ; CHECK-NEXT: cmtst v1.4s, v2.4s, v2.4s
269 ; CHECK-NEXT: sturh w8, [x0, #9]
270 ; CHECK-NEXT: lsr w8, w8, #16
271 ; CHECK-NEXT: strh w9, [x0, #6]
272 ; CHECK-NEXT: sturh w10, [x0, #3]
273 ; CHECK-NEXT: lsr w9, w9, #16
274 ; CHECK-NEXT: lsr w10, w10, #16
275 ; CHECK-NEXT: orn v0.16b, v1.16b, v0.16b
276 ; CHECK-NEXT: strb w8, [x0, #11]
277 ; CHECK-NEXT: lsr w8, w11, #16
278 ; CHECK-NEXT: strh w11, [x0]
279 ; CHECK-NEXT: strb w9, [x0, #8]
280 ; CHECK-NEXT: strb w10, [x0, #5]
281 ; CHECK-NEXT: strb w8, [x0, #2]
283 %t = call {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
284 %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
285 %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
286 %res = sext <4 x i1> %obit to <4 x i32>
287 store <4 x i24> %val, <4 x i24>* %p2
291 define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
292 ; CHECK-LABEL: umulo_v4i1:
294 ; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
295 ; CHECK-NEXT: umov w9, v0.h[1]
296 ; CHECK-NEXT: umov w8, v0.h[0]
297 ; CHECK-NEXT: and w9, w9, #0x1
298 ; CHECK-NEXT: bfi w8, w9, #1, #1
299 ; CHECK-NEXT: umov w9, v0.h[2]
300 ; CHECK-NEXT: and w9, w9, #0x1
301 ; CHECK-NEXT: bfi w8, w9, #2, #1
302 ; CHECK-NEXT: umov w9, v0.h[3]
303 ; CHECK-NEXT: bfi w8, w9, #3, #29
304 ; CHECK-NEXT: and w8, w8, #0xf
305 ; CHECK-NEXT: movi v0.2d, #0000000000000000
306 ; CHECK-NEXT: strb w8, [x0]
308 %t = call {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
309 %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
310 %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
311 %res = sext <4 x i1> %obit to <4 x i32>
312 store <4 x i1> %val, <4 x i1>* %p2
316 define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
317 ; CHECK-LABEL: umulo_v2i128:
319 ; CHECK-NEXT: cmp x7, #0
320 ; CHECK-NEXT: umulh x8, x3, x6
321 ; CHECK-NEXT: cset w13, ne
322 ; CHECK-NEXT: cmp x3, #0
323 ; CHECK-NEXT: umulh x9, x7, x2
324 ; CHECK-NEXT: mul x10, x7, x2
325 ; CHECK-NEXT: cset w14, ne
326 ; CHECK-NEXT: cmp xzr, x8
327 ; CHECK-NEXT: ldr x8, [sp]
328 ; CHECK-NEXT: umulh x11, x2, x6
329 ; CHECK-NEXT: madd x10, x3, x6, x10
330 ; CHECK-NEXT: and w13, w14, w13
331 ; CHECK-NEXT: cset w14, ne
332 ; CHECK-NEXT: cmp xzr, x9
333 ; CHECK-NEXT: orr w13, w13, w14
334 ; CHECK-NEXT: cset w14, ne
335 ; CHECK-NEXT: adds x10, x11, x10
336 ; CHECK-NEXT: mul x12, x2, x6
337 ; CHECK-NEXT: orr w13, w13, w14
338 ; CHECK-NEXT: cset w14, hs
339 ; CHECK-NEXT: cmp x5, #0
340 ; CHECK-NEXT: umulh x17, x1, x4
341 ; CHECK-NEXT: stp x12, x10, [x8, #16]
342 ; CHECK-NEXT: cset w10, ne
343 ; CHECK-NEXT: cmp x1, #0
344 ; CHECK-NEXT: umulh x9, x5, x0
345 ; CHECK-NEXT: mul x11, x5, x0
346 ; CHECK-NEXT: cset w12, ne
347 ; CHECK-NEXT: cmp xzr, x17
348 ; CHECK-NEXT: umulh x15, x0, x4
349 ; CHECK-NEXT: madd x11, x1, x4, x11
350 ; CHECK-NEXT: and w10, w12, w10
351 ; CHECK-NEXT: cset w12, ne
352 ; CHECK-NEXT: cmp xzr, x9
353 ; CHECK-NEXT: orr w9, w10, w12
354 ; CHECK-NEXT: cset w10, ne
355 ; CHECK-NEXT: adds x11, x15, x11
356 ; CHECK-NEXT: orr w9, w9, w10
357 ; CHECK-NEXT: cset w10, hs
358 ; CHECK-NEXT: orr w9, w9, w10
359 ; CHECK-NEXT: orr w10, w13, w14
360 ; CHECK-NEXT: fmov s0, w9
361 ; CHECK-NEXT: mov v0.s[1], w10
362 ; CHECK-NEXT: shl v0.2s, v0.2s, #31
363 ; CHECK-NEXT: mul x16, x0, x4
364 ; CHECK-NEXT: sshr v0.2s, v0.2s, #31
365 ; CHECK-NEXT: stp x16, x11, [x8]
367 %t = call {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
368 %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
369 %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
370 %res = sext <2 x i1> %obit to <2 x i32>
371 store <2 x i128> %val, <2 x i128>* %p2