| blob | b0edf0ddebd4e05a8a34c7f3b1c15e6a9245ac98 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -aarch64-sve-vector-bits-min=128 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_128
3 ; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
4 ; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
5 ; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
7 target triple = "aarch64-unknown-linux-gnu"
9 ;
10 ; SREM
11 ;
13 ; Vector vXi8 sdiv are not legal for NEON so use SVE when available.
14 ; FIXME: We should be able to improve the codegen for >= 256 bits here.
15 define <8 x i8> @srem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 {
16 ; VBITS_GE_128-LABEL: srem_v8i8:
17 ; VBITS_GE_128: // %bb.0:
18 ; VBITS_GE_128-NEXT: ptrue p0.s, vl4
19 ; VBITS_GE_128-NEXT: sshll v2.8h, v1.8b, #0
20 ; VBITS_GE_128-NEXT: sshll v3.8h, v0.8b, #0
21 ; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0
22 ; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #0
23 ; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0
24 ; VBITS_GE_128-NEXT: sshll v3.4s, v3.4h, #0
25 ; VBITS_GE_128-NEXT: sdivr z4.s, p0/m, z4.s, z5.s
26 ; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s
27 ; VBITS_GE_128-NEXT: uzp1 v2.8h, v2.8h, v4.8h
28 ; VBITS_GE_128-NEXT: xtn v2.8b, v2.8h
29 ; VBITS_GE_128-NEXT: mls v0.8b, v2.8b, v1.8b
30 ; VBITS_GE_128-NEXT: ret
31 ;
32 ; VBITS_GE_256-LABEL: srem_v8i8:
33 ; VBITS_GE_256: // %bb.0:
34 ; VBITS_GE_256-NEXT: // kill: def $d1 killed $d1 def $z1
35 ; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 def $z0
36 ; VBITS_GE_256-NEXT: sunpklo z2.h, z1.b
37 ; VBITS_GE_256-NEXT: sunpklo z3.h, z0.b
38 ; VBITS_GE_256-NEXT: ptrue p0.s, vl8
39 ; VBITS_GE_256-NEXT: sunpklo z2.s, z2.h
40 ; VBITS_GE_256-NEXT: sunpklo z3.s, z3.h
41 ; VBITS_GE_256-NEXT: sdivr z2.s, p0/m, z2.s, z3.s
42 ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
43 ; VBITS_GE_256-NEXT: umov w8, v2.h[0]
44 ; VBITS_GE_256-NEXT: umov w9, v2.h[1]
45 ; VBITS_GE_256-NEXT: fmov s3, w8
46 ; VBITS_GE_256-NEXT: umov w8, v2.h[2]
47 ; VBITS_GE_256-NEXT: mov v3.b[1], w9
48 ; VBITS_GE_256-NEXT: mov v3.b[2], w8
49 ; VBITS_GE_256-NEXT: umov w8, v2.h[3]
50 ; VBITS_GE_256-NEXT: mov v3.b[3], w8
51 ; VBITS_GE_256-NEXT: umov w8, v2.h[4]
52 ; VBITS_GE_256-NEXT: mov v3.b[4], w8
53 ; VBITS_GE_256-NEXT: umov w8, v2.h[5]
54 ; VBITS_GE_256-NEXT: mov v3.b[5], w8
55 ; VBITS_GE_256-NEXT: umov w8, v2.h[6]
56 ; VBITS_GE_256-NEXT: mov v3.b[6], w8
57 ; VBITS_GE_256-NEXT: umov w8, v2.h[7]
58 ; VBITS_GE_256-NEXT: mov v3.b[7], w8
59 ; VBITS_GE_256-NEXT: mls v0.8b, v3.8b, v1.8b
60 ; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 killed $z0
61 ; VBITS_GE_256-NEXT: ret
62 ;
63 ; VBITS_GE_512-LABEL: srem_v8i8:
64 ; VBITS_GE_512: // %bb.0:
65 ; VBITS_GE_512-NEXT: // kill: def $d1 killed $d1 def $z1
66 ; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z0
67 ; VBITS_GE_512-NEXT: sunpklo z2.h, z1.b
68 ; VBITS_GE_512-NEXT: sunpklo z3.h, z0.b
69 ; VBITS_GE_512-NEXT: ptrue p0.s, vl8
70 ; VBITS_GE_512-NEXT: sunpklo z2.s, z2.h
71 ; VBITS_GE_512-NEXT: sunpklo z3.s, z3.h
72 ; VBITS_GE_512-NEXT: sdivr z2.s, p0/m, z2.s, z3.s
73 ; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
74 ; VBITS_GE_512-NEXT: umov w8, v2.h[0]
75 ; VBITS_GE_512-NEXT: umov w9, v2.h[1]
76 ; VBITS_GE_512-NEXT: fmov s3, w8
77 ; VBITS_GE_512-NEXT: umov w8, v2.h[2]
78 ; VBITS_GE_512-NEXT: mov v3.b[1], w9
79 ; VBITS_GE_512-NEXT: mov v3.b[2], w8
80 ; VBITS_GE_512-NEXT: umov w8, v2.h[3]
81 ; VBITS_GE_512-NEXT: mov v3.b[3], w8
82 ; VBITS_GE_512-NEXT: umov w8, v2.h[4]
83 ; VBITS_GE_512-NEXT: mov v3.b[4], w8
84 ; VBITS_GE_512-NEXT: umov w8, v2.h[5]
85 ; VBITS_GE_512-NEXT: mov v3.b[5], w8
86 ; VBITS_GE_512-NEXT: umov w8, v2.h[6]
87 ; VBITS_GE_512-NEXT: mov v3.b[6], w8
88 ; VBITS_GE_512-NEXT: umov w8, v2.h[7]
89 ; VBITS_GE_512-NEXT: mov v3.b[7], w8
90 ; VBITS_GE_512-NEXT: mls v0.8b, v3.8b, v1.8b
91 ; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 killed $z0
92 ; VBITS_GE_512-NEXT: ret
93 %res = srem <8 x i8> %op1, %op2
94 ret <8 x i8> %res
95 }
97 define <16 x i8> @srem_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 {
98 ; VBITS_GE_128-LABEL: srem_v16i8:
99 ; VBITS_GE_128: // %bb.0:
100 ; VBITS_GE_128-NEXT: ptrue p0.s, vl4
101 ; VBITS_GE_128-NEXT: sshll2 v2.8h, v1.16b, #0
102 ; VBITS_GE_128-NEXT: sshll2 v3.8h, v0.16b, #0
103 ; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0
104 ; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #0
105 ; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0
106 ; VBITS_GE_128-NEXT: sshll v3.4s, v3.4h, #0
107 ; VBITS_GE_128-NEXT: sdivr z4.s, p0/m, z4.s, z5.s
108 ; VBITS_GE_128-NEXT: sshll v5.8h, v0.8b, #0
109 ; VBITS_GE_128-NEXT: sshll2 v7.4s, v5.8h, #0
110 ; VBITS_GE_128-NEXT: sshll v5.4s, v5.4h, #0
111 ; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s
112 ; VBITS_GE_128-NEXT: sshll v3.8h, v1.8b, #0
113 ; VBITS_GE_128-NEXT: sshll2 v6.4s, v3.8h, #0
114 ; VBITS_GE_128-NEXT: sshll v3.4s, v3.4h, #0
115 ; VBITS_GE_128-NEXT: sdivr z6.s, p0/m, z6.s, z7.s
116 ; VBITS_GE_128-NEXT: uzp1 v2.8h, v2.8h, v4.8h
117 ; VBITS_GE_128-NEXT: sdivr z3.s, p0/m, z3.s, z5.s
118 ; VBITS_GE_128-NEXT: uzp1 v3.8h, v3.8h, v6.8h
119 ; VBITS_GE_128-NEXT: uzp1 v2.16b, v3.16b, v2.16b
120 ; VBITS_GE_128-NEXT: mls v0.16b, v2.16b, v1.16b
121 ; VBITS_GE_128-NEXT: ret
122 ;
123 ; VBITS_GE_256-LABEL: srem_v16i8:
124 ; VBITS_GE_256: // %bb.0:
125 ; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1
126 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0
127 ; VBITS_GE_256-NEXT: sunpklo z2.h, z1.b
128 ; VBITS_GE_256-NEXT: sunpklo z3.h, z0.b
129 ; VBITS_GE_256-NEXT: ptrue p0.s, vl8
130 ; VBITS_GE_256-NEXT: sunpklo z4.s, z2.h
131 ; VBITS_GE_256-NEXT: sunpklo z5.s, z3.h
132 ; VBITS_GE_256-NEXT: ext z2.b, z2.b, z2.b, #16
133 ; VBITS_GE_256-NEXT: ext z3.b, z3.b, z3.b, #16
134 ; VBITS_GE_256-NEXT: sunpklo z2.s, z2.h
135 ; VBITS_GE_256-NEXT: sunpklo z3.s, z3.h
136 ; VBITS_GE_256-NEXT: sdivr z4.s, p0/m, z4.s, z5.s
137 ; VBITS_GE_256-NEXT: sdivr z2.s, p0/m, z2.s, z3.s
138 ; VBITS_GE_256-NEXT: ptrue p0.h, vl8
139 ; VBITS_GE_256-NEXT: uzp1 z3.h, z4.h, z4.h
140 ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
141 ; VBITS_GE_256-NEXT: splice z3.h, p0, z3.h, z2.h
142 ; VBITS_GE_256-NEXT: uzp1 z2.b, z3.b, z3.b
143 ; VBITS_GE_256-NEXT: mls v0.16b, v2.16b, v1.16b
144 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0
145 ; VBITS_GE_256-NEXT: ret
146 ;
147 ; VBITS_GE_512-LABEL: srem_v16i8:
148 ; VBITS_GE_512: // %bb.0:
149 ; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1
150 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0
151 ; VBITS_GE_512-NEXT: sunpklo z2.h, z1.b
152 ; VBITS_GE_512-NEXT: sunpklo z3.h, z0.b
153 ; VBITS_GE_512-NEXT: ptrue p0.s, vl16
154 ; VBITS_GE_512-NEXT: sunpklo z2.s, z2.h
155 ; VBITS_GE_512-NEXT: sunpklo z3.s, z3.h
156 ; VBITS_GE_512-NEXT: sdivr z2.s, p0/m, z2.s, z3.s
157 ; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
158 ; VBITS_GE_512-NEXT: uzp1 z2.b, z2.b, z2.b
159 ; VBITS_GE_512-NEXT: mls v0.16b, v2.16b, v1.16b
160 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0
161 ; VBITS_GE_512-NEXT: ret
162 %res = srem <16 x i8> %op1, %op2
163 ret <16 x i8> %res
164 }
166 define void @srem_v32i8(ptr %a, ptr %b) vscale_range(8,0) #0 {
167 ; CHECK-LABEL: srem_v32i8:
168 ; CHECK: // %bb.0:
169 ; CHECK-NEXT: ptrue p0.b, vl32
170 ; CHECK-NEXT: ptrue p1.s, vl32
171 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
172 ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
173 ; CHECK-NEXT: sunpklo z2.h, z1.b
174 ; CHECK-NEXT: sunpklo z3.h, z0.b
175 ; CHECK-NEXT: sunpklo z2.s, z2.h
176 ; CHECK-NEXT: sunpklo z3.s, z3.h
177 ; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
178 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
179 ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b
180 ; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
181 ; CHECK-NEXT: st1b { z0.b }, p0, [x0]
182 ; CHECK-NEXT: ret
183 %op1 = load <32 x i8>, ptr %a
184 %op2 = load <32 x i8>, ptr %b
185 %res = srem <32 x i8> %op1, %op2
186 store <32 x i8> %res, ptr %a
187 ret void
188 }
190 define void @srem_v64i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
191 ; CHECK-LABEL: srem_v64i8:
192 ; CHECK: // %bb.0:
193 ; CHECK-NEXT: ptrue p0.b, vl64
194 ; CHECK-NEXT: ptrue p1.s, vl64
195 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
196 ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
197 ; CHECK-NEXT: sunpklo z2.h, z1.b
198 ; CHECK-NEXT: sunpklo z3.h, z0.b
199 ; CHECK-NEXT: sunpklo z2.s, z2.h
200 ; CHECK-NEXT: sunpklo z3.s, z3.h
201 ; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
202 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
203 ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b
204 ; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
205 ; CHECK-NEXT: st1b { z0.b }, p0, [x0]
206 ; CHECK-NEXT: ret
207 %op1 = load <64 x i8>, ptr %a
208 %op2 = load <64 x i8>, ptr %b
209 %res = srem <64 x i8> %op1, %op2
210 store <64 x i8> %res, ptr %a
211 ret void
212 }
214 define void @srem_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
215 ; CHECK-LABEL: srem_v128i8:
216 ; CHECK: // %bb.0:
217 ; CHECK-NEXT: ptrue p0.b, vl128
218 ; CHECK-NEXT: ptrue p1.s, vl64
219 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
220 ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
221 ; CHECK-NEXT: sunpklo z2.h, z1.b
222 ; CHECK-NEXT: sunpklo z3.h, z0.b
223 ; CHECK-NEXT: sunpklo z4.s, z2.h
224 ; CHECK-NEXT: sunpklo z5.s, z3.h
225 ; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128
226 ; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
227 ; CHECK-NEXT: sunpklo z2.s, z2.h
228 ; CHECK-NEXT: sunpklo z3.s, z3.h
229 ; CHECK-NEXT: sdivr z4.s, p1/m, z4.s, z5.s
230 ; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
231 ; CHECK-NEXT: ptrue p1.h, vl64
232 ; CHECK-NEXT: uzp1 z3.h, z4.h, z4.h
233 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
234 ; CHECK-NEXT: splice z3.h, p1, z3.h, z2.h
235 ; CHECK-NEXT: uzp1 z2.b, z3.b, z3.b
236 ; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
237 ; CHECK-NEXT: st1b { z0.b }, p0, [x0]
238 ; CHECK-NEXT: ret
239 %op1 = load <128 x i8>, ptr %a
240 %op2 = load <128 x i8>, ptr %b
241 %res = srem <128 x i8> %op1, %op2
242 store <128 x i8> %res, ptr %a
243 ret void
244 }
246 define void @srem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
247 ; CHECK-LABEL: srem_v256i8:
248 ; CHECK: // %bb.0:
249 ; CHECK-NEXT: ptrue p0.b, vl256
250 ; CHECK-NEXT: ptrue p1.s, vl64
251 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
252 ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
253 ; CHECK-NEXT: sunpklo z2.h, z1.b
254 ; CHECK-NEXT: sunpklo z3.h, z0.b
255 ; CHECK-NEXT: sunpklo z4.s, z2.h
256 ; CHECK-NEXT: sunpklo z5.s, z3.h
257 ; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128
258 ; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
259 ; CHECK-NEXT: sunpklo z2.s, z2.h
260 ; CHECK-NEXT: sunpklo z3.s, z3.h
261 ; CHECK-NEXT: sdivr z4.s, p1/m, z4.s, z5.s
262 ; CHECK-NEXT: mov z5.d, z0.d
263 ; CHECK-NEXT: ext z5.b, z5.b, z0.b, #128
264 ; CHECK-NEXT: sunpklo z5.h, z5.b
265 ; CHECK-NEXT: sunpklo z7.s, z5.h
266 ; CHECK-NEXT: ext z5.b, z5.b, z5.b, #128
267 ; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
268 ; CHECK-NEXT: mov z3.d, z1.d
269 ; CHECK-NEXT: sunpklo z5.s, z5.h
270 ; CHECK-NEXT: ext z3.b, z3.b, z1.b, #128
271 ; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h
272 ; CHECK-NEXT: sunpklo z3.h, z3.b
273 ; CHECK-NEXT: sunpklo z6.s, z3.h
274 ; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
275 ; CHECK-NEXT: sunpklo z3.s, z3.h
276 ; CHECK-NEXT: sdivr z6.s, p1/m, z6.s, z7.s
277 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
278 ; CHECK-NEXT: sdivr z3.s, p1/m, z3.s, z5.s
279 ; CHECK-NEXT: ptrue p1.h, vl64
280 ; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h
281 ; CHECK-NEXT: splice z4.h, p1, z4.h, z2.h
282 ; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b
283 ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h
284 ; CHECK-NEXT: splice z5.h, p1, z5.h, z3.h
285 ; CHECK-NEXT: ptrue p1.b, vl128
286 ; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b
287 ; CHECK-NEXT: splice z2.b, p1, z2.b, z3.b
288 ; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
289 ; CHECK-NEXT: st1b { z0.b }, p0, [x0]
290 ; CHECK-NEXT: ret
291 %op1 = load <256 x i8>, ptr %a
292 %op2 = load <256 x i8>, ptr %b
293 %res = srem <256 x i8> %op1, %op2
294 store <256 x i8> %res, ptr %a
295 ret void
296 }
298 ; Vector vXi16 sdiv are not legal for NEON so use SVE when available.
299 ; FIXME: We should be able to improve the codegen for >= 256 bits here.
300 define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 {
301 ; VBITS_GE_128-LABEL: srem_v4i16:
302 ; VBITS_GE_128: // %bb.0:
303 ; VBITS_GE_128-NEXT: ptrue p0.s, vl4
304 ; VBITS_GE_128-NEXT: sshll v2.4s, v1.4h, #0
305 ; VBITS_GE_128-NEXT: sshll v3.4s, v0.4h, #0
306 ; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s
307 ; VBITS_GE_128-NEXT: xtn v2.4h, v2.4s
308 ; VBITS_GE_128-NEXT: mls v0.4h, v2.4h, v1.4h
309 ; VBITS_GE_128-NEXT: ret
310 ;
311 ; VBITS_GE_256-LABEL: srem_v4i16:
312 ; VBITS_GE_256: // %bb.0:
313 ; VBITS_GE_256-NEXT: ptrue p0.s, vl4
314 ; VBITS_GE_256-NEXT: sshll v2.4s, v1.4h, #0
315 ; VBITS_GE_256-NEXT: sshll v3.4s, v0.4h, #0
316 ; VBITS_GE_256-NEXT: sdivr z2.s, p0/m, z2.s, z3.s
317 ; VBITS_GE_256-NEXT: mov w8, v2.s[1]
318 ; VBITS_GE_256-NEXT: mov v3.16b, v2.16b
319 ; VBITS_GE_256-NEXT: mov w9, v2.s[2]
320 ; VBITS_GE_256-NEXT: mov v3.h[1], w8
321 ; VBITS_GE_256-NEXT: mov w8, v2.s[3]
322 ; VBITS_GE_256-NEXT: mov v3.h[2], w9
323 ; VBITS_GE_256-NEXT: mov v3.h[3], w8
324 ; VBITS_GE_256-NEXT: mls v0.4h, v3.4h, v1.4h
325 ; VBITS_GE_256-NEXT: ret
326 ;
327 ; VBITS_GE_512-LABEL: srem_v4i16:
328 ; VBITS_GE_512: // %bb.0:
329 ; VBITS_GE_512-NEXT: ptrue p0.s, vl4
330 ; VBITS_GE_512-NEXT: sshll v2.4s, v1.4h, #0
331 ; VBITS_GE_512-NEXT: sshll v3.4s, v0.4h, #0
332 ; VBITS_GE_512-NEXT: sdivr z2.s, p0/m, z2.s, z3.s
333 ; VBITS_GE_512-NEXT: mov w8, v2.s[1]
334 ; VBITS_GE_512-NEXT: mov v3.16b, v2.16b
335 ; VBITS_GE_512-NEXT: mov w9, v2.s[2]
336 ; VBITS_GE_512-NEXT: mov v3.h[1], w8
337 ; VBITS_GE_512-NEXT: mov w8, v2.s[3]
338 ; VBITS_GE_512-NEXT: mov v3.h[2], w9
339 ; VBITS_GE_512-NEXT: mov v3.h[3], w8
340 ; VBITS_GE_512-NEXT: mls v0.4h, v3.4h, v1.4h
341 ; VBITS_GE_512-NEXT: ret
342 %res = srem <4 x i16> %op1, %op2
343 ret <4 x i16> %res
344 }
346 define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 {
347 ; VBITS_GE_128-LABEL: srem_v8i16:
348 ; VBITS_GE_128: // %bb.0:
349 ; VBITS_GE_128-NEXT: ptrue p0.s, vl4
350 ; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0
351 ; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0
352 ; VBITS_GE_128-NEXT: sshll v4.4s, v0.4h, #0
353 ; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s
354 ; VBITS_GE_128-NEXT: sshll v3.4s, v1.4h, #0
355 ; VBITS_GE_128-NEXT: sdivr z3.s, p0/m, z3.s, z4.s
356 ; VBITS_GE_128-NEXT: uzp1 v2.8h, v3.8h, v2.8h
357 ; VBITS_GE_128-NEXT: mls v0.8h, v2.8h, v1.8h
358 ; VBITS_GE_128-NEXT: ret
359 ;
360 ; VBITS_GE_256-LABEL: srem_v8i16:
361 ; VBITS_GE_256: // %bb.0:
362 ; VBITS_GE_256-NEXT: ptrue p0.s, vl8
363 ; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1
364 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0
365 ; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h
366 ; VBITS_GE_256-NEXT: sunpklo z3.s, z0.h
367 ; VBITS_GE_256-NEXT: sdivr z2.s, p0/m, z2.s, z3.s
368 ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
369 ; VBITS_GE_256-NEXT: mls v0.8h, v2.8h, v1.8h
370 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0
371 ; VBITS_GE_256-NEXT: ret
372 ;
373 ; VBITS_GE_512-LABEL: srem_v8i16:
374 ; VBITS_GE_512: // %bb.0:
375 ; VBITS_GE_512-NEXT: ptrue p0.s, vl8
376 ; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1
377 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0
378 ; VBITS_GE_512-NEXT: sunpklo z2.s, z1.h
379 ; VBITS_GE_512-NEXT: sunpklo z3.s, z0.h
380 ; VBITS_GE_512-NEXT: sdivr z2.s, p0/m, z2.s, z3.s
381 ; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
382 ; VBITS_GE_512-NEXT: mls v0.8h, v2.8h, v1.8h
383 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0
384 ; VBITS_GE_512-NEXT: ret
385 %res = srem <8 x i16> %op1, %op2
386 ret <8 x i16> %res
387 }
389 define void @srem_v16i16(ptr %a, ptr %b) #0 {
390 ; VBITS_GE_128-LABEL: srem_v16i16:
391 ; VBITS_GE_128: // %bb.0:
392 ; VBITS_GE_128-NEXT: ptrue p0.s, vl4
393 ; VBITS_GE_128-NEXT: ldp q4, q1, [x1]
394 ; VBITS_GE_128-NEXT: ldr q0, [x0, #16]
395 ; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0
396 ; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0
397 ; VBITS_GE_128-NEXT: sshll2 v5.4s, v4.8h, #0
398 ; VBITS_GE_128-NEXT: sshll v16.4s, v0.4h, #0
399 ; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s
400 ; VBITS_GE_128-NEXT: ldr q3, [x0]
401 ; VBITS_GE_128-NEXT: sshll2 v6.4s, v3.8h, #0
402 ; VBITS_GE_128-NEXT: sshll v7.4s, v3.4h, #0
403 ; VBITS_GE_128-NEXT: sdivr z5.s, p0/m, z5.s, z6.s
404 ; VBITS_GE_128-NEXT: sshll v6.4s, v4.4h, #0
405 ; VBITS_GE_128-NEXT: sdivr z6.s, p0/m, z6.s, z7.s
406 ; VBITS_GE_128-NEXT: sshll v7.4s, v1.4h, #0
407 ; VBITS_GE_128-NEXT: sdivr z7.s, p0/m, z7.s, z16.s
408 ; VBITS_GE_128-NEXT: uzp1 v5.8h, v6.8h, v5.8h
409 ; VBITS_GE_128-NEXT: mls v3.8h, v5.8h, v4.8h
410 ; VBITS_GE_128-NEXT: uzp1 v2.8h, v7.8h, v2.8h
411 ; VBITS_GE_128-NEXT: mls v0.8h, v2.8h, v1.8h
412 ; VBITS_GE_128-NEXT: stp q3, q0, [x0]
413 ; VBITS_GE_128-NEXT: ret
414 ;
415 ; VBITS_GE_256-LABEL: srem_v16i16:
416 ; VBITS_GE_256: // %bb.0:
417 ; VBITS_GE_256-NEXT: ptrue p0.h, vl16
418 ; VBITS_GE_256-NEXT: ptrue p1.s, vl8
419 ; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]
420 ; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]
421 ; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h
422 ; VBITS_GE_256-NEXT: sunpklo z3.s, z0.h
423 ; VBITS_GE_256-NEXT: mov z4.d, z0.d
424 ; VBITS_GE_256-NEXT: ext z4.b, z4.b, z0.b, #16
425 ; VBITS_GE_256-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
426 ; VBITS_GE_256-NEXT: mov z3.d, z1.d
427 ; VBITS_GE_256-NEXT: sunpklo z4.s, z4.h
428 ; VBITS_GE_256-NEXT: ext z3.b, z3.b, z1.b, #16
429 ; VBITS_GE_256-NEXT: sunpklo z3.s, z3.h
430 ; VBITS_GE_256-NEXT: sdivr z3.s, p1/m, z3.s, z4.s
431 ; VBITS_GE_256-NEXT: ptrue p1.h, vl8
432 ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
433 ; VBITS_GE_256-NEXT: uzp1 z3.h, z3.h, z3.h
434 ; VBITS_GE_256-NEXT: splice z2.h, p1, z2.h, z3.h
435 ; VBITS_GE_256-NEXT: mls z0.h, p0/m, z2.h, z1.h
436 ; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]
437 ; VBITS_GE_256-NEXT: ret
438 ;
439 ; VBITS_GE_512-LABEL: srem_v16i16:
440 ; VBITS_GE_512: // %bb.0:
441 ; VBITS_GE_512-NEXT: ptrue p0.h, vl16
442 ; VBITS_GE_512-NEXT: ptrue p1.s, vl16
443 ; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]
444 ; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]
445 ; VBITS_GE_512-NEXT: sunpklo z2.s, z1.h
446 ; VBITS_GE_512-NEXT: sunpklo z3.s, z0.h
447 ; VBITS_GE_512-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
448 ; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
449 ; VBITS_GE_512-NEXT: mls z0.h, p0/m, z2.h, z1.h
450 ; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]
451 ; VBITS_GE_512-NEXT: ret
452 %op1 = load <16 x i16>, ptr %a
453 %op2 = load <16 x i16>, ptr %b
454 %res = srem <16 x i16> %op1, %op2
455 store <16 x i16> %res, ptr %a
456 ret void
457 }
459 define void @srem_v32i16(ptr %a, ptr %b) vscale_range(8,0) #0 {
460 ; CHECK-LABEL: srem_v32i16:
461 ; CHECK: // %bb.0:
462 ; CHECK-NEXT: ptrue p0.h, vl32
463 ; CHECK-NEXT: ptrue p1.s, vl32
464 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
465 ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
466 ; CHECK-NEXT: sunpklo z2.s, z1.h
467 ; CHECK-NEXT: sunpklo z3.s, z0.h
468 ; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
469 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
470 ; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h
471 ; CHECK-NEXT: st1h { z0.h }, p0, [x0]
472 ; CHECK-NEXT: ret
473 %op1 = load <32 x i16>, ptr %a
474 %op2 = load <32 x i16>, ptr %b
475 %res = srem <32 x i16> %op1, %op2
476 store <32 x i16> %res, ptr %a
477 ret void
478 }
480 define void @srem_v64i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
481 ; CHECK-LABEL: srem_v64i16:
482 ; CHECK: // %bb.0:
483 ; CHECK-NEXT: ptrue p0.h, vl64
484 ; CHECK-NEXT: ptrue p1.s, vl64
485 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
486 ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
487 ; CHECK-NEXT: sunpklo z2.s, z1.h
488 ; CHECK-NEXT: sunpklo z3.s, z0.h
489 ; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
490 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
491 ; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h
492 ; CHECK-NEXT: st1h { z0.h }, p0, [x0]
493 ; CHECK-NEXT: ret
494 %op1 = load <64 x i16>, ptr %a
495 %op2 = load <64 x i16>, ptr %b
496 %res = srem <64 x i16> %op1, %op2
497 store <64 x i16> %res, ptr %a
498 ret void
499 }
501 define void @srem_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
502 ; CHECK-LABEL: srem_v128i16:
503 ; CHECK: // %bb.0:
504 ; CHECK-NEXT: ptrue p0.h, vl128
505 ; CHECK-NEXT: ptrue p1.s, vl64
506 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
507 ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
508 ; CHECK-NEXT: sunpklo z2.s, z1.h
509 ; CHECK-NEXT: sunpklo z3.s, z0.h
510 ; CHECK-NEXT: mov z4.d, z0.d
511 ; CHECK-NEXT: ext z4.b, z4.b, z0.b, #128
512 ; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
513 ; CHECK-NEXT: mov z3.d, z1.d
514 ; CHECK-NEXT: sunpklo z4.s, z4.h
515 ; CHECK-NEXT: ext z3.b, z3.b, z1.b, #128
516 ; CHECK-NEXT: sunpklo z3.s, z3.h
517 ; CHECK-NEXT: sdivr z3.s, p1/m, z3.s, z4.s
518 ; CHECK-NEXT: ptrue p1.h, vl64
519 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
520 ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h
521 ; CHECK-NEXT: splice z2.h, p1, z2.h, z3.h
522 ; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h
523 ; CHECK-NEXT: st1h { z0.h }, p0, [x0]
524 ; CHECK-NEXT: ret
525 %op1 = load <128 x i16>, ptr %a
526 %op2 = load <128 x i16>, ptr %b
527 %res = srem <128 x i16> %op1, %op2
528 store <128 x i16> %res, ptr %a
529 ret void
530 }
532 ; Vector v2i32 sdiv are not legal for NEON so use SVE when available.
533 define <2 x i32> @srem_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #0 {
534 ; CHECK-LABEL: srem_v2i32:
535 ; CHECK: // %bb.0:
536 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
537 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
538 ; CHECK-NEXT: ptrue p0.s, vl2
539 ; CHECK-NEXT: movprfx z2, z0
540 ; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s
541 ; CHECK-NEXT: mls v0.2s, v2.2s, v1.2s
542 ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
543 ; CHECK-NEXT: ret
544 %res = srem <2 x i32> %op1, %op2
545 ret <2 x i32> %res
546 }
548 ; Vector v4i32 sdiv are not legal for NEON so use SVE when available.
549 define <4 x i32> @srem_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(1,0) #0 {
550 ; CHECK-LABEL: srem_v4i32:
551 ; CHECK: // %bb.0:
552 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
553 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
554 ; CHECK-NEXT: ptrue p0.s, vl4
555 ; CHECK-NEXT: movprfx z2, z0
556 ; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s
557 ; CHECK-NEXT: mls v0.4s, v2.4s, v1.4s
558 ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
559 ; CHECK-NEXT: ret
560 %res = srem <4 x i32> %op1, %op2
561 ret <4 x i32> %res
562 }
564 define void @srem_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {
565 ; CHECK-LABEL: srem_v8i32:
566 ; CHECK: // %bb.0:
567 ; CHECK-NEXT: ptrue p0.s, vl8
568 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
569 ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
570 ; CHECK-NEXT: movprfx z2, z0
571 ; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s
572 ; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s
573 ; CHECK-NEXT: st1w { z0.s }, p0, [x0]
574 ; CHECK-NEXT: ret
575 %op1 = load <8 x i32>, ptr %a
576 %op2 = load <8 x i32>, ptr %b
577 %res = srem <8 x i32> %op1, %op2
578 store <8 x i32> %res, ptr %a
579 ret void
580 }
582 define void @srem_v16i32(ptr %a, ptr %b) #0 {
583 ; VBITS_GE_128-LABEL: srem_v16i32:
584 ; VBITS_GE_128: // %bb.0:
585 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1]
586 ; VBITS_GE_128-NEXT: ptrue p0.s, vl4
587 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0]
588 ; VBITS_GE_128-NEXT: movprfx z4, z1
589 ; VBITS_GE_128-NEXT: sdiv z4.s, p0/m, z4.s, z0.s
590 ; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32]
591 ; VBITS_GE_128-NEXT: movprfx z19, z2
592 ; VBITS_GE_128-NEXT: sdiv z19.s, p0/m, z19.s, z3.s
593 ; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32]
594 ; VBITS_GE_128-NEXT: movprfx z7, z5
595 ; VBITS_GE_128-NEXT: sdiv z7.s, p0/m, z7.s, z6.s
596 ; VBITS_GE_128-NEXT: movprfx z18, z16
597 ; VBITS_GE_128-NEXT: sdiv z18.s, p0/m, z18.s, z17.s
598 ; VBITS_GE_128-NEXT: mls v1.4s, v4.4s, v0.4s
599 ; VBITS_GE_128-NEXT: mls v2.4s, v19.4s, v3.4s
600 ; VBITS_GE_128-NEXT: mls v16.4s, v18.4s, v17.4s
601 ; VBITS_GE_128-NEXT: mls v5.4s, v7.4s, v6.4s
602 ; VBITS_GE_128-NEXT: stp q1, q2, [x0]
603 ; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32]
604 ; VBITS_GE_128-NEXT: ret
605 ;
606 ; VBITS_GE_256-LABEL: srem_v16i32:
607 ; VBITS_GE_256: // %bb.0:
608 ; VBITS_GE_256-NEXT: ptrue p0.s, vl8
609 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
610 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
611 ; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]
612 ; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2]
613 ; VBITS_GE_256-NEXT: ld1w { z4.s }, p0/z, [x1]
614 ; VBITS_GE_256-NEXT: movprfx z3, z0
615 ; VBITS_GE_256-NEXT: sdiv z3.s, p0/m, z3.s, z2.s
616 ; VBITS_GE_256-NEXT: movprfx z5, z1
617 ; VBITS_GE_256-NEXT: sdiv z5.s, p0/m, z5.s, z4.s
618 ; VBITS_GE_256-NEXT: mls z0.s, p0/m, z3.s, z2.s
619 ; VBITS_GE_256-NEXT: mls z1.s, p0/m, z5.s, z4.s
620 ; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
621 ; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
622 ; VBITS_GE_256-NEXT: ret
623 ;
624 ; VBITS_GE_512-LABEL: srem_v16i32:
625 ; VBITS_GE_512: // %bb.0:
626 ; VBITS_GE_512-NEXT: ptrue p0.s, vl16
627 ; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
628 ; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]
629 ; VBITS_GE_512-NEXT: movprfx z2, z0
630 ; VBITS_GE_512-NEXT: sdiv z2.s, p0/m, z2.s, z1.s
631 ; VBITS_GE_512-NEXT: mls z0.s, p0/m, z2.s, z1.s
632 ; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
633 ; VBITS_GE_512-NEXT: ret
634 %op1 = load <16 x i32>, ptr %a
635 %op2 = load <16 x i32>, ptr %b
636 %res = srem <16 x i32> %op1, %op2
637 store <16 x i32> %res, ptr %a
638 ret void
639 }
641 define void @srem_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {
642 ; CHECK-LABEL: srem_v32i32:
643 ; CHECK: // %bb.0:
644 ; CHECK-NEXT: ptrue p0.s, vl32
645 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
646 ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
647 ; CHECK-NEXT: movprfx z2, z0
648 ; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s
649 ; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s
650 ; CHECK-NEXT: st1w { z0.s }, p0, [x0]
651 ; CHECK-NEXT: ret
652 %op1 = load <32 x i32>, ptr %a
653 %op2 = load <32 x i32>, ptr %b
654 %res = srem <32 x i32> %op1, %op2
655 store <32 x i32> %res, ptr %a
656 ret void
657 }
659 define void @srem_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {
660 ; CHECK-LABEL: srem_v64i32:
661 ; CHECK: // %bb.0:
662 ; CHECK-NEXT: ptrue p0.s, vl64
663 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
664 ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
665 ; CHECK-NEXT: movprfx z2, z0
666 ; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s
667 ; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s
668 ; CHECK-NEXT: st1w { z0.s }, p0, [x0]
669 ; CHECK-NEXT: ret
670 %op1 = load <64 x i32>, ptr %a
671 %op2 = load <64 x i32>, ptr %b
672 %res = srem <64 x i32> %op1, %op2
673 store <64 x i32> %res, ptr %a
674 ret void
675 }
677 ; Vector i64 sdiv are not legal for NEON so use SVE when available.
678 ; FIXME: We should be able to improve the codegen for the 128 bits case here.
679 define <1 x i64> @srem_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #0 {
680 ; CHECK-LABEL: srem_v1i64:
681 ; CHECK: // %bb.0:
682 ; CHECK-NEXT: ptrue p0.d, vl1
683 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
684 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
685 ; CHECK-NEXT: movprfx z2, z0
686 ; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d
687 ; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
688 ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
689 ; CHECK-NEXT: ret
690 %res = srem <1 x i64> %op1, %op2
691 ret <1 x i64> %res
692 }
694 ; Vector i64 sdiv are not legal for NEON so use SVE when available.
695 ; FIXME: We should be able to improve the codegen for the 128 bits case here.
696 define <2 x i64> @srem_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #0 {
697 ; CHECK-LABEL: srem_v2i64:
698 ; CHECK: // %bb.0:
699 ; CHECK-NEXT: ptrue p0.d, vl2
700 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
701 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
702 ; CHECK-NEXT: movprfx z2, z0
703 ; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d
704 ; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
705 ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
706 ; CHECK-NEXT: ret
707 %res = srem <2 x i64> %op1, %op2
708 ret <2 x i64> %res
709 }
711 define void @srem_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {
712 ; CHECK-LABEL: srem_v4i64:
713 ; CHECK: // %bb.0:
714 ; CHECK-NEXT: ptrue p0.d, vl4
715 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
716 ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
717 ; CHECK-NEXT: movprfx z2, z0
718 ; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d
719 ; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
720 ; CHECK-NEXT: st1d { z0.d }, p0, [x0]
721 ; CHECK-NEXT: ret
722 %op1 = load <4 x i64>, ptr %a
723 %op2 = load <4 x i64>, ptr %b
724 %res = srem <4 x i64> %op1, %op2
725 store <4 x i64> %res, ptr %a
726 ret void
727 }
729 define void @srem_v8i64(ptr %a, ptr %b) #0 {
730 ; VBITS_GE_128-LABEL: srem_v8i64:
731 ; VBITS_GE_128: // %bb.0:
732 ; VBITS_GE_128-NEXT: ptrue p0.d, vl2
733 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1]
734 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0]
735 ; VBITS_GE_128-NEXT: movprfx z4, z1
736 ; VBITS_GE_128-NEXT: sdiv z4.d, p0/m, z4.d, z0.d
737 ; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32]
738 ; VBITS_GE_128-NEXT: movprfx z19, z2
739 ; VBITS_GE_128-NEXT: sdiv z19.d, p0/m, z19.d, z3.d
740 ; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32]
741 ; VBITS_GE_128-NEXT: movprfx z7, z5
742 ; VBITS_GE_128-NEXT: sdiv z7.d, p0/m, z7.d, z6.d
743 ; VBITS_GE_128-NEXT: movprfx z18, z16
744 ; VBITS_GE_128-NEXT: sdiv z18.d, p0/m, z18.d, z17.d
745 ; VBITS_GE_128-NEXT: mls z16.d, p0/m, z18.d, z17.d
746 ; VBITS_GE_128-NEXT: mls z5.d, p0/m, z7.d, z6.d
747 ; VBITS_GE_128-NEXT: msb z0.d, p0/m, z4.d, z1.d
748 ; VBITS_GE_128-NEXT: movprfx z1, z2
749 ; VBITS_GE_128-NEXT: mls z1.d, p0/m, z19.d, z3.d
750 ; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32]
751 ; VBITS_GE_128-NEXT: stp q0, q1, [x0]
752 ; VBITS_GE_128-NEXT: ret
753 ;
754 ; VBITS_GE_256-LABEL: srem_v8i64:
755 ; VBITS_GE_256: // %bb.0:
756 ; VBITS_GE_256-NEXT: ptrue p0.d, vl4
757 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
758 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
759 ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
760 ; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3]
761 ; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1]
762 ; VBITS_GE_256-NEXT: movprfx z3, z0
763 ; VBITS_GE_256-NEXT: sdiv z3.d, p0/m, z3.d, z2.d
764 ; VBITS_GE_256-NEXT: movprfx z5, z1
765 ; VBITS_GE_256-NEXT: sdiv z5.d, p0/m, z5.d, z4.d
766 ; VBITS_GE_256-NEXT: mls z0.d, p0/m, z3.d, z2.d
767 ; VBITS_GE_256-NEXT: mls z1.d, p0/m, z5.d, z4.d
768 ; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
769 ; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
770 ; VBITS_GE_256-NEXT: ret
771 ;
772 ; VBITS_GE_512-LABEL: srem_v8i64:
773 ; VBITS_GE_512: // %bb.0:
774 ; VBITS_GE_512-NEXT: ptrue p0.d, vl8
775 ; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
776 ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
777 ; VBITS_GE_512-NEXT: movprfx z2, z0
778 ; VBITS_GE_512-NEXT: sdiv z2.d, p0/m, z2.d, z1.d
779 ; VBITS_GE_512-NEXT: mls z0.d, p0/m, z2.d, z1.d
780 ; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
781 ; VBITS_GE_512-NEXT: ret
782 %op1 = load <8 x i64>, ptr %a
783 %op2 = load <8 x i64>, ptr %b
784 %res = srem <8 x i64> %op1, %op2
785 store <8 x i64> %res, ptr %a
786 ret void
787 }
789 define void @srem_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {
790 ; CHECK-LABEL: srem_v16i64:
791 ; CHECK: // %bb.0:
792 ; CHECK-NEXT: ptrue p0.d, vl16
793 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
794 ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
795 ; CHECK-NEXT: movprfx z2, z0
796 ; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d
797 ; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
798 ; CHECK-NEXT: st1d { z0.d }, p0, [x0]
799 ; CHECK-NEXT: ret
800 %op1 = load <16 x i64>, ptr %a
801 %op2 = load <16 x i64>, ptr %b
802 %res = srem <16 x i64> %op1, %op2
803 store <16 x i64> %res, ptr %a
804 ret void
805 }
807 define void @srem_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {
808 ; CHECK-LABEL: srem_v32i64:
809 ; CHECK: // %bb.0:
810 ; CHECK-NEXT: ptrue p0.d, vl32
811 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
812 ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
813 ; CHECK-NEXT: movprfx z2, z0
814 ; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d
815 ; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
816 ; CHECK-NEXT: st1d { z0.d }, p0, [x0]
817 ; CHECK-NEXT: ret
818 %op1 = load <32 x i64>, ptr %a
819 %op2 = load <32 x i64>, ptr %b
820 %res = srem <32 x i64> %op1, %op2
821 store <32 x i64> %res, ptr %a
822 ret void
823 }
825 ;
826 ; UREM
827 ;
829 ; Vector vXi8 udiv are not legal for NEON so use SVE when available.
830 ; FIXME: We should be able to improve the codegen for >= 256 bits here.
831 define <8 x i8> @urem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 {
832 ; VBITS_GE_128-LABEL: urem_v8i8:
833 ; VBITS_GE_128: // %bb.0:
834 ; VBITS_GE_128-NEXT: ptrue p0.s, vl4
835 ; VBITS_GE_128-NEXT: ushll v2.8h, v1.8b, #0
836 ; VBITS_GE_128-NEXT: ushll v3.8h, v0.8b, #0
837 ; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0
838 ; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0
839 ; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0
840 ; VBITS_GE_128-NEXT: ushll v3.4s, v3.4h, #0
841 ; VBITS_GE_128-NEXT: udivr z4.s, p0/m, z4.s, z5.s
842 ; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s
843 ; VBITS_GE_128-NEXT: uzp1 v2.8h, v2.8h, v4.8h
844 ; VBITS_GE_128-NEXT: xtn v2.8b, v2.8h
845 ; VBITS_GE_128-NEXT: mls v0.8b, v2.8b, v1.8b
846 ; VBITS_GE_128-NEXT: ret
847 ;
848 ; VBITS_GE_256-LABEL: urem_v8i8:
849 ; VBITS_GE_256: // %bb.0:
850 ; VBITS_GE_256-NEXT: // kill: def $d1 killed $d1 def $z1
851 ; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 def $z0
852 ; VBITS_GE_256-NEXT: uunpklo z2.h, z1.b
853 ; VBITS_GE_256-NEXT: uunpklo z3.h, z0.b
854 ; VBITS_GE_256-NEXT: ptrue p0.s, vl8
855 ; VBITS_GE_256-NEXT: uunpklo z2.s, z2.h
856 ; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h
857 ; VBITS_GE_256-NEXT: udivr z2.s, p0/m, z2.s, z3.s
858 ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
859 ; VBITS_GE_256-NEXT: umov w8, v2.h[0]
860 ; VBITS_GE_256-NEXT: umov w9, v2.h[1]
861 ; VBITS_GE_256-NEXT: fmov s3, w8
862 ; VBITS_GE_256-NEXT: umov w8, v2.h[2]
863 ; VBITS_GE_256-NEXT: mov v3.b[1], w9
864 ; VBITS_GE_256-NEXT: mov v3.b[2], w8
865 ; VBITS_GE_256-NEXT: umov w8, v2.h[3]
866 ; VBITS_GE_256-NEXT: mov v3.b[3], w8
867 ; VBITS_GE_256-NEXT: umov w8, v2.h[4]
868 ; VBITS_GE_256-NEXT: mov v3.b[4], w8
869 ; VBITS_GE_256-NEXT: umov w8, v2.h[5]
870 ; VBITS_GE_256-NEXT: mov v3.b[5], w8
871 ; VBITS_GE_256-NEXT: umov w8, v2.h[6]
872 ; VBITS_GE_256-NEXT: mov v3.b[6], w8
873 ; VBITS_GE_256-NEXT: umov w8, v2.h[7]
874 ; VBITS_GE_256-NEXT: mov v3.b[7], w8
875 ; VBITS_GE_256-NEXT: mls v0.8b, v3.8b, v1.8b
876 ; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 killed $z0
877 ; VBITS_GE_256-NEXT: ret
878 ;
879 ; VBITS_GE_512-LABEL: urem_v8i8:
880 ; VBITS_GE_512: // %bb.0:
881 ; VBITS_GE_512-NEXT: // kill: def $d1 killed $d1 def $z1
882 ; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z0
883 ; VBITS_GE_512-NEXT: uunpklo z2.h, z1.b
884 ; VBITS_GE_512-NEXT: uunpklo z3.h, z0.b
885 ; VBITS_GE_512-NEXT: ptrue p0.s, vl8
886 ; VBITS_GE_512-NEXT: uunpklo z2.s, z2.h
887 ; VBITS_GE_512-NEXT: uunpklo z3.s, z3.h
888 ; VBITS_GE_512-NEXT: udivr z2.s, p0/m, z2.s, z3.s
889 ; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
890 ; VBITS_GE_512-NEXT: umov w8, v2.h[0]
891 ; VBITS_GE_512-NEXT: umov w9, v2.h[1]
892 ; VBITS_GE_512-NEXT: fmov s3, w8
893 ; VBITS_GE_512-NEXT: umov w8, v2.h[2]
894 ; VBITS_GE_512-NEXT: mov v3.b[1], w9
895 ; VBITS_GE_512-NEXT: mov v3.b[2], w8
896 ; VBITS_GE_512-NEXT: umov w8, v2.h[3]
897 ; VBITS_GE_512-NEXT: mov v3.b[3], w8
898 ; VBITS_GE_512-NEXT: umov w8, v2.h[4]
899 ; VBITS_GE_512-NEXT: mov v3.b[4], w8
900 ; VBITS_GE_512-NEXT: umov w8, v2.h[5]
901 ; VBITS_GE_512-NEXT: mov v3.b[5], w8
902 ; VBITS_GE_512-NEXT: umov w8, v2.h[6]
903 ; VBITS_GE_512-NEXT: mov v3.b[6], w8
904 ; VBITS_GE_512-NEXT: umov w8, v2.h[7]
905 ; VBITS_GE_512-NEXT: mov v3.b[7], w8
906 ; VBITS_GE_512-NEXT: mls v0.8b, v3.8b, v1.8b
907 ; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 killed $z0
908 ; VBITS_GE_512-NEXT: ret
909 %res = urem <8 x i8> %op1, %op2
910 ret <8 x i8> %res
911 }
913 define <16 x i8> @urem_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 {
914 ; VBITS_GE_128-LABEL: urem_v16i8:
915 ; VBITS_GE_128: // %bb.0:
916 ; VBITS_GE_128-NEXT: ptrue p0.s, vl4
917 ; VBITS_GE_128-NEXT: ushll2 v2.8h, v1.16b, #0
918 ; VBITS_GE_128-NEXT: ushll2 v3.8h, v0.16b, #0
919 ; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0
920 ; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0
921 ; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0
922 ; VBITS_GE_128-NEXT: ushll v3.4s, v3.4h, #0
923 ; VBITS_GE_128-NEXT: udivr z4.s, p0/m, z4.s, z5.s
924 ; VBITS_GE_128-NEXT: ushll v5.8h, v0.8b, #0
925 ; VBITS_GE_128-NEXT: ushll2 v7.4s, v5.8h, #0
926 ; VBITS_GE_128-NEXT: ushll v5.4s, v5.4h, #0
927 ; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s
928 ; VBITS_GE_128-NEXT: ushll v3.8h, v1.8b, #0
929 ; VBITS_GE_128-NEXT: ushll2 v6.4s, v3.8h, #0
930 ; VBITS_GE_128-NEXT: ushll v3.4s, v3.4h, #0
931 ; VBITS_GE_128-NEXT: udivr z6.s, p0/m, z6.s, z7.s
932 ; VBITS_GE_128-NEXT: uzp1 v2.8h, v2.8h, v4.8h
933 ; VBITS_GE_128-NEXT: udivr z3.s, p0/m, z3.s, z5.s
934 ; VBITS_GE_128-NEXT: uzp1 v3.8h, v3.8h, v6.8h
935 ; VBITS_GE_128-NEXT: uzp1 v2.16b, v3.16b, v2.16b
936 ; VBITS_GE_128-NEXT: mls v0.16b, v2.16b, v1.16b
937 ; VBITS_GE_128-NEXT: ret
938 ;
939 ; VBITS_GE_256-LABEL: urem_v16i8:
940 ; VBITS_GE_256: // %bb.0:
941 ; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1
942 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0
943 ; VBITS_GE_256-NEXT: uunpklo z2.h, z1.b
944 ; VBITS_GE_256-NEXT: uunpklo z3.h, z0.b
945 ; VBITS_GE_256-NEXT: ptrue p0.s, vl8
946 ; VBITS_GE_256-NEXT: uunpklo z4.s, z2.h
947 ; VBITS_GE_256-NEXT: uunpklo z5.s, z3.h
948 ; VBITS_GE_256-NEXT: ext z2.b, z2.b, z2.b, #16
949 ; VBITS_GE_256-NEXT: ext z3.b, z3.b, z3.b, #16
950 ; VBITS_GE_256-NEXT: uunpklo z2.s, z2.h
951 ; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h
952 ; VBITS_GE_256-NEXT: udivr z4.s, p0/m, z4.s, z5.s
953 ; VBITS_GE_256-NEXT: udivr z2.s, p0/m, z2.s, z3.s
954 ; VBITS_GE_256-NEXT: ptrue p0.h, vl8
955 ; VBITS_GE_256-NEXT: uzp1 z3.h, z4.h, z4.h
956 ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
957 ; VBITS_GE_256-NEXT: splice z3.h, p0, z3.h, z2.h
958 ; VBITS_GE_256-NEXT: uzp1 z2.b, z3.b, z3.b
959 ; VBITS_GE_256-NEXT: mls v0.16b, v2.16b, v1.16b
960 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0
961 ; VBITS_GE_256-NEXT: ret
962 ;
963 ; VBITS_GE_512-LABEL: urem_v16i8:
964 ; VBITS_GE_512: // %bb.0:
965 ; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1
966 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0
967 ; VBITS_GE_512-NEXT: uunpklo z2.h, z1.b
968 ; VBITS_GE_512-NEXT: uunpklo z3.h, z0.b
969 ; VBITS_GE_512-NEXT: ptrue p0.s, vl16
970 ; VBITS_GE_512-NEXT: uunpklo z2.s, z2.h
971 ; VBITS_GE_512-NEXT: uunpklo z3.s, z3.h
972 ; VBITS_GE_512-NEXT: udivr z2.s, p0/m, z2.s, z3.s
973 ; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
974 ; VBITS_GE_512-NEXT: uzp1 z2.b, z2.b, z2.b
975 ; VBITS_GE_512-NEXT: mls v0.16b, v2.16b, v1.16b
976 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0
977 ; VBITS_GE_512-NEXT: ret
978 %res = urem <16 x i8> %op1, %op2
979 ret <16 x i8> %res
980 }
982 define void @urem_v32i8(ptr %a, ptr %b) vscale_range(8,0) #0 {
983 ; CHECK-LABEL: urem_v32i8:
984 ; CHECK: // %bb.0:
985 ; CHECK-NEXT: ptrue p0.b, vl32
986 ; CHECK-NEXT: ptrue p1.s, vl32
987 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
988 ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
989 ; CHECK-NEXT: uunpklo z2.h, z1.b
990 ; CHECK-NEXT: uunpklo z3.h, z0.b
991 ; CHECK-NEXT: uunpklo z2.s, z2.h
992 ; CHECK-NEXT: uunpklo z3.s, z3.h
993 ; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s
994 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
995 ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b
996 ; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
997 ; CHECK-NEXT: st1b { z0.b }, p0, [x0]
998 ; CHECK-NEXT: ret
999 %op1 = load <32 x i8>, ptr %a
1000 %op2 = load <32 x i8>, ptr %b
1001 %res = urem <32 x i8> %op1, %op2
1002 store <32 x i8> %res, ptr %a
1003 ret void
1004 }
1006 define void @urem_v64i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
1007 ; CHECK-LABEL: urem_v64i8:
1008 ; CHECK: // %bb.0:
1009 ; CHECK-NEXT: ptrue p0.b, vl64
1010 ; CHECK-NEXT: ptrue p1.s, vl64
1011 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
1012 ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
1013 ; CHECK-NEXT: uunpklo z2.h, z1.b
1014 ; CHECK-NEXT: uunpklo z3.h, z0.b
1015 ; CHECK-NEXT: uunpklo z2.s, z2.h
1016 ; CHECK-NEXT: uunpklo z3.s, z3.h
1017 ; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s
1018 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
1019 ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b
1020 ; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
1021 ; CHECK-NEXT: st1b { z0.b }, p0, [x0]
1022 ; CHECK-NEXT: ret
1023 %op1 = load <64 x i8>, ptr %a
1024 %op2 = load <64 x i8>, ptr %b
1025 %res = urem <64 x i8> %op1, %op2
1026 store <64 x i8> %res, ptr %a
1027 ret void
1028 }
1030 define void @urem_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
1031 ; CHECK-LABEL: urem_v128i8:
1032 ; CHECK: // %bb.0:
1033 ; CHECK-NEXT: ptrue p0.b, vl128
1034 ; CHECK-NEXT: ptrue p1.s, vl64
1035 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
1036 ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
1037 ; CHECK-NEXT: uunpklo z2.h, z1.b
1038 ; CHECK-NEXT: uunpklo z3.h, z0.b
1039 ; CHECK-NEXT: uunpklo z4.s, z2.h
1040 ; CHECK-NEXT: uunpklo z5.s, z3.h
1041 ; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128
1042 ; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
1043 ; CHECK-NEXT: uunpklo z2.s, z2.h
1044 ; CHECK-NEXT: uunpklo z3.s, z3.h
1045 ; CHECK-NEXT: udivr z4.s, p1/m, z4.s, z5.s
1046 ; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s
1047 ; CHECK-NEXT: ptrue p1.h, vl64
1048 ; CHECK-NEXT: uzp1 z3.h, z4.h, z4.h
1049 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
1050 ; CHECK-NEXT: splice z3.h, p1, z3.h, z2.h
1051 ; CHECK-NEXT: uzp1 z2.b, z3.b, z3.b
1052 ; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
1053 ; CHECK-NEXT: st1b { z0.b }, p0, [x0]
1054 ; CHECK-NEXT: ret
1055 %op1 = load <128 x i8>, ptr %a
1056 %op2 = load <128 x i8>, ptr %b
1057 %res = urem <128 x i8> %op1, %op2
1058 store <128 x i8> %res, ptr %a
1059 ret void
1060 }
1062 define void @urem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
1063 ; CHECK-LABEL: urem_v256i8:
1064 ; CHECK: // %bb.0:
1065 ; CHECK-NEXT: ptrue p0.b, vl256
1066 ; CHECK-NEXT: ptrue p1.s, vl64
1067 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
1068 ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
1069 ; CHECK-NEXT: uunpklo z2.h, z1.b
1070 ; CHECK-NEXT: uunpklo z3.h, z0.b
1071 ; CHECK-NEXT: uunpklo z4.s, z2.h
1072 ; CHECK-NEXT: uunpklo z5.s, z3.h
1073 ; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128
1074 ; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
1075 ; CHECK-NEXT: uunpklo z2.s, z2.h
1076 ; CHECK-NEXT: uunpklo z3.s, z3.h
1077 ; CHECK-NEXT: udivr z4.s, p1/m, z4.s, z5.s
1078 ; CHECK-NEXT: mov z5.d, z0.d
1079 ; CHECK-NEXT: ext z5.b, z5.b, z0.b, #128
1080 ; CHECK-NEXT: uunpklo z5.h, z5.b
1081 ; CHECK-NEXT: uunpklo z7.s, z5.h
1082 ; CHECK-NEXT: ext z5.b, z5.b, z5.b, #128
1083 ; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s
1084 ; CHECK-NEXT: mov z3.d, z1.d
1085 ; CHECK-NEXT: uunpklo z5.s, z5.h
1086 ; CHECK-NEXT: ext z3.b, z3.b, z1.b, #128
1087 ; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h
1088 ; CHECK-NEXT: uunpklo z3.h, z3.b
1089 ; CHECK-NEXT: uunpklo z6.s, z3.h
1090 ; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
1091 ; CHECK-NEXT: uunpklo z3.s, z3.h
1092 ; CHECK-NEXT: udivr z6.s, p1/m, z6.s, z7.s
1093 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
1094 ; CHECK-NEXT: udivr z3.s, p1/m, z3.s, z5.s
1095 ; CHECK-NEXT: ptrue p1.h, vl64
1096 ; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h
1097 ; CHECK-NEXT: splice z4.h, p1, z4.h, z2.h
1098 ; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b
1099 ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h
1100 ; CHECK-NEXT: splice z5.h, p1, z5.h, z3.h
1101 ; CHECK-NEXT: ptrue p1.b, vl128
1102 ; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b
1103 ; CHECK-NEXT: splice z2.b, p1, z2.b, z3.b
1104 ; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
1105 ; CHECK-NEXT: st1b { z0.b }, p0, [x0]
1106 ; CHECK-NEXT: ret
1107 %op1 = load <256 x i8>, ptr %a
1108 %op2 = load <256 x i8>, ptr %b
1109 %res = urem <256 x i8> %op1, %op2
1110 store <256 x i8> %res, ptr %a
1111 ret void
1112 }
1114 ; Vector vXi16 udiv are not legal for NEON so use SVE when available.
1115 ; FIXME: We should be able to improve the codegen for >= 256 bits here.
1116 define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 {
1117 ; VBITS_GE_128-LABEL: urem_v4i16:
1118 ; VBITS_GE_128: // %bb.0:
1119 ; VBITS_GE_128-NEXT: ptrue p0.s, vl4
1120 ; VBITS_GE_128-NEXT: ushll v2.4s, v1.4h, #0
1121 ; VBITS_GE_128-NEXT: ushll v3.4s, v0.4h, #0
1122 ; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s
1123 ; VBITS_GE_128-NEXT: xtn v2.4h, v2.4s
1124 ; VBITS_GE_128-NEXT: mls v0.4h, v2.4h, v1.4h
1125 ; VBITS_GE_128-NEXT: ret
1126 ;
1127 ; VBITS_GE_256-LABEL: urem_v4i16:
1128 ; VBITS_GE_256: // %bb.0:
1129 ; VBITS_GE_256-NEXT: ptrue p0.s, vl4
1130 ; VBITS_GE_256-NEXT: ushll v2.4s, v1.4h, #0
1131 ; VBITS_GE_256-NEXT: ushll v3.4s, v0.4h, #0
1132 ; VBITS_GE_256-NEXT: udivr z2.s, p0/m, z2.s, z3.s
1133 ; VBITS_GE_256-NEXT: mov w8, v2.s[1]
1134 ; VBITS_GE_256-NEXT: mov v3.16b, v2.16b
1135 ; VBITS_GE_256-NEXT: mov w9, v2.s[2]
1136 ; VBITS_GE_256-NEXT: mov v3.h[1], w8
1137 ; VBITS_GE_256-NEXT: mov w8, v2.s[3]
1138 ; VBITS_GE_256-NEXT: mov v3.h[2], w9
1139 ; VBITS_GE_256-NEXT: mov v3.h[3], w8
1140 ; VBITS_GE_256-NEXT: mls v0.4h, v3.4h, v1.4h
1141 ; VBITS_GE_256-NEXT: ret
1142 ;
1143 ; VBITS_GE_512-LABEL: urem_v4i16:
1144 ; VBITS_GE_512: // %bb.0:
1145 ; VBITS_GE_512-NEXT: ptrue p0.s, vl4
1146 ; VBITS_GE_512-NEXT: ushll v2.4s, v1.4h, #0
1147 ; VBITS_GE_512-NEXT: ushll v3.4s, v0.4h, #0
1148 ; VBITS_GE_512-NEXT: udivr z2.s, p0/m, z2.s, z3.s
1149 ; VBITS_GE_512-NEXT: mov w8, v2.s[1]
1150 ; VBITS_GE_512-NEXT: mov v3.16b, v2.16b
1151 ; VBITS_GE_512-NEXT: mov w9, v2.s[2]
1152 ; VBITS_GE_512-NEXT: mov v3.h[1], w8
1153 ; VBITS_GE_512-NEXT: mov w8, v2.s[3]
1154 ; VBITS_GE_512-NEXT: mov v3.h[2], w9
1155 ; VBITS_GE_512-NEXT: mov v3.h[3], w8
1156 ; VBITS_GE_512-NEXT: mls v0.4h, v3.4h, v1.4h
1157 ; VBITS_GE_512-NEXT: ret
1158 %res = urem <4 x i16> %op1, %op2
1159 ret <4 x i16> %res
1160 }
1162 define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 {
1163 ; VBITS_GE_128-LABEL: urem_v8i16:
1164 ; VBITS_GE_128: // %bb.0:
1165 ; VBITS_GE_128-NEXT: ptrue p0.s, vl4
1166 ; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #0
1167 ; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0
1168 ; VBITS_GE_128-NEXT: ushll v4.4s, v0.4h, #0
1169 ; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s
1170 ; VBITS_GE_128-NEXT: ushll v3.4s, v1.4h, #0
1171 ; VBITS_GE_128-NEXT: udivr z3.s, p0/m, z3.s, z4.s
1172 ; VBITS_GE_128-NEXT: uzp1 v2.8h, v3.8h, v2.8h
1173 ; VBITS_GE_128-NEXT: mls v0.8h, v2.8h, v1.8h
1174 ; VBITS_GE_128-NEXT: ret
1175 ;
1176 ; VBITS_GE_256-LABEL: urem_v8i16:
1177 ; VBITS_GE_256: // %bb.0:
1178 ; VBITS_GE_256-NEXT: ptrue p0.s, vl8
1179 ; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1
1180 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0
1181 ; VBITS_GE_256-NEXT: uunpklo z2.s, z1.h
1182 ; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h
1183 ; VBITS_GE_256-NEXT: udivr z2.s, p0/m, z2.s, z3.s
1184 ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
1185 ; VBITS_GE_256-NEXT: mls v0.8h, v2.8h, v1.8h
1186 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0
1187 ; VBITS_GE_256-NEXT: ret
1188 ;
1189 ; VBITS_GE_512-LABEL: urem_v8i16:
1190 ; VBITS_GE_512: // %bb.0:
1191 ; VBITS_GE_512-NEXT: ptrue p0.s, vl8
1192 ; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1
1193 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0
1194 ; VBITS_GE_512-NEXT: uunpklo z2.s, z1.h
1195 ; VBITS_GE_512-NEXT: uunpklo z3.s, z0.h
1196 ; VBITS_GE_512-NEXT: udivr z2.s, p0/m, z2.s, z3.s
1197 ; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
1198 ; VBITS_GE_512-NEXT: mls v0.8h, v2.8h, v1.8h
1199 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0
1200 ; VBITS_GE_512-NEXT: ret
1201 %res = urem <8 x i16> %op1, %op2
1202 ret <8 x i16> %res
1203 }
1205 define void @urem_v16i16(ptr %a, ptr %b) #0 {
1206 ; VBITS_GE_128-LABEL: urem_v16i16:
1207 ; VBITS_GE_128: // %bb.0:
1208 ; VBITS_GE_128-NEXT: ptrue p0.s, vl4
1209 ; VBITS_GE_128-NEXT: ldp q4, q1, [x1]
1210 ; VBITS_GE_128-NEXT: ldr q0, [x0, #16]
1211 ; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #0
1212 ; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0
1213 ; VBITS_GE_128-NEXT: ushll2 v5.4s, v4.8h, #0
1214 ; VBITS_GE_128-NEXT: ushll v16.4s, v0.4h, #0
1215 ; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s
1216 ; VBITS_GE_128-NEXT: ldr q3, [x0]
1217 ; VBITS_GE_128-NEXT: ushll2 v6.4s, v3.8h, #0
1218 ; VBITS_GE_128-NEXT: ushll v7.4s, v3.4h, #0
1219 ; VBITS_GE_128-NEXT: udivr z5.s, p0/m, z5.s, z6.s
1220 ; VBITS_GE_128-NEXT: ushll v6.4s, v4.4h, #0
1221 ; VBITS_GE_128-NEXT: udivr z6.s, p0/m, z6.s, z7.s
1222 ; VBITS_GE_128-NEXT: ushll v7.4s, v1.4h, #0
1223 ; VBITS_GE_128-NEXT: udivr z7.s, p0/m, z7.s, z16.s
1224 ; VBITS_GE_128-NEXT: uzp1 v5.8h, v6.8h, v5.8h
1225 ; VBITS_GE_128-NEXT: mls v3.8h, v5.8h, v4.8h
1226 ; VBITS_GE_128-NEXT: uzp1 v2.8h, v7.8h, v2.8h
1227 ; VBITS_GE_128-NEXT: mls v0.8h, v2.8h, v1.8h
1228 ; VBITS_GE_128-NEXT: stp q3, q0, [x0]
1229 ; VBITS_GE_128-NEXT: ret
1230 ;
1231 ; VBITS_GE_256-LABEL: urem_v16i16:
1232 ; VBITS_GE_256: // %bb.0:
1233 ; VBITS_GE_256-NEXT: ptrue p0.h, vl16
1234 ; VBITS_GE_256-NEXT: ptrue p1.s, vl8
1235 ; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]
1236 ; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]
1237 ; VBITS_GE_256-NEXT: uunpklo z2.s, z1.h
1238 ; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h
1239 ; VBITS_GE_256-NEXT: mov z4.d, z0.d
1240 ; VBITS_GE_256-NEXT: ext z4.b, z4.b, z0.b, #16
1241 ; VBITS_GE_256-NEXT: udivr z2.s, p1/m, z2.s, z3.s
1242 ; VBITS_GE_256-NEXT: mov z3.d, z1.d
1243 ; VBITS_GE_256-NEXT: uunpklo z4.s, z4.h
1244 ; VBITS_GE_256-NEXT: ext z3.b, z3.b, z1.b, #16
1245 ; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h
1246 ; VBITS_GE_256-NEXT: udivr z3.s, p1/m, z3.s, z4.s
1247 ; VBITS_GE_256-NEXT: ptrue p1.h, vl8
1248 ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
1249 ; VBITS_GE_256-NEXT: uzp1 z3.h, z3.h, z3.h
1250 ; VBITS_GE_256-NEXT: splice z2.h, p1, z2.h, z3.h
1251 ; VBITS_GE_256-NEXT: mls z0.h, p0/m, z2.h, z1.h
1252 ; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]
1253 ; VBITS_GE_256-NEXT: ret
1254 ;
1255 ; VBITS_GE_512-LABEL: urem_v16i16:
1256 ; VBITS_GE_512: // %bb.0:
1257 ; VBITS_GE_512-NEXT: ptrue p0.h, vl16
1258 ; VBITS_GE_512-NEXT: ptrue p1.s, vl16
1259 ; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]
1260 ; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]
1261 ; VBITS_GE_512-NEXT: uunpklo z2.s, z1.h
1262 ; VBITS_GE_512-NEXT: uunpklo z3.s, z0.h
1263 ; VBITS_GE_512-NEXT: udivr z2.s, p1/m, z2.s, z3.s
1264 ; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
1265 ; VBITS_GE_512-NEXT: mls z0.h, p0/m, z2.h, z1.h
1266 ; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]
1267 ; VBITS_GE_512-NEXT: ret
1268 %op1 = load <16 x i16>, ptr %a
1269 %op2 = load <16 x i16>, ptr %b
1270 %res = urem <16 x i16> %op1, %op2
1271 store <16 x i16> %res, ptr %a
1272 ret void
1273 }
1275 define void @urem_v32i16(ptr %a, ptr %b) vscale_range(8,0) #0 {
1276 ; CHECK-LABEL: urem_v32i16:
1277 ; CHECK: // %bb.0:
1278 ; CHECK-NEXT: ptrue p0.h, vl32
1279 ; CHECK-NEXT: ptrue p1.s, vl32
1280 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
1281 ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
1282 ; CHECK-NEXT: uunpklo z2.s, z1.h
1283 ; CHECK-NEXT: uunpklo z3.s, z0.h
1284 ; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s
1285 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
1286 ; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h
1287 ; CHECK-NEXT: st1h { z0.h }, p0, [x0]
1288 ; CHECK-NEXT: ret
1289 %op1 = load <32 x i16>, ptr %a
1290 %op2 = load <32 x i16>, ptr %b
1291 %res = urem <32 x i16> %op1, %op2
1292 store <32 x i16> %res, ptr %a
1293 ret void
1294 }
1296 define void @urem_v64i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
1297 ; CHECK-LABEL: urem_v64i16:
1298 ; CHECK: // %bb.0:
1299 ; CHECK-NEXT: ptrue p0.h, vl64
1300 ; CHECK-NEXT: ptrue p1.s, vl64
1301 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
1302 ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
1303 ; CHECK-NEXT: uunpklo z2.s, z1.h
1304 ; CHECK-NEXT: uunpklo z3.s, z0.h
1305 ; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s
1306 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
1307 ; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h
1308 ; CHECK-NEXT: st1h { z0.h }, p0, [x0]
1309 ; CHECK-NEXT: ret
1310 %op1 = load <64 x i16>, ptr %a
1311 %op2 = load <64 x i16>, ptr %b
1312 %res = urem <64 x i16> %op1, %op2
1313 store <64 x i16> %res, ptr %a
1314 ret void
1315 }
1317 define void @urem_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
1318 ; CHECK-LABEL: urem_v128i16:
1319 ; CHECK: // %bb.0:
1320 ; CHECK-NEXT: ptrue p0.h, vl128
1321 ; CHECK-NEXT: ptrue p1.s, vl64
1322 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
1323 ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
1324 ; CHECK-NEXT: uunpklo z2.s, z1.h
1325 ; CHECK-NEXT: uunpklo z3.s, z0.h
1326 ; CHECK-NEXT: mov z4.d, z0.d
1327 ; CHECK-NEXT: ext z4.b, z4.b, z0.b, #128
1328 ; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s
1329 ; CHECK-NEXT: mov z3.d, z1.d
1330 ; CHECK-NEXT: uunpklo z4.s, z4.h
1331 ; CHECK-NEXT: ext z3.b, z3.b, z1.b, #128
1332 ; CHECK-NEXT: uunpklo z3.s, z3.h
1333 ; CHECK-NEXT: udivr z3.s, p1/m, z3.s, z4.s
1334 ; CHECK-NEXT: ptrue p1.h, vl64
1335 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
1336 ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h
1337 ; CHECK-NEXT: splice z2.h, p1, z2.h, z3.h
1338 ; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h
1339 ; CHECK-NEXT: st1h { z0.h }, p0, [x0]
1340 ; CHECK-NEXT: ret
1341 %op1 = load <128 x i16>, ptr %a
1342 %op2 = load <128 x i16>, ptr %b
1343 %res = urem <128 x i16> %op1, %op2
1344 store <128 x i16> %res, ptr %a
1345 ret void
1346 }
1348 ; Vector v2i32 udiv are not legal for NEON so use SVE when available.
1349 define <2 x i32> @urem_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #0 {
1350 ; CHECK-LABEL: urem_v2i32:
1351 ; CHECK: // %bb.0:
1352 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
1353 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
1354 ; CHECK-NEXT: ptrue p0.s, vl2
1355 ; CHECK-NEXT: movprfx z2, z0
1356 ; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s
1357 ; CHECK-NEXT: mls v0.2s, v2.2s, v1.2s
1358 ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
1359 ; CHECK-NEXT: ret
1360 %res = urem <2 x i32> %op1, %op2
1361 ret <2 x i32> %res
1362 }
1364 ; Vector v4i32 udiv are not legal for NEON so use SVE when available.
1365 define <4 x i32> @urem_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(1,0) #0 {
1366 ; CHECK-LABEL: urem_v4i32:
1367 ; CHECK: // %bb.0:
1368 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
1369 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
1370 ; CHECK-NEXT: ptrue p0.s, vl4
1371 ; CHECK-NEXT: movprfx z2, z0
1372 ; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s
1373 ; CHECK-NEXT: mls v0.4s, v2.4s, v1.4s
1374 ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
1375 ; CHECK-NEXT: ret
1376 %res = urem <4 x i32> %op1, %op2
1377 ret <4 x i32> %res
1378 }
1380 define void @urem_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {
1381 ; CHECK-LABEL: urem_v8i32:
1382 ; CHECK: // %bb.0:
1383 ; CHECK-NEXT: ptrue p0.s, vl8
1384 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
1385 ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
1386 ; CHECK-NEXT: movprfx z2, z0
1387 ; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s
1388 ; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s
1389 ; CHECK-NEXT: st1w { z0.s }, p0, [x0]
1390 ; CHECK-NEXT: ret
1391 %op1 = load <8 x i32>, ptr %a
1392 %op2 = load <8 x i32>, ptr %b
1393 %res = urem <8 x i32> %op1, %op2
1394 store <8 x i32> %res, ptr %a
1395 ret void
1396 }
1398 define void @urem_v16i32(ptr %a, ptr %b) #0 {
1399 ; VBITS_GE_128-LABEL: urem_v16i32:
1400 ; VBITS_GE_128: // %bb.0:
1401 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1]
1402 ; VBITS_GE_128-NEXT: ptrue p0.s, vl4
1403 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0]
1404 ; VBITS_GE_128-NEXT: movprfx z4, z1
1405 ; VBITS_GE_128-NEXT: udiv z4.s, p0/m, z4.s, z0.s
1406 ; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32]
1407 ; VBITS_GE_128-NEXT: movprfx z19, z2
1408 ; VBITS_GE_128-NEXT: udiv z19.s, p0/m, z19.s, z3.s
1409 ; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32]
1410 ; VBITS_GE_128-NEXT: movprfx z7, z5
1411 ; VBITS_GE_128-NEXT: udiv z7.s, p0/m, z7.s, z6.s
1412 ; VBITS_GE_128-NEXT: movprfx z18, z16
1413 ; VBITS_GE_128-NEXT: udiv z18.s, p0/m, z18.s, z17.s
1414 ; VBITS_GE_128-NEXT: mls v1.4s, v4.4s, v0.4s
1415 ; VBITS_GE_128-NEXT: mls v2.4s, v19.4s, v3.4s
1416 ; VBITS_GE_128-NEXT: mls v16.4s, v18.4s, v17.4s
1417 ; VBITS_GE_128-NEXT: mls v5.4s, v7.4s, v6.4s
1418 ; VBITS_GE_128-NEXT: stp q1, q2, [x0]
1419 ; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32]
1420 ; VBITS_GE_128-NEXT: ret
1421 ;
1422 ; VBITS_GE_256-LABEL: urem_v16i32:
1423 ; VBITS_GE_256: // %bb.0:
1424 ; VBITS_GE_256-NEXT: ptrue p0.s, vl8
1425 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
1426 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
1427 ; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]
1428 ; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2]
1429 ; VBITS_GE_256-NEXT: ld1w { z4.s }, p0/z, [x1]
1430 ; VBITS_GE_256-NEXT: movprfx z3, z0
1431 ; VBITS_GE_256-NEXT: udiv z3.s, p0/m, z3.s, z2.s
1432 ; VBITS_GE_256-NEXT: movprfx z5, z1
1433 ; VBITS_GE_256-NEXT: udiv z5.s, p0/m, z5.s, z4.s
1434 ; VBITS_GE_256-NEXT: mls z0.s, p0/m, z3.s, z2.s
1435 ; VBITS_GE_256-NEXT: mls z1.s, p0/m, z5.s, z4.s
1436 ; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
1437 ; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
1438 ; VBITS_GE_256-NEXT: ret
1439 ;
1440 ; VBITS_GE_512-LABEL: urem_v16i32:
1441 ; VBITS_GE_512: // %bb.0:
1442 ; VBITS_GE_512-NEXT: ptrue p0.s, vl16
1443 ; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
1444 ; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]
1445 ; VBITS_GE_512-NEXT: movprfx z2, z0
1446 ; VBITS_GE_512-NEXT: udiv z2.s, p0/m, z2.s, z1.s
1447 ; VBITS_GE_512-NEXT: mls z0.s, p0/m, z2.s, z1.s
1448 ; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
1449 ; VBITS_GE_512-NEXT: ret
1450 %op1 = load <16 x i32>, ptr %a
1451 %op2 = load <16 x i32>, ptr %b
1452 %res = urem <16 x i32> %op1, %op2
1453 store <16 x i32> %res, ptr %a
1454 ret void
1455 }
1457 define void @urem_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {
1458 ; CHECK-LABEL: urem_v32i32:
1459 ; CHECK: // %bb.0:
1460 ; CHECK-NEXT: ptrue p0.s, vl32
1461 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
1462 ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
1463 ; CHECK-NEXT: movprfx z2, z0
1464 ; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s
1465 ; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s
1466 ; CHECK-NEXT: st1w { z0.s }, p0, [x0]
1467 ; CHECK-NEXT: ret
1468 %op1 = load <32 x i32>, ptr %a
1469 %op2 = load <32 x i32>, ptr %b
1470 %res = urem <32 x i32> %op1, %op2
1471 store <32 x i32> %res, ptr %a
1472 ret void
1473 }
1475 define void @urem_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {
1476 ; CHECK-LABEL: urem_v64i32:
1477 ; CHECK: // %bb.0:
1478 ; CHECK-NEXT: ptrue p0.s, vl64
1479 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
1480 ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
1481 ; CHECK-NEXT: movprfx z2, z0
1482 ; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s
1483 ; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s
1484 ; CHECK-NEXT: st1w { z0.s }, p0, [x0]
1485 ; CHECK-NEXT: ret
1486 %op1 = load <64 x i32>, ptr %a
1487 %op2 = load <64 x i32>, ptr %b
1488 %res = urem <64 x i32> %op1, %op2
1489 store <64 x i32> %res, ptr %a
1490 ret void
1491 }
1493 ; Vector i64 udiv are not legal for NEON so use SVE when available.
1494 ; FIXME: We should be able to improve the codegen for the 128 bits case here.
1495 define <1 x i64> @urem_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #0 {
1496 ; CHECK-LABEL: urem_v1i64:
1497 ; CHECK: // %bb.0:
1498 ; CHECK-NEXT: ptrue p0.d, vl1
1499 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
1500 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
1501 ; CHECK-NEXT: movprfx z2, z0
1502 ; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d
1503 ; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
1504 ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
1505 ; CHECK-NEXT: ret
1506 %res = urem <1 x i64> %op1, %op2
1507 ret <1 x i64> %res
1508 }
1510 ; Vector i64 udiv are not legal for NEON so use SVE when available.
1511 ; FIXME: We should be able to improve the codegen for the 128 bits case here.
1512 define <2 x i64> @urem_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #0 {
1513 ; CHECK-LABEL: urem_v2i64:
1514 ; CHECK: // %bb.0:
1515 ; CHECK-NEXT: ptrue p0.d, vl2
1516 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
1517 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
1518 ; CHECK-NEXT: movprfx z2, z0
1519 ; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d
1520 ; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
1521 ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
1522 ; CHECK-NEXT: ret
1523 %res = urem <2 x i64> %op1, %op2
1524 ret <2 x i64> %res
1525 }
1527 define void @urem_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {
1528 ; CHECK-LABEL: urem_v4i64:
1529 ; CHECK: // %bb.0:
1530 ; CHECK-NEXT: ptrue p0.d, vl4
1531 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
1532 ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
1533 ; CHECK-NEXT: movprfx z2, z0
1534 ; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d
1535 ; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
1536 ; CHECK-NEXT: st1d { z0.d }, p0, [x0]
1537 ; CHECK-NEXT: ret
1538 %op1 = load <4 x i64>, ptr %a
1539 %op2 = load <4 x i64>, ptr %b
1540 %res = urem <4 x i64> %op1, %op2
1541 store <4 x i64> %res, ptr %a
1542 ret void
1543 }
1545 define void @urem_v8i64(ptr %a, ptr %b) #0 {
1546 ; VBITS_GE_128-LABEL: urem_v8i64:
1547 ; VBITS_GE_128: // %bb.0:
1548 ; VBITS_GE_128-NEXT: ptrue p0.d, vl2
1549 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1]
1550 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0]
1551 ; VBITS_GE_128-NEXT: movprfx z4, z1
1552 ; VBITS_GE_128-NEXT: udiv z4.d, p0/m, z4.d, z0.d
1553 ; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32]
1554 ; VBITS_GE_128-NEXT: movprfx z19, z2
1555 ; VBITS_GE_128-NEXT: udiv z19.d, p0/m, z19.d, z3.d
1556 ; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32]
1557 ; VBITS_GE_128-NEXT: movprfx z7, z5
1558 ; VBITS_GE_128-NEXT: udiv z7.d, p0/m, z7.d, z6.d
1559 ; VBITS_GE_128-NEXT: movprfx z18, z16
1560 ; VBITS_GE_128-NEXT: udiv z18.d, p0/m, z18.d, z17.d
1561 ; VBITS_GE_128-NEXT: mls z16.d, p0/m, z18.d, z17.d
1562 ; VBITS_GE_128-NEXT: mls z5.d, p0/m, z7.d, z6.d
1563 ; VBITS_GE_128-NEXT: msb z0.d, p0/m, z4.d, z1.d
1564 ; VBITS_GE_128-NEXT: movprfx z1, z2
1565 ; VBITS_GE_128-NEXT: mls z1.d, p0/m, z19.d, z3.d
1566 ; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32]
1567 ; VBITS_GE_128-NEXT: stp q0, q1, [x0]
1568 ; VBITS_GE_128-NEXT: ret
1569 ;
1570 ; VBITS_GE_256-LABEL: urem_v8i64:
1571 ; VBITS_GE_256: // %bb.0:
1572 ; VBITS_GE_256-NEXT: ptrue p0.d, vl4
1573 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
1574 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
1575 ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
1576 ; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3]
1577 ; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1]
1578 ; VBITS_GE_256-NEXT: movprfx z3, z0
1579 ; VBITS_GE_256-NEXT: udiv z3.d, p0/m, z3.d, z2.d
1580 ; VBITS_GE_256-NEXT: movprfx z5, z1
1581 ; VBITS_GE_256-NEXT: udiv z5.d, p0/m, z5.d, z4.d
1582 ; VBITS_GE_256-NEXT: mls z0.d, p0/m, z3.d, z2.d
1583 ; VBITS_GE_256-NEXT: mls z1.d, p0/m, z5.d, z4.d
1584 ; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
1585 ; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
1586 ; VBITS_GE_256-NEXT: ret
1587 ;
1588 ; VBITS_GE_512-LABEL: urem_v8i64:
1589 ; VBITS_GE_512: // %bb.0:
1590 ; VBITS_GE_512-NEXT: ptrue p0.d, vl8
1591 ; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
1592 ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
1593 ; VBITS_GE_512-NEXT: movprfx z2, z0
1594 ; VBITS_GE_512-NEXT: udiv z2.d, p0/m, z2.d, z1.d
1595 ; VBITS_GE_512-NEXT: mls z0.d, p0/m, z2.d, z1.d
1596 ; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
1597 ; VBITS_GE_512-NEXT: ret
1598 %op1 = load <8 x i64>, ptr %a
1599 %op2 = load <8 x i64>, ptr %b
1600 %res = urem <8 x i64> %op1, %op2
1601 store <8 x i64> %res, ptr %a
1602 ret void
1603 }
1605 define void @urem_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {
1606 ; CHECK-LABEL: urem_v16i64:
1607 ; CHECK: // %bb.0:
1608 ; CHECK-NEXT: ptrue p0.d, vl16
1609 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
1610 ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
1611 ; CHECK-NEXT: movprfx z2, z0
1612 ; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d
1613 ; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
1614 ; CHECK-NEXT: st1d { z0.d }, p0, [x0]
1615 ; CHECK-NEXT: ret
1616 %op1 = load <16 x i64>, ptr %a
1617 %op2 = load <16 x i64>, ptr %b
1618 %res = urem <16 x i64> %op1, %op2
1619 store <16 x i64> %res, ptr %a
1620 ret void
1621 }
1623 define void @urem_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {
1624 ; CHECK-LABEL: urem_v32i64:
1625 ; CHECK: // %bb.0:
1626 ; CHECK-NEXT: ptrue p0.d, vl32
1627 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
1628 ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
1629 ; CHECK-NEXT: movprfx z2, z0
1630 ; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d
1631 ; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
1632 ; CHECK-NEXT: st1d { z0.d }, p0, [x0]
1633 ; CHECK-NEXT: ret
1634 %op1 = load <32 x i64>, ptr %a
1635 %op2 = load <32 x i64>, ptr %b
1636 %res = urem <32 x i64> %op1, %op2
1637 store <32 x i64> %res, ptr %a
1638 ret void
1639 }
1641 attributes #0 = { "target-features"="+sve" }