| blob | 603f667d0c61521d0f95f035a2bc7f8e392ede08 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp,+fp-armv8d16sp,+fp16,+fullfp16 -tail-predication=enabled %s -o - | FileCheck %s
4 define arm_aapcs_vfpcc void @fast_float_mul(float* nocapture %a, float* nocapture readonly %b, float* nocapture readonly %c, i32 %N) {
5 ; CHECK-LABEL: fast_float_mul:
6 ; CHECK: @ %bb.0: @ %entry
7 ; CHECK-NEXT: push {r4, r5, r6, r7, lr}
8 ; CHECK-NEXT: cmp r3, #0
9 ; CHECK-NEXT: beq .LBB0_11
10 ; CHECK-NEXT: @ %bb.1: @ %vector.memcheck
11 ; CHECK-NEXT: add.w r5, r0, r3, lsl #2
12 ; CHECK-NEXT: add.w r4, r2, r3, lsl #2
13 ; CHECK-NEXT: cmp r5, r2
14 ; CHECK-NEXT: cset r12, hi
15 ; CHECK-NEXT: cmp r4, r0
16 ; CHECK-NEXT: cset lr, hi
17 ; CHECK-NEXT: cmp r5, r1
18 ; CHECK-NEXT: add.w r5, r1, r3, lsl #2
19 ; CHECK-NEXT: cset r4, hi
20 ; CHECK-NEXT: cmp r5, r0
21 ; CHECK-NEXT: cset r5, hi
22 ; CHECK-NEXT: tst r5, r4
23 ; CHECK-NEXT: it eq
24 ; CHECK-NEXT: andseq.w r5, lr, r12
25 ; CHECK-NEXT: beq .LBB0_4
26 ; CHECK-NEXT: @ %bb.2: @ %for.body.preheader
27 ; CHECK-NEXT: subs r5, r3, #1
28 ; CHECK-NEXT: and r12, r3, #3
29 ; CHECK-NEXT: cmp r5, #3
30 ; CHECK-NEXT: bhs .LBB0_6
31 ; CHECK-NEXT: @ %bb.3:
32 ; CHECK-NEXT: movs r3, #0
33 ; CHECK-NEXT: b .LBB0_8
34 ; CHECK-NEXT: .LBB0_4: @ %vector.ph
35 ; CHECK-NEXT: dlstp.32 lr, r3
36 ; CHECK-NEXT: .LBB0_5: @ %vector.body
37 ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
38 ; CHECK-NEXT: vldrw.u32 q0, [r1], #16
39 ; CHECK-NEXT: vldrw.u32 q1, [r2], #16
40 ; CHECK-NEXT: vmul.f32 q0, q1, q0
41 ; CHECK-NEXT: vstrw.32 q0, [r0], #16
42 ; CHECK-NEXT: letp lr, .LBB0_5
43 ; CHECK-NEXT: b .LBB0_11
44 ; CHECK-NEXT: .LBB0_6: @ %for.body.preheader.new
45 ; CHECK-NEXT: sub.w lr, r3, r12
46 ; CHECK-NEXT: movs r4, #0
47 ; CHECK-NEXT: movs r3, #0
48 ; CHECK-NEXT: .LBB0_7: @ %for.body
49 ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
50 ; CHECK-NEXT: adds r5, r1, r4
51 ; CHECK-NEXT: adds r6, r2, r4
52 ; CHECK-NEXT: adds r7, r0, r4
53 ; CHECK-NEXT: adds r3, #4
54 ; CHECK-NEXT: vldr s0, [r5]
55 ; CHECK-NEXT: adds r4, #16
56 ; CHECK-NEXT: vldr s2, [r6]
57 ; CHECK-NEXT: cmp lr, r3
58 ; CHECK-NEXT: vmul.f32 s0, s2, s0
59 ; CHECK-NEXT: vstr s0, [r7]
60 ; CHECK-NEXT: vldr s0, [r5, #4]
61 ; CHECK-NEXT: vldr s2, [r6, #4]
62 ; CHECK-NEXT: vmul.f32 s0, s2, s0
63 ; CHECK-NEXT: vstr s0, [r7, #4]
64 ; CHECK-NEXT: vldr s0, [r5, #8]
65 ; CHECK-NEXT: vldr s2, [r6, #8]
66 ; CHECK-NEXT: vmul.f32 s0, s2, s0
67 ; CHECK-NEXT: vstr s0, [r7, #8]
68 ; CHECK-NEXT: vldr s0, [r5, #12]
69 ; CHECK-NEXT: vldr s2, [r6, #12]
70 ; CHECK-NEXT: vmul.f32 s0, s2, s0
71 ; CHECK-NEXT: vstr s0, [r7, #12]
72 ; CHECK-NEXT: bne .LBB0_7
73 ; CHECK-NEXT: .LBB0_8: @ %for.cond.cleanup.loopexit.unr-lcssa
74 ; CHECK-NEXT: wls lr, r12, .LBB0_11
75 ; CHECK-NEXT: @ %bb.9: @ %for.body.epil.preheader
76 ; CHECK-NEXT: add.w r1, r1, r3, lsl #2
77 ; CHECK-NEXT: add.w r2, r2, r3, lsl #2
78 ; CHECK-NEXT: add.w r0, r0, r3, lsl #2
79 ; CHECK-NEXT: .LBB0_10: @ %for.body.epil
80 ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
81 ; CHECK-NEXT: vldmia r1!, {s0}
82 ; CHECK-NEXT: vldmia r2!, {s2}
83 ; CHECK-NEXT: vmul.f32 s0, s2, s0
84 ; CHECK-NEXT: vstmia r0!, {s0}
85 ; CHECK-NEXT: le lr, .LBB0_10
86 ; CHECK-NEXT: .LBB0_11: @ %for.cond.cleanup
87 ; CHECK-NEXT: pop {r4, r5, r6, r7, pc}
88 entry:
89 %cmp8 = icmp eq i32 %N, 0
90 br i1 %cmp8, label %for.cond.cleanup, label %vector.memcheck
92 vector.memcheck: ; preds = %entry
93 %scevgep = getelementptr float, float* %a, i32 %N
94 %scevgep13 = getelementptr float, float* %b, i32 %N
95 %scevgep16 = getelementptr float, float* %c, i32 %N
96 %bound0 = icmp ugt float* %scevgep13, %a
97 %bound1 = icmp ugt float* %scevgep, %b
98 %found.conflict = and i1 %bound0, %bound1
99 %bound018 = icmp ugt float* %scevgep16, %a
100 %bound119 = icmp ugt float* %scevgep, %c
101 %found.conflict20 = and i1 %bound018, %bound119
102 %conflict.rdx = or i1 %found.conflict, %found.conflict20
103 br i1 %conflict.rdx, label %for.body.preheader, label %vector.ph
105 for.body.preheader: ; preds = %vector.memcheck
106 %0 = add i32 %N, -1
107 %xtraiter = and i32 %N, 3
108 %1 = icmp ult i32 %0, 3
109 br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
111 for.body.preheader.new: ; preds = %for.body.preheader
112 %unroll_iter = sub i32 %N, %xtraiter
113 br label %for.body
115 vector.ph: ; preds = %vector.memcheck
116 %n.rnd.up = add i32 %N, 3
117 %n.vec = and i32 %n.rnd.up, -4
118 br label %vector.body
120 vector.body: ; preds = %vector.body, %vector.ph
121 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
122 %2 = getelementptr inbounds float, float* %b, i32 %index
123 %3 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)
124 %4 = bitcast float* %2 to <4 x float>*
125 %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %4, i32 4, <4 x i1> %3, <4 x float> undef)
126 %5 = getelementptr inbounds float, float* %c, i32 %index
127 %6 = bitcast float* %5 to <4 x float>*
128 %wide.masked.load23 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %6, i32 4, <4 x i1> %3, <4 x float> undef)
129 %7 = fmul fast <4 x float> %wide.masked.load23, %wide.masked.load
130 %8 = getelementptr inbounds float, float* %a, i32 %index
131 %9 = bitcast float* %8 to <4 x float>*
132 call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %7, <4 x float>* %9, i32 4, <4 x i1> %3)
133 %index.next = add i32 %index, 4
134 %10 = icmp eq i32 %index.next, %n.vec
135 br i1 %10, label %for.cond.cleanup, label %vector.body
137 for.cond.cleanup.loopexit.unr-lcssa: ; preds = %for.body, %for.body.preheader
138 %i.09.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
139 %lcmp.mod = icmp eq i32 %xtraiter, 0
140 br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
142 for.body.epil: ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
143 %i.09.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.09.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
144 %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
145 %arrayidx.epil = getelementptr inbounds float, float* %b, i32 %i.09.epil
146 %11 = load float, float* %arrayidx.epil, align 4
147 %arrayidx1.epil = getelementptr inbounds float, float* %c, i32 %i.09.epil
148 %12 = load float, float* %arrayidx1.epil, align 4
149 %mul.epil = fmul fast float %12, %11
150 %arrayidx2.epil = getelementptr inbounds float, float* %a, i32 %i.09.epil
151 store float %mul.epil, float* %arrayidx2.epil, align 4
152 %inc.epil = add nuw i32 %i.09.epil, 1
153 %epil.iter.sub = add i32 %epil.iter, -1
154 %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
155 br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
157 for.cond.cleanup: ; preds = %vector.body, %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
158 ret void
160 for.body: ; preds = %for.body, %for.body.preheader.new
161 %i.09 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
162 %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
163 %arrayidx = getelementptr inbounds float, float* %b, i32 %i.09
164 %13 = load float, float* %arrayidx, align 4
165 %arrayidx1 = getelementptr inbounds float, float* %c, i32 %i.09
166 %14 = load float, float* %arrayidx1, align 4
167 %mul = fmul fast float %14, %13
168 %arrayidx2 = getelementptr inbounds float, float* %a, i32 %i.09
169 store float %mul, float* %arrayidx2, align 4
170 %inc = or i32 %i.09, 1
171 %arrayidx.1 = getelementptr inbounds float, float* %b, i32 %inc
172 %15 = load float, float* %arrayidx.1, align 4
173 %arrayidx1.1 = getelementptr inbounds float, float* %c, i32 %inc
174 %16 = load float, float* %arrayidx1.1, align 4
175 %mul.1 = fmul fast float %16, %15
176 %arrayidx2.1 = getelementptr inbounds float, float* %a, i32 %inc
177 store float %mul.1, float* %arrayidx2.1, align 4
178 %inc.1 = or i32 %i.09, 2
179 %arrayidx.2 = getelementptr inbounds float, float* %b, i32 %inc.1
180 %17 = load float, float* %arrayidx.2, align 4
181 %arrayidx1.2 = getelementptr inbounds float, float* %c, i32 %inc.1
182 %18 = load float, float* %arrayidx1.2, align 4
183 %mul.2 = fmul fast float %18, %17
184 %arrayidx2.2 = getelementptr inbounds float, float* %a, i32 %inc.1
185 store float %mul.2, float* %arrayidx2.2, align 4
186 %inc.2 = or i32 %i.09, 3
187 %arrayidx.3 = getelementptr inbounds float, float* %b, i32 %inc.2
188 %19 = load float, float* %arrayidx.3, align 4
189 %arrayidx1.3 = getelementptr inbounds float, float* %c, i32 %inc.2
190 %20 = load float, float* %arrayidx1.3, align 4
191 %mul.3 = fmul fast float %20, %19
192 %arrayidx2.3 = getelementptr inbounds float, float* %a, i32 %inc.2
193 store float %mul.3, float* %arrayidx2.3, align 4
194 %inc.3 = add nuw i32 %i.09, 4
195 %niter.nsub.3 = add i32 %niter, -4
196 %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
197 br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
198 }
200 define arm_aapcs_vfpcc float @fast_float_mac(float* nocapture readonly %b, float* nocapture readonly %c, i32 %N) {
201 ; CHECK-LABEL: fast_float_mac:
202 ; CHECK: @ %bb.0: @ %entry
203 ; CHECK-NEXT: push {r7, lr}
204 ; CHECK-NEXT: cbz r2, .LBB1_4
205 ; CHECK-NEXT: @ %bb.1: @ %vector.ph
206 ; CHECK-NEXT: adds r3, r2, #3
207 ; CHECK-NEXT: mov.w r12, #1
208 ; CHECK-NEXT: bic r3, r3, #3
209 ; CHECK-NEXT: vmov.i32 q0, #0x0
210 ; CHECK-NEXT: subs r3, #4
211 ; CHECK-NEXT: add.w r3, r12, r3, lsr #2
212 ; CHECK-NEXT: dls lr, r3
213 ; CHECK-NEXT: .LBB1_2: @ %vector.body
214 ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
215 ; CHECK-NEXT: vctp.32 r2
216 ; CHECK-NEXT: subs r2, #4
217 ; CHECK-NEXT: vmov q1, q0
218 ; CHECK-NEXT: vpstt
219 ; CHECK-NEXT: vldrwt.u32 q2, [r0], #16
220 ; CHECK-NEXT: vldrwt.u32 q3, [r1], #16
221 ; CHECK-NEXT: vfma.f32 q0, q3, q2
222 ; CHECK-NEXT: le lr, .LBB1_2
223 ; CHECK-NEXT: @ %bb.3: @ %middle.block
224 ; CHECK-NEXT: vpsel q0, q0, q1
225 ; CHECK-NEXT: vmov.f32 s4, s2
226 ; CHECK-NEXT: vmov.f32 s5, s3
227 ; CHECK-NEXT: vadd.f32 q0, q0, q1
228 ; CHECK-NEXT: vmov r0, s1
229 ; CHECK-NEXT: vadd.f32 q0, q0, r0
230 ; CHECK-NEXT: pop {r7, pc}
231 ; CHECK-NEXT: .LBB1_4:
232 ; CHECK-NEXT: vldr s0, .LCPI1_0
233 ; CHECK-NEXT: pop {r7, pc}
234 ; CHECK-NEXT: .p2align 2
235 ; CHECK-NEXT: @ %bb.5:
236 ; CHECK-NEXT: .LCPI1_0:
237 ; CHECK-NEXT: .long 0x00000000 @ float 0
238 entry:
239 %cmp8 = icmp eq i32 %N, 0
240 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
242 vector.ph: ; preds = %entry
243 %n.rnd.up = add i32 %N, 3
244 %n.vec = and i32 %n.rnd.up, -4
245 br label %vector.body
247 vector.body: ; preds = %vector.body, %vector.ph
248 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
249 %vec.phi = phi <4 x float> [ zeroinitializer, %vector.ph ], [ %6, %vector.body ]
250 %0 = getelementptr inbounds float, float* %b, i32 %index
251 %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)
252 %2 = bitcast float* %0 to <4 x float>*
253 %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %2, i32 4, <4 x i1> %1, <4 x float> undef)
254 %3 = getelementptr inbounds float, float* %c, i32 %index
255 %4 = bitcast float* %3 to <4 x float>*
256 %wide.masked.load13 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %4, i32 4, <4 x i1> %1, <4 x float> undef)
257 %5 = fmul fast <4 x float> %wide.masked.load13, %wide.masked.load
258 %6 = fadd fast <4 x float> %5, %vec.phi
259 %index.next = add i32 %index, 4
260 %7 = icmp eq i32 %index.next, %n.vec
261 br i1 %7, label %middle.block, label %vector.body
263 middle.block: ; preds = %vector.body
264 %8 = select <4 x i1> %1, <4 x float> %6, <4 x float> %vec.phi
265 %rdx.shuf = shufflevector <4 x float> %8, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
266 %bin.rdx = fadd fast <4 x float> %8, %rdx.shuf
267 %rdx.shuf14 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
268 %bin.rdx15 = fadd fast <4 x float> %bin.rdx, %rdx.shuf14
269 %9 = extractelement <4 x float> %bin.rdx15, i32 0
270 br label %for.cond.cleanup
272 for.cond.cleanup: ; preds = %middle.block, %entry
273 %a.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %9, %middle.block ]
274 ret float %a.0.lcssa
275 }
277 define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, half* nocapture readonly %c, i32 %N) {
278 ; CHECK-LABEL: fast_float_half_mac:
279 ; CHECK: @ %bb.0: @ %entry
280 ; CHECK-NEXT: push {r4, r5, r7, lr}
281 ; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13}
282 ; CHECK-NEXT: sub sp, #8
283 ; CHECK-NEXT: cmp r2, #0
284 ; CHECK-NEXT: beq.w .LBB2_20
285 ; CHECK-NEXT: @ %bb.1: @ %vector.ph
286 ; CHECK-NEXT: adds r3, r2, #3
287 ; CHECK-NEXT: vmov.i32 q5, #0x0
288 ; CHECK-NEXT: bic r3, r3, #3
289 ; CHECK-NEXT: sub.w r12, r3, #4
290 ; CHECK-NEXT: movs r3, #1
291 ; CHECK-NEXT: add.w lr, r3, r12, lsr #2
292 ; CHECK-NEXT: sub.w r12, r2, #1
293 ; CHECK-NEXT: adr r2, .LCPI2_1
294 ; CHECK-NEXT: mov lr, lr
295 ; CHECK-NEXT: vldrw.u32 q0, [r2]
296 ; CHECK-NEXT: movs r3, #0
297 ; CHECK-NEXT: vdup.32 q1, r12
298 ; CHECK-NEXT: vdup.32 q2, r12
299 ; CHECK-NEXT: b .LBB2_3
300 ; CHECK-NEXT: .LBB2_2: @ %else26
301 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
302 ; CHECK-NEXT: vmul.f16 q5, q6, q5
303 ; CHECK-NEXT: adds r0, #8
304 ; CHECK-NEXT: vcvtt.f32.f16 s23, s21
305 ; CHECK-NEXT: vcvtb.f32.f16 s22, s21
306 ; CHECK-NEXT: vcvtt.f32.f16 s21, s20
307 ; CHECK-NEXT: vcvtb.f32.f16 s20, s20
308 ; CHECK-NEXT: adds r1, #8
309 ; CHECK-NEXT: adds r3, #4
310 ; CHECK-NEXT: vadd.f32 q5, q3, q5
311 ; CHECK-NEXT: subs.w lr, lr, #1
312 ; CHECK-NEXT: bne .LBB2_3
313 ; CHECK-NEXT: b .LBB2_19
314 ; CHECK-NEXT: .LBB2_3: @ %vector.body
315 ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
316 ; CHECK-NEXT: vadd.i32 q4, q0, r3
317 ; CHECK-NEXT: vmov q3, q5
318 ; CHECK-NEXT: vcmp.u32 cs, q1, q4
319 ; CHECK-NEXT: @ implicit-def: $q5
320 ; CHECK-NEXT: vmrs r4, p0
321 ; CHECK-NEXT: and r2, r4, #1
322 ; CHECK-NEXT: rsbs r5, r2, #0
323 ; CHECK-NEXT: movs r2, #0
324 ; CHECK-NEXT: bfi r2, r5, #0, #1
325 ; CHECK-NEXT: ubfx r5, r4, #4, #1
326 ; CHECK-NEXT: rsbs r5, r5, #0
327 ; CHECK-NEXT: bfi r2, r5, #1, #1
328 ; CHECK-NEXT: ubfx r5, r4, #8, #1
329 ; CHECK-NEXT: ubfx r4, r4, #12, #1
330 ; CHECK-NEXT: rsbs r5, r5, #0
331 ; CHECK-NEXT: bfi r2, r5, #2, #1
332 ; CHECK-NEXT: rsbs r4, r4, #0
333 ; CHECK-NEXT: bfi r2, r4, #3, #1
334 ; CHECK-NEXT: lsls r4, r2, #31
335 ; CHECK-NEXT: bne .LBB2_12
336 ; CHECK-NEXT: @ %bb.4: @ %else
337 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
338 ; CHECK-NEXT: lsls r4, r2, #30
339 ; CHECK-NEXT: bmi .LBB2_13
340 ; CHECK-NEXT: .LBB2_5: @ %else7
341 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
342 ; CHECK-NEXT: lsls r4, r2, #29
343 ; CHECK-NEXT: bmi .LBB2_14
344 ; CHECK-NEXT: .LBB2_6: @ %else10
345 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
346 ; CHECK-NEXT: lsls r2, r2, #28
347 ; CHECK-NEXT: bpl .LBB2_8
348 ; CHECK-NEXT: .LBB2_7: @ %cond.load12
349 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
350 ; CHECK-NEXT: vldr.16 s22, [r0, #6]
351 ; CHECK-NEXT: vins.f16 s21, s22
352 ; CHECK-NEXT: .LBB2_8: @ %else13
353 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
354 ; CHECK-NEXT: vcmp.u32 cs, q2, q4
355 ; CHECK-NEXT: @ implicit-def: $q6
356 ; CHECK-NEXT: vmrs r4, p0
357 ; CHECK-NEXT: and r2, r4, #1
358 ; CHECK-NEXT: rsbs r5, r2, #0
359 ; CHECK-NEXT: movs r2, #0
360 ; CHECK-NEXT: bfi r2, r5, #0, #1
361 ; CHECK-NEXT: ubfx r5, r4, #4, #1
362 ; CHECK-NEXT: rsbs r5, r5, #0
363 ; CHECK-NEXT: bfi r2, r5, #1, #1
364 ; CHECK-NEXT: ubfx r5, r4, #8, #1
365 ; CHECK-NEXT: ubfx r4, r4, #12, #1
366 ; CHECK-NEXT: rsbs r5, r5, #0
367 ; CHECK-NEXT: bfi r2, r5, #2, #1
368 ; CHECK-NEXT: rsbs r4, r4, #0
369 ; CHECK-NEXT: bfi r2, r4, #3, #1
370 ; CHECK-NEXT: lsls r4, r2, #31
371 ; CHECK-NEXT: bne .LBB2_15
372 ; CHECK-NEXT: @ %bb.9: @ %else17
373 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
374 ; CHECK-NEXT: lsls r4, r2, #30
375 ; CHECK-NEXT: bmi .LBB2_16
376 ; CHECK-NEXT: .LBB2_10: @ %else20
377 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
378 ; CHECK-NEXT: lsls r4, r2, #29
379 ; CHECK-NEXT: bmi .LBB2_17
380 ; CHECK-NEXT: .LBB2_11: @ %else23
381 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
382 ; CHECK-NEXT: lsls r2, r2, #28
383 ; CHECK-NEXT: bpl .LBB2_2
384 ; CHECK-NEXT: b .LBB2_18
385 ; CHECK-NEXT: .LBB2_12: @ %cond.load
386 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
387 ; CHECK-NEXT: vldr.16 s20, [r0]
388 ; CHECK-NEXT: lsls r4, r2, #30
389 ; CHECK-NEXT: bpl .LBB2_5
390 ; CHECK-NEXT: .LBB2_13: @ %cond.load6
391 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
392 ; CHECK-NEXT: vldr.16 s22, [r0, #2]
393 ; CHECK-NEXT: vins.f16 s20, s22
394 ; CHECK-NEXT: lsls r4, r2, #29
395 ; CHECK-NEXT: bpl .LBB2_6
396 ; CHECK-NEXT: .LBB2_14: @ %cond.load9
397 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
398 ; CHECK-NEXT: vldr.16 s21, [r0, #4]
399 ; CHECK-NEXT: vmovx.f16 s22, s0
400 ; CHECK-NEXT: vins.f16 s21, s22
401 ; CHECK-NEXT: lsls r2, r2, #28
402 ; CHECK-NEXT: bmi .LBB2_7
403 ; CHECK-NEXT: b .LBB2_8
404 ; CHECK-NEXT: .LBB2_15: @ %cond.load16
405 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
406 ; CHECK-NEXT: vldr.16 s24, [r1]
407 ; CHECK-NEXT: lsls r4, r2, #30
408 ; CHECK-NEXT: bpl .LBB2_10
409 ; CHECK-NEXT: .LBB2_16: @ %cond.load19
410 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
411 ; CHECK-NEXT: vldr.16 s26, [r1, #2]
412 ; CHECK-NEXT: vins.f16 s24, s26
413 ; CHECK-NEXT: lsls r4, r2, #29
414 ; CHECK-NEXT: bpl .LBB2_11
415 ; CHECK-NEXT: .LBB2_17: @ %cond.load22
416 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
417 ; CHECK-NEXT: vldr.16 s25, [r1, #4]
418 ; CHECK-NEXT: vmovx.f16 s26, s0
419 ; CHECK-NEXT: vins.f16 s25, s26
420 ; CHECK-NEXT: lsls r2, r2, #28
421 ; CHECK-NEXT: bpl.w .LBB2_2
422 ; CHECK-NEXT: .LBB2_18: @ %cond.load25
423 ; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
424 ; CHECK-NEXT: vldr.16 s26, [r1, #6]
425 ; CHECK-NEXT: vins.f16 s25, s26
426 ; CHECK-NEXT: b .LBB2_2
427 ; CHECK-NEXT: .LBB2_19: @ %middle.block
428 ; CHECK-NEXT: vdup.32 q0, r12
429 ; CHECK-NEXT: vcmp.u32 cs, q0, q4
430 ; CHECK-NEXT: vpsel q0, q5, q3
431 ; CHECK-NEXT: vmov.f32 s4, s2
432 ; CHECK-NEXT: vmov.f32 s5, s3
433 ; CHECK-NEXT: vadd.f32 q0, q0, q1
434 ; CHECK-NEXT: vmov r0, s1
435 ; CHECK-NEXT: vadd.f32 q0, q0, r0
436 ; CHECK-NEXT: b .LBB2_21
437 ; CHECK-NEXT: .LBB2_20:
438 ; CHECK-NEXT: vldr s0, .LCPI2_0
439 ; CHECK-NEXT: .LBB2_21: @ %for.cond.cleanup
440 ; CHECK-NEXT: add sp, #8
441 ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13}
442 ; CHECK-NEXT: pop {r4, r5, r7, pc}
443 ; CHECK-NEXT: .p2align 4
444 ; CHECK-NEXT: @ %bb.22:
445 ; CHECK-NEXT: .LCPI2_1:
446 ; CHECK-NEXT: .long 0 @ 0x0
447 ; CHECK-NEXT: .long 1 @ 0x1
448 ; CHECK-NEXT: .long 2 @ 0x2
449 ; CHECK-NEXT: .long 3 @ 0x3
450 ; CHECK-NEXT: .LCPI2_0:
451 ; CHECK-NEXT: .long 0x00000000 @ float 0
452 entry:
453 %cmp8 = icmp eq i32 %N, 0
454 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
456 vector.ph: ; preds = %entry
457 %n.rnd.up = add i32 %N, 3
458 %n.vec = and i32 %n.rnd.up, -4
459 %trip.count.minus.1 = add i32 %N, -1
460 %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
461 %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
462 br label %vector.body
464 vector.body: ; preds = %vector.body, %vector.ph
465 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
466 %vec.phi = phi <4 x float> [ zeroinitializer, %vector.ph ], [ %7, %vector.body ]
467 %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
468 %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
469 %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
470 %0 = getelementptr inbounds half, half* %b, i32 %index
471 %1 = icmp ule <4 x i32> %induction, %broadcast.splat12
472 %2 = bitcast half* %0 to <4 x half>*
473 %wide.masked.load = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* %2, i32 2, <4 x i1> %1, <4 x half> undef)
474 %3 = getelementptr inbounds half, half* %c, i32 %index
475 %4 = bitcast half* %3 to <4 x half>*
476 %wide.masked.load13 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* %4, i32 2, <4 x i1> %1, <4 x half> undef)
477 %5 = fmul fast <4 x half> %wide.masked.load13, %wide.masked.load
478 %6 = fpext <4 x half> %5 to <4 x float>
479 %7 = fadd fast <4 x float> %vec.phi, %6
480 %index.next = add i32 %index, 4
481 %8 = icmp eq i32 %index.next, %n.vec
482 br i1 %8, label %middle.block, label %vector.body
484 middle.block: ; preds = %vector.body
485 %9 = select <4 x i1> %1, <4 x float> %7, <4 x float> %vec.phi
486 %rdx.shuf = shufflevector <4 x float> %9, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
487 %bin.rdx = fadd fast <4 x float> %9, %rdx.shuf
488 %rdx.shuf14 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
489 %bin.rdx15 = fadd fast <4 x float> %bin.rdx, %rdx.shuf14
490 %10 = extractelement <4 x float> %bin.rdx15, i32 0
491 br label %for.cond.cleanup
493 for.cond.cleanup: ; preds = %middle.block, %entry
494 %a.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %10, %middle.block ]
495 ret float %a.0.lcssa
496 }
498 ; Function Attrs: argmemonly nounwind readonly willreturn
499 declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32 immarg, <4 x i1>, <4 x float>)
501 ; Function Attrs: argmemonly nounwind willreturn
502 declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32 immarg, <4 x i1>)
504 ; Function Attrs: argmemonly nounwind readonly willreturn
505 declare <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>*, i32 immarg, <4 x i1>, <4 x half>)
507 declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)