1 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2 ; RUN: opt -passes=slp-vectorizer -mtriple=arm64-apple-ios -S %s | FileCheck %s
4 ; Test case where not vectorizing is more profitable because multiple
5 ; fmul/{fadd,fsub} pairs can be lowered to fma instructions.
6 define void @slp_not_profitable_with_fast_fmf(ptr %A, ptr %B) {
7 ; CHECK-LABEL: @slp_not_profitable_with_fast_fmf(
8 ; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 1
9 ; CHECK-NEXT: [[A_0:%.*]] = load float, ptr [[A:%.*]], align 4
10 ; CHECK-NEXT: [[B_1:%.*]] = load float, ptr [[GEP_B_1]], align 4
11 ; CHECK-NEXT: [[MUL_0:%.*]] = fmul fast float [[B_1]], [[A_0]]
12 ; CHECK-NEXT: [[B_0:%.*]] = load float, ptr [[B]], align 4
13 ; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
14 ; CHECK-NEXT: [[B_2:%.*]] = load float, ptr [[GEP_B_2]], align 4
15 ; CHECK-NEXT: [[MUL_1:%.*]] = fmul fast float [[B_2]], [[B_0]]
16 ; CHECK-NEXT: [[SUB:%.*]] = fsub fast float [[MUL_0]], [[MUL_1]]
17 ; CHECK-NEXT: [[MUL_2:%.*]] = fmul fast float [[B_0]], [[B_1]]
18 ; CHECK-NEXT: [[MUL_3:%.*]] = fmul fast float [[B_2]], [[A_0]]
19 ; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[MUL_3]], [[MUL_2]]
20 ; CHECK-NEXT: store float [[SUB]], ptr [[A]], align 4
21 ; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
22 ; CHECK-NEXT: store float [[ADD]], ptr [[GEP_A_1]], align 4
23 ; CHECK-NEXT: store float [[B_2]], ptr [[B]], align 4
24 ; CHECK-NEXT: ret void
26 %gep.B.1 = getelementptr inbounds float, ptr %B, i64 1
27 %A.0 = load float, ptr %A, align 4
28 %B.1 = load float, ptr %gep.B.1, align 4
29 %mul.0 = fmul fast float %B.1, %A.0
30 %B.0 = load float, ptr %B, align 4
31 %gep.B.2 = getelementptr inbounds float, ptr %B, i64 2
32 %B.2 = load float, ptr %gep.B.2, align 4
33 %mul.1 = fmul fast float %B.2, %B.0
34 %sub = fsub fast float %mul.0, %mul.1
35 %mul.2 = fmul fast float %B.0, %B.1
36 %mul.3 = fmul fast float %B.2, %A.0
37 %add = fadd fast float %mul.3, %mul.2
38 store float %sub, ptr %A, align 4
39 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 1
40 store float %add, ptr %gep.A.1, align 4
41 store float %B.2, ptr %B, align 4
45 define void @slp_not_profitable_with_reassoc_fmf(ptr %A, ptr %B) {
46 ; CHECK-LABEL: @slp_not_profitable_with_reassoc_fmf(
47 ; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 1
48 ; CHECK-NEXT: [[A_0:%.*]] = load float, ptr [[A:%.*]], align 4
49 ; CHECK-NEXT: [[B_1:%.*]] = load float, ptr [[GEP_B_1]], align 4
50 ; CHECK-NEXT: [[MUL_0:%.*]] = fmul reassoc float [[B_1]], [[A_0]]
51 ; CHECK-NEXT: [[B_0:%.*]] = load float, ptr [[B]], align 4
52 ; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
53 ; CHECK-NEXT: [[B_2:%.*]] = load float, ptr [[GEP_B_2]], align 4
54 ; CHECK-NEXT: [[MUL_1:%.*]] = fmul float [[B_2]], [[B_0]]
55 ; CHECK-NEXT: [[SUB:%.*]] = fsub reassoc float [[MUL_0]], [[MUL_1]]
56 ; CHECK-NEXT: [[MUL_2:%.*]] = fmul float [[B_0]], [[B_1]]
57 ; CHECK-NEXT: [[MUL_3:%.*]] = fmul reassoc float [[B_2]], [[A_0]]
58 ; CHECK-NEXT: [[ADD:%.*]] = fadd reassoc float [[MUL_3]], [[MUL_2]]
59 ; CHECK-NEXT: store float [[SUB]], ptr [[A]], align 4
60 ; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
61 ; CHECK-NEXT: store float [[ADD]], ptr [[GEP_A_1]], align 4
62 ; CHECK-NEXT: store float [[B_2]], ptr [[B]], align 4
63 ; CHECK-NEXT: ret void
65 %gep.B.1 = getelementptr inbounds float, ptr %B, i64 1
66 %A.0 = load float, ptr %A, align 4
67 %B.1 = load float, ptr %gep.B.1, align 4
68 %mul.0 = fmul reassoc float %B.1, %A.0
69 %B.0 = load float, ptr %B, align 4
70 %gep.B.2 = getelementptr inbounds float, ptr %B, i64 2
71 %B.2 = load float, ptr %gep.B.2, align 4
72 %mul.1 = fmul float %B.2, %B.0
73 %sub = fsub reassoc float %mul.0, %mul.1
74 %mul.2 = fmul float %B.0, %B.1
75 %mul.3 = fmul reassoc float %B.2, %A.0
76 %add = fadd reassoc float %mul.3, %mul.2
77 store float %sub, ptr %A, align 4
78 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 1
79 store float %add, ptr %gep.A.1, align 4
80 store float %B.2, ptr %B, align 4
84 ; FMA cannot be used due to missing fast-math flags, so SLP should kick in.
85 define void @slp_profitable_missing_fmf_on_fadd_fsub(ptr %A, ptr %B) {
86 ; CHECK-LABEL: @slp_profitable_missing_fmf_on_fadd_fsub(
87 ; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 1
88 ; CHECK-NEXT: [[A_0:%.*]] = load float, ptr [[A:%.*]], align 4
89 ; CHECK-NEXT: [[B_1:%.*]] = load float, ptr [[GEP_B_1]], align 4
90 ; CHECK-NEXT: [[MUL_0:%.*]] = fmul fast float [[B_1]], [[A_0]]
91 ; CHECK-NEXT: [[B_0:%.*]] = load float, ptr [[B]], align 4
92 ; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
93 ; CHECK-NEXT: [[B_2:%.*]] = load float, ptr [[GEP_B_2]], align 4
94 ; CHECK-NEXT: [[MUL_1:%.*]] = fmul fast float [[B_2]], [[B_0]]
95 ; CHECK-NEXT: [[SUB:%.*]] = fsub float [[MUL_0]], [[MUL_1]]
96 ; CHECK-NEXT: [[MUL_2:%.*]] = fmul fast float [[B_0]], [[B_1]]
97 ; CHECK-NEXT: [[MUL_3:%.*]] = fmul fast float [[B_2]], [[A_0]]
98 ; CHECK-NEXT: [[ADD:%.*]] = fadd float [[MUL_3]], [[MUL_2]]
99 ; CHECK-NEXT: store float [[SUB]], ptr [[A]], align 4
100 ; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
101 ; CHECK-NEXT: store float [[ADD]], ptr [[GEP_A_1]], align 4
102 ; CHECK-NEXT: store float [[B_2]], ptr [[B]], align 4
103 ; CHECK-NEXT: ret void
105 %gep.B.1 = getelementptr inbounds float, ptr %B, i64 1
106 %A.0 = load float, ptr %A, align 4
107 %B.1 = load float, ptr %gep.B.1, align 4
108 %mul.0 = fmul fast float %B.1, %A.0
109 %B.0 = load float, ptr %B, align 4
110 %gep.B.2 = getelementptr inbounds float, ptr %B, i64 2
111 %B.2 = load float, ptr %gep.B.2, align 4
112 %mul.1 = fmul fast float %B.2, %B.0
113 %sub = fsub float %mul.0, %mul.1
114 %mul.2 = fmul fast float %B.0, %B.1
115 %mul.3 = fmul fast float %B.2, %A.0
116 %add = fadd float %mul.3, %mul.2
117 store float %sub, ptr %A, align 4
118 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 1
119 store float %add, ptr %gep.A.1, align 4
120 store float %B.2, ptr %B, align 4
124 ; FMA cannot be used due to missing fast-math flags, so SLP should kick in.
125 define void @slp_profitable_missing_fmf_on_fmul_fadd_fsub(ptr %A, ptr %B) {
126 ; CHECK-LABEL: @slp_profitable_missing_fmf_on_fmul_fadd_fsub(
127 ; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 1
128 ; CHECK-NEXT: [[A_0:%.*]] = load float, ptr [[A:%.*]], align 4
129 ; CHECK-NEXT: [[B_1:%.*]] = load float, ptr [[GEP_B_1]], align 4
130 ; CHECK-NEXT: [[MUL_0:%.*]] = fmul float [[B_1]], [[A_0]]
131 ; CHECK-NEXT: [[B_0:%.*]] = load float, ptr [[B]], align 4
132 ; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
133 ; CHECK-NEXT: [[B_2:%.*]] = load float, ptr [[GEP_B_2]], align 4
134 ; CHECK-NEXT: [[MUL_1:%.*]] = fmul float [[B_2]], [[B_0]]
135 ; CHECK-NEXT: [[SUB:%.*]] = fsub float [[MUL_0]], [[MUL_1]]
136 ; CHECK-NEXT: [[MUL_2:%.*]] = fmul float [[B_0]], [[B_1]]
137 ; CHECK-NEXT: [[MUL_3:%.*]] = fmul float [[B_2]], [[A_0]]
138 ; CHECK-NEXT: [[ADD:%.*]] = fadd float [[MUL_3]], [[MUL_2]]
139 ; CHECK-NEXT: store float [[SUB]], ptr [[A]], align 4
140 ; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
141 ; CHECK-NEXT: store float [[ADD]], ptr [[GEP_A_1]], align 4
142 ; CHECK-NEXT: store float [[B_2]], ptr [[B]], align 4
143 ; CHECK-NEXT: ret void
145 %gep.B.1 = getelementptr inbounds float, ptr %B, i64 1
146 %A.0 = load float, ptr %A, align 4
147 %B.1 = load float, ptr %gep.B.1, align 4
148 %mul.0 = fmul float %B.1, %A.0
149 %B.0 = load float, ptr %B, align 4
150 %gep.B.2 = getelementptr inbounds float, ptr %B, i64 2
151 %B.2 = load float, ptr %gep.B.2, align 4
152 %mul.1 = fmul float %B.2, %B.0
153 %sub = fsub float %mul.0, %mul.1
154 %mul.2 = fmul float %B.0, %B.1
155 %mul.3 = fmul float %B.2, %A.0
156 %add = fadd float %mul.3, %mul.2
157 store float %sub, ptr %A, align 4
158 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 1
159 store float %add, ptr %gep.A.1, align 4
160 store float %B.2, ptr %B, align 4
164 ; FMA cannot be used due to missing fast-math flags, so SLP should kick in.
165 define void @slp_profitable_missing_fmf_nnans_only(ptr %A, ptr %B) {
166 ; CHECK-LABEL: @slp_profitable_missing_fmf_nnans_only(
167 ; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 1
168 ; CHECK-NEXT: [[A_0:%.*]] = load float, ptr [[A:%.*]], align 4
169 ; CHECK-NEXT: [[B_1:%.*]] = load float, ptr [[GEP_B_1]], align 4
170 ; CHECK-NEXT: [[MUL_0:%.*]] = fmul nnan float [[B_1]], [[A_0]]
171 ; CHECK-NEXT: [[B_0:%.*]] = load float, ptr [[B]], align 4
172 ; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
173 ; CHECK-NEXT: [[B_2:%.*]] = load float, ptr [[GEP_B_2]], align 4
174 ; CHECK-NEXT: [[MUL_1:%.*]] = fmul nnan float [[B_2]], [[B_0]]
175 ; CHECK-NEXT: [[SUB:%.*]] = fsub nnan float [[MUL_0]], [[MUL_1]]
176 ; CHECK-NEXT: [[MUL_2:%.*]] = fmul nnan float [[B_0]], [[B_1]]
177 ; CHECK-NEXT: [[MUL_3:%.*]] = fmul nnan float [[B_2]], [[A_0]]
178 ; CHECK-NEXT: [[ADD:%.*]] = fadd nnan float [[MUL_3]], [[MUL_2]]
179 ; CHECK-NEXT: store float [[SUB]], ptr [[A]], align 4
180 ; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
181 ; CHECK-NEXT: store float [[ADD]], ptr [[GEP_A_1]], align 4
182 ; CHECK-NEXT: store float [[B_2]], ptr [[B]], align 4
183 ; CHECK-NEXT: ret void
185 %gep.B.1 = getelementptr inbounds float, ptr %B, i64 1
186 %A.0 = load float, ptr %A, align 4
187 %B.1 = load float, ptr %gep.B.1, align 4
188 %mul.0 = fmul nnan float %B.1, %A.0
189 %B.0 = load float, ptr %B, align 4
190 %gep.B.2 = getelementptr inbounds float, ptr %B, i64 2
191 %B.2 = load float, ptr %gep.B.2, align 4
192 %mul.1 = fmul nnan float %B.2, %B.0
193 %sub = fsub nnan float %mul.0, %mul.1
194 %mul.2 = fmul nnan float %B.0, %B.1
195 %mul.3 = fmul nnan float %B.2, %A.0
196 %add = fadd nnan float %mul.3, %mul.2
197 store float %sub, ptr %A, align 4
198 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 1
199 store float %add, ptr %gep.A.1, align 4
200 store float %B.2, ptr %B, align 4
204 ; Test case where not vectorizing is more profitable because multiple
205 ; fmul/{fadd,fsub} pairs can be lowered to fma instructions.
206 define float @slp_not_profitable_in_loop(float %x, ptr %A) {
207 ; CHECK-LABEL: @slp_not_profitable_in_loop(
209 ; CHECK-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 2
210 ; CHECK-NEXT: [[L_1:%.*]] = load float, ptr [[GEP_A_2]], align 4
211 ; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A]], align 4
212 ; CHECK-NEXT: [[L_3:%.*]] = load float, ptr [[A]], align 4
213 ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> <float poison, float 3.000000e+00>, float [[X:%.*]], i32 0
214 ; CHECK-NEXT: br label [[LOOP:%.*]]
216 ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
217 ; CHECK-NEXT: [[RED:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[RED_NEXT:%.*]], [[LOOP]] ]
218 ; CHECK-NEXT: [[TMP2:%.*]] = fmul fast <2 x float> [[TMP1]], [[TMP0]]
219 ; CHECK-NEXT: [[MUL12:%.*]] = fmul fast float 3.000000e+00, [[L_1]]
220 ; CHECK-NEXT: [[MUL16:%.*]] = fmul fast float 3.000000e+00, [[L_3]]
221 ; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
222 ; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[MUL12]], [[TMP3]]
223 ; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
224 ; CHECK-NEXT: [[ADD13:%.*]] = fadd fast float [[ADD]], [[TMP4]]
225 ; CHECK-NEXT: [[RED_NEXT]] = fadd fast float [[ADD13]], [[MUL16]]
226 ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
227 ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], 10
228 ; CHECK-NEXT: br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]]
230 ; CHECK-NEXT: ret float [[RED_NEXT]]
233 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 1
234 %l.0 = load float, ptr %gep.A.1, align 4
235 %gep.A.2 = getelementptr inbounds float, ptr %A, i64 2
236 %l.1 = load float, ptr %gep.A.2, align 4
237 %l.2 = load float, ptr %A, align 4
238 %l.3 = load float, ptr %A, align 4
242 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
243 %red = phi float [ 0.000000e+00, %entry ], [ %red.next, %loop ]
244 %mul11 = fmul fast float 3.000000e+00, %l.0
245 %mul12 = fmul fast float 3.000000e+00, %l.1
246 %mul14 = fmul fast float %x, %l.2
247 %mul16 = fmul fast float 3.000000e+00, %l.3
248 %add = fadd fast float %mul12, %mul11
249 %add13 = fadd fast float %add, %mul14
250 %red.next = fadd fast float %add13, %mul16
251 %iv.next = add nuw nsw i64 %iv, 1
252 %cmp = icmp eq i64 %iv, 10
253 br i1 %cmp, label %exit, label %loop
259 define void @slp_profitable(ptr %A, ptr %B, float %0) {
260 ; CHECK-LABEL: @slp_profitable(
262 ; CHECK-NEXT: [[SUB_I1096:%.*]] = fsub fast float 1.000000e+00, [[TMP0:%.*]]
263 ; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[A:%.*]], align 4
264 ; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
265 ; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer
266 ; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <2 x float> [[TMP1]], [[TMP3]]
267 ; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
268 ; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[SUB_I1096]], i32 0
269 ; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <2 x i32> zeroinitializer
270 ; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <2 x float> [[TMP1]], [[TMP7]]
271 ; CHECK-NEXT: [[TMP9:%.*]] = fadd fast <2 x float> [[TMP5]], [[TMP8]]
272 ; CHECK-NEXT: [[TMP10:%.*]] = fsub fast <2 x float> [[TMP5]], [[TMP8]]
273 ; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
274 ; CHECK-NEXT: store <2 x float> [[TMP11]], ptr [[B:%.*]], align 4
275 ; CHECK-NEXT: ret void
278 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 1
279 %sub.i1096 = fsub fast float 1.000000e+00, %0
280 %1 = load float, ptr %A, align 4
281 %mul.i1100 = fmul fast float %1, %sub.i1096
282 %2 = load float, ptr %gep.A.1, align 4
283 %mul7.i1101 = fmul fast float %2, %0
284 %add.i1102 = fadd fast float %mul7.i1101, %mul.i1100
285 %mul14.i = fmul fast float %1, %0
286 %3 = fmul fast float %2, %sub.i1096
287 %add15.i = fsub fast float %mul14.i, %3
288 store float %add.i1102, ptr %B, align 4
289 %gep.B.1 = getelementptr inbounds float, ptr %B, i64 1
290 store float %add15.i, ptr %gep.B.1, align 4