1 ; RUN: opt -S -force-vector-width=2 -force-vector-interleave=1 -loop-vectorize -verify-loop-info -simplifycfg -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck %s
2 ; RUN: opt -S -force-vector-width=1 -force-vector-interleave=2 -loop-vectorize -verify-loop-info < %s | FileCheck %s --check-prefix=UNROLL-NO-VF
4 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
6 ; Test predication of non-void instructions, specifically (i) that these
7 ; instructions permit vectorization and (ii) the creation of an insertelement
8 ; and a Phi node. We check the full 2-element sequence for all predicate instructions.
9 define void @test(i32* nocapture %asd, i32* nocapture %aud,
10 i32* nocapture %asr, i32* nocapture %aur) {
14 for.cond.cleanup: ; preds = %if.end
19 ; CHECK: %[[SDEE:[a-zA-Z0-9]+]] = extractelement <2 x i1> %{{.*}}, i32 0
20 ; CHECK: br i1 %[[SDEE]], label %[[CSD:[a-zA-Z0-9.]+]], label %[[ESD:[a-zA-Z0-9.]+]]
22 ; CHECK: %[[SDA0:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
23 ; CHECK: %[[SDA1:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
24 ; CHECK: %[[SD0:[a-zA-Z0-9]+]] = sdiv i32 %[[SDA0]], %[[SDA1]]
25 ; CHECK: %[[SD1:[a-zA-Z0-9]+]] = insertelement <2 x i32> poison, i32 %[[SD0]], i32 0
26 ; CHECK: %[[UDA0:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
27 ; CHECK: %[[UDA1:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
28 ; CHECK: %[[UD0:[a-zA-Z0-9]+]] = udiv i32 %[[UDA0]], %[[UDA1]]
29 ; CHECK: %[[UD1:[a-zA-Z0-9]+]] = insertelement <2 x i32> poison, i32 %[[UD0]], i32 0
30 ; CHECK: %[[SRA0:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
31 ; CHECK: %[[SRA1:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
32 ; CHECK: %[[SR0:[a-zA-Z0-9]+]] = srem i32 %[[SRA0]], %[[SRA1]]
33 ; CHECK: %[[SR1:[a-zA-Z0-9]+]] = insertelement <2 x i32> poison, i32 %[[SR0]], i32 0
34 ; CHECK: %[[URA0:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
35 ; CHECK: %[[URA1:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
36 ; CHECK: %[[UR0:[a-zA-Z0-9]+]] = urem i32 %[[URA0]], %[[URA1]]
37 ; CHECK: %[[UR1:[a-zA-Z0-9]+]] = insertelement <2 x i32> poison, i32 %[[UR0]], i32 0
38 ; CHECK: br label %[[ESD]]
40 ; CHECK: [[SDR:%[a-zA-Z0-9]+]] = phi <2 x i32> [ poison, %vector.body ], [ %[[SD1]], %[[CSD]] ]
41 ; CHECK: [[UDR:%.+]] = phi <2 x i32> [ poison, %{{.*}} ], [ %[[UD1]], %[[CSD]] ]
42 ; CHECK: [[SRR:%.+]] = phi <2 x i32> [ poison, %{{.*}} ], [ %[[SR1]], %[[CSD]] ]
43 ; CHECK: [[URR:%.+]] = phi <2 x i32> [ poison, %{{.*}} ], [ %[[UR1]], %[[CSD]] ]
44 ; CHECK: %[[SDEEH:[a-zA-Z0-9]+]] = extractelement <2 x i1> %{{.*}}, i32 1
45 ; CHECK: br i1 %[[SDEEH]], label %[[CSDH:[a-zA-Z0-9.]+]], label %[[ESDH:[a-zA-Z0-9.]+]]
47 ; CHECK: %[[SD1_A0H:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 1
48 ; CHECK: %[[SD1_A1H:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 1
49 ; CHECK: %[[SD1_0H:[a-zA-Z0-9]+]] = sdiv i32 %[[SD1_A0H]], %[[SD1_A1H]]
50 ; CHECK: %[[SD1_1H:[a-zA-Z0-9]+]] = insertelement <2 x i32> [[SDR]], i32 %[[SD1_0H]], i32 1
51 ; CHECK: %[[UD1_A0:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 1
52 ; CHECK: %[[UD1_A1:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 1
53 ; CHECK: %[[UD1_0:[a-zA-Z0-9]+]] = udiv i32 %[[UD1_A0]], %[[UD1_A1]]
54 ; CHECK: %[[UD1_1:[a-zA-Z0-9]+]] = insertelement <2 x i32> [[UDR]], i32 %[[UD1_0]], i32 1
55 ; CHECK: %[[SR1_A0:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 1
56 ; CHECK: %[[SR1_A1:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 1
57 ; CHECK: %[[SR1_0:[a-zA-Z0-9]+]] = srem i32 %[[SR1_A0]], %[[SR1_A1]]
58 ; CHECK: %[[SR1_1:[a-zA-Z0-9]+]] = insertelement <2 x i32> [[SRR]], i32 %[[SR1_0]], i32 1
59 ; CHECK: %[[UR1_A0:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 1
60 ; CHECK: %[[UR1_A1:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 1
61 ; CHECK: %[[UR1_0:[a-zA-Z0-9]+]] = urem i32 %[[UR1_A0]], %[[UR1_A1]]
62 ; CHECK: %[[UR1_1:[a-zA-Z0-9]+]] = insertelement <2 x i32> [[URR]], i32 %[[UR1_0]], i32 1
63 ; CHECK: br label %[[ESDH]]
65 ; CHECK: [[SDR1:%[a-zA-Z0-9]+]] = phi <2 x i32> [ [[SDR]], %[[ESD]] ], [ %[[SD1_1H]], %[[CSDH]] ]
66 ; CHECK: [[UDR1:%.+]] = phi <2 x i32> [ [[UDR]], %{{.*}} ], [ %[[UD1_1]], %[[CSDH]] ]
67 ; CHECK: [[SRR1:%.+]] = phi <2 x i32> [ [[SRR]], %{{.*}} ], [ %[[SR1_1]], %[[CSDH]] ]
68 ; CHECK: [[URR1:%.+]] = phi <2 x i32> [ [[URR]], %{{.*}} ], [ %[[UR1_1]], %[[CSDH]] ]
70 for.body: ; preds = %if.end, %entry
71 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end ]
72 %isd = getelementptr inbounds i32, i32* %asd, i64 %indvars.iv
73 %iud = getelementptr inbounds i32, i32* %aud, i64 %indvars.iv
74 %isr = getelementptr inbounds i32, i32* %asr, i64 %indvars.iv
75 %iur = getelementptr inbounds i32, i32* %aur, i64 %indvars.iv
76 %lsd = load i32, i32* %isd, align 4
77 %lud = load i32, i32* %iud, align 4
78 %lsr = load i32, i32* %isr, align 4
79 %lur = load i32, i32* %iur, align 4
80 %psd = add nsw i32 %lsd, 23
81 %pud = add nsw i32 %lud, 24
82 %psr = add nsw i32 %lsr, 25
83 %pur = add nsw i32 %lur, 26
84 %cmp1 = icmp slt i32 %lsd, 100
85 br i1 %cmp1, label %if.then, label %if.end
87 if.then: ; preds = %for.body
88 %rsd = sdiv i32 %psd, %lsd
89 %rud = udiv i32 %pud, %lud
90 %rsr = srem i32 %psr, %lsr
91 %rur = urem i32 %pur, %lur
94 if.end: ; preds = %if.then, %for.body
95 %ysd.0 = phi i32 [ %rsd, %if.then ], [ %psd, %for.body ]
96 %yud.0 = phi i32 [ %rud, %if.then ], [ %pud, %for.body ]
97 %ysr.0 = phi i32 [ %rsr, %if.then ], [ %psr, %for.body ]
98 %yur.0 = phi i32 [ %rur, %if.then ], [ %pur, %for.body ]
99 store i32 %ysd.0, i32* %isd, align 4
100 store i32 %yud.0, i32* %iud, align 4
101 store i32 %ysr.0, i32* %isr, align 4
102 store i32 %yur.0, i32* %iur, align 4
103 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
104 %exitcond = icmp eq i64 %indvars.iv.next, 128
105 br i1 %exitcond, label %for.cond.cleanup, label %for.body
108 define void @test_scalar2scalar(i32* nocapture %asd, i32* nocapture %bsd) {
112 for.cond.cleanup: ; preds = %if.end
115 ; CHECK-LABEL: test_scalar2scalar
116 ; CHECK: vector.body:
117 ; CHECK: br i1 %{{.*}}, label %[[THEN:[a-zA-Z0-9.]+]], label %[[FI:[a-zA-Z0-9.]+]]
119 ; CHECK: %[[PD:[a-zA-Z0-9]+]] = sdiv i32 %{{.*}}, %{{.*}}
120 ; CHECK: br label %[[FI]]
122 ; CHECK: %{{.*}} = phi i32 [ poison, %vector.body ], [ %[[PD]], %[[THEN]] ]
124 for.body: ; preds = %if.end, %entry
125 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end ]
126 %isd = getelementptr inbounds i32, i32* %asd, i64 %indvars.iv
127 %lsd = load i32, i32* %isd, align 4
128 %isd.b = getelementptr inbounds i32, i32* %bsd, i64 %indvars.iv
129 %lsd.b = load i32, i32* %isd.b, align 4
130 %psd = add nsw i32 %lsd, 23
131 %cmp1 = icmp slt i32 %lsd, 100
132 br i1 %cmp1, label %if.then, label %if.end
134 if.then: ; preds = %for.body
135 %sd1 = sdiv i32 %psd, %lsd
136 %rsd = sdiv i32 %lsd.b, %sd1
139 if.end: ; preds = %if.then, %for.body
140 %ysd.0 = phi i32 [ %rsd, %if.then ], [ %psd, %for.body ]
141 store i32 %ysd.0, i32* %isd, align 4
142 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
143 %exitcond = icmp eq i64 %indvars.iv.next, 128
144 br i1 %exitcond, label %for.cond.cleanup, label %for.body
147 define void @pr30172(i32* nocapture %asd, i32* nocapture %bsd) {
151 for.cond.cleanup: ; preds = %if.end
154 ; CHECK-LABEL: pr30172
155 ; CHECK: vector.body:
156 ; CHECK: %[[CMP1:.+]] = icmp slt <2 x i32> %[[VAL:.+]], <i32 100, i32 100>
157 ; CHECK: %[[CMP2:.+]] = icmp sge <2 x i32> %[[VAL]], <i32 200, i32 200>
158 ; CHECK: %[[NOT:.+]] = xor <2 x i1> %[[CMP1]], <i1 true, i1 true>
159 ; CHECK: %[[AND:.+]] = select <2 x i1> %[[NOT]], <2 x i1> %[[CMP2]], <2 x i1> zeroinitializer
160 ; CHECK: %[[OR:.+]] = or <2 x i1> %[[AND]], %[[CMP1]]
161 ; CHECK: %[[EXTRACT:.+]] = extractelement <2 x i1> %[[OR]], i32 0
162 ; CHECK: br i1 %[[EXTRACT]], label %[[THEN:[a-zA-Z0-9.]+]], label %[[FI:[a-zA-Z0-9.]+]]
164 ; CHECK: %[[PD:[a-zA-Z0-9]+]] = sdiv i32 %{{.*}}, %{{.*}}
165 ; CHECK: br label %[[FI]]
167 ; CHECK: %{{.*}} = phi i32 [ poison, %vector.body ], [ %[[PD]], %[[THEN]] ]
170 for.body: ; preds = %if.end, %entry
171 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end ]
172 %isd = getelementptr inbounds i32, i32* %asd, i64 %indvars.iv
173 %lsd = load i32, i32* %isd, align 4
174 %isd.b = getelementptr inbounds i32, i32* %bsd, i64 %indvars.iv
175 %lsd.b = load i32, i32* %isd.b, align 4
176 %psd = add nsw i32 %lsd, 23
177 %cmp1 = icmp slt i32 %lsd, 100
178 br i1 %cmp1, label %if.then, label %check
180 check: ; preds = %for.body
181 %cmp2 = icmp sge i32 %lsd, 200
182 br i1 %cmp2, label %if.then, label %if.end
184 if.then: ; preds = %check, %for.body
185 %sd1 = sdiv i32 %psd, %lsd
186 %rsd = sdiv i32 %lsd.b, %sd1
189 if.end: ; preds = %if.then, %check
190 %ysd.0 = phi i32 [ %rsd, %if.then ], [ %psd, %check ]
191 store i32 %ysd.0, i32* %isd, align 4
192 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
193 %exitcond = icmp eq i64 %indvars.iv.next, 128
194 br i1 %exitcond, label %for.cond.cleanup, label %for.body
198 define i32 @predicated_udiv_scalarized_operand(i32* %a, i1 %c, i32 %x, i64 %n) {
202 ; CHECK-LABEL: predicated_udiv_scalarized_operand
203 ; CHECK: vector.body:
204 ; CHECK: %wide.load = load <2 x i32>, <2 x i32>* {{.*}}, align 4
205 ; CHECK: br i1 {{.*}}, label %[[IF0:.+]], label %[[CONT0:.+]]
207 ; CHECK: %[[T00:.+]] = extractelement <2 x i32> %wide.load, i32 0
208 ; CHECK: %[[T01:.+]] = add nsw i32 %[[T00]], %x
209 ; CHECK: %[[T02:.+]] = extractelement <2 x i32> %wide.load, i32 0
210 ; CHECK: %[[T03:.+]] = udiv i32 %[[T02]], %[[T01]]
211 ; CHECK: %[[T04:.+]] = insertelement <2 x i32> poison, i32 %[[T03]], i32 0
212 ; CHECK: br label %[[CONT0]]
214 ; CHECK: %[[T05:.+]] = phi <2 x i32> [ poison, %vector.body ], [ %[[T04]], %[[IF0]] ]
215 ; CHECK: br i1 {{.*}}, label %[[IF1:.+]], label %[[CONT1:.+]]
217 ; CHECK: %[[T06:.+]] = extractelement <2 x i32> %wide.load, i32 1
218 ; CHECK: %[[T07:.+]] = add nsw i32 %[[T06]], %x
219 ; CHECK: %[[T08:.+]] = extractelement <2 x i32> %wide.load, i32 1
220 ; CHECK: %[[T09:.+]] = udiv i32 %[[T08]], %[[T07]]
221 ; CHECK: %[[T10:.+]] = insertelement <2 x i32> %[[T05]], i32 %[[T09]], i32 1
222 ; CHECK: br label %[[CONT1]]
224 ; CHECK: phi <2 x i32> [ %[[T05]], %[[CONT0]] ], [ %[[T10]], %[[IF1]] ]
225 ; CHECK: br i1 {{.*}}, label %middle.block, label %vector.body
227 ; Test predicating an instruction that feeds a vectorizable use, when unrolled
228 ; but not vectorized. Derived from pr34248 reproducer.
230 ; UNROLL-NO-VF-LABEL: predicated_udiv_scalarized_operand
231 ; UNROLL-NO-VF: vector.body:
232 ; UNROLL-NO-VF: %[[LOAD0:.+]] = load i32, i32*
233 ; UNROLL-NO-VF: %[[LOAD1:.+]] = load i32, i32*
234 ; UNROLL-NO-VF: br i1 {{.*}}, label %[[IF0:.+]], label %[[CONT0:.+]]
235 ; UNROLL-NO-VF: [[IF0]]:
236 ; UNROLL-NO-VF: %[[ADD0:.+]] = add nsw i32 %[[LOAD0]], %x
237 ; UNROLL-NO-VF: %[[DIV0:.+]] = udiv i32 %[[LOAD0]], %[[ADD0]]
238 ; UNROLL-NO-VF: br label %[[CONT0]]
239 ; UNROLL-NO-VF: [[CONT0]]:
240 ; UNROLL-NO-VF: phi i32 [ poison, %vector.body ], [ %[[DIV0]], %[[IF0]] ]
241 ; UNROLL-NO-VF: br i1 {{.*}}, label %[[IF1:.+]], label %[[CONT1:.+]]
242 ; UNROLL-NO-VF: [[IF1]]:
243 ; UNROLL-NO-VF: %[[ADD1:.+]] = add nsw i32 %[[LOAD1]], %x
244 ; UNROLL-NO-VF: %[[DIV1:.+]] = udiv i32 %[[LOAD1]], %[[ADD1]]
245 ; UNROLL-NO-VF: br label %[[CONT1]]
246 ; UNROLL-NO-VF: [[CONT1]]:
247 ; UNROLL-NO-VF: phi i32 [ poison, %[[CONT0]] ], [ %[[DIV1]], %[[IF1]] ]
248 ; UNROLL-NO-VF: br i1 {{.*}}, label %middle.block, label %vector.body
251 %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
252 %r = phi i32 [ 0, %entry ], [ %tmp6, %for.inc ]
253 %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i
254 %tmp2 = load i32, i32* %tmp0, align 4
255 br i1 %c, label %if.then, label %for.inc
258 %tmp3 = add nsw i32 %tmp2, %x
259 %tmp4 = udiv i32 %tmp2, %tmp3
263 %tmp5 = phi i32 [ %tmp2, %for.body ], [ %tmp4, %if.then]
264 %tmp6 = add i32 %r, %tmp5
265 %i.next = add nuw nsw i64 %i, 1
266 %cond = icmp slt i64 %i.next, %n
267 br i1 %cond, label %for.body, label %for.end
270 %tmp7 = phi i32 [ %tmp6, %for.inc ]