1 ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S \
2 ; RUN: -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s --check-prefix=CHECK-VF4IC1
3 ; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S \
4 ; RUN: -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s --check-prefix=CHECK-VF4IC4
6 target triple = "aarch64-linux-gnu"
8 define i32 @select_const_i32_from_icmp(ptr nocapture readonly %v, i64 %n) #0 {
9 ; CHECK-VF4IC1-LABEL: @select_const_i32_from_icmp
10 ; CHECK-VF4IC1: vector.body:
11 ; CHECK-VF4IC1: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]
12 ; CHECK-VF4IC1: [[VEC_LOAD:%.*]] = load <vscale x 4 x i32>
13 ; CHECK-VF4IC1-NEXT: [[VEC_ICMP:%.*]] = icmp eq <vscale x 4 x i32> [[VEC_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
14 ; CHECK-VF4IC1-NEXT: [[VEC_SEL]] = select <vscale x 4 x i1> [[VEC_ICMP]], <vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 7, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
15 ; CHECK-VF4IC1: middle.block:
16 ; CHECK-VF4IC1-NEXT: [[FIN_ICMP:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
17 ; CHECK-VF4IC1-NEXT: [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[FIN_ICMP]])
18 ; CHECK-VF4IC1-NEXT: {{.*}} = select i1 [[OR_RDX]], i32 7, i32 3
20 ; CHECK-VF4IC4-LABEL: @select_const_i32_from_icmp
21 ; CHECK-VF4IC4: vector.body:
22 ; CHECK-VF4IC4: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), %vector.ph ], [ [[VEC_SEL1:%.*]], %vector.body ]
23 ; CHECK-VF4IC4: [[VEC_PHI2:%.*]] = phi <vscale x 4 x i32> [ shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), %vector.ph ], [ [[VEC_SEL2:%.*]], %vector.body ]
24 ; CHECK-VF4IC4: [[VEC_PHI3:%.*]] = phi <vscale x 4 x i32> [ shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), %vector.ph ], [ [[VEC_SEL3:%.*]], %vector.body ]
25 ; CHECK-VF4IC4: [[VEC_PHI4:%.*]] = phi <vscale x 4 x i32> [ shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), %vector.ph ], [ [[VEC_SEL4:%.*]], %vector.body ]
26 ; CHECK-VF4IC4: [[VEC_ICMP1:%.*]] = icmp eq <vscale x 4 x i32> {{.*}}, shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
27 ; CHECK-VF4IC4-NEXT: [[VEC_ICMP2:%.*]] = icmp eq <vscale x 4 x i32> {{.*}}, shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
28 ; CHECK-VF4IC4-NEXT: [[VEC_ICMP3:%.*]] = icmp eq <vscale x 4 x i32> {{.*}}, shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
29 ; CHECK-VF4IC4-NEXT: [[VEC_ICMP4:%.*]] = icmp eq <vscale x 4 x i32> {{.*}}, shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
30 ; CHECK-VF4IC4-NEXT: [[VEC_SEL1]] = select <vscale x 4 x i1> [[VEC_ICMP1]], <vscale x 4 x i32> [[VEC_PHI1]], <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 7, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
31 ; CHECK-VF4IC4-NEXT: [[VEC_SEL2]] = select <vscale x 4 x i1> [[VEC_ICMP2]], <vscale x 4 x i32> [[VEC_PHI2]], <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 7, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
32 ; CHECK-VF4IC4-NEXT: [[VEC_SEL3]] = select <vscale x 4 x i1> [[VEC_ICMP3]], <vscale x 4 x i32> [[VEC_PHI3]], <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 7, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
33 ; CHECK-VF4IC4-NEXT: [[VEC_SEL4]] = select <vscale x 4 x i1> [[VEC_ICMP4]], <vscale x 4 x i32> [[VEC_PHI4]], <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 7, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
34 ; CHECK-VF4IC4: middle.block:
35 ; CHECK-VF4IC4-NEXT: [[VEC_ICMP5:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL1]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
36 ; CHECK-VF4IC4-NEXT: [[VEC_SEL5:%.*]] = select <vscale x 4 x i1> [[VEC_ICMP5]], <vscale x 4 x i32> [[VEC_SEL1]], <vscale x 4 x i32> [[VEC_SEL2]]
37 ; CHECK-VF4IC4-NEXT: [[VEC_ICMP6:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL5]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
38 ; CHECK-VF4IC4-NEXT: [[VEC_SEL6:%.*]] = select <vscale x 4 x i1> [[VEC_ICMP6]], <vscale x 4 x i32> [[VEC_SEL5]], <vscale x 4 x i32> [[VEC_SEL3]]
39 ; CHECK-VF4IC4-NEXT: [[VEC_ICMP7:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL6]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
40 ; CHECK-VF4IC4-NEXT: [[VEC_SEL7:%.*]] = select <vscale x 4 x i1> [[VEC_ICMP7]], <vscale x 4 x i32> [[VEC_SEL6]], <vscale x 4 x i32> [[VEC_SEL4]]
41 ; CHECK-VF4IC4-NEXT: [[FIN_ICMP:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL7]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
42 ; CHECK-VF4IC4-NEXT: [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[FIN_ICMP]])
43 ; CHECK-VF4IC4-NEXT: {{.*}} = select i1 [[OR_RDX]], i32 7, i32 3
47 for.body: ; preds = %entry, %for.body
48 %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
49 %1 = phi i32 [ 3, %entry ], [ %5, %for.body ]
50 %2 = getelementptr inbounds i32, ptr %v, i64 %0
51 %3 = load i32, ptr %2, align 4
52 %4 = icmp eq i32 %3, 3
53 %5 = select i1 %4, i32 %1, i32 7
54 %6 = add nuw nsw i64 %0, 1
55 %7 = icmp eq i64 %6, %n
56 br i1 %7, label %exit, label %for.body, !llvm.loop !0
58 exit: ; preds = %for.body
62 define i32 @select_i32_from_icmp(ptr nocapture readonly %v, i32 %a, i32 %b, i64 %n) #0 {
63 ; CHECK-VF4IC1-LABEL: @select_i32_from_icmp
64 ; CHECK-VF4IC1: vector.ph:
65 ; CHECK-VF4IC1: [[TMP1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 %a, i64 0
66 ; CHECK-VF4IC1-NEXT: [[SPLAT_OF_A:%.*]] = shufflevector <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
67 ; CHECK-VF4IC1-NEXT: [[TMP2:%.*]] = insertelement <vscale x 4 x i32> poison, i32 %b, i64 0
68 ; CHECK-VF4IC1-NEXT: [[SPLAT_OF_B:%.*]] = shufflevector <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
69 ; CHECK-VF4IC1: vector.body:
70 ; CHECK-VF4IC1: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[SPLAT_OF_A]], %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]
71 ; CHECK-VF4IC1: [[VEC_LOAD:%.*]] = load <vscale x 4 x i32>
72 ; CHECK-VF4IC1-NEXT: [[VEC_ICMP:%.*]] = icmp eq <vscale x 4 x i32> [[VEC_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
73 ; CHECK-VF4IC1-NEXT: [[VEC_SEL]] = select <vscale x 4 x i1> [[VEC_ICMP]], <vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[SPLAT_OF_B]]
74 ; CHECK-VF4IC1: middle.block:
75 ; CHECK-VF4IC1-NEXT: [[FIN_INS:%.*]] = insertelement <vscale x 4 x i32> poison, i32 %a, i64 0
76 ; CHECK-VF4IC1-NEXT: [[FIN_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[FIN_INS]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
77 ; CHECK-VF4IC1-NEXT: [[FIN_CMP:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL]], [[FIN_SPLAT]]
78 ; CHECK-VF4IC1-NEXT: [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[FIN_CMP]])
79 ; CHECK-VF4IC1-NEXT: {{.*}} = select i1 [[OR_RDX]], i32 %b, i32 %a
81 ; CHECK-VF4IC4-LABEL: @select_i32_from_icmp
82 ; CHECK-VF4IC4: vector.body:
86 for.body: ; preds = %entry, %for.body
87 %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
88 %1 = phi i32 [ %a, %entry ], [ %5, %for.body ]
89 %2 = getelementptr inbounds i32, ptr %v, i64 %0
90 %3 = load i32, ptr %2, align 4
91 %4 = icmp eq i32 %3, 3
92 %5 = select i1 %4, i32 %1, i32 %b
93 %6 = add nuw nsw i64 %0, 1
94 %7 = icmp eq i64 %6, %n
95 br i1 %7, label %exit, label %for.body, !llvm.loop !0
97 exit: ; preds = %for.body
101 define i32 @select_const_i32_from_fcmp(ptr nocapture readonly %v, i64 %n) #0 {
102 ; CHECK-VF4IC1-LABEL: @select_const_i32_from_fcmp
103 ; CHECK-VF4IC1: vector.body:
104 ; CHECK-VF4IC1: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 2, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]
105 ; CHECK-VF4IC1: [[VEC_LOAD:%.*]] = load <vscale x 4 x float>
106 ; CHECK-VF4IC1-NEXT: [[VEC_ICMP:%.*]] = fcmp fast ueq <vscale x 4 x float> [[VEC_LOAD]], shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float 3.000000e+00, i64 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer)
107 ; CHECK-VF4IC1-NEXT: [[VEC_SEL]] = select <vscale x 4 x i1> [[VEC_ICMP]], <vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
108 ; CHECK-VF4IC1: middle.block:
109 ; CHECK-VF4IC1-NEXT: [[FIN_ICMP:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 2, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
110 ; CHECK-VF4IC1-NEXT: [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[FIN_ICMP]])
111 ; CHECK-VF4IC1-NEXT: {{.*}} = select i1 [[OR_RDX]], i32 1, i32 2
113 ; CHECK-VF4IC4-LABEL: @select_const_i32_from_fcmp
114 ; CHECK-VF4IC4: vector.body:
118 for.body: ; preds = %entry, %for.body
119 %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
120 %1 = phi i32 [ 2, %entry ], [ %5, %for.body ]
121 %2 = getelementptr inbounds float, ptr %v, i64 %0
122 %3 = load float, ptr %2, align 4
123 %4 = fcmp fast ueq float %3, 3.0
124 %5 = select i1 %4, i32 %1, i32 1
125 %6 = add nuw nsw i64 %0, 1
126 %7 = icmp eq i64 %6, %n
127 br i1 %7, label %exit, label %for.body, !llvm.loop !0
129 exit: ; preds = %for.body
133 define float @select_const_f32_from_icmp(ptr nocapture readonly %v, i64 %n) #0 {
134 ; CHECK-VF4IC1-LABEL: @select_const_f32_from_icmp
135 ; CHECK-VF4IC1-NOT: vector.body
136 ; CHECK-VF4IC4-LABEL: @select_const_f32_from_icmp
137 ; CHECK-VF4IC4-NOT: vector.body
141 for.body: ; preds = %entry, %for.body
142 %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]
143 %1 = phi fast float [ 3.0, %entry ], [ %5, %for.body ]
144 %2 = getelementptr inbounds i32, ptr %v, i64 %0
145 %3 = load i32, ptr %2, align 4
146 %4 = icmp eq i32 %3, 3
147 %5 = select fast i1 %4, float %1, float 7.0
148 %6 = add nuw nsw i64 %0, 1
149 %7 = icmp eq i64 %6, %n
150 br i1 %7, label %exit, label %for.body, !llvm.loop !0
152 exit: ; preds = %for.body
156 define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i64 %n) #0 {
157 ; CHECK-VF4IC1-LABEL: @pred_select_const_i32_from_icmp
158 ; CHECK-VF4IC1: vector.body:
159 ; CHECK-VF4IC1: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]
160 ; CHECK-VF4IC1: [[VEC_LOAD:%.*]] = load <vscale x 4 x i32>
161 ; CHECK-VF4IC1: [[MASK:%.*]] = icmp sgt <vscale x 4 x i32> [[VEC_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 35, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
162 ; CHECK-VF4IC1: [[MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr {{%.*}}, i32 4, <vscale x 4 x i1> [[MASK]], <vscale x 4 x i32> poison)
163 ; CHECK-VF4IC1-NEXT: [[VEC_ICMP:%.*]] = icmp eq <vscale x 4 x i32> [[MASKED_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 2, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
164 ; CHECK-VF4IC1-NEXT: [[VEC_SEL_TMP:%.*]] = select <vscale x 4 x i1> [[VEC_ICMP]], <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> [[VEC_PHI]]
165 ; CHECK-VF4IC1: [[VEC_SEL:%.*]] = select <vscale x 4 x i1> [[MASK]], <vscale x 4 x i32> [[VEC_SEL_TMP]], <vscale x 4 x i32> [[VEC_PHI]]
166 ; CHECK-VF4IC1: middle.block:
167 ; CHECK-VF4IC1-NEXT: [[FIN_ICMP:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_SEL]], zeroinitializer
168 ; CHECK-VF4IC1-NEXT: [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[FIN_ICMP]])
169 ; CHECK-VF4IC1-NEXT: {{.*}} = select i1 [[OR_RDX]], i32 1, i32 0
171 ; CHECK-VF4IC4-LABEL: @pred_select_const_i32_from_icmp
172 ; CHECK-VF4IC4: vector.body:
176 for.body: ; preds = %entry, %for.inc
177 %i.013 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
178 %r.012 = phi i32 [ %r.1, %for.inc ], [ 0, %entry ]
179 %arrayidx = getelementptr inbounds i32, ptr %src1, i64 %i.013
180 %0 = load i32, ptr %arrayidx, align 4
181 %cmp1 = icmp sgt i32 %0, 35
182 br i1 %cmp1, label %if.then, label %for.inc
184 if.then: ; preds = %for.body
185 %arrayidx2 = getelementptr inbounds i32, ptr %src2, i64 %i.013
186 %1 = load i32, ptr %arrayidx2, align 4
187 %cmp3 = icmp eq i32 %1, 2
188 %spec.select = select i1 %cmp3, i32 1, i32 %r.012
191 for.inc: ; preds = %if.then, %for.body
192 %r.1 = phi i32 [ %r.012, %for.body ], [ %spec.select, %if.then ]
193 %inc = add nuw nsw i64 %i.013, 1
194 %exitcond.not = icmp eq i64 %inc, %n
195 br i1 %exitcond.not, label %for.end.loopexit, label %for.body, !llvm.loop !0
197 for.end.loopexit: ; preds = %for.inc
198 %r.1.lcssa = phi i32 [ %r.1, %for.inc ]
203 attributes #0 = { "target-features"="+sve" }
205 !0 = distinct !{!0, !1}
206 !1 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}