[llvm-project.git] / llvm / test / Transforms / LoopVectorize / AArch64 / sve-interleaved-accesses.ll
| blob | fa8e35f4697e917972f06cf3219168acc2594634 |
1 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2 ; RUN: opt -mtriple=aarch64-none-linux-gnu -S -passes=loop-vectorize,instcombine -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true -mattr=+sve -scalable-vectorization=on -runtime-memory-check-threshold=24 < %s | FileCheck %s
4 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
6 ; Check vectorization on an interleaved load group of factor 2 and an interleaved
7 ; store group of factor 2.
9 ; int AB[1024];
10 ; int CD[1024];
11 ; void test_array_load2_store2(int C, int D) {
12 ; for (int i = 0; i < 1024; i+=2) {
13 ; int A = AB[i];
14 ; int B = AB[i+1];
15 ; CD[i] = A + C;
16 ; CD[i+1] = B * D;
17 ; }
18 ; }
21 @AB = common global [1024 x i32] zeroinitializer, align 4
22 @CD = common global [1024 x i32] zeroinitializer, align 4
24 define void @test_array_load2_store2(i32 %C, i32 %D) #1 {
25 ; CHECK-LABEL: @test_array_load2_store2(
26 ; CHECK-NEXT: entry:
27 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
28 ; CHECK: vector.ph:
29 ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
30 ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2
31 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[C:%.*]], i64 0
32 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
33 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[D:%.*]], i64 0
34 ; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
35 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
36 ; CHECK: vector.body:
37 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
38 ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
39 ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 [[OFFSET_IDX]]
40 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP0]], align 4
41 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
42 ; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
43 ; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
44 ; CHECK-NEXT: [[TMP3:%.*]] = or disjoint i64 [[OFFSET_IDX]], 1
45 ; CHECK-NEXT: [[TMP4:%.*]] = add nsw <vscale x 4 x i32> [[TMP1]], [[BROADCAST_SPLAT]]
46 ; CHECK-NEXT: [[TMP5:%.*]] = mul nsw <vscale x 4 x i32> [[TMP2]], [[BROADCAST_SPLAT2]]
47 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 [[TMP3]]
48 ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 -1
49 ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.experimental.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]])
50 ; CHECK-NEXT: store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
51 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
52 ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
53 ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
54 ; CHECK: middle.block:
55 ; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
56 ; CHECK: scalar.ph:
57 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
58 ; CHECK: for.body:
59 ; CHECK-NEXT: br i1 poison, label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]]
60 ; CHECK: for.end:
61 ; CHECK-NEXT: ret void
62 ;
63 entry:
64 br label %for.body
66 for.body: ; preds = %for.body, %entry
67 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
68 %arrayidx0 = getelementptr inbounds [1024 x i32], [1024 x i32]* @AB, i64 0, i64 %indvars.iv
69 %load1 = load i32, i32* %arrayidx0, align 4
70 %or = or disjoint i64 %indvars.iv, 1
71 %arrayidx1 = getelementptr inbounds [1024 x i32], [1024 x i32]* @AB, i64 0, i64 %or
72 %load2 = load i32, i32* %arrayidx1, align 4
73 %add = add nsw i32 %load1, %C
74 %mul = mul nsw i32 %load2, %D
75 %arrayidx2 = getelementptr inbounds [1024 x i32], [1024 x i32]* @CD, i64 0, i64 %indvars.iv
76 store i32 %add, i32* %arrayidx2, align 4
77 %arrayidx3 = getelementptr inbounds [1024 x i32], [1024 x i32]* @CD, i64 0, i64 %or
78 store i32 %mul, i32* %arrayidx3, align 4
79 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
80 %cmp = icmp slt i64 %indvars.iv.next, 1024
81 br i1 %cmp, label %for.body, label %for.end
83 for.end: ; preds = %for.body
84 ret void
85 }
87 ; Check vectorization on an interleaved load group of factor 2 with narrower types and an interleaved
88 ; store group of factor 2.
90 ; short AB[1024];
91 ; int CD[1024];
92 ; void test_array_load2_store2(int C, int D) {
93 ; for (int i = 0; i < 1024; i+=2) {
94 ; short A = AB[i];
95 ; short B = AB[i+1];
96 ; CD[i] = A + C;
97 ; CD[i+1] = B * D;
98 ; }
99 ; }
102 @AB_i16 = common global [1024 x i16] zeroinitializer, align 4
104 define void @test_array_load2_i16_store2(i32 %C, i32 %D) #1 {
105 ; CHECK-LABEL: @test_array_load2_i16_store2(
106 ; CHECK-NEXT: entry:
107 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
108 ; CHECK: vector.ph:
109 ; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
110 ; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2
111 ; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
112 ; CHECK-NEXT: [[TMP1:%.*]] = shl <vscale x 4 x i64> [[TMP0]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
113 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
114 ; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 3
115 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
116 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
117 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[C:%.*]], i64 0
118 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
119 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[D:%.*]], i64 0
120 ; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
121 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
122 ; CHECK: vector.body:
123 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
124 ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
125 ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [1024 x i16], ptr @AB_i16, i64 0, <vscale x 4 x i64> [[VEC_IND]]
126 ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> [[TMP4]], i32 2, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i16> poison)
127 ; CHECK-NEXT: [[TMP5:%.*]] = or disjoint <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
128 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [1024 x i16], ptr @AB_i16, i64 0, <vscale x 4 x i64> [[TMP5]]
129 ; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> [[TMP6]], i32 2, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i16> poison)
130 ; CHECK-NEXT: [[TMP7:%.*]] = sext <vscale x 4 x i16> [[WIDE_MASKED_GATHER]] to <vscale x 4 x i32>
131 ; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[BROADCAST_SPLAT]], [[TMP7]]
132 ; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i16> [[WIDE_MASKED_GATHER1]] to <vscale x 4 x i32>
133 ; CHECK-NEXT: [[TMP10:%.*]] = mul nsw <vscale x 4 x i32> [[BROADCAST_SPLAT3]], [[TMP9]]
134 ; CHECK-NEXT: [[TMP11:%.*]] = extractelement <vscale x 4 x i64> [[TMP5]], i64 0
135 ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 [[TMP11]]
136 ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i64 -1
137 ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.experimental.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> [[TMP10]])
138 ; CHECK-NEXT: store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP13]], align 4
139 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]]
140 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
141 ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
142 ; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
143 ; CHECK: middle.block:
144 ; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
145 ; CHECK: scalar.ph:
146 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
147 ; CHECK: for.body:
148 ; CHECK-NEXT: br i1 poison, label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP5:![0-9]+]]
149 ; CHECK: for.end:
150 ; CHECK-NEXT: ret void
151 ;
152 entry:
153 br label %for.body
155 for.body: ; preds = %entry, %for.body
156 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
157 %arrayidx = getelementptr inbounds [1024 x i16], [1024 x i16]* @AB_i16, i64 0, i64 %indvars.iv
158 %0 = load i16, i16* %arrayidx, align 2
159 %1 = or disjoint i64 %indvars.iv, 1
160 %arrayidx2 = getelementptr inbounds [1024 x i16], [1024 x i16]* @AB_i16, i64 0, i64 %1
161 %2 = load i16, i16* %arrayidx2, align 2
162 %conv = sext i16 %0 to i32
163 %add3 = add nsw i32 %conv, %C
164 %arrayidx5 = getelementptr inbounds [1024 x i32], [1024 x i32]* @CD, i64 0, i64 %indvars.iv
165 store i32 %add3, i32* %arrayidx5, align 4
166 %conv6 = sext i16 %2 to i32
167 %mul = mul nsw i32 %conv6, %D
168 %arrayidx9 = getelementptr inbounds [1024 x i32], [1024 x i32]* @CD, i64 0, i64 %1
169 store i32 %mul, i32* %arrayidx9, align 4
170 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
171 %cmp = icmp ult i64 %indvars.iv, 1022
172 br i1 %cmp, label %for.body, label %for.end
174 for.end: ; preds = %for.body
175 ret void
176 }
178 ; Check vectorization on an interleaved load group of factor 2 and an interleaved
179 ; store group of factor 2 with narrower types.
181 ; int AB[1024];
182 ; short CD[1024];
183 ; void test_array_load2_store2(int C, int D) {
184 ; for (int i = 0; i < 1024; i+=2) {
185 ; short A = AB[i];
186 ; short B = AB[i+1];
187 ; CD[i] = A + C;
188 ; CD[i+1] = B * D;
189 ; }
190 ; }
193 @CD_i16 = dso_local local_unnamed_addr global [1024 x i16] zeroinitializer, align 2
195 define void @test_array_load2_store2_i16(i32 noundef %C, i32 noundef %D) #1 {
196 ; CHECK-LABEL: @test_array_load2_store2_i16(
197 ; CHECK-NEXT: entry:
198 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
199 ; CHECK: vector.ph:
200 ; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
201 ; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2
202 ; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
203 ; CHECK-NEXT: [[TMP1:%.*]] = shl <vscale x 4 x i64> [[TMP0]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
204 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
205 ; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 3
206 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
207 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
208 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[C:%.*]], i64 0
209 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
210 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[D:%.*]], i64 0
211 ; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
212 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
213 ; CHECK: vector.body:
214 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
215 ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
216 ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
217 ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 [[OFFSET_IDX]]
218 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP4]], align 4
219 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
220 ; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
221 ; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
222 ; CHECK-NEXT: [[TMP7:%.*]] = or disjoint <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
223 ; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[TMP5]], [[BROADCAST_SPLAT]]
224 ; CHECK-NEXT: [[TMP9:%.*]] = trunc <vscale x 4 x i32> [[TMP8]] to <vscale x 4 x i16>
225 ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [1024 x i16], ptr @CD_i16, i64 0, <vscale x 4 x i64> [[VEC_IND]]
226 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> [[TMP9]], <vscale x 4 x ptr> [[TMP10]], i32 2, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
227 ; CHECK-NEXT: [[TMP11:%.*]] = mul nsw <vscale x 4 x i32> [[TMP6]], [[BROADCAST_SPLAT2]]
228 ; CHECK-NEXT: [[TMP12:%.*]] = trunc <vscale x 4 x i32> [[TMP11]] to <vscale x 4 x i16>
229 ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [1024 x i16], ptr @CD_i16, i64 0, <vscale x 4 x i64> [[TMP7]]
230 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> [[TMP12]], <vscale x 4 x ptr> [[TMP13]], i32 2, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
231 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]]
232 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
233 ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
234 ; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
235 ; CHECK: middle.block:
236 ; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
237 ; CHECK: scalar.ph:
238 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
239 ; CHECK: for.body:
240 ; CHECK-NEXT: br i1 poison, label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP7:![0-9]+]]
241 ; CHECK: for.end:
242 ; CHECK-NEXT: ret void
243 ;
244 entry:
245 br label %for.body
247 for.body: ; preds = %entry, %for.body
248 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
249 %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @AB, i64 0, i64 %indvars.iv
250 %0 = load i32, i32* %arrayidx, align 4
251 %1 = or disjoint i64 %indvars.iv, 1
252 %arrayidx2 = getelementptr inbounds [1024 x i32], [1024 x i32]* @AB, i64 0, i64 %1
253 %2 = load i32, i32* %arrayidx2, align 4
254 %add3 = add nsw i32 %0, %C
255 %conv = trunc i32 %add3 to i16
256 %arrayidx5 = getelementptr inbounds [1024 x i16], [1024 x i16]* @CD_i16, i64 0, i64 %indvars.iv
257 store i16 %conv, i16* %arrayidx5, align 2
258 %mul = mul nsw i32 %2, %D
259 %conv6 = trunc i32 %mul to i16
260 %arrayidx9 = getelementptr inbounds [1024 x i16], [1024 x i16]* @CD_i16, i64 0, i64 %1
261 store i16 %conv6, i16* %arrayidx9, align 2
262 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
263 %cmp = icmp ult i64 %indvars.iv, 1022
264 br i1 %cmp, label %for.body, label %for.end
266 for.end: ; preds = %for.body
267 ret void
268 }
270 ; Check vectorization on an interleaved load group of factor 6.
271 ; There is no dedicated ldN/stN so use gather instead
273 %struct.ST6 = type { i32, i32, i32, i32, i32, i32 }
275 define i32 @test_struct_load6(%struct.ST6* %S) #1 {
276 ; CHECK-LABEL: @test_struct_load6(
277 ; CHECK-NEXT: entry:
278 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
279 ; CHECK: vector.ph:
280 ; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
281 ; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 2
282 ; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
283 ; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
284 ; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
285 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
286 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
287 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
288 ; CHECK: vector.body:
289 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
290 ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP0]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
291 ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
292 ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_ST6:%.*]], ptr [[S:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
293 ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP3]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> poison)
294 ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_ST6]], ptr [[S]], <vscale x 4 x i64> [[VEC_IND]], i32 1
295 ; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP4]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> poison)
296 ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_ST6]], ptr [[S]], <vscale x 4 x i64> [[VEC_IND]], i32 2
297 ; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP5]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> poison)
298 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_ST6]], ptr [[S]], <vscale x 4 x i64> [[VEC_IND]], i32 3
299 ; CHECK-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP6]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> poison)
300 ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_ST6]], ptr [[S]], <vscale x 4 x i64> [[VEC_IND]], i32 4
301 ; CHECK-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP7]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> poison)
302 ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_ST6]], ptr [[S]], <vscale x 4 x i64> [[VEC_IND]], i32 5
303 ; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP8]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> poison)
304 ; CHECK-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], [[VEC_PHI]]
305 ; CHECK-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[WIDE_MASKED_GATHER2]]
306 ; CHECK-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER1]], [[WIDE_MASKED_GATHER3]]
307 ; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER4]]
308 ; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[TMP12]], [[WIDE_MASKED_GATHER5]]
309 ; CHECK-NEXT: [[TMP14]] = sub <vscale x 4 x i32> [[TMP10]], [[TMP13]]
310 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]]
311 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
312 ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
313 ; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
314 ; CHECK: middle.block:
315 ; CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP14]])
316 ; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
317 ; CHECK: scalar.ph:
318 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
319 ; CHECK: for.body:
320 ; CHECK-NEXT: br i1 poison, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
321 ; CHECK: for.cond.cleanup:
322 ; CHECK-NEXT: [[SUB14_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
323 ; CHECK-NEXT: ret i32 [[SUB14_LCSSA]]
324 ;
325 entry:
326 br label %for.body
328 for.body: ; preds = %entry, %for.body
329 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
330 %r.041 = phi i32 [ 0, %entry ], [ %sub14, %for.body ]
331 %x = getelementptr inbounds %struct.ST6, %struct.ST6* %S, i64 %indvars.iv, i32 0
332 %0 = load i32, i32* %x, align 4
333 %y = getelementptr inbounds %struct.ST6, %struct.ST6* %S, i64 %indvars.iv, i32 1
334 %1 = load i32, i32* %y, align 4
335 %z = getelementptr inbounds %struct.ST6, %struct.ST6* %S, i64 %indvars.iv, i32 2
336 %2 = load i32, i32* %z, align 4
337 %w = getelementptr inbounds %struct.ST6, %struct.ST6* %S, i64 %indvars.iv, i32 3
338 %3 = load i32, i32* %w, align 4
339 %a = getelementptr inbounds %struct.ST6, %struct.ST6* %S, i64 %indvars.iv, i32 4
340 %4 = load i32, i32* %a, align 4
341 %b = getelementptr inbounds %struct.ST6, %struct.ST6* %S, i64 %indvars.iv, i32 5
342 %5 = load i32, i32* %b, align 4
343 %.neg36 = add i32 %0, %r.041
344 %6 = add i32 %.neg36, %2
345 %7 = add i32 %1, %3
346 %8 = add i32 %7, %4
347 %9 = add i32 %8, %5
348 %sub14 = sub i32 %6, %9
349 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
350 %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
351 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
353 for.cond.cleanup: ; preds = %for.body
354 %sub14.lcssa = phi i32 [ %sub14, %for.body ]
355 ret i32 %sub14.lcssa
356 }
359 ; Check vectorization on a reverse interleaved load group of factor 2 and
360 ; a reverse interleaved store group of factor 2.
362 ; struct ST2 {
363 ; int x;
364 ; int y;
365 ; };
366 ;
367 ; void test_reversed_load2_store2(struct ST2 *A, struct ST2 *B) {
368 ; for (int i = 1023; i >= 0; i--) {
369 ; int a = A[i].x + i; // interleaved load of index 0
370 ; int b = A[i].y - i; // interleaved load of index 1
371 ; B[i].x = a; // interleaved store of index 0
372 ; B[i].y = b; // interleaved store of index 1
373 ; }
374 ; }
377 %struct.ST2 = type { i32, i32 }
379 define void @test_reversed_load2_store2(%struct.ST2* noalias nocapture readonly %A, %struct.ST2* noalias nocapture %B) #1 {
380 ; CHECK-LABEL: @test_reversed_load2_store2(
381 ; CHECK-NEXT: entry:
382 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
383 ; CHECK: vector.ph:
384 ; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
385 ; CHECK-NEXT: [[TMP19:%.*]] = shl nuw nsw i64 [[TMP18]], 2
386 ; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
387 ; CHECK-NEXT: [[INDUCTION:%.*]] = sub <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1023, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), [[TMP0]]
388 ; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
389 ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i32 [[TMP1]], -4
390 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTNEG]], i64 0
391 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
392 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
393 ; CHECK: vector.body:
394 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
395 ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
396 ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
397 ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_ST2:%.*]], ptr [[A:%.*]], i64 [[OFFSET_IDX]], i32 0
398 ; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
399 ; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i32 [[TMP3]], 3
400 ; CHECK-NEXT: [[TMP5:%.*]] = sub nsw i32 2, [[TMP4]]
401 ; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
402 ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 [[TMP6]]
403 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP7]], align 4
404 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
405 ; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
406 ; CHECK-NEXT: [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP8]])
407 ; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
408 ; CHECK-NEXT: [[REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP9]])
409 ; CHECK-NEXT: [[TMP10:%.*]] = add nsw <vscale x 4 x i32> [[REVERSE]], [[VEC_IND]]
410 ; CHECK-NEXT: [[TMP11:%.*]] = sub nsw <vscale x 4 x i32> [[REVERSE1]], [[VEC_IND]]
411 ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_ST2]], ptr [[B:%.*]], i64 [[OFFSET_IDX]], i32 1
412 ; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vscale.i32()
413 ; CHECK-NEXT: [[TMP14:%.*]] = shl nuw nsw i32 [[TMP13]], 3
414 ; CHECK-NEXT: [[TMP15:%.*]] = sub nsw i32 1, [[TMP14]]
415 ; CHECK-NEXT: [[TMP16:%.*]] = sext i32 [[TMP15]] to i64
416 ; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i64 [[TMP16]]
417 ; CHECK-NEXT: [[REVERSE2:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP10]])
418 ; CHECK-NEXT: [[REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP11]])
419 ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.experimental.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[REVERSE2]], <vscale x 4 x i32> [[REVERSE3]])
420 ; CHECK-NEXT: store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
421 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
422 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
423 ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
424 ; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
425 ; CHECK: middle.block:
426 ; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
427 ; CHECK: scalar.ph:
428 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
429 ; CHECK: for.cond.cleanup:
430 ; CHECK-NEXT: ret void
431 ; CHECK: for.body:
432 ; CHECK-NEXT: br i1 poison, label [[FOR_BODY]], label [[FOR_COND_CLEANUP]], !llvm.loop [[LOOP11:![0-9]+]]
433 ;
434 entry:
435 br label %for.body
437 for.cond.cleanup: ; preds = %for.body
438 ret void
440 for.body: ; preds = %for.body, %entry
441 %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %for.body ]
442 %x = getelementptr inbounds %struct.ST2, %struct.ST2* %A, i64 %indvars.iv, i32 0
443 %load1 = load i32, i32* %x, align 4
444 %trunc = trunc i64 %indvars.iv to i32
445 %add = add nsw i32 %load1, %trunc
446 %y = getelementptr inbounds %struct.ST2, %struct.ST2* %A, i64 %indvars.iv, i32 1
447 %load2 = load i32, i32* %y, align 4
448 %sub = sub nsw i32 %load2, %trunc
449 %x5 = getelementptr inbounds %struct.ST2, %struct.ST2* %B, i64 %indvars.iv, i32 0
450 store i32 %add, i32* %x5, align 4
451 %y8 = getelementptr inbounds %struct.ST2, %struct.ST2* %B, i64 %indvars.iv, i32 1
452 store i32 %sub, i32* %y8, align 4
453 %indvars.iv.next = add nsw i64 %indvars.iv, -1
454 %cmp = icmp sgt i64 %indvars.iv, 0
455 br i1 %cmp, label %for.body, label %for.cond.cleanup
456 }
458 ; Check vectorization on an interleaved load group of factor 2 with 1 gap
459 ; (missing the load of odd elements). Because the vectorized loop would
460 ; speculatively access memory out-of-bounds, we must execute at least one
461 ; iteration of the scalar loop.
463 ; void even_load_static_tc(int *A, int *B) {
464 ; for (unsigned i = 0; i < 1024; i+=2)
465 ; B[i/2] = A[i] * 2;
466 ; }
469 define void @even_load_static_tc(i32* noalias nocapture readonly %A, i32* noalias nocapture %B) #1 {
470 ; CHECK-LABEL: @even_load_static_tc(
471 ; CHECK-NEXT: entry:
472 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
473 ; CHECK: vector.ph:
474 ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
475 ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
476 ; CHECK-NEXT: [[N_VEC:%.*]] = sub nuw nsw i64 512, [[TMP1]]
477 ; CHECK-NEXT: [[IND_END:%.*]] = shl nuw nsw i64 [[N_VEC]], 1
478 ; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
479 ; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
480 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
481 ; CHECK: vector.body:
482 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
483 ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
484 ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
485 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP2]], align 4
486 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
487 ; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
488 ; CHECK-NEXT: [[TMP4:%.*]] = shl nsw <vscale x 4 x i32> [[TMP3]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
489 ; CHECK-NEXT: [[TMP5:%.*]] = and i64 [[INDEX]], 9223372036854775804
490 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP5]]
491 ; CHECK-NEXT: store <vscale x 4 x i32> [[TMP4]], ptr [[TMP6]], align 4
492 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
493 ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
494 ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
495 ; CHECK: middle.block:
496 ; CHECK-NEXT: br label [[SCALAR_PH]]
497 ; CHECK: scalar.ph:
498 ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ poison, [[ENTRY:%.*]] ]
499 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
500 ; CHECK: for.cond.cleanup:
501 ; CHECK-NEXT: ret void
502 ; CHECK: for.body:
503 ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
504 ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
505 ; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
506 ; CHECK-NEXT: [[MUL:%.*]] = shl nsw i32 [[LOAD]], 1
507 ; CHECK-NEXT: [[LSHR:%.*]] = lshr exact i64 [[INDVARS_IV]], 1
508 ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[LSHR]]
509 ; CHECK-NEXT: store i32 [[MUL]], ptr [[ARRAYIDX2]], align 4
510 ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 2
511 ; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV]], 1022
512 ; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP13:![0-9]+]]
513 ;
514 entry:
515 br label %for.body
517 for.cond.cleanup: ; preds = %for.body
518 ret void
520 for.body: ; preds = %for.body, %entry
521 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
522 %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
523 %load = load i32, i32* %arrayidx, align 4
524 %mul = shl nsw i32 %load, 1
525 %lshr = lshr exact i64 %indvars.iv, 1
526 %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %lshr
527 store i32 %mul, i32* %arrayidx2, align 4
528 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
529 %cmp = icmp ult i64 %indvars.iv.next, 1024
530 br i1 %cmp, label %for.body, label %for.cond.cleanup
531 }
533 ; Check vectorization on an interleaved load group of factor 2 with 1 gap
534 ; (missing the load of odd elements). Because the vectorized loop would
535 ; speculatively access memory out-of-bounds, we must execute at least one
536 ; iteration of the scalar loop.
538 ; void even_load_dynamic_tc(int *A, int *B, unsigned N) {
539 ; for (unsigned i = 0; i < N; i+=2)
540 ; B[i/2] = A[i] * 2;
541 ; }
544 define void @even_load_dynamic_tc(i32* noalias nocapture readonly %A, i32* noalias nocapture %B, i64 %N) #1 {
545 ; CHECK-LABEL: @even_load_dynamic_tc(
546 ; CHECK-NEXT: entry:
547 ; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 2)
548 ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[UMAX]], -1
549 ; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
550 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
551 ; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2
552 ; CHECK-NEXT: [[MIN_ITERS_CHECK_NOT_NOT:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
553 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK_NOT_NOT]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
554 ; CHECK: vector.ph:
555 ; CHECK-NEXT: [[TMP4:%.*]] = add nuw i64 [[TMP1]], 1
556 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
557 ; CHECK-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 2
558 ; CHECK-NEXT: [[TMP7:%.*]] = add nsw i64 [[TMP6]], -1
559 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP4]], [[TMP7]]
560 ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
561 ; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP6]], i64 [[N_MOD_VF]]
562 ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[TMP9]]
563 ; CHECK-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 1
564 ; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
565 ; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 2
566 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
567 ; CHECK: vector.body:
568 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
569 ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
570 ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
571 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP10]], align 4
572 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
573 ; CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
574 ; CHECK-NEXT: [[TMP12:%.*]] = shl nsw <vscale x 4 x i32> [[TMP11]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
575 ; CHECK-NEXT: [[TMP13:%.*]] = and i64 [[INDEX]], 9223372036854775804
576 ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP13]]
577 ; CHECK-NEXT: store <vscale x 4 x i32> [[TMP12]], ptr [[TMP14]], align 4
578 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]]
579 ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
580 ; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
581 ; CHECK: middle.block:
582 ; CHECK-NEXT: br label [[SCALAR_PH]]
583 ; CHECK: scalar.ph:
584 ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
585 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
586 ; CHECK: for.cond.cleanup:
587 ; CHECK-NEXT: ret void
588 ; CHECK: for.body:
589 ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
590 ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
591 ; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
592 ; CHECK-NEXT: [[MUL:%.*]] = shl nsw i32 [[LOAD]], 1
593 ; CHECK-NEXT: [[LSHR:%.*]] = lshr exact i64 [[INDVARS_IV]], 1
594 ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[LSHR]]
595 ; CHECK-NEXT: store i32 [[MUL]], ptr [[ARRAYIDX2]], align 4
596 ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 2
597 ; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], [[N]]
598 ; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP15:![0-9]+]]
599 ;
600 entry:
601 br label %for.body
603 for.cond.cleanup: ; preds = %for.body
604 ret void
606 for.body: ; preds = %for.body, %entry
607 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
608 %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
609 %load = load i32, i32* %arrayidx, align 4
610 %mul = shl nsw i32 %load, 1
611 %lshr = lshr exact i64 %indvars.iv, 1
612 %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %lshr
613 store i32 %mul, i32* %arrayidx2, align 4
614 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
615 %cmp = icmp ult i64 %indvars.iv.next, %N
616 br i1 %cmp, label %for.body, label %for.cond.cleanup
617 }
619 ; Check vectorization on a reverse interleaved load group of factor 2 with 1
620 ; gap and a reverse interleaved store group of factor 2. The interleaved load
621 ; group should be removed since it has a gap and is reverse.
623 ; struct pair {
624 ; int x;
625 ; int y;
626 ; };
627 ;
628 ; void load_gap_reverse(struct pair *P1, struct pair *P2, int X) {
629 ; for (int i = 1023; i >= 0; i--) {
630 ; int a = X + i;
631 ; int b = A[i].y - i;
632 ; B[i].x = a;
633 ; B[i].y = b;
634 ; }
635 ; }
637 %pair = type { i64, i64 }
638 define void @load_gap_reverse(%pair* noalias nocapture readonly %P1, %pair* noalias nocapture readonly %P2, i64 %X) #1 {
639 ; CHECK-LABEL: @load_gap_reverse(
640 ; CHECK-NEXT: entry:
641 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
642 ; CHECK: vector.ph:
643 ; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
644 ; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
645 ; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
646 ; CHECK-NEXT: [[INDUCTION:%.*]] = sub <vscale x 4 x i64> shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1023, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer), [[TMP0]]
647 ; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
648 ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP1]], -4
649 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[DOTNEG]], i64 0
650 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
651 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[X:%.*]], i64 0
652 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
653 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
654 ; CHECK: vector.body:
655 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
656 ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
657 ; CHECK-NEXT: [[TMP2:%.*]] = add nsw <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[VEC_IND]]
658 ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P1:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
659 ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P2:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 1
660 ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> [[TMP4]], i32 8, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i64> poison)
661 ; CHECK-NEXT: [[TMP5:%.*]] = sub nsw <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], [[VEC_IND]]
662 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i64.nxv4p0(<vscale x 4 x i64> [[TMP2]], <vscale x 4 x ptr> [[TMP3]], i32 8, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
663 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i64.nxv4p0(<vscale x 4 x i64> [[TMP5]], <vscale x 4 x ptr> [[TMP4]], i32 8, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
664 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
665 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
666 ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
667 ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
668 ; CHECK: middle.block:
669 ; CHECK-NEXT: br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
670 ; CHECK: scalar.ph:
671 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
672 ; CHECK: for.body:
673 ; CHECK-NEXT: br i1 poison, label [[FOR_BODY]], label [[FOR_EXIT]], !llvm.loop [[LOOP17:![0-9]+]]
674 ; CHECK: for.exit:
675 ; CHECK-NEXT: ret void
676 ;
677 entry:
678 br label %for.body
680 for.body:
681 %i = phi i64 [ 1023, %entry ], [ %i.next, %for.body ]
682 %0 = add nsw i64 %X, %i
683 %1 = getelementptr inbounds %pair, %pair* %P1, i64 %i, i32 0
684 %2 = getelementptr inbounds %pair, %pair* %P2, i64 %i, i32 1
685 %3 = load i64, i64* %2, align 8
686 %4 = sub nsw i64 %3, %i
687 store i64 %0, i64* %1, align 8
688 store i64 %4, i64* %2, align 8
689 %i.next = add nsw i64 %i, -1
690 %cond = icmp sgt i64 %i, 0
691 br i1 %cond, label %for.body, label %for.exit
693 for.exit:
694 ret void
695 }
697 ; Check vectorization on interleaved access groups identified from mixed
698 ; loads/stores.
699 ; void mixed_load2_store2(int *A, int *B) {
700 ; for (unsigned i = 0; i < 1024; i+=2) {
701 ; B[i] = A[i] * A[i+1];
702 ; B[i+1] = A[i] + A[i+1];
703 ; }
704 ; }
707 define void @mixed_load2_store2(i32* noalias nocapture readonly %A, i32* noalias nocapture %B) #1 {
708 ; CHECK-LABEL: @mixed_load2_store2(
709 ; CHECK-NEXT: entry:
710 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
711 ; CHECK: vector.ph:
712 ; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
713 ; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 2
714 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
715 ; CHECK: vector.body:
716 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
717 ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
718 ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
719 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP0]], align 4
720 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
721 ; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
722 ; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
723 ; CHECK-NEXT: [[TMP3:%.*]] = or disjoint i64 [[OFFSET_IDX]], 1
724 ; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <vscale x 4 x i32> [[TMP2]], [[TMP1]]
725 ; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
726 ; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
727 ; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 1
728 ; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 4 x i32> [[TMP6]], [[TMP5]]
729 ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP3]]
730 ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 -1
731 ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.experimental.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP7]])
732 ; CHECK-NEXT: store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP9]], align 4
733 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
734 ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
735 ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
736 ; CHECK: middle.block:
737 ; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
738 ; CHECK: scalar.ph:
739 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
740 ; CHECK: for.cond.cleanup:
741 ; CHECK-NEXT: ret void
742 ; CHECK: for.body:
743 ; CHECK-NEXT: br i1 poison, label [[FOR_BODY]], label [[FOR_COND_CLEANUP]], !llvm.loop [[LOOP19:![0-9]+]]
744 ;
745 entry:
746 br label %for.body
748 for.cond.cleanup: ; preds = %for.body
749 ret void
751 for.body: ; preds = %for.body, %entry
752 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
753 %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
754 %load1 = load i32, i32* %arrayidx, align 4
755 %or = or disjoint i64 %indvars.iv, 1
756 %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %or
757 %load2 = load i32, i32* %arrayidx2, align 4
758 %mul = mul nsw i32 %load2, %load1
759 %arrayidx4 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
760 store i32 %mul, i32* %arrayidx4, align 4
761 %load3 = load i32, i32* %arrayidx, align 4
762 %load4 = load i32, i32* %arrayidx2, align 4
763 %add10 = add nsw i32 %load4, %load3
764 %arrayidx13 = getelementptr inbounds i32, i32* %B, i64 %or
765 store i32 %add10, i32* %arrayidx13, align 4
766 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
767 %cmp = icmp ult i64 %indvars.iv.next, 1024
768 br i1 %cmp, label %for.body, label %for.cond.cleanup
769 }
772 ; Check vectorization on interleaved access groups with members having different
773 ; kinds of type.
775 ; struct IntFloat {
776 ; int a;
777 ; float b;
778 ; };
779 ;
780 ; int SA;
781 ; float SB;
782 ;
783 ; void int_float_struct(struct IntFloat *A) {
784 ; int SumA;
785 ; float SumB;
786 ; for (unsigned i = 0; i < 1024; i++) {
787 ; SumA += A[i].a;
788 ; SumB += A[i].b;
789 ; }
790 ; SA = SumA;
791 ; SB = SumB;
792 ; }
795 %struct.IntFloat = type { i32, float }
797 @SA = common global i32 0, align 4
798 @SB = common global float 0.000000e+00, align 4
800 define void @int_float_struct(%struct.IntFloat* nocapture readonly %p) #0 {
801 ; CHECK-LABEL: @int_float_struct(
802 ; CHECK-NEXT: entry:
803 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
804 ; CHECK: vector.ph:
805 ; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
806 ; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
807 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
808 ; CHECK: vector.body:
809 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
810 ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ insertelement (<vscale x 4 x float> zeroinitializer, float undef, i32 0), [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
811 ; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> zeroinitializer, i32 undef, i32 0), [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
812 ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_INTFLOAT:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 0
813 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP0]], align 4
814 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
815 ; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
816 ; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
817 ; CHECK-NEXT: [[TMP3:%.*]] = bitcast <vscale x 4 x i32> [[TMP2]] to <vscale x 4 x float>
818 ; CHECK-NEXT: [[TMP4]] = add <vscale x 4 x i32> [[TMP1]], [[VEC_PHI1]]
819 ; CHECK-NEXT: [[TMP5]] = fadd fast <vscale x 4 x float> [[VEC_PHI]], [[TMP3]]
820 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
821 ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
822 ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
823 ; CHECK: middle.block:
824 ; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP5]])
825 ; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP4]])
826 ; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
827 ; CHECK: scalar.ph:
828 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
829 ; CHECK: for.cond.cleanup:
830 ; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
831 ; CHECK-NEXT: [[ADD3_LCSSA:%.*]] = phi float [ poison, [[FOR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
832 ; CHECK-NEXT: store i32 [[ADD_LCSSA]], ptr @SA, align 4
833 ; CHECK-NEXT: store float [[ADD3_LCSSA]], ptr @SB, align 4
834 ; CHECK-NEXT: ret void
835 ; CHECK: for.body:
836 ; CHECK-NEXT: br i1 poison, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
837 ;
838 entry:
839 br label %for.body
841 for.cond.cleanup: ; preds = %for.body
842 store i32 %add, i32* @SA, align 4
843 store float %add3, float* @SB, align 4
844 ret void
846 for.body: ; preds = %for.body, %entry
847 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
848 %SumB.014 = phi float [ undef, %entry ], [ %add3, %for.body ]
849 %SumA.013 = phi i32 [ undef, %entry ], [ %add, %for.body ]
850 %a = getelementptr inbounds %struct.IntFloat, %struct.IntFloat* %p, i64 %indvars.iv, i32 0
851 %load1 = load i32, i32* %a, align 4
852 %add = add nsw i32 %load1, %SumA.013
853 %b = getelementptr inbounds %struct.IntFloat, %struct.IntFloat* %p, i64 %indvars.iv, i32 1
854 %load2 = load float, float* %b, align 4
855 %add3 = fadd fast float %SumB.014, %load2
856 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
857 %exitcond = icmp eq i64 %indvars.iv.next, 1024
858 br i1 %exitcond, label %for.cond.cleanup, label %for.body
859 }
861 ; Check vectorization of interleaved access groups in the presence of
862 ; dependences (PR27626). The following tests check that we don't reorder
863 ; dependent loads and stores when generating code for interleaved access
864 ; groups. Stores should be scalarized because the required code motion would
865 ; break dependences, and the remaining interleaved load groups should have
866 ; gaps.
868 ; PR27626_0: Ensure a strided store is not moved after a dependent (zero
869 ; distance) strided load.
871 ; void PR27626_0(struct pair *p, int z, int n) {
872 ; for (int i = 0; i < n; i++) {
873 ; p[i].x = z;
874 ; p[i].y = p[i].x;
875 ; }
876 ; }
878 %pair.i32 = type { i32, i32 }
879 define void @PR27626_0(%pair.i32 *%p, i32 %z, i64 %n) #1 {
880 ; CHECK-LABEL: @PR27626_0(
881 ; CHECK-NEXT: entry:
882 ; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
883 ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
884 ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
885 ; CHECK-NEXT: [[MIN_ITERS_CHECK_NOT:%.*]] = icmp ugt i64 [[SMAX]], [[TMP1]]
886 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK_NOT]], label [[VECTOR_PH:%.*]], label [[SCALAR_PH:%.*]]
887 ; CHECK: vector.ph:
888 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
889 ; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2
890 ; CHECK-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP3]], -1
891 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], [[TMP4]]
892 ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
893 ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]]
894 ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]]
895 ; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
896 ; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2
897 ; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
898 ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
899 ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2
900 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
901 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
902 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z:%.*]], i64 0
903 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
904 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
905 ; CHECK: vector.body:
906 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
907 ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
908 ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
909 ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], <vscale x 4 x i64> [[VEC_IND]], i32 1
910 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
911 ; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x ptr> [[TMP10]], i64 0
912 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP12]], align 4
913 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
914 ; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
915 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP13]], <vscale x 4 x ptr> [[TMP11]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
916 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]]
917 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
918 ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
919 ; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
920 ; CHECK: middle.block:
921 ; CHECK-NEXT: br label [[SCALAR_PH]]
922 ; CHECK: scalar.ph:
923 ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
924 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
925 ; CHECK: for.body:
926 ; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
927 ; CHECK-NEXT: [[P_I_X:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 0
928 ; CHECK-NEXT: [[P_I_Y:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1
929 ; CHECK-NEXT: store i32 [[Z]], ptr [[P_I_X]], align 4
930 ; CHECK-NEXT: store i32 [[Z]], ptr [[P_I_Y]], align 4
931 ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
932 ; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
933 ; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP23:![0-9]+]]
934 ; CHECK: for.end:
935 ; CHECK-NEXT: ret void
936 ;
937 entry:
938 br label %for.body
940 for.body:
941 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
942 %p_i.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 0
943 %p_i.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1
944 store i32 %z, i32* %p_i.x, align 4
945 %0 = load i32, i32* %p_i.x, align 4
946 store i32 %0, i32 *%p_i.y, align 4
947 %i.next = add nuw nsw i64 %i, 1
948 %cond = icmp slt i64 %i.next, %n
949 br i1 %cond, label %for.body, label %for.end
951 for.end:
952 ret void
953 }
955 ; PR27626_1: Ensure a strided load is not moved before a dependent (zero
956 ; distance) strided store.
958 ; void PR27626_1(struct pair *p, int n) {
959 ; int s = 0;
960 ; for (int i = 0; i < n; i++) {
961 ; p[i].y = p[i].x;
962 ; s += p[i].y
963 ; }
964 ; }
966 define i32 @PR27626_1(%pair.i32 *%p, i64 %n) #1 {
967 ; CHECK-LABEL: @PR27626_1(
968 ; CHECK-NEXT: entry:
969 ; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
970 ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
971 ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
972 ; CHECK-NEXT: [[MIN_ITERS_CHECK_NOT:%.*]] = icmp ugt i64 [[SMAX]], [[TMP1]]
973 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK_NOT]], label [[VECTOR_PH:%.*]], label [[SCALAR_PH:%.*]]
974 ; CHECK: vector.ph:
975 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
976 ; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2
977 ; CHECK-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP3]], -1
978 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], [[TMP4]]
979 ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
980 ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]]
981 ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]]
982 ; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
983 ; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i64 [[TMP16]], 2
984 ; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
985 ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
986 ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2
987 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
988 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
989 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
990 ; CHECK: vector.body:
991 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
992 ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
993 ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
994 ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 0
995 ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], <vscale x 4 x i64> [[VEC_IND]], i32 1
996 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP10]], align 4
997 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
998 ; CHECK-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
999 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP12]], <vscale x 4 x ptr> [[TMP11]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
1000 ; CHECK-NEXT: [[TMP13:%.*]] = extractelement <vscale x 4 x ptr> [[TMP11]], i64 0
1001 ; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x i32>, ptr [[TMP13]], align 4
1002 ; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC1]])
1003 ; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
1004 ; CHECK-NEXT: [[TMP15]] = add <vscale x 4 x i32> [[TMP14]], [[VEC_PHI]]
1005 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]]
1006 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
1007 ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
1008 ; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
1009 ; CHECK: middle.block:
1010 ; CHECK-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP15]])
1011 ; CHECK-NEXT: br label [[SCALAR_PH]]
1012 ; CHECK: scalar.ph:
1013 ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
1014 ; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
1015 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
1016 ; CHECK: for.body:
1017 ; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
1018 ; CHECK-NEXT: [[S:%.*]] = phi i32 [ [[TMP21:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
1019 ; CHECK-NEXT: [[P_I_X:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 0
1020 ; CHECK-NEXT: [[P_I_Y:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1
1021 ; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[P_I_X]], align 4
1022 ; CHECK-NEXT: store i32 [[TMP20]], ptr [[P_I_Y]], align 4
1023 ; CHECK-NEXT: [[TMP21]] = add nsw i32 [[TMP20]], [[S]]
1024 ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
1025 ; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
1026 ; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP25:![0-9]+]]
1027 ; CHECK: for.end:
1028 ; CHECK-NEXT: ret i32 [[TMP21]]
1029 ;
1030 entry:
1031 br label %for.body
1033 for.body:
1034 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
1035 %s = phi i32 [ %2, %for.body ], [ 0, %entry ]
1036 %p_i.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 0
1037 %p_i.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1
1038 %0 = load i32, i32* %p_i.x, align 4
1039 store i32 %0, i32* %p_i.y, align 4
1040 %1 = load i32, i32* %p_i.y, align 4
1041 %2 = add nsw i32 %1, %s
1042 %i.next = add nuw nsw i64 %i, 1
1043 %cond = icmp slt i64 %i.next, %n
1044 br i1 %cond, label %for.body, label %for.end
1046 for.end:
1047 %3 = phi i32 [ %2, %for.body ]
1048 ret i32 %3
1049 }
1051 ; PR27626_2: Ensure a strided store is not moved after a dependent (negative
1052 ; distance) strided load.
1054 ; void PR27626_2(struct pair *p, int z, int n) {
1055 ; for (int i = 0; i < n; i++) {
1056 ; p[i].x = z;
1057 ; p[i].y = p[i - 1].x;
1058 ; }
1059 ; }
1061 define void @PR27626_2(%pair.i32 *%p, i64 %n, i32 %z) #1 {
1062 ; CHECK-LABEL: @PR27626_2(
1063 ; CHECK-NEXT: entry:
1064 ; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
1065 ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
1066 ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
1067 ; CHECK-NEXT: [[MIN_ITERS_CHECK_NOT:%.*]] = icmp ugt i64 [[SMAX]], [[TMP1]]
1068 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK_NOT]], label [[VECTOR_PH:%.*]], label [[SCALAR_PH:%.*]]
1069 ; CHECK: vector.ph:
1070 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
1071 ; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2
1072 ; CHECK-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP3]], -1
1073 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], [[TMP4]]
1074 ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
1075 ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]]
1076 ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]]
1077 ; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
1078 ; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2
1079 ; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
1080 ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
1081 ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2
1082 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
1083 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
1084 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z:%.*]], i64 0
1085 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
1086 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
1087 ; CHECK: vector.body:
1088 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1089 ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
1090 ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
1091 ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 -1, i32 0
1092 ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], <vscale x 4 x i64> [[VEC_IND]], i32 1
1093 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
1094 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP11]], align 4
1095 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
1096 ; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
1097 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP13]], <vscale x 4 x ptr> [[TMP12]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
1098 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]]
1099 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
1100 ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
1101 ; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
1102 ; CHECK: middle.block:
1103 ; CHECK-NEXT: br label [[SCALAR_PH]]
1104 ; CHECK: scalar.ph:
1105 ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
1106 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
1107 ; CHECK: for.body:
1108 ; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
1109 ; CHECK-NEXT: [[P_I_X:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 0
1110 ; CHECK-NEXT: [[P_I_MINUS_1_X:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 -1, i32 0
1111 ; CHECK-NEXT: [[P_I_Y:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1
1112 ; CHECK-NEXT: store i32 [[Z]], ptr [[P_I_X]], align 4
1113 ; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[P_I_MINUS_1_X]], align 4
1114 ; CHECK-NEXT: store i32 [[TMP17]], ptr [[P_I_Y]], align 4
1115 ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
1116 ; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
1117 ; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP27:![0-9]+]]
1118 ; CHECK: for.end:
1119 ; CHECK-NEXT: ret void
1120 ;
1121 entry:
1122 br label %for.body
1124 for.body:
1125 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
1126 %i_minus_1 = add nuw nsw i64 %i, -1
1127 %p_i.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 0
1128 %p_i_minus_1.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i_minus_1, i32 0
1129 %p_i.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1
1130 store i32 %z, i32* %p_i.x, align 4
1131 %0 = load i32, i32* %p_i_minus_1.x, align 4
1132 store i32 %0, i32 *%p_i.y, align 4
1133 %i.next = add nuw nsw i64 %i, 1
1134 %cond = icmp slt i64 %i.next, %n
1135 br i1 %cond, label %for.body, label %for.end
1137 for.end:
1138 ret void
1139 }
1141 ; PR27626_3: Ensure a strided load is not moved before a dependent (negative
1142 ; distance) strided store.
1144 ; void PR27626_3(struct pair *p, int z, int n) {
1145 ; for (int i = 0; i < n; i++) {
1146 ; p[i + 1].y = p[i].x;
1147 ; s += p[i].y;
1148 ; }
1149 ; }
1151 define i32 @PR27626_3(%pair.i32 *%p, i64 %n, i32 %z) #1 {
1152 ; CHECK-LABEL: @PR27626_3(
1153 ; CHECK-NEXT: entry:
1154 ; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
1155 ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
1156 ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
1157 ; CHECK-NEXT: [[MIN_ITERS_CHECK_NOT:%.*]] = icmp ugt i64 [[SMAX]], [[TMP1]]
1158 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK_NOT]], label [[VECTOR_PH:%.*]], label [[SCALAR_PH:%.*]]
1159 ; CHECK: vector.ph:
1160 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
1161 ; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2
1162 ; CHECK-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP3]], -1
1163 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], [[TMP4]]
1164 ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
1165 ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]]
1166 ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]]
1167 ; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
1168 ; CHECK-NEXT: [[TMP18:%.*]] = shl nuw nsw i64 [[TMP17]], 2
1169 ; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
1170 ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
1171 ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2
1172 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
1173 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
1174 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
1175 ; CHECK: vector.body:
1176 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1177 ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
1178 ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
1179 ; CHECK-NEXT: [[TMP10:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
1180 ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 0
1181 ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[INDEX]], i32 1
1182 ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], <vscale x 4 x i64> [[TMP10]], i32 1
1183 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP11]], align 4
1184 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
1185 ; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
1186 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP14]], <vscale x 4 x ptr> [[TMP13]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
1187 ; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x i32>, ptr [[TMP12]], align 4
1188 ; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC1]])
1189 ; CHECK-NEXT: [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
1190 ; CHECK-NEXT: [[TMP16]] = add <vscale x 4 x i32> [[TMP15]], [[VEC_PHI]]
1191 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]]
1192 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
1193 ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
1194 ; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
1195 ; CHECK: middle.block:
1196 ; CHECK-NEXT: [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP16]])
1197 ; CHECK-NEXT: br label [[SCALAR_PH]]
1198 ; CHECK: scalar.ph:
1199 ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
1200 ; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP20]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
1201 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
1202 ; CHECK: for.body:
1203 ; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
1204 ; CHECK-NEXT: [[S:%.*]] = phi i32 [ [[TMP23:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
1205 ; CHECK-NEXT: [[I_PLUS_1:%.*]] = add nuw nsw i64 [[I]], 1
1206 ; CHECK-NEXT: [[P_I_X:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 0
1207 ; CHECK-NEXT: [[P_I_Y:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1
1208 ; CHECK-NEXT: [[P_I_PLUS_1_Y:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I_PLUS_1]], i32 1
1209 ; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[P_I_X]], align 4
1210 ; CHECK-NEXT: store i32 [[TMP21]], ptr [[P_I_PLUS_1_Y]], align 4
1211 ; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[P_I_Y]], align 4
1212 ; CHECK-NEXT: [[TMP23]] = add nsw i32 [[TMP22]], [[S]]
1213 ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
1214 ; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
1215 ; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP29:![0-9]+]]
1216 ; CHECK: for.end:
1217 ; CHECK-NEXT: ret i32 [[TMP23]]
1218 ;
1219 entry:
1220 br label %for.body
1222 for.body:
1223 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
1224 %s = phi i32 [ %2, %for.body ], [ 0, %entry ]
1225 %i_plus_1 = add nuw nsw i64 %i, 1
1226 %p_i.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 0
1227 %p_i.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1
1228 %p_i_plus_1.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i_plus_1, i32 1
1229 %0 = load i32, i32* %p_i.x, align 4
1230 store i32 %0, i32* %p_i_plus_1.y, align 4
1231 %1 = load i32, i32* %p_i.y, align 4
1232 %2 = add nsw i32 %1, %s
1233 %i.next = add nuw nsw i64 %i, 1
1234 %cond = icmp slt i64 %i.next, %n
1235 br i1 %cond, label %for.body, label %for.end
1237 for.end:
1238 %3 = phi i32 [ %2, %for.body ]
1239 ret i32 %3
1240 }
1242 ; PR27626_4: Ensure we form an interleaved group for strided stores in the
1243 ; presence of a write-after-write dependence. We create a group for
1244 ; (2) and (3) while excluding (1).
1246 ; void PR27626_4(int *a, int x, int y, int z, int n) {
1247 ; for (int i = 0; i < n; i += 2) {
1248 ; a[i] = x; // (1)
1249 ; a[i] = y; // (2)
1250 ; a[i + 1] = z; // (3)
1251 ; }
1252 ; }
1254 define void @PR27626_4(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) #1 {
1255 ; CHECK-LABEL: @PR27626_4(
1256 ; CHECK-NEXT: entry:
1257 ; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 2)
1258 ; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1
1259 ; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
1260 ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
1261 ; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
1262 ; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
1263 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
1264 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
1265 ; CHECK: vector.ph:
1266 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
1267 ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP5]], -4
1268 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], [[DOTNEG]]
1269 ; CHECK-NEXT: [[IND_END:%.*]] = shl nuw i64 [[N_VEC]], 1
1270 ; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
1271 ; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2
1272 ; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
1273 ; CHECK-NEXT: [[TMP7:%.*]] = shl <vscale x 4 x i64> [[TMP6]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
1274 ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
1275 ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 3
1276 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
1277 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
1278 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X:%.*]], i64 0
1279 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
1280 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Y:%.*]], i64 0
1281 ; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
1282 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z:%.*]], i64 0
1283 ; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
1284 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
1285 ; CHECK: vector.body:
1286 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1287 ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
1288 ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
1289 ; CHECK-NEXT: [[TMP10:%.*]] = or disjoint i64 [[OFFSET_IDX]], 1
1290 ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]]
1291 ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
1292 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x ptr> [[TMP11]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
1293 ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i64 -1
1294 ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.experimental.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[BROADCAST_SPLAT2]], <vscale x 4 x i32> [[BROADCAST_SPLAT4]])
1295 ; CHECK-NEXT: store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP13]], align 4
1296 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]]
1297 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
1298 ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
1299 ; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
1300 ; CHECK: middle.block:
1301 ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
1302 ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
1303 ; CHECK: scalar.ph:
1304 ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
1305 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
1306 ; CHECK: for.body:
1307 ; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
1308 ; CHECK-NEXT: [[I_PLUS_1:%.*]] = or disjoint i64 [[I]], 1
1309 ; CHECK-NEXT: [[A_I:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
1310 ; CHECK-NEXT: [[A_I_PLUS_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_PLUS_1]]
1311 ; CHECK-NEXT: store i32 [[Y]], ptr [[A_I]], align 4
1312 ; CHECK-NEXT: store i32 [[Z]], ptr [[A_I_PLUS_1]], align 4
1313 ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2
1314 ; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
1315 ; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP31:![0-9]+]]
1316 ; CHECK: for.end:
1317 ; CHECK-NEXT: ret void
1318 ;
1319 entry:
1320 br label %for.body
1322 for.body:
1323 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
1324 %i_plus_1 = add i64 %i, 1
1325 %a_i = getelementptr inbounds i32, i32* %a, i64 %i
1326 %a_i_plus_1 = getelementptr inbounds i32, i32* %a, i64 %i_plus_1
1327 store i32 %x, i32* %a_i, align 4
1328 store i32 %y, i32* %a_i, align 4
1329 store i32 %z, i32* %a_i_plus_1, align 4
1330 %i.next = add nuw nsw i64 %i, 2
1331 %cond = icmp slt i64 %i.next, %n
1332 br i1 %cond, label %for.body, label %for.end
1334 for.end:
1335 ret void
1336 }
1338 ; PR27626_5: Ensure we do not form an interleaved group for strided stores in
1339 ; the presence of a write-after-write dependence.
1341 ; void PR27626_5(int *a, int x, int y, int z, int n) {
1342 ; for (int i = 3; i < n; i += 2) {
1343 ; a[i - 1] = x;
1344 ; a[i - 3] = y;
1345 ; a[i] = z;
1346 ; }
1347 ; }
1349 define void @PR27626_5(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) #1 {
1350 ; CHECK-LABEL: @PR27626_5(
1351 ; CHECK-NEXT: entry:
1352 ; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 5)
1353 ; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -4
1354 ; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
1355 ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
1356 ; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
1357 ; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
1358 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
1359 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
1360 ; CHECK: vector.ph:
1361 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
1362 ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP5]], -4
1363 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], [[DOTNEG]]
1364 ; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[N_VEC]], 1
1365 ; CHECK-NEXT: [[IND_END:%.*]] = or disjoint i64 [[TMP6]], 3
1366 ; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
1367 ; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i64 [[TMP16]], 2
1368 ; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
1369 ; CHECK-NEXT: [[TMP8:%.*]] = shl <vscale x 4 x i64> [[TMP7]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
1370 ; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[TMP8]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 3, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
1371 ; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
1372 ; CHECK-NEXT: [[TMP10:%.*]] = shl nuw nsw i64 [[TMP9]], 3
1373 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP10]], i64 0
1374 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
1375 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X:%.*]], i64 0
1376 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
1377 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Y:%.*]], i64 0
1378 ; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
1379 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z:%.*]], i64 0
1380 ; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
1381 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
1382 ; CHECK: vector.body:
1383 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1384 ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
1385 ; CHECK-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 -1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
1386 ; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 -3, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
1387 ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]]
1388 ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], <vscale x 4 x i64> [[TMP11]]
1389 ; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], <vscale x 4 x i64> [[TMP12]]
1390 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x ptr> [[TMP14]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
1391 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT2]], <vscale x 4 x ptr> [[TMP15]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
1392 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT4]], <vscale x 4 x ptr> [[TMP13]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
1393 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]]
1394 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
1395 ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
1396 ; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
1397 ; CHECK: middle.block:
1398 ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
1399 ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
1400 ; CHECK: scalar.ph:
1401 ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
1402 ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
1403 ; CHECK: for.body:
1404 ; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
1405 ; CHECK-NEXT: [[A_I:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
1406 ; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[A]], i64 [[I]]
1407 ; CHECK-NEXT: [[A_I_MINUS_1:%.*]] = getelementptr i32, ptr [[TMP19]], i64 -1
1408 ; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[A]], i64 [[I]]
1409 ; CHECK-NEXT: [[A_I_MINUS_3:%.*]] = getelementptr i32, ptr [[TMP20]], i64 -3
1410 ; CHECK-NEXT: store i32 [[X]], ptr [[A_I_MINUS_1]], align 4
1411 ; CHECK-NEXT: store i32 [[Y]], ptr [[A_I_MINUS_3]], align 4
1412 ; CHECK-NEXT: store i32 [[Z]], ptr [[A_I]], align 4
1413 ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2
1414 ; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
1415 ; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP33:![0-9]+]]
1416 ; CHECK: for.end:
1417 ; CHECK-NEXT: ret void
1418 ;
1419 entry:
1420 br label %for.body
1422 for.body:
1423 %i = phi i64 [ %i.next, %for.body ], [ 3, %entry ]
1424 %i_minus_1 = sub i64 %i, 1
1425 %i_minus_3 = sub i64 %i_minus_1, 2
1426 %a_i = getelementptr inbounds i32, i32* %a, i64 %i
1427 %a_i_minus_1 = getelementptr inbounds i32, i32* %a, i64 %i_minus_1
1428 %a_i_minus_3 = getelementptr inbounds i32, i32* %a, i64 %i_minus_3
1429 store i32 %x, i32* %a_i_minus_1, align 4
1430 store i32 %y, i32* %a_i_minus_3, align 4
1431 store i32 %z, i32* %a_i, align 4
1432 %i.next = add nuw nsw i64 %i, 2
1433 %cond = icmp slt i64 %i.next, %n
1434 br i1 %cond, label %for.body, label %for.end
1436 for.end:
1437 ret void
1438 }
1440 ; PR34743: Ensure that a cast which needs to sink after a load that belongs to
1441 ; an interleaved group, indeeded gets sunk.
1443 ; void PR34743(short *a, int *b, int n) {
1444 ; for (int i = 0, iv = 0; iv < n; i++, iv += 2) {
1445 ; b[i] = a[iv] * a[iv+1] * a[iv+2];
1446 ; }
1447 ; }
1449 define void @PR34743(i16* %a, i32* %b, i64 %n) #1 {
1450 ; CHECK-LABEL: @PR34743(
1451 ; CHECK-NEXT: entry:
1452 ; CHECK-NEXT: [[DOTPRE:%.*]] = load i16, ptr [[A:%.*]], align 2
1453 ; CHECK-NEXT: [[TMP0:%.*]] = lshr i64 [[N:%.*]], 1
1454 ; CHECK-NEXT: [[TMP1:%.*]] = add nuw i64 [[TMP0]], 1
1455 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
1456 ; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2
1457 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
1458 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
1459 ; CHECK: vector.memcheck:
1460 ; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[N]], 1
1461 ; CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], -4
1462 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP5]]
1463 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP6]], i64 4
1464 ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 2
1465 ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
1466 ; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[TMP7]], i64 6
1467 ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP2]], [[B]]
1468 ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP]]
1469 ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
1470 ; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
1471 ; CHECK: vector.ph:
1472 ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
1473 ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP8]], -4
1474 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP1]], [[DOTNEG]]
1475 ; CHECK-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 1
1476 ; CHECK-NEXT: [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
1477 ; CHECK-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP27]], 2
1478 ; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
1479 ; CHECK-NEXT: [[TMP10:%.*]] = shl nuw nsw i32 [[TMP9]], 2
1480 ; CHECK-NEXT: [[TMP11:%.*]] = add nsw i32 [[TMP10]], -1
1481 ; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[DOTPRE]], i32 [[TMP11]]
1482 ; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
1483 ; CHECK-NEXT: [[TMP13:%.*]] = shl <vscale x 4 x i64> [[TMP12]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
1484 ; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
1485 ; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 3
1486 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP15]], i64 0
1487 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
1488 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
1489 ; CHECK: vector.body:
1490 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1491 ; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_MASKED_GATHER4:%.*]], [[VECTOR_BODY]] ]
1492 ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP13]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
1493 ; CHECK-NEXT: [[TMP16:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
1494 ; CHECK-NEXT: [[TMP17:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 2, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
1495 ; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i16, ptr [[A]], <vscale x 4 x i64> [[TMP16]]
1496 ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> [[TMP18]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i16> poison), !alias.scope [[META34:![0-9]+]]
1497 ; CHECK-NEXT: [[TMP19:%.*]] = sext <vscale x 4 x i16> [[WIDE_MASKED_GATHER]] to <vscale x 4 x i32>
1498 ; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i16, ptr [[A]], <vscale x 4 x i64> [[TMP17]]
1499 ; CHECK-NEXT: [[WIDE_MASKED_GATHER4]] = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> [[TMP20]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i16> poison), !alias.scope [[META34]]
1500 ; CHECK-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i16> @llvm.experimental.vector.splice.nxv4i16(<vscale x 4 x i16> [[VECTOR_RECUR]], <vscale x 4 x i16> [[WIDE_MASKED_GATHER4]], i32 -1)
1501 ; CHECK-NEXT: [[TMP22:%.*]] = sext <vscale x 4 x i16> [[TMP21]] to <vscale x 4 x i32>
1502 ; CHECK-NEXT: [[TMP23:%.*]] = sext <vscale x 4 x i16> [[WIDE_MASKED_GATHER4]] to <vscale x 4 x i32>
1503 ; CHECK-NEXT: [[TMP24:%.*]] = mul nsw <vscale x 4 x i32> [[TMP22]], [[TMP19]]
1504 ; CHECK-NEXT: [[TMP25:%.*]] = mul nsw <vscale x 4 x i32> [[TMP24]], [[TMP23]]
1505 ; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
1506 ; CHECK-NEXT: store <vscale x 4 x i32> [[TMP25]], ptr [[TMP26]], align 4, !alias.scope [[META37:![0-9]+]], !noalias [[META34]]
1507 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP28]]
1508 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
1509 ; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
1510 ; CHECK-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
1511 ; CHECK: middle.block:
1512 ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
1513 ; CHECK-NEXT: [[TMP30:%.*]] = call i32 @llvm.vscale.i32()
1514 ; CHECK-NEXT: [[TMP31:%.*]] = shl nuw nsw i32 [[TMP30]], 2
1515 ; CHECK-NEXT: [[TMP32:%.*]] = add nsw i32 [[TMP31]], -1
1516 ; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_MASKED_GATHER4]], i32 [[TMP32]]
1517 ; CHECK-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]
1518 ; CHECK: scalar.ph:
1519 ; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
1520 ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ]
1521 ; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i64 [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
1522 ; CHECK-NEXT: br label [[LOOP:%.*]]
1523 ; CHECK: loop:
1524 ; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[LOAD2:%.*]], [[LOOP]] ]
1525 ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV2:%.*]], [[LOOP]] ]
1526 ; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ], [ [[I1:%.*]], [[LOOP]] ]
1527 ; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[SCALAR_RECUR]] to i32
1528 ; CHECK-NEXT: [[I1]] = add nuw nsw i64 [[I]], 1
1529 ; CHECK-NEXT: [[IV1:%.*]] = or disjoint i64 [[IV]], 1
1530 ; CHECK-NEXT: [[IV2]] = add nuw nsw i64 [[IV]], 2
1531 ; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV1]]
1532 ; CHECK-NEXT: [[LOAD1:%.*]] = load i16, ptr [[GEP1]], align 4
1533 ; CHECK-NEXT: [[CONV1:%.*]] = sext i16 [[LOAD1]] to i32
1534 ; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV2]]
1535 ; CHECK-NEXT: [[LOAD2]] = load i16, ptr [[GEP2]], align 4
1536 ; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[LOAD2]] to i32
1537 ; CHECK-NEXT: [[MUL01:%.*]] = mul nsw i32 [[CONV]], [[CONV1]]
1538 ; CHECK-NEXT: [[MUL012:%.*]] = mul nsw i32 [[MUL01]], [[CONV2]]
1539 ; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]
1540 ; CHECK-NEXT: store i32 [[MUL012]], ptr [[ARRAYIDX5]], align 4
1541 ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV]], [[N]]
1542 ; CHECK-NEXT: br i1 [[EXITCOND]], label [[END]], label [[LOOP]], !llvm.loop [[LOOP40:![0-9]+]]
1543 ; CHECK: end:
1544 ; CHECK-NEXT: ret void
1545 ;
1546 entry:
1547 %.pre = load i16, i16* %a
1548 br label %loop
1550 loop:
1551 %0 = phi i16 [ %.pre, %entry ], [ %load2, %loop ]
1552 %iv = phi i64 [ 0, %entry ], [ %iv2, %loop ]
1553 %i = phi i64 [ 0, %entry ], [ %i1, %loop ]
1554 %conv = sext i16 %0 to i32
1555 %i1 = add nuw nsw i64 %i, 1
1556 %iv1 = add nuw nsw i64 %iv, 1
1557 %iv2 = add nuw nsw i64 %iv, 2
1558 %gep1 = getelementptr inbounds i16, i16* %a, i64 %iv1
1559 %load1 = load i16, i16* %gep1, align 4
1560 %conv1 = sext i16 %load1 to i32
1561 %gep2 = getelementptr inbounds i16, i16* %a, i64 %iv2
1562 %load2 = load i16, i16* %gep2, align 4
1563 %conv2 = sext i16 %load2 to i32
1564 %mul01 = mul nsw i32 %conv, %conv1
1565 %mul012 = mul nsw i32 %mul01, %conv2
1566 %arrayidx5 = getelementptr inbounds i32, i32* %b, i64 %i
1567 store i32 %mul012, i32* %arrayidx5
1568 %exitcond = icmp eq i64 %iv, %n
1569 br i1 %exitcond, label %end, label %loop
1571 end:
1572 ret void
1573 }
1575 attributes #1 = { "target-features"="+sve" vscale_range(1, 16) }
1576 attributes #0 = { "unsafe-fp-math"="true" "target-features"="+sve" vscale_range(1, 16) }