| blob | e333e47219116de8e59b15f643664fddf0ab7911 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE
3 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX
4 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
5 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FP
6 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FCP
7 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512
8 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512-FCP
9 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ
10 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-FCP
11 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW
12 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW-FCP
13 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512DQ-BW
14 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-BW-FCP
16 ; These patterns are produced by LoopVectorizer for interleaved stores.
18 define void @store_i32_stride2_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.vec) nounwind {
19 ; SSE-LABEL: store_i32_stride2_vf2:
20 ; SSE: # %bb.0:
21 ; SSE-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
22 ; SSE-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero
23 ; SSE-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
24 ; SSE-NEXT: movaps %xmm1, (%rdx)
25 ; SSE-NEXT: retq
26 ;
27 ; AVX-LABEL: store_i32_stride2_vf2:
28 ; AVX: # %bb.0:
29 ; AVX-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
30 ; AVX-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
31 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
32 ; AVX-NEXT: vmovaps %xmm0, (%rdx)
33 ; AVX-NEXT: retq
34 ;
35 ; AVX2-LABEL: store_i32_stride2_vf2:
36 ; AVX2: # %bb.0:
37 ; AVX2-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
38 ; AVX2-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
39 ; AVX2-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
40 ; AVX2-NEXT: vmovaps %xmm0, (%rdx)
41 ; AVX2-NEXT: retq
42 ;
43 ; AVX2-FP-LABEL: store_i32_stride2_vf2:
44 ; AVX2-FP: # %bb.0:
45 ; AVX2-FP-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
46 ; AVX2-FP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
47 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
48 ; AVX2-FP-NEXT: vmovaps %xmm0, (%rdx)
49 ; AVX2-FP-NEXT: retq
50 ;
51 ; AVX2-FCP-LABEL: store_i32_stride2_vf2:
52 ; AVX2-FCP: # %bb.0:
53 ; AVX2-FCP-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
54 ; AVX2-FCP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
55 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
56 ; AVX2-FCP-NEXT: vmovaps %xmm0, (%rdx)
57 ; AVX2-FCP-NEXT: retq
58 ;
59 ; AVX512-LABEL: store_i32_stride2_vf2:
60 ; AVX512: # %bb.0:
61 ; AVX512-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
62 ; AVX512-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
63 ; AVX512-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
64 ; AVX512-NEXT: vmovaps %xmm0, (%rdx)
65 ; AVX512-NEXT: retq
66 ;
67 ; AVX512-FCP-LABEL: store_i32_stride2_vf2:
68 ; AVX512-FCP: # %bb.0:
69 ; AVX512-FCP-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
70 ; AVX512-FCP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
71 ; AVX512-FCP-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
72 ; AVX512-FCP-NEXT: vmovaps %xmm0, (%rdx)
73 ; AVX512-FCP-NEXT: retq
74 ;
75 ; AVX512DQ-LABEL: store_i32_stride2_vf2:
76 ; AVX512DQ: # %bb.0:
77 ; AVX512DQ-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
78 ; AVX512DQ-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
79 ; AVX512DQ-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
80 ; AVX512DQ-NEXT: vmovaps %xmm0, (%rdx)
81 ; AVX512DQ-NEXT: retq
82 ;
83 ; AVX512DQ-FCP-LABEL: store_i32_stride2_vf2:
84 ; AVX512DQ-FCP: # %bb.0:
85 ; AVX512DQ-FCP-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
86 ; AVX512DQ-FCP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
87 ; AVX512DQ-FCP-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
88 ; AVX512DQ-FCP-NEXT: vmovaps %xmm0, (%rdx)
89 ; AVX512DQ-FCP-NEXT: retq
90 ;
91 ; AVX512BW-LABEL: store_i32_stride2_vf2:
92 ; AVX512BW: # %bb.0:
93 ; AVX512BW-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
94 ; AVX512BW-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
95 ; AVX512BW-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
96 ; AVX512BW-NEXT: vmovaps %xmm0, (%rdx)
97 ; AVX512BW-NEXT: retq
98 ;
99 ; AVX512BW-FCP-LABEL: store_i32_stride2_vf2:
100 ; AVX512BW-FCP: # %bb.0:
101 ; AVX512BW-FCP-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
102 ; AVX512BW-FCP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
103 ; AVX512BW-FCP-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
104 ; AVX512BW-FCP-NEXT: vmovaps %xmm0, (%rdx)
105 ; AVX512BW-FCP-NEXT: retq
106 ;
107 ; AVX512DQ-BW-LABEL: store_i32_stride2_vf2:
108 ; AVX512DQ-BW: # %bb.0:
109 ; AVX512DQ-BW-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
110 ; AVX512DQ-BW-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
111 ; AVX512DQ-BW-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
112 ; AVX512DQ-BW-NEXT: vmovaps %xmm0, (%rdx)
113 ; AVX512DQ-BW-NEXT: retq
114 ;
115 ; AVX512DQ-BW-FCP-LABEL: store_i32_stride2_vf2:
116 ; AVX512DQ-BW-FCP: # %bb.0:
117 ; AVX512DQ-BW-FCP-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
118 ; AVX512DQ-BW-FCP-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
119 ; AVX512DQ-BW-FCP-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
120 ; AVX512DQ-BW-FCP-NEXT: vmovaps %xmm0, (%rdx)
121 ; AVX512DQ-BW-FCP-NEXT: retq
122 %in.vec0 = load <2 x i32>, ptr %in.vecptr0, align 64
123 %in.vec1 = load <2 x i32>, ptr %in.vecptr1, align 64
124 %1 = shufflevector <2 x i32> %in.vec0, <2 x i32> %in.vec1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
125 %interleaved.vec = shufflevector <4 x i32> %1, <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
126 store <4 x i32> %interleaved.vec, ptr %out.vec, align 64
127 ret void
128 }
130 define void @store_i32_stride2_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.vec) nounwind {
131 ; SSE-LABEL: store_i32_stride2_vf4:
132 ; SSE: # %bb.0:
133 ; SSE-NEXT: movaps (%rdi), %xmm0
134 ; SSE-NEXT: movaps (%rsi), %xmm1
135 ; SSE-NEXT: movaps %xmm0, %xmm2
136 ; SSE-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
137 ; SSE-NEXT: unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
138 ; SSE-NEXT: movaps %xmm0, 16(%rdx)
139 ; SSE-NEXT: movaps %xmm2, (%rdx)
140 ; SSE-NEXT: retq
141 ;
142 ; AVX-LABEL: store_i32_stride2_vf4:
143 ; AVX: # %bb.0:
144 ; AVX-NEXT: vmovaps (%rdi), %xmm0
145 ; AVX-NEXT: vmovaps (%rsi), %xmm1
146 ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm2
147 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
148 ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0,3,1,6,4,7,5]
149 ; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm2[0,2,1,3,4,6,5,7]
150 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6],ymm1[7]
151 ; AVX-NEXT: vmovaps %ymm0, (%rdx)
152 ; AVX-NEXT: vzeroupper
153 ; AVX-NEXT: retq
154 ;
155 ; AVX2-LABEL: store_i32_stride2_vf4:
156 ; AVX2: # %bb.0:
157 ; AVX2-NEXT: vmovaps (%rdi), %xmm0
158 ; AVX2-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0
159 ; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,4,1,5,2,6,3,7]
160 ; AVX2-NEXT: vpermps %ymm0, %ymm1, %ymm0
161 ; AVX2-NEXT: vmovaps %ymm0, (%rdx)
162 ; AVX2-NEXT: vzeroupper
163 ; AVX2-NEXT: retq
164 ;
165 ; AVX2-FP-LABEL: store_i32_stride2_vf4:
166 ; AVX2-FP: # %bb.0:
167 ; AVX2-FP-NEXT: vmovaps (%rdi), %xmm0
168 ; AVX2-FP-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0
169 ; AVX2-FP-NEXT: vmovaps {{.*#+}} ymm1 = [0,4,1,5,2,6,3,7]
170 ; AVX2-FP-NEXT: vpermps %ymm0, %ymm1, %ymm0
171 ; AVX2-FP-NEXT: vmovaps %ymm0, (%rdx)
172 ; AVX2-FP-NEXT: vzeroupper
173 ; AVX2-FP-NEXT: retq
174 ;
175 ; AVX2-FCP-LABEL: store_i32_stride2_vf4:
176 ; AVX2-FCP: # %bb.0:
177 ; AVX2-FCP-NEXT: vmovaps (%rdi), %xmm0
178 ; AVX2-FCP-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0
179 ; AVX2-FCP-NEXT: vmovaps {{.*#+}} ymm1 = [0,4,1,5,2,6,3,7]
180 ; AVX2-FCP-NEXT: vpermps %ymm0, %ymm1, %ymm0
181 ; AVX2-FCP-NEXT: vmovaps %ymm0, (%rdx)
182 ; AVX2-FCP-NEXT: vzeroupper
183 ; AVX2-FCP-NEXT: retq
184 ;
185 ; AVX512-LABEL: store_i32_stride2_vf4:
186 ; AVX512: # %bb.0:
187 ; AVX512-NEXT: vmovaps (%rdi), %xmm0
188 ; AVX512-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0
189 ; AVX512-NEXT: vmovaps {{.*#+}} ymm1 = [0,4,1,5,2,6,3,7]
190 ; AVX512-NEXT: vpermps %ymm0, %ymm1, %ymm0
191 ; AVX512-NEXT: vmovaps %ymm0, (%rdx)
192 ; AVX512-NEXT: vzeroupper
193 ; AVX512-NEXT: retq
194 ;
195 ; AVX512-FCP-LABEL: store_i32_stride2_vf4:
196 ; AVX512-FCP: # %bb.0:
197 ; AVX512-FCP-NEXT: vmovaps (%rdi), %xmm0
198 ; AVX512-FCP-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0
199 ; AVX512-FCP-NEXT: vmovaps {{.*#+}} ymm1 = [0,4,1,5,2,6,3,7]
200 ; AVX512-FCP-NEXT: vpermps %ymm0, %ymm1, %ymm0
201 ; AVX512-FCP-NEXT: vmovaps %ymm0, (%rdx)
202 ; AVX512-FCP-NEXT: vzeroupper
203 ; AVX512-FCP-NEXT: retq
204 ;
205 ; AVX512DQ-LABEL: store_i32_stride2_vf4:
206 ; AVX512DQ: # %bb.0:
207 ; AVX512DQ-NEXT: vmovaps (%rdi), %xmm0
208 ; AVX512DQ-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0
209 ; AVX512DQ-NEXT: vmovaps {{.*#+}} ymm1 = [0,4,1,5,2,6,3,7]
210 ; AVX512DQ-NEXT: vpermps %ymm0, %ymm1, %ymm0
211 ; AVX512DQ-NEXT: vmovaps %ymm0, (%rdx)
212 ; AVX512DQ-NEXT: vzeroupper
213 ; AVX512DQ-NEXT: retq
214 ;
215 ; AVX512DQ-FCP-LABEL: store_i32_stride2_vf4:
216 ; AVX512DQ-FCP: # %bb.0:
217 ; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %xmm0
218 ; AVX512DQ-FCP-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0
219 ; AVX512DQ-FCP-NEXT: vmovaps {{.*#+}} ymm1 = [0,4,1,5,2,6,3,7]
220 ; AVX512DQ-FCP-NEXT: vpermps %ymm0, %ymm1, %ymm0
221 ; AVX512DQ-FCP-NEXT: vmovaps %ymm0, (%rdx)
222 ; AVX512DQ-FCP-NEXT: vzeroupper
223 ; AVX512DQ-FCP-NEXT: retq
224 ;
225 ; AVX512BW-LABEL: store_i32_stride2_vf4:
226 ; AVX512BW: # %bb.0:
227 ; AVX512BW-NEXT: vmovaps (%rdi), %xmm0
228 ; AVX512BW-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0
229 ; AVX512BW-NEXT: vmovaps {{.*#+}} ymm1 = [0,4,1,5,2,6,3,7]
230 ; AVX512BW-NEXT: vpermps %ymm0, %ymm1, %ymm0
231 ; AVX512BW-NEXT: vmovaps %ymm0, (%rdx)
232 ; AVX512BW-NEXT: vzeroupper
233 ; AVX512BW-NEXT: retq
234 ;
235 ; AVX512BW-FCP-LABEL: store_i32_stride2_vf4:
236 ; AVX512BW-FCP: # %bb.0:
237 ; AVX512BW-FCP-NEXT: vmovaps (%rdi), %xmm0
238 ; AVX512BW-FCP-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0
239 ; AVX512BW-FCP-NEXT: vmovaps {{.*#+}} ymm1 = [0,4,1,5,2,6,3,7]
240 ; AVX512BW-FCP-NEXT: vpermps %ymm0, %ymm1, %ymm0
241 ; AVX512BW-FCP-NEXT: vmovaps %ymm0, (%rdx)
242 ; AVX512BW-FCP-NEXT: vzeroupper
243 ; AVX512BW-FCP-NEXT: retq
244 ;
245 ; AVX512DQ-BW-LABEL: store_i32_stride2_vf4:
246 ; AVX512DQ-BW: # %bb.0:
247 ; AVX512DQ-BW-NEXT: vmovaps (%rdi), %xmm0
248 ; AVX512DQ-BW-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0
249 ; AVX512DQ-BW-NEXT: vmovaps {{.*#+}} ymm1 = [0,4,1,5,2,6,3,7]
250 ; AVX512DQ-BW-NEXT: vpermps %ymm0, %ymm1, %ymm0
251 ; AVX512DQ-BW-NEXT: vmovaps %ymm0, (%rdx)
252 ; AVX512DQ-BW-NEXT: vzeroupper
253 ; AVX512DQ-BW-NEXT: retq
254 ;
255 ; AVX512DQ-BW-FCP-LABEL: store_i32_stride2_vf4:
256 ; AVX512DQ-BW-FCP: # %bb.0:
257 ; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %xmm0
258 ; AVX512DQ-BW-FCP-NEXT: vinsertf128 $1, (%rsi), %ymm0, %ymm0
259 ; AVX512DQ-BW-FCP-NEXT: vmovaps {{.*#+}} ymm1 = [0,4,1,5,2,6,3,7]
260 ; AVX512DQ-BW-FCP-NEXT: vpermps %ymm0, %ymm1, %ymm0
261 ; AVX512DQ-BW-FCP-NEXT: vmovaps %ymm0, (%rdx)
262 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
263 ; AVX512DQ-BW-FCP-NEXT: retq
264 %in.vec0 = load <4 x i32>, ptr %in.vecptr0, align 64
265 %in.vec1 = load <4 x i32>, ptr %in.vecptr1, align 64
266 %1 = shufflevector <4 x i32> %in.vec0, <4 x i32> %in.vec1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
267 %interleaved.vec = shufflevector <8 x i32> %1, <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
268 store <8 x i32> %interleaved.vec, ptr %out.vec, align 64
269 ret void
270 }
272 define void @store_i32_stride2_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.vec) nounwind {
273 ; SSE-LABEL: store_i32_stride2_vf8:
274 ; SSE: # %bb.0:
275 ; SSE-NEXT: movaps (%rdi), %xmm0
276 ; SSE-NEXT: movaps 16(%rdi), %xmm1
277 ; SSE-NEXT: movaps (%rsi), %xmm2
278 ; SSE-NEXT: movaps 16(%rsi), %xmm3
279 ; SSE-NEXT: movaps %xmm0, %xmm4
280 ; SSE-NEXT: unpckhps {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
281 ; SSE-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
282 ; SSE-NEXT: movaps %xmm1, %xmm2
283 ; SSE-NEXT: unpckhps {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
284 ; SSE-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
285 ; SSE-NEXT: movaps %xmm1, 32(%rdx)
286 ; SSE-NEXT: movaps %xmm2, 48(%rdx)
287 ; SSE-NEXT: movaps %xmm0, (%rdx)
288 ; SSE-NEXT: movaps %xmm4, 16(%rdx)
289 ; SSE-NEXT: retq
290 ;
291 ; AVX-LABEL: store_i32_stride2_vf8:
292 ; AVX: # %bb.0:
293 ; AVX-NEXT: vmovaps (%rsi), %xmm0
294 ; AVX-NEXT: vmovaps 16(%rsi), %xmm1
295 ; AVX-NEXT: vmovaps (%rdi), %xmm2
296 ; AVX-NEXT: vmovaps 16(%rdi), %xmm3
297 ; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm2[2],xmm0[2],xmm2[3],xmm0[3]
298 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
299 ; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0
300 ; AVX-NEXT: vunpckhps {{.*#+}} xmm2 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
301 ; AVX-NEXT: vunpcklps {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
302 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
303 ; AVX-NEXT: vmovaps %ymm1, 32(%rdx)
304 ; AVX-NEXT: vmovaps %ymm0, (%rdx)
305 ; AVX-NEXT: vzeroupper
306 ; AVX-NEXT: retq
307 ;
308 ; AVX2-LABEL: store_i32_stride2_vf8:
309 ; AVX2: # %bb.0:
310 ; AVX2-NEXT: vmovaps (%rdi), %ymm0
311 ; AVX2-NEXT: vmovaps (%rsi), %ymm1
312 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
313 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
314 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm0[0,1],ymm2[0,1]
315 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
316 ; AVX2-NEXT: vmovaps %ymm0, 32(%rdx)
317 ; AVX2-NEXT: vmovaps %ymm1, (%rdx)
318 ; AVX2-NEXT: vzeroupper
319 ; AVX2-NEXT: retq
320 ;
321 ; AVX2-FP-LABEL: store_i32_stride2_vf8:
322 ; AVX2-FP: # %bb.0:
323 ; AVX2-FP-NEXT: vmovaps (%rdi), %ymm0
324 ; AVX2-FP-NEXT: vmovaps (%rsi), %ymm1
325 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
326 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
327 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm0[0,1],ymm2[0,1]
328 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
329 ; AVX2-FP-NEXT: vmovaps %ymm0, 32(%rdx)
330 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rdx)
331 ; AVX2-FP-NEXT: vzeroupper
332 ; AVX2-FP-NEXT: retq
333 ;
334 ; AVX2-FCP-LABEL: store_i32_stride2_vf8:
335 ; AVX2-FCP: # %bb.0:
336 ; AVX2-FCP-NEXT: vmovaps (%rdi), %ymm0
337 ; AVX2-FCP-NEXT: vmovaps (%rsi), %ymm1
338 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
339 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
340 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm0[0,1],ymm2[0,1]
341 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3]
342 ; AVX2-FCP-NEXT: vmovaps %ymm0, 32(%rdx)
343 ; AVX2-FCP-NEXT: vmovaps %ymm1, (%rdx)
344 ; AVX2-FCP-NEXT: vzeroupper
345 ; AVX2-FCP-NEXT: retq
346 ;
347 ; AVX512-LABEL: store_i32_stride2_vf8:
348 ; AVX512: # %bb.0:
349 ; AVX512-NEXT: vmovaps (%rdi), %ymm0
350 ; AVX512-NEXT: vinsertf64x4 $1, (%rsi), %zmm0, %zmm0
351 ; AVX512-NEXT: vmovaps {{.*#+}} zmm1 = [0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15]
352 ; AVX512-NEXT: vpermps %zmm0, %zmm1, %zmm0
353 ; AVX512-NEXT: vmovaps %zmm0, (%rdx)
354 ; AVX512-NEXT: vzeroupper
355 ; AVX512-NEXT: retq
356 ;
357 ; AVX512-FCP-LABEL: store_i32_stride2_vf8:
358 ; AVX512-FCP: # %bb.0:
359 ; AVX512-FCP-NEXT: vmovaps (%rdi), %ymm0
360 ; AVX512-FCP-NEXT: vinsertf64x4 $1, (%rsi), %zmm0, %zmm0
361 ; AVX512-FCP-NEXT: vmovaps {{.*#+}} zmm1 = [0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15]
362 ; AVX512-FCP-NEXT: vpermps %zmm0, %zmm1, %zmm0
363 ; AVX512-FCP-NEXT: vmovaps %zmm0, (%rdx)
364 ; AVX512-FCP-NEXT: vzeroupper
365 ; AVX512-FCP-NEXT: retq
366 ;
367 ; AVX512DQ-LABEL: store_i32_stride2_vf8:
368 ; AVX512DQ: # %bb.0:
369 ; AVX512DQ-NEXT: vmovaps (%rdi), %ymm0
370 ; AVX512DQ-NEXT: vinsertf64x4 $1, (%rsi), %zmm0, %zmm0
371 ; AVX512DQ-NEXT: vmovaps {{.*#+}} zmm1 = [0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15]
372 ; AVX512DQ-NEXT: vpermps %zmm0, %zmm1, %zmm0
373 ; AVX512DQ-NEXT: vmovaps %zmm0, (%rdx)
374 ; AVX512DQ-NEXT: vzeroupper
375 ; AVX512DQ-NEXT: retq
376 ;
377 ; AVX512DQ-FCP-LABEL: store_i32_stride2_vf8:
378 ; AVX512DQ-FCP: # %bb.0:
379 ; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %ymm0
380 ; AVX512DQ-FCP-NEXT: vinsertf64x4 $1, (%rsi), %zmm0, %zmm0
381 ; AVX512DQ-FCP-NEXT: vmovaps {{.*#+}} zmm1 = [0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15]
382 ; AVX512DQ-FCP-NEXT: vpermps %zmm0, %zmm1, %zmm0
383 ; AVX512DQ-FCP-NEXT: vmovaps %zmm0, (%rdx)
384 ; AVX512DQ-FCP-NEXT: vzeroupper
385 ; AVX512DQ-FCP-NEXT: retq
386 ;
387 ; AVX512BW-LABEL: store_i32_stride2_vf8:
388 ; AVX512BW: # %bb.0:
389 ; AVX512BW-NEXT: vmovaps (%rdi), %ymm0
390 ; AVX512BW-NEXT: vinsertf64x4 $1, (%rsi), %zmm0, %zmm0
391 ; AVX512BW-NEXT: vmovaps {{.*#+}} zmm1 = [0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15]
392 ; AVX512BW-NEXT: vpermps %zmm0, %zmm1, %zmm0
393 ; AVX512BW-NEXT: vmovaps %zmm0, (%rdx)
394 ; AVX512BW-NEXT: vzeroupper
395 ; AVX512BW-NEXT: retq
396 ;
397 ; AVX512BW-FCP-LABEL: store_i32_stride2_vf8:
398 ; AVX512BW-FCP: # %bb.0:
399 ; AVX512BW-FCP-NEXT: vmovaps (%rdi), %ymm0
400 ; AVX512BW-FCP-NEXT: vinsertf64x4 $1, (%rsi), %zmm0, %zmm0
401 ; AVX512BW-FCP-NEXT: vmovaps {{.*#+}} zmm1 = [0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15]
402 ; AVX512BW-FCP-NEXT: vpermps %zmm0, %zmm1, %zmm0
403 ; AVX512BW-FCP-NEXT: vmovaps %zmm0, (%rdx)
404 ; AVX512BW-FCP-NEXT: vzeroupper
405 ; AVX512BW-FCP-NEXT: retq
406 ;
407 ; AVX512DQ-BW-LABEL: store_i32_stride2_vf8:
408 ; AVX512DQ-BW: # %bb.0:
409 ; AVX512DQ-BW-NEXT: vmovaps (%rdi), %ymm0
410 ; AVX512DQ-BW-NEXT: vinsertf64x4 $1, (%rsi), %zmm0, %zmm0
411 ; AVX512DQ-BW-NEXT: vmovaps {{.*#+}} zmm1 = [0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15]
412 ; AVX512DQ-BW-NEXT: vpermps %zmm0, %zmm1, %zmm0
413 ; AVX512DQ-BW-NEXT: vmovaps %zmm0, (%rdx)
414 ; AVX512DQ-BW-NEXT: vzeroupper
415 ; AVX512DQ-BW-NEXT: retq
416 ;
417 ; AVX512DQ-BW-FCP-LABEL: store_i32_stride2_vf8:
418 ; AVX512DQ-BW-FCP: # %bb.0:
419 ; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %ymm0
420 ; AVX512DQ-BW-FCP-NEXT: vinsertf64x4 $1, (%rsi), %zmm0, %zmm0
421 ; AVX512DQ-BW-FCP-NEXT: vmovaps {{.*#+}} zmm1 = [0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15]
422 ; AVX512DQ-BW-FCP-NEXT: vpermps %zmm0, %zmm1, %zmm0
423 ; AVX512DQ-BW-FCP-NEXT: vmovaps %zmm0, (%rdx)
424 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
425 ; AVX512DQ-BW-FCP-NEXT: retq
426 %in.vec0 = load <8 x i32>, ptr %in.vecptr0, align 64
427 %in.vec1 = load <8 x i32>, ptr %in.vecptr1, align 64
428 %1 = shufflevector <8 x i32> %in.vec0, <8 x i32> %in.vec1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
429 %interleaved.vec = shufflevector <16 x i32> %1, <16 x i32> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
430 store <16 x i32> %interleaved.vec, ptr %out.vec, align 64
431 ret void
432 }
434 define void @store_i32_stride2_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.vec) nounwind {
435 ; SSE-LABEL: store_i32_stride2_vf16:
436 ; SSE: # %bb.0:
437 ; SSE-NEXT: movaps (%rdi), %xmm0
438 ; SSE-NEXT: movaps 16(%rdi), %xmm1
439 ; SSE-NEXT: movaps 32(%rdi), %xmm2
440 ; SSE-NEXT: movaps 48(%rdi), %xmm3
441 ; SSE-NEXT: movaps (%rsi), %xmm4
442 ; SSE-NEXT: movaps 16(%rsi), %xmm5
443 ; SSE-NEXT: movaps 32(%rsi), %xmm6
444 ; SSE-NEXT: movaps 48(%rsi), %xmm7
445 ; SSE-NEXT: movaps %xmm0, %xmm8
446 ; SSE-NEXT: unpckhps {{.*#+}} xmm8 = xmm8[2],xmm4[2],xmm8[3],xmm4[3]
447 ; SSE-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
448 ; SSE-NEXT: movaps %xmm1, %xmm4
449 ; SSE-NEXT: unpckhps {{.*#+}} xmm4 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
450 ; SSE-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
451 ; SSE-NEXT: movaps %xmm2, %xmm5
452 ; SSE-NEXT: unpckhps {{.*#+}} xmm5 = xmm5[2],xmm6[2],xmm5[3],xmm6[3]
453 ; SSE-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1]
454 ; SSE-NEXT: movaps %xmm3, %xmm6
455 ; SSE-NEXT: unpckhps {{.*#+}} xmm6 = xmm6[2],xmm7[2],xmm6[3],xmm7[3]
456 ; SSE-NEXT: unpcklps {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1]
457 ; SSE-NEXT: movaps %xmm3, 96(%rdx)
458 ; SSE-NEXT: movaps %xmm6, 112(%rdx)
459 ; SSE-NEXT: movaps %xmm2, 64(%rdx)
460 ; SSE-NEXT: movaps %xmm5, 80(%rdx)
461 ; SSE-NEXT: movaps %xmm1, 32(%rdx)
462 ; SSE-NEXT: movaps %xmm4, 48(%rdx)
463 ; SSE-NEXT: movaps %xmm0, (%rdx)
464 ; SSE-NEXT: movaps %xmm8, 16(%rdx)
465 ; SSE-NEXT: retq
466 ;
467 ; AVX-LABEL: store_i32_stride2_vf16:
468 ; AVX: # %bb.0:
469 ; AVX-NEXT: vmovaps (%rsi), %xmm0
470 ; AVX-NEXT: vmovaps 16(%rsi), %xmm1
471 ; AVX-NEXT: vmovaps 32(%rsi), %xmm2
472 ; AVX-NEXT: vmovaps 48(%rsi), %xmm3
473 ; AVX-NEXT: vmovaps (%rdi), %xmm4
474 ; AVX-NEXT: vmovaps 16(%rdi), %xmm5
475 ; AVX-NEXT: vmovaps 32(%rdi), %xmm6
476 ; AVX-NEXT: vmovaps 48(%rdi), %xmm7
477 ; AVX-NEXT: vunpckhps {{.*#+}} xmm8 = xmm5[2],xmm1[2],xmm5[3],xmm1[3]
478 ; AVX-NEXT: vunpcklps {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
479 ; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm1, %ymm1
480 ; AVX-NEXT: vunpckhps {{.*#+}} xmm5 = xmm4[2],xmm0[2],xmm4[3],xmm0[3]
481 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
482 ; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm0
483 ; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
484 ; AVX-NEXT: vunpcklps {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
485 ; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm2, %ymm2
486 ; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm7[2],xmm3[2],xmm7[3],xmm3[3]
487 ; AVX-NEXT: vunpcklps {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1]
488 ; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
489 ; AVX-NEXT: vmovaps %ymm3, 96(%rdx)
490 ; AVX-NEXT: vmovaps %ymm2, 64(%rdx)
491 ; AVX-NEXT: vmovaps %ymm0, (%rdx)
492 ; AVX-NEXT: vmovaps %ymm1, 32(%rdx)
493 ; AVX-NEXT: vzeroupper
494 ; AVX-NEXT: retq
495 ;
496 ; AVX2-LABEL: store_i32_stride2_vf16:
497 ; AVX2: # %bb.0:
498 ; AVX2-NEXT: vmovaps (%rdi), %ymm0
499 ; AVX2-NEXT: vmovaps 32(%rdi), %ymm1
500 ; AVX2-NEXT: vmovaps (%rsi), %ymm2
501 ; AVX2-NEXT: vmovaps 32(%rsi), %ymm3
502 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm4 = ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[6],ymm2[6],ymm0[7],ymm2[7]
503 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
504 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm4[2,3]
505 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm4[0,1]
506 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm4 = ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[6],ymm3[6],ymm1[7],ymm3[7]
507 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5]
508 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm1[2,3],ymm4[2,3]
509 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm4[0,1]
510 ; AVX2-NEXT: vmovaps %ymm1, 64(%rdx)
511 ; AVX2-NEXT: vmovaps %ymm3, 96(%rdx)
512 ; AVX2-NEXT: vmovaps %ymm0, (%rdx)
513 ; AVX2-NEXT: vmovaps %ymm2, 32(%rdx)
514 ; AVX2-NEXT: vzeroupper
515 ; AVX2-NEXT: retq
516 ;
517 ; AVX2-FP-LABEL: store_i32_stride2_vf16:
518 ; AVX2-FP: # %bb.0:
519 ; AVX2-FP-NEXT: vmovaps (%rdi), %ymm0
520 ; AVX2-FP-NEXT: vmovaps 32(%rdi), %ymm1
521 ; AVX2-FP-NEXT: vmovaps (%rsi), %ymm2
522 ; AVX2-FP-NEXT: vmovaps 32(%rsi), %ymm3
523 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm4 = ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[6],ymm2[6],ymm0[7],ymm2[7]
524 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
525 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm4[2,3]
526 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm4[0,1]
527 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm4 = ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[6],ymm3[6],ymm1[7],ymm3[7]
528 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5]
529 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm1[2,3],ymm4[2,3]
530 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm4[0,1]
531 ; AVX2-FP-NEXT: vmovaps %ymm1, 64(%rdx)
532 ; AVX2-FP-NEXT: vmovaps %ymm3, 96(%rdx)
533 ; AVX2-FP-NEXT: vmovaps %ymm0, (%rdx)
534 ; AVX2-FP-NEXT: vmovaps %ymm2, 32(%rdx)
535 ; AVX2-FP-NEXT: vzeroupper
536 ; AVX2-FP-NEXT: retq
537 ;
538 ; AVX2-FCP-LABEL: store_i32_stride2_vf16:
539 ; AVX2-FCP: # %bb.0:
540 ; AVX2-FCP-NEXT: vmovaps (%rdi), %ymm0
541 ; AVX2-FCP-NEXT: vmovaps 32(%rdi), %ymm1
542 ; AVX2-FCP-NEXT: vmovaps (%rsi), %ymm2
543 ; AVX2-FCP-NEXT: vmovaps 32(%rsi), %ymm3
544 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm4 = ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[6],ymm2[6],ymm0[7],ymm2[7]
545 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[4],ymm2[4],ymm0[5],ymm2[5]
546 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm4[2,3]
547 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm4[0,1]
548 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm4 = ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[6],ymm3[6],ymm1[7],ymm3[7]
549 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5]
550 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm1[2,3],ymm4[2,3]
551 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm4[0,1]
552 ; AVX2-FCP-NEXT: vmovaps %ymm1, 64(%rdx)
553 ; AVX2-FCP-NEXT: vmovaps %ymm3, 96(%rdx)
554 ; AVX2-FCP-NEXT: vmovaps %ymm0, (%rdx)
555 ; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rdx)
556 ; AVX2-FCP-NEXT: vzeroupper
557 ; AVX2-FCP-NEXT: retq
558 ;
559 ; AVX512-LABEL: store_i32_stride2_vf16:
560 ; AVX512: # %bb.0:
561 ; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
562 ; AVX512-NEXT: vmovdqa64 (%rsi), %zmm1
563 ; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm2 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
564 ; AVX512-NEXT: vpermi2d %zmm1, %zmm0, %zmm2
565 ; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm3 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
566 ; AVX512-NEXT: vpermi2d %zmm1, %zmm0, %zmm3
567 ; AVX512-NEXT: vmovdqa64 %zmm3, 64(%rdx)
568 ; AVX512-NEXT: vmovdqa64 %zmm2, (%rdx)
569 ; AVX512-NEXT: vzeroupper
570 ; AVX512-NEXT: retq
571 ;
572 ; AVX512-FCP-LABEL: store_i32_stride2_vf16:
573 ; AVX512-FCP: # %bb.0:
574 ; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
575 ; AVX512-FCP-NEXT: vmovdqa64 (%rsi), %zmm1
576 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm2 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
577 ; AVX512-FCP-NEXT: vpermi2d %zmm1, %zmm0, %zmm2
578 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm3 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
579 ; AVX512-FCP-NEXT: vpermi2d %zmm1, %zmm0, %zmm3
580 ; AVX512-FCP-NEXT: vmovdqa64 %zmm3, 64(%rdx)
581 ; AVX512-FCP-NEXT: vmovdqa64 %zmm2, (%rdx)
582 ; AVX512-FCP-NEXT: vzeroupper
583 ; AVX512-FCP-NEXT: retq
584 ;
585 ; AVX512DQ-LABEL: store_i32_stride2_vf16:
586 ; AVX512DQ: # %bb.0:
587 ; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
588 ; AVX512DQ-NEXT: vmovdqa64 (%rsi), %zmm1
589 ; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm2 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
590 ; AVX512DQ-NEXT: vpermi2d %zmm1, %zmm0, %zmm2
591 ; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm3 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
592 ; AVX512DQ-NEXT: vpermi2d %zmm1, %zmm0, %zmm3
593 ; AVX512DQ-NEXT: vmovdqa64 %zmm3, 64(%rdx)
594 ; AVX512DQ-NEXT: vmovdqa64 %zmm2, (%rdx)
595 ; AVX512DQ-NEXT: vzeroupper
596 ; AVX512DQ-NEXT: retq
597 ;
598 ; AVX512DQ-FCP-LABEL: store_i32_stride2_vf16:
599 ; AVX512DQ-FCP: # %bb.0:
600 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
601 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rsi), %zmm1
602 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm2 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
603 ; AVX512DQ-FCP-NEXT: vpermi2d %zmm1, %zmm0, %zmm2
604 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm3 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
605 ; AVX512DQ-FCP-NEXT: vpermi2d %zmm1, %zmm0, %zmm3
606 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, 64(%rdx)
607 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, (%rdx)
608 ; AVX512DQ-FCP-NEXT: vzeroupper
609 ; AVX512DQ-FCP-NEXT: retq
610 ;
611 ; AVX512BW-LABEL: store_i32_stride2_vf16:
612 ; AVX512BW: # %bb.0:
613 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
614 ; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm1
615 ; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm2 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
616 ; AVX512BW-NEXT: vpermi2d %zmm1, %zmm0, %zmm2
617 ; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm3 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
618 ; AVX512BW-NEXT: vpermi2d %zmm1, %zmm0, %zmm3
619 ; AVX512BW-NEXT: vmovdqa64 %zmm3, 64(%rdx)
620 ; AVX512BW-NEXT: vmovdqa64 %zmm2, (%rdx)
621 ; AVX512BW-NEXT: vzeroupper
622 ; AVX512BW-NEXT: retq
623 ;
624 ; AVX512BW-FCP-LABEL: store_i32_stride2_vf16:
625 ; AVX512BW-FCP: # %bb.0:
626 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
627 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm1
628 ; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm2 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
629 ; AVX512BW-FCP-NEXT: vpermi2d %zmm1, %zmm0, %zmm2
630 ; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm3 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
631 ; AVX512BW-FCP-NEXT: vpermi2d %zmm1, %zmm0, %zmm3
632 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, 64(%rdx)
633 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, (%rdx)
634 ; AVX512BW-FCP-NEXT: vzeroupper
635 ; AVX512BW-FCP-NEXT: retq
636 ;
637 ; AVX512DQ-BW-LABEL: store_i32_stride2_vf16:
638 ; AVX512DQ-BW: # %bb.0:
639 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
640 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rsi), %zmm1
641 ; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} zmm2 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
642 ; AVX512DQ-BW-NEXT: vpermi2d %zmm1, %zmm0, %zmm2
643 ; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} zmm3 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
644 ; AVX512DQ-BW-NEXT: vpermi2d %zmm1, %zmm0, %zmm3
645 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, 64(%rdx)
646 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, (%rdx)
647 ; AVX512DQ-BW-NEXT: vzeroupper
648 ; AVX512DQ-BW-NEXT: retq
649 ;
650 ; AVX512DQ-BW-FCP-LABEL: store_i32_stride2_vf16:
651 ; AVX512DQ-BW-FCP: # %bb.0:
652 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
653 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm1
654 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm2 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
655 ; AVX512DQ-BW-FCP-NEXT: vpermi2d %zmm1, %zmm0, %zmm2
656 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm3 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
657 ; AVX512DQ-BW-FCP-NEXT: vpermi2d %zmm1, %zmm0, %zmm3
658 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, 64(%rdx)
659 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, (%rdx)
660 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
661 ; AVX512DQ-BW-FCP-NEXT: retq
662 %in.vec0 = load <16 x i32>, ptr %in.vecptr0, align 64
663 %in.vec1 = load <16 x i32>, ptr %in.vecptr1, align 64
664 %1 = shufflevector <16 x i32> %in.vec0, <16 x i32> %in.vec1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
665 %interleaved.vec = shufflevector <32 x i32> %1, <32 x i32> poison, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
666 store <32 x i32> %interleaved.vec, ptr %out.vec, align 64
667 ret void
668 }
670 define void @store_i32_stride2_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.vec) nounwind {
671 ; SSE-LABEL: store_i32_stride2_vf32:
672 ; SSE: # %bb.0:
673 ; SSE-NEXT: movaps 112(%rdi), %xmm0
674 ; SSE-NEXT: movaps 96(%rdi), %xmm6
675 ; SSE-NEXT: movaps 80(%rdi), %xmm4
676 ; SSE-NEXT: movaps 64(%rdi), %xmm3
677 ; SSE-NEXT: movaps (%rdi), %xmm8
678 ; SSE-NEXT: movaps 16(%rdi), %xmm1
679 ; SSE-NEXT: movaps 32(%rdi), %xmm2
680 ; SSE-NEXT: movaps 48(%rdi), %xmm5
681 ; SSE-NEXT: movaps 96(%rsi), %xmm11
682 ; SSE-NEXT: movaps 80(%rsi), %xmm12
683 ; SSE-NEXT: movaps 64(%rsi), %xmm13
684 ; SSE-NEXT: movaps (%rsi), %xmm9
685 ; SSE-NEXT: movaps 16(%rsi), %xmm10
686 ; SSE-NEXT: movaps 32(%rsi), %xmm14
687 ; SSE-NEXT: movaps 48(%rsi), %xmm15
688 ; SSE-NEXT: movaps %xmm8, %xmm7
689 ; SSE-NEXT: unpckhps {{.*#+}} xmm7 = xmm7[2],xmm9[2],xmm7[3],xmm9[3]
690 ; SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
691 ; SSE-NEXT: unpcklps {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1]
692 ; SSE-NEXT: movaps %xmm1, %xmm9
693 ; SSE-NEXT: unpckhps {{.*#+}} xmm9 = xmm9[2],xmm10[2],xmm9[3],xmm10[3]
694 ; SSE-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1]
695 ; SSE-NEXT: movaps %xmm2, %xmm10
696 ; SSE-NEXT: unpckhps {{.*#+}} xmm10 = xmm10[2],xmm14[2],xmm10[3],xmm14[3]
697 ; SSE-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm14[0],xmm2[1],xmm14[1]
698 ; SSE-NEXT: movaps %xmm5, %xmm14
699 ; SSE-NEXT: unpckhps {{.*#+}} xmm14 = xmm14[2],xmm15[2],xmm14[3],xmm15[3]
700 ; SSE-NEXT: unpcklps {{.*#+}} xmm5 = xmm5[0],xmm15[0],xmm5[1],xmm15[1]
701 ; SSE-NEXT: movaps %xmm3, %xmm15
702 ; SSE-NEXT: unpckhps {{.*#+}} xmm15 = xmm15[2],xmm13[2],xmm15[3],xmm13[3]
703 ; SSE-NEXT: unpcklps {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1]
704 ; SSE-NEXT: movaps %xmm4, %xmm13
705 ; SSE-NEXT: unpckhps {{.*#+}} xmm13 = xmm13[2],xmm12[2],xmm13[3],xmm12[3]
706 ; SSE-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1]
707 ; SSE-NEXT: movaps %xmm6, %xmm12
708 ; SSE-NEXT: unpckhps {{.*#+}} xmm12 = xmm12[2],xmm11[2],xmm12[3],xmm11[3]
709 ; SSE-NEXT: unpcklps {{.*#+}} xmm6 = xmm6[0],xmm11[0],xmm6[1],xmm11[1]
710 ; SSE-NEXT: movaps 112(%rsi), %xmm11
711 ; SSE-NEXT: movaps %xmm0, %xmm7
712 ; SSE-NEXT: unpckhps {{.*#+}} xmm7 = xmm7[2],xmm11[2],xmm7[3],xmm11[3]
713 ; SSE-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm11[0],xmm0[1],xmm11[1]
714 ; SSE-NEXT: movaps %xmm0, 224(%rdx)
715 ; SSE-NEXT: movaps %xmm7, 240(%rdx)
716 ; SSE-NEXT: movaps %xmm6, 192(%rdx)
717 ; SSE-NEXT: movaps %xmm12, 208(%rdx)
718 ; SSE-NEXT: movaps %xmm4, 160(%rdx)
719 ; SSE-NEXT: movaps %xmm13, 176(%rdx)
720 ; SSE-NEXT: movaps %xmm3, 128(%rdx)
721 ; SSE-NEXT: movaps %xmm15, 144(%rdx)
722 ; SSE-NEXT: movaps %xmm5, 96(%rdx)
723 ; SSE-NEXT: movaps %xmm14, 112(%rdx)
724 ; SSE-NEXT: movaps %xmm2, 64(%rdx)
725 ; SSE-NEXT: movaps %xmm10, 80(%rdx)
726 ; SSE-NEXT: movaps %xmm1, 32(%rdx)
727 ; SSE-NEXT: movaps %xmm9, 48(%rdx)
728 ; SSE-NEXT: movaps %xmm8, (%rdx)
729 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
730 ; SSE-NEXT: movaps %xmm0, 16(%rdx)
731 ; SSE-NEXT: retq
732 ;
733 ; AVX-LABEL: store_i32_stride2_vf32:
734 ; AVX: # %bb.0:
735 ; AVX-NEXT: vmovaps 112(%rsi), %xmm0
736 ; AVX-NEXT: vmovaps 112(%rdi), %xmm1
737 ; AVX-NEXT: vunpckhps {{.*#+}} xmm2 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
738 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
739 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
740 ; AVX-NEXT: vmovaps 96(%rsi), %xmm1
741 ; AVX-NEXT: vmovaps 96(%rdi), %xmm2
742 ; AVX-NEXT: vunpckhps {{.*#+}} xmm3 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
743 ; AVX-NEXT: vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
744 ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
745 ; AVX-NEXT: vmovaps 64(%rsi), %xmm2
746 ; AVX-NEXT: vmovaps 64(%rdi), %xmm3
747 ; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
748 ; AVX-NEXT: vunpcklps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
749 ; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm2, %ymm2
750 ; AVX-NEXT: vmovaps 80(%rsi), %xmm3
751 ; AVX-NEXT: vmovaps 80(%rdi), %xmm4
752 ; AVX-NEXT: vunpckhps {{.*#+}} xmm5 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
753 ; AVX-NEXT: vunpcklps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
754 ; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm3, %ymm3
755 ; AVX-NEXT: vmovaps (%rsi), %xmm4
756 ; AVX-NEXT: vmovaps 16(%rsi), %xmm5
757 ; AVX-NEXT: vmovaps 32(%rsi), %xmm6
758 ; AVX-NEXT: vmovaps 48(%rsi), %xmm7
759 ; AVX-NEXT: vmovaps (%rdi), %xmm8
760 ; AVX-NEXT: vmovaps 16(%rdi), %xmm9
761 ; AVX-NEXT: vmovaps 32(%rdi), %xmm10
762 ; AVX-NEXT: vmovaps 48(%rdi), %xmm11
763 ; AVX-NEXT: vunpckhps {{.*#+}} xmm12 = xmm8[2],xmm4[2],xmm8[3],xmm4[3]
764 ; AVX-NEXT: vunpcklps {{.*#+}} xmm4 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
765 ; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm4, %ymm4
766 ; AVX-NEXT: vunpckhps {{.*#+}} xmm8 = xmm10[2],xmm6[2],xmm10[3],xmm6[3]
767 ; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm10[0],xmm6[0],xmm10[1],xmm6[1]
768 ; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm6, %ymm6
769 ; AVX-NEXT: vunpckhps {{.*#+}} xmm8 = xmm11[2],xmm7[2],xmm11[3],xmm7[3]
770 ; AVX-NEXT: vunpcklps {{.*#+}} xmm7 = xmm11[0],xmm7[0],xmm11[1],xmm7[1]
771 ; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm7, %ymm7
772 ; AVX-NEXT: vunpckhps {{.*#+}} xmm8 = xmm9[2],xmm5[2],xmm9[3],xmm5[3]
773 ; AVX-NEXT: vunpcklps {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1]
774 ; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm5, %ymm5
775 ; AVX-NEXT: vmovaps %ymm5, 32(%rdx)
776 ; AVX-NEXT: vmovaps %ymm7, 96(%rdx)
777 ; AVX-NEXT: vmovaps %ymm6, 64(%rdx)
778 ; AVX-NEXT: vmovaps %ymm4, (%rdx)
779 ; AVX-NEXT: vmovaps %ymm3, 160(%rdx)
780 ; AVX-NEXT: vmovaps %ymm2, 128(%rdx)
781 ; AVX-NEXT: vmovaps %ymm1, 192(%rdx)
782 ; AVX-NEXT: vmovaps %ymm0, 224(%rdx)
783 ; AVX-NEXT: vzeroupper
784 ; AVX-NEXT: retq
785 ;
786 ; AVX2-LABEL: store_i32_stride2_vf32:
787 ; AVX2: # %bb.0:
788 ; AVX2-NEXT: vmovaps (%rdi), %ymm0
789 ; AVX2-NEXT: vmovaps 32(%rdi), %ymm1
790 ; AVX2-NEXT: vmovaps 64(%rdi), %ymm2
791 ; AVX2-NEXT: vmovaps 96(%rdi), %ymm3
792 ; AVX2-NEXT: vmovaps (%rsi), %ymm4
793 ; AVX2-NEXT: vmovaps 32(%rsi), %ymm5
794 ; AVX2-NEXT: vmovaps 64(%rsi), %ymm6
795 ; AVX2-NEXT: vmovaps 96(%rsi), %ymm7
796 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm8 = ymm0[2],ymm4[2],ymm0[3],ymm4[3],ymm0[6],ymm4[6],ymm0[7],ymm4[7]
797 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm4[0],ymm0[1],ymm4[1],ymm0[4],ymm4[4],ymm0[5],ymm4[5]
798 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[2,3],ymm8[2,3]
799 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm8[0,1]
800 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm8 = ymm1[2],ymm5[2],ymm1[3],ymm5[3],ymm1[6],ymm5[6],ymm1[7],ymm5[7]
801 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[1],ymm5[1],ymm1[4],ymm5[4],ymm1[5],ymm5[5]
802 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm1[2,3],ymm8[2,3]
803 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm8[0,1]
804 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm8 = ymm2[2],ymm6[2],ymm2[3],ymm6[3],ymm2[6],ymm6[6],ymm2[7],ymm6[7]
805 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm2 = ymm2[0],ymm6[0],ymm2[1],ymm6[1],ymm2[4],ymm6[4],ymm2[5],ymm6[5]
806 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm6 = ymm2[2,3],ymm8[2,3]
807 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm2[0,1],ymm8[0,1]
808 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm8 = ymm3[2],ymm7[2],ymm3[3],ymm7[3],ymm3[6],ymm7[6],ymm3[7],ymm7[7]
809 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm3 = ymm3[0],ymm7[0],ymm3[1],ymm7[1],ymm3[4],ymm7[4],ymm3[5],ymm7[5]
810 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm7 = ymm3[2,3],ymm8[2,3]
811 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm3[0,1],ymm8[0,1]
812 ; AVX2-NEXT: vmovaps %ymm3, 192(%rdx)
813 ; AVX2-NEXT: vmovaps %ymm7, 224(%rdx)
814 ; AVX2-NEXT: vmovaps %ymm2, 128(%rdx)
815 ; AVX2-NEXT: vmovaps %ymm6, 160(%rdx)
816 ; AVX2-NEXT: vmovaps %ymm1, 64(%rdx)
817 ; AVX2-NEXT: vmovaps %ymm5, 96(%rdx)
818 ; AVX2-NEXT: vmovaps %ymm0, (%rdx)
819 ; AVX2-NEXT: vmovaps %ymm4, 32(%rdx)
820 ; AVX2-NEXT: vzeroupper
821 ; AVX2-NEXT: retq
822 ;
823 ; AVX2-FP-LABEL: store_i32_stride2_vf32:
824 ; AVX2-FP: # %bb.0:
825 ; AVX2-FP-NEXT: vmovaps (%rdi), %ymm0
826 ; AVX2-FP-NEXT: vmovaps 32(%rdi), %ymm1
827 ; AVX2-FP-NEXT: vmovaps 64(%rdi), %ymm2
828 ; AVX2-FP-NEXT: vmovaps 96(%rdi), %ymm3
829 ; AVX2-FP-NEXT: vmovaps (%rsi), %ymm4
830 ; AVX2-FP-NEXT: vmovaps 32(%rsi), %ymm5
831 ; AVX2-FP-NEXT: vmovaps 64(%rsi), %ymm6
832 ; AVX2-FP-NEXT: vmovaps 96(%rsi), %ymm7
833 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm8 = ymm0[2],ymm4[2],ymm0[3],ymm4[3],ymm0[6],ymm4[6],ymm0[7],ymm4[7]
834 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm4[0],ymm0[1],ymm4[1],ymm0[4],ymm4[4],ymm0[5],ymm4[5]
835 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[2,3],ymm8[2,3]
836 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm8[0,1]
837 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm8 = ymm1[2],ymm5[2],ymm1[3],ymm5[3],ymm1[6],ymm5[6],ymm1[7],ymm5[7]
838 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[1],ymm5[1],ymm1[4],ymm5[4],ymm1[5],ymm5[5]
839 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm1[2,3],ymm8[2,3]
840 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm8[0,1]
841 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm8 = ymm2[2],ymm6[2],ymm2[3],ymm6[3],ymm2[6],ymm6[6],ymm2[7],ymm6[7]
842 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm2 = ymm2[0],ymm6[0],ymm2[1],ymm6[1],ymm2[4],ymm6[4],ymm2[5],ymm6[5]
843 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm6 = ymm2[2,3],ymm8[2,3]
844 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm2[0,1],ymm8[0,1]
845 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm8 = ymm3[2],ymm7[2],ymm3[3],ymm7[3],ymm3[6],ymm7[6],ymm3[7],ymm7[7]
846 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm3 = ymm3[0],ymm7[0],ymm3[1],ymm7[1],ymm3[4],ymm7[4],ymm3[5],ymm7[5]
847 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm7 = ymm3[2,3],ymm8[2,3]
848 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm3[0,1],ymm8[0,1]
849 ; AVX2-FP-NEXT: vmovaps %ymm3, 192(%rdx)
850 ; AVX2-FP-NEXT: vmovaps %ymm7, 224(%rdx)
851 ; AVX2-FP-NEXT: vmovaps %ymm2, 128(%rdx)
852 ; AVX2-FP-NEXT: vmovaps %ymm6, 160(%rdx)
853 ; AVX2-FP-NEXT: vmovaps %ymm1, 64(%rdx)
854 ; AVX2-FP-NEXT: vmovaps %ymm5, 96(%rdx)
855 ; AVX2-FP-NEXT: vmovaps %ymm0, (%rdx)
856 ; AVX2-FP-NEXT: vmovaps %ymm4, 32(%rdx)
857 ; AVX2-FP-NEXT: vzeroupper
858 ; AVX2-FP-NEXT: retq
859 ;
860 ; AVX2-FCP-LABEL: store_i32_stride2_vf32:
861 ; AVX2-FCP: # %bb.0:
862 ; AVX2-FCP-NEXT: vmovaps (%rdi), %ymm0
863 ; AVX2-FCP-NEXT: vmovaps 32(%rdi), %ymm1
864 ; AVX2-FCP-NEXT: vmovaps 64(%rdi), %ymm2
865 ; AVX2-FCP-NEXT: vmovaps 96(%rdi), %ymm3
866 ; AVX2-FCP-NEXT: vmovaps (%rsi), %ymm4
867 ; AVX2-FCP-NEXT: vmovaps 32(%rsi), %ymm5
868 ; AVX2-FCP-NEXT: vmovaps 64(%rsi), %ymm6
869 ; AVX2-FCP-NEXT: vmovaps 96(%rsi), %ymm7
870 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm8 = ymm0[2],ymm4[2],ymm0[3],ymm4[3],ymm0[6],ymm4[6],ymm0[7],ymm4[7]
871 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm4[0],ymm0[1],ymm4[1],ymm0[4],ymm4[4],ymm0[5],ymm4[5]
872 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[2,3],ymm8[2,3]
873 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm8[0,1]
874 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm8 = ymm1[2],ymm5[2],ymm1[3],ymm5[3],ymm1[6],ymm5[6],ymm1[7],ymm5[7]
875 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[1],ymm5[1],ymm1[4],ymm5[4],ymm1[5],ymm5[5]
876 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm1[2,3],ymm8[2,3]
877 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm8[0,1]
878 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm8 = ymm2[2],ymm6[2],ymm2[3],ymm6[3],ymm2[6],ymm6[6],ymm2[7],ymm6[7]
879 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm2 = ymm2[0],ymm6[0],ymm2[1],ymm6[1],ymm2[4],ymm6[4],ymm2[5],ymm6[5]
880 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm6 = ymm2[2,3],ymm8[2,3]
881 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm2[0,1],ymm8[0,1]
882 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm8 = ymm3[2],ymm7[2],ymm3[3],ymm7[3],ymm3[6],ymm7[6],ymm3[7],ymm7[7]
883 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm3 = ymm3[0],ymm7[0],ymm3[1],ymm7[1],ymm3[4],ymm7[4],ymm3[5],ymm7[5]
884 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm7 = ymm3[2,3],ymm8[2,3]
885 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm3[0,1],ymm8[0,1]
886 ; AVX2-FCP-NEXT: vmovaps %ymm3, 192(%rdx)
887 ; AVX2-FCP-NEXT: vmovaps %ymm7, 224(%rdx)
888 ; AVX2-FCP-NEXT: vmovaps %ymm2, 128(%rdx)
889 ; AVX2-FCP-NEXT: vmovaps %ymm6, 160(%rdx)
890 ; AVX2-FCP-NEXT: vmovaps %ymm1, 64(%rdx)
891 ; AVX2-FCP-NEXT: vmovaps %ymm5, 96(%rdx)
892 ; AVX2-FCP-NEXT: vmovaps %ymm0, (%rdx)
893 ; AVX2-FCP-NEXT: vmovaps %ymm4, 32(%rdx)
894 ; AVX2-FCP-NEXT: vzeroupper
895 ; AVX2-FCP-NEXT: retq
896 ;
897 ; AVX512-LABEL: store_i32_stride2_vf32:
898 ; AVX512: # %bb.0:
899 ; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
900 ; AVX512-NEXT: vmovdqa64 64(%rdi), %zmm1
901 ; AVX512-NEXT: vmovdqa64 (%rsi), %zmm2
902 ; AVX512-NEXT: vmovdqa64 64(%rsi), %zmm3
903 ; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm4 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
904 ; AVX512-NEXT: vmovdqa64 %zmm0, %zmm5
905 ; AVX512-NEXT: vpermt2d %zmm2, %zmm4, %zmm5
906 ; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm6 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
907 ; AVX512-NEXT: vpermt2d %zmm2, %zmm6, %zmm0
908 ; AVX512-NEXT: vpermi2d %zmm3, %zmm1, %zmm4
909 ; AVX512-NEXT: vpermt2d %zmm3, %zmm6, %zmm1
910 ; AVX512-NEXT: vmovdqa64 %zmm1, 128(%rdx)
911 ; AVX512-NEXT: vmovdqa64 %zmm4, 192(%rdx)
912 ; AVX512-NEXT: vmovdqa64 %zmm0, (%rdx)
913 ; AVX512-NEXT: vmovdqa64 %zmm5, 64(%rdx)
914 ; AVX512-NEXT: vzeroupper
915 ; AVX512-NEXT: retq
916 ;
917 ; AVX512-FCP-LABEL: store_i32_stride2_vf32:
918 ; AVX512-FCP: # %bb.0:
919 ; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
920 ; AVX512-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
921 ; AVX512-FCP-NEXT: vmovdqa64 (%rsi), %zmm2
922 ; AVX512-FCP-NEXT: vmovdqa64 64(%rsi), %zmm3
923 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm4 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
924 ; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm5
925 ; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm4, %zmm5
926 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm6 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
927 ; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm6, %zmm0
928 ; AVX512-FCP-NEXT: vpermi2d %zmm3, %zmm1, %zmm4
929 ; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm6, %zmm1
930 ; AVX512-FCP-NEXT: vmovdqa64 %zmm1, 128(%rdx)
931 ; AVX512-FCP-NEXT: vmovdqa64 %zmm4, 192(%rdx)
932 ; AVX512-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
933 ; AVX512-FCP-NEXT: vmovdqa64 %zmm5, 64(%rdx)
934 ; AVX512-FCP-NEXT: vzeroupper
935 ; AVX512-FCP-NEXT: retq
936 ;
937 ; AVX512DQ-LABEL: store_i32_stride2_vf32:
938 ; AVX512DQ: # %bb.0:
939 ; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
940 ; AVX512DQ-NEXT: vmovdqa64 64(%rdi), %zmm1
941 ; AVX512DQ-NEXT: vmovdqa64 (%rsi), %zmm2
942 ; AVX512DQ-NEXT: vmovdqa64 64(%rsi), %zmm3
943 ; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm4 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
944 ; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm5
945 ; AVX512DQ-NEXT: vpermt2d %zmm2, %zmm4, %zmm5
946 ; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm6 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
947 ; AVX512DQ-NEXT: vpermt2d %zmm2, %zmm6, %zmm0
948 ; AVX512DQ-NEXT: vpermi2d %zmm3, %zmm1, %zmm4
949 ; AVX512DQ-NEXT: vpermt2d %zmm3, %zmm6, %zmm1
950 ; AVX512DQ-NEXT: vmovdqa64 %zmm1, 128(%rdx)
951 ; AVX512DQ-NEXT: vmovdqa64 %zmm4, 192(%rdx)
952 ; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%rdx)
953 ; AVX512DQ-NEXT: vmovdqa64 %zmm5, 64(%rdx)
954 ; AVX512DQ-NEXT: vzeroupper
955 ; AVX512DQ-NEXT: retq
956 ;
957 ; AVX512DQ-FCP-LABEL: store_i32_stride2_vf32:
958 ; AVX512DQ-FCP: # %bb.0:
959 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
960 ; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
961 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rsi), %zmm2
962 ; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rsi), %zmm3
963 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm4 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
964 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm5
965 ; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm4, %zmm5
966 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm6 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
967 ; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm6, %zmm0
968 ; AVX512DQ-FCP-NEXT: vpermi2d %zmm3, %zmm1, %zmm4
969 ; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm6, %zmm1
970 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, 128(%rdx)
971 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm4, 192(%rdx)
972 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
973 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, 64(%rdx)
974 ; AVX512DQ-FCP-NEXT: vzeroupper
975 ; AVX512DQ-FCP-NEXT: retq
976 ;
977 ; AVX512BW-LABEL: store_i32_stride2_vf32:
978 ; AVX512BW: # %bb.0:
979 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
980 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
981 ; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm2
982 ; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm3
983 ; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm4 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
984 ; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm5
985 ; AVX512BW-NEXT: vpermt2d %zmm2, %zmm4, %zmm5
986 ; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm6 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
987 ; AVX512BW-NEXT: vpermt2d %zmm2, %zmm6, %zmm0
988 ; AVX512BW-NEXT: vpermi2d %zmm3, %zmm1, %zmm4
989 ; AVX512BW-NEXT: vpermt2d %zmm3, %zmm6, %zmm1
990 ; AVX512BW-NEXT: vmovdqa64 %zmm1, 128(%rdx)
991 ; AVX512BW-NEXT: vmovdqa64 %zmm4, 192(%rdx)
992 ; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
993 ; AVX512BW-NEXT: vmovdqa64 %zmm5, 64(%rdx)
994 ; AVX512BW-NEXT: vzeroupper
995 ; AVX512BW-NEXT: retq
996 ;
997 ; AVX512BW-FCP-LABEL: store_i32_stride2_vf32:
998 ; AVX512BW-FCP: # %bb.0:
999 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1000 ; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1001 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm2
1002 ; AVX512BW-FCP-NEXT: vmovdqa64 64(%rsi), %zmm3
1003 ; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm4 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
1004 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm5
1005 ; AVX512BW-FCP-NEXT: vpermt2d %zmm2, %zmm4, %zmm5
1006 ; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm6 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
1007 ; AVX512BW-FCP-NEXT: vpermt2d %zmm2, %zmm6, %zmm0
1008 ; AVX512BW-FCP-NEXT: vpermi2d %zmm3, %zmm1, %zmm4
1009 ; AVX512BW-FCP-NEXT: vpermt2d %zmm3, %zmm6, %zmm1
1010 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, 128(%rdx)
1011 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, 192(%rdx)
1012 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
1013 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, 64(%rdx)
1014 ; AVX512BW-FCP-NEXT: vzeroupper
1015 ; AVX512BW-FCP-NEXT: retq
1016 ;
1017 ; AVX512DQ-BW-LABEL: store_i32_stride2_vf32:
1018 ; AVX512DQ-BW: # %bb.0:
1019 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
1020 ; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm1
1021 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rsi), %zmm2
1022 ; AVX512DQ-BW-NEXT: vmovdqa64 64(%rsi), %zmm3
1023 ; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} zmm4 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
1024 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm5
1025 ; AVX512DQ-BW-NEXT: vpermt2d %zmm2, %zmm4, %zmm5
1026 ; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} zmm6 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
1027 ; AVX512DQ-BW-NEXT: vpermt2d %zmm2, %zmm6, %zmm0
1028 ; AVX512DQ-BW-NEXT: vpermi2d %zmm3, %zmm1, %zmm4
1029 ; AVX512DQ-BW-NEXT: vpermt2d %zmm3, %zmm6, %zmm1
1030 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, 128(%rdx)
1031 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, 192(%rdx)
1032 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, (%rdx)
1033 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, 64(%rdx)
1034 ; AVX512DQ-BW-NEXT: vzeroupper
1035 ; AVX512DQ-BW-NEXT: retq
1036 ;
1037 ; AVX512DQ-BW-FCP-LABEL: store_i32_stride2_vf32:
1038 ; AVX512DQ-BW-FCP: # %bb.0:
1039 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1040 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1041 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm2
1042 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rsi), %zmm3
1043 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm4 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
1044 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm5
1045 ; AVX512DQ-BW-FCP-NEXT: vpermt2d %zmm2, %zmm4, %zmm5
1046 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm6 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
1047 ; AVX512DQ-BW-FCP-NEXT: vpermt2d %zmm2, %zmm6, %zmm0
1048 ; AVX512DQ-BW-FCP-NEXT: vpermi2d %zmm3, %zmm1, %zmm4
1049 ; AVX512DQ-BW-FCP-NEXT: vpermt2d %zmm3, %zmm6, %zmm1
1050 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, 128(%rdx)
1051 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, 192(%rdx)
1052 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
1053 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, 64(%rdx)
1054 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
1055 ; AVX512DQ-BW-FCP-NEXT: retq
1056 %in.vec0 = load <32 x i32>, ptr %in.vecptr0, align 64
1057 %in.vec1 = load <32 x i32>, ptr %in.vecptr1, align 64
1058 %1 = shufflevector <32 x i32> %in.vec0, <32 x i32> %in.vec1, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
1059 %interleaved.vec = shufflevector <64 x i32> %1, <64 x i32> poison, <64 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
1060 store <64 x i32> %interleaved.vec, ptr %out.vec, align 64
1061 ret void
1062 }
1064 define void @store_i32_stride2_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.vec) nounwind {
1065 ; SSE-LABEL: store_i32_stride2_vf64:
1066 ; SSE: # %bb.0:
1067 ; SSE-NEXT: subq $152, %rsp
1068 ; SSE-NEXT: movaps 112(%rdi), %xmm14
1069 ; SSE-NEXT: movaps 96(%rdi), %xmm13
1070 ; SSE-NEXT: movaps 80(%rdi), %xmm11
1071 ; SSE-NEXT: movaps 64(%rdi), %xmm10
1072 ; SSE-NEXT: movaps (%rdi), %xmm7
1073 ; SSE-NEXT: movaps 16(%rdi), %xmm8
1074 ; SSE-NEXT: movaps 32(%rdi), %xmm9
1075 ; SSE-NEXT: movaps 48(%rdi), %xmm12
1076 ; SSE-NEXT: movaps 96(%rsi), %xmm0
1077 ; SSE-NEXT: movaps 80(%rsi), %xmm1
1078 ; SSE-NEXT: movaps 64(%rsi), %xmm2
1079 ; SSE-NEXT: movaps (%rsi), %xmm3
1080 ; SSE-NEXT: movaps 16(%rsi), %xmm4
1081 ; SSE-NEXT: movaps 32(%rsi), %xmm5
1082 ; SSE-NEXT: movaps 48(%rsi), %xmm6
1083 ; SSE-NEXT: movaps %xmm7, %xmm15
1084 ; SSE-NEXT: unpcklps {{.*#+}} xmm15 = xmm15[0],xmm3[0],xmm15[1],xmm3[1]
1085 ; SSE-NEXT: movaps %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1086 ; SSE-NEXT: unpckhps {{.*#+}} xmm7 = xmm7[2],xmm3[2],xmm7[3],xmm3[3]
1087 ; SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1088 ; SSE-NEXT: movaps %xmm8, %xmm7
1089 ; SSE-NEXT: unpcklps {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1]
1090 ; SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1091 ; SSE-NEXT: unpckhps {{.*#+}} xmm8 = xmm8[2],xmm4[2],xmm8[3],xmm4[3]
1092 ; SSE-NEXT: movaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1093 ; SSE-NEXT: movaps %xmm9, %xmm4
1094 ; SSE-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
1095 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1096 ; SSE-NEXT: unpckhps {{.*#+}} xmm9 = xmm9[2],xmm5[2],xmm9[3],xmm5[3]
1097 ; SSE-NEXT: movaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1098 ; SSE-NEXT: movaps %xmm12, %xmm4
1099 ; SSE-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
1100 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1101 ; SSE-NEXT: unpckhps {{.*#+}} xmm12 = xmm12[2],xmm6[2],xmm12[3],xmm6[3]
1102 ; SSE-NEXT: movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1103 ; SSE-NEXT: movaps %xmm10, %xmm3
1104 ; SSE-NEXT: unpcklps {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
1105 ; SSE-NEXT: movaps %xmm3, (%rsp) # 16-byte Spill
1106 ; SSE-NEXT: unpckhps {{.*#+}} xmm10 = xmm10[2],xmm2[2],xmm10[3],xmm2[3]
1107 ; SSE-NEXT: movaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1108 ; SSE-NEXT: movaps %xmm11, %xmm2
1109 ; SSE-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1110 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1111 ; SSE-NEXT: unpckhps {{.*#+}} xmm11 = xmm11[2],xmm1[2],xmm11[3],xmm1[3]
1112 ; SSE-NEXT: movaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1113 ; SSE-NEXT: movaps %xmm13, %xmm1
1114 ; SSE-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1115 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1116 ; SSE-NEXT: unpckhps {{.*#+}} xmm13 = xmm13[2],xmm0[2],xmm13[3],xmm0[3]
1117 ; SSE-NEXT: movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1118 ; SSE-NEXT: movaps 112(%rsi), %xmm0
1119 ; SSE-NEXT: movaps %xmm14, %xmm1
1120 ; SSE-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1121 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1122 ; SSE-NEXT: unpckhps {{.*#+}} xmm14 = xmm14[2],xmm0[2],xmm14[3],xmm0[3]
1123 ; SSE-NEXT: movaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1124 ; SSE-NEXT: movaps 128(%rdi), %xmm15
1125 ; SSE-NEXT: movaps 128(%rsi), %xmm0
1126 ; SSE-NEXT: movaps %xmm15, %xmm1
1127 ; SSE-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1128 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1129 ; SSE-NEXT: unpckhps {{.*#+}} xmm15 = xmm15[2],xmm0[2],xmm15[3],xmm0[3]
1130 ; SSE-NEXT: movaps 144(%rdi), %xmm13
1131 ; SSE-NEXT: movaps 144(%rsi), %xmm0
1132 ; SSE-NEXT: movaps %xmm13, %xmm14
1133 ; SSE-NEXT: unpcklps {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
1134 ; SSE-NEXT: unpckhps {{.*#+}} xmm13 = xmm13[2],xmm0[2],xmm13[3],xmm0[3]
1135 ; SSE-NEXT: movaps 160(%rdi), %xmm10
1136 ; SSE-NEXT: movaps 160(%rsi), %xmm0
1137 ; SSE-NEXT: movaps %xmm10, %xmm12
1138 ; SSE-NEXT: unpcklps {{.*#+}} xmm12 = xmm12[0],xmm0[0],xmm12[1],xmm0[1]
1139 ; SSE-NEXT: unpckhps {{.*#+}} xmm10 = xmm10[2],xmm0[2],xmm10[3],xmm0[3]
1140 ; SSE-NEXT: movaps 176(%rdi), %xmm8
1141 ; SSE-NEXT: movaps 176(%rsi), %xmm0
1142 ; SSE-NEXT: movaps %xmm8, %xmm11
1143 ; SSE-NEXT: unpcklps {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1]
1144 ; SSE-NEXT: unpckhps {{.*#+}} xmm8 = xmm8[2],xmm0[2],xmm8[3],xmm0[3]
1145 ; SSE-NEXT: movaps 192(%rdi), %xmm6
1146 ; SSE-NEXT: movaps 192(%rsi), %xmm0
1147 ; SSE-NEXT: movaps %xmm6, %xmm9
1148 ; SSE-NEXT: unpcklps {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]
1149 ; SSE-NEXT: unpckhps {{.*#+}} xmm6 = xmm6[2],xmm0[2],xmm6[3],xmm0[3]
1150 ; SSE-NEXT: movaps 208(%rdi), %xmm5
1151 ; SSE-NEXT: movaps 208(%rsi), %xmm1
1152 ; SSE-NEXT: movaps %xmm5, %xmm7
1153 ; SSE-NEXT: unpcklps {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
1154 ; SSE-NEXT: unpckhps {{.*#+}} xmm5 = xmm5[2],xmm1[2],xmm5[3],xmm1[3]
1155 ; SSE-NEXT: movaps 224(%rdi), %xmm1
1156 ; SSE-NEXT: movaps 224(%rsi), %xmm3
1157 ; SSE-NEXT: movaps %xmm1, %xmm2
1158 ; SSE-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
1159 ; SSE-NEXT: unpckhps {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
1160 ; SSE-NEXT: movaps 240(%rdi), %xmm3
1161 ; SSE-NEXT: movaps 240(%rsi), %xmm4
1162 ; SSE-NEXT: movaps %xmm3, %xmm0
1163 ; SSE-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
1164 ; SSE-NEXT: unpckhps {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
1165 ; SSE-NEXT: movaps %xmm3, 496(%rdx)
1166 ; SSE-NEXT: movaps %xmm0, 480(%rdx)
1167 ; SSE-NEXT: movaps %xmm1, 464(%rdx)
1168 ; SSE-NEXT: movaps %xmm2, 448(%rdx)
1169 ; SSE-NEXT: movaps %xmm5, 432(%rdx)
1170 ; SSE-NEXT: movaps %xmm7, 416(%rdx)
1171 ; SSE-NEXT: movaps %xmm6, 400(%rdx)
1172 ; SSE-NEXT: movaps %xmm9, 384(%rdx)
1173 ; SSE-NEXT: movaps %xmm8, 368(%rdx)
1174 ; SSE-NEXT: movaps %xmm11, 352(%rdx)
1175 ; SSE-NEXT: movaps %xmm10, 336(%rdx)
1176 ; SSE-NEXT: movaps %xmm12, 320(%rdx)
1177 ; SSE-NEXT: movaps %xmm13, 304(%rdx)
1178 ; SSE-NEXT: movaps %xmm14, 288(%rdx)
1179 ; SSE-NEXT: movaps %xmm15, 272(%rdx)
1180 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1181 ; SSE-NEXT: movaps %xmm0, 256(%rdx)
1182 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1183 ; SSE-NEXT: movaps %xmm0, 240(%rdx)
1184 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1185 ; SSE-NEXT: movaps %xmm0, 224(%rdx)
1186 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1187 ; SSE-NEXT: movaps %xmm0, 208(%rdx)
1188 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1189 ; SSE-NEXT: movaps %xmm0, 192(%rdx)
1190 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1191 ; SSE-NEXT: movaps %xmm0, 176(%rdx)
1192 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1193 ; SSE-NEXT: movaps %xmm0, 160(%rdx)
1194 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1195 ; SSE-NEXT: movaps %xmm0, 144(%rdx)
1196 ; SSE-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
1197 ; SSE-NEXT: movaps %xmm0, 128(%rdx)
1198 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1199 ; SSE-NEXT: movaps %xmm0, 112(%rdx)
1200 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1201 ; SSE-NEXT: movaps %xmm0, 96(%rdx)
1202 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1203 ; SSE-NEXT: movaps %xmm0, 80(%rdx)
1204 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1205 ; SSE-NEXT: movaps %xmm0, 64(%rdx)
1206 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1207 ; SSE-NEXT: movaps %xmm0, 48(%rdx)
1208 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1209 ; SSE-NEXT: movaps %xmm0, 32(%rdx)
1210 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1211 ; SSE-NEXT: movaps %xmm0, 16(%rdx)
1212 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1213 ; SSE-NEXT: movaps %xmm0, (%rdx)
1214 ; SSE-NEXT: addq $152, %rsp
1215 ; SSE-NEXT: retq
1216 ;
1217 ; AVX-LABEL: store_i32_stride2_vf64:
1218 ; AVX: # %bb.0:
1219 ; AVX-NEXT: vmovaps 240(%rsi), %xmm0
1220 ; AVX-NEXT: vmovaps 240(%rdi), %xmm1
1221 ; AVX-NEXT: vunpckhps {{.*#+}} xmm2 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
1222 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1223 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
1224 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1225 ; AVX-NEXT: vmovaps 224(%rsi), %xmm1
1226 ; AVX-NEXT: vmovaps 224(%rdi), %xmm2
1227 ; AVX-NEXT: vunpckhps {{.*#+}} xmm3 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
1228 ; AVX-NEXT: vunpcklps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1229 ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm0
1230 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1231 ; AVX-NEXT: vmovaps 128(%rsi), %xmm2
1232 ; AVX-NEXT: vmovaps 128(%rdi), %xmm3
1233 ; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
1234 ; AVX-NEXT: vunpcklps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
1235 ; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm2, %ymm2
1236 ; AVX-NEXT: vmovaps 144(%rsi), %xmm3
1237 ; AVX-NEXT: vmovaps 144(%rdi), %xmm4
1238 ; AVX-NEXT: vunpckhps {{.*#+}} xmm5 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
1239 ; AVX-NEXT: vunpcklps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
1240 ; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm3, %ymm3
1241 ; AVX-NEXT: vmovaps (%rsi), %xmm4
1242 ; AVX-NEXT: vmovaps 16(%rsi), %xmm8
1243 ; AVX-NEXT: vmovaps 32(%rsi), %xmm5
1244 ; AVX-NEXT: vmovaps 48(%rsi), %xmm11
1245 ; AVX-NEXT: vmovaps (%rdi), %xmm6
1246 ; AVX-NEXT: vmovaps 16(%rdi), %xmm9
1247 ; AVX-NEXT: vmovaps 32(%rdi), %xmm7
1248 ; AVX-NEXT: vunpckhps {{.*#+}} xmm10 = xmm6[2],xmm4[2],xmm6[3],xmm4[3]
1249 ; AVX-NEXT: vunpcklps {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
1250 ; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm4, %ymm4
1251 ; AVX-NEXT: vunpckhps {{.*#+}} xmm6 = xmm7[2],xmm5[2],xmm7[3],xmm5[3]
1252 ; AVX-NEXT: vunpcklps {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
1253 ; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5
1254 ; AVX-NEXT: vmovaps 64(%rsi), %xmm6
1255 ; AVX-NEXT: vmovaps 64(%rdi), %xmm7
1256 ; AVX-NEXT: vunpckhps {{.*#+}} xmm10 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]
1257 ; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
1258 ; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm6, %ymm6
1259 ; AVX-NEXT: vmovaps 96(%rsi), %xmm7
1260 ; AVX-NEXT: vmovaps 96(%rdi), %xmm10
1261 ; AVX-NEXT: vunpckhps {{.*#+}} xmm12 = xmm10[2],xmm7[2],xmm10[3],xmm7[3]
1262 ; AVX-NEXT: vunpcklps {{.*#+}} xmm7 = xmm10[0],xmm7[0],xmm10[1],xmm7[1]
1263 ; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm7, %ymm7
1264 ; AVX-NEXT: vmovaps 160(%rsi), %xmm10
1265 ; AVX-NEXT: vmovaps 160(%rdi), %xmm12
1266 ; AVX-NEXT: vunpckhps {{.*#+}} xmm13 = xmm12[2],xmm10[2],xmm12[3],xmm10[3]
1267 ; AVX-NEXT: vunpcklps {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]
1268 ; AVX-NEXT: vinsertf128 $1, %xmm13, %ymm10, %ymm10
1269 ; AVX-NEXT: vmovaps 192(%rsi), %xmm12
1270 ; AVX-NEXT: vmovaps 192(%rdi), %xmm13
1271 ; AVX-NEXT: vunpckhps {{.*#+}} xmm14 = xmm13[2],xmm12[2],xmm13[3],xmm12[3]
1272 ; AVX-NEXT: vunpcklps {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]
1273 ; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm12, %ymm12
1274 ; AVX-NEXT: vmovaps 80(%rsi), %xmm13
1275 ; AVX-NEXT: vmovaps 80(%rdi), %xmm14
1276 ; AVX-NEXT: vunpckhps {{.*#+}} xmm15 = xmm14[2],xmm13[2],xmm14[3],xmm13[3]
1277 ; AVX-NEXT: vunpcklps {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
1278 ; AVX-NEXT: vmovaps 48(%rdi), %xmm14
1279 ; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm13, %ymm13
1280 ; AVX-NEXT: vunpckhps {{.*#+}} xmm15 = xmm14[2],xmm11[2],xmm14[3],xmm11[3]
1281 ; AVX-NEXT: vunpcklps {{.*#+}} xmm11 = xmm14[0],xmm11[0],xmm14[1],xmm11[1]
1282 ; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm11, %ymm11
1283 ; AVX-NEXT: vunpckhps {{.*#+}} xmm14 = xmm9[2],xmm8[2],xmm9[3],xmm8[3]
1284 ; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
1285 ; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm8, %ymm8
1286 ; AVX-NEXT: vmovaps 112(%rsi), %xmm9
1287 ; AVX-NEXT: vmovaps 112(%rdi), %xmm14
1288 ; AVX-NEXT: vunpckhps {{.*#+}} xmm15 = xmm14[2],xmm9[2],xmm14[3],xmm9[3]
1289 ; AVX-NEXT: vunpcklps {{.*#+}} xmm9 = xmm14[0],xmm9[0],xmm14[1],xmm9[1]
1290 ; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm9, %ymm9
1291 ; AVX-NEXT: vmovaps 176(%rsi), %xmm14
1292 ; AVX-NEXT: vmovaps 176(%rdi), %xmm15
1293 ; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm15[2],xmm14[2],xmm15[3],xmm14[3]
1294 ; AVX-NEXT: vunpcklps {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
1295 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm14, %ymm1
1296 ; AVX-NEXT: vmovaps 208(%rsi), %xmm14
1297 ; AVX-NEXT: vmovaps 208(%rdi), %xmm15
1298 ; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm15[2],xmm14[2],xmm15[3],xmm14[3]
1299 ; AVX-NEXT: vunpcklps {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
1300 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm14, %ymm0
1301 ; AVX-NEXT: vmovaps %ymm0, 416(%rdx)
1302 ; AVX-NEXT: vmovaps %ymm1, 352(%rdx)
1303 ; AVX-NEXT: vmovaps %ymm9, 224(%rdx)
1304 ; AVX-NEXT: vmovaps %ymm8, 32(%rdx)
1305 ; AVX-NEXT: vmovaps %ymm11, 96(%rdx)
1306 ; AVX-NEXT: vmovaps %ymm13, 160(%rdx)
1307 ; AVX-NEXT: vmovaps %ymm12, 384(%rdx)
1308 ; AVX-NEXT: vmovaps %ymm10, 320(%rdx)
1309 ; AVX-NEXT: vmovaps %ymm7, 192(%rdx)
1310 ; AVX-NEXT: vmovaps %ymm6, 128(%rdx)
1311 ; AVX-NEXT: vmovaps %ymm5, 64(%rdx)
1312 ; AVX-NEXT: vmovaps %ymm4, (%rdx)
1313 ; AVX-NEXT: vmovaps %ymm3, 288(%rdx)
1314 ; AVX-NEXT: vmovaps %ymm2, 256(%rdx)
1315 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
1316 ; AVX-NEXT: vmovaps %ymm0, 448(%rdx)
1317 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
1318 ; AVX-NEXT: vmovaps %ymm0, 480(%rdx)
1319 ; AVX-NEXT: vzeroupper
1320 ; AVX-NEXT: retq
1321 ;
1322 ; AVX2-LABEL: store_i32_stride2_vf64:
1323 ; AVX2: # %bb.0:
1324 ; AVX2-NEXT: vmovaps 224(%rdi), %ymm0
1325 ; AVX2-NEXT: vmovaps 192(%rdi), %ymm3
1326 ; AVX2-NEXT: vmovaps 160(%rdi), %ymm6
1327 ; AVX2-NEXT: vmovaps 128(%rdi), %ymm8
1328 ; AVX2-NEXT: vmovaps (%rdi), %ymm1
1329 ; AVX2-NEXT: vmovaps 32(%rdi), %ymm4
1330 ; AVX2-NEXT: vmovaps 64(%rdi), %ymm7
1331 ; AVX2-NEXT: vmovaps 96(%rdi), %ymm9
1332 ; AVX2-NEXT: vmovaps 192(%rsi), %ymm10
1333 ; AVX2-NEXT: vmovaps 160(%rsi), %ymm11
1334 ; AVX2-NEXT: vmovaps 128(%rsi), %ymm12
1335 ; AVX2-NEXT: vmovaps (%rsi), %ymm2
1336 ; AVX2-NEXT: vmovaps 32(%rsi), %ymm5
1337 ; AVX2-NEXT: vmovaps 64(%rsi), %ymm13
1338 ; AVX2-NEXT: vmovaps 96(%rsi), %ymm14
1339 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm15 = ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[6],ymm2[6],ymm1[7],ymm2[7]
1340 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm2 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[4],ymm2[4],ymm1[5],ymm2[5]
1341 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm2[2,3],ymm15[2,3]
1342 ; AVX2-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1343 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm2[0,1],ymm15[0,1]
1344 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm15 = ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[6],ymm5[6],ymm4[7],ymm5[7]
1345 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm5 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5]
1346 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm5[2,3],ymm15[2,3]
1347 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm5[0,1],ymm15[0,1]
1348 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm15 = ymm7[2],ymm13[2],ymm7[3],ymm13[3],ymm7[6],ymm13[6],ymm7[7],ymm13[7]
1349 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm13 = ymm7[0],ymm13[0],ymm7[1],ymm13[1],ymm7[4],ymm13[4],ymm7[5],ymm13[5]
1350 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm7 = ymm13[2,3],ymm15[2,3]
1351 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm13 = ymm13[0,1],ymm15[0,1]
1352 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm15 = ymm9[2],ymm14[2],ymm9[3],ymm14[3],ymm9[6],ymm14[6],ymm9[7],ymm14[7]
1353 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm9 = ymm9[0],ymm14[0],ymm9[1],ymm14[1],ymm9[4],ymm14[4],ymm9[5],ymm14[5]
1354 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm14 = ymm9[2,3],ymm15[2,3]
1355 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm9 = ymm9[0,1],ymm15[0,1]
1356 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm15 = ymm8[2],ymm12[2],ymm8[3],ymm12[3],ymm8[6],ymm12[6],ymm8[7],ymm12[7]
1357 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm8 = ymm8[0],ymm12[0],ymm8[1],ymm12[1],ymm8[4],ymm12[4],ymm8[5],ymm12[5]
1358 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm12 = ymm8[2,3],ymm15[2,3]
1359 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm8 = ymm8[0,1],ymm15[0,1]
1360 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm15 = ymm6[2],ymm11[2],ymm6[3],ymm11[3],ymm6[6],ymm11[6],ymm6[7],ymm11[7]
1361 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm6 = ymm6[0],ymm11[0],ymm6[1],ymm11[1],ymm6[4],ymm11[4],ymm6[5],ymm11[5]
1362 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm11 = ymm6[2,3],ymm15[2,3]
1363 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm6 = ymm6[0,1],ymm15[0,1]
1364 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm15 = ymm3[2],ymm10[2],ymm3[3],ymm10[3],ymm3[6],ymm10[6],ymm3[7],ymm10[7]
1365 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm3 = ymm3[0],ymm10[0],ymm3[1],ymm10[1],ymm3[4],ymm10[4],ymm3[5],ymm10[5]
1366 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm10 = ymm3[2,3],ymm15[2,3]
1367 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm3[0,1],ymm15[0,1]
1368 ; AVX2-NEXT: vmovaps 224(%rsi), %ymm15
1369 ; AVX2-NEXT: vmovaps %ymm0, %ymm1
1370 ; AVX2-NEXT: vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm15[2],ymm0[3],ymm15[3],ymm0[6],ymm15[6],ymm0[7],ymm15[7]
1371 ; AVX2-NEXT: vunpcklps {{.*#+}} ymm15 = ymm1[0],ymm15[0],ymm1[1],ymm15[1],ymm1[4],ymm15[4],ymm1[5],ymm15[5]
1372 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm15[2,3],ymm0[2,3]
1373 ; AVX2-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm15[0,1],ymm0[0,1]
1374 ; AVX2-NEXT: vmovaps %ymm0, 448(%rdx)
1375 ; AVX2-NEXT: vmovaps %ymm1, 480(%rdx)
1376 ; AVX2-NEXT: vmovaps %ymm3, 384(%rdx)
1377 ; AVX2-NEXT: vmovaps %ymm10, 416(%rdx)
1378 ; AVX2-NEXT: vmovaps %ymm6, 320(%rdx)
1379 ; AVX2-NEXT: vmovaps %ymm11, 352(%rdx)
1380 ; AVX2-NEXT: vmovaps %ymm8, 256(%rdx)
1381 ; AVX2-NEXT: vmovaps %ymm12, 288(%rdx)
1382 ; AVX2-NEXT: vmovaps %ymm9, 192(%rdx)
1383 ; AVX2-NEXT: vmovaps %ymm14, 224(%rdx)
1384 ; AVX2-NEXT: vmovaps %ymm13, 128(%rdx)
1385 ; AVX2-NEXT: vmovaps %ymm7, 160(%rdx)
1386 ; AVX2-NEXT: vmovaps %ymm5, 64(%rdx)
1387 ; AVX2-NEXT: vmovaps %ymm4, 96(%rdx)
1388 ; AVX2-NEXT: vmovaps %ymm2, (%rdx)
1389 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
1390 ; AVX2-NEXT: vmovaps %ymm0, 32(%rdx)
1391 ; AVX2-NEXT: vzeroupper
1392 ; AVX2-NEXT: retq
1393 ;
1394 ; AVX2-FP-LABEL: store_i32_stride2_vf64:
1395 ; AVX2-FP: # %bb.0:
1396 ; AVX2-FP-NEXT: vmovaps 224(%rdi), %ymm0
1397 ; AVX2-FP-NEXT: vmovaps 192(%rdi), %ymm3
1398 ; AVX2-FP-NEXT: vmovaps 160(%rdi), %ymm6
1399 ; AVX2-FP-NEXT: vmovaps 128(%rdi), %ymm8
1400 ; AVX2-FP-NEXT: vmovaps (%rdi), %ymm1
1401 ; AVX2-FP-NEXT: vmovaps 32(%rdi), %ymm4
1402 ; AVX2-FP-NEXT: vmovaps 64(%rdi), %ymm7
1403 ; AVX2-FP-NEXT: vmovaps 96(%rdi), %ymm9
1404 ; AVX2-FP-NEXT: vmovaps 192(%rsi), %ymm10
1405 ; AVX2-FP-NEXT: vmovaps 160(%rsi), %ymm11
1406 ; AVX2-FP-NEXT: vmovaps 128(%rsi), %ymm12
1407 ; AVX2-FP-NEXT: vmovaps (%rsi), %ymm2
1408 ; AVX2-FP-NEXT: vmovaps 32(%rsi), %ymm5
1409 ; AVX2-FP-NEXT: vmovaps 64(%rsi), %ymm13
1410 ; AVX2-FP-NEXT: vmovaps 96(%rsi), %ymm14
1411 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[6],ymm2[6],ymm1[7],ymm2[7]
1412 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm2 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[4],ymm2[4],ymm1[5],ymm2[5]
1413 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm2[2,3],ymm15[2,3]
1414 ; AVX2-FP-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1415 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm2[0,1],ymm15[0,1]
1416 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[6],ymm5[6],ymm4[7],ymm5[7]
1417 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm5 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5]
1418 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm5[2,3],ymm15[2,3]
1419 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm5[0,1],ymm15[0,1]
1420 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm7[2],ymm13[2],ymm7[3],ymm13[3],ymm7[6],ymm13[6],ymm7[7],ymm13[7]
1421 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm13 = ymm7[0],ymm13[0],ymm7[1],ymm13[1],ymm7[4],ymm13[4],ymm7[5],ymm13[5]
1422 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm7 = ymm13[2,3],ymm15[2,3]
1423 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm13 = ymm13[0,1],ymm15[0,1]
1424 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm9[2],ymm14[2],ymm9[3],ymm14[3],ymm9[6],ymm14[6],ymm9[7],ymm14[7]
1425 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm9 = ymm9[0],ymm14[0],ymm9[1],ymm14[1],ymm9[4],ymm14[4],ymm9[5],ymm14[5]
1426 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm14 = ymm9[2,3],ymm15[2,3]
1427 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm9 = ymm9[0,1],ymm15[0,1]
1428 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm8[2],ymm12[2],ymm8[3],ymm12[3],ymm8[6],ymm12[6],ymm8[7],ymm12[7]
1429 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm8 = ymm8[0],ymm12[0],ymm8[1],ymm12[1],ymm8[4],ymm12[4],ymm8[5],ymm12[5]
1430 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm12 = ymm8[2,3],ymm15[2,3]
1431 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm8 = ymm8[0,1],ymm15[0,1]
1432 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm6[2],ymm11[2],ymm6[3],ymm11[3],ymm6[6],ymm11[6],ymm6[7],ymm11[7]
1433 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm6 = ymm6[0],ymm11[0],ymm6[1],ymm11[1],ymm6[4],ymm11[4],ymm6[5],ymm11[5]
1434 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm11 = ymm6[2,3],ymm15[2,3]
1435 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm6 = ymm6[0,1],ymm15[0,1]
1436 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm3[2],ymm10[2],ymm3[3],ymm10[3],ymm3[6],ymm10[6],ymm3[7],ymm10[7]
1437 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm3 = ymm3[0],ymm10[0],ymm3[1],ymm10[1],ymm3[4],ymm10[4],ymm3[5],ymm10[5]
1438 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm10 = ymm3[2,3],ymm15[2,3]
1439 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm3[0,1],ymm15[0,1]
1440 ; AVX2-FP-NEXT: vmovaps 224(%rsi), %ymm15
1441 ; AVX2-FP-NEXT: vmovaps %ymm0, %ymm1
1442 ; AVX2-FP-NEXT: vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm15[2],ymm0[3],ymm15[3],ymm0[6],ymm15[6],ymm0[7],ymm15[7]
1443 ; AVX2-FP-NEXT: vunpcklps {{.*#+}} ymm15 = ymm1[0],ymm15[0],ymm1[1],ymm15[1],ymm1[4],ymm15[4],ymm1[5],ymm15[5]
1444 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm15[2,3],ymm0[2,3]
1445 ; AVX2-FP-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm15[0,1],ymm0[0,1]
1446 ; AVX2-FP-NEXT: vmovaps %ymm0, 448(%rdx)
1447 ; AVX2-FP-NEXT: vmovaps %ymm1, 480(%rdx)
1448 ; AVX2-FP-NEXT: vmovaps %ymm3, 384(%rdx)
1449 ; AVX2-FP-NEXT: vmovaps %ymm10, 416(%rdx)
1450 ; AVX2-FP-NEXT: vmovaps %ymm6, 320(%rdx)
1451 ; AVX2-FP-NEXT: vmovaps %ymm11, 352(%rdx)
1452 ; AVX2-FP-NEXT: vmovaps %ymm8, 256(%rdx)
1453 ; AVX2-FP-NEXT: vmovaps %ymm12, 288(%rdx)
1454 ; AVX2-FP-NEXT: vmovaps %ymm9, 192(%rdx)
1455 ; AVX2-FP-NEXT: vmovaps %ymm14, 224(%rdx)
1456 ; AVX2-FP-NEXT: vmovaps %ymm13, 128(%rdx)
1457 ; AVX2-FP-NEXT: vmovaps %ymm7, 160(%rdx)
1458 ; AVX2-FP-NEXT: vmovaps %ymm5, 64(%rdx)
1459 ; AVX2-FP-NEXT: vmovaps %ymm4, 96(%rdx)
1460 ; AVX2-FP-NEXT: vmovaps %ymm2, (%rdx)
1461 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
1462 ; AVX2-FP-NEXT: vmovaps %ymm0, 32(%rdx)
1463 ; AVX2-FP-NEXT: vzeroupper
1464 ; AVX2-FP-NEXT: retq
1465 ;
1466 ; AVX2-FCP-LABEL: store_i32_stride2_vf64:
1467 ; AVX2-FCP: # %bb.0:
1468 ; AVX2-FCP-NEXT: vmovaps 224(%rdi), %ymm0
1469 ; AVX2-FCP-NEXT: vmovaps 192(%rdi), %ymm3
1470 ; AVX2-FCP-NEXT: vmovaps 160(%rdi), %ymm6
1471 ; AVX2-FCP-NEXT: vmovaps 128(%rdi), %ymm8
1472 ; AVX2-FCP-NEXT: vmovaps (%rdi), %ymm1
1473 ; AVX2-FCP-NEXT: vmovaps 32(%rdi), %ymm4
1474 ; AVX2-FCP-NEXT: vmovaps 64(%rdi), %ymm7
1475 ; AVX2-FCP-NEXT: vmovaps 96(%rdi), %ymm9
1476 ; AVX2-FCP-NEXT: vmovaps 192(%rsi), %ymm10
1477 ; AVX2-FCP-NEXT: vmovaps 160(%rsi), %ymm11
1478 ; AVX2-FCP-NEXT: vmovaps 128(%rsi), %ymm12
1479 ; AVX2-FCP-NEXT: vmovaps (%rsi), %ymm2
1480 ; AVX2-FCP-NEXT: vmovaps 32(%rsi), %ymm5
1481 ; AVX2-FCP-NEXT: vmovaps 64(%rsi), %ymm13
1482 ; AVX2-FCP-NEXT: vmovaps 96(%rsi), %ymm14
1483 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[6],ymm2[6],ymm1[7],ymm2[7]
1484 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm2 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[4],ymm2[4],ymm1[5],ymm2[5]
1485 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm2[2,3],ymm15[2,3]
1486 ; AVX2-FCP-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1487 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm2[0,1],ymm15[0,1]
1488 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[6],ymm5[6],ymm4[7],ymm5[7]
1489 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm5 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5]
1490 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm5[2,3],ymm15[2,3]
1491 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm5[0,1],ymm15[0,1]
1492 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm7[2],ymm13[2],ymm7[3],ymm13[3],ymm7[6],ymm13[6],ymm7[7],ymm13[7]
1493 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm13 = ymm7[0],ymm13[0],ymm7[1],ymm13[1],ymm7[4],ymm13[4],ymm7[5],ymm13[5]
1494 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm7 = ymm13[2,3],ymm15[2,3]
1495 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm13 = ymm13[0,1],ymm15[0,1]
1496 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm9[2],ymm14[2],ymm9[3],ymm14[3],ymm9[6],ymm14[6],ymm9[7],ymm14[7]
1497 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm9 = ymm9[0],ymm14[0],ymm9[1],ymm14[1],ymm9[4],ymm14[4],ymm9[5],ymm14[5]
1498 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm14 = ymm9[2,3],ymm15[2,3]
1499 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm9 = ymm9[0,1],ymm15[0,1]
1500 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm8[2],ymm12[2],ymm8[3],ymm12[3],ymm8[6],ymm12[6],ymm8[7],ymm12[7]
1501 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm8 = ymm8[0],ymm12[0],ymm8[1],ymm12[1],ymm8[4],ymm12[4],ymm8[5],ymm12[5]
1502 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm12 = ymm8[2,3],ymm15[2,3]
1503 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm8 = ymm8[0,1],ymm15[0,1]
1504 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm6[2],ymm11[2],ymm6[3],ymm11[3],ymm6[6],ymm11[6],ymm6[7],ymm11[7]
1505 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm6 = ymm6[0],ymm11[0],ymm6[1],ymm11[1],ymm6[4],ymm11[4],ymm6[5],ymm11[5]
1506 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm11 = ymm6[2,3],ymm15[2,3]
1507 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm6 = ymm6[0,1],ymm15[0,1]
1508 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm15 = ymm3[2],ymm10[2],ymm3[3],ymm10[3],ymm3[6],ymm10[6],ymm3[7],ymm10[7]
1509 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm3 = ymm3[0],ymm10[0],ymm3[1],ymm10[1],ymm3[4],ymm10[4],ymm3[5],ymm10[5]
1510 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm10 = ymm3[2,3],ymm15[2,3]
1511 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm3[0,1],ymm15[0,1]
1512 ; AVX2-FCP-NEXT: vmovaps 224(%rsi), %ymm15
1513 ; AVX2-FCP-NEXT: vmovaps %ymm0, %ymm1
1514 ; AVX2-FCP-NEXT: vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm15[2],ymm0[3],ymm15[3],ymm0[6],ymm15[6],ymm0[7],ymm15[7]
1515 ; AVX2-FCP-NEXT: vunpcklps {{.*#+}} ymm15 = ymm1[0],ymm15[0],ymm1[1],ymm15[1],ymm1[4],ymm15[4],ymm1[5],ymm15[5]
1516 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm15[2,3],ymm0[2,3]
1517 ; AVX2-FCP-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm15[0,1],ymm0[0,1]
1518 ; AVX2-FCP-NEXT: vmovaps %ymm0, 448(%rdx)
1519 ; AVX2-FCP-NEXT: vmovaps %ymm1, 480(%rdx)
1520 ; AVX2-FCP-NEXT: vmovaps %ymm3, 384(%rdx)
1521 ; AVX2-FCP-NEXT: vmovaps %ymm10, 416(%rdx)
1522 ; AVX2-FCP-NEXT: vmovaps %ymm6, 320(%rdx)
1523 ; AVX2-FCP-NEXT: vmovaps %ymm11, 352(%rdx)
1524 ; AVX2-FCP-NEXT: vmovaps %ymm8, 256(%rdx)
1525 ; AVX2-FCP-NEXT: vmovaps %ymm12, 288(%rdx)
1526 ; AVX2-FCP-NEXT: vmovaps %ymm9, 192(%rdx)
1527 ; AVX2-FCP-NEXT: vmovaps %ymm14, 224(%rdx)
1528 ; AVX2-FCP-NEXT: vmovaps %ymm13, 128(%rdx)
1529 ; AVX2-FCP-NEXT: vmovaps %ymm7, 160(%rdx)
1530 ; AVX2-FCP-NEXT: vmovaps %ymm5, 64(%rdx)
1531 ; AVX2-FCP-NEXT: vmovaps %ymm4, 96(%rdx)
1532 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rdx)
1533 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
1534 ; AVX2-FCP-NEXT: vmovaps %ymm0, 32(%rdx)
1535 ; AVX2-FCP-NEXT: vzeroupper
1536 ; AVX2-FCP-NEXT: retq
1537 ;
1538 ; AVX512-LABEL: store_i32_stride2_vf64:
1539 ; AVX512: # %bb.0:
1540 ; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
1541 ; AVX512-NEXT: vmovdqa64 64(%rdi), %zmm1
1542 ; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm2
1543 ; AVX512-NEXT: vmovdqa64 192(%rdi), %zmm3
1544 ; AVX512-NEXT: vmovdqa64 (%rsi), %zmm4
1545 ; AVX512-NEXT: vmovdqa64 64(%rsi), %zmm5
1546 ; AVX512-NEXT: vmovdqa64 128(%rsi), %zmm6
1547 ; AVX512-NEXT: vmovdqa64 192(%rsi), %zmm7
1548 ; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm8 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
1549 ; AVX512-NEXT: vmovdqa64 %zmm0, %zmm9
1550 ; AVX512-NEXT: vpermt2d %zmm4, %zmm8, %zmm9
1551 ; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm10 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
1552 ; AVX512-NEXT: vpermt2d %zmm4, %zmm10, %zmm0
1553 ; AVX512-NEXT: vmovdqa64 %zmm1, %zmm4
1554 ; AVX512-NEXT: vpermt2d %zmm5, %zmm8, %zmm4
1555 ; AVX512-NEXT: vpermt2d %zmm5, %zmm10, %zmm1
1556 ; AVX512-NEXT: vmovdqa64 %zmm2, %zmm5
1557 ; AVX512-NEXT: vpermt2d %zmm6, %zmm8, %zmm5
1558 ; AVX512-NEXT: vpermt2d %zmm6, %zmm10, %zmm2
1559 ; AVX512-NEXT: vpermi2d %zmm7, %zmm3, %zmm8
1560 ; AVX512-NEXT: vpermt2d %zmm7, %zmm10, %zmm3
1561 ; AVX512-NEXT: vmovdqa64 %zmm3, 384(%rdx)
1562 ; AVX512-NEXT: vmovdqa64 %zmm8, 448(%rdx)
1563 ; AVX512-NEXT: vmovdqa64 %zmm2, 256(%rdx)
1564 ; AVX512-NEXT: vmovdqa64 %zmm5, 320(%rdx)
1565 ; AVX512-NEXT: vmovdqa64 %zmm1, 128(%rdx)
1566 ; AVX512-NEXT: vmovdqa64 %zmm4, 192(%rdx)
1567 ; AVX512-NEXT: vmovdqa64 %zmm0, (%rdx)
1568 ; AVX512-NEXT: vmovdqa64 %zmm9, 64(%rdx)
1569 ; AVX512-NEXT: vzeroupper
1570 ; AVX512-NEXT: retq
1571 ;
1572 ; AVX512-FCP-LABEL: store_i32_stride2_vf64:
1573 ; AVX512-FCP: # %bb.0:
1574 ; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1575 ; AVX512-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1576 ; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
1577 ; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %zmm3
1578 ; AVX512-FCP-NEXT: vmovdqa64 (%rsi), %zmm4
1579 ; AVX512-FCP-NEXT: vmovdqa64 64(%rsi), %zmm5
1580 ; AVX512-FCP-NEXT: vmovdqa64 128(%rsi), %zmm6
1581 ; AVX512-FCP-NEXT: vmovdqa64 192(%rsi), %zmm7
1582 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm8 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
1583 ; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm9
1584 ; AVX512-FCP-NEXT: vpermt2d %zmm4, %zmm8, %zmm9
1585 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm10 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
1586 ; AVX512-FCP-NEXT: vpermt2d %zmm4, %zmm10, %zmm0
1587 ; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm4
1588 ; AVX512-FCP-NEXT: vpermt2d %zmm5, %zmm8, %zmm4
1589 ; AVX512-FCP-NEXT: vpermt2d %zmm5, %zmm10, %zmm1
1590 ; AVX512-FCP-NEXT: vmovdqa64 %zmm2, %zmm5
1591 ; AVX512-FCP-NEXT: vpermt2d %zmm6, %zmm8, %zmm5
1592 ; AVX512-FCP-NEXT: vpermt2d %zmm6, %zmm10, %zmm2
1593 ; AVX512-FCP-NEXT: vpermi2d %zmm7, %zmm3, %zmm8
1594 ; AVX512-FCP-NEXT: vpermt2d %zmm7, %zmm10, %zmm3
1595 ; AVX512-FCP-NEXT: vmovdqa64 %zmm3, 384(%rdx)
1596 ; AVX512-FCP-NEXT: vmovdqa64 %zmm8, 448(%rdx)
1597 ; AVX512-FCP-NEXT: vmovdqa64 %zmm2, 256(%rdx)
1598 ; AVX512-FCP-NEXT: vmovdqa64 %zmm5, 320(%rdx)
1599 ; AVX512-FCP-NEXT: vmovdqa64 %zmm1, 128(%rdx)
1600 ; AVX512-FCP-NEXT: vmovdqa64 %zmm4, 192(%rdx)
1601 ; AVX512-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
1602 ; AVX512-FCP-NEXT: vmovdqa64 %zmm9, 64(%rdx)
1603 ; AVX512-FCP-NEXT: vzeroupper
1604 ; AVX512-FCP-NEXT: retq
1605 ;
1606 ; AVX512DQ-LABEL: store_i32_stride2_vf64:
1607 ; AVX512DQ: # %bb.0:
1608 ; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
1609 ; AVX512DQ-NEXT: vmovdqa64 64(%rdi), %zmm1
1610 ; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm2
1611 ; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %zmm3
1612 ; AVX512DQ-NEXT: vmovdqa64 (%rsi), %zmm4
1613 ; AVX512DQ-NEXT: vmovdqa64 64(%rsi), %zmm5
1614 ; AVX512DQ-NEXT: vmovdqa64 128(%rsi), %zmm6
1615 ; AVX512DQ-NEXT: vmovdqa64 192(%rsi), %zmm7
1616 ; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm8 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
1617 ; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm9
1618 ; AVX512DQ-NEXT: vpermt2d %zmm4, %zmm8, %zmm9
1619 ; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm10 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
1620 ; AVX512DQ-NEXT: vpermt2d %zmm4, %zmm10, %zmm0
1621 ; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm4
1622 ; AVX512DQ-NEXT: vpermt2d %zmm5, %zmm8, %zmm4
1623 ; AVX512DQ-NEXT: vpermt2d %zmm5, %zmm10, %zmm1
1624 ; AVX512DQ-NEXT: vmovdqa64 %zmm2, %zmm5
1625 ; AVX512DQ-NEXT: vpermt2d %zmm6, %zmm8, %zmm5
1626 ; AVX512DQ-NEXT: vpermt2d %zmm6, %zmm10, %zmm2
1627 ; AVX512DQ-NEXT: vpermi2d %zmm7, %zmm3, %zmm8
1628 ; AVX512DQ-NEXT: vpermt2d %zmm7, %zmm10, %zmm3
1629 ; AVX512DQ-NEXT: vmovdqa64 %zmm3, 384(%rdx)
1630 ; AVX512DQ-NEXT: vmovdqa64 %zmm8, 448(%rdx)
1631 ; AVX512DQ-NEXT: vmovdqa64 %zmm2, 256(%rdx)
1632 ; AVX512DQ-NEXT: vmovdqa64 %zmm5, 320(%rdx)
1633 ; AVX512DQ-NEXT: vmovdqa64 %zmm1, 128(%rdx)
1634 ; AVX512DQ-NEXT: vmovdqa64 %zmm4, 192(%rdx)
1635 ; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%rdx)
1636 ; AVX512DQ-NEXT: vmovdqa64 %zmm9, 64(%rdx)
1637 ; AVX512DQ-NEXT: vzeroupper
1638 ; AVX512DQ-NEXT: retq
1639 ;
1640 ; AVX512DQ-FCP-LABEL: store_i32_stride2_vf64:
1641 ; AVX512DQ-FCP: # %bb.0:
1642 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1643 ; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1644 ; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
1645 ; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %zmm3
1646 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rsi), %zmm4
1647 ; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rsi), %zmm5
1648 ; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rsi), %zmm6
1649 ; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rsi), %zmm7
1650 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm8 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
1651 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm9
1652 ; AVX512DQ-FCP-NEXT: vpermt2d %zmm4, %zmm8, %zmm9
1653 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm10 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
1654 ; AVX512DQ-FCP-NEXT: vpermt2d %zmm4, %zmm10, %zmm0
1655 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm4
1656 ; AVX512DQ-FCP-NEXT: vpermt2d %zmm5, %zmm8, %zmm4
1657 ; AVX512DQ-FCP-NEXT: vpermt2d %zmm5, %zmm10, %zmm1
1658 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, %zmm5
1659 ; AVX512DQ-FCP-NEXT: vpermt2d %zmm6, %zmm8, %zmm5
1660 ; AVX512DQ-FCP-NEXT: vpermt2d %zmm6, %zmm10, %zmm2
1661 ; AVX512DQ-FCP-NEXT: vpermi2d %zmm7, %zmm3, %zmm8
1662 ; AVX512DQ-FCP-NEXT: vpermt2d %zmm7, %zmm10, %zmm3
1663 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, 384(%rdx)
1664 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm8, 448(%rdx)
1665 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, 256(%rdx)
1666 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, 320(%rdx)
1667 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, 128(%rdx)
1668 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm4, 192(%rdx)
1669 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
1670 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm9, 64(%rdx)
1671 ; AVX512DQ-FCP-NEXT: vzeroupper
1672 ; AVX512DQ-FCP-NEXT: retq
1673 ;
1674 ; AVX512BW-LABEL: store_i32_stride2_vf64:
1675 ; AVX512BW: # %bb.0:
1676 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
1677 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
1678 ; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm2
1679 ; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm3
1680 ; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm4
1681 ; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm5
1682 ; AVX512BW-NEXT: vmovdqa64 128(%rsi), %zmm6
1683 ; AVX512BW-NEXT: vmovdqa64 192(%rsi), %zmm7
1684 ; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm8 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
1685 ; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm9
1686 ; AVX512BW-NEXT: vpermt2d %zmm4, %zmm8, %zmm9
1687 ; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm10 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
1688 ; AVX512BW-NEXT: vpermt2d %zmm4, %zmm10, %zmm0
1689 ; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm4
1690 ; AVX512BW-NEXT: vpermt2d %zmm5, %zmm8, %zmm4
1691 ; AVX512BW-NEXT: vpermt2d %zmm5, %zmm10, %zmm1
1692 ; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm5
1693 ; AVX512BW-NEXT: vpermt2d %zmm6, %zmm8, %zmm5
1694 ; AVX512BW-NEXT: vpermt2d %zmm6, %zmm10, %zmm2
1695 ; AVX512BW-NEXT: vpermi2d %zmm7, %zmm3, %zmm8
1696 ; AVX512BW-NEXT: vpermt2d %zmm7, %zmm10, %zmm3
1697 ; AVX512BW-NEXT: vmovdqa64 %zmm3, 384(%rdx)
1698 ; AVX512BW-NEXT: vmovdqa64 %zmm8, 448(%rdx)
1699 ; AVX512BW-NEXT: vmovdqa64 %zmm2, 256(%rdx)
1700 ; AVX512BW-NEXT: vmovdqa64 %zmm5, 320(%rdx)
1701 ; AVX512BW-NEXT: vmovdqa64 %zmm1, 128(%rdx)
1702 ; AVX512BW-NEXT: vmovdqa64 %zmm4, 192(%rdx)
1703 ; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
1704 ; AVX512BW-NEXT: vmovdqa64 %zmm9, 64(%rdx)
1705 ; AVX512BW-NEXT: vzeroupper
1706 ; AVX512BW-NEXT: retq
1707 ;
1708 ; AVX512BW-FCP-LABEL: store_i32_stride2_vf64:
1709 ; AVX512BW-FCP: # %bb.0:
1710 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1711 ; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1712 ; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
1713 ; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm3
1714 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm4
1715 ; AVX512BW-FCP-NEXT: vmovdqa64 64(%rsi), %zmm5
1716 ; AVX512BW-FCP-NEXT: vmovdqa64 128(%rsi), %zmm6
1717 ; AVX512BW-FCP-NEXT: vmovdqa64 192(%rsi), %zmm7
1718 ; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm8 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
1719 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm9
1720 ; AVX512BW-FCP-NEXT: vpermt2d %zmm4, %zmm8, %zmm9
1721 ; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm10 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
1722 ; AVX512BW-FCP-NEXT: vpermt2d %zmm4, %zmm10, %zmm0
1723 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm4
1724 ; AVX512BW-FCP-NEXT: vpermt2d %zmm5, %zmm8, %zmm4
1725 ; AVX512BW-FCP-NEXT: vpermt2d %zmm5, %zmm10, %zmm1
1726 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm5
1727 ; AVX512BW-FCP-NEXT: vpermt2d %zmm6, %zmm8, %zmm5
1728 ; AVX512BW-FCP-NEXT: vpermt2d %zmm6, %zmm10, %zmm2
1729 ; AVX512BW-FCP-NEXT: vpermi2d %zmm7, %zmm3, %zmm8
1730 ; AVX512BW-FCP-NEXT: vpermt2d %zmm7, %zmm10, %zmm3
1731 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, 384(%rdx)
1732 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm8, 448(%rdx)
1733 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, 256(%rdx)
1734 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, 320(%rdx)
1735 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, 128(%rdx)
1736 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, 192(%rdx)
1737 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
1738 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm9, 64(%rdx)
1739 ; AVX512BW-FCP-NEXT: vzeroupper
1740 ; AVX512BW-FCP-NEXT: retq
1741 ;
1742 ; AVX512DQ-BW-LABEL: store_i32_stride2_vf64:
1743 ; AVX512DQ-BW: # %bb.0:
1744 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
1745 ; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm1
1746 ; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm2
1747 ; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %zmm3
1748 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rsi), %zmm4
1749 ; AVX512DQ-BW-NEXT: vmovdqa64 64(%rsi), %zmm5
1750 ; AVX512DQ-BW-NEXT: vmovdqa64 128(%rsi), %zmm6
1751 ; AVX512DQ-BW-NEXT: vmovdqa64 192(%rsi), %zmm7
1752 ; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} zmm8 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
1753 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm9
1754 ; AVX512DQ-BW-NEXT: vpermt2d %zmm4, %zmm8, %zmm9
1755 ; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} zmm10 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
1756 ; AVX512DQ-BW-NEXT: vpermt2d %zmm4, %zmm10, %zmm0
1757 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm4
1758 ; AVX512DQ-BW-NEXT: vpermt2d %zmm5, %zmm8, %zmm4
1759 ; AVX512DQ-BW-NEXT: vpermt2d %zmm5, %zmm10, %zmm1
1760 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, %zmm5
1761 ; AVX512DQ-BW-NEXT: vpermt2d %zmm6, %zmm8, %zmm5
1762 ; AVX512DQ-BW-NEXT: vpermt2d %zmm6, %zmm10, %zmm2
1763 ; AVX512DQ-BW-NEXT: vpermi2d %zmm7, %zmm3, %zmm8
1764 ; AVX512DQ-BW-NEXT: vpermt2d %zmm7, %zmm10, %zmm3
1765 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, 384(%rdx)
1766 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm8, 448(%rdx)
1767 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, 256(%rdx)
1768 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, 320(%rdx)
1769 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, 128(%rdx)
1770 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, 192(%rdx)
1771 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, (%rdx)
1772 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm9, 64(%rdx)
1773 ; AVX512DQ-BW-NEXT: vzeroupper
1774 ; AVX512DQ-BW-NEXT: retq
1775 ;
1776 ; AVX512DQ-BW-FCP-LABEL: store_i32_stride2_vf64:
1777 ; AVX512DQ-BW-FCP: # %bb.0:
1778 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1779 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1780 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
1781 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm3
1782 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm4
1783 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rsi), %zmm5
1784 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rsi), %zmm6
1785 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rsi), %zmm7
1786 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm8 = [8,24,9,25,10,26,11,27,12,28,13,29,14,30,15,31]
1787 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm9
1788 ; AVX512DQ-BW-FCP-NEXT: vpermt2d %zmm4, %zmm8, %zmm9
1789 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm10 = [0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23]
1790 ; AVX512DQ-BW-FCP-NEXT: vpermt2d %zmm4, %zmm10, %zmm0
1791 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm4
1792 ; AVX512DQ-BW-FCP-NEXT: vpermt2d %zmm5, %zmm8, %zmm4
1793 ; AVX512DQ-BW-FCP-NEXT: vpermt2d %zmm5, %zmm10, %zmm1
1794 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm5
1795 ; AVX512DQ-BW-FCP-NEXT: vpermt2d %zmm6, %zmm8, %zmm5
1796 ; AVX512DQ-BW-FCP-NEXT: vpermt2d %zmm6, %zmm10, %zmm2
1797 ; AVX512DQ-BW-FCP-NEXT: vpermi2d %zmm7, %zmm3, %zmm8
1798 ; AVX512DQ-BW-FCP-NEXT: vpermt2d %zmm7, %zmm10, %zmm3
1799 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, 384(%rdx)
1800 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, 448(%rdx)
1801 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, 256(%rdx)
1802 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, 320(%rdx)
1803 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, 128(%rdx)
1804 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, 192(%rdx)
1805 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
1806 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm9, 64(%rdx)
1807 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
1808 ; AVX512DQ-BW-FCP-NEXT: retq
1809 %in.vec0 = load <64 x i32>, ptr %in.vecptr0, align 64
1810 %in.vec1 = load <64 x i32>, ptr %in.vecptr1, align 64
1811 %1 = shufflevector <64 x i32> %in.vec0, <64 x i32> %in.vec1, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
1812 %interleaved.vec = shufflevector <128 x i32> %1, <128 x i32> poison, <128 x i32> <i32 0, i32 64, i32 1, i32 65, i32 2, i32 66, i32 3, i32 67, i32 4, i32 68, i32 5, i32 69, i32 6, i32 70, i32 7, i32 71, i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 16, i32 80, i32 17, i32 81, i32 18, i32 82, i32 19, i32 83, i32 20, i32 84, i32 21, i32 85, i32 22, i32 86, i32 23, i32 87, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 32, i32 96, i32 33, i32 97, i32 34, i32 98, i32 35, i32 99, i32 36, i32 100, i32 37, i32 101, i32 38, i32 102, i32 39, i32 103, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 48, i32 112, i32 49, i32 113, i32 50, i32 114, i32 51, i32 115, i32 52, i32 116, i32 53, i32 117, i32 54, i32 118, i32 55, i32 119, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>
1813 store <128 x i32> %interleaved.vec, ptr %out.vec, align 64
1814 ret void
1815 }