| blob | b3d8d05f699472589a1b72fc14c66e34e70857ff |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE
3 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX
4 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
5 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FP
6 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FCP
7 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512-VL
8 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512-FCP
9 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512,AVX512DQ
10 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512DQ-FCP
11 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
12 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW-FCP
13 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512DQ-BW
14 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512DQ-BW-FCP
16 ; These patterns are produced by LoopVectorizer for interleaved loads.
18 define void @load_i16_stride2_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nounwind {
19 ; SSE-LABEL: load_i16_stride2_vf2:
20 ; SSE: # %bb.0:
21 ; SSE-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
22 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
23 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
24 ; SSE-NEXT: movd %xmm1, (%rsi)
25 ; SSE-NEXT: movd %xmm0, (%rdx)
26 ; SSE-NEXT: retq
27 ;
28 ; AVX-LABEL: load_i16_stride2_vf2:
29 ; AVX: # %bb.0:
30 ; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
31 ; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
32 ; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
33 ; AVX-NEXT: vmovd %xmm1, (%rsi)
34 ; AVX-NEXT: vmovd %xmm0, (%rdx)
35 ; AVX-NEXT: retq
36 ;
37 ; AVX2-LABEL: load_i16_stride2_vf2:
38 ; AVX2: # %bb.0:
39 ; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
40 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
41 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
42 ; AVX2-NEXT: vmovd %xmm1, (%rsi)
43 ; AVX2-NEXT: vmovd %xmm0, (%rdx)
44 ; AVX2-NEXT: retq
45 ;
46 ; AVX2-FP-LABEL: load_i16_stride2_vf2:
47 ; AVX2-FP: # %bb.0:
48 ; AVX2-FP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
49 ; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
50 ; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
51 ; AVX2-FP-NEXT: vmovd %xmm1, (%rsi)
52 ; AVX2-FP-NEXT: vmovd %xmm0, (%rdx)
53 ; AVX2-FP-NEXT: retq
54 ;
55 ; AVX2-FCP-LABEL: load_i16_stride2_vf2:
56 ; AVX2-FCP: # %bb.0:
57 ; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
58 ; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
59 ; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
60 ; AVX2-FCP-NEXT: vmovd %xmm1, (%rsi)
61 ; AVX2-FCP-NEXT: vmovd %xmm0, (%rdx)
62 ; AVX2-FCP-NEXT: retq
63 ;
64 ; AVX512-LABEL: load_i16_stride2_vf2:
65 ; AVX512: # %bb.0:
66 ; AVX512-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
67 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
68 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
69 ; AVX512-NEXT: vmovd %xmm1, (%rsi)
70 ; AVX512-NEXT: vmovd %xmm0, (%rdx)
71 ; AVX512-NEXT: retq
72 %wide.vec = load <4 x i16>, ptr %in.vec, align 64
73 %strided.vec0 = shufflevector <4 x i16> %wide.vec, <4 x i16> poison, <2 x i32> <i32 0, i32 2>
74 %strided.vec1 = shufflevector <4 x i16> %wide.vec, <4 x i16> poison, <2 x i32> <i32 1, i32 3>
75 store <2 x i16> %strided.vec0, ptr %out.vec0, align 64
76 store <2 x i16> %strided.vec1, ptr %out.vec1, align 64
77 ret void
78 }
80 define void @load_i16_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nounwind {
81 ; SSE-LABEL: load_i16_stride2_vf4:
82 ; SSE: # %bb.0:
83 ; SSE-NEXT: movdqa (%rdi), %xmm0
84 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
85 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
86 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
87 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]
88 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,7]
89 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
90 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7]
91 ; SSE-NEXT: movq %xmm1, (%rsi)
92 ; SSE-NEXT: movq %xmm0, (%rdx)
93 ; SSE-NEXT: retq
94 ;
95 ; AVX-LABEL: load_i16_stride2_vf4:
96 ; AVX: # %bb.0:
97 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
98 ; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u]
99 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
100 ; AVX-NEXT: vmovq %xmm1, (%rsi)
101 ; AVX-NEXT: vmovq %xmm0, (%rdx)
102 ; AVX-NEXT: retq
103 ;
104 ; AVX2-LABEL: load_i16_stride2_vf4:
105 ; AVX2: # %bb.0:
106 ; AVX2-NEXT: vmovdqa (%rdi), %xmm0
107 ; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u]
108 ; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
109 ; AVX2-NEXT: vmovq %xmm1, (%rsi)
110 ; AVX2-NEXT: vmovq %xmm0, (%rdx)
111 ; AVX2-NEXT: retq
112 ;
113 ; AVX2-FP-LABEL: load_i16_stride2_vf4:
114 ; AVX2-FP: # %bb.0:
115 ; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
116 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u]
117 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
118 ; AVX2-FP-NEXT: vmovq %xmm1, (%rsi)
119 ; AVX2-FP-NEXT: vmovq %xmm0, (%rdx)
120 ; AVX2-FP-NEXT: retq
121 ;
122 ; AVX2-FCP-LABEL: load_i16_stride2_vf4:
123 ; AVX2-FCP: # %bb.0:
124 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0
125 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u]
126 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
127 ; AVX2-FCP-NEXT: vmovq %xmm1, (%rsi)
128 ; AVX2-FCP-NEXT: vmovq %xmm0, (%rdx)
129 ; AVX2-FCP-NEXT: retq
130 ;
131 ; AVX512-LABEL: load_i16_stride2_vf4:
132 ; AVX512: # %bb.0:
133 ; AVX512-NEXT: vmovdqa (%rdi), %xmm0
134 ; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
135 ; AVX512-NEXT: vpmovdw %xmm0, (%rsi)
136 ; AVX512-NEXT: vmovq %xmm1, (%rdx)
137 ; AVX512-NEXT: retq
138 %wide.vec = load <8 x i16>, ptr %in.vec, align 64
139 %strided.vec0 = shufflevector <8 x i16> %wide.vec, <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
140 %strided.vec1 = shufflevector <8 x i16> %wide.vec, <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
141 store <4 x i16> %strided.vec0, ptr %out.vec0, align 64
142 store <4 x i16> %strided.vec1, ptr %out.vec1, align 64
143 ret void
144 }
146 define void @load_i16_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nounwind {
147 ; SSE-LABEL: load_i16_stride2_vf8:
148 ; SSE: # %bb.0:
149 ; SSE-NEXT: movdqa (%rdi), %xmm0
150 ; SSE-NEXT: movdqa 16(%rdi), %xmm1
151 ; SSE-NEXT: movdqa %xmm1, %xmm2
152 ; SSE-NEXT: pslld $16, %xmm2
153 ; SSE-NEXT: psrad $16, %xmm2
154 ; SSE-NEXT: movdqa %xmm0, %xmm3
155 ; SSE-NEXT: pslld $16, %xmm3
156 ; SSE-NEXT: psrad $16, %xmm3
157 ; SSE-NEXT: packssdw %xmm2, %xmm3
158 ; SSE-NEXT: psrad $16, %xmm1
159 ; SSE-NEXT: psrad $16, %xmm0
160 ; SSE-NEXT: packssdw %xmm1, %xmm0
161 ; SSE-NEXT: movdqa %xmm3, (%rsi)
162 ; SSE-NEXT: movdqa %xmm0, (%rdx)
163 ; SSE-NEXT: retq
164 ;
165 ; AVX-LABEL: load_i16_stride2_vf8:
166 ; AVX: # %bb.0:
167 ; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
168 ; AVX-NEXT: vmovdqa (%rdi), %xmm1
169 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
170 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
171 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
172 ; AVX-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
173 ; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
174 ; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
175 ; AVX-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
176 ; AVX-NEXT: vmovdqa %xmm0, (%rsi)
177 ; AVX-NEXT: vmovdqa %xmm1, (%rdx)
178 ; AVX-NEXT: retq
179 ;
180 ; AVX2-LABEL: load_i16_stride2_vf8:
181 ; AVX2: # %bb.0:
182 ; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
183 ; AVX2-NEXT: vmovdqa (%rdi), %xmm1
184 ; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2
185 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
186 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
187 ; AVX2-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
188 ; AVX2-NEXT: vpsrld $16, %xmm2, %xmm2
189 ; AVX2-NEXT: vpsrld $16, %xmm1, %xmm1
190 ; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
191 ; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
192 ; AVX2-NEXT: vmovdqa %xmm1, (%rdx)
193 ; AVX2-NEXT: retq
194 ;
195 ; AVX2-FP-LABEL: load_i16_stride2_vf8:
196 ; AVX2-FP: # %bb.0:
197 ; AVX2-FP-NEXT: vpxor %xmm0, %xmm0, %xmm0
198 ; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1
199 ; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2
200 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
201 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
202 ; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
203 ; AVX2-FP-NEXT: vpsrld $16, %xmm2, %xmm2
204 ; AVX2-FP-NEXT: vpsrld $16, %xmm1, %xmm1
205 ; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
206 ; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsi)
207 ; AVX2-FP-NEXT: vmovdqa %xmm1, (%rdx)
208 ; AVX2-FP-NEXT: retq
209 ;
210 ; AVX2-FCP-LABEL: load_i16_stride2_vf8:
211 ; AVX2-FCP: # %bb.0:
212 ; AVX2-FCP-NEXT: vpxor %xmm0, %xmm0, %xmm0
213 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1
214 ; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
215 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
216 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
217 ; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
218 ; AVX2-FCP-NEXT: vpsrld $16, %xmm2, %xmm2
219 ; AVX2-FCP-NEXT: vpsrld $16, %xmm1, %xmm1
220 ; AVX2-FCP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
221 ; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rsi)
222 ; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rdx)
223 ; AVX2-FCP-NEXT: retq
224 ;
225 ; AVX512-LABEL: load_i16_stride2_vf8:
226 ; AVX512: # %bb.0:
227 ; AVX512-NEXT: vmovdqa (%rdi), %ymm0
228 ; AVX512-NEXT: vpsrld $16, %ymm0, %ymm1
229 ; AVX512-NEXT: vpmovdw %ymm0, (%rsi)
230 ; AVX512-NEXT: vpmovdw %ymm1, (%rdx)
231 ; AVX512-NEXT: vzeroupper
232 ; AVX512-NEXT: retq
233 %wide.vec = load <16 x i16>, ptr %in.vec, align 64
234 %strided.vec0 = shufflevector <16 x i16> %wide.vec, <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
235 %strided.vec1 = shufflevector <16 x i16> %wide.vec, <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
236 store <8 x i16> %strided.vec0, ptr %out.vec0, align 64
237 store <8 x i16> %strided.vec1, ptr %out.vec1, align 64
238 ret void
239 }
241 define void @load_i16_stride2_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nounwind {
242 ; SSE-LABEL: load_i16_stride2_vf16:
243 ; SSE: # %bb.0:
244 ; SSE-NEXT: movdqa (%rdi), %xmm0
245 ; SSE-NEXT: movdqa 16(%rdi), %xmm2
246 ; SSE-NEXT: movdqa 32(%rdi), %xmm1
247 ; SSE-NEXT: movdqa 48(%rdi), %xmm3
248 ; SSE-NEXT: movdqa %xmm3, %xmm4
249 ; SSE-NEXT: pslld $16, %xmm4
250 ; SSE-NEXT: psrad $16, %xmm4
251 ; SSE-NEXT: movdqa %xmm1, %xmm5
252 ; SSE-NEXT: pslld $16, %xmm5
253 ; SSE-NEXT: psrad $16, %xmm5
254 ; SSE-NEXT: packssdw %xmm4, %xmm5
255 ; SSE-NEXT: movdqa %xmm2, %xmm4
256 ; SSE-NEXT: pslld $16, %xmm4
257 ; SSE-NEXT: psrad $16, %xmm4
258 ; SSE-NEXT: movdqa %xmm0, %xmm6
259 ; SSE-NEXT: pslld $16, %xmm6
260 ; SSE-NEXT: psrad $16, %xmm6
261 ; SSE-NEXT: packssdw %xmm4, %xmm6
262 ; SSE-NEXT: psrad $16, %xmm3
263 ; SSE-NEXT: psrad $16, %xmm1
264 ; SSE-NEXT: packssdw %xmm3, %xmm1
265 ; SSE-NEXT: psrad $16, %xmm2
266 ; SSE-NEXT: psrad $16, %xmm0
267 ; SSE-NEXT: packssdw %xmm2, %xmm0
268 ; SSE-NEXT: movdqa %xmm6, (%rsi)
269 ; SSE-NEXT: movdqa %xmm5, 16(%rsi)
270 ; SSE-NEXT: movdqa %xmm0, (%rdx)
271 ; SSE-NEXT: movdqa %xmm1, 16(%rdx)
272 ; SSE-NEXT: retq
273 ;
274 ; AVX-LABEL: load_i16_stride2_vf16:
275 ; AVX: # %bb.0:
276 ; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
277 ; AVX-NEXT: vmovdqa (%rdi), %xmm1
278 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
279 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm3
280 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm4
281 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm0[1],xmm4[2],xmm0[3],xmm4[4],xmm0[5],xmm4[6],xmm0[7]
282 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
283 ; AVX-NEXT: vpackusdw %xmm5, %xmm6, %xmm5
284 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
285 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
286 ; AVX-NEXT: vpackusdw %xmm6, %xmm0, %xmm0
287 ; AVX-NEXT: vpsrld $16, %xmm4, %xmm4
288 ; AVX-NEXT: vpsrld $16, %xmm3, %xmm3
289 ; AVX-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
290 ; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
291 ; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
292 ; AVX-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
293 ; AVX-NEXT: vmovdqa %xmm0, (%rsi)
294 ; AVX-NEXT: vmovdqa %xmm5, 16(%rsi)
295 ; AVX-NEXT: vmovdqa %xmm1, (%rdx)
296 ; AVX-NEXT: vmovdqa %xmm3, 16(%rdx)
297 ; AVX-NEXT: retq
298 ;
299 ; AVX2-LABEL: load_i16_stride2_vf16:
300 ; AVX2: # %bb.0:
301 ; AVX2-NEXT: vmovdqa (%rdi), %ymm0
302 ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1
303 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
304 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
305 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm3 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
306 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
307 ; AVX2-NEXT: vshufps {{.*#+}} ymm2 = ymm3[0,2],ymm2[0,2],ymm3[4,6],ymm2[4,6]
308 ; AVX2-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
309 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
310 ; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1
311 ; AVX2-NEXT: vpshufb %ymm3, %ymm0, %ymm0
312 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
313 ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
314 ; AVX2-NEXT: vmovaps %ymm2, (%rsi)
315 ; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
316 ; AVX2-NEXT: vzeroupper
317 ; AVX2-NEXT: retq
318 ;
319 ; AVX2-FP-LABEL: load_i16_stride2_vf16:
320 ; AVX2-FP: # %bb.0:
321 ; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm0
322 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm1
323 ; AVX2-FP-NEXT: vpmovsxdq {{.*#+}} ymm2 = [84148480,218892552,353636624,488380696]
324 ; AVX2-FP-NEXT: vpshufb %ymm2, %ymm1, %ymm3
325 ; AVX2-FP-NEXT: vpshufb %ymm2, %ymm0, %ymm2
326 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm3[0,2],ymm2[4,6],ymm3[4,6]
327 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
328 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm3 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
329 ; AVX2-FP-NEXT: vpshufb %ymm3, %ymm1, %ymm1
330 ; AVX2-FP-NEXT: vpshufb %ymm3, %ymm0, %ymm0
331 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
332 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
333 ; AVX2-FP-NEXT: vmovaps %ymm2, (%rsi)
334 ; AVX2-FP-NEXT: vmovdqa %ymm0, (%rdx)
335 ; AVX2-FP-NEXT: vzeroupper
336 ; AVX2-FP-NEXT: retq
337 ;
338 ; AVX2-FCP-LABEL: load_i16_stride2_vf16:
339 ; AVX2-FCP: # %bb.0:
340 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm0
341 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
342 ; AVX2-FCP-NEXT: vpmovsxdq {{.*#+}} ymm2 = [84148480,218892552,353636624,488380696]
343 ; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm3
344 ; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm2
345 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm3[0,2],ymm2[4,6],ymm3[4,6]
346 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
347 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
348 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm1
349 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm0, %ymm0
350 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
351 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
352 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rsi)
353 ; AVX2-FCP-NEXT: vmovdqa %ymm0, (%rdx)
354 ; AVX2-FCP-NEXT: vzeroupper
355 ; AVX2-FCP-NEXT: retq
356 ;
357 ; AVX512-LABEL: load_i16_stride2_vf16:
358 ; AVX512: # %bb.0:
359 ; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
360 ; AVX512-NEXT: vpsrld $16, %zmm0, %zmm1
361 ; AVX512-NEXT: vpmovdw %zmm0, (%rsi)
362 ; AVX512-NEXT: vpmovdw %zmm1, (%rdx)
363 ; AVX512-NEXT: vzeroupper
364 ; AVX512-NEXT: retq
365 %wide.vec = load <32 x i16>, ptr %in.vec, align 64
366 %strided.vec0 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
367 %strided.vec1 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
368 store <16 x i16> %strided.vec0, ptr %out.vec0, align 64
369 store <16 x i16> %strided.vec1, ptr %out.vec1, align 64
370 ret void
371 }
373 define void @load_i16_stride2_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nounwind {
374 ; SSE-LABEL: load_i16_stride2_vf32:
375 ; SSE: # %bb.0:
376 ; SSE-NEXT: movdqa 64(%rdi), %xmm0
377 ; SSE-NEXT: movdqa 80(%rdi), %xmm4
378 ; SSE-NEXT: movdqa 96(%rdi), %xmm1
379 ; SSE-NEXT: movdqa 112(%rdi), %xmm6
380 ; SSE-NEXT: movdqa (%rdi), %xmm2
381 ; SSE-NEXT: movdqa 16(%rdi), %xmm7
382 ; SSE-NEXT: movdqa 32(%rdi), %xmm3
383 ; SSE-NEXT: movdqa 48(%rdi), %xmm9
384 ; SSE-NEXT: movdqa %xmm9, %xmm8
385 ; SSE-NEXT: pslld $16, %xmm8
386 ; SSE-NEXT: psrad $16, %xmm8
387 ; SSE-NEXT: movdqa %xmm3, %xmm5
388 ; SSE-NEXT: pslld $16, %xmm5
389 ; SSE-NEXT: psrad $16, %xmm5
390 ; SSE-NEXT: packssdw %xmm8, %xmm5
391 ; SSE-NEXT: movdqa %xmm7, %xmm10
392 ; SSE-NEXT: pslld $16, %xmm10
393 ; SSE-NEXT: psrad $16, %xmm10
394 ; SSE-NEXT: movdqa %xmm2, %xmm8
395 ; SSE-NEXT: pslld $16, %xmm8
396 ; SSE-NEXT: psrad $16, %xmm8
397 ; SSE-NEXT: packssdw %xmm10, %xmm8
398 ; SSE-NEXT: movdqa %xmm6, %xmm11
399 ; SSE-NEXT: pslld $16, %xmm11
400 ; SSE-NEXT: psrad $16, %xmm11
401 ; SSE-NEXT: movdqa %xmm1, %xmm10
402 ; SSE-NEXT: pslld $16, %xmm10
403 ; SSE-NEXT: psrad $16, %xmm10
404 ; SSE-NEXT: packssdw %xmm11, %xmm10
405 ; SSE-NEXT: movdqa %xmm4, %xmm11
406 ; SSE-NEXT: pslld $16, %xmm11
407 ; SSE-NEXT: psrad $16, %xmm11
408 ; SSE-NEXT: movdqa %xmm0, %xmm12
409 ; SSE-NEXT: pslld $16, %xmm12
410 ; SSE-NEXT: psrad $16, %xmm12
411 ; SSE-NEXT: packssdw %xmm11, %xmm12
412 ; SSE-NEXT: psrad $16, %xmm9
413 ; SSE-NEXT: psrad $16, %xmm3
414 ; SSE-NEXT: packssdw %xmm9, %xmm3
415 ; SSE-NEXT: psrad $16, %xmm7
416 ; SSE-NEXT: psrad $16, %xmm2
417 ; SSE-NEXT: packssdw %xmm7, %xmm2
418 ; SSE-NEXT: psrad $16, %xmm6
419 ; SSE-NEXT: psrad $16, %xmm1
420 ; SSE-NEXT: packssdw %xmm6, %xmm1
421 ; SSE-NEXT: psrad $16, %xmm4
422 ; SSE-NEXT: psrad $16, %xmm0
423 ; SSE-NEXT: packssdw %xmm4, %xmm0
424 ; SSE-NEXT: movdqa %xmm12, 32(%rsi)
425 ; SSE-NEXT: movdqa %xmm10, 48(%rsi)
426 ; SSE-NEXT: movdqa %xmm8, (%rsi)
427 ; SSE-NEXT: movdqa %xmm5, 16(%rsi)
428 ; SSE-NEXT: movdqa %xmm0, 32(%rdx)
429 ; SSE-NEXT: movdqa %xmm1, 48(%rdx)
430 ; SSE-NEXT: movdqa %xmm2, (%rdx)
431 ; SSE-NEXT: movdqa %xmm3, 16(%rdx)
432 ; SSE-NEXT: retq
433 ;
434 ; AVX-LABEL: load_i16_stride2_vf32:
435 ; AVX: # %bb.0:
436 ; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
437 ; AVX-NEXT: vmovdqa (%rdi), %xmm1
438 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
439 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm3
440 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm4
441 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm0[1],xmm4[2],xmm0[3],xmm4[4],xmm0[5],xmm4[6],xmm0[7]
442 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
443 ; AVX-NEXT: vpackusdw %xmm5, %xmm6, %xmm5
444 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
445 ; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
446 ; AVX-NEXT: vpackusdw %xmm6, %xmm7, %xmm6
447 ; AVX-NEXT: vmovdqa 112(%rdi), %xmm7
448 ; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm7[0],xmm0[1],xmm7[2],xmm0[3],xmm7[4],xmm0[5],xmm7[6],xmm0[7]
449 ; AVX-NEXT: vmovdqa 96(%rdi), %xmm9
450 ; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm9[0],xmm0[1],xmm9[2],xmm0[3],xmm9[4],xmm0[5],xmm9[6],xmm0[7]
451 ; AVX-NEXT: vpackusdw %xmm8, %xmm10, %xmm8
452 ; AVX-NEXT: vmovdqa 80(%rdi), %xmm10
453 ; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm10[0],xmm0[1],xmm10[2],xmm0[3],xmm10[4],xmm0[5],xmm10[6],xmm0[7]
454 ; AVX-NEXT: vmovdqa 64(%rdi), %xmm12
455 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm12[0],xmm0[1],xmm12[2],xmm0[3],xmm12[4],xmm0[5],xmm12[6],xmm0[7]
456 ; AVX-NEXT: vpackusdw %xmm11, %xmm0, %xmm0
457 ; AVX-NEXT: vpsrld $16, %xmm4, %xmm4
458 ; AVX-NEXT: vpsrld $16, %xmm3, %xmm3
459 ; AVX-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
460 ; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
461 ; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
462 ; AVX-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
463 ; AVX-NEXT: vpsrld $16, %xmm7, %xmm2
464 ; AVX-NEXT: vpsrld $16, %xmm9, %xmm4
465 ; AVX-NEXT: vpackusdw %xmm2, %xmm4, %xmm2
466 ; AVX-NEXT: vpsrld $16, %xmm10, %xmm4
467 ; AVX-NEXT: vpsrld $16, %xmm12, %xmm7
468 ; AVX-NEXT: vpackusdw %xmm4, %xmm7, %xmm4
469 ; AVX-NEXT: vmovdqa %xmm0, 32(%rsi)
470 ; AVX-NEXT: vmovdqa %xmm8, 48(%rsi)
471 ; AVX-NEXT: vmovdqa %xmm6, (%rsi)
472 ; AVX-NEXT: vmovdqa %xmm5, 16(%rsi)
473 ; AVX-NEXT: vmovdqa %xmm4, 32(%rdx)
474 ; AVX-NEXT: vmovdqa %xmm2, 48(%rdx)
475 ; AVX-NEXT: vmovdqa %xmm1, (%rdx)
476 ; AVX-NEXT: vmovdqa %xmm3, 16(%rdx)
477 ; AVX-NEXT: retq
478 ;
479 ; AVX2-LABEL: load_i16_stride2_vf32:
480 ; AVX2: # %bb.0:
481 ; AVX2-NEXT: vmovdqa (%rdi), %ymm0
482 ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1
483 ; AVX2-NEXT: vmovdqa 64(%rdi), %ymm2
484 ; AVX2-NEXT: vmovdqa 96(%rdi), %ymm3
485 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm4 = ymm3[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
486 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm4 = ymm4[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
487 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm5 = ymm2[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
488 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
489 ; AVX2-NEXT: vshufps {{.*#+}} ymm4 = ymm5[0,2],ymm4[0,2],ymm5[4,6],ymm4[4,6]
490 ; AVX2-NEXT: vpermpd {{.*#+}} ymm4 = ymm4[0,2,1,3]
491 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm5 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
492 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
493 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm6 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
494 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm6 = ymm6[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
495 ; AVX2-NEXT: vshufps {{.*#+}} ymm5 = ymm6[0,2],ymm5[0,2],ymm6[4,6],ymm5[4,6]
496 ; AVX2-NEXT: vpermpd {{.*#+}} ymm5 = ymm5[0,2,1,3]
497 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm6 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
498 ; AVX2-NEXT: vpshufb %ymm6, %ymm3, %ymm3
499 ; AVX2-NEXT: vpshufb %ymm6, %ymm2, %ymm2
500 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
501 ; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
502 ; AVX2-NEXT: vpshufb %ymm6, %ymm1, %ymm1
503 ; AVX2-NEXT: vpshufb %ymm6, %ymm0, %ymm0
504 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
505 ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
506 ; AVX2-NEXT: vmovaps %ymm5, (%rsi)
507 ; AVX2-NEXT: vmovaps %ymm4, 32(%rsi)
508 ; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
509 ; AVX2-NEXT: vmovdqa %ymm2, 32(%rdx)
510 ; AVX2-NEXT: vzeroupper
511 ; AVX2-NEXT: retq
512 ;
513 ; AVX2-FP-LABEL: load_i16_stride2_vf32:
514 ; AVX2-FP: # %bb.0:
515 ; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm0
516 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm1
517 ; AVX2-FP-NEXT: vmovdqa 64(%rdi), %ymm2
518 ; AVX2-FP-NEXT: vmovdqa 96(%rdi), %ymm3
519 ; AVX2-FP-NEXT: vpmovsxdq {{.*#+}} ymm4 = [84148480,218892552,353636624,488380696]
520 ; AVX2-FP-NEXT: vpshufb %ymm4, %ymm3, %ymm5
521 ; AVX2-FP-NEXT: vpshufb %ymm4, %ymm2, %ymm6
522 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm5 = ymm6[0,2],ymm5[0,2],ymm6[4,6],ymm5[4,6]
523 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm5 = ymm5[0,2,1,3]
524 ; AVX2-FP-NEXT: vpshufb %ymm4, %ymm1, %ymm6
525 ; AVX2-FP-NEXT: vpshufb %ymm4, %ymm0, %ymm4
526 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm4 = ymm4[0,2],ymm6[0,2],ymm4[4,6],ymm6[4,6]
527 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm4 = ymm4[0,2,1,3]
528 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm6 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
529 ; AVX2-FP-NEXT: vpshufb %ymm6, %ymm3, %ymm3
530 ; AVX2-FP-NEXT: vpshufb %ymm6, %ymm2, %ymm2
531 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
532 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
533 ; AVX2-FP-NEXT: vpshufb %ymm6, %ymm1, %ymm1
534 ; AVX2-FP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
535 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
536 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
537 ; AVX2-FP-NEXT: vmovaps %ymm4, (%rsi)
538 ; AVX2-FP-NEXT: vmovaps %ymm5, 32(%rsi)
539 ; AVX2-FP-NEXT: vmovdqa %ymm0, (%rdx)
540 ; AVX2-FP-NEXT: vmovdqa %ymm2, 32(%rdx)
541 ; AVX2-FP-NEXT: vzeroupper
542 ; AVX2-FP-NEXT: retq
543 ;
544 ; AVX2-FCP-LABEL: load_i16_stride2_vf32:
545 ; AVX2-FCP: # %bb.0:
546 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm0
547 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
548 ; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %ymm2
549 ; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %ymm3
550 ; AVX2-FCP-NEXT: vpmovsxdq {{.*#+}} ymm4 = [84148480,218892552,353636624,488380696]
551 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm5
552 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm6
553 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm5 = ymm6[0,2],ymm5[0,2],ymm6[4,6],ymm5[4,6]
554 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm5 = ymm5[0,2,1,3]
555 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm6
556 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm0, %ymm4
557 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm4 = ymm4[0,2],ymm6[0,2],ymm4[4,6],ymm6[4,6]
558 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm4 = ymm4[0,2,1,3]
559 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
560 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm3
561 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm2
562 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
563 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
564 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm1
565 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
566 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
567 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
568 ; AVX2-FCP-NEXT: vmovaps %ymm4, (%rsi)
569 ; AVX2-FCP-NEXT: vmovaps %ymm5, 32(%rsi)
570 ; AVX2-FCP-NEXT: vmovdqa %ymm0, (%rdx)
571 ; AVX2-FCP-NEXT: vmovdqa %ymm2, 32(%rdx)
572 ; AVX2-FCP-NEXT: vzeroupper
573 ; AVX2-FCP-NEXT: retq
574 ;
575 ; AVX512-VL-LABEL: load_i16_stride2_vf32:
576 ; AVX512-VL: # %bb.0:
577 ; AVX512-VL-NEXT: vmovdqa64 (%rdi), %zmm0
578 ; AVX512-VL-NEXT: vmovdqa64 64(%rdi), %zmm1
579 ; AVX512-VL-NEXT: vpsrld $16, %zmm0, %zmm2
580 ; AVX512-VL-NEXT: vpsrld $16, %zmm1, %zmm3
581 ; AVX512-VL-NEXT: vpmovdw %zmm1, 32(%rsi)
582 ; AVX512-VL-NEXT: vpmovdw %zmm0, (%rsi)
583 ; AVX512-VL-NEXT: vpmovdw %zmm3, 32(%rdx)
584 ; AVX512-VL-NEXT: vpmovdw %zmm2, (%rdx)
585 ; AVX512-VL-NEXT: vzeroupper
586 ; AVX512-VL-NEXT: retq
587 ;
588 ; AVX512-FCP-LABEL: load_i16_stride2_vf32:
589 ; AVX512-FCP: # %bb.0:
590 ; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
591 ; AVX512-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
592 ; AVX512-FCP-NEXT: vpsrld $16, %zmm0, %zmm2
593 ; AVX512-FCP-NEXT: vpsrld $16, %zmm1, %zmm3
594 ; AVX512-FCP-NEXT: vpmovdw %zmm1, 32(%rsi)
595 ; AVX512-FCP-NEXT: vpmovdw %zmm0, (%rsi)
596 ; AVX512-FCP-NEXT: vpmovdw %zmm3, 32(%rdx)
597 ; AVX512-FCP-NEXT: vpmovdw %zmm2, (%rdx)
598 ; AVX512-FCP-NEXT: vzeroupper
599 ; AVX512-FCP-NEXT: retq
600 ;
601 ; AVX512DQ-LABEL: load_i16_stride2_vf32:
602 ; AVX512DQ: # %bb.0:
603 ; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
604 ; AVX512DQ-NEXT: vmovdqa64 64(%rdi), %zmm1
605 ; AVX512DQ-NEXT: vpsrld $16, %zmm0, %zmm2
606 ; AVX512DQ-NEXT: vpsrld $16, %zmm1, %zmm3
607 ; AVX512DQ-NEXT: vpmovdw %zmm1, 32(%rsi)
608 ; AVX512DQ-NEXT: vpmovdw %zmm0, (%rsi)
609 ; AVX512DQ-NEXT: vpmovdw %zmm3, 32(%rdx)
610 ; AVX512DQ-NEXT: vpmovdw %zmm2, (%rdx)
611 ; AVX512DQ-NEXT: vzeroupper
612 ; AVX512DQ-NEXT: retq
613 ;
614 ; AVX512DQ-FCP-LABEL: load_i16_stride2_vf32:
615 ; AVX512DQ-FCP: # %bb.0:
616 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
617 ; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
618 ; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm0, %zmm2
619 ; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm1, %zmm3
620 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm1, 32(%rsi)
621 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm0, (%rsi)
622 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm3, 32(%rdx)
623 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm2, (%rdx)
624 ; AVX512DQ-FCP-NEXT: vzeroupper
625 ; AVX512DQ-FCP-NEXT: retq
626 ;
627 ; AVX512BW-LABEL: load_i16_stride2_vf32:
628 ; AVX512BW: # %bb.0:
629 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
630 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
631 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
632 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
633 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
634 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm3
635 ; AVX512BW-NEXT: vmovdqa64 %zmm2, (%rsi)
636 ; AVX512BW-NEXT: vmovdqa64 %zmm3, (%rdx)
637 ; AVX512BW-NEXT: vzeroupper
638 ; AVX512BW-NEXT: retq
639 ;
640 ; AVX512BW-FCP-LABEL: load_i16_stride2_vf32:
641 ; AVX512BW-FCP: # %bb.0:
642 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
643 ; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
644 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
645 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
646 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
647 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm3
648 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, (%rsi)
649 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, (%rdx)
650 ; AVX512BW-FCP-NEXT: vzeroupper
651 ; AVX512BW-FCP-NEXT: retq
652 ;
653 ; AVX512DQ-BW-LABEL: load_i16_stride2_vf32:
654 ; AVX512DQ-BW: # %bb.0:
655 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
656 ; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm1
657 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
658 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
659 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
660 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm3
661 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, (%rsi)
662 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, (%rdx)
663 ; AVX512DQ-BW-NEXT: vzeroupper
664 ; AVX512DQ-BW-NEXT: retq
665 ;
666 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride2_vf32:
667 ; AVX512DQ-BW-FCP: # %bb.0:
668 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
669 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
670 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
671 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
672 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
673 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm3
674 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, (%rsi)
675 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, (%rdx)
676 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
677 ; AVX512DQ-BW-FCP-NEXT: retq
678 %wide.vec = load <64 x i16>, ptr %in.vec, align 64
679 %strided.vec0 = shufflevector <64 x i16> %wide.vec, <64 x i16> poison, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>
680 %strided.vec1 = shufflevector <64 x i16> %wide.vec, <64 x i16> poison, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>
681 store <32 x i16> %strided.vec0, ptr %out.vec0, align 64
682 store <32 x i16> %strided.vec1, ptr %out.vec1, align 64
683 ret void
684 }
686 define void @load_i16_stride2_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nounwind {
687 ; SSE-LABEL: load_i16_stride2_vf64:
688 ; SSE: # %bb.0:
689 ; SSE-NEXT: subq $40, %rsp
690 ; SSE-NEXT: movdqa 160(%rdi), %xmm14
691 ; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
692 ; SSE-NEXT: movdqa 176(%rdi), %xmm2
693 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
694 ; SSE-NEXT: movdqa 64(%rdi), %xmm11
695 ; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
696 ; SSE-NEXT: movdqa 80(%rdi), %xmm1
697 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
698 ; SSE-NEXT: movdqa 96(%rdi), %xmm9
699 ; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
700 ; SSE-NEXT: movdqa 112(%rdi), %xmm4
701 ; SSE-NEXT: movdqa %xmm4, (%rsp) # 16-byte Spill
702 ; SSE-NEXT: movdqa (%rdi), %xmm10
703 ; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
704 ; SSE-NEXT: movdqa 16(%rdi), %xmm7
705 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
706 ; SSE-NEXT: movdqa 32(%rdi), %xmm13
707 ; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
708 ; SSE-NEXT: movdqa 48(%rdi), %xmm0
709 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
710 ; SSE-NEXT: pslld $16, %xmm0
711 ; SSE-NEXT: psrad $16, %xmm0
712 ; SSE-NEXT: pslld $16, %xmm13
713 ; SSE-NEXT: psrad $16, %xmm13
714 ; SSE-NEXT: packssdw %xmm0, %xmm13
715 ; SSE-NEXT: movdqa %xmm7, %xmm0
716 ; SSE-NEXT: pslld $16, %xmm0
717 ; SSE-NEXT: psrad $16, %xmm0
718 ; SSE-NEXT: pslld $16, %xmm10
719 ; SSE-NEXT: psrad $16, %xmm10
720 ; SSE-NEXT: packssdw %xmm0, %xmm10
721 ; SSE-NEXT: movdqa %xmm4, %xmm0
722 ; SSE-NEXT: pslld $16, %xmm0
723 ; SSE-NEXT: psrad $16, %xmm0
724 ; SSE-NEXT: pslld $16, %xmm9
725 ; SSE-NEXT: psrad $16, %xmm9
726 ; SSE-NEXT: packssdw %xmm0, %xmm9
727 ; SSE-NEXT: movdqa %xmm1, %xmm0
728 ; SSE-NEXT: pslld $16, %xmm0
729 ; SSE-NEXT: psrad $16, %xmm0
730 ; SSE-NEXT: pslld $16, %xmm11
731 ; SSE-NEXT: psrad $16, %xmm11
732 ; SSE-NEXT: packssdw %xmm0, %xmm11
733 ; SSE-NEXT: movdqa %xmm2, %xmm0
734 ; SSE-NEXT: pslld $16, %xmm0
735 ; SSE-NEXT: psrad $16, %xmm0
736 ; SSE-NEXT: pslld $16, %xmm14
737 ; SSE-NEXT: psrad $16, %xmm14
738 ; SSE-NEXT: packssdw %xmm0, %xmm14
739 ; SSE-NEXT: movdqa 144(%rdi), %xmm7
740 ; SSE-NEXT: movdqa %xmm7, %xmm0
741 ; SSE-NEXT: pslld $16, %xmm0
742 ; SSE-NEXT: psrad $16, %xmm0
743 ; SSE-NEXT: movdqa 128(%rdi), %xmm8
744 ; SSE-NEXT: movdqa %xmm8, %xmm15
745 ; SSE-NEXT: pslld $16, %xmm15
746 ; SSE-NEXT: psrad $16, %xmm15
747 ; SSE-NEXT: packssdw %xmm0, %xmm15
748 ; SSE-NEXT: movdqa 240(%rdi), %xmm12
749 ; SSE-NEXT: movdqa %xmm12, %xmm1
750 ; SSE-NEXT: pslld $16, %xmm1
751 ; SSE-NEXT: psrad $16, %xmm1
752 ; SSE-NEXT: movdqa 224(%rdi), %xmm5
753 ; SSE-NEXT: movdqa %xmm5, %xmm3
754 ; SSE-NEXT: pslld $16, %xmm3
755 ; SSE-NEXT: psrad $16, %xmm3
756 ; SSE-NEXT: packssdw %xmm1, %xmm3
757 ; SSE-NEXT: movdqa 208(%rdi), %xmm6
758 ; SSE-NEXT: movdqa %xmm6, %xmm4
759 ; SSE-NEXT: pslld $16, %xmm4
760 ; SSE-NEXT: psrad $16, %xmm4
761 ; SSE-NEXT: movdqa 192(%rdi), %xmm2
762 ; SSE-NEXT: movdqa %xmm2, %xmm1
763 ; SSE-NEXT: pslld $16, %xmm1
764 ; SSE-NEXT: psrad $16, %xmm1
765 ; SSE-NEXT: packssdw %xmm4, %xmm1
766 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
767 ; SSE-NEXT: psrad $16, %xmm0
768 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
769 ; SSE-NEXT: psrad $16, %xmm4
770 ; SSE-NEXT: packssdw %xmm0, %xmm4
771 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
772 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
773 ; SSE-NEXT: psrad $16, %xmm0
774 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
775 ; SSE-NEXT: psrad $16, %xmm4
776 ; SSE-NEXT: packssdw %xmm0, %xmm4
777 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
778 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
779 ; SSE-NEXT: psrad $16, %xmm0
780 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
781 ; SSE-NEXT: psrad $16, %xmm4
782 ; SSE-NEXT: packssdw %xmm0, %xmm4
783 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
784 ; SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
785 ; SSE-NEXT: psrad $16, %xmm0
786 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
787 ; SSE-NEXT: psrad $16, %xmm4
788 ; SSE-NEXT: packssdw %xmm0, %xmm4
789 ; SSE-NEXT: psrad $16, %xmm7
790 ; SSE-NEXT: psrad $16, %xmm8
791 ; SSE-NEXT: packssdw %xmm7, %xmm8
792 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
793 ; SSE-NEXT: psrad $16, %xmm0
794 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
795 ; SSE-NEXT: psrad $16, %xmm7
796 ; SSE-NEXT: packssdw %xmm0, %xmm7
797 ; SSE-NEXT: psrad $16, %xmm6
798 ; SSE-NEXT: psrad $16, %xmm2
799 ; SSE-NEXT: packssdw %xmm6, %xmm2
800 ; SSE-NEXT: psrad $16, %xmm12
801 ; SSE-NEXT: psrad $16, %xmm5
802 ; SSE-NEXT: packssdw %xmm12, %xmm5
803 ; SSE-NEXT: movdqa %xmm1, 96(%rsi)
804 ; SSE-NEXT: movdqa %xmm3, 112(%rsi)
805 ; SSE-NEXT: movdqa %xmm15, 64(%rsi)
806 ; SSE-NEXT: movdqa %xmm14, 80(%rsi)
807 ; SSE-NEXT: movdqa %xmm11, 32(%rsi)
808 ; SSE-NEXT: movdqa %xmm9, 48(%rsi)
809 ; SSE-NEXT: movdqa %xmm10, (%rsi)
810 ; SSE-NEXT: movdqa %xmm13, 16(%rsi)
811 ; SSE-NEXT: movdqa %xmm5, 112(%rdx)
812 ; SSE-NEXT: movdqa %xmm2, 96(%rdx)
813 ; SSE-NEXT: movdqa %xmm7, 80(%rdx)
814 ; SSE-NEXT: movdqa %xmm8, 64(%rdx)
815 ; SSE-NEXT: movdqa %xmm4, 48(%rdx)
816 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
817 ; SSE-NEXT: movaps %xmm0, 32(%rdx)
818 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
819 ; SSE-NEXT: movaps %xmm0, 16(%rdx)
820 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
821 ; SSE-NEXT: movaps %xmm0, (%rdx)
822 ; SSE-NEXT: addq $40, %rsp
823 ; SSE-NEXT: retq
824 ;
825 ; AVX-LABEL: load_i16_stride2_vf64:
826 ; AVX: # %bb.0:
827 ; AVX-NEXT: subq $24, %rsp
828 ; AVX-NEXT: vpxor %xmm11, %xmm11, %xmm11
829 ; AVX-NEXT: vmovdqa 176(%rdi), %xmm0
830 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
831 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm11[1],xmm0[2],xmm11[3],xmm0[4],xmm11[5],xmm0[6],xmm11[7]
832 ; AVX-NEXT: vmovdqa 160(%rdi), %xmm9
833 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm9[0],xmm11[1],xmm9[2],xmm11[3],xmm9[4],xmm11[5],xmm9[6],xmm11[7]
834 ; AVX-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
835 ; AVX-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
836 ; AVX-NEXT: vmovdqa (%rdi), %xmm6
837 ; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
838 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm8
839 ; AVX-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
840 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm7
841 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm10
842 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm10[0],xmm11[1],xmm10[2],xmm11[3],xmm10[4],xmm11[5],xmm10[6],xmm11[7]
843 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0],xmm11[1],xmm7[2],xmm11[3],xmm7[4],xmm11[5],xmm7[6],xmm11[7]
844 ; AVX-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
845 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
846 ; AVX-NEXT: vmovdqa 144(%rdi), %xmm5
847 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0],xmm11[1],xmm5[2],xmm11[3],xmm5[4],xmm11[5],xmm5[6],xmm11[7]
848 ; AVX-NEXT: vmovdqa 128(%rdi), %xmm4
849 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm11[1],xmm4[2],xmm11[3],xmm4[4],xmm11[5],xmm4[6],xmm11[7]
850 ; AVX-NEXT: vpackusdw %xmm2, %xmm3, %xmm0
851 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
852 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm8[0],xmm11[1],xmm8[2],xmm11[3],xmm8[4],xmm11[5],xmm8[6],xmm11[7]
853 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm6[0],xmm11[1],xmm6[2],xmm11[3],xmm6[4],xmm11[5],xmm6[6],xmm11[7]
854 ; AVX-NEXT: vpackusdw %xmm2, %xmm3, %xmm0
855 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
856 ; AVX-NEXT: vmovdqa 112(%rdi), %xmm12
857 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm12[0],xmm11[1],xmm12[2],xmm11[3],xmm12[4],xmm11[5],xmm12[6],xmm11[7]
858 ; AVX-NEXT: vmovdqa 96(%rdi), %xmm13
859 ; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm13[0],xmm11[1],xmm13[2],xmm11[3],xmm13[4],xmm11[5],xmm13[6],xmm11[7]
860 ; AVX-NEXT: vpackusdw %xmm2, %xmm8, %xmm0
861 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
862 ; AVX-NEXT: vmovdqa 80(%rdi), %xmm15
863 ; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm15[0],xmm11[1],xmm15[2],xmm11[3],xmm15[4],xmm11[5],xmm15[6],xmm11[7]
864 ; AVX-NEXT: vmovdqa 64(%rdi), %xmm6
865 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm6[0],xmm11[1],xmm6[2],xmm11[3],xmm6[4],xmm11[5],xmm6[6],xmm11[7]
866 ; AVX-NEXT: vpackusdw %xmm14, %xmm3, %xmm0
867 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
868 ; AVX-NEXT: vmovdqa 240(%rdi), %xmm3
869 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm11[1],xmm3[2],xmm11[3],xmm3[4],xmm11[5],xmm3[6],xmm11[7]
870 ; AVX-NEXT: vmovdqa 224(%rdi), %xmm0
871 ; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm0[0],xmm11[1],xmm0[2],xmm11[3],xmm0[4],xmm11[5],xmm0[6],xmm11[7]
872 ; AVX-NEXT: vpackusdw %xmm1, %xmm8, %xmm14
873 ; AVX-NEXT: vmovdqa 208(%rdi), %xmm8
874 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm8[0],xmm11[1],xmm8[2],xmm11[3],xmm8[4],xmm11[5],xmm8[6],xmm11[7]
875 ; AVX-NEXT: vmovdqa 192(%rdi), %xmm2
876 ; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm2[0],xmm11[1],xmm2[2],xmm11[3],xmm2[4],xmm11[5],xmm2[6],xmm11[7]
877 ; AVX-NEXT: vpackusdw %xmm1, %xmm11, %xmm1
878 ; AVX-NEXT: vpsrld $16, %xmm3, %xmm3
879 ; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
880 ; AVX-NEXT: vpackusdw %xmm3, %xmm0, %xmm11
881 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
882 ; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
883 ; AVX-NEXT: vpsrld $16, %xmm9, %xmm3
884 ; AVX-NEXT: vpackusdw %xmm0, %xmm3, %xmm0
885 ; AVX-NEXT: vpsrld $16, %xmm5, %xmm3
886 ; AVX-NEXT: vpsrld $16, %xmm4, %xmm4
887 ; AVX-NEXT: vpackusdw %xmm3, %xmm4, %xmm3
888 ; AVX-NEXT: vpsrld $16, %xmm10, %xmm4
889 ; AVX-NEXT: vpsrld $16, %xmm7, %xmm5
890 ; AVX-NEXT: vpackusdw %xmm4, %xmm5, %xmm4
891 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
892 ; AVX-NEXT: vpsrld $16, %xmm5, %xmm5
893 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
894 ; AVX-NEXT: vpsrld $16, %xmm7, %xmm7
895 ; AVX-NEXT: vpackusdw %xmm5, %xmm7, %xmm5
896 ; AVX-NEXT: vpsrld $16, %xmm8, %xmm7
897 ; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
898 ; AVX-NEXT: vpackusdw %xmm7, %xmm2, %xmm2
899 ; AVX-NEXT: vpsrld $16, %xmm12, %xmm7
900 ; AVX-NEXT: vpsrld $16, %xmm13, %xmm8
901 ; AVX-NEXT: vpackusdw %xmm7, %xmm8, %xmm7
902 ; AVX-NEXT: vpsrld $16, %xmm15, %xmm8
903 ; AVX-NEXT: vpsrld $16, %xmm6, %xmm6
904 ; AVX-NEXT: vpackusdw %xmm8, %xmm6, %xmm6
905 ; AVX-NEXT: vmovdqa %xmm1, 96(%rsi)
906 ; AVX-NEXT: vmovdqa %xmm14, 112(%rsi)
907 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
908 ; AVX-NEXT: vmovaps %xmm1, 32(%rsi)
909 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
910 ; AVX-NEXT: vmovaps %xmm1, 48(%rsi)
911 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
912 ; AVX-NEXT: vmovaps %xmm1, (%rsi)
913 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
914 ; AVX-NEXT: vmovaps %xmm1, 64(%rsi)
915 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
916 ; AVX-NEXT: vmovaps %xmm1, 16(%rsi)
917 ; AVX-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload
918 ; AVX-NEXT: vmovaps %xmm1, 80(%rsi)
919 ; AVX-NEXT: vmovdqa %xmm6, 32(%rdx)
920 ; AVX-NEXT: vmovdqa %xmm7, 48(%rdx)
921 ; AVX-NEXT: vmovdqa %xmm2, 96(%rdx)
922 ; AVX-NEXT: vmovdqa %xmm5, (%rdx)
923 ; AVX-NEXT: vmovdqa %xmm4, 16(%rdx)
924 ; AVX-NEXT: vmovdqa %xmm3, 64(%rdx)
925 ; AVX-NEXT: vmovdqa %xmm0, 80(%rdx)
926 ; AVX-NEXT: vmovdqa %xmm11, 112(%rdx)
927 ; AVX-NEXT: addq $24, %rsp
928 ; AVX-NEXT: retq
929 ;
930 ; AVX2-LABEL: load_i16_stride2_vf64:
931 ; AVX2: # %bb.0:
932 ; AVX2-NEXT: vmovdqa 128(%rdi), %ymm0
933 ; AVX2-NEXT: vmovdqa 160(%rdi), %ymm1
934 ; AVX2-NEXT: vmovdqa 192(%rdi), %ymm4
935 ; AVX2-NEXT: vmovdqa 224(%rdi), %ymm6
936 ; AVX2-NEXT: vmovdqa (%rdi), %ymm3
937 ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm5
938 ; AVX2-NEXT: vmovdqa 64(%rdi), %ymm7
939 ; AVX2-NEXT: vmovdqa 96(%rdi), %ymm8
940 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm8[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
941 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
942 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm9 = ymm7[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
943 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm9 = ymm9[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
944 ; AVX2-NEXT: vshufps {{.*#+}} ymm2 = ymm9[0,2],ymm2[0,2],ymm9[4,6],ymm2[4,6]
945 ; AVX2-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
946 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm9 = ymm6[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
947 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm9 = ymm9[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
948 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm10 = ymm4[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
949 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm10 = ymm10[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
950 ; AVX2-NEXT: vshufps {{.*#+}} ymm9 = ymm10[0,2],ymm9[0,2],ymm10[4,6],ymm9[4,6]
951 ; AVX2-NEXT: vpermpd {{.*#+}} ymm9 = ymm9[0,2,1,3]
952 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm10 = ymm5[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
953 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm10 = ymm10[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
954 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm11 = ymm3[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
955 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm11 = ymm11[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
956 ; AVX2-NEXT: vshufps {{.*#+}} ymm10 = ymm11[0,2],ymm10[0,2],ymm11[4,6],ymm10[4,6]
957 ; AVX2-NEXT: vpermpd {{.*#+}} ymm10 = ymm10[0,2,1,3]
958 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm11 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
959 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm11 = ymm11[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
960 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm12 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
961 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm12 = ymm12[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
962 ; AVX2-NEXT: vshufps {{.*#+}} ymm11 = ymm12[0,2],ymm11[0,2],ymm12[4,6],ymm11[4,6]
963 ; AVX2-NEXT: vpermpd {{.*#+}} ymm11 = ymm11[0,2,1,3]
964 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm12 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
965 ; AVX2-NEXT: vpshufb %ymm12, %ymm8, %ymm8
966 ; AVX2-NEXT: vpshufb %ymm12, %ymm7, %ymm7
967 ; AVX2-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1],ymm8[2,3],ymm7[4,5],ymm8[6,7]
968 ; AVX2-NEXT: vpermq {{.*#+}} ymm7 = ymm7[0,2,1,3]
969 ; AVX2-NEXT: vpshufb %ymm12, %ymm6, %ymm6
970 ; AVX2-NEXT: vpshufb %ymm12, %ymm4, %ymm4
971 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm6[2,3],ymm4[4,5],ymm6[6,7]
972 ; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
973 ; AVX2-NEXT: vpshufb %ymm12, %ymm5, %ymm5
974 ; AVX2-NEXT: vpshufb %ymm12, %ymm3, %ymm3
975 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm5[2,3],ymm3[4,5],ymm5[6,7]
976 ; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
977 ; AVX2-NEXT: vpshufb %ymm12, %ymm1, %ymm1
978 ; AVX2-NEXT: vpshufb %ymm12, %ymm0, %ymm0
979 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
980 ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
981 ; AVX2-NEXT: vmovaps %ymm11, 64(%rsi)
982 ; AVX2-NEXT: vmovaps %ymm10, (%rsi)
983 ; AVX2-NEXT: vmovaps %ymm9, 96(%rsi)
984 ; AVX2-NEXT: vmovaps %ymm2, 32(%rsi)
985 ; AVX2-NEXT: vmovdqa %ymm0, 64(%rdx)
986 ; AVX2-NEXT: vmovdqa %ymm3, (%rdx)
987 ; AVX2-NEXT: vmovdqa %ymm4, 96(%rdx)
988 ; AVX2-NEXT: vmovdqa %ymm7, 32(%rdx)
989 ; AVX2-NEXT: vzeroupper
990 ; AVX2-NEXT: retq
991 ;
992 ; AVX2-FP-LABEL: load_i16_stride2_vf64:
993 ; AVX2-FP: # %bb.0:
994 ; AVX2-FP-NEXT: vmovdqa 128(%rdi), %ymm0
995 ; AVX2-FP-NEXT: vmovdqa 160(%rdi), %ymm1
996 ; AVX2-FP-NEXT: vmovdqa 192(%rdi), %ymm3
997 ; AVX2-FP-NEXT: vmovdqa 224(%rdi), %ymm4
998 ; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm5
999 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm6
1000 ; AVX2-FP-NEXT: vmovdqa 64(%rdi), %ymm7
1001 ; AVX2-FP-NEXT: vmovdqa 96(%rdi), %ymm8
1002 ; AVX2-FP-NEXT: vpmovsxdq {{.*#+}} ymm9 = [84148480,218892552,353636624,488380696]
1003 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm8, %ymm2
1004 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm7, %ymm10
1005 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm2 = ymm10[0,2],ymm2[0,2],ymm10[4,6],ymm2[4,6]
1006 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
1007 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm4, %ymm10
1008 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm3, %ymm11
1009 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm10 = ymm11[0,2],ymm10[0,2],ymm11[4,6],ymm10[4,6]
1010 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm10 = ymm10[0,2,1,3]
1011 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm6, %ymm11
1012 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm5, %ymm12
1013 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm11 = ymm12[0,2],ymm11[0,2],ymm12[4,6],ymm11[4,6]
1014 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm11 = ymm11[0,2,1,3]
1015 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm1, %ymm12
1016 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm0, %ymm9
1017 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm9 = ymm9[0,2],ymm12[0,2],ymm9[4,6],ymm12[4,6]
1018 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm9 = ymm9[0,2,1,3]
1019 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm12 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
1020 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm8, %ymm8
1021 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm7, %ymm7
1022 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1],ymm8[2,3],ymm7[4,5],ymm8[6,7]
1023 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm7 = ymm7[0,2,1,3]
1024 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm4, %ymm4
1025 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm3, %ymm3
1026 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
1027 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
1028 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm6, %ymm4
1029 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm5, %ymm5
1030 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]
1031 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
1032 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm1, %ymm1
1033 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm0, %ymm0
1034 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
1035 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
1036 ; AVX2-FP-NEXT: vmovaps %ymm9, 64(%rsi)
1037 ; AVX2-FP-NEXT: vmovaps %ymm11, (%rsi)
1038 ; AVX2-FP-NEXT: vmovaps %ymm10, 96(%rsi)
1039 ; AVX2-FP-NEXT: vmovaps %ymm2, 32(%rsi)
1040 ; AVX2-FP-NEXT: vmovdqa %ymm0, 64(%rdx)
1041 ; AVX2-FP-NEXT: vmovdqa %ymm4, (%rdx)
1042 ; AVX2-FP-NEXT: vmovdqa %ymm3, 96(%rdx)
1043 ; AVX2-FP-NEXT: vmovdqa %ymm7, 32(%rdx)
1044 ; AVX2-FP-NEXT: vzeroupper
1045 ; AVX2-FP-NEXT: retq
1046 ;
1047 ; AVX2-FCP-LABEL: load_i16_stride2_vf64:
1048 ; AVX2-FCP: # %bb.0:
1049 ; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %ymm0
1050 ; AVX2-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
1051 ; AVX2-FCP-NEXT: vmovdqa 192(%rdi), %ymm3
1052 ; AVX2-FCP-NEXT: vmovdqa 224(%rdi), %ymm4
1053 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm5
1054 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm6
1055 ; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %ymm7
1056 ; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %ymm8
1057 ; AVX2-FCP-NEXT: vpmovsxdq {{.*#+}} ymm9 = [84148480,218892552,353636624,488380696]
1058 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm8, %ymm2
1059 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm7, %ymm10
1060 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm2 = ymm10[0,2],ymm2[0,2],ymm10[4,6],ymm2[4,6]
1061 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
1062 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm4, %ymm10
1063 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm11
1064 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm10 = ymm11[0,2],ymm10[0,2],ymm11[4,6],ymm10[4,6]
1065 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm10 = ymm10[0,2,1,3]
1066 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm6, %ymm11
1067 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm5, %ymm12
1068 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm11 = ymm12[0,2],ymm11[0,2],ymm12[4,6],ymm11[4,6]
1069 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm11 = ymm11[0,2,1,3]
1070 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm1, %ymm12
1071 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm0, %ymm9
1072 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm9 = ymm9[0,2],ymm12[0,2],ymm9[4,6],ymm12[4,6]
1073 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm9 = ymm9[0,2,1,3]
1074 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm12 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
1075 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm8, %ymm8
1076 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm7, %ymm7
1077 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1],ymm8[2,3],ymm7[4,5],ymm8[6,7]
1078 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm7 = ymm7[0,2,1,3]
1079 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm4, %ymm4
1080 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm3, %ymm3
1081 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
1082 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
1083 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm6, %ymm4
1084 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm5, %ymm5
1085 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]
1086 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
1087 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm1, %ymm1
1088 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm0
1089 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
1090 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
1091 ; AVX2-FCP-NEXT: vmovaps %ymm9, 64(%rsi)
1092 ; AVX2-FCP-NEXT: vmovaps %ymm11, (%rsi)
1093 ; AVX2-FCP-NEXT: vmovaps %ymm10, 96(%rsi)
1094 ; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rsi)
1095 ; AVX2-FCP-NEXT: vmovdqa %ymm0, 64(%rdx)
1096 ; AVX2-FCP-NEXT: vmovdqa %ymm4, (%rdx)
1097 ; AVX2-FCP-NEXT: vmovdqa %ymm3, 96(%rdx)
1098 ; AVX2-FCP-NEXT: vmovdqa %ymm7, 32(%rdx)
1099 ; AVX2-FCP-NEXT: vzeroupper
1100 ; AVX2-FCP-NEXT: retq
1101 ;
1102 ; AVX512-VL-LABEL: load_i16_stride2_vf64:
1103 ; AVX512-VL: # %bb.0:
1104 ; AVX512-VL-NEXT: vmovdqa64 (%rdi), %zmm0
1105 ; AVX512-VL-NEXT: vmovdqa64 64(%rdi), %zmm1
1106 ; AVX512-VL-NEXT: vmovdqa64 128(%rdi), %zmm2
1107 ; AVX512-VL-NEXT: vmovdqa64 192(%rdi), %zmm3
1108 ; AVX512-VL-NEXT: vpmovdw %zmm1, %ymm4
1109 ; AVX512-VL-NEXT: vpsrld $16, %zmm1, %zmm1
1110 ; AVX512-VL-NEXT: vpsrld $16, %zmm0, %zmm5
1111 ; AVX512-VL-NEXT: vpsrld $16, %zmm3, %zmm6
1112 ; AVX512-VL-NEXT: vpsrld $16, %zmm2, %zmm7
1113 ; AVX512-VL-NEXT: vpmovdw %zmm0, (%rsi)
1114 ; AVX512-VL-NEXT: vmovdqa %ymm4, 32(%rsi)
1115 ; AVX512-VL-NEXT: vpmovdw %zmm2, 64(%rsi)
1116 ; AVX512-VL-NEXT: vpmovdw %zmm3, 96(%rsi)
1117 ; AVX512-VL-NEXT: vpmovdw %zmm7, 64(%rdx)
1118 ; AVX512-VL-NEXT: vpmovdw %zmm6, 96(%rdx)
1119 ; AVX512-VL-NEXT: vpmovdw %zmm5, (%rdx)
1120 ; AVX512-VL-NEXT: vpmovdw %zmm1, 32(%rdx)
1121 ; AVX512-VL-NEXT: vzeroupper
1122 ; AVX512-VL-NEXT: retq
1123 ;
1124 ; AVX512-FCP-LABEL: load_i16_stride2_vf64:
1125 ; AVX512-FCP: # %bb.0:
1126 ; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1127 ; AVX512-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1128 ; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
1129 ; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %zmm3
1130 ; AVX512-FCP-NEXT: vpmovdw %zmm1, %ymm4
1131 ; AVX512-FCP-NEXT: vpsrld $16, %zmm1, %zmm1
1132 ; AVX512-FCP-NEXT: vpsrld $16, %zmm0, %zmm5
1133 ; AVX512-FCP-NEXT: vpsrld $16, %zmm3, %zmm6
1134 ; AVX512-FCP-NEXT: vpsrld $16, %zmm2, %zmm7
1135 ; AVX512-FCP-NEXT: vpmovdw %zmm0, (%rsi)
1136 ; AVX512-FCP-NEXT: vmovdqa %ymm4, 32(%rsi)
1137 ; AVX512-FCP-NEXT: vpmovdw %zmm2, 64(%rsi)
1138 ; AVX512-FCP-NEXT: vpmovdw %zmm3, 96(%rsi)
1139 ; AVX512-FCP-NEXT: vpmovdw %zmm7, 64(%rdx)
1140 ; AVX512-FCP-NEXT: vpmovdw %zmm6, 96(%rdx)
1141 ; AVX512-FCP-NEXT: vpmovdw %zmm5, (%rdx)
1142 ; AVX512-FCP-NEXT: vpmovdw %zmm1, 32(%rdx)
1143 ; AVX512-FCP-NEXT: vzeroupper
1144 ; AVX512-FCP-NEXT: retq
1145 ;
1146 ; AVX512DQ-LABEL: load_i16_stride2_vf64:
1147 ; AVX512DQ: # %bb.0:
1148 ; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
1149 ; AVX512DQ-NEXT: vmovdqa64 64(%rdi), %zmm1
1150 ; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm2
1151 ; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %zmm3
1152 ; AVX512DQ-NEXT: vpmovdw %zmm1, %ymm4
1153 ; AVX512DQ-NEXT: vpsrld $16, %zmm1, %zmm1
1154 ; AVX512DQ-NEXT: vpsrld $16, %zmm0, %zmm5
1155 ; AVX512DQ-NEXT: vpsrld $16, %zmm3, %zmm6
1156 ; AVX512DQ-NEXT: vpsrld $16, %zmm2, %zmm7
1157 ; AVX512DQ-NEXT: vpmovdw %zmm0, (%rsi)
1158 ; AVX512DQ-NEXT: vmovdqa %ymm4, 32(%rsi)
1159 ; AVX512DQ-NEXT: vpmovdw %zmm2, 64(%rsi)
1160 ; AVX512DQ-NEXT: vpmovdw %zmm3, 96(%rsi)
1161 ; AVX512DQ-NEXT: vpmovdw %zmm7, 64(%rdx)
1162 ; AVX512DQ-NEXT: vpmovdw %zmm6, 96(%rdx)
1163 ; AVX512DQ-NEXT: vpmovdw %zmm5, (%rdx)
1164 ; AVX512DQ-NEXT: vpmovdw %zmm1, 32(%rdx)
1165 ; AVX512DQ-NEXT: vzeroupper
1166 ; AVX512DQ-NEXT: retq
1167 ;
1168 ; AVX512DQ-FCP-LABEL: load_i16_stride2_vf64:
1169 ; AVX512DQ-FCP: # %bb.0:
1170 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1171 ; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1172 ; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
1173 ; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %zmm3
1174 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm1, %ymm4
1175 ; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm1, %zmm1
1176 ; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm0, %zmm5
1177 ; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm3, %zmm6
1178 ; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm2, %zmm7
1179 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm0, (%rsi)
1180 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm4, 32(%rsi)
1181 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm2, 64(%rsi)
1182 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm3, 96(%rsi)
1183 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm7, 64(%rdx)
1184 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm6, 96(%rdx)
1185 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm5, (%rdx)
1186 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm1, 32(%rdx)
1187 ; AVX512DQ-FCP-NEXT: vzeroupper
1188 ; AVX512DQ-FCP-NEXT: retq
1189 ;
1190 ; AVX512BW-LABEL: load_i16_stride2_vf64:
1191 ; AVX512BW: # %bb.0:
1192 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
1193 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
1194 ; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm2
1195 ; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm3
1196 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
1197 ; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm5
1198 ; AVX512BW-NEXT: vpermt2w %zmm1, %zmm4, %zmm5
1199 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm4
1200 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
1201 ; AVX512BW-NEXT: vpermt2w %zmm1, %zmm6, %zmm0
1202 ; AVX512BW-NEXT: vpermt2w %zmm3, %zmm6, %zmm2
1203 ; AVX512BW-NEXT: vmovdqa64 %zmm4, 64(%rsi)
1204 ; AVX512BW-NEXT: vmovdqa64 %zmm5, (%rsi)
1205 ; AVX512BW-NEXT: vmovdqa64 %zmm2, 64(%rdx)
1206 ; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
1207 ; AVX512BW-NEXT: vzeroupper
1208 ; AVX512BW-NEXT: retq
1209 ;
1210 ; AVX512BW-FCP-LABEL: load_i16_stride2_vf64:
1211 ; AVX512BW-FCP: # %bb.0:
1212 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1213 ; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1214 ; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
1215 ; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm3
1216 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
1217 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm5
1218 ; AVX512BW-FCP-NEXT: vpermt2w %zmm1, %zmm4, %zmm5
1219 ; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm4
1220 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm6 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
1221 ; AVX512BW-FCP-NEXT: vpermt2w %zmm1, %zmm6, %zmm0
1222 ; AVX512BW-FCP-NEXT: vpermt2w %zmm3, %zmm6, %zmm2
1223 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, 64(%rsi)
1224 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, (%rsi)
1225 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, 64(%rdx)
1226 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
1227 ; AVX512BW-FCP-NEXT: vzeroupper
1228 ; AVX512BW-FCP-NEXT: retq
1229 ;
1230 ; AVX512DQ-BW-LABEL: load_i16_stride2_vf64:
1231 ; AVX512DQ-BW: # %bb.0:
1232 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
1233 ; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm1
1234 ; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm2
1235 ; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %zmm3
1236 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
1237 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm5
1238 ; AVX512DQ-BW-NEXT: vpermt2w %zmm1, %zmm4, %zmm5
1239 ; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm4
1240 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
1241 ; AVX512DQ-BW-NEXT: vpermt2w %zmm1, %zmm6, %zmm0
1242 ; AVX512DQ-BW-NEXT: vpermt2w %zmm3, %zmm6, %zmm2
1243 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, 64(%rsi)
1244 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, (%rsi)
1245 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, 64(%rdx)
1246 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, (%rdx)
1247 ; AVX512DQ-BW-NEXT: vzeroupper
1248 ; AVX512DQ-BW-NEXT: retq
1249 ;
1250 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride2_vf64:
1251 ; AVX512DQ-BW-FCP: # %bb.0:
1252 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1253 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1254 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
1255 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm3
1256 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
1257 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm5
1258 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm1, %zmm4, %zmm5
1259 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm4
1260 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm6 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
1261 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm1, %zmm6, %zmm0
1262 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm3, %zmm6, %zmm2
1263 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, 64(%rsi)
1264 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, (%rsi)
1265 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, 64(%rdx)
1266 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
1267 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
1268 ; AVX512DQ-BW-FCP-NEXT: retq
1269 %wide.vec = load <128 x i16>, ptr %in.vec, align 64
1270 %strided.vec0 = shufflevector <128 x i16> %wide.vec, <128 x i16> poison, <64 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62, i32 64, i32 66, i32 68, i32 70, i32 72, i32 74, i32 76, i32 78, i32 80, i32 82, i32 84, i32 86, i32 88, i32 90, i32 92, i32 94, i32 96, i32 98, i32 100, i32 102, i32 104, i32 106, i32 108, i32 110, i32 112, i32 114, i32 116, i32 118, i32 120, i32 122, i32 124, i32 126>
1271 %strided.vec1 = shufflevector <128 x i16> %wide.vec, <128 x i16> poison, <64 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63, i32 65, i32 67, i32 69, i32 71, i32 73, i32 75, i32 77, i32 79, i32 81, i32 83, i32 85, i32 87, i32 89, i32 91, i32 93, i32 95, i32 97, i32 99, i32 101, i32 103, i32 105, i32 107, i32 109, i32 111, i32 113, i32 115, i32 117, i32 119, i32 121, i32 123, i32 125, i32 127>
1272 store <64 x i16> %strided.vec0, ptr %out.vec0, align 64
1273 store <64 x i16> %strided.vec1, ptr %out.vec1, align 64
1274 ret void
1275 }