| blob | 00e43df15deea0d02c12d2a1163dbf12a9a8f8f3 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE
3 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX
4 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
5 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FP
6 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FCP
7 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512
8 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512-FCP
9 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ
10 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-FCP
11 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW
12 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW-FCP
13 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512DQ-BW
14 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-BW-FCP
16 ; These patterns are produced by LoopVectorizer for interleaved loads.
18 define void @load_i16_stride2_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nounwind {
19 ; SSE-LABEL: load_i16_stride2_vf2:
20 ; SSE: # %bb.0:
21 ; SSE-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
22 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
23 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
24 ; SSE-NEXT: movd %xmm1, (%rsi)
25 ; SSE-NEXT: movd %xmm0, (%rdx)
26 ; SSE-NEXT: retq
27 ;
28 ; AVX-LABEL: load_i16_stride2_vf2:
29 ; AVX: # %bb.0:
30 ; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
31 ; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
32 ; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
33 ; AVX-NEXT: vmovd %xmm1, (%rsi)
34 ; AVX-NEXT: vmovd %xmm0, (%rdx)
35 ; AVX-NEXT: retq
36 ;
37 ; AVX2-LABEL: load_i16_stride2_vf2:
38 ; AVX2: # %bb.0:
39 ; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
40 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
41 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
42 ; AVX2-NEXT: vmovd %xmm1, (%rsi)
43 ; AVX2-NEXT: vmovd %xmm0, (%rdx)
44 ; AVX2-NEXT: retq
45 ;
46 ; AVX2-FP-LABEL: load_i16_stride2_vf2:
47 ; AVX2-FP: # %bb.0:
48 ; AVX2-FP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
49 ; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
50 ; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
51 ; AVX2-FP-NEXT: vmovd %xmm1, (%rsi)
52 ; AVX2-FP-NEXT: vmovd %xmm0, (%rdx)
53 ; AVX2-FP-NEXT: retq
54 ;
55 ; AVX2-FCP-LABEL: load_i16_stride2_vf2:
56 ; AVX2-FCP: # %bb.0:
57 ; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
58 ; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
59 ; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
60 ; AVX2-FCP-NEXT: vmovd %xmm1, (%rsi)
61 ; AVX2-FCP-NEXT: vmovd %xmm0, (%rdx)
62 ; AVX2-FCP-NEXT: retq
63 ;
64 ; AVX512-LABEL: load_i16_stride2_vf2:
65 ; AVX512: # %bb.0:
66 ; AVX512-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
67 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
68 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
69 ; AVX512-NEXT: vmovd %xmm1, (%rsi)
70 ; AVX512-NEXT: vmovd %xmm0, (%rdx)
71 ; AVX512-NEXT: retq
72 ;
73 ; AVX512-FCP-LABEL: load_i16_stride2_vf2:
74 ; AVX512-FCP: # %bb.0:
75 ; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
76 ; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
77 ; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
78 ; AVX512-FCP-NEXT: vmovd %xmm1, (%rsi)
79 ; AVX512-FCP-NEXT: vmovd %xmm0, (%rdx)
80 ; AVX512-FCP-NEXT: retq
81 ;
82 ; AVX512DQ-LABEL: load_i16_stride2_vf2:
83 ; AVX512DQ: # %bb.0:
84 ; AVX512DQ-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
85 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
86 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
87 ; AVX512DQ-NEXT: vmovd %xmm1, (%rsi)
88 ; AVX512DQ-NEXT: vmovd %xmm0, (%rdx)
89 ; AVX512DQ-NEXT: retq
90 ;
91 ; AVX512DQ-FCP-LABEL: load_i16_stride2_vf2:
92 ; AVX512DQ-FCP: # %bb.0:
93 ; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
94 ; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
95 ; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
96 ; AVX512DQ-FCP-NEXT: vmovd %xmm1, (%rsi)
97 ; AVX512DQ-FCP-NEXT: vmovd %xmm0, (%rdx)
98 ; AVX512DQ-FCP-NEXT: retq
99 ;
100 ; AVX512BW-LABEL: load_i16_stride2_vf2:
101 ; AVX512BW: # %bb.0:
102 ; AVX512BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
103 ; AVX512BW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
104 ; AVX512BW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
105 ; AVX512BW-NEXT: vmovd %xmm1, (%rsi)
106 ; AVX512BW-NEXT: vmovd %xmm0, (%rdx)
107 ; AVX512BW-NEXT: retq
108 ;
109 ; AVX512BW-FCP-LABEL: load_i16_stride2_vf2:
110 ; AVX512BW-FCP: # %bb.0:
111 ; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
112 ; AVX512BW-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
113 ; AVX512BW-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
114 ; AVX512BW-FCP-NEXT: vmovd %xmm1, (%rsi)
115 ; AVX512BW-FCP-NEXT: vmovd %xmm0, (%rdx)
116 ; AVX512BW-FCP-NEXT: retq
117 ;
118 ; AVX512DQ-BW-LABEL: load_i16_stride2_vf2:
119 ; AVX512DQ-BW: # %bb.0:
120 ; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
121 ; AVX512DQ-BW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
122 ; AVX512DQ-BW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
123 ; AVX512DQ-BW-NEXT: vmovd %xmm1, (%rsi)
124 ; AVX512DQ-BW-NEXT: vmovd %xmm0, (%rdx)
125 ; AVX512DQ-BW-NEXT: retq
126 ;
127 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride2_vf2:
128 ; AVX512DQ-BW-FCP: # %bb.0:
129 ; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
130 ; AVX512DQ-BW-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
131 ; AVX512DQ-BW-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
132 ; AVX512DQ-BW-FCP-NEXT: vmovd %xmm1, (%rsi)
133 ; AVX512DQ-BW-FCP-NEXT: vmovd %xmm0, (%rdx)
134 ; AVX512DQ-BW-FCP-NEXT: retq
135 %wide.vec = load <4 x i16>, ptr %in.vec, align 64
136 %strided.vec0 = shufflevector <4 x i16> %wide.vec, <4 x i16> poison, <2 x i32> <i32 0, i32 2>
137 %strided.vec1 = shufflevector <4 x i16> %wide.vec, <4 x i16> poison, <2 x i32> <i32 1, i32 3>
138 store <2 x i16> %strided.vec0, ptr %out.vec0, align 64
139 store <2 x i16> %strided.vec1, ptr %out.vec1, align 64
140 ret void
141 }
143 define void @load_i16_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nounwind {
144 ; SSE-LABEL: load_i16_stride2_vf4:
145 ; SSE: # %bb.0:
146 ; SSE-NEXT: movdqa (%rdi), %xmm0
147 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
148 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
149 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
150 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]
151 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,7]
152 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
153 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7]
154 ; SSE-NEXT: movq %xmm1, (%rsi)
155 ; SSE-NEXT: movq %xmm0, (%rdx)
156 ; SSE-NEXT: retq
157 ;
158 ; AVX-LABEL: load_i16_stride2_vf4:
159 ; AVX: # %bb.0:
160 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
161 ; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u]
162 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
163 ; AVX-NEXT: vmovq %xmm1, (%rsi)
164 ; AVX-NEXT: vmovq %xmm0, (%rdx)
165 ; AVX-NEXT: retq
166 ;
167 ; AVX2-LABEL: load_i16_stride2_vf4:
168 ; AVX2: # %bb.0:
169 ; AVX2-NEXT: vmovdqa (%rdi), %xmm0
170 ; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u]
171 ; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
172 ; AVX2-NEXT: vmovq %xmm1, (%rsi)
173 ; AVX2-NEXT: vmovq %xmm0, (%rdx)
174 ; AVX2-NEXT: retq
175 ;
176 ; AVX2-FP-LABEL: load_i16_stride2_vf4:
177 ; AVX2-FP: # %bb.0:
178 ; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
179 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u]
180 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
181 ; AVX2-FP-NEXT: vmovq %xmm1, (%rsi)
182 ; AVX2-FP-NEXT: vmovq %xmm0, (%rdx)
183 ; AVX2-FP-NEXT: retq
184 ;
185 ; AVX2-FCP-LABEL: load_i16_stride2_vf4:
186 ; AVX2-FCP: # %bb.0:
187 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0
188 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u]
189 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
190 ; AVX2-FCP-NEXT: vmovq %xmm1, (%rsi)
191 ; AVX2-FCP-NEXT: vmovq %xmm0, (%rdx)
192 ; AVX2-FCP-NEXT: retq
193 ;
194 ; AVX512-LABEL: load_i16_stride2_vf4:
195 ; AVX512: # %bb.0:
196 ; AVX512-NEXT: vmovdqa (%rdi), %xmm0
197 ; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
198 ; AVX512-NEXT: vpmovdw %xmm0, (%rsi)
199 ; AVX512-NEXT: vmovq %xmm1, (%rdx)
200 ; AVX512-NEXT: retq
201 ;
202 ; AVX512-FCP-LABEL: load_i16_stride2_vf4:
203 ; AVX512-FCP: # %bb.0:
204 ; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0
205 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
206 ; AVX512-FCP-NEXT: vpmovdw %xmm0, (%rsi)
207 ; AVX512-FCP-NEXT: vmovq %xmm1, (%rdx)
208 ; AVX512-FCP-NEXT: retq
209 ;
210 ; AVX512DQ-LABEL: load_i16_stride2_vf4:
211 ; AVX512DQ: # %bb.0:
212 ; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
213 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
214 ; AVX512DQ-NEXT: vpmovdw %xmm0, (%rsi)
215 ; AVX512DQ-NEXT: vmovq %xmm1, (%rdx)
216 ; AVX512DQ-NEXT: retq
217 ;
218 ; AVX512DQ-FCP-LABEL: load_i16_stride2_vf4:
219 ; AVX512DQ-FCP: # %bb.0:
220 ; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0
221 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
222 ; AVX512DQ-FCP-NEXT: vpmovdw %xmm0, (%rsi)
223 ; AVX512DQ-FCP-NEXT: vmovq %xmm1, (%rdx)
224 ; AVX512DQ-FCP-NEXT: retq
225 ;
226 ; AVX512BW-LABEL: load_i16_stride2_vf4:
227 ; AVX512BW: # %bb.0:
228 ; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0
229 ; AVX512BW-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
230 ; AVX512BW-NEXT: vpmovdw %xmm0, (%rsi)
231 ; AVX512BW-NEXT: vmovq %xmm1, (%rdx)
232 ; AVX512BW-NEXT: retq
233 ;
234 ; AVX512BW-FCP-LABEL: load_i16_stride2_vf4:
235 ; AVX512BW-FCP: # %bb.0:
236 ; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm0
237 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
238 ; AVX512BW-FCP-NEXT: vpmovdw %xmm0, (%rsi)
239 ; AVX512BW-FCP-NEXT: vmovq %xmm1, (%rdx)
240 ; AVX512BW-FCP-NEXT: retq
241 ;
242 ; AVX512DQ-BW-LABEL: load_i16_stride2_vf4:
243 ; AVX512DQ-BW: # %bb.0:
244 ; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm0
245 ; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
246 ; AVX512DQ-BW-NEXT: vpmovdw %xmm0, (%rsi)
247 ; AVX512DQ-BW-NEXT: vmovq %xmm1, (%rdx)
248 ; AVX512DQ-BW-NEXT: retq
249 ;
250 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride2_vf4:
251 ; AVX512DQ-BW-FCP: # %bb.0:
252 ; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm0
253 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]
254 ; AVX512DQ-BW-FCP-NEXT: vpmovdw %xmm0, (%rsi)
255 ; AVX512DQ-BW-FCP-NEXT: vmovq %xmm1, (%rdx)
256 ; AVX512DQ-BW-FCP-NEXT: retq
257 %wide.vec = load <8 x i16>, ptr %in.vec, align 64
258 %strided.vec0 = shufflevector <8 x i16> %wide.vec, <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
259 %strided.vec1 = shufflevector <8 x i16> %wide.vec, <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
260 store <4 x i16> %strided.vec0, ptr %out.vec0, align 64
261 store <4 x i16> %strided.vec1, ptr %out.vec1, align 64
262 ret void
263 }
265 define void @load_i16_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nounwind {
266 ; SSE-LABEL: load_i16_stride2_vf8:
267 ; SSE: # %bb.0:
268 ; SSE-NEXT: movdqa (%rdi), %xmm0
269 ; SSE-NEXT: movdqa 16(%rdi), %xmm1
270 ; SSE-NEXT: movdqa %xmm1, %xmm2
271 ; SSE-NEXT: pslld $16, %xmm2
272 ; SSE-NEXT: psrad $16, %xmm2
273 ; SSE-NEXT: movdqa %xmm0, %xmm3
274 ; SSE-NEXT: pslld $16, %xmm3
275 ; SSE-NEXT: psrad $16, %xmm3
276 ; SSE-NEXT: packssdw %xmm2, %xmm3
277 ; SSE-NEXT: psrad $16, %xmm1
278 ; SSE-NEXT: psrad $16, %xmm0
279 ; SSE-NEXT: packssdw %xmm1, %xmm0
280 ; SSE-NEXT: movdqa %xmm3, (%rsi)
281 ; SSE-NEXT: movdqa %xmm0, (%rdx)
282 ; SSE-NEXT: retq
283 ;
284 ; AVX-LABEL: load_i16_stride2_vf8:
285 ; AVX: # %bb.0:
286 ; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
287 ; AVX-NEXT: vmovdqa (%rdi), %xmm1
288 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
289 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
290 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
291 ; AVX-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
292 ; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
293 ; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
294 ; AVX-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
295 ; AVX-NEXT: vmovdqa %xmm0, (%rsi)
296 ; AVX-NEXT: vmovdqa %xmm1, (%rdx)
297 ; AVX-NEXT: retq
298 ;
299 ; AVX2-LABEL: load_i16_stride2_vf8:
300 ; AVX2: # %bb.0:
301 ; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
302 ; AVX2-NEXT: vmovdqa (%rdi), %xmm1
303 ; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2
304 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
305 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
306 ; AVX2-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
307 ; AVX2-NEXT: vpsrld $16, %xmm2, %xmm2
308 ; AVX2-NEXT: vpsrld $16, %xmm1, %xmm1
309 ; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
310 ; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
311 ; AVX2-NEXT: vmovdqa %xmm1, (%rdx)
312 ; AVX2-NEXT: retq
313 ;
314 ; AVX2-FP-LABEL: load_i16_stride2_vf8:
315 ; AVX2-FP: # %bb.0:
316 ; AVX2-FP-NEXT: vpxor %xmm0, %xmm0, %xmm0
317 ; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1
318 ; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2
319 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
320 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
321 ; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
322 ; AVX2-FP-NEXT: vpsrld $16, %xmm2, %xmm2
323 ; AVX2-FP-NEXT: vpsrld $16, %xmm1, %xmm1
324 ; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
325 ; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsi)
326 ; AVX2-FP-NEXT: vmovdqa %xmm1, (%rdx)
327 ; AVX2-FP-NEXT: retq
328 ;
329 ; AVX2-FCP-LABEL: load_i16_stride2_vf8:
330 ; AVX2-FCP: # %bb.0:
331 ; AVX2-FCP-NEXT: vpxor %xmm0, %xmm0, %xmm0
332 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1
333 ; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
334 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
335 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
336 ; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
337 ; AVX2-FCP-NEXT: vpsrld $16, %xmm2, %xmm2
338 ; AVX2-FCP-NEXT: vpsrld $16, %xmm1, %xmm1
339 ; AVX2-FCP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
340 ; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rsi)
341 ; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rdx)
342 ; AVX2-FCP-NEXT: retq
343 ;
344 ; AVX512-LABEL: load_i16_stride2_vf8:
345 ; AVX512: # %bb.0:
346 ; AVX512-NEXT: vmovdqa (%rdi), %ymm0
347 ; AVX512-NEXT: vpsrld $16, %ymm0, %ymm1
348 ; AVX512-NEXT: vpmovdw %ymm0, (%rsi)
349 ; AVX512-NEXT: vpmovdw %ymm1, (%rdx)
350 ; AVX512-NEXT: vzeroupper
351 ; AVX512-NEXT: retq
352 ;
353 ; AVX512-FCP-LABEL: load_i16_stride2_vf8:
354 ; AVX512-FCP: # %bb.0:
355 ; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm0
356 ; AVX512-FCP-NEXT: vpsrld $16, %ymm0, %ymm1
357 ; AVX512-FCP-NEXT: vpmovdw %ymm0, (%rsi)
358 ; AVX512-FCP-NEXT: vpmovdw %ymm1, (%rdx)
359 ; AVX512-FCP-NEXT: vzeroupper
360 ; AVX512-FCP-NEXT: retq
361 ;
362 ; AVX512DQ-LABEL: load_i16_stride2_vf8:
363 ; AVX512DQ: # %bb.0:
364 ; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
365 ; AVX512DQ-NEXT: vpsrld $16, %ymm0, %ymm1
366 ; AVX512DQ-NEXT: vpmovdw %ymm0, (%rsi)
367 ; AVX512DQ-NEXT: vpmovdw %ymm1, (%rdx)
368 ; AVX512DQ-NEXT: vzeroupper
369 ; AVX512DQ-NEXT: retq
370 ;
371 ; AVX512DQ-FCP-LABEL: load_i16_stride2_vf8:
372 ; AVX512DQ-FCP: # %bb.0:
373 ; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm0
374 ; AVX512DQ-FCP-NEXT: vpsrld $16, %ymm0, %ymm1
375 ; AVX512DQ-FCP-NEXT: vpmovdw %ymm0, (%rsi)
376 ; AVX512DQ-FCP-NEXT: vpmovdw %ymm1, (%rdx)
377 ; AVX512DQ-FCP-NEXT: vzeroupper
378 ; AVX512DQ-FCP-NEXT: retq
379 ;
380 ; AVX512BW-LABEL: load_i16_stride2_vf8:
381 ; AVX512BW: # %bb.0:
382 ; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0
383 ; AVX512BW-NEXT: vpsrld $16, %ymm0, %ymm1
384 ; AVX512BW-NEXT: vpmovdw %ymm0, (%rsi)
385 ; AVX512BW-NEXT: vpmovdw %ymm1, (%rdx)
386 ; AVX512BW-NEXT: vzeroupper
387 ; AVX512BW-NEXT: retq
388 ;
389 ; AVX512BW-FCP-LABEL: load_i16_stride2_vf8:
390 ; AVX512BW-FCP: # %bb.0:
391 ; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm0
392 ; AVX512BW-FCP-NEXT: vpsrld $16, %ymm0, %ymm1
393 ; AVX512BW-FCP-NEXT: vpmovdw %ymm0, (%rsi)
394 ; AVX512BW-FCP-NEXT: vpmovdw %ymm1, (%rdx)
395 ; AVX512BW-FCP-NEXT: vzeroupper
396 ; AVX512BW-FCP-NEXT: retq
397 ;
398 ; AVX512DQ-BW-LABEL: load_i16_stride2_vf8:
399 ; AVX512DQ-BW: # %bb.0:
400 ; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %ymm0
401 ; AVX512DQ-BW-NEXT: vpsrld $16, %ymm0, %ymm1
402 ; AVX512DQ-BW-NEXT: vpmovdw %ymm0, (%rsi)
403 ; AVX512DQ-BW-NEXT: vpmovdw %ymm1, (%rdx)
404 ; AVX512DQ-BW-NEXT: vzeroupper
405 ; AVX512DQ-BW-NEXT: retq
406 ;
407 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride2_vf8:
408 ; AVX512DQ-BW-FCP: # %bb.0:
409 ; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm0
410 ; AVX512DQ-BW-FCP-NEXT: vpsrld $16, %ymm0, %ymm1
411 ; AVX512DQ-BW-FCP-NEXT: vpmovdw %ymm0, (%rsi)
412 ; AVX512DQ-BW-FCP-NEXT: vpmovdw %ymm1, (%rdx)
413 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
414 ; AVX512DQ-BW-FCP-NEXT: retq
415 %wide.vec = load <16 x i16>, ptr %in.vec, align 64
416 %strided.vec0 = shufflevector <16 x i16> %wide.vec, <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
417 %strided.vec1 = shufflevector <16 x i16> %wide.vec, <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
418 store <8 x i16> %strided.vec0, ptr %out.vec0, align 64
419 store <8 x i16> %strided.vec1, ptr %out.vec1, align 64
420 ret void
421 }
423 define void @load_i16_stride2_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nounwind {
424 ; SSE-LABEL: load_i16_stride2_vf16:
425 ; SSE: # %bb.0:
426 ; SSE-NEXT: movdqa (%rdi), %xmm0
427 ; SSE-NEXT: movdqa 16(%rdi), %xmm2
428 ; SSE-NEXT: movdqa 32(%rdi), %xmm1
429 ; SSE-NEXT: movdqa 48(%rdi), %xmm3
430 ; SSE-NEXT: movdqa %xmm3, %xmm4
431 ; SSE-NEXT: pslld $16, %xmm4
432 ; SSE-NEXT: psrad $16, %xmm4
433 ; SSE-NEXT: movdqa %xmm1, %xmm5
434 ; SSE-NEXT: pslld $16, %xmm5
435 ; SSE-NEXT: psrad $16, %xmm5
436 ; SSE-NEXT: packssdw %xmm4, %xmm5
437 ; SSE-NEXT: movdqa %xmm2, %xmm4
438 ; SSE-NEXT: pslld $16, %xmm4
439 ; SSE-NEXT: psrad $16, %xmm4
440 ; SSE-NEXT: movdqa %xmm0, %xmm6
441 ; SSE-NEXT: pslld $16, %xmm6
442 ; SSE-NEXT: psrad $16, %xmm6
443 ; SSE-NEXT: packssdw %xmm4, %xmm6
444 ; SSE-NEXT: psrad $16, %xmm3
445 ; SSE-NEXT: psrad $16, %xmm1
446 ; SSE-NEXT: packssdw %xmm3, %xmm1
447 ; SSE-NEXT: psrad $16, %xmm2
448 ; SSE-NEXT: psrad $16, %xmm0
449 ; SSE-NEXT: packssdw %xmm2, %xmm0
450 ; SSE-NEXT: movdqa %xmm6, (%rsi)
451 ; SSE-NEXT: movdqa %xmm5, 16(%rsi)
452 ; SSE-NEXT: movdqa %xmm0, (%rdx)
453 ; SSE-NEXT: movdqa %xmm1, 16(%rdx)
454 ; SSE-NEXT: retq
455 ;
456 ; AVX-LABEL: load_i16_stride2_vf16:
457 ; AVX: # %bb.0:
458 ; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
459 ; AVX-NEXT: vmovdqa (%rdi), %xmm1
460 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
461 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm3
462 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm4
463 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm0[1],xmm4[2],xmm0[3],xmm4[4],xmm0[5],xmm4[6],xmm0[7]
464 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
465 ; AVX-NEXT: vpackusdw %xmm5, %xmm6, %xmm5
466 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
467 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
468 ; AVX-NEXT: vpackusdw %xmm6, %xmm0, %xmm0
469 ; AVX-NEXT: vpsrld $16, %xmm4, %xmm4
470 ; AVX-NEXT: vpsrld $16, %xmm3, %xmm3
471 ; AVX-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
472 ; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
473 ; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
474 ; AVX-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
475 ; AVX-NEXT: vmovdqa %xmm0, (%rsi)
476 ; AVX-NEXT: vmovdqa %xmm5, 16(%rsi)
477 ; AVX-NEXT: vmovdqa %xmm1, (%rdx)
478 ; AVX-NEXT: vmovdqa %xmm3, 16(%rdx)
479 ; AVX-NEXT: retq
480 ;
481 ; AVX2-LABEL: load_i16_stride2_vf16:
482 ; AVX2: # %bb.0:
483 ; AVX2-NEXT: vmovdqa (%rdi), %ymm0
484 ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1
485 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
486 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
487 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm3 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
488 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
489 ; AVX2-NEXT: vshufps {{.*#+}} ymm2 = ymm3[0,2],ymm2[0,2],ymm3[4,6],ymm2[4,6]
490 ; AVX2-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
491 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
492 ; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1
493 ; AVX2-NEXT: vpshufb %ymm3, %ymm0, %ymm0
494 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
495 ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
496 ; AVX2-NEXT: vmovaps %ymm2, (%rsi)
497 ; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
498 ; AVX2-NEXT: vzeroupper
499 ; AVX2-NEXT: retq
500 ;
501 ; AVX2-FP-LABEL: load_i16_stride2_vf16:
502 ; AVX2-FP: # %bb.0:
503 ; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm0
504 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm1
505 ; AVX2-FP-NEXT: vpmovsxdq {{.*#+}} ymm2 = [84148480,218892552,353636624,488380696]
506 ; AVX2-FP-NEXT: vpshufb %ymm2, %ymm1, %ymm3
507 ; AVX2-FP-NEXT: vpshufb %ymm2, %ymm0, %ymm2
508 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm3[0,2],ymm2[4,6],ymm3[4,6]
509 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
510 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm3 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
511 ; AVX2-FP-NEXT: vpshufb %ymm3, %ymm1, %ymm1
512 ; AVX2-FP-NEXT: vpshufb %ymm3, %ymm0, %ymm0
513 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
514 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
515 ; AVX2-FP-NEXT: vmovaps %ymm2, (%rsi)
516 ; AVX2-FP-NEXT: vmovdqa %ymm0, (%rdx)
517 ; AVX2-FP-NEXT: vzeroupper
518 ; AVX2-FP-NEXT: retq
519 ;
520 ; AVX2-FCP-LABEL: load_i16_stride2_vf16:
521 ; AVX2-FCP: # %bb.0:
522 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm0
523 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
524 ; AVX2-FCP-NEXT: vpmovsxdq {{.*#+}} ymm2 = [84148480,218892552,353636624,488380696]
525 ; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm3
526 ; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm2
527 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm3[0,2],ymm2[4,6],ymm3[4,6]
528 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
529 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
530 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm1
531 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm0, %ymm0
532 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
533 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
534 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rsi)
535 ; AVX2-FCP-NEXT: vmovdqa %ymm0, (%rdx)
536 ; AVX2-FCP-NEXT: vzeroupper
537 ; AVX2-FCP-NEXT: retq
538 ;
539 ; AVX512-LABEL: load_i16_stride2_vf16:
540 ; AVX512: # %bb.0:
541 ; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
542 ; AVX512-NEXT: vpsrld $16, %zmm0, %zmm1
543 ; AVX512-NEXT: vpmovdw %zmm0, (%rsi)
544 ; AVX512-NEXT: vpmovdw %zmm1, (%rdx)
545 ; AVX512-NEXT: vzeroupper
546 ; AVX512-NEXT: retq
547 ;
548 ; AVX512-FCP-LABEL: load_i16_stride2_vf16:
549 ; AVX512-FCP: # %bb.0:
550 ; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
551 ; AVX512-FCP-NEXT: vpsrld $16, %zmm0, %zmm1
552 ; AVX512-FCP-NEXT: vpmovdw %zmm0, (%rsi)
553 ; AVX512-FCP-NEXT: vpmovdw %zmm1, (%rdx)
554 ; AVX512-FCP-NEXT: vzeroupper
555 ; AVX512-FCP-NEXT: retq
556 ;
557 ; AVX512DQ-LABEL: load_i16_stride2_vf16:
558 ; AVX512DQ: # %bb.0:
559 ; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
560 ; AVX512DQ-NEXT: vpsrld $16, %zmm0, %zmm1
561 ; AVX512DQ-NEXT: vpmovdw %zmm0, (%rsi)
562 ; AVX512DQ-NEXT: vpmovdw %zmm1, (%rdx)
563 ; AVX512DQ-NEXT: vzeroupper
564 ; AVX512DQ-NEXT: retq
565 ;
566 ; AVX512DQ-FCP-LABEL: load_i16_stride2_vf16:
567 ; AVX512DQ-FCP: # %bb.0:
568 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
569 ; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm0, %zmm1
570 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm0, (%rsi)
571 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm1, (%rdx)
572 ; AVX512DQ-FCP-NEXT: vzeroupper
573 ; AVX512DQ-FCP-NEXT: retq
574 ;
575 ; AVX512BW-LABEL: load_i16_stride2_vf16:
576 ; AVX512BW: # %bb.0:
577 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
578 ; AVX512BW-NEXT: vpsrld $16, %zmm0, %zmm1
579 ; AVX512BW-NEXT: vpmovdw %zmm0, (%rsi)
580 ; AVX512BW-NEXT: vpmovdw %zmm1, (%rdx)
581 ; AVX512BW-NEXT: vzeroupper
582 ; AVX512BW-NEXT: retq
583 ;
584 ; AVX512BW-FCP-LABEL: load_i16_stride2_vf16:
585 ; AVX512BW-FCP: # %bb.0:
586 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
587 ; AVX512BW-FCP-NEXT: vpsrld $16, %zmm0, %zmm1
588 ; AVX512BW-FCP-NEXT: vpmovdw %zmm0, (%rsi)
589 ; AVX512BW-FCP-NEXT: vpmovdw %zmm1, (%rdx)
590 ; AVX512BW-FCP-NEXT: vzeroupper
591 ; AVX512BW-FCP-NEXT: retq
592 ;
593 ; AVX512DQ-BW-LABEL: load_i16_stride2_vf16:
594 ; AVX512DQ-BW: # %bb.0:
595 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
596 ; AVX512DQ-BW-NEXT: vpsrld $16, %zmm0, %zmm1
597 ; AVX512DQ-BW-NEXT: vpmovdw %zmm0, (%rsi)
598 ; AVX512DQ-BW-NEXT: vpmovdw %zmm1, (%rdx)
599 ; AVX512DQ-BW-NEXT: vzeroupper
600 ; AVX512DQ-BW-NEXT: retq
601 ;
602 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride2_vf16:
603 ; AVX512DQ-BW-FCP: # %bb.0:
604 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
605 ; AVX512DQ-BW-FCP-NEXT: vpsrld $16, %zmm0, %zmm1
606 ; AVX512DQ-BW-FCP-NEXT: vpmovdw %zmm0, (%rsi)
607 ; AVX512DQ-BW-FCP-NEXT: vpmovdw %zmm1, (%rdx)
608 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
609 ; AVX512DQ-BW-FCP-NEXT: retq
610 %wide.vec = load <32 x i16>, ptr %in.vec, align 64
611 %strided.vec0 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
612 %strided.vec1 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
613 store <16 x i16> %strided.vec0, ptr %out.vec0, align 64
614 store <16 x i16> %strided.vec1, ptr %out.vec1, align 64
615 ret void
616 }
618 define void @load_i16_stride2_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nounwind {
619 ; SSE-LABEL: load_i16_stride2_vf32:
620 ; SSE: # %bb.0:
621 ; SSE-NEXT: movdqa 64(%rdi), %xmm0
622 ; SSE-NEXT: movdqa 80(%rdi), %xmm4
623 ; SSE-NEXT: movdqa 96(%rdi), %xmm1
624 ; SSE-NEXT: movdqa 112(%rdi), %xmm6
625 ; SSE-NEXT: movdqa (%rdi), %xmm2
626 ; SSE-NEXT: movdqa 16(%rdi), %xmm7
627 ; SSE-NEXT: movdqa 32(%rdi), %xmm3
628 ; SSE-NEXT: movdqa 48(%rdi), %xmm9
629 ; SSE-NEXT: movdqa %xmm9, %xmm8
630 ; SSE-NEXT: pslld $16, %xmm8
631 ; SSE-NEXT: psrad $16, %xmm8
632 ; SSE-NEXT: movdqa %xmm3, %xmm5
633 ; SSE-NEXT: pslld $16, %xmm5
634 ; SSE-NEXT: psrad $16, %xmm5
635 ; SSE-NEXT: packssdw %xmm8, %xmm5
636 ; SSE-NEXT: movdqa %xmm7, %xmm10
637 ; SSE-NEXT: pslld $16, %xmm10
638 ; SSE-NEXT: psrad $16, %xmm10
639 ; SSE-NEXT: movdqa %xmm2, %xmm8
640 ; SSE-NEXT: pslld $16, %xmm8
641 ; SSE-NEXT: psrad $16, %xmm8
642 ; SSE-NEXT: packssdw %xmm10, %xmm8
643 ; SSE-NEXT: movdqa %xmm6, %xmm11
644 ; SSE-NEXT: pslld $16, %xmm11
645 ; SSE-NEXT: psrad $16, %xmm11
646 ; SSE-NEXT: movdqa %xmm1, %xmm10
647 ; SSE-NEXT: pslld $16, %xmm10
648 ; SSE-NEXT: psrad $16, %xmm10
649 ; SSE-NEXT: packssdw %xmm11, %xmm10
650 ; SSE-NEXT: movdqa %xmm4, %xmm11
651 ; SSE-NEXT: pslld $16, %xmm11
652 ; SSE-NEXT: psrad $16, %xmm11
653 ; SSE-NEXT: movdqa %xmm0, %xmm12
654 ; SSE-NEXT: pslld $16, %xmm12
655 ; SSE-NEXT: psrad $16, %xmm12
656 ; SSE-NEXT: packssdw %xmm11, %xmm12
657 ; SSE-NEXT: psrad $16, %xmm9
658 ; SSE-NEXT: psrad $16, %xmm3
659 ; SSE-NEXT: packssdw %xmm9, %xmm3
660 ; SSE-NEXT: psrad $16, %xmm7
661 ; SSE-NEXT: psrad $16, %xmm2
662 ; SSE-NEXT: packssdw %xmm7, %xmm2
663 ; SSE-NEXT: psrad $16, %xmm6
664 ; SSE-NEXT: psrad $16, %xmm1
665 ; SSE-NEXT: packssdw %xmm6, %xmm1
666 ; SSE-NEXT: psrad $16, %xmm4
667 ; SSE-NEXT: psrad $16, %xmm0
668 ; SSE-NEXT: packssdw %xmm4, %xmm0
669 ; SSE-NEXT: movdqa %xmm12, 32(%rsi)
670 ; SSE-NEXT: movdqa %xmm10, 48(%rsi)
671 ; SSE-NEXT: movdqa %xmm8, (%rsi)
672 ; SSE-NEXT: movdqa %xmm5, 16(%rsi)
673 ; SSE-NEXT: movdqa %xmm0, 32(%rdx)
674 ; SSE-NEXT: movdqa %xmm1, 48(%rdx)
675 ; SSE-NEXT: movdqa %xmm2, (%rdx)
676 ; SSE-NEXT: movdqa %xmm3, 16(%rdx)
677 ; SSE-NEXT: retq
678 ;
679 ; AVX-LABEL: load_i16_stride2_vf32:
680 ; AVX: # %bb.0:
681 ; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
682 ; AVX-NEXT: vmovdqa (%rdi), %xmm1
683 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
684 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm3
685 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm4
686 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm0[1],xmm4[2],xmm0[3],xmm4[4],xmm0[5],xmm4[6],xmm0[7]
687 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
688 ; AVX-NEXT: vpackusdw %xmm5, %xmm6, %xmm5
689 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
690 ; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
691 ; AVX-NEXT: vpackusdw %xmm6, %xmm7, %xmm6
692 ; AVX-NEXT: vmovdqa 112(%rdi), %xmm7
693 ; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm7[0],xmm0[1],xmm7[2],xmm0[3],xmm7[4],xmm0[5],xmm7[6],xmm0[7]
694 ; AVX-NEXT: vmovdqa 96(%rdi), %xmm9
695 ; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm9[0],xmm0[1],xmm9[2],xmm0[3],xmm9[4],xmm0[5],xmm9[6],xmm0[7]
696 ; AVX-NEXT: vpackusdw %xmm8, %xmm10, %xmm8
697 ; AVX-NEXT: vmovdqa 80(%rdi), %xmm10
698 ; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm10[0],xmm0[1],xmm10[2],xmm0[3],xmm10[4],xmm0[5],xmm10[6],xmm0[7]
699 ; AVX-NEXT: vmovdqa 64(%rdi), %xmm12
700 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm12[0],xmm0[1],xmm12[2],xmm0[3],xmm12[4],xmm0[5],xmm12[6],xmm0[7]
701 ; AVX-NEXT: vpackusdw %xmm11, %xmm0, %xmm0
702 ; AVX-NEXT: vpsrld $16, %xmm4, %xmm4
703 ; AVX-NEXT: vpsrld $16, %xmm3, %xmm3
704 ; AVX-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
705 ; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
706 ; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
707 ; AVX-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
708 ; AVX-NEXT: vpsrld $16, %xmm7, %xmm2
709 ; AVX-NEXT: vpsrld $16, %xmm9, %xmm4
710 ; AVX-NEXT: vpackusdw %xmm2, %xmm4, %xmm2
711 ; AVX-NEXT: vpsrld $16, %xmm10, %xmm4
712 ; AVX-NEXT: vpsrld $16, %xmm12, %xmm7
713 ; AVX-NEXT: vpackusdw %xmm4, %xmm7, %xmm4
714 ; AVX-NEXT: vmovdqa %xmm0, 32(%rsi)
715 ; AVX-NEXT: vmovdqa %xmm8, 48(%rsi)
716 ; AVX-NEXT: vmovdqa %xmm6, (%rsi)
717 ; AVX-NEXT: vmovdqa %xmm5, 16(%rsi)
718 ; AVX-NEXT: vmovdqa %xmm4, 32(%rdx)
719 ; AVX-NEXT: vmovdqa %xmm2, 48(%rdx)
720 ; AVX-NEXT: vmovdqa %xmm1, (%rdx)
721 ; AVX-NEXT: vmovdqa %xmm3, 16(%rdx)
722 ; AVX-NEXT: retq
723 ;
724 ; AVX2-LABEL: load_i16_stride2_vf32:
725 ; AVX2: # %bb.0:
726 ; AVX2-NEXT: vmovdqa (%rdi), %ymm0
727 ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1
728 ; AVX2-NEXT: vmovdqa 64(%rdi), %ymm2
729 ; AVX2-NEXT: vmovdqa 96(%rdi), %ymm3
730 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm4 = ymm3[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
731 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm4 = ymm4[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
732 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm5 = ymm2[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
733 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
734 ; AVX2-NEXT: vshufps {{.*#+}} ymm4 = ymm5[0,2],ymm4[0,2],ymm5[4,6],ymm4[4,6]
735 ; AVX2-NEXT: vpermpd {{.*#+}} ymm4 = ymm4[0,2,1,3]
736 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm5 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
737 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
738 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm6 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
739 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm6 = ymm6[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
740 ; AVX2-NEXT: vshufps {{.*#+}} ymm5 = ymm6[0,2],ymm5[0,2],ymm6[4,6],ymm5[4,6]
741 ; AVX2-NEXT: vpermpd {{.*#+}} ymm5 = ymm5[0,2,1,3]
742 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm6 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
743 ; AVX2-NEXT: vpshufb %ymm6, %ymm3, %ymm3
744 ; AVX2-NEXT: vpshufb %ymm6, %ymm2, %ymm2
745 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
746 ; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
747 ; AVX2-NEXT: vpshufb %ymm6, %ymm1, %ymm1
748 ; AVX2-NEXT: vpshufb %ymm6, %ymm0, %ymm0
749 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
750 ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
751 ; AVX2-NEXT: vmovaps %ymm5, (%rsi)
752 ; AVX2-NEXT: vmovaps %ymm4, 32(%rsi)
753 ; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
754 ; AVX2-NEXT: vmovdqa %ymm2, 32(%rdx)
755 ; AVX2-NEXT: vzeroupper
756 ; AVX2-NEXT: retq
757 ;
758 ; AVX2-FP-LABEL: load_i16_stride2_vf32:
759 ; AVX2-FP: # %bb.0:
760 ; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm0
761 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm1
762 ; AVX2-FP-NEXT: vmovdqa 64(%rdi), %ymm2
763 ; AVX2-FP-NEXT: vmovdqa 96(%rdi), %ymm3
764 ; AVX2-FP-NEXT: vpmovsxdq {{.*#+}} ymm4 = [84148480,218892552,353636624,488380696]
765 ; AVX2-FP-NEXT: vpshufb %ymm4, %ymm3, %ymm5
766 ; AVX2-FP-NEXT: vpshufb %ymm4, %ymm2, %ymm6
767 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm5 = ymm6[0,2],ymm5[0,2],ymm6[4,6],ymm5[4,6]
768 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm5 = ymm5[0,2,1,3]
769 ; AVX2-FP-NEXT: vpshufb %ymm4, %ymm1, %ymm6
770 ; AVX2-FP-NEXT: vpshufb %ymm4, %ymm0, %ymm4
771 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm4 = ymm4[0,2],ymm6[0,2],ymm4[4,6],ymm6[4,6]
772 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm4 = ymm4[0,2,1,3]
773 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm6 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
774 ; AVX2-FP-NEXT: vpshufb %ymm6, %ymm3, %ymm3
775 ; AVX2-FP-NEXT: vpshufb %ymm6, %ymm2, %ymm2
776 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
777 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
778 ; AVX2-FP-NEXT: vpshufb %ymm6, %ymm1, %ymm1
779 ; AVX2-FP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
780 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
781 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
782 ; AVX2-FP-NEXT: vmovaps %ymm4, (%rsi)
783 ; AVX2-FP-NEXT: vmovaps %ymm5, 32(%rsi)
784 ; AVX2-FP-NEXT: vmovdqa %ymm0, (%rdx)
785 ; AVX2-FP-NEXT: vmovdqa %ymm2, 32(%rdx)
786 ; AVX2-FP-NEXT: vzeroupper
787 ; AVX2-FP-NEXT: retq
788 ;
789 ; AVX2-FCP-LABEL: load_i16_stride2_vf32:
790 ; AVX2-FCP: # %bb.0:
791 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm0
792 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
793 ; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %ymm2
794 ; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %ymm3
795 ; AVX2-FCP-NEXT: vpmovsxdq {{.*#+}} ymm4 = [84148480,218892552,353636624,488380696]
796 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm5
797 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm6
798 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm5 = ymm6[0,2],ymm5[0,2],ymm6[4,6],ymm5[4,6]
799 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm5 = ymm5[0,2,1,3]
800 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm6
801 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm0, %ymm4
802 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm4 = ymm4[0,2],ymm6[0,2],ymm4[4,6],ymm6[4,6]
803 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm4 = ymm4[0,2,1,3]
804 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
805 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm3
806 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm2
807 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
808 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
809 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm1
810 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
811 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
812 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
813 ; AVX2-FCP-NEXT: vmovaps %ymm4, (%rsi)
814 ; AVX2-FCP-NEXT: vmovaps %ymm5, 32(%rsi)
815 ; AVX2-FCP-NEXT: vmovdqa %ymm0, (%rdx)
816 ; AVX2-FCP-NEXT: vmovdqa %ymm2, 32(%rdx)
817 ; AVX2-FCP-NEXT: vzeroupper
818 ; AVX2-FCP-NEXT: retq
819 ;
820 ; AVX512-LABEL: load_i16_stride2_vf32:
821 ; AVX512: # %bb.0:
822 ; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
823 ; AVX512-NEXT: vmovdqa64 64(%rdi), %zmm1
824 ; AVX512-NEXT: vpsrld $16, %zmm0, %zmm2
825 ; AVX512-NEXT: vpsrld $16, %zmm1, %zmm3
826 ; AVX512-NEXT: vpmovdw %zmm1, 32(%rsi)
827 ; AVX512-NEXT: vpmovdw %zmm0, (%rsi)
828 ; AVX512-NEXT: vpmovdw %zmm3, 32(%rdx)
829 ; AVX512-NEXT: vpmovdw %zmm2, (%rdx)
830 ; AVX512-NEXT: vzeroupper
831 ; AVX512-NEXT: retq
832 ;
833 ; AVX512-FCP-LABEL: load_i16_stride2_vf32:
834 ; AVX512-FCP: # %bb.0:
835 ; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
836 ; AVX512-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
837 ; AVX512-FCP-NEXT: vpsrld $16, %zmm0, %zmm2
838 ; AVX512-FCP-NEXT: vpsrld $16, %zmm1, %zmm3
839 ; AVX512-FCP-NEXT: vpmovdw %zmm1, 32(%rsi)
840 ; AVX512-FCP-NEXT: vpmovdw %zmm0, (%rsi)
841 ; AVX512-FCP-NEXT: vpmovdw %zmm3, 32(%rdx)
842 ; AVX512-FCP-NEXT: vpmovdw %zmm2, (%rdx)
843 ; AVX512-FCP-NEXT: vzeroupper
844 ; AVX512-FCP-NEXT: retq
845 ;
846 ; AVX512DQ-LABEL: load_i16_stride2_vf32:
847 ; AVX512DQ: # %bb.0:
848 ; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
849 ; AVX512DQ-NEXT: vmovdqa64 64(%rdi), %zmm1
850 ; AVX512DQ-NEXT: vpsrld $16, %zmm0, %zmm2
851 ; AVX512DQ-NEXT: vpsrld $16, %zmm1, %zmm3
852 ; AVX512DQ-NEXT: vpmovdw %zmm1, 32(%rsi)
853 ; AVX512DQ-NEXT: vpmovdw %zmm0, (%rsi)
854 ; AVX512DQ-NEXT: vpmovdw %zmm3, 32(%rdx)
855 ; AVX512DQ-NEXT: vpmovdw %zmm2, (%rdx)
856 ; AVX512DQ-NEXT: vzeroupper
857 ; AVX512DQ-NEXT: retq
858 ;
859 ; AVX512DQ-FCP-LABEL: load_i16_stride2_vf32:
860 ; AVX512DQ-FCP: # %bb.0:
861 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
862 ; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
863 ; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm0, %zmm2
864 ; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm1, %zmm3
865 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm1, 32(%rsi)
866 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm0, (%rsi)
867 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm3, 32(%rdx)
868 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm2, (%rdx)
869 ; AVX512DQ-FCP-NEXT: vzeroupper
870 ; AVX512DQ-FCP-NEXT: retq
871 ;
872 ; AVX512BW-LABEL: load_i16_stride2_vf32:
873 ; AVX512BW: # %bb.0:
874 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
875 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
876 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
877 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
878 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
879 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm3
880 ; AVX512BW-NEXT: vmovdqa64 %zmm2, (%rsi)
881 ; AVX512BW-NEXT: vmovdqa64 %zmm3, (%rdx)
882 ; AVX512BW-NEXT: vzeroupper
883 ; AVX512BW-NEXT: retq
884 ;
885 ; AVX512BW-FCP-LABEL: load_i16_stride2_vf32:
886 ; AVX512BW-FCP: # %bb.0:
887 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
888 ; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
889 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
890 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
891 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
892 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm3
893 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, (%rsi)
894 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, (%rdx)
895 ; AVX512BW-FCP-NEXT: vzeroupper
896 ; AVX512BW-FCP-NEXT: retq
897 ;
898 ; AVX512DQ-BW-LABEL: load_i16_stride2_vf32:
899 ; AVX512DQ-BW: # %bb.0:
900 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
901 ; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm1
902 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
903 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
904 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
905 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm3
906 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, (%rsi)
907 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, (%rdx)
908 ; AVX512DQ-BW-NEXT: vzeroupper
909 ; AVX512DQ-BW-NEXT: retq
910 ;
911 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride2_vf32:
912 ; AVX512DQ-BW-FCP: # %bb.0:
913 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
914 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
915 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
916 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
917 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
918 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm3
919 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, (%rsi)
920 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, (%rdx)
921 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
922 ; AVX512DQ-BW-FCP-NEXT: retq
923 %wide.vec = load <64 x i16>, ptr %in.vec, align 64
924 %strided.vec0 = shufflevector <64 x i16> %wide.vec, <64 x i16> poison, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>
925 %strided.vec1 = shufflevector <64 x i16> %wide.vec, <64 x i16> poison, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>
926 store <32 x i16> %strided.vec0, ptr %out.vec0, align 64
927 store <32 x i16> %strided.vec1, ptr %out.vec1, align 64
928 ret void
929 }
931 define void @load_i16_stride2_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nounwind {
932 ; SSE-LABEL: load_i16_stride2_vf64:
933 ; SSE: # %bb.0:
934 ; SSE-NEXT: subq $40, %rsp
935 ; SSE-NEXT: movdqa 160(%rdi), %xmm14
936 ; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
937 ; SSE-NEXT: movdqa 176(%rdi), %xmm2
938 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
939 ; SSE-NEXT: movdqa 64(%rdi), %xmm11
940 ; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
941 ; SSE-NEXT: movdqa 80(%rdi), %xmm1
942 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
943 ; SSE-NEXT: movdqa 96(%rdi), %xmm9
944 ; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
945 ; SSE-NEXT: movdqa 112(%rdi), %xmm4
946 ; SSE-NEXT: movdqa %xmm4, (%rsp) # 16-byte Spill
947 ; SSE-NEXT: movdqa (%rdi), %xmm10
948 ; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
949 ; SSE-NEXT: movdqa 16(%rdi), %xmm7
950 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
951 ; SSE-NEXT: movdqa 32(%rdi), %xmm13
952 ; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
953 ; SSE-NEXT: movdqa 48(%rdi), %xmm0
954 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
955 ; SSE-NEXT: pslld $16, %xmm0
956 ; SSE-NEXT: psrad $16, %xmm0
957 ; SSE-NEXT: pslld $16, %xmm13
958 ; SSE-NEXT: psrad $16, %xmm13
959 ; SSE-NEXT: packssdw %xmm0, %xmm13
960 ; SSE-NEXT: movdqa %xmm7, %xmm0
961 ; SSE-NEXT: pslld $16, %xmm0
962 ; SSE-NEXT: psrad $16, %xmm0
963 ; SSE-NEXT: pslld $16, %xmm10
964 ; SSE-NEXT: psrad $16, %xmm10
965 ; SSE-NEXT: packssdw %xmm0, %xmm10
966 ; SSE-NEXT: movdqa %xmm4, %xmm0
967 ; SSE-NEXT: pslld $16, %xmm0
968 ; SSE-NEXT: psrad $16, %xmm0
969 ; SSE-NEXT: pslld $16, %xmm9
970 ; SSE-NEXT: psrad $16, %xmm9
971 ; SSE-NEXT: packssdw %xmm0, %xmm9
972 ; SSE-NEXT: movdqa %xmm1, %xmm0
973 ; SSE-NEXT: pslld $16, %xmm0
974 ; SSE-NEXT: psrad $16, %xmm0
975 ; SSE-NEXT: pslld $16, %xmm11
976 ; SSE-NEXT: psrad $16, %xmm11
977 ; SSE-NEXT: packssdw %xmm0, %xmm11
978 ; SSE-NEXT: movdqa %xmm2, %xmm0
979 ; SSE-NEXT: pslld $16, %xmm0
980 ; SSE-NEXT: psrad $16, %xmm0
981 ; SSE-NEXT: pslld $16, %xmm14
982 ; SSE-NEXT: psrad $16, %xmm14
983 ; SSE-NEXT: packssdw %xmm0, %xmm14
984 ; SSE-NEXT: movdqa 144(%rdi), %xmm7
985 ; SSE-NEXT: movdqa %xmm7, %xmm0
986 ; SSE-NEXT: pslld $16, %xmm0
987 ; SSE-NEXT: psrad $16, %xmm0
988 ; SSE-NEXT: movdqa 128(%rdi), %xmm8
989 ; SSE-NEXT: movdqa %xmm8, %xmm15
990 ; SSE-NEXT: pslld $16, %xmm15
991 ; SSE-NEXT: psrad $16, %xmm15
992 ; SSE-NEXT: packssdw %xmm0, %xmm15
993 ; SSE-NEXT: movdqa 240(%rdi), %xmm12
994 ; SSE-NEXT: movdqa %xmm12, %xmm1
995 ; SSE-NEXT: pslld $16, %xmm1
996 ; SSE-NEXT: psrad $16, %xmm1
997 ; SSE-NEXT: movdqa 224(%rdi), %xmm5
998 ; SSE-NEXT: movdqa %xmm5, %xmm3
999 ; SSE-NEXT: pslld $16, %xmm3
1000 ; SSE-NEXT: psrad $16, %xmm3
1001 ; SSE-NEXT: packssdw %xmm1, %xmm3
1002 ; SSE-NEXT: movdqa 208(%rdi), %xmm6
1003 ; SSE-NEXT: movdqa %xmm6, %xmm4
1004 ; SSE-NEXT: pslld $16, %xmm4
1005 ; SSE-NEXT: psrad $16, %xmm4
1006 ; SSE-NEXT: movdqa 192(%rdi), %xmm2
1007 ; SSE-NEXT: movdqa %xmm2, %xmm1
1008 ; SSE-NEXT: pslld $16, %xmm1
1009 ; SSE-NEXT: psrad $16, %xmm1
1010 ; SSE-NEXT: packssdw %xmm4, %xmm1
1011 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1012 ; SSE-NEXT: psrad $16, %xmm0
1013 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
1014 ; SSE-NEXT: psrad $16, %xmm4
1015 ; SSE-NEXT: packssdw %xmm0, %xmm4
1016 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1017 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1018 ; SSE-NEXT: psrad $16, %xmm0
1019 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
1020 ; SSE-NEXT: psrad $16, %xmm4
1021 ; SSE-NEXT: packssdw %xmm0, %xmm4
1022 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1023 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1024 ; SSE-NEXT: psrad $16, %xmm0
1025 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
1026 ; SSE-NEXT: psrad $16, %xmm4
1027 ; SSE-NEXT: packssdw %xmm0, %xmm4
1028 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1029 ; SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
1030 ; SSE-NEXT: psrad $16, %xmm0
1031 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
1032 ; SSE-NEXT: psrad $16, %xmm4
1033 ; SSE-NEXT: packssdw %xmm0, %xmm4
1034 ; SSE-NEXT: psrad $16, %xmm7
1035 ; SSE-NEXT: psrad $16, %xmm8
1036 ; SSE-NEXT: packssdw %xmm7, %xmm8
1037 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1038 ; SSE-NEXT: psrad $16, %xmm0
1039 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
1040 ; SSE-NEXT: psrad $16, %xmm7
1041 ; SSE-NEXT: packssdw %xmm0, %xmm7
1042 ; SSE-NEXT: psrad $16, %xmm6
1043 ; SSE-NEXT: psrad $16, %xmm2
1044 ; SSE-NEXT: packssdw %xmm6, %xmm2
1045 ; SSE-NEXT: psrad $16, %xmm12
1046 ; SSE-NEXT: psrad $16, %xmm5
1047 ; SSE-NEXT: packssdw %xmm12, %xmm5
1048 ; SSE-NEXT: movdqa %xmm1, 96(%rsi)
1049 ; SSE-NEXT: movdqa %xmm3, 112(%rsi)
1050 ; SSE-NEXT: movdqa %xmm15, 64(%rsi)
1051 ; SSE-NEXT: movdqa %xmm14, 80(%rsi)
1052 ; SSE-NEXT: movdqa %xmm11, 32(%rsi)
1053 ; SSE-NEXT: movdqa %xmm9, 48(%rsi)
1054 ; SSE-NEXT: movdqa %xmm10, (%rsi)
1055 ; SSE-NEXT: movdqa %xmm13, 16(%rsi)
1056 ; SSE-NEXT: movdqa %xmm5, 112(%rdx)
1057 ; SSE-NEXT: movdqa %xmm2, 96(%rdx)
1058 ; SSE-NEXT: movdqa %xmm7, 80(%rdx)
1059 ; SSE-NEXT: movdqa %xmm8, 64(%rdx)
1060 ; SSE-NEXT: movdqa %xmm4, 48(%rdx)
1061 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1062 ; SSE-NEXT: movaps %xmm0, 32(%rdx)
1063 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1064 ; SSE-NEXT: movaps %xmm0, 16(%rdx)
1065 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1066 ; SSE-NEXT: movaps %xmm0, (%rdx)
1067 ; SSE-NEXT: addq $40, %rsp
1068 ; SSE-NEXT: retq
1069 ;
1070 ; AVX-LABEL: load_i16_stride2_vf64:
1071 ; AVX: # %bb.0:
1072 ; AVX-NEXT: subq $24, %rsp
1073 ; AVX-NEXT: vpxor %xmm11, %xmm11, %xmm11
1074 ; AVX-NEXT: vmovdqa 176(%rdi), %xmm0
1075 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1076 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm11[1],xmm0[2],xmm11[3],xmm0[4],xmm11[5],xmm0[6],xmm11[7]
1077 ; AVX-NEXT: vmovdqa 160(%rdi), %xmm9
1078 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm9[0],xmm11[1],xmm9[2],xmm11[3],xmm9[4],xmm11[5],xmm9[6],xmm11[7]
1079 ; AVX-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
1080 ; AVX-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
1081 ; AVX-NEXT: vmovdqa (%rdi), %xmm6
1082 ; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1083 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm8
1084 ; AVX-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1085 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm7
1086 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm10
1087 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm10[0],xmm11[1],xmm10[2],xmm11[3],xmm10[4],xmm11[5],xmm10[6],xmm11[7]
1088 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0],xmm11[1],xmm7[2],xmm11[3],xmm7[4],xmm11[5],xmm7[6],xmm11[7]
1089 ; AVX-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
1090 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1091 ; AVX-NEXT: vmovdqa 144(%rdi), %xmm5
1092 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0],xmm11[1],xmm5[2],xmm11[3],xmm5[4],xmm11[5],xmm5[6],xmm11[7]
1093 ; AVX-NEXT: vmovdqa 128(%rdi), %xmm4
1094 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm11[1],xmm4[2],xmm11[3],xmm4[4],xmm11[5],xmm4[6],xmm11[7]
1095 ; AVX-NEXT: vpackusdw %xmm2, %xmm3, %xmm0
1096 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1097 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm8[0],xmm11[1],xmm8[2],xmm11[3],xmm8[4],xmm11[5],xmm8[6],xmm11[7]
1098 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm6[0],xmm11[1],xmm6[2],xmm11[3],xmm6[4],xmm11[5],xmm6[6],xmm11[7]
1099 ; AVX-NEXT: vpackusdw %xmm2, %xmm3, %xmm0
1100 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1101 ; AVX-NEXT: vmovdqa 112(%rdi), %xmm12
1102 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm12[0],xmm11[1],xmm12[2],xmm11[3],xmm12[4],xmm11[5],xmm12[6],xmm11[7]
1103 ; AVX-NEXT: vmovdqa 96(%rdi), %xmm13
1104 ; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm13[0],xmm11[1],xmm13[2],xmm11[3],xmm13[4],xmm11[5],xmm13[6],xmm11[7]
1105 ; AVX-NEXT: vpackusdw %xmm2, %xmm8, %xmm0
1106 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1107 ; AVX-NEXT: vmovdqa 80(%rdi), %xmm15
1108 ; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm15[0],xmm11[1],xmm15[2],xmm11[3],xmm15[4],xmm11[5],xmm15[6],xmm11[7]
1109 ; AVX-NEXT: vmovdqa 64(%rdi), %xmm6
1110 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm6[0],xmm11[1],xmm6[2],xmm11[3],xmm6[4],xmm11[5],xmm6[6],xmm11[7]
1111 ; AVX-NEXT: vpackusdw %xmm14, %xmm3, %xmm0
1112 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1113 ; AVX-NEXT: vmovdqa 240(%rdi), %xmm3
1114 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm11[1],xmm3[2],xmm11[3],xmm3[4],xmm11[5],xmm3[6],xmm11[7]
1115 ; AVX-NEXT: vmovdqa 224(%rdi), %xmm0
1116 ; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm0[0],xmm11[1],xmm0[2],xmm11[3],xmm0[4],xmm11[5],xmm0[6],xmm11[7]
1117 ; AVX-NEXT: vpackusdw %xmm1, %xmm8, %xmm14
1118 ; AVX-NEXT: vmovdqa 208(%rdi), %xmm8
1119 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm8[0],xmm11[1],xmm8[2],xmm11[3],xmm8[4],xmm11[5],xmm8[6],xmm11[7]
1120 ; AVX-NEXT: vmovdqa 192(%rdi), %xmm2
1121 ; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm2[0],xmm11[1],xmm2[2],xmm11[3],xmm2[4],xmm11[5],xmm2[6],xmm11[7]
1122 ; AVX-NEXT: vpackusdw %xmm1, %xmm11, %xmm1
1123 ; AVX-NEXT: vpsrld $16, %xmm3, %xmm3
1124 ; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
1125 ; AVX-NEXT: vpackusdw %xmm3, %xmm0, %xmm11
1126 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1127 ; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
1128 ; AVX-NEXT: vpsrld $16, %xmm9, %xmm3
1129 ; AVX-NEXT: vpackusdw %xmm0, %xmm3, %xmm0
1130 ; AVX-NEXT: vpsrld $16, %xmm5, %xmm3
1131 ; AVX-NEXT: vpsrld $16, %xmm4, %xmm4
1132 ; AVX-NEXT: vpackusdw %xmm3, %xmm4, %xmm3
1133 ; AVX-NEXT: vpsrld $16, %xmm10, %xmm4
1134 ; AVX-NEXT: vpsrld $16, %xmm7, %xmm5
1135 ; AVX-NEXT: vpackusdw %xmm4, %xmm5, %xmm4
1136 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
1137 ; AVX-NEXT: vpsrld $16, %xmm5, %xmm5
1138 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
1139 ; AVX-NEXT: vpsrld $16, %xmm7, %xmm7
1140 ; AVX-NEXT: vpackusdw %xmm5, %xmm7, %xmm5
1141 ; AVX-NEXT: vpsrld $16, %xmm8, %xmm7
1142 ; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
1143 ; AVX-NEXT: vpackusdw %xmm7, %xmm2, %xmm2
1144 ; AVX-NEXT: vpsrld $16, %xmm12, %xmm7
1145 ; AVX-NEXT: vpsrld $16, %xmm13, %xmm8
1146 ; AVX-NEXT: vpackusdw %xmm7, %xmm8, %xmm7
1147 ; AVX-NEXT: vpsrld $16, %xmm15, %xmm8
1148 ; AVX-NEXT: vpsrld $16, %xmm6, %xmm6
1149 ; AVX-NEXT: vpackusdw %xmm8, %xmm6, %xmm6
1150 ; AVX-NEXT: vmovdqa %xmm1, 96(%rsi)
1151 ; AVX-NEXT: vmovdqa %xmm14, 112(%rsi)
1152 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
1153 ; AVX-NEXT: vmovaps %xmm1, 32(%rsi)
1154 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
1155 ; AVX-NEXT: vmovaps %xmm1, 48(%rsi)
1156 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
1157 ; AVX-NEXT: vmovaps %xmm1, (%rsi)
1158 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
1159 ; AVX-NEXT: vmovaps %xmm1, 64(%rsi)
1160 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
1161 ; AVX-NEXT: vmovaps %xmm1, 16(%rsi)
1162 ; AVX-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload
1163 ; AVX-NEXT: vmovaps %xmm1, 80(%rsi)
1164 ; AVX-NEXT: vmovdqa %xmm6, 32(%rdx)
1165 ; AVX-NEXT: vmovdqa %xmm7, 48(%rdx)
1166 ; AVX-NEXT: vmovdqa %xmm2, 96(%rdx)
1167 ; AVX-NEXT: vmovdqa %xmm5, (%rdx)
1168 ; AVX-NEXT: vmovdqa %xmm4, 16(%rdx)
1169 ; AVX-NEXT: vmovdqa %xmm3, 64(%rdx)
1170 ; AVX-NEXT: vmovdqa %xmm0, 80(%rdx)
1171 ; AVX-NEXT: vmovdqa %xmm11, 112(%rdx)
1172 ; AVX-NEXT: addq $24, %rsp
1173 ; AVX-NEXT: retq
1174 ;
1175 ; AVX2-LABEL: load_i16_stride2_vf64:
1176 ; AVX2: # %bb.0:
1177 ; AVX2-NEXT: vmovdqa 128(%rdi), %ymm0
1178 ; AVX2-NEXT: vmovdqa 160(%rdi), %ymm1
1179 ; AVX2-NEXT: vmovdqa 192(%rdi), %ymm4
1180 ; AVX2-NEXT: vmovdqa 224(%rdi), %ymm6
1181 ; AVX2-NEXT: vmovdqa (%rdi), %ymm3
1182 ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm5
1183 ; AVX2-NEXT: vmovdqa 64(%rdi), %ymm7
1184 ; AVX2-NEXT: vmovdqa 96(%rdi), %ymm8
1185 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm8[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
1186 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
1187 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm9 = ymm7[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
1188 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm9 = ymm9[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
1189 ; AVX2-NEXT: vshufps {{.*#+}} ymm2 = ymm9[0,2],ymm2[0,2],ymm9[4,6],ymm2[4,6]
1190 ; AVX2-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
1191 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm9 = ymm6[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
1192 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm9 = ymm9[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
1193 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm10 = ymm4[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
1194 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm10 = ymm10[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
1195 ; AVX2-NEXT: vshufps {{.*#+}} ymm9 = ymm10[0,2],ymm9[0,2],ymm10[4,6],ymm9[4,6]
1196 ; AVX2-NEXT: vpermpd {{.*#+}} ymm9 = ymm9[0,2,1,3]
1197 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm10 = ymm5[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
1198 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm10 = ymm10[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
1199 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm11 = ymm3[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
1200 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm11 = ymm11[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
1201 ; AVX2-NEXT: vshufps {{.*#+}} ymm10 = ymm11[0,2],ymm10[0,2],ymm11[4,6],ymm10[4,6]
1202 ; AVX2-NEXT: vpermpd {{.*#+}} ymm10 = ymm10[0,2,1,3]
1203 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm11 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
1204 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm11 = ymm11[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
1205 ; AVX2-NEXT: vpshuflw {{.*#+}} ymm12 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
1206 ; AVX2-NEXT: vpshufhw {{.*#+}} ymm12 = ymm12[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
1207 ; AVX2-NEXT: vshufps {{.*#+}} ymm11 = ymm12[0,2],ymm11[0,2],ymm12[4,6],ymm11[4,6]
1208 ; AVX2-NEXT: vpermpd {{.*#+}} ymm11 = ymm11[0,2,1,3]
1209 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm12 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
1210 ; AVX2-NEXT: vpshufb %ymm12, %ymm8, %ymm8
1211 ; AVX2-NEXT: vpshufb %ymm12, %ymm7, %ymm7
1212 ; AVX2-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1],ymm8[2,3],ymm7[4,5],ymm8[6,7]
1213 ; AVX2-NEXT: vpermq {{.*#+}} ymm7 = ymm7[0,2,1,3]
1214 ; AVX2-NEXT: vpshufb %ymm12, %ymm6, %ymm6
1215 ; AVX2-NEXT: vpshufb %ymm12, %ymm4, %ymm4
1216 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm6[2,3],ymm4[4,5],ymm6[6,7]
1217 ; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
1218 ; AVX2-NEXT: vpshufb %ymm12, %ymm5, %ymm5
1219 ; AVX2-NEXT: vpshufb %ymm12, %ymm3, %ymm3
1220 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm5[2,3],ymm3[4,5],ymm5[6,7]
1221 ; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
1222 ; AVX2-NEXT: vpshufb %ymm12, %ymm1, %ymm1
1223 ; AVX2-NEXT: vpshufb %ymm12, %ymm0, %ymm0
1224 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
1225 ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
1226 ; AVX2-NEXT: vmovaps %ymm11, 64(%rsi)
1227 ; AVX2-NEXT: vmovaps %ymm10, (%rsi)
1228 ; AVX2-NEXT: vmovaps %ymm9, 96(%rsi)
1229 ; AVX2-NEXT: vmovaps %ymm2, 32(%rsi)
1230 ; AVX2-NEXT: vmovdqa %ymm0, 64(%rdx)
1231 ; AVX2-NEXT: vmovdqa %ymm3, (%rdx)
1232 ; AVX2-NEXT: vmovdqa %ymm4, 96(%rdx)
1233 ; AVX2-NEXT: vmovdqa %ymm7, 32(%rdx)
1234 ; AVX2-NEXT: vzeroupper
1235 ; AVX2-NEXT: retq
1236 ;
1237 ; AVX2-FP-LABEL: load_i16_stride2_vf64:
1238 ; AVX2-FP: # %bb.0:
1239 ; AVX2-FP-NEXT: vmovdqa 128(%rdi), %ymm0
1240 ; AVX2-FP-NEXT: vmovdqa 160(%rdi), %ymm1
1241 ; AVX2-FP-NEXT: vmovdqa 192(%rdi), %ymm3
1242 ; AVX2-FP-NEXT: vmovdqa 224(%rdi), %ymm4
1243 ; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm5
1244 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm6
1245 ; AVX2-FP-NEXT: vmovdqa 64(%rdi), %ymm7
1246 ; AVX2-FP-NEXT: vmovdqa 96(%rdi), %ymm8
1247 ; AVX2-FP-NEXT: vpmovsxdq {{.*#+}} ymm9 = [84148480,218892552,353636624,488380696]
1248 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm8, %ymm2
1249 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm7, %ymm10
1250 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm2 = ymm10[0,2],ymm2[0,2],ymm10[4,6],ymm2[4,6]
1251 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
1252 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm4, %ymm10
1253 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm3, %ymm11
1254 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm10 = ymm11[0,2],ymm10[0,2],ymm11[4,6],ymm10[4,6]
1255 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm10 = ymm10[0,2,1,3]
1256 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm6, %ymm11
1257 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm5, %ymm12
1258 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm11 = ymm12[0,2],ymm11[0,2],ymm12[4,6],ymm11[4,6]
1259 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm11 = ymm11[0,2,1,3]
1260 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm1, %ymm12
1261 ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm0, %ymm9
1262 ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm9 = ymm9[0,2],ymm12[0,2],ymm9[4,6],ymm12[4,6]
1263 ; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm9 = ymm9[0,2,1,3]
1264 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm12 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
1265 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm8, %ymm8
1266 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm7, %ymm7
1267 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1],ymm8[2,3],ymm7[4,5],ymm8[6,7]
1268 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm7 = ymm7[0,2,1,3]
1269 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm4, %ymm4
1270 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm3, %ymm3
1271 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
1272 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
1273 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm6, %ymm4
1274 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm5, %ymm5
1275 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]
1276 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
1277 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm1, %ymm1
1278 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm0, %ymm0
1279 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
1280 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
1281 ; AVX2-FP-NEXT: vmovaps %ymm9, 64(%rsi)
1282 ; AVX2-FP-NEXT: vmovaps %ymm11, (%rsi)
1283 ; AVX2-FP-NEXT: vmovaps %ymm10, 96(%rsi)
1284 ; AVX2-FP-NEXT: vmovaps %ymm2, 32(%rsi)
1285 ; AVX2-FP-NEXT: vmovdqa %ymm0, 64(%rdx)
1286 ; AVX2-FP-NEXT: vmovdqa %ymm4, (%rdx)
1287 ; AVX2-FP-NEXT: vmovdqa %ymm3, 96(%rdx)
1288 ; AVX2-FP-NEXT: vmovdqa %ymm7, 32(%rdx)
1289 ; AVX2-FP-NEXT: vzeroupper
1290 ; AVX2-FP-NEXT: retq
1291 ;
1292 ; AVX2-FCP-LABEL: load_i16_stride2_vf64:
1293 ; AVX2-FCP: # %bb.0:
1294 ; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %ymm0
1295 ; AVX2-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
1296 ; AVX2-FCP-NEXT: vmovdqa 192(%rdi), %ymm3
1297 ; AVX2-FCP-NEXT: vmovdqa 224(%rdi), %ymm4
1298 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm5
1299 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm6
1300 ; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %ymm7
1301 ; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %ymm8
1302 ; AVX2-FCP-NEXT: vpmovsxdq {{.*#+}} ymm9 = [84148480,218892552,353636624,488380696]
1303 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm8, %ymm2
1304 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm7, %ymm10
1305 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm2 = ymm10[0,2],ymm2[0,2],ymm10[4,6],ymm2[4,6]
1306 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
1307 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm4, %ymm10
1308 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm11
1309 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm10 = ymm11[0,2],ymm10[0,2],ymm11[4,6],ymm10[4,6]
1310 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm10 = ymm10[0,2,1,3]
1311 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm6, %ymm11
1312 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm5, %ymm12
1313 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm11 = ymm12[0,2],ymm11[0,2],ymm12[4,6],ymm11[4,6]
1314 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm11 = ymm11[0,2,1,3]
1315 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm1, %ymm12
1316 ; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm0, %ymm9
1317 ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm9 = ymm9[0,2],ymm12[0,2],ymm9[4,6],ymm12[4,6]
1318 ; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm9 = ymm9[0,2,1,3]
1319 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm12 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
1320 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm8, %ymm8
1321 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm7, %ymm7
1322 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1],ymm8[2,3],ymm7[4,5],ymm8[6,7]
1323 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm7 = ymm7[0,2,1,3]
1324 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm4, %ymm4
1325 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm3, %ymm3
1326 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
1327 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
1328 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm6, %ymm4
1329 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm5, %ymm5
1330 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]
1331 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
1332 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm1, %ymm1
1333 ; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm0, %ymm0
1334 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
1335 ; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
1336 ; AVX2-FCP-NEXT: vmovaps %ymm9, 64(%rsi)
1337 ; AVX2-FCP-NEXT: vmovaps %ymm11, (%rsi)
1338 ; AVX2-FCP-NEXT: vmovaps %ymm10, 96(%rsi)
1339 ; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rsi)
1340 ; AVX2-FCP-NEXT: vmovdqa %ymm0, 64(%rdx)
1341 ; AVX2-FCP-NEXT: vmovdqa %ymm4, (%rdx)
1342 ; AVX2-FCP-NEXT: vmovdqa %ymm3, 96(%rdx)
1343 ; AVX2-FCP-NEXT: vmovdqa %ymm7, 32(%rdx)
1344 ; AVX2-FCP-NEXT: vzeroupper
1345 ; AVX2-FCP-NEXT: retq
1346 ;
1347 ; AVX512-LABEL: load_i16_stride2_vf64:
1348 ; AVX512: # %bb.0:
1349 ; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
1350 ; AVX512-NEXT: vmovdqa64 64(%rdi), %zmm1
1351 ; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm2
1352 ; AVX512-NEXT: vmovdqa64 192(%rdi), %zmm3
1353 ; AVX512-NEXT: vpmovdw %zmm1, %ymm4
1354 ; AVX512-NEXT: vpsrld $16, %zmm1, %zmm1
1355 ; AVX512-NEXT: vpsrld $16, %zmm0, %zmm5
1356 ; AVX512-NEXT: vpsrld $16, %zmm3, %zmm6
1357 ; AVX512-NEXT: vpsrld $16, %zmm2, %zmm7
1358 ; AVX512-NEXT: vpmovdw %zmm0, (%rsi)
1359 ; AVX512-NEXT: vmovdqa %ymm4, 32(%rsi)
1360 ; AVX512-NEXT: vpmovdw %zmm2, 64(%rsi)
1361 ; AVX512-NEXT: vpmovdw %zmm3, 96(%rsi)
1362 ; AVX512-NEXT: vpmovdw %zmm7, 64(%rdx)
1363 ; AVX512-NEXT: vpmovdw %zmm6, 96(%rdx)
1364 ; AVX512-NEXT: vpmovdw %zmm5, (%rdx)
1365 ; AVX512-NEXT: vpmovdw %zmm1, 32(%rdx)
1366 ; AVX512-NEXT: vzeroupper
1367 ; AVX512-NEXT: retq
1368 ;
1369 ; AVX512-FCP-LABEL: load_i16_stride2_vf64:
1370 ; AVX512-FCP: # %bb.0:
1371 ; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1372 ; AVX512-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1373 ; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
1374 ; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %zmm3
1375 ; AVX512-FCP-NEXT: vpmovdw %zmm1, %ymm4
1376 ; AVX512-FCP-NEXT: vpsrld $16, %zmm1, %zmm1
1377 ; AVX512-FCP-NEXT: vpsrld $16, %zmm0, %zmm5
1378 ; AVX512-FCP-NEXT: vpsrld $16, %zmm3, %zmm6
1379 ; AVX512-FCP-NEXT: vpsrld $16, %zmm2, %zmm7
1380 ; AVX512-FCP-NEXT: vpmovdw %zmm0, (%rsi)
1381 ; AVX512-FCP-NEXT: vmovdqa %ymm4, 32(%rsi)
1382 ; AVX512-FCP-NEXT: vpmovdw %zmm2, 64(%rsi)
1383 ; AVX512-FCP-NEXT: vpmovdw %zmm3, 96(%rsi)
1384 ; AVX512-FCP-NEXT: vpmovdw %zmm7, 64(%rdx)
1385 ; AVX512-FCP-NEXT: vpmovdw %zmm6, 96(%rdx)
1386 ; AVX512-FCP-NEXT: vpmovdw %zmm5, (%rdx)
1387 ; AVX512-FCP-NEXT: vpmovdw %zmm1, 32(%rdx)
1388 ; AVX512-FCP-NEXT: vzeroupper
1389 ; AVX512-FCP-NEXT: retq
1390 ;
1391 ; AVX512DQ-LABEL: load_i16_stride2_vf64:
1392 ; AVX512DQ: # %bb.0:
1393 ; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
1394 ; AVX512DQ-NEXT: vmovdqa64 64(%rdi), %zmm1
1395 ; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm2
1396 ; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %zmm3
1397 ; AVX512DQ-NEXT: vpmovdw %zmm1, %ymm4
1398 ; AVX512DQ-NEXT: vpsrld $16, %zmm1, %zmm1
1399 ; AVX512DQ-NEXT: vpsrld $16, %zmm0, %zmm5
1400 ; AVX512DQ-NEXT: vpsrld $16, %zmm3, %zmm6
1401 ; AVX512DQ-NEXT: vpsrld $16, %zmm2, %zmm7
1402 ; AVX512DQ-NEXT: vpmovdw %zmm0, (%rsi)
1403 ; AVX512DQ-NEXT: vmovdqa %ymm4, 32(%rsi)
1404 ; AVX512DQ-NEXT: vpmovdw %zmm2, 64(%rsi)
1405 ; AVX512DQ-NEXT: vpmovdw %zmm3, 96(%rsi)
1406 ; AVX512DQ-NEXT: vpmovdw %zmm7, 64(%rdx)
1407 ; AVX512DQ-NEXT: vpmovdw %zmm6, 96(%rdx)
1408 ; AVX512DQ-NEXT: vpmovdw %zmm5, (%rdx)
1409 ; AVX512DQ-NEXT: vpmovdw %zmm1, 32(%rdx)
1410 ; AVX512DQ-NEXT: vzeroupper
1411 ; AVX512DQ-NEXT: retq
1412 ;
1413 ; AVX512DQ-FCP-LABEL: load_i16_stride2_vf64:
1414 ; AVX512DQ-FCP: # %bb.0:
1415 ; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1416 ; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1417 ; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
1418 ; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %zmm3
1419 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm1, %ymm4
1420 ; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm1, %zmm1
1421 ; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm0, %zmm5
1422 ; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm3, %zmm6
1423 ; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm2, %zmm7
1424 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm0, (%rsi)
1425 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm4, 32(%rsi)
1426 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm2, 64(%rsi)
1427 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm3, 96(%rsi)
1428 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm7, 64(%rdx)
1429 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm6, 96(%rdx)
1430 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm5, (%rdx)
1431 ; AVX512DQ-FCP-NEXT: vpmovdw %zmm1, 32(%rdx)
1432 ; AVX512DQ-FCP-NEXT: vzeroupper
1433 ; AVX512DQ-FCP-NEXT: retq
1434 ;
1435 ; AVX512BW-LABEL: load_i16_stride2_vf64:
1436 ; AVX512BW: # %bb.0:
1437 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
1438 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
1439 ; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm2
1440 ; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm3
1441 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
1442 ; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm5
1443 ; AVX512BW-NEXT: vpermt2w %zmm1, %zmm4, %zmm5
1444 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm4
1445 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
1446 ; AVX512BW-NEXT: vpermt2w %zmm1, %zmm6, %zmm0
1447 ; AVX512BW-NEXT: vpermt2w %zmm3, %zmm6, %zmm2
1448 ; AVX512BW-NEXT: vmovdqa64 %zmm4, 64(%rsi)
1449 ; AVX512BW-NEXT: vmovdqa64 %zmm5, (%rsi)
1450 ; AVX512BW-NEXT: vmovdqa64 %zmm2, 64(%rdx)
1451 ; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
1452 ; AVX512BW-NEXT: vzeroupper
1453 ; AVX512BW-NEXT: retq
1454 ;
1455 ; AVX512BW-FCP-LABEL: load_i16_stride2_vf64:
1456 ; AVX512BW-FCP: # %bb.0:
1457 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1458 ; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1459 ; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
1460 ; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm3
1461 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
1462 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm5
1463 ; AVX512BW-FCP-NEXT: vpermt2w %zmm1, %zmm4, %zmm5
1464 ; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm4
1465 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm6 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
1466 ; AVX512BW-FCP-NEXT: vpermt2w %zmm1, %zmm6, %zmm0
1467 ; AVX512BW-FCP-NEXT: vpermt2w %zmm3, %zmm6, %zmm2
1468 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, 64(%rsi)
1469 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, (%rsi)
1470 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, 64(%rdx)
1471 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
1472 ; AVX512BW-FCP-NEXT: vzeroupper
1473 ; AVX512BW-FCP-NEXT: retq
1474 ;
1475 ; AVX512DQ-BW-LABEL: load_i16_stride2_vf64:
1476 ; AVX512DQ-BW: # %bb.0:
1477 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
1478 ; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm1
1479 ; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm2
1480 ; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %zmm3
1481 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
1482 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm5
1483 ; AVX512DQ-BW-NEXT: vpermt2w %zmm1, %zmm4, %zmm5
1484 ; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm4
1485 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
1486 ; AVX512DQ-BW-NEXT: vpermt2w %zmm1, %zmm6, %zmm0
1487 ; AVX512DQ-BW-NEXT: vpermt2w %zmm3, %zmm6, %zmm2
1488 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, 64(%rsi)
1489 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, (%rsi)
1490 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, 64(%rdx)
1491 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, (%rdx)
1492 ; AVX512DQ-BW-NEXT: vzeroupper
1493 ; AVX512DQ-BW-NEXT: retq
1494 ;
1495 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride2_vf64:
1496 ; AVX512DQ-BW-FCP: # %bb.0:
1497 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1498 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1499 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
1500 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm3
1501 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
1502 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm5
1503 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm1, %zmm4, %zmm5
1504 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm4
1505 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm6 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
1506 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm1, %zmm6, %zmm0
1507 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm3, %zmm6, %zmm2
1508 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, 64(%rsi)
1509 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, (%rsi)
1510 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, 64(%rdx)
1511 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
1512 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
1513 ; AVX512DQ-BW-FCP-NEXT: retq
1514 %wide.vec = load <128 x i16>, ptr %in.vec, align 64
1515 %strided.vec0 = shufflevector <128 x i16> %wide.vec, <128 x i16> poison, <64 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62, i32 64, i32 66, i32 68, i32 70, i32 72, i32 74, i32 76, i32 78, i32 80, i32 82, i32 84, i32 86, i32 88, i32 90, i32 92, i32 94, i32 96, i32 98, i32 100, i32 102, i32 104, i32 106, i32 108, i32 110, i32 112, i32 114, i32 116, i32 118, i32 120, i32 122, i32 124, i32 126>
1516 %strided.vec1 = shufflevector <128 x i16> %wide.vec, <128 x i16> poison, <64 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63, i32 65, i32 67, i32 69, i32 71, i32 73, i32 75, i32 77, i32 79, i32 81, i32 83, i32 85, i32 87, i32 89, i32 91, i32 93, i32 95, i32 97, i32 99, i32 101, i32 103, i32 105, i32 107, i32 109, i32 111, i32 113, i32 115, i32 117, i32 119, i32 121, i32 123, i32 125, i32 127>
1517 store <64 x i16> %strided.vec0, ptr %out.vec0, align 64
1518 store <64 x i16> %strided.vec1, ptr %out.vec1, align 64
1519 ret void
1520 }