| blob | 5a7abf8cdf9fbfec71f88dde80a74f6b5403000c |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,FALLBACK0
3 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1,AVX1-ONLY,FALLBACK1
4 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX1,AVX2,AVX2-ONLY,AVX2-SLOW,FALLBACK2
5 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX1,AVX2,AVX2-ONLY,AVX2-FAST,FALLBACK3
6 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX1,AVX2,AVX2-ONLY,AVX2-FAST-PERLANE,FALLBACK4
7 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512F,AVX512-SLOW,AVX512F-SLOW,AVX512F-ONLY-SLOW,FALLBACK5
8 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512F,AVX512-FAST,AVX512F-FAST,AVX512F-ONLY-FAST,FALLBACK6
9 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512F,AVX512-SLOW,AVX512F-SLOW,AVX512DQ-SLOW,FALLBACK7
10 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512F,AVX512-FAST,AVX512F-FAST,AVX512DQ-FAST,FALLBACK8
11 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512BW,AVX512-SLOW,AVX512BW-SLOW,AVX512BW-ONLY-SLOW,FALLBACK9
12 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512BW,AVX512-FAST,AVX512BW-FAST,AVX512BW-ONLY-FAST,FALLBACK10
13 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512BW,AVX512-SLOW,AVX512BW-SLOW,AVX512DQBW-SLOW,FALLBACK11
14 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX512,AVX512BW,AVX512-FAST,AVX512BW-FAST,AVX512DQBW-FAST,FALLBACK12
16 ; These patterns are produced by LoopVectorizer for interleaved loads.
18 define void @load_i16_stride4_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind {
19 ; SSE-LABEL: load_i16_stride4_vf2:
20 ; SSE: # %bb.0:
21 ; SSE-NEXT: movdqa (%rdi), %xmm0
22 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
23 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[0,2,2,3,4,5,6,7]
24 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]
25 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
26 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm0[2,0,2,3,4,5,6,7]
27 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]
28 ; SSE-NEXT: movd %xmm2, (%rsi)
29 ; SSE-NEXT: movd %xmm1, (%rdx)
30 ; SSE-NEXT: movd %xmm3, (%rcx)
31 ; SSE-NEXT: movd %xmm0, (%r8)
32 ; SSE-NEXT: retq
33 ;
34 ; AVX1-LABEL: load_i16_stride4_vf2:
35 ; AVX1: # %bb.0:
36 ; AVX1-NEXT: vmovdqa (%rdi), %xmm0
37 ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
38 ; AVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm1[0,2,2,3,4,5,6,7]
39 ; AVX1-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]
40 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
41 ; AVX1-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[2,0,2,3,4,5,6,7]
42 ; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]
43 ; AVX1-NEXT: vmovd %xmm2, (%rsi)
44 ; AVX1-NEXT: vmovd %xmm1, (%rdx)
45 ; AVX1-NEXT: vmovd %xmm3, (%rcx)
46 ; AVX1-NEXT: vmovd %xmm0, (%r8)
47 ; AVX1-NEXT: retq
48 ;
49 ; AVX512-SLOW-LABEL: load_i16_stride4_vf2:
50 ; AVX512-SLOW: # %bb.0:
51 ; AVX512-SLOW-NEXT: vmovdqa (%rdi), %xmm0
52 ; AVX512-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
53 ; AVX512-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]
54 ; AVX512-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
55 ; AVX512-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]
56 ; AVX512-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
57 ; AVX512-SLOW-NEXT: vpmovqw %xmm0, (%rsi)
58 ; AVX512-SLOW-NEXT: vmovd %xmm1, (%rdx)
59 ; AVX512-SLOW-NEXT: vmovd %xmm3, (%rcx)
60 ; AVX512-SLOW-NEXT: vmovd %xmm2, (%r8)
61 ; AVX512-SLOW-NEXT: retq
62 ;
63 ; AVX512-FAST-LABEL: load_i16_stride4_vf2:
64 ; AVX512-FAST: # %bb.0:
65 ; AVX512-FAST-NEXT: vmovdqa (%rdi), %xmm0
66 ; AVX512-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
67 ; AVX512-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
68 ; AVX512-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]
69 ; AVX512-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
70 ; AVX512-FAST-NEXT: vpmovqw %xmm0, (%rsi)
71 ; AVX512-FAST-NEXT: vmovd %xmm1, (%rdx)
72 ; AVX512-FAST-NEXT: vmovd %xmm3, (%rcx)
73 ; AVX512-FAST-NEXT: vmovd %xmm2, (%r8)
74 ; AVX512-FAST-NEXT: retq
75 %wide.vec = load <8 x i16>, ptr %in.vec, align 64
76 %strided.vec0 = shufflevector <8 x i16> %wide.vec, <8 x i16> poison, <2 x i32> <i32 0, i32 4>
77 %strided.vec1 = shufflevector <8 x i16> %wide.vec, <8 x i16> poison, <2 x i32> <i32 1, i32 5>
78 %strided.vec2 = shufflevector <8 x i16> %wide.vec, <8 x i16> poison, <2 x i32> <i32 2, i32 6>
79 %strided.vec3 = shufflevector <8 x i16> %wide.vec, <8 x i16> poison, <2 x i32> <i32 3, i32 7>
80 store <2 x i16> %strided.vec0, ptr %out.vec0, align 64
81 store <2 x i16> %strided.vec1, ptr %out.vec1, align 64
82 store <2 x i16> %strided.vec2, ptr %out.vec2, align 64
83 store <2 x i16> %strided.vec3, ptr %out.vec3, align 64
84 ret void
85 }
87 define void @load_i16_stride4_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind {
88 ; SSE-LABEL: load_i16_stride4_vf4:
89 ; SSE: # %bb.0:
90 ; SSE-NEXT: movdqa (%rdi), %xmm0
91 ; SSE-NEXT: movdqa 16(%rdi), %xmm1
92 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
93 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm2[0,2,2,3,4,5,6,7]
94 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]
95 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm4[0,2,2,3,4,5,6,7]
96 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
97 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
98 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm4[1,3,2,3,4,5,6,7]
99 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
100 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
101 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[2,0,2,3,4,5,6,7]
102 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
103 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm0[2,0,2,3,4,5,6,7]
104 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
105 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
106 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]
107 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
108 ; SSE-NEXT: movq %xmm5, (%rsi)
109 ; SSE-NEXT: movq %xmm3, (%rdx)
110 ; SSE-NEXT: movq %xmm4, (%rcx)
111 ; SSE-NEXT: movq %xmm0, (%r8)
112 ; SSE-NEXT: retq
113 ;
114 ; AVX1-ONLY-LABEL: load_i16_stride4_vf4:
115 ; AVX1-ONLY: # %bb.0:
116 ; AVX1-ONLY-NEXT: vpxor %xmm0, %xmm0, %xmm0
117 ; AVX1-ONLY-NEXT: vmovdqa (%rdi), %xmm1
118 ; AVX1-ONLY-NEXT: vmovdqa 16(%rdi), %xmm2
119 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1,2,3],xmm2[4],xmm0[5,6,7]
120 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3],xmm1[4],xmm0[5,6,7]
121 ; AVX1-ONLY-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
122 ; AVX1-ONLY-NEXT: vpackusdw %xmm0, %xmm0, %xmm0
123 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
124 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7]
125 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[0,2,2,3]
126 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
127 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
128 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
129 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm4 = xmm2[2,0,2,3,4,5,6,7]
130 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
131 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm5 = xmm1[2,0,2,3,4,5,6,7]
132 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
133 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
134 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
135 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
136 ; AVX1-ONLY-NEXT: vmovq %xmm0, (%rsi)
137 ; AVX1-ONLY-NEXT: vmovq %xmm3, (%rdx)
138 ; AVX1-ONLY-NEXT: vmovq %xmm4, (%rcx)
139 ; AVX1-ONLY-NEXT: vmovq %xmm1, (%r8)
140 ; AVX1-ONLY-NEXT: retq
141 ;
142 ; AVX2-SLOW-LABEL: load_i16_stride4_vf4:
143 ; AVX2-SLOW: # %bb.0:
144 ; AVX2-SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
145 ; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm1
146 ; AVX2-SLOW-NEXT: vmovdqa 16(%rdi), %xmm2
147 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1,2,3],xmm2[4],xmm0[5,6,7]
148 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3],xmm1[4],xmm0[5,6,7]
149 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
150 ; AVX2-SLOW-NEXT: vpackusdw %xmm0, %xmm0, %xmm0
151 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
152 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7]
153 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[0,2,2,3]
154 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
155 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
156 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
157 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm2[2,0,2,3,4,5,6,7]
158 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
159 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm1[2,0,2,3,4,5,6,7]
160 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
161 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
162 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
163 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
164 ; AVX2-SLOW-NEXT: vmovq %xmm0, (%rsi)
165 ; AVX2-SLOW-NEXT: vmovq %xmm3, (%rdx)
166 ; AVX2-SLOW-NEXT: vmovq %xmm4, (%rcx)
167 ; AVX2-SLOW-NEXT: vmovq %xmm1, (%r8)
168 ; AVX2-SLOW-NEXT: retq
169 ;
170 ; AVX2-FAST-LABEL: load_i16_stride4_vf4:
171 ; AVX2-FAST: # %bb.0:
172 ; AVX2-FAST-NEXT: vpxor %xmm0, %xmm0, %xmm0
173 ; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm1
174 ; AVX2-FAST-NEXT: vmovdqa 16(%rdi), %xmm2
175 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1,2,3],xmm2[4],xmm0[5,6,7]
176 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3],xmm1[4],xmm0[5,6,7]
177 ; AVX2-FAST-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
178 ; AVX2-FAST-NEXT: vpackusdw %xmm0, %xmm0, %xmm0
179 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm3 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
180 ; AVX2-FAST-NEXT: vpshufb %xmm3, %xmm2, %xmm4
181 ; AVX2-FAST-NEXT: vpshufb %xmm3, %xmm1, %xmm3
182 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
183 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
184 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm4 = xmm2[2,0,2,3,4,5,6,7]
185 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
186 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm5 = xmm1[2,0,2,3,4,5,6,7]
187 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
188 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
189 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
190 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
191 ; AVX2-FAST-NEXT: vmovq %xmm0, (%rsi)
192 ; AVX2-FAST-NEXT: vmovq %xmm3, (%rdx)
193 ; AVX2-FAST-NEXT: vmovq %xmm4, (%rcx)
194 ; AVX2-FAST-NEXT: vmovq %xmm1, (%r8)
195 ; AVX2-FAST-NEXT: retq
196 ;
197 ; AVX2-FAST-PERLANE-LABEL: load_i16_stride4_vf4:
198 ; AVX2-FAST-PERLANE: # %bb.0:
199 ; AVX2-FAST-PERLANE-NEXT: vpxor %xmm0, %xmm0, %xmm0
200 ; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rdi), %xmm1
201 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm2
202 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1,2,3],xmm2[4],xmm0[5,6,7]
203 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3],xmm1[4],xmm0[5,6,7]
204 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
205 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm0, %xmm0, %xmm0
206 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm3 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
207 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm3, %xmm2, %xmm4
208 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm3, %xmm1, %xmm3
209 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
210 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
211 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm4 = xmm2[2,0,2,3,4,5,6,7]
212 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
213 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm5 = xmm1[2,0,2,3,4,5,6,7]
214 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
215 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
216 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
217 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
218 ; AVX2-FAST-PERLANE-NEXT: vmovq %xmm0, (%rsi)
219 ; AVX2-FAST-PERLANE-NEXT: vmovq %xmm3, (%rdx)
220 ; AVX2-FAST-PERLANE-NEXT: vmovq %xmm4, (%rcx)
221 ; AVX2-FAST-PERLANE-NEXT: vmovq %xmm1, (%r8)
222 ; AVX2-FAST-PERLANE-NEXT: retq
223 ;
224 ; AVX512-LABEL: load_i16_stride4_vf4:
225 ; AVX512: # %bb.0:
226 ; AVX512-NEXT: vmovdqa (%rdi), %ymm0
227 ; AVX512-NEXT: vpsrlq $16, %ymm0, %ymm1
228 ; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm2
229 ; AVX512-NEXT: vpsrlq $48, %ymm0, %ymm3
230 ; AVX512-NEXT: vpmovqw %ymm0, (%rsi)
231 ; AVX512-NEXT: vpmovqw %ymm1, (%rdx)
232 ; AVX512-NEXT: vpmovqw %ymm2, (%rcx)
233 ; AVX512-NEXT: vpmovqw %ymm3, (%r8)
234 ; AVX512-NEXT: vzeroupper
235 ; AVX512-NEXT: retq
236 %wide.vec = load <16 x i16>, ptr %in.vec, align 64
237 %strided.vec0 = shufflevector <16 x i16> %wide.vec, <16 x i16> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
238 %strided.vec1 = shufflevector <16 x i16> %wide.vec, <16 x i16> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
239 %strided.vec2 = shufflevector <16 x i16> %wide.vec, <16 x i16> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
240 %strided.vec3 = shufflevector <16 x i16> %wide.vec, <16 x i16> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
241 store <4 x i16> %strided.vec0, ptr %out.vec0, align 64
242 store <4 x i16> %strided.vec1, ptr %out.vec1, align 64
243 store <4 x i16> %strided.vec2, ptr %out.vec2, align 64
244 store <4 x i16> %strided.vec3, ptr %out.vec3, align 64
245 ret void
246 }
248 define void @load_i16_stride4_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind {
249 ; SSE-LABEL: load_i16_stride4_vf8:
250 ; SSE: # %bb.0:
251 ; SSE-NEXT: movdqa (%rdi), %xmm2
252 ; SSE-NEXT: movdqa 16(%rdi), %xmm3
253 ; SSE-NEXT: movdqa 32(%rdi), %xmm1
254 ; SSE-NEXT: movdqa 48(%rdi), %xmm4
255 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,2,2,3]
256 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[0,2,2,3,4,5,6,7]
257 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[0,2,2,3]
258 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm6[0,2,2,3,4,5,6,7]
259 ; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1]
260 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm4[0,2,2,3]
261 ; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm8[0,1,0,2,4,5,6,7]
262 ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm1[0,2,2,3]
263 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[0,1,0,2,4,5,6,7]
264 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1]
265 ; SSE-NEXT: movsd {{.*#+}} xmm0 = xmm7[0],xmm0[1]
266 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
267 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
268 ; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
269 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm8[0,1,1,3,4,5,6,7]
270 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm10[0,1,1,3,4,5,6,7]
271 ; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
272 ; SSE-NEXT: movsd {{.*#+}} xmm7 = xmm6[0],xmm7[1]
273 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
274 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm3[2,0,2,3,4,5,6,7]
275 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
276 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm2[2,0,2,3,4,5,6,7]
277 ; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
278 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
279 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm4[0,1,2,0,4,5,6,7]
280 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
281 ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm1[0,1,2,0,4,5,6,7]
282 ; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
283 ; SSE-NEXT: movsd {{.*#+}} xmm8 = xmm6[0],xmm8[1]
284 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
285 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
286 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
287 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm4[0,1,3,1,4,5,6,7]
288 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]
289 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
290 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
291 ; SSE-NEXT: movapd %xmm0, (%rsi)
292 ; SSE-NEXT: movapd %xmm7, (%rdx)
293 ; SSE-NEXT: movapd %xmm8, (%rcx)
294 ; SSE-NEXT: movapd %xmm1, (%r8)
295 ; SSE-NEXT: retq
296 ;
297 ; AVX1-ONLY-LABEL: load_i16_stride4_vf8:
298 ; AVX1-ONLY: # %bb.0:
299 ; AVX1-ONLY-NEXT: vpxor %xmm0, %xmm0, %xmm0
300 ; AVX1-ONLY-NEXT: vmovdqa (%rdi), %xmm1
301 ; AVX1-ONLY-NEXT: vmovdqa 16(%rdi), %xmm2
302 ; AVX1-ONLY-NEXT: vmovdqa 32(%rdi), %xmm3
303 ; AVX1-ONLY-NEXT: vmovdqa 48(%rdi), %xmm4
304 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm0[1,2,3],xmm4[4],xmm0[5,6,7]
305 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0],xmm0[1,2,3],xmm3[4],xmm0[5,6,7]
306 ; AVX1-ONLY-NEXT: vpackusdw %xmm5, %xmm6, %xmm5
307 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm6 = xmm2[0],xmm0[1,2,3],xmm2[4],xmm0[5,6,7]
308 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3],xmm1[4],xmm0[5,6,7]
309 ; AVX1-ONLY-NEXT: vpackusdw %xmm6, %xmm0, %xmm0
310 ; AVX1-ONLY-NEXT: vpackusdw %xmm5, %xmm0, %xmm0
311 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[0,2,2,3]
312 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]
313 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[0,2,2,3]
314 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
315 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
316 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[0,2,2,3]
317 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
318 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[0,2,2,3]
319 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]
320 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
321 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3],xmm5[4,5,6,7]
322 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
323 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm6 = xmm4[0,1,2,0,4,5,6,7]
324 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
325 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm7 = xmm3[0,1,2,0,4,5,6,7]
326 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
327 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
328 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[2,0,2,3,4,5,6,7]
329 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
330 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm8 = xmm1[2,0,2,3,4,5,6,7]
331 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
332 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2,3],xmm6[4,5,6,7]
333 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
334 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
335 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
336 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
337 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
338 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
339 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5,6,7]
340 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, (%rsi)
341 ; AVX1-ONLY-NEXT: vmovdqa %xmm5, (%rdx)
342 ; AVX1-ONLY-NEXT: vmovdqa %xmm6, (%rcx)
343 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, (%r8)
344 ; AVX1-ONLY-NEXT: retq
345 ;
346 ; AVX2-SLOW-LABEL: load_i16_stride4_vf8:
347 ; AVX2-SLOW: # %bb.0:
348 ; AVX2-SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
349 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
350 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
351 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
352 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
353 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm2
354 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
355 ; AVX2-SLOW-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
356 ; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm1
357 ; AVX2-SLOW-NEXT: vmovdqa 16(%rdi), %xmm2
358 ; AVX2-SLOW-NEXT: vmovdqa 32(%rdi), %xmm3
359 ; AVX2-SLOW-NEXT: vmovdqa 48(%rdi), %xmm4
360 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[0,2,2,3]
361 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]
362 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[0,2,2,3]
363 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
364 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
365 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[0,2,2,3]
366 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
367 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[0,2,2,3]
368 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]
369 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
370 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
371 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
372 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm4[0,1,2,0,4,5,6,7]
373 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
374 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm3[0,1,2,0,4,5,6,7]
375 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
376 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
377 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[2,0,2,3,4,5,6,7]
378 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
379 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm1[2,0,2,3,4,5,6,7]
380 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
381 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
382 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
383 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
384 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
385 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
386 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
387 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
388 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]
389 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, (%rsi)
390 ; AVX2-SLOW-NEXT: vmovdqa %xmm5, (%rdx)
391 ; AVX2-SLOW-NEXT: vmovdqa %xmm6, (%rcx)
392 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, (%r8)
393 ; AVX2-SLOW-NEXT: vzeroupper
394 ; AVX2-SLOW-NEXT: retq
395 ;
396 ; AVX2-FAST-LABEL: load_i16_stride4_vf8:
397 ; AVX2-FAST: # %bb.0:
398 ; AVX2-FAST-NEXT: vpxor %xmm0, %xmm0, %xmm0
399 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
400 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
401 ; AVX2-FAST-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
402 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
403 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm2
404 ; AVX2-FAST-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
405 ; AVX2-FAST-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
406 ; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm1
407 ; AVX2-FAST-NEXT: vmovdqa 16(%rdi), %xmm2
408 ; AVX2-FAST-NEXT: vmovdqa 32(%rdi), %xmm3
409 ; AVX2-FAST-NEXT: vmovdqa 48(%rdi), %xmm4
410 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]
411 ; AVX2-FAST-NEXT: vpshufb %xmm5, %xmm4, %xmm6
412 ; AVX2-FAST-NEXT: vpshufb %xmm5, %xmm3, %xmm5
413 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
414 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm6 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
415 ; AVX2-FAST-NEXT: vpshufb %xmm6, %xmm2, %xmm7
416 ; AVX2-FAST-NEXT: vpshufb %xmm6, %xmm1, %xmm6
417 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
418 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
419 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
420 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm6 = xmm4[0,1,2,0,4,5,6,7]
421 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
422 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm7 = xmm3[0,1,2,0,4,5,6,7]
423 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
424 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
425 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[2,0,2,3,4,5,6,7]
426 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
427 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm8 = xmm1[2,0,2,3,4,5,6,7]
428 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
429 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
430 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
431 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
432 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
433 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
434 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
435 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
436 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]
437 ; AVX2-FAST-NEXT: vmovdqa %xmm0, (%rsi)
438 ; AVX2-FAST-NEXT: vmovdqa %xmm5, (%rdx)
439 ; AVX2-FAST-NEXT: vmovdqa %xmm6, (%rcx)
440 ; AVX2-FAST-NEXT: vmovdqa %xmm1, (%r8)
441 ; AVX2-FAST-NEXT: vzeroupper
442 ; AVX2-FAST-NEXT: retq
443 ;
444 ; AVX2-FAST-PERLANE-LABEL: load_i16_stride4_vf8:
445 ; AVX2-FAST-PERLANE: # %bb.0:
446 ; AVX2-FAST-PERLANE-NEXT: vpxor %xmm0, %xmm0, %xmm0
447 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
448 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm1, %xmm2
449 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
450 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
451 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm2
452 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
453 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
454 ; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rdi), %xmm1
455 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm2
456 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %xmm3
457 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 48(%rdi), %xmm4
458 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]
459 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm5, %xmm4, %xmm6
460 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm5, %xmm3, %xmm5
461 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
462 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm6 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
463 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm2, %xmm7
464 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm1, %xmm6
465 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
466 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
467 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
468 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm6 = xmm4[0,1,2,0,4,5,6,7]
469 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
470 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm7 = xmm3[0,1,2,0,4,5,6,7]
471 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
472 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
473 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[2,0,2,3,4,5,6,7]
474 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
475 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm8 = xmm1[2,0,2,3,4,5,6,7]
476 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
477 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
478 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
479 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
480 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
481 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
482 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
483 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
484 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]
485 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, (%rsi)
486 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm5, (%rdx)
487 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm6, (%rcx)
488 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, (%r8)
489 ; AVX2-FAST-PERLANE-NEXT: vzeroupper
490 ; AVX2-FAST-PERLANE-NEXT: retq
491 ;
492 ; AVX512-LABEL: load_i16_stride4_vf8:
493 ; AVX512: # %bb.0:
494 ; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
495 ; AVX512-NEXT: vpsrlq $16, %zmm0, %zmm1
496 ; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm2
497 ; AVX512-NEXT: vpsrlq $48, %zmm0, %zmm3
498 ; AVX512-NEXT: vpmovqw %zmm0, (%rsi)
499 ; AVX512-NEXT: vpmovqw %zmm1, (%rdx)
500 ; AVX512-NEXT: vpmovqw %zmm2, (%rcx)
501 ; AVX512-NEXT: vpmovqw %zmm3, (%r8)
502 ; AVX512-NEXT: vzeroupper
503 ; AVX512-NEXT: retq
504 %wide.vec = load <32 x i16>, ptr %in.vec, align 64
505 %strided.vec0 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
506 %strided.vec1 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
507 %strided.vec2 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
508 %strided.vec3 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
509 store <8 x i16> %strided.vec0, ptr %out.vec0, align 64
510 store <8 x i16> %strided.vec1, ptr %out.vec1, align 64
511 store <8 x i16> %strided.vec2, ptr %out.vec2, align 64
512 store <8 x i16> %strided.vec3, ptr %out.vec3, align 64
513 ret void
514 }
516 define void @load_i16_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind {
517 ; SSE-LABEL: load_i16_stride4_vf16:
518 ; SSE: # %bb.0:
519 ; SSE-NEXT: movdqa 96(%rdi), %xmm4
520 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
521 ; SSE-NEXT: movdqa 64(%rdi), %xmm1
522 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
523 ; SSE-NEXT: movdqa 80(%rdi), %xmm3
524 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
525 ; SSE-NEXT: movdqa (%rdi), %xmm8
526 ; SSE-NEXT: movdqa 16(%rdi), %xmm10
527 ; SSE-NEXT: movdqa 32(%rdi), %xmm7
528 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
529 ; SSE-NEXT: movdqa 48(%rdi), %xmm6
530 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm10[0,2,2,3]
531 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[0,2,2,3,4,5,6,7]
532 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,2,2,3]
533 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm9[0,2,2,3,4,5,6,7]
534 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
535 ; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm6[0,2,2,3]
536 ; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm12[0,1,0,2,4,5,6,7]
537 ; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm7[0,2,2,3]
538 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm13[0,1,0,2,4,5,6,7]
539 ; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm11[0],xmm7[1],xmm11[1]
540 ; SSE-NEXT: movsd {{.*#+}} xmm7 = xmm2[0],xmm7[1]
541 ; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm3[0,2,2,3]
542 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm14[0,2,2,3,4,5,6,7]
543 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm1[0,2,2,3]
544 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm15[0,2,2,3,4,5,6,7]
545 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
546 ; SSE-NEXT: movdqa 112(%rdi), %xmm11
547 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm11[0,2,2,3]
548 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[0,1,0,2,4,5,6,7]
549 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
550 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[0,1,0,2,4,5,6,7]
551 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
552 ; SSE-NEXT: movsd {{.*#+}} xmm4 = xmm0[0],xmm4[1]
553 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[1,3,2,3,4,5,6,7]
554 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm9[1,3,2,3,4,5,6,7]
555 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
556 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm12[0,1,1,3,4,5,6,7]
557 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm13[0,1,1,3,4,5,6,7]
558 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
559 ; SSE-NEXT: movsd {{.*#+}} xmm5 = xmm1[0],xmm5[1]
560 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm14[1,3,2,3,4,5,6,7]
561 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm15[1,3,2,3,4,5,6,7]
562 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
563 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[0,1,1,3,4,5,6,7]
564 ; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm2[0,1,1,3,4,5,6,7]
565 ; SSE-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]
566 ; SSE-NEXT: movsd {{.*#+}} xmm9 = xmm1[0],xmm9[1]
567 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm10[3,1,2,3]
568 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[2,0,2,3,4,5,6,7]
569 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm8[3,1,2,3]
570 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]
571 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
572 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[3,1,2,3]
573 ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm1[0,1,2,0,4,5,6,7]
574 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
575 ; SSE-NEXT: # xmm10 = mem[3,1,2,3]
576 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm10[0,1,2,0,4,5,6,7]
577 ; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
578 ; SSE-NEXT: movsd {{.*#+}} xmm6 = xmm3[0],xmm6[1]
579 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
580 ; SSE-NEXT: # xmm3 = mem[3,1,2,3]
581 ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm3[2,0,2,3,4,5,6,7]
582 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
583 ; SSE-NEXT: # xmm12 = mem[3,1,2,3]
584 ; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm12[2,0,2,3,4,5,6,7]
585 ; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm8[0],xmm13[1],xmm8[1]
586 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm11[3,1,2,3]
587 ; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm8[0,1,2,0,4,5,6,7]
588 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
589 ; SSE-NEXT: # xmm14 = mem[3,1,2,3]
590 ; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm14[0,1,2,0,4,5,6,7]
591 ; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm11[0],xmm15[1],xmm11[1]
592 ; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm13[0],xmm15[1]
593 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]
594 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
595 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
596 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm1[0,1,3,1,4,5,6,7]
597 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm10[0,1,3,1,4,5,6,7]
598 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
599 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
600 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[3,1,2,3,4,5,6,7]
601 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm12[3,1,2,3,4,5,6,7]
602 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
603 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm8[0,1,3,1,4,5,6,7]
604 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm14[0,1,3,1,4,5,6,7]
605 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
606 ; SSE-NEXT: movsd {{.*#+}} xmm3 = xmm2[0],xmm3[1]
607 ; SSE-NEXT: movapd %xmm4, 16(%rsi)
608 ; SSE-NEXT: movapd %xmm7, (%rsi)
609 ; SSE-NEXT: movapd %xmm9, 16(%rdx)
610 ; SSE-NEXT: movapd %xmm5, (%rdx)
611 ; SSE-NEXT: movapd %xmm15, 16(%rcx)
612 ; SSE-NEXT: movapd %xmm6, (%rcx)
613 ; SSE-NEXT: movapd %xmm3, 16(%r8)
614 ; SSE-NEXT: movapd %xmm1, (%r8)
615 ; SSE-NEXT: retq
616 ;
617 ; AVX1-ONLY-LABEL: load_i16_stride4_vf16:
618 ; AVX1-ONLY: # %bb.0:
619 ; AVX1-ONLY-NEXT: vpxor %xmm1, %xmm1, %xmm1
620 ; AVX1-ONLY-NEXT: vmovdqa 112(%rdi), %xmm5
621 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm5[0],xmm1[1,2,3],xmm5[4],xmm1[5,6,7]
622 ; AVX1-ONLY-NEXT: vmovdqa 96(%rdi), %xmm6
623 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm6[0],xmm1[1,2,3],xmm6[4],xmm1[5,6,7]
624 ; AVX1-ONLY-NEXT: vpackusdw %xmm0, %xmm2, %xmm0
625 ; AVX1-ONLY-NEXT: vmovdqa 80(%rdi), %xmm7
626 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm7[0],xmm1[1,2,3],xmm7[4],xmm1[5,6,7]
627 ; AVX1-ONLY-NEXT: vmovdqa 64(%rdi), %xmm8
628 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm8[0],xmm1[1,2,3],xmm8[4],xmm1[5,6,7]
629 ; AVX1-ONLY-NEXT: vpackusdw %xmm2, %xmm3, %xmm2
630 ; AVX1-ONLY-NEXT: vpackusdw %xmm0, %xmm2, %xmm0
631 ; AVX1-ONLY-NEXT: vmovdqa (%rdi), %xmm2
632 ; AVX1-ONLY-NEXT: vmovdqa 16(%rdi), %xmm3
633 ; AVX1-ONLY-NEXT: vmovdqa 32(%rdi), %xmm4
634 ; AVX1-ONLY-NEXT: vmovdqa 48(%rdi), %xmm9
635 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm10 = xmm9[0],xmm1[1,2,3],xmm9[4],xmm1[5,6,7]
636 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm11 = xmm4[0],xmm1[1,2,3],xmm4[4],xmm1[5,6,7]
637 ; AVX1-ONLY-NEXT: vpackusdw %xmm10, %xmm11, %xmm10
638 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm11 = xmm3[0],xmm1[1,2,3],xmm3[4],xmm1[5,6,7]
639 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3],xmm2[4],xmm1[5,6,7]
640 ; AVX1-ONLY-NEXT: vpackusdw %xmm11, %xmm1, %xmm1
641 ; AVX1-ONLY-NEXT: vpackusdw %xmm10, %xmm1, %xmm1
642 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm10 = xmm5[0,2,2,3]
643 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]
644 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm11 = xmm6[0,2,2,3]
645 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]
646 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
647 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm11 = xmm7[0,2,2,3]
648 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]
649 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm12 = xmm8[0,2,2,3]
650 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]
651 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
652 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3],xmm10[4,5,6,7]
653 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm10, %ymm0, %ymm10
654 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm11 = xmm9[0,2,2,3]
655 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]
656 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm12 = xmm4[0,2,2,3]
657 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[0,1,1,3,4,5,6,7]
658 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
659 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm12 = xmm3[0,2,2,3]
660 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]
661 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm13 = xmm2[0,2,2,3]
662 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[1,3,2,3,4,5,6,7]
663 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]
664 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm11 = xmm12[0,1,2,3],xmm11[4,5,6,7]
665 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
666 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
667 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm5[0,1,2,0,4,5,6,7]
668 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
669 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm12 = xmm6[0,1,2,0,4,5,6,7]
670 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
671 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
672 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm12 = xmm7[2,0,2,3,4,5,6,7]
673 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,1,2,3]
674 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm13 = xmm8[2,0,2,3,4,5,6,7]
675 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]
676 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm11 = xmm12[0,1,2,3],xmm11[4,5,6,7]
677 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm11, %ymm0, %ymm11
678 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[3,1,2,3]
679 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm12 = xmm9[0,1,2,0,4,5,6,7]
680 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
681 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm13 = xmm4[0,1,2,0,4,5,6,7]
682 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]
683 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
684 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm13 = xmm3[2,0,2,3,4,5,6,7]
685 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
686 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm14 = xmm2[2,0,2,3,4,5,6,7]
687 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
688 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm12 = xmm13[0,1,2,3],xmm12[4,5,6,7]
689 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm11 = ymm12[0,1,2,3],ymm11[4,5,6,7]
690 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
691 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,1,4,5,6,7]
692 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
693 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm6 = xmm7[3,1,2,3,4,5,6,7]
694 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm7 = xmm8[3,1,2,3,4,5,6,7]
695 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
696 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3],xmm5[4,5,6,7]
697 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5
698 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm6 = xmm9[0,1,3,1,4,5,6,7]
699 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
700 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
701 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
702 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
703 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
704 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm4[4,5,6,7]
705 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4,5,6,7]
706 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, (%rsi)
707 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, 16(%rsi)
708 ; AVX1-ONLY-NEXT: vmovaps %ymm10, (%rdx)
709 ; AVX1-ONLY-NEXT: vmovaps %ymm11, (%rcx)
710 ; AVX1-ONLY-NEXT: vmovaps %ymm2, (%r8)
711 ; AVX1-ONLY-NEXT: vzeroupper
712 ; AVX1-ONLY-NEXT: retq
713 ;
714 ; AVX2-SLOW-LABEL: load_i16_stride4_vf16:
715 ; AVX2-SLOW: # %bb.0:
716 ; AVX2-SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
717 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
718 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
719 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
720 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
721 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
722 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
723 ; AVX2-SLOW-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
724 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
725 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
726 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
727 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
728 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
729 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
730 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm3
731 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
732 ; AVX2-SLOW-NEXT: vpackusdw %xmm0, %xmm0, %xmm0
733 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
734 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
735 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
736 ; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm1
737 ; AVX2-SLOW-NEXT: vmovdqa 16(%rdi), %xmm2
738 ; AVX2-SLOW-NEXT: vmovdqa 32(%rdi), %xmm3
739 ; AVX2-SLOW-NEXT: vmovdqa 48(%rdi), %xmm5
740 ; AVX2-SLOW-NEXT: vmovdqa 112(%rdi), %xmm6
741 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
742 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7]
743 ; AVX2-SLOW-NEXT: vmovdqa 96(%rdi), %xmm7
744 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[0,2,2,3]
745 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7]
746 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
747 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
748 ; AVX2-SLOW-NEXT: vmovdqa 80(%rdi), %xmm8
749 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm8[0,2,2,3]
750 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
751 ; AVX2-SLOW-NEXT: vmovdqa 64(%rdi), %xmm10
752 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm10[0,2,2,3]
753 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]
754 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
755 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
756 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5],ymm4[6,7]
757 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm5[0,2,2,3]
758 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
759 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm3[0,2,2,3]
760 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]
761 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
762 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm2[0,2,2,3]
763 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]
764 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm1[0,2,2,3]
765 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]
766 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
767 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1],xmm9[2,3]
768 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3],ymm4[4,5,6,7]
769 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
770 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm6[0,1,2,0,4,5,6,7]
771 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
772 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm7[0,1,2,0,4,5,6,7]
773 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
774 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
775 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,1,2,3]
776 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm8[2,0,2,3,4,5,6,7]
777 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,1,2,3]
778 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm10[2,0,2,3,4,5,6,7]
779 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
780 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
781 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5],ymm9[6,7]
782 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
783 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm5[0,1,2,0,4,5,6,7]
784 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
785 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm3[0,1,2,0,4,5,6,7]
786 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
787 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
788 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm2[2,0,2,3,4,5,6,7]
789 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
790 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm13 = xmm1[2,0,2,3,4,5,6,7]
791 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]
792 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
793 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3],ymm9[4,5,6,7]
794 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,1,4,5,6,7]
795 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]
796 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
797 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
798 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm8[3,1,2,3,4,5,6,7]
799 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm10[3,1,2,3,4,5,6,7]
800 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
801 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
802 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]
803 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
804 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
805 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
806 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
807 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
808 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
809 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]
810 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]
811 ; AVX2-SLOW-NEXT: vmovdqa %ymm0, (%rsi)
812 ; AVX2-SLOW-NEXT: vmovdqa %ymm4, (%rdx)
813 ; AVX2-SLOW-NEXT: vmovdqa %ymm9, (%rcx)
814 ; AVX2-SLOW-NEXT: vmovdqa %ymm1, (%r8)
815 ; AVX2-SLOW-NEXT: vzeroupper
816 ; AVX2-SLOW-NEXT: retq
817 ;
818 ; AVX2-FAST-LABEL: load_i16_stride4_vf16:
819 ; AVX2-FAST: # %bb.0:
820 ; AVX2-FAST-NEXT: vmovdqa 64(%rdi), %ymm1
821 ; AVX2-FAST-NEXT: vmovdqa 96(%rdi), %ymm2
822 ; AVX2-FAST-NEXT: vpxor %xmm0, %xmm0, %xmm0
823 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
824 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm3, %xmm4
825 ; AVX2-FAST-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
826 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
827 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm4
828 ; AVX2-FAST-NEXT: vpackusdw %xmm4, %xmm0, %xmm0
829 ; AVX2-FAST-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
830 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm3 = [0,2,2,3,0,2,4,6]
831 ; AVX2-FAST-NEXT: vpermd %ymm2, %ymm3, %ymm4
832 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} ymm5 = [16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29]
833 ; AVX2-FAST-NEXT: vpshufb %ymm5, %ymm4, %ymm6
834 ; AVX2-FAST-NEXT: vpermd %ymm1, %ymm3, %ymm7
835 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm8 = <0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u>
836 ; AVX2-FAST-NEXT: vpshufb %ymm8, %ymm7, %ymm3
837 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm6[6,7]
838 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
839 ; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm3
840 ; AVX2-FAST-NEXT: vmovdqa 16(%rdi), %xmm6
841 ; AVX2-FAST-NEXT: vmovdqa 32(%rdi), %xmm9
842 ; AVX2-FAST-NEXT: vmovdqa 48(%rdi), %xmm10
843 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm11 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]
844 ; AVX2-FAST-NEXT: vpshufb %xmm11, %xmm10, %xmm12
845 ; AVX2-FAST-NEXT: vpshufb %xmm11, %xmm9, %xmm11
846 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
847 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm12 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
848 ; AVX2-FAST-NEXT: vpshufb %xmm12, %xmm6, %xmm13
849 ; AVX2-FAST-NEXT: vpshufb %xmm12, %xmm3, %xmm12
850 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
851 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]
852 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} ymm12 = [18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
853 ; AVX2-FAST-NEXT: vpshufb %ymm12, %ymm4, %ymm4
854 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm13 = <2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,u,u,u,u,u,u,u,u>
855 ; AVX2-FAST-NEXT: vpshufb %ymm13, %ymm7, %ymm7
856 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm7[0,1,2,3,4,5],ymm4[6,7]
857 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm4[4,5,6,7]
858 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm7 = [1,3,2,3,1,3,5,7]
859 ; AVX2-FAST-NEXT: vpermd %ymm2, %ymm7, %ymm2
860 ; AVX2-FAST-NEXT: vpshufb %ymm5, %ymm2, %ymm5
861 ; AVX2-FAST-NEXT: vpermd %ymm1, %ymm7, %ymm1
862 ; AVX2-FAST-NEXT: vpshufb %ymm8, %ymm1, %ymm7
863 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3,4,5],ymm5[6,7]
864 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm7 = xmm10[3,1,2,3]
865 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm8 = xmm7[0,1,2,0,4,5,6,7]
866 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[3,1,2,3]
867 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm10 = xmm9[0,1,2,0,4,5,6,7]
868 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm10[0],xmm8[0],xmm10[1],xmm8[1]
869 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
870 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm10 = xmm6[2,0,2,3,4,5,6,7]
871 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
872 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm11 = xmm3[2,0,2,3,4,5,6,7]
873 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
874 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3]
875 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7]
876 ; AVX2-FAST-NEXT: vpshufb %ymm12, %ymm2, %ymm2
877 ; AVX2-FAST-NEXT: vpshufb %ymm13, %ymm1, %ymm1
878 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
879 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm7[0,1,3,1,4,5,6,7]
880 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm7 = xmm9[0,1,3,1,4,5,6,7]
881 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
882 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]
883 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
884 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
885 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
886 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
887 ; AVX2-FAST-NEXT: vmovdqa %ymm0, (%rsi)
888 ; AVX2-FAST-NEXT: vmovdqa %ymm4, (%rdx)
889 ; AVX2-FAST-NEXT: vmovdqa %ymm5, (%rcx)
890 ; AVX2-FAST-NEXT: vmovdqa %ymm1, (%r8)
891 ; AVX2-FAST-NEXT: vzeroupper
892 ; AVX2-FAST-NEXT: retq
893 ;
894 ; AVX2-FAST-PERLANE-LABEL: load_i16_stride4_vf16:
895 ; AVX2-FAST-PERLANE: # %bb.0:
896 ; AVX2-FAST-PERLANE-NEXT: vpxor %xmm0, %xmm0, %xmm0
897 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
898 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm1, %xmm2
899 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
900 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
901 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
902 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
903 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
904 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
905 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
906 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
907 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
908 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
909 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
910 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm3
911 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
912 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm0, %xmm0, %xmm0
913 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
914 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
915 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
916 ; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rdi), %xmm1
917 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm3
918 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %xmm4
919 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 48(%rdi), %xmm5
920 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]
921 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 112(%rdi), %xmm6
922 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm2, %xmm6, %xmm7
923 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 96(%rdi), %xmm8
924 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm2, %xmm8, %xmm9
925 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1]
926 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
927 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm9 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
928 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 80(%rdi), %xmm10
929 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm10, %xmm11
930 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 64(%rdi), %xmm12
931 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm12, %xmm13
932 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]
933 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
934 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm7 = ymm11[0,1,2,3,4,5],ymm7[6,7]
935 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm2, %xmm5, %xmm11
936 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm2, %xmm4, %xmm2
937 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm11[0],xmm2[1],xmm11[1]
938 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm3, %xmm11
939 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm9, %xmm1, %xmm9
940 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
941 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm2 = xmm9[0,1],xmm2[2,3]
942 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm7[4,5,6,7]
943 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
944 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm7 = xmm6[0,1,2,0,4,5,6,7]
945 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,1,2,3]
946 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm9 = xmm8[0,1,2,0,4,5,6,7]
947 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1]
948 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
949 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm9 = xmm10[3,1,2,3]
950 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm10 = xmm9[2,0,2,3,4,5,6,7]
951 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm11 = xmm12[3,1,2,3]
952 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm12 = xmm11[2,0,2,3,4,5,6,7]
953 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]
954 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
955 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm7 = ymm10[0,1,2,3,4,5],ymm7[6,7]
956 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
957 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm10 = xmm5[0,1,2,0,4,5,6,7]
958 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
959 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm12 = xmm4[0,1,2,0,4,5,6,7]
960 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]
961 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
962 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm12 = xmm3[2,0,2,3,4,5,6,7]
963 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
964 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm13 = xmm1[2,0,2,3,4,5,6,7]
965 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]
966 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm10 = xmm12[0,1],xmm10[2,3]
967 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm7 = ymm10[0,1,2,3],ymm7[4,5,6,7]
968 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,1,4,5,6,7]
969 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,3,1,4,5,6,7]
970 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]
971 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
972 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm8 = xmm9[3,1,2,3,4,5,6,7]
973 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm9 = xmm11[3,1,2,3,4,5,6,7]
974 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
975 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
976 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3,4,5],ymm6[6,7]
977 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
978 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
979 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
980 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
981 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
982 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
983 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3]
984 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]
985 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm0, (%rsi)
986 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm2, (%rdx)
987 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm7, (%rcx)
988 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm1, (%r8)
989 ; AVX2-FAST-PERLANE-NEXT: vzeroupper
990 ; AVX2-FAST-PERLANE-NEXT: retq
991 ;
992 ; AVX512F-SLOW-LABEL: load_i16_stride4_vf16:
993 ; AVX512F-SLOW: # %bb.0:
994 ; AVX512F-SLOW-NEXT: vmovdqa 64(%rdi), %ymm0
995 ; AVX512F-SLOW-NEXT: vpmovqw %ymm0, %xmm0
996 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
997 ; AVX512F-SLOW-NEXT: vmovdqa 112(%rdi), %xmm1
998 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
999 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,1,0,2,4,5,6,7]
1000 ; AVX512F-SLOW-NEXT: vmovdqa 96(%rdi), %xmm4
1001 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[0,2,2,3]
1002 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm5[0,1,0,2,4,5,6,7]
1003 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
1004 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
1005 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6,7]
1006 ; AVX512F-SLOW-NEXT: vmovdqa64 (%rdi), %zmm3
1007 ; AVX512F-SLOW-NEXT: vpmovqw %zmm3, %xmm6
1008 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
1009 ; AVX512F-SLOW-NEXT: vmovdqa 64(%rdi), %xmm6
1010 ; AVX512F-SLOW-NEXT: vmovdqa 80(%rdi), %xmm7
1011 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
1012 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]
1013 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
1014 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
1015 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm7[0,2,2,3]
1016 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
1017 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[0,2,2,3]
1018 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]
1019 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
1020 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
1021 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
1022 ; AVX512F-SLOW-NEXT: vpsrlq $16, %zmm3, %zmm5
1023 ; AVX512F-SLOW-NEXT: vpmovqw %zmm5, %xmm5
1024 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
1025 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
1026 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm1[0,1,2,0,4,5,6,7]
1027 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
1028 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm4[0,1,2,0,4,5,6,7]
1029 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
1030 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
1031 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
1032 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm7[2,0,2,3,4,5,6,7]
1033 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
1034 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm6[2,0,2,3,4,5,6,7]
1035 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
1036 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
1037 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3,4,5],ymm5[6,7]
1038 ; AVX512F-SLOW-NEXT: vpsrlq $32, %zmm3, %zmm8
1039 ; AVX512F-SLOW-NEXT: vpmovqw %zmm8, %xmm8
1040 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7]
1041 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]
1042 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
1043 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
1044 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
1045 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm7[3,1,2,3,4,5,6,7]
1046 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]
1047 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
1048 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
1049 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
1050 ; AVX512F-SLOW-NEXT: vpsrlq $48, %zmm3, %zmm3
1051 ; AVX512F-SLOW-NEXT: vpmovqw %zmm3, %xmm3
1052 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
1053 ; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rsi)
1054 ; AVX512F-SLOW-NEXT: vmovdqa %ymm2, (%rdx)
1055 ; AVX512F-SLOW-NEXT: vmovdqa %ymm5, (%rcx)
1056 ; AVX512F-SLOW-NEXT: vmovdqa %ymm1, (%r8)
1057 ; AVX512F-SLOW-NEXT: vzeroupper
1058 ; AVX512F-SLOW-NEXT: retq
1059 ;
1060 ; AVX512F-FAST-LABEL: load_i16_stride4_vf16:
1061 ; AVX512F-FAST: # %bb.0:
1062 ; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0,16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0]
1063 ; AVX512F-FAST-NEXT: # ymm0 = mem[0,1,0,1]
1064 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm1 = [0,2,2,3,0,2,4,6]
1065 ; AVX512F-FAST-NEXT: vmovdqa 96(%rdi), %ymm2
1066 ; AVX512F-FAST-NEXT: vpermd %ymm2, %ymm1, %ymm3
1067 ; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm3, %ymm4
1068 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm5 = <0,1,4,5,u,u,u,u,8,9,12,13,12,13,14,15,16,17,20,21,u,u,u,u,24,25,28,29,u,u,u,u>
1069 ; AVX512F-FAST-NEXT: vmovdqa 64(%rdi), %ymm6
1070 ; AVX512F-FAST-NEXT: vpermd %ymm6, %ymm1, %ymm1
1071 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm1, %ymm7
1072 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm8 = [0,2,2,3,4,6,12,14]
1073 ; AVX512F-FAST-NEXT: vpermt2d %ymm4, %ymm8, %ymm7
1074 ; AVX512F-FAST-NEXT: vmovdqa64 (%rdi), %zmm4
1075 ; AVX512F-FAST-NEXT: vpmovqw %zmm4, %xmm9
1076 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3],ymm7[4,5,6,7]
1077 ; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm9 = [18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
1078 ; AVX512F-FAST-NEXT: vpshufb %ymm9, %ymm3, %ymm3
1079 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm10 = <2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,u,u,u,u,u,u,u,u>
1080 ; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm1, %ymm1
1081 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm3[6,7]
1082 ; AVX512F-FAST-NEXT: vpsrlq $16, %zmm4, %zmm3
1083 ; AVX512F-FAST-NEXT: vpmovqw %zmm3, %xmm3
1084 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
1085 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm3 = [1,3,2,3,1,3,5,7]
1086 ; AVX512F-FAST-NEXT: vpermd %ymm2, %ymm3, %ymm2
1087 ; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm2, %ymm0
1088 ; AVX512F-FAST-NEXT: vpermd %ymm6, %ymm3, %ymm3
1089 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm3, %ymm5
1090 ; AVX512F-FAST-NEXT: vpermt2d %ymm0, %ymm8, %ymm5
1091 ; AVX512F-FAST-NEXT: vpsrlq $32, %zmm4, %zmm0
1092 ; AVX512F-FAST-NEXT: vpmovqw %zmm0, %xmm0
1093 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
1094 ; AVX512F-FAST-NEXT: vpshufb %ymm9, %ymm2, %ymm2
1095 ; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm3, %ymm3
1096 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
1097 ; AVX512F-FAST-NEXT: vpsrlq $48, %zmm4, %zmm3
1098 ; AVX512F-FAST-NEXT: vpmovqw %zmm3, %xmm3
1099 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
1100 ; AVX512F-FAST-NEXT: vmovdqa %ymm7, (%rsi)
1101 ; AVX512F-FAST-NEXT: vmovdqa %ymm1, (%rdx)
1102 ; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rcx)
1103 ; AVX512F-FAST-NEXT: vmovdqa %ymm2, (%r8)
1104 ; AVX512F-FAST-NEXT: vzeroupper
1105 ; AVX512F-FAST-NEXT: retq
1106 ;
1107 ; AVX512BW-LABEL: load_i16_stride4_vf16:
1108 ; AVX512BW: # %bb.0:
1109 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm0 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]
1110 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm1
1111 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm2
1112 ; AVX512BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm0
1113 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm3 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]
1114 ; AVX512BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm3
1115 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm4 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]
1116 ; AVX512BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm4
1117 ; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm5 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]
1118 ; AVX512BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm5
1119 ; AVX512BW-NEXT: vmovdqa %ymm0, (%rsi)
1120 ; AVX512BW-NEXT: vmovdqa %ymm3, (%rdx)
1121 ; AVX512BW-NEXT: vmovdqa %ymm4, (%rcx)
1122 ; AVX512BW-NEXT: vmovdqa %ymm5, (%r8)
1123 ; AVX512BW-NEXT: vzeroupper
1124 ; AVX512BW-NEXT: retq
1125 %wide.vec = load <64 x i16>, ptr %in.vec, align 64
1126 %strided.vec0 = shufflevector <64 x i16> %wide.vec, <64 x i16> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
1127 %strided.vec1 = shufflevector <64 x i16> %wide.vec, <64 x i16> poison, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
1128 %strided.vec2 = shufflevector <64 x i16> %wide.vec, <64 x i16> poison, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
1129 %strided.vec3 = shufflevector <64 x i16> %wide.vec, <64 x i16> poison, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
1130 store <16 x i16> %strided.vec0, ptr %out.vec0, align 64
1131 store <16 x i16> %strided.vec1, ptr %out.vec1, align 64
1132 store <16 x i16> %strided.vec2, ptr %out.vec2, align 64
1133 store <16 x i16> %strided.vec3, ptr %out.vec3, align 64
1134 ret void
1135 }
1137 define void @load_i16_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind {
1138 ; SSE-LABEL: load_i16_stride4_vf32:
1139 ; SSE: # %bb.0:
1140 ; SSE-NEXT: subq $248, %rsp
1141 ; SSE-NEXT: movdqa 224(%rdi), %xmm3
1142 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1143 ; SSE-NEXT: movdqa 192(%rdi), %xmm4
1144 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1145 ; SSE-NEXT: movdqa 208(%rdi), %xmm5
1146 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1147 ; SSE-NEXT: movdqa 96(%rdi), %xmm2
1148 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1149 ; SSE-NEXT: movdqa 112(%rdi), %xmm6
1150 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1151 ; SSE-NEXT: movdqa 64(%rdi), %xmm1
1152 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1153 ; SSE-NEXT: movdqa 80(%rdi), %xmm0
1154 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1155 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1156 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1157 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
1158 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
1159 ; SSE-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
1160 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
1161 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1162 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
1163 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1164 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
1165 ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm2[0,2,2,3]
1166 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm10[0,1,0,2,4,5,6,7]
1167 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1168 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
1169 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1170 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm5[0,2,2,3]
1171 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[0,2,2,3,4,5,6,7]
1172 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm4[0,2,2,3]
1173 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm8[0,2,2,3,4,5,6,7]
1174 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1175 ; SSE-NEXT: movdqa 240(%rdi), %xmm0
1176 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1177 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,2,2,3]
1178 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm6[0,1,0,2,4,5,6,7]
1179 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm3[0,2,2,3]
1180 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm7[0,1,0,2,4,5,6,7]
1181 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1182 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
1183 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1184 ; SSE-NEXT: movdqa (%rdi), %xmm1
1185 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1186 ; SSE-NEXT: movdqa 16(%rdi), %xmm0
1187 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1188 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,2,2,3]
1189 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[0,2,2,3,4,5,6,7]
1190 ; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm1[0,2,2,3]
1191 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[0,2,2,3,4,5,6,7]
1192 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1193 ; SSE-NEXT: movdqa 32(%rdi), %xmm2
1194 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1195 ; SSE-NEXT: movdqa 48(%rdi), %xmm0
1196 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1197 ; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm0[0,2,2,3]
1198 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[0,1,0,2,4,5,6,7]
1199 ; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm2[0,2,2,3]
1200 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm14[0,1,0,2,4,5,6,7]
1201 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1202 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
1203 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1204 ; SSE-NEXT: movdqa 128(%rdi), %xmm0
1205 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1206 ; SSE-NEXT: movdqa 144(%rdi), %xmm1
1207 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1208 ; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm1[0,2,2,3]
1209 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm13[0,2,2,3,4,5,6,7]
1210 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]
1211 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm4[0,2,2,3,4,5,6,7]
1212 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1213 ; SSE-NEXT: movdqa 160(%rdi), %xmm2
1214 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1215 ; SSE-NEXT: movdqa 176(%rdi), %xmm1
1216 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1217 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]
1218 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[0,1,0,2,4,5,6,7]
1219 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
1220 ; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm2[0,1,0,2,4,5,6,7]
1221 ; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm1[0],xmm15[1],xmm1[1]
1222 ; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm0[0],xmm15[1]
1223 ; SSE-NEXT: movapd %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1224 ; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1225 ; SSE-NEXT: # xmm0 = mem[1,3,2,3,4,5,6,7]
1226 ; SSE-NEXT: pshuflw $237, (%rsp), %xmm1 # 16-byte Folded Reload
1227 ; SSE-NEXT: # xmm1 = mem[1,3,2,3,4,5,6,7]
1228 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1229 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1230 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
1231 ; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]
1232 ; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]
1233 ; SSE-NEXT: movsd {{.*#+}} xmm10 = xmm1[0],xmm10[1]
1234 ; SSE-NEXT: movapd %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1235 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[1,3,2,3,4,5,6,7]
1236 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm8[1,3,2,3,4,5,6,7]
1237 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1238 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm6[0,1,1,3,4,5,6,7]
1239 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm7[0,1,1,3,4,5,6,7]
1240 ; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1]
1241 ; SSE-NEXT: movsd {{.*#+}} xmm6 = xmm1[0],xmm6[1]
1242 ; SSE-NEXT: movapd %xmm6, (%rsp) # 16-byte Spill
1243 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[1,3,2,3,4,5,6,7]
1244 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[1,3,2,3,4,5,6,7]
1245 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1246 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[0,1,1,3,4,5,6,7]
1247 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm14[0,1,1,3,4,5,6,7]
1248 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
1249 ; SSE-NEXT: movsd {{.*#+}} xmm5 = xmm1[0],xmm5[1]
1250 ; SSE-NEXT: movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1251 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm13[1,3,2,3,4,5,6,7]
1252 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm4[1,3,2,3,4,5,6,7]
1253 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1254 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[0,1,1,3,4,5,6,7]
1255 ; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm2[0,1,1,3,4,5,6,7]
1256 ; SSE-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm0[0],xmm12[1],xmm0[1]
1257 ; SSE-NEXT: movsd {{.*#+}} xmm12 = xmm1[0],xmm12[1]
1258 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1259 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
1260 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1261 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1262 ; SSE-NEXT: # xmm1 = mem[3,1,2,3]
1263 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1264 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
1265 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
1266 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1267 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1268 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
1269 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1270 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
1271 ; SSE-NEXT: # xmm11 = mem[3,1,2,3]
1272 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
1273 ; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm11[0,1,2,0,4,5,6,7]
1274 ; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]
1275 ; SSE-NEXT: movsd {{.*#+}} xmm10 = xmm1[0],xmm10[1]
1276 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1277 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
1278 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1279 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1280 ; SSE-NEXT: # xmm1 = mem[3,1,2,3]
1281 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1282 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
1283 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
1284 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1285 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1286 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
1287 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1288 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
1289 ; SSE-NEXT: # xmm14 = mem[3,1,2,3]
1290 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
1291 ; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm14[0,1,2,0,4,5,6,7]
1292 ; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
1293 ; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm1[0],xmm15[1]
1294 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1295 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
1296 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1297 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload
1298 ; SSE-NEXT: # xmm13 = mem[3,1,2,3]
1299 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
1300 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm13[2,0,2,3,4,5,6,7]
1301 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1302 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
1303 ; SSE-NEXT: # xmm9 = mem[3,1,2,3]
1304 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
1305 ; SSE-NEXT: # xmm7 = mem[3,1,2,3]
1306 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[0,1,2,0,4,5,6,7]
1307 ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm7[0,1,2,0,4,5,6,7]
1308 ; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
1309 ; SSE-NEXT: movsd {{.*#+}} xmm8 = xmm1[0],xmm8[1]
1310 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
1311 ; SSE-NEXT: # xmm6 = mem[3,1,2,3]
1312 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
1313 ; SSE-NEXT: # xmm5 = mem[3,1,2,3]
1314 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm6[2,0,2,3,4,5,6,7]
1315 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[2,0,2,3,4,5,6,7]
1316 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
1317 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
1318 ; SSE-NEXT: # xmm4 = mem[3,1,2,3]
1319 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
1320 ; SSE-NEXT: # xmm3 = mem[3,1,2,3]
1321 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm4[0,1,2,0,4,5,6,7]
1322 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[0,1,2,0,4,5,6,7]
1323 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
1324 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
1325 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1326 ; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]
1327 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
1328 ; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]
1329 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1330 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1331 ; SSE-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
1332 ; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[0,1,3,1,4,5,6,7]
1333 ; SSE-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1]
1334 ; SSE-NEXT: movsd {{.*#+}} xmm11 = xmm2[0],xmm11[1]
1335 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1336 ; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]
1337 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
1338 ; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]
1339 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1340 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1341 ; SSE-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
1342 ; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm14[0,1,3,1,4,5,6,7]
1343 ; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
1344 ; SSE-NEXT: movsd {{.*#+}} xmm14 = xmm2[0],xmm14[1]
1345 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1346 ; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]
1347 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm13[3,1,2,3,4,5,6,7]
1348 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1349 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[0,1,3,1,4,5,6,7]
1350 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]
1351 ; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1]
1352 ; SSE-NEXT: movsd {{.*#+}} xmm7 = xmm2[0],xmm7[1]
1353 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm6[3,1,2,3,4,5,6,7]
1354 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm5[3,1,2,3,4,5,6,7]
1355 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1356 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm4[0,1,3,1,4,5,6,7]
1357 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
1358 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
1359 ; SSE-NEXT: movsd {{.*#+}} xmm3 = xmm2[0],xmm3[1]
1360 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1361 ; SSE-NEXT: movaps %xmm0, 32(%rsi)
1362 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1363 ; SSE-NEXT: movaps %xmm0, (%rsi)
1364 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1365 ; SSE-NEXT: movaps %xmm0, 48(%rsi)
1366 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1367 ; SSE-NEXT: movaps %xmm0, 16(%rsi)
1368 ; SSE-NEXT: movapd %xmm12, 32(%rdx)
1369 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1370 ; SSE-NEXT: movaps %xmm0, (%rdx)
1371 ; SSE-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
1372 ; SSE-NEXT: movaps %xmm0, 48(%rdx)
1373 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
1374 ; SSE-NEXT: movaps %xmm0, 16(%rdx)
1375 ; SSE-NEXT: movapd %xmm1, 32(%rcx)
1376 ; SSE-NEXT: movapd %xmm8, (%rcx)
1377 ; SSE-NEXT: movapd %xmm15, 48(%rcx)
1378 ; SSE-NEXT: movapd %xmm10, 16(%rcx)
1379 ; SSE-NEXT: movapd %xmm3, 32(%r8)
1380 ; SSE-NEXT: movapd %xmm7, (%r8)
1381 ; SSE-NEXT: movapd %xmm14, 48(%r8)
1382 ; SSE-NEXT: movapd %xmm11, 16(%r8)
1383 ; SSE-NEXT: addq $248, %rsp
1384 ; SSE-NEXT: retq
1385 ;
1386 ; AVX1-ONLY-LABEL: load_i16_stride4_vf32:
1387 ; AVX1-ONLY: # %bb.0:
1388 ; AVX1-ONLY-NEXT: subq $280, %rsp # imm = 0x118
1389 ; AVX1-ONLY-NEXT: vpxor %xmm6, %xmm6, %xmm6
1390 ; AVX1-ONLY-NEXT: vmovdqa 112(%rdi), %xmm4
1391 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0],xmm6[1,2,3],xmm4[4],xmm6[5,6,7]
1392 ; AVX1-ONLY-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1393 ; AVX1-ONLY-NEXT: vmovdqa 96(%rdi), %xmm11
1394 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm11[0],xmm6[1,2,3],xmm11[4],xmm6[5,6,7]
1395 ; AVX1-ONLY-NEXT: vpackusdw %xmm2, %xmm3, %xmm2
1396 ; AVX1-ONLY-NEXT: vmovdqa 80(%rdi), %xmm3
1397 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm3[0],xmm6[1,2,3],xmm3[4],xmm6[5,6,7]
1398 ; AVX1-ONLY-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1399 ; AVX1-ONLY-NEXT: vmovdqa 64(%rdi), %xmm0
1400 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1401 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm0[0],xmm6[1,2,3],xmm0[4],xmm6[5,6,7]
1402 ; AVX1-ONLY-NEXT: vpackusdw %xmm5, %xmm7, %xmm5
1403 ; AVX1-ONLY-NEXT: vpackusdw %xmm2, %xmm5, %xmm2
1404 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1405 ; AVX1-ONLY-NEXT: vmovdqa (%rdi), %xmm10
1406 ; AVX1-ONLY-NEXT: vmovdqa 16(%rdi), %xmm0
1407 ; AVX1-ONLY-NEXT: vmovdqa 32(%rdi), %xmm1
1408 ; AVX1-ONLY-NEXT: vmovdqa 48(%rdi), %xmm2
1409 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1410 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm2[0],xmm6[1,2,3],xmm2[4],xmm6[5,6,7]
1411 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm8 = xmm1[0],xmm6[1,2,3],xmm1[4],xmm6[5,6,7]
1412 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1413 ; AVX1-ONLY-NEXT: vpackusdw %xmm7, %xmm8, %xmm7
1414 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm8 = xmm0[0],xmm6[1,2,3],xmm0[4],xmm6[5,6,7]
1415 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, %xmm2
1416 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1417 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0],xmm6[1,2,3],xmm10[4],xmm6[5,6,7]
1418 ; AVX1-ONLY-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1419 ; AVX1-ONLY-NEXT: vpackusdw %xmm8, %xmm9, %xmm8
1420 ; AVX1-ONLY-NEXT: vpackusdw %xmm7, %xmm8, %xmm7
1421 ; AVX1-ONLY-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1422 ; AVX1-ONLY-NEXT: vmovdqa 240(%rdi), %xmm0
1423 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1424 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm0[0],xmm6[1,2,3],xmm0[4],xmm6[5,6,7]
1425 ; AVX1-ONLY-NEXT: vmovdqa 224(%rdi), %xmm12
1426 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm8 = xmm12[0],xmm6[1,2,3],xmm12[4],xmm6[5,6,7]
1427 ; AVX1-ONLY-NEXT: vmovdqa %xmm12, (%rsp) # 16-byte Spill
1428 ; AVX1-ONLY-NEXT: vpackusdw %xmm7, %xmm8, %xmm7
1429 ; AVX1-ONLY-NEXT: vmovdqa 208(%rdi), %xmm5
1430 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm8 = xmm5[0],xmm6[1,2,3],xmm5[4],xmm6[5,6,7]
1431 ; AVX1-ONLY-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1432 ; AVX1-ONLY-NEXT: vmovdqa 192(%rdi), %xmm0
1433 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm0[0],xmm6[1,2,3],xmm0[4],xmm6[5,6,7]
1434 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1435 ; AVX1-ONLY-NEXT: vpackusdw %xmm8, %xmm9, %xmm8
1436 ; AVX1-ONLY-NEXT: vpackusdw %xmm7, %xmm8, %xmm7
1437 ; AVX1-ONLY-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1438 ; AVX1-ONLY-NEXT: vmovdqa 176(%rdi), %xmm14
1439 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm14[0],xmm6[1,2,3],xmm14[4],xmm6[5,6,7]
1440 ; AVX1-ONLY-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1441 ; AVX1-ONLY-NEXT: vmovdqa 160(%rdi), %xmm13
1442 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm15 = xmm13[0],xmm6[1,2,3],xmm13[4],xmm6[5,6,7]
1443 ; AVX1-ONLY-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1444 ; AVX1-ONLY-NEXT: vpackusdw %xmm7, %xmm15, %xmm7
1445 ; AVX1-ONLY-NEXT: vmovdqa 144(%rdi), %xmm8
1446 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm15 = xmm8[0],xmm6[1,2,3],xmm8[4],xmm6[5,6,7]
1447 ; AVX1-ONLY-NEXT: vmovdqa %xmm8, %xmm9
1448 ; AVX1-ONLY-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1449 ; AVX1-ONLY-NEXT: vmovdqa 128(%rdi), %xmm8
1450 ; AVX1-ONLY-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1451 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm6 = xmm8[0],xmm6[1,2,3],xmm8[4],xmm6[5,6,7]
1452 ; AVX1-ONLY-NEXT: vpackusdw %xmm15, %xmm6, %xmm6
1453 ; AVX1-ONLY-NEXT: vpackusdw %xmm7, %xmm6, %xmm6
1454 ; AVX1-ONLY-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1455 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[0,2,2,3]
1456 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
1457 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm11[0,2,2,3]
1458 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
1459 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
1460 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[0,2,2,3]
1461 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]
1462 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
1463 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm15 = xmm3[0,2,2,3]
1464 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[1,3,2,3,4,5,6,7]
1465 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm15[0],xmm7[0],xmm15[1],xmm7[1]
1466 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2,3],xmm6[4,5,6,7]
1467 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
1468 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[0,2,2,3]
1469 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
1470 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm15 = xmm1[0,2,2,3]
1471 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[0,1,1,3,4,5,6,7]
1472 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm15[0],xmm7[0],xmm15[1],xmm7[1]
1473 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm15 = xmm2[0,2,2,3]
1474 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[1,3,2,3,4,5,6,7]
1475 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm8 = xmm10[0,2,2,3]
1476 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]
1477 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm15[0],xmm8[1],xmm15[1]
1478 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1,2,3],xmm7[4,5,6,7]
1479 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6
1480 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
1481 ; AVX1-ONLY-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1482 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
1483 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm6 = xmm10[0,2,2,3]
1484 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
1485 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm12[0,2,2,3]
1486 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
1487 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
1488 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[0,2,2,3]
1489 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]
1490 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[0,2,2,3]
1491 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]
1492 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
1493 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2,3],xmm6[4,5,6,7]
1494 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm14[0,2,2,3]
1495 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
1496 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm8 = xmm13[0,2,2,3]
1497 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7]
1498 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
1499 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm8 = xmm9[0,2,2,3]
1500 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]
1501 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
1502 ; AVX1-ONLY-NEXT: # xmm15 = mem[0,2,2,3]
1503 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[1,3,2,3,4,5,6,7]
1504 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm15[0],xmm8[0],xmm15[1],xmm8[1]
1505 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1,2,3],xmm7[4,5,6,7]
1506 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6
1507 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm6[4,5,6,7]
1508 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1509 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1510 ; AVX1-ONLY-NEXT: # xmm0 = mem[3,1,2,3]
1511 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1512 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm11[3,1,2,3]
1513 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1514 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
1515 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
1516 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1517 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1518 ; AVX1-ONLY-NEXT: # xmm1 = mem[3,1,2,3]
1519 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1520 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[3,1,2,3]
1521 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1522 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
1523 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]
1524 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
1525 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
1526 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[3,1,2,3]
1527 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
1528 ; AVX1-ONLY-NEXT: # xmm6 = mem[3,1,2,3]
1529 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[0,1,2,0,4,5,6,7]
1530 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[0,1,2,0,4,5,6,7]
1531 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1532 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
1533 ; AVX1-ONLY-NEXT: # xmm12 = mem[3,1,2,3]
1534 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
1535 ; AVX1-ONLY-NEXT: # xmm5 = mem[3,1,2,3]
1536 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm8 = xmm12[2,0,2,3,4,5,6,7]
1537 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm9 = xmm5[2,0,2,3,4,5,6,7]
1538 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
1539 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm8[0,1,2,3],xmm1[4,5,6,7]
1540 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
1541 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
1542 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1543 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[3,1,2,3]
1544 ; AVX1-ONLY-NEXT: vpshufd $231, (%rsp), %xmm14 # 16-byte Folded Reload
1545 ; AVX1-ONLY-NEXT: # xmm14 = mem[3,1,2,3]
1546 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm0 = xmm13[0,1,2,0,4,5,6,7]
1547 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm14[0,1,2,0,4,5,6,7]
1548 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1549 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
1550 ; AVX1-ONLY-NEXT: # xmm10 = mem[3,1,2,3]
1551 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
1552 ; AVX1-ONLY-NEXT: # xmm11 = mem[3,1,2,3]
1553 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm10[2,0,2,3,4,5,6,7]
1554 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm8 = xmm11[2,0,2,3,4,5,6,7]
1555 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm8[0],xmm1[0],xmm8[1],xmm1[1]
1556 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0,1,2,3],xmm0[4,5,6,7]
1557 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
1558 ; AVX1-ONLY-NEXT: # xmm8 = mem[3,1,2,3]
1559 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
1560 ; AVX1-ONLY-NEXT: # xmm9 = mem[3,1,2,3]
1561 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[0,1,2,0,4,5,6,7]
1562 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm9[0,1,2,0,4,5,6,7]
1563 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1564 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
1565 ; AVX1-ONLY-NEXT: # xmm4 = mem[3,1,2,3]
1566 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
1567 ; AVX1-ONLY-NEXT: # xmm2 = mem[3,1,2,3]
1568 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[2,0,2,3,4,5,6,7]
1569 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm15 = xmm2[2,0,2,3,4,5,6,7]
1570 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
1571 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
1572 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm1
1573 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm15 = ymm0[0,1,2,3],ymm1[4,5,6,7]
1574 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1575 ; AVX1-ONLY-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
1576 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1577 ; AVX1-ONLY-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
1578 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1579 ; AVX1-ONLY-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1580 ; AVX1-ONLY-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
1581 ; AVX1-ONLY-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
1582 ; AVX1-ONLY-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
1583 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
1584 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
1585 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[0,1,3,1,4,5,6,7]
1586 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm6[0,1,3,1,4,5,6,7]
1587 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
1588 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[3,1,2,3,4,5,6,7]
1589 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7]
1590 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
1591 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
1592 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
1593 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
1594 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm13[0,1,3,1,4,5,6,7]
1595 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm14[0,1,3,1,4,5,6,7]
1596 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
1597 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm10[3,1,2,3,4,5,6,7]
1598 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm5 = xmm11[3,1,2,3,4,5,6,7]
1599 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
1600 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
1601 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm8[0,1,3,1,4,5,6,7]
1602 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm5 = xmm9[0,1,3,1,4,5,6,7]
1603 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
1604 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7]
1605 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
1606 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
1607 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5,6,7]
1608 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
1609 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
1610 ; AVX1-ONLY-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1611 ; AVX1-ONLY-NEXT: vmovaps %xmm2, 32(%rsi)
1612 ; AVX1-ONLY-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1613 ; AVX1-ONLY-NEXT: vmovaps %xmm2, 48(%rsi)
1614 ; AVX1-ONLY-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1615 ; AVX1-ONLY-NEXT: vmovaps %xmm2, (%rsi)
1616 ; AVX1-ONLY-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1617 ; AVX1-ONLY-NEXT: vmovaps %xmm2, 16(%rsi)
1618 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
1619 ; AVX1-ONLY-NEXT: vmovaps %ymm2, 32(%rdx)
1620 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
1621 ; AVX1-ONLY-NEXT: vmovaps %ymm2, (%rdx)
1622 ; AVX1-ONLY-NEXT: vmovaps %ymm15, 32(%rcx)
1623 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
1624 ; AVX1-ONLY-NEXT: vmovaps %ymm2, (%rcx)
1625 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 32(%r8)
1626 ; AVX1-ONLY-NEXT: vmovaps %ymm0, (%r8)
1627 ; AVX1-ONLY-NEXT: addq $280, %rsp # imm = 0x118
1628 ; AVX1-ONLY-NEXT: vzeroupper
1629 ; AVX1-ONLY-NEXT: retq
1630 ;
1631 ; AVX2-SLOW-LABEL: load_i16_stride4_vf32:
1632 ; AVX2-SLOW: # %bb.0:
1633 ; AVX2-SLOW-NEXT: subq $168, %rsp
1634 ; AVX2-SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
1635 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
1636 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
1637 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
1638 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
1639 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
1640 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
1641 ; AVX2-SLOW-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
1642 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
1643 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
1644 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
1645 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
1646 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
1647 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
1648 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm4
1649 ; AVX2-SLOW-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
1650 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm3, %xmm3
1651 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
1652 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
1653 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
1654 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1655 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
1656 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
1657 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
1658 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
1659 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
1660 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
1661 ; AVX2-SLOW-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
1662 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
1663 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
1664 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
1665 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
1666 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
1667 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
1668 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm3
1669 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
1670 ; AVX2-SLOW-NEXT: vpackusdw %xmm0, %xmm0, %xmm0
1671 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
1672 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
1673 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
1674 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1675 ; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm1
1676 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill
1677 ; AVX2-SLOW-NEXT: vmovdqa 16(%rdi), %xmm5
1678 ; AVX2-SLOW-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1679 ; AVX2-SLOW-NEXT: vmovdqa 32(%rdi), %xmm6
1680 ; AVX2-SLOW-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1681 ; AVX2-SLOW-NEXT: vmovdqa 48(%rdi), %xmm7
1682 ; AVX2-SLOW-NEXT: vmovdqa 112(%rdi), %xmm9
1683 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm9[0,2,2,3]
1684 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
1685 ; AVX2-SLOW-NEXT: vmovdqa 96(%rdi), %xmm13
1686 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm13[0,2,2,3]
1687 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
1688 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1689 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
1690 ; AVX2-SLOW-NEXT: vmovdqa 80(%rdi), %xmm14
1691 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[0,2,2,3]
1692 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[1,3,2,3,4,5,6,7]
1693 ; AVX2-SLOW-NEXT: vmovdqa 64(%rdi), %xmm2
1694 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[0,2,2,3]
1695 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
1696 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
1697 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
1698 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
1699 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm7[0,2,2,3]
1700 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
1701 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]
1702 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7]
1703 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
1704 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm5[0,2,2,3]
1705 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
1706 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm1[0,2,2,3]
1707 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]
1708 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
1709 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
1710 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
1711 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1712 ; AVX2-SLOW-NEXT: vmovdqa 240(%rdi), %xmm0
1713 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1714 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
1715 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
1716 ; AVX2-SLOW-NEXT: vmovdqa 224(%rdi), %xmm10
1717 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm10[0,2,2,3]
1718 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
1719 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
1720 ; AVX2-SLOW-NEXT: vmovdqa 208(%rdi), %xmm11
1721 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm11[0,2,2,3]
1722 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7]
1723 ; AVX2-SLOW-NEXT: vmovdqa 192(%rdi), %xmm15
1724 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm15[0,2,2,3]
1725 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
1726 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
1727 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
1728 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
1729 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
1730 ; AVX2-SLOW-NEXT: vmovdqa 176(%rdi), %xmm12
1731 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm12[0,2,2,3]
1732 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
1733 ; AVX2-SLOW-NEXT: vmovdqa 160(%rdi), %xmm4
1734 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
1735 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
1736 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
1737 ; AVX2-SLOW-NEXT: vmovdqa 144(%rdi), %xmm3
1738 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[0,2,2,3]
1739 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
1740 ; AVX2-SLOW-NEXT: vmovdqa 128(%rdi), %xmm6
1741 ; AVX2-SLOW-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1742 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,2,2,3]
1743 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
1744 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
1745 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm1 = xmm5[0,1],xmm1[2,3]
1746 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
1747 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1748 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm9[3,1,2,3]
1749 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1750 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm13[3,1,2,3]
1751 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1752 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
1753 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
1754 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1755 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm14[3,1,2,3]
1756 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1757 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
1758 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1759 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
1760 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[2,0,2,3,4,5,6,7]
1761 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
1762 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
1763 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
1764 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3,4,5],ymm0[6,7]
1765 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm7[3,1,2,3]
1766 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
1767 ; AVX2-SLOW-NEXT: # xmm9 = mem[3,1,2,3]
1768 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm13[0,1,2,0,4,5,6,7]
1769 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm9[0,1,2,0,4,5,6,7]
1770 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
1771 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
1772 ; AVX2-SLOW-NEXT: # xmm7 = mem[3,1,2,3]
1773 ; AVX2-SLOW-NEXT: vpshufd $231, (%rsp), %xmm6 # 16-byte Folded Reload
1774 ; AVX2-SLOW-NEXT: # xmm6 = mem[3,1,2,3]
1775 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm7[2,0,2,3,4,5,6,7]
1776 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm6[2,0,2,3,4,5,6,7]
1777 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
1778 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
1779 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
1780 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
1781 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
1782 ; AVX2-SLOW-NEXT: # xmm8 = mem[3,1,2,3]
1783 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,1,2,3]
1784 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm8[0,1,2,0,4,5,6,7]
1785 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm10[0,1,2,0,4,5,6,7]
1786 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1787 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,1,2,3]
1788 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[3,1,2,3]
1789 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm11[2,0,2,3,4,5,6,7]
1790 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm15[2,0,2,3,4,5,6,7]
1791 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1792 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
1793 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
1794 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm0[6,7]
1795 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[3,1,2,3]
1796 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[3,1,2,3]
1797 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm12[0,1,2,0,4,5,6,7]
1798 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[0,1,2,0,4,5,6,7]
1799 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1800 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[3,1,2,3]
1801 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
1802 ; AVX2-SLOW-NEXT: # xmm3 = mem[3,1,2,3]
1803 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[2,0,2,3,4,5,6,7]
1804 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm14 = xmm3[2,0,2,3,4,5,6,7]
1805 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
1806 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
1807 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm14 = ymm0[0,1,2,3],ymm1[4,5,6,7]
1808 ; AVX2-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
1809 ; AVX2-SLOW-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
1810 ; AVX2-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1811 ; AVX2-SLOW-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
1812 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1813 ; AVX2-SLOW-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
1814 ; AVX2-SLOW-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
1815 ; AVX2-SLOW-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
1816 ; AVX2-SLOW-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]
1817 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1818 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
1819 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
1820 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
1821 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm13[0,1,3,1,4,5,6,7]
1822 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm9[0,1,3,1,4,5,6,7]
1823 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1824 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm7[3,1,2,3,4,5,6,7]
1825 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]
1826 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
1827 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
1828 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
1829 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[0,1,3,1,4,5,6,7]
1830 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm10[0,1,3,1,4,5,6,7]
1831 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1832 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[3,1,2,3,4,5,6,7]
1833 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm15[3,1,2,3,4,5,6,7]
1834 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
1835 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
1836 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
1837 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
1838 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[0,1,3,1,4,5,6,7]
1839 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
1840 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
1841 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7]
1842 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
1843 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
1844 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
1845 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
1846 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
1847 ; AVX2-SLOW-NEXT: vmovaps %ymm2, 32(%rsi)
1848 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
1849 ; AVX2-SLOW-NEXT: vmovaps %ymm2, (%rsi)
1850 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
1851 ; AVX2-SLOW-NEXT: vmovaps %ymm2, 32(%rdx)
1852 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
1853 ; AVX2-SLOW-NEXT: vmovaps %ymm2, (%rdx)
1854 ; AVX2-SLOW-NEXT: vmovdqa %ymm14, 32(%rcx)
1855 ; AVX2-SLOW-NEXT: vmovups (%rsp), %ymm2 # 32-byte Reload
1856 ; AVX2-SLOW-NEXT: vmovaps %ymm2, (%rcx)
1857 ; AVX2-SLOW-NEXT: vmovdqa %ymm1, 32(%r8)
1858 ; AVX2-SLOW-NEXT: vmovdqa %ymm0, (%r8)
1859 ; AVX2-SLOW-NEXT: addq $168, %rsp
1860 ; AVX2-SLOW-NEXT: vzeroupper
1861 ; AVX2-SLOW-NEXT: retq
1862 ;
1863 ; AVX2-FAST-LABEL: load_i16_stride4_vf32:
1864 ; AVX2-FAST: # %bb.0:
1865 ; AVX2-FAST-NEXT: subq $104, %rsp
1866 ; AVX2-FAST-NEXT: vmovdqa 192(%rdi), %ymm5
1867 ; AVX2-FAST-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1868 ; AVX2-FAST-NEXT: vmovdqa 224(%rdi), %ymm6
1869 ; AVX2-FAST-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1870 ; AVX2-FAST-NEXT: vmovdqa 64(%rdi), %ymm7
1871 ; AVX2-FAST-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1872 ; AVX2-FAST-NEXT: vmovdqa 96(%rdi), %ymm8
1873 ; AVX2-FAST-NEXT: vpxor %xmm0, %xmm0, %xmm0
1874 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
1875 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
1876 ; AVX2-FAST-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
1877 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
1878 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm3
1879 ; AVX2-FAST-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
1880 ; AVX2-FAST-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
1881 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = [0,2,2,3,0,2,4,6]
1882 ; AVX2-FAST-NEXT: vpermd %ymm8, %ymm2, %ymm10
1883 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} ymm4 = [16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29]
1884 ; AVX2-FAST-NEXT: vpshufb %ymm4, %ymm10, %ymm3
1885 ; AVX2-FAST-NEXT: vmovdqa %ymm4, %ymm9
1886 ; AVX2-FAST-NEXT: vpermd %ymm7, %ymm2, %ymm11
1887 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm7 = <0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u>
1888 ; AVX2-FAST-NEXT: vpshufb %ymm7, %ymm11, %ymm4
1889 ; AVX2-FAST-NEXT: vmovdqa %ymm7, %ymm12
1890 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
1891 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
1892 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1893 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
1894 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm3
1895 ; AVX2-FAST-NEXT: vpackusdw %xmm3, %xmm1, %xmm1
1896 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
1897 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
1898 ; AVX2-FAST-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
1899 ; AVX2-FAST-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
1900 ; AVX2-FAST-NEXT: vpermd %ymm6, %ymm2, %ymm7
1901 ; AVX2-FAST-NEXT: vpshufb %ymm9, %ymm7, %ymm1
1902 ; AVX2-FAST-NEXT: vpermd %ymm5, %ymm2, %ymm4
1903 ; AVX2-FAST-NEXT: vpshufb %ymm12, %ymm4, %ymm3
1904 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
1905 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
1906 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1907 ; AVX2-FAST-NEXT: vmovdqa 32(%rdi), %xmm15
1908 ; AVX2-FAST-NEXT: vmovdqa 48(%rdi), %xmm5
1909 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]
1910 ; AVX2-FAST-NEXT: vpshufb %xmm3, %xmm5, %xmm0
1911 ; AVX2-FAST-NEXT: vpshufb %xmm3, %xmm15, %xmm9
1912 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]
1913 ; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm14
1914 ; AVX2-FAST-NEXT: vmovdqa 16(%rdi), %xmm6
1915 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
1916 ; AVX2-FAST-NEXT: vpshufb %xmm2, %xmm6, %xmm12
1917 ; AVX2-FAST-NEXT: vpshufb %xmm2, %xmm14, %xmm13
1918 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]
1919 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm12 = xmm12[0,1],xmm9[2,3]
1920 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} ymm0 = [18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
1921 ; AVX2-FAST-NEXT: vpshufb %ymm0, %ymm10, %ymm13
1922 ; AVX2-FAST-NEXT: vmovdqa %ymm0, %ymm9
1923 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm0 = <2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,u,u,u,u,u,u,u,u>
1924 ; AVX2-FAST-NEXT: vpshufb %ymm0, %ymm11, %ymm11
1925 ; AVX2-FAST-NEXT: vmovdqa %ymm0, %ymm10
1926 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3,4,5],ymm13[6,7]
1927 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm11[4,5,6,7]
1928 ; AVX2-FAST-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
1929 ; AVX2-FAST-NEXT: vmovdqa 160(%rdi), %xmm13
1930 ; AVX2-FAST-NEXT: vmovdqa 176(%rdi), %xmm11
1931 ; AVX2-FAST-NEXT: vpshufb %xmm3, %xmm11, %xmm12
1932 ; AVX2-FAST-NEXT: vpshufb %xmm3, %xmm13, %xmm3
1933 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm12[0],xmm3[1],xmm12[1]
1934 ; AVX2-FAST-NEXT: vmovdqa 128(%rdi), %xmm12
1935 ; AVX2-FAST-NEXT: vmovdqa 144(%rdi), %xmm3
1936 ; AVX2-FAST-NEXT: vpshufb %xmm2, %xmm3, %xmm0
1937 ; AVX2-FAST-NEXT: vpshufb %xmm2, %xmm12, %xmm2
1938 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
1939 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
1940 ; AVX2-FAST-NEXT: vpshufb %ymm9, %ymm7, %ymm1
1941 ; AVX2-FAST-NEXT: vpshufb %ymm10, %ymm4, %ymm2
1942 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
1943 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
1944 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1945 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm7 = [1,3,2,3,1,3,5,7]
1946 ; AVX2-FAST-NEXT: vpermd %ymm8, %ymm7, %ymm4
1947 ; AVX2-FAST-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm8 # 32-byte Folded Reload
1948 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} ymm10 = [16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29]
1949 ; AVX2-FAST-NEXT: vpshufb %ymm10, %ymm4, %ymm0
1950 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm8[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
1951 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
1952 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[3,1,2,3]
1953 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[3,1,2,3]
1954 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm5 = xmm1[0,1,2,0,4,5,6,7]
1955 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm15[0,1,2,0,4,5,6,7]
1956 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
1957 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
1958 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[3,1,2,3]
1959 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm5 = xmm6[2,0,2,3,4,5,6,7]
1960 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm9 = xmm14[2,0,2,3,4,5,6,7]
1961 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1]
1962 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
1963 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
1964 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
1965 ; AVX2-FAST-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm5 # 32-byte Folded Reload
1966 ; AVX2-FAST-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm2 # 32-byte Folded Reload
1967 ; AVX2-FAST-NEXT: vpshufb %ymm10, %ymm5, %ymm7
1968 ; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm9 = ymm2[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
1969 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3,4,5],ymm7[6,7]
1970 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm9 = xmm11[3,1,2,3]
1971 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm11 = xmm13[3,1,2,3]
1972 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm13 = xmm9[0,1,2,0,4,5,6,7]
1973 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm10 = xmm11[0,1,2,0,4,5,6,7]
1974 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm13[0],xmm10[1],xmm13[1]
1975 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
1976 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[3,1,2,3]
1977 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm13 = xmm3[2,0,2,3,4,5,6,7]
1978 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm0 = xmm12[2,0,2,3,4,5,6,7]
1979 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1]
1980 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm10[2,3]
1981 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm7[4,5,6,7]
1982 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} ymm10 = [18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
1983 ; AVX2-FAST-NEXT: vpshufb %ymm10, %ymm4, %ymm4
1984 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm13 = <2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,u,u,u,u,u,u,u,u>
1985 ; AVX2-FAST-NEXT: vpshufb %ymm13, %ymm8, %ymm7
1986 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm7[0,1,2,3,4,5],ymm4[6,7]
1987 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]
1988 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm7 = xmm15[0,1,3,1,4,5,6,7]
1989 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
1990 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]
1991 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm7 = xmm14[3,1,2,3,4,5,6,7]
1992 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
1993 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm1 = xmm6[0,1],xmm1[2,3]
1994 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm4[4,5,6,7]
1995 ; AVX2-FAST-NEXT: vpshufb %ymm10, %ymm5, %ymm4
1996 ; AVX2-FAST-NEXT: vpshufb %ymm13, %ymm2, %ymm2
1997 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
1998 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm4 = xmm9[0,1,3,1,4,5,6,7]
1999 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm5 = xmm11[0,1,3,1,4,5,6,7]
2000 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
2001 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
2002 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[3,1,2,3,4,5,6,7]
2003 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
2004 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3]
2005 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
2006 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
2007 ; AVX2-FAST-NEXT: vmovaps %ymm3, 32(%rsi)
2008 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
2009 ; AVX2-FAST-NEXT: vmovaps %ymm3, (%rsi)
2010 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
2011 ; AVX2-FAST-NEXT: vmovaps %ymm3, 32(%rdx)
2012 ; AVX2-FAST-NEXT: vmovups (%rsp), %ymm3 # 32-byte Reload
2013 ; AVX2-FAST-NEXT: vmovaps %ymm3, (%rdx)
2014 ; AVX2-FAST-NEXT: vmovdqa %ymm0, 32(%rcx)
2015 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
2016 ; AVX2-FAST-NEXT: vmovaps %ymm0, (%rcx)
2017 ; AVX2-FAST-NEXT: vmovdqa %ymm2, 32(%r8)
2018 ; AVX2-FAST-NEXT: vmovdqa %ymm1, (%r8)
2019 ; AVX2-FAST-NEXT: addq $104, %rsp
2020 ; AVX2-FAST-NEXT: vzeroupper
2021 ; AVX2-FAST-NEXT: retq
2022 ;
2023 ; AVX2-FAST-PERLANE-LABEL: load_i16_stride4_vf32:
2024 ; AVX2-FAST-PERLANE: # %bb.0:
2025 ; AVX2-FAST-PERLANE-NEXT: subq $184, %rsp
2026 ; AVX2-FAST-PERLANE-NEXT: vpxor %xmm0, %xmm0, %xmm0
2027 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
2028 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm1, %xmm2
2029 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
2030 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
2031 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
2032 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
2033 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
2034 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
2035 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
2036 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
2037 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
2038 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
2039 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
2040 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm3, %xmm4
2041 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
2042 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm3, %xmm3
2043 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
2044 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
2045 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
2046 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
2047 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
2048 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm1, %xmm2
2049 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
2050 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
2051 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
2052 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
2053 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
2054 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
2055 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
2056 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
2057 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
2058 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
2059 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
2060 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm3
2061 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
2062 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm0, %xmm0, %xmm0
2063 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
2064 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
2065 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
2066 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
2067 ; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rdi), %xmm13
2068 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2069 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm12
2070 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2071 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %xmm14
2072 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2073 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 48(%rdi), %xmm7
2074 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm10 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]
2075 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 112(%rdi), %xmm2
2076 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm2, %xmm0
2077 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 96(%rdi), %xmm4
2078 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm4, %xmm1
2079 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2080 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm3
2081 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm11 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
2082 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 80(%rdi), %xmm5
2083 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm11, %xmm5, %xmm8
2084 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 64(%rdi), %xmm6
2085 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm11, %xmm6, %xmm9
2086 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
2087 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
2088 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm3 = ymm8[0,1,2,3,4,5],ymm3[6,7]
2089 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm7, %xmm8
2090 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm14, %xmm9
2091 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
2092 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm11, %xmm12, %xmm9
2093 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm11, %xmm13, %xmm12
2094 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1]
2095 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
2096 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm3[4,5,6,7]
2097 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
2098 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 240(%rdi), %xmm14
2099 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm14, %xmm9
2100 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 224(%rdi), %xmm8
2101 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm8, %xmm12
2102 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1]
2103 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 208(%rdi), %xmm0
2104 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2105 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm11, %xmm0, %xmm12
2106 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 192(%rdi), %xmm15
2107 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm11, %xmm15, %xmm13
2108 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]
2109 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
2110 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
2111 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3,4,5],ymm9[6,7]
2112 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 160(%rdi), %xmm0
2113 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2114 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 176(%rdi), %xmm13
2115 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm13, %xmm12
2116 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm10, %xmm0, %xmm10
2117 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1]
2118 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 128(%rdi), %xmm3
2119 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm3, (%rsp) # 16-byte Spill
2120 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 144(%rdi), %xmm12
2121 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm11, %xmm12, %xmm0
2122 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm11, %xmm3, %xmm11
2123 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm11[0],xmm0[0],xmm11[1],xmm0[1]
2124 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm10[2,3]
2125 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
2126 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
2127 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
2128 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2129 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[3,1,2,3]
2130 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2131 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
2132 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
2133 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2134 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[3,1,2,3]
2135 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2136 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm2 = xmm6[3,1,2,3]
2137 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2138 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
2139 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
2140 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
2141 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
2142 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
2143 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm0[6,7]
2144 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
2145 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
2146 ; AVX2-FAST-PERLANE-NEXT: # xmm6 = mem[3,1,2,3]
2147 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[0,1,2,0,4,5,6,7]
2148 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[0,1,2,0,4,5,6,7]
2149 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
2150 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
2151 ; AVX2-FAST-PERLANE-NEXT: # xmm5 = mem[3,1,2,3]
2152 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
2153 ; AVX2-FAST-PERLANE-NEXT: # xmm4 = mem[3,1,2,3]
2154 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm0 = xmm5[2,0,2,3,4,5,6,7]
2155 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm9 = xmm4[2,0,2,3,4,5,6,7]
2156 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]
2157 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
2158 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
2159 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
2160 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[3,1,2,3]
2161 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm3 = xmm8[3,1,2,3]
2162 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm0 = xmm2[0,1,2,0,4,5,6,7]
2163 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm8 = xmm3[0,1,2,0,4,5,6,7]
2164 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
2165 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
2166 ; AVX2-FAST-PERLANE-NEXT: # xmm8 = mem[3,1,2,3]
2167 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm14 = xmm15[3,1,2,3]
2168 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm9 = xmm8[2,0,2,3,4,5,6,7]
2169 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm15 = xmm14[2,0,2,3,4,5,6,7]
2170 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1]
2171 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
2172 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
2173 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm9[0,1,2,3,4,5],ymm0[6,7]
2174 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[3,1,2,3]
2175 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
2176 ; AVX2-FAST-PERLANE-NEXT: # xmm9 = mem[3,1,2,3]
2177 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm15 = xmm13[0,1,2,0,4,5,6,7]
2178 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm11 = xmm9[0,1,2,0,4,5,6,7]
2179 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm15[0],xmm11[1],xmm15[1]
2180 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm15 = xmm12[3,1,2,3]
2181 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, (%rsp), %xmm12 # 16-byte Folded Reload
2182 ; AVX2-FAST-PERLANE-NEXT: # xmm12 = mem[3,1,2,3]
2183 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm0 = xmm15[2,0,2,3,4,5,6,7]
2184 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm10 = xmm12[2,0,2,3,4,5,6,7]
2185 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]
2186 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm11[2,3]
2187 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
2188 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2189 ; AVX2-FAST-PERLANE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
2190 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
2191 ; AVX2-FAST-PERLANE-NEXT: # xmm10 = mem[0,1,3,1,4,5,6,7]
2192 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm10[0],xmm1[0],xmm10[1],xmm1[1]
2193 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
2194 ; AVX2-FAST-PERLANE-NEXT: # xmm10 = mem[3,1,2,3,4,5,6,7]
2195 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
2196 ; AVX2-FAST-PERLANE-NEXT: # xmm11 = mem[3,1,2,3,4,5,6,7]
2197 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
2198 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
2199 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
2200 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm10[0,1,2,3,4,5],ymm1[6,7]
2201 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]
2202 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,1,4,5,6,7]
2203 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
2204 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7]
2205 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7]
2206 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
2207 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm6[2,3]
2208 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
2209 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7]
2210 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
2211 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
2212 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm8[3,1,2,3,4,5,6,7]
2213 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm4 = xmm14[3,1,2,3,4,5,6,7]
2214 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
2215 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
2216 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
2217 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
2218 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm13[0,1,3,1,4,5,6,7]
2219 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm4 = xmm9[0,1,3,1,4,5,6,7]
2220 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
2221 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm4 = xmm15[3,1,2,3,4,5,6,7]
2222 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[3,1,2,3,4,5,6,7]
2223 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
2224 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
2225 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
2226 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
2227 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm3, 32(%rsi)
2228 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
2229 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm3, (%rsi)
2230 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
2231 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm3, 32(%rdx)
2232 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
2233 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm3, (%rdx)
2234 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm0, 32(%rcx)
2235 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
2236 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm0, (%rcx)
2237 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm2, 32(%r8)
2238 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm1, (%r8)
2239 ; AVX2-FAST-PERLANE-NEXT: addq $184, %rsp
2240 ; AVX2-FAST-PERLANE-NEXT: vzeroupper
2241 ; AVX2-FAST-PERLANE-NEXT: retq
2242 ;
2243 ; AVX512F-SLOW-LABEL: load_i16_stride4_vf32:
2244 ; AVX512F-SLOW: # %bb.0:
2245 ; AVX512F-SLOW-NEXT: vmovdqa64 (%rdi), %zmm0
2246 ; AVX512F-SLOW-NEXT: vmovdqa64 128(%rdi), %zmm1
2247 ; AVX512F-SLOW-NEXT: vmovdqa 192(%rdi), %ymm2
2248 ; AVX512F-SLOW-NEXT: vpmovqw %ymm2, %xmm2
2249 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
2250 ; AVX512F-SLOW-NEXT: vmovdqa 240(%rdi), %xmm6
2251 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
2252 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm5[0,1,0,2,4,5,6,7]
2253 ; AVX512F-SLOW-NEXT: vmovdqa 224(%rdi), %xmm7
2254 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[0,2,2,3]
2255 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm9[0,1,0,2,4,5,6,7]
2256 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
2257 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
2258 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
2259 ; AVX512F-SLOW-NEXT: vpmovqw %zmm1, %xmm3
2260 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
2261 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm2
2262 ; AVX512F-SLOW-NEXT: vmovdqa 64(%rdi), %ymm3
2263 ; AVX512F-SLOW-NEXT: vpmovqw %ymm3, %xmm3
2264 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm10
2265 ; AVX512F-SLOW-NEXT: vmovdqa 112(%rdi), %xmm3
2266 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm3[0,2,2,3]
2267 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm8[0,1,0,2,4,5,6,7]
2268 ; AVX512F-SLOW-NEXT: vmovdqa 96(%rdi), %xmm4
2269 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm4[0,2,2,3]
2270 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm13 = xmm12[0,1,0,2,4,5,6,7]
2271 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]
2272 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
2273 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm11[6,7]
2274 ; AVX512F-SLOW-NEXT: vpmovqw %zmm0, %xmm11
2275 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
2276 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm10[0,1,2,3],zmm2[4,5,6,7]
2277 ; AVX512F-SLOW-NEXT: vmovdqa 64(%rdi), %xmm10
2278 ; AVX512F-SLOW-NEXT: vmovdqa 80(%rdi), %xmm11
2279 ; AVX512F-SLOW-NEXT: vmovdqa 192(%rdi), %xmm13
2280 ; AVX512F-SLOW-NEXT: vmovdqa 208(%rdi), %xmm14
2281 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]
2282 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
2283 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1]
2284 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
2285 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm14[0,2,2,3]
2286 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
2287 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm15 = xmm13[0,2,2,3]
2288 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[1,3,2,3,4,5,6,7]
2289 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1]
2290 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
2291 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
2292 ; AVX512F-SLOW-NEXT: vpsrlq $16, %zmm1, %zmm9
2293 ; AVX512F-SLOW-NEXT: vpmovqw %zmm9, %xmm9
2294 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3],ymm5[4,5,6,7]
2295 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm5
2296 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7]
2297 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm12[0,1,1,3,4,5,6,7]
2298 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
2299 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
2300 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm11[0,2,2,3]
2301 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
2302 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[0,2,2,3]
2303 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]
2304 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1]
2305 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
2306 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4,5],ymm8[6,7]
2307 ; AVX512F-SLOW-NEXT: vpsrlq $16, %zmm0, %zmm9
2308 ; AVX512F-SLOW-NEXT: vpmovqw %zmm9, %xmm9
2309 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
2310 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm8[0,1,2,3],zmm5[4,5,6,7]
2311 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
2312 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm6[0,1,2,0,4,5,6,7]
2313 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
2314 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm7[0,1,2,0,4,5,6,7]
2315 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
2316 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm12
2317 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm14[3,1,2,3]
2318 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm14 = xmm8[2,0,2,3,4,5,6,7]
2319 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm13[3,1,2,3]
2320 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm13 = xmm9[2,0,2,3,4,5,6,7]
2321 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1]
2322 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
2323 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5],ymm12[6,7]
2324 ; AVX512F-SLOW-NEXT: vpsrlq $32, %zmm1, %zmm13
2325 ; AVX512F-SLOW-NEXT: vpmovqw %zmm13, %xmm13
2326 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3],ymm12[4,5,6,7]
2327 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm12, %zmm0, %zmm12
2328 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
2329 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm13 = xmm3[0,1,2,0,4,5,6,7]
2330 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
2331 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm14 = xmm4[0,1,2,0,4,5,6,7]
2332 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
2333 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
2334 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,1,2,3]
2335 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm14 = xmm11[2,0,2,3,4,5,6,7]
2336 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,1,2,3]
2337 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm15 = xmm10[2,0,2,3,4,5,6,7]
2338 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
2339 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
2340 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3,4,5],ymm13[6,7]
2341 ; AVX512F-SLOW-NEXT: vpsrlq $32, %zmm0, %zmm14
2342 ; AVX512F-SLOW-NEXT: vpmovqw %zmm14, %xmm14
2343 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3],ymm13[4,5,6,7]
2344 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm12 = zmm13[0,1,2,3],zmm12[4,5,6,7]
2345 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,1,4,5,6,7]
2346 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]
2347 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
2348 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
2349 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm8[3,1,2,3,4,5,6,7]
2350 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm9[3,1,2,3,4,5,6,7]
2351 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
2352 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
2353 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]
2354 ; AVX512F-SLOW-NEXT: vpsrlq $48, %zmm1, %zmm1
2355 ; AVX512F-SLOW-NEXT: vpmovqw %zmm1, %xmm1
2356 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]
2357 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1
2358 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
2359 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
2360 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
2361 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
2362 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm11[3,1,2,3,4,5,6,7]
2363 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm10[3,1,2,3,4,5,6,7]
2364 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
2365 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
2366 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
2367 ; AVX512F-SLOW-NEXT: vpsrlq $48, %zmm0, %zmm0
2368 ; AVX512F-SLOW-NEXT: vpmovqw %zmm0, %xmm0
2369 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
2370 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
2371 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm2, (%rsi)
2372 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm5, (%rdx)
2373 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm12, (%rcx)
2374 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm0, (%r8)
2375 ; AVX512F-SLOW-NEXT: vzeroupper
2376 ; AVX512F-SLOW-NEXT: retq
2377 ;
2378 ; AVX512F-FAST-LABEL: load_i16_stride4_vf32:
2379 ; AVX512F-FAST: # %bb.0:
2380 ; AVX512F-FAST-NEXT: vmovdqa64 (%rdi), %zmm0
2381 ; AVX512F-FAST-NEXT: vmovdqa64 128(%rdi), %zmm1
2382 ; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0,16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0]
2383 ; AVX512F-FAST-NEXT: # ymm2 = mem[0,1,0,1]
2384 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm4 = [0,2,2,3,0,2,4,6]
2385 ; AVX512F-FAST-NEXT: vmovdqa 224(%rdi), %ymm6
2386 ; AVX512F-FAST-NEXT: vpermd %ymm6, %ymm4, %ymm5
2387 ; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm5, %ymm7
2388 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm3 = <0,1,4,5,u,u,u,u,8,9,12,13,12,13,14,15,16,17,20,21,u,u,u,u,24,25,28,29,u,u,u,u>
2389 ; AVX512F-FAST-NEXT: vmovdqa 192(%rdi), %ymm8
2390 ; AVX512F-FAST-NEXT: vpermd %ymm8, %ymm4, %ymm9
2391 ; AVX512F-FAST-NEXT: vpshufb %ymm3, %ymm9, %ymm10
2392 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm11 = [0,2,2,3,4,6,12,14]
2393 ; AVX512F-FAST-NEXT: vpermt2d %ymm7, %ymm11, %ymm10
2394 ; AVX512F-FAST-NEXT: vpmovqw %zmm1, %xmm7
2395 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm10[4,5,6,7]
2396 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm0, %zmm7
2397 ; AVX512F-FAST-NEXT: vmovdqa 96(%rdi), %ymm10
2398 ; AVX512F-FAST-NEXT: vpermd %ymm10, %ymm4, %ymm12
2399 ; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm12, %ymm13
2400 ; AVX512F-FAST-NEXT: vmovdqa 64(%rdi), %ymm14
2401 ; AVX512F-FAST-NEXT: vpermd %ymm14, %ymm4, %ymm15
2402 ; AVX512F-FAST-NEXT: vpshufb %ymm3, %ymm15, %ymm4
2403 ; AVX512F-FAST-NEXT: vpermt2d %ymm13, %ymm11, %ymm4
2404 ; AVX512F-FAST-NEXT: vpmovqw %zmm0, %xmm13
2405 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3],ymm4[4,5,6,7]
2406 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm7 = zmm4[0,1,2,3],zmm7[4,5,6,7]
2407 ; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm4 = [18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
2408 ; AVX512F-FAST-NEXT: vpshufb %ymm4, %ymm5, %ymm13
2409 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm5 = <2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,u,u,u,u,u,u,u,u>
2410 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm9, %ymm9
2411 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm13[6,7]
2412 ; AVX512F-FAST-NEXT: vpsrlq $16, %zmm1, %zmm13
2413 ; AVX512F-FAST-NEXT: vpmovqw %zmm13, %xmm13
2414 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm9 = ymm13[0,1,2,3],ymm9[4,5,6,7]
2415 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm0, %zmm9
2416 ; AVX512F-FAST-NEXT: vpshufb %ymm4, %ymm12, %ymm12
2417 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm15, %ymm13
2418 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5],ymm12[6,7]
2419 ; AVX512F-FAST-NEXT: vpsrlq $16, %zmm0, %zmm13
2420 ; AVX512F-FAST-NEXT: vpmovqw %zmm13, %xmm13
2421 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3],ymm12[4,5,6,7]
2422 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm9 = zmm12[0,1,2,3],zmm9[4,5,6,7]
2423 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm12 = [1,3,2,3,1,3,5,7]
2424 ; AVX512F-FAST-NEXT: vpermd %ymm6, %ymm12, %ymm6
2425 ; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm6, %ymm13
2426 ; AVX512F-FAST-NEXT: vpermd %ymm8, %ymm12, %ymm8
2427 ; AVX512F-FAST-NEXT: vpshufb %ymm3, %ymm8, %ymm15
2428 ; AVX512F-FAST-NEXT: vpermt2d %ymm13, %ymm11, %ymm15
2429 ; AVX512F-FAST-NEXT: vpsrlq $32, %zmm1, %zmm13
2430 ; AVX512F-FAST-NEXT: vpmovqw %zmm13, %xmm13
2431 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3],ymm15[4,5,6,7]
2432 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm13, %zmm0, %zmm13
2433 ; AVX512F-FAST-NEXT: vpermd %ymm10, %ymm12, %ymm10
2434 ; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm10, %ymm2
2435 ; AVX512F-FAST-NEXT: vpermd %ymm14, %ymm12, %ymm12
2436 ; AVX512F-FAST-NEXT: vpshufb %ymm3, %ymm12, %ymm3
2437 ; AVX512F-FAST-NEXT: vpermt2d %ymm2, %ymm11, %ymm3
2438 ; AVX512F-FAST-NEXT: vpsrlq $32, %zmm0, %zmm2
2439 ; AVX512F-FAST-NEXT: vpmovqw %zmm2, %xmm2
2440 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
2441 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm13[4,5,6,7]
2442 ; AVX512F-FAST-NEXT: vpshufb %ymm4, %ymm6, %ymm3
2443 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm8, %ymm6
2444 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3,4,5],ymm3[6,7]
2445 ; AVX512F-FAST-NEXT: vpsrlq $48, %zmm1, %zmm1
2446 ; AVX512F-FAST-NEXT: vpmovqw %zmm1, %xmm1
2447 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
2448 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1
2449 ; AVX512F-FAST-NEXT: vpshufb %ymm4, %ymm10, %ymm3
2450 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm12, %ymm4
2451 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
2452 ; AVX512F-FAST-NEXT: vpsrlq $48, %zmm0, %zmm0
2453 ; AVX512F-FAST-NEXT: vpmovqw %zmm0, %xmm0
2454 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
2455 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
2456 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm7, (%rsi)
2457 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm9, (%rdx)
2458 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm2, (%rcx)
2459 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm0, (%r8)
2460 ; AVX512F-FAST-NEXT: vzeroupper
2461 ; AVX512F-FAST-NEXT: retq
2462 ;
2463 ; AVX512BW-LABEL: load_i16_stride4_vf32:
2464 ; AVX512BW: # %bb.0:
2465 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
2466 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
2467 ; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm2
2468 ; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm3
2469 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]
2470 ; AVX512BW-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3]
2471 ; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm5
2472 ; AVX512BW-NEXT: vpermt2w %zmm3, %zmm4, %zmm5
2473 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm4
2474 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm4[0,1,2,3],zmm5[4,5,6,7]
2475 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61,1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]
2476 ; AVX512BW-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]
2477 ; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm6
2478 ; AVX512BW-NEXT: vpermt2w %zmm3, %zmm5, %zmm6
2479 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm5
2480 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,2,3],zmm6[4,5,6,7]
2481 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm6 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62,2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]
2482 ; AVX512BW-NEXT: # zmm6 = mem[0,1,2,3,0,1,2,3]
2483 ; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm7
2484 ; AVX512BW-NEXT: vpermt2w %zmm3, %zmm6, %zmm7
2485 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm6
2486 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm7[4,5,6,7]
2487 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63,3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]
2488 ; AVX512BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]
2489 ; AVX512BW-NEXT: vpermt2w %zmm3, %zmm7, %zmm2
2490 ; AVX512BW-NEXT: vpermt2w %zmm1, %zmm7, %zmm0
2491 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[4,5,6,7]
2492 ; AVX512BW-NEXT: vmovdqa64 %zmm4, (%rsi)
2493 ; AVX512BW-NEXT: vmovdqa64 %zmm5, (%rdx)
2494 ; AVX512BW-NEXT: vmovdqa64 %zmm6, (%rcx)
2495 ; AVX512BW-NEXT: vmovdqa64 %zmm0, (%r8)
2496 ; AVX512BW-NEXT: vzeroupper
2497 ; AVX512BW-NEXT: retq
2498 %wide.vec = load <128 x i16>, ptr %in.vec, align 64
2499 %strided.vec0 = shufflevector <128 x i16> %wide.vec, <128 x i16> poison, <32 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60, i32 64, i32 68, i32 72, i32 76, i32 80, i32 84, i32 88, i32 92, i32 96, i32 100, i32 104, i32 108, i32 112, i32 116, i32 120, i32 124>
2500 %strided.vec1 = shufflevector <128 x i16> %wide.vec, <128 x i16> poison, <32 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61, i32 65, i32 69, i32 73, i32 77, i32 81, i32 85, i32 89, i32 93, i32 97, i32 101, i32 105, i32 109, i32 113, i32 117, i32 121, i32 125>
2501 %strided.vec2 = shufflevector <128 x i16> %wide.vec, <128 x i16> poison, <32 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62, i32 66, i32 70, i32 74, i32 78, i32 82, i32 86, i32 90, i32 94, i32 98, i32 102, i32 106, i32 110, i32 114, i32 118, i32 122, i32 126>
2502 %strided.vec3 = shufflevector <128 x i16> %wide.vec, <128 x i16> poison, <32 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63, i32 67, i32 71, i32 75, i32 79, i32 83, i32 87, i32 91, i32 95, i32 99, i32 103, i32 107, i32 111, i32 115, i32 119, i32 123, i32 127>
2503 store <32 x i16> %strided.vec0, ptr %out.vec0, align 64
2504 store <32 x i16> %strided.vec1, ptr %out.vec1, align 64
2505 store <32 x i16> %strided.vec2, ptr %out.vec2, align 64
2506 store <32 x i16> %strided.vec3, ptr %out.vec3, align 64
2507 ret void
2508 }
2510 define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind {
2511 ; SSE-LABEL: load_i16_stride4_vf64:
2512 ; SSE: # %bb.0:
2513 ; SSE-NEXT: subq $824, %rsp # imm = 0x338
2514 ; SSE-NEXT: movdqa 352(%rdi), %xmm3
2515 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2516 ; SSE-NEXT: movdqa 320(%rdi), %xmm4
2517 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2518 ; SSE-NEXT: movdqa 336(%rdi), %xmm5
2519 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2520 ; SSE-NEXT: movdqa 96(%rdi), %xmm2
2521 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2522 ; SSE-NEXT: movdqa 112(%rdi), %xmm6
2523 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2524 ; SSE-NEXT: movdqa 64(%rdi), %xmm1
2525 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2526 ; SSE-NEXT: movdqa 80(%rdi), %xmm0
2527 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2528 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2529 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2530 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2531 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
2532 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2533 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
2534 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2535 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
2536 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2537 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
2538 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
2539 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2540 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
2541 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2542 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
2543 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2544 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
2545 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2546 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2547 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
2548 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2549 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
2550 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2551 ; SSE-NEXT: movdqa 368(%rdi), %xmm0
2552 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2553 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2554 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2555 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
2556 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
2557 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2558 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
2559 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2560 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
2561 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2562 ; SSE-NEXT: movdqa (%rdi), %xmm1
2563 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2564 ; SSE-NEXT: movdqa 16(%rdi), %xmm0
2565 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2566 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2567 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2568 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2569 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
2570 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2571 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
2572 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2573 ; SSE-NEXT: movdqa 32(%rdi), %xmm2
2574 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2575 ; SSE-NEXT: movdqa 48(%rdi), %xmm0
2576 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2577 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2578 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2579 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
2580 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
2581 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2582 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
2583 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2584 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
2585 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2586 ; SSE-NEXT: movdqa 256(%rdi), %xmm1
2587 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2588 ; SSE-NEXT: movdqa 272(%rdi), %xmm0
2589 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2590 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2591 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2592 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2593 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
2594 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2595 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
2596 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2597 ; SSE-NEXT: movdqa 288(%rdi), %xmm2
2598 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2599 ; SSE-NEXT: movdqa 304(%rdi), %xmm0
2600 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2601 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2602 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2603 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
2604 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
2605 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2606 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
2607 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2608 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
2609 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2610 ; SSE-NEXT: movdqa 192(%rdi), %xmm1
2611 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2612 ; SSE-NEXT: movdqa 208(%rdi), %xmm0
2613 ; SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
2614 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2615 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2616 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
2617 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
2618 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2619 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
2620 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2621 ; SSE-NEXT: movdqa 224(%rdi), %xmm2
2622 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2623 ; SSE-NEXT: movdqa 240(%rdi), %xmm0
2624 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2625 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2626 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2627 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
2628 ; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm2[0,2,2,3]
2629 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm14[0,1,0,2,4,5,6,7]
2630 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2631 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
2632 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2633 ; SSE-NEXT: movdqa 448(%rdi), %xmm1
2634 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2635 ; SSE-NEXT: movdqa 464(%rdi), %xmm0
2636 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2637 ; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm0[0,2,2,3]
2638 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm13[0,2,2,3,4,5,6,7]
2639 ; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm1[0,2,2,3]
2640 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[0,2,2,3,4,5,6,7]
2641 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2642 ; SSE-NEXT: movdqa 480(%rdi), %xmm2
2643 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2644 ; SSE-NEXT: movdqa 496(%rdi), %xmm0
2645 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2646 ; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm0[0,2,2,3]
2647 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[0,1,0,2,4,5,6,7]
2648 ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm2[0,2,2,3]
2649 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm10[0,1,0,2,4,5,6,7]
2650 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2651 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
2652 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2653 ; SSE-NEXT: movdqa 128(%rdi), %xmm1
2654 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2655 ; SSE-NEXT: movdqa 144(%rdi), %xmm0
2656 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2657 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,2,2,3]
2658 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[0,2,2,3,4,5,6,7]
2659 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm1[0,2,2,3]
2660 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm8[0,2,2,3,4,5,6,7]
2661 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2662 ; SSE-NEXT: movdqa 160(%rdi), %xmm2
2663 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2664 ; SSE-NEXT: movdqa 176(%rdi), %xmm0
2665 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2666 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,2,2,3]
2667 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[0,1,0,2,4,5,6,7]
2668 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[0,2,2,3]
2669 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm6[0,1,0,2,4,5,6,7]
2670 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2671 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
2672 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2673 ; SSE-NEXT: movdqa 384(%rdi), %xmm0
2674 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2675 ; SSE-NEXT: movdqa 400(%rdi), %xmm1
2676 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2677 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3]
2678 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm5[0,2,2,3,4,5,6,7]
2679 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]
2680 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm4[0,2,2,3,4,5,6,7]
2681 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2682 ; SSE-NEXT: movdqa 416(%rdi), %xmm2
2683 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2684 ; SSE-NEXT: movdqa 432(%rdi), %xmm1
2685 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2686 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]
2687 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[0,1,0,2,4,5,6,7]
2688 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
2689 ; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm2[0,1,0,2,4,5,6,7]
2690 ; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm1[0],xmm15[1],xmm1[1]
2691 ; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm0[0],xmm15[1]
2692 ; SSE-NEXT: movapd %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2693 ; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2694 ; SSE-NEXT: # xmm0 = mem[1,3,2,3,4,5,6,7]
2695 ; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2696 ; SSE-NEXT: # xmm1 = mem[1,3,2,3,4,5,6,7]
2697 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2698 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2699 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
2700 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
2701 ; SSE-NEXT: # xmm15 = mem[0,1,1,3,4,5,6,7]
2702 ; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
2703 ; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm1[0],xmm15[1]
2704 ; SSE-NEXT: movapd %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2705 ; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2706 ; SSE-NEXT: # xmm0 = mem[1,3,2,3,4,5,6,7]
2707 ; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2708 ; SSE-NEXT: # xmm1 = mem[1,3,2,3,4,5,6,7]
2709 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2710 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2711 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
2712 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
2713 ; SSE-NEXT: # xmm15 = mem[0,1,1,3,4,5,6,7]
2714 ; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
2715 ; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm1[0],xmm15[1]
2716 ; SSE-NEXT: movapd %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2717 ; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2718 ; SSE-NEXT: # xmm0 = mem[1,3,2,3,4,5,6,7]
2719 ; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2720 ; SSE-NEXT: # xmm1 = mem[1,3,2,3,4,5,6,7]
2721 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2722 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2723 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
2724 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
2725 ; SSE-NEXT: # xmm15 = mem[0,1,1,3,4,5,6,7]
2726 ; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
2727 ; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm1[0],xmm15[1]
2728 ; SSE-NEXT: movapd %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2729 ; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2730 ; SSE-NEXT: # xmm0 = mem[1,3,2,3,4,5,6,7]
2731 ; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2732 ; SSE-NEXT: # xmm1 = mem[1,3,2,3,4,5,6,7]
2733 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2734 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2735 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
2736 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
2737 ; SSE-NEXT: # xmm15 = mem[0,1,1,3,4,5,6,7]
2738 ; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
2739 ; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm1[0],xmm15[1]
2740 ; SSE-NEXT: movapd %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2741 ; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2742 ; SSE-NEXT: # xmm0 = mem[1,3,2,3,4,5,6,7]
2743 ; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2744 ; SSE-NEXT: # xmm1 = mem[1,3,2,3,4,5,6,7]
2745 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2746 ; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2747 ; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]
2748 ; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm14[0,1,1,3,4,5,6,7]
2749 ; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
2750 ; SSE-NEXT: movsd {{.*#+}} xmm14 = xmm1[0],xmm14[1]
2751 ; SSE-NEXT: movapd %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2752 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm13[1,3,2,3,4,5,6,7]
2753 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[1,3,2,3,4,5,6,7]
2754 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2755 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[0,1,1,3,4,5,6,7]
2756 ; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]
2757 ; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]
2758 ; SSE-NEXT: movsd {{.*#+}} xmm10 = xmm1[0],xmm10[1]
2759 ; SSE-NEXT: movapd %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2760 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[1,3,2,3,4,5,6,7]
2761 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm8[1,3,2,3,4,5,6,7]
2762 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2763 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[0,1,1,3,4,5,6,7]
2764 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
2765 ; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1]
2766 ; SSE-NEXT: movsd {{.*#+}} xmm6 = xmm1[0],xmm6[1]
2767 ; SSE-NEXT: movapd %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2768 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[1,3,2,3,4,5,6,7]
2769 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm4[1,3,2,3,4,5,6,7]
2770 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2771 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[0,1,1,3,4,5,6,7]
2772 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
2773 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2774 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
2775 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2776 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2777 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
2778 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2779 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2780 ; SSE-NEXT: # xmm1 = mem[3,1,2,3]
2781 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2782 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
2783 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
2784 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2785 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2786 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
2787 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2788 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2789 ; SSE-NEXT: # xmm2 = mem[3,1,2,3]
2790 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2791 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
2792 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
2793 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2794 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
2795 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2796 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2797 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
2798 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2799 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2800 ; SSE-NEXT: # xmm1 = mem[3,1,2,3]
2801 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2802 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
2803 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
2804 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2805 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2806 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
2807 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2808 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2809 ; SSE-NEXT: # xmm2 = mem[3,1,2,3]
2810 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2811 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
2812 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
2813 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2814 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
2815 ; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2816 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2817 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
2818 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2819 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2820 ; SSE-NEXT: # xmm1 = mem[3,1,2,3]
2821 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2822 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
2823 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[2,0,2,3,4,5,6,7]
2824 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2825 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload
2826 ; SSE-NEXT: # xmm13 = mem[3,1,2,3]
2827 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
2828 ; SSE-NEXT: # xmm8 = mem[3,1,2,3]
2829 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm13[0,1,2,0,4,5,6,7]
2830 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm8[0,1,2,0,4,5,6,7]
2831 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2832 ; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
2833 ; SSE-NEXT: movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2834 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2835 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
2836 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2837 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2838 ; SSE-NEXT: # xmm1 = mem[3,1,2,3]
2839 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2840 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
2841 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[2,0,2,3,4,5,6,7]
2842 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2843 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2844 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
2845 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2846 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2847 ; SSE-NEXT: # xmm1 = mem[3,1,2,3]
2848 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2849 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
2850 ; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm1[0,1,2,0,4,5,6,7]
2851 ; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
2852 ; SSE-NEXT: movsd {{.*#+}} xmm14 = xmm2[0],xmm14[1]
2853 ; SSE-NEXT: pshufd $231, (%rsp), %xmm0 # 16-byte Folded Reload
2854 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
2855 ; SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
2856 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2857 ; SSE-NEXT: # xmm1 = mem[3,1,2,3]
2858 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2859 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
2860 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[2,0,2,3,4,5,6,7]
2861 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2862 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
2863 ; SSE-NEXT: # xmm15 = mem[3,1,2,3]
2864 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
2865 ; SSE-NEXT: # xmm12 = mem[3,1,2,3]
2866 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm15[0,1,2,0,4,5,6,7]
2867 ; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm12[0,1,2,0,4,5,6,7]
2868 ; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]
2869 ; SSE-NEXT: movsd {{.*#+}} xmm10 = xmm2[0],xmm10[1]
2870 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2871 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
2872 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2873 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2874 ; SSE-NEXT: # xmm1 = mem[3,1,2,3]
2875 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2876 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
2877 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[2,0,2,3,4,5,6,7]
2878 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2879 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2880 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
2881 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2882 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2883 ; SSE-NEXT: # xmm1 = mem[3,1,2,3]
2884 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2885 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
2886 ; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm1[0,1,2,0,4,5,6,7]
2887 ; SSE-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]
2888 ; SSE-NEXT: movsd {{.*#+}} xmm9 = xmm2[0],xmm9[1]
2889 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
2890 ; SSE-NEXT: # xmm7 = mem[3,1,2,3]
2891 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
2892 ; SSE-NEXT: # xmm6 = mem[3,1,2,3]
2893 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[2,0,2,3,4,5,6,7]
2894 ; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm6[2,0,2,3,4,5,6,7]
2895 ; SSE-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1]
2896 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
2897 ; SSE-NEXT: # xmm4 = mem[3,1,2,3]
2898 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
2899 ; SSE-NEXT: # xmm3 = mem[3,1,2,3]
2900 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm4[0,1,2,0,4,5,6,7]
2901 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm3[0,1,2,0,4,5,6,7]
2902 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
2903 ; SSE-NEXT: movsd {{.*#+}} xmm5 = xmm11[0],xmm5[1]
2904 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2905 ; SSE-NEXT: # xmm0 = mem[3,1,2,3]
2906 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2907 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2908 ; SSE-NEXT: # xmm2 = mem[3,1,2,3]
2909 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2910 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[2,0,2,3,4,5,6,7]
2911 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm2[2,0,2,3,4,5,6,7]
2912 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
2913 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2914 ; SSE-NEXT: # xmm1 = mem[3,1,2,3]
2915 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2916 ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
2917 ; SSE-NEXT: # xmm11 = mem[3,1,2,3]
2918 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
2919 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm11[0,1,2,0,4,5,6,7]
2920 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
2921 ; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
2922 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
2923 ; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]
2924 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2925 ; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
2926 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
2927 ; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm13[0,1,3,1,4,5,6,7]
2928 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm8[0,1,3,1,4,5,6,7]
2929 ; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1]
2930 ; SSE-NEXT: movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
2931 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2932 ; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
2933 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload
2934 ; SSE-NEXT: # xmm13 = mem[3,1,2,3,4,5,6,7]
2935 ; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm1[0],xmm13[1],xmm1[1]
2936 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2937 ; SSE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
2938 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
2939 ; SSE-NEXT: # xmm8 = mem[0,1,3,1,4,5,6,7]
2940 ; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm1[0],xmm8[1],xmm1[1]
2941 ; SSE-NEXT: movsd {{.*#+}} xmm8 = xmm13[0],xmm8[1]
2942 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm7[3,1,2,3,4,5,6,7]
2943 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]
2944 ; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
2945 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm4[0,1,3,1,4,5,6,7]
2946 ; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm3[0,1,3,1,4,5,6,7]
2947 ; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm1[0],xmm13[1],xmm1[1]
2948 ; SSE-NEXT: movsd {{.*#+}} xmm13 = xmm6[0],xmm13[1]
2949 ; SSE-NEXT: pshuflw $231, (%rsp), %xmm1 # 16-byte Folded Reload
2950 ; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
2951 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
2952 ; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
2953 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
2954 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm15[0,1,3,1,4,5,6,7]
2955 ; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[0,1,3,1,4,5,6,7]
2956 ; SSE-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm1[0],xmm12[1],xmm1[1]
2957 ; SSE-NEXT: movsd {{.*#+}} xmm12 = xmm3[0],xmm12[1]
2958 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2959 ; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
2960 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
2961 ; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
2962 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
2963 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2964 ; SSE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
2965 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
2966 ; SSE-NEXT: # xmm15 = mem[0,1,3,1,4,5,6,7]
2967 ; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm1[0],xmm15[1],xmm1[1]
2968 ; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm3[0],xmm15[1]
2969 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2970 ; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
2971 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
2972 ; SSE-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7]
2973 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
2974 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2975 ; SSE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
2976 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
2977 ; SSE-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]
2978 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
2979 ; SSE-NEXT: movsd {{.*#+}} xmm3 = xmm4[0],xmm3[1]
2980 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2981 ; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
2982 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
2983 ; SSE-NEXT: # xmm6 = mem[3,1,2,3,4,5,6,7]
2984 ; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
2985 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2986 ; SSE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
2987 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm11[0,1,3,1,4,5,6,7]
2988 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
2989 ; SSE-NEXT: movsd {{.*#+}} xmm4 = xmm6[0],xmm4[1]
2990 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2991 ; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
2992 ; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
2993 ; SSE-NEXT: # xmm6 = mem[3,1,2,3,4,5,6,7]
2994 ; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
2995 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2996 ; SSE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
2997 ; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
2998 ; SSE-NEXT: # xmm7 = mem[0,1,3,1,4,5,6,7]
2999 ; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
3000 ; SSE-NEXT: movsd {{.*#+}} xmm7 = xmm6[0],xmm7[1]
3001 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3002 ; SSE-NEXT: movaps %xmm1, 96(%rsi)
3003 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3004 ; SSE-NEXT: movaps %xmm1, 32(%rsi)
3005 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3006 ; SSE-NEXT: movaps %xmm1, 112(%rsi)
3007 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3008 ; SSE-NEXT: movaps %xmm1, 48(%rsi)
3009 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3010 ; SSE-NEXT: movaps %xmm1, 64(%rsi)
3011 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3012 ; SSE-NEXT: movaps %xmm1, (%rsi)
3013 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3014 ; SSE-NEXT: movaps %xmm1, 80(%rsi)
3015 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3016 ; SSE-NEXT: movaps %xmm1, 16(%rsi)
3017 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3018 ; SSE-NEXT: movaps %xmm1, 96(%rdx)
3019 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3020 ; SSE-NEXT: movaps %xmm1, 32(%rdx)
3021 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3022 ; SSE-NEXT: movaps %xmm1, 112(%rdx)
3023 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3024 ; SSE-NEXT: movaps %xmm1, 48(%rdx)
3025 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3026 ; SSE-NEXT: movaps %xmm1, 64(%rdx)
3027 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3028 ; SSE-NEXT: movaps %xmm1, (%rdx)
3029 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3030 ; SSE-NEXT: movaps %xmm1, 80(%rdx)
3031 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3032 ; SSE-NEXT: movaps %xmm1, 16(%rdx)
3033 ; SSE-NEXT: movapd %xmm2, 96(%rcx)
3034 ; SSE-NEXT: movapd %xmm5, 32(%rcx)
3035 ; SSE-NEXT: movapd %xmm9, 112(%rcx)
3036 ; SSE-NEXT: movapd %xmm10, 48(%rcx)
3037 ; SSE-NEXT: movapd %xmm14, 64(%rcx)
3038 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3039 ; SSE-NEXT: movaps %xmm1, (%rcx)
3040 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3041 ; SSE-NEXT: movaps %xmm1, 80(%rcx)
3042 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3043 ; SSE-NEXT: movaps %xmm1, 16(%rcx)
3044 ; SSE-NEXT: movapd %xmm7, 112(%r8)
3045 ; SSE-NEXT: movapd %xmm4, 96(%r8)
3046 ; SSE-NEXT: movapd %xmm3, 80(%r8)
3047 ; SSE-NEXT: movapd %xmm15, 64(%r8)
3048 ; SSE-NEXT: movapd %xmm12, 48(%r8)
3049 ; SSE-NEXT: movapd %xmm13, 32(%r8)
3050 ; SSE-NEXT: movapd %xmm8, 16(%r8)
3051 ; SSE-NEXT: movapd %xmm0, (%r8)
3052 ; SSE-NEXT: addq $824, %rsp # imm = 0x338
3053 ; SSE-NEXT: retq
3054 ;
3055 ; AVX1-ONLY-LABEL: load_i16_stride4_vf64:
3056 ; AVX1-ONLY: # %bb.0:
3057 ; AVX1-ONLY-NEXT: subq $776, %rsp # imm = 0x308
3058 ; AVX1-ONLY-NEXT: vpxor %xmm10, %xmm10, %xmm10
3059 ; AVX1-ONLY-NEXT: vmovdqa 368(%rdi), %xmm8
3060 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm8[0],xmm10[1,2,3],xmm8[4],xmm10[5,6,7]
3061 ; AVX1-ONLY-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3062 ; AVX1-ONLY-NEXT: vmovdqa 352(%rdi), %xmm1
3063 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3064 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]
3065 ; AVX1-ONLY-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
3066 ; AVX1-ONLY-NEXT: vmovdqa 336(%rdi), %xmm4
3067 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0],xmm10[1,2,3],xmm4[4],xmm10[5,6,7]
3068 ; AVX1-ONLY-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3069 ; AVX1-ONLY-NEXT: vmovdqa 320(%rdi), %xmm7
3070 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm7[0],xmm10[1,2,3],xmm7[4],xmm10[5,6,7]
3071 ; AVX1-ONLY-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3072 ; AVX1-ONLY-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
3073 ; AVX1-ONLY-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
3074 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3075 ; AVX1-ONLY-NEXT: vmovdqa 304(%rdi), %xmm13
3076 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm13[0],xmm10[1,2,3],xmm13[4],xmm10[5,6,7]
3077 ; AVX1-ONLY-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3078 ; AVX1-ONLY-NEXT: vmovdqa 288(%rdi), %xmm1
3079 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3080 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]
3081 ; AVX1-ONLY-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
3082 ; AVX1-ONLY-NEXT: vmovdqa 272(%rdi), %xmm14
3083 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm14[0],xmm10[1,2,3],xmm14[4],xmm10[5,6,7]
3084 ; AVX1-ONLY-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3085 ; AVX1-ONLY-NEXT: vmovdqa 256(%rdi), %xmm2
3086 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3087 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm10[1,2,3],xmm2[4],xmm10[5,6,7]
3088 ; AVX1-ONLY-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
3089 ; AVX1-ONLY-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
3090 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3091 ; AVX1-ONLY-NEXT: vmovdqa 112(%rdi), %xmm0
3092 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3093 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]
3094 ; AVX1-ONLY-NEXT: vmovdqa 96(%rdi), %xmm0
3095 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3096 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]
3097 ; AVX1-ONLY-NEXT: vpackusdw %xmm2, %xmm3, %xmm2
3098 ; AVX1-ONLY-NEXT: vmovdqa 80(%rdi), %xmm0
3099 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]
3100 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, %xmm3
3101 ; AVX1-ONLY-NEXT: vmovdqa 64(%rdi), %xmm0
3102 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
3103 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm6 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]
3104 ; AVX1-ONLY-NEXT: vpackusdw %xmm5, %xmm6, %xmm5
3105 ; AVX1-ONLY-NEXT: vpackusdw %xmm2, %xmm5, %xmm2
3106 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3107 ; AVX1-ONLY-NEXT: vmovdqa 32(%rdi), %xmm0
3108 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3109 ; AVX1-ONLY-NEXT: vmovdqa 48(%rdi), %xmm1
3110 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3111 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]
3112 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm6 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]
3113 ; AVX1-ONLY-NEXT: vpackusdw %xmm5, %xmm6, %xmm5
3114 ; AVX1-ONLY-NEXT: vmovdqa (%rdi), %xmm0
3115 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3116 ; AVX1-ONLY-NEXT: vmovdqa 16(%rdi), %xmm1
3117 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3118 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]
3119 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm11 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]
3120 ; AVX1-ONLY-NEXT: vpackusdw %xmm9, %xmm11, %xmm9
3121 ; AVX1-ONLY-NEXT: vpackusdw %xmm5, %xmm9, %xmm5
3122 ; AVX1-ONLY-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3123 ; AVX1-ONLY-NEXT: vmovdqa 240(%rdi), %xmm0
3124 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3125 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]
3126 ; AVX1-ONLY-NEXT: vmovdqa 224(%rdi), %xmm15
3127 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm15[0],xmm10[1,2,3],xmm15[4],xmm10[5,6,7]
3128 ; AVX1-ONLY-NEXT: vmovdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3129 ; AVX1-ONLY-NEXT: vpackusdw %xmm5, %xmm9, %xmm5
3130 ; AVX1-ONLY-NEXT: vmovdqa 208(%rdi), %xmm0
3131 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3132 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]
3133 ; AVX1-ONLY-NEXT: vmovdqa 192(%rdi), %xmm0
3134 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm11 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]
3135 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3136 ; AVX1-ONLY-NEXT: vpackusdw %xmm9, %xmm11, %xmm9
3137 ; AVX1-ONLY-NEXT: vpackusdw %xmm5, %xmm9, %xmm5
3138 ; AVX1-ONLY-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3139 ; AVX1-ONLY-NEXT: vmovdqa 176(%rdi), %xmm2
3140 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3141 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm5 = xmm2[0],xmm10[1,2,3],xmm2[4],xmm10[5,6,7]
3142 ; AVX1-ONLY-NEXT: vmovdqa 160(%rdi), %xmm2
3143 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3144 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm2[0],xmm10[1,2,3],xmm2[4],xmm10[5,6,7]
3145 ; AVX1-ONLY-NEXT: vpackusdw %xmm5, %xmm9, %xmm5
3146 ; AVX1-ONLY-NEXT: vmovdqa 144(%rdi), %xmm9
3147 ; AVX1-ONLY-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3148 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0],xmm10[1,2,3],xmm9[4],xmm10[5,6,7]
3149 ; AVX1-ONLY-NEXT: vmovdqa 128(%rdi), %xmm11
3150 ; AVX1-ONLY-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3151 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0],xmm10[1,2,3],xmm11[4],xmm10[5,6,7]
3152 ; AVX1-ONLY-NEXT: vpackusdw %xmm9, %xmm11, %xmm9
3153 ; AVX1-ONLY-NEXT: vpackusdw %xmm5, %xmm9, %xmm5
3154 ; AVX1-ONLY-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3155 ; AVX1-ONLY-NEXT: vmovdqa 496(%rdi), %xmm1
3156 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3157 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]
3158 ; AVX1-ONLY-NEXT: vmovdqa 480(%rdi), %xmm1
3159 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3160 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm11 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]
3161 ; AVX1-ONLY-NEXT: vpackusdw %xmm9, %xmm11, %xmm9
3162 ; AVX1-ONLY-NEXT: vmovdqa 464(%rdi), %xmm1
3163 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3164 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm11 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]
3165 ; AVX1-ONLY-NEXT: vmovdqa 448(%rdi), %xmm1
3166 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3167 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm12 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]
3168 ; AVX1-ONLY-NEXT: vpackusdw %xmm11, %xmm12, %xmm11
3169 ; AVX1-ONLY-NEXT: vpackusdw %xmm9, %xmm11, %xmm9
3170 ; AVX1-ONLY-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3171 ; AVX1-ONLY-NEXT: vmovdqa 432(%rdi), %xmm2
3172 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3173 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm2[0],xmm10[1,2,3],xmm2[4],xmm10[5,6,7]
3174 ; AVX1-ONLY-NEXT: vmovdqa 416(%rdi), %xmm1
3175 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3176 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm11 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]
3177 ; AVX1-ONLY-NEXT: vpackusdw %xmm9, %xmm11, %xmm9
3178 ; AVX1-ONLY-NEXT: vmovdqa 400(%rdi), %xmm5
3179 ; AVX1-ONLY-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3180 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm11 = xmm5[0],xmm10[1,2,3],xmm5[4],xmm10[5,6,7]
3181 ; AVX1-ONLY-NEXT: vmovdqa 384(%rdi), %xmm2
3182 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3183 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm10 = xmm2[0],xmm10[1,2,3],xmm2[4],xmm10[5,6,7]
3184 ; AVX1-ONLY-NEXT: vpackusdw %xmm11, %xmm10, %xmm10
3185 ; AVX1-ONLY-NEXT: vpackusdw %xmm9, %xmm10, %xmm9
3186 ; AVX1-ONLY-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3187 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm9 = xmm8[0,2,2,3]
3188 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
3189 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
3190 ; AVX1-ONLY-NEXT: # xmm10 = mem[0,2,2,3]
3191 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]
3192 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
3193 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm10 = xmm4[0,2,2,3]
3194 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7]
3195 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm11 = xmm7[0,2,2,3]
3196 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]
3197 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
3198 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1,2,3],xmm9[4,5,6,7]
3199 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm10 = xmm13[0,2,2,3]
3200 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]
3201 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
3202 ; AVX1-ONLY-NEXT: # xmm11 = mem[0,2,2,3]
3203 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]
3204 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
3205 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm11 = xmm14[0,2,2,3]
3206 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]
3207 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
3208 ; AVX1-ONLY-NEXT: # xmm12 = mem[0,2,2,3]
3209 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]
3210 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
3211 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3],xmm10[4,5,6,7]
3212 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm9
3213 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]
3214 ; AVX1-ONLY-NEXT: vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3215 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
3216 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[0,2,2,3]
3217 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
3218 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
3219 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm10 = xmm2[0,2,2,3]
3220 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]
3221 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
3222 ; AVX1-ONLY-NEXT: vmovdqa %xmm3, %xmm4
3223 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm10 = xmm3[0,2,2,3]
3224 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7]
3225 ; AVX1-ONLY-NEXT: vmovdqa (%rsp), %xmm3 # 16-byte Reload
3226 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm11 = xmm3[0,2,2,3]
3227 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]
3228 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
3229 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1,2,3],xmm9[4,5,6,7]
3230 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
3231 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[0,2,2,3]
3232 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]
3233 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
3234 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm11 = xmm5[0,2,2,3]
3235 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]
3236 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
3237 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
3238 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm11 = xmm8[0,2,2,3]
3239 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]
3240 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
3241 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm12 = xmm7[0,2,2,3]
3242 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]
3243 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
3244 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3],xmm10[4,5,6,7]
3245 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm9
3246 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]
3247 ; AVX1-ONLY-NEXT: vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3248 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
3249 ; AVX1-ONLY-NEXT: # xmm9 = mem[0,2,2,3]
3250 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
3251 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm10 = xmm15[0,2,2,3]
3252 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]
3253 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
3254 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
3255 ; AVX1-ONLY-NEXT: # xmm10 = mem[0,2,2,3]
3256 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7]
3257 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm11 = xmm0[0,2,2,3]
3258 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]
3259 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
3260 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1,2,3],xmm9[4,5,6,7]
3261 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
3262 ; AVX1-ONLY-NEXT: # xmm10 = mem[0,2,2,3]
3263 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]
3264 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
3265 ; AVX1-ONLY-NEXT: # xmm11 = mem[0,2,2,3]
3266 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]
3267 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
3268 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
3269 ; AVX1-ONLY-NEXT: # xmm11 = mem[0,2,2,3]
3270 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]
3271 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
3272 ; AVX1-ONLY-NEXT: # xmm12 = mem[0,2,2,3]
3273 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]
3274 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
3275 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3],xmm10[4,5,6,7]
3276 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm9
3277 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm9[4,5,6,7]
3278 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3279 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
3280 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm9 = xmm13[0,2,2,3]
3281 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
3282 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
3283 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm10 = xmm14[0,2,2,3]
3284 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]
3285 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
3286 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
3287 ; AVX1-ONLY-NEXT: # xmm10 = mem[0,2,2,3]
3288 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7]
3289 ; AVX1-ONLY-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
3290 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm11 = xmm15[0,2,2,3]
3291 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]
3292 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
3293 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1,2,3],xmm9[4,5,6,7]
3294 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
3295 ; AVX1-ONLY-NEXT: # xmm10 = mem[0,2,2,3]
3296 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]
3297 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
3298 ; AVX1-ONLY-NEXT: # xmm11 = mem[0,2,2,3]
3299 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]
3300 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
3301 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
3302 ; AVX1-ONLY-NEXT: # xmm11 = mem[0,2,2,3]
3303 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]
3304 ; AVX1-ONLY-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
3305 ; AVX1-ONLY-NEXT: # xmm12 = mem[0,2,2,3]
3306 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]
3307 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
3308 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3],xmm10[4,5,6,7]
3309 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm9
3310 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm9[4,5,6,7]
3311 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3312 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]
3313 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3314 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
3315 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3316 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
3317 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
3318 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3319 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[3,1,2,3]
3320 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3321 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[3,1,2,3]
3322 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, (%rsp) # 16-byte Spill
3323 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
3324 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]
3325 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
3326 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
3327 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm1 = xmm6[3,1,2,3]
3328 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3329 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm5[3,1,2,3]
3330 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3331 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
3332 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
3333 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3334 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm8[3,1,2,3]
3335 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3336 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm3 = xmm7[3,1,2,3]
3337 ; AVX1-ONLY-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3338 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
3339 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,0,2,3,4,5,6,7]
3340 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
3341 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
3342 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
3343 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3344 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3345 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
3346 ; AVX1-ONLY-NEXT: # xmm0 = mem[3,1,2,3]
3347 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3348 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3349 ; AVX1-ONLY-NEXT: # xmm1 = mem[3,1,2,3]
3350 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3351 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
3352 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
3353 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3354 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3355 ; AVX1-ONLY-NEXT: # xmm1 = mem[3,1,2,3]
3356 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3357 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
3358 ; AVX1-ONLY-NEXT: # xmm2 = mem[3,1,2,3]
3359 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3360 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
3361 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
3362 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3363 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
3364 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3365 ; AVX1-ONLY-NEXT: # xmm1 = mem[3,1,2,3]
3366 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3367 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
3368 ; AVX1-ONLY-NEXT: # xmm2 = mem[3,1,2,3]
3369 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3370 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
3371 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
3372 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3373 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
3374 ; AVX1-ONLY-NEXT: # xmm12 = mem[3,1,2,3]
3375 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
3376 ; AVX1-ONLY-NEXT: # xmm11 = mem[3,1,2,3]
3377 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[2,0,2,3,4,5,6,7]
3378 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm11[2,0,2,3,4,5,6,7]
3379 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
3380 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
3381 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
3382 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3383 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3384 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm0 = xmm13[3,1,2,3]
3385 ; AVX1-ONLY-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3386 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[3,1,2,3]
3387 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3388 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
3389 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[0,1,2,0,4,5,6,7]
3390 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3391 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3392 ; AVX1-ONLY-NEXT: # xmm1 = mem[3,1,2,3]
3393 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3394 ; AVX1-ONLY-NEXT: vpshufd {{.*#+}} xmm2 = xmm15[3,1,2,3]
3395 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3396 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
3397 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
3398 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3399 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
3400 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3401 ; AVX1-ONLY-NEXT: # xmm1 = mem[3,1,2,3]
3402 ; AVX1-ONLY-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3403 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
3404 ; AVX1-ONLY-NEXT: # xmm2 = mem[3,1,2,3]
3405 ; AVX1-ONLY-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3406 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
3407 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
3408 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3409 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload
3410 ; AVX1-ONLY-NEXT: # xmm13 = mem[3,1,2,3]
3411 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
3412 ; AVX1-ONLY-NEXT: # xmm14 = mem[3,1,2,3]
3413 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm13[2,0,2,3,4,5,6,7]
3414 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm14[2,0,2,3,4,5,6,7]
3415 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
3416 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
3417 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
3418 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3419 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3420 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
3421 ; AVX1-ONLY-NEXT: # xmm10 = mem[3,1,2,3]
3422 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
3423 ; AVX1-ONLY-NEXT: # xmm9 = mem[3,1,2,3]
3424 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm0 = xmm10[0,1,2,0,4,5,6,7]
3425 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm9[0,1,2,0,4,5,6,7]
3426 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3427 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
3428 ; AVX1-ONLY-NEXT: # xmm8 = mem[3,1,2,3]
3429 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
3430 ; AVX1-ONLY-NEXT: # xmm7 = mem[3,1,2,3]
3431 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[2,0,2,3,4,5,6,7]
3432 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm7[2,0,2,3,4,5,6,7]
3433 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3434 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0,1,2,3],xmm0[4,5,6,7]
3435 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
3436 ; AVX1-ONLY-NEXT: # xmm6 = mem[3,1,2,3]
3437 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
3438 ; AVX1-ONLY-NEXT: # xmm5 = mem[3,1,2,3]
3439 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm6[0,1,2,0,4,5,6,7]
3440 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[0,1,2,0,4,5,6,7]
3441 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3442 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
3443 ; AVX1-ONLY-NEXT: # xmm4 = mem[3,1,2,3]
3444 ; AVX1-ONLY-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
3445 ; AVX1-ONLY-NEXT: # xmm2 = mem[3,1,2,3]
3446 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[2,0,2,3,4,5,6,7]
3447 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm15 = xmm2[2,0,2,3,4,5,6,7]
3448 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
3449 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
3450 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm1
3451 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
3452 ; AVX1-ONLY-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3453 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
3454 ; AVX1-ONLY-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
3455 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3456 ; AVX1-ONLY-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
3457 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3458 ; AVX1-ONLY-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3459 ; AVX1-ONLY-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
3460 ; AVX1-ONLY-NEXT: vpshuflw $231, (%rsp), %xmm3 # 16-byte Folded Reload
3461 ; AVX1-ONLY-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
3462 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
3463 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
3464 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3465 ; AVX1-ONLY-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
3466 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
3467 ; AVX1-ONLY-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]
3468 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
3469 ; AVX1-ONLY-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
3470 ; AVX1-ONLY-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
3471 ; AVX1-ONLY-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
3472 ; AVX1-ONLY-NEXT: # xmm15 = mem[3,1,2,3,4,5,6,7]
3473 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm15[0],xmm3[0],xmm15[1],xmm3[1]
3474 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
3475 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
3476 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm15 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3477 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
3478 ; AVX1-ONLY-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
3479 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3480 ; AVX1-ONLY-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
3481 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3482 ; AVX1-ONLY-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3483 ; AVX1-ONLY-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
3484 ; AVX1-ONLY-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
3485 ; AVX1-ONLY-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
3486 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
3487 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
3488 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3489 ; AVX1-ONLY-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
3490 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
3491 ; AVX1-ONLY-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]
3492 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
3493 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[3,1,2,3,4,5,6,7]
3494 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[3,1,2,3,4,5,6,7]
3495 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm11[0],xmm3[0],xmm11[1],xmm3[1]
3496 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
3497 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
3498 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm3 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3499 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm0 = xmm10[0,1,3,1,4,5,6,7]
3500 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm9[0,1,3,1,4,5,6,7]
3501 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3502 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[3,1,2,3,4,5,6,7]
3503 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[3,1,2,3,4,5,6,7]
3504 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
3505 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
3506 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm1 = xmm6[0,1,3,1,4,5,6,7]
3507 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
3508 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
3509 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7]
3510 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
3511 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
3512 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
3513 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
3514 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3515 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3516 ; AVX1-ONLY-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
3517 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
3518 ; AVX1-ONLY-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]
3519 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3520 ; AVX1-ONLY-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
3521 ; AVX1-ONLY-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]
3522 ; AVX1-ONLY-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
3523 ; AVX1-ONLY-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7]
3524 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
3525 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
3526 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
3527 ; AVX1-ONLY-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]
3528 ; AVX1-ONLY-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
3529 ; AVX1-ONLY-NEXT: # xmm4 = mem[0,1,3,1,4,5,6,7]
3530 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
3531 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[3,1,2,3,4,5,6,7]
3532 ; AVX1-ONLY-NEXT: vpshuflw {{.*#+}} xmm5 = xmm14[3,1,2,3,4,5,6,7]
3533 ; AVX1-ONLY-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
3534 ; AVX1-ONLY-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
3535 ; AVX1-ONLY-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
3536 ; AVX1-ONLY-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
3537 ; AVX1-ONLY-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
3538 ; AVX1-ONLY-NEXT: vmovaps %xmm2, 96(%rsi)
3539 ; AVX1-ONLY-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
3540 ; AVX1-ONLY-NEXT: vmovaps %xmm2, 112(%rsi)
3541 ; AVX1-ONLY-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
3542 ; AVX1-ONLY-NEXT: vmovaps %xmm2, 32(%rsi)
3543 ; AVX1-ONLY-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
3544 ; AVX1-ONLY-NEXT: vmovaps %xmm2, 48(%rsi)
3545 ; AVX1-ONLY-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
3546 ; AVX1-ONLY-NEXT: vmovaps %xmm2, (%rsi)
3547 ; AVX1-ONLY-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
3548 ; AVX1-ONLY-NEXT: vmovaps %xmm2, 16(%rsi)
3549 ; AVX1-ONLY-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
3550 ; AVX1-ONLY-NEXT: vmovaps %xmm2, 64(%rsi)
3551 ; AVX1-ONLY-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
3552 ; AVX1-ONLY-NEXT: vmovaps %xmm2, 80(%rsi)
3553 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3554 ; AVX1-ONLY-NEXT: vmovaps %ymm2, 96(%rdx)
3555 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3556 ; AVX1-ONLY-NEXT: vmovaps %ymm2, 32(%rdx)
3557 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3558 ; AVX1-ONLY-NEXT: vmovaps %ymm2, (%rdx)
3559 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3560 ; AVX1-ONLY-NEXT: vmovaps %ymm2, 64(%rdx)
3561 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3562 ; AVX1-ONLY-NEXT: vmovaps %ymm2, 32(%rcx)
3563 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3564 ; AVX1-ONLY-NEXT: vmovaps %ymm2, 96(%rcx)
3565 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3566 ; AVX1-ONLY-NEXT: vmovaps %ymm2, 64(%rcx)
3567 ; AVX1-ONLY-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3568 ; AVX1-ONLY-NEXT: vmovaps %ymm2, (%rcx)
3569 ; AVX1-ONLY-NEXT: vmovaps %ymm1, 96(%r8)
3570 ; AVX1-ONLY-NEXT: vmovaps %ymm0, 32(%r8)
3571 ; AVX1-ONLY-NEXT: vmovaps %ymm3, 64(%r8)
3572 ; AVX1-ONLY-NEXT: vmovaps %ymm15, (%r8)
3573 ; AVX1-ONLY-NEXT: addq $776, %rsp # imm = 0x308
3574 ; AVX1-ONLY-NEXT: vzeroupper
3575 ; AVX1-ONLY-NEXT: retq
3576 ;
3577 ; AVX2-SLOW-LABEL: load_i16_stride4_vf64:
3578 ; AVX2-SLOW: # %bb.0:
3579 ; AVX2-SLOW-NEXT: subq $696, %rsp # imm = 0x2B8
3580 ; AVX2-SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
3581 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3582 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
3583 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
3584 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3585 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
3586 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
3587 ; AVX2-SLOW-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
3588 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3589 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
3590 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
3591 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
3592 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3593 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3594 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm4
3595 ; AVX2-SLOW-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
3596 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm3, %xmm3
3597 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
3598 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
3599 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
3600 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3601 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3602 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
3603 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
3604 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3605 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
3606 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
3607 ; AVX2-SLOW-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
3608 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3609 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
3610 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
3611 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
3612 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3613 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3614 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm4
3615 ; AVX2-SLOW-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
3616 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm3, %xmm3
3617 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
3618 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
3619 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
3620 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3621 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3622 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
3623 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
3624 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3625 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
3626 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
3627 ; AVX2-SLOW-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
3628 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3629 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
3630 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
3631 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
3632 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3633 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3634 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm4
3635 ; AVX2-SLOW-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
3636 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm3, %xmm3
3637 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
3638 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
3639 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
3640 ; AVX2-SLOW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3641 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3642 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
3643 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
3644 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3645 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
3646 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
3647 ; AVX2-SLOW-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
3648 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3649 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
3650 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
3651 ; AVX2-SLOW-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
3652 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3653 ; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
3654 ; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm3
3655 ; AVX2-SLOW-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
3656 ; AVX2-SLOW-NEXT: vpackusdw %xmm0, %xmm0, %xmm0
3657 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3658 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
3659 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3660 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3661 ; AVX2-SLOW-NEXT: vmovdqa 256(%rdi), %xmm4
3662 ; AVX2-SLOW-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3663 ; AVX2-SLOW-NEXT: vmovdqa 272(%rdi), %xmm3
3664 ; AVX2-SLOW-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3665 ; AVX2-SLOW-NEXT: vmovdqa 288(%rdi), %xmm5
3666 ; AVX2-SLOW-NEXT: vmovdqa %xmm5, (%rsp) # 16-byte Spill
3667 ; AVX2-SLOW-NEXT: vmovdqa 304(%rdi), %xmm6
3668 ; AVX2-SLOW-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3669 ; AVX2-SLOW-NEXT: vmovdqa 368(%rdi), %xmm0
3670 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3671 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3672 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
3673 ; AVX2-SLOW-NEXT: vmovdqa 352(%rdi), %xmm1
3674 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3675 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
3676 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
3677 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3678 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3679 ; AVX2-SLOW-NEXT: vmovdqa 336(%rdi), %xmm1
3680 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3681 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
3682 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]
3683 ; AVX2-SLOW-NEXT: vmovdqa 320(%rdi), %xmm2
3684 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3685 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
3686 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
3687 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3688 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3689 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
3690 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm6[0,2,2,3]
3691 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
3692 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm5[0,2,2,3]
3693 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
3694 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3695 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
3696 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
3697 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
3698 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7]
3699 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
3700 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
3701 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3702 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3703 ; AVX2-SLOW-NEXT: vmovdqa 112(%rdi), %xmm0
3704 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3705 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3706 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
3707 ; AVX2-SLOW-NEXT: vmovdqa 96(%rdi), %xmm1
3708 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3709 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
3710 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
3711 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3712 ; AVX2-SLOW-NEXT: vmovdqa 80(%rdi), %xmm1
3713 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3714 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
3715 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]
3716 ; AVX2-SLOW-NEXT: vmovdqa 64(%rdi), %xmm2
3717 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3718 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
3719 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
3720 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3721 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3722 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3723 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
3724 ; AVX2-SLOW-NEXT: vmovdqa 48(%rdi), %xmm14
3725 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm14[0,2,2,3]
3726 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
3727 ; AVX2-SLOW-NEXT: vmovdqa 32(%rdi), %xmm8
3728 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm8[0,2,2,3]
3729 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
3730 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3731 ; AVX2-SLOW-NEXT: vmovdqa 16(%rdi), %xmm12
3732 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm12[0,2,2,3]
3733 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
3734 ; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm9
3735 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm9[0,2,2,3]
3736 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
3737 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
3738 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
3739 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3740 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3741 ; AVX2-SLOW-NEXT: vmovdqa 240(%rdi), %xmm0
3742 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3743 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
3744 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
3745 ; AVX2-SLOW-NEXT: vmovdqa 224(%rdi), %xmm1
3746 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3747 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
3748 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
3749 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3750 ; AVX2-SLOW-NEXT: vmovdqa 208(%rdi), %xmm1
3751 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3752 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
3753 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]
3754 ; AVX2-SLOW-NEXT: vmovdqa 192(%rdi), %xmm2
3755 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3756 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
3757 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
3758 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3759 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3760 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3761 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
3762 ; AVX2-SLOW-NEXT: vmovdqa 176(%rdi), %xmm1
3763 ; AVX2-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3764 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
3765 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
3766 ; AVX2-SLOW-NEXT: vmovdqa 160(%rdi), %xmm2
3767 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3768 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
3769 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
3770 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3771 ; AVX2-SLOW-NEXT: vmovdqa 144(%rdi), %xmm2
3772 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3773 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
3774 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
3775 ; AVX2-SLOW-NEXT: vmovdqa 128(%rdi), %xmm3
3776 ; AVX2-SLOW-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3777 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[0,2,2,3]
3778 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
3779 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
3780 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
3781 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3782 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3783 ; AVX2-SLOW-NEXT: vmovdqa 496(%rdi), %xmm11
3784 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm11[0,2,2,3]
3785 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
3786 ; AVX2-SLOW-NEXT: vmovdqa 480(%rdi), %xmm13
3787 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm13[0,2,2,3]
3788 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
3789 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3790 ; AVX2-SLOW-NEXT: vmovdqa 464(%rdi), %xmm15
3791 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm15[0,2,2,3]
3792 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]
3793 ; AVX2-SLOW-NEXT: vmovdqa 448(%rdi), %xmm6
3794 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
3795 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
3796 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
3797 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3798 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3799 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm0[6,7]
3800 ; AVX2-SLOW-NEXT: vmovdqa 432(%rdi), %xmm5
3801 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[0,2,2,3]
3802 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
3803 ; AVX2-SLOW-NEXT: vmovdqa 416(%rdi), %xmm4
3804 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[0,2,2,3]
3805 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
3806 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
3807 ; AVX2-SLOW-NEXT: vmovdqa 400(%rdi), %xmm7
3808 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm7[0,2,2,3]
3809 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm1[1,3,2,3,4,5,6,7]
3810 ; AVX2-SLOW-NEXT: vmovdqa 384(%rdi), %xmm1
3811 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm1[0,2,2,3]
3812 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7]
3813 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]
3814 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
3815 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
3816 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3817 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
3818 ; AVX2-SLOW-NEXT: # xmm0 = mem[3,1,2,3]
3819 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3820 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
3821 ; AVX2-SLOW-NEXT: # xmm2 = mem[3,1,2,3]
3822 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3823 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
3824 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
3825 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
3826 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
3827 ; AVX2-SLOW-NEXT: # xmm2 = mem[3,1,2,3]
3828 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3829 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
3830 ; AVX2-SLOW-NEXT: # xmm3 = mem[3,1,2,3]
3831 ; AVX2-SLOW-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3832 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
3833 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,0,2,3,4,5,6,7]
3834 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
3835 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3836 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3837 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
3838 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[3,1,2,3]
3839 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3840 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm8[3,1,2,3]
3841 ; AVX2-SLOW-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3842 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
3843 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,2,0,4,5,6,7]
3844 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
3845 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm12[3,1,2,3]
3846 ; AVX2-SLOW-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3847 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm9[3,1,2,3]
3848 ; AVX2-SLOW-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3849 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,0,2,3,4,5,6,7]
3850 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[2,0,2,3,4,5,6,7]
3851 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1]
3852 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
3853 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
3854 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3855 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
3856 ; AVX2-SLOW-NEXT: # xmm0 = mem[3,1,2,3]
3857 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3858 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
3859 ; AVX2-SLOW-NEXT: # xmm2 = mem[3,1,2,3]
3860 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3861 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
3862 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
3863 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
3864 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
3865 ; AVX2-SLOW-NEXT: # xmm2 = mem[3,1,2,3]
3866 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3867 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
3868 ; AVX2-SLOW-NEXT: # xmm3 = mem[3,1,2,3]
3869 ; AVX2-SLOW-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3870 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
3871 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,0,2,3,4,5,6,7]
3872 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
3873 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3874 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3875 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
3876 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
3877 ; AVX2-SLOW-NEXT: # xmm2 = mem[3,1,2,3]
3878 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3879 ; AVX2-SLOW-NEXT: vpshufd $231, (%rsp), %xmm3 # 16-byte Folded Reload
3880 ; AVX2-SLOW-NEXT: # xmm3 = mem[3,1,2,3]
3881 ; AVX2-SLOW-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3882 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
3883 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,2,0,4,5,6,7]
3884 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
3885 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
3886 ; AVX2-SLOW-NEXT: # xmm12 = mem[3,1,2,3]
3887 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
3888 ; AVX2-SLOW-NEXT: # xmm10 = mem[3,1,2,3]
3889 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[2,0,2,3,4,5,6,7]
3890 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm14 = xmm10[2,0,2,3,4,5,6,7]
3891 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm14[0],xmm3[0],xmm14[1],xmm3[1]
3892 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
3893 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
3894 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3895 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm11[3,1,2,3]
3896 ; AVX2-SLOW-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
3897 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm13[3,1,2,3]
3898 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3899 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
3900 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
3901 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
3902 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm15[3,1,2,3]
3903 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3904 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm6[3,1,2,3]
3905 ; AVX2-SLOW-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3906 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
3907 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,0,2,3,4,5,6,7]
3908 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
3909 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3910 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3911 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
3912 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm5[3,1,2,3]
3913 ; AVX2-SLOW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3914 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[3,1,2,3]
3915 ; AVX2-SLOW-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3916 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
3917 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,2,0,4,5,6,7]
3918 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
3919 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm7[3,1,2,3]
3920 ; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm1[3,1,2,3]
3921 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm13[2,0,2,3,4,5,6,7]
3922 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm11[2,0,2,3,4,5,6,7]
3923 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
3924 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3]
3925 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3926 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3927 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
3928 ; AVX2-SLOW-NEXT: # xmm9 = mem[3,1,2,3]
3929 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
3930 ; AVX2-SLOW-NEXT: # xmm8 = mem[3,1,2,3]
3931 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm9[0,1,2,0,4,5,6,7]
3932 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[0,1,2,0,4,5,6,7]
3933 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3934 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
3935 ; AVX2-SLOW-NEXT: # xmm7 = mem[3,1,2,3]
3936 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
3937 ; AVX2-SLOW-NEXT: # xmm6 = mem[3,1,2,3]
3938 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[2,0,2,3,4,5,6,7]
3939 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[2,0,2,3,4,5,6,7]
3940 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
3941 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3942 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3943 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm14 = ymm1[0,1,2,3,4,5],ymm0[6,7]
3944 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
3945 ; AVX2-SLOW-NEXT: # xmm5 = mem[3,1,2,3]
3946 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
3947 ; AVX2-SLOW-NEXT: # xmm4 = mem[3,1,2,3]
3948 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm5[0,1,2,0,4,5,6,7]
3949 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[0,1,2,0,4,5,6,7]
3950 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3951 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
3952 ; AVX2-SLOW-NEXT: # xmm3 = mem[3,1,2,3]
3953 ; AVX2-SLOW-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
3954 ; AVX2-SLOW-NEXT: # xmm2 = mem[3,1,2,3]
3955 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm3[2,0,2,3,4,5,6,7]
3956 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm15 = xmm2[2,0,2,3,4,5,6,7]
3957 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
3958 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
3959 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm14[4,5,6,7]
3960 ; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3961 ; AVX2-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
3962 ; AVX2-SLOW-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
3963 ; AVX2-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3964 ; AVX2-SLOW-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
3965 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3966 ; AVX2-SLOW-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3967 ; AVX2-SLOW-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
3968 ; AVX2-SLOW-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
3969 ; AVX2-SLOW-NEXT: # xmm14 = mem[3,1,2,3,4,5,6,7]
3970 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm14[0],xmm1[0],xmm14[1],xmm1[1]
3971 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3972 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3973 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
3974 ; AVX2-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3975 ; AVX2-SLOW-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
3976 ; AVX2-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
3977 ; AVX2-SLOW-NEXT: # xmm14 = mem[0,1,3,1,4,5,6,7]
3978 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm14[0],xmm1[0],xmm14[1],xmm1[1]
3979 ; AVX2-SLOW-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
3980 ; AVX2-SLOW-NEXT: # xmm14 = mem[3,1,2,3,4,5,6,7]
3981 ; AVX2-SLOW-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
3982 ; AVX2-SLOW-NEXT: # xmm15 = mem[3,1,2,3,4,5,6,7]
3983 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
3984 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm1 = xmm14[0,1],xmm1[2,3]
3985 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm15 = ymm1[0,1,2,3],ymm0[4,5,6,7]
3986 ; AVX2-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
3987 ; AVX2-SLOW-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
3988 ; AVX2-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3989 ; AVX2-SLOW-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
3990 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
3991 ; AVX2-SLOW-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
3992 ; AVX2-SLOW-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
3993 ; AVX2-SLOW-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
3994 ; AVX2-SLOW-NEXT: # xmm14 = mem[3,1,2,3,4,5,6,7]
3995 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm14[0],xmm1[0],xmm14[1],xmm1[1]
3996 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3997 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3998 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
3999 ; AVX2-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4000 ; AVX2-SLOW-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
4001 ; AVX2-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
4002 ; AVX2-SLOW-NEXT: # xmm14 = mem[0,1,3,1,4,5,6,7]
4003 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm14[0],xmm1[0],xmm14[1],xmm1[1]
4004 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[3,1,2,3,4,5,6,7]
4005 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[3,1,2,3,4,5,6,7]
4006 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1]
4007 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm1 = xmm10[0,1],xmm1[2,3]
4008 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4009 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm9[0,1,3,1,4,5,6,7]
4010 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,3,1,4,5,6,7]
4011 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
4012 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[3,1,2,3,4,5,6,7]
4013 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]
4014 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
4015 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4016 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
4017 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3,4,5],ymm0[6,7]
4018 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
4019 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
4020 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
4021 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
4022 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
4023 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
4024 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3]
4025 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
4026 ; AVX2-SLOW-NEXT: vpshuflw $116, (%rsp), %xmm2 # 16-byte Folded Reload
4027 ; AVX2-SLOW-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]
4028 ; AVX2-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4029 ; AVX2-SLOW-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]
4030 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
4031 ; AVX2-SLOW-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4032 ; AVX2-SLOW-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
4033 ; AVX2-SLOW-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
4034 ; AVX2-SLOW-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7]
4035 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
4036 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
4037 ; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
4038 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
4039 ; AVX2-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4040 ; AVX2-SLOW-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]
4041 ; AVX2-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
4042 ; AVX2-SLOW-NEXT: # xmm4 = mem[0,1,3,1,4,5,6,7]
4043 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
4044 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[3,1,2,3,4,5,6,7]
4045 ; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm11[3,1,2,3,4,5,6,7]
4046 ; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
4047 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
4048 ; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
4049 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4050 ; AVX2-SLOW-NEXT: vmovaps %ymm3, 32(%rsi)
4051 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4052 ; AVX2-SLOW-NEXT: vmovaps %ymm3, 96(%rsi)
4053 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4054 ; AVX2-SLOW-NEXT: vmovaps %ymm3, 64(%rsi)
4055 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4056 ; AVX2-SLOW-NEXT: vmovaps %ymm3, (%rsi)
4057 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4058 ; AVX2-SLOW-NEXT: vmovaps %ymm3, 96(%rdx)
4059 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4060 ; AVX2-SLOW-NEXT: vmovaps %ymm3, 32(%rdx)
4061 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4062 ; AVX2-SLOW-NEXT: vmovaps %ymm3, (%rdx)
4063 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4064 ; AVX2-SLOW-NEXT: vmovaps %ymm3, 64(%rdx)
4065 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4066 ; AVX2-SLOW-NEXT: vmovaps %ymm3, 32(%rcx)
4067 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4068 ; AVX2-SLOW-NEXT: vmovaps %ymm3, 96(%rcx)
4069 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4070 ; AVX2-SLOW-NEXT: vmovaps %ymm3, 64(%rcx)
4071 ; AVX2-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4072 ; AVX2-SLOW-NEXT: vmovaps %ymm3, (%rcx)
4073 ; AVX2-SLOW-NEXT: vmovdqa %ymm2, 96(%r8)
4074 ; AVX2-SLOW-NEXT: vmovdqa %ymm0, 32(%r8)
4075 ; AVX2-SLOW-NEXT: vmovdqa %ymm1, 64(%r8)
4076 ; AVX2-SLOW-NEXT: vmovdqa %ymm15, (%r8)
4077 ; AVX2-SLOW-NEXT: addq $696, %rsp # imm = 0x2B8
4078 ; AVX2-SLOW-NEXT: vzeroupper
4079 ; AVX2-SLOW-NEXT: retq
4080 ;
4081 ; AVX2-FAST-LABEL: load_i16_stride4_vf64:
4082 ; AVX2-FAST: # %bb.0:
4083 ; AVX2-FAST-NEXT: subq $680, %rsp # imm = 0x2A8
4084 ; AVX2-FAST-NEXT: vmovdqa 320(%rdi), %ymm6
4085 ; AVX2-FAST-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4086 ; AVX2-FAST-NEXT: vmovdqa 352(%rdi), %ymm7
4087 ; AVX2-FAST-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4088 ; AVX2-FAST-NEXT: vmovdqa 64(%rdi), %ymm5
4089 ; AVX2-FAST-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4090 ; AVX2-FAST-NEXT: vmovdqa 96(%rdi), %ymm4
4091 ; AVX2-FAST-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4092 ; AVX2-FAST-NEXT: vpxor %xmm1, %xmm1, %xmm1
4093 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]
4094 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm2
4095 ; AVX2-FAST-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
4096 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]
4097 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm3
4098 ; AVX2-FAST-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
4099 ; AVX2-FAST-NEXT: vpackusdw %xmm0, %xmm2, %xmm0
4100 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = [0,2,2,3,0,2,4,6]
4101 ; AVX2-FAST-NEXT: vpermd %ymm4, %ymm2, %ymm3
4102 ; AVX2-FAST-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4103 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} ymm4 = [16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29]
4104 ; AVX2-FAST-NEXT: vpshufb %ymm4, %ymm3, %ymm3
4105 ; AVX2-FAST-NEXT: vmovdqa %ymm4, %ymm8
4106 ; AVX2-FAST-NEXT: vpermd %ymm5, %ymm2, %ymm10
4107 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm5 = <0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u>
4108 ; AVX2-FAST-NEXT: vpshufb %ymm5, %ymm10, %ymm4
4109 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
4110 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
4111 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4112 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]
4113 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
4114 ; AVX2-FAST-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
4115 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]
4116 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm3, %xmm4
4117 ; AVX2-FAST-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
4118 ; AVX2-FAST-NEXT: vpackusdw %xmm0, %xmm3, %xmm0
4119 ; AVX2-FAST-NEXT: vpermd %ymm7, %ymm2, %ymm3
4120 ; AVX2-FAST-NEXT: vpshufb %ymm8, %ymm3, %ymm4
4121 ; AVX2-FAST-NEXT: vpermd %ymm6, %ymm2, %ymm7
4122 ; AVX2-FAST-NEXT: vpshufb %ymm5, %ymm7, %ymm6
4123 ; AVX2-FAST-NEXT: vmovdqa %ymm5, %ymm9
4124 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1,2,3,4,5],ymm4[6,7]
4125 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
4126 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4127 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]
4128 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm4
4129 ; AVX2-FAST-NEXT: vpackusdw %xmm4, %xmm0, %xmm0
4130 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm4 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]
4131 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm4, %xmm6
4132 ; AVX2-FAST-NEXT: vpackusdw %xmm6, %xmm4, %xmm4
4133 ; AVX2-FAST-NEXT: vpackusdw %xmm0, %xmm4, %xmm0
4134 ; AVX2-FAST-NEXT: vmovdqa 448(%rdi), %ymm5
4135 ; AVX2-FAST-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4136 ; AVX2-FAST-NEXT: vmovdqa 480(%rdi), %ymm4
4137 ; AVX2-FAST-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4138 ; AVX2-FAST-NEXT: vpermd %ymm4, %ymm2, %ymm4
4139 ; AVX2-FAST-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4140 ; AVX2-FAST-NEXT: vpshufb %ymm8, %ymm4, %ymm4
4141 ; AVX2-FAST-NEXT: vpermd %ymm5, %ymm2, %ymm12
4142 ; AVX2-FAST-NEXT: vpshufb %ymm9, %ymm12, %ymm6
4143 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1,2,3,4,5],ymm4[6,7]
4144 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
4145 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4146 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]
4147 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm4
4148 ; AVX2-FAST-NEXT: vpackusdw %xmm4, %xmm0, %xmm0
4149 ; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]
4150 ; AVX2-FAST-NEXT: vextracti128 $1, %ymm1, %xmm4
4151 ; AVX2-FAST-NEXT: vpackusdw %xmm4, %xmm1, %xmm1
4152 ; AVX2-FAST-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
4153 ; AVX2-FAST-NEXT: vmovdqa 192(%rdi), %ymm1
4154 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4155 ; AVX2-FAST-NEXT: vmovdqa 224(%rdi), %ymm4
4156 ; AVX2-FAST-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4157 ; AVX2-FAST-NEXT: vpermd %ymm4, %ymm2, %ymm5
4158 ; AVX2-FAST-NEXT: vpermd %ymm1, %ymm2, %ymm11
4159 ; AVX2-FAST-NEXT: vpshufb %ymm8, %ymm5, %ymm2
4160 ; AVX2-FAST-NEXT: vpshufb %ymm9, %ymm11, %ymm4
4161 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
4162 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
4163 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4164 ; AVX2-FAST-NEXT: vmovdqa 288(%rdi), %xmm1
4165 ; AVX2-FAST-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4166 ; AVX2-FAST-NEXT: vmovdqa 304(%rdi), %xmm0
4167 ; AVX2-FAST-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4168 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]
4169 ; AVX2-FAST-NEXT: vpshufb %xmm4, %xmm0, %xmm0
4170 ; AVX2-FAST-NEXT: vpshufb %xmm4, %xmm1, %xmm6
4171 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm6[0],xmm0[0],xmm6[1],xmm0[1]
4172 ; AVX2-FAST-NEXT: vmovdqa 256(%rdi), %xmm0
4173 ; AVX2-FAST-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4174 ; AVX2-FAST-NEXT: vmovdqa 272(%rdi), %xmm9
4175 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
4176 ; AVX2-FAST-NEXT: vpshufb %xmm1, %xmm9, %xmm14
4177 ; AVX2-FAST-NEXT: vpshufb %xmm1, %xmm0, %xmm15
4178 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
4179 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3]
4180 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} ymm6 = [18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
4181 ; AVX2-FAST-NEXT: vpshufb %ymm6, %ymm3, %ymm3
4182 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm8 = <2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,u,u,u,u,u,u,u,u>
4183 ; AVX2-FAST-NEXT: vpshufb %ymm8, %ymm7, %ymm7
4184 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0,1,2,3,4,5],ymm3[6,7]
4185 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3],ymm3[4,5,6,7]
4186 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4187 ; AVX2-FAST-NEXT: vmovdqa 32(%rdi), %xmm13
4188 ; AVX2-FAST-NEXT: vmovdqa 48(%rdi), %xmm7
4189 ; AVX2-FAST-NEXT: vpshufb %xmm4, %xmm7, %xmm3
4190 ; AVX2-FAST-NEXT: vpshufb %xmm4, %xmm13, %xmm14
4191 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm15 = xmm14[0],xmm3[0],xmm14[1],xmm3[1]
4192 ; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm3
4193 ; AVX2-FAST-NEXT: vmovdqa 16(%rdi), %xmm14
4194 ; AVX2-FAST-NEXT: vpshufb %xmm1, %xmm14, %xmm0
4195 ; AVX2-FAST-NEXT: vpshufb %xmm1, %xmm3, %xmm2
4196 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
4197 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm15[2,3]
4198 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4199 ; AVX2-FAST-NEXT: vpshufb %ymm6, %ymm2, %ymm2
4200 ; AVX2-FAST-NEXT: vpshufb %ymm8, %ymm10, %ymm10
4201 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4,5],ymm2[6,7]
4202 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
4203 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4204 ; AVX2-FAST-NEXT: vmovdqa 160(%rdi), %xmm10
4205 ; AVX2-FAST-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4206 ; AVX2-FAST-NEXT: vmovdqa 176(%rdi), %xmm0
4207 ; AVX2-FAST-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
4208 ; AVX2-FAST-NEXT: vpshufb %xmm4, %xmm0, %xmm0
4209 ; AVX2-FAST-NEXT: vpshufb %xmm4, %xmm10, %xmm2
4210 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
4211 ; AVX2-FAST-NEXT: vmovdqa 128(%rdi), %xmm15
4212 ; AVX2-FAST-NEXT: vmovdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4213 ; AVX2-FAST-NEXT: vmovdqa 144(%rdi), %xmm2
4214 ; AVX2-FAST-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4215 ; AVX2-FAST-NEXT: vpshufb %xmm1, %xmm2, %xmm2
4216 ; AVX2-FAST-NEXT: vpshufb %xmm1, %xmm15, %xmm10
4217 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm10[0],xmm2[0],xmm10[1],xmm2[1]
4218 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
4219 ; AVX2-FAST-NEXT: vpshufb %ymm6, %ymm5, %ymm2
4220 ; AVX2-FAST-NEXT: vpshufb %ymm8, %ymm11, %ymm5
4221 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
4222 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
4223 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4224 ; AVX2-FAST-NEXT: vmovdqa 416(%rdi), %xmm5
4225 ; AVX2-FAST-NEXT: vmovdqa 432(%rdi), %xmm10
4226 ; AVX2-FAST-NEXT: vpshufb %xmm4, %xmm10, %xmm0
4227 ; AVX2-FAST-NEXT: vpshufb %xmm4, %xmm5, %xmm2
4228 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
4229 ; AVX2-FAST-NEXT: vmovdqa 384(%rdi), %xmm4
4230 ; AVX2-FAST-NEXT: vmovdqa 400(%rdi), %xmm2
4231 ; AVX2-FAST-NEXT: vpshufb %xmm1, %xmm2, %xmm11
4232 ; AVX2-FAST-NEXT: vpshufb %xmm1, %xmm4, %xmm1
4233 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm11[0],xmm1[1],xmm11[1]
4234 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
4235 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
4236 ; AVX2-FAST-NEXT: vpshufb %ymm6, %ymm1, %ymm1
4237 ; AVX2-FAST-NEXT: vpshufb %ymm8, %ymm12, %ymm11
4238 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5],ymm1[6,7]
4239 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
4240 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4241 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,2,3,1,3,5,7]
4242 ; AVX2-FAST-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
4243 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4244 ; AVX2-FAST-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
4245 ; AVX2-FAST-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4246 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} ymm12 = [16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29]
4247 ; AVX2-FAST-NEXT: vpshufb %ymm12, %ymm1, %ymm1
4248 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm15 = <0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u>
4249 ; AVX2-FAST-NEXT: vpshufb %ymm15, %ymm6, %ymm11
4250 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5],ymm1[6,7]
4251 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm7[3,1,2,3]
4252 ; AVX2-FAST-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4253 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm8 = xmm13[3,1,2,3]
4254 ; AVX2-FAST-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4255 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm7 = xmm6[0,1,2,0,4,5,6,7]
4256 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm11 = xmm8[0,1,2,0,4,5,6,7]
4257 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm11[0],xmm7[0],xmm11[1],xmm7[1]
4258 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm14[3,1,2,3]
4259 ; AVX2-FAST-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4260 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm8 = xmm3[3,1,2,3]
4261 ; AVX2-FAST-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4262 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm6[2,0,2,3,4,5,6,7]
4263 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm11 = xmm8[2,0,2,3,4,5,6,7]
4264 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm11[0],xmm3[0],xmm11[1],xmm3[1]
4265 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm7[2,3]
4266 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
4267 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4268 ; AVX2-FAST-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
4269 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4270 ; AVX2-FAST-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
4271 ; AVX2-FAST-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4272 ; AVX2-FAST-NEXT: vpshufb %ymm12, %ymm1, %ymm1
4273 ; AVX2-FAST-NEXT: vmovdqa %ymm12, %ymm6
4274 ; AVX2-FAST-NEXT: vpshufb %ymm15, %ymm3, %ymm3
4275 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
4276 ; AVX2-FAST-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4277 ; AVX2-FAST-NEXT: # xmm3 = mem[3,1,2,3]
4278 ; AVX2-FAST-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4279 ; AVX2-FAST-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
4280 ; AVX2-FAST-NEXT: # xmm14 = mem[3,1,2,3]
4281 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,2,0,4,5,6,7]
4282 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm7 = xmm14[0,1,2,0,4,5,6,7]
4283 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1]
4284 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm13 = xmm9[3,1,2,3]
4285 ; AVX2-FAST-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
4286 ; AVX2-FAST-NEXT: # xmm11 = mem[3,1,2,3]
4287 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm9 = xmm13[2,0,2,3,4,5,6,7]
4288 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm12 = xmm11[2,0,2,3,4,5,6,7]
4289 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1]
4290 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm3 = xmm9[0,1],xmm3[2,3]
4291 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
4292 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4293 ; AVX2-FAST-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
4294 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4295 ; AVX2-FAST-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
4296 ; AVX2-FAST-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4297 ; AVX2-FAST-NEXT: vpshufb %ymm6, %ymm1, %ymm1
4298 ; AVX2-FAST-NEXT: vmovdqa %ymm15, %ymm7
4299 ; AVX2-FAST-NEXT: vpshufb %ymm15, %ymm3, %ymm3
4300 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
4301 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm10[3,1,2,3]
4302 ; AVX2-FAST-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4303 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
4304 ; AVX2-FAST-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4305 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,2,0,4,5,6,7]
4306 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,2,0,4,5,6,7]
4307 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
4308 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
4309 ; AVX2-FAST-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4310 ; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm15 = xmm4[3,1,2,3]
4311 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
4312 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm4 = xmm15[2,0,2,3,4,5,6,7]
4313 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
4314 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3]
4315 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
4316 ; AVX2-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4317 ; AVX2-FAST-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
4318 ; AVX2-FAST-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
4319 ; AVX2-FAST-NEXT: vpshufb %ymm6, %ymm9, %ymm0
4320 ; AVX2-FAST-NEXT: vpshufb %ymm7, %ymm8, %ymm1
4321 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm0[6,7]
4322 ; AVX2-FAST-NEXT: vpshufd $231, (%rsp), %xmm7 # 16-byte Folded Reload
4323 ; AVX2-FAST-NEXT: # xmm7 = mem[3,1,2,3]
4324 ; AVX2-FAST-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
4325 ; AVX2-FAST-NEXT: # xmm5 = mem[3,1,2,3]
4326 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[0,1,2,0,4,5,6,7]
4327 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[0,1,2,0,4,5,6,7]
4328 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
4329 ; AVX2-FAST-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
4330 ; AVX2-FAST-NEXT: # xmm4 = mem[3,1,2,3]
4331 ; AVX2-FAST-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
4332 ; AVX2-FAST-NEXT: # xmm6 = mem[3,1,2,3]
4333 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[2,0,2,3,4,5,6,7]
4334 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm12 = xmm6[2,0,2,3,4,5,6,7]
4335 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm12[0],xmm0[0],xmm12[1],xmm0[1]
4336 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
4337 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
4338 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4339 ; AVX2-FAST-NEXT: vpbroadcastq {{.*#+}} ymm10 = [18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
4340 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
4341 ; AVX2-FAST-NEXT: vpshufb %ymm10, %ymm0, %ymm0
4342 ; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = <2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,u,u,u,u,u,u,u,u>
4343 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
4344 ; AVX2-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm1
4345 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
4346 ; AVX2-FAST-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4347 ; AVX2-FAST-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
4348 ; AVX2-FAST-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4349 ; AVX2-FAST-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]
4350 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
4351 ; AVX2-FAST-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4352 ; AVX2-FAST-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
4353 ; AVX2-FAST-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
4354 ; AVX2-FAST-NEXT: # xmm12 = mem[3,1,2,3,4,5,6,7]
4355 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm12[0],xmm3[0],xmm12[1],xmm3[1]
4356 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
4357 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4358 ; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4359 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
4360 ; AVX2-FAST-NEXT: vpshufb %ymm10, %ymm0, %ymm0
4361 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
4362 ; AVX2-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm1
4363 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
4364 ; AVX2-FAST-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4365 ; AVX2-FAST-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
4366 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm14[0,1,3,1,4,5,6,7]
4367 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
4368 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm13[3,1,2,3,4,5,6,7]
4369 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[3,1,2,3,4,5,6,7]
4370 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm11[0],xmm3[0],xmm11[1],xmm3[1]
4371 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
4372 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4373 ; AVX2-FAST-NEXT: vpshufb %ymm10, %ymm9, %ymm1
4374 ; AVX2-FAST-NEXT: vmovdqa %ymm2, %ymm9
4375 ; AVX2-FAST-NEXT: vpshufb %ymm2, %ymm8, %ymm3
4376 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
4377 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm3 = xmm7[0,1,3,1,4,5,6,7]
4378 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
4379 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
4380 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7]
4381 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[3,1,2,3,4,5,6,7]
4382 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
4383 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3]
4384 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
4385 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4386 ; AVX2-FAST-NEXT: vpshufb %ymm10, %ymm2, %ymm2
4387 ; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4388 ; AVX2-FAST-NEXT: vpshufb %ymm9, %ymm3, %ymm3
4389 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
4390 ; AVX2-FAST-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4391 ; AVX2-FAST-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]
4392 ; AVX2-FAST-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
4393 ; AVX2-FAST-NEXT: # xmm4 = mem[0,1,3,1,4,5,6,7]
4394 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
4395 ; AVX2-FAST-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
4396 ; AVX2-FAST-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7]
4397 ; AVX2-FAST-NEXT: vpshuflw {{.*#+}} xmm5 = xmm15[3,1,2,3,4,5,6,7]
4398 ; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
4399 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
4400 ; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
4401 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4402 ; AVX2-FAST-NEXT: vmovaps %ymm3, 32(%rsi)
4403 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4404 ; AVX2-FAST-NEXT: vmovaps %ymm3, 96(%rsi)
4405 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4406 ; AVX2-FAST-NEXT: vmovaps %ymm3, 64(%rsi)
4407 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4408 ; AVX2-FAST-NEXT: vmovaps %ymm3, (%rsi)
4409 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4410 ; AVX2-FAST-NEXT: vmovaps %ymm3, 96(%rdx)
4411 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4412 ; AVX2-FAST-NEXT: vmovaps %ymm3, 32(%rdx)
4413 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4414 ; AVX2-FAST-NEXT: vmovaps %ymm3, (%rdx)
4415 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4416 ; AVX2-FAST-NEXT: vmovaps %ymm3, 64(%rdx)
4417 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4418 ; AVX2-FAST-NEXT: vmovaps %ymm3, 32(%rcx)
4419 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4420 ; AVX2-FAST-NEXT: vmovaps %ymm3, 96(%rcx)
4421 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4422 ; AVX2-FAST-NEXT: vmovaps %ymm3, 64(%rcx)
4423 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
4424 ; AVX2-FAST-NEXT: vmovaps %ymm3, (%rcx)
4425 ; AVX2-FAST-NEXT: vmovdqa %ymm2, 96(%r8)
4426 ; AVX2-FAST-NEXT: vmovdqa %ymm1, 32(%r8)
4427 ; AVX2-FAST-NEXT: vmovdqa %ymm0, 64(%r8)
4428 ; AVX2-FAST-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
4429 ; AVX2-FAST-NEXT: vmovaps %ymm0, (%r8)
4430 ; AVX2-FAST-NEXT: addq $680, %rsp # imm = 0x2A8
4431 ; AVX2-FAST-NEXT: vzeroupper
4432 ; AVX2-FAST-NEXT: retq
4433 ;
4434 ; AVX2-FAST-PERLANE-LABEL: load_i16_stride4_vf64:
4435 ; AVX2-FAST-PERLANE: # %bb.0:
4436 ; AVX2-FAST-PERLANE-NEXT: subq $712, %rsp # imm = 0x2C8
4437 ; AVX2-FAST-PERLANE-NEXT: vpxor %xmm0, %xmm0, %xmm0
4438 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4439 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm1, %xmm2
4440 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
4441 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4442 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
4443 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
4444 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
4445 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4446 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
4447 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
4448 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
4449 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
4450 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4451 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm3, %xmm4
4452 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
4453 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm3, %xmm3
4454 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
4455 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
4456 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
4457 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4458 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4459 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm1, %xmm2
4460 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
4461 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4462 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
4463 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
4464 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
4465 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4466 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
4467 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
4468 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
4469 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
4470 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4471 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm3, %xmm4
4472 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
4473 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm3, %xmm3
4474 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
4475 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
4476 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
4477 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4478 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4479 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm1, %xmm2
4480 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
4481 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4482 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
4483 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
4484 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
4485 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4486 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
4487 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
4488 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
4489 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
4490 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4491 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm3, %xmm4
4492 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
4493 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm3, %xmm3
4494 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
4495 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
4496 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
4497 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4498 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4499 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm1, %xmm2
4500 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
4501 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4502 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
4503 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
4504 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
4505 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4506 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm2, %xmm3
4507 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
4508 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm2, %xmm2, %xmm2
4509 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
4510 ; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]
4511 ; AVX2-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm3
4512 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
4513 ; AVX2-FAST-PERLANE-NEXT: vpackusdw %xmm0, %xmm0, %xmm0
4514 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4515 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
4516 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4517 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4518 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 256(%rdi), %xmm5
4519 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4520 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 288(%rdi), %xmm3
4521 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm3, (%rsp) # 16-byte Spill
4522 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 304(%rdi), %xmm7
4523 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4524 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]
4525 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 368(%rdi), %xmm0
4526 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4527 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm0, %xmm0
4528 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 352(%rdi), %xmm1
4529 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4530 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm1, %xmm1
4531 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
4532 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4533 ; AVX2-FAST-PERLANE-NEXT: vmovdqa {{.*#+}} xmm6 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
4534 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 336(%rdi), %xmm1
4535 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4536 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm1, %xmm1
4537 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 320(%rdi), %xmm2
4538 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4539 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm2, %xmm2
4540 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
4541 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4542 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
4543 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm7, %xmm1
4544 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm3, %xmm2
4545 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
4546 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 272(%rdi), %xmm2
4547 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4548 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm2, %xmm2
4549 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm5, %xmm3
4550 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
4551 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
4552 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4553 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4554 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 112(%rdi), %xmm5
4555 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm5, %xmm2
4556 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 96(%rdi), %xmm11
4557 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm11, %xmm3
4558 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
4559 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 80(%rdi), %xmm12
4560 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm12, %xmm7
4561 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 64(%rdi), %xmm15
4562 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm15, %xmm8
4563 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
4564 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
4565 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
4566 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],ymm3[6,7]
4567 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %xmm0
4568 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4569 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 48(%rdi), %xmm3
4570 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm3, %xmm8
4571 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm0, %xmm9
4572 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
4573 ; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rdi), %xmm0
4574 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4575 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm1
4576 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4577 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm1, %xmm9
4578 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm0, %xmm10
4579 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
4580 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
4581 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm7[4,5,6,7]
4582 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4583 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 240(%rdi), %xmm0
4584 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4585 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm0, %xmm7
4586 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 224(%rdi), %xmm0
4587 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4588 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm0, %xmm8
4589 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
4590 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 208(%rdi), %xmm0
4591 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4592 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm0, %xmm8
4593 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 192(%rdi), %xmm0
4594 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4595 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm0, %xmm9
4596 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
4597 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
4598 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
4599 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5],ymm7[6,7]
4600 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 160(%rdi), %xmm0
4601 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4602 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 176(%rdi), %xmm1
4603 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4604 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm1, %xmm8
4605 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm0, %xmm9
4606 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
4607 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 128(%rdi), %xmm0
4608 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4609 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 144(%rdi), %xmm1
4610 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4611 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm1, %xmm9
4612 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm0, %xmm10
4613 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
4614 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
4615 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm7[4,5,6,7]
4616 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4617 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 496(%rdi), %xmm8
4618 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm8, %xmm9
4619 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 480(%rdi), %xmm7
4620 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm7, %xmm10
4621 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
4622 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 464(%rdi), %xmm10
4623 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm10, %xmm14
4624 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 448(%rdi), %xmm0
4625 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4626 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm0, %xmm9
4627 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm14[0],xmm9[1],xmm14[1]
4628 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
4629 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
4630 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm13[6,7]
4631 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 416(%rdi), %xmm14
4632 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 432(%rdi), %xmm13
4633 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm13, %xmm0
4634 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm4, %xmm14, %xmm4
4635 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
4636 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 384(%rdi), %xmm0
4637 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4638 ; AVX2-FAST-PERLANE-NEXT: vmovdqa 400(%rdi), %xmm4
4639 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm4, %xmm1
4640 ; AVX2-FAST-PERLANE-NEXT: vpshufb %xmm6, %xmm0, %xmm6
4641 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
4642 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
4643 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm9[4,5,6,7]
4644 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4645 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm0 = xmm5[3,1,2,3]
4646 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4647 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm11[3,1,2,3]
4648 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4649 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
4650 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
4651 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
4652 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm12[3,1,2,3]
4653 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4654 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm2 = xmm15[3,1,2,3]
4655 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4656 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
4657 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
4658 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
4659 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4660 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4661 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
4662 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[3,1,2,3]
4663 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4664 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4665 ; AVX2-FAST-PERLANE-NEXT: # xmm2 = mem[3,1,2,3]
4666 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4667 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
4668 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
4669 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
4670 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4671 ; AVX2-FAST-PERLANE-NEXT: # xmm2 = mem[3,1,2,3]
4672 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4673 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4674 ; AVX2-FAST-PERLANE-NEXT: # xmm3 = mem[3,1,2,3]
4675 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4676 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
4677 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,0,2,3,4,5,6,7]
4678 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
4679 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
4680 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4681 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4682 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4683 ; AVX2-FAST-PERLANE-NEXT: # xmm0 = mem[3,1,2,3]
4684 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4685 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4686 ; AVX2-FAST-PERLANE-NEXT: # xmm1 = mem[3,1,2,3]
4687 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4688 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
4689 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
4690 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
4691 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4692 ; AVX2-FAST-PERLANE-NEXT: # xmm1 = mem[3,1,2,3]
4693 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4694 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4695 ; AVX2-FAST-PERLANE-NEXT: # xmm2 = mem[3,1,2,3]
4696 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4697 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
4698 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
4699 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
4700 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4701 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4702 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
4703 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4704 ; AVX2-FAST-PERLANE-NEXT: # xmm1 = mem[3,1,2,3]
4705 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4706 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, (%rsp), %xmm15 # 16-byte Folded Reload
4707 ; AVX2-FAST-PERLANE-NEXT: # xmm15 = mem[3,1,2,3]
4708 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
4709 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm15[0,1,2,0,4,5,6,7]
4710 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
4711 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
4712 ; AVX2-FAST-PERLANE-NEXT: # xmm12 = mem[3,1,2,3]
4713 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
4714 ; AVX2-FAST-PERLANE-NEXT: # xmm11 = mem[3,1,2,3]
4715 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[2,0,2,3,4,5,6,7]
4716 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm11[2,0,2,3,4,5,6,7]
4717 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
4718 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
4719 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4720 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4721 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm0 = xmm8[3,1,2,3]
4722 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
4723 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm7[3,1,2,3]
4724 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4725 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
4726 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
4727 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
4728 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm10[3,1,2,3]
4729 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4730 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4731 ; AVX2-FAST-PERLANE-NEXT: # xmm2 = mem[3,1,2,3]
4732 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4733 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]
4734 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
4735 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
4736 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4737 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4738 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
4739 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm1 = xmm13[3,1,2,3]
4740 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4741 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[3,1,2,3]
4742 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4743 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
4744 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
4745 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
4746 ; AVX2-FAST-PERLANE-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[3,1,2,3]
4747 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4748 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
4749 ; AVX2-FAST-PERLANE-NEXT: # xmm10 = mem[3,1,2,3]
4750 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
4751 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm10[2,0,2,3,4,5,6,7]
4752 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
4753 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
4754 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4755 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4756 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
4757 ; AVX2-FAST-PERLANE-NEXT: # xmm14 = mem[3,1,2,3]
4758 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
4759 ; AVX2-FAST-PERLANE-NEXT: # xmm9 = mem[3,1,2,3]
4760 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm0 = xmm14[0,1,2,0,4,5,6,7]
4761 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm9[0,1,2,0,4,5,6,7]
4762 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
4763 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
4764 ; AVX2-FAST-PERLANE-NEXT: # xmm8 = mem[3,1,2,3]
4765 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
4766 ; AVX2-FAST-PERLANE-NEXT: # xmm7 = mem[3,1,2,3]
4767 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[2,0,2,3,4,5,6,7]
4768 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm7[2,0,2,3,4,5,6,7]
4769 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
4770 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4771 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4772 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm0[6,7]
4773 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
4774 ; AVX2-FAST-PERLANE-NEXT: # xmm6 = mem[3,1,2,3]
4775 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
4776 ; AVX2-FAST-PERLANE-NEXT: # xmm5 = mem[3,1,2,3]
4777 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm6[0,1,2,0,4,5,6,7]
4778 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[0,1,2,0,4,5,6,7]
4779 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
4780 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
4781 ; AVX2-FAST-PERLANE-NEXT: # xmm4 = mem[3,1,2,3]
4782 ; AVX2-FAST-PERLANE-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4783 ; AVX2-FAST-PERLANE-NEXT: # xmm2 = mem[3,1,2,3]
4784 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[2,0,2,3,4,5,6,7]
4785 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm13 = xmm2[2,0,2,3,4,5,6,7]
4786 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1]
4787 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
4788 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
4789 ; AVX2-FAST-PERLANE-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
4790 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4791 ; AVX2-FAST-PERLANE-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
4792 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4793 ; AVX2-FAST-PERLANE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
4794 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
4795 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4796 ; AVX2-FAST-PERLANE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
4797 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4798 ; AVX2-FAST-PERLANE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
4799 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
4800 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4801 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4802 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
4803 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4804 ; AVX2-FAST-PERLANE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
4805 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4806 ; AVX2-FAST-PERLANE-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]
4807 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
4808 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4809 ; AVX2-FAST-PERLANE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
4810 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload
4811 ; AVX2-FAST-PERLANE-NEXT: # xmm13 = mem[3,1,2,3,4,5,6,7]
4812 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm13[0],xmm3[0],xmm13[1],xmm3[1]
4813 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
4814 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm13 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4815 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
4816 ; AVX2-FAST-PERLANE-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]
4817 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4818 ; AVX2-FAST-PERLANE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
4819 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
4820 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4821 ; AVX2-FAST-PERLANE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]
4822 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4823 ; AVX2-FAST-PERLANE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
4824 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
4825 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4826 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4827 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
4828 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
4829 ; AVX2-FAST-PERLANE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
4830 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm15[0,1,3,1,4,5,6,7]
4831 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
4832 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[3,1,2,3,4,5,6,7]
4833 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[3,1,2,3,4,5,6,7]
4834 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm11[0],xmm3[0],xmm11[1],xmm3[1]
4835 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
4836 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm11 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4837 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm0 = xmm14[0,1,3,1,4,5,6,7]
4838 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm9[0,1,3,1,4,5,6,7]
4839 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
4840 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[3,1,2,3,4,5,6,7]
4841 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm7[3,1,2,3,4,5,6,7]
4842 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
4843 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4844 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4845 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
4846 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm1 = xmm6[0,1,3,1,4,5,6,7]
4847 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm5[0,1,3,1,4,5,6,7]
4848 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
4849 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm3 = xmm4[3,1,2,3,4,5,6,7]
4850 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
4851 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
4852 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
4853 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
4854 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $116, (%rsp), %xmm1 # 16-byte Folded Reload
4855 ; AVX2-FAST-PERLANE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
4856 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4857 ; AVX2-FAST-PERLANE-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]
4858 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
4859 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4860 ; AVX2-FAST-PERLANE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]
4861 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4862 ; AVX2-FAST-PERLANE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
4863 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
4864 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4865 ; AVX2-FAST-PERLANE-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
4866 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
4867 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
4868 ; AVX2-FAST-PERLANE-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]
4869 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4870 ; AVX2-FAST-PERLANE-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]
4871 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
4872 ; AVX2-FAST-PERLANE-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
4873 ; AVX2-FAST-PERLANE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
4874 ; AVX2-FAST-PERLANE-NEXT: vpshuflw {{.*#+}} xmm4 = xmm10[3,1,2,3,4,5,6,7]
4875 ; AVX2-FAST-PERLANE-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
4876 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
4877 ; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
4878 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4879 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm2, 32(%rsi)
4880 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4881 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm2, 96(%rsi)
4882 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4883 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm2, 64(%rsi)
4884 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4885 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm2, (%rsi)
4886 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4887 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm2, 96(%rdx)
4888 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4889 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm2, 32(%rdx)
4890 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4891 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm2, (%rdx)
4892 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4893 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm2, 64(%rdx)
4894 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4895 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm2, 32(%rcx)
4896 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4897 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm2, 96(%rcx)
4898 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4899 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm2, 64(%rcx)
4900 ; AVX2-FAST-PERLANE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
4901 ; AVX2-FAST-PERLANE-NEXT: vmovaps %ymm2, (%rcx)
4902 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm1, 96(%r8)
4903 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm0, 32(%r8)
4904 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm11, 64(%r8)
4905 ; AVX2-FAST-PERLANE-NEXT: vmovdqa %ymm13, (%r8)
4906 ; AVX2-FAST-PERLANE-NEXT: addq $712, %rsp # imm = 0x2C8
4907 ; AVX2-FAST-PERLANE-NEXT: vzeroupper
4908 ; AVX2-FAST-PERLANE-NEXT: retq
4909 ;
4910 ; AVX512F-SLOW-LABEL: load_i16_stride4_vf64:
4911 ; AVX512F-SLOW: # %bb.0:
4912 ; AVX512F-SLOW-NEXT: subq $104, %rsp
4913 ; AVX512F-SLOW-NEXT: vmovdqa 240(%rdi), %xmm0
4914 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm0[0,2,2,3]
4915 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm10[0,1,0,2,4,5,6,7]
4916 ; AVX512F-SLOW-NEXT: vmovdqa 224(%rdi), %xmm2
4917 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[0,2,2,3]
4918 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm5[0,1,0,2,4,5,6,7]
4919 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
4920 ; AVX512F-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4921 ; AVX512F-SLOW-NEXT: vmovdqa64 112(%rdi), %xmm20
4922 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm20[0,2,2,3]
4923 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm14[0,1,0,2,4,5,6,7]
4924 ; AVX512F-SLOW-NEXT: vmovdqa 96(%rdi), %xmm3
4925 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[0,2,2,3]
4926 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm7[0,1,0,2,4,5,6,7]
4927 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
4928 ; AVX512F-SLOW-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4929 ; AVX512F-SLOW-NEXT: vmovdqa64 496(%rdi), %xmm19
4930 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm15 = xmm19[0,2,2,3]
4931 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm15[0,1,0,2,4,5,6,7]
4932 ; AVX512F-SLOW-NEXT: vmovdqa64 480(%rdi), %xmm21
4933 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm8 = xmm21[0,2,2,3]
4934 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm8[0,1,0,2,4,5,6,7]
4935 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
4936 ; AVX512F-SLOW-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill
4937 ; AVX512F-SLOW-NEXT: vmovdqa64 368(%rdi), %xmm17
4938 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm17[0,2,2,3]
4939 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm4[0,1,0,2,4,5,6,7]
4940 ; AVX512F-SLOW-NEXT: vmovdqa64 352(%rdi), %xmm24
4941 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm24[0,2,2,3]
4942 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm1[0,1,0,2,4,5,6,7]
4943 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm18 = xmm9[0],xmm6[0],xmm9[1],xmm6[1]
4944 ; AVX512F-SLOW-NEXT: vmovdqa64 320(%rdi), %xmm25
4945 ; AVX512F-SLOW-NEXT: vmovdqa64 336(%rdi), %xmm29
4946 ; AVX512F-SLOW-NEXT: vmovdqa 448(%rdi), %xmm11
4947 ; AVX512F-SLOW-NEXT: vmovdqa64 464(%rdi), %xmm16
4948 ; AVX512F-SLOW-NEXT: vmovdqa 64(%rdi), %xmm9
4949 ; AVX512F-SLOW-NEXT: vmovdqa 80(%rdi), %xmm12
4950 ; AVX512F-SLOW-NEXT: vmovdqa 192(%rdi), %xmm13
4951 ; AVX512F-SLOW-NEXT: vmovdqa 208(%rdi), %xmm6
4952 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]
4953 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]
4954 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm10[0],xmm5[1],xmm10[1]
4955 ; AVX512F-SLOW-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4956 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
4957 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
4958 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm13[0,2,2,3]
4959 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7]
4960 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1]
4961 ; AVX512F-SLOW-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4962 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm14[0,1,1,3,4,5,6,7]
4963 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]
4964 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
4965 ; AVX512F-SLOW-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4966 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm12[0,2,2,3]
4967 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
4968 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm9[0,2,2,3]
4969 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]
4970 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm27 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
4971 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm15[0,1,1,3,4,5,6,7]
4972 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm8[0,1,1,3,4,5,6,7]
4973 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
4974 ; AVX512F-SLOW-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4975 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm16[0,2,2,3]
4976 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
4977 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm11[0,2,2,3]
4978 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]
4979 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm22 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
4980 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7]
4981 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
4982 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm26 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
4983 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm29[0,2,2,3]
4984 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]
4985 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm25[0,2,2,3]
4986 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
4987 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm30 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
4988 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm10 = xmm0[3,1,2,3]
4989 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm10[0,1,2,0,4,5,6,7]
4990 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm14 = xmm2[3,1,2,3]
4991 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm14[0,1,2,0,4,5,6,7]
4992 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm28 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
4993 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
4994 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm6[2,0,2,3,4,5,6,7]
4995 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm13[3,1,2,3]
4996 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[2,0,2,3,4,5,6,7]
4997 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm31 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
4998 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm20[3,1,2,3]
4999 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm1[0,1,2,0,4,5,6,7]
5000 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
5001 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm3[0,1,2,0,4,5,6,7]
5002 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm23 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
5003 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm15 = xmm12[3,1,2,3]
5004 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm15[2,0,2,3,4,5,6,7]
5005 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[3,1,2,3]
5006 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm9[2,0,2,3,4,5,6,7]
5007 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm20 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
5008 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm12 = xmm19[3,1,2,3]
5009 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm13 = xmm21[3,1,2,3]
5010 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm12[0,1,2,0,4,5,6,7]
5011 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[0,1,2,0,4,5,6,7]
5012 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm21 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
5013 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm16[3,1,2,3]
5014 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,1,2,3]
5015 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm7[2,0,2,3,4,5,6,7]
5016 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm11[2,0,2,3,4,5,6,7]
5017 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm19 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
5018 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm17[3,1,2,3]
5019 ; AVX512F-SLOW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5020 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm24[3,1,2,3]
5021 ; AVX512F-SLOW-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5022 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]
5023 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm4[0,1,2,0,4,5,6,7]
5024 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm17 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
5025 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm29[3,1,2,3]
5026 ; AVX512F-SLOW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5027 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]
5028 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm25[3,1,2,3]
5029 ; AVX512F-SLOW-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5030 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm4[2,0,2,3,4,5,6,7]
5031 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm16 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
5032 ; AVX512F-SLOW-NEXT: vmovdqa 192(%rdi), %ymm0
5033 ; AVX512F-SLOW-NEXT: vpmovqw %ymm0, %xmm0
5034 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
5035 ; AVX512F-SLOW-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 16-byte Folded Reload
5036 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3,4,5],ymm5[6,7]
5037 ; AVX512F-SLOW-NEXT: vmovdqa64 128(%rdi), %zmm0
5038 ; AVX512F-SLOW-NEXT: vpmovqw %zmm0, %xmm4
5039 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
5040 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm4
5041 ; AVX512F-SLOW-NEXT: vmovdqa 64(%rdi), %ymm5
5042 ; AVX512F-SLOW-NEXT: vpmovqw %ymm5, %xmm5
5043 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
5044 ; AVX512F-SLOW-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 16-byte Folded Reload
5045 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm8[6,7]
5046 ; AVX512F-SLOW-NEXT: vmovdqa64 (%rdi), %zmm29
5047 ; AVX512F-SLOW-NEXT: vpmovqw %zmm29, %xmm8
5048 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7]
5049 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm5[0,1,2,3],zmm4[4,5,6,7]
5050 ; AVX512F-SLOW-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
5051 ; AVX512F-SLOW-NEXT: vmovdqa 448(%rdi), %ymm4
5052 ; AVX512F-SLOW-NEXT: vpmovqw %ymm4, %xmm4
5053 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
5054 ; AVX512F-SLOW-NEXT: vinserti128 $1, (%rsp), %ymm0, %ymm5 # 16-byte Folded Reload
5055 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm5[6,7]
5056 ; AVX512F-SLOW-NEXT: vmovdqa64 384(%rdi), %zmm25
5057 ; AVX512F-SLOW-NEXT: vpmovqw %zmm25, %xmm5
5058 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
5059 ; AVX512F-SLOW-NEXT: vmovdqa 320(%rdi), %ymm5
5060 ; AVX512F-SLOW-NEXT: vpmovqw %ymm5, %xmm5
5061 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
5062 ; AVX512F-SLOW-NEXT: vinserti32x4 $1, %xmm18, %ymm0, %ymm8
5063 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm8[6,7]
5064 ; AVX512F-SLOW-NEXT: vmovdqa64 256(%rdi), %zmm18
5065 ; AVX512F-SLOW-NEXT: vpmovqw %zmm18, %xmm8
5066 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7]
5067 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm4
5068 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm24 = zmm5[0,1,2,3],zmm4[4,5,6,7]
5069 ; AVX512F-SLOW-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
5070 ; AVX512F-SLOW-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 16-byte Folded Reload
5071 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
5072 ; AVX512F-SLOW-NEXT: vpsrlq $16, %zmm0, %zmm5
5073 ; AVX512F-SLOW-NEXT: vpmovqw %zmm5, %xmm5
5074 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
5075 ; AVX512F-SLOW-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 16-byte Folded Reload
5076 ; AVX512F-SLOW-NEXT: vinserti32x4 $1, %xmm27, %ymm0, %ymm8
5077 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3,4,5],ymm5[6,7]
5078 ; AVX512F-SLOW-NEXT: vpsrlq $16, %zmm29, %zmm8
5079 ; AVX512F-SLOW-NEXT: vpmovqw %zmm8, %xmm8
5080 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7]
5081 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm4
5082 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm27 = zmm5[0,1,2,3],zmm4[4,5,6,7]
5083 ; AVX512F-SLOW-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
5084 ; AVX512F-SLOW-NEXT: vinserti32x4 $1, %xmm22, %ymm0, %ymm5
5085 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
5086 ; AVX512F-SLOW-NEXT: vpsrlq $16, %zmm25, %zmm5
5087 ; AVX512F-SLOW-NEXT: vpmovqw %zmm5, %xmm5
5088 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
5089 ; AVX512F-SLOW-NEXT: vinserti32x4 $1, %xmm26, %ymm0, %ymm5
5090 ; AVX512F-SLOW-NEXT: vinserti32x4 $1, %xmm30, %ymm0, %ymm8
5091 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3,4,5],ymm5[6,7]
5092 ; AVX512F-SLOW-NEXT: vpsrlq $16, %zmm18, %zmm8
5093 ; AVX512F-SLOW-NEXT: vpmovqw %zmm8, %xmm8
5094 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7]
5095 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm4
5096 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm22 = zmm5[0,1,2,3],zmm4[4,5,6,7]
5097 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm10[0,1,3,1,4,5,6,7]
5098 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm14[0,1,3,1,4,5,6,7]
5099 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
5100 ; AVX512F-SLOW-NEXT: vinserti32x4 $1, %xmm28, %ymm0, %ymm5
5101 ; AVX512F-SLOW-NEXT: vinserti32x4 $1, %xmm31, %ymm0, %ymm8
5102 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3,4,5],ymm5[6,7]
5103 ; AVX512F-SLOW-NEXT: vpsrlq $32, %zmm0, %zmm8
5104 ; AVX512F-SLOW-NEXT: vpmovqw %zmm8, %xmm8
5105 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7]
5106 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm5
5107 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]
5108 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
5109 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm2[0],xmm6[0],xmm2[1],xmm6[1]
5110 ; AVX512F-SLOW-NEXT: vinserti32x4 $1, %xmm23, %ymm0, %ymm2
5111 ; AVX512F-SLOW-NEXT: vinserti32x4 $1, %xmm20, %ymm0, %ymm8
5112 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm8[0,1,2,3,4,5],ymm2[6,7]
5113 ; AVX512F-SLOW-NEXT: vpsrlq $32, %zmm29, %zmm8
5114 ; AVX512F-SLOW-NEXT: vpmovqw %zmm8, %xmm8
5115 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm8[0,1,2,3],ymm2[4,5,6,7]
5116 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm5[4,5,6,7]
5117 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]
5118 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
5119 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
5120 ; AVX512F-SLOW-NEXT: vinserti32x4 $1, %xmm21, %ymm0, %ymm3
5121 ; AVX512F-SLOW-NEXT: vinserti32x4 $1, %xmm19, %ymm0, %ymm5
5122 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4,5],ymm3[6,7]
5123 ; AVX512F-SLOW-NEXT: vpsrlq $32, %zmm25, %zmm5
5124 ; AVX512F-SLOW-NEXT: vpmovqw %zmm5, %xmm5
5125 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3],ymm3[4,5,6,7]
5126 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm3
5127 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm5 = xmm15[3,1,2,3,4,5,6,7]
5128 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm9[3,1,2,3,4,5,6,7]
5129 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
5130 ; AVX512F-SLOW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm8
5131 ; AVX512F-SLOW-NEXT: vinserti32x4 $1, %xmm16, %ymm0, %ymm9
5132 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4,5],ymm8[6,7]
5133 ; AVX512F-SLOW-NEXT: vpsrlq $32, %zmm18, %zmm9
5134 ; AVX512F-SLOW-NEXT: vpmovqw %zmm9, %xmm9
5135 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
5136 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm3 = zmm8[0,1,2,3],zmm3[4,5,6,7]
5137 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm12[0,1,3,1,4,5,6,7]
5138 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm9 = xmm13[0,1,3,1,4,5,6,7]
5139 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
5140 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
5141 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
5142 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1,2,3,4,5],ymm4[6,7]
5143 ; AVX512F-SLOW-NEXT: vpsrlq $48, %zmm0, %zmm0
5144 ; AVX512F-SLOW-NEXT: vpmovqw %zmm0, %xmm0
5145 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
5146 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0
5147 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm7[3,1,2,3,4,5,6,7]
5148 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm6 = xmm11[3,1,2,3,4,5,6,7]
5149 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
5150 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
5151 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
5152 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5],ymm1[6,7]
5153 ; AVX512F-SLOW-NEXT: vpsrlq $48, %zmm29, %zmm5
5154 ; AVX512F-SLOW-NEXT: vpmovqw %zmm5, %xmm5
5155 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
5156 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
5157 ; AVX512F-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5158 ; AVX512F-SLOW-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
5159 ; AVX512F-SLOW-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5160 ; AVX512F-SLOW-NEXT: # xmm5 = mem[0,1,3,1,4,5,6,7]
5161 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
5162 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm5
5163 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
5164 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm5[6,7]
5165 ; AVX512F-SLOW-NEXT: vpsrlq $48, %zmm25, %zmm5
5166 ; AVX512F-SLOW-NEXT: vpmovqw %zmm5, %xmm5
5167 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
5168 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm4
5169 ; AVX512F-SLOW-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5170 ; AVX512F-SLOW-NEXT: # xmm5 = mem[3,1,2,3,4,5,6,7]
5171 ; AVX512F-SLOW-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload
5172 ; AVX512F-SLOW-NEXT: # xmm6 = mem[3,1,2,3,4,5,6,7]
5173 ; AVX512F-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
5174 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
5175 ; AVX512F-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
5176 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5],ymm1[6,7]
5177 ; AVX512F-SLOW-NEXT: vpsrlq $48, %zmm18, %zmm5
5178 ; AVX512F-SLOW-NEXT: vpmovqw %zmm5, %xmm5
5179 ; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
5180 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm4[4,5,6,7]
5181 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm24, 64(%rsi)
5182 ; AVX512F-SLOW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
5183 ; AVX512F-SLOW-NEXT: vmovaps %zmm4, (%rsi)
5184 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm22, 64(%rdx)
5185 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm27, (%rdx)
5186 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm3, 64(%rcx)
5187 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm2, (%rcx)
5188 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm1, 64(%r8)
5189 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm0, (%r8)
5190 ; AVX512F-SLOW-NEXT: addq $104, %rsp
5191 ; AVX512F-SLOW-NEXT: vzeroupper
5192 ; AVX512F-SLOW-NEXT: retq
5193 ;
5194 ; AVX512F-FAST-LABEL: load_i16_stride4_vf64:
5195 ; AVX512F-FAST: # %bb.0:
5196 ; AVX512F-FAST-NEXT: vmovdqa64 256(%rdi), %zmm23
5197 ; AVX512F-FAST-NEXT: vmovdqa64 384(%rdi), %zmm26
5198 ; AVX512F-FAST-NEXT: vmovdqa64 (%rdi), %zmm30
5199 ; AVX512F-FAST-NEXT: vmovdqa64 128(%rdi), %zmm4
5200 ; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0,16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0]
5201 ; AVX512F-FAST-NEXT: # ymm5 = mem[0,1,0,1]
5202 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm1 = [0,2,2,3,0,2,4,6]
5203 ; AVX512F-FAST-NEXT: vmovdqa64 224(%rdi), %ymm24
5204 ; AVX512F-FAST-NEXT: vpermd %ymm24, %ymm1, %ymm10
5205 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm10, %ymm0
5206 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm6 = <0,1,4,5,u,u,u,u,8,9,12,13,12,13,14,15,16,17,20,21,u,u,u,u,24,25,28,29,u,u,u,u>
5207 ; AVX512F-FAST-NEXT: vmovdqa64 192(%rdi), %ymm25
5208 ; AVX512F-FAST-NEXT: vpermd %ymm25, %ymm1, %ymm11
5209 ; AVX512F-FAST-NEXT: vpshufb %ymm6, %ymm11, %ymm3
5210 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm7 = [0,2,2,3,4,6,12,14]
5211 ; AVX512F-FAST-NEXT: vpermt2d %ymm0, %ymm7, %ymm3
5212 ; AVX512F-FAST-NEXT: vpmovqw %zmm4, %xmm0
5213 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
5214 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0
5215 ; AVX512F-FAST-NEXT: vmovdqa64 96(%rdi), %ymm27
5216 ; AVX512F-FAST-NEXT: vpermd %ymm27, %ymm1, %ymm3
5217 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm3, %ymm9
5218 ; AVX512F-FAST-NEXT: vmovdqa64 64(%rdi), %ymm28
5219 ; AVX512F-FAST-NEXT: vpermd %ymm28, %ymm1, %ymm8
5220 ; AVX512F-FAST-NEXT: vpshufb %ymm6, %ymm8, %ymm12
5221 ; AVX512F-FAST-NEXT: vpermt2d %ymm9, %ymm7, %ymm12
5222 ; AVX512F-FAST-NEXT: vpmovqw %zmm30, %xmm9
5223 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm12[4,5,6,7]
5224 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm9[0,1,2,3],zmm0[4,5,6,7]
5225 ; AVX512F-FAST-NEXT: vmovdqa64 480(%rdi), %ymm16
5226 ; AVX512F-FAST-NEXT: vpermd %ymm16, %ymm1, %ymm0
5227 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm0, %ymm9
5228 ; AVX512F-FAST-NEXT: vmovdqa64 448(%rdi), %ymm17
5229 ; AVX512F-FAST-NEXT: vpermd %ymm17, %ymm1, %ymm12
5230 ; AVX512F-FAST-NEXT: vpshufb %ymm6, %ymm12, %ymm13
5231 ; AVX512F-FAST-NEXT: vpermt2d %ymm9, %ymm7, %ymm13
5232 ; AVX512F-FAST-NEXT: vpmovqw %zmm26, %xmm9
5233 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm13[4,5,6,7]
5234 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm0, %zmm9
5235 ; AVX512F-FAST-NEXT: vmovdqa64 352(%rdi), %ymm18
5236 ; AVX512F-FAST-NEXT: vpermd %ymm18, %ymm1, %ymm13
5237 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm13, %ymm14
5238 ; AVX512F-FAST-NEXT: vmovdqa64 320(%rdi), %ymm20
5239 ; AVX512F-FAST-NEXT: vpermd %ymm20, %ymm1, %ymm1
5240 ; AVX512F-FAST-NEXT: vpshufb %ymm6, %ymm1, %ymm15
5241 ; AVX512F-FAST-NEXT: vpermt2d %ymm14, %ymm7, %ymm15
5242 ; AVX512F-FAST-NEXT: vpmovqw %zmm23, %xmm14
5243 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm15[4,5,6,7]
5244 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm22 = zmm14[0,1,2,3],zmm9[4,5,6,7]
5245 ; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm9 = [18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
5246 ; AVX512F-FAST-NEXT: vpshufb %ymm9, %ymm10, %ymm14
5247 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm10 = <2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,u,u,u,u,u,u,u,u>
5248 ; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm11, %ymm11
5249 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3,4,5],ymm14[6,7]
5250 ; AVX512F-FAST-NEXT: vpsrlq $16, %zmm4, %zmm14
5251 ; AVX512F-FAST-NEXT: vpmovqw %zmm14, %xmm14
5252 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm11 = ymm14[0,1,2,3],ymm11[4,5,6,7]
5253 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm11, %zmm0, %zmm11
5254 ; AVX512F-FAST-NEXT: vpshufb %ymm9, %ymm3, %ymm3
5255 ; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm8, %ymm8
5256 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm8[0,1,2,3,4,5],ymm3[6,7]
5257 ; AVX512F-FAST-NEXT: vpsrlq $16, %zmm30, %zmm8
5258 ; AVX512F-FAST-NEXT: vpmovqw %zmm8, %xmm8
5259 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm8[0,1,2,3],ymm3[4,5,6,7]
5260 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm29 = zmm3[0,1,2,3],zmm11[4,5,6,7]
5261 ; AVX512F-FAST-NEXT: vpshufb %ymm9, %ymm0, %ymm0
5262 ; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm12, %ymm3
5263 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
5264 ; AVX512F-FAST-NEXT: vpsrlq $16, %zmm26, %zmm3
5265 ; AVX512F-FAST-NEXT: vpmovqw %zmm3, %xmm3
5266 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
5267 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0
5268 ; AVX512F-FAST-NEXT: vpshufb %ymm9, %ymm13, %ymm3
5269 ; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm1, %ymm1
5270 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm3[6,7]
5271 ; AVX512F-FAST-NEXT: vpsrlq $16, %zmm23, %zmm3
5272 ; AVX512F-FAST-NEXT: vpmovqw %zmm3, %xmm3
5273 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
5274 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm19 = zmm1[0,1,2,3],zmm0[4,5,6,7]
5275 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm15 = [1,3,2,3,1,3,5,7]
5276 ; AVX512F-FAST-NEXT: vpermd %ymm24, %ymm15, %ymm3
5277 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm3, %ymm0
5278 ; AVX512F-FAST-NEXT: vpermd %ymm25, %ymm15, %ymm8
5279 ; AVX512F-FAST-NEXT: vpshufb %ymm6, %ymm8, %ymm1
5280 ; AVX512F-FAST-NEXT: vpermt2d %ymm0, %ymm7, %ymm1
5281 ; AVX512F-FAST-NEXT: vpsrlq $32, %zmm4, %zmm0
5282 ; AVX512F-FAST-NEXT: vpmovqw %zmm0, %xmm0
5283 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
5284 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm1
5285 ; AVX512F-FAST-NEXT: vpermd %ymm27, %ymm15, %ymm0
5286 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm0, %ymm13
5287 ; AVX512F-FAST-NEXT: vpermd %ymm28, %ymm15, %ymm12
5288 ; AVX512F-FAST-NEXT: vpshufb %ymm6, %ymm12, %ymm14
5289 ; AVX512F-FAST-NEXT: vpermt2d %ymm13, %ymm7, %ymm14
5290 ; AVX512F-FAST-NEXT: vpsrlq $32, %zmm30, %zmm13
5291 ; AVX512F-FAST-NEXT: vpmovqw %zmm13, %xmm13
5292 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3],ymm14[4,5,6,7]
5293 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm24 = zmm13[0,1,2,3],zmm1[4,5,6,7]
5294 ; AVX512F-FAST-NEXT: vpermd %ymm16, %ymm15, %ymm13
5295 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm13, %ymm1
5296 ; AVX512F-FAST-NEXT: vpermd %ymm17, %ymm15, %ymm14
5297 ; AVX512F-FAST-NEXT: vpshufb %ymm6, %ymm14, %ymm11
5298 ; AVX512F-FAST-NEXT: vpermt2d %ymm1, %ymm7, %ymm11
5299 ; AVX512F-FAST-NEXT: vpsrlq $32, %zmm26, %zmm1
5300 ; AVX512F-FAST-NEXT: vpmovqw %zmm1, %xmm1
5301 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm11[4,5,6,7]
5302 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm11
5303 ; AVX512F-FAST-NEXT: vpermd %ymm18, %ymm15, %ymm1
5304 ; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm1, %ymm2
5305 ; AVX512F-FAST-NEXT: vpermd %ymm20, %ymm15, %ymm5
5306 ; AVX512F-FAST-NEXT: vpshufb %ymm6, %ymm5, %ymm6
5307 ; AVX512F-FAST-NEXT: vpermt2d %ymm2, %ymm7, %ymm6
5308 ; AVX512F-FAST-NEXT: vpsrlq $32, %zmm23, %zmm2
5309 ; AVX512F-FAST-NEXT: vpmovqw %zmm2, %xmm2
5310 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
5311 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm11[4,5,6,7]
5312 ; AVX512F-FAST-NEXT: vpshufb %ymm9, %ymm3, %ymm3
5313 ; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm8, %ymm6
5314 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3,4,5],ymm3[6,7]
5315 ; AVX512F-FAST-NEXT: vpsrlq $48, %zmm4, %zmm4
5316 ; AVX512F-FAST-NEXT: vpmovqw %zmm4, %xmm4
5317 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
5318 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm3
5319 ; AVX512F-FAST-NEXT: vpshufb %ymm9, %ymm0, %ymm0
5320 ; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm12, %ymm4
5321 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
5322 ; AVX512F-FAST-NEXT: vpsrlq $48, %zmm30, %zmm4
5323 ; AVX512F-FAST-NEXT: vpmovqw %zmm4, %xmm4
5324 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
5325 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[4,5,6,7]
5326 ; AVX512F-FAST-NEXT: vpshufb %ymm9, %ymm13, %ymm3
5327 ; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm14, %ymm4
5328 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
5329 ; AVX512F-FAST-NEXT: vpsrlq $48, %zmm26, %zmm4
5330 ; AVX512F-FAST-NEXT: vpmovqw %zmm4, %xmm4
5331 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
5332 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm3
5333 ; AVX512F-FAST-NEXT: vpshufb %ymm9, %ymm1, %ymm1
5334 ; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm5, %ymm4
5335 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
5336 ; AVX512F-FAST-NEXT: vpsrlq $48, %zmm23, %zmm4
5337 ; AVX512F-FAST-NEXT: vpmovqw %zmm4, %xmm4
5338 ; AVX512F-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
5339 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm3[4,5,6,7]
5340 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm22, 64(%rsi)
5341 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm21, (%rsi)
5342 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm19, 64(%rdx)
5343 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm29, (%rdx)
5344 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm2, 64(%rcx)
5345 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm24, (%rcx)
5346 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm1, 64(%r8)
5347 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm0, (%r8)
5348 ; AVX512F-FAST-NEXT: vzeroupper
5349 ; AVX512F-FAST-NEXT: retq
5350 ;
5351 ; AVX512BW-LABEL: load_i16_stride4_vf64:
5352 ; AVX512BW: # %bb.0:
5353 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
5354 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
5355 ; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm2
5356 ; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm3
5357 ; AVX512BW-NEXT: vmovdqa64 320(%rdi), %zmm4
5358 ; AVX512BW-NEXT: vmovdqa64 256(%rdi), %zmm5
5359 ; AVX512BW-NEXT: vmovdqa64 448(%rdi), %zmm6
5360 ; AVX512BW-NEXT: vmovdqa64 384(%rdi), %zmm7
5361 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]
5362 ; AVX512BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]
5363 ; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm9
5364 ; AVX512BW-NEXT: vpermt2w %zmm6, %zmm8, %zmm9
5365 ; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm10
5366 ; AVX512BW-NEXT: vpermt2w %zmm4, %zmm8, %zmm10
5367 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm9 = zmm10[0,1,2,3],zmm9[4,5,6,7]
5368 ; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm10
5369 ; AVX512BW-NEXT: vpermt2w %zmm3, %zmm8, %zmm10
5370 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm8
5371 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],zmm10[4,5,6,7]
5372 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61,1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]
5373 ; AVX512BW-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]
5374 ; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm11
5375 ; AVX512BW-NEXT: vpermt2w %zmm6, %zmm10, %zmm11
5376 ; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm12
5377 ; AVX512BW-NEXT: vpermt2w %zmm4, %zmm10, %zmm12
5378 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm11 = zmm12[0,1,2,3],zmm11[4,5,6,7]
5379 ; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm12
5380 ; AVX512BW-NEXT: vpermt2w %zmm3, %zmm10, %zmm12
5381 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm10
5382 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm10 = zmm10[0,1,2,3],zmm12[4,5,6,7]
5383 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm12 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62,2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]
5384 ; AVX512BW-NEXT: # zmm12 = mem[0,1,2,3,0,1,2,3]
5385 ; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm13
5386 ; AVX512BW-NEXT: vpermt2w %zmm6, %zmm12, %zmm13
5387 ; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm14
5388 ; AVX512BW-NEXT: vpermt2w %zmm4, %zmm12, %zmm14
5389 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm14[0,1,2,3],zmm13[4,5,6,7]
5390 ; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm14
5391 ; AVX512BW-NEXT: vpermt2w %zmm3, %zmm12, %zmm14
5392 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm12
5393 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm12 = zmm12[0,1,2,3],zmm14[4,5,6,7]
5394 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm14 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63,3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]
5395 ; AVX512BW-NEXT: # zmm14 = mem[0,1,2,3,0,1,2,3]
5396 ; AVX512BW-NEXT: vpermt2w %zmm6, %zmm14, %zmm7
5397 ; AVX512BW-NEXT: vpermt2w %zmm4, %zmm14, %zmm5
5398 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm5[0,1,2,3],zmm7[4,5,6,7]
5399 ; AVX512BW-NEXT: vpermt2w %zmm3, %zmm14, %zmm2
5400 ; AVX512BW-NEXT: vpermt2w %zmm1, %zmm14, %zmm0
5401 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[4,5,6,7]
5402 ; AVX512BW-NEXT: vmovdqa64 %zmm9, 64(%rsi)
5403 ; AVX512BW-NEXT: vmovdqa64 %zmm8, (%rsi)
5404 ; AVX512BW-NEXT: vmovdqa64 %zmm11, 64(%rdx)
5405 ; AVX512BW-NEXT: vmovdqa64 %zmm10, (%rdx)
5406 ; AVX512BW-NEXT: vmovdqa64 %zmm13, 64(%rcx)
5407 ; AVX512BW-NEXT: vmovdqa64 %zmm12, (%rcx)
5408 ; AVX512BW-NEXT: vmovdqa64 %zmm4, 64(%r8)
5409 ; AVX512BW-NEXT: vmovdqa64 %zmm0, (%r8)
5410 ; AVX512BW-NEXT: vzeroupper
5411 ; AVX512BW-NEXT: retq
5412 %wide.vec = load <256 x i16>, ptr %in.vec, align 64
5413 %strided.vec0 = shufflevector <256 x i16> %wide.vec, <256 x i16> poison, <64 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60, i32 64, i32 68, i32 72, i32 76, i32 80, i32 84, i32 88, i32 92, i32 96, i32 100, i32 104, i32 108, i32 112, i32 116, i32 120, i32 124, i32 128, i32 132, i32 136, i32 140, i32 144, i32 148, i32 152, i32 156, i32 160, i32 164, i32 168, i32 172, i32 176, i32 180, i32 184, i32 188, i32 192, i32 196, i32 200, i32 204, i32 208, i32 212, i32 216, i32 220, i32 224, i32 228, i32 232, i32 236, i32 240, i32 244, i32 248, i32 252>
5414 %strided.vec1 = shufflevector <256 x i16> %wide.vec, <256 x i16> poison, <64 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61, i32 65, i32 69, i32 73, i32 77, i32 81, i32 85, i32 89, i32 93, i32 97, i32 101, i32 105, i32 109, i32 113, i32 117, i32 121, i32 125, i32 129, i32 133, i32 137, i32 141, i32 145, i32 149, i32 153, i32 157, i32 161, i32 165, i32 169, i32 173, i32 177, i32 181, i32 185, i32 189, i32 193, i32 197, i32 201, i32 205, i32 209, i32 213, i32 217, i32 221, i32 225, i32 229, i32 233, i32 237, i32 241, i32 245, i32 249, i32 253>
5415 %strided.vec2 = shufflevector <256 x i16> %wide.vec, <256 x i16> poison, <64 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62, i32 66, i32 70, i32 74, i32 78, i32 82, i32 86, i32 90, i32 94, i32 98, i32 102, i32 106, i32 110, i32 114, i32 118, i32 122, i32 126, i32 130, i32 134, i32 138, i32 142, i32 146, i32 150, i32 154, i32 158, i32 162, i32 166, i32 170, i32 174, i32 178, i32 182, i32 186, i32 190, i32 194, i32 198, i32 202, i32 206, i32 210, i32 214, i32 218, i32 222, i32 226, i32 230, i32 234, i32 238, i32 242, i32 246, i32 250, i32 254>
5416 %strided.vec3 = shufflevector <256 x i16> %wide.vec, <256 x i16> poison, <64 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63, i32 67, i32 71, i32 75, i32 79, i32 83, i32 87, i32 91, i32 95, i32 99, i32 103, i32 107, i32 111, i32 115, i32 119, i32 123, i32 127, i32 131, i32 135, i32 139, i32 143, i32 147, i32 151, i32 155, i32 159, i32 163, i32 167, i32 171, i32 175, i32 179, i32 183, i32 187, i32 191, i32 195, i32 199, i32 203, i32 207, i32 211, i32 215, i32 219, i32 223, i32 227, i32 231, i32 235, i32 239, i32 243, i32 247, i32 251, i32 255>
5417 store <64 x i16> %strided.vec0, ptr %out.vec0, align 64
5418 store <64 x i16> %strided.vec1, ptr %out.vec1, align 64
5419 store <64 x i16> %strided.vec2, ptr %out.vec2, align 64
5420 store <64 x i16> %strided.vec3, ptr %out.vec3, align 64
5421 ret void
5422 }
5423 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
5424 ; AVX: {{.*}}
5425 ; AVX2: {{.*}}
5426 ; AVX2-ONLY: {{.*}}
5427 ; AVX512BW-FAST: {{.*}}
5428 ; AVX512BW-ONLY-FAST: {{.*}}
5429 ; AVX512BW-ONLY-SLOW: {{.*}}
5430 ; AVX512BW-SLOW: {{.*}}
5431 ; AVX512DQ-FAST: {{.*}}
5432 ; AVX512DQ-SLOW: {{.*}}
5433 ; AVX512DQBW-FAST: {{.*}}
5434 ; AVX512DQBW-SLOW: {{.*}}
5435 ; AVX512F: {{.*}}
5436 ; AVX512F-ONLY-FAST: {{.*}}
5437 ; AVX512F-ONLY-SLOW: {{.*}}
5438 ; FALLBACK0: {{.*}}
5439 ; FALLBACK1: {{.*}}
5440 ; FALLBACK10: {{.*}}
5441 ; FALLBACK11: {{.*}}
5442 ; FALLBACK12: {{.*}}
5443 ; FALLBACK2: {{.*}}
5444 ; FALLBACK3: {{.*}}
5445 ; FALLBACK4: {{.*}}
5446 ; FALLBACK5: {{.*}}
5447 ; FALLBACK6: {{.*}}
5448 ; FALLBACK7: {{.*}}
5449 ; FALLBACK8: {{.*}}
5450 ; FALLBACK9: {{.*}}