| blob | b18f08b62f0d4cc2ee1ce9a6686ba4453c072c2c |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE
3 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX
4 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
5 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FP
6 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FCP
7 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512
8 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512-FCP
9 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ
10 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-FCP
11 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW
12 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW-FCP
13 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512DQ-BW
14 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-BW-FCP
16 ; These patterns are produced by LoopVectorizer for interleaved loads.
18 define void @load_i16_stride5_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4) nounwind {
19 ; SSE-LABEL: load_i16_stride5_vf2:
20 ; SSE: # %bb.0:
21 ; SSE-NEXT: movdqa (%rdi), %xmm0
22 ; SSE-NEXT: movdqa 16(%rdi), %xmm1
23 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
24 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,3,2,3,4,5,6,7]
25 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,3,2,3]
26 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
27 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[3,1,2,3]
28 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7]
29 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
30 ; SSE-NEXT: psrlq $48, %xmm0
31 ; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
32 ; SSE-NEXT: psrld $16, %xmm1
33 ; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
34 ; SSE-NEXT: movd %xmm2, (%rsi)
35 ; SSE-NEXT: movd %xmm3, (%rdx)
36 ; SSE-NEXT: movd %xmm4, (%rcx)
37 ; SSE-NEXT: movd %xmm0, (%r8)
38 ; SSE-NEXT: movd %xmm5, (%r9)
39 ; SSE-NEXT: retq
40 ;
41 ; AVX-LABEL: load_i16_stride5_vf2:
42 ; AVX: # %bb.0:
43 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
44 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
45 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
46 ; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,3,2,3,4,5,6,7]
47 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,3,2,3]
48 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
49 ; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[3,1,2,3]
50 ; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7]
51 ; AVX-NEXT: vpsrlq $48, %xmm0, %xmm5
52 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
53 ; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
54 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
55 ; AVX-NEXT: vmovd %xmm2, (%rsi)
56 ; AVX-NEXT: vmovd %xmm3, (%rdx)
57 ; AVX-NEXT: vmovd %xmm4, (%rcx)
58 ; AVX-NEXT: vmovd %xmm5, (%r8)
59 ; AVX-NEXT: vmovd %xmm0, (%r9)
60 ; AVX-NEXT: retq
61 ;
62 ; AVX2-LABEL: load_i16_stride5_vf2:
63 ; AVX2: # %bb.0:
64 ; AVX2-NEXT: vmovdqa (%rdi), %xmm0
65 ; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
66 ; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
67 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,3,2,3,4,5,6,7]
68 ; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,3,2,3]
69 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
70 ; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[3,1,2,3]
71 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7]
72 ; AVX2-NEXT: vpsrlq $48, %xmm0, %xmm0
73 ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
74 ; AVX2-NEXT: vpbroadcastw 8(%rdi), %xmm5
75 ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2,3,4,5,6,7]
76 ; AVX2-NEXT: vmovd %xmm2, (%rsi)
77 ; AVX2-NEXT: vmovd %xmm3, (%rdx)
78 ; AVX2-NEXT: vmovd %xmm4, (%rcx)
79 ; AVX2-NEXT: vmovd %xmm0, (%r8)
80 ; AVX2-NEXT: vmovd %xmm1, (%r9)
81 ; AVX2-NEXT: retq
82 ;
83 ; AVX2-FP-LABEL: load_i16_stride5_vf2:
84 ; AVX2-FP: # %bb.0:
85 ; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
86 ; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm1
87 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
88 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[2,3,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
89 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,5,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
90 ; AVX2-FP-NEXT: vpsrlq $48, %xmm0, %xmm0
91 ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
92 ; AVX2-FP-NEXT: vpbroadcastw 8(%rdi), %xmm5
93 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2,3,4,5,6,7]
94 ; AVX2-FP-NEXT: vmovd %xmm2, (%rsi)
95 ; AVX2-FP-NEXT: vmovd %xmm3, (%rdx)
96 ; AVX2-FP-NEXT: vmovd %xmm4, (%rcx)
97 ; AVX2-FP-NEXT: vmovd %xmm0, (%r8)
98 ; AVX2-FP-NEXT: vmovd %xmm1, (%r9)
99 ; AVX2-FP-NEXT: retq
100 ;
101 ; AVX2-FCP-LABEL: load_i16_stride5_vf2:
102 ; AVX2-FCP: # %bb.0:
103 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0
104 ; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
105 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
106 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[2,3,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
107 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,5,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
108 ; AVX2-FCP-NEXT: vpsrlq $48, %xmm0, %xmm0
109 ; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
110 ; AVX2-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm5
111 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2,3,4,5,6,7]
112 ; AVX2-FCP-NEXT: vmovd %xmm2, (%rsi)
113 ; AVX2-FCP-NEXT: vmovd %xmm3, (%rdx)
114 ; AVX2-FCP-NEXT: vmovd %xmm4, (%rcx)
115 ; AVX2-FCP-NEXT: vmovd %xmm0, (%r8)
116 ; AVX2-FCP-NEXT: vmovd %xmm1, (%r9)
117 ; AVX2-FCP-NEXT: retq
118 ;
119 ; AVX512-LABEL: load_i16_stride5_vf2:
120 ; AVX512: # %bb.0:
121 ; AVX512-NEXT: vmovdqa (%rdi), %xmm0
122 ; AVX512-NEXT: vmovdqa 16(%rdi), %xmm1
123 ; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
124 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,3,2,3,4,5,6,7]
125 ; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,3,2,3]
126 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
127 ; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[3,1,2,3]
128 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7]
129 ; AVX512-NEXT: vpsrlq $48, %xmm0, %xmm0
130 ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
131 ; AVX512-NEXT: vpbroadcastw 8(%rdi), %xmm5
132 ; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2,3,4,5,6,7]
133 ; AVX512-NEXT: vmovd %xmm2, (%rsi)
134 ; AVX512-NEXT: vmovd %xmm3, (%rdx)
135 ; AVX512-NEXT: vmovd %xmm4, (%rcx)
136 ; AVX512-NEXT: vmovd %xmm0, (%r8)
137 ; AVX512-NEXT: vmovd %xmm1, (%r9)
138 ; AVX512-NEXT: retq
139 ;
140 ; AVX512-FCP-LABEL: load_i16_stride5_vf2:
141 ; AVX512-FCP: # %bb.0:
142 ; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0
143 ; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
144 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
145 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[2,3,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
146 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,5,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
147 ; AVX512-FCP-NEXT: vpsrlq $48, %xmm0, %xmm0
148 ; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
149 ; AVX512-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm5
150 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2,3,4,5,6,7]
151 ; AVX512-FCP-NEXT: vmovd %xmm2, (%rsi)
152 ; AVX512-FCP-NEXT: vmovd %xmm3, (%rdx)
153 ; AVX512-FCP-NEXT: vmovd %xmm4, (%rcx)
154 ; AVX512-FCP-NEXT: vmovd %xmm0, (%r8)
155 ; AVX512-FCP-NEXT: vmovd %xmm1, (%r9)
156 ; AVX512-FCP-NEXT: retq
157 ;
158 ; AVX512DQ-LABEL: load_i16_stride5_vf2:
159 ; AVX512DQ: # %bb.0:
160 ; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
161 ; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm1
162 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
163 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,3,2,3,4,5,6,7]
164 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,3,2,3]
165 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
166 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[3,1,2,3]
167 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7]
168 ; AVX512DQ-NEXT: vpsrlq $48, %xmm0, %xmm0
169 ; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
170 ; AVX512DQ-NEXT: vpbroadcastw 8(%rdi), %xmm5
171 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2,3,4,5,6,7]
172 ; AVX512DQ-NEXT: vmovd %xmm2, (%rsi)
173 ; AVX512DQ-NEXT: vmovd %xmm3, (%rdx)
174 ; AVX512DQ-NEXT: vmovd %xmm4, (%rcx)
175 ; AVX512DQ-NEXT: vmovd %xmm0, (%r8)
176 ; AVX512DQ-NEXT: vmovd %xmm1, (%r9)
177 ; AVX512DQ-NEXT: retq
178 ;
179 ; AVX512DQ-FCP-LABEL: load_i16_stride5_vf2:
180 ; AVX512DQ-FCP: # %bb.0:
181 ; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0
182 ; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
183 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
184 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[2,3,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
185 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,5,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
186 ; AVX512DQ-FCP-NEXT: vpsrlq $48, %xmm0, %xmm0
187 ; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
188 ; AVX512DQ-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm5
189 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2,3,4,5,6,7]
190 ; AVX512DQ-FCP-NEXT: vmovd %xmm2, (%rsi)
191 ; AVX512DQ-FCP-NEXT: vmovd %xmm3, (%rdx)
192 ; AVX512DQ-FCP-NEXT: vmovd %xmm4, (%rcx)
193 ; AVX512DQ-FCP-NEXT: vmovd %xmm0, (%r8)
194 ; AVX512DQ-FCP-NEXT: vmovd %xmm1, (%r9)
195 ; AVX512DQ-FCP-NEXT: retq
196 ;
197 ; AVX512BW-LABEL: load_i16_stride5_vf2:
198 ; AVX512BW: # %bb.0:
199 ; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0
200 ; AVX512BW-NEXT: vmovdqa 16(%rdi), %xmm1
201 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
202 ; AVX512BW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,3,2,3,4,5,6,7]
203 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,3,2,3]
204 ; AVX512BW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
205 ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[3,1,2,3]
206 ; AVX512BW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7]
207 ; AVX512BW-NEXT: vpsrlq $48, %xmm0, %xmm0
208 ; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
209 ; AVX512BW-NEXT: vpbroadcastw 8(%rdi), %xmm5
210 ; AVX512BW-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2,3,4,5,6,7]
211 ; AVX512BW-NEXT: vmovd %xmm2, (%rsi)
212 ; AVX512BW-NEXT: vmovd %xmm3, (%rdx)
213 ; AVX512BW-NEXT: vmovd %xmm4, (%rcx)
214 ; AVX512BW-NEXT: vmovd %xmm0, (%r8)
215 ; AVX512BW-NEXT: vmovd %xmm1, (%r9)
216 ; AVX512BW-NEXT: retq
217 ;
218 ; AVX512BW-FCP-LABEL: load_i16_stride5_vf2:
219 ; AVX512BW-FCP: # %bb.0:
220 ; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm0
221 ; AVX512BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
222 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
223 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[2,3,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
224 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,5,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
225 ; AVX512BW-FCP-NEXT: vpsrlq $48, %xmm0, %xmm0
226 ; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
227 ; AVX512BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm5
228 ; AVX512BW-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2,3,4,5,6,7]
229 ; AVX512BW-FCP-NEXT: vmovd %xmm2, (%rsi)
230 ; AVX512BW-FCP-NEXT: vmovd %xmm3, (%rdx)
231 ; AVX512BW-FCP-NEXT: vmovd %xmm4, (%rcx)
232 ; AVX512BW-FCP-NEXT: vmovd %xmm0, (%r8)
233 ; AVX512BW-FCP-NEXT: vmovd %xmm1, (%r9)
234 ; AVX512BW-FCP-NEXT: retq
235 ;
236 ; AVX512DQ-BW-LABEL: load_i16_stride5_vf2:
237 ; AVX512DQ-BW: # %bb.0:
238 ; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm0
239 ; AVX512DQ-BW-NEXT: vmovdqa 16(%rdi), %xmm1
240 ; AVX512DQ-BW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
241 ; AVX512DQ-BW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,3,2,3,4,5,6,7]
242 ; AVX512DQ-BW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,3,2,3]
243 ; AVX512DQ-BW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
244 ; AVX512DQ-BW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[3,1,2,3]
245 ; AVX512DQ-BW-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7]
246 ; AVX512DQ-BW-NEXT: vpsrlq $48, %xmm0, %xmm0
247 ; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
248 ; AVX512DQ-BW-NEXT: vpbroadcastw 8(%rdi), %xmm5
249 ; AVX512DQ-BW-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2,3,4,5,6,7]
250 ; AVX512DQ-BW-NEXT: vmovd %xmm2, (%rsi)
251 ; AVX512DQ-BW-NEXT: vmovd %xmm3, (%rdx)
252 ; AVX512DQ-BW-NEXT: vmovd %xmm4, (%rcx)
253 ; AVX512DQ-BW-NEXT: vmovd %xmm0, (%r8)
254 ; AVX512DQ-BW-NEXT: vmovd %xmm1, (%r9)
255 ; AVX512DQ-BW-NEXT: retq
256 ;
257 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride5_vf2:
258 ; AVX512DQ-BW-FCP: # %bb.0:
259 ; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm0
260 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
261 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
262 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[2,3,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
263 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[4,5,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
264 ; AVX512DQ-BW-FCP-NEXT: vpsrlq $48, %xmm0, %xmm0
265 ; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
266 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm5
267 ; AVX512DQ-BW-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2,3,4,5,6,7]
268 ; AVX512DQ-BW-FCP-NEXT: vmovd %xmm2, (%rsi)
269 ; AVX512DQ-BW-FCP-NEXT: vmovd %xmm3, (%rdx)
270 ; AVX512DQ-BW-FCP-NEXT: vmovd %xmm4, (%rcx)
271 ; AVX512DQ-BW-FCP-NEXT: vmovd %xmm0, (%r8)
272 ; AVX512DQ-BW-FCP-NEXT: vmovd %xmm1, (%r9)
273 ; AVX512DQ-BW-FCP-NEXT: retq
274 %wide.vec = load <10 x i16>, ptr %in.vec, align 64
275 %strided.vec0 = shufflevector <10 x i16> %wide.vec, <10 x i16> poison, <2 x i32> <i32 0, i32 5>
276 %strided.vec1 = shufflevector <10 x i16> %wide.vec, <10 x i16> poison, <2 x i32> <i32 1, i32 6>
277 %strided.vec2 = shufflevector <10 x i16> %wide.vec, <10 x i16> poison, <2 x i32> <i32 2, i32 7>
278 %strided.vec3 = shufflevector <10 x i16> %wide.vec, <10 x i16> poison, <2 x i32> <i32 3, i32 8>
279 %strided.vec4 = shufflevector <10 x i16> %wide.vec, <10 x i16> poison, <2 x i32> <i32 4, i32 9>
280 store <2 x i16> %strided.vec0, ptr %out.vec0, align 64
281 store <2 x i16> %strided.vec1, ptr %out.vec1, align 64
282 store <2 x i16> %strided.vec2, ptr %out.vec2, align 64
283 store <2 x i16> %strided.vec3, ptr %out.vec3, align 64
284 store <2 x i16> %strided.vec4, ptr %out.vec4, align 64
285 ret void
286 }
288 define void @load_i16_stride5_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4) nounwind {
289 ; SSE-LABEL: load_i16_stride5_vf4:
290 ; SSE: # %bb.0:
291 ; SSE-NEXT: movdqa (%rdi), %xmm2
292 ; SSE-NEXT: movdqa 16(%rdi), %xmm3
293 ; SSE-NEXT: movdqa 32(%rdi), %xmm0
294 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[3,1,2,3]
295 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm1[2,1,2,3,4,5,6,7]
296 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
297 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]
298 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
299 ; SSE-NEXT: movdqa %xmm3, %xmm4
300 ; SSE-NEXT: psrlq $48, %xmm4
301 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,3,2,3]
302 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,2,2,3,4,5,6,7]
303 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
304 ; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
305 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm5[0,3,2,1]
306 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,6,6,7]
307 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
308 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,3,2,1,4,5,6,7]
309 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,1,1,3]
310 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,7]
311 ; SSE-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]
312 ; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
313 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[3,1,2,0]
314 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,6,7]
315 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,1,2,3]
316 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,2,0,3,4,5,6,7]
317 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[1,1,1,1]
318 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm3[0,2,2,3]
319 ; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
320 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm7[0,3,2,3,4,5,6,7]
321 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3]
322 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[1,0,3,3,4,5,6,7]
323 ; SSE-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
324 ; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,2,2,3,4,5,6,7]
325 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3]
326 ; SSE-NEXT: movdqa {{.*#+}} xmm7 = [65535,65535,65535,0,65535,65535,65535,65535]
327 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm3[3,0]
328 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[0,2]
329 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm3[0,1,2,3,4,6,6,7]
330 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
331 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,3,3,4,5,6,7]
332 ; SSE-NEXT: pand %xmm7, %xmm2
333 ; SSE-NEXT: pandn %xmm0, %xmm7
334 ; SSE-NEXT: por %xmm2, %xmm7
335 ; SSE-NEXT: movq %xmm1, (%rsi)
336 ; SSE-NEXT: movq %xmm4, (%rdx)
337 ; SSE-NEXT: movq %xmm5, (%rcx)
338 ; SSE-NEXT: movq %xmm6, (%r8)
339 ; SSE-NEXT: movq %xmm7, (%r9)
340 ; SSE-NEXT: retq
341 ;
342 ; AVX-LABEL: load_i16_stride5_vf4:
343 ; AVX: # %bb.0:
344 ; AVX-NEXT: vpshufd {{.*#+}} xmm0 = mem[3,1,2,3]
345 ; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]
346 ; AVX-NEXT: vmovdqa (%rdi), %xmm1
347 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
348 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm3
349 ; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[0,2,2,3]
350 ; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,3,2,3,4,5,6,7]
351 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
352 ; AVX-NEXT: vpsrlq $48, %xmm2, %xmm4
353 ; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[0,3,2,3]
354 ; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,2,2,3,4,5,6,7]
355 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
356 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
357 ; AVX-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[0,1,4,5,8,9,2,3,u,u,u,u,u,u,u,u]
358 ; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[0,1,1,3]
359 ; AVX-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,7]
360 ; AVX-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]
361 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
362 ; AVX-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[0,1,4,5,8,9,6,7,u,u,u,u,u,u,u,u]
363 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm1[2,3],xmm2[4,5,6,7]
364 ; AVX-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,10,11,u,u,u,u,u,u,u,u]
365 ; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]
366 ; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,2,2,3,4,5,6,7]
367 ; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,2,2,3]
368 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
369 ; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
370 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[3],xmm1[4,5,6,7]
371 ; AVX-NEXT: vmovq %xmm0, (%rsi)
372 ; AVX-NEXT: vmovq %xmm4, (%rdx)
373 ; AVX-NEXT: vmovq %xmm5, (%rcx)
374 ; AVX-NEXT: vmovq %xmm6, (%r8)
375 ; AVX-NEXT: vmovq %xmm1, (%r9)
376 ; AVX-NEXT: retq
377 ;
378 ; AVX2-LABEL: load_i16_stride5_vf4:
379 ; AVX2: # %bb.0:
380 ; AVX2-NEXT: vmovdqa (%rdi), %xmm0
381 ; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
382 ; AVX2-NEXT: vmovdqa 32(%rdi), %xmm2
383 ; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[3,1,2,3]
384 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,1,2,3,4,5,6,7]
385 ; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]
386 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,3,2,3,4,5,6,7]
387 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
388 ; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1,2,3,4,5,6,7]
389 ; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0],xmm1[1],xmm4[2,3]
390 ; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,12,13,6,7,0,1,u,u,u,u,u,u,u,u]
391 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm2[0],xmm0[1,2,3]
392 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],xmm1[2],xmm5[3]
393 ; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[4,5,14,15,8,9,2,3,u,u,u,u,u,u,u,u]
394 ; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm0[0,1],xmm2[2],xmm0[3,4,5,6,7]
395 ; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm1[0],xmm6[1],xmm1[2,3]
396 ; AVX2-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,4,5,u,u,u,u,u,u,u,u]
397 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3]
398 ; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
399 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3],xmm0[4,5,6,7]
400 ; AVX2-NEXT: vmovq %xmm3, (%rsi)
401 ; AVX2-NEXT: vmovq %xmm4, (%rdx)
402 ; AVX2-NEXT: vmovq %xmm5, (%rcx)
403 ; AVX2-NEXT: vmovq %xmm6, (%r8)
404 ; AVX2-NEXT: vmovq %xmm0, (%r9)
405 ; AVX2-NEXT: retq
406 ;
407 ; AVX2-FP-LABEL: load_i16_stride5_vf4:
408 ; AVX2-FP: # %bb.0:
409 ; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
410 ; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm1
411 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm2
412 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm3 = xmm1[4,5,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
413 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[0,1,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
414 ; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
415 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1,2,3,4,5,6,7]
416 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0],xmm1[1],xmm4[2,3]
417 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,12,13,6,7,0,1,u,u,u,u,u,u,u,u]
418 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm2[0],xmm0[1,2,3]
419 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],xmm1[2],xmm5[3]
420 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[4,5,14,15,8,9,2,3,u,u,u,u,u,u,u,u]
421 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm6 = xmm0[0,1],xmm2[2],xmm0[3,4,5,6,7]
422 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm6 = xmm1[0],xmm6[1],xmm1[2,3]
423 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,4,5,u,u,u,u,u,u,u,u]
424 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3]
425 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
426 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3],xmm0[4,5,6,7]
427 ; AVX2-FP-NEXT: vmovq %xmm3, (%rsi)
428 ; AVX2-FP-NEXT: vmovq %xmm4, (%rdx)
429 ; AVX2-FP-NEXT: vmovq %xmm5, (%rcx)
430 ; AVX2-FP-NEXT: vmovq %xmm6, (%r8)
431 ; AVX2-FP-NEXT: vmovq %xmm0, (%r9)
432 ; AVX2-FP-NEXT: retq
433 ;
434 ; AVX2-FCP-LABEL: load_i16_stride5_vf4:
435 ; AVX2-FCP: # %bb.0:
436 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0
437 ; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
438 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm2
439 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm1[4,5,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
440 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[0,1,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
441 ; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
442 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1,2,3,4,5,6,7]
443 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0],xmm1[1],xmm4[2,3]
444 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,12,13,6,7,0,1,u,u,u,u,u,u,u,u]
445 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm2[0],xmm0[1,2,3]
446 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],xmm1[2],xmm5[3]
447 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[4,5,14,15,8,9,2,3,u,u,u,u,u,u,u,u]
448 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm0[0,1],xmm2[2],xmm0[3,4,5,6,7]
449 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm1[0],xmm6[1],xmm1[2,3]
450 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,4,5,u,u,u,u,u,u,u,u]
451 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3]
452 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
453 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3],xmm0[4,5,6,7]
454 ; AVX2-FCP-NEXT: vmovq %xmm3, (%rsi)
455 ; AVX2-FCP-NEXT: vmovq %xmm4, (%rdx)
456 ; AVX2-FCP-NEXT: vmovq %xmm5, (%rcx)
457 ; AVX2-FCP-NEXT: vmovq %xmm6, (%r8)
458 ; AVX2-FCP-NEXT: vmovq %xmm0, (%r9)
459 ; AVX2-FCP-NEXT: retq
460 ;
461 ; AVX512-LABEL: load_i16_stride5_vf4:
462 ; AVX512: # %bb.0:
463 ; AVX512-NEXT: vmovdqa (%rdi), %xmm0
464 ; AVX512-NEXT: vmovdqa 16(%rdi), %xmm1
465 ; AVX512-NEXT: vmovdqa 32(%rdi), %xmm2
466 ; AVX512-NEXT: vpextrw $5, %xmm0, %eax
467 ; AVX512-NEXT: vpinsrw $1, %eax, %xmm0, %xmm3
468 ; AVX512-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm1[2],xmm3[3,4,5,6,7]
469 ; AVX512-NEXT: vpextrw $7, %xmm1, %eax
470 ; AVX512-NEXT: vpinsrw $3, %eax, %xmm3, %xmm3
471 ; AVX512-NEXT: vpextrw $6, %xmm0, %eax
472 ; AVX512-NEXT: vpextrw $1, %xmm0, %r10d
473 ; AVX512-NEXT: vmovd %r10d, %xmm4
474 ; AVX512-NEXT: vpinsrw $1, %eax, %xmm4, %xmm4
475 ; AVX512-NEXT: vpextrw $3, %xmm1, %eax
476 ; AVX512-NEXT: vpinsrw $2, %eax, %xmm4, %xmm1
477 ; AVX512-NEXT: vmovd %xmm2, %eax
478 ; AVX512-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1
479 ; AVX512-NEXT: vmovdqa 16(%rdi), %xmm4
480 ; AVX512-NEXT: vpblendd {{.*#+}} xmm5 = xmm2[0],xmm0[1,2,3]
481 ; AVX512-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],xmm4[2],xmm5[3]
482 ; AVX512-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[4,5,14,15,8,9,2,3,u,u,u,u,u,u,u,u]
483 ; AVX512-NEXT: vpblendw {{.*#+}} xmm6 = xmm0[0,1],xmm2[2],xmm0[3,4,5,6,7]
484 ; AVX512-NEXT: vpblendd {{.*#+}} xmm6 = xmm4[0],xmm6[1],xmm4[2,3]
485 ; AVX512-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,4,5,u,u,u,u,u,u,u,u]
486 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2],xmm4[3]
487 ; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
488 ; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3],xmm0[4,5,6,7]
489 ; AVX512-NEXT: vmovq %xmm3, (%rsi)
490 ; AVX512-NEXT: vmovq %xmm1, (%rdx)
491 ; AVX512-NEXT: vmovq %xmm5, (%rcx)
492 ; AVX512-NEXT: vmovq %xmm6, (%r8)
493 ; AVX512-NEXT: vmovq %xmm0, (%r9)
494 ; AVX512-NEXT: retq
495 ;
496 ; AVX512-FCP-LABEL: load_i16_stride5_vf4:
497 ; AVX512-FCP: # %bb.0:
498 ; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0
499 ; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
500 ; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm2
501 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,1,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
502 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm1[2],xmm3[3,4,5,6,7]
503 ; AVX512-FCP-NEXT: vpextrw $7, %xmm1, %eax
504 ; AVX512-FCP-NEXT: vpinsrw $3, %eax, %xmm3, %xmm3
505 ; AVX512-FCP-NEXT: vpextrw $3, %xmm1, %eax
506 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
507 ; AVX512-FCP-NEXT: vpinsrw $2, %eax, %xmm1, %xmm1
508 ; AVX512-FCP-NEXT: vmovd %xmm2, %eax
509 ; AVX512-FCP-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1
510 ; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm4
511 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm2[0],xmm0[1,2,3]
512 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],xmm4[2],xmm5[3]
513 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[4,5,14,15,8,9,2,3,u,u,u,u,u,u,u,u]
514 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm0[0,1],xmm2[2],xmm0[3,4,5,6,7]
515 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm4[0],xmm6[1],xmm4[2,3]
516 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,4,5,u,u,u,u,u,u,u,u]
517 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2],xmm4[3]
518 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
519 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3],xmm0[4,5,6,7]
520 ; AVX512-FCP-NEXT: vmovq %xmm3, (%rsi)
521 ; AVX512-FCP-NEXT: vmovq %xmm1, (%rdx)
522 ; AVX512-FCP-NEXT: vmovq %xmm5, (%rcx)
523 ; AVX512-FCP-NEXT: vmovq %xmm6, (%r8)
524 ; AVX512-FCP-NEXT: vmovq %xmm0, (%r9)
525 ; AVX512-FCP-NEXT: retq
526 ;
527 ; AVX512DQ-LABEL: load_i16_stride5_vf4:
528 ; AVX512DQ: # %bb.0:
529 ; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
530 ; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm1
531 ; AVX512DQ-NEXT: vmovdqa 32(%rdi), %xmm2
532 ; AVX512DQ-NEXT: vpextrw $5, %xmm0, %eax
533 ; AVX512DQ-NEXT: vpinsrw $1, %eax, %xmm0, %xmm3
534 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm1[2],xmm3[3,4,5,6,7]
535 ; AVX512DQ-NEXT: vpextrw $7, %xmm1, %eax
536 ; AVX512DQ-NEXT: vpinsrw $3, %eax, %xmm3, %xmm3
537 ; AVX512DQ-NEXT: vpextrw $6, %xmm0, %eax
538 ; AVX512DQ-NEXT: vpextrw $1, %xmm0, %r10d
539 ; AVX512DQ-NEXT: vmovd %r10d, %xmm4
540 ; AVX512DQ-NEXT: vpinsrw $1, %eax, %xmm4, %xmm4
541 ; AVX512DQ-NEXT: vpextrw $3, %xmm1, %eax
542 ; AVX512DQ-NEXT: vpinsrw $2, %eax, %xmm4, %xmm1
543 ; AVX512DQ-NEXT: vmovd %xmm2, %eax
544 ; AVX512DQ-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1
545 ; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm4
546 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm5 = xmm2[0],xmm0[1,2,3]
547 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],xmm4[2],xmm5[3]
548 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[4,5,14,15,8,9,2,3,u,u,u,u,u,u,u,u]
549 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm6 = xmm0[0,1],xmm2[2],xmm0[3,4,5,6,7]
550 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm6 = xmm4[0],xmm6[1],xmm4[2,3]
551 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,4,5,u,u,u,u,u,u,u,u]
552 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2],xmm4[3]
553 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
554 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3],xmm0[4,5,6,7]
555 ; AVX512DQ-NEXT: vmovq %xmm3, (%rsi)
556 ; AVX512DQ-NEXT: vmovq %xmm1, (%rdx)
557 ; AVX512DQ-NEXT: vmovq %xmm5, (%rcx)
558 ; AVX512DQ-NEXT: vmovq %xmm6, (%r8)
559 ; AVX512DQ-NEXT: vmovq %xmm0, (%r9)
560 ; AVX512DQ-NEXT: retq
561 ;
562 ; AVX512DQ-FCP-LABEL: load_i16_stride5_vf4:
563 ; AVX512DQ-FCP: # %bb.0:
564 ; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0
565 ; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
566 ; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm2
567 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,1,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
568 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm1[2],xmm3[3,4,5,6,7]
569 ; AVX512DQ-FCP-NEXT: vpextrw $7, %xmm1, %eax
570 ; AVX512DQ-FCP-NEXT: vpinsrw $3, %eax, %xmm3, %xmm3
571 ; AVX512DQ-FCP-NEXT: vpextrw $3, %xmm1, %eax
572 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
573 ; AVX512DQ-FCP-NEXT: vpinsrw $2, %eax, %xmm1, %xmm1
574 ; AVX512DQ-FCP-NEXT: vmovd %xmm2, %eax
575 ; AVX512DQ-FCP-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1
576 ; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm4
577 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm2[0],xmm0[1,2,3]
578 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],xmm4[2],xmm5[3]
579 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[4,5,14,15,8,9,2,3,u,u,u,u,u,u,u,u]
580 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm0[0,1],xmm2[2],xmm0[3,4,5,6,7]
581 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm4[0],xmm6[1],xmm4[2,3]
582 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,4,5,u,u,u,u,u,u,u,u]
583 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2],xmm4[3]
584 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
585 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3],xmm0[4,5,6,7]
586 ; AVX512DQ-FCP-NEXT: vmovq %xmm3, (%rsi)
587 ; AVX512DQ-FCP-NEXT: vmovq %xmm1, (%rdx)
588 ; AVX512DQ-FCP-NEXT: vmovq %xmm5, (%rcx)
589 ; AVX512DQ-FCP-NEXT: vmovq %xmm6, (%r8)
590 ; AVX512DQ-FCP-NEXT: vmovq %xmm0, (%r9)
591 ; AVX512DQ-FCP-NEXT: retq
592 ;
593 ; AVX512BW-LABEL: load_i16_stride5_vf4:
594 ; AVX512BW: # %bb.0:
595 ; AVX512BW-NEXT: vmovq {{.*#+}} xmm0 = [1,6,11,0,0,0,0,0]
596 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm1
597 ; AVX512BW-NEXT: vpermw %zmm1, %zmm0, %zmm0
598 ; AVX512BW-NEXT: vmovq {{.*#+}} xmm2 = [0,5,10,0,0,0,0,0]
599 ; AVX512BW-NEXT: vpermw %zmm1, %zmm2, %zmm1
600 ; AVX512BW-NEXT: vmovdqa 16(%rdi), %xmm2
601 ; AVX512BW-NEXT: vpextrw $7, %xmm2, %eax
602 ; AVX512BW-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1
603 ; AVX512BW-NEXT: vpinsrw $3, 32(%rdi), %xmm0, %xmm0
604 ; AVX512BW-NEXT: vmovq {{.*#+}} xmm2 = [2,7,12,17,0,0,0,0]
605 ; AVX512BW-NEXT: vmovdqa 32(%rdi), %ymm3
606 ; AVX512BW-NEXT: vmovdqa (%rdi), %ymm4
607 ; AVX512BW-NEXT: vpermi2w %ymm3, %ymm4, %ymm2
608 ; AVX512BW-NEXT: vmovq {{.*#+}} xmm5 = [3,8,13,18,0,0,0,0]
609 ; AVX512BW-NEXT: vpermi2w %ymm3, %ymm4, %ymm5
610 ; AVX512BW-NEXT: vmovq {{.*#+}} xmm6 = [4,9,14,19,0,0,0,0]
611 ; AVX512BW-NEXT: vpermi2w %ymm3, %ymm4, %ymm6
612 ; AVX512BW-NEXT: vmovq %xmm1, (%rsi)
613 ; AVX512BW-NEXT: vmovq %xmm0, (%rdx)
614 ; AVX512BW-NEXT: vmovq %xmm2, (%rcx)
615 ; AVX512BW-NEXT: vmovq %xmm5, (%r8)
616 ; AVX512BW-NEXT: vmovq %xmm6, (%r9)
617 ; AVX512BW-NEXT: vzeroupper
618 ; AVX512BW-NEXT: retq
619 ;
620 ; AVX512BW-FCP-LABEL: load_i16_stride5_vf4:
621 ; AVX512BW-FCP: # %bb.0:
622 ; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm0 = [1,6,11,0,0,0,0,0]
623 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm1
624 ; AVX512BW-FCP-NEXT: vpermw %zmm1, %zmm0, %zmm0
625 ; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = [0,5,10,0,0,0,0,0]
626 ; AVX512BW-FCP-NEXT: vpermw %zmm1, %zmm2, %zmm1
627 ; AVX512BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
628 ; AVX512BW-FCP-NEXT: vpextrw $7, %xmm2, %eax
629 ; AVX512BW-FCP-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1
630 ; AVX512BW-FCP-NEXT: vpinsrw $3, 32(%rdi), %xmm0, %xmm0
631 ; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = [2,7,12,17,0,0,0,0]
632 ; AVX512BW-FCP-NEXT: vmovdqa 32(%rdi), %ymm3
633 ; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm4
634 ; AVX512BW-FCP-NEXT: vpermi2w %ymm3, %ymm4, %ymm2
635 ; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm5 = [3,8,13,18,0,0,0,0]
636 ; AVX512BW-FCP-NEXT: vpermi2w %ymm3, %ymm4, %ymm5
637 ; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm6 = [4,9,14,19,0,0,0,0]
638 ; AVX512BW-FCP-NEXT: vpermi2w %ymm3, %ymm4, %ymm6
639 ; AVX512BW-FCP-NEXT: vmovq %xmm1, (%rsi)
640 ; AVX512BW-FCP-NEXT: vmovq %xmm0, (%rdx)
641 ; AVX512BW-FCP-NEXT: vmovq %xmm2, (%rcx)
642 ; AVX512BW-FCP-NEXT: vmovq %xmm5, (%r8)
643 ; AVX512BW-FCP-NEXT: vmovq %xmm6, (%r9)
644 ; AVX512BW-FCP-NEXT: vzeroupper
645 ; AVX512BW-FCP-NEXT: retq
646 ;
647 ; AVX512DQ-BW-LABEL: load_i16_stride5_vf4:
648 ; AVX512DQ-BW: # %bb.0:
649 ; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm0 = [1,6,11,0,0,0,0,0]
650 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm1
651 ; AVX512DQ-BW-NEXT: vpermw %zmm1, %zmm0, %zmm0
652 ; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm2 = [0,5,10,0,0,0,0,0]
653 ; AVX512DQ-BW-NEXT: vpermw %zmm1, %zmm2, %zmm1
654 ; AVX512DQ-BW-NEXT: vmovdqa 16(%rdi), %xmm2
655 ; AVX512DQ-BW-NEXT: vpextrw $7, %xmm2, %eax
656 ; AVX512DQ-BW-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1
657 ; AVX512DQ-BW-NEXT: vpinsrw $3, 32(%rdi), %xmm0, %xmm0
658 ; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm2 = [2,7,12,17,0,0,0,0]
659 ; AVX512DQ-BW-NEXT: vmovdqa 32(%rdi), %ymm3
660 ; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %ymm4
661 ; AVX512DQ-BW-NEXT: vpermi2w %ymm3, %ymm4, %ymm2
662 ; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm5 = [3,8,13,18,0,0,0,0]
663 ; AVX512DQ-BW-NEXT: vpermi2w %ymm3, %ymm4, %ymm5
664 ; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm6 = [4,9,14,19,0,0,0,0]
665 ; AVX512DQ-BW-NEXT: vpermi2w %ymm3, %ymm4, %ymm6
666 ; AVX512DQ-BW-NEXT: vmovq %xmm1, (%rsi)
667 ; AVX512DQ-BW-NEXT: vmovq %xmm0, (%rdx)
668 ; AVX512DQ-BW-NEXT: vmovq %xmm2, (%rcx)
669 ; AVX512DQ-BW-NEXT: vmovq %xmm5, (%r8)
670 ; AVX512DQ-BW-NEXT: vmovq %xmm6, (%r9)
671 ; AVX512DQ-BW-NEXT: vzeroupper
672 ; AVX512DQ-BW-NEXT: retq
673 ;
674 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride5_vf4:
675 ; AVX512DQ-BW-FCP: # %bb.0:
676 ; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm0 = [1,6,11,0,0,0,0,0]
677 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm1
678 ; AVX512DQ-BW-FCP-NEXT: vpermw %zmm1, %zmm0, %zmm0
679 ; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = [0,5,10,0,0,0,0,0]
680 ; AVX512DQ-BW-FCP-NEXT: vpermw %zmm1, %zmm2, %zmm1
681 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
682 ; AVX512DQ-BW-FCP-NEXT: vpextrw $7, %xmm2, %eax
683 ; AVX512DQ-BW-FCP-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1
684 ; AVX512DQ-BW-FCP-NEXT: vpinsrw $3, 32(%rdi), %xmm0, %xmm0
685 ; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = [2,7,12,17,0,0,0,0]
686 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 32(%rdi), %ymm3
687 ; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm4
688 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %ymm3, %ymm4, %ymm2
689 ; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm5 = [3,8,13,18,0,0,0,0]
690 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %ymm3, %ymm4, %ymm5
691 ; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm6 = [4,9,14,19,0,0,0,0]
692 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %ymm3, %ymm4, %ymm6
693 ; AVX512DQ-BW-FCP-NEXT: vmovq %xmm1, (%rsi)
694 ; AVX512DQ-BW-FCP-NEXT: vmovq %xmm0, (%rdx)
695 ; AVX512DQ-BW-FCP-NEXT: vmovq %xmm2, (%rcx)
696 ; AVX512DQ-BW-FCP-NEXT: vmovq %xmm5, (%r8)
697 ; AVX512DQ-BW-FCP-NEXT: vmovq %xmm6, (%r9)
698 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
699 ; AVX512DQ-BW-FCP-NEXT: retq
700 %wide.vec = load <20 x i16>, ptr %in.vec, align 64
701 %strided.vec0 = shufflevector <20 x i16> %wide.vec, <20 x i16> poison, <4 x i32> <i32 0, i32 5, i32 10, i32 15>
702 %strided.vec1 = shufflevector <20 x i16> %wide.vec, <20 x i16> poison, <4 x i32> <i32 1, i32 6, i32 11, i32 16>
703 %strided.vec2 = shufflevector <20 x i16> %wide.vec, <20 x i16> poison, <4 x i32> <i32 2, i32 7, i32 12, i32 17>
704 %strided.vec3 = shufflevector <20 x i16> %wide.vec, <20 x i16> poison, <4 x i32> <i32 3, i32 8, i32 13, i32 18>
705 %strided.vec4 = shufflevector <20 x i16> %wide.vec, <20 x i16> poison, <4 x i32> <i32 4, i32 9, i32 14, i32 19>
706 store <4 x i16> %strided.vec0, ptr %out.vec0, align 64
707 store <4 x i16> %strided.vec1, ptr %out.vec1, align 64
708 store <4 x i16> %strided.vec2, ptr %out.vec2, align 64
709 store <4 x i16> %strided.vec3, ptr %out.vec3, align 64
710 store <4 x i16> %strided.vec4, ptr %out.vec4, align 64
711 ret void
712 }
714 define void @load_i16_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4) nounwind {
715 ; SSE-LABEL: load_i16_stride5_vf8:
716 ; SSE: # %bb.0:
717 ; SSE-NEXT: movdqa 64(%rdi), %xmm6
718 ; SSE-NEXT: movdqa (%rdi), %xmm4
719 ; SSE-NEXT: movdqa 16(%rdi), %xmm3
720 ; SSE-NEXT: movdqa 32(%rdi), %xmm0
721 ; SSE-NEXT: movdqa 48(%rdi), %xmm5
722 ; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535,0,65535,65535,65535]
723 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm5[0,1,0,3]
724 ; SSE-NEXT: pand %xmm1, %xmm2
725 ; SSE-NEXT: pandn %xmm0, %xmm1
726 ; SSE-NEXT: por %xmm2, %xmm1
727 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[3,1,2,3]
728 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7]
729 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm4[0,2,2,3]
730 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,3,2,3,4,5,6,7]
731 ; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
732 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,1],xmm1[2,3]
733 ; SSE-NEXT: movaps {{.*#+}} xmm1 = [65535,65535,65535,65535,65535,65535,65535,0]
734 ; SSE-NEXT: andps %xmm1, %xmm7
735 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,1,0,1]
736 ; SSE-NEXT: movaps %xmm1, %xmm2
737 ; SSE-NEXT: pandn %xmm8, %xmm2
738 ; SSE-NEXT: por %xmm7, %xmm2
739 ; SSE-NEXT: movdqa %xmm3, %xmm7
740 ; SSE-NEXT: psrlq $48, %xmm7
741 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm4[0,3,2,3]
742 ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[1,2,2,3,4,5,6,7]
743 ; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
744 ; SSE-NEXT: movdqa {{.*#+}} xmm7 = [0,0,0,65535,65535,65535,65535,65535]
745 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm5[1,3,2,3]
746 ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm0[0,2,2,3]
747 ; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]
748 ; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm10[0,1,2,3,7,5,6,7]
749 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,1,2,1]
750 ; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[0,0,0,0,4,5,6,7]
751 ; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,6,4,7]
752 ; SSE-NEXT: pand %xmm7, %xmm9
753 ; SSE-NEXT: pandn %xmm8, %xmm7
754 ; SSE-NEXT: por %xmm9, %xmm7
755 ; SSE-NEXT: pand %xmm1, %xmm7
756 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,1,2,0]
757 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm6[0,1,0,3]
758 ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm6[0,1,1,3]
759 ; SSE-NEXT: psllq $48, %xmm6
760 ; SSE-NEXT: pandn %xmm6, %xmm1
761 ; SSE-NEXT: por %xmm7, %xmm1
762 ; SSE-NEXT: movdqa %xmm5, %xmm7
763 ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]
764 ; SSE-NEXT: movdqa %xmm5, %xmm12
765 ; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[1,0],xmm0[0,0]
766 ; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[2,0],xmm0[2,3]
767 ; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[0,0,1,3]
768 ; SSE-NEXT: movdqa {{.*#+}} xmm13 = [65535,65535,65535,0,0,0,65535,65535]
769 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,1,1,3]
770 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,7]
771 ; SSE-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]
772 ; SSE-NEXT: pand %xmm13, %xmm5
773 ; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm4[1,1,1,1]
774 ; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm3[0,2,2,3]
775 ; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm11[0],xmm14[1],xmm11[1]
776 ; SSE-NEXT: movdqa %xmm13, %xmm15
777 ; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm14[0,3,2,3,4,5,6,7]
778 ; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[0,2,2,3]
779 ; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[1,0,3,3,4,5,6,7]
780 ; SSE-NEXT: pand %xmm13, %xmm11
781 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm3[3,0]
782 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0,2]
783 ; SSE-NEXT: movdqa %xmm13, %xmm4
784 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
785 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
786 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[2,1,3,3,4,5,6,7]
787 ; SSE-NEXT: pand %xmm13, %xmm3
788 ; SSE-NEXT: pandn %xmm12, %xmm13
789 ; SSE-NEXT: por %xmm13, %xmm5
790 ; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,6,5,6,7]
791 ; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,6,5]
792 ; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[3,1],xmm12[2,3]
793 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm8[2,0]
794 ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm0[2,2,2,2,4,5,6,7]
795 ; SSE-NEXT: pandn %xmm8, %xmm15
796 ; SSE-NEXT: por %xmm15, %xmm11
797 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,0],xmm0[3,0]
798 ; SSE-NEXT: pandn %xmm0, %xmm4
799 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm7[0,2]
800 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,4,6,7]
801 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm9[0,1,2,3,4,5,5,6]
802 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[3,1],xmm0[2,3]
803 ; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[0,1],xmm7[2,0]
804 ; SSE-NEXT: por %xmm4, %xmm3
805 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm6[0,1,0,3,4,5,6,7]
806 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm10[0,1,2,3,4,5,4,7]
807 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm0[1,3]
808 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,0]
809 ; SSE-NEXT: movdqa %xmm2, (%rsi)
810 ; SSE-NEXT: movdqa %xmm1, (%rdx)
811 ; SSE-NEXT: movaps %xmm5, (%rcx)
812 ; SSE-NEXT: movaps %xmm11, (%r8)
813 ; SSE-NEXT: movaps %xmm3, (%r9)
814 ; SSE-NEXT: retq
815 ;
816 ; AVX-LABEL: load_i16_stride5_vf8:
817 ; AVX: # %bb.0:
818 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
819 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
820 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm2
821 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm3
822 ; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[0,1,0,3]
823 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm2[4],xmm4[5,6,7]
824 ; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[3,1,2,3]
825 ; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[2,1,2,3,4,5,6,7]
826 ; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[0,2,2,3]
827 ; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,3,2,3,4,5,6,7]
828 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
829 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
830 ; AVX-NEXT: vmovdqa 64(%rdi), %xmm5
831 ; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[0,1,0,1]
832 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6],xmm6[7]
833 ; AVX-NEXT: vpsrlq $48, %xmm1, %xmm6
834 ; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[0,3,2,3]
835 ; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,2,2,3,4,5,6,7]
836 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
837 ; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
838 ; AVX-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,0,1,10,11,4,5,14,15,u,u]
839 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[3,4,5,6,7]
840 ; AVX-NEXT: vpsllq $48, %xmm5, %xmm7
841 ; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,5,6],xmm7[7]
842 ; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[0,1,1,3]
843 ; AVX-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,7,6,7]
844 ; AVX-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm7[2],xmm1[2],xmm7[3],xmm1[3]
845 ; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm2[0,1],xmm3[2,3],xmm2[4,5,6,7]
846 ; AVX-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,u,2,3,12,13,6,7,u,u,u,u]
847 ; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm8[3,4,5],xmm7[6,7]
848 ; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[0,1,2,0]
849 ; AVX-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,6,5]
850 ; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5],xmm8[6,7]
851 ; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm1[0,1],xmm0[2,3],xmm1[4,5,6,7]
852 ; AVX-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[6,7,0,1,10,11,u,u,u,u,u,u,12,13,14,15]
853 ; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm2[0,1,2,3],xmm3[4,5],xmm2[6,7]
854 ; AVX-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[2,2,2,2,4,5,6,7]
855 ; AVX-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,4,6,7]
856 ; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2],xmm9[3,4,5],xmm8[6,7]
857 ; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm5[0,1,0,3]
858 ; AVX-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,5,5,6]
859 ; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5],xmm9[6,7]
860 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
861 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
862 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,0,3,4,5,6,7]
863 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
864 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
865 ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,12,13,u,u,u,u,u,u,12,13,14,15]
866 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3,4,5],xmm0[6,7]
867 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[0,1,1,3]
868 ; AVX-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7]
869 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
870 ; AVX-NEXT: vmovdqa %xmm4, (%rsi)
871 ; AVX-NEXT: vmovdqa %xmm6, (%rdx)
872 ; AVX-NEXT: vmovdqa %xmm7, (%rcx)
873 ; AVX-NEXT: vmovdqa %xmm8, (%r8)
874 ; AVX-NEXT: vmovdqa %xmm0, (%r9)
875 ; AVX-NEXT: retq
876 ;
877 ; AVX2-LABEL: load_i16_stride5_vf8:
878 ; AVX2: # %bb.0:
879 ; AVX2-NEXT: vmovdqa (%rdi), %ymm0
880 ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm2
881 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5],ymm2[6],ymm0[7,8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13],ymm2[14],ymm0[15]
882 ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
883 ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1,2,3],xmm1[4,5],xmm3[6,7]
884 ; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,10,11,4,5,14,15,8,9,2,3,12,13,u,u]
885 ; AVX2-NEXT: vpbroadcastw 70(%rdi), %xmm3
886 ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6],xmm3[7]
887 ; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0],ymm0[1],ymm2[2],ymm0[3],ymm2[4,5],ymm0[6],ymm2[7,8],ymm0[9],ymm2[10],ymm0[11],ymm2[12,13],ymm0[14],ymm2[15]
888 ; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
889 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5,6],xmm4[7]
890 ; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[2,3,12,13,6,7,0,1,10,11,4,5,14,15,u,u]
891 ; AVX2-NEXT: vmovdqa 64(%rdi), %xmm4
892 ; AVX2-NEXT: vpsllq $48, %xmm4, %xmm5
893 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,6],xmm5[7]
894 ; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm2[0,1],ymm0[2],ymm2[3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8,9],ymm0[10],ymm2[11],ymm0[12],ymm2[13,14],ymm0[15]
895 ; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm6
896 ; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3,4],xmm5[5,6,7]
897 ; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
898 ; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[0,1,2,0]
899 ; AVX2-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,5,6,5]
900 ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3]
901 ; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm0[0],ymm2[1,2],ymm0[3],ymm2[4],ymm0[5],ymm2[6,7],ymm0[8],ymm2[9,10],ymm0[11],ymm2[12],ymm0[13],ymm2[14,15]
902 ; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm7
903 ; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0],xmm6[1],xmm7[2],xmm6[3]
904 ; AVX2-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
905 ; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[0,1,0,3]
906 ; AVX2-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,5,6]
907 ; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[3]
908 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5],ymm0[6],ymm2[7,8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]
909 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
910 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3,4],xmm2[5,6,7]
911 ; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
912 ; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[0,1,1,3]
913 ; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]
914 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3]
915 ; AVX2-NEXT: vmovdqa %xmm1, (%rsi)
916 ; AVX2-NEXT: vmovdqa %xmm3, (%rdx)
917 ; AVX2-NEXT: vmovdqa %xmm5, (%rcx)
918 ; AVX2-NEXT: vmovdqa %xmm6, (%r8)
919 ; AVX2-NEXT: vmovdqa %xmm0, (%r9)
920 ; AVX2-NEXT: vzeroupper
921 ; AVX2-NEXT: retq
922 ;
923 ; AVX2-FP-LABEL: load_i16_stride5_vf8:
924 ; AVX2-FP: # %bb.0:
925 ; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm0
926 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm1
927 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
928 ; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm3
929 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
930 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,10,11,4,5,14,15,8,9,2,3,12,13,u,u]
931 ; AVX2-FP-NEXT: vpbroadcastw 70(%rdi), %xmm3
932 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6],xmm3[7]
933 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
934 ; AVX2-FP-NEXT: vextracti128 $1, %ymm3, %xmm4
935 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5,6],xmm4[7]
936 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[2,3,12,13,6,7,0,1,10,11,4,5,14,15,u,u]
937 ; AVX2-FP-NEXT: vmovdqa 64(%rdi), %xmm4
938 ; AVX2-FP-NEXT: vpsllq $48, %xmm4, %xmm5
939 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,6],xmm5[7]
940 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm5 = [4,5,14,15,8,9,2,3,12,13,6,7,0,1,10,11]
941 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm4, %xmm6
942 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm7 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
943 ; AVX2-FP-NEXT: vextracti128 $1, %ymm7, %xmm8
944 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm8[3,4],xmm7[5,6,7]
945 ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm7, %xmm5
946 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3]
947 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm6 = [6,7,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
948 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm4, %xmm7
949 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
950 ; AVX2-FP-NEXT: vextracti128 $1, %ymm8, %xmm9
951 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0],xmm8[1],xmm9[2],xmm8[3]
952 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm8, %xmm6
953 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[3]
954 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm7 = [8,9,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
955 ; AVX2-FP-NEXT: vpshufb %xmm7, %xmm4, %xmm4
956 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
957 ; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm1
958 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3,4],xmm1[5,6,7]
959 ; AVX2-FP-NEXT: vpshufb %xmm7, %xmm0, %xmm0
960 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm4[3]
961 ; AVX2-FP-NEXT: vmovdqa %xmm2, (%rsi)
962 ; AVX2-FP-NEXT: vmovdqa %xmm3, (%rdx)
963 ; AVX2-FP-NEXT: vmovdqa %xmm5, (%rcx)
964 ; AVX2-FP-NEXT: vmovdqa %xmm6, (%r8)
965 ; AVX2-FP-NEXT: vmovdqa %xmm0, (%r9)
966 ; AVX2-FP-NEXT: vzeroupper
967 ; AVX2-FP-NEXT: retq
968 ;
969 ; AVX2-FCP-LABEL: load_i16_stride5_vf8:
970 ; AVX2-FCP: # %bb.0:
971 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm0
972 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
973 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
974 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm2, %xmm3
975 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
976 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,10,11,4,5,14,15,8,9,2,3,12,13,u,u]
977 ; AVX2-FCP-NEXT: vpbroadcastw 70(%rdi), %xmm3
978 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6],xmm3[7]
979 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
980 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm3, %xmm4
981 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5,6],xmm4[7]
982 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[2,3,12,13,6,7,0,1,10,11,4,5,14,15,u,u]
983 ; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %xmm4
984 ; AVX2-FCP-NEXT: vpsllq $48, %xmm4, %xmm5
985 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,6],xmm5[7]
986 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [4,5,14,15,8,9,2,3,12,13,6,7,0,1,10,11]
987 ; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm4, %xmm6
988 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
989 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm7, %xmm8
990 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm8[3,4],xmm7[5,6,7]
991 ; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm7, %xmm5
992 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3]
993 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [6,7,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
994 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm4, %xmm7
995 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
996 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm8, %xmm9
997 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0],xmm8[1],xmm9[2],xmm8[3]
998 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm8, %xmm6
999 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[3]
1000 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm7 = [8,9,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
1001 ; AVX2-FCP-NEXT: vpshufb %xmm7, %xmm4, %xmm4
1002 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
1003 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1
1004 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3,4],xmm1[5,6,7]
1005 ; AVX2-FCP-NEXT: vpshufb %xmm7, %xmm0, %xmm0
1006 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm4[3]
1007 ; AVX2-FCP-NEXT: vmovdqa %xmm2, (%rsi)
1008 ; AVX2-FCP-NEXT: vmovdqa %xmm3, (%rdx)
1009 ; AVX2-FCP-NEXT: vmovdqa %xmm5, (%rcx)
1010 ; AVX2-FCP-NEXT: vmovdqa %xmm6, (%r8)
1011 ; AVX2-FCP-NEXT: vmovdqa %xmm0, (%r9)
1012 ; AVX2-FCP-NEXT: vzeroupper
1013 ; AVX2-FCP-NEXT: retq
1014 ;
1015 ; AVX512-LABEL: load_i16_stride5_vf8:
1016 ; AVX512: # %bb.0:
1017 ; AVX512-NEXT: vmovdqa (%rdi), %ymm0
1018 ; AVX512-NEXT: vmovdqa 32(%rdi), %ymm2
1019 ; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5],ymm2[6],ymm0[7,8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13],ymm2[14],ymm0[15]
1020 ; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm3
1021 ; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1,2,3],xmm1[4,5],xmm3[6,7]
1022 ; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,10,11,4,5,14,15,8,9,2,3,12,13,u,u]
1023 ; AVX512-NEXT: vpbroadcastw 70(%rdi), %xmm3
1024 ; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6],xmm3[7]
1025 ; AVX512-NEXT: vmovdqa 64(%rdi), %xmm3
1026 ; AVX512-NEXT: vpblendw {{.*#+}} ymm4 = ymm2[0],ymm0[1],ymm2[2],ymm0[3],ymm2[4,5],ymm0[6],ymm2[7,8],ymm0[9],ymm2[10],ymm0[11],ymm2[12,13],ymm0[14],ymm2[15]
1027 ; AVX512-NEXT: vextracti128 $1, %ymm4, %xmm5
1028 ; AVX512-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5,6],xmm5[7]
1029 ; AVX512-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,12,13,6,7,0,1,10,11,4,5,14,15,u,u]
1030 ; AVX512-NEXT: vpsllq $48, %xmm3, %xmm5
1031 ; AVX512-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6],xmm5[7]
1032 ; AVX512-NEXT: vpblendw {{.*#+}} ymm5 = ymm2[0,1],ymm0[2],ymm2[3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8,9],ymm0[10],ymm2[11],ymm0[12],ymm2[13,14],ymm0[15]
1033 ; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm6
1034 ; AVX512-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3,4],xmm5[5,6,7]
1035 ; AVX512-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
1036 ; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[0,1,2,0]
1037 ; AVX512-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,5,6,5]
1038 ; AVX512-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3]
1039 ; AVX512-NEXT: vpblendw {{.*#+}} ymm6 = ymm0[0],ymm2[1,2],ymm0[3],ymm2[4],ymm0[5],ymm2[6,7],ymm0[8],ymm2[9,10],ymm0[11],ymm2[12],ymm0[13],ymm2[14,15]
1040 ; AVX512-NEXT: vextracti128 $1, %ymm6, %xmm7
1041 ; AVX512-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0],xmm6[1],xmm7[2],xmm6[3]
1042 ; AVX512-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
1043 ; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[0,1,0,3]
1044 ; AVX512-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,5,6]
1045 ; AVX512-NEXT: vpblendd {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[3]
1046 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5],ymm0[6],ymm2[7,8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]
1047 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
1048 ; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3,4],xmm2[5,6,7]
1049 ; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
1050 ; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[0,1,1,3]
1051 ; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]
1052 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3]
1053 ; AVX512-NEXT: vmovdqa %xmm1, (%rsi)
1054 ; AVX512-NEXT: vmovdqa %xmm4, (%rdx)
1055 ; AVX512-NEXT: vmovdqa %xmm5, (%rcx)
1056 ; AVX512-NEXT: vmovdqa %xmm6, (%r8)
1057 ; AVX512-NEXT: vmovdqa %xmm0, (%r9)
1058 ; AVX512-NEXT: vzeroupper
1059 ; AVX512-NEXT: retq
1060 ;
1061 ; AVX512-FCP-LABEL: load_i16_stride5_vf8:
1062 ; AVX512-FCP: # %bb.0:
1063 ; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm0
1064 ; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
1065 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
1066 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm2, %xmm3
1067 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
1068 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,10,11,4,5,14,15,8,9,2,3,12,13,u,u]
1069 ; AVX512-FCP-NEXT: vpbroadcastw 70(%rdi), %xmm3
1070 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6],xmm3[7]
1071 ; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm3
1072 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
1073 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm4, %xmm5
1074 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5,6],xmm5[7]
1075 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,12,13,6,7,0,1,10,11,4,5,14,15,u,u]
1076 ; AVX512-FCP-NEXT: vpsllq $48, %xmm3, %xmm5
1077 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6],xmm5[7]
1078 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [4,5,14,15,8,9,2,3,12,13,6,7,0,1,10,11]
1079 ; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm3, %xmm6
1080 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
1081 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm7, %xmm8
1082 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm8[3,4],xmm7[5,6,7]
1083 ; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm7, %xmm5
1084 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3]
1085 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [6,7,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
1086 ; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm3, %xmm7
1087 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
1088 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm8, %xmm9
1089 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0],xmm8[1],xmm9[2],xmm8[3]
1090 ; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm8, %xmm6
1091 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[3]
1092 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm7 = [8,9,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
1093 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm3, %xmm3
1094 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
1095 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1
1096 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3,4],xmm1[5,6,7]
1097 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm0, %xmm0
1098 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3]
1099 ; AVX512-FCP-NEXT: vmovdqa %xmm2, (%rsi)
1100 ; AVX512-FCP-NEXT: vmovdqa %xmm4, (%rdx)
1101 ; AVX512-FCP-NEXT: vmovdqa %xmm5, (%rcx)
1102 ; AVX512-FCP-NEXT: vmovdqa %xmm6, (%r8)
1103 ; AVX512-FCP-NEXT: vmovdqa %xmm0, (%r9)
1104 ; AVX512-FCP-NEXT: vzeroupper
1105 ; AVX512-FCP-NEXT: retq
1106 ;
1107 ; AVX512DQ-LABEL: load_i16_stride5_vf8:
1108 ; AVX512DQ: # %bb.0:
1109 ; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
1110 ; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm2
1111 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5],ymm2[6],ymm0[7,8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13],ymm2[14],ymm0[15]
1112 ; AVX512DQ-NEXT: vextracti128 $1, %ymm1, %xmm3
1113 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1,2,3],xmm1[4,5],xmm3[6,7]
1114 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,10,11,4,5,14,15,8,9,2,3,12,13,u,u]
1115 ; AVX512DQ-NEXT: vpbroadcastw 70(%rdi), %xmm3
1116 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6],xmm3[7]
1117 ; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm3
1118 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm4 = ymm2[0],ymm0[1],ymm2[2],ymm0[3],ymm2[4,5],ymm0[6],ymm2[7,8],ymm0[9],ymm2[10],ymm0[11],ymm2[12,13],ymm0[14],ymm2[15]
1119 ; AVX512DQ-NEXT: vextracti128 $1, %ymm4, %xmm5
1120 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5,6],xmm5[7]
1121 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,12,13,6,7,0,1,10,11,4,5,14,15,u,u]
1122 ; AVX512DQ-NEXT: vpsllq $48, %xmm3, %xmm5
1123 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6],xmm5[7]
1124 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm5 = ymm2[0,1],ymm0[2],ymm2[3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8,9],ymm0[10],ymm2[11],ymm0[12],ymm2[13,14],ymm0[15]
1125 ; AVX512DQ-NEXT: vextracti128 $1, %ymm5, %xmm6
1126 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3,4],xmm5[5,6,7]
1127 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
1128 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[0,1,2,0]
1129 ; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,5,6,5]
1130 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3]
1131 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm6 = ymm0[0],ymm2[1,2],ymm0[3],ymm2[4],ymm0[5],ymm2[6,7],ymm0[8],ymm2[9,10],ymm0[11],ymm2[12],ymm0[13],ymm2[14,15]
1132 ; AVX512DQ-NEXT: vextracti128 $1, %ymm6, %xmm7
1133 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0],xmm6[1],xmm7[2],xmm6[3]
1134 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
1135 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[0,1,0,3]
1136 ; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,5,6]
1137 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[3]
1138 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5],ymm0[6],ymm2[7,8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]
1139 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm2
1140 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3,4],xmm2[5,6,7]
1141 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
1142 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[0,1,1,3]
1143 ; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]
1144 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm2[3]
1145 ; AVX512DQ-NEXT: vmovdqa %xmm1, (%rsi)
1146 ; AVX512DQ-NEXT: vmovdqa %xmm4, (%rdx)
1147 ; AVX512DQ-NEXT: vmovdqa %xmm5, (%rcx)
1148 ; AVX512DQ-NEXT: vmovdqa %xmm6, (%r8)
1149 ; AVX512DQ-NEXT: vmovdqa %xmm0, (%r9)
1150 ; AVX512DQ-NEXT: vzeroupper
1151 ; AVX512DQ-NEXT: retq
1152 ;
1153 ; AVX512DQ-FCP-LABEL: load_i16_stride5_vf8:
1154 ; AVX512DQ-FCP: # %bb.0:
1155 ; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm0
1156 ; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
1157 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
1158 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm2, %xmm3
1159 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
1160 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,10,11,4,5,14,15,8,9,2,3,12,13,u,u]
1161 ; AVX512DQ-FCP-NEXT: vpbroadcastw 70(%rdi), %xmm3
1162 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6],xmm3[7]
1163 ; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm3
1164 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
1165 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm4, %xmm5
1166 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5,6],xmm5[7]
1167 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,12,13,6,7,0,1,10,11,4,5,14,15,u,u]
1168 ; AVX512DQ-FCP-NEXT: vpsllq $48, %xmm3, %xmm5
1169 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6],xmm5[7]
1170 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [4,5,14,15,8,9,2,3,12,13,6,7,0,1,10,11]
1171 ; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm3, %xmm6
1172 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
1173 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm7, %xmm8
1174 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm8[3,4],xmm7[5,6,7]
1175 ; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm7, %xmm5
1176 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3]
1177 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [6,7,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
1178 ; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm3, %xmm7
1179 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
1180 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm8, %xmm9
1181 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0],xmm8[1],xmm9[2],xmm8[3]
1182 ; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm8, %xmm6
1183 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[3]
1184 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm7 = [8,9,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
1185 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm3, %xmm3
1186 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
1187 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1
1188 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3,4],xmm1[5,6,7]
1189 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm0, %xmm0
1190 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3]
1191 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, (%rsi)
1192 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm4, (%rdx)
1193 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm5, (%rcx)
1194 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm6, (%r8)
1195 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%r9)
1196 ; AVX512DQ-FCP-NEXT: vzeroupper
1197 ; AVX512DQ-FCP-NEXT: retq
1198 ;
1199 ; AVX512BW-LABEL: load_i16_stride5_vf8:
1200 ; AVX512BW: # %bb.0:
1201 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
1202 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
1203 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} xmm2 = [0,5,10,15,20,25,30,35]
1204 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
1205 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} xmm3 = [1,6,11,16,21,26,31,36]
1206 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm3
1207 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} xmm4 = [2,7,12,17,22,27,32,37]
1208 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm4
1209 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} xmm5 = [3,8,13,18,23,28,33,38]
1210 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm5
1211 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} xmm6 = [4,9,14,19,24,29,34,39]
1212 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm6
1213 ; AVX512BW-NEXT: vmovdqa %xmm2, (%rsi)
1214 ; AVX512BW-NEXT: vmovdqa %xmm3, (%rdx)
1215 ; AVX512BW-NEXT: vmovdqa %xmm4, (%rcx)
1216 ; AVX512BW-NEXT: vmovdqa %xmm5, (%r8)
1217 ; AVX512BW-NEXT: vmovdqa %xmm6, (%r9)
1218 ; AVX512BW-NEXT: vzeroupper
1219 ; AVX512BW-NEXT: retq
1220 ;
1221 ; AVX512BW-FCP-LABEL: load_i16_stride5_vf8:
1222 ; AVX512BW-FCP: # %bb.0:
1223 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1224 ; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1225 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm2 = [0,5,10,15,20,25,30,35]
1226 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
1227 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm3 = [1,6,11,16,21,26,31,36]
1228 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm3
1229 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm4 = [2,7,12,17,22,27,32,37]
1230 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm4
1231 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm5 = [3,8,13,18,23,28,33,38]
1232 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm5
1233 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm6 = [4,9,14,19,24,29,34,39]
1234 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm6
1235 ; AVX512BW-FCP-NEXT: vmovdqa %xmm2, (%rsi)
1236 ; AVX512BW-FCP-NEXT: vmovdqa %xmm3, (%rdx)
1237 ; AVX512BW-FCP-NEXT: vmovdqa %xmm4, (%rcx)
1238 ; AVX512BW-FCP-NEXT: vmovdqa %xmm5, (%r8)
1239 ; AVX512BW-FCP-NEXT: vmovdqa %xmm6, (%r9)
1240 ; AVX512BW-FCP-NEXT: vzeroupper
1241 ; AVX512BW-FCP-NEXT: retq
1242 ;
1243 ; AVX512DQ-BW-LABEL: load_i16_stride5_vf8:
1244 ; AVX512DQ-BW: # %bb.0:
1245 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
1246 ; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm1
1247 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} xmm2 = [0,5,10,15,20,25,30,35]
1248 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
1249 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} xmm3 = [1,6,11,16,21,26,31,36]
1250 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm3
1251 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} xmm4 = [2,7,12,17,22,27,32,37]
1252 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm4
1253 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} xmm5 = [3,8,13,18,23,28,33,38]
1254 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm5
1255 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} xmm6 = [4,9,14,19,24,29,34,39]
1256 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm6
1257 ; AVX512DQ-BW-NEXT: vmovdqa %xmm2, (%rsi)
1258 ; AVX512DQ-BW-NEXT: vmovdqa %xmm3, (%rdx)
1259 ; AVX512DQ-BW-NEXT: vmovdqa %xmm4, (%rcx)
1260 ; AVX512DQ-BW-NEXT: vmovdqa %xmm5, (%r8)
1261 ; AVX512DQ-BW-NEXT: vmovdqa %xmm6, (%r9)
1262 ; AVX512DQ-BW-NEXT: vzeroupper
1263 ; AVX512DQ-BW-NEXT: retq
1264 ;
1265 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride5_vf8:
1266 ; AVX512DQ-BW-FCP: # %bb.0:
1267 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
1268 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
1269 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm2 = [0,5,10,15,20,25,30,35]
1270 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
1271 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm3 = [1,6,11,16,21,26,31,36]
1272 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm3
1273 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm4 = [2,7,12,17,22,27,32,37]
1274 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm4
1275 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm5 = [3,8,13,18,23,28,33,38]
1276 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm5
1277 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm6 = [4,9,14,19,24,29,34,39]
1278 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm6
1279 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm2, (%rsi)
1280 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm3, (%rdx)
1281 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm4, (%rcx)
1282 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm5, (%r8)
1283 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm6, (%r9)
1284 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
1285 ; AVX512DQ-BW-FCP-NEXT: retq
1286 %wide.vec = load <40 x i16>, ptr %in.vec, align 64
1287 %strided.vec0 = shufflevector <40 x i16> %wide.vec, <40 x i16> poison, <8 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35>
1288 %strided.vec1 = shufflevector <40 x i16> %wide.vec, <40 x i16> poison, <8 x i32> <i32 1, i32 6, i32 11, i32 16, i32 21, i32 26, i32 31, i32 36>
1289 %strided.vec2 = shufflevector <40 x i16> %wide.vec, <40 x i16> poison, <8 x i32> <i32 2, i32 7, i32 12, i32 17, i32 22, i32 27, i32 32, i32 37>
1290 %strided.vec3 = shufflevector <40 x i16> %wide.vec, <40 x i16> poison, <8 x i32> <i32 3, i32 8, i32 13, i32 18, i32 23, i32 28, i32 33, i32 38>
1291 %strided.vec4 = shufflevector <40 x i16> %wide.vec, <40 x i16> poison, <8 x i32> <i32 4, i32 9, i32 14, i32 19, i32 24, i32 29, i32 34, i32 39>
1292 store <8 x i16> %strided.vec0, ptr %out.vec0, align 64
1293 store <8 x i16> %strided.vec1, ptr %out.vec1, align 64
1294 store <8 x i16> %strided.vec2, ptr %out.vec2, align 64
1295 store <8 x i16> %strided.vec3, ptr %out.vec3, align 64
1296 store <8 x i16> %strided.vec4, ptr %out.vec4, align 64
1297 ret void
1298 }
1300 define void @load_i16_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4) nounwind {
1301 ; SSE-LABEL: load_i16_stride5_vf16:
1302 ; SSE: # %bb.0:
1303 ; SSE-NEXT: movdqa 144(%rdi), %xmm14
1304 ; SSE-NEXT: movdqa 80(%rdi), %xmm8
1305 ; SSE-NEXT: movdqa 96(%rdi), %xmm7
1306 ; SSE-NEXT: movdqa 128(%rdi), %xmm15
1307 ; SSE-NEXT: movdqa 112(%rdi), %xmm12
1308 ; SSE-NEXT: movdqa 64(%rdi), %xmm10
1309 ; SSE-NEXT: movdqa (%rdi), %xmm11
1310 ; SSE-NEXT: movdqa 16(%rdi), %xmm9
1311 ; SSE-NEXT: movdqa 32(%rdi), %xmm13
1312 ; SSE-NEXT: movdqa 48(%rdi), %xmm3
1313 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,65535,65535,0,65535,65535,65535]
1314 ; SSE-NEXT: movdqa %xmm0, %xmm1
1315 ; SSE-NEXT: pandn %xmm13, %xmm1
1316 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,1,0,3]
1317 ; SSE-NEXT: movdqa %xmm3, %xmm5
1318 ; SSE-NEXT: pand %xmm0, %xmm2
1319 ; SSE-NEXT: por %xmm1, %xmm2
1320 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm9[3,1,2,3]
1321 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
1322 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm11[0,2,2,3]
1323 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm3[0,3,2,3,4,5,6,7]
1324 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
1325 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3]
1326 ; SSE-NEXT: movaps {{.*#+}} xmm6 = [65535,65535,65535,65535,65535,65535,65535,0]
1327 ; SSE-NEXT: andps %xmm6, %xmm4
1328 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm10[0,1,0,1]
1329 ; SSE-NEXT: movaps %xmm6, %xmm2
1330 ; SSE-NEXT: pandn %xmm1, %xmm2
1331 ; SSE-NEXT: por %xmm4, %xmm2
1332 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1333 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm15[0,1,0,3]
1334 ; SSE-NEXT: pand %xmm0, %xmm1
1335 ; SSE-NEXT: pandn %xmm12, %xmm0
1336 ; SSE-NEXT: por %xmm1, %xmm0
1337 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[3,1,2,3]
1338 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
1339 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm8[0,2,2,3]
1340 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,3,2,3,4,5,6,7]
1341 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1342 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3]
1343 ; SSE-NEXT: andps %xmm6, %xmm2
1344 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm14[0,1,0,1]
1345 ; SSE-NEXT: movaps %xmm6, %xmm1
1346 ; SSE-NEXT: andnps %xmm0, %xmm1
1347 ; SSE-NEXT: orps %xmm2, %xmm1
1348 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1349 ; SSE-NEXT: movdqa %xmm9, %xmm0
1350 ; SSE-NEXT: psrlq $48, %xmm0
1351 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm11[0,3,2,3]
1352 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,2,3,4,5,6,7]
1353 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
1354 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,0,0,65535,65535,65535,65535,65535]
1355 ; SSE-NEXT: movdqa %xmm0, %xmm2
1356 ; SSE-NEXT: pandn %xmm1, %xmm2
1357 ; SSE-NEXT: movdqa %xmm5, %xmm3
1358 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm5[1,3,2,3]
1359 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm13[0,2,2,3]
1360 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
1361 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm4[0,1,2,3,7,5,6,7]
1362 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]
1363 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
1364 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,4,7]
1365 ; SSE-NEXT: pand %xmm0, %xmm1
1366 ; SSE-NEXT: por %xmm2, %xmm1
1367 ; SSE-NEXT: movdqa %xmm10, %xmm5
1368 ; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1369 ; SSE-NEXT: movdqa %xmm10, %xmm2
1370 ; SSE-NEXT: psllq $48, %xmm2
1371 ; SSE-NEXT: movaps %xmm6, %xmm4
1372 ; SSE-NEXT: andnps %xmm2, %xmm4
1373 ; SSE-NEXT: pand %xmm6, %xmm1
1374 ; SSE-NEXT: orps %xmm1, %xmm4
1375 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1376 ; SSE-NEXT: movdqa %xmm7, %xmm1
1377 ; SSE-NEXT: psrlq $48, %xmm1
1378 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm8[0,3,2,3]
1379 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]
1380 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1381 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm15[1,3,2,3]
1382 ; SSE-NEXT: movdqa %xmm15, %xmm10
1383 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm12[0,2,2,3]
1384 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
1385 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm4[0,1,2,3,7,5,6,7]
1386 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]
1387 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
1388 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,4,7]
1389 ; SSE-NEXT: pand %xmm0, %xmm1
1390 ; SSE-NEXT: pandn %xmm2, %xmm0
1391 ; SSE-NEXT: por %xmm1, %xmm0
1392 ; SSE-NEXT: pand %xmm6, %xmm0
1393 ; SSE-NEXT: movdqa %xmm14, %xmm4
1394 ; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1395 ; SSE-NEXT: movdqa %xmm14, %xmm1
1396 ; SSE-NEXT: psllq $48, %xmm1
1397 ; SSE-NEXT: pandn %xmm1, %xmm6
1398 ; SSE-NEXT: por %xmm0, %xmm6
1399 ; SSE-NEXT: movdqa %xmm3, %xmm0
1400 ; SSE-NEXT: movdqa %xmm3, %xmm14
1401 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
1402 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm13[0,0]
1403 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm13[2,3]
1404 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,0,1,3]
1405 ; SSE-NEXT: movaps {{.*#+}} xmm3 = [65535,65535,65535,0,0,0,65535,65535]
1406 ; SSE-NEXT: movaps %xmm3, %xmm1
1407 ; SSE-NEXT: andnps %xmm0, %xmm1
1408 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm11[0,1,1,3]
1409 ; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm2[0,1,2,3,4,7,6,7]
1410 ; SSE-NEXT: punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm9[2],xmm15[3],xmm9[3]
1411 ; SSE-NEXT: pand %xmm3, %xmm15
1412 ; SSE-NEXT: por %xmm1, %xmm15
1413 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]
1414 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm5[0,1,2,0]
1415 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6,5]
1416 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[2,3]
1417 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[0,1],xmm1[2,0]
1418 ; SSE-NEXT: movdqa %xmm10, %xmm5
1419 ; SSE-NEXT: movdqa %xmm10, %xmm1
1420 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm12[0,0]
1421 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm12[2,3]
1422 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,0,1,3]
1423 ; SSE-NEXT: movaps %xmm3, %xmm2
1424 ; SSE-NEXT: andnps %xmm1, %xmm2
1425 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm8[0,1,1,3]
1426 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7]
1427 ; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm7[2],xmm0[3],xmm7[3]
1428 ; SSE-NEXT: pand %xmm3, %xmm0
1429 ; SSE-NEXT: por %xmm2, %xmm0
1430 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]
1431 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,1,2,0]
1432 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6,5]
1433 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,1],xmm1[2,3]
1434 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
1435 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm13[2,2,2,2,4,5,6,7]
1436 ; SSE-NEXT: movdqa %xmm3, %xmm2
1437 ; SSE-NEXT: pandn %xmm1, %xmm2
1438 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm11[1,1,1,1]
1439 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm9[0,2,2,3]
1440 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
1441 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm4[0,3,2,3,4,5,6,7]
1442 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
1443 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,3,4,5,6,7]
1444 ; SSE-NEXT: pand %xmm3, %xmm1
1445 ; SSE-NEXT: por %xmm2, %xmm1
1446 ; SSE-NEXT: movdqa %xmm14, %xmm4
1447 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm13[3,0]
1448 ; SSE-NEXT: movdqa %xmm3, %xmm2
1449 ; SSE-NEXT: pandn %xmm13, %xmm2
1450 ; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,1],xmm4[0,2]
1451 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm13[0,1,2,3,7,4,6,7]
1452 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
1453 ; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm10[0,1,0,3]
1454 ; SSE-NEXT: pshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,5,5,6]
1455 ; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[3,1],xmm4[2,3]
1456 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm13[2,0]
1457 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm12[2,2,2,2,4,5,6,7]
1458 ; SSE-NEXT: movdqa %xmm3, %xmm14
1459 ; SSE-NEXT: pandn %xmm4, %xmm14
1460 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm8[1,1,1,1]
1461 ; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm7[0,2,2,3]
1462 ; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm4[0],xmm13[1],xmm4[1]
1463 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm13[0,3,2,3,4,5,6,7]
1464 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
1465 ; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm4[1,0,3,3,4,5,6,7]
1466 ; SSE-NEXT: pand %xmm3, %xmm13
1467 ; SSE-NEXT: por %xmm14, %xmm13
1468 ; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,0],xmm9[3,0]
1469 ; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[0,1],xmm11[0,2]
1470 ; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,0],xmm7[3,0]
1471 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,1],xmm8[0,2]
1472 ; SSE-NEXT: movdqa %xmm5, %xmm11
1473 ; SSE-NEXT: movdqa %xmm5, %xmm4
1474 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm12[3,0]
1475 ; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm9[0,1,2,3,4,6,6,7]
1476 ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3]
1477 ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[2,1,3,3,4,5,6,7]
1478 ; SSE-NEXT: pand %xmm3, %xmm8
1479 ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,6,6,7]
1480 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,2,2,3]
1481 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[2,1,3,3,4,5,6,7]
1482 ; SSE-NEXT: pand %xmm3, %xmm7
1483 ; SSE-NEXT: pandn %xmm12, %xmm3
1484 ; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[0,1],xmm4[0,2]
1485 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm12[0,1,2,3,7,4,6,7]
1486 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
1487 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm9[0,1,0,3]
1488 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,5,6]
1489 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[3,1],xmm4[2,3]
1490 ; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,1],xmm5[2,0]
1491 ; SSE-NEXT: por %xmm2, %xmm8
1492 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
1493 ; SSE-NEXT: # xmm2 = mem[0,2,2,3]
1494 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm10[0,1,1,3]
1495 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7]
1496 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,7]
1497 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[1,3]
1498 ; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[0,1],xmm4[2,0]
1499 ; SSE-NEXT: por %xmm7, %xmm3
1500 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm11[0,2,2,3]
1501 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm9[0,1,1,3]
1502 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7]
1503 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,7]
1504 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[1,3]
1505 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,0]
1506 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1507 ; SSE-NEXT: movaps %xmm2, 16(%rsi)
1508 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1509 ; SSE-NEXT: movaps %xmm2, (%rsi)
1510 ; SSE-NEXT: movdqa %xmm6, 16(%rdx)
1511 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
1512 ; SSE-NEXT: movaps %xmm2, (%rdx)
1513 ; SSE-NEXT: movaps %xmm0, 16(%rcx)
1514 ; SSE-NEXT: movaps %xmm15, (%rcx)
1515 ; SSE-NEXT: movaps %xmm13, 16(%r8)
1516 ; SSE-NEXT: movaps %xmm1, (%r8)
1517 ; SSE-NEXT: movaps %xmm3, 16(%r9)
1518 ; SSE-NEXT: movaps %xmm8, (%r9)
1519 ; SSE-NEXT: retq
1520 ;
1521 ; AVX-LABEL: load_i16_stride5_vf16:
1522 ; AVX: # %bb.0:
1523 ; AVX-NEXT: vmovdqa 96(%rdi), %xmm0
1524 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,1,1,3]
1525 ; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,5,4,7]
1526 ; AVX-NEXT: vmovdqa 112(%rdi), %xmm1
1527 ; AVX-NEXT: vpunpckhqdq {{.*#+}} xmm3 = xmm2[1],xmm1[1]
1528 ; AVX-NEXT: vmovdqa 80(%rdi), %xmm2
1529 ; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[0,2,2,3]
1530 ; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,3,2,3,4,5,6,7]
1531 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3,4,5,6,7]
1532 ; AVX-NEXT: vmovdqa 144(%rdi), %xmm8
1533 ; AVX-NEXT: vmovdqa 128(%rdi), %xmm7
1534 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm7[0,1],xmm8[2,3],xmm7[4,5,6,7]
1535 ; AVX-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u,u,u,u,u,2,3,12,13,6,7]
1536 ; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm3[0,1,2,3,4],xmm4[5,6,7]
1537 ; AVX-NEXT: vmovdqa (%rdi), %xmm3
1538 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm4
1539 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm5
1540 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm6
1541 ; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[0,1,0,3]
1542 ; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1,2,3],xmm5[4],xmm9[5,6,7]
1543 ; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm4[3,1,2,3]
1544 ; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[2,1,2,3,4,5,6,7]
1545 ; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm3[0,2,2,3]
1546 ; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[0,3,2,3,4,5,6,7]
1547 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
1548 ; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3],xmm9[4,5,6,7]
1549 ; AVX-NEXT: vmovaps {{.*#+}} ymm11 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,65535,65535,65535]
1550 ; AVX-NEXT: vandps %ymm11, %ymm9, %ymm12
1551 ; AVX-NEXT: vmovaps 64(%rdi), %xmm9
1552 ; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm9[0,1,0,1]
1553 ; AVX-NEXT: vandnps %ymm13, %ymm11, %ymm13
1554 ; AVX-NEXT: vorps %ymm13, %ymm12, %ymm12
1555 ; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm12, %ymm10
1556 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm1[0,1],xmm0[2,3],xmm1[4,5,6,7]
1557 ; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[0,1,3,0,4,5,6,7]
1558 ; AVX-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,5,5,5,5]
1559 ; AVX-NEXT: vpshufd {{.*#+}} xmm13 = xmm2[0,3,2,3]
1560 ; AVX-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[1,2,2,3,4,5,6,7]
1561 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3,4,5,6,7]
1562 ; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm7[0,1,2,3],xmm8[4,5],xmm7[6,7]
1563 ; AVX-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u,u,u,u,u,u,u,4,5,14,15,8,9]
1564 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm13[5,6,7]
1565 ; AVX-NEXT: vpsllq $48, %xmm9, %xmm13
1566 ; AVX-NEXT: vandnps %ymm13, %ymm11, %ymm13
1567 ; AVX-NEXT: vpsrlq $48, %xmm4, %xmm14
1568 ; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm3[0,3,2,3]
1569 ; AVX-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[1,2,2,3,4,5,6,7]
1570 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
1571 ; AVX-NEXT: vpblendw {{.*#+}} xmm15 = xmm5[0,1],xmm6[2,3],xmm5[4,5],xmm6[6,7]
1572 ; AVX-NEXT: vpshufb {{.*#+}} xmm15 = xmm15[u,u,u,u,u,u,0,1,10,11,4,5,14,15,6,7]
1573 ; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0,1,2],xmm15[3,4,5,6,7]
1574 ; AVX-NEXT: vandps %ymm11, %ymm14, %ymm11
1575 ; AVX-NEXT: vorps %ymm13, %ymm11, %ymm11
1576 ; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm11, %ymm11
1577 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
1578 ; AVX-NEXT: vpshufb {{.*#+}} xmm12 = xmm12[u,u,u,u,8,9,2,3,12,13,12,13,u,u,u,u]
1579 ; AVX-NEXT: vpshufd {{.*#+}} xmm13 = xmm2[3,1,2,3]
1580 ; AVX-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[2,1,2,3,4,5,6,7]
1581 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3,4,5,6,7]
1582 ; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm8[0,1],xmm7[2,3],xmm8[4,5,6,7]
1583 ; AVX-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u,u,u,u,u,u,u,6,7,0,1,10,11]
1584 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm13[5,6,7]
1585 ; AVX-NEXT: vpshufd {{.*#+}} xmm13 = xmm3[0,1,1,3]
1586 ; AVX-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,7,6,7]
1587 ; AVX-NEXT: vpunpckhdq {{.*#+}} xmm13 = xmm13[2],xmm4[2],xmm13[3],xmm4[3]
1588 ; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm5[0,1],xmm6[2,3],xmm5[4,5,6,7]
1589 ; AVX-NEXT: vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,u,u,2,3,12,13,6,7,u,u,u,u]
1590 ; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2],xmm14[3,4,5],xmm13[6,7]
1591 ; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm9[0,1,2,0]
1592 ; AVX-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,5,6,5]
1593 ; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,5],xmm14[6,7]
1594 ; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm13, %ymm12
1595 ; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
1596 ; AVX-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[u,u,0,1,10,11,4,5,14,15,u,u,u,u,u,u]
1597 ; AVX-NEXT: vpsrlq $48, %xmm2, %xmm14
1598 ; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm14[0],xmm13[1,2,3,4,5,6,7]
1599 ; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm8[0,1,2,3],xmm7[4,5],xmm8[6,7]
1600 ; AVX-NEXT: vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,u,u,u,u,u,u,8,9,2,3,12,13]
1601 ; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2,3,4],xmm14[5,6,7]
1602 ; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm4[0,1],xmm3[2,3],xmm4[4,5,6,7]
1603 ; AVX-NEXT: vpshufb {{.*#+}} xmm14 = xmm14[6,7,0,1,10,11,u,u,u,u,u,u,u,u,u,u]
1604 ; AVX-NEXT: vpblendw {{.*#+}} xmm15 = xmm5[0,1,2,3],xmm6[4,5],xmm5[6,7]
1605 ; AVX-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[2,2,2,2,4,5,6,7]
1606 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,7,4,6,7]
1607 ; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0,1,2],xmm15[3,4,5],xmm14[6,7]
1608 ; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm9[0,1,0,3]
1609 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,5,6]
1610 ; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,5],xmm15[6,7]
1611 ; AVX-NEXT: vinsertf128 $1, %xmm13, %ymm14, %ymm13
1612 ; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,1,2,3]
1613 ; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,2,1,4,5,6,7]
1614 ; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,2,2,3]
1615 ; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,0,3,4,5,6,7]
1616 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
1617 ; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]
1618 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5,6,7]
1619 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
1620 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3,4,5,6,7]
1621 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm7[4,5,6,7]
1622 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[1,1,1,1]
1623 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm6[0,2,2,3]
1624 ; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7]
1625 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1626 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm3[4,5],xmm4[6,7]
1627 ; AVX-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
1628 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3,4,5],xmm2[6,7]
1629 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm9[0,1,1,3]
1630 ; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]
1631 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm2[6,7]
1632 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
1633 ; AVX-NEXT: vmovaps %ymm10, (%rsi)
1634 ; AVX-NEXT: vmovaps %ymm11, (%rdx)
1635 ; AVX-NEXT: vmovaps %ymm12, (%rcx)
1636 ; AVX-NEXT: vmovaps %ymm13, (%r8)
1637 ; AVX-NEXT: vmovaps %ymm0, (%r9)
1638 ; AVX-NEXT: vzeroupper
1639 ; AVX-NEXT: retq
1640 ;
1641 ; AVX2-LABEL: load_i16_stride5_vf16:
1642 ; AVX2: # %bb.0:
1643 ; AVX2-NEXT: vmovdqa (%rdi), %ymm2
1644 ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm3
1645 ; AVX2-NEXT: vmovdqa 64(%rdi), %ymm0
1646 ; AVX2-NEXT: vmovdqa 96(%rdi), %ymm1
1647 ; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]
1648 ; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5
1649 ; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1,2,3],xmm4[4,5],xmm5[6,7]
1650 ; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[0,1,10,11,4,5,14,15,8,9,2,3,12,13,6,7]
1651 ; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
1652 ; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm5[2,3,0,1]
1653 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm6[5],ymm5[6],ymm6[7]
1654 ; AVX2-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[0,1,10,11,4,5,14,15,8,9,10,11,4,5,6,7,16,17,26,27,20,21,30,31,24,25,26,27,20,21,22,23]
1655 ; AVX2-NEXT: vpmovsxbw {{.*#+}} xmm7 = [65535,65535,65535,65535,65535,65535,65535,0]
1656 ; AVX2-NEXT: vpblendvb %ymm7, %ymm4, %ymm5, %ymm5
1657 ; AVX2-NEXT: vmovdqa 144(%rdi), %xmm6
1658 ; AVX2-NEXT: vmovdqa 128(%rdi), %xmm4
1659 ; AVX2-NEXT: vpblendd {{.*#+}} xmm8 = xmm4[0],xmm6[1],xmm4[2,3]
1660 ; AVX2-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,u,u,u,u,u,2,3,12,13,6,7]
1661 ; AVX2-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
1662 ; AVX2-NEXT: vpblendw {{.*#+}} ymm8 = ymm5[0,1,2,3,4],ymm8[5,6,7],ymm5[8,9,10,11,12],ymm8[13,14,15]
1663 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm8[4,5,6,7]
1664 ; AVX2-NEXT: vpblendw {{.*#+}} ymm8 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4,5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10],ymm2[11],ymm3[12,13],ymm2[14],ymm3[15]
1665 ; AVX2-NEXT: vextracti128 $1, %ymm8, %xmm9
1666 ; AVX2-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5,6],xmm9[7]
1667 ; AVX2-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[2,3,12,13,6,7,0,1,10,11,4,5,14,15,10,11]
1668 ; AVX2-NEXT: vpblendw {{.*#+}} ymm9 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
1669 ; AVX2-NEXT: vpermq {{.*#+}} ymm10 = ymm9[2,3,0,1]
1670 ; AVX2-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4],ymm10[5],ymm9[6,7]
1671 ; AVX2-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[2,3,12,13,6,7,0,1,10,11,6,7,8,9,8,9,18,19,28,29,22,23,16,17,26,27,22,23,24,25,24,25]
1672 ; AVX2-NEXT: vpblendvb %ymm7, %ymm8, %ymm9, %ymm7
1673 ; AVX2-NEXT: vpblendd {{.*#+}} xmm8 = xmm4[0,1],xmm6[2],xmm4[3]
1674 ; AVX2-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,u,u,u,u,u,4,5,14,15,8,9]
1675 ; AVX2-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
1676 ; AVX2-NEXT: vpblendw {{.*#+}} ymm8 = ymm7[0,1,2,3,4],ymm8[5,6,7],ymm7[8,9,10,11,12],ymm8[13,14,15]
1677 ; AVX2-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5,6,7]
1678 ; AVX2-NEXT: vpblendw {{.*#+}} ymm8 = ymm3[0,1],ymm2[2],ymm3[3],ymm2[4],ymm3[5,6],ymm2[7],ymm3[8,9],ymm2[10],ymm3[11],ymm2[12],ymm3[13,14],ymm2[15]
1679 ; AVX2-NEXT: vextracti128 $1, %ymm8, %xmm9
1680 ; AVX2-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2],xmm9[3,4],xmm8[5,6,7]
1681 ; AVX2-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
1682 ; AVX2-NEXT: vpblendw {{.*#+}} ymm9 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
1683 ; AVX2-NEXT: vpermq {{.*#+}} ymm10 = ymm9[2,3,0,1]
1684 ; AVX2-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm10[6],ymm9[7]
1685 ; AVX2-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
1686 ; AVX2-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2],ymm9[3,4,5,6,7]
1687 ; AVX2-NEXT: vpblendd {{.*#+}} xmm9 = xmm6[0],xmm4[1],xmm6[2,3]
1688 ; AVX2-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[u,u,u,u,u,u,u,u,u,u,6,7,0,1,10,11]
1689 ; AVX2-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
1690 ; AVX2-NEXT: vpblendw {{.*#+}} ymm9 = ymm8[0,1,2,3,4],ymm9[5,6,7],ymm8[8,9,10,11,12],ymm9[13,14,15]
1691 ; AVX2-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm9[4,5,6,7]
1692 ; AVX2-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0],ymm3[1,2],ymm2[3],ymm3[4],ymm2[5],ymm3[6,7],ymm2[8],ymm3[9,10],ymm2[11],ymm3[12],ymm2[13],ymm3[14,15]
1693 ; AVX2-NEXT: vextracti128 $1, %ymm9, %xmm10
1694 ; AVX2-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2],xmm9[3]
1695 ; AVX2-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
1696 ; AVX2-NEXT: vpblendw {{.*#+}} ymm10 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
1697 ; AVX2-NEXT: vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
1698 ; AVX2-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4],ymm10[5],ymm11[6],ymm10[7]
1699 ; AVX2-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
1700 ; AVX2-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
1701 ; AVX2-NEXT: vpblendd {{.*#+}} xmm10 = xmm6[0,1],xmm4[2],xmm6[3]
1702 ; AVX2-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[u,u,u,u,u,u,u,u,u,u,8,9,2,3,12,13]
1703 ; AVX2-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
1704 ; AVX2-NEXT: vpblendw {{.*#+}} ymm10 = ymm9[0,1,2,3,4],ymm10[5,6,7],ymm9[8,9,10,11,12],ymm10[13,14,15]
1705 ; AVX2-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5,6,7]
1706 ; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13],ymm2[14],ymm3[15]
1707 ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
1708 ; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
1709 ; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
1710 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
1711 ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
1712 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6],ymm1[7]
1713 ; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,14,15,24,25,18,19,28,29,22,23,u,u,u,u,u,u,u,u]
1714 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
1715 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm6[3,1,2,3]
1716 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,1,4,5,6,7]
1717 ; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
1718 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7]
1719 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1720 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
1721 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
1722 ; AVX2-NEXT: vmovdqa %ymm5, (%rsi)
1723 ; AVX2-NEXT: vmovdqa %ymm7, (%rdx)
1724 ; AVX2-NEXT: vmovdqa %ymm8, (%rcx)
1725 ; AVX2-NEXT: vmovdqa %ymm9, (%r8)
1726 ; AVX2-NEXT: vmovdqa %ymm0, (%r9)
1727 ; AVX2-NEXT: vzeroupper
1728 ; AVX2-NEXT: retq
1729 ;
1730 ; AVX2-FP-LABEL: load_i16_stride5_vf16:
1731 ; AVX2-FP: # %bb.0:
1732 ; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm2
1733 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm3
1734 ; AVX2-FP-NEXT: vmovdqa 64(%rdi), %ymm0
1735 ; AVX2-FP-NEXT: vmovdqa 96(%rdi), %ymm1
1736 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm4 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]
1737 ; AVX2-FP-NEXT: vextracti128 $1, %ymm4, %xmm5
1738 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1,2,3],xmm4[4,5],xmm5[6,7]
1739 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[0,1,10,11,4,5,14,15,8,9,2,3,12,13,6,7]
1740 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm5 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
1741 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm6 = ymm5[2,3,0,1]
1742 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm6[5],ymm5[6],ymm6[7]
1743 ; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[0,1,10,11,4,5,14,15,8,9,10,11,4,5,6,7,16,17,26,27,20,21,30,31,24,25,26,27,20,21,22,23]
1744 ; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} xmm7 = [65535,65535,65535,65535,65535,65535,65535,0]
1745 ; AVX2-FP-NEXT: vpblendvb %ymm7, %ymm4, %ymm5, %ymm6
1746 ; AVX2-FP-NEXT: vmovdqa 144(%rdi), %xmm4
1747 ; AVX2-FP-NEXT: vmovdqa 128(%rdi), %xmm5
1748 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm8 = xmm5[0],xmm4[1],xmm5[2,3]
1749 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,u,u,u,u,u,2,3,12,13,6,7]
1750 ; AVX2-FP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
1751 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm6[0,1,2,3,4],ymm8[5,6,7],ymm6[8,9,10,11,12],ymm8[13,14,15]
1752 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm8[4,5,6,7]
1753 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4,5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10],ymm2[11],ymm3[12,13],ymm2[14],ymm3[15]
1754 ; AVX2-FP-NEXT: vextracti128 $1, %ymm8, %xmm9
1755 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5,6],xmm9[7]
1756 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[2,3,12,13,6,7,0,1,10,11,4,5,14,15,10,11]
1757 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
1758 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm10 = ymm9[2,3,0,1]
1759 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4],ymm10[5],ymm9[6,7]
1760 ; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[2,3,12,13,6,7,0,1,10,11,6,7,8,9,8,9,18,19,28,29,22,23,16,17,26,27,22,23,24,25,24,25]
1761 ; AVX2-FP-NEXT: vpblendvb %ymm7, %ymm8, %ymm9, %ymm7
1762 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm8 = xmm5[0,1],xmm4[2],xmm5[3]
1763 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,u,u,u,u,u,4,5,14,15,8,9]
1764 ; AVX2-FP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
1765 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm7[0,1,2,3,4],ymm8[5,6,7],ymm7[8,9,10,11,12],ymm8[13,14,15]
1766 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5,6,7]
1767 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm3[0,1],ymm2[2],ymm3[3],ymm2[4],ymm3[5,6],ymm2[7],ymm3[8,9],ymm2[10],ymm3[11],ymm2[12],ymm3[13,14],ymm2[15]
1768 ; AVX2-FP-NEXT: vextracti128 $1, %ymm8, %xmm9
1769 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2],xmm9[3,4],xmm8[5,6,7]
1770 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
1771 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
1772 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm10 = ymm9[2,3,0,1]
1773 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm10[6],ymm9[7]
1774 ; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
1775 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2],ymm9[3,4,5,6,7]
1776 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm9 = xmm4[0],xmm5[1],xmm4[2,3]
1777 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[u,u,u,u,u,u,u,u,u,u,6,7,0,1,10,11]
1778 ; AVX2-FP-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
1779 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm8[0,1,2,3,4],ymm9[5,6,7],ymm8[8,9,10,11,12],ymm9[13,14,15]
1780 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm9[4,5,6,7]
1781 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0],ymm3[1,2],ymm2[3],ymm3[4],ymm2[5],ymm3[6,7],ymm2[8],ymm3[9,10],ymm2[11],ymm3[12],ymm2[13],ymm3[14,15]
1782 ; AVX2-FP-NEXT: vextracti128 $1, %ymm9, %xmm10
1783 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2],xmm9[3]
1784 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
1785 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm10 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
1786 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
1787 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4],ymm10[5],ymm11[6],ymm10[7]
1788 ; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
1789 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
1790 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm10 = xmm4[0,1],xmm5[2],xmm4[3]
1791 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[u,u,u,u,u,u,u,u,u,u,8,9,2,3,12,13]
1792 ; AVX2-FP-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
1793 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm10 = ymm9[0,1,2,3,4],ymm10[5,6,7],ymm9[8,9,10,11,12],ymm10[13,14,15]
1794 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5,6,7]
1795 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13],ymm2[14],ymm3[15]
1796 ; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm3
1797 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
1798 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
1799 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
1800 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
1801 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6],ymm1[7]
1802 ; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,14,15,24,25,18,19,28,29,22,23,u,u,u,u,u,u,u,u]
1803 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
1804 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm1 = xmm4[12,13,14,15,4,5,14,15,u,u,u,u,u,u,u,u]
1805 ; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm2 = xmm5[0,1,2,3,0,1,10,11,u,u,u,u,u,u,u,u]
1806 ; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1807 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
1808 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
1809 ; AVX2-FP-NEXT: vmovdqa %ymm6, (%rsi)
1810 ; AVX2-FP-NEXT: vmovdqa %ymm7, (%rdx)
1811 ; AVX2-FP-NEXT: vmovdqa %ymm8, (%rcx)
1812 ; AVX2-FP-NEXT: vmovdqa %ymm9, (%r8)
1813 ; AVX2-FP-NEXT: vmovdqa %ymm0, (%r9)
1814 ; AVX2-FP-NEXT: vzeroupper
1815 ; AVX2-FP-NEXT: retq
1816 ;
1817 ; AVX2-FCP-LABEL: load_i16_stride5_vf16:
1818 ; AVX2-FCP: # %bb.0:
1819 ; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %ymm0
1820 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm3
1821 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm4
1822 ; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %ymm1
1823 ; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %ymm2
1824 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5],ymm4[6],ymm3[7,8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13],ymm4[14],ymm3[15]
1825 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm5, %xmm6
1826 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm6[1,2,3],xmm5[4,5],xmm6[6,7]
1827 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[0,1,10,11,4,5,14,15,8,9,2,3,12,13,6,7]
1828 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm1[0],ymm2[1,2],ymm1[3],ymm2[4],ymm1[5],ymm2[6,7],ymm1[8],ymm2[9,10],ymm1[11],ymm2[12],ymm1[13],ymm2[14,15]
1829 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm7 = [1,3,0,2,4,6,1,3]
1830 ; AVX2-FCP-NEXT: vpermd %ymm6, %ymm7, %ymm6
1831 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm6 = ymm6[0,1,6,7,8,9,14,15,4,5,14,15,4,5,2,3,16,17,22,23,24,25,30,31,20,21,30,31,20,21,18,19]
1832 ; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} xmm8 = [65535,65535,65535,65535,65535,65535,65535,0]
1833 ; AVX2-FCP-NEXT: vpblendvb %ymm8, %ymm5, %ymm6, %ymm5
1834 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [0,0,0,0,0,3,5,0]
1835 ; AVX2-FCP-NEXT: vpermd %ymm0, %ymm6, %ymm6
1836 ; AVX2-FCP-NEXT: vpbroadcastq {{.*#+}} ymm7 = [0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27]
1837 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm6, %ymm6
1838 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm5[0,1,2,3,4],ymm6[5,6,7],ymm5[8,9,10,11,12],ymm6[13,14,15]
1839 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
1840 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4,5],ymm3[6],ymm4[7,8],ymm3[9],ymm4[10],ymm3[11],ymm4[12,13],ymm3[14],ymm4[15]
1841 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm6, %xmm9
1842 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm9[2,3],xmm6[4,5,6],xmm9[7]
1843 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[2,3,12,13,6,7,0,1,10,11,4,5,14,15,10,11]
1844 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5],ymm1[6],ymm2[7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13],ymm1[14],ymm2[15]
1845 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm10 = [2,0,0,0,4,7,1,6]
1846 ; AVX2-FCP-NEXT: vpermd %ymm9, %ymm10, %ymm9
1847 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[2,3,4,5,10,11,0,1,14,15,2,3,12,13,0,1,18,19,20,21,26,27,16,17,30,31,18,19,28,29,16,17]
1848 ; AVX2-FCP-NEXT: vpblendvb %ymm8, %ymm6, %ymm9, %ymm6
1849 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [0,0,0,0,1,3,6,0]
1850 ; AVX2-FCP-NEXT: vpermd %ymm0, %ymm8, %ymm9
1851 ; AVX2-FCP-NEXT: vpbroadcastq {{.*#+}} ymm8 = [0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25]
1852 ; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm9, %ymm9
1853 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm9 = ymm6[0,1,2,3,4],ymm9[5,6,7],ymm6[8,9,10,11,12],ymm9[13,14,15]
1854 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm9[4,5,6,7]
1855 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm9 = ymm4[0,1],ymm3[2],ymm4[3],ymm3[4],ymm4[5,6],ymm3[7],ymm4[8,9],ymm3[10],ymm4[11],ymm3[12],ymm4[13,14],ymm3[15]
1856 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm9, %xmm10
1857 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1,2],xmm10[3,4],xmm9[5,6,7]
1858 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
1859 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5],ymm2[6],ymm1[7,8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
1860 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm11 = [0,2,0,0,5,7,2,4]
1861 ; AVX2-FCP-NEXT: vpermd %ymm10, %ymm11, %ymm10
1862 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
1863 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
1864 ; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [1,4,6,0,1,4,6,0]
1865 ; AVX2-FCP-NEXT: # ymm10 = mem[0,1,0,1]
1866 ; AVX2-FCP-NEXT: vpermd %ymm0, %ymm10, %ymm10
1867 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm10, %ymm7
1868 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm9[0,1,2,3,4],ymm7[5,6,7],ymm9[8,9,10,11,12],ymm7[13,14,15]
1869 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3],ymm7[4,5,6,7]
1870 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm9 = ymm3[0],ymm4[1,2],ymm3[3],ymm4[4],ymm3[5],ymm4[6,7],ymm3[8],ymm4[9,10],ymm3[11],ymm4[12],ymm3[13],ymm4[14,15]
1871 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm9, %xmm10
1872 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2],xmm9[3]
1873 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
1874 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7,8],ymm1[9],ymm2[10],ymm1[11],ymm2[12,13],ymm1[14],ymm2[15]
1875 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm11 = [0,3,0,0,5,0,2,7]
1876 ; AVX2-FCP-NEXT: vpermd %ymm10, %ymm11, %ymm10
1877 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,2,3,4,5,18,19,20,21,26,27,16,17,30,31,30,31,18,19,20,21]
1878 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
1879 ; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [2,4,7,0,2,4,7,0]
1880 ; AVX2-FCP-NEXT: # ymm10 = mem[0,1,0,1]
1881 ; AVX2-FCP-NEXT: vpermd %ymm0, %ymm10, %ymm10
1882 ; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm10, %ymm8
1883 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5,6,7],ymm9[8,9,10,11,12],ymm8[13,14,15]
1884 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
1885 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5],ymm3[6],ymm4[7,8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13],ymm3[14],ymm4[15]
1886 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm3, %xmm4
1887 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3,4],xmm4[5,6,7]
1888 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
1889 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3],ymm1[4],ymm2[5,6],ymm1[7],ymm2[8,9],ymm1[10],ymm2[11],ymm1[12],ymm2[13,14],ymm1[15]
1890 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,3,0,0,6,0,3,5]
1891 ; AVX2-FCP-NEXT: vpermd %ymm1, %ymm2, %ymm1
1892 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,u,u,u,u,u,u,u,u]
1893 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3,4,5,6,7]
1894 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,0,0,0,0,2,5,7]
1895 ; AVX2-FCP-NEXT: vpermd %ymm0, %ymm2, %ymm0
1896 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,17,22,23,24,25,30,31]
1897 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
1898 ; AVX2-FCP-NEXT: vmovdqa %ymm5, (%rsi)
1899 ; AVX2-FCP-NEXT: vmovdqa %ymm6, (%rdx)
1900 ; AVX2-FCP-NEXT: vmovdqa %ymm7, (%rcx)
1901 ; AVX2-FCP-NEXT: vmovdqa %ymm8, (%r8)
1902 ; AVX2-FCP-NEXT: vmovdqa %ymm0, (%r9)
1903 ; AVX2-FCP-NEXT: vzeroupper
1904 ; AVX2-FCP-NEXT: retq
1905 ;
1906 ; AVX512-LABEL: load_i16_stride5_vf16:
1907 ; AVX512: # %bb.0:
1908 ; AVX512-NEXT: vmovdqa (%rdi), %ymm2
1909 ; AVX512-NEXT: vmovdqa 32(%rdi), %ymm3
1910 ; AVX512-NEXT: vmovdqa 64(%rdi), %ymm0
1911 ; AVX512-NEXT: vmovdqa 96(%rdi), %ymm1
1912 ; AVX512-NEXT: vpblendw {{.*#+}} ymm4 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
1913 ; AVX512-NEXT: vpermq {{.*#+}} ymm5 = ymm4[2,3,0,1]
1914 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm5[5],ymm4[6],ymm5[7]
1915 ; AVX512-NEXT: vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[6,7,16,17,26,27,20,21,30,31,24,25],zero,zero,zero,zero,zero,zero
1916 ; AVX512-NEXT: vpblendw {{.*#+}} ymm5 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]
1917 ; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm6
1918 ; AVX512-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm6[1,2,3],xmm5[4,5],xmm6[6,7]
1919 ; AVX512-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[u,u,u,u,u,u]
1920 ; AVX512-NEXT: vpor %ymm4, %ymm5, %ymm5
1921 ; AVX512-NEXT: vmovdqa 144(%rdi), %xmm6
1922 ; AVX512-NEXT: vmovdqa 128(%rdi), %xmm4
1923 ; AVX512-NEXT: vpblendd {{.*#+}} xmm7 = xmm4[0],xmm6[1],xmm4[2,3]
1924 ; AVX512-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,u,u,u,u,2,3,12,13,6,7]
1925 ; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
1926 ; AVX512-NEXT: vpblendw {{.*#+}} ymm7 = ymm5[0,1,2,3,4],ymm7[5,6,7],ymm5[8,9,10,11,12],ymm7[13,14,15]
1927 ; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm7[4,5,6,7]
1928 ; AVX512-NEXT: vpblendw {{.*#+}} ymm7 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
1929 ; AVX512-NEXT: vpermq {{.*#+}} ymm8 = ymm7[2,3,0,1]
1930 ; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4],ymm8[5],ymm7[6,7]
1931 ; AVX512-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[8,9,18,19,28,29,22,23,16,17,26,27],zero,zero,zero,zero,zero,zero
1932 ; AVX512-NEXT: vpblendw {{.*#+}} ymm8 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4,5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10],ymm2[11],ymm3[12,13],ymm2[14],ymm3[15]
1933 ; AVX512-NEXT: vextracti128 $1, %ymm8, %xmm9
1934 ; AVX512-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5,6],xmm9[7]
1935 ; AVX512-NEXT: vpshufb {{.*#+}} ymm8 = ymm8[2,3,12,13,6,7,0,1,10,11,4,5,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[u,u,u,u,u,u]
1936 ; AVX512-NEXT: vpor %ymm7, %ymm8, %ymm7
1937 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm4[0,1],xmm6[2],xmm4[3]
1938 ; AVX512-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,u,u,u,u,u,4,5,14,15,8,9]
1939 ; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
1940 ; AVX512-NEXT: vpblendw {{.*#+}} ymm8 = ymm7[0,1,2,3,4],ymm8[5,6,7],ymm7[8,9,10,11,12],ymm8[13,14,15]
1941 ; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5,6,7]
1942 ; AVX512-NEXT: vpblendw {{.*#+}} ymm8 = ymm3[0,1],ymm2[2],ymm3[3],ymm2[4],ymm3[5,6],ymm2[7],ymm3[8,9],ymm2[10],ymm3[11],ymm2[12],ymm3[13,14],ymm2[15]
1943 ; AVX512-NEXT: vextracti128 $1, %ymm8, %xmm9
1944 ; AVX512-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2],xmm9[3,4],xmm8[5,6,7]
1945 ; AVX512-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
1946 ; AVX512-NEXT: vpblendw {{.*#+}} ymm9 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
1947 ; AVX512-NEXT: vpermq {{.*#+}} ymm10 = ymm9[2,3,0,1]
1948 ; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm10[6],ymm9[7]
1949 ; AVX512-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
1950 ; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2],ymm9[3,4,5,6,7]
1951 ; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm6[0],xmm4[1],xmm6[2,3]
1952 ; AVX512-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[u,u,u,u,u,u,u,u,u,u,6,7,0,1,10,11]
1953 ; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
1954 ; AVX512-NEXT: vpblendw {{.*#+}} ymm9 = ymm8[0,1,2,3,4],ymm9[5,6,7],ymm8[8,9,10,11,12],ymm9[13,14,15]
1955 ; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm9[4,5,6,7]
1956 ; AVX512-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0],ymm3[1,2],ymm2[3],ymm3[4],ymm2[5],ymm3[6,7],ymm2[8],ymm3[9,10],ymm2[11],ymm3[12],ymm2[13],ymm3[14,15]
1957 ; AVX512-NEXT: vextracti128 $1, %ymm9, %xmm10
1958 ; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2],xmm9[3]
1959 ; AVX512-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
1960 ; AVX512-NEXT: vpblendw {{.*#+}} ymm10 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
1961 ; AVX512-NEXT: vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
1962 ; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4],ymm10[5],ymm11[6],ymm10[7]
1963 ; AVX512-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
1964 ; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
1965 ; AVX512-NEXT: vpblendd {{.*#+}} xmm10 = xmm6[0,1],xmm4[2],xmm6[3]
1966 ; AVX512-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[u,u,u,u,u,u,u,u,u,u,8,9,2,3,12,13]
1967 ; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
1968 ; AVX512-NEXT: vpblendw {{.*#+}} ymm10 = ymm9[0,1,2,3,4],ymm10[5,6,7],ymm9[8,9,10,11,12],ymm10[13,14,15]
1969 ; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5,6,7]
1970 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13],ymm2[14],ymm3[15]
1971 ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm3
1972 ; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
1973 ; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
1974 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
1975 ; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
1976 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6],ymm1[7]
1977 ; AVX512-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,14,15,24,25,18,19,28,29,22,23,u,u,u,u,u,u,u,u]
1978 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
1979 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm6[3,1,2,3]
1980 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,1,4,5,6,7]
1981 ; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
1982 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7]
1983 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
1984 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
1985 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
1986 ; AVX512-NEXT: vmovdqa %ymm5, (%rsi)
1987 ; AVX512-NEXT: vmovdqa %ymm7, (%rdx)
1988 ; AVX512-NEXT: vmovdqa %ymm8, (%rcx)
1989 ; AVX512-NEXT: vmovdqa %ymm9, (%r8)
1990 ; AVX512-NEXT: vmovdqa %ymm0, (%r9)
1991 ; AVX512-NEXT: vzeroupper
1992 ; AVX512-NEXT: retq
1993 ;
1994 ; AVX512-FCP-LABEL: load_i16_stride5_vf16:
1995 ; AVX512-FCP: # %bb.0:
1996 ; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm2
1997 ; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm3
1998 ; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm0
1999 ; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm1
2000 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
2001 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [1,0,0,0,4,6,1,3]
2002 ; AVX512-FCP-NEXT: vpermd %ymm4, %ymm5, %ymm4
2003 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[2,3,16,17,22,23,24,25,30,31,20,21],zero,zero,zero,zero,zero,zero
2004 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]
2005 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm5, %xmm6
2006 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm6[1,2,3],xmm5[4,5],xmm6[6,7]
2007 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[u,u,u,u,u,u]
2008 ; AVX512-FCP-NEXT: vpor %ymm4, %ymm5, %ymm5
2009 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [0,0,0,0,0,3,5,0]
2010 ; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
2011 ; AVX512-FCP-NEXT: vpermd %ymm4, %ymm6, %ymm6
2012 ; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm7 = [0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27]
2013 ; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm6, %ymm6
2014 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm5[0,1,2,3,4],ymm6[5,6,7],ymm5[8,9,10,11,12],ymm6[13,14,15]
2015 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
2016 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
2017 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [2,0,0,0,4,7,1,6]
2018 ; AVX512-FCP-NEXT: vpermd %ymm6, %ymm8, %ymm6
2019 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[0,1,18,19,20,21,26,27,16,17,30,31],zero,zero,zero,zero,zero,zero
2020 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4,5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10],ymm2[11],ymm3[12,13],ymm2[14],ymm3[15]
2021 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm8, %xmm9
2022 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5,6],xmm9[7]
2023 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm8[2,3,12,13,6,7,0,1,10,11,4,5,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[u,u,u,u,u,u]
2024 ; AVX512-FCP-NEXT: vpor %ymm6, %ymm8, %ymm6
2025 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [0,0,0,0,1,3,6,0]
2026 ; AVX512-FCP-NEXT: vpermd %ymm4, %ymm8, %ymm9
2027 ; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm8 = [0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25]
2028 ; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm9, %ymm9
2029 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm9 = ymm6[0,1,2,3,4],ymm9[5,6,7],ymm6[8,9,10,11,12],ymm9[13,14,15]
2030 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm9[4,5,6,7]
2031 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm9 = ymm3[0,1],ymm2[2],ymm3[3],ymm2[4],ymm3[5,6],ymm2[7],ymm3[8,9],ymm2[10],ymm3[11],ymm2[12],ymm3[13,14],ymm2[15]
2032 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm9, %xmm10
2033 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1,2],xmm10[3,4],xmm9[5,6,7]
2034 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
2035 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
2036 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm11 = [0,2,0,0,5,7,2,4]
2037 ; AVX512-FCP-NEXT: vpermd %ymm10, %ymm11, %ymm10
2038 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
2039 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
2040 ; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [1,4,6,0,1,4,6,0]
2041 ; AVX512-FCP-NEXT: # ymm10 = mem[0,1,0,1]
2042 ; AVX512-FCP-NEXT: vpermd %ymm4, %ymm10, %ymm10
2043 ; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm10, %ymm7
2044 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm9[0,1,2,3,4],ymm7[5,6,7],ymm9[8,9,10,11,12],ymm7[13,14,15]
2045 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3],ymm7[4,5,6,7]
2046 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0],ymm3[1,2],ymm2[3],ymm3[4],ymm2[5],ymm3[6,7],ymm2[8],ymm3[9,10],ymm2[11],ymm3[12],ymm2[13],ymm3[14,15]
2047 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm9, %xmm10
2048 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2],xmm9[3]
2049 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
2050 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
2051 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm11 = [0,3,0,0,5,0,2,7]
2052 ; AVX512-FCP-NEXT: vpermd %ymm10, %ymm11, %ymm10
2053 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,2,3,4,5,18,19,20,21,26,27,16,17,30,31,30,31,18,19,20,21]
2054 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
2055 ; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [2,4,7,0,2,4,7,0]
2056 ; AVX512-FCP-NEXT: # ymm10 = mem[0,1,0,1]
2057 ; AVX512-FCP-NEXT: vpermd %ymm4, %ymm10, %ymm10
2058 ; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm10, %ymm8
2059 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5,6,7],ymm9[8,9,10,11,12],ymm8[13,14,15]
2060 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
2061 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13],ymm2[14],ymm3[15]
2062 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm2, %xmm3
2063 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
2064 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
2065 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
2066 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,0,0,6,0,3,5]
2067 ; AVX512-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm0
2068 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,u,u,u,u,u,u,u,u]
2069 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
2070 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,0,0,0,0,2,5,7]
2071 ; AVX512-FCP-NEXT: vpermd %ymm4, %ymm1, %ymm1
2072 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,17,22,23,u,u,u,u,24,25,30,31,u,u,u,u]
2073 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,1,2,3,4,5,12,14]
2074 ; AVX512-FCP-NEXT: vpermi2d %ymm1, %ymm0, %ymm2
2075 ; AVX512-FCP-NEXT: vmovdqa %ymm5, (%rsi)
2076 ; AVX512-FCP-NEXT: vmovdqa %ymm6, (%rdx)
2077 ; AVX512-FCP-NEXT: vmovdqa %ymm7, (%rcx)
2078 ; AVX512-FCP-NEXT: vmovdqa %ymm8, (%r8)
2079 ; AVX512-FCP-NEXT: vmovdqa %ymm2, (%r9)
2080 ; AVX512-FCP-NEXT: vzeroupper
2081 ; AVX512-FCP-NEXT: retq
2082 ;
2083 ; AVX512DQ-LABEL: load_i16_stride5_vf16:
2084 ; AVX512DQ: # %bb.0:
2085 ; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm2
2086 ; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm3
2087 ; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm0
2088 ; AVX512DQ-NEXT: vmovdqa 96(%rdi), %ymm1
2089 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm4 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
2090 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm5 = ymm4[2,3,0,1]
2091 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm5[5],ymm4[6],ymm5[7]
2092 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[6,7,16,17,26,27,20,21,30,31,24,25],zero,zero,zero,zero,zero,zero
2093 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm5 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]
2094 ; AVX512DQ-NEXT: vextracti128 $1, %ymm5, %xmm6
2095 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm6[1,2,3],xmm5[4,5],xmm6[6,7]
2096 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[u,u,u,u,u,u]
2097 ; AVX512DQ-NEXT: vpor %ymm4, %ymm5, %ymm5
2098 ; AVX512DQ-NEXT: vmovdqa 144(%rdi), %xmm6
2099 ; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm4
2100 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm7 = xmm4[0],xmm6[1],xmm4[2,3]
2101 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,u,u,u,u,2,3,12,13,6,7]
2102 ; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
2103 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm7 = ymm5[0,1,2,3,4],ymm7[5,6,7],ymm5[8,9,10,11,12],ymm7[13,14,15]
2104 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm7[4,5,6,7]
2105 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm7 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
2106 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm7[2,3,0,1]
2107 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4],ymm8[5],ymm7[6,7]
2108 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[8,9,18,19,28,29,22,23,16,17,26,27],zero,zero,zero,zero,zero,zero
2109 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm8 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4,5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10],ymm2[11],ymm3[12,13],ymm2[14],ymm3[15]
2110 ; AVX512DQ-NEXT: vextracti128 $1, %ymm8, %xmm9
2111 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5,6],xmm9[7]
2112 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm8 = ymm8[2,3,12,13,6,7,0,1,10,11,4,5,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[u,u,u,u,u,u]
2113 ; AVX512DQ-NEXT: vpor %ymm7, %ymm8, %ymm7
2114 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm4[0,1],xmm6[2],xmm4[3]
2115 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,u,u,u,u,u,4,5,14,15,8,9]
2116 ; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
2117 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm8 = ymm7[0,1,2,3,4],ymm8[5,6,7],ymm7[8,9,10,11,12],ymm8[13,14,15]
2118 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5,6,7]
2119 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm8 = ymm3[0,1],ymm2[2],ymm3[3],ymm2[4],ymm3[5,6],ymm2[7],ymm3[8,9],ymm2[10],ymm3[11],ymm2[12],ymm3[13,14],ymm2[15]
2120 ; AVX512DQ-NEXT: vextracti128 $1, %ymm8, %xmm9
2121 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2],xmm9[3,4],xmm8[5,6,7]
2122 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
2123 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm9 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
2124 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm10 = ymm9[2,3,0,1]
2125 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm10[6],ymm9[7]
2126 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
2127 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2],ymm9[3,4,5,6,7]
2128 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm6[0],xmm4[1],xmm6[2,3]
2129 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[u,u,u,u,u,u,u,u,u,u,6,7,0,1,10,11]
2130 ; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
2131 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm9 = ymm8[0,1,2,3,4],ymm9[5,6,7],ymm8[8,9,10,11,12],ymm9[13,14,15]
2132 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm9[4,5,6,7]
2133 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0],ymm3[1,2],ymm2[3],ymm3[4],ymm2[5],ymm3[6,7],ymm2[8],ymm3[9,10],ymm2[11],ymm3[12],ymm2[13],ymm3[14,15]
2134 ; AVX512DQ-NEXT: vextracti128 $1, %ymm9, %xmm10
2135 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2],xmm9[3]
2136 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
2137 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm10 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
2138 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
2139 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4],ymm10[5],ymm11[6],ymm10[7]
2140 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
2141 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
2142 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm10 = xmm6[0,1],xmm4[2],xmm6[3]
2143 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[u,u,u,u,u,u,u,u,u,u,8,9,2,3,12,13]
2144 ; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
2145 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm10 = ymm9[0,1,2,3,4],ymm10[5,6,7],ymm9[8,9,10,11,12],ymm10[13,14,15]
2146 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5,6,7]
2147 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13],ymm2[14],ymm3[15]
2148 ; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
2149 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
2150 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
2151 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
2152 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
2153 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6],ymm1[7]
2154 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,14,15,24,25,18,19,28,29,22,23,u,u,u,u,u,u,u,u]
2155 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
2156 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm6[3,1,2,3]
2157 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,1,4,5,6,7]
2158 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
2159 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7]
2160 ; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
2161 ; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
2162 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
2163 ; AVX512DQ-NEXT: vmovdqa %ymm5, (%rsi)
2164 ; AVX512DQ-NEXT: vmovdqa %ymm7, (%rdx)
2165 ; AVX512DQ-NEXT: vmovdqa %ymm8, (%rcx)
2166 ; AVX512DQ-NEXT: vmovdqa %ymm9, (%r8)
2167 ; AVX512DQ-NEXT: vmovdqa %ymm0, (%r9)
2168 ; AVX512DQ-NEXT: vzeroupper
2169 ; AVX512DQ-NEXT: retq
2170 ;
2171 ; AVX512DQ-FCP-LABEL: load_i16_stride5_vf16:
2172 ; AVX512DQ-FCP: # %bb.0:
2173 ; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm2
2174 ; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm3
2175 ; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm0
2176 ; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm1
2177 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
2178 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [1,0,0,0,4,6,1,3]
2179 ; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm5, %ymm4
2180 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[2,3,16,17,22,23,24,25,30,31,20,21],zero,zero,zero,zero,zero,zero
2181 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]
2182 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm5, %xmm6
2183 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm6[1,2,3],xmm5[4,5],xmm6[6,7]
2184 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[u,u,u,u,u,u]
2185 ; AVX512DQ-FCP-NEXT: vpor %ymm4, %ymm5, %ymm5
2186 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [0,0,0,0,0,3,5,0]
2187 ; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
2188 ; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm6, %ymm6
2189 ; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm7 = [0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27]
2190 ; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm6, %ymm6
2191 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm5[0,1,2,3,4],ymm6[5,6,7],ymm5[8,9,10,11,12],ymm6[13,14,15]
2192 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
2193 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
2194 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [2,0,0,0,4,7,1,6]
2195 ; AVX512DQ-FCP-NEXT: vpermd %ymm6, %ymm8, %ymm6
2196 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[0,1,18,19,20,21,26,27,16,17,30,31],zero,zero,zero,zero,zero,zero
2197 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4,5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10],ymm2[11],ymm3[12,13],ymm2[14],ymm3[15]
2198 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm8, %xmm9
2199 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5,6],xmm9[7]
2200 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm8[2,3,12,13,6,7,0,1,10,11,4,5,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[u,u,u,u,u,u]
2201 ; AVX512DQ-FCP-NEXT: vpor %ymm6, %ymm8, %ymm6
2202 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [0,0,0,0,1,3,6,0]
2203 ; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm8, %ymm9
2204 ; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm8 = [0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25]
2205 ; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm9, %ymm9
2206 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm9 = ymm6[0,1,2,3,4],ymm9[5,6,7],ymm6[8,9,10,11,12],ymm9[13,14,15]
2207 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm9[4,5,6,7]
2208 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm9 = ymm3[0,1],ymm2[2],ymm3[3],ymm2[4],ymm3[5,6],ymm2[7],ymm3[8,9],ymm2[10],ymm3[11],ymm2[12],ymm3[13,14],ymm2[15]
2209 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm9, %xmm10
2210 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1,2],xmm10[3,4],xmm9[5,6,7]
2211 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
2212 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
2213 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm11 = [0,2,0,0,5,7,2,4]
2214 ; AVX512DQ-FCP-NEXT: vpermd %ymm10, %ymm11, %ymm10
2215 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
2216 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
2217 ; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [1,4,6,0,1,4,6,0]
2218 ; AVX512DQ-FCP-NEXT: # ymm10 = mem[0,1,0,1]
2219 ; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm10, %ymm10
2220 ; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm10, %ymm7
2221 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm9[0,1,2,3,4],ymm7[5,6,7],ymm9[8,9,10,11,12],ymm7[13,14,15]
2222 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3],ymm7[4,5,6,7]
2223 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0],ymm3[1,2],ymm2[3],ymm3[4],ymm2[5],ymm3[6,7],ymm2[8],ymm3[9,10],ymm2[11],ymm3[12],ymm2[13],ymm3[14,15]
2224 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm9, %xmm10
2225 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2],xmm9[3]
2226 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
2227 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
2228 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm11 = [0,3,0,0,5,0,2,7]
2229 ; AVX512DQ-FCP-NEXT: vpermd %ymm10, %ymm11, %ymm10
2230 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,2,3,4,5,18,19,20,21,26,27,16,17,30,31,30,31,18,19,20,21]
2231 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
2232 ; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [2,4,7,0,2,4,7,0]
2233 ; AVX512DQ-FCP-NEXT: # ymm10 = mem[0,1,0,1]
2234 ; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm10, %ymm10
2235 ; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm10, %ymm8
2236 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5,6,7],ymm9[8,9,10,11,12],ymm8[13,14,15]
2237 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
2238 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13],ymm2[14],ymm3[15]
2239 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm2, %xmm3
2240 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
2241 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
2242 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
2243 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,0,0,6,0,3,5]
2244 ; AVX512DQ-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm0
2245 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,u,u,u,u,u,u,u,u]
2246 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
2247 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,0,0,0,0,2,5,7]
2248 ; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm1, %ymm1
2249 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,17,22,23,u,u,u,u,24,25,30,31,u,u,u,u]
2250 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,1,2,3,4,5,12,14]
2251 ; AVX512DQ-FCP-NEXT: vpermi2d %ymm1, %ymm0, %ymm2
2252 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm5, (%rsi)
2253 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm6, (%rdx)
2254 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm7, (%rcx)
2255 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm8, (%r8)
2256 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm2, (%r9)
2257 ; AVX512DQ-FCP-NEXT: vzeroupper
2258 ; AVX512DQ-FCP-NEXT: retq
2259 ;
2260 ; AVX512BW-LABEL: load_i16_stride5_vf16:
2261 ; AVX512BW: # %bb.0:
2262 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
2263 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
2264 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
2265 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
2266 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm3 = [0,1,2,3,4,5,6,7,8,9,10,11,12,17,22,27]
2267 ; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm4
2268 ; AVX512BW-NEXT: vpermi2w %ymm4, %ymm2, %ymm3
2269 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
2270 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
2271 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,18,23,28]
2272 ; AVX512BW-NEXT: vpermi2w %ymm4, %ymm2, %ymm5
2273 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0]
2274 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm2
2275 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,1,2,3,4,5,6,7,8,9,10,11,12,19,24,29]
2276 ; AVX512BW-NEXT: vpermi2w %ymm4, %ymm2, %ymm6
2277 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0]
2278 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm2
2279 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm7 = [0,1,2,3,4,5,6,7,8,9,10,11,12,20,25,30]
2280 ; AVX512BW-NEXT: vpermi2w %ymm4, %ymm2, %ymm7
2281 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0]
2282 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
2283 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm0 = [0,1,2,3,4,5,6,7,8,9,10,11,16,21,26,31]
2284 ; AVX512BW-NEXT: vpermi2w %ymm4, %ymm2, %ymm0
2285 ; AVX512BW-NEXT: vmovdqa %ymm3, (%rsi)
2286 ; AVX512BW-NEXT: vmovdqa %ymm5, (%rdx)
2287 ; AVX512BW-NEXT: vmovdqa %ymm6, (%rcx)
2288 ; AVX512BW-NEXT: vmovdqa %ymm7, (%r8)
2289 ; AVX512BW-NEXT: vmovdqa %ymm0, (%r9)
2290 ; AVX512BW-NEXT: vzeroupper
2291 ; AVX512BW-NEXT: retq
2292 ;
2293 ; AVX512BW-FCP-LABEL: load_i16_stride5_vf16:
2294 ; AVX512BW-FCP: # %bb.0:
2295 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
2296 ; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
2297 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
2298 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
2299 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm3 = [0,1,2,3,4,5,6,7,8,9,10,11,12,17,22,27]
2300 ; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
2301 ; AVX512BW-FCP-NEXT: vpermi2w %ymm4, %ymm2, %ymm3
2302 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
2303 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
2304 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,18,23,28]
2305 ; AVX512BW-FCP-NEXT: vpermi2w %ymm4, %ymm2, %ymm5
2306 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0]
2307 ; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm1, %zmm2
2308 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,1,2,3,4,5,6,7,8,9,10,11,12,19,24,29]
2309 ; AVX512BW-FCP-NEXT: vpermi2w %ymm4, %ymm2, %ymm6
2310 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0]
2311 ; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm1, %zmm2
2312 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm7 = [0,1,2,3,4,5,6,7,8,9,10,11,12,20,25,30]
2313 ; AVX512BW-FCP-NEXT: vpermi2w %ymm4, %ymm2, %ymm7
2314 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0]
2315 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
2316 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm0 = [0,1,2,3,4,5,6,7,8,9,10,11,16,21,26,31]
2317 ; AVX512BW-FCP-NEXT: vpermi2w %ymm4, %ymm2, %ymm0
2318 ; AVX512BW-FCP-NEXT: vmovdqa %ymm3, (%rsi)
2319 ; AVX512BW-FCP-NEXT: vmovdqa %ymm5, (%rdx)
2320 ; AVX512BW-FCP-NEXT: vmovdqa %ymm6, (%rcx)
2321 ; AVX512BW-FCP-NEXT: vmovdqa %ymm7, (%r8)
2322 ; AVX512BW-FCP-NEXT: vmovdqa %ymm0, (%r9)
2323 ; AVX512BW-FCP-NEXT: vzeroupper
2324 ; AVX512BW-FCP-NEXT: retq
2325 ;
2326 ; AVX512DQ-BW-LABEL: load_i16_stride5_vf16:
2327 ; AVX512DQ-BW: # %bb.0:
2328 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
2329 ; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm1
2330 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
2331 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
2332 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm3 = [0,1,2,3,4,5,6,7,8,9,10,11,12,17,22,27]
2333 ; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm4
2334 ; AVX512DQ-BW-NEXT: vpermi2w %ymm4, %ymm2, %ymm3
2335 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
2336 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
2337 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,18,23,28]
2338 ; AVX512DQ-BW-NEXT: vpermi2w %ymm4, %ymm2, %ymm5
2339 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0]
2340 ; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm2
2341 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,1,2,3,4,5,6,7,8,9,10,11,12,19,24,29]
2342 ; AVX512DQ-BW-NEXT: vpermi2w %ymm4, %ymm2, %ymm6
2343 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0]
2344 ; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm2
2345 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm7 = [0,1,2,3,4,5,6,7,8,9,10,11,12,20,25,30]
2346 ; AVX512DQ-BW-NEXT: vpermi2w %ymm4, %ymm2, %ymm7
2347 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0]
2348 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
2349 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm0 = [0,1,2,3,4,5,6,7,8,9,10,11,16,21,26,31]
2350 ; AVX512DQ-BW-NEXT: vpermi2w %ymm4, %ymm2, %ymm0
2351 ; AVX512DQ-BW-NEXT: vmovdqa %ymm3, (%rsi)
2352 ; AVX512DQ-BW-NEXT: vmovdqa %ymm5, (%rdx)
2353 ; AVX512DQ-BW-NEXT: vmovdqa %ymm6, (%rcx)
2354 ; AVX512DQ-BW-NEXT: vmovdqa %ymm7, (%r8)
2355 ; AVX512DQ-BW-NEXT: vmovdqa %ymm0, (%r9)
2356 ; AVX512DQ-BW-NEXT: vzeroupper
2357 ; AVX512DQ-BW-NEXT: retq
2358 ;
2359 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride5_vf16:
2360 ; AVX512DQ-BW-FCP: # %bb.0:
2361 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
2362 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm1
2363 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
2364 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
2365 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm3 = [0,1,2,3,4,5,6,7,8,9,10,11,12,17,22,27]
2366 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
2367 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %ymm4, %ymm2, %ymm3
2368 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
2369 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
2370 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,18,23,28]
2371 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %ymm4, %ymm2, %ymm5
2372 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0]
2373 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm1, %zmm2
2374 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,1,2,3,4,5,6,7,8,9,10,11,12,19,24,29]
2375 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %ymm4, %ymm2, %ymm6
2376 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0]
2377 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm1, %zmm2
2378 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm7 = [0,1,2,3,4,5,6,7,8,9,10,11,12,20,25,30]
2379 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %ymm4, %ymm2, %ymm7
2380 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0]
2381 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
2382 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm0 = [0,1,2,3,4,5,6,7,8,9,10,11,16,21,26,31]
2383 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %ymm4, %ymm2, %ymm0
2384 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm3, (%rsi)
2385 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm5, (%rdx)
2386 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm6, (%rcx)
2387 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm7, (%r8)
2388 ; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm0, (%r9)
2389 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
2390 ; AVX512DQ-BW-FCP-NEXT: retq
2391 %wide.vec = load <80 x i16>, ptr %in.vec, align 64
2392 %strided.vec0 = shufflevector <80 x i16> %wide.vec, <80 x i16> poison, <16 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35, i32 40, i32 45, i32 50, i32 55, i32 60, i32 65, i32 70, i32 75>
2393 %strided.vec1 = shufflevector <80 x i16> %wide.vec, <80 x i16> poison, <16 x i32> <i32 1, i32 6, i32 11, i32 16, i32 21, i32 26, i32 31, i32 36, i32 41, i32 46, i32 51, i32 56, i32 61, i32 66, i32 71, i32 76>
2394 %strided.vec2 = shufflevector <80 x i16> %wide.vec, <80 x i16> poison, <16 x i32> <i32 2, i32 7, i32 12, i32 17, i32 22, i32 27, i32 32, i32 37, i32 42, i32 47, i32 52, i32 57, i32 62, i32 67, i32 72, i32 77>
2395 %strided.vec3 = shufflevector <80 x i16> %wide.vec, <80 x i16> poison, <16 x i32> <i32 3, i32 8, i32 13, i32 18, i32 23, i32 28, i32 33, i32 38, i32 43, i32 48, i32 53, i32 58, i32 63, i32 68, i32 73, i32 78>
2396 %strided.vec4 = shufflevector <80 x i16> %wide.vec, <80 x i16> poison, <16 x i32> <i32 4, i32 9, i32 14, i32 19, i32 24, i32 29, i32 34, i32 39, i32 44, i32 49, i32 54, i32 59, i32 64, i32 69, i32 74, i32 79>
2397 store <16 x i16> %strided.vec0, ptr %out.vec0, align 64
2398 store <16 x i16> %strided.vec1, ptr %out.vec1, align 64
2399 store <16 x i16> %strided.vec2, ptr %out.vec2, align 64
2400 store <16 x i16> %strided.vec3, ptr %out.vec3, align 64
2401 store <16 x i16> %strided.vec4, ptr %out.vec4, align 64
2402 ret void
2403 }
2405 define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4) nounwind {
2406 ; SSE-LABEL: load_i16_stride5_vf32:
2407 ; SSE: # %bb.0:
2408 ; SSE-NEXT: subq $408, %rsp # imm = 0x198
2409 ; SSE-NEXT: movdqa 64(%rdi), %xmm4
2410 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2411 ; SSE-NEXT: movdqa (%rdi), %xmm10
2412 ; SSE-NEXT: movdqa 16(%rdi), %xmm13
2413 ; SSE-NEXT: movdqa 32(%rdi), %xmm9
2414 ; SSE-NEXT: movdqa 48(%rdi), %xmm5
2415 ; SSE-NEXT: movdqa 224(%rdi), %xmm7
2416 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2417 ; SSE-NEXT: movdqa 160(%rdi), %xmm11
2418 ; SSE-NEXT: movdqa 176(%rdi), %xmm12
2419 ; SSE-NEXT: movdqa 208(%rdi), %xmm3
2420 ; SSE-NEXT: movdqa 192(%rdi), %xmm2
2421 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2422 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,65535,65535,0,65535,65535,65535]
2423 ; SSE-NEXT: movdqa %xmm0, %xmm1
2424 ; SSE-NEXT: pandn %xmm2, %xmm1
2425 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,1,0,3]
2426 ; SSE-NEXT: movdqa %xmm3, %xmm8
2427 ; SSE-NEXT: pand %xmm0, %xmm2
2428 ; SSE-NEXT: por %xmm1, %xmm2
2429 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm12[3,1,2,3]
2430 ; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2431 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
2432 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm11[0,2,2,3]
2433 ; SSE-NEXT: movdqa %xmm11, (%rsp) # 16-byte Spill
2434 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,3,2,3,4,5,6,7]
2435 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
2436 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3]
2437 ; SSE-NEXT: movaps {{.*#+}} xmm15 = [65535,65535,65535,65535,65535,65535,65535,0]
2438 ; SSE-NEXT: andps %xmm15, %xmm3
2439 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[0,1,0,1]
2440 ; SSE-NEXT: movaps %xmm15, %xmm2
2441 ; SSE-NEXT: pandn %xmm1, %xmm2
2442 ; SSE-NEXT: por %xmm3, %xmm2
2443 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2444 ; SSE-NEXT: movdqa %xmm0, %xmm1
2445 ; SSE-NEXT: pandn %xmm9, %xmm1
2446 ; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2447 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm5[0,1,0,3]
2448 ; SSE-NEXT: movdqa %xmm5, %xmm7
2449 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2450 ; SSE-NEXT: pand %xmm0, %xmm2
2451 ; SSE-NEXT: por %xmm1, %xmm2
2452 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm13[3,1,2,3]
2453 ; SSE-NEXT: movdqa %xmm13, %xmm5
2454 ; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2455 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
2456 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm10[0,2,2,3]
2457 ; SSE-NEXT: movdqa %xmm10, %xmm6
2458 ; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2459 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,3,2,3,4,5,6,7]
2460 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
2461 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3]
2462 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[0,1,0,1]
2463 ; SSE-NEXT: movaps %xmm15, %xmm2
2464 ; SSE-NEXT: andnps %xmm1, %xmm2
2465 ; SSE-NEXT: movdqa 272(%rdi), %xmm4
2466 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2467 ; SSE-NEXT: andps %xmm15, %xmm3
2468 ; SSE-NEXT: orps %xmm3, %xmm2
2469 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2470 ; SSE-NEXT: movdqa %xmm0, %xmm1
2471 ; SSE-NEXT: pandn %xmm4, %xmm1
2472 ; SSE-NEXT: movdqa 288(%rdi), %xmm2
2473 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2474 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]
2475 ; SSE-NEXT: pand %xmm0, %xmm2
2476 ; SSE-NEXT: por %xmm1, %xmm2
2477 ; SSE-NEXT: movdqa 256(%rdi), %xmm3
2478 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[3,1,2,3]
2479 ; SSE-NEXT: movdqa %xmm3, %xmm14
2480 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2481 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
2482 ; SSE-NEXT: movdqa 240(%rdi), %xmm13
2483 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm13[0,2,2,3]
2484 ; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2485 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,3,2,3,4,5,6,7]
2486 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
2487 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3]
2488 ; SSE-NEXT: movdqa 304(%rdi), %xmm1
2489 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2490 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
2491 ; SSE-NEXT: movaps %xmm15, %xmm2
2492 ; SSE-NEXT: andnps %xmm1, %xmm2
2493 ; SSE-NEXT: andps %xmm15, %xmm3
2494 ; SSE-NEXT: orps %xmm3, %xmm2
2495 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2496 ; SSE-NEXT: movdqa 128(%rdi), %xmm1
2497 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2498 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
2499 ; SSE-NEXT: pand %xmm0, %xmm1
2500 ; SSE-NEXT: movdqa 112(%rdi), %xmm2
2501 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2502 ; SSE-NEXT: pandn %xmm2, %xmm0
2503 ; SSE-NEXT: por %xmm1, %xmm0
2504 ; SSE-NEXT: movdqa 96(%rdi), %xmm1
2505 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2506 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
2507 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
2508 ; SSE-NEXT: movdqa 80(%rdi), %xmm4
2509 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
2510 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2511 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,3,2,3,4,5,6,7]
2512 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
2513 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3]
2514 ; SSE-NEXT: movdqa 144(%rdi), %xmm0
2515 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2516 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
2517 ; SSE-NEXT: movaps %xmm15, %xmm1
2518 ; SSE-NEXT: andnps %xmm0, %xmm1
2519 ; SSE-NEXT: andps %xmm15, %xmm2
2520 ; SSE-NEXT: orps %xmm2, %xmm1
2521 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2522 ; SSE-NEXT: psrlq $48, %xmm12
2523 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm11[0,3,2,3]
2524 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,2,3,4,5,6,7]
2525 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1]
2526 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,0,0,65535,65535,65535,65535,65535]
2527 ; SSE-NEXT: movdqa %xmm0, %xmm2
2528 ; SSE-NEXT: pandn %xmm1, %xmm2
2529 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2530 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[1,3,2,3]
2531 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
2532 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm11[0,2,2,3]
2533 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
2534 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm3[0,1,2,3,7,5,6,7]
2535 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]
2536 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
2537 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,4,7]
2538 ; SSE-NEXT: pand %xmm0, %xmm1
2539 ; SSE-NEXT: por %xmm2, %xmm1
2540 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
2541 ; SSE-NEXT: movdqa %xmm10, %xmm2
2542 ; SSE-NEXT: psllq $48, %xmm2
2543 ; SSE-NEXT: movaps %xmm15, %xmm3
2544 ; SSE-NEXT: andnps %xmm2, %xmm3
2545 ; SSE-NEXT: pand %xmm15, %xmm1
2546 ; SSE-NEXT: orps %xmm1, %xmm3
2547 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2548 ; SSE-NEXT: psrlq $48, %xmm5
2549 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm6[0,3,2,3]
2550 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]
2551 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
2552 ; SSE-NEXT: movdqa %xmm0, %xmm1
2553 ; SSE-NEXT: pandn %xmm2, %xmm1
2554 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm7[1,3,2,3]
2555 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm9[0,2,2,3]
2556 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
2557 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm3[0,1,2,3,7,5,6,7]
2558 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]
2559 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
2560 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,4,7]
2561 ; SSE-NEXT: pand %xmm0, %xmm2
2562 ; SSE-NEXT: por %xmm1, %xmm2
2563 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
2564 ; SSE-NEXT: movdqa %xmm6, %xmm1
2565 ; SSE-NEXT: psllq $48, %xmm1
2566 ; SSE-NEXT: movdqa %xmm15, %xmm3
2567 ; SSE-NEXT: pandn %xmm1, %xmm3
2568 ; SSE-NEXT: pand %xmm15, %xmm2
2569 ; SSE-NEXT: por %xmm2, %xmm3
2570 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2571 ; SSE-NEXT: movdqa %xmm14, %xmm1
2572 ; SSE-NEXT: psrlq $48, %xmm1
2573 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm13[0,3,2,3]
2574 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]
2575 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
2576 ; SSE-NEXT: movdqa %xmm0, %xmm1
2577 ; SSE-NEXT: pandn %xmm2, %xmm1
2578 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
2579 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm14[1,3,2,3]
2580 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
2581 ; SSE-NEXT: # xmm3 = mem[0,2,2,3]
2582 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
2583 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm3[0,1,2,3,7,5,6,7]
2584 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]
2585 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
2586 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,4,7]
2587 ; SSE-NEXT: pand %xmm0, %xmm2
2588 ; SSE-NEXT: por %xmm1, %xmm2
2589 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
2590 ; SSE-NEXT: movdqa %xmm7, %xmm1
2591 ; SSE-NEXT: psllq $48, %xmm1
2592 ; SSE-NEXT: movdqa %xmm15, %xmm3
2593 ; SSE-NEXT: pandn %xmm1, %xmm3
2594 ; SSE-NEXT: pand %xmm15, %xmm2
2595 ; SSE-NEXT: por %xmm2, %xmm3
2596 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2597 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
2598 ; SSE-NEXT: movdqa %xmm13, %xmm1
2599 ; SSE-NEXT: psrlq $48, %xmm1
2600 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,3,2,3]
2601 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]
2602 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
2603 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
2604 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm12[1,3,2,3]
2605 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
2606 ; SSE-NEXT: # xmm3 = mem[0,2,2,3]
2607 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
2608 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm3[0,1,2,3,7,5,6,7]
2609 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]
2610 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
2611 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,4,7]
2612 ; SSE-NEXT: pand %xmm0, %xmm1
2613 ; SSE-NEXT: pandn %xmm2, %xmm0
2614 ; SSE-NEXT: por %xmm1, %xmm0
2615 ; SSE-NEXT: pand %xmm15, %xmm0
2616 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
2617 ; SSE-NEXT: movdqa %xmm5, %xmm1
2618 ; SSE-NEXT: psllq $48, %xmm1
2619 ; SSE-NEXT: pandn %xmm1, %xmm15
2620 ; SSE-NEXT: por %xmm0, %xmm15
2621 ; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2622 ; SSE-NEXT: movdqa %xmm8, %xmm0
2623 ; SSE-NEXT: movdqa %xmm11, %xmm8
2624 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm11[0,0]
2625 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm11[2,3]
2626 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,0,1,3]
2627 ; SSE-NEXT: movaps {{.*#+}} xmm11 = [65535,65535,65535,0,0,0,65535,65535]
2628 ; SSE-NEXT: movaps %xmm11, %xmm1
2629 ; SSE-NEXT: andnps %xmm0, %xmm1
2630 ; SSE-NEXT: movdqa (%rsp), %xmm4 # 16-byte Reload
2631 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,1,1,3]
2632 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,7]
2633 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
2634 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
2635 ; SSE-NEXT: pand %xmm11, %xmm2
2636 ; SSE-NEXT: por %xmm1, %xmm2
2637 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]
2638 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm10[0,1,2,0]
2639 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6,5]
2640 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[2,3]
2641 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,0]
2642 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2643 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2644 ; SSE-NEXT: movaps %xmm1, %xmm0
2645 ; SSE-NEXT: movaps %xmm1, %xmm15
2646 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
2647 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm9[0,0]
2648 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm9[2,3]
2649 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,0,1,3]
2650 ; SSE-NEXT: movaps %xmm11, %xmm1
2651 ; SSE-NEXT: andnps %xmm0, %xmm1
2652 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2653 ; SSE-NEXT: # xmm2 = mem[0,1,1,3]
2654 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,7]
2655 ; SSE-NEXT: punpckhdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2656 ; SSE-NEXT: # xmm2 = xmm2[2],mem[2],xmm2[3],mem[3]
2657 ; SSE-NEXT: pand %xmm11, %xmm2
2658 ; SSE-NEXT: por %xmm1, %xmm2
2659 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]
2660 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[0,1,2,0]
2661 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6,5]
2662 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[2,3]
2663 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,0]
2664 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2665 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
2666 ; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[1,0],xmm10[0,0]
2667 ; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[2,0],xmm10[2,3]
2668 ; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[0,0,1,3]
2669 ; SSE-NEXT: movaps %xmm11, %xmm1
2670 ; SSE-NEXT: andnps %xmm14, %xmm1
2671 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2672 ; SSE-NEXT: # xmm2 = mem[0,1,1,3]
2673 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,7]
2674 ; SSE-NEXT: punpckhdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2675 ; SSE-NEXT: # xmm2 = xmm2[2],mem[2],xmm2[3],mem[3]
2676 ; SSE-NEXT: pand %xmm11, %xmm2
2677 ; SSE-NEXT: por %xmm1, %xmm2
2678 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm14[0,1,2,3,6,5,6,7]
2679 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[0,1,2,0]
2680 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6,5]
2681 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[2,3]
2682 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,0]
2683 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2684 ; SSE-NEXT: movdqa %xmm12, %xmm0
2685 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2686 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm1[0,0]
2687 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,3]
2688 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,0,1,3]
2689 ; SSE-NEXT: movaps %xmm11, %xmm1
2690 ; SSE-NEXT: andnps %xmm0, %xmm1
2691 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
2692 ; SSE-NEXT: # xmm2 = mem[0,1,1,3]
2693 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,7]
2694 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm13[2],xmm2[3],xmm13[3]
2695 ; SSE-NEXT: pand %xmm11, %xmm2
2696 ; SSE-NEXT: por %xmm1, %xmm2
2697 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]
2698 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm5[0,1,2,0]
2699 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6,5]
2700 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[2,3]
2701 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,0]
2702 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2703 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm8[2,2,2,2,4,5,6,7]
2704 ; SSE-NEXT: movdqa %xmm11, %xmm1
2705 ; SSE-NEXT: pandn %xmm0, %xmm1
2706 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,1,1]
2707 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
2708 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2709 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm2[0,3,2,3,4,5,6,7]
2710 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2711 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,3,4,5,6,7]
2712 ; SSE-NEXT: pand %xmm11, %xmm0
2713 ; SSE-NEXT: por %xmm1, %xmm0
2714 ; SSE-NEXT: movdqa %xmm0, %xmm2
2715 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
2716 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm8[3,0]
2717 ; SSE-NEXT: movaps %xmm11, %xmm0
2718 ; SSE-NEXT: andnps %xmm8, %xmm0
2719 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2720 ; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[0,1],xmm3[0,2]
2721 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm8[0,1,2,3,7,4,6,7]
2722 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2723 ; SSE-NEXT: # xmm1 = mem[0,1,0,3]
2724 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,6]
2725 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[2,3]
2726 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,0]
2727 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2728 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[2,2,2,2,4,5,6,7]
2729 ; SSE-NEXT: movdqa %xmm11, %xmm1
2730 ; SSE-NEXT: pandn %xmm0, %xmm1
2731 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
2732 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,1,1]
2733 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
2734 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm14[0,2,2,3]
2735 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2736 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm2[0,3,2,3,4,5,6,7]
2737 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2738 ; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm0[1,0,3,3,4,5,6,7]
2739 ; SSE-NEXT: pand %xmm11, %xmm13
2740 ; SSE-NEXT: por %xmm1, %xmm13
2741 ; SSE-NEXT: movaps %xmm15, %xmm0
2742 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm9[3,0]
2743 ; SSE-NEXT: movdqa %xmm11, %xmm12
2744 ; SSE-NEXT: pandn %xmm9, %xmm12
2745 ; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[0,1],xmm0[0,2]
2746 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm9[0,1,2,3,7,4,6,7]
2747 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2748 ; SSE-NEXT: # xmm1 = mem[0,1,0,3]
2749 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,6]
2750 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[2,3]
2751 ; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,1],xmm1[2,0]
2752 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[2,2,2,2,4,5,6,7]
2753 ; SSE-NEXT: movdqa %xmm11, %xmm1
2754 ; SSE-NEXT: pandn %xmm0, %xmm1
2755 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
2756 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,1,1]
2757 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
2758 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
2759 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2760 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm2[0,3,2,3,4,5,6,7]
2761 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2762 ; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm0[1,0,3,3,4,5,6,7]
2763 ; SSE-NEXT: pand %xmm11, %xmm9
2764 ; SSE-NEXT: por %xmm1, %xmm9
2765 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
2766 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm10[3,0]
2767 ; SSE-NEXT: movdqa %xmm11, %xmm15
2768 ; SSE-NEXT: pandn %xmm10, %xmm15
2769 ; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[0,1],xmm0[0,2]
2770 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm10[0,1,2,3,7,4,6,7]
2771 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[0,1,0,3]
2772 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,6]
2773 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1],xmm0[2,3]
2774 ; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[0,1],xmm1[2,0]
2775 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
2776 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[2,2,2,2,4,5,6,7]
2777 ; SSE-NEXT: movdqa %xmm11, %xmm1
2778 ; SSE-NEXT: pandn %xmm0, %xmm1
2779 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
2780 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,1,1]
2781 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
2782 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
2783 ; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
2784 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm2[0,3,2,3,4,5,6,7]
2785 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2786 ; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm0[1,0,3,3,4,5,6,7]
2787 ; SSE-NEXT: pand %xmm11, %xmm10
2788 ; SSE-NEXT: por %xmm1, %xmm10
2789 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
2790 ; SSE-NEXT: movaps (%rsp), %xmm1 # 16-byte Reload
2791 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[3,0]
2792 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,2]
2793 ; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,0],xmm14[3,0]
2794 ; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[0,1],xmm8[0,2]
2795 ; SSE-NEXT: movaps %xmm14, %xmm2
2796 ; SSE-NEXT: movdqa %xmm3, %xmm1
2797 ; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,0],xmm3[3,0]
2798 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm6[0,2]
2799 ; SSE-NEXT: movaps %xmm1, %xmm14
2800 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0],xmm4[3,0]
2801 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm5[0,2]
2802 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
2803 ; SSE-NEXT: movaps %xmm8, %xmm1
2804 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm7[3,0]
2805 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
2806 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2807 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm0[2,1,3,3,4,5,6,7]
2808 ; SSE-NEXT: pand %xmm11, %xmm3
2809 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm2[0,1,2,3,4,6,6,7]
2810 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2811 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm0[2,1,3,3,4,5,6,7]
2812 ; SSE-NEXT: pand %xmm11, %xmm2
2813 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm14[0,1,2,3,4,6,6,7]
2814 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
2815 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,3,3,4,5,6,7]
2816 ; SSE-NEXT: pand %xmm11, %xmm0
2817 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,6,6,7]
2818 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
2819 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[2,1,3,3,4,5,6,7]
2820 ; SSE-NEXT: pand %xmm11, %xmm4
2821 ; SSE-NEXT: pandn %xmm7, %xmm11
2822 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,1],xmm1[0,2]
2823 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm7[0,1,2,3,7,4,6,7]
2824 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
2825 ; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm6[0,1,0,3]
2826 ; SSE-NEXT: pshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,5,5,6]
2827 ; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[3,1],xmm1[2,3]
2828 ; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[0,1],xmm14[2,0]
2829 ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
2830 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2831 ; SSE-NEXT: # xmm1 = mem[0,2,2,3]
2832 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
2833 ; SSE-NEXT: # xmm14 = mem[0,1,1,3]
2834 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
2835 ; SSE-NEXT: pshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,5,4,7]
2836 ; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[3,1],xmm1[1,3]
2837 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm14[2,0]
2838 ; SSE-NEXT: por %xmm12, %xmm2
2839 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2840 ; SSE-NEXT: # xmm1 = mem[0,2,2,3]
2841 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
2842 ; SSE-NEXT: # xmm12 = mem[0,1,1,3]
2843 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
2844 ; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5,4,7]
2845 ; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[3,1],xmm1[1,3]
2846 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm12[2,0]
2847 ; SSE-NEXT: por %xmm15, %xmm0
2848 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
2849 ; SSE-NEXT: # xmm1 = mem[0,2,2,3]
2850 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
2851 ; SSE-NEXT: # xmm5 = mem[0,1,1,3]
2852 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
2853 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,4,7]
2854 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[3,1],xmm1[1,3]
2855 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,0]
2856 ; SSE-NEXT: por %xmm4, %xmm11
2857 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[0,2,2,3]
2858 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm6[0,1,1,3]
2859 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
2860 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,7]
2861 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm1[1,3]
2862 ; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[0,1],xmm4[2,0]
2863 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2864 ; SSE-NEXT: movaps %xmm1, 16(%rsi)
2865 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2866 ; SSE-NEXT: movaps %xmm1, 48(%rsi)
2867 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2868 ; SSE-NEXT: movaps %xmm1, (%rsi)
2869 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2870 ; SSE-NEXT: movaps %xmm1, 32(%rsi)
2871 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2872 ; SSE-NEXT: movaps %xmm1, 16(%rdx)
2873 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2874 ; SSE-NEXT: movaps %xmm1, 48(%rdx)
2875 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2876 ; SSE-NEXT: movaps %xmm1, (%rdx)
2877 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2878 ; SSE-NEXT: movaps %xmm1, 32(%rdx)
2879 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2880 ; SSE-NEXT: movaps %xmm1, 16(%rcx)
2881 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2882 ; SSE-NEXT: movaps %xmm1, 48(%rcx)
2883 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2884 ; SSE-NEXT: movaps %xmm1, (%rcx)
2885 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2886 ; SSE-NEXT: movaps %xmm1, 32(%rcx)
2887 ; SSE-NEXT: movaps %xmm10, 16(%r8)
2888 ; SSE-NEXT: movaps %xmm9, 48(%r8)
2889 ; SSE-NEXT: movaps %xmm13, (%r8)
2890 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
2891 ; SSE-NEXT: movaps %xmm1, 32(%r8)
2892 ; SSE-NEXT: movaps %xmm11, 16(%r9)
2893 ; SSE-NEXT: movaps %xmm0, 48(%r9)
2894 ; SSE-NEXT: movaps %xmm2, (%r9)
2895 ; SSE-NEXT: movaps %xmm3, 32(%r9)
2896 ; SSE-NEXT: addq $408, %rsp # imm = 0x198
2897 ; SSE-NEXT: retq
2898 ;
2899 ; AVX-LABEL: load_i16_stride5_vf32:
2900 ; AVX: # %bb.0:
2901 ; AVX-NEXT: subq $424, %rsp # imm = 0x1A8
2902 ; AVX-NEXT: vmovdqa 144(%rdi), %xmm2
2903 ; AVX-NEXT: vmovdqa 128(%rdi), %xmm1
2904 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm2[2,3],xmm1[4,5,6,7]
2905 ; AVX-NEXT: vmovdqa %xmm2, %xmm9
2906 ; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2907 ; AVX-NEXT: vmovdqa %xmm1, %xmm7
2908 ; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [6,7,2,3,4,5,6,7,6,7,2,3,12,13,6,7]
2909 ; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm0
2910 ; AVX-NEXT: vmovdqa 96(%rdi), %xmm3
2911 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[0,1,1,3]
2912 ; AVX-NEXT: vmovdqa %xmm3, %xmm11
2913 ; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]
2914 ; AVX-NEXT: vmovdqa 112(%rdi), %xmm3
2915 ; AVX-NEXT: vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1]
2916 ; AVX-NEXT: vmovdqa %xmm3, %xmm10
2917 ; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2918 ; AVX-NEXT: vmovdqa 80(%rdi), %xmm3
2919 ; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2920 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
2921 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,3,2,3,4,5,6,7]
2922 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3,4,5,6,7]
2923 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm0[5,6,7]
2924 ; AVX-NEXT: vmovdqa (%rdi), %xmm5
2925 ; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2926 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm6
2927 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm3
2928 ; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2929 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm4
2930 ; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm4[0,1,0,3]
2931 ; AVX-NEXT: vmovdqa %xmm4, %xmm15
2932 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4],xmm0[5,6,7]
2933 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm6[3,1,2,3]
2934 ; AVX-NEXT: vmovdqa %xmm6, %xmm12
2935 ; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2936 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,1,2,3,4,5,6,7]
2937 ; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm5[0,2,2,3]
2938 ; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,3,2,3,4,5,6,7]
2939 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
2940 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm0[4,5,6,7]
2941 ; AVX-NEXT: vmovaps {{.*#+}} ymm6 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,65535,65535,65535]
2942 ; AVX-NEXT: vandps %ymm6, %ymm3, %ymm3
2943 ; AVX-NEXT: vmovaps 64(%rdi), %xmm0
2944 ; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm0[0,1,0,1]
2945 ; AVX-NEXT: vmovaps %xmm0, %xmm5
2946 ; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2947 ; AVX-NEXT: vandnps %ymm4, %ymm6, %ymm4
2948 ; AVX-NEXT: vorps %ymm4, %ymm3, %ymm3
2949 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm0
2950 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
2951 ; AVX-NEXT: vmovdqa 304(%rdi), %xmm2
2952 ; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2953 ; AVX-NEXT: vmovdqa 288(%rdi), %xmm13
2954 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm13[0,1],xmm2[2,3],xmm13[4,5,6,7]
2955 ; AVX-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2956 ; AVX-NEXT: vpshufb %xmm1, %xmm2, %xmm1
2957 ; AVX-NEXT: vmovdqa 256(%rdi), %xmm0
2958 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2959 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
2960 ; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]
2961 ; AVX-NEXT: vmovdqa 272(%rdi), %xmm0
2962 ; AVX-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
2963 ; AVX-NEXT: vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm0[1]
2964 ; AVX-NEXT: vmovdqa 240(%rdi), %xmm0
2965 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2966 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
2967 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,3,2,3,4,5,6,7]
2968 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3,4,5,6,7]
2969 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0,1,2,3,4],xmm1[5,6,7]
2970 ; AVX-NEXT: vmovdqa 176(%rdi), %xmm0
2971 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2972 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[3,1,2,3]
2973 ; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
2974 ; AVX-NEXT: vmovdqa 160(%rdi), %xmm0
2975 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2976 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
2977 ; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,3,2,3,4,5,6,7]
2978 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
2979 ; AVX-NEXT: vmovdqa 208(%rdi), %xmm0
2980 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2981 ; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
2982 ; AVX-NEXT: vmovdqa 192(%rdi), %xmm2
2983 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4],xmm0[5,6,7]
2984 ; AVX-NEXT: vmovdqa %xmm2, %xmm14
2985 ; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2986 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
2987 ; AVX-NEXT: vandps %ymm6, %ymm0, %ymm0
2988 ; AVX-NEXT: vmovaps 224(%rdi), %xmm1
2989 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2990 ; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm1[0,1,0,1]
2991 ; AVX-NEXT: vandnps %ymm8, %ymm6, %ymm8
2992 ; AVX-NEXT: vorps %ymm0, %ymm8, %ymm0
2993 ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
2994 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
2995 ; AVX-NEXT: vmovdqa %xmm11, %xmm6
2996 ; AVX-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
2997 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm10[0,1],xmm11[2,3],xmm10[4,5,6,7]
2998 ; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,0,4,5,6,7]
2999 ; AVX-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
3000 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
3001 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm11[0,3,2,3]
3002 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
3003 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3,4,5,6,7]
3004 ; AVX-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3005 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm7[0,1,2,3],xmm9[4,5],xmm7[6,7]
3006 ; AVX-NEXT: vmovdqa {{.*#+}} xmm8 = [0,1,2,3,4,5,6,7,8,9,4,5,14,15,8,9]
3007 ; AVX-NEXT: vpshufb %xmm8, %xmm3, %xmm3
3008 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm3[5,6,7]
3009 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
3010 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[0,3,2,3]
3011 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
3012 ; AVX-NEXT: vpsrlq $48, %xmm12, %xmm9
3013 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1]
3014 ; AVX-NEXT: vmovdqa %xmm15, %xmm12
3015 ; AVX-NEXT: vmovdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
3016 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
3017 ; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm2[0,1],xmm15[2,3],xmm2[4,5],xmm15[6,7]
3018 ; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,0,1,0,1,0,1,10,11,4,5,14,15,6,7]
3019 ; AVX-NEXT: vpshufb %xmm1, %xmm9, %xmm9
3020 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm9[3,4,5,6,7]
3021 ; AVX-NEXT: vmovaps {{.*#+}} ymm10 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,65535,65535,65535]
3022 ; AVX-NEXT: vandps %ymm3, %ymm10, %ymm3
3023 ; AVX-NEXT: vpsllq $48, %xmm5, %xmm9
3024 ; AVX-NEXT: vandnps %ymm9, %ymm10, %ymm9
3025 ; AVX-NEXT: vorps %ymm3, %ymm9, %ymm3
3026 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm3, %ymm0
3027 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3028 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
3029 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm13[0,1,2,3],xmm5[4,5],xmm13[6,7]
3030 ; AVX-NEXT: vpshufb %xmm8, %xmm0, %xmm0
3031 ; AVX-NEXT: vmovdqa (%rsp), %xmm15 # 16-byte Reload
3032 ; AVX-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm3 # 16-byte Folded Reload
3033 ; AVX-NEXT: # xmm3 = xmm15[0,1],mem[2,3],xmm15[4,5,6,7]
3034 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,0,4,5,6,7]
3035 ; AVX-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
3036 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
3037 ; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm10[0,3,2,3]
3038 ; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,2,2,3,4,5,6,7]
3039 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm8[0,1],xmm3[2,3,4,5,6,7]
3040 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3,4],xmm0[5,6,7]
3041 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
3042 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm14[0,1],xmm13[2,3],xmm14[4,5],xmm13[6,7]
3043 ; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm1
3044 ; AVX-NEXT: vpshufd $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
3045 ; AVX-NEXT: # xmm3 = mem[0,3,2,3]
3046 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
3047 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
3048 ; AVX-NEXT: vpsrlq $48, %xmm8, %xmm8
3049 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1]
3050 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3,4,5,6,7]
3051 ; AVX-NEXT: vmovaps {{.*#+}} ymm8 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,65535,65535,65535]
3052 ; AVX-NEXT: vandps %ymm1, %ymm8, %ymm1
3053 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
3054 ; AVX-NEXT: vpsllq $48, %xmm14, %xmm3
3055 ; AVX-NEXT: vandnps %ymm3, %ymm8, %ymm3
3056 ; AVX-NEXT: vorps %ymm3, %ymm1, %ymm1
3057 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
3058 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3059 ; AVX-NEXT: vpblendw $207, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm0 # 16-byte Folded Reload
3060 ; AVX-NEXT: # xmm0 = mem[0,1,2,3],xmm6[4,5],mem[6,7]
3061 ; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [u,u,u,u,8,9,2,3,12,13,12,13,12,13,12,13]
3062 ; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm0
3063 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm11[3,1,2,3]
3064 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,1,2,3,4,5,6,7]
3065 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3,4,5,6,7]
3066 ; AVX-NEXT: vpblendw $243, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm3 # 16-byte Folded Reload
3067 ; AVX-NEXT: # xmm3 = mem[0,1],xmm7[2,3],mem[4,5,6,7]
3068 ; AVX-NEXT: vmovdqa {{.*#+}} xmm7 = [0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
3069 ; AVX-NEXT: vpshufb %xmm7, %xmm3, %xmm3
3070 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm3[5,6,7]
3071 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0,1],xmm12[2,3],xmm2[4,5,6,7]
3072 ; AVX-NEXT: vmovdqa {{.*#+}} xmm8 = [2,3,2,3,2,3,2,3,12,13,6,7,12,13,14,15]
3073 ; AVX-NEXT: vpshufb %xmm8, %xmm3, %xmm3
3074 ; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm4[0,1,1,3]
3075 ; AVX-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,7,6,7]
3076 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
3077 ; AVX-NEXT: vpunpckhdq {{.*#+}} xmm9 = xmm9[2],xmm11[2],xmm9[3],xmm11[3]
3078 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm9[0,1,2],xmm3[3,4,5],xmm9[6,7]
3079 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
3080 ; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm12[0,1,2,0]
3081 ; AVX-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,5,6,5]
3082 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5],xmm9[6,7]
3083 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm3, %ymm0
3084 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3085 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
3086 ; AVX-NEXT: vmovdqa %xmm5, %xmm9
3087 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm5[0,1],xmm6[2,3],xmm5[4,5,6,7]
3088 ; AVX-NEXT: vpshufb %xmm7, %xmm0, %xmm4
3089 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
3090 ; AVX-NEXT: vmovdqa %xmm15, %xmm0
3091 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm15[0,1,2,3],xmm5[4,5],xmm15[6,7]
3092 ; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm1
3093 ; AVX-NEXT: vmovdqa %xmm10, %xmm2
3094 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm10[3,1,2,3]
3095 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,1,2,3,4,5,6,7]
3096 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3,4,5,6,7]
3097 ; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm1[0,1,2,3,4],xmm4[5,6,7]
3098 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
3099 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm15[0,1],xmm13[2,3],xmm15[4,5,6,7]
3100 ; AVX-NEXT: vpshufb %xmm8, %xmm1, %xmm1
3101 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
3102 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[0,1,1,3]
3103 ; AVX-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,7]
3104 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
3105 ; AVX-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm3[2],xmm8[2],xmm3[3],xmm8[3]
3106 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3,4,5],xmm3[6,7]
3107 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm14[0,1,2,0]
3108 ; AVX-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,6,5]
3109 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm3[6,7]
3110 ; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm1, %ymm1
3111 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3112 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm0[2,3],xmm5[4,5],xmm0[6,7]
3113 ; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [u,u,0,1,10,11,4,5,14,15,14,15,14,15,14,15]
3114 ; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm0
3115 ; AVX-NEXT: vpsrlq $48, %xmm2, %xmm3
3116 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0],xmm0[1,2,3,4,5,6,7]
3117 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm9[0,1,2,3],xmm6[4,5],xmm9[6,7]
3118 ; AVX-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
3119 ; AVX-NEXT: vpshufb %xmm5, %xmm3, %xmm3
3120 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm3[5,6,7]
3121 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm8[0,1],xmm4[2,3],xmm8[4,5,6,7]
3122 ; AVX-NEXT: vmovdqa {{.*#+}} xmm6 = [6,7,0,1,10,11,10,11,8,9,10,11,12,13,14,15]
3123 ; AVX-NEXT: vpshufb %xmm6, %xmm3, %xmm3
3124 ; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm15[0,1,2,3],xmm13[4,5],xmm15[6,7]
3125 ; AVX-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[2,2,2,2,4,5,6,7]
3126 ; AVX-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,4,6,7]
3127 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm9[3,4,5],xmm3[6,7]
3128 ; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm14[0,1,0,3]
3129 ; AVX-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,5,5,6]
3130 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5],xmm9[6,7]
3131 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm3, %ymm0
3132 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3133 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
3134 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
3135 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm8[0,1,2,3],xmm9[4,5],xmm8[6,7]
3136 ; AVX-NEXT: vpshufb %xmm5, %xmm0, %xmm0
3137 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
3138 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
3139 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm4[2,3],xmm5[4,5],xmm4[6,7]
3140 ; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm1
3141 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
3142 ; AVX-NEXT: vpsrlq $48, %xmm2, %xmm3
3143 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3,4,5,6,7]
3144 ; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm1[0,1,2,3,4],xmm0[5,6,7]
3145 ; AVX-NEXT: vmovdqa %xmm11, %xmm0
3146 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
3147 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm11[0,1],xmm10[2,3],xmm11[4,5,6,7]
3148 ; AVX-NEXT: vpshufb %xmm6, %xmm1, %xmm1
3149 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
3150 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
3151 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm6[0,1,2,3],xmm11[4,5],xmm6[6,7]
3152 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,2,2,2,4,5,6,7]
3153 ; AVX-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,4,6,7]
3154 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[3,4,5],xmm1[6,7]
3155 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm12[0,1,0,3]
3156 ; AVX-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,5,6]
3157 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm3[6,7]
3158 ; AVX-NEXT: vinsertf128 $1, %xmm7, %ymm1, %ymm7
3159 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm8[3,1,2,3]
3160 ; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,1,4,5,6,7]
3161 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm9[0,2,2,3]
3162 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,0,3,4,5,6,7]
3163 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
3164 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[0,3,2,3]
3165 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[3],xmm3[4,5,6,7]
3166 ; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
3167 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0],xmm3[1,2,3,4,5,6,7]
3168 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
3169 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0,1,2,3],xmm10[4,5],xmm0[6,7]
3170 ; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm6[1,1,1,1]
3171 ; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm11[0,2,2,3]
3172 ; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,0,3,4,5,6,7]
3173 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
3174 ; AVX-NEXT: vmovdqa {{.*#+}} xmm8 = [8,9,2,3,12,13,12,13,8,9,12,13,12,13,14,15]
3175 ; AVX-NEXT: vpshufb %xmm8, %xmm3, %xmm3
3176 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm5[3,4,5],xmm3[6,7]
3177 ; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm12[0,1,1,3]
3178 ; AVX-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,4,7]
3179 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5],xmm5[6,7]
3180 ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1
3181 ; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
3182 ; AVX-NEXT: # xmm3 = mem[3,1,2,3]
3183 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,2,1,4,5,6,7]
3184 ; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
3185 ; AVX-NEXT: # xmm5 = mem[0,2,2,3]
3186 ; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,0,3,4,5,6,7]
3187 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
3188 ; AVX-NEXT: vpshufd $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
3189 ; AVX-NEXT: # xmm5 = mem[0,3,2,3]
3190 ; AVX-NEXT: vpblendw $8, (%rsp), %xmm5, %xmm5 # 16-byte Folded Reload
3191 ; AVX-NEXT: # xmm5 = xmm5[0,1,2],mem[3],xmm5[4,5,6,7]
3192 ; AVX-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
3193 ; AVX-NEXT: # xmm9 = mem[2,3,2,3]
3194 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm9[0],xmm5[1,2,3,4,5,6,7]
3195 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1,2,3],xmm3[4,5,6,7]
3196 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
3197 ; AVX-NEXT: vpblendw $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm5 # 16-byte Folded Reload
3198 ; AVX-NEXT: # xmm5 = xmm0[0,1,2,3],mem[4,5],xmm0[6,7]
3199 ; AVX-NEXT: vpshufb %xmm8, %xmm5, %xmm5
3200 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm15[1,1,1,1]
3201 ; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm13[0,2,2,3]
3202 ; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,0,3,4,5,6,7]
3203 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
3204 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3,4,5],xmm5[6,7]
3205 ; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm14[0,1,1,3]
3206 ; AVX-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,7]
3207 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5],xmm4[6,7]
3208 ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
3209 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
3210 ; AVX-NEXT: vmovaps %ymm3, 32(%rsi)
3211 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
3212 ; AVX-NEXT: vmovaps %ymm3, (%rsi)
3213 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
3214 ; AVX-NEXT: vmovaps %ymm3, 32(%rdx)
3215 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
3216 ; AVX-NEXT: vmovaps %ymm3, (%rdx)
3217 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3218 ; AVX-NEXT: vmovaps %ymm0, 32(%rcx)
3219 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3220 ; AVX-NEXT: vmovaps %ymm0, (%rcx)
3221 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3222 ; AVX-NEXT: vmovaps %ymm0, 32(%r8)
3223 ; AVX-NEXT: vmovaps %ymm7, (%r8)
3224 ; AVX-NEXT: vmovaps %ymm2, 32(%r9)
3225 ; AVX-NEXT: vmovaps %ymm1, (%r9)
3226 ; AVX-NEXT: addq $424, %rsp # imm = 0x1A8
3227 ; AVX-NEXT: vzeroupper
3228 ; AVX-NEXT: retq
3229 ;
3230 ; AVX2-LABEL: load_i16_stride5_vf32:
3231 ; AVX2: # %bb.0:
3232 ; AVX2-NEXT: subq $264, %rsp # imm = 0x108
3233 ; AVX2-NEXT: vmovdqa (%rdi), %ymm1
3234 ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm2
3235 ; AVX2-NEXT: vmovdqa 64(%rdi), %ymm3
3236 ; AVX2-NEXT: vmovdqa 96(%rdi), %ymm12
3237 ; AVX2-NEXT: vmovdqa 192(%rdi), %ymm4
3238 ; AVX2-NEXT: vmovdqa 160(%rdi), %ymm5
3239 ; AVX2-NEXT: vmovdqa 224(%rdi), %ymm7
3240 ; AVX2-NEXT: vmovdqa 256(%rdi), %ymm6
3241 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm7[0],ymm6[1,2],ymm7[3],ymm6[4],ymm7[5],ymm6[6,7],ymm7[8],ymm6[9,10],ymm7[11],ymm6[12],ymm7[13],ymm6[14,15]
3242 ; AVX2-NEXT: vpermq {{.*#+}} ymm8 = ymm0[2,3,0,1]
3243 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm8[5],ymm0[6],ymm8[7]
3244 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm10 = [0,1,10,11,4,5,14,15,8,9,10,11,4,5,6,7,16,17,26,27,20,21,30,31,24,25,26,27,20,21,22,23]
3245 ; AVX2-NEXT: vpshufb %ymm10, %ymm0, %ymm0
3246 ; AVX2-NEXT: vpblendw {{.*#+}} ymm8 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
3247 ; AVX2-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3248 ; AVX2-NEXT: vextracti128 $1, %ymm8, %xmm9
3249 ; AVX2-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm9[1,2,3],xmm8[4,5],xmm9[6,7]
3250 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm11 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,6,7]
3251 ; AVX2-NEXT: vpshufb %xmm11, %xmm8, %xmm8
3252 ; AVX2-NEXT: vpmovsxbw {{.*#+}} xmm9 = [65535,65535,65535,65535,65535,65535,65535,0]
3253 ; AVX2-NEXT: vpblendvb %ymm9, %ymm8, %ymm0, %ymm8
3254 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0],ymm12[1,2],ymm3[3],ymm12[4],ymm3[5],ymm12[6,7],ymm3[8],ymm12[9,10],ymm3[11],ymm12[12],ymm3[13],ymm12[14,15]
3255 ; AVX2-NEXT: vmovdqa %ymm12, %ymm15
3256 ; AVX2-NEXT: vpermq {{.*#+}} ymm12 = ymm0[2,3,0,1]
3257 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm12[5],ymm0[6],ymm12[7]
3258 ; AVX2-NEXT: vpshufb %ymm10, %ymm0, %ymm0
3259 ; AVX2-NEXT: vpblendw {{.*#+}} ymm10 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5],ymm2[6],ymm1[7,8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
3260 ; AVX2-NEXT: vmovdqu %ymm1, (%rsp) # 32-byte Spill
3261 ; AVX2-NEXT: vextracti128 $1, %ymm10, %xmm12
3262 ; AVX2-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0],xmm12[1,2,3],xmm10[4,5],xmm12[6,7]
3263 ; AVX2-NEXT: vpshufb %xmm11, %xmm10, %xmm10
3264 ; AVX2-NEXT: vpblendvb %ymm9, %ymm10, %ymm0, %ymm12
3265 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5],ymm7[6],ymm6[7,8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13],ymm7[14],ymm6[15]
3266 ; AVX2-NEXT: vpermq {{.*#+}} ymm10 = ymm0[2,3,0,1]
3267 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm10[5],ymm0[6,7]
3268 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm10 = [2,3,12,13,6,7,0,1,10,11,6,7,8,9,8,9,18,19,28,29,22,23,16,17,26,27,22,23,24,25,24,25]
3269 ; AVX2-NEXT: vpshufb %ymm10, %ymm0, %ymm0
3270 ; AVX2-NEXT: vpblendw {{.*#+}} ymm11 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4,5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10],ymm5[11],ymm4[12,13],ymm5[14],ymm4[15]
3271 ; AVX2-NEXT: vextracti128 $1, %ymm11, %xmm13
3272 ; AVX2-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0,1],xmm13[2,3],xmm11[4,5,6],xmm13[7]
3273 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm14 = [2,3,12,13,6,7,0,1,10,11,4,5,14,15,10,11]
3274 ; AVX2-NEXT: vpshufb %xmm14, %xmm11, %xmm11
3275 ; AVX2-NEXT: vpblendvb %ymm9, %ymm11, %ymm0, %ymm0
3276 ; AVX2-NEXT: vpblendw {{.*#+}} ymm11 = ymm15[0],ymm3[1],ymm15[2,3],ymm3[4],ymm15[5],ymm3[6],ymm15[7,8],ymm3[9],ymm15[10,11],ymm3[12],ymm15[13],ymm3[14],ymm15[15]
3277 ; AVX2-NEXT: vmovdqa %ymm15, %ymm5
3278 ; AVX2-NEXT: vpermq {{.*#+}} ymm13 = ymm11[2,3,0,1]
3279 ; AVX2-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm13[5],ymm11[6,7]
3280 ; AVX2-NEXT: vpshufb %ymm10, %ymm11, %ymm10
3281 ; AVX2-NEXT: vpblendw {{.*#+}} ymm11 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7,8],ymm1[9],ymm2[10],ymm1[11],ymm2[12,13],ymm1[14],ymm2[15]
3282 ; AVX2-NEXT: vmovdqa %ymm2, %ymm15
3283 ; AVX2-NEXT: vextracti128 $1, %ymm11, %xmm13
3284 ; AVX2-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0,1],xmm13[2,3],xmm11[4,5,6],xmm13[7]
3285 ; AVX2-NEXT: vmovdqa 304(%rdi), %xmm13
3286 ; AVX2-NEXT: vpshufb %xmm14, %xmm11, %xmm11
3287 ; AVX2-NEXT: vmovdqa 288(%rdi), %xmm14
3288 ; AVX2-NEXT: vpblendvb %ymm9, %ymm11, %ymm10, %ymm9
3289 ; AVX2-NEXT: vpblendd {{.*#+}} xmm10 = xmm14[0],xmm13[1],xmm14[2,3]
3290 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [6,7,2,3,4,5,6,7,6,7,2,3,12,13,6,7]
3291 ; AVX2-NEXT: vpshufb %xmm1, %xmm10, %xmm10
3292 ; AVX2-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
3293 ; AVX2-NEXT: vpblendw {{.*#+}} ymm10 = ymm8[0,1,2,3,4],ymm10[5,6,7],ymm8[8,9,10,11,12],ymm10[13,14,15]
3294 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm8[0,1,2,3],ymm10[4,5,6,7]
3295 ; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3296 ; AVX2-NEXT: vmovdqa 144(%rdi), %xmm11
3297 ; AVX2-NEXT: vmovdqa 128(%rdi), %xmm10
3298 ; AVX2-NEXT: vpblendd {{.*#+}} xmm8 = xmm10[0],xmm11[1],xmm10[2,3]
3299 ; AVX2-NEXT: vpshufb %xmm1, %xmm8, %xmm1
3300 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3301 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm12[0,1,2,3,4],ymm1[5,6,7],ymm12[8,9,10,11,12],ymm1[13,14,15]
3302 ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
3303 ; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3304 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm14[0,1],xmm13[2],xmm14[3]
3305 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm8 = [0,1,2,3,4,5,6,7,8,9,4,5,14,15,8,9]
3306 ; AVX2-NEXT: vpshufb %xmm8, %xmm1, %xmm1
3307 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3308 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
3309 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
3310 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3311 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm11[2],xmm10[3]
3312 ; AVX2-NEXT: vpshufb %xmm8, %xmm0, %xmm0
3313 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3314 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0,1,2,3,4],ymm0[5,6,7],ymm9[8,9,10,11,12],ymm0[13,14,15]
3315 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
3316 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3317 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm7[0],ymm6[1],ymm7[2,3],ymm6[4],ymm7[5],ymm6[6],ymm7[7,8],ymm6[9],ymm7[10,11],ymm6[12],ymm7[13],ymm6[14],ymm7[15]
3318 ; AVX2-NEXT: vmovdqa %ymm6, %ymm9
3319 ; AVX2-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3320 ; AVX2-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3321 ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
3322 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6],ymm0[7]
3323 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3324 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0,1],ymm2[2],ymm4[3],ymm2[4],ymm4[5,6],ymm2[7],ymm4[8,9],ymm2[10],ymm4[11],ymm2[12],ymm4[13,14],ymm2[15]
3325 ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm8
3326 ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm8[3,4],xmm1[5,6,7]
3327 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm8 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
3328 ; AVX2-NEXT: vpshufb %ymm8, %ymm0, %ymm0
3329 ; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
3330 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
3331 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm13[0],xmm14[1],xmm13[2,3]
3332 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm12 = [0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
3333 ; AVX2-NEXT: vpshufb %xmm12, %xmm1, %xmm1
3334 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3335 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
3336 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
3337 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3338 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0],ymm5[1],ymm3[2,3],ymm5[4],ymm3[5],ymm5[6],ymm3[7,8],ymm5[9],ymm3[10,11],ymm5[12],ymm3[13],ymm5[14],ymm3[15]
3339 ; AVX2-NEXT: vmovdqa %ymm5, %ymm6
3340 ; AVX2-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3341 ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
3342 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6],ymm0[7]
3343 ; AVX2-NEXT: vpshufb %ymm8, %ymm0, %ymm0
3344 ; AVX2-NEXT: vmovdqu (%rsp), %ymm5 # 32-byte Reload
3345 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm15[0,1],ymm5[2],ymm15[3],ymm5[4],ymm15[5,6],ymm5[7],ymm15[8,9],ymm5[10],ymm15[11],ymm5[12],ymm15[13,14],ymm5[15]
3346 ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm8
3347 ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm8[3,4],xmm1[5,6,7]
3348 ; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
3349 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
3350 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm11[0],xmm10[1],xmm11[2,3]
3351 ; AVX2-NEXT: vpshufb %xmm12, %xmm1, %xmm1
3352 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3353 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
3354 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
3355 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3356 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4,5],ymm7[6],ymm9[7,8],ymm7[9],ymm9[10],ymm7[11],ymm9[12,13],ymm7[14],ymm9[15]
3357 ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
3358 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
3359 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm4[1,2],ymm2[3],ymm4[4],ymm2[5],ymm4[6,7],ymm2[8],ymm4[9,10],ymm2[11],ymm4[12],ymm2[13],ymm4[14,15]
3360 ; AVX2-NEXT: vmovdqa %ymm4, %ymm7
3361 ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm8
3362 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm8[0],xmm1[1],xmm8[2],xmm1[3]
3363 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm12 = [u,u,u,u,u,u,u,u,u,u,u,u,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
3364 ; AVX2-NEXT: vpshufb %ymm12, %ymm0, %ymm0
3365 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm8 = [6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
3366 ; AVX2-NEXT: vpshufb %xmm8, %xmm1, %xmm1
3367 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
3368 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm13[0,1],xmm14[2],xmm13[3]
3369 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
3370 ; AVX2-NEXT: vpshufb %xmm4, %xmm1, %xmm1
3371 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3372 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
3373 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
3374 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3375 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4,5],ymm3[6],ymm6[7,8],ymm3[9],ymm6[10],ymm3[11],ymm6[12,13],ymm3[14],ymm6[15]
3376 ; AVX2-NEXT: vpermq {{.*#+}} ymm9 = ymm0[2,3,0,1]
3377 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm9[4],ymm0[5],ymm9[6],ymm0[7]
3378 ; AVX2-NEXT: vpshufb %ymm12, %ymm0, %ymm0
3379 ; AVX2-NEXT: vpblendw {{.*#+}} ymm9 = ymm5[0],ymm15[1,2],ymm5[3],ymm15[4],ymm5[5],ymm15[6,7],ymm5[8],ymm15[9,10],ymm5[11],ymm15[12],ymm5[13],ymm15[14,15]
3380 ; AVX2-NEXT: vmovdqa %ymm5, %ymm1
3381 ; AVX2-NEXT: vextracti128 $1, %ymm9, %xmm12
3382 ; AVX2-NEXT: vpblendd {{.*#+}} xmm9 = xmm12[0],xmm9[1],xmm12[2],xmm9[3]
3383 ; AVX2-NEXT: vpshufb %xmm8, %xmm9, %xmm8
3384 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2],ymm0[3,4,5,6,7]
3385 ; AVX2-NEXT: vpblendd {{.*#+}} xmm8 = xmm11[0,1],xmm10[2],xmm11[3]
3386 ; AVX2-NEXT: vpshufb %xmm4, %xmm8, %xmm2
3387 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3388 ; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0,1,2,3,4],ymm2[5,6,7],ymm0[8,9,10,11,12],ymm2[13,14,15]
3389 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
3390 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3391 ; AVX2-NEXT: vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
3392 ; AVX2-NEXT: # ymm2 = ymm2[0,1],mem[2],ymm2[3],mem[4],ymm2[5,6],mem[7],ymm2[8,9],mem[10],ymm2[11],mem[12],ymm2[13,14],mem[15]
3393 ; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]
3394 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4],ymm2[5,6],ymm6[7]
3395 ; AVX2-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm4 # 32-byte Folded Reload
3396 ; AVX2-NEXT: # ymm4 = ymm7[0],mem[1],ymm7[2,3],mem[4],ymm7[5],mem[6],ymm7[7,8],mem[9],ymm7[10,11],mem[12],ymm7[13],mem[14],ymm7[15]
3397 ; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5
3398 ; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3,4],xmm5[5,6,7]
3399 ; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [24,25,18,19,28,29,22,23,0,0,0,0,4,5,14,15,24,25,18,19,28,29,22,23,0,0,0,0,4,5,14,15]
3400 ; AVX2-NEXT: # ymm5 = mem[0,1,0,1]
3401 ; AVX2-NEXT: vpshufb %ymm5, %ymm2, %ymm2
3402 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm6 = [8,9,2,3,12,13,6,7,0,1,10,11,0,1,6,7]
3403 ; AVX2-NEXT: vpshufb %xmm6, %xmm4, %xmm4
3404 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3,4,5,6,7]
3405 ; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm13[3,1,2,3]
3406 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,2,1,4,5,6,7]
3407 ; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm14[0,2,2,3]
3408 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,0,3,4,5,6,7]
3409 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm7[0],xmm4[0],xmm7[1],xmm4[1]
3410 ; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
3411 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
3412 ; AVX2-NEXT: vpblendw $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
3413 ; AVX2-NEXT: # ymm3 = mem[0,1],ymm3[2],mem[3],ymm3[4],mem[5,6],ymm3[7],mem[8,9],ymm3[10],mem[11],ymm3[12],mem[13,14],ymm3[15]
3414 ; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
3415 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4],ymm3[5,6],ymm4[7]
3416 ; AVX2-NEXT: vpshufb %ymm5, %ymm3, %ymm3
3417 ; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm15[0],ymm1[1],ymm15[2,3],ymm1[4],ymm15[5],ymm1[6],ymm15[7,8],ymm1[9],ymm15[10,11],ymm1[12],ymm15[13],ymm1[14],ymm15[15]
3418 ; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5
3419 ; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3,4],xmm5[5,6,7]
3420 ; AVX2-NEXT: vpshufb %xmm6, %xmm4, %xmm4
3421 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7]
3422 ; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm11[3,1,2,3]
3423 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,2,1,4,5,6,7]
3424 ; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm10[0,2,2,3]
3425 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,0,3,4,5,6,7]
3426 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
3427 ; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
3428 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]
3429 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
3430 ; AVX2-NEXT: vmovaps %ymm4, 32(%rsi)
3431 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
3432 ; AVX2-NEXT: vmovaps %ymm1, (%rsi)
3433 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
3434 ; AVX2-NEXT: vmovaps %ymm1, 32(%rdx)
3435 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
3436 ; AVX2-NEXT: vmovaps %ymm1, (%rdx)
3437 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
3438 ; AVX2-NEXT: vmovaps %ymm1, 32(%rcx)
3439 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
3440 ; AVX2-NEXT: vmovaps %ymm1, (%rcx)
3441 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
3442 ; AVX2-NEXT: vmovaps %ymm1, 32(%r8)
3443 ; AVX2-NEXT: vmovdqa %ymm0, (%r8)
3444 ; AVX2-NEXT: vmovdqa %ymm2, 32(%r9)
3445 ; AVX2-NEXT: vmovdqa %ymm3, (%r9)
3446 ; AVX2-NEXT: addq $264, %rsp # imm = 0x108
3447 ; AVX2-NEXT: vzeroupper
3448 ; AVX2-NEXT: retq
3449 ;
3450 ; AVX2-FP-LABEL: load_i16_stride5_vf32:
3451 ; AVX2-FP: # %bb.0:
3452 ; AVX2-FP-NEXT: subq $264, %rsp # imm = 0x108
3453 ; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm13
3454 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm6
3455 ; AVX2-FP-NEXT: vmovdqa 64(%rdi), %ymm7
3456 ; AVX2-FP-NEXT: vmovdqa 96(%rdi), %ymm10
3457 ; AVX2-FP-NEXT: vmovdqa 192(%rdi), %ymm2
3458 ; AVX2-FP-NEXT: vmovdqa 160(%rdi), %ymm3
3459 ; AVX2-FP-NEXT: vmovdqa 224(%rdi), %ymm1
3460 ; AVX2-FP-NEXT: vmovdqa 256(%rdi), %ymm4
3461 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm4[1,2],ymm1[3],ymm4[4],ymm1[5],ymm4[6,7],ymm1[8],ymm4[9,10],ymm1[11],ymm4[12],ymm1[13],ymm4[14,15]
3462 ; AVX2-FP-NEXT: vmovdqa %ymm4, %ymm5
3463 ; AVX2-FP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3464 ; AVX2-FP-NEXT: vmovdqa %ymm1, %ymm4
3465 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
3466 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6],ymm1[7]
3467 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,10,11,4,5,14,15,8,9,10,11,4,5,6,7,16,17,26,27,20,21,30,31,24,25,26,27,20,21,22,23]
3468 ; AVX2-FP-NEXT: vpshufb %ymm1, %ymm0, %ymm0
3469 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13],ymm2[14],ymm3[15]
3470 ; AVX2-FP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3471 ; AVX2-FP-NEXT: vextracti128 $1, %ymm8, %xmm9
3472 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm9[1,2,3],xmm8[4,5],xmm9[6,7]
3473 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm9 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,6,7]
3474 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm8, %xmm8
3475 ; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} xmm11 = [65535,65535,65535,65535,65535,65535,65535,0]
3476 ; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm8, %ymm0, %ymm0
3477 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm7[0],ymm10[1,2],ymm7[3],ymm10[4],ymm7[5],ymm10[6,7],ymm7[8],ymm10[9,10],ymm7[11],ymm10[12],ymm7[13],ymm10[14,15]
3478 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm12 = ymm8[2,3,0,1]
3479 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm12[5],ymm8[6],ymm12[7]
3480 ; AVX2-FP-NEXT: vpshufb %ymm1, %ymm8, %ymm1
3481 ; AVX2-FP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3482 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm13[0],ymm6[1],ymm13[2,3],ymm6[4],ymm13[5],ymm6[6],ymm13[7,8],ymm6[9],ymm13[10,11],ymm6[12],ymm13[13],ymm6[14],ymm13[15]
3483 ; AVX2-FP-NEXT: vextracti128 $1, %ymm8, %xmm12
3484 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm12[1,2,3],xmm8[4,5],xmm12[6,7]
3485 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm8, %xmm8
3486 ; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm8, %ymm1, %ymm12
3487 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
3488 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
3489 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm8[5],ymm1[6,7]
3490 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm8 = [2,3,12,13,6,7,0,1,10,11,6,7,8,9,8,9,18,19,28,29,22,23,16,17,26,27,22,23,24,25,24,25]
3491 ; AVX2-FP-NEXT: vpshufb %ymm8, %ymm1, %ymm1
3492 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0],ymm3[1],ymm2[2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10],ymm3[11],ymm2[12,13],ymm3[14],ymm2[15]
3493 ; AVX2-FP-NEXT: vmovdqa %ymm2, %ymm5
3494 ; AVX2-FP-NEXT: vextracti128 $1, %ymm9, %xmm14
3495 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1],xmm14[2,3],xmm9[4,5,6],xmm14[7]
3496 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm14 = [2,3,12,13,6,7,0,1,10,11,4,5,14,15,10,11]
3497 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm9, %xmm9
3498 ; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm9, %ymm1, %ymm1
3499 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm10[0],ymm7[1],ymm10[2,3],ymm7[4],ymm10[5],ymm7[6],ymm10[7,8],ymm7[9],ymm10[10,11],ymm7[12],ymm10[13],ymm7[14],ymm10[15]
3500 ; AVX2-FP-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3501 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm15 = ymm9[2,3,0,1]
3502 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4],ymm15[5],ymm9[6,7]
3503 ; AVX2-FP-NEXT: vpshufb %ymm8, %ymm9, %ymm15
3504 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm6[0],ymm13[1],ymm6[2],ymm13[3],ymm6[4,5],ymm13[6],ymm6[7,8],ymm13[9],ymm6[10],ymm13[11],ymm6[12,13],ymm13[14],ymm6[15]
3505 ; AVX2-FP-NEXT: vextracti128 $1, %ymm8, %xmm9
3506 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm9 = xmm8[0,1],xmm9[2,3],xmm8[4,5,6],xmm9[7]
3507 ; AVX2-FP-NEXT: vmovdqa 304(%rdi), %xmm8
3508 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm9, %xmm14
3509 ; AVX2-FP-NEXT: vmovdqa 288(%rdi), %xmm9
3510 ; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm14, %ymm15, %ymm11
3511 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm14 = xmm9[0],xmm8[1],xmm9[2,3]
3512 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm2 = [6,7,2,3,4,5,6,7,6,7,2,3,12,13,6,7]
3513 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm14, %xmm14
3514 ; AVX2-FP-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
3515 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm14 = ymm0[0,1,2,3,4],ymm14[5,6,7],ymm0[8,9,10,11,12],ymm14[13,14,15]
3516 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm14[4,5,6,7]
3517 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3518 ; AVX2-FP-NEXT: vmovdqa 144(%rdi), %xmm6
3519 ; AVX2-FP-NEXT: vmovdqa 128(%rdi), %xmm15
3520 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm14 = xmm15[0],xmm6[1],xmm15[2,3]
3521 ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm14, %xmm2
3522 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3523 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm12[0,1,2,3,4],ymm2[5,6,7],ymm12[8,9,10,11,12],ymm2[13,14,15]
3524 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm2[4,5,6,7]
3525 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3526 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm9[0,1],xmm8[2],xmm9[3]
3527 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm12 = [0,1,2,3,4,5,6,7,8,9,4,5,14,15,8,9]
3528 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm2, %xmm2
3529 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3530 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3,4],ymm2[5,6,7],ymm1[8,9,10,11,12],ymm2[13,14,15]
3531 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm2[4,5,6,7]
3532 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3533 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm15[0,1],xmm6[2],xmm15[3]
3534 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm1, %xmm1
3535 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
3536 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm11[0,1,2,3,4],ymm1[5,6,7],ymm11[8,9,10,11,12],ymm1[13,14,15]
3537 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm1[4,5,6,7]
3538 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3539 ; AVX2-FP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3540 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
3541 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5],ymm3[6],ymm4[7,8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13],ymm3[14],ymm4[15]
3542 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
3543 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6],ymm1[7]
3544 ; AVX2-FP-NEXT: vmovdqu %ymm5, (%rsp) # 32-byte Spill
3545 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3546 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm5[0,1],ymm0[2],ymm5[3],ymm0[4],ymm5[5,6],ymm0[7],ymm5[8,9],ymm0[10],ymm5[11],ymm0[12],ymm5[13,14],ymm0[15]
3547 ; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm11
3548 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm11[3,4],xmm2[5,6,7]
3549 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm11 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
3550 ; AVX2-FP-NEXT: vpshufb %ymm11, %ymm1, %ymm1
3551 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm12 = [4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
3552 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm2, %xmm2
3553 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
3554 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm8[0],xmm9[1],xmm8[2,3]
3555 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm14 = [0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
3556 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm2, %xmm2
3557 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3558 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3,4],ymm2[5,6,7],ymm1[8,9,10,11,12],ymm2[13,14,15]
3559 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
3560 ; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3561 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm7[0],ymm10[1],ymm7[2,3],ymm10[4],ymm7[5],ymm10[6],ymm7[7,8],ymm10[9],ymm7[10,11],ymm10[12],ymm7[13],ymm10[14],ymm7[15]
3562 ; AVX2-FP-NEXT: vmovdqa %ymm7, %ymm10
3563 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
3564 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6],ymm1[7]
3565 ; AVX2-FP-NEXT: vpshufb %ymm11, %ymm1, %ymm1
3566 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
3567 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm7[0,1],ymm13[2],ymm7[3],ymm13[4],ymm7[5,6],ymm13[7],ymm7[8,9],ymm13[10],ymm7[11],ymm13[12],ymm7[13,14],ymm13[15]
3568 ; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm11
3569 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm11[3,4],xmm2[5,6,7]
3570 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm2, %xmm2
3571 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
3572 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm6[0],xmm15[1],xmm6[2,3]
3573 ; AVX2-FP-NEXT: vpshufb %xmm14, %xmm2, %xmm2
3574 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3575 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3,4],ymm2[5,6,7],ymm1[8,9,10,11,12],ymm2[13,14,15]
3576 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
3577 ; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3578 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7,8],ymm4[9],ymm3[10],ymm4[11],ymm3[12,13],ymm4[14],ymm3[15]
3579 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
3580 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]
3581 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0],ymm5[1,2],ymm0[3],ymm5[4],ymm0[5],ymm5[6,7],ymm0[8],ymm5[9,10],ymm0[11],ymm5[12],ymm0[13],ymm5[14,15]
3582 ; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm11
3583 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm11[0],xmm2[1],xmm11[2],xmm2[3]
3584 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm11 = [u,u,u,u,u,u,u,u,u,u,u,u,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
3585 ; AVX2-FP-NEXT: vpshufb %ymm11, %ymm1, %ymm1
3586 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm12 = [6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
3587 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm2, %xmm2
3588 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
3589 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm8[0,1],xmm9[2],xmm8[3]
3590 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
3591 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
3592 ; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
3593 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3,4],ymm2[5,6,7],ymm1[8,9,10,11,12],ymm2[13,14,15]
3594 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm2[4,5,6,7]
3595 ; AVX2-FP-NEXT: vmovdqa %ymm10, %ymm5
3596 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
3597 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm10[0],ymm5[1],ymm10[2],ymm5[3],ymm10[4,5],ymm5[6],ymm10[7,8],ymm5[9],ymm10[10],ymm5[11],ymm10[12,13],ymm5[14],ymm10[15]
3598 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm14 = ymm1[2,3,0,1]
3599 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4],ymm1[5],ymm14[6],ymm1[7]
3600 ; AVX2-FP-NEXT: vpshufb %ymm11, %ymm1, %ymm1
3601 ; AVX2-FP-NEXT: vmovdqa %ymm7, %ymm4
3602 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm11 = ymm13[0],ymm7[1,2],ymm13[3],ymm7[4],ymm13[5],ymm7[6,7],ymm13[8],ymm7[9,10],ymm13[11],ymm7[12],ymm13[13],ymm7[14,15]
3603 ; AVX2-FP-NEXT: vextracti128 $1, %ymm11, %xmm14
3604 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm11 = xmm14[0],xmm11[1],xmm14[2],xmm11[3]
3605 ; AVX2-FP-NEXT: vpshufb %xmm12, %xmm11, %xmm11
3606 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2],ymm1[3,4,5,6,7]
3607 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm11 = xmm6[0,1],xmm15[2],xmm6[3]
3608 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm11, %xmm3
3609 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
3610 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm1[0,1,2,3,4],ymm3[5,6,7],ymm1[8,9,10,11,12],ymm3[13,14,15]
3611 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
3612 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm11 = [12,13,14,15,4,5,14,15,8,9,10,11,12,13,14,15]
3613 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm6, %xmm3
3614 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,2,3,0,1,10,11,8,9,10,11,12,13,14,15]
3615 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm15, %xmm12
3616 ; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm12[0],xmm3[0],xmm12[1],xmm3[1]
3617 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm6 = ymm10[0,1],ymm5[2],ymm10[3],ymm5[4],ymm10[5,6],ymm5[7],ymm10[8,9],ymm5[10],ymm10[11],ymm5[12],ymm10[13,14],ymm5[15]
3618 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
3619 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4],ymm6[5,6],ymm7[7]
3620 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm13[1],ymm4[2,3],ymm13[4],ymm4[5],ymm13[6],ymm4[7,8],ymm13[9],ymm4[10,11],ymm13[12],ymm4[13],ymm13[14],ymm4[15]
3621 ; AVX2-FP-NEXT: vextracti128 $1, %ymm4, %xmm5
3622 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3,4],xmm5[5,6,7]
3623 ; AVX2-FP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [24,25,18,19,28,29,22,23,0,0,0,0,4,5,14,15,24,25,18,19,28,29,22,23,0,0,0,0,4,5,14,15]
3624 ; AVX2-FP-NEXT: # ymm5 = mem[0,1,0,1]
3625 ; AVX2-FP-NEXT: vpshufb %ymm5, %ymm6, %ymm6
3626 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm7 = [8,9,2,3,12,13,6,7,0,1,10,11,0,1,6,7]
3627 ; AVX2-FP-NEXT: vpshufb %xmm7, %xmm4, %xmm4
3628 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2],ymm6[3,4,5,6,7]
3629 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
3630 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
3631 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
3632 ; AVX2-FP-NEXT: vpblendw $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
3633 ; AVX2-FP-NEXT: # ymm4 = mem[0,1],ymm4[2],mem[3],ymm4[4],mem[5,6],ymm4[7],mem[8,9],ymm4[10],mem[11],ymm4[12],mem[13,14],ymm4[15]
3634 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm6 = ymm4[2,3,0,1]
3635 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4],ymm4[5,6],ymm6[7]
3636 ; AVX2-FP-NEXT: vpshufb %ymm5, %ymm4, %ymm4
3637 ; AVX2-FP-NEXT: vmovdqu (%rsp), %ymm5 # 32-byte Reload
3638 ; AVX2-FP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
3639 ; AVX2-FP-NEXT: # ymm5 = ymm5[0],mem[1],ymm5[2,3],mem[4],ymm5[5],mem[6],ymm5[7,8],mem[9],ymm5[10,11],mem[12],ymm5[13],mem[14],ymm5[15]
3640 ; AVX2-FP-NEXT: vextracti128 $1, %ymm5, %xmm6
3641 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3,4],xmm6[5,6,7]
3642 ; AVX2-FP-NEXT: vpshufb %xmm7, %xmm5, %xmm5
3643 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3,4,5,6,7]
3644 ; AVX2-FP-NEXT: vpshufb %xmm11, %xmm8, %xmm5
3645 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm9, %xmm0
3646 ; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
3647 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
3648 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
3649 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
3650 ; AVX2-FP-NEXT: vmovaps %ymm4, 32(%rsi)
3651 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
3652 ; AVX2-FP-NEXT: vmovaps %ymm4, (%rsi)
3653 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
3654 ; AVX2-FP-NEXT: vmovaps %ymm4, 32(%rdx)
3655 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
3656 ; AVX2-FP-NEXT: vmovaps %ymm4, (%rdx)
3657 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
3658 ; AVX2-FP-NEXT: vmovaps %ymm4, 32(%rcx)
3659 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
3660 ; AVX2-FP-NEXT: vmovaps %ymm4, (%rcx)
3661 ; AVX2-FP-NEXT: vmovdqa %ymm2, 32(%r8)
3662 ; AVX2-FP-NEXT: vmovdqa %ymm1, (%r8)
3663 ; AVX2-FP-NEXT: vmovdqa %ymm0, 32(%r9)
3664 ; AVX2-FP-NEXT: vmovdqa %ymm3, (%r9)
3665 ; AVX2-FP-NEXT: addq $264, %rsp # imm = 0x108
3666 ; AVX2-FP-NEXT: vzeroupper
3667 ; AVX2-FP-NEXT: retq
3668 ;
3669 ; AVX2-FCP-LABEL: load_i16_stride5_vf32:
3670 ; AVX2-FCP: # %bb.0:
3671 ; AVX2-FCP-NEXT: subq $296, %rsp # imm = 0x128
3672 ; AVX2-FCP-NEXT: vmovdqa 224(%rdi), %ymm15
3673 ; AVX2-FCP-NEXT: vmovdqa 256(%rdi), %ymm1
3674 ; AVX2-FCP-NEXT: vmovdqa 192(%rdi), %ymm3
3675 ; AVX2-FCP-NEXT: vmovdqa 160(%rdi), %ymm14
3676 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm2
3677 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm0
3678 ; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %ymm4
3679 ; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %ymm5
3680 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm4[0],ymm5[1,2],ymm4[3],ymm5[4],ymm4[5],ymm5[6,7],ymm4[8],ymm5[9,10],ymm4[11],ymm5[12],ymm4[13],ymm5[14,15]
3681 ; AVX2-FCP-NEXT: vmovdqa %ymm5, %ymm7
3682 ; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3683 ; AVX2-FCP-NEXT: vmovdqa %ymm4, %ymm6
3684 ; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3685 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm10 = [1,3,0,2,4,6,1,3]
3686 ; AVX2-FCP-NEXT: vpermd %ymm8, %ymm10, %ymm8
3687 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm11 = [0,1,6,7,8,9,14,15,4,5,14,15,4,5,2,3,16,17,22,23,24,25,30,31,20,21,30,31,20,21,18,19]
3688 ; AVX2-FCP-NEXT: vpshufb %ymm11, %ymm8, %ymm8
3689 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5],ymm0[6],ymm2[7,8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]
3690 ; AVX2-FCP-NEXT: vmovdqa %ymm0, %ymm5
3691 ; AVX2-FCP-NEXT: vmovdqa %ymm2, %ymm4
3692 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm9, %xmm12
3693 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0],xmm12[1,2,3],xmm9[4,5],xmm12[6,7]
3694 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm12 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,6,7]
3695 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm9, %xmm13
3696 ; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} xmm9 = [65535,65535,65535,65535,65535,65535,65535,0]
3697 ; AVX2-FCP-NEXT: vpblendvb %ymm9, %ymm13, %ymm8, %ymm8
3698 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm14[0],ymm3[1],ymm14[2,3],ymm3[4],ymm14[5],ymm3[6],ymm14[7,8],ymm3[9],ymm14[10,11],ymm3[12],ymm14[13],ymm3[14],ymm14[15]
3699 ; AVX2-FCP-NEXT: vmovdqa %ymm14, %ymm0
3700 ; AVX2-FCP-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3701 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm13, %xmm14
3702 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0],xmm14[1,2,3],xmm13[4,5],xmm14[6,7]
3703 ; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm13, %xmm12
3704 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm15[0],ymm1[1,2],ymm15[3],ymm1[4],ymm15[5],ymm1[6,7],ymm15[8],ymm1[9,10],ymm15[11],ymm1[12],ymm15[13],ymm1[14,15]
3705 ; AVX2-FCP-NEXT: vmovdqu %ymm1, (%rsp) # 32-byte Spill
3706 ; AVX2-FCP-NEXT: vmovdqa %ymm15, %ymm2
3707 ; AVX2-FCP-NEXT: vpermd %ymm13, %ymm10, %ymm10
3708 ; AVX2-FCP-NEXT: vpshufb %ymm11, %ymm10, %ymm10
3709 ; AVX2-FCP-NEXT: vpblendvb %ymm9, %ymm12, %ymm10, %ymm11
3710 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm7[0],ymm6[1],ymm7[2,3],ymm6[4],ymm7[5],ymm6[6],ymm7[7,8],ymm6[9],ymm7[10,11],ymm6[12],ymm7[13],ymm6[14],ymm7[15]
3711 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm12 = [2,0,0,0,4,7,1,6]
3712 ; AVX2-FCP-NEXT: vpermd %ymm10, %ymm12, %ymm10
3713 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm14 = [2,3,4,5,10,11,0,1,14,15,2,3,12,13,0,1,18,19,20,21,26,27,16,17,30,31,18,19,28,29,16,17]
3714 ; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm10, %ymm10
3715 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
3716 ; AVX2-FCP-NEXT: vmovdqa %ymm4, %ymm7
3717 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm13, %xmm15
3718 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1],xmm15[2,3],xmm13[4,5,6],xmm15[7]
3719 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm15 = [2,3,12,13,6,7,0,1,10,11,4,5,14,15,10,11]
3720 ; AVX2-FCP-NEXT: vpshufb %xmm15, %xmm13, %xmm13
3721 ; AVX2-FCP-NEXT: vpblendvb %ymm9, %ymm13, %ymm10, %ymm13
3722 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm3[0],ymm0[1],ymm3[2],ymm0[3],ymm3[4,5],ymm0[6],ymm3[7,8],ymm0[9],ymm3[10],ymm0[11],ymm3[12,13],ymm0[14],ymm3[15]
3723 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm10, %xmm0
3724 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3],xmm10[4,5,6],xmm0[7]
3725 ; AVX2-FCP-NEXT: vpshufb %xmm15, %xmm0, %xmm0
3726 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5],ymm2[6],ymm1[7,8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
3727 ; AVX2-FCP-NEXT: vmovdqa %ymm2, %ymm15
3728 ; AVX2-FCP-NEXT: vpermd %ymm10, %ymm12, %ymm10
3729 ; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %ymm12
3730 ; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm10, %ymm10
3731 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm14 = [0,3,1,3,0,3,5,7]
3732 ; AVX2-FCP-NEXT: vpblendvb %ymm9, %ymm0, %ymm10, %ymm0
3733 ; AVX2-FCP-NEXT: vpermd %ymm12, %ymm14, %ymm9
3734 ; AVX2-FCP-NEXT: vpbroadcastq {{.*#+}} ymm6 = [0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27]
3735 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm9, %ymm9
3736 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm9 = ymm8[0,1,2,3,4],ymm9[5,6,7],ymm8[8,9,10,11,12],ymm9[13,14,15]
3737 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3],ymm9[4,5,6,7]
3738 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3739 ; AVX2-FCP-NEXT: vmovdqa 288(%rdi), %ymm10
3740 ; AVX2-FCP-NEXT: vpermd %ymm10, %ymm14, %ymm8
3741 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm8, %ymm8
3742 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm11[0,1,2,3,4],ymm8[5,6,7],ymm11[8,9,10,11,12],ymm8[13,14,15]
3743 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3],ymm8[4,5,6,7]
3744 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3745 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [1,3,2,3,1,3,6,7]
3746 ; AVX2-FCP-NEXT: vpermd %ymm12, %ymm8, %ymm11
3747 ; AVX2-FCP-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25]
3748 ; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm11, %ymm11
3749 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm13[0,1,2,3,4],ymm11[5,6,7],ymm13[8,9,10,11,12],ymm11[13,14,15]
3750 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3],ymm11[4,5,6,7]
3751 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3752 ; AVX2-FCP-NEXT: vpermd %ymm10, %ymm8, %ymm8
3753 ; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm8, %ymm8
3754 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm0[0,1,2,3,4],ymm8[5,6,7],ymm0[8,9,10,11,12],ymm8[13,14,15]
3755 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
3756 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3757 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm5[0,1],ymm4[2],ymm5[3],ymm4[4],ymm5[5,6],ymm4[7],ymm5[8,9],ymm4[10],ymm5[11],ymm4[12],ymm5[13,14],ymm4[15]
3758 ; AVX2-FCP-NEXT: vmovdqa %ymm5, %ymm9
3759 ; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3760 ; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3761 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm8
3762 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm8[3,4],xmm0[5,6,7]
3763 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
3764 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
3765 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
3766 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm14 = [0,2,0,0,5,7,2,4]
3767 ; AVX2-FCP-NEXT: vpermd %ymm8, %ymm14, %ymm8
3768 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
3769 ; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm8, %ymm8
3770 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
3771 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm0
3772 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm8[3,4,5,6,7]
3773 ; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [1,4,6,0,1,4,6,0]
3774 ; AVX2-FCP-NEXT: # ymm8 = mem[0,1,0,1]
3775 ; AVX2-FCP-NEXT: vpermd %ymm12, %ymm8, %ymm11
3776 ; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm11, %ymm11
3777 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm0[0,1,2,3,4],ymm11[5,6,7],ymm0[8,9,10,11,12],ymm11[13,14,15]
3778 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm11[4,5,6,7]
3779 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3780 ; AVX2-FCP-NEXT: vmovdqa %ymm3, %ymm6
3781 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
3782 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm3[0,1],ymm0[2],ymm3[3],ymm0[4],ymm3[5,6],ymm0[7],ymm3[8,9],ymm0[10],ymm3[11],ymm0[12],ymm3[13,14],ymm0[15]
3783 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm11, %xmm13
3784 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0,1,2],xmm13[3,4],xmm11[5,6,7]
3785 ; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm11, %xmm1
3786 ; AVX2-FCP-NEXT: vmovdqu (%rsp), %ymm3 # 32-byte Reload
3787 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm15[0],ymm3[1],ymm15[2,3],ymm3[4],ymm15[5],ymm3[6],ymm15[7,8],ymm3[9],ymm15[10,11],ymm3[12],ymm15[13],ymm3[14],ymm15[15]
3788 ; AVX2-FCP-NEXT: vpermd %ymm11, %ymm14, %ymm11
3789 ; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm11, %ymm2
3790 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
3791 ; AVX2-FCP-NEXT: vpermd %ymm10, %ymm8, %ymm2
3792 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27]
3793 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3,4],ymm2[5,6,7],ymm1[8,9,10,11,12],ymm2[13,14,15]
3794 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
3795 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3796 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm7[0],ymm9[1,2],ymm7[3],ymm9[4],ymm7[5],ymm9[6,7],ymm7[8],ymm9[9,10],ymm7[11],ymm9[12],ymm7[13],ymm9[14,15]
3797 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm1, %xmm2
3798 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
3799 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4,5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10],ymm5[11],ymm4[12,13],ymm5[14],ymm4[15]
3800 ; AVX2-FCP-NEXT: vmovdqa %ymm4, %ymm7
3801 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm11 = [0,3,0,0,5,0,2,7]
3802 ; AVX2-FCP-NEXT: vpermd %ymm2, %ymm11, %ymm2
3803 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm13 = [u,u,u,u,u,u,u,u,u,u,u,u,2,3,4,5,18,19,20,21,26,27,16,17,30,31,30,31,18,19,20,21]
3804 ; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm2, %ymm2
3805 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm14 = [6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
3806 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm1, %xmm1
3807 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
3808 ; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [2,4,7,0,2,4,7,0]
3809 ; AVX2-FCP-NEXT: # ymm2 = mem[0,1,0,1]
3810 ; AVX2-FCP-NEXT: vpermd %ymm12, %ymm2, %ymm8
3811 ; AVX2-FCP-NEXT: vpbroadcastq {{.*#+}} ymm4 = [0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25]
3812 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm8, %ymm8
3813 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm1[0,1,2,3,4],ymm8[5,6,7],ymm1[8,9,10,11,12],ymm8[13,14,15]
3814 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm8[4,5,6,7]
3815 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
3816 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm6[1,2],ymm0[3],ymm6[4],ymm0[5],ymm6[6,7],ymm0[8],ymm6[9,10],ymm0[11],ymm6[12],ymm0[13],ymm6[14,15]
3817 ; AVX2-FCP-NEXT: vmovdqa %ymm6, %ymm8
3818 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm1, %xmm0
3819 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
3820 ; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm0, %xmm0
3821 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0],ymm15[1],ymm3[2],ymm15[3],ymm3[4,5],ymm15[6],ymm3[7,8],ymm15[9],ymm3[10],ymm15[11],ymm3[12,13],ymm15[14],ymm3[15]
3822 ; AVX2-FCP-NEXT: vmovdqa %ymm3, %ymm9
3823 ; AVX2-FCP-NEXT: vmovdqa %ymm15, %ymm14
3824 ; AVX2-FCP-NEXT: vpermd %ymm1, %ymm11, %ymm1
3825 ; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm1, %ymm1
3826 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
3827 ; AVX2-FCP-NEXT: vpermd %ymm10, %ymm2, %ymm1
3828 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm1
3829 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
3830 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm15 = ymm0[0,1,2,3],ymm1[4,5,6,7]
3831 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm7[0,1],ymm5[2],ymm7[3],ymm5[4],ymm7[5,6],ymm5[7],ymm7[8,9],ymm5[10],ymm7[11],ymm5[12],ymm7[13,14],ymm5[15]
3832 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
3833 ; AVX2-FCP-NEXT: vpblendw $173, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
3834 ; AVX2-FCP-NEXT: # ymm1 = mem[0],ymm1[1],mem[2,3],ymm1[4],mem[5],ymm1[6],mem[7,8],ymm1[9],mem[10,11],ymm1[12],mem[13],ymm1[14],mem[15]
3835 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm1, %xmm2
3836 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3,4],xmm2[5,6,7]
3837 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,3,0,0,6,0,3,5]
3838 ; AVX2-FCP-NEXT: vpermd %ymm0, %ymm2, %ymm0
3839 ; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [16,17,22,23,24,25,30,31,0,0,0,0,0,1,6,7,16,17,22,23,24,25,30,31,0,0,0,0,0,1,6,7]
3840 ; AVX2-FCP-NEXT: # ymm4 = mem[0,1,0,1]
3841 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm0, %ymm0
3842 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [8,9,2,3,12,13,6,7,0,1,10,11,0,1,6,7]
3843 ; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm1, %xmm1
3844 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
3845 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,2,1,3,0,2,5,7]
3846 ; AVX2-FCP-NEXT: vpermd %ymm12, %ymm1, %ymm6
3847 ; AVX2-FCP-NEXT: vpbroadcastq {{.*#+}} ymm7 = [16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31]
3848 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm6, %ymm6
3849 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm6[6,7]
3850 ; AVX2-FCP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm3 # 32-byte Folded Reload
3851 ; AVX2-FCP-NEXT: # ymm3 = ymm8[0],mem[1],ymm8[2,3],mem[4],ymm8[5],mem[6],ymm8[7,8],mem[9],ymm8[10,11],mem[12],ymm8[13],mem[14],ymm8[15]
3852 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm3, %xmm6
3853 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm6[0,1,2],xmm3[3,4],xmm6[5,6,7]
3854 ; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm3, %xmm3
3855 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm9[0,1],ymm14[2],ymm9[3],ymm14[4],ymm9[5,6],ymm14[7],ymm9[8,9],ymm14[10],ymm9[11],ymm14[12],ymm9[13,14],ymm14[15]
3856 ; AVX2-FCP-NEXT: vpermd %ymm5, %ymm2, %ymm2
3857 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm2
3858 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7]
3859 ; AVX2-FCP-NEXT: vpermd %ymm10, %ymm1, %ymm1
3860 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm1, %ymm1
3861 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
3862 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3863 ; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rsi)
3864 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3865 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rsi)
3866 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3867 ; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rdx)
3868 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3869 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rdx)
3870 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3871 ; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rcx)
3872 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3873 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%rcx)
3874 ; AVX2-FCP-NEXT: vmovdqa %ymm15, 32(%r8)
3875 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
3876 ; AVX2-FCP-NEXT: vmovaps %ymm2, (%r8)
3877 ; AVX2-FCP-NEXT: vmovdqa %ymm1, 32(%r9)
3878 ; AVX2-FCP-NEXT: vmovdqa %ymm0, (%r9)
3879 ; AVX2-FCP-NEXT: addq $296, %rsp # imm = 0x128
3880 ; AVX2-FCP-NEXT: vzeroupper
3881 ; AVX2-FCP-NEXT: retq
3882 ;
3883 ; AVX512-LABEL: load_i16_stride5_vf32:
3884 ; AVX512: # %bb.0:
3885 ; AVX512-NEXT: vmovdqa 256(%rdi), %ymm0
3886 ; AVX512-NEXT: vmovdqa 288(%rdi), %ymm1
3887 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
3888 ; AVX512-NEXT: vmovdqa64 %ymm1, %ymm17
3889 ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm3
3890 ; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[3,4],xmm2[5,6,7]
3891 ; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,4,5,14,15,8,9,2,3,12,13,6,7]
3892 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm4
3893 ; AVX512-NEXT: vmovdqa 192(%rdi), %ymm3
3894 ; AVX512-NEXT: vmovdqa 224(%rdi), %ymm9
3895 ; AVX512-NEXT: vpblendw {{.*#+}} ymm5 = ymm9[0],ymm3[1],ymm9[2,3],ymm3[4],ymm9[5],ymm3[6],ymm9[7,8],ymm3[9],ymm9[10,11],ymm3[12],ymm9[13],ymm3[14],ymm9[15]
3896 ; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm6
3897 ; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1,2],ymm6[3],ymm5[4,5,6,7]
3898 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm7 = [u,u,u,u,u,u,u,u,8,9,2,3,12,13,6,7,16,17,26,27,26,27,30,31,24,25,18,19,28,29,22,23]
3899 ; AVX512-NEXT: vpshufb %ymm7, %ymm5, %ymm5
3900 ; AVX512-NEXT: vmovdqa 176(%rdi), %xmm1
3901 ; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm1[3,1,2,3]
3902 ; AVX512-NEXT: vmovdqa64 %xmm1, %xmm20
3903 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[2,1,2,3,4,5,6,7]
3904 ; AVX512-NEXT: vmovdqa 160(%rdi), %xmm6
3905 ; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[0,2,2,3]
3906 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,3,2,3,4,5,6,7]
3907 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm10[0],xmm8[0],xmm10[1],xmm8[1]
3908 ; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1],ymm5[2,3,4,5,6,7]
3909 ; AVX512-NEXT: vpblendd {{.*#+}} ymm14 = ymm5[0,1,2,3,4],ymm4[5,6,7]
3910 ; AVX512-NEXT: vmovdqa (%rdi), %ymm8
3911 ; AVX512-NEXT: vmovdqa 32(%rdi), %ymm10
3912 ; AVX512-NEXT: vmovdqa 64(%rdi), %ymm4
3913 ; AVX512-NEXT: vmovdqa 96(%rdi), %ymm5
3914 ; AVX512-NEXT: vpblendw {{.*#+}} ymm11 = ymm4[0],ymm5[1,2],ymm4[3],ymm5[4],ymm4[5],ymm5[6,7],ymm4[8],ymm5[9,10],ymm4[11],ymm5[12],ymm4[13],ymm5[14,15]
3915 ; AVX512-NEXT: vpermq {{.*#+}} ymm12 = ymm11[2,3,0,1]
3916 ; AVX512-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm12[5],ymm11[6],ymm12[7]
3917 ; AVX512-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[6,7,16,17,26,27,20,21,30,31,24,25],zero,zero,zero,zero,zero,zero
3918 ; AVX512-NEXT: vpblendw {{.*#+}} ymm12 = ymm8[0],ymm10[1],ymm8[2,3],ymm10[4],ymm8[5],ymm10[6],ymm8[7,8],ymm10[9],ymm8[10,11],ymm10[12],ymm8[13],ymm10[14],ymm8[15]
3919 ; AVX512-NEXT: vextracti128 $1, %ymm12, %xmm13
3920 ; AVX512-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0],xmm13[1,2,3],xmm12[4,5],xmm13[6,7]
3921 ; AVX512-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[u,u,u,u,u,u]
3922 ; AVX512-NEXT: vpor %ymm11, %ymm12, %ymm15
3923 ; AVX512-NEXT: vmovdqa 144(%rdi), %xmm11
3924 ; AVX512-NEXT: vmovdqa 128(%rdi), %xmm12
3925 ; AVX512-NEXT: vpblendd {{.*#+}} xmm13 = xmm12[0],xmm11[1],xmm12[2,3]
3926 ; AVX512-NEXT: vpshufb %xmm7, %xmm13, %xmm7
3927 ; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
3928 ; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
3929 ; AVX512-NEXT: vpternlogq $184, %zmm15, %zmm18, %zmm7
3930 ; AVX512-NEXT: vinserti64x4 $1, %ymm14, %zmm7, %zmm16
3931 ; AVX512-NEXT: vpblendw {{.*#+}} ymm7 = ymm3[0],ymm9[1],ymm3[2,3],ymm9[4],ymm3[5],ymm9[6],ymm3[7,8],ymm9[9],ymm3[10,11],ymm9[12],ymm3[13],ymm9[14],ymm3[15]
3932 ; AVX512-NEXT: vextracti128 $1, %ymm7, %xmm14
3933 ; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0],ymm14[1],ymm7[2],ymm14[3],ymm7[4,5,6,7]
3934 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm14 = [2,3,12,13,0,1,0,1,10,11,4,5,14,15,8,9,18,19,28,29,16,17,16,17,26,27,20,21,30,31,24,25]
3935 ; AVX512-NEXT: vpshufb %ymm14, %ymm7, %ymm7
3936 ; AVX512-NEXT: vpsrlq $48, %xmm1, %xmm15
3937 ; AVX512-NEXT: vpshufd {{.*#+}} xmm13 = xmm6[0,3,2,3]
3938 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[1,2,2,3,4,5,6,7]
3939 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1]
3940 ; AVX512-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2],xmm7[3,4,5,6,7]
3941 ; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm13[0,1,2,3],ymm7[4,5,6,7]
3942 ; AVX512-NEXT: vmovdqa %ymm0, %ymm2
3943 ; AVX512-NEXT: vmovdqa64 %ymm17, %ymm1
3944 ; AVX512-NEXT: vpblendw {{.*#+}} ymm13 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
3945 ; AVX512-NEXT: vextracti128 $1, %ymm13, %xmm15
3946 ; AVX512-NEXT: vpblendd {{.*#+}} xmm13 = xmm15[0],xmm13[1],xmm15[2],xmm13[3]
3947 ; AVX512-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u,6,7,0,1,10,11,4,5,14,15,8,9]
3948 ; AVX512-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
3949 ; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4],ymm13[5,6,7]
3950 ; AVX512-NEXT: vpblendw {{.*#+}} ymm13 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
3951 ; AVX512-NEXT: vpermq {{.*#+}} ymm15 = ymm13[2,3,0,1]
3952 ; AVX512-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3,4],ymm15[5],ymm13[6,7]
3953 ; AVX512-NEXT: vpblendw {{.*#+}} ymm15 = ymm10[0],ymm8[1],ymm10[2],ymm8[3],ymm10[4,5],ymm8[6],ymm10[7,8],ymm8[9],ymm10[10],ymm8[11],ymm10[12,13],ymm8[14],ymm10[15]
3954 ; AVX512-NEXT: vextracti128 $1, %ymm15, %xmm0
3955 ; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm15[0,1],xmm0[2,3],xmm15[4,5,6],xmm0[7]
3956 ; AVX512-NEXT: vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[8,9,18,19,28,29,22,23,16,17,26,27],zero,zero,zero,zero,zero,zero
3957 ; AVX512-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[2,3,12,13,6,7,0,1,10,11,4,5,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[u,u,u,u,u,u]
3958 ; AVX512-NEXT: vpor %ymm0, %ymm13, %ymm0
3959 ; AVX512-NEXT: vpblendd {{.*#+}} xmm13 = xmm12[0,1],xmm11[2],xmm12[3]
3960 ; AVX512-NEXT: vpshufb %xmm14, %xmm13, %xmm13
3961 ; AVX512-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
3962 ; AVX512-NEXT: vpternlogq $184, %zmm0, %zmm18, %zmm13
3963 ; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm13, %zmm19
3964 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0],ymm3[1],ymm9[2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7,8],ymm3[9],ymm9[10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
3965 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm7
3966 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm7[1],ymm0[2,3,4,5,6,7]
3967 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm7 = [4,5,14,15,12,13,2,3,12,13,6,7,0,1,10,11,20,21,30,31,28,29,18,19,28,29,22,23,16,17,26,27]
3968 ; AVX512-NEXT: vpshufb %ymm7, %ymm0, %ymm0
3969 ; AVX512-NEXT: vmovdqa64 %ymm7, %ymm21
3970 ; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[0,1,1,3]
3971 ; AVX512-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,7,6,7]
3972 ; AVX512-NEXT: vmovdqa64 %xmm20, %xmm15
3973 ; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm7[2],xmm20[2],xmm7[3],xmm20[3]
3974 ; AVX512-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm0[3,4,5,6,7]
3975 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
3976 ; AVX512-NEXT: vpblendw {{.*#+}} ymm7 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5],ymm2[6],ymm1[7,8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
3977 ; AVX512-NEXT: vextracti128 $1, %ymm7, %xmm13
3978 ; AVX512-NEXT: vpblendw {{.*#+}} xmm7 = xmm13[0,1,2],xmm7[3,4],xmm13[5,6,7]
3979 ; AVX512-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,8,9,2,3,12,13,6,7,0,1,10,11]
3980 ; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
3981 ; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm0[0,1,2,3,4],ymm7[5,6,7]
3982 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0,1],ymm3[2],ymm9[3],ymm3[4],ymm9[5,6],ymm3[7],ymm9[8,9],ymm3[10],ymm9[11],ymm3[12],ymm9[13,14],ymm3[15]
3983 ; AVX512-NEXT: vmovdqa64 %ymm9, %ymm20
3984 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm13
3985 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm13[2],ymm0[3,4,5,6,7]
3986 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm13 = [6,7,6,7,8,9,4,5,14,15,8,9,2,3,12,13,22,23,22,23,24,25,20,21,30,31,24,25,18,19,28,29]
3987 ; AVX512-NEXT: vpshufb %ymm13, %ymm0, %ymm0
3988 ; AVX512-NEXT: vpblendd {{.*#+}} xmm14 = xmm15[0],xmm6[1],xmm15[2,3]
3989 ; AVX512-NEXT: vmovdqa64 %xmm15, %xmm22
3990 ; AVX512-NEXT: vpshufb {{.*#+}} xmm14 = xmm14[6,7,0,1,10,11,u,u,u,u,u,u,u,u,u,u]
3991 ; AVX512-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0,1,2],xmm0[3,4,5,6,7]
3992 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm0[4,5,6,7]
3993 ; AVX512-NEXT: vpblendd {{.*#+}} xmm14 = xmm11[0,1],xmm12[2],xmm11[3]
3994 ; AVX512-NEXT: vpshufb %xmm13, %xmm14, %xmm13
3995 ; AVX512-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
3996 ; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm13, %zmm0
3997 ; AVX512-NEXT: vpblendw {{.*#+}} ymm13 = ymm8[0],ymm10[1,2],ymm8[3],ymm10[4],ymm8[5],ymm10[6,7],ymm8[8],ymm10[9,10],ymm8[11],ymm10[12],ymm8[13],ymm10[14,15]
3998 ; AVX512-NEXT: vextracti128 $1, %ymm13, %xmm14
3999 ; AVX512-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0],xmm13[1],xmm14[2],xmm13[3]
4000 ; AVX512-NEXT: vpblendw {{.*#+}} ymm14 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
4001 ; AVX512-NEXT: vpermq {{.*#+}} ymm15 = ymm14[2,3,0,1]
4002 ; AVX512-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm15[4],ymm14[5],ymm15[6],ymm14[7]
4003 ; AVX512-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
4004 ; AVX512-NEXT: vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
4005 ; AVX512-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1,2],ymm14[3,4,5,6,7]
4006 ; AVX512-NEXT: vpblendw {{.*#+}} ymm14 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5],ymm1[6],ymm2[7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13],ymm1[14],ymm2[15]
4007 ; AVX512-NEXT: vmovdqa %ymm2, %ymm9
4008 ; AVX512-NEXT: vextracti128 $1, %ymm14, %xmm15
4009 ; AVX512-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0],xmm15[1,2,3],xmm14[4,5],xmm15[6,7]
4010 ; AVX512-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm13
4011 ; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm14[u,u,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
4012 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4013 ; AVX512-NEXT: vextracti64x4 $1, %zmm13, %ymm14
4014 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm14[0],ymm0[1,2,3,4,5,6,7],ymm14[8],ymm0[9,10,11,12,13,14,15]
4015 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm0[4,5,6,7]
4016 ; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm13, %zmm17
4017 ; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm11[3,1,2,3]
4018 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
4019 ; AVX512-NEXT: vpshufd {{.*#+}} xmm13 = xmm12[0,2,2,3]
4020 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[0,1,0,3,4,5,6,7]
4021 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1]
4022 ; AVX512-NEXT: vpblendw {{.*#+}} ymm13 = ymm10[0,1],ymm8[2],ymm10[3],ymm8[4],ymm10[5,6],ymm8[7],ymm10[8,9],ymm8[10],ymm10[11],ymm8[12],ymm10[13,14],ymm8[15]
4023 ; AVX512-NEXT: vextracti128 $1, %ymm13, %xmm14
4024 ; AVX512-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2],xmm14[3,4],xmm13[5,6,7]
4025 ; AVX512-NEXT: vpblendw {{.*#+}} ymm14 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10,11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]
4026 ; AVX512-NEXT: vpermq {{.*#+}} ymm15 = ymm14[2,3,0,1]
4027 ; AVX512-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3,4,5],ymm15[6],ymm14[7]
4028 ; AVX512-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
4029 ; AVX512-NEXT: vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
4030 ; AVX512-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1,2],ymm14[3,4,5,6,7]
4031 ; AVX512-NEXT: vpblendd {{.*#+}} xmm11 = xmm11[0],xmm12[1],xmm11[2,3]
4032 ; AVX512-NEXT: vmovdqa64 %ymm21, %ymm2
4033 ; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm11
4034 ; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
4035 ; AVX512-NEXT: vpternlogq $184, %zmm13, %zmm18, %zmm11
4036 ; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm11, %zmm7
4037 ; AVX512-NEXT: vmovdqa64 %ymm20, %ymm2
4038 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4],ymm3[5],ymm2[6,7],ymm3[8],ymm2[9,10],ymm3[11],ymm2[12],ymm3[13],ymm2[14,15]
4039 ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm3
4040 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3,4,5,6,7]
4041 ; AVX512-NEXT: vmovdqa64 %xmm22, %xmm3
4042 ; AVX512-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm6[2],xmm3[3]
4043 ; AVX512-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[8,9,14,15,4,5,6,7,0,1,10,11,4,5,14,15,24,25,30,31,20,21,22,23,16,17,26,27,20,21,30,31]
4044 ; AVX512-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
4045 ; AVX512-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm2[3,4,5,6,7]
4046 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
4047 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4048 ; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
4049 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm10[0],ymm8[1],ymm10[2,3],ymm8[4],ymm10[5],ymm8[6],ymm10[7,8],ymm8[9],ymm10[10,11],ymm8[12],ymm10[13],ymm8[14],ymm10[15]
4050 ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm3
4051 ; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
4052 ; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0,1],ymm4[2],ymm5[3],ymm4[4],ymm5[5,6],ymm4[7],ymm5[8,9],ymm4[10],ymm5[11],ymm4[12],ymm5[13,14],ymm4[15]
4053 ; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
4054 ; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4],ymm3[5,6],ymm4[7]
4055 ; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
4056 ; AVX512-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,4,5,14,15,24,25,18,19,28,29,22,23,20,21,22,23,20,21,30,31]
4057 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
4058 ; AVX512-NEXT: movb $7, %al
4059 ; AVX512-NEXT: kmovw %eax, %k1
4060 ; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm0 {%k1}
4061 ; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm9[1],ymm1[2],ymm9[3],ymm1[4,5],ymm9[6],ymm1[7,8],ymm9[9],ymm1[10],ymm9[11],ymm1[12,13],ymm9[14],ymm1[15]
4062 ; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2
4063 ; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5,6],xmm2[7]
4064 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm2
4065 ; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
4066 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4067 ; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3,4,5,6,7],ymm2[8],ymm1[9,10,11,12,13,14,15]
4068 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
4069 ; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
4070 ; AVX512-NEXT: vmovdqa64 %zmm16, (%rsi)
4071 ; AVX512-NEXT: vmovdqa64 %zmm19, (%rdx)
4072 ; AVX512-NEXT: vmovdqa64 %zmm7, (%rcx)
4073 ; AVX512-NEXT: vmovdqa64 %zmm17, (%r8)
4074 ; AVX512-NEXT: vmovdqa64 %zmm0, (%r9)
4075 ; AVX512-NEXT: vzeroupper
4076 ; AVX512-NEXT: retq
4077 ;
4078 ; AVX512-FCP-LABEL: load_i16_stride5_vf32:
4079 ; AVX512-FCP: # %bb.0:
4080 ; AVX512-FCP-NEXT: vmovdqa 176(%rdi), %xmm2
4081 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm2[4,5,14,15,4,5,6,7,u,u,u,u,u,u,u,u]
4082 ; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %xmm3
4083 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm3[0,1,10,11,8,9,10,11,u,u,u,u,u,u,u,u]
4084 ; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
4085 ; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm4
4086 ; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm5
4087 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
4088 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [2,4,7,1,4,6,0,0]
4089 ; AVX512-FCP-NEXT: vpermd %ymm1, %ymm6, %ymm1
4090 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,8,9,14,15,0,1,6,7,16,17,22,23,u,u,u,u,u,u,u,u,u,u,u,u]
4091 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [8,9,3,2,4,0,0,0]
4092 ; AVX512-FCP-NEXT: vpermi2d %ymm0, %ymm1, %ymm6
4093 ; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %ymm0
4094 ; AVX512-FCP-NEXT: vmovdqa 288(%rdi), %ymm1
4095 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
4096 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm7, %xmm8
4097 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm8[3,4],xmm7[5,6,7]
4098 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,4,5,14,15,8,9,2,3,12,13,6,7]
4099 ; AVX512-FCP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
4100 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm7[5,6,7]
4101 ; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm10
4102 ; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm11
4103 ; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm8
4104 ; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm9
4105 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm8[0],ymm9[1,2],ymm8[3],ymm9[4],ymm8[5],ymm9[6,7],ymm8[8],ymm9[9,10],ymm8[11],ymm9[12],ymm8[13],ymm9[14,15]
4106 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm12 = [1,0,0,0,4,6,1,3]
4107 ; AVX512-FCP-NEXT: vpermd %ymm7, %ymm12, %ymm7
4108 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[2,3,16,17,22,23,24,25,30,31,20,21],zero,zero,zero,zero,zero,zero
4109 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm10[0],ymm11[1],ymm10[2,3],ymm11[4],ymm10[5],ymm11[6],ymm10[7,8],ymm11[9],ymm10[10,11],ymm11[12],ymm10[13],ymm11[14],ymm10[15]
4110 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm12, %xmm13
4111 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0],xmm13[1,2,3],xmm12[4,5],xmm13[6,7]
4112 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[u,u,u,u,u,u]
4113 ; AVX512-FCP-NEXT: vpor %ymm7, %ymm12, %ymm12
4114 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm13 = [0,3,1,0,0,3,5,0]
4115 ; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm7
4116 ; AVX512-FCP-NEXT: vpermd %ymm7, %ymm13, %ymm13
4117 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm14 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
4118 ; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm13, %ymm13
4119 ; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
4120 ; AVX512-FCP-NEXT: vpternlogq $184, %zmm12, %zmm18, %zmm13
4121 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm13, %zmm16
4122 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm11[0],ymm10[1],ymm11[2],ymm10[3],ymm11[4,5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10],ymm10[11],ymm11[12,13],ymm10[14],ymm11[15]
4123 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm12, %xmm13
4124 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1],xmm13[2,3],xmm12[4,5,6],xmm13[7]
4125 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [2,3,12,13,6,7,0,1,10,11,4,5,14,15,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
4126 ; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm12, %ymm12
4127 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm9[0],ymm8[1],ymm9[2,3],ymm8[4],ymm9[5],ymm8[6],ymm9[7,8],ymm8[9],ymm9[10,11],ymm8[12],ymm9[13],ymm8[14],ymm9[15]
4128 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm17 = [2,0,0,0,4,7,1,6]
4129 ; AVX512-FCP-NEXT: vpermd %ymm13, %ymm17, %ymm13
4130 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[0,1,18,19,20,21,26,27,16,17,30,31],zero,zero,zero,zero,zero,zero
4131 ; AVX512-FCP-NEXT: vpor %ymm13, %ymm12, %ymm12
4132 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm13 = [1,3,2,0,1,3,6,0]
4133 ; AVX512-FCP-NEXT: vpermd %ymm7, %ymm13, %ymm15
4134 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm13 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
4135 ; AVX512-FCP-NEXT: vpshufb %ymm13, %ymm15, %ymm15
4136 ; AVX512-FCP-NEXT: vpternlogq $184, %zmm12, %zmm18, %zmm15
4137 ; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm3, %xmm6
4138 ; AVX512-FCP-NEXT: vpsrlq $48, %xmm2, %xmm12
4139 ; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm12[0],xmm6[1],xmm12[1]
4140 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10,11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]
4141 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm17 = [0,2,5,7,4,7,0,0]
4142 ; AVX512-FCP-NEXT: vpermd %ymm12, %ymm17, %ymm12
4143 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[2,3,4,5,4,5,0,1,6,7,8,9,14,15,4,5,18,19,20,21,20,21,16,17,22,23,24,25,30,31,20,21]
4144 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm12[3,4,5,6,7]
4145 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm12[4,5,6,7]
4146 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
4147 ; AVX512-FCP-NEXT: vbroadcasti32x4 {{.*#+}} ymm17 = [1,4,6,3,1,4,6,3]
4148 ; AVX512-FCP-NEXT: # ymm17 = mem[0,1,2,3,0,1,2,3]
4149 ; AVX512-FCP-NEXT: vpermd %ymm12, %ymm17, %ymm12
4150 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27,16,17,30,31,24,25]
4151 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm12[5,6,7]
4152 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm15, %zmm17
4153 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
4154 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm6, %xmm15
4155 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm15[0,1,2],xmm6[3,4],xmm15[5,6,7]
4156 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,8,9,2,3,12,13,6,7,0,1,10,11]
4157 ; AVX512-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
4158 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm15 = xmm3[u,u,u,u,u,u,u,u,4,5,14,15,u,u,u,u]
4159 ; AVX512-FCP-NEXT: vpunpckhdq {{.*#+}} xmm15 = xmm15[2],xmm2[2],xmm15[3],xmm2[3]
4160 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
4161 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm19 = [0,3,5,2,5,7,0,0]
4162 ; AVX512-FCP-NEXT: vpermd %ymm12, %ymm19, %ymm12
4163 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[0,1,6,7,2,3,2,3,4,5,10,11,0,1,14,15,16,17,22,23,18,19,18,19,20,21,26,27,16,17,30,31]
4164 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm15 = xmm15[0,1,2],xmm12[3,4,5,6,7]
4165 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm15[0,1,2,3],ymm12[4,5,6,7]
4166 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm12[0,1,2,3,4],ymm6[5,6,7]
4167 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm11[0,1],ymm10[2],ymm11[3],ymm10[4],ymm11[5,6],ymm10[7],ymm11[8,9],ymm10[10],ymm11[11],ymm10[12],ymm11[13,14],ymm10[15]
4168 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm12, %xmm15
4169 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm15[3,4],xmm12[5,6,7]
4170 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm12 = xmm12[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
4171 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5],ymm9[6],ymm8[7,8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13],ymm9[14],ymm8[15]
4172 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm19 = [0,2,0,0,5,7,2,4]
4173 ; AVX512-FCP-NEXT: vpermd %ymm15, %ymm19, %ymm15
4174 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
4175 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0,1,2],ymm15[3,4,5,6,7]
4176 ; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [1,4,6,0,1,4,6,0]
4177 ; AVX512-FCP-NEXT: # ymm15 = mem[0,1,0,1]
4178 ; AVX512-FCP-NEXT: vpermd %ymm7, %ymm15, %ymm15
4179 ; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm15, %ymm14
4180 ; AVX512-FCP-NEXT: vpternlogq $184, %zmm12, %zmm18, %zmm14
4181 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm14, %zmm14
4182 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm10[0],ymm11[1,2],ymm10[3],ymm11[4],ymm10[5],ymm11[6,7],ymm10[8],ymm11[9,10],ymm10[11],ymm11[12],ymm10[13],ymm11[14,15]
4183 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm6, %xmm12
4184 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm12[0],xmm6[1],xmm12[2],xmm6[3]
4185 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
4186 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm9[0],ymm8[1],ymm9[2],ymm8[3],ymm9[4,5],ymm8[6],ymm9[7,8],ymm8[9],ymm9[10],ymm8[11],ymm9[12,13],ymm8[14],ymm9[15]
4187 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm15 = [0,3,0,0,5,0,2,7]
4188 ; AVX512-FCP-NEXT: vpermd %ymm12, %ymm15, %ymm12
4189 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,2,3,4,5,18,19,20,21,26,27,16,17,30,31,30,31,18,19,20,21]
4190 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2],ymm12[3,4,5,6,7]
4191 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm2[0],xmm3[1],xmm2[2,3]
4192 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm12 = xmm12[6,7,0,1,10,11,u,u,u,u,u,u,u,u,u,u]
4193 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm5[0,1],ymm4[2],ymm5[3],ymm4[4],ymm5[5,6],ymm4[7],ymm5[8,9],ymm4[10],ymm5[11],ymm4[12],ymm5[13,14],ymm4[15]
4194 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm18 = [1,3,6,0,5,0,0,0]
4195 ; AVX512-FCP-NEXT: vpermd %ymm15, %ymm18, %ymm15
4196 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm15[2,3,2,3,4,5,0,1,6,7,8,9,14,15,4,5,18,19,18,19,20,21,16,17,22,23,24,25,30,31,20,21]
4197 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm15[3,4,5,6,7]
4198 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm15[4,5,6,7]
4199 ; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [2,4,7,0,2,4,7,0]
4200 ; AVX512-FCP-NEXT: # ymm15 = mem[0,1,0,1]
4201 ; AVX512-FCP-NEXT: vpermd %ymm7, %ymm15, %ymm15
4202 ; AVX512-FCP-NEXT: vpshufb %ymm13, %ymm15, %ymm13
4203 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm12, %zmm13, %zmm12
4204 ; AVX512-FCP-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm12
4205 ; AVX512-FCP-NEXT: vextracti64x4 $1, %zmm12, %ymm6
4206 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
4207 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm13, %xmm15
4208 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0],xmm15[1,2,3],xmm13[4,5],xmm15[6,7]
4209 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[u,u,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
4210 ; AVX512-FCP-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
4211 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm6[0],ymm13[1,2,3,4,5,6,7],ymm6[8],ymm13[9,10,11,12,13,14,15]
4212 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm13[4,5,6,7]
4213 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm12, %zmm6
4214 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm11[0],ymm10[1],ymm11[2,3],ymm10[4],ymm11[5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10,11],ymm10[12],ymm11[13],ymm10[14],ymm11[15]
4215 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm10, %xmm11
4216 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3,4],xmm11[5,6,7]
4217 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
4218 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm9[0,1],ymm8[2],ymm9[3],ymm8[4],ymm9[5,6],ymm8[7],ymm9[8,9],ymm8[10],ymm9[11],ymm8[12],ymm9[13,14],ymm8[15]
4219 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [1,3,0,0,6,0,3,5]
4220 ; AVX512-FCP-NEXT: vpermd %ymm8, %ymm9, %ymm8
4221 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,16,17,22,23,16,17,22,23]
4222 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm10[0,1,2],ymm8[3,4,5,6,7]
4223 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3]
4224 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
4225 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm5[1,2],ymm4[3],ymm5[4],ymm4[5],ymm5[6,7],ymm4[8],ymm5[9,10],ymm4[11],ymm5[12],ymm4[13],ymm5[14,15]
4226 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [1,4,6,3,6,0,0,0]
4227 ; AVX512-FCP-NEXT: vpermd %ymm3, %ymm4, %ymm3
4228 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,2,3,4,5,10,11,0,1,14,15,16,17,18,19,20,21,18,19,20,21,26,27,16,17,30,31]
4229 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[3,4,5,6,7]
4230 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
4231 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,2,1,3,0,2,5,7]
4232 ; AVX512-FCP-NEXT: vpermd %ymm7, %ymm3, %ymm3
4233 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[0,1,6,7,4,5,6,7,0,1,6,7,8,9,14,15,16,17,22,23,20,21,22,23,16,17,22,23,24,25,30,31]
4234 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2
4235 ; AVX512-FCP-NEXT: movb $7, %al
4236 ; AVX512-FCP-NEXT: kmovw %eax, %k1
4237 ; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm2 {%k1}
4238 ; AVX512-FCP-NEXT: vextracti64x4 $1, %zmm2, %ymm3
4239 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
4240 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1
4241 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6],xmm1[7]
4242 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
4243 ; AVX512-FCP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4244 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0],ymm0[1,2,3,4,5,6,7],ymm3[8],ymm0[9,10,11,12,13,14,15]
4245 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
4246 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
4247 ; AVX512-FCP-NEXT: vmovdqa64 %zmm16, (%rsi)
4248 ; AVX512-FCP-NEXT: vmovdqa64 %zmm17, (%rdx)
4249 ; AVX512-FCP-NEXT: vmovdqa64 %zmm14, (%rcx)
4250 ; AVX512-FCP-NEXT: vmovdqa64 %zmm6, (%r8)
4251 ; AVX512-FCP-NEXT: vmovdqa64 %zmm0, (%r9)
4252 ; AVX512-FCP-NEXT: vzeroupper
4253 ; AVX512-FCP-NEXT: retq
4254 ;
4255 ; AVX512DQ-LABEL: load_i16_stride5_vf32:
4256 ; AVX512DQ: # %bb.0:
4257 ; AVX512DQ-NEXT: vmovdqa 256(%rdi), %ymm0
4258 ; AVX512DQ-NEXT: vmovdqa 288(%rdi), %ymm1
4259 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
4260 ; AVX512DQ-NEXT: vmovdqa64 %ymm1, %ymm17
4261 ; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
4262 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[3,4],xmm2[5,6,7]
4263 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,4,5,14,15,8,9,2,3,12,13,6,7]
4264 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm4
4265 ; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm3
4266 ; AVX512DQ-NEXT: vmovdqa 224(%rdi), %ymm9
4267 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm5 = ymm9[0],ymm3[1],ymm9[2,3],ymm3[4],ymm9[5],ymm3[6],ymm9[7,8],ymm3[9],ymm9[10,11],ymm3[12],ymm9[13],ymm3[14],ymm9[15]
4268 ; AVX512DQ-NEXT: vextracti128 $1, %ymm5, %xmm6
4269 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1,2],ymm6[3],ymm5[4,5,6,7]
4270 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm7 = [u,u,u,u,u,u,u,u,8,9,2,3,12,13,6,7,16,17,26,27,26,27,30,31,24,25,18,19,28,29,22,23]
4271 ; AVX512DQ-NEXT: vpshufb %ymm7, %ymm5, %ymm5
4272 ; AVX512DQ-NEXT: vmovdqa 176(%rdi), %xmm1
4273 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm1[3,1,2,3]
4274 ; AVX512DQ-NEXT: vmovdqa64 %xmm1, %xmm20
4275 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[2,1,2,3,4,5,6,7]
4276 ; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm6
4277 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[0,2,2,3]
4278 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,3,2,3,4,5,6,7]
4279 ; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm10[0],xmm8[0],xmm10[1],xmm8[1]
4280 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1],ymm5[2,3,4,5,6,7]
4281 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm14 = ymm5[0,1,2,3,4],ymm4[5,6,7]
4282 ; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm8
4283 ; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm10
4284 ; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm4
4285 ; AVX512DQ-NEXT: vmovdqa 96(%rdi), %ymm5
4286 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm11 = ymm4[0],ymm5[1,2],ymm4[3],ymm5[4],ymm4[5],ymm5[6,7],ymm4[8],ymm5[9,10],ymm4[11],ymm5[12],ymm4[13],ymm5[14,15]
4287 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm12 = ymm11[2,3,0,1]
4288 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm12[5],ymm11[6],ymm12[7]
4289 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[6,7,16,17,26,27,20,21,30,31,24,25],zero,zero,zero,zero,zero,zero
4290 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm12 = ymm8[0],ymm10[1],ymm8[2,3],ymm10[4],ymm8[5],ymm10[6],ymm8[7,8],ymm10[9],ymm8[10,11],ymm10[12],ymm8[13],ymm10[14],ymm8[15]
4291 ; AVX512DQ-NEXT: vextracti128 $1, %ymm12, %xmm13
4292 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0],xmm13[1,2,3],xmm12[4,5],xmm13[6,7]
4293 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[u,u,u,u,u,u]
4294 ; AVX512DQ-NEXT: vpor %ymm11, %ymm12, %ymm15
4295 ; AVX512DQ-NEXT: vmovdqa 144(%rdi), %xmm11
4296 ; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm12
4297 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm13 = xmm12[0],xmm11[1],xmm12[2,3]
4298 ; AVX512DQ-NEXT: vpshufb %xmm7, %xmm13, %xmm7
4299 ; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
4300 ; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
4301 ; AVX512DQ-NEXT: vpternlogq $184, %zmm15, %zmm18, %zmm7
4302 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm14, %zmm7, %zmm16
4303 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm7 = ymm3[0],ymm9[1],ymm3[2,3],ymm9[4],ymm3[5],ymm9[6],ymm3[7,8],ymm9[9],ymm3[10,11],ymm9[12],ymm3[13],ymm9[14],ymm3[15]
4304 ; AVX512DQ-NEXT: vextracti128 $1, %ymm7, %xmm14
4305 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0],ymm14[1],ymm7[2],ymm14[3],ymm7[4,5,6,7]
4306 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm14 = [2,3,12,13,0,1,0,1,10,11,4,5,14,15,8,9,18,19,28,29,16,17,16,17,26,27,20,21,30,31,24,25]
4307 ; AVX512DQ-NEXT: vpshufb %ymm14, %ymm7, %ymm7
4308 ; AVX512DQ-NEXT: vpsrlq $48, %xmm1, %xmm15
4309 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm13 = xmm6[0,3,2,3]
4310 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[1,2,2,3,4,5,6,7]
4311 ; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1]
4312 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2],xmm7[3,4,5,6,7]
4313 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm13[0,1,2,3],ymm7[4,5,6,7]
4314 ; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm2
4315 ; AVX512DQ-NEXT: vmovdqa64 %ymm17, %ymm1
4316 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm13 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
4317 ; AVX512DQ-NEXT: vextracti128 $1, %ymm13, %xmm15
4318 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm13 = xmm15[0],xmm13[1],xmm15[2],xmm13[3]
4319 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u,6,7,0,1,10,11,4,5,14,15,8,9]
4320 ; AVX512DQ-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
4321 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4],ymm13[5,6,7]
4322 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm13 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
4323 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm15 = ymm13[2,3,0,1]
4324 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3,4],ymm15[5],ymm13[6,7]
4325 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm15 = ymm10[0],ymm8[1],ymm10[2],ymm8[3],ymm10[4,5],ymm8[6],ymm10[7,8],ymm8[9],ymm10[10],ymm8[11],ymm10[12,13],ymm8[14],ymm10[15]
4326 ; AVX512DQ-NEXT: vextracti128 $1, %ymm15, %xmm0
4327 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm15[0,1],xmm0[2,3],xmm15[4,5,6],xmm0[7]
4328 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[8,9,18,19,28,29,22,23,16,17,26,27],zero,zero,zero,zero,zero,zero
4329 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[2,3,12,13,6,7,0,1,10,11,4,5,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[u,u,u,u,u,u]
4330 ; AVX512DQ-NEXT: vpor %ymm0, %ymm13, %ymm0
4331 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm13 = xmm12[0,1],xmm11[2],xmm12[3]
4332 ; AVX512DQ-NEXT: vpshufb %xmm14, %xmm13, %xmm13
4333 ; AVX512DQ-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
4334 ; AVX512DQ-NEXT: vpternlogq $184, %zmm0, %zmm18, %zmm13
4335 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm7, %zmm13, %zmm19
4336 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0],ymm3[1],ymm9[2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7,8],ymm3[9],ymm9[10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
4337 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm7
4338 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm7[1],ymm0[2,3,4,5,6,7]
4339 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm7 = [4,5,14,15,12,13,2,3,12,13,6,7,0,1,10,11,20,21,30,31,28,29,18,19,28,29,22,23,16,17,26,27]
4340 ; AVX512DQ-NEXT: vpshufb %ymm7, %ymm0, %ymm0
4341 ; AVX512DQ-NEXT: vmovdqa64 %ymm7, %ymm21
4342 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[0,1,1,3]
4343 ; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,7,6,7]
4344 ; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm15
4345 ; AVX512DQ-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm7[2],xmm20[2],xmm7[3],xmm20[3]
4346 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm0[3,4,5,6,7]
4347 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
4348 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm7 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5],ymm2[6],ymm1[7,8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
4349 ; AVX512DQ-NEXT: vextracti128 $1, %ymm7, %xmm13
4350 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm7 = xmm13[0,1,2],xmm7[3,4],xmm13[5,6,7]
4351 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,8,9,2,3,12,13,6,7,0,1,10,11]
4352 ; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
4353 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm0[0,1,2,3,4],ymm7[5,6,7]
4354 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0,1],ymm3[2],ymm9[3],ymm3[4],ymm9[5,6],ymm3[7],ymm9[8,9],ymm3[10],ymm9[11],ymm3[12],ymm9[13,14],ymm3[15]
4355 ; AVX512DQ-NEXT: vmovdqa64 %ymm9, %ymm20
4356 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm13
4357 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm13[2],ymm0[3,4,5,6,7]
4358 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm13 = [6,7,6,7,8,9,4,5,14,15,8,9,2,3,12,13,22,23,22,23,24,25,20,21,30,31,24,25,18,19,28,29]
4359 ; AVX512DQ-NEXT: vpshufb %ymm13, %ymm0, %ymm0
4360 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm14 = xmm15[0],xmm6[1],xmm15[2,3]
4361 ; AVX512DQ-NEXT: vmovdqa64 %xmm15, %xmm22
4362 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm14 = xmm14[6,7,0,1,10,11,u,u,u,u,u,u,u,u,u,u]
4363 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0,1,2],xmm0[3,4,5,6,7]
4364 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm0[4,5,6,7]
4365 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm14 = xmm11[0,1],xmm12[2],xmm11[3]
4366 ; AVX512DQ-NEXT: vpshufb %xmm13, %xmm14, %xmm13
4367 ; AVX512DQ-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
4368 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm13, %zmm0
4369 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm13 = ymm8[0],ymm10[1,2],ymm8[3],ymm10[4],ymm8[5],ymm10[6,7],ymm8[8],ymm10[9,10],ymm8[11],ymm10[12],ymm8[13],ymm10[14,15]
4370 ; AVX512DQ-NEXT: vextracti128 $1, %ymm13, %xmm14
4371 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0],xmm13[1],xmm14[2],xmm13[3]
4372 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm14 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
4373 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm15 = ymm14[2,3,0,1]
4374 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm15[4],ymm14[5],ymm15[6],ymm14[7]
4375 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
4376 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
4377 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1,2],ymm14[3,4,5,6,7]
4378 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm14 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5],ymm1[6],ymm2[7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13],ymm1[14],ymm2[15]
4379 ; AVX512DQ-NEXT: vmovdqa %ymm2, %ymm9
4380 ; AVX512DQ-NEXT: vextracti128 $1, %ymm14, %xmm15
4381 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0],xmm15[1,2,3],xmm14[4,5],xmm15[6,7]
4382 ; AVX512DQ-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm13
4383 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm14[u,u,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
4384 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4385 ; AVX512DQ-NEXT: vextracti64x4 $1, %zmm13, %ymm14
4386 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm14[0],ymm0[1,2,3,4,5,6,7],ymm14[8],ymm0[9,10,11,12,13,14,15]
4387 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm0[4,5,6,7]
4388 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm13, %zmm17
4389 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm11[3,1,2,3]
4390 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
4391 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm13 = xmm12[0,2,2,3]
4392 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[0,1,0,3,4,5,6,7]
4393 ; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1]
4394 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm13 = ymm10[0,1],ymm8[2],ymm10[3],ymm8[4],ymm10[5,6],ymm8[7],ymm10[8,9],ymm8[10],ymm10[11],ymm8[12],ymm10[13,14],ymm8[15]
4395 ; AVX512DQ-NEXT: vextracti128 $1, %ymm13, %xmm14
4396 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2],xmm14[3,4],xmm13[5,6,7]
4397 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm14 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10,11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]
4398 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm15 = ymm14[2,3,0,1]
4399 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3,4,5],ymm15[6],ymm14[7]
4400 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
4401 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
4402 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1,2],ymm14[3,4,5,6,7]
4403 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm11 = xmm11[0],xmm12[1],xmm11[2,3]
4404 ; AVX512DQ-NEXT: vmovdqa64 %ymm21, %ymm2
4405 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm11
4406 ; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
4407 ; AVX512DQ-NEXT: vpternlogq $184, %zmm13, %zmm18, %zmm11
4408 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm7, %zmm11, %zmm7
4409 ; AVX512DQ-NEXT: vmovdqa64 %ymm20, %ymm2
4410 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4],ymm3[5],ymm2[6,7],ymm3[8],ymm2[9,10],ymm3[11],ymm2[12],ymm3[13],ymm2[14,15]
4411 ; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
4412 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3,4,5,6,7]
4413 ; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm3
4414 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm6[2],xmm3[3]
4415 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[8,9,14,15,4,5,6,7,0,1,10,11,4,5,14,15,24,25,30,31,20,21,22,23,16,17,26,27,20,21,30,31]
4416 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
4417 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm2[3,4,5,6,7]
4418 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
4419 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4420 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
4421 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm10[0],ymm8[1],ymm10[2,3],ymm8[4],ymm10[5],ymm8[6],ymm10[7,8],ymm8[9],ymm10[10,11],ymm8[12],ymm10[13],ymm8[14],ymm10[15]
4422 ; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
4423 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
4424 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0,1],ymm4[2],ymm5[3],ymm4[4],ymm5[5,6],ymm4[7],ymm5[8,9],ymm4[10],ymm5[11],ymm4[12],ymm5[13,14],ymm4[15]
4425 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
4426 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4],ymm3[5,6],ymm4[7]
4427 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
4428 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,4,5,14,15,24,25,18,19,28,29,22,23,20,21,22,23,20,21,30,31]
4429 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
4430 ; AVX512DQ-NEXT: movb $7, %al
4431 ; AVX512DQ-NEXT: kmovw %eax, %k1
4432 ; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm0 {%k1}
4433 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm9[1],ymm1[2],ymm9[3],ymm1[4,5],ymm9[6],ymm1[7,8],ymm9[9],ymm1[10],ymm9[11],ymm1[12,13],ymm9[14],ymm1[15]
4434 ; AVX512DQ-NEXT: vextracti128 $1, %ymm1, %xmm2
4435 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5,6],xmm2[7]
4436 ; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm2
4437 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
4438 ; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
4439 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3,4,5,6,7],ymm2[8],ymm1[9,10,11,12,13,14,15]
4440 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
4441 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
4442 ; AVX512DQ-NEXT: vmovdqa64 %zmm16, (%rsi)
4443 ; AVX512DQ-NEXT: vmovdqa64 %zmm19, (%rdx)
4444 ; AVX512DQ-NEXT: vmovdqa64 %zmm7, (%rcx)
4445 ; AVX512DQ-NEXT: vmovdqa64 %zmm17, (%r8)
4446 ; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%r9)
4447 ; AVX512DQ-NEXT: vzeroupper
4448 ; AVX512DQ-NEXT: retq
4449 ;
4450 ; AVX512DQ-FCP-LABEL: load_i16_stride5_vf32:
4451 ; AVX512DQ-FCP: # %bb.0:
4452 ; AVX512DQ-FCP-NEXT: vmovdqa 176(%rdi), %xmm2
4453 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm2[4,5,14,15,4,5,6,7,u,u,u,u,u,u,u,u]
4454 ; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %xmm3
4455 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm3[0,1,10,11,8,9,10,11,u,u,u,u,u,u,u,u]
4456 ; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
4457 ; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm4
4458 ; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm5
4459 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
4460 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [2,4,7,1,4,6,0,0]
4461 ; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm6, %ymm1
4462 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,8,9,14,15,0,1,6,7,16,17,22,23,u,u,u,u,u,u,u,u,u,u,u,u]
4463 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [8,9,3,2,4,0,0,0]
4464 ; AVX512DQ-FCP-NEXT: vpermi2d %ymm0, %ymm1, %ymm6
4465 ; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %ymm0
4466 ; AVX512DQ-FCP-NEXT: vmovdqa 288(%rdi), %ymm1
4467 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
4468 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm7, %xmm8
4469 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm8[3,4],xmm7[5,6,7]
4470 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,4,5,14,15,8,9,2,3,12,13,6,7]
4471 ; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
4472 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm7[5,6,7]
4473 ; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm10
4474 ; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm11
4475 ; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm8
4476 ; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm9
4477 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm8[0],ymm9[1,2],ymm8[3],ymm9[4],ymm8[5],ymm9[6,7],ymm8[8],ymm9[9,10],ymm8[11],ymm9[12],ymm8[13],ymm9[14,15]
4478 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm12 = [1,0,0,0,4,6,1,3]
4479 ; AVX512DQ-FCP-NEXT: vpermd %ymm7, %ymm12, %ymm7
4480 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[2,3,16,17,22,23,24,25,30,31,20,21],zero,zero,zero,zero,zero,zero
4481 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm10[0],ymm11[1],ymm10[2,3],ymm11[4],ymm10[5],ymm11[6],ymm10[7,8],ymm11[9],ymm10[10,11],ymm11[12],ymm10[13],ymm11[14],ymm10[15]
4482 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm12, %xmm13
4483 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0],xmm13[1,2,3],xmm12[4,5],xmm13[6,7]
4484 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[u,u,u,u,u,u]
4485 ; AVX512DQ-FCP-NEXT: vpor %ymm7, %ymm12, %ymm12
4486 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm13 = [0,3,1,0,0,3,5,0]
4487 ; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm7
4488 ; AVX512DQ-FCP-NEXT: vpermd %ymm7, %ymm13, %ymm13
4489 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm14 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
4490 ; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm13, %ymm13
4491 ; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
4492 ; AVX512DQ-FCP-NEXT: vpternlogq $184, %zmm12, %zmm18, %zmm13
4493 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm13, %zmm16
4494 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm11[0],ymm10[1],ymm11[2],ymm10[3],ymm11[4,5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10],ymm10[11],ymm11[12,13],ymm10[14],ymm11[15]
4495 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm12, %xmm13
4496 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1],xmm13[2,3],xmm12[4,5,6],xmm13[7]
4497 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [2,3,12,13,6,7,0,1,10,11,4,5,14,15,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
4498 ; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm12, %ymm12
4499 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm9[0],ymm8[1],ymm9[2,3],ymm8[4],ymm9[5],ymm8[6],ymm9[7,8],ymm8[9],ymm9[10,11],ymm8[12],ymm9[13],ymm8[14],ymm9[15]
4500 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm17 = [2,0,0,0,4,7,1,6]
4501 ; AVX512DQ-FCP-NEXT: vpermd %ymm13, %ymm17, %ymm13
4502 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[0,1,18,19,20,21,26,27,16,17,30,31],zero,zero,zero,zero,zero,zero
4503 ; AVX512DQ-FCP-NEXT: vpor %ymm13, %ymm12, %ymm12
4504 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm13 = [1,3,2,0,1,3,6,0]
4505 ; AVX512DQ-FCP-NEXT: vpermd %ymm7, %ymm13, %ymm15
4506 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm13 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
4507 ; AVX512DQ-FCP-NEXT: vpshufb %ymm13, %ymm15, %ymm15
4508 ; AVX512DQ-FCP-NEXT: vpternlogq $184, %zmm12, %zmm18, %zmm15
4509 ; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm3, %xmm6
4510 ; AVX512DQ-FCP-NEXT: vpsrlq $48, %xmm2, %xmm12
4511 ; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm12[0],xmm6[1],xmm12[1]
4512 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10,11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]
4513 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm17 = [0,2,5,7,4,7,0,0]
4514 ; AVX512DQ-FCP-NEXT: vpermd %ymm12, %ymm17, %ymm12
4515 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[2,3,4,5,4,5,0,1,6,7,8,9,14,15,4,5,18,19,20,21,20,21,16,17,22,23,24,25,30,31,20,21]
4516 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm12[3,4,5,6,7]
4517 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm12[4,5,6,7]
4518 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
4519 ; AVX512DQ-FCP-NEXT: vbroadcasti32x4 {{.*#+}} ymm17 = [1,4,6,3,1,4,6,3]
4520 ; AVX512DQ-FCP-NEXT: # ymm17 = mem[0,1,2,3,0,1,2,3]
4521 ; AVX512DQ-FCP-NEXT: vpermd %ymm12, %ymm17, %ymm12
4522 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27,16,17,30,31,24,25]
4523 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm12[5,6,7]
4524 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm15, %zmm17
4525 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
4526 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm6, %xmm15
4527 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm15[0,1,2],xmm6[3,4],xmm15[5,6,7]
4528 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,8,9,2,3,12,13,6,7,0,1,10,11]
4529 ; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
4530 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm15 = xmm3[u,u,u,u,u,u,u,u,4,5,14,15,u,u,u,u]
4531 ; AVX512DQ-FCP-NEXT: vpunpckhdq {{.*#+}} xmm15 = xmm15[2],xmm2[2],xmm15[3],xmm2[3]
4532 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
4533 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm19 = [0,3,5,2,5,7,0,0]
4534 ; AVX512DQ-FCP-NEXT: vpermd %ymm12, %ymm19, %ymm12
4535 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[0,1,6,7,2,3,2,3,4,5,10,11,0,1,14,15,16,17,22,23,18,19,18,19,20,21,26,27,16,17,30,31]
4536 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm15 = xmm15[0,1,2],xmm12[3,4,5,6,7]
4537 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm15[0,1,2,3],ymm12[4,5,6,7]
4538 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm12[0,1,2,3,4],ymm6[5,6,7]
4539 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm11[0,1],ymm10[2],ymm11[3],ymm10[4],ymm11[5,6],ymm10[7],ymm11[8,9],ymm10[10],ymm11[11],ymm10[12],ymm11[13,14],ymm10[15]
4540 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm12, %xmm15
4541 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm15[3,4],xmm12[5,6,7]
4542 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm12 = xmm12[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
4543 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5],ymm9[6],ymm8[7,8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13],ymm9[14],ymm8[15]
4544 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm19 = [0,2,0,0,5,7,2,4]
4545 ; AVX512DQ-FCP-NEXT: vpermd %ymm15, %ymm19, %ymm15
4546 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
4547 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0,1,2],ymm15[3,4,5,6,7]
4548 ; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [1,4,6,0,1,4,6,0]
4549 ; AVX512DQ-FCP-NEXT: # ymm15 = mem[0,1,0,1]
4550 ; AVX512DQ-FCP-NEXT: vpermd %ymm7, %ymm15, %ymm15
4551 ; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm15, %ymm14
4552 ; AVX512DQ-FCP-NEXT: vpternlogq $184, %zmm12, %zmm18, %zmm14
4553 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm14, %zmm14
4554 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm10[0],ymm11[1,2],ymm10[3],ymm11[4],ymm10[5],ymm11[6,7],ymm10[8],ymm11[9,10],ymm10[11],ymm11[12],ymm10[13],ymm11[14,15]
4555 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm6, %xmm12
4556 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm12[0],xmm6[1],xmm12[2],xmm6[3]
4557 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
4558 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm9[0],ymm8[1],ymm9[2],ymm8[3],ymm9[4,5],ymm8[6],ymm9[7,8],ymm8[9],ymm9[10],ymm8[11],ymm9[12,13],ymm8[14],ymm9[15]
4559 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm15 = [0,3,0,0,5,0,2,7]
4560 ; AVX512DQ-FCP-NEXT: vpermd %ymm12, %ymm15, %ymm12
4561 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,2,3,4,5,18,19,20,21,26,27,16,17,30,31,30,31,18,19,20,21]
4562 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2],ymm12[3,4,5,6,7]
4563 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm2[0],xmm3[1],xmm2[2,3]
4564 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm12 = xmm12[6,7,0,1,10,11,u,u,u,u,u,u,u,u,u,u]
4565 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm5[0,1],ymm4[2],ymm5[3],ymm4[4],ymm5[5,6],ymm4[7],ymm5[8,9],ymm4[10],ymm5[11],ymm4[12],ymm5[13,14],ymm4[15]
4566 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm18 = [1,3,6,0,5,0,0,0]
4567 ; AVX512DQ-FCP-NEXT: vpermd %ymm15, %ymm18, %ymm15
4568 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm15[2,3,2,3,4,5,0,1,6,7,8,9,14,15,4,5,18,19,18,19,20,21,16,17,22,23,24,25,30,31,20,21]
4569 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm15[3,4,5,6,7]
4570 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm15[4,5,6,7]
4571 ; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [2,4,7,0,2,4,7,0]
4572 ; AVX512DQ-FCP-NEXT: # ymm15 = mem[0,1,0,1]
4573 ; AVX512DQ-FCP-NEXT: vpermd %ymm7, %ymm15, %ymm15
4574 ; AVX512DQ-FCP-NEXT: vpshufb %ymm13, %ymm15, %ymm13
4575 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm12, %zmm13, %zmm12
4576 ; AVX512DQ-FCP-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm12
4577 ; AVX512DQ-FCP-NEXT: vextracti64x4 $1, %zmm12, %ymm6
4578 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
4579 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm13, %xmm15
4580 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0],xmm15[1,2,3],xmm13[4,5],xmm15[6,7]
4581 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[u,u,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
4582 ; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
4583 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm6[0],ymm13[1,2,3,4,5,6,7],ymm6[8],ymm13[9,10,11,12,13,14,15]
4584 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm13[4,5,6,7]
4585 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm12, %zmm6
4586 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm11[0],ymm10[1],ymm11[2,3],ymm10[4],ymm11[5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10,11],ymm10[12],ymm11[13],ymm10[14],ymm11[15]
4587 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm10, %xmm11
4588 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3,4],xmm11[5,6,7]
4589 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
4590 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm9[0,1],ymm8[2],ymm9[3],ymm8[4],ymm9[5,6],ymm8[7],ymm9[8,9],ymm8[10],ymm9[11],ymm8[12],ymm9[13,14],ymm8[15]
4591 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [1,3,0,0,6,0,3,5]
4592 ; AVX512DQ-FCP-NEXT: vpermd %ymm8, %ymm9, %ymm8
4593 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,16,17,22,23,16,17,22,23]
4594 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm10[0,1,2],ymm8[3,4,5,6,7]
4595 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3]
4596 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
4597 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm5[1,2],ymm4[3],ymm5[4],ymm4[5],ymm5[6,7],ymm4[8],ymm5[9,10],ymm4[11],ymm5[12],ymm4[13],ymm5[14,15]
4598 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [1,4,6,3,6,0,0,0]
4599 ; AVX512DQ-FCP-NEXT: vpermd %ymm3, %ymm4, %ymm3
4600 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,2,3,4,5,10,11,0,1,14,15,16,17,18,19,20,21,18,19,20,21,26,27,16,17,30,31]
4601 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[3,4,5,6,7]
4602 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
4603 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,2,1,3,0,2,5,7]
4604 ; AVX512DQ-FCP-NEXT: vpermd %ymm7, %ymm3, %ymm3
4605 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[0,1,6,7,4,5,6,7,0,1,6,7,8,9,14,15,16,17,22,23,20,21,22,23,16,17,22,23,24,25,30,31]
4606 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2
4607 ; AVX512DQ-FCP-NEXT: movb $7, %al
4608 ; AVX512DQ-FCP-NEXT: kmovw %eax, %k1
4609 ; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm2 {%k1}
4610 ; AVX512DQ-FCP-NEXT: vextracti64x4 $1, %zmm2, %ymm3
4611 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
4612 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1
4613 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6],xmm1[7]
4614 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
4615 ; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
4616 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0],ymm0[1,2,3,4,5,6,7],ymm3[8],ymm0[9,10,11,12,13,14,15]
4617 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
4618 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
4619 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm16, (%rsi)
4620 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm17, (%rdx)
4621 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm14, (%rcx)
4622 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, (%r8)
4623 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, (%r9)
4624 ; AVX512DQ-FCP-NEXT: vzeroupper
4625 ; AVX512DQ-FCP-NEXT: retq
4626 ;
4627 ; AVX512BW-LABEL: load_i16_stride5_vf32:
4628 ; AVX512BW: # %bb.0:
4629 ; AVX512BW-NEXT: vmovdqa64 256(%rdi), %zmm0
4630 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm1
4631 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm2
4632 ; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm3
4633 ; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm4
4634 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11]
4635 ; AVX512BW-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]
4636 ; AVX512BW-NEXT: vpermi2w %zmm4, %zmm3, %zmm5
4637 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
4638 ; AVX512BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm6
4639 ; AVX512BW-NEXT: movl $67100672, %eax # imm = 0x3FFE000
4640 ; AVX512BW-NEXT: kmovd %eax, %k1
4641 ; AVX512BW-NEXT: vmovdqu16 %zmm5, %zmm6 {%k1}
4642 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,34,39,44,49,54,59]
4643 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm6, %zmm5
4644 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm6 = [49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44]
4645 ; AVX512BW-NEXT: # zmm6 = mem[0,1,2,3,0,1,2,3]
4646 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm4, %zmm6
4647 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm7 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
4648 ; AVX512BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm7
4649 ; AVX512BW-NEXT: vmovdqu16 %zmm6, %zmm7 {%k1}
4650 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,35,40,45,50,55,60]
4651 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm7, %zmm6
4652 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45]
4653 ; AVX512BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]
4654 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm4, %zmm7
4655 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm8 = [34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0]
4656 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm2, %zmm8
4657 ; AVX512BW-NEXT: vmovdqu16 %zmm7, %zmm8 {%k1}
4658 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,36,41,46,51,56,61]
4659 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm8, %zmm7
4660 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14]
4661 ; AVX512BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]
4662 ; AVX512BW-NEXT: vpermi2w %zmm4, %zmm3, %zmm8
4663 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm9 = [35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0]
4664 ; AVX512BW-NEXT: vpermi2w %zmm1, %zmm2, %zmm9
4665 ; AVX512BW-NEXT: movl $33546240, %eax # imm = 0x1FFE000
4666 ; AVX512BW-NEXT: kmovd %eax, %k1
4667 ; AVX512BW-NEXT: vmovdqu16 %zmm8, %zmm9 {%k1}
4668 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,32,37,42,47,52,57,62]
4669 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm9, %zmm8
4670 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm9 = [4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0]
4671 ; AVX512BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm9
4672 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm1 = [20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15]
4673 ; AVX512BW-NEXT: # zmm1 = mem[0,1,2,3,0,1,2,3]
4674 ; AVX512BW-NEXT: vpermi2w %zmm4, %zmm3, %zmm1
4675 ; AVX512BW-NEXT: movb $7, %al
4676 ; AVX512BW-NEXT: kmovd %eax, %k1
4677 ; AVX512BW-NEXT: vmovdqa64 %zmm9, %zmm1 {%k1}
4678 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,33,38,43,48,53,58,63]
4679 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm2
4680 ; AVX512BW-NEXT: vmovdqa64 %zmm5, (%rsi)
4681 ; AVX512BW-NEXT: vmovdqa64 %zmm6, (%rdx)
4682 ; AVX512BW-NEXT: vmovdqa64 %zmm7, (%rcx)
4683 ; AVX512BW-NEXT: vmovdqa64 %zmm8, (%r8)
4684 ; AVX512BW-NEXT: vmovdqa64 %zmm2, (%r9)
4685 ; AVX512BW-NEXT: vzeroupper
4686 ; AVX512BW-NEXT: retq
4687 ;
4688 ; AVX512BW-FCP-LABEL: load_i16_stride5_vf32:
4689 ; AVX512BW-FCP: # %bb.0:
4690 ; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdi), %zmm0
4691 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm1
4692 ; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm2
4693 ; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm3
4694 ; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm4
4695 ; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11]
4696 ; AVX512BW-FCP-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]
4697 ; AVX512BW-FCP-NEXT: vpermi2w %zmm4, %zmm3, %zmm5
4698 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
4699 ; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm6
4700 ; AVX512BW-FCP-NEXT: movl $67100672, %eax # imm = 0x3FFE000
4701 ; AVX512BW-FCP-NEXT: kmovd %eax, %k1
4702 ; AVX512BW-FCP-NEXT: vmovdqu16 %zmm5, %zmm6 {%k1}
4703 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,34,39,44,49,54,59]
4704 ; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm6, %zmm5
4705 ; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm6 = [49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44]
4706 ; AVX512BW-FCP-NEXT: # zmm6 = mem[0,1,2,3,0,1,2,3]
4707 ; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm4, %zmm6
4708 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm7 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
4709 ; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm7
4710 ; AVX512BW-FCP-NEXT: vmovdqu16 %zmm6, %zmm7 {%k1}
4711 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,35,40,45,50,55,60]
4712 ; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm7, %zmm6
4713 ; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45]
4714 ; AVX512BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]
4715 ; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm4, %zmm7
4716 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm8 = [34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0]
4717 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm2, %zmm8
4718 ; AVX512BW-FCP-NEXT: vmovdqu16 %zmm7, %zmm8 {%k1}
4719 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,36,41,46,51,56,61]
4720 ; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm8, %zmm7
4721 ; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14]
4722 ; AVX512BW-FCP-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]
4723 ; AVX512BW-FCP-NEXT: vpermi2w %zmm4, %zmm3, %zmm8
4724 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm9 = [35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0]
4725 ; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm2, %zmm9
4726 ; AVX512BW-FCP-NEXT: movl $33546240, %eax # imm = 0x1FFE000
4727 ; AVX512BW-FCP-NEXT: kmovd %eax, %k1
4728 ; AVX512BW-FCP-NEXT: vmovdqu16 %zmm8, %zmm9 {%k1}
4729 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,32,37,42,47,52,57,62]
4730 ; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm9, %zmm8
4731 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm9 = [4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0]
4732 ; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm9
4733 ; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm1 = [20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15]
4734 ; AVX512BW-FCP-NEXT: # zmm1 = mem[0,1,2,3,0,1,2,3]
4735 ; AVX512BW-FCP-NEXT: vpermi2w %zmm4, %zmm3, %zmm1
4736 ; AVX512BW-FCP-NEXT: movb $7, %al
4737 ; AVX512BW-FCP-NEXT: kmovd %eax, %k1
4738 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm9, %zmm1 {%k1}
4739 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,33,38,43,48,53,58,63]
4740 ; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm1, %zmm2
4741 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, (%rsi)
4742 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm6, (%rdx)
4743 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, (%rcx)
4744 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm8, (%r8)
4745 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, (%r9)
4746 ; AVX512BW-FCP-NEXT: vzeroupper
4747 ; AVX512BW-FCP-NEXT: retq
4748 ;
4749 ; AVX512DQ-BW-LABEL: load_i16_stride5_vf32:
4750 ; AVX512DQ-BW: # %bb.0:
4751 ; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdi), %zmm0
4752 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm1
4753 ; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm2
4754 ; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm3
4755 ; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %zmm4
4756 ; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11]
4757 ; AVX512DQ-BW-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]
4758 ; AVX512DQ-BW-NEXT: vpermi2w %zmm4, %zmm3, %zmm5
4759 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
4760 ; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm6
4761 ; AVX512DQ-BW-NEXT: movl $67100672, %eax # imm = 0x3FFE000
4762 ; AVX512DQ-BW-NEXT: kmovd %eax, %k1
4763 ; AVX512DQ-BW-NEXT: vmovdqu16 %zmm5, %zmm6 {%k1}
4764 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,34,39,44,49,54,59]
4765 ; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm6, %zmm5
4766 ; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm6 = [49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44]
4767 ; AVX512DQ-BW-NEXT: # zmm6 = mem[0,1,2,3,0,1,2,3]
4768 ; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm4, %zmm6
4769 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm7 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
4770 ; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm7
4771 ; AVX512DQ-BW-NEXT: vmovdqu16 %zmm6, %zmm7 {%k1}
4772 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,35,40,45,50,55,60]
4773 ; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm7, %zmm6
4774 ; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45]
4775 ; AVX512DQ-BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]
4776 ; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm4, %zmm7
4777 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm8 = [34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0]
4778 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm2, %zmm8
4779 ; AVX512DQ-BW-NEXT: vmovdqu16 %zmm7, %zmm8 {%k1}
4780 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,36,41,46,51,56,61]
4781 ; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm8, %zmm7
4782 ; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14]
4783 ; AVX512DQ-BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]
4784 ; AVX512DQ-BW-NEXT: vpermi2w %zmm4, %zmm3, %zmm8
4785 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm9 = [35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0]
4786 ; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm2, %zmm9
4787 ; AVX512DQ-BW-NEXT: movl $33546240, %eax # imm = 0x1FFE000
4788 ; AVX512DQ-BW-NEXT: kmovd %eax, %k1
4789 ; AVX512DQ-BW-NEXT: vmovdqu16 %zmm8, %zmm9 {%k1}
4790 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,32,37,42,47,52,57,62]
4791 ; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm9, %zmm8
4792 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm9 = [4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0]
4793 ; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm9
4794 ; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm1 = [20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15]
4795 ; AVX512DQ-BW-NEXT: # zmm1 = mem[0,1,2,3,0,1,2,3]
4796 ; AVX512DQ-BW-NEXT: vpermi2w %zmm4, %zmm3, %zmm1
4797 ; AVX512DQ-BW-NEXT: movb $7, %al
4798 ; AVX512DQ-BW-NEXT: kmovd %eax, %k1
4799 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm9, %zmm1 {%k1}
4800 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,33,38,43,48,53,58,63]
4801 ; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm2
4802 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, (%rsi)
4803 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm6, (%rdx)
4804 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, (%rcx)
4805 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm8, (%r8)
4806 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, (%r9)
4807 ; AVX512DQ-BW-NEXT: vzeroupper
4808 ; AVX512DQ-BW-NEXT: retq
4809 ;
4810 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride5_vf32:
4811 ; AVX512DQ-BW-FCP: # %bb.0:
4812 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdi), %zmm0
4813 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm1
4814 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm2
4815 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm3
4816 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm4
4817 ; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11]
4818 ; AVX512DQ-BW-FCP-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]
4819 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm4, %zmm3, %zmm5
4820 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
4821 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm6
4822 ; AVX512DQ-BW-FCP-NEXT: movl $67100672, %eax # imm = 0x3FFE000
4823 ; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k1
4824 ; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm5, %zmm6 {%k1}
4825 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,34,39,44,49,54,59]
4826 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm6, %zmm5
4827 ; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm6 = [49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44]
4828 ; AVX512DQ-BW-FCP-NEXT: # zmm6 = mem[0,1,2,3,0,1,2,3]
4829 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm4, %zmm6
4830 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm7 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
4831 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm7
4832 ; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm6, %zmm7 {%k1}
4833 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,35,40,45,50,55,60]
4834 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm7, %zmm6
4835 ; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45]
4836 ; AVX512DQ-BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]
4837 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm4, %zmm7
4838 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm8 = [34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0]
4839 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm2, %zmm8
4840 ; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm7, %zmm8 {%k1}
4841 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,36,41,46,51,56,61]
4842 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm8, %zmm7
4843 ; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14]
4844 ; AVX512DQ-BW-FCP-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]
4845 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm4, %zmm3, %zmm8
4846 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm9 = [35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0]
4847 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm2, %zmm9
4848 ; AVX512DQ-BW-FCP-NEXT: movl $33546240, %eax # imm = 0x1FFE000
4849 ; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k1
4850 ; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm8, %zmm9 {%k1}
4851 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,32,37,42,47,52,57,62]
4852 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm9, %zmm8
4853 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm9 = [4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0]
4854 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm9
4855 ; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm1 = [20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15]
4856 ; AVX512DQ-BW-FCP-NEXT: # zmm1 = mem[0,1,2,3,0,1,2,3]
4857 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm4, %zmm3, %zmm1
4858 ; AVX512DQ-BW-FCP-NEXT: movb $7, %al
4859 ; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k1
4860 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm9, %zmm1 {%k1}
4861 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,33,38,43,48,53,58,63]
4862 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm1, %zmm2
4863 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, (%rsi)
4864 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm6, (%rdx)
4865 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, (%rcx)
4866 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, (%r8)
4867 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, (%r9)
4868 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
4869 ; AVX512DQ-BW-FCP-NEXT: retq
4870 %wide.vec = load <160 x i16>, ptr %in.vec, align 64
4871 %strided.vec0 = shufflevector <160 x i16> %wide.vec, <160 x i16> poison, <32 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35, i32 40, i32 45, i32 50, i32 55, i32 60, i32 65, i32 70, i32 75, i32 80, i32 85, i32 90, i32 95, i32 100, i32 105, i32 110, i32 115, i32 120, i32 125, i32 130, i32 135, i32 140, i32 145, i32 150, i32 155>
4872 %strided.vec1 = shufflevector <160 x i16> %wide.vec, <160 x i16> poison, <32 x i32> <i32 1, i32 6, i32 11, i32 16, i32 21, i32 26, i32 31, i32 36, i32 41, i32 46, i32 51, i32 56, i32 61, i32 66, i32 71, i32 76, i32 81, i32 86, i32 91, i32 96, i32 101, i32 106, i32 111, i32 116, i32 121, i32 126, i32 131, i32 136, i32 141, i32 146, i32 151, i32 156>
4873 %strided.vec2 = shufflevector <160 x i16> %wide.vec, <160 x i16> poison, <32 x i32> <i32 2, i32 7, i32 12, i32 17, i32 22, i32 27, i32 32, i32 37, i32 42, i32 47, i32 52, i32 57, i32 62, i32 67, i32 72, i32 77, i32 82, i32 87, i32 92, i32 97, i32 102, i32 107, i32 112, i32 117, i32 122, i32 127, i32 132, i32 137, i32 142, i32 147, i32 152, i32 157>
4874 %strided.vec3 = shufflevector <160 x i16> %wide.vec, <160 x i16> poison, <32 x i32> <i32 3, i32 8, i32 13, i32 18, i32 23, i32 28, i32 33, i32 38, i32 43, i32 48, i32 53, i32 58, i32 63, i32 68, i32 73, i32 78, i32 83, i32 88, i32 93, i32 98, i32 103, i32 108, i32 113, i32 118, i32 123, i32 128, i32 133, i32 138, i32 143, i32 148, i32 153, i32 158>
4875 %strided.vec4 = shufflevector <160 x i16> %wide.vec, <160 x i16> poison, <32 x i32> <i32 4, i32 9, i32 14, i32 19, i32 24, i32 29, i32 34, i32 39, i32 44, i32 49, i32 54, i32 59, i32 64, i32 69, i32 74, i32 79, i32 84, i32 89, i32 94, i32 99, i32 104, i32 109, i32 114, i32 119, i32 124, i32 129, i32 134, i32 139, i32 144, i32 149, i32 154, i32 159>
4876 store <32 x i16> %strided.vec0, ptr %out.vec0, align 64
4877 store <32 x i16> %strided.vec1, ptr %out.vec1, align 64
4878 store <32 x i16> %strided.vec2, ptr %out.vec2, align 64
4879 store <32 x i16> %strided.vec3, ptr %out.vec3, align 64
4880 store <32 x i16> %strided.vec4, ptr %out.vec4, align 64
4881 ret void
4882 }
4884 define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4) nounwind {
4885 ; SSE-LABEL: load_i16_stride5_vf64:
4886 ; SSE: # %bb.0:
4887 ; SSE-NEXT: subq $1016, %rsp # imm = 0x3F8
4888 ; SSE-NEXT: movdqa 464(%rdi), %xmm5
4889 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4890 ; SSE-NEXT: movdqa 400(%rdi), %xmm8
4891 ; SSE-NEXT: movdqa 416(%rdi), %xmm11
4892 ; SSE-NEXT: movdqa 448(%rdi), %xmm4
4893 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4894 ; SSE-NEXT: movdqa 432(%rdi), %xmm7
4895 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4896 ; SSE-NEXT: movdqa 144(%rdi), %xmm6
4897 ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4898 ; SSE-NEXT: movdqa 80(%rdi), %xmm15
4899 ; SSE-NEXT: movdqa 96(%rdi), %xmm10
4900 ; SSE-NEXT: movdqa 128(%rdi), %xmm14
4901 ; SSE-NEXT: movdqa 112(%rdi), %xmm2
4902 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4903 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,65535,65535,0,65535,65535,65535]
4904 ; SSE-NEXT: movdqa %xmm0, %xmm1
4905 ; SSE-NEXT: pandn %xmm2, %xmm1
4906 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm14[0,1,0,3]
4907 ; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4908 ; SSE-NEXT: pand %xmm0, %xmm2
4909 ; SSE-NEXT: por %xmm1, %xmm2
4910 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm10[3,1,2,3]
4911 ; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4912 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
4913 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm15[0,2,2,3]
4914 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,3,2,3,4,5,6,7]
4915 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
4916 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3]
4917 ; SSE-NEXT: movaps {{.*#+}} xmm13 = [65535,65535,65535,65535,65535,65535,65535,0]
4918 ; SSE-NEXT: andps %xmm13, %xmm3
4919 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[0,1,0,1]
4920 ; SSE-NEXT: movaps %xmm13, %xmm2
4921 ; SSE-NEXT: pandn %xmm1, %xmm2
4922 ; SSE-NEXT: por %xmm3, %xmm2
4923 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4924 ; SSE-NEXT: movdqa %xmm0, %xmm1
4925 ; SSE-NEXT: pandn %xmm7, %xmm1
4926 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,1,0,3]
4927 ; SSE-NEXT: pand %xmm0, %xmm2
4928 ; SSE-NEXT: por %xmm1, %xmm2
4929 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm11[3,1,2,3]
4930 ; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4931 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
4932 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm8[0,2,2,3]
4933 ; SSE-NEXT: movdqa %xmm8, %xmm6
4934 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4935 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm3[0,3,2,3,4,5,6,7]
4936 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
4937 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3]
4938 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm5[0,1,0,1]
4939 ; SSE-NEXT: movaps %xmm13, %xmm2
4940 ; SSE-NEXT: andnps %xmm1, %xmm2
4941 ; SSE-NEXT: movdqa 32(%rdi), %xmm3
4942 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4943 ; SSE-NEXT: andps %xmm13, %xmm4
4944 ; SSE-NEXT: orps %xmm4, %xmm2
4945 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4946 ; SSE-NEXT: movdqa %xmm0, %xmm1
4947 ; SSE-NEXT: pandn %xmm3, %xmm1
4948 ; SSE-NEXT: movdqa 48(%rdi), %xmm2
4949 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4950 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]
4951 ; SSE-NEXT: pand %xmm0, %xmm2
4952 ; SSE-NEXT: por %xmm1, %xmm2
4953 ; SSE-NEXT: movdqa 16(%rdi), %xmm1
4954 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4955 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
4956 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
4957 ; SSE-NEXT: movdqa (%rdi), %xmm5
4958 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm5[0,2,2,3]
4959 ; SSE-NEXT: movdqa %xmm5, %xmm9
4960 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4961 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,3,2,3,4,5,6,7]
4962 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
4963 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3]
4964 ; SSE-NEXT: movdqa 64(%rdi), %xmm1
4965 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4966 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
4967 ; SSE-NEXT: movaps %xmm13, %xmm2
4968 ; SSE-NEXT: andnps %xmm1, %xmm2
4969 ; SSE-NEXT: andps %xmm13, %xmm4
4970 ; SSE-NEXT: orps %xmm4, %xmm2
4971 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4972 ; SSE-NEXT: movdqa 352(%rdi), %xmm2
4973 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4974 ; SSE-NEXT: movdqa %xmm0, %xmm1
4975 ; SSE-NEXT: pandn %xmm2, %xmm1
4976 ; SSE-NEXT: movdqa 368(%rdi), %xmm2
4977 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4978 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]
4979 ; SSE-NEXT: pand %xmm0, %xmm2
4980 ; SSE-NEXT: por %xmm1, %xmm2
4981 ; SSE-NEXT: movdqa 336(%rdi), %xmm1
4982 ; SSE-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
4983 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
4984 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
4985 ; SSE-NEXT: movdqa 320(%rdi), %xmm7
4986 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,2,2,3]
4987 ; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4988 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,3,2,3,4,5,6,7]
4989 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
4990 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3]
4991 ; SSE-NEXT: movdqa 384(%rdi), %xmm1
4992 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4993 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
4994 ; SSE-NEXT: movaps %xmm13, %xmm2
4995 ; SSE-NEXT: andnps %xmm1, %xmm2
4996 ; SSE-NEXT: andps %xmm13, %xmm4
4997 ; SSE-NEXT: orps %xmm4, %xmm2
4998 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
4999 ; SSE-NEXT: movdqa 272(%rdi), %xmm2
5000 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5001 ; SSE-NEXT: movdqa %xmm0, %xmm1
5002 ; SSE-NEXT: pandn %xmm2, %xmm1
5003 ; SSE-NEXT: movdqa 288(%rdi), %xmm2
5004 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5005 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]
5006 ; SSE-NEXT: pand %xmm0, %xmm2
5007 ; SSE-NEXT: por %xmm1, %xmm2
5008 ; SSE-NEXT: movdqa 256(%rdi), %xmm12
5009 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm12[3,1,2,3]
5010 ; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5011 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
5012 ; SSE-NEXT: movdqa 240(%rdi), %xmm3
5013 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5014 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,2,2,3]
5015 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,3,2,3,4,5,6,7]
5016 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
5017 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3]
5018 ; SSE-NEXT: movdqa 304(%rdi), %xmm1
5019 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5020 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
5021 ; SSE-NEXT: movaps %xmm13, %xmm2
5022 ; SSE-NEXT: andnps %xmm1, %xmm2
5023 ; SSE-NEXT: andps %xmm13, %xmm4
5024 ; SSE-NEXT: orps %xmm4, %xmm2
5025 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5026 ; SSE-NEXT: movdqa 592(%rdi), %xmm2
5027 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5028 ; SSE-NEXT: movdqa %xmm0, %xmm1
5029 ; SSE-NEXT: pandn %xmm2, %xmm1
5030 ; SSE-NEXT: movdqa 608(%rdi), %xmm2
5031 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5032 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]
5033 ; SSE-NEXT: pand %xmm0, %xmm2
5034 ; SSE-NEXT: por %xmm1, %xmm2
5035 ; SSE-NEXT: movdqa 576(%rdi), %xmm1
5036 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5037 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
5038 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
5039 ; SSE-NEXT: movdqa 560(%rdi), %xmm3
5040 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,2,2,3]
5041 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5042 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,3,2,3,4,5,6,7]
5043 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
5044 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3]
5045 ; SSE-NEXT: movdqa 624(%rdi), %xmm1
5046 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5047 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
5048 ; SSE-NEXT: movaps %xmm13, %xmm2
5049 ; SSE-NEXT: andnps %xmm1, %xmm2
5050 ; SSE-NEXT: andps %xmm13, %xmm4
5051 ; SSE-NEXT: orps %xmm4, %xmm2
5052 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5053 ; SSE-NEXT: movdqa 192(%rdi), %xmm2
5054 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5055 ; SSE-NEXT: movdqa %xmm0, %xmm1
5056 ; SSE-NEXT: pandn %xmm2, %xmm1
5057 ; SSE-NEXT: movdqa 208(%rdi), %xmm2
5058 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5059 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,1,0,3]
5060 ; SSE-NEXT: pand %xmm0, %xmm4
5061 ; SSE-NEXT: por %xmm1, %xmm4
5062 ; SSE-NEXT: movdqa 176(%rdi), %xmm1
5063 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5064 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
5065 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
5066 ; SSE-NEXT: movdqa 160(%rdi), %xmm2
5067 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5068 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,2,2,3]
5069 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,3,2,3,4,5,6,7]
5070 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
5071 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm4[2,3]
5072 ; SSE-NEXT: movdqa 224(%rdi), %xmm1
5073 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5074 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
5075 ; SSE-NEXT: movaps %xmm13, %xmm4
5076 ; SSE-NEXT: andnps %xmm1, %xmm4
5077 ; SSE-NEXT: andps %xmm13, %xmm5
5078 ; SSE-NEXT: orps %xmm5, %xmm4
5079 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5080 ; SSE-NEXT: movdqa 528(%rdi), %xmm1
5081 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5082 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
5083 ; SSE-NEXT: pand %xmm0, %xmm1
5084 ; SSE-NEXT: movdqa 512(%rdi), %xmm2
5085 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5086 ; SSE-NEXT: pandn %xmm2, %xmm0
5087 ; SSE-NEXT: por %xmm1, %xmm0
5088 ; SSE-NEXT: movdqa 496(%rdi), %xmm1
5089 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5090 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
5091 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]
5092 ; SSE-NEXT: movdqa 480(%rdi), %xmm2
5093 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5094 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,2,2,3]
5095 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,3,2,3,4,5,6,7]
5096 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
5097 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm0[2,3]
5098 ; SSE-NEXT: movdqa 544(%rdi), %xmm0
5099 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5100 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
5101 ; SSE-NEXT: movaps %xmm13, %xmm1
5102 ; SSE-NEXT: andnps %xmm0, %xmm1
5103 ; SSE-NEXT: andps %xmm13, %xmm4
5104 ; SSE-NEXT: orps %xmm4, %xmm1
5105 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5106 ; SSE-NEXT: psrlq $48, %xmm10
5107 ; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5108 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm15[0,3,2,3]
5109 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,2,3,4,5,6,7]
5110 ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1]
5111 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,0,0,65535,65535,65535,65535,65535]
5112 ; SSE-NEXT: movdqa %xmm0, %xmm4
5113 ; SSE-NEXT: pandn %xmm1, %xmm4
5114 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm14[1,3,2,3]
5115 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5116 ; SSE-NEXT: # xmm5 = mem[0,2,2,3]
5117 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
5118 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm5[0,1,2,3,7,5,6,7]
5119 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]
5120 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
5121 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,4,7]
5122 ; SSE-NEXT: pand %xmm0, %xmm1
5123 ; SSE-NEXT: por %xmm4, %xmm1
5124 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5125 ; SSE-NEXT: movdqa %xmm8, %xmm4
5126 ; SSE-NEXT: psllq $48, %xmm4
5127 ; SSE-NEXT: movaps %xmm13, %xmm2
5128 ; SSE-NEXT: andnps %xmm4, %xmm2
5129 ; SSE-NEXT: pand %xmm13, %xmm1
5130 ; SSE-NEXT: orps %xmm1, %xmm2
5131 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5132 ; SSE-NEXT: psrlq $48, %xmm11
5133 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm6[0,3,2,3]
5134 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,2,2,3,4,5,6,7]
5135 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm11[0],xmm4[1],xmm11[1]
5136 ; SSE-NEXT: movdqa %xmm0, %xmm1
5137 ; SSE-NEXT: pandn %xmm4, %xmm1
5138 ; SSE-NEXT: pshufd $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
5139 ; SSE-NEXT: # xmm4 = mem[1,3,2,3]
5140 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5141 ; SSE-NEXT: # xmm5 = mem[0,2,2,3]
5142 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
5143 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm5[0,1,2,3,7,5,6,7]
5144 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,1]
5145 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
5146 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,4,7]
5147 ; SSE-NEXT: pand %xmm0, %xmm4
5148 ; SSE-NEXT: por %xmm1, %xmm4
5149 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5150 ; SSE-NEXT: psllq $48, %xmm1
5151 ; SSE-NEXT: movdqa %xmm13, %xmm2
5152 ; SSE-NEXT: pandn %xmm1, %xmm2
5153 ; SSE-NEXT: pand %xmm13, %xmm4
5154 ; SSE-NEXT: por %xmm4, %xmm2
5155 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5156 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5157 ; SSE-NEXT: psrlq $48, %xmm1
5158 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm9[0,3,2,3]
5159 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,2,2,3,4,5,6,7]
5160 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
5161 ; SSE-NEXT: movdqa %xmm0, %xmm1
5162 ; SSE-NEXT: pandn %xmm4, %xmm1
5163 ; SSE-NEXT: pshufd $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
5164 ; SSE-NEXT: # xmm4 = mem[1,3,2,3]
5165 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5166 ; SSE-NEXT: # xmm5 = mem[0,2,2,3]
5167 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
5168 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm5[0,1,2,3,7,5,6,7]
5169 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,1]
5170 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
5171 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,4,7]
5172 ; SSE-NEXT: pand %xmm0, %xmm4
5173 ; SSE-NEXT: por %xmm1, %xmm4
5174 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5175 ; SSE-NEXT: psllq $48, %xmm1
5176 ; SSE-NEXT: movdqa %xmm13, %xmm2
5177 ; SSE-NEXT: pandn %xmm1, %xmm2
5178 ; SSE-NEXT: pand %xmm13, %xmm4
5179 ; SSE-NEXT: por %xmm4, %xmm2
5180 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5181 ; SSE-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
5182 ; SSE-NEXT: psrlq $48, %xmm1
5183 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,3,2,3]
5184 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,2,2,3,4,5,6,7]
5185 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
5186 ; SSE-NEXT: movdqa %xmm0, %xmm1
5187 ; SSE-NEXT: pandn %xmm4, %xmm1
5188 ; SSE-NEXT: pshufd $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
5189 ; SSE-NEXT: # xmm4 = mem[1,3,2,3]
5190 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5191 ; SSE-NEXT: # xmm5 = mem[0,2,2,3]
5192 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
5193 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm5[0,1,2,3,7,5,6,7]
5194 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,1]
5195 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
5196 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,4,7]
5197 ; SSE-NEXT: pand %xmm0, %xmm4
5198 ; SSE-NEXT: por %xmm1, %xmm4
5199 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5200 ; SSE-NEXT: psllq $48, %xmm1
5201 ; SSE-NEXT: movdqa %xmm13, %xmm2
5202 ; SSE-NEXT: pandn %xmm1, %xmm2
5203 ; SSE-NEXT: pand %xmm13, %xmm4
5204 ; SSE-NEXT: por %xmm4, %xmm2
5205 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5206 ; SSE-NEXT: psrlq $48, %xmm12
5207 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
5208 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm14[0,3,2,3]
5209 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,2,2,3,4,5,6,7]
5210 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1]
5211 ; SSE-NEXT: movdqa %xmm0, %xmm1
5212 ; SSE-NEXT: pandn %xmm4, %xmm1
5213 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
5214 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm10[1,3,2,3]
5215 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5216 ; SSE-NEXT: # xmm5 = mem[0,2,2,3]
5217 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
5218 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm5[0,1,2,3,7,5,6,7]
5219 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,1]
5220 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
5221 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,4,7]
5222 ; SSE-NEXT: pand %xmm0, %xmm4
5223 ; SSE-NEXT: por %xmm1, %xmm4
5224 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5225 ; SSE-NEXT: movdqa %xmm12, %xmm1
5226 ; SSE-NEXT: psllq $48, %xmm1
5227 ; SSE-NEXT: movdqa %xmm13, %xmm2
5228 ; SSE-NEXT: pandn %xmm1, %xmm2
5229 ; SSE-NEXT: pand %xmm13, %xmm4
5230 ; SSE-NEXT: por %xmm4, %xmm2
5231 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5232 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5233 ; SSE-NEXT: psrlq $48, %xmm1
5234 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,3,2,3]
5235 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,2,2,3,4,5,6,7]
5236 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
5237 ; SSE-NEXT: movdqa %xmm0, %xmm1
5238 ; SSE-NEXT: pandn %xmm4, %xmm1
5239 ; SSE-NEXT: pshufd $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
5240 ; SSE-NEXT: # xmm4 = mem[1,3,2,3]
5241 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5242 ; SSE-NEXT: # xmm5 = mem[0,2,2,3]
5243 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
5244 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm5[0,1,2,3,7,5,6,7]
5245 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,1]
5246 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
5247 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,4,7]
5248 ; SSE-NEXT: pand %xmm0, %xmm4
5249 ; SSE-NEXT: por %xmm1, %xmm4
5250 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5251 ; SSE-NEXT: psllq $48, %xmm1
5252 ; SSE-NEXT: movdqa %xmm13, %xmm2
5253 ; SSE-NEXT: pandn %xmm1, %xmm2
5254 ; SSE-NEXT: pand %xmm13, %xmm4
5255 ; SSE-NEXT: por %xmm4, %xmm2
5256 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5257 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
5258 ; SSE-NEXT: movdqa %xmm7, %xmm1
5259 ; SSE-NEXT: psrlq $48, %xmm1
5260 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
5261 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm11[0,3,2,3]
5262 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,2,2,3,4,5,6,7]
5263 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
5264 ; SSE-NEXT: movdqa %xmm0, %xmm1
5265 ; SSE-NEXT: pandn %xmm4, %xmm1
5266 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
5267 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]
5268 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
5269 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm9[0,2,2,3]
5270 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
5271 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm5[0,1,2,3,7,5,6,7]
5272 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,1]
5273 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
5274 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,4,7]
5275 ; SSE-NEXT: pand %xmm0, %xmm4
5276 ; SSE-NEXT: por %xmm1, %xmm4
5277 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5278 ; SSE-NEXT: movdqa %xmm3, %xmm1
5279 ; SSE-NEXT: psllq $48, %xmm1
5280 ; SSE-NEXT: movdqa %xmm13, %xmm5
5281 ; SSE-NEXT: pandn %xmm1, %xmm5
5282 ; SSE-NEXT: pand %xmm13, %xmm4
5283 ; SSE-NEXT: por %xmm4, %xmm5
5284 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5285 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5286 ; SSE-NEXT: psrlq $48, %xmm1
5287 ; SSE-NEXT: pshufd $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
5288 ; SSE-NEXT: # xmm4 = mem[0,3,2,3]
5289 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,2,2,3,4,5,6,7]
5290 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
5291 ; SSE-NEXT: pshufd $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5292 ; SSE-NEXT: # xmm1 = mem[1,3,2,3]
5293 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5294 ; SSE-NEXT: # xmm5 = mem[0,2,2,3]
5295 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
5296 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm5[0,1,2,3,7,5,6,7]
5297 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]
5298 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
5299 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,4,7]
5300 ; SSE-NEXT: pand %xmm0, %xmm1
5301 ; SSE-NEXT: pandn %xmm4, %xmm0
5302 ; SSE-NEXT: por %xmm1, %xmm0
5303 ; SSE-NEXT: pand %xmm13, %xmm0
5304 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5305 ; SSE-NEXT: psllq $48, %xmm1
5306 ; SSE-NEXT: pandn %xmm1, %xmm13
5307 ; SSE-NEXT: por %xmm0, %xmm13
5308 ; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5309 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5310 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5311 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm0[0,0]
5312 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
5313 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,0,1,3]
5314 ; SSE-NEXT: movaps {{.*#+}} xmm6 = [65535,65535,65535,0,0,0,65535,65535]
5315 ; SSE-NEXT: movaps %xmm6, %xmm4
5316 ; SSE-NEXT: andnps %xmm1, %xmm4
5317 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm15[0,1,1,3]
5318 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,7]
5319 ; SSE-NEXT: punpckhdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5320 ; SSE-NEXT: # xmm5 = xmm5[2],mem[2],xmm5[3],mem[3]
5321 ; SSE-NEXT: pand %xmm6, %xmm5
5322 ; SSE-NEXT: por %xmm4, %xmm5
5323 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]
5324 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm8[0,1,2,0]
5325 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
5326 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm1[2,3]
5327 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm4[2,0]
5328 ; SSE-NEXT: movaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5329 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5330 ; SSE-NEXT: movaps %xmm0, %xmm1
5331 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
5332 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm13[0,0]
5333 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm13[2,3]
5334 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,0,1,3]
5335 ; SSE-NEXT: movaps %xmm6, %xmm4
5336 ; SSE-NEXT: andnps %xmm1, %xmm4
5337 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5338 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm8[0,1,1,3]
5339 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,7]
5340 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
5341 ; SSE-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm15[2],xmm5[3],xmm15[3]
5342 ; SSE-NEXT: pand %xmm6, %xmm5
5343 ; SSE-NEXT: por %xmm4, %xmm5
5344 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]
5345 ; SSE-NEXT: pshufd $36, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
5346 ; SSE-NEXT: # xmm4 = mem[0,1,2,0]
5347 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
5348 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm1[2,3]
5349 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm4[2,0]
5350 ; SSE-NEXT: movaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5351 ; SSE-NEXT: movdqa %xmm10, %xmm1
5352 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5353 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm4[0,0]
5354 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm4[2,3]
5355 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,0,1,3]
5356 ; SSE-NEXT: movaps %xmm6, %xmm4
5357 ; SSE-NEXT: andnps %xmm1, %xmm4
5358 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm14[0,1,1,3]
5359 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,7]
5360 ; SSE-NEXT: punpckhdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5361 ; SSE-NEXT: # xmm5 = xmm5[2],mem[2],xmm5[3],mem[3]
5362 ; SSE-NEXT: pand %xmm6, %xmm5
5363 ; SSE-NEXT: por %xmm4, %xmm5
5364 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]
5365 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm12[0,1,2,0]
5366 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
5367 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm1[2,3]
5368 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm4[2,0]
5369 ; SSE-NEXT: movaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5370 ; SSE-NEXT: movdqa %xmm2, %xmm1
5371 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm9[0,0]
5372 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm9[2,3]
5373 ; SSE-NEXT: movdqa %xmm9, %xmm12
5374 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,0,1,3]
5375 ; SSE-NEXT: movaps %xmm6, %xmm4
5376 ; SSE-NEXT: andnps %xmm1, %xmm4
5377 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm11[0,1,1,3]
5378 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm5[0,1,2,3,4,7,6,7]
5379 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm7[2],xmm2[3],xmm7[3]
5380 ; SSE-NEXT: pand %xmm6, %xmm2
5381 ; SSE-NEXT: por %xmm4, %xmm2
5382 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]
5383 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,1,2,0]
5384 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
5385 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm1[2,3]
5386 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,0]
5387 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5388 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5389 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
5390 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm2[0,0]
5391 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[2,3]
5392 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,0,1,3]
5393 ; SSE-NEXT: movaps %xmm6, %xmm4
5394 ; SSE-NEXT: andnps %xmm1, %xmm4
5395 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5396 ; SSE-NEXT: # xmm5 = mem[0,1,1,3]
5397 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm5[0,1,2,3,4,7,6,7]
5398 ; SSE-NEXT: punpckhdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
5399 ; SSE-NEXT: # xmm2 = xmm2[2],mem[2],xmm2[3],mem[3]
5400 ; SSE-NEXT: pand %xmm6, %xmm2
5401 ; SSE-NEXT: por %xmm4, %xmm2
5402 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]
5403 ; SSE-NEXT: pshufd $36, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
5404 ; SSE-NEXT: # xmm4 = mem[0,1,2,0]
5405 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
5406 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm1[2,3]
5407 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,0]
5408 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5409 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5410 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
5411 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm7[0,0]
5412 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm7[2,3]
5413 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,0,1,3]
5414 ; SSE-NEXT: movaps %xmm6, %xmm4
5415 ; SSE-NEXT: andnps %xmm1, %xmm4
5416 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5417 ; SSE-NEXT: # xmm5 = mem[0,1,1,3]
5418 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm5[0,1,2,3,4,7,6,7]
5419 ; SSE-NEXT: movdqa (%rsp), %xmm9 # 16-byte Reload
5420 ; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm9[2],xmm2[3],xmm9[3]
5421 ; SSE-NEXT: pand %xmm6, %xmm2
5422 ; SSE-NEXT: por %xmm4, %xmm2
5423 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]
5424 ; SSE-NEXT: pshufd $36, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
5425 ; SSE-NEXT: # xmm4 = mem[0,1,2,0]
5426 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
5427 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm1[2,3]
5428 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,0]
5429 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5430 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5431 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
5432 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm2[0,0]
5433 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[2,3]
5434 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,0,1,3]
5435 ; SSE-NEXT: movaps %xmm6, %xmm4
5436 ; SSE-NEXT: andnps %xmm1, %xmm4
5437 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5438 ; SSE-NEXT: # xmm5 = mem[0,1,1,3]
5439 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm5[0,1,2,3,4,7,6,7]
5440 ; SSE-NEXT: punpckhdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
5441 ; SSE-NEXT: # xmm2 = xmm2[2],mem[2],xmm2[3],mem[3]
5442 ; SSE-NEXT: pand %xmm6, %xmm2
5443 ; SSE-NEXT: por %xmm4, %xmm2
5444 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]
5445 ; SSE-NEXT: pshufd $36, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
5446 ; SSE-NEXT: # xmm4 = mem[0,1,2,0]
5447 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
5448 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm1[2,3]
5449 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,0]
5450 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5451 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5452 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
5453 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm10[0,0]
5454 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm10[2,3]
5455 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,0,1,3]
5456 ; SSE-NEXT: movaps %xmm6, %xmm4
5457 ; SSE-NEXT: andnps %xmm1, %xmm4
5458 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5459 ; SSE-NEXT: # xmm5 = mem[0,1,1,3]
5460 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm5[0,1,2,3,4,7,6,7]
5461 ; SSE-NEXT: punpckhdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
5462 ; SSE-NEXT: # xmm2 = xmm2[2],mem[2],xmm2[3],mem[3]
5463 ; SSE-NEXT: pand %xmm6, %xmm2
5464 ; SSE-NEXT: por %xmm4, %xmm2
5465 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]
5466 ; SSE-NEXT: pshufd $36, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
5467 ; SSE-NEXT: # xmm4 = mem[0,1,2,0]
5468 ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]
5469 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm1[2,3]
5470 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,0]
5471 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5472 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm13[2,2,2,2,4,5,6,7]
5473 ; SSE-NEXT: movdqa %xmm6, %xmm4
5474 ; SSE-NEXT: pandn %xmm1, %xmm4
5475 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[1,1,1,1]
5476 ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm15[0,2,2,3]
5477 ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
5478 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm5[0,3,2,3,4,5,6,7]
5479 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
5480 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,3,4,5,6,7]
5481 ; SSE-NEXT: pand %xmm6, %xmm1
5482 ; SSE-NEXT: por %xmm4, %xmm1
5483 ; SSE-NEXT: movdqa %xmm1, %xmm2
5484 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm13[3,0]
5485 ; SSE-NEXT: movaps %xmm6, %xmm3
5486 ; SSE-NEXT: andnps %xmm13, %xmm3
5487 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5488 ; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,1],xmm0[0,2]
5489 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm13[0,1,2,3,7,4,6,7]
5490 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
5491 ; SSE-NEXT: # xmm3 = mem[0,1,0,3]
5492 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,5,6]
5493 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1],xmm1[2,3]
5494 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,0]
5495 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5496 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
5497 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm14[2,2,2,2,4,5,6,7]
5498 ; SSE-NEXT: movdqa %xmm6, %xmm3
5499 ; SSE-NEXT: pandn %xmm1, %xmm3
5500 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5501 ; SSE-NEXT: # xmm1 = mem[1,1,1,1]
5502 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
5503 ; SSE-NEXT: # xmm4 = mem[0,2,2,3]
5504 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
5505 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm4[0,3,2,3,4,5,6,7]
5506 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
5507 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,3,4,5,6,7]
5508 ; SSE-NEXT: pand %xmm6, %xmm1
5509 ; SSE-NEXT: por %xmm3, %xmm1
5510 ; SSE-NEXT: movdqa %xmm1, %xmm4
5511 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5512 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm14[3,0]
5513 ; SSE-NEXT: movaps %xmm6, %xmm2
5514 ; SSE-NEXT: andnps %xmm14, %xmm2
5515 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5516 ; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[0,1],xmm1[0,2]
5517 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm14[0,1,2,3,7,4,6,7]
5518 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
5519 ; SSE-NEXT: # xmm3 = mem[0,1,0,3]
5520 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,5,6]
5521 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1],xmm1[2,3]
5522 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,0]
5523 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5524 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[2,2,2,2,4,5,6,7]
5525 ; SSE-NEXT: movdqa %xmm6, %xmm3
5526 ; SSE-NEXT: pandn %xmm1, %xmm3
5527 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm11[1,1,1,1]
5528 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
5529 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm13[0,2,2,3]
5530 ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
5531 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm4[0,3,2,3,4,5,6,7]
5532 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
5533 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,3,4,5,6,7]
5534 ; SSE-NEXT: pand %xmm6, %xmm1
5535 ; SSE-NEXT: por %xmm3, %xmm1
5536 ; SSE-NEXT: movdqa %xmm1, %xmm3
5537 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5538 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm12[3,0]
5539 ; SSE-NEXT: movaps %xmm6, %xmm2
5540 ; SSE-NEXT: andnps %xmm12, %xmm2
5541 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5542 ; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[0,1],xmm1[0,2]
5543 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm12[0,1,2,3,7,4,6,7]
5544 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
5545 ; SSE-NEXT: # xmm2 = mem[0,1,0,3]
5546 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,5,6]
5547 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,1],xmm1[2,3]
5548 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,0]
5549 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5550 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5551 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm4[2,2,2,2,4,5,6,7]
5552 ; SSE-NEXT: movdqa %xmm6, %xmm2
5553 ; SSE-NEXT: pandn %xmm1, %xmm2
5554 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5555 ; SSE-NEXT: # xmm1 = mem[1,1,1,1]
5556 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
5557 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
5558 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
5559 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[0,3,2,3,4,5,6,7]
5560 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
5561 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,3,4,5,6,7]
5562 ; SSE-NEXT: pand %xmm6, %xmm1
5563 ; SSE-NEXT: por %xmm2, %xmm1
5564 ; SSE-NEXT: movdqa %xmm1, %xmm3
5565 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5566 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm4[3,0]
5567 ; SSE-NEXT: movaps %xmm4, %xmm2
5568 ; SSE-NEXT: movaps %xmm6, %xmm4
5569 ; SSE-NEXT: andnps %xmm2, %xmm4
5570 ; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5571 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2]
5572 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,7,4,6,7]
5573 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
5574 ; SSE-NEXT: # xmm2 = mem[0,1,0,3]
5575 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,5,6]
5576 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,1],xmm1[2,3]
5577 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,0]
5578 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5579 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm7[2,2,2,2,4,5,6,7]
5580 ; SSE-NEXT: movdqa %xmm6, %xmm2
5581 ; SSE-NEXT: pandn %xmm1, %xmm2
5582 ; SSE-NEXT: pshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5583 ; SSE-NEXT: # xmm1 = mem[1,1,1,1]
5584 ; SSE-NEXT: movdqa %xmm9, %xmm11
5585 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm9[0,2,2,3]
5586 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
5587 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[0,3,2,3,4,5,6,7]
5588 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
5589 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm1[1,0,3,3,4,5,6,7]
5590 ; SSE-NEXT: pand %xmm6, %xmm0
5591 ; SSE-NEXT: por %xmm2, %xmm0
5592 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5593 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm7[3,0]
5594 ; SSE-NEXT: movaps %xmm6, %xmm2
5595 ; SSE-NEXT: andnps %xmm7, %xmm2
5596 ; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5597 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,1],xmm1[0,2]
5598 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm7[0,1,2,3,7,4,6,7]
5599 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
5600 ; SSE-NEXT: # xmm2 = mem[0,1,0,3]
5601 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,5,6]
5602 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,1],xmm1[2,3]
5603 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
5604 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5605 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5606 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm4[2,2,2,2,4,5,6,7]
5607 ; SSE-NEXT: movdqa %xmm6, %xmm2
5608 ; SSE-NEXT: pandn %xmm1, %xmm2
5609 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
5610 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm15[1,1,1,1]
5611 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
5612 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm9[0,2,2,3]
5613 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
5614 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[0,3,2,3,4,5,6,7]
5615 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
5616 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm1[1,0,3,3,4,5,6,7]
5617 ; SSE-NEXT: pand %xmm6, %xmm0
5618 ; SSE-NEXT: por %xmm2, %xmm0
5619 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5620 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm4[3,0]
5621 ; SSE-NEXT: movaps %xmm4, %xmm2
5622 ; SSE-NEXT: movaps %xmm6, %xmm3
5623 ; SSE-NEXT: andnps %xmm4, %xmm3
5624 ; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5625 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2]
5626 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,7,4,6,7]
5627 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
5628 ; SSE-NEXT: # xmm2 = mem[0,1,0,3]
5629 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,5,6]
5630 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,1],xmm1[2,3]
5631 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
5632 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5633 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm10[2,2,2,2,4,5,6,7]
5634 ; SSE-NEXT: movdqa %xmm6, %xmm2
5635 ; SSE-NEXT: pandn %xmm1, %xmm2
5636 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
5637 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm12[1,1,1,1]
5638 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
5639 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm7[0,2,2,3]
5640 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
5641 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[0,3,2,3,4,5,6,7]
5642 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
5643 ; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm1[1,0,3,3,4,5,6,7]
5644 ; SSE-NEXT: pand %xmm6, %xmm14
5645 ; SSE-NEXT: por %xmm2, %xmm14
5646 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5647 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm10[3,0]
5648 ; SSE-NEXT: movaps %xmm6, %xmm0
5649 ; SSE-NEXT: andnps %xmm10, %xmm0
5650 ; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5651 ; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[0,1],xmm1[0,2]
5652 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm10[0,1,2,3,7,4,6,7]
5653 ; SSE-NEXT: pshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
5654 ; SSE-NEXT: # xmm2 = mem[0,1,0,3]
5655 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,5,6]
5656 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,1],xmm1[2,3]
5657 ; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[0,1],xmm2[2,0]
5658 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
5659 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[2,2,2,2,4,5,6,7]
5660 ; SSE-NEXT: movdqa %xmm6, %xmm2
5661 ; SSE-NEXT: pandn %xmm1, %xmm2
5662 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
5663 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[1,1,1,1]
5664 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5665 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
5666 ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
5667 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[0,3,2,3,4,5,6,7]
5668 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
5669 ; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm1[1,0,3,3,4,5,6,7]
5670 ; SSE-NEXT: pand %xmm6, %xmm10
5671 ; SSE-NEXT: por %xmm2, %xmm10
5672 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
5673 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5674 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,0]
5675 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2]
5676 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
5677 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5678 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm3[3,0]
5679 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm1[0,2]
5680 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5681 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm13[3,0]
5682 ; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,1],xmm1[0,2]
5683 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5684 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm5[3,0]
5685 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm1[0,2]
5686 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5687 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm11[3,0]
5688 ; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[0,1],xmm1[0,2]
5689 ; SSE-NEXT: movaps %xmm11, (%rsp) # 16-byte Spill
5690 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[2,0],xmm9[3,0]
5691 ; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[0,1],xmm15[0,2]
5692 ; SSE-NEXT: movdqa %xmm7, %xmm1
5693 ; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[2,0],xmm7[3,0]
5694 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm12[0,2]
5695 ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5696 ; SSE-NEXT: movdqa %xmm4, %xmm1
5697 ; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,0],xmm4[3,0]
5698 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm8[0,2]
5699 ; SSE-NEXT: movaps %xmm1, %xmm15
5700 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
5701 ; SSE-NEXT: movaps %xmm4, %xmm1
5702 ; SSE-NEXT: movaps %xmm4, %xmm12
5703 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[3,0]
5704 ; SSE-NEXT: movaps %xmm0, %xmm11
5705 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
5706 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
5707 ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm2[2,1,3,3,4,5,6,7]
5708 ; SSE-NEXT: pand %xmm6, %xmm8
5709 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm3[0,1,2,3,4,6,6,7]
5710 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
5711 ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm2[2,1,3,3,4,5,6,7]
5712 ; SSE-NEXT: pand %xmm6, %xmm7
5713 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm13[0,1,2,3,4,6,6,7]
5714 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
5715 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm2[2,1,3,3,4,5,6,7]
5716 ; SSE-NEXT: pand %xmm6, %xmm0
5717 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm5[0,1,2,3,4,6,6,7]
5718 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
5719 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm2[2,1,3,3,4,5,6,7]
5720 ; SSE-NEXT: pand %xmm6, %xmm5
5721 ; SSE-NEXT: pshufhw $232, (%rsp), %xmm2 # 16-byte Folded Reload
5722 ; SSE-NEXT: # xmm2 = mem[0,1,2,3,4,6,6,7]
5723 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
5724 ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[2,1,3,3,4,5,6,7]
5725 ; SSE-NEXT: pand %xmm6, %xmm4
5726 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm9[0,1,2,3,4,6,6,7]
5727 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
5728 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm2[2,1,3,3,4,5,6,7]
5729 ; SSE-NEXT: pand %xmm6, %xmm3
5730 ; SSE-NEXT: pshufhw $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
5731 ; SSE-NEXT: # xmm2 = mem[0,1,2,3,4,6,6,7]
5732 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
5733 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,3,3,4,5,6,7]
5734 ; SSE-NEXT: pand %xmm6, %xmm2
5735 ; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm15[0,1,2,3,4,6,6,7]
5736 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,2,2,3]
5737 ; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[2,1,3,3,4,5,6,7]
5738 ; SSE-NEXT: pand %xmm6, %xmm9
5739 ; SSE-NEXT: andnps %xmm11, %xmm6
5740 ; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[0,1],xmm1[0,2]
5741 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm11[0,1,2,3,7,4,6,7]
5742 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
5743 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm11[0,1,0,3]
5744 ; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,5,6]
5745 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[3,1],xmm1[2,3]
5746 ; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[0,1],xmm15[2,0]
5747 ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload
5748 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5749 ; SSE-NEXT: # xmm1 = mem[0,2,2,3]
5750 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
5751 ; SSE-NEXT: # xmm15 = mem[0,1,1,3]
5752 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
5753 ; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,4,7]
5754 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[3,1],xmm1[1,3]
5755 ; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[0,1],xmm15[2,0]
5756 ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
5757 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5758 ; SSE-NEXT: # xmm1 = mem[0,2,2,3]
5759 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
5760 ; SSE-NEXT: # xmm15 = mem[0,1,1,3]
5761 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
5762 ; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,4,7]
5763 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[3,1],xmm1[1,3]
5764 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,1],xmm15[2,0]
5765 ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
5766 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5767 ; SSE-NEXT: # xmm1 = mem[0,2,2,3]
5768 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
5769 ; SSE-NEXT: # xmm15 = mem[0,1,1,3]
5770 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
5771 ; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,4,7]
5772 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[3,1],xmm1[1,3]
5773 ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm15[2,0]
5774 ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
5775 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5776 ; SSE-NEXT: # xmm1 = mem[0,2,2,3]
5777 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
5778 ; SSE-NEXT: # xmm15 = mem[0,1,1,3]
5779 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
5780 ; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,4,7]
5781 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[3,1],xmm1[1,3]
5782 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm15[2,0]
5783 ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
5784 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5785 ; SSE-NEXT: # xmm1 = mem[0,2,2,3]
5786 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
5787 ; SSE-NEXT: # xmm15 = mem[0,1,1,3]
5788 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
5789 ; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,4,7]
5790 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[3,1],xmm1[1,3]
5791 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm15[2,0]
5792 ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
5793 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5794 ; SSE-NEXT: # xmm1 = mem[0,2,2,3]
5795 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
5796 ; SSE-NEXT: # xmm15 = mem[0,1,1,3]
5797 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
5798 ; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,4,7]
5799 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[3,1],xmm1[1,3]
5800 ; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm15[2,0]
5801 ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
5802 ; SSE-NEXT: pshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
5803 ; SSE-NEXT: # xmm1 = mem[0,2,2,3]
5804 ; SSE-NEXT: pshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
5805 ; SSE-NEXT: # xmm15 = mem[0,1,1,3]
5806 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
5807 ; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,4,7]
5808 ; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[3,1],xmm1[1,3]
5809 ; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm15[2,0]
5810 ; SSE-NEXT: orps %xmm9, %xmm6
5811 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm12[0,2,2,3]
5812 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm11[0,1,1,3]
5813 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
5814 ; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,5,4,7]
5815 ; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[3,1],xmm1[1,3]
5816 ; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm9[2,0]
5817 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5818 ; SSE-NEXT: movaps %xmm1, 96(%rsi)
5819 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5820 ; SSE-NEXT: movaps %xmm1, 32(%rsi)
5821 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5822 ; SSE-NEXT: movaps %xmm1, 112(%rsi)
5823 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5824 ; SSE-NEXT: movaps %xmm1, 48(%rsi)
5825 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5826 ; SSE-NEXT: movaps %xmm1, 64(%rsi)
5827 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5828 ; SSE-NEXT: movaps %xmm1, (%rsi)
5829 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5830 ; SSE-NEXT: movaps %xmm1, 80(%rsi)
5831 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5832 ; SSE-NEXT: movaps %xmm1, 16(%rsi)
5833 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5834 ; SSE-NEXT: movaps %xmm1, 96(%rdx)
5835 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5836 ; SSE-NEXT: movaps %xmm1, 32(%rdx)
5837 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5838 ; SSE-NEXT: movaps %xmm1, 112(%rdx)
5839 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5840 ; SSE-NEXT: movaps %xmm1, 48(%rdx)
5841 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5842 ; SSE-NEXT: movaps %xmm1, 64(%rdx)
5843 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5844 ; SSE-NEXT: movaps %xmm1, (%rdx)
5845 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5846 ; SSE-NEXT: movaps %xmm1, 80(%rdx)
5847 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5848 ; SSE-NEXT: movaps %xmm1, 16(%rdx)
5849 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5850 ; SSE-NEXT: movaps %xmm1, 96(%rcx)
5851 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5852 ; SSE-NEXT: movaps %xmm1, 112(%rcx)
5853 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5854 ; SSE-NEXT: movaps %xmm1, 64(%rcx)
5855 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5856 ; SSE-NEXT: movaps %xmm1, 80(%rcx)
5857 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5858 ; SSE-NEXT: movaps %xmm1, 32(%rcx)
5859 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5860 ; SSE-NEXT: movaps %xmm1, 48(%rcx)
5861 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5862 ; SSE-NEXT: movaps %xmm1, (%rcx)
5863 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5864 ; SSE-NEXT: movaps %xmm1, 16(%rcx)
5865 ; SSE-NEXT: movaps %xmm10, 112(%r8)
5866 ; SSE-NEXT: movaps %xmm14, 96(%r8)
5867 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5868 ; SSE-NEXT: movaps %xmm1, 80(%r8)
5869 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5870 ; SSE-NEXT: movaps %xmm1, 64(%r8)
5871 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5872 ; SSE-NEXT: movaps %xmm1, 48(%r8)
5873 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5874 ; SSE-NEXT: movaps %xmm1, 32(%r8)
5875 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5876 ; SSE-NEXT: movaps %xmm1, 16(%r8)
5877 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
5878 ; SSE-NEXT: movaps %xmm1, (%r8)
5879 ; SSE-NEXT: movaps %xmm6, 112(%r9)
5880 ; SSE-NEXT: movaps %xmm2, 96(%r9)
5881 ; SSE-NEXT: movaps %xmm3, 80(%r9)
5882 ; SSE-NEXT: movaps %xmm4, 64(%r9)
5883 ; SSE-NEXT: movaps %xmm5, 48(%r9)
5884 ; SSE-NEXT: movaps %xmm0, 32(%r9)
5885 ; SSE-NEXT: movaps %xmm7, 16(%r9)
5886 ; SSE-NEXT: movaps %xmm8, (%r9)
5887 ; SSE-NEXT: addq $1016, %rsp # imm = 0x3F8
5888 ; SSE-NEXT: retq
5889 ;
5890 ; AVX-LABEL: load_i16_stride5_vf64:
5891 ; AVX: # %bb.0:
5892 ; AVX-NEXT: subq $1032, %rsp # imm = 0x408
5893 ; AVX-NEXT: vmovdqa 304(%rdi), %xmm0
5894 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5895 ; AVX-NEXT: vmovdqa 288(%rdi), %xmm1
5896 ; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5897 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5,6,7]
5898 ; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [6,7,2,3,4,5,6,7,6,7,2,3,12,13,6,7]
5899 ; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm0
5900 ; AVX-NEXT: vmovdqa 256(%rdi), %xmm2
5901 ; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5902 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,1,3]
5903 ; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]
5904 ; AVX-NEXT: vmovdqa 272(%rdi), %xmm15
5905 ; AVX-NEXT: vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm15[1]
5906 ; AVX-NEXT: vmovdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5907 ; AVX-NEXT: vmovdqa 240(%rdi), %xmm3
5908 ; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5909 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
5910 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,3,2,3,4,5,6,7]
5911 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3,4,5,6,7]
5912 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm0[5,6,7]
5913 ; AVX-NEXT: vmovdqa 208(%rdi), %xmm0
5914 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5915 ; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
5916 ; AVX-NEXT: vmovdqa 192(%rdi), %xmm3
5917 ; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5918 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4],xmm0[5,6,7]
5919 ; AVX-NEXT: vmovdqa 176(%rdi), %xmm3
5920 ; AVX-NEXT: vmovdqa %xmm3, (%rsp) # 16-byte Spill
5921 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
5922 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,1,2,3,4,5,6,7]
5923 ; AVX-NEXT: vmovdqa 160(%rdi), %xmm4
5924 ; AVX-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5925 ; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
5926 ; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,3,2,3,4,5,6,7]
5927 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
5928 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm0[4,5,6,7]
5929 ; AVX-NEXT: vmovaps {{.*#+}} ymm5 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,65535,65535,65535]
5930 ; AVX-NEXT: vandps %ymm5, %ymm3, %ymm3
5931 ; AVX-NEXT: vmovaps 224(%rdi), %xmm0
5932 ; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5933 ; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm0[0,1,0,1]
5934 ; AVX-NEXT: vandnps %ymm4, %ymm5, %ymm4
5935 ; AVX-NEXT: vorps %ymm4, %ymm3, %ymm3
5936 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm0
5937 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5938 ; AVX-NEXT: vmovdqa 576(%rdi), %xmm0
5939 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5940 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
5941 ; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]
5942 ; AVX-NEXT: vmovdqa 592(%rdi), %xmm12
5943 ; AVX-NEXT: vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm12[1]
5944 ; AVX-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5945 ; AVX-NEXT: vmovdqa 560(%rdi), %xmm0
5946 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5947 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
5948 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,3,2,3,4,5,6,7]
5949 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3,4,5,6,7]
5950 ; AVX-NEXT: vmovdqa 624(%rdi), %xmm3
5951 ; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5952 ; AVX-NEXT: vmovdqa 608(%rdi), %xmm0
5953 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5954 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0,1],xmm3[2,3],xmm0[4,5,6,7]
5955 ; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm3
5956 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm3[5,6,7]
5957 ; AVX-NEXT: vmovdqa 496(%rdi), %xmm0
5958 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5959 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
5960 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,1,2,3,4,5,6,7]
5961 ; AVX-NEXT: vmovdqa 480(%rdi), %xmm9
5962 ; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm9[0,2,2,3]
5963 ; AVX-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5964 ; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,3,2,3,4,5,6,7]
5965 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
5966 ; AVX-NEXT: vmovdqa 528(%rdi), %xmm0
5967 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5968 ; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[0,1,0,3]
5969 ; AVX-NEXT: vmovdqa 512(%rdi), %xmm13
5970 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm13[4],xmm4[5,6,7]
5971 ; AVX-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5972 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4,5,6,7]
5973 ; AVX-NEXT: vandps %ymm5, %ymm3, %ymm3
5974 ; AVX-NEXT: vmovaps 544(%rdi), %xmm0
5975 ; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm0[0,1,0,1]
5976 ; AVX-NEXT: vmovaps %xmm0, %xmm11
5977 ; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5978 ; AVX-NEXT: vandnps %ymm4, %ymm5, %ymm4
5979 ; AVX-NEXT: vorps %ymm4, %ymm3, %ymm3
5980 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm0
5981 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
5982 ; AVX-NEXT: vmovdqa 96(%rdi), %xmm10
5983 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm10[0,1,1,3]
5984 ; AVX-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5985 ; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]
5986 ; AVX-NEXT: vmovdqa 112(%rdi), %xmm0
5987 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5988 ; AVX-NEXT: vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm0[1]
5989 ; AVX-NEXT: vmovdqa 80(%rdi), %xmm0
5990 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5991 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
5992 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,3,2,3,4,5,6,7]
5993 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3,4,5,6,7]
5994 ; AVX-NEXT: vmovdqa 144(%rdi), %xmm7
5995 ; AVX-NEXT: vmovdqa 128(%rdi), %xmm6
5996 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm6[0,1],xmm7[2,3],xmm6[4,5,6,7]
5997 ; AVX-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5998 ; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
5999 ; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm3
6000 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm3[5,6,7]
6001 ; AVX-NEXT: vmovdqa 16(%rdi), %xmm0
6002 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6003 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,1,2,3]
6004 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,1,2,3,4,5,6,7]
6005 ; AVX-NEXT: vmovdqa (%rdi), %xmm0
6006 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6007 ; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]
6008 ; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,3,2,3,4,5,6,7]
6009 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
6010 ; AVX-NEXT: vmovdqa 32(%rdi), %xmm0
6011 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6012 ; AVX-NEXT: vmovdqa 48(%rdi), %xmm4
6013 ; AVX-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6014 ; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,3]
6015 ; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm0[4],xmm4[5,6,7]
6016 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4,5,6,7]
6017 ; AVX-NEXT: vandps %ymm5, %ymm3, %ymm3
6018 ; AVX-NEXT: vmovaps 64(%rdi), %xmm0
6019 ; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6020 ; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm0[0,1,0,1]
6021 ; AVX-NEXT: vandnps %ymm4, %ymm5, %ymm4
6022 ; AVX-NEXT: vorps %ymm4, %ymm3, %ymm3
6023 ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm0
6024 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6025 ; AVX-NEXT: vmovdqa 464(%rdi), %xmm8
6026 ; AVX-NEXT: vmovdqa 448(%rdi), %xmm0
6027 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6028 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0,1],xmm8[2,3],xmm0[4,5,6,7]
6029 ; AVX-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6030 ; AVX-NEXT: vpshufb %xmm1, %xmm2, %xmm1
6031 ; AVX-NEXT: vmovdqa 416(%rdi), %xmm0
6032 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6033 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
6034 ; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]
6035 ; AVX-NEXT: vmovdqa 432(%rdi), %xmm0
6036 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6037 ; AVX-NEXT: vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm0[1]
6038 ; AVX-NEXT: vmovdqa 400(%rdi), %xmm0
6039 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6040 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
6041 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,3,2,3,4,5,6,7]
6042 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3,4,5,6,7]
6043 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3,4],xmm1[5,6,7]
6044 ; AVX-NEXT: vmovdqa 336(%rdi), %xmm0
6045 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6046 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
6047 ; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7]
6048 ; AVX-NEXT: vmovdqa 320(%rdi), %xmm0
6049 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6050 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
6051 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,3,2,3,4,5,6,7]
6052 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
6053 ; AVX-NEXT: vmovdqa 368(%rdi), %xmm0
6054 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6055 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,1,0,3]
6056 ; AVX-NEXT: vmovdqa 352(%rdi), %xmm0
6057 ; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6058 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm0[4],xmm3[5,6,7]
6059 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5,6,7]
6060 ; AVX-NEXT: vandps %ymm5, %ymm2, %ymm2
6061 ; AVX-NEXT: vmovaps 384(%rdi), %xmm0
6062 ; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6063 ; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm0[0,1,0,1]
6064 ; AVX-NEXT: vandnps %ymm3, %ymm5, %ymm3
6065 ; AVX-NEXT: vorps %ymm3, %ymm2, %ymm2
6066 ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm0
6067 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6068 ; AVX-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm1 # 16-byte Folded Reload
6069 ; AVX-NEXT: # xmm1 = xmm15[0,1],mem[2,3],xmm15[4,5,6,7]
6070 ; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,0,4,5,6,7]
6071 ; AVX-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]
6072 ; AVX-NEXT: vpshufd $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6073 ; AVX-NEXT: # xmm2 = mem[0,3,2,3]
6074 ; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]
6075 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3,4,5,6,7]
6076 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6077 ; AVX-NEXT: vpblendw $207, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
6078 ; AVX-NEXT: # xmm3 = mem[0,1,2,3],xmm0[4,5],mem[6,7]
6079 ; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,8,9,4,5,14,15,8,9]
6080 ; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm3
6081 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0,1,2,3,4],xmm3[5,6,7]
6082 ; AVX-NEXT: vpshufd $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6083 ; AVX-NEXT: # xmm1 = mem[0,3,2,3]
6084 ; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,2,2,3,4,5,6,7]
6085 ; AVX-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload
6086 ; AVX-NEXT: vpsrlq $48, %xmm0, %xmm15
6087 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm15 = xmm1[0],xmm15[0],xmm1[1],xmm15[1]
6088 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6089 ; AVX-NEXT: vpblendw $204, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
6090 ; AVX-NEXT: # xmm0 = xmm0[0,1],mem[2,3],xmm0[4,5],mem[6,7]
6091 ; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,0,1,0,1,0,1,10,11,4,5,14,15,6,7]
6092 ; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm0
6093 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm15[0,1,2],xmm0[3,4,5,6,7]
6094 ; AVX-NEXT: vandps %ymm5, %ymm0, %ymm0
6095 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
6096 ; AVX-NEXT: vpsllq $48, %xmm4, %xmm15
6097 ; AVX-NEXT: vandnps %ymm15, %ymm5, %ymm15
6098 ; AVX-NEXT: vorps %ymm0, %ymm15, %ymm0
6099 ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
6100 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6101 ; AVX-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm0 # 16-byte Folded Reload
6102 ; AVX-NEXT: # xmm0 = xmm12[0,1],mem[2,3],xmm12[4,5,6,7]
6103 ; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,0,4,5,6,7]
6104 ; AVX-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
6105 ; AVX-NEXT: vpshufd $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6106 ; AVX-NEXT: # xmm3 = mem[0,3,2,3]
6107 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
6108 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3,4,5,6,7]
6109 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
6110 ; AVX-NEXT: vpblendw $207, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm3 # 16-byte Folded Reload
6111 ; AVX-NEXT: # xmm3 = mem[0,1,2,3],xmm14[4,5],mem[6,7]
6112 ; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm3
6113 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm3[5,6,7]
6114 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm9[0,3,2,3]
6115 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
6116 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
6117 ; AVX-NEXT: vpsrlq $48, %xmm4, %xmm15
6118 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm15[0],xmm3[1],xmm15[1]
6119 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
6120 ; AVX-NEXT: vpblendw {{.*#+}} xmm15 = xmm13[0,1],xmm9[2,3],xmm13[4,5],xmm9[6,7]
6121 ; AVX-NEXT: vpshufb %xmm1, %xmm15, %xmm15
6122 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm15[3,4,5,6,7]
6123 ; AVX-NEXT: vandps %ymm5, %ymm3, %ymm3
6124 ; AVX-NEXT: vpsllq $48, %xmm11, %xmm15
6125 ; AVX-NEXT: vandnps %ymm15, %ymm5, %ymm15
6126 ; AVX-NEXT: vorps %ymm3, %ymm15, %ymm3
6127 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm3, %ymm0
6128 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6129 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
6130 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm13[0,1],xmm10[2,3],xmm13[4,5,6,7]
6131 ; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,0,4,5,6,7]
6132 ; AVX-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
6133 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
6134 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm12[0,3,2,3]
6135 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
6136 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3,4,5,6,7]
6137 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm6[0,1,2,3],xmm7[4,5],xmm6[6,7]
6138 ; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm3
6139 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm3[5,6,7]
6140 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
6141 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm11[0,3,2,3]
6142 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
6143 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
6144 ; AVX-NEXT: vpsrlq $48, %xmm10, %xmm15
6145 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm15[0],xmm3[1],xmm15[1]
6146 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
6147 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
6148 ; AVX-NEXT: vpblendw {{.*#+}} xmm15 = xmm4[0,1],xmm6[2,3],xmm4[4,5],xmm6[6,7]
6149 ; AVX-NEXT: vpshufb %xmm1, %xmm15, %xmm15
6150 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm15[3,4,5,6,7]
6151 ; AVX-NEXT: vandps %ymm5, %ymm3, %ymm3
6152 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
6153 ; AVX-NEXT: vpsllq $48, %xmm7, %xmm15
6154 ; AVX-NEXT: vandnps %ymm15, %ymm5, %ymm15
6155 ; AVX-NEXT: vorps %ymm3, %ymm15, %ymm3
6156 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm3, %ymm0
6157 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6158 ; AVX-NEXT: vpblendw $207, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm0 # 16-byte Folded Reload
6159 ; AVX-NEXT: # xmm0 = mem[0,1,2,3],xmm8[4,5],mem[6,7]
6160 ; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm0
6161 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
6162 ; AVX-NEXT: vpblendw $243, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm2 # 16-byte Folded Reload
6163 ; AVX-NEXT: # xmm2 = mem[0,1],xmm8[2,3],mem[4,5,6,7]
6164 ; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,0,4,5,6,7]
6165 ; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]
6166 ; AVX-NEXT: vpshufd $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
6167 ; AVX-NEXT: # xmm3 = mem[0,3,2,3]
6168 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,2,2,3,4,5,6,7]
6169 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3,4,5,6,7]
6170 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3,4],xmm0[5,6,7]
6171 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6172 ; AVX-NEXT: vpblendw $204, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
6173 ; AVX-NEXT: # xmm2 = xmm2[0,1],mem[2,3],xmm2[4,5],mem[6,7]
6174 ; AVX-NEXT: vpshufb %xmm1, %xmm2, %xmm1
6175 ; AVX-NEXT: vpshufd $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6176 ; AVX-NEXT: # xmm2 = mem[0,3,2,3]
6177 ; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]
6178 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6179 ; AVX-NEXT: vpsrlq $48, %xmm3, %xmm3
6180 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
6181 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3,4,5,6,7]
6182 ; AVX-NEXT: vandps %ymm5, %ymm1, %ymm1
6183 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6184 ; AVX-NEXT: vpsllq $48, %xmm2, %xmm2
6185 ; AVX-NEXT: vandnps %ymm2, %ymm5, %ymm2
6186 ; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
6187 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
6188 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6189 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6190 ; AVX-NEXT: vpblendw $207, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
6191 ; AVX-NEXT: # xmm1 = mem[0,1,2,3],xmm0[4,5],mem[6,7]
6192 ; AVX-NEXT: vmovdqa {{.*#+}} xmm0 = [u,u,u,u,8,9,2,3,12,13,12,13,12,13,12,13]
6193 ; AVX-NEXT: vpshufb %xmm0, %xmm1, %xmm1
6194 ; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6195 ; AVX-NEXT: # xmm2 = mem[3,1,2,3]
6196 ; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7]
6197 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3,4,5,6,7]
6198 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6199 ; AVX-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm3 # 16-byte Folded Reload
6200 ; AVX-NEXT: # xmm3 = xmm2[0,1],mem[2,3],xmm2[4,5,6,7]
6201 ; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
6202 ; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm3
6203 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0,1,2,3,4],xmm3[5,6,7]
6204 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
6205 ; AVX-NEXT: vpblendw $243, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm5 # 16-byte Folded Reload
6206 ; AVX-NEXT: # xmm5 = mem[0,1],xmm1[2,3],mem[4,5,6,7]
6207 ; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [2,3,2,3,2,3,2,3,12,13,6,7,12,13,14,15]
6208 ; AVX-NEXT: vpshufb %xmm1, %xmm5, %xmm5
6209 ; AVX-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
6210 ; AVX-NEXT: # xmm15 = mem[0,1,1,3]
6211 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,7,6,7]
6212 ; AVX-NEXT: vpunpckhdq (%rsp), %xmm15, %xmm15 # 16-byte Folded Reload
6213 ; AVX-NEXT: # xmm15 = xmm15[2],mem[2],xmm15[3],mem[3]
6214 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm15[0,1,2],xmm5[3,4,5],xmm15[6,7]
6215 ; AVX-NEXT: vpshufd $36, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
6216 ; AVX-NEXT: # xmm15 = mem[0,1,2,0]
6217 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,6,5]
6218 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5],xmm15[6,7]
6219 ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm5, %ymm3
6220 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6221 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6222 ; AVX-NEXT: vpblendw $207, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
6223 ; AVX-NEXT: # xmm3 = mem[0,1,2,3],xmm3[4,5],mem[6,7]
6224 ; AVX-NEXT: vpshufb %xmm0, %xmm3, %xmm3
6225 ; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
6226 ; AVX-NEXT: # xmm5 = mem[3,1,2,3]
6227 ; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[2,1,2,3,4,5,6,7]
6228 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3,4,5,6,7]
6229 ; AVX-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm5 # 16-byte Folded Reload
6230 ; AVX-NEXT: # xmm5 = xmm14[0,1],mem[2,3],xmm14[4,5,6,7]
6231 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
6232 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm5[5,6,7]
6233 ; AVX-NEXT: vpblendw $243, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm5 # 16-byte Folded Reload
6234 ; AVX-NEXT: # xmm5 = mem[0,1],xmm9[2,3],mem[4,5,6,7]
6235 ; AVX-NEXT: vpshufb %xmm1, %xmm5, %xmm5
6236 ; AVX-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
6237 ; AVX-NEXT: # xmm15 = mem[0,1,1,3]
6238 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,7,6,7]
6239 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
6240 ; AVX-NEXT: vpunpckhdq {{.*#+}} xmm15 = xmm15[2],xmm14[2],xmm15[3],xmm14[3]
6241 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm15[0,1,2],xmm5[3,4,5],xmm15[6,7]
6242 ; AVX-NEXT: vpshufd $36, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
6243 ; AVX-NEXT: # xmm15 = mem[0,1,2,0]
6244 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,6,5]
6245 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5],xmm15[6,7]
6246 ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm5, %ymm3
6247 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6248 ; AVX-NEXT: vpblendw $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13, %xmm3 # 16-byte Folded Reload
6249 ; AVX-NEXT: # xmm3 = xmm13[0,1,2,3],mem[4,5],xmm13[6,7]
6250 ; AVX-NEXT: vpshufb %xmm0, %xmm3, %xmm3
6251 ; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm12[3,1,2,3]
6252 ; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[2,1,2,3,4,5,6,7]
6253 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3,4,5,6,7]
6254 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
6255 ; AVX-NEXT: vpblendw $243, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
6256 ; AVX-NEXT: # xmm5 = mem[0,1],xmm5[2,3],mem[4,5,6,7]
6257 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
6258 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm5[5,6,7]
6259 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0,1],xmm6[2,3],xmm4[4,5,6,7]
6260 ; AVX-NEXT: vpshufb %xmm1, %xmm5, %xmm5
6261 ; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm11[0,1,1,3]
6262 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,7,6,7]
6263 ; AVX-NEXT: vpunpckhdq {{.*#+}} xmm15 = xmm15[2],xmm10[2],xmm15[3],xmm10[3]
6264 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm15[0,1,2],xmm5[3,4,5],xmm15[6,7]
6265 ; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm7[0,1,2,0]
6266 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,6,5]
6267 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5],xmm15[6,7]
6268 ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm5, %ymm3
6269 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6270 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
6271 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
6272 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm12[0,1],xmm13[2,3],xmm12[4,5,6,7]
6273 ; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2
6274 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
6275 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm7[0,1,2,3],xmm8[4,5],xmm7[6,7]
6276 ; AVX-NEXT: vpshufb %xmm0, %xmm3, %xmm0
6277 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
6278 ; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm8[3,1,2,3]
6279 ; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,1,2,3,4,5,6,7]
6280 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3,4,5,6,7]
6281 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm2[5,6,7]
6282 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
6283 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
6284 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm10[0,1],xmm11[2,3],xmm10[4,5,6,7]
6285 ; AVX-NEXT: vpshufb %xmm1, %xmm2, %xmm1
6286 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
6287 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm6[0,1,1,3]
6288 ; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,7]
6289 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
6290 ; AVX-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
6291 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3,4,5],xmm2[6,7]
6292 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
6293 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm9[0,1,2,0]
6294 ; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6,5]
6295 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm2[6,7]
6296 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
6297 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6298 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6299 ; AVX-NEXT: vpblendw $204, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
6300 ; AVX-NEXT: # xmm1 = xmm0[0,1],mem[2,3],xmm0[4,5],mem[6,7]
6301 ; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,0,1,10,11,4,5,14,15,14,15,14,15,14,15]
6302 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6303 ; AVX-NEXT: vpsrlq $48, %xmm2, %xmm2
6304 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3,4,5,6,7]
6305 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6306 ; AVX-NEXT: vpblendw $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm3 # 16-byte Folded Reload
6307 ; AVX-NEXT: # xmm3 = xmm2[0,1,2,3],mem[4,5],xmm2[6,7]
6308 ; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
6309 ; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm3
6310 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0,1,2,3,4],xmm3[5,6,7]
6311 ; AVX-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
6312 ; AVX-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm5 # 16-byte Folded Reload
6313 ; AVX-NEXT: # xmm5 = xmm1[0,1],mem[2,3],xmm1[4,5,6,7]
6314 ; AVX-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[6,7,0,1,10,11,10,11,8,9,10,11,12,13,14,15]
6315 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6316 ; AVX-NEXT: vpblendw $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm15 # 16-byte Folded Reload
6317 ; AVX-NEXT: # xmm15 = xmm0[0,1,2,3],mem[4,5],xmm0[6,7]
6318 ; AVX-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[2,2,2,2,4,5,6,7]
6319 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,7,4,6,7]
6320 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm15[3,4,5],xmm5[6,7]
6321 ; AVX-NEXT: vpshufd $196, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
6322 ; AVX-NEXT: # xmm15 = mem[0,1,0,3]
6323 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,5,6]
6324 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5],xmm15[6,7]
6325 ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm5, %ymm3
6326 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6327 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6328 ; AVX-NEXT: vpblendw $204, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
6329 ; AVX-NEXT: # xmm3 = xmm0[0,1],mem[2,3],xmm0[4,5],mem[6,7]
6330 ; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [u,u,0,1,10,11,4,5,14,15,14,15,14,15,14,15]
6331 ; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm3
6332 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6333 ; AVX-NEXT: vpsrlq $48, %xmm0, %xmm5
6334 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0],xmm3[1,2,3,4,5,6,7]
6335 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
6336 ; AVX-NEXT: vpblendw $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm5 # 16-byte Folded Reload
6337 ; AVX-NEXT: # xmm5 = xmm0[0,1,2,3],mem[4,5],xmm0[6,7]
6338 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
6339 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm5[5,6,7]
6340 ; AVX-NEXT: vpblendw $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm5 # 16-byte Folded Reload
6341 ; AVX-NEXT: # xmm5 = xmm14[0,1],mem[2,3],xmm14[4,5,6,7]
6342 ; AVX-NEXT: vmovdqa {{.*#+}} xmm0 = [6,7,0,1,10,11,10,11,8,9,10,11,12,13,14,15]
6343 ; AVX-NEXT: vpshufb %xmm0, %xmm5, %xmm5
6344 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
6345 ; AVX-NEXT: vpblendw $207, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm15 # 16-byte Folded Reload
6346 ; AVX-NEXT: # xmm15 = mem[0,1,2,3],xmm14[4,5],mem[6,7]
6347 ; AVX-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[2,2,2,2,4,5,6,7]
6348 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,7,4,6,7]
6349 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm15[3,4,5],xmm5[6,7]
6350 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
6351 ; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm14[0,1,0,3]
6352 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,5,6]
6353 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5],xmm15[6,7]
6354 ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm5, %ymm3
6355 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6356 ; AVX-NEXT: vpblendw $51, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm3 # 16-byte Folded Reload
6357 ; AVX-NEXT: # xmm3 = mem[0,1],xmm7[2,3],mem[4,5],xmm7[6,7]
6358 ; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm3
6359 ; AVX-NEXT: vpsrlq $48, %xmm8, %xmm5
6360 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0],xmm3[1,2,3,4,5,6,7]
6361 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm12[0,1,2,3],xmm13[4,5],xmm12[6,7]
6362 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
6363 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm5[5,6,7]
6364 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0,1],xmm6[2,3],xmm4[4,5,6,7]
6365 ; AVX-NEXT: vpshufb %xmm0, %xmm5, %xmm5
6366 ; AVX-NEXT: vmovdqa %xmm0, %xmm4
6367 ; AVX-NEXT: vpblendw {{.*#+}} xmm15 = xmm10[0,1,2,3],xmm11[4,5],xmm10[6,7]
6368 ; AVX-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[2,2,2,2,4,5,6,7]
6369 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,7,4,6,7]
6370 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm15[3,4,5],xmm5[6,7]
6371 ; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm9[0,1,0,3]
6372 ; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,5,6]
6373 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5],xmm15[6,7]
6374 ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm5, %ymm3
6375 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6376 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
6377 ; AVX-NEXT: vpblendw $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13, %xmm3 # 16-byte Folded Reload
6378 ; AVX-NEXT: # xmm3 = xmm13[0,1,2,3],mem[4,5],xmm13[6,7]
6379 ; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2
6380 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
6381 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
6382 ; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm10[0,1],xmm11[2,3],xmm10[4,5],xmm11[6,7]
6383 ; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm0
6384 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
6385 ; AVX-NEXT: vpsrlq $48, %xmm7, %xmm3
6386 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0],xmm0[1,2,3,4,5,6,7]
6387 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm2[5,6,7]
6388 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
6389 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6390 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm6[0,1],xmm3[2,3],xmm6[4,5,6,7]
6391 ; AVX-NEXT: vpshufb %xmm4, %xmm2, %xmm1
6392 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
6393 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
6394 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm8[4,5],xmm4[6,7]
6395 ; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,2,2,2,4,5,6,7]
6396 ; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,4,6,7]
6397 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3,4,5],xmm1[6,7]
6398 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
6399 ; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm9[0,1,0,3]
6400 ; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,5,6]
6401 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm2[6,7]
6402 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
6403 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6404 ; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
6405 ; AVX-NEXT: # xmm0 = mem[3,1,2,3]
6406 ; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
6407 ; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6408 ; AVX-NEXT: # xmm1 = mem[0,2,2,3]
6409 ; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
6410 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
6411 ; AVX-NEXT: vpshufd $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
6412 ; AVX-NEXT: # xmm1 = mem[0,3,2,3]
6413 ; AVX-NEXT: vpblendw $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
6414 ; AVX-NEXT: # xmm1 = xmm1[0,1,2],mem[3],xmm1[4,5,6,7]
6415 ; AVX-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6416 ; AVX-NEXT: # xmm2 = mem[2,3,2,3]
6417 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3,4,5,6,7]
6418 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
6419 ; AVX-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
6420 ; AVX-NEXT: vpblendw $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
6421 ; AVX-NEXT: # xmm1 = xmm1[0,1,2,3],mem[4,5],xmm1[6,7]
6422 ; AVX-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6423 ; AVX-NEXT: # xmm2 = mem[1,1,1,1]
6424 ; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
6425 ; AVX-NEXT: # xmm5 = mem[0,2,2,3]
6426 ; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,0,3,4,5,6,7]
6427 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
6428 ; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [8,9,2,3,12,13,12,13,8,9,12,13,12,13,14,15]
6429 ; AVX-NEXT: vpshufb %xmm2, %xmm1, %xmm1
6430 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[3,4,5],xmm1[6,7]
6431 ; AVX-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
6432 ; AVX-NEXT: # xmm5 = mem[0,1,1,3]
6433 ; AVX-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,4,7]
6434 ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm5[6,7]
6435 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm1
6436 ; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
6437 ; AVX-NEXT: # xmm0 = mem[3,1,2,3]
6438 ; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
6439 ; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
6440 ; AVX-NEXT: # xmm5 = mem[0,2,2,3]
6441 ; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,0,3,4,5,6,7]
6442 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
6443 ; AVX-NEXT: vpshufd $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
6444 ; AVX-NEXT: # xmm5 = mem[0,3,2,3]
6445 ; AVX-NEXT: vpblendw $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
6446 ; AVX-NEXT: # xmm5 = xmm5[0,1,2],mem[3],xmm5[4,5,6,7]
6447 ; AVX-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
6448 ; AVX-NEXT: # xmm15 = mem[2,3,2,3]
6449 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm15[0],xmm5[1,2,3,4,5,6,7]
6450 ; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm5[0,1,2,3],xmm0[4,5,6,7]
6451 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
6452 ; AVX-NEXT: vpblendw $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
6453 ; AVX-NEXT: # xmm5 = xmm5[0,1,2,3],mem[4,5],xmm5[6,7]
6454 ; AVX-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
6455 ; AVX-NEXT: # xmm15 = mem[1,1,1,1]
6456 ; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
6457 ; AVX-NEXT: # xmm12 = mem[0,2,2,3]
6458 ; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[0,1,0,3,4,5,6,7]
6459 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm12[0],xmm15[0],xmm12[1],xmm15[1]
6460 ; AVX-NEXT: vpshufb %xmm2, %xmm5, %xmm5
6461 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm12[3,4,5],xmm5[6,7]
6462 ; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm14[0,1,1,3]
6463 ; AVX-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5,4,7]
6464 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5],xmm12[6,7]
6465 ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm5, %ymm0
6466 ; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm13[3,1,2,3]
6467 ; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,2,1,4,5,6,7]
6468 ; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
6469 ; AVX-NEXT: # xmm12 = mem[0,2,2,3]
6470 ; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[0,1,0,3,4,5,6,7]
6471 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm12[0],xmm5[0],xmm12[1],xmm5[1]
6472 ; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[0,3,2,3]
6473 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm11[3],xmm12[4,5,6,7]
6474 ; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm7[2,3,2,3]
6475 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm15[0],xmm12[1,2,3,4,5,6,7]
6476 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm12[0,1,2,3],xmm5[4,5,6,7]
6477 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm6[0,1,2,3],xmm3[4,5],xmm6[6,7]
6478 ; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm4[1,1,1,1]
6479 ; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm8[0,2,2,3]
6480 ; AVX-NEXT: vpshuflw {{.*#+}} xmm14 = xmm14[0,1,0,3,4,5,6,7]
6481 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1]
6482 ; AVX-NEXT: vpshufb %xmm2, %xmm12, %xmm12
6483 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm14[3,4,5],xmm12[6,7]
6484 ; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm9[0,1,1,3]
6485 ; AVX-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,5,4,7]
6486 ; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5],xmm14[6,7]
6487 ; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm12, %ymm12
6488 ; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
6489 ; AVX-NEXT: # xmm5 = mem[3,1,2,3]
6490 ; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,2,1,4,5,6,7]
6491 ; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload
6492 ; AVX-NEXT: # xmm13 = mem[0,2,2,3]
6493 ; AVX-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[0,1,0,3,4,5,6,7]
6494 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1]
6495 ; AVX-NEXT: vpshufd $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload
6496 ; AVX-NEXT: # xmm13 = mem[0,3,2,3]
6497 ; AVX-NEXT: vpblendw $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13, %xmm11 # 16-byte Folded Reload
6498 ; AVX-NEXT: # xmm11 = xmm13[0,1,2],mem[3],xmm13[4,5,6,7]
6499 ; AVX-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
6500 ; AVX-NEXT: # xmm10 = mem[2,3,2,3]
6501 ; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0],xmm11[1,2,3,4,5,6,7]
6502 ; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm10[0,1,2,3],xmm5[4,5,6,7]
6503 ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
6504 ; AVX-NEXT: vpblendw $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm8 # 16-byte Folded Reload
6505 ; AVX-NEXT: # xmm8 = xmm3[0,1,2,3],mem[4,5],xmm3[6,7]
6506 ; AVX-NEXT: vpshufb %xmm2, %xmm8, %xmm2
6507 ; AVX-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6508 ; AVX-NEXT: # xmm4 = mem[1,1,1,1]
6509 ; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload
6510 ; AVX-NEXT: # xmm7 = mem[0,2,2,3]
6511 ; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,0,3,4,5,6,7]
6512 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm7[0],xmm4[0],xmm7[1],xmm4[1]
6513 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[3,4,5],xmm2[6,7]
6514 ; AVX-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6515 ; AVX-NEXT: # xmm4 = mem[0,1,1,3]
6516 ; AVX-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,7]
6517 ; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5],xmm4[6,7]
6518 ; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm2, %ymm2
6519 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6520 ; AVX-NEXT: vmovaps %ymm3, 64(%rsi)
6521 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
6522 ; AVX-NEXT: vmovaps %ymm4, (%rsi)
6523 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
6524 ; AVX-NEXT: vmovaps %ymm4, 96(%rsi)
6525 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
6526 ; AVX-NEXT: vmovaps %ymm4, 32(%rsi)
6527 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6528 ; AVX-NEXT: vmovaps %ymm3, 64(%rdx)
6529 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6530 ; AVX-NEXT: vmovaps %ymm3, (%rdx)
6531 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6532 ; AVX-NEXT: vmovaps %ymm3, 96(%rdx)
6533 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6534 ; AVX-NEXT: vmovaps %ymm3, 32(%rdx)
6535 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6536 ; AVX-NEXT: vmovaps %ymm3, 64(%rcx)
6537 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6538 ; AVX-NEXT: vmovaps %ymm3, (%rcx)
6539 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6540 ; AVX-NEXT: vmovaps %ymm3, 96(%rcx)
6541 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6542 ; AVX-NEXT: vmovaps %ymm3, 32(%rcx)
6543 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6544 ; AVX-NEXT: vmovaps %ymm3, 64(%r8)
6545 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6546 ; AVX-NEXT: vmovaps %ymm3, (%r8)
6547 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6548 ; AVX-NEXT: vmovaps %ymm3, 96(%r8)
6549 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6550 ; AVX-NEXT: vmovaps %ymm3, 32(%r8)
6551 ; AVX-NEXT: vmovaps %ymm2, 64(%r9)
6552 ; AVX-NEXT: vmovaps %ymm12, (%r9)
6553 ; AVX-NEXT: vmovaps %ymm0, 96(%r9)
6554 ; AVX-NEXT: vmovaps %ymm1, 32(%r9)
6555 ; AVX-NEXT: addq $1032, %rsp # imm = 0x408
6556 ; AVX-NEXT: vzeroupper
6557 ; AVX-NEXT: retq
6558 ;
6559 ; AVX2-LABEL: load_i16_stride5_vf64:
6560 ; AVX2: # %bb.0:
6561 ; AVX2-NEXT: subq $1048, %rsp # imm = 0x418
6562 ; AVX2-NEXT: vmovdqa 384(%rdi), %ymm10
6563 ; AVX2-NEXT: vmovdqa 512(%rdi), %ymm4
6564 ; AVX2-NEXT: vmovdqa 480(%rdi), %ymm14
6565 ; AVX2-NEXT: vmovdqa 544(%rdi), %ymm7
6566 ; AVX2-NEXT: vmovdqa 576(%rdi), %ymm8
6567 ; AVX2-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6568 ; AVX2-NEXT: vmovdqa 192(%rdi), %ymm3
6569 ; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6570 ; AVX2-NEXT: vmovdqa 160(%rdi), %ymm5
6571 ; AVX2-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6572 ; AVX2-NEXT: vmovdqa 224(%rdi), %ymm0
6573 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6574 ; AVX2-NEXT: vmovdqa 256(%rdi), %ymm1
6575 ; AVX2-NEXT: vmovdqu %ymm1, (%rsp) # 32-byte Spill
6576 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
6577 ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
6578 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6],ymm1[7]
6579 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,10,11,4,5,14,15,8,9,10,11,4,5,6,7,16,17,26,27,20,21,30,31,24,25,26,27,20,21,22,23]
6580 ; AVX2-NEXT: vpshufb %ymm1, %ymm0, %ymm2
6581 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm5[0],ymm3[1],ymm5[2,3],ymm3[4],ymm5[5],ymm3[6],ymm5[7,8],ymm3[9],ymm5[10,11],ymm3[12],ymm5[13],ymm3[14],ymm5[15]
6582 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3
6583 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0],xmm3[1,2,3],xmm0[4,5],xmm3[6,7]
6584 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,6,7]
6585 ; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm3
6586 ; AVX2-NEXT: vpmovsxbw {{.*#+}} xmm5 = [65535,65535,65535,65535,65535,65535,65535,0]
6587 ; AVX2-NEXT: vpblendvb %ymm5, %ymm3, %ymm2, %ymm2
6588 ; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6589 ; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm7[0],ymm8[1,2],ymm7[3],ymm8[4],ymm7[5],ymm8[6,7],ymm7[8],ymm8[9,10],ymm7[11],ymm8[12],ymm7[13],ymm8[14,15]
6590 ; AVX2-NEXT: vmovdqa %ymm7, %ymm11
6591 ; AVX2-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6592 ; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
6593 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6],ymm3[7]
6594 ; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm14[0],ymm4[1],ymm14[2,3],ymm4[4],ymm14[5],ymm4[6],ymm14[7,8],ymm4[9],ymm14[10,11],ymm4[12],ymm14[13],ymm4[14],ymm14[15]
6595 ; AVX2-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6596 ; AVX2-NEXT: vmovdqa %ymm4, %ymm8
6597 ; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6598 ; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
6599 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1,2,3],xmm3[4,5],xmm4[6,7]
6600 ; AVX2-NEXT: vmovdqa 416(%rdi), %ymm13
6601 ; AVX2-NEXT: vpshufb %ymm1, %ymm2, %ymm2
6602 ; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm3
6603 ; AVX2-NEXT: vpblendvb %ymm5, %ymm3, %ymm2, %ymm2
6604 ; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6605 ; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm10[0],ymm13[1,2],ymm10[3],ymm13[4],ymm10[5],ymm13[6,7],ymm10[8],ymm13[9,10],ymm10[11],ymm13[12],ymm10[13],ymm13[14,15]
6606 ; AVX2-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6607 ; AVX2-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6608 ; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
6609 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6],ymm3[7]
6610 ; AVX2-NEXT: vmovdqa 352(%rdi), %ymm4
6611 ; AVX2-NEXT: vmovdqa 320(%rdi), %ymm6
6612 ; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm6[0],ymm4[1],ymm6[2,3],ymm4[4],ymm6[5],ymm4[6],ymm6[7,8],ymm4[9],ymm6[10,11],ymm4[12],ymm6[13],ymm4[14],ymm6[15]
6613 ; AVX2-NEXT: vmovdqa %ymm6, %ymm15
6614 ; AVX2-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6615 ; AVX2-NEXT: vmovdqa %ymm4, %ymm6
6616 ; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6617 ; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
6618 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1,2,3],xmm3[4,5],xmm4[6,7]
6619 ; AVX2-NEXT: vpshufb %ymm1, %ymm2, %ymm2
6620 ; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm3
6621 ; AVX2-NEXT: vpblendvb %ymm5, %ymm3, %ymm2, %ymm2
6622 ; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6623 ; AVX2-NEXT: vmovdqa 64(%rdi), %ymm9
6624 ; AVX2-NEXT: vmovdqa 96(%rdi), %ymm4
6625 ; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm9[0],ymm4[1,2],ymm9[3],ymm4[4],ymm9[5],ymm4[6,7],ymm9[8],ymm4[9,10],ymm9[11],ymm4[12],ymm9[13],ymm4[14,15]
6626 ; AVX2-NEXT: vmovdqa %ymm4, %ymm12
6627 ; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6628 ; AVX2-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6629 ; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
6630 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6],ymm3[7]
6631 ; AVX2-NEXT: vpshufb %ymm1, %ymm2, %ymm1
6632 ; AVX2-NEXT: vmovdqa (%rdi), %ymm5
6633 ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm7
6634 ; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm5[0],ymm7[1],ymm5[2,3],ymm7[4],ymm5[5],ymm7[6],ymm5[7,8],ymm7[9],ymm5[10,11],ymm7[12],ymm5[13],ymm7[14],ymm5[15]
6635 ; AVX2-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6636 ; AVX2-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6637 ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
6638 ; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
6639 ; AVX2-NEXT: vpshufb %xmm0, %xmm2, %xmm0
6640 ; AVX2-NEXT: vpmovsxbw {{.*#+}} xmm3 = [65535,65535,65535,65535,65535,65535,65535,0]
6641 ; AVX2-NEXT: vpblendvb %ymm3, %ymm0, %ymm1, %ymm0
6642 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6643 ; AVX2-NEXT: vmovdqu (%rsp), %ymm0 # 32-byte Reload
6644 ; AVX2-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
6645 ; AVX2-NEXT: # ymm0 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5],mem[6],ymm0[7,8],mem[9],ymm0[10,11],mem[12],ymm0[13],mem[14],ymm0[15]
6646 ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
6647 ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
6648 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
6649 ; AVX2-NEXT: vpblendw $74, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
6650 ; AVX2-NEXT: # ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4,5],mem[6],ymm0[7,8],mem[9],ymm0[10],mem[11],ymm0[12,13],mem[14],ymm0[15]
6651 ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
6652 ; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5,6],xmm2[7]
6653 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm0 = [2,3,12,13,6,7,0,1,10,11,6,7,8,9,8,9,18,19,28,29,22,23,16,17,26,27,22,23,24,25,24,25]
6654 ; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm1
6655 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [2,3,12,13,6,7,0,1,10,11,4,5,14,15,10,11]
6656 ; AVX2-NEXT: vpshufb %xmm4, %xmm2, %xmm2
6657 ; AVX2-NEXT: vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
6658 ; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6659 ; AVX2-NEXT: vpblendw $173, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm1 # 32-byte Folded Reload
6660 ; AVX2-NEXT: # ymm1 = mem[0],ymm11[1],mem[2,3],ymm11[4],mem[5],ymm11[6],mem[7,8],ymm11[9],mem[10,11],ymm11[12],mem[13],ymm11[14],mem[15]
6661 ; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
6662 ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6,7]
6663 ; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm8[0],ymm14[1],ymm8[2],ymm14[3],ymm8[4,5],ymm14[6],ymm8[7,8],ymm14[9],ymm8[10],ymm14[11],ymm8[12,13],ymm14[14],ymm8[15]
6664 ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm8
6665 ; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm8[2,3],xmm2[4,5,6],xmm8[7]
6666 ; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm1
6667 ; AVX2-NEXT: vpshufb %xmm4, %xmm2, %xmm2
6668 ; AVX2-NEXT: vpblendvb %ymm3, %ymm2, %ymm1, %ymm11
6669 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm13[0],ymm10[1],ymm13[2,3],ymm10[4],ymm13[5],ymm10[6],ymm13[7,8],ymm10[9],ymm13[10,11],ymm10[12],ymm13[13],ymm10[14],ymm13[15]
6670 ; AVX2-NEXT: vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
6671 ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm8[5],ymm1[6,7]
6672 ; AVX2-NEXT: vpblendw {{.*#+}} ymm8 = ymm6[0],ymm15[1],ymm6[2],ymm15[3],ymm6[4,5],ymm15[6],ymm6[7,8],ymm15[9],ymm6[10],ymm15[11],ymm6[12,13],ymm15[14],ymm6[15]
6673 ; AVX2-NEXT: vextracti128 $1, %ymm8, %xmm10
6674 ; AVX2-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3],xmm8[4,5,6],xmm10[7]
6675 ; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm1
6676 ; AVX2-NEXT: vpshufb %xmm4, %xmm8, %xmm8
6677 ; AVX2-NEXT: vpblendvb %ymm3, %ymm8, %ymm1, %ymm6
6678 ; AVX2-NEXT: vpblendw {{.*#+}} ymm8 = ymm12[0],ymm9[1],ymm12[2,3],ymm9[4],ymm12[5],ymm9[6],ymm12[7,8],ymm9[9],ymm12[10,11],ymm9[12],ymm12[13],ymm9[14],ymm12[15]
6679 ; AVX2-NEXT: vpermq {{.*#+}} ymm10 = ymm8[2,3,0,1]
6680 ; AVX2-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm10[5],ymm8[6,7]
6681 ; AVX2-NEXT: vpshufb %ymm0, %ymm8, %ymm0
6682 ; AVX2-NEXT: vpblendw {{.*#+}} ymm8 = ymm7[0],ymm5[1],ymm7[2],ymm5[3],ymm7[4,5],ymm5[6],ymm7[7,8],ymm5[9],ymm7[10],ymm5[11],ymm7[12,13],ymm5[14],ymm7[15]
6683 ; AVX2-NEXT: vextracti128 $1, %ymm8, %xmm10
6684 ; AVX2-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3],xmm8[4,5,6],xmm10[7]
6685 ; AVX2-NEXT: vpshufb %xmm4, %xmm8, %xmm4
6686 ; AVX2-NEXT: vpblendvb %ymm3, %ymm4, %ymm0, %ymm2
6687 ; AVX2-NEXT: vmovdqa 304(%rdi), %xmm9
6688 ; AVX2-NEXT: vmovdqa 288(%rdi), %xmm3
6689 ; AVX2-NEXT: vpblendd {{.*#+}} xmm8 = xmm3[0],xmm9[1],xmm3[2,3]
6690 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm0 = [6,7,2,3,4,5,6,7,6,7,2,3,12,13,6,7]
6691 ; AVX2-NEXT: vpshufb %xmm0, %xmm8, %xmm8
6692 ; AVX2-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
6693 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
6694 ; AVX2-NEXT: vpblendw {{.*#+}} ymm8 = ymm4[0,1,2,3,4],ymm8[5,6,7],ymm4[8,9,10,11,12],ymm8[13,14,15]
6695 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
6696 ; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6697 ; AVX2-NEXT: vmovdqa 624(%rdi), %xmm5
6698 ; AVX2-NEXT: vmovdqa 608(%rdi), %xmm12
6699 ; AVX2-NEXT: vpblendd {{.*#+}} xmm10 = xmm12[0],xmm5[1],xmm12[2,3]
6700 ; AVX2-NEXT: vmovdqa %xmm5, %xmm15
6701 ; AVX2-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6702 ; AVX2-NEXT: vpshufb %xmm0, %xmm10, %xmm10
6703 ; AVX2-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
6704 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
6705 ; AVX2-NEXT: vpblendw {{.*#+}} ymm10 = ymm4[0,1,2,3,4],ymm10[5,6,7],ymm4[8,9,10,11,12],ymm10[13,14,15]
6706 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm10[4,5,6,7]
6707 ; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6708 ; AVX2-NEXT: vmovdqa 464(%rdi), %xmm5
6709 ; AVX2-NEXT: vmovdqa 448(%rdi), %xmm4
6710 ; AVX2-NEXT: vpblendd {{.*#+}} xmm13 = xmm4[0],xmm5[1],xmm4[2,3]
6711 ; AVX2-NEXT: vmovdqa %xmm5, %xmm10
6712 ; AVX2-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6713 ; AVX2-NEXT: vmovdqa %xmm4, %xmm8
6714 ; AVX2-NEXT: vpshufb %xmm0, %xmm13, %xmm13
6715 ; AVX2-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
6716 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
6717 ; AVX2-NEXT: vpblendw {{.*#+}} ymm13 = ymm4[0,1,2,3,4],ymm13[5,6,7],ymm4[8,9,10,11,12],ymm13[13,14,15]
6718 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm13[4,5,6,7]
6719 ; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6720 ; AVX2-NEXT: vmovdqa 144(%rdi), %xmm5
6721 ; AVX2-NEXT: vmovdqa 128(%rdi), %xmm4
6722 ; AVX2-NEXT: vpblendd {{.*#+}} xmm14 = xmm4[0],xmm5[1],xmm4[2,3]
6723 ; AVX2-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6724 ; AVX2-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6725 ; AVX2-NEXT: vpshufb %xmm0, %xmm14, %xmm0
6726 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
6727 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
6728 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm7[0,1,2,3,4],ymm0[5,6,7],ymm7[8,9,10,11,12],ymm0[13,14,15]
6729 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
6730 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6731 ; AVX2-NEXT: vmovdqa %xmm3, %xmm14
6732 ; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6733 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm9[2],xmm3[3]
6734 ; AVX2-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6735 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm7 = [0,1,2,3,4,5,6,7,8,9,4,5,14,15,8,9]
6736 ; AVX2-NEXT: vpshufb %xmm7, %xmm0, %xmm0
6737 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
6738 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
6739 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7],ymm1[8,9,10,11,12],ymm0[13,14,15]
6740 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
6741 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6742 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm12[0,1],xmm15[2],xmm12[3]
6743 ; AVX2-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6744 ; AVX2-NEXT: vpshufb %xmm7, %xmm0, %xmm0
6745 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
6746 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm11[0,1,2,3,4],ymm0[5,6,7],ymm11[8,9,10,11,12],ymm0[13,14,15]
6747 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm0[4,5,6,7]
6748 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6749 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm8[0,1],xmm10[2],xmm8[3]
6750 ; AVX2-NEXT: vmovdqa %xmm8, %xmm10
6751 ; AVX2-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
6752 ; AVX2-NEXT: vpshufb %xmm7, %xmm0, %xmm0
6753 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
6754 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm6[0,1,2,3,4],ymm0[5,6,7],ymm6[8,9,10,11,12],ymm0[13,14,15]
6755 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
6756 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6757 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm5[2],xmm4[3]
6758 ; AVX2-NEXT: vpshufb %xmm7, %xmm0, %xmm0
6759 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
6760 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7],ymm2[8,9,10,11,12],ymm0[13,14,15]
6761 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
6762 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6763 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
6764 ; AVX2-NEXT: vpblendw $82, (%rsp), %ymm13, %ymm0 # 32-byte Folded Reload
6765 ; AVX2-NEXT: # ymm0 = ymm13[0],mem[1],ymm13[2,3],mem[4],ymm13[5],mem[6],ymm13[7,8],mem[9],ymm13[10,11],mem[12],ymm13[13],mem[14],ymm13[15]
6766 ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
6767 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6],ymm0[7]
6768 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
6769 ; AVX2-NEXT: vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm1 # 32-byte Folded Reload
6770 ; AVX2-NEXT: # ymm1 = ymm15[0,1],mem[2],ymm15[3],mem[4],ymm15[5,6],mem[7],ymm15[8,9],mem[10],ymm15[11],mem[12],ymm15[13,14],mem[15]
6771 ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
6772 ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3,4],xmm1[5,6,7]
6773 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
6774 ; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm3
6775 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
6776 ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm1
6777 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0,1,2],ymm3[3,4,5,6,7]
6778 ; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm9[0],xmm14[1],xmm9[2,3]
6779 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
6780 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
6781 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
6782 ; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm3[0,1,2,3,4],ymm6[5,6,7],ymm3[8,9,10,11,12],ymm6[13,14,15]
6783 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5,6,7]
6784 ; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6785 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
6786 ; AVX2-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
6787 ; AVX2-NEXT: # ymm3 = ymm3[0],mem[1],ymm3[2,3],mem[4],ymm3[5],mem[6],ymm3[7,8],mem[9],ymm3[10,11],mem[12],ymm3[13],mem[14],ymm3[15]
6788 ; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
6789 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm6[6],ymm3[7]
6790 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
6791 ; AVX2-NEXT: vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm6 # 32-byte Folded Reload
6792 ; AVX2-NEXT: # ymm6 = ymm11[0,1],mem[2],ymm11[3],mem[4],ymm11[5,6],mem[7],ymm11[8,9],mem[10],ymm11[11],mem[12],ymm11[13,14],mem[15]
6793 ; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm7
6794 ; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[3,4],xmm6[5,6,7]
6795 ; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm3
6796 ; AVX2-NEXT: vpshufb %xmm0, %xmm6, %xmm6
6797 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2],ymm3[3,4,5,6,7]
6798 ; AVX2-NEXT: vpblendd $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm6 # 16-byte Folded Reload
6799 ; AVX2-NEXT: # xmm6 = mem[0],xmm12[1],mem[2,3]
6800 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
6801 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
6802 ; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm3[0,1,2,3,4],ymm6[5,6,7],ymm3[8,9,10,11,12],ymm6[13,14,15]
6803 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5,6,7]
6804 ; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6805 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
6806 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
6807 ; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm14[0],ymm4[1],ymm14[2,3],ymm4[4],ymm14[5],ymm4[6],ymm14[7,8],ymm4[9],ymm14[10,11],ymm4[12],ymm14[13],ymm4[14],ymm14[15]
6808 ; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
6809 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm6[6],ymm3[7]
6810 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
6811 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
6812 ; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm9[0,1],ymm8[2],ymm9[3],ymm8[4],ymm9[5,6],ymm8[7],ymm9[8,9],ymm8[10],ymm9[11],ymm8[12],ymm9[13,14],ymm8[15]
6813 ; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm7
6814 ; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm7[3,4],xmm6[5,6,7]
6815 ; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm3
6816 ; AVX2-NEXT: vpshufb %xmm0, %xmm6, %xmm6
6817 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2],ymm3[3,4,5,6,7]
6818 ; AVX2-NEXT: vpblendd $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10, %xmm6 # 16-byte Folded Reload
6819 ; AVX2-NEXT: # xmm6 = mem[0],xmm10[1],mem[2,3]
6820 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
6821 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
6822 ; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm3[0,1,2,3,4],ymm6[5,6,7],ymm3[8,9,10,11,12],ymm6[13,14,15]
6823 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5,6,7]
6824 ; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6825 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
6826 ; AVX2-NEXT: vpblendw $173, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm3 # 32-byte Folded Reload
6827 ; AVX2-NEXT: # ymm3 = mem[0],ymm12[1],mem[2,3],ymm12[4],mem[5],ymm12[6],mem[7,8],ymm12[9],mem[10,11],ymm12[12],mem[13],ymm12[14],mem[15]
6828 ; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
6829 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm6[6],ymm3[7]
6830 ; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm2
6831 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
6832 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
6833 ; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm10[0,1],ymm5[2],ymm10[3],ymm5[4],ymm10[5,6],ymm5[7],ymm10[8,9],ymm5[10],ymm10[11],ymm5[12],ymm10[13,14],ymm5[15]
6834 ; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm6
6835 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm6[3,4],xmm3[5,6,7]
6836 ; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0
6837 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
6838 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
6839 ; AVX2-NEXT: vpblendd $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
6840 ; AVX2-NEXT: # xmm2 = mem[0],xmm2[1],mem[2,3]
6841 ; AVX2-NEXT: vpshufb %xmm1, %xmm2, %xmm1
6842 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
6843 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
6844 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
6845 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6846 ; AVX2-NEXT: vpblendw $181, (%rsp), %ymm13, %ymm0 # 32-byte Folded Reload
6847 ; AVX2-NEXT: # ymm0 = mem[0],ymm13[1],mem[2],ymm13[3],mem[4,5],ymm13[6],mem[7,8],ymm13[9],mem[10],ymm13[11],mem[12,13],ymm13[14],mem[15]
6848 ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
6849 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
6850 ; AVX2-NEXT: vpblendw $41, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm1 # 32-byte Folded Reload
6851 ; AVX2-NEXT: # ymm1 = mem[0],ymm15[1,2],mem[3],ymm15[4],mem[5],ymm15[6,7],mem[8],ymm15[9,10],mem[11],ymm15[12],mem[13],ymm15[14,15]
6852 ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
6853 ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
6854 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,u,u,u,u,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
6855 ; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm3
6856 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm0 = [6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
6857 ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm1
6858 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0,1,2],ymm3[3,4,5,6,7]
6859 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
6860 ; AVX2-NEXT: vpblendd $11, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm6 # 16-byte Folded Reload
6861 ; AVX2-NEXT: # xmm6 = mem[0,1],xmm15[2],mem[3]
6862 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
6863 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
6864 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
6865 ; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm3[0,1,2,3,4],ymm6[5,6,7],ymm3[8,9,10,11,12],ymm6[13,14,15]
6866 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5,6,7]
6867 ; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6868 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
6869 ; AVX2-NEXT: vpblendw $74, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm3 # 32-byte Folded Reload
6870 ; AVX2-NEXT: # ymm3 = ymm13[0],mem[1],ymm13[2],mem[3],ymm13[4,5],mem[6],ymm13[7,8],mem[9],ymm13[10],mem[11],ymm13[12,13],mem[14],ymm13[15]
6871 ; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
6872 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4],ymm3[5],ymm6[6],ymm3[7]
6873 ; AVX2-NEXT: vpblendw $41, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm6 # 32-byte Folded Reload
6874 ; AVX2-NEXT: # ymm6 = mem[0],ymm11[1,2],mem[3],ymm11[4],mem[5],ymm11[6,7],mem[8],ymm11[9,10],mem[11],ymm11[12],mem[13],ymm11[14,15]
6875 ; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm7
6876 ; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0],xmm6[1],xmm7[2],xmm6[3]
6877 ; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm3
6878 ; AVX2-NEXT: vpshufb %xmm0, %xmm6, %xmm6
6879 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2],ymm3[3,4,5,6,7]
6880 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
6881 ; AVX2-NEXT: vpblendd $11, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm6 # 16-byte Folded Reload
6882 ; AVX2-NEXT: # xmm6 = mem[0,1],xmm11[2],mem[3]
6883 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
6884 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
6885 ; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm3[0,1,2,3,4],ymm6[5,6,7],ymm3[8,9,10,11,12],ymm6[13,14,15]
6886 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5,6,7]
6887 ; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6888 ; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm14[1],ymm4[2],ymm14[3],ymm4[4,5],ymm14[6],ymm4[7,8],ymm14[9],ymm4[10],ymm14[11],ymm4[12,13],ymm14[14],ymm4[15]
6889 ; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
6890 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4],ymm3[5],ymm6[6],ymm3[7]
6891 ; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm8[0],ymm9[1,2],ymm8[3],ymm9[4],ymm8[5],ymm9[6,7],ymm8[8],ymm9[9,10],ymm8[11],ymm9[12],ymm8[13],ymm9[14,15]
6892 ; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm7
6893 ; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0],xmm6[1],xmm7[2],xmm6[3]
6894 ; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm3
6895 ; AVX2-NEXT: vpshufb %xmm0, %xmm6, %xmm6
6896 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2],ymm3[3,4,5,6,7]
6897 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
6898 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
6899 ; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm8[0,1],xmm14[2],xmm8[3]
6900 ; AVX2-NEXT: vpshufb %xmm1, %xmm6, %xmm6
6901 ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
6902 ; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm3[0,1,2,3,4],ymm6[5,6,7],ymm3[8,9,10,11,12],ymm6[13,14,15]
6903 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5,6,7]
6904 ; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
6905 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
6906 ; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm12[0],ymm9[1],ymm12[2],ymm9[3],ymm12[4,5],ymm9[6],ymm12[7,8],ymm9[9],ymm12[10],ymm9[11],ymm12[12,13],ymm9[14],ymm12[15]
6907 ; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
6908 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4],ymm3[5],ymm6[6],ymm3[7]
6909 ; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm2
6910 ; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0],ymm10[1,2],ymm5[3],ymm10[4],ymm5[5],ymm10[6,7],ymm5[8],ymm10[9,10],ymm5[11],ymm10[12],ymm5[13],ymm10[14,15]
6911 ; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm6
6912 ; AVX2-NEXT: vpblendd {{.*#+}} xmm3 = xmm6[0],xmm3[1],xmm6[2],xmm3[3]
6913 ; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0
6914 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
6915 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
6916 ; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
6917 ; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm10[0,1],xmm12[2],xmm10[3]
6918 ; AVX2-NEXT: vpshufb %xmm1, %xmm2, %xmm1
6919 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
6920 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
6921 ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
6922 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
6923 ; AVX2-NEXT: vpblendw $107, (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload
6924 ; AVX2-NEXT: # ymm0 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7],mem[8,9],ymm0[10],mem[11],ymm0[12],mem[13,14],ymm0[15]
6925 ; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
6926 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4],ymm0[5,6],ymm2[7]
6927 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
6928 ; AVX2-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
6929 ; AVX2-NEXT: # ymm2 = ymm2[0],mem[1],ymm2[2,3],mem[4],ymm2[5],mem[6],ymm2[7,8],mem[9],ymm2[10,11],mem[12],ymm2[13],mem[14],ymm2[15]
6930 ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
6931 ; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
6932 ; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm7 = [24,25,18,19,28,29,22,23,0,0,0,0,4,5,14,15,24,25,18,19,28,29,22,23,0,0,0,0,4,5,14,15]
6933 ; AVX2-NEXT: # ymm7 = mem[0,1,0,1]
6934 ; AVX2-NEXT: vpshufb %ymm7, %ymm0, %ymm0
6935 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [8,9,2,3,12,13,6,7,0,1,10,11,0,1,6,7]
6936 ; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2
6937 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
6938 ; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
6939 ; AVX2-NEXT: # xmm2 = mem[3,1,2,3]
6940 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,1,4,5,6,7]
6941 ; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm15[0,2,2,3]
6942 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,0,3,4,5,6,7]
6943 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
6944 ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
6945 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
6946 ; AVX2-NEXT: vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm2 # 32-byte Folded Reload
6947 ; AVX2-NEXT: # ymm2 = ymm13[0,1],mem[2],ymm13[3],mem[4],ymm13[5,6],mem[7],ymm13[8,9],mem[10],ymm13[11],mem[12],ymm13[13,14],mem[15]
6948 ; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
6949 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4],ymm2[5,6],ymm4[7]
6950 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
6951 ; AVX2-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
6952 ; AVX2-NEXT: # ymm4 = ymm4[0],mem[1],ymm4[2,3],mem[4],ymm4[5],mem[6],ymm4[7,8],mem[9],ymm4[10,11],mem[12],ymm4[13],mem[14],ymm4[15]
6953 ; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5
6954 ; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3,4],xmm5[5,6,7]
6955 ; AVX2-NEXT: vpshufb %ymm7, %ymm2, %ymm2
6956 ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4
6957 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3,4,5,6,7]
6958 ; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
6959 ; AVX2-NEXT: # xmm4 = mem[3,1,2,3]
6960 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,2,1,4,5,6,7]
6961 ; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm11[0,2,2,3]
6962 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,0,3,4,5,6,7]
6963 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
6964 ; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
6965 ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
6966 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
6967 ; AVX2-NEXT: vpblendw $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
6968 ; AVX2-NEXT: # ymm4 = mem[0,1],ymm4[2],mem[3],ymm4[4],mem[5,6],ymm4[7],mem[8,9],ymm4[10],mem[11],ymm4[12],mem[13,14],ymm4[15]
6969 ; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm4[2,3,0,1]
6970 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4],ymm4[5,6],ymm5[7]
6971 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
6972 ; AVX2-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
6973 ; AVX2-NEXT: # ymm5 = ymm5[0],mem[1],ymm5[2,3],mem[4],ymm5[5],mem[6],ymm5[7,8],mem[9],ymm5[10,11],mem[12],ymm5[13],mem[14],ymm5[15]
6974 ; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm6
6975 ; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3,4],xmm6[5,6,7]
6976 ; AVX2-NEXT: vpshufb %ymm7, %ymm4, %ymm4
6977 ; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5
6978 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3,4,5,6,7]
6979 ; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm8[3,1,2,3]
6980 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,2,1,4,5,6,7]
6981 ; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm14[0,2,2,3]
6982 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,0,3,4,5,6,7]
6983 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
6984 ; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
6985 ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm5[6,7]
6986 ; AVX2-NEXT: vpblendw $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm5 # 32-byte Folded Reload
6987 ; AVX2-NEXT: # ymm5 = mem[0,1],ymm9[2],mem[3],ymm9[4],mem[5,6],ymm9[7],mem[8,9],ymm9[10],mem[11],ymm9[12],mem[13,14],ymm9[15]
6988 ; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm5[2,3,0,1]
6989 ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4],ymm5[5,6],ymm6[7]
6990 ; AVX2-NEXT: vpshufb %ymm7, %ymm5, %ymm5
6991 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
6992 ; AVX2-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
6993 ; AVX2-NEXT: # ymm6 = ymm6[0],mem[1],ymm6[2,3],mem[4],ymm6[5],mem[6],ymm6[7,8],mem[9],ymm6[10,11],mem[12],ymm6[13],mem[14],ymm6[15]
6994 ; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm7
6995 ; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3,4],xmm7[5,6,7]
6996 ; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm3
6997 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3,4,5,6,7]
6998 ; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm10[3,1,2,3]
6999 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,2,1,4,5,6,7]
7000 ; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm12[0,2,2,3]
7001 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,0,3,4,5,6,7]
7002 ; AVX2-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
7003 ; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
7004 ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm5[6,7]
7005 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7006 ; AVX2-NEXT: vmovaps %ymm5, 64(%rsi)
7007 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7008 ; AVX2-NEXT: vmovaps %ymm5, (%rsi)
7009 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7010 ; AVX2-NEXT: vmovaps %ymm5, 96(%rsi)
7011 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7012 ; AVX2-NEXT: vmovaps %ymm5, 32(%rsi)
7013 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7014 ; AVX2-NEXT: vmovaps %ymm5, 64(%rdx)
7015 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7016 ; AVX2-NEXT: vmovaps %ymm5, (%rdx)
7017 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7018 ; AVX2-NEXT: vmovaps %ymm5, 96(%rdx)
7019 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7020 ; AVX2-NEXT: vmovaps %ymm5, 32(%rdx)
7021 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7022 ; AVX2-NEXT: vmovaps %ymm5, 64(%rcx)
7023 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7024 ; AVX2-NEXT: vmovaps %ymm5, (%rcx)
7025 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7026 ; AVX2-NEXT: vmovaps %ymm5, 96(%rcx)
7027 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7028 ; AVX2-NEXT: vmovaps %ymm5, 32(%rcx)
7029 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7030 ; AVX2-NEXT: vmovaps %ymm5, 64(%r8)
7031 ; AVX2-NEXT: vmovdqa %ymm1, (%r8)
7032 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7033 ; AVX2-NEXT: vmovaps %ymm1, 96(%r8)
7034 ; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7035 ; AVX2-NEXT: vmovaps %ymm1, 32(%r8)
7036 ; AVX2-NEXT: vmovdqa %ymm4, 64(%r9)
7037 ; AVX2-NEXT: vmovdqa %ymm3, (%r9)
7038 ; AVX2-NEXT: vmovdqa %ymm2, 96(%r9)
7039 ; AVX2-NEXT: vmovdqa %ymm0, 32(%r9)
7040 ; AVX2-NEXT: addq $1048, %rsp # imm = 0x418
7041 ; AVX2-NEXT: vzeroupper
7042 ; AVX2-NEXT: retq
7043 ;
7044 ; AVX2-FP-LABEL: load_i16_stride5_vf64:
7045 ; AVX2-FP: # %bb.0:
7046 ; AVX2-FP-NEXT: subq $1080, %rsp # imm = 0x438
7047 ; AVX2-FP-NEXT: vmovdqa 384(%rdi), %ymm13
7048 ; AVX2-FP-NEXT: vmovdqa 512(%rdi), %ymm5
7049 ; AVX2-FP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7050 ; AVX2-FP-NEXT: vmovdqa 480(%rdi), %ymm6
7051 ; AVX2-FP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7052 ; AVX2-FP-NEXT: vmovdqa 544(%rdi), %ymm7
7053 ; AVX2-FP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7054 ; AVX2-FP-NEXT: vmovdqa 576(%rdi), %ymm8
7055 ; AVX2-FP-NEXT: vmovdqa 192(%rdi), %ymm2
7056 ; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7057 ; AVX2-FP-NEXT: vmovdqa 160(%rdi), %ymm4
7058 ; AVX2-FP-NEXT: vmovdqa 224(%rdi), %ymm1
7059 ; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7060 ; AVX2-FP-NEXT: vmovdqa 256(%rdi), %ymm0
7061 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7062 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4],ymm1[5],ymm0[6,7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12],ymm1[13],ymm0[14,15]
7063 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
7064 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6],ymm1[7]
7065 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,10,11,4,5,14,15,8,9,10,11,4,5,6,7,16,17,26,27,20,21,30,31,24,25,26,27,20,21,22,23]
7066 ; AVX2-FP-NEXT: vpshufb %ymm1, %ymm0, %ymm3
7067 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0],ymm2[1],ymm4[2,3],ymm2[4],ymm4[5],ymm2[6],ymm4[7,8],ymm2[9],ymm4[10,11],ymm2[12],ymm4[13],ymm2[14],ymm4[15]
7068 ; AVX2-FP-NEXT: vmovdqa %ymm4, %ymm15
7069 ; AVX2-FP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7070 ; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm2
7071 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3],xmm0[4,5],xmm2[6,7]
7072 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,6,7]
7073 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm2, %xmm4
7074 ; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} xmm10 = [65535,65535,65535,65535,65535,65535,65535,0]
7075 ; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm4, %ymm3, %ymm2
7076 ; AVX2-FP-NEXT: vmovdqu %ymm2, (%rsp) # 32-byte Spill
7077 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm7[0],ymm8[1,2],ymm7[3],ymm8[4],ymm7[5],ymm8[6,7],ymm7[8],ymm8[9,10],ymm7[11],ymm8[12],ymm7[13],ymm8[14,15]
7078 ; AVX2-FP-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7079 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
7080 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm4[5],ymm3[6],ymm4[7]
7081 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm4 = ymm6[0],ymm5[1],ymm6[2,3],ymm5[4],ymm6[5],ymm5[6],ymm6[7,8],ymm5[9],ymm6[10,11],ymm5[12],ymm6[13],ymm5[14],ymm6[15]
7082 ; AVX2-FP-NEXT: vextracti128 $1, %ymm4, %xmm6
7083 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm6[1,2,3],xmm4[4,5],xmm6[6,7]
7084 ; AVX2-FP-NEXT: vmovdqa 416(%rdi), %ymm14
7085 ; AVX2-FP-NEXT: vpshufb %ymm1, %ymm3, %ymm3
7086 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm4, %xmm4
7087 ; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm4, %ymm3, %ymm3
7088 ; AVX2-FP-NEXT: vmovdqa %ymm10, %ymm7
7089 ; AVX2-FP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7090 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm13[0],ymm14[1,2],ymm13[3],ymm14[4],ymm13[5],ymm14[6,7],ymm13[8],ymm14[9,10],ymm13[11],ymm14[12],ymm13[13],ymm14[14,15]
7091 ; AVX2-FP-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7092 ; AVX2-FP-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7093 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
7094 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm4[5],ymm3[6],ymm4[7]
7095 ; AVX2-FP-NEXT: vmovdqa 352(%rdi), %ymm2
7096 ; AVX2-FP-NEXT: vmovdqa 320(%rdi), %ymm5
7097 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm4 = ymm5[0],ymm2[1],ymm5[2,3],ymm2[4],ymm5[5],ymm2[6],ymm5[7,8],ymm2[9],ymm5[10,11],ymm2[12],ymm5[13],ymm2[14],ymm5[15]
7098 ; AVX2-FP-NEXT: vmovdqa %ymm5, %ymm11
7099 ; AVX2-FP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7100 ; AVX2-FP-NEXT: vmovdqa %ymm2, %ymm12
7101 ; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7102 ; AVX2-FP-NEXT: vextracti128 $1, %ymm4, %xmm6
7103 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm6[1,2,3],xmm4[4,5],xmm6[6,7]
7104 ; AVX2-FP-NEXT: vpshufb %ymm1, %ymm3, %ymm3
7105 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm4, %xmm4
7106 ; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm4, %ymm3, %ymm2
7107 ; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7108 ; AVX2-FP-NEXT: vmovdqa 64(%rdi), %ymm9
7109 ; AVX2-FP-NEXT: vmovdqa 96(%rdi), %ymm10
7110 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm9[0],ymm10[1,2],ymm9[3],ymm10[4],ymm9[5],ymm10[6,7],ymm9[8],ymm10[9,10],ymm9[11],ymm10[12],ymm9[13],ymm10[14,15]
7111 ; AVX2-FP-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7112 ; AVX2-FP-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7113 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
7114 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm6[5],ymm3[6],ymm6[7]
7115 ; AVX2-FP-NEXT: vpshufb %ymm1, %ymm3, %ymm1
7116 ; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm4
7117 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm5
7118 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10,11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]
7119 ; AVX2-FP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7120 ; AVX2-FP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7121 ; AVX2-FP-NEXT: vextracti128 $1, %ymm3, %xmm6
7122 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm6[1,2,3],xmm3[4,5],xmm6[6,7]
7123 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm3, %xmm0
7124 ; AVX2-FP-NEXT: vmovdqa %ymm7, %ymm2
7125 ; AVX2-FP-NEXT: vpblendvb %ymm7, %ymm0, %ymm1, %ymm0
7126 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7127 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7128 ; AVX2-FP-NEXT: vpblendw $173, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
7129 ; AVX2-FP-NEXT: # ymm0 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5],ymm0[6],mem[7,8],ymm0[9],mem[10,11],ymm0[12],mem[13],ymm0[14],mem[15]
7130 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
7131 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
7132 ; AVX2-FP-NEXT: vpblendw $181, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm1 # 32-byte Folded Reload
7133 ; AVX2-FP-NEXT: # ymm1 = mem[0],ymm15[1],mem[2],ymm15[3],mem[4,5],ymm15[6],mem[7,8],ymm15[9],mem[10],ymm15[11],mem[12,13],ymm15[14],mem[15]
7134 ; AVX2-FP-NEXT: vextracti128 $1, %ymm1, %xmm3
7135 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6],xmm3[7]
7136 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm7 = [2,3,12,13,6,7,0,1,10,11,6,7,8,9,8,9,18,19,28,29,22,23,16,17,26,27,22,23,24,25,24,25]
7137 ; AVX2-FP-NEXT: vpshufb %ymm7, %ymm0, %ymm0
7138 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm6 = [2,3,12,13,6,7,0,1,10,11,4,5,14,15,10,11]
7139 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm1, %xmm1
7140 ; AVX2-FP-NEXT: vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
7141 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7142 ; AVX2-FP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm0 # 32-byte Folded Reload
7143 ; AVX2-FP-NEXT: # ymm0 = ymm8[0],mem[1],ymm8[2,3],mem[4],ymm8[5],mem[6],ymm8[7,8],mem[9],ymm8[10,11],mem[12],ymm8[13],mem[14],ymm8[15]
7144 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
7145 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
7146 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
7147 ; AVX2-FP-NEXT: vpblendw $181, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm1 # 32-byte Folded Reload
7148 ; AVX2-FP-NEXT: # ymm1 = mem[0],ymm15[1],mem[2],ymm15[3],mem[4,5],ymm15[6],mem[7,8],ymm15[9],mem[10],ymm15[11],mem[12,13],ymm15[14],mem[15]
7149 ; AVX2-FP-NEXT: vextracti128 $1, %ymm1, %xmm8
7150 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm8[2,3],xmm1[4,5,6],xmm8[7]
7151 ; AVX2-FP-NEXT: vpshufb %ymm7, %ymm0, %ymm0
7152 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm1, %xmm1
7153 ; AVX2-FP-NEXT: vpblendvb %ymm2, %ymm1, %ymm0, %ymm3
7154 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm14[0],ymm13[1],ymm14[2,3],ymm13[4],ymm14[5],ymm13[6],ymm14[7,8],ymm13[9],ymm14[10,11],ymm13[12],ymm14[13],ymm13[14],ymm14[15]
7155 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm8 = ymm0[2,3,0,1]
7156 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm8[5],ymm0[6,7]
7157 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm12[0],ymm11[1],ymm12[2],ymm11[3],ymm12[4,5],ymm11[6],ymm12[7,8],ymm11[9],ymm12[10],ymm11[11],ymm12[12,13],ymm11[14],ymm12[15]
7158 ; AVX2-FP-NEXT: vextracti128 $1, %ymm8, %xmm11
7159 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm11[2,3],xmm8[4,5,6],xmm11[7]
7160 ; AVX2-FP-NEXT: vpshufb %ymm7, %ymm0, %ymm0
7161 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm8, %xmm8
7162 ; AVX2-FP-NEXT: vmovdqa %ymm2, %ymm1
7163 ; AVX2-FP-NEXT: vpblendvb %ymm2, %ymm8, %ymm0, %ymm2
7164 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm10[0],ymm9[1],ymm10[2,3],ymm9[4],ymm10[5],ymm9[6],ymm10[7,8],ymm9[9],ymm10[10,11],ymm9[12],ymm10[13],ymm9[14],ymm10[15]
7165 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm11 = ymm8[2,3,0,1]
7166 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm11[5],ymm8[6,7]
7167 ; AVX2-FP-NEXT: vpshufb %ymm7, %ymm8, %ymm7
7168 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
7169 ; AVX2-FP-NEXT: vextracti128 $1, %ymm8, %xmm11
7170 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm11[2,3],xmm8[4,5,6],xmm11[7]
7171 ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm8, %xmm6
7172 ; AVX2-FP-NEXT: vpblendvb %ymm1, %ymm6, %ymm7, %ymm12
7173 ; AVX2-FP-NEXT: vmovdqa 304(%rdi), %xmm1
7174 ; AVX2-FP-NEXT: vmovdqa 288(%rdi), %xmm0
7175 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm7 = xmm0[0],xmm1[1],xmm0[2,3]
7176 ; AVX2-FP-NEXT: vmovdqa %xmm0, %xmm9
7177 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm0 = [6,7,2,3,4,5,6,7,6,7,2,3,12,13,6,7]
7178 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm7, %xmm7
7179 ; AVX2-FP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
7180 ; AVX2-FP-NEXT: vmovdqu (%rsp), %ymm4 # 32-byte Reload
7181 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm7 = ymm4[0,1,2,3,4],ymm7[5,6,7],ymm4[8,9,10,11,12],ymm7[13,14,15]
7182 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm7[4,5,6,7]
7183 ; AVX2-FP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7184 ; AVX2-FP-NEXT: vmovdqa 624(%rdi), %xmm4
7185 ; AVX2-FP-NEXT: vmovdqa 608(%rdi), %xmm5
7186 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm7 = xmm5[0],xmm4[1],xmm5[2,3]
7187 ; AVX2-FP-NEXT: vmovdqa %xmm5, %xmm8
7188 ; AVX2-FP-NEXT: vmovdqa %xmm4, %xmm10
7189 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm7, %xmm7
7190 ; AVX2-FP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
7191 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7192 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm7 = ymm4[0,1,2,3,4],ymm7[5,6,7],ymm4[8,9,10,11,12],ymm7[13,14,15]
7193 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm7[4,5,6,7]
7194 ; AVX2-FP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7195 ; AVX2-FP-NEXT: vmovdqa 464(%rdi), %xmm4
7196 ; AVX2-FP-NEXT: vmovdqa 448(%rdi), %xmm5
7197 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm7 = xmm5[0],xmm4[1],xmm5[2,3]
7198 ; AVX2-FP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7199 ; AVX2-FP-NEXT: vmovdqa %xmm4, %xmm6
7200 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm7, %xmm7
7201 ; AVX2-FP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
7202 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7203 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm7 = ymm4[0,1,2,3,4],ymm7[5,6,7],ymm4[8,9,10,11,12],ymm7[13,14,15]
7204 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm7[4,5,6,7]
7205 ; AVX2-FP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7206 ; AVX2-FP-NEXT: vmovdqa 144(%rdi), %xmm4
7207 ; AVX2-FP-NEXT: vmovdqa 128(%rdi), %xmm7
7208 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm14 = xmm7[0],xmm4[1],xmm7[2,3]
7209 ; AVX2-FP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7210 ; AVX2-FP-NEXT: vmovdqa %xmm4, %xmm11
7211 ; AVX2-FP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7212 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm14, %xmm0
7213 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7214 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7215 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0,1,2,3,4],ymm0[5,6,7],ymm4[8,9,10,11,12],ymm0[13,14,15]
7216 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
7217 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7218 ; AVX2-FP-NEXT: vmovdqa %xmm9, %xmm4
7219 ; AVX2-FP-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7220 ; AVX2-FP-NEXT: vmovdqa %xmm1, %xmm14
7221 ; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7222 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm9[0,1],xmm1[2],xmm9[3]
7223 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm9 = [0,1,2,3,4,5,6,7,8,9,4,5,14,15,8,9]
7224 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm0, %xmm0
7225 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7226 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7227 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7],ymm1[8,9,10,11,12],ymm0[13,14,15]
7228 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
7229 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7230 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm8[0,1],xmm10[2],xmm8[3]
7231 ; AVX2-FP-NEXT: vmovdqa %xmm10, %xmm13
7232 ; AVX2-FP-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7233 ; AVX2-FP-NEXT: vmovdqa %xmm8, %xmm10
7234 ; AVX2-FP-NEXT: vmovdqa %xmm8, (%rsp) # 16-byte Spill
7235 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm0, %xmm0
7236 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7237 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5,6,7],ymm3[8,9,10,11,12],ymm0[13,14,15]
7238 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
7239 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7240 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm5[0,1],xmm6[2],xmm5[3]
7241 ; AVX2-FP-NEXT: vmovdqa %xmm6, %xmm3
7242 ; AVX2-FP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
7243 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm0, %xmm0
7244 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7245 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7],ymm2[8,9,10,11,12],ymm0[13,14,15]
7246 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
7247 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7248 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm7[0,1],xmm11[2],xmm7[3]
7249 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm0, %xmm0
7250 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
7251 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm12[0,1,2,3,4],ymm0[5,6,7],ymm12[8,9,10,11,12],ymm0[13,14,15]
7252 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm0[4,5,6,7]
7253 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7254 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
7255 ; AVX2-FP-NEXT: vpblendw $173, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm0 # 32-byte Folded Reload
7256 ; AVX2-FP-NEXT: # ymm0 = mem[0],ymm8[1],mem[2,3],ymm8[4],mem[5],ymm8[6],mem[7,8],ymm8[9],mem[10,11],ymm8[12],mem[13],ymm8[14],mem[15]
7257 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
7258 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6],ymm0[7]
7259 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7260 ; AVX2-FP-NEXT: vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm1 # 32-byte Folded Reload
7261 ; AVX2-FP-NEXT: # ymm1 = ymm5[0,1],mem[2],ymm5[3],mem[4],ymm5[5,6],mem[7],ymm5[8,9],mem[10],ymm5[11],mem[12],ymm5[13,14],mem[15]
7262 ; AVX2-FP-NEXT: vextracti128 $1, %ymm1, %xmm2
7263 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3,4],xmm1[5,6,7]
7264 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm6 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
7265 ; AVX2-FP-NEXT: vpshufb %ymm6, %ymm0, %ymm2
7266 ; AVX2-FP-NEXT: vmovdqa %ymm6, %ymm11
7267 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
7268 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm1
7269 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3,4,5,6,7]
7270 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm9 = xmm14[0],xmm4[1],xmm14[2,3]
7271 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
7272 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm9, %xmm9
7273 ; AVX2-FP-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
7274 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0,1,2,3,4],ymm9[5,6,7],ymm2[8,9,10,11,12],ymm9[13,14,15]
7275 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm9[4,5,6,7]
7276 ; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7277 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
7278 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7279 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm6[1],ymm4[2,3],ymm6[4],ymm4[5],ymm6[6],ymm4[7,8],ymm6[9],ymm4[10,11],ymm6[12],ymm4[13],ymm6[14],ymm4[15]
7280 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm9 = ymm2[2,3,0,1]
7281 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm9[6],ymm2[7]
7282 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
7283 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm7[0,1],ymm15[2],ymm7[3],ymm15[4],ymm7[5,6],ymm15[7],ymm7[8,9],ymm15[10],ymm7[11],ymm15[12],ymm7[13,14],ymm15[15]
7284 ; AVX2-FP-NEXT: vextracti128 $1, %ymm9, %xmm12
7285 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1,2],xmm12[3,4],xmm9[5,6,7]
7286 ; AVX2-FP-NEXT: vpshufb %ymm11, %ymm2, %ymm2
7287 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm9, %xmm9
7288 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm9[0,1,2],ymm2[3,4,5,6,7]
7289 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm9 = xmm13[0],xmm10[1],xmm13[2,3]
7290 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm9, %xmm9
7291 ; AVX2-FP-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
7292 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0,1,2,3,4],ymm9[5,6,7],ymm2[8,9,10,11,12],ymm9[13,14,15]
7293 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm9[4,5,6,7]
7294 ; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7295 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
7296 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
7297 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm15[0],ymm14[1],ymm15[2,3],ymm14[4],ymm15[5],ymm14[6],ymm15[7,8],ymm14[9],ymm15[10,11],ymm14[12],ymm15[13],ymm14[14],ymm15[15]
7298 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm9 = ymm2[2,3,0,1]
7299 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm9[6],ymm2[7]
7300 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
7301 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
7302 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm10[0,1],ymm11[2],ymm10[3],ymm11[4],ymm10[5,6],ymm11[7],ymm10[8,9],ymm11[10],ymm10[11],ymm11[12],ymm10[13,14],ymm11[15]
7303 ; AVX2-FP-NEXT: vextracti128 $1, %ymm9, %xmm12
7304 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1,2],xmm12[3,4],xmm9[5,6,7]
7305 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm12 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
7306 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm2, %ymm2
7307 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm9, %xmm9
7308 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm9[0,1,2],ymm2[3,4,5,6,7]
7309 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
7310 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm9 = xmm3[0],xmm13[1],xmm3[2,3]
7311 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm9, %xmm9
7312 ; AVX2-FP-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
7313 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0,1,2,3,4],ymm9[5,6,7],ymm2[8,9,10,11,12],ymm9[13,14,15]
7314 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm9[4,5,6,7]
7315 ; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7316 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
7317 ; AVX2-FP-NEXT: vpblendw $173, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
7318 ; AVX2-FP-NEXT: # ymm2 = mem[0],ymm2[1],mem[2,3],ymm2[4],mem[5],ymm2[6],mem[7,8],ymm2[9],mem[10,11],ymm2[12],mem[13],ymm2[14],mem[15]
7319 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm9 = ymm2[2,3,0,1]
7320 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm9[6],ymm2[7]
7321 ; AVX2-FP-NEXT: vpshufb %ymm12, %ymm2, %ymm2
7322 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
7323 ; AVX2-FP-NEXT: vpblendw $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
7324 ; AVX2-FP-NEXT: # ymm3 = mem[0,1],ymm3[2],mem[3],ymm3[4],mem[5,6],ymm3[7],mem[8,9],ymm3[10],mem[11],ymm3[12],mem[13,14],ymm3[15]
7325 ; AVX2-FP-NEXT: vextracti128 $1, %ymm3, %xmm9
7326 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm9[3,4],xmm3[5,6,7]
7327 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm3, %xmm0
7328 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
7329 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
7330 ; AVX2-FP-NEXT: vpblendd $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
7331 ; AVX2-FP-NEXT: # xmm2 = mem[0],xmm2[1],mem[2,3]
7332 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm2, %xmm1
7333 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
7334 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
7335 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
7336 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7337 ; AVX2-FP-NEXT: vpblendw $74, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm0 # 32-byte Folded Reload
7338 ; AVX2-FP-NEXT: # ymm0 = ymm8[0],mem[1],ymm8[2],mem[3],ymm8[4,5],mem[6],ymm8[7,8],mem[9],ymm8[10],mem[11],ymm8[12,13],mem[14],ymm8[15]
7339 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
7340 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
7341 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
7342 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm8[0],ymm5[1,2],ymm8[3],ymm5[4],ymm8[5],ymm5[6,7],ymm8[8],ymm5[9,10],ymm8[11],ymm5[12],ymm8[13],ymm5[14,15]
7343 ; AVX2-FP-NEXT: vextracti128 $1, %ymm1, %xmm2
7344 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
7345 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm3 = [u,u,u,u,u,u,u,u,u,u,u,u,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
7346 ; AVX2-FP-NEXT: vpshufb %ymm3, %ymm0, %ymm2
7347 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm0 = [6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
7348 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm1
7349 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3,4,5,6,7]
7350 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7351 ; AVX2-FP-NEXT: vpblendd $4, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm9 # 16-byte Folded Reload
7352 ; AVX2-FP-NEXT: # xmm9 = xmm1[0,1],mem[2],xmm1[3]
7353 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
7354 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm9, %xmm9
7355 ; AVX2-FP-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
7356 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0,1,2,3,4],ymm9[5,6,7],ymm2[8,9,10,11,12],ymm9[13,14,15]
7357 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm9[4,5,6,7]
7358 ; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7359 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4,5],ymm4[6],ymm6[7,8],ymm4[9],ymm6[10],ymm4[11],ymm6[12,13],ymm4[14],ymm6[15]
7360 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm9 = ymm2[2,3,0,1]
7361 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm9[4],ymm2[5],ymm9[6],ymm2[7]
7362 ; AVX2-FP-NEXT: vpblendw $41, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm9 # 32-byte Folded Reload
7363 ; AVX2-FP-NEXT: # ymm9 = mem[0],ymm7[1,2],mem[3],ymm7[4],mem[5],ymm7[6,7],mem[8],ymm7[9,10],mem[11],ymm7[12],mem[13],ymm7[14,15]
7364 ; AVX2-FP-NEXT: vextracti128 $1, %ymm9, %xmm12
7365 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm9 = xmm12[0],xmm9[1],xmm12[2],xmm9[3]
7366 ; AVX2-FP-NEXT: vpshufb %ymm3, %ymm2, %ymm2
7367 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm9, %xmm9
7368 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm9[0,1,2],ymm2[3,4,5,6,7]
7369 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
7370 ; AVX2-FP-NEXT: vpblendd $4, (%rsp), %xmm4, %xmm9 # 16-byte Folded Reload
7371 ; AVX2-FP-NEXT: # xmm9 = xmm4[0,1],mem[2],xmm4[3]
7372 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm9, %xmm9
7373 ; AVX2-FP-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
7374 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0,1,2,3,4],ymm9[5,6,7],ymm2[8,9,10,11,12],ymm9[13,14,15]
7375 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm9[4,5,6,7]
7376 ; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7377 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm14[0],ymm15[1],ymm14[2],ymm15[3],ymm14[4,5],ymm15[6],ymm14[7,8],ymm15[9],ymm14[10],ymm15[11],ymm14[12,13],ymm15[14],ymm14[15]
7378 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm9 = ymm2[2,3,0,1]
7379 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm9[4],ymm2[5],ymm9[6],ymm2[7]
7380 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm11[0],ymm10[1,2],ymm11[3],ymm10[4],ymm11[5],ymm10[6,7],ymm11[8],ymm10[9,10],ymm11[11],ymm10[12],ymm11[13],ymm10[14,15]
7381 ; AVX2-FP-NEXT: vextracti128 $1, %ymm9, %xmm12
7382 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm9 = xmm12[0],xmm9[1],xmm12[2],xmm9[3]
7383 ; AVX2-FP-NEXT: vpshufb %ymm3, %ymm2, %ymm2
7384 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm9, %xmm9
7385 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm9[0,1,2],ymm2[3,4,5,6,7]
7386 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
7387 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm9 = xmm15[0,1],xmm13[2],xmm15[3]
7388 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm9, %xmm9
7389 ; AVX2-FP-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
7390 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm2[0,1,2,3,4],ymm9[5,6,7],ymm2[8,9,10,11,12],ymm9[13,14,15]
7391 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm9[4,5,6,7]
7392 ; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7393 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
7394 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
7395 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm12[0],ymm7[1],ymm12[2],ymm7[3],ymm12[4,5],ymm7[6],ymm12[7,8],ymm7[9],ymm12[10],ymm7[11],ymm12[12,13],ymm7[14],ymm12[15]
7396 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm9 = ymm2[2,3,0,1]
7397 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm9[4],ymm2[5],ymm9[6],ymm2[7]
7398 ; AVX2-FP-NEXT: vpshufb %ymm3, %ymm2, %ymm2
7399 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
7400 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
7401 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm10[0],ymm11[1,2],ymm10[3],ymm11[4],ymm10[5],ymm11[6,7],ymm10[8],ymm11[9,10],ymm10[11],ymm11[12],ymm10[13],ymm11[14,15]
7402 ; AVX2-FP-NEXT: vextracti128 $1, %ymm3, %xmm9
7403 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm3 = xmm9[0],xmm3[1],xmm9[2],xmm3[3]
7404 ; AVX2-FP-NEXT: vpshufb %xmm0, %xmm3, %xmm0
7405 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
7406 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
7407 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
7408 ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm5[2],xmm3[3]
7409 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm2, %xmm1
7410 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
7411 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
7412 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
7413 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7414 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7415 ; AVX2-FP-NEXT: vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
7416 ; AVX2-FP-NEXT: # ymm0 = ymm0[0,1],mem[2],ymm0[3],mem[4],ymm0[5,6],mem[7],ymm0[8,9],mem[10],ymm0[11],mem[12],ymm0[13,14],mem[15]
7417 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
7418 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6],ymm1[7]
7419 ; AVX2-FP-NEXT: vpblendw $173, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm0 # 32-byte Folded Reload
7420 ; AVX2-FP-NEXT: # ymm0 = mem[0],ymm8[1],mem[2,3],ymm8[4],mem[5],ymm8[6],mem[7,8],ymm8[9],mem[10,11],ymm8[12],mem[13],ymm8[14],mem[15]
7421 ; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm2
7422 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm0[3,4],xmm2[5,6,7]
7423 ; AVX2-FP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [24,25,18,19,28,29,22,23,0,0,0,0,4,5,14,15,24,25,18,19,28,29,22,23,0,0,0,0,4,5,14,15]
7424 ; AVX2-FP-NEXT: # ymm0 = mem[0,1,0,1]
7425 ; AVX2-FP-NEXT: vpshufb %ymm0, %ymm1, %ymm1
7426 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm13 = [8,9,2,3,12,13,6,7,0,1,10,11,0,1,6,7]
7427 ; AVX2-FP-NEXT: vpshufb %xmm13, %xmm2, %xmm2
7428 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2],ymm1[3,4,5,6,7]
7429 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm9 = [12,13,14,15,4,5,14,15,8,9,10,11,12,13,14,15]
7430 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
7431 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm1, %xmm4
7432 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,0,1,10,11,8,9,10,11,12,13,14,15]
7433 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
7434 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm6, %xmm6
7435 ; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
7436 ; AVX2-FP-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
7437 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm2[0,1,2,3,4,5],ymm4[6,7]
7438 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
7439 ; AVX2-FP-NEXT: vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
7440 ; AVX2-FP-NEXT: # ymm2 = ymm2[0,1],mem[2],ymm2[3],mem[4],ymm2[5,6],mem[7],ymm2[8,9],mem[10],ymm2[11],mem[12],ymm2[13,14],mem[15]
7441 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
7442 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4],ymm2[5,6],ymm4[7]
7443 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7444 ; AVX2-FP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
7445 ; AVX2-FP-NEXT: # ymm4 = ymm4[0],mem[1],ymm4[2,3],mem[4],ymm4[5],mem[6],ymm4[7,8],mem[9],ymm4[10,11],mem[12],ymm4[13],mem[14],ymm4[15]
7446 ; AVX2-FP-NEXT: vextracti128 $1, %ymm4, %xmm14
7447 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm14[0,1,2],xmm4[3,4],xmm14[5,6,7]
7448 ; AVX2-FP-NEXT: vpshufb %ymm0, %ymm2, %ymm2
7449 ; AVX2-FP-NEXT: vpshufb %xmm13, %xmm4, %xmm4
7450 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3,4,5,6,7]
7451 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
7452 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm4, %xmm4
7453 ; AVX2-FP-NEXT: vmovdqa (%rsp), %xmm8 # 16-byte Reload
7454 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm8, %xmm8
7455 ; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
7456 ; AVX2-FP-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
7457 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
7458 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm3, %xmm4
7459 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm5, %xmm5
7460 ; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
7461 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm5 = ymm12[0,1],ymm7[2],ymm12[3],ymm7[4],ymm12[5,6],ymm7[7],ymm12[8,9],ymm7[10],ymm12[11],ymm7[12],ymm12[13,14],ymm7[15]
7462 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm7 = ymm5[2,3,0,1]
7463 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm7[4],ymm5[5,6],ymm7[7]
7464 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm7 = ymm11[0],ymm10[1],ymm11[2,3],ymm10[4],ymm11[5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10,11],ymm10[12],ymm11[13],ymm10[14],ymm11[15]
7465 ; AVX2-FP-NEXT: vextracti128 $1, %ymm7, %xmm8
7466 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3,4],xmm8[5,6,7]
7467 ; AVX2-FP-NEXT: vpshufb %ymm0, %ymm5, %ymm5
7468 ; AVX2-FP-NEXT: vpshufb %xmm13, %xmm7, %xmm7
7469 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1,2],ymm5[3,4,5,6,7]
7470 ; AVX2-FP-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
7471 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
7472 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
7473 ; AVX2-FP-NEXT: vpblendw $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm5 # 32-byte Folded Reload
7474 ; AVX2-FP-NEXT: # ymm5 = mem[0,1],ymm3[2],mem[3],ymm3[4],mem[5,6],ymm3[7],mem[8,9],ymm3[10],mem[11],ymm3[12],mem[13,14],ymm3[15]
7475 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm7 = ymm5[2,3,0,1]
7476 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm7[4],ymm5[5,6],ymm7[7]
7477 ; AVX2-FP-NEXT: vpshufb %ymm0, %ymm5, %ymm0
7478 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
7479 ; AVX2-FP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm5 # 32-byte Folded Reload
7480 ; AVX2-FP-NEXT: # ymm5 = ymm3[0],mem[1],ymm3[2,3],mem[4],ymm3[5],mem[6],ymm3[7,8],mem[9],ymm3[10,11],mem[12],ymm3[13],mem[14],ymm3[15]
7481 ; AVX2-FP-NEXT: vextracti128 $1, %ymm5, %xmm7
7482 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm5 = xmm7[0,1,2],xmm5[3,4],xmm7[5,6,7]
7483 ; AVX2-FP-NEXT: vpshufb %xmm13, %xmm5, %xmm3
7484 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3,4,5,6,7]
7485 ; AVX2-FP-NEXT: vpshufb %xmm9, %xmm15, %xmm3
7486 ; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
7487 ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm5, %xmm1
7488 ; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
7489 ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
7490 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
7491 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7492 ; AVX2-FP-NEXT: vmovaps %ymm1, 64(%rsi)
7493 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7494 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rsi)
7495 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7496 ; AVX2-FP-NEXT: vmovaps %ymm1, 96(%rsi)
7497 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7498 ; AVX2-FP-NEXT: vmovaps %ymm1, 32(%rsi)
7499 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7500 ; AVX2-FP-NEXT: vmovaps %ymm1, 64(%rdx)
7501 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7502 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rdx)
7503 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7504 ; AVX2-FP-NEXT: vmovaps %ymm1, 96(%rdx)
7505 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7506 ; AVX2-FP-NEXT: vmovaps %ymm1, 32(%rdx)
7507 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7508 ; AVX2-FP-NEXT: vmovaps %ymm1, 64(%rcx)
7509 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7510 ; AVX2-FP-NEXT: vmovaps %ymm1, (%rcx)
7511 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7512 ; AVX2-FP-NEXT: vmovaps %ymm1, 96(%rcx)
7513 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7514 ; AVX2-FP-NEXT: vmovaps %ymm1, 32(%rcx)
7515 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7516 ; AVX2-FP-NEXT: vmovaps %ymm1, 64(%r8)
7517 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7518 ; AVX2-FP-NEXT: vmovaps %ymm1, (%r8)
7519 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7520 ; AVX2-FP-NEXT: vmovaps %ymm1, 96(%r8)
7521 ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7522 ; AVX2-FP-NEXT: vmovaps %ymm1, 32(%r8)
7523 ; AVX2-FP-NEXT: vmovdqa %ymm0, 64(%r9)
7524 ; AVX2-FP-NEXT: vmovdqa %ymm4, (%r9)
7525 ; AVX2-FP-NEXT: vmovdqa %ymm2, 96(%r9)
7526 ; AVX2-FP-NEXT: vmovdqa %ymm6, 32(%r9)
7527 ; AVX2-FP-NEXT: addq $1080, %rsp # imm = 0x438
7528 ; AVX2-FP-NEXT: vzeroupper
7529 ; AVX2-FP-NEXT: retq
7530 ;
7531 ; AVX2-FCP-LABEL: load_i16_stride5_vf64:
7532 ; AVX2-FCP: # %bb.0:
7533 ; AVX2-FCP-NEXT: subq $1000, %rsp # imm = 0x3E8
7534 ; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm9
7535 ; AVX2-FCP-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7536 ; AVX2-FCP-NEXT: vmovdqa 544(%rdi), %ymm6
7537 ; AVX2-FCP-NEXT: vmovdqa 576(%rdi), %ymm11
7538 ; AVX2-FCP-NEXT: vmovdqa 512(%rdi), %ymm8
7539 ; AVX2-FCP-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7540 ; AVX2-FCP-NEXT: vmovdqa 480(%rdi), %ymm10
7541 ; AVX2-FCP-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7542 ; AVX2-FCP-NEXT: vmovdqa 224(%rdi), %ymm4
7543 ; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7544 ; AVX2-FCP-NEXT: vmovdqa 256(%rdi), %ymm3
7545 ; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7546 ; AVX2-FCP-NEXT: vmovdqa 192(%rdi), %ymm0
7547 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7548 ; AVX2-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
7549 ; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7550 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
7551 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1
7552 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4,5],xmm1[6,7]
7553 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,6,7]
7554 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm2
7555 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1,2],ymm4[3],ymm3[4],ymm4[5],ymm3[6,7],ymm4[8],ymm3[9,10],ymm4[11],ymm3[12],ymm4[13],ymm3[14,15]
7556 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,0,2,4,6,1,3]
7557 ; AVX2-FCP-NEXT: vpermd %ymm3, %ymm1, %ymm4
7558 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,6,7,8,9,14,15,4,5,14,15,4,5,2,3,16,17,22,23,24,25,30,31,20,21,30,31,20,21,18,19]
7559 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm4
7560 ; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} xmm13 = [65535,65535,65535,65535,65535,65535,65535,0]
7561 ; AVX2-FCP-NEXT: vpblendvb %ymm13, %ymm2, %ymm4, %ymm2
7562 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7563 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm10[0],ymm8[1],ymm10[2,3],ymm8[4],ymm10[5],ymm8[6],ymm10[7,8],ymm8[9],ymm10[10,11],ymm8[12],ymm10[13],ymm8[14],ymm10[15]
7564 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm2, %xmm4
7565 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1,2,3],xmm2[4,5],xmm4[6,7]
7566 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm2, %xmm2
7567 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm6[0],ymm11[1,2],ymm6[3],ymm11[4],ymm6[5],ymm11[6,7],ymm6[8],ymm11[9,10],ymm6[11],ymm11[12],ymm6[13],ymm11[14,15]
7568 ; AVX2-FCP-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7569 ; AVX2-FCP-NEXT: vmovdqa %ymm6, %ymm14
7570 ; AVX2-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7571 ; AVX2-FCP-NEXT: vpermd %ymm4, %ymm1, %ymm4
7572 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm4
7573 ; AVX2-FCP-NEXT: vpblendvb %ymm13, %ymm2, %ymm4, %ymm2
7574 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7575 ; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm4
7576 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm9[0],ymm4[1],ymm9[2,3],ymm4[4],ymm9[5],ymm4[6],ymm9[7,8],ymm4[9],ymm9[10,11],ymm4[12],ymm9[13],ymm4[14],ymm9[15]
7577 ; AVX2-FCP-NEXT: vmovdqa %ymm4, %ymm15
7578 ; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7579 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm2, %xmm4
7580 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1,2,3],xmm2[4,5],xmm4[6,7]
7581 ; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %ymm8
7582 ; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %ymm12
7583 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm8[0],ymm12[1,2],ymm8[3],ymm12[4],ymm8[5],ymm12[6,7],ymm8[8],ymm12[9,10],ymm8[11],ymm12[12],ymm8[13],ymm12[14,15]
7584 ; AVX2-FCP-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7585 ; AVX2-FCP-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7586 ; AVX2-FCP-NEXT: vpermd %ymm4, %ymm1, %ymm4
7587 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm4
7588 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm2, %xmm2
7589 ; AVX2-FCP-NEXT: vpblendvb %ymm13, %ymm2, %ymm4, %ymm2
7590 ; AVX2-FCP-NEXT: vmovdqu %ymm2, (%rsp) # 32-byte Spill
7591 ; AVX2-FCP-NEXT: vmovdqa 352(%rdi), %ymm4
7592 ; AVX2-FCP-NEXT: vmovdqa 320(%rdi), %ymm7
7593 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm7[0],ymm4[1],ymm7[2,3],ymm4[4],ymm7[5],ymm4[6],ymm7[7,8],ymm4[9],ymm7[10,11],ymm4[12],ymm7[13],ymm4[14],ymm7[15]
7594 ; AVX2-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7595 ; AVX2-FCP-NEXT: vmovdqa %ymm4, %ymm9
7596 ; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7597 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm2, %xmm4
7598 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1,2,3],xmm2[4,5],xmm4[6,7]
7599 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm2, %xmm0
7600 ; AVX2-FCP-NEXT: vmovdqa 384(%rdi), %ymm5
7601 ; AVX2-FCP-NEXT: vmovdqa 416(%rdi), %ymm6
7602 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm5[0],ymm6[1,2],ymm5[3],ymm6[4],ymm5[5],ymm6[6,7],ymm5[8],ymm6[9,10],ymm5[11],ymm6[12],ymm5[13],ymm6[14,15]
7603 ; AVX2-FCP-NEXT: vmovdqa %ymm6, %ymm10
7604 ; AVX2-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7605 ; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7606 ; AVX2-FCP-NEXT: vpermd %ymm2, %ymm1, %ymm1
7607 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm1
7608 ; AVX2-FCP-NEXT: vpblendvb %ymm13, %ymm0, %ymm1, %ymm0
7609 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7610 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7611 ; AVX2-FCP-NEXT: vpblendw $181, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
7612 ; AVX2-FCP-NEXT: # ymm0 = mem[0],ymm0[1],mem[2],ymm0[3],mem[4,5],ymm0[6],mem[7,8],ymm0[9],mem[10],ymm0[11],mem[12,13],ymm0[14],mem[15]
7613 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1
7614 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6],xmm1[7]
7615 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm4 = [2,3,12,13,6,7,0,1,10,11,4,5,14,15,10,11]
7616 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm0, %xmm0
7617 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7618 ; AVX2-FCP-NEXT: vpblendw $173, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
7619 ; AVX2-FCP-NEXT: # ymm1 = mem[0],ymm1[1],mem[2,3],ymm1[4],mem[5],ymm1[6],mem[7,8],ymm1[9],mem[10,11],ymm1[12],mem[13],ymm1[14],mem[15]
7620 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [2,0,0,0,4,7,1,6]
7621 ; AVX2-FCP-NEXT: vpermd %ymm1, %ymm2, %ymm3
7622 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [2,3,4,5,10,11,0,1,14,15,2,3,12,13,0,1,18,19,20,21,26,27,16,17,30,31,18,19,28,29,16,17]
7623 ; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm3, %ymm3
7624 ; AVX2-FCP-NEXT: vpblendvb %ymm13, %ymm0, %ymm3, %ymm6
7625 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7626 ; AVX2-FCP-NEXT: vpblendw $74, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
7627 ; AVX2-FCP-NEXT: # ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4,5],mem[6],ymm0[7,8],mem[9],ymm0[10],mem[11],ymm0[12,13],mem[14],ymm0[15]
7628 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm3
7629 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5,6],xmm3[7]
7630 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm0, %xmm0
7631 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm11[0],ymm14[1],ymm11[2,3],ymm14[4],ymm11[5],ymm14[6],ymm11[7,8],ymm14[9],ymm11[10,11],ymm14[12],ymm11[13],ymm14[14],ymm11[15]
7632 ; AVX2-FCP-NEXT: vpermd %ymm3, %ymm2, %ymm3
7633 ; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm3, %ymm3
7634 ; AVX2-FCP-NEXT: vpblendvb %ymm13, %ymm0, %ymm3, %ymm3
7635 ; AVX2-FCP-NEXT: vpblendw $74, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm0 # 32-byte Folded Reload
7636 ; AVX2-FCP-NEXT: # ymm0 = ymm15[0],mem[1],ymm15[2],mem[3],ymm15[4,5],mem[6],ymm15[7,8],mem[9],ymm15[10],mem[11],ymm15[12,13],mem[14],ymm15[15]
7637 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm11
7638 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm11[2,3],xmm0[4,5,6],xmm11[7]
7639 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm12[0],ymm8[1],ymm12[2,3],ymm8[4],ymm12[5],ymm8[6],ymm12[7,8],ymm8[9],ymm12[10,11],ymm8[12],ymm12[13],ymm8[14],ymm12[15]
7640 ; AVX2-FCP-NEXT: vpermd %ymm11, %ymm2, %ymm11
7641 ; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm11, %ymm11
7642 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm0, %xmm0
7643 ; AVX2-FCP-NEXT: vpblendvb %ymm13, %ymm0, %ymm11, %ymm0
7644 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4,5],ymm7[6],ymm9[7,8],ymm7[9],ymm9[10],ymm7[11],ymm9[12,13],ymm7[14],ymm9[15]
7645 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm11, %xmm12
7646 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0,1],xmm12[2,3],xmm11[4,5,6],xmm12[7]
7647 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm11, %xmm4
7648 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm10[0],ymm5[1],ymm10[2,3],ymm5[4],ymm10[5],ymm5[6],ymm10[7,8],ymm5[9],ymm10[10,11],ymm5[12],ymm10[13],ymm5[14],ymm10[15]
7649 ; AVX2-FCP-NEXT: vpermd %ymm11, %ymm2, %ymm2
7650 ; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm1
7651 ; AVX2-FCP-NEXT: vpblendvb %ymm13, %ymm4, %ymm1, %ymm1
7652 ; AVX2-FCP-NEXT: vmovdqa 288(%rdi), %ymm2
7653 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,3,1,3,0,3,5,7]
7654 ; AVX2-FCP-NEXT: vpermd %ymm2, %ymm5, %ymm11
7655 ; AVX2-FCP-NEXT: vmovdqa %ymm2, %ymm12
7656 ; AVX2-FCP-NEXT: vpbroadcastq {{.*#+}} ymm7 = [0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27]
7657 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm11, %ymm11
7658 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
7659 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm2[0,1,2,3,4],ymm11[5,6,7],ymm2[8,9,10,11,12],ymm11[13,14,15]
7660 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm11[4,5,6,7]
7661 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7662 ; AVX2-FCP-NEXT: vmovdqa 608(%rdi), %ymm2
7663 ; AVX2-FCP-NEXT: vpermd %ymm2, %ymm5, %ymm11
7664 ; AVX2-FCP-NEXT: vmovdqa %ymm2, %ymm10
7665 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm11, %ymm11
7666 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
7667 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm2[0,1,2,3,4],ymm11[5,6,7],ymm2[8,9,10,11,12],ymm11[13,14,15]
7668 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm11[4,5,6,7]
7669 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7670 ; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %ymm2
7671 ; AVX2-FCP-NEXT: vpermd %ymm2, %ymm5, %ymm11
7672 ; AVX2-FCP-NEXT: vmovdqa %ymm2, %ymm9
7673 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm11, %ymm11
7674 ; AVX2-FCP-NEXT: vmovdqu (%rsp), %ymm2 # 32-byte Reload
7675 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm2[0,1,2,3,4],ymm11[5,6,7],ymm2[8,9,10,11,12],ymm11[13,14,15]
7676 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm11[4,5,6,7]
7677 ; AVX2-FCP-NEXT: vmovdqu %ymm2, (%rsp) # 32-byte Spill
7678 ; AVX2-FCP-NEXT: vmovdqa 448(%rdi), %ymm2
7679 ; AVX2-FCP-NEXT: vpermd %ymm2, %ymm5, %ymm5
7680 ; AVX2-FCP-NEXT: vmovdqa %ymm2, %ymm4
7681 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7682 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm5, %ymm5
7683 ; AVX2-FCP-NEXT: vmovdqa %ymm7, %ymm14
7684 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
7685 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm2[0,1,2,3,4],ymm5[5,6,7],ymm2[8,9,10,11,12],ymm5[13,14,15]
7686 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4,5,6,7]
7687 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7688 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm7 = [1,3,2,3,1,3,6,7]
7689 ; AVX2-FCP-NEXT: vpermd %ymm12, %ymm7, %ymm11
7690 ; AVX2-FCP-NEXT: vmovdqa %ymm12, %ymm8
7691 ; AVX2-FCP-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7692 ; AVX2-FCP-NEXT: vpbroadcastq {{.*#+}} ymm13 = [0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25]
7693 ; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm11, %ymm11
7694 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm6[0,1,2,3,4],ymm11[5,6,7],ymm6[8,9,10,11,12],ymm11[13,14,15]
7695 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm11[4,5,6,7]
7696 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7697 ; AVX2-FCP-NEXT: vpermd %ymm10, %ymm7, %ymm6
7698 ; AVX2-FCP-NEXT: vmovdqa %ymm10, %ymm12
7699 ; AVX2-FCP-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7700 ; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm6, %ymm6
7701 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm3[0,1,2,3,4],ymm6[5,6,7],ymm3[8,9,10,11,12],ymm6[13,14,15]
7702 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm6[4,5,6,7]
7703 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7704 ; AVX2-FCP-NEXT: vpermd %ymm9, %ymm7, %ymm3
7705 ; AVX2-FCP-NEXT: vmovdqa %ymm9, %ymm5
7706 ; AVX2-FCP-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7707 ; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm3, %ymm3
7708 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm0[0,1,2,3,4],ymm3[5,6,7],ymm0[8,9,10,11,12],ymm3[13,14,15]
7709 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
7710 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7711 ; AVX2-FCP-NEXT: vpermd %ymm4, %ymm7, %ymm0
7712 ; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm0, %ymm0
7713 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7],ymm1[8,9,10,11,12],ymm0[13,14,15]
7714 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
7715 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7716 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
7717 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7718 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0,1],ymm4[2],ymm9[3],ymm4[4],ymm9[5,6],ymm4[7],ymm9[8,9],ymm4[10],ymm9[11],ymm4[12],ymm9[13,14],ymm4[15]
7719 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1
7720 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1,2],xmm1[3,4],xmm0[5,6,7]
7721 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
7722 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm6
7723 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7724 ; AVX2-FCP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload
7725 ; AVX2-FCP-NEXT: # ymm3 = ymm1[0],mem[1],ymm1[2,3],mem[4],ymm1[5],mem[6],ymm1[7,8],mem[9],ymm1[10,11],mem[12],ymm1[13],mem[14],ymm1[15]
7726 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,2,0,0,5,7,2,4]
7727 ; AVX2-FCP-NEXT: vpermd %ymm3, %ymm1, %ymm7
7728 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
7729 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm7, %ymm7
7730 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm6[0,1,2],ymm7[3,4,5,6,7]
7731 ; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [1,4,6,0,1,4,6,0]
7732 ; AVX2-FCP-NEXT: # ymm6 = mem[0,1,0,1]
7733 ; AVX2-FCP-NEXT: vpermd %ymm8, %ymm6, %ymm11
7734 ; AVX2-FCP-NEXT: vmovdqa %ymm14, %ymm8
7735 ; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm11, %ymm11
7736 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm7[0,1,2,3,4],ymm11[5,6,7],ymm7[8,9,10,11,12],ymm11[13,14,15]
7737 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm11[4,5,6,7]
7738 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7739 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
7740 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
7741 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm15[0,1],ymm14[2],ymm15[3],ymm14[4],ymm15[5,6],ymm14[7],ymm15[8,9],ymm14[10],ymm15[11],ymm14[12],ymm15[13,14],ymm14[15]
7742 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm7, %xmm11
7743 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm11[3,4],xmm7[5,6,7]
7744 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm7, %xmm7
7745 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
7746 ; AVX2-FCP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm11 # 32-byte Folded Reload
7747 ; AVX2-FCP-NEXT: # ymm11 = ymm10[0],mem[1],ymm10[2,3],mem[4],ymm10[5],mem[6],ymm10[7,8],mem[9],ymm10[10,11],mem[12],ymm10[13],mem[14],ymm10[15]
7748 ; AVX2-FCP-NEXT: vpermd %ymm11, %ymm1, %ymm11
7749 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm11, %ymm11
7750 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2],ymm11[3,4,5,6,7]
7751 ; AVX2-FCP-NEXT: vpermd %ymm12, %ymm6, %ymm11
7752 ; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm11, %ymm11
7753 ; AVX2-FCP-NEXT: vmovdqa %ymm8, %ymm13
7754 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm7[0,1,2,3,4],ymm11[5,6,7],ymm7[8,9,10,11,12],ymm11[13,14,15]
7755 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm11[4,5,6,7]
7756 ; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7757 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
7758 ; AVX2-FCP-NEXT: vpblendw $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm7 # 32-byte Folded Reload
7759 ; AVX2-FCP-NEXT: # ymm7 = mem[0,1],ymm12[2],mem[3],ymm12[4],mem[5,6],ymm12[7],mem[8,9],ymm12[10],mem[11],ymm12[12],mem[13,14],ymm12[15]
7760 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm7, %xmm11
7761 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm11[3,4],xmm7[5,6,7]
7762 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
7763 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
7764 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm2[0],ymm8[1],ymm2[2,3],ymm8[4],ymm2[5],ymm8[6],ymm2[7,8],ymm8[9],ymm2[10,11],ymm8[12],ymm2[13],ymm8[14],ymm2[15]
7765 ; AVX2-FCP-NEXT: vpermd %ymm11, %ymm1, %ymm11
7766 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm11, %ymm11
7767 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm7, %xmm7
7768 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2],ymm11[3,4,5,6,7]
7769 ; AVX2-FCP-NEXT: vpermd %ymm5, %ymm6, %ymm11
7770 ; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm11, %ymm11
7771 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm7[0,1,2,3,4],ymm11[5,6,7],ymm7[8,9,10,11,12],ymm11[13,14,15]
7772 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3],ymm11[4,5,6,7]
7773 ; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7774 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
7775 ; AVX2-FCP-NEXT: vpblendw $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm7 # 32-byte Folded Reload
7776 ; AVX2-FCP-NEXT: # ymm7 = mem[0,1],ymm13[2],mem[3],ymm13[4],mem[5,6],ymm13[7],mem[8,9],ymm13[10],mem[11],ymm13[12],mem[13,14],ymm13[15]
7777 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm7, %xmm11
7778 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm11[3,4],xmm7[5,6,7]
7779 ; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm7, %xmm0
7780 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
7781 ; AVX2-FCP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm7 # 32-byte Folded Reload
7782 ; AVX2-FCP-NEXT: # ymm7 = ymm5[0],mem[1],ymm5[2,3],mem[4],ymm5[5],mem[6],ymm5[7,8],mem[9],ymm5[10,11],mem[12],ymm5[13],mem[14],ymm5[15]
7783 ; AVX2-FCP-NEXT: vpermd %ymm7, %ymm1, %ymm1
7784 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm1
7785 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
7786 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm1 # 32-byte Folded Reload
7787 ; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27]
7788 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
7789 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
7790 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7791 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0],ymm9[1,2],ymm4[3],ymm9[4],ymm4[5],ymm9[6,7],ymm4[8],ymm9[9,10],ymm4[11],ymm9[12],ymm4[13],ymm9[14,15]
7792 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1
7793 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3]
7794 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm4 = [6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
7795 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm0, %xmm0
7796 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
7797 ; AVX2-FCP-NEXT: vpblendw $74, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm3 # 32-byte Folded Reload
7798 ; AVX2-FCP-NEXT: # ymm3 = ymm11[0],mem[1],ymm11[2],mem[3],ymm11[4,5],mem[6],ymm11[7,8],mem[9],ymm11[10],mem[11],ymm11[12,13],mem[14],ymm11[15]
7799 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [0,3,0,0,5,0,2,7]
7800 ; AVX2-FCP-NEXT: vpermd %ymm3, %ymm9, %ymm6
7801 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [u,u,u,u,u,u,u,u,u,u,u,u,2,3,4,5,18,19,20,21,26,27,16,17,30,31,30,31,18,19,20,21]
7802 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm6, %ymm6
7803 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm6[3,4,5,6,7]
7804 ; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [2,4,7,0,2,4,7,0]
7805 ; AVX2-FCP-NEXT: # ymm6 = mem[0,1,0,1]
7806 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm7 # 32-byte Folded Reload
7807 ; AVX2-FCP-NEXT: vpbroadcastq {{.*#+}} ymm1 = [0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25,0,0,16,17,22,23,24,25]
7808 ; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm7
7809 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm0[0,1,2,3,4],ymm7[5,6,7],ymm0[8,9,10,11,12],ymm7[13,14,15]
7810 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm7[4,5,6,7]
7811 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7812 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm14[0],ymm15[1,2],ymm14[3],ymm15[4],ymm14[5],ymm15[6,7],ymm14[8],ymm15[9,10],ymm14[11],ymm15[12],ymm14[13],ymm15[14,15]
7813 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm7
7814 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm7[0],xmm0[1],xmm7[2],xmm0[3]
7815 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm0, %xmm0
7816 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
7817 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm15[0],ymm10[1],ymm15[2],ymm10[3],ymm15[4,5],ymm10[6],ymm15[7,8],ymm10[9],ymm15[10],ymm10[11],ymm15[12,13],ymm10[14],ymm15[15]
7818 ; AVX2-FCP-NEXT: vpermd %ymm7, %ymm9, %ymm7
7819 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm7, %ymm7
7820 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm7[3,4,5,6,7]
7821 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm7 # 32-byte Folded Reload
7822 ; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm7
7823 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm0[0,1,2,3,4],ymm7[5,6,7],ymm0[8,9,10,11,12],ymm7[13,14,15]
7824 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm7[4,5,6,7]
7825 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7826 ; AVX2-FCP-NEXT: vpblendw $214, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm0 # 32-byte Folded Reload
7827 ; AVX2-FCP-NEXT: # ymm0 = ymm12[0],mem[1,2],ymm12[3],mem[4],ymm12[5],mem[6,7],ymm12[8],mem[9,10],ymm12[11],mem[12],ymm12[13],mem[14,15]
7828 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm7
7829 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm7[0],xmm0[1],xmm7[2],xmm0[3]
7830 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm8[0],ymm2[1],ymm8[2],ymm2[3],ymm8[4,5],ymm2[6],ymm8[7,8],ymm2[9],ymm8[10],ymm2[11],ymm8[12,13],ymm2[14],ymm8[15]
7831 ; AVX2-FCP-NEXT: vmovdqa %ymm8, %ymm12
7832 ; AVX2-FCP-NEXT: vpermd %ymm7, %ymm9, %ymm7
7833 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm7, %ymm7
7834 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm0, %xmm0
7835 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm7[3,4,5,6,7]
7836 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
7837 ; AVX2-FCP-NEXT: vpermd %ymm8, %ymm6, %ymm7
7838 ; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm7
7839 ; AVX2-FCP-NEXT: vmovdqa %ymm1, %ymm2
7840 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm0[0,1,2,3,4],ymm7[5,6,7],ymm0[8,9,10,11,12],ymm7[13,14,15]
7841 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm7[4,5,6,7]
7842 ; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7843 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
7844 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm13[0],ymm10[1,2],ymm13[3],ymm10[4],ymm13[5],ymm10[6,7],ymm13[8],ymm10[9,10],ymm13[11],ymm10[12],ymm13[13],ymm10[14,15]
7845 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm7
7846 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm7[0],xmm0[1],xmm7[2],xmm0[3]
7847 ; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm0, %xmm0
7848 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
7849 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm13[0],ymm5[1],ymm13[2],ymm5[3],ymm13[4,5],ymm5[6],ymm13[7,8],ymm5[9],ymm13[10],ymm5[11],ymm13[12,13],ymm5[14],ymm13[15]
7850 ; AVX2-FCP-NEXT: vpermd %ymm1, %ymm9, %ymm1
7851 ; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm1
7852 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
7853 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
7854 ; AVX2-FCP-NEXT: vpermd %ymm14, %ymm6, %ymm1
7855 ; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm1
7856 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
7857 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
7858 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
7859 ; AVX2-FCP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
7860 ; AVX2-FCP-NEXT: # ymm0 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5],mem[6],ymm0[7,8],mem[9],ymm0[10,11],mem[12],ymm0[13],mem[14],ymm0[15]
7861 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm2
7862 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3,4],xmm2[5,6,7]
7863 ; AVX2-FCP-NEXT: vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm2 # 32-byte Folded Reload
7864 ; AVX2-FCP-NEXT: # ymm2 = ymm11[0,1],mem[2],ymm11[3],mem[4],ymm11[5,6],mem[7],ymm11[8,9],mem[10],ymm11[11],mem[12],ymm11[13,14],mem[15]
7865 ; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [8,9,2,3,12,13,6,7,0,1,10,11,0,1,6,7]
7866 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm0
7867 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [1,3,0,0,6,0,3,5]
7868 ; AVX2-FCP-NEXT: vpermd %ymm2, %ymm5, %ymm2
7869 ; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm7 = [16,17,22,23,24,25,30,31,0,0,0,0,0,1,6,7,16,17,22,23,24,25,30,31,0,0,0,0,0,1,6,7]
7870 ; AVX2-FCP-NEXT: # ymm7 = mem[0,1,0,1]
7871 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm2
7872 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0,1,2],ymm2[3,4,5,6,7]
7873 ; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,1,3,0,2,5,7]
7874 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
7875 ; AVX2-FCP-NEXT: vpbroadcastq {{.*#+}} ymm4 = [16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31]
7876 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm3
7877 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
7878 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
7879 ; AVX2-FCP-NEXT: vpblendw $173, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
7880 ; AVX2-FCP-NEXT: # ymm3 = mem[0],ymm3[1],mem[2,3],ymm3[4],mem[5],ymm3[6],mem[7,8],ymm3[9],mem[10,11],ymm3[12],mem[13],ymm3[14],mem[15]
7881 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm3, %xmm11
7882 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm11[0,1,2],xmm3[3,4],xmm11[5,6,7]
7883 ; AVX2-FCP-NEXT: vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm11 # 32-byte Folded Reload
7884 ; AVX2-FCP-NEXT: # ymm11 = ymm15[0,1],mem[2],ymm15[3],mem[4],ymm15[5,6],mem[7],ymm15[8,9],mem[10],ymm15[11],mem[12],ymm15[13,14],mem[15]
7885 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm3, %xmm3
7886 ; AVX2-FCP-NEXT: vpermd %ymm11, %ymm5, %ymm11
7887 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm11, %ymm11
7888 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm11[3,4,5,6,7]
7889 ; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
7890 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm9, %ymm9
7891 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm9[6,7]
7892 ; AVX2-FCP-NEXT: vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm9 # 32-byte Folded Reload
7893 ; AVX2-FCP-NEXT: # ymm9 = ymm12[0,1],mem[2],ymm12[3],mem[4],ymm12[5,6],mem[7],ymm12[8,9],mem[10],ymm12[11],mem[12],ymm12[13,14],mem[15]
7894 ; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
7895 ; AVX2-FCP-NEXT: vpblendw $173, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm11 # 32-byte Folded Reload
7896 ; AVX2-FCP-NEXT: # ymm11 = mem[0],ymm11[1],mem[2,3],ymm11[4],mem[5],ymm11[6],mem[7,8],ymm11[9],mem[10,11],ymm11[12],mem[13],ymm11[14],mem[15]
7897 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm11, %xmm12
7898 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3,4],xmm12[5,6,7]
7899 ; AVX2-FCP-NEXT: vpermd %ymm9, %ymm5, %ymm9
7900 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm9, %ymm9
7901 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm11, %xmm11
7902 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2],ymm9[3,4,5,6,7]
7903 ; AVX2-FCP-NEXT: vpermd %ymm8, %ymm0, %ymm8
7904 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm8, %ymm8
7905 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4,5],ymm8[6,7]
7906 ; AVX2-FCP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm9 # 32-byte Folded Reload
7907 ; AVX2-FCP-NEXT: # ymm9 = ymm10[0],mem[1],ymm10[2,3],mem[4],ymm10[5],mem[6],ymm10[7,8],mem[9],ymm10[10,11],mem[12],ymm10[13],mem[14],ymm10[15]
7908 ; AVX2-FCP-NEXT: vextracti128 $1, %ymm9, %xmm11
7909 ; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1,2],xmm9[3,4],xmm11[5,6,7]
7910 ; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm9, %xmm6
7911 ; AVX2-FCP-NEXT: vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm9 # 32-byte Folded Reload
7912 ; AVX2-FCP-NEXT: # ymm9 = ymm13[0,1],mem[2],ymm13[3],mem[4],ymm13[5,6],mem[7],ymm13[8,9],mem[10],ymm13[11],mem[12],ymm13[13,14],mem[15]
7913 ; AVX2-FCP-NEXT: vpermd %ymm9, %ymm5, %ymm5
7914 ; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm5, %ymm5
7915 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2],ymm5[3,4,5,6,7]
7916 ; AVX2-FCP-NEXT: vpermd %ymm14, %ymm0, %ymm0
7917 ; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm0, %ymm0
7918 ; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5],ymm0[6,7]
7919 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7920 ; AVX2-FCP-NEXT: vmovaps %ymm4, 64(%rsi)
7921 ; AVX2-FCP-NEXT: vmovups (%rsp), %ymm4 # 32-byte Reload
7922 ; AVX2-FCP-NEXT: vmovaps %ymm4, (%rsi)
7923 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7924 ; AVX2-FCP-NEXT: vmovaps %ymm4, 96(%rsi)
7925 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7926 ; AVX2-FCP-NEXT: vmovaps %ymm4, 32(%rsi)
7927 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7928 ; AVX2-FCP-NEXT: vmovaps %ymm4, 64(%rdx)
7929 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7930 ; AVX2-FCP-NEXT: vmovaps %ymm4, (%rdx)
7931 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7932 ; AVX2-FCP-NEXT: vmovaps %ymm4, 96(%rdx)
7933 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7934 ; AVX2-FCP-NEXT: vmovaps %ymm4, 32(%rdx)
7935 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7936 ; AVX2-FCP-NEXT: vmovaps %ymm4, 64(%rcx)
7937 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7938 ; AVX2-FCP-NEXT: vmovaps %ymm4, (%rcx)
7939 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7940 ; AVX2-FCP-NEXT: vmovaps %ymm4, 96(%rcx)
7941 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
7942 ; AVX2-FCP-NEXT: vmovaps %ymm4, 32(%rcx)
7943 ; AVX2-FCP-NEXT: vmovdqa %ymm1, 64(%r8)
7944 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7945 ; AVX2-FCP-NEXT: vmovaps %ymm1, (%r8)
7946 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7947 ; AVX2-FCP-NEXT: vmovaps %ymm1, 96(%r8)
7948 ; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
7949 ; AVX2-FCP-NEXT: vmovaps %ymm1, 32(%r8)
7950 ; AVX2-FCP-NEXT: vmovdqa %ymm0, 64(%r9)
7951 ; AVX2-FCP-NEXT: vmovdqa %ymm8, (%r9)
7952 ; AVX2-FCP-NEXT: vmovdqa %ymm3, 96(%r9)
7953 ; AVX2-FCP-NEXT: vmovdqa %ymm2, 32(%r9)
7954 ; AVX2-FCP-NEXT: addq $1000, %rsp # imm = 0x3E8
7955 ; AVX2-FCP-NEXT: vzeroupper
7956 ; AVX2-FCP-NEXT: retq
7957 ;
7958 ; AVX512-LABEL: load_i16_stride5_vf64:
7959 ; AVX512: # %bb.0:
7960 ; AVX512-NEXT: subq $552, %rsp # imm = 0x228
7961 ; AVX512-NEXT: vmovdqa 384(%rdi), %ymm1
7962 ; AVX512-NEXT: vmovdqa 416(%rdi), %ymm2
7963 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm2[1,2],ymm1[3],ymm2[4],ymm1[5],ymm2[6,7],ymm1[8],ymm2[9,10],ymm1[11],ymm2[12],ymm1[13],ymm2[14,15]
7964 ; AVX512-NEXT: vmovdqa %ymm2, %ymm11
7965 ; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7966 ; AVX512-NEXT: vmovdqa %ymm1, %ymm6
7967 ; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7968 ; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
7969 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6],ymm1[7]
7970 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,6,7,16,17,26,27,20,21,30,31,24,25,128,128,128,128,128,128]
7971 ; AVX512-NEXT: vpshufb %ymm0, %ymm1, %ymm2
7972 ; AVX512-NEXT: vmovdqa 352(%rdi), %ymm3
7973 ; AVX512-NEXT: vmovdqa 320(%rdi), %ymm4
7974 ; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5],ymm3[6],ymm4[7,8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13],ymm3[14],ymm4[15]
7975 ; AVX512-NEXT: vmovdqa %ymm4, %ymm7
7976 ; AVX512-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7977 ; AVX512-NEXT: vmovdqa %ymm3, %ymm8
7978 ; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
7979 ; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm3
7980 ; AVX512-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm3[1,2,3],xmm1[4,5],xmm3[6,7]
7981 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
7982 ; AVX512-NEXT: vpshufb %ymm1, %ymm3, %ymm3
7983 ; AVX512-NEXT: vporq %ymm2, %ymm3, %ymm19
7984 ; AVX512-NEXT: vmovdqa 192(%rdi), %ymm15
7985 ; AVX512-NEXT: vmovdqa 224(%rdi), %ymm13
7986 ; AVX512-NEXT: vmovdqa 176(%rdi), %xmm12
7987 ; AVX512-NEXT: vmovdqa 160(%rdi), %xmm14
7988 ; AVX512-NEXT: vmovdqa (%rdi), %ymm4
7989 ; AVX512-NEXT: vmovdqa 32(%rdi), %ymm5
7990 ; AVX512-NEXT: vmovdqa 64(%rdi), %ymm3
7991 ; AVX512-NEXT: vmovdqa 96(%rdi), %ymm9
7992 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4],ymm3[5],ymm9[6,7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12],ymm3[13],ymm9[14,15]
7993 ; AVX512-NEXT: vmovdqa %ymm3, %ymm10
7994 ; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
7995 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6],ymm3[7]
7996 ; AVX512-NEXT: vpshufb %ymm0, %ymm2, %ymm0
7997 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10,11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]
7998 ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm3
7999 ; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
8000 ; AVX512-NEXT: vpshufb %ymm1, %ymm2, %ymm1
8001 ; AVX512-NEXT: vpor %ymm0, %ymm1, %ymm0
8002 ; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8003 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm11[0],ymm6[1],ymm11[2,3],ymm6[4],ymm11[5],ymm6[6],ymm11[7,8],ymm6[9],ymm11[10,11],ymm6[12],ymm11[13],ymm6[14],ymm11[15]
8004 ; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
8005 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
8006 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,8,9,18,19,28,29,22,23,16,17,26,27,128,128,128,128,128,128]
8007 ; AVX512-NEXT: vpshufb %ymm0, %ymm1, %ymm2
8008 ; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4,5],ymm7[6],ymm8[7,8],ymm7[9],ymm8[10],ymm7[11],ymm8[12,13],ymm7[14],ymm8[15]
8009 ; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm3
8010 ; AVX512-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6],xmm3[7]
8011 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [2,3,12,13,6,7,0,1,10,11,4,5,14,15,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
8012 ; AVX512-NEXT: vpshufb %ymm1, %ymm3, %ymm3
8013 ; AVX512-NEXT: vpor %ymm2, %ymm3, %ymm2
8014 ; AVX512-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8015 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm9[0],ymm10[1],ymm9[2,3],ymm10[4],ymm9[5],ymm10[6],ymm9[7,8],ymm10[9],ymm9[10,11],ymm10[12],ymm9[13],ymm10[14],ymm9[15]
8016 ; AVX512-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8017 ; AVX512-NEXT: vmovdqu %ymm9, (%rsp) # 32-byte Spill
8018 ; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
8019 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
8020 ; AVX512-NEXT: vpshufb %ymm0, %ymm2, %ymm0
8021 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
8022 ; AVX512-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8023 ; AVX512-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8024 ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm3
8025 ; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5,6],xmm3[7]
8026 ; AVX512-NEXT: vpshufb %ymm1, %ymm2, %ymm1
8027 ; AVX512-NEXT: vporq %ymm0, %ymm1, %ymm28
8028 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm6 = [6,7,0,1,10,11,4,5,14,15,14,15,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
8029 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm13[0,1],ymm15[2],ymm13[3],ymm15[4],ymm13[5,6],ymm15[7],ymm13[8,9],ymm15[10],ymm13[11],ymm15[12],ymm13[13,14],ymm15[15]
8030 ; AVX512-NEXT: vmovdqa64 %ymm15, %ymm18
8031 ; AVX512-NEXT: vmovdqa64 %ymm13, %ymm24
8032 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
8033 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4,5,6,7]
8034 ; AVX512-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[6,7,6,7,8,9,4,5,14,15,8,9,2,3,12,13,22,23,22,23,24,25,20,21,30,31,24,25,18,19,28,29]
8035 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm12[0],xmm14[1],xmm12[2,3]
8036 ; AVX512-NEXT: vmovdqa64 %xmm12, %xmm16
8037 ; AVX512-NEXT: vmovdqa64 %xmm14, %xmm30
8038 ; AVX512-NEXT: vpshufb %xmm6, %xmm1, %xmm1
8039 ; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[3,4,5,6,7]
8040 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
8041 ; AVX512-NEXT: vmovdqa 144(%rdi), %xmm2
8042 ; AVX512-NEXT: vmovdqa 128(%rdi), %xmm3
8043 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm3[2],xmm2[3]
8044 ; AVX512-NEXT: vmovdqa %xmm3, %xmm7
8045 ; AVX512-NEXT: vmovdqa %xmm2, %xmm11
8046 ; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
8047 ; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
8048 ; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
8049 ; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm9[0],ymm10[1],ymm9[2],ymm10[3],ymm9[4,5],ymm10[6],ymm9[7,8],ymm10[9],ymm9[10],ymm10[11],ymm9[12,13],ymm10[14],ymm9[15]
8050 ; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
8051 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]
8052 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm5[1,2],ymm4[3],ymm5[4],ymm4[5],ymm5[6,7],ymm4[8],ymm5[9,10],ymm4[11],ymm5[12],ymm4[13],ymm5[14,15]
8053 ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm3
8054 ; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
8055 ; AVX512-NEXT: vpshufb %ymm6, %ymm1, %ymm1
8056 ; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[6,7,0,1,10,11,4,5,14,15,8,9,12,13,14,15]
8057 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
8058 ; AVX512-NEXT: vmovdqa 256(%rdi), %ymm12
8059 ; AVX512-NEXT: vmovdqa 288(%rdi), %ymm15
8060 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm12[0],ymm15[1],ymm12[2,3],ymm15[4],ymm12[5],ymm15[6],ymm12[7,8],ymm15[9],ymm12[10,11],ymm15[12],ymm12[13],ymm15[14],ymm12[15]
8061 ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm3
8062 ; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
8063 ; AVX512-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
8064 ; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm2[0,1,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
8065 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8066 ; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm2
8067 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15]
8068 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
8069 ; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
8070 ; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8071 ; AVX512-NEXT: vmovdqa 464(%rdi), %xmm8
8072 ; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm8[3,1,2,3]
8073 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
8074 ; AVX512-NEXT: vmovdqa 448(%rdi), %xmm3
8075 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
8076 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
8077 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
8078 ; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8079 ; AVX512-NEXT: vmovdqa %xmm11, %xmm6
8080 ; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm11[3,1,2,3]
8081 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
8082 ; AVX512-NEXT: vmovdqa %xmm7, %xmm9
8083 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm7[0,2,2,3]
8084 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
8085 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
8086 ; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8087 ; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0],xmm8[1],xmm3[2,3]
8088 ; AVX512-NEXT: vmovdqa64 %xmm3, %xmm22
8089 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [6,7,2,3,4,5,6,7,6,7,2,3,12,13,6,7]
8090 ; AVX512-NEXT: vpshufb %xmm1, %xmm0, %xmm0
8091 ; AVX512-NEXT: vmovdqa64 %xmm1, %xmm23
8092 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm10
8093 ; AVX512-NEXT: vmovdqa 576(%rdi), %ymm1
8094 ; AVX512-NEXT: vmovdqa 608(%rdi), %ymm2
8095 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1],ymm1[2],ymm2[3],ymm1[4],ymm2[5,6],ymm1[7],ymm2[8,9],ymm1[10],ymm2[11],ymm1[12],ymm2[13,14],ymm1[15]
8096 ; AVX512-NEXT: vmovdqa64 %ymm2, %ymm20
8097 ; AVX512-NEXT: vmovdqa64 %ymm1, %ymm17
8098 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
8099 ; AVX512-NEXT: vpblendw {{.*#+}} xmm11 = xmm0[0,1,2],xmm1[3,4],xmm0[5,6,7]
8100 ; AVX512-NEXT: vmovdqa 512(%rdi), %ymm5
8101 ; AVX512-NEXT: vmovdqa 544(%rdi), %ymm13
8102 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm13[0],ymm5[1],ymm13[2,3],ymm5[4],ymm13[5],ymm5[6],ymm13[7,8],ymm5[9],ymm13[10,11],ymm5[12],ymm13[13],ymm5[14],ymm13[15]
8103 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
8104 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5,6,7]
8105 ; AVX512-NEXT: vmovdqa64 496(%rdi), %xmm21
8106 ; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm21[3,1,2,3]
8107 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm1[2,1,2,3,4,5,6,7]
8108 ; AVX512-NEXT: vmovdqa 480(%rdi), %xmm1
8109 ; AVX512-NEXT: vpshufd {{.*#+}} xmm14 = xmm1[0,2,2,3]
8110 ; AVX512-NEXT: vmovdqa %xmm1, %xmm7
8111 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm14 = xmm14[0,3,2,3,4,5,6,7]
8112 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
8113 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm14 = [u,u,u,u,u,u,u,u,8,9,2,3,12,13,6,7,16,17,26,27,26,27,30,31,24,25,18,19,28,29,22,23]
8114 ; AVX512-NEXT: vpshufb %ymm14, %ymm2, %ymm2
8115 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3,4,5,6,7]
8116 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm2 = [12,13,14,15,4,5,14,15,8,9,2,3,12,13,6,7]
8117 ; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm11
8118 ; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
8119 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm11[5,6,7]
8120 ; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm27 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
8121 ; AVX512-NEXT: vpternlogq $184, %zmm19, %zmm27, %zmm10
8122 ; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm10, %zmm0
8123 ; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8124 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm15[0,1],ymm12[2],ymm15[3],ymm12[4],ymm15[5,6],ymm12[7],ymm15[8,9],ymm12[10],ymm15[11],ymm12[12],ymm15[13,14],ymm12[15]
8125 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm10
8126 ; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm10[3,4],xmm0[5,6,7]
8127 ; AVX512-NEXT: vpshufb %xmm2, %xmm0, %xmm0
8128 ; AVX512-NEXT: vmovdqa64 %ymm18, %ymm3
8129 ; AVX512-NEXT: vmovdqa64 %ymm24, %ymm1
8130 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5],ymm3[6],ymm1[7,8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13],ymm3[14],ymm1[15]
8131 ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm10
8132 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0],ymm2[1,2],ymm10[3],ymm2[4,5,6,7]
8133 ; AVX512-NEXT: vpshufb %ymm14, %ymm2, %ymm2
8134 ; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm16[3,1,2,3]
8135 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[2,1,2,3,4,5,6,7]
8136 ; AVX512-NEXT: vmovdqa64 %xmm30, %xmm24
8137 ; AVX512-NEXT: vpshufd {{.*#+}} xmm11 = xmm30[0,2,2,3]
8138 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,3,2,3,4,5,6,7]
8139 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
8140 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1],ymm2[2,3,4,5,6,7]
8141 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8142 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
8143 ; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm9[0],xmm6[1],xmm9[2,3]
8144 ; AVX512-NEXT: vmovdqa64 %xmm9, %xmm25
8145 ; AVX512-NEXT: vmovdqa64 %xmm23, %xmm4
8146 ; AVX512-NEXT: vpshufb %xmm4, %xmm2, %xmm2
8147 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
8148 ; AVX512-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm2 # 64-byte Folded Reload
8149 ; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
8150 ; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8151 ; AVX512-NEXT: vmovdqa64 %xmm22, %xmm4
8152 ; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm4[0,1],xmm8[2],xmm4[3]
8153 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,8,9,4,5,14,15,8,9]
8154 ; AVX512-NEXT: vpshufb %xmm0, %xmm2, %xmm2
8155 ; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
8156 ; AVX512-NEXT: vpblendw {{.*#+}} ymm10 = ymm5[0],ymm13[1],ymm5[2,3],ymm13[4],ymm5[5],ymm13[6],ymm5[7,8],ymm13[9],ymm5[10,11],ymm13[12],ymm5[13],ymm13[14],ymm5[15]
8157 ; AVX512-NEXT: vmovdqa64 %ymm13, %ymm26
8158 ; AVX512-NEXT: vmovdqa64 %ymm5, %ymm22
8159 ; AVX512-NEXT: vextracti128 $1, %ymm10, %xmm11
8160 ; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4,5,6,7]
8161 ; AVX512-NEXT: vpshufd {{.*#+}} xmm11 = xmm7[0,3,2,3]
8162 ; AVX512-NEXT: vmovdqa64 %xmm7, %xmm30
8163 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,2,2,3,4,5,6,7]
8164 ; AVX512-NEXT: vpsrlq $48, %xmm21, %xmm13
8165 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm13[0],xmm11[1],xmm13[1]
8166 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm13 = [2,3,12,13,0,1,0,1,10,11,4,5,14,15,8,9,18,19,28,29,16,17,16,17,26,27,20,21,30,31,24,25]
8167 ; AVX512-NEXT: vpshufb %ymm13, %ymm10, %ymm10
8168 ; AVX512-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0,1,2],xmm10[3,4,5,6,7]
8169 ; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
8170 ; AVX512-NEXT: vmovdqa64 %ymm20, %ymm9
8171 ; AVX512-NEXT: vmovdqa64 %ymm17, %ymm7
8172 ; AVX512-NEXT: vpblendw {{.*#+}} ymm11 = ymm7[0],ymm9[1,2],ymm7[3],ymm9[4],ymm7[5],ymm9[6,7],ymm7[8],ymm9[9,10],ymm7[11],ymm9[12],ymm7[13],ymm9[14,15]
8173 ; AVX512-NEXT: vextracti128 $1, %ymm11, %xmm14
8174 ; AVX512-NEXT: vpblendd {{.*#+}} xmm11 = xmm14[0],xmm11[1],xmm14[2],xmm11[3]
8175 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm14 = [0,1,2,3,6,7,0,1,10,11,4,5,14,15,8,9]
8176 ; AVX512-NEXT: vpshufb %xmm14, %xmm11, %xmm11
8177 ; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
8178 ; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm11[5,6,7]
8179 ; AVX512-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm2 # 64-byte Folded Reload
8180 ; AVX512-NEXT: vinserti64x4 $1, %ymm10, %zmm2, %zmm2
8181 ; AVX512-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8182 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5],ymm1[6],ymm3[7,8],ymm1[9],ymm3[10,11],ymm1[12],ymm3[13],ymm1[14],ymm3[15]
8183 ; AVX512-NEXT: vmovdqa64 %ymm1, %ymm17
8184 ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm10
8185 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm10[1],ymm2[2],ymm10[3],ymm2[4,5,6,7]
8186 ; AVX512-NEXT: vpshufb %ymm13, %ymm2, %ymm2
8187 ; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm24[0,3,2,3]
8188 ; AVX512-NEXT: vmovdqa64 %xmm24, %xmm20
8189 ; AVX512-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[1,2,2,3,4,5,6,7]
8190 ; AVX512-NEXT: vmovdqa64 %xmm16, %xmm1
8191 ; AVX512-NEXT: vpsrlq $48, %xmm16, %xmm11
8192 ; AVX512-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]
8193 ; AVX512-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm2[3,4,5,6,7]
8194 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3],ymm2[4,5,6,7]
8195 ; AVX512-NEXT: vpblendw {{.*#+}} ymm10 = ymm12[0],ymm15[1,2],ymm12[3],ymm15[4],ymm12[5],ymm15[6,7],ymm12[8],ymm15[9,10],ymm12[11],ymm15[12],ymm12[13],ymm15[14,15]
8196 ; AVX512-NEXT: vmovdqa64 %ymm12, %ymm19
8197 ; AVX512-NEXT: vextracti128 $1, %ymm10, %xmm11
8198 ; AVX512-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2],xmm10[3]
8199 ; AVX512-NEXT: vpshufb %xmm14, %xmm10, %xmm10
8200 ; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
8201 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm10[5,6,7]
8202 ; AVX512-NEXT: vmovdqa64 %xmm25, %xmm5
8203 ; AVX512-NEXT: vpblendd {{.*#+}} xmm10 = xmm5[0,1],xmm6[2],xmm5[3]
8204 ; AVX512-NEXT: vmovdqa64 %xmm6, %xmm23
8205 ; AVX512-NEXT: vpshufb %xmm0, %xmm10, %xmm0
8206 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8207 ; AVX512-NEXT: vpternlogq $184, %zmm28, %zmm27, %zmm0
8208 ; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
8209 ; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8210 ; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
8211 ; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm18 # 32-byte Reload
8212 ; AVX512-NEXT: vmovdqa64 %ymm18, %ymm0
8213 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm12[1],ymm0[2,3],ymm12[4],ymm0[5],ymm12[6],ymm0[7,8],ymm12[9],ymm0[10,11],ymm12[12],ymm0[13],ymm12[14],ymm0[15]
8214 ; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
8215 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm2[6],ymm0[7]
8216 ; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Reload
8217 ; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Reload
8218 ; AVX512-NEXT: vmovdqa64 %ymm24, %ymm0
8219 ; AVX512-NEXT: vmovdqa64 %ymm16, %ymm5
8220 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm5[2],ymm0[3],ymm5[4],ymm0[5,6],ymm5[7],ymm0[8,9],ymm5[10],ymm0[11],ymm5[12],ymm0[13,14],ymm5[15]
8221 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm10
8222 ; AVX512-NEXT: vpblendw {{.*#+}} xmm10 = xmm0[0,1,2],xmm10[3,4],xmm0[5,6,7]
8223 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm0 = [4,5,14,15,8,9,2,3,12,13,10,11,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
8224 ; AVX512-NEXT: vpshufb %ymm0, %ymm2, %ymm2
8225 ; AVX512-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[4,5,14,15,8,9,2,3,12,13,6,7,12,13,14,15]
8226 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2],ymm2[3,4,5,6,7]
8227 ; AVX512-NEXT: vpblendd {{.*#+}} xmm13 = xmm8[0],xmm4[1],xmm8[2,3]
8228 ; AVX512-NEXT: vmovdqa64 %xmm4, %xmm29
8229 ; AVX512-NEXT: vmovdqa64 %xmm8, %xmm28
8230 ; AVX512-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
8231 ; AVX512-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
8232 ; AVX512-NEXT: vpternlogq $184, %zmm2, %zmm27, %zmm13
8233 ; AVX512-NEXT: vmovdqa64 %ymm26, %ymm8
8234 ; AVX512-NEXT: vmovdqa64 %ymm22, %ymm4
8235 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm8[0],ymm4[1],ymm8[2],ymm4[3],ymm8[4,5],ymm4[6],ymm8[7,8],ymm4[9],ymm8[10],ymm4[11],ymm8[12,13],ymm4[14],ymm8[15]
8236 ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm14
8237 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm14[1],ymm2[2,3,4,5,6,7]
8238 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm14 = [4,5,14,15,12,13,2,3,12,13,6,7,0,1,10,11,20,21,30,31,28,29,18,19,28,29,22,23,16,17,26,27]
8239 ; AVX512-NEXT: vpshufb %ymm14, %ymm2, %ymm2
8240 ; AVX512-NEXT: vmovdqa64 %xmm30, %xmm6
8241 ; AVX512-NEXT: vpshufd {{.*#+}} xmm11 = xmm30[0,1,1,3]
8242 ; AVX512-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,7,6,7]
8243 ; AVX512-NEXT: vmovdqa64 %xmm21, %xmm5
8244 ; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm11 = xmm11[2],xmm21[2],xmm11[3],xmm21[3]
8245 ; AVX512-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0,1,2],xmm2[3,4,5,6,7]
8246 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm11[0,1,2,3],ymm2[4,5,6,7]
8247 ; AVX512-NEXT: vpblendw {{.*#+}} ymm11 = ymm9[0],ymm7[1],ymm9[2,3],ymm7[4],ymm9[5],ymm7[6],ymm9[7,8],ymm7[9],ymm9[10,11],ymm7[12],ymm9[13],ymm7[14],ymm9[15]
8248 ; AVX512-NEXT: vmovdqa64 %ymm7, %ymm21
8249 ; AVX512-NEXT: vextracti128 $1, %ymm11, %xmm10
8250 ; AVX512-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm11[3,4],xmm10[5,6,7]
8251 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm11 = [0,1,2,3,8,9,2,3,12,13,6,7,0,1,10,11]
8252 ; AVX512-NEXT: vpshufb %xmm11, %xmm10, %xmm10
8253 ; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
8254 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm10[5,6,7]
8255 ; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm13, %zmm2
8256 ; AVX512-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8257 ; AVX512-NEXT: vmovdqa64 %ymm17, %ymm2
8258 ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10],ymm3[11],ymm2[12,13],ymm3[14],ymm2[15]
8259 ; AVX512-NEXT: vmovdqa64 %ymm3, %ymm22
8260 ; AVX512-NEXT: vmovdqa64 %ymm17, %ymm26
8261 ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm10
8262 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm10[1],ymm2[2,3,4,5,6,7]
8263 ; AVX512-NEXT: vpshufb %ymm14, %ymm2, %ymm2
8264 ; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm20[0,1,1,3]
8265 ; AVX512-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,7,6,7]
8266 ; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm10 = xmm10[2],xmm1[2],xmm10[3],xmm1[3]
8267 ; AVX512-NEXT: vmovdqa64 %xmm1, %xmm17
8268 ; AVX512-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm2[3,4,5,6,7]
8269 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3],ymm2[4,5,6,7]
8270 ; AVX512-NEXT: vmovdqa64 %ymm19, %ymm1
8271 ; AVX512-NEXT: vpblendw {{.*#+}} ymm10 = ymm15[0],ymm1[1],ymm15[2,3],ymm1[4],ymm15[5],ymm1[6],ymm15[7,8],ymm1[9],ymm15[10,11],ymm1[12],ymm15[13],ymm1[14],ymm15[15]
8272 ; AVX512-NEXT: vmovdqa64 %ymm15, %ymm31
8273 ; AVX512-NEXT: vextracti128 $1, %ymm10, %xmm13
8274 ; AVX512-NEXT: vpblendw {{.*#+}} xmm10 = xmm13[0,1,2],xmm10[3,4],xmm13[5,6,7]
8275 ; AVX512-NEXT: vpshufb %xmm11, %xmm10, %xmm10
8276 ; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
8277 ; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm10[5,6,7]
8278 ; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8279 ; AVX512-NEXT: vpblendw $82, (%rsp), %ymm1, %ymm10 # 32-byte Folded Reload
8280 ; AVX512-NEXT: # ymm10 = ymm1[0],mem[1],ymm1[2,3],mem[4],ymm1[5],mem[6],ymm1[7,8],mem[9],ymm1[10,11],mem[12],ymm1[13],mem[14],ymm1[15]
8281 ; AVX512-NEXT: vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
8282 ; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm11[6],ymm10[7]
8283 ; AVX512-NEXT: vpshufb %ymm0, %ymm10, %ymm0
8284 ; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
8285 ; AVX512-NEXT: vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload
8286 ; AVX512-NEXT: # ymm10 = ymm1[0,1],mem[2],ymm1[3],mem[4],ymm1[5,6],mem[7],ymm1[8,9],mem[10],ymm1[11],mem[12],ymm1[13,14],mem[15]
8287 ; AVX512-NEXT: vextracti128 $1, %ymm10, %xmm11
8288 ; AVX512-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm11[3,4],xmm10[5,6,7]
8289 ; AVX512-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[4,5,14,15,8,9,2,3,12,13,6,7,12,13,14,15]
8290 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2],ymm0[3,4,5,6,7]
8291 ; AVX512-NEXT: vmovdqa64 %xmm25, %xmm1
8292 ; AVX512-NEXT: vmovdqa64 %xmm23, %xmm3
8293 ; AVX512-NEXT: vpblendd {{.*#+}} xmm10 = xmm3[0],xmm1[1],xmm3[2,3]
8294 ; AVX512-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
8295 ; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
8296 ; AVX512-NEXT: vpternlogq $184, %zmm0, %zmm27, %zmm10
8297 ; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm10, %zmm23
8298 ; AVX512-NEXT: vmovdqa64 %ymm18, %ymm13
8299 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm12[0],ymm13[1],ymm12[2],ymm13[3],ymm12[4,5],ymm13[6],ymm12[7,8],ymm13[9],ymm12[10],ymm13[11],ymm12[12,13],ymm13[14],ymm12[15]
8300 ; AVX512-NEXT: vpermq {{.*#+}} ymm10 = ymm0[2,3,0,1]
8301 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm10[4],ymm0[5],ymm10[6],ymm0[7]
8302 ; AVX512-NEXT: vmovdqa64 %ymm16, %ymm14
8303 ; AVX512-NEXT: vmovdqa64 %ymm24, %ymm15
8304 ; AVX512-NEXT: vpblendw {{.*#+}} ymm10 = ymm14[0],ymm15[1,2],ymm14[3],ymm15[4],ymm14[5],ymm15[6,7],ymm14[8],ymm15[9,10],ymm14[11],ymm15[12],ymm14[13],ymm15[14,15]
8305 ; AVX512-NEXT: vextracti128 $1, %ymm10, %xmm11
8306 ; AVX512-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2],xmm10[3]
8307 ; AVX512-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[6,7,0,1,10,11,4,5,14,15,8,9,12,13,14,15]
8308 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [6,7,0,1,10,11,4,5,14,15,14,15,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
8309 ; AVX512-NEXT: vpshufb %ymm1, %ymm0, %ymm0
8310 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2],ymm0[3,4,5,6,7]
8311 ; AVX512-NEXT: vmovdqa %ymm4, %ymm3
8312 ; AVX512-NEXT: vmovdqa %ymm8, %ymm4
8313 ; AVX512-NEXT: vpblendw {{.*#+}} ymm10 = ymm8[0,1],ymm3[2],ymm8[3],ymm3[4],ymm8[5,6],ymm3[7],ymm8[8,9],ymm3[10],ymm8[11],ymm3[12],ymm8[13,14],ymm3[15]
8314 ; AVX512-NEXT: vextracti128 $1, %ymm10, %xmm11
8315 ; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4,5,6,7]
8316 ; AVX512-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[6,7,6,7,8,9,4,5,14,15,8,9,2,3,12,13,22,23,22,23,24,25,20,21,30,31,24,25,18,19,28,29]
8317 ; AVX512-NEXT: vpblendd {{.*#+}} xmm11 = xmm5[0],xmm6[1],xmm5[2,3]
8318 ; AVX512-NEXT: vmovdqa64 %xmm30, %xmm7
8319 ; AVX512-NEXT: vmovdqa %xmm5, %xmm6
8320 ; AVX512-NEXT: vpshufb %xmm1, %xmm11, %xmm11
8321 ; AVX512-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0,1,2],xmm10[3,4,5,6,7]
8322 ; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
8323 ; AVX512-NEXT: vmovdqa64 %xmm29, %xmm1
8324 ; AVX512-NEXT: vmovdqa64 %xmm28, %xmm2
8325 ; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm2[0,1],xmm1[2],xmm2[3]
8326 ; AVX512-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
8327 ; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
8328 ; AVX512-NEXT: vinserti64x4 $1, %ymm10, %zmm8, %zmm8
8329 ; AVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm8
8330 ; AVX512-NEXT: vmovdqa %ymm9, %ymm2
8331 ; AVX512-NEXT: vmovdqa64 %ymm21, %ymm10
8332 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm10[0],ymm9[1],ymm10[2,3],ymm9[4],ymm10[5],ymm9[6],ymm10[7,8],ymm9[9],ymm10[10,11],ymm9[12],ymm10[13],ymm9[14],ymm10[15]
8333 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm9
8334 ; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm9[1,2,3],xmm0[4,5],xmm9[6,7]
8335 ; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
8336 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8337 ; AVX512-NEXT: vextracti64x4 $1, %zmm8, %ymm9
8338 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0],ymm0[1,2,3,4,5,6,7],ymm9[8],ymm0[9,10,11,12,13,14,15]
8339 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
8340 ; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm8, %zmm25
8341 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm12[0,1],ymm13[2],ymm12[3],ymm13[4],ymm12[5,6],ymm13[7],ymm12[8,9],ymm13[10],ymm12[11],ymm13[12],ymm12[13,14],ymm13[15]
8342 ; AVX512-NEXT: vpermq {{.*#+}} ymm8 = ymm0[2,3,0,1]
8343 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4],ymm0[5,6],ymm8[7]
8344 ; AVX512-NEXT: vpblendw {{.*#+}} ymm8 = ymm15[0],ymm14[1],ymm15[2,3],ymm14[4],ymm15[5],ymm14[6],ymm15[7,8],ymm14[9],ymm15[10,11],ymm14[12],ymm15[13],ymm14[14],ymm15[15]
8345 ; AVX512-NEXT: vextracti128 $1, %ymm8, %xmm9
8346 ; AVX512-NEXT: vpblendw {{.*#+}} xmm11 = xmm9[0,1,2],xmm8[3,4],xmm9[5,6,7]
8347 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm9 = [8,9,2,3,12,13,6,7,4,5,6,7,4,5,14,15,24,25,18,19,28,29,22,23,20,21,22,23,20,21,30,31]
8348 ; AVX512-NEXT: vpshufb %ymm9, %ymm0, %ymm0
8349 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm8 = [8,9,2,3,12,13,6,7,0,1,10,11,0,1,6,7]
8350 ; AVX512-NEXT: vpshufb %xmm8, %xmm11, %xmm11
8351 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2],ymm0[3,4,5,6,7]
8352 ; AVX512-NEXT: vpblendw {{.*#+}} ymm4 = ymm3[0],ymm4[1,2],ymm3[3],ymm4[4],ymm3[5],ymm4[6,7],ymm3[8],ymm4[9,10],ymm3[11],ymm4[12],ymm3[13],ymm4[14,15]
8353 ; AVX512-NEXT: vextracti128 $1, %ymm4, %xmm5
8354 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3,4,5,6,7]
8355 ; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm6[0,1],xmm7[2],xmm6[3]
8356 ; AVX512-NEXT: vmovdqa {{.*#+}} ymm3 = [8,9,14,15,4,5,6,7,0,1,10,11,4,5,14,15,24,25,30,31,20,21,22,23,16,17,26,27,20,21,30,31]
8357 ; AVX512-NEXT: vpshufb %ymm3, %ymm4, %ymm4
8358 ; AVX512-NEXT: vpshufb %xmm9, %xmm1, %xmm1
8359 ; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3,4,5,6,7]
8360 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm4[4,5,6,7]
8361 ; AVX512-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
8362 ; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm4, %zmm1
8363 ; AVX512-NEXT: movb $7, %al
8364 ; AVX512-NEXT: kmovw %eax, %k1
8365 ; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm1 {%k1}
8366 ; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm4
8367 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm10[1],ymm2[2],ymm10[3],ymm2[4,5],ymm10[6],ymm2[7,8],ymm10[9],ymm2[10],ymm10[11],ymm2[12,13],ymm10[14],ymm2[15]
8368 ; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm5
8369 ; AVX512-NEXT: vpblendw {{.*#+}} xmm5 = xmm0[0,1],xmm5[2,3],xmm0[4,5,6],xmm5[7]
8370 ; AVX512-NEXT: vmovdqa {{.*#+}} xmm0 = [6,7,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
8371 ; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm5
8372 ; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
8373 ; AVX512-NEXT: vpblendw {{.*#+}} ymm5 = ymm4[0],ymm5[1,2,3,4,5,6,7],ymm4[8],ymm5[9,10,11,12,13,14,15]
8374 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
8375 ; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm1
8376 ; AVX512-NEXT: vmovdqa64 %ymm22, %ymm2
8377 ; AVX512-NEXT: vmovdqa64 %ymm26, %ymm4
8378 ; AVX512-NEXT: vpblendw {{.*#+}} ymm4 = ymm2[0],ymm4[1,2],ymm2[3],ymm4[4],ymm2[5],ymm4[6,7],ymm2[8],ymm4[9,10],ymm2[11],ymm4[12],ymm2[13],ymm4[14,15]
8379 ; AVX512-NEXT: vextracti128 $1, %ymm4, %xmm5
8380 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3,4,5,6,7]
8381 ; AVX512-NEXT: vpshufb %ymm3, %ymm4, %ymm3
8382 ; AVX512-NEXT: vmovdqa64 %xmm20, %xmm2
8383 ; AVX512-NEXT: vmovdqa64 %xmm17, %xmm4
8384 ; AVX512-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm2[2],xmm4[3]
8385 ; AVX512-NEXT: vpshufb %xmm9, %xmm4, %xmm4
8386 ; AVX512-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm3[3,4,5,6,7]
8387 ; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
8388 ; AVX512-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
8389 ; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3
8390 ; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
8391 ; AVX512-NEXT: vpblendw $107, (%rsp), %ymm2, %ymm4 # 32-byte Folded Reload
8392 ; AVX512-NEXT: # ymm4 = mem[0,1],ymm2[2],mem[3],ymm2[4],mem[5,6],ymm2[7],mem[8,9],ymm2[10],mem[11],ymm2[12],mem[13,14],ymm2[15]
8393 ; AVX512-NEXT: vpermq {{.*#+}} ymm5 = ymm4[2,3,0,1]
8394 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4],ymm4[5,6],ymm5[7]
8395 ; AVX512-NEXT: vpshufb %ymm9, %ymm4, %ymm4
8396 ; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
8397 ; AVX512-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm5 # 32-byte Folded Reload
8398 ; AVX512-NEXT: # ymm5 = ymm2[0],mem[1],ymm2[2,3],mem[4],ymm2[5],mem[6],ymm2[7,8],mem[9],ymm2[10,11],mem[12],ymm2[13],mem[14],ymm2[15]
8399 ; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm6
8400 ; AVX512-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3,4],xmm6[5,6,7]
8401 ; AVX512-NEXT: vpshufb %xmm8, %xmm5, %xmm5
8402 ; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3,4,5,6,7]
8403 ; AVX512-NEXT: vinserti64x4 $0, %ymm4, %zmm0, %zmm3 {%k1}
8404 ; AVX512-NEXT: vmovdqa64 %ymm19, %ymm2
8405 ; AVX512-NEXT: vmovdqa64 %ymm31, %ymm4
8406 ; AVX512-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm2[1],ymm4[2],ymm2[3],ymm4[4,5],ymm2[6],ymm4[7,8],ymm2[9],ymm4[10],ymm2[11],ymm4[12,13],ymm2[14],ymm4[15]
8407 ; AVX512-NEXT: vextracti128 $1, %ymm4, %xmm5
8408 ; AVX512-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5,6],xmm5[7]
8409 ; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm0
8410 ; AVX512-NEXT: vextracti64x4 $1, %zmm3, %ymm4
8411 ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8412 ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2,3,4,5,6,7],ymm4[8],ymm0[9,10,11,12,13,14,15]
8413 ; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
8414 ; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0
8415 ; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
8416 ; AVX512-NEXT: vmovaps %zmm2, (%rsi)
8417 ; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
8418 ; AVX512-NEXT: vmovaps %zmm2, 64(%rsi)
8419 ; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
8420 ; AVX512-NEXT: vmovaps %zmm2, 64(%rdx)
8421 ; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
8422 ; AVX512-NEXT: vmovaps %zmm2, (%rdx)
8423 ; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
8424 ; AVX512-NEXT: vmovaps %zmm2, 64(%rcx)
8425 ; AVX512-NEXT: vmovdqa64 %zmm23, (%rcx)
8426 ; AVX512-NEXT: vmovdqa64 %zmm25, 64(%r8)
8427 ; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
8428 ; AVX512-NEXT: vmovaps %zmm2, (%r8)
8429 ; AVX512-NEXT: vmovdqa64 %zmm1, 64(%r9)
8430 ; AVX512-NEXT: vmovdqa64 %zmm0, (%r9)
8431 ; AVX512-NEXT: addq $552, %rsp # imm = 0x228
8432 ; AVX512-NEXT: vzeroupper
8433 ; AVX512-NEXT: retq
8434 ;
8435 ; AVX512-FCP-LABEL: load_i16_stride5_vf64:
8436 ; AVX512-FCP: # %bb.0:
8437 ; AVX512-FCP-NEXT: subq $552, %rsp # imm = 0x228
8438 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,14,15,4,5,6,7,8,9,10,11,12,13,14,15]
8439 ; AVX512-FCP-NEXT: vmovdqa 496(%rdi), %xmm1
8440 ; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8441 ; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm1
8442 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
8443 ; AVX512-FCP-NEXT: vmovdqa 480(%rdi), %xmm2
8444 ; AVX512-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
8445 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
8446 ; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
8447 ; AVX512-FCP-NEXT: vmovdqa 512(%rdi), %ymm10
8448 ; AVX512-FCP-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8449 ; AVX512-FCP-NEXT: vmovdqa 544(%rdi), %ymm11
8450 ; AVX512-FCP-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8451 ; AVX512-FCP-NEXT: vmovdqa 576(%rdi), %ymm7
8452 ; AVX512-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8453 ; AVX512-FCP-NEXT: vmovdqa 608(%rdi), %ymm8
8454 ; AVX512-FCP-NEXT: vmovdqu %ymm8, (%rsp) # 32-byte Spill
8455 ; AVX512-FCP-NEXT: vmovdqa 352(%rdi), %ymm4
8456 ; AVX512-FCP-NEXT: vmovdqa 320(%rdi), %ymm5
8457 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
8458 ; AVX512-FCP-NEXT: vmovdqa64 %ymm5, %ymm28
8459 ; AVX512-FCP-NEXT: vmovdqa64 %ymm4, %ymm30
8460 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm1, %xmm4
8461 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm1[0],xmm4[1,2,3],xmm1[4,5],xmm4[6,7]
8462 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
8463 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm4
8464 ; AVX512-FCP-NEXT: vmovdqa 384(%rdi), %ymm6
8465 ; AVX512-FCP-NEXT: vmovdqa 416(%rdi), %ymm9
8466 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm6[0],ymm9[1,2],ymm6[3],ymm9[4],ymm6[5],ymm9[6,7],ymm6[8],ymm9[9,10],ymm6[11],ymm9[12],ymm6[13],ymm9[14,15]
8467 ; AVX512-FCP-NEXT: vmovdqa64 %ymm9, %ymm26
8468 ; AVX512-FCP-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8469 ; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm27
8470 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [1,3,0,2,4,6,1,3]
8471 ; AVX512-FCP-NEXT: vpermd %ymm5, %ymm6, %ymm5
8472 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,2,3,16,17,22,23,24,25,30,31,20,21,128,128,128,128,128,128]
8473 ; AVX512-FCP-NEXT: vpshufb %ymm13, %ymm5, %ymm5
8474 ; AVX512-FCP-NEXT: vpor %ymm5, %ymm4, %ymm4
8475 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm8[0,1],ymm7[2],ymm8[3],ymm7[4],ymm8[5,6],ymm7[7],ymm8[8,9],ymm7[10],ymm8[11],ymm7[12],ymm8[13,14],ymm7[15]
8476 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm5, %xmm7
8477 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm7[3,4],xmm5[5,6,7]
8478 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm11[0],ymm10[1],ymm11[2,3],ymm10[4],ymm11[5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10,11],ymm10[12],ymm11[13],ymm10[14],ymm11[15]
8479 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm10 = [2,4,7,1,4,6,0,0]
8480 ; AVX512-FCP-NEXT: vpermd %ymm7, %ymm10, %ymm7
8481 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [u,u,u,u,u,u,u,u,8,9,14,15,0,1,6,7,16,17,22,23,20,21,22,23,24,25,30,31,16,17,22,23]
8482 ; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm7, %ymm7
8483 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm17 = [8,9,3,2,4,5,7,6]
8484 ; AVX512-FCP-NEXT: vpermt2d %ymm2, %ymm17, %ymm7
8485 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [12,13,14,15,4,5,14,15,8,9,2,3,12,13,6,7]
8486 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
8487 ; AVX512-FCP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
8488 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3,4],ymm5[5,6,7]
8489 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm18 = [0,3,1,3,0,3,5,7]
8490 ; AVX512-FCP-NEXT: vmovdqa 448(%rdi), %ymm7
8491 ; AVX512-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8492 ; AVX512-FCP-NEXT: vpermd %ymm7, %ymm18, %ymm7
8493 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm11 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
8494 ; AVX512-FCP-NEXT: vpshufb %ymm11, %ymm7, %ymm7
8495 ; AVX512-FCP-NEXT: vmovdqa %ymm11, %ymm14
8496 ; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
8497 ; AVX512-FCP-NEXT: vpternlogq $184, %zmm4, %zmm8, %zmm7
8498 ; AVX512-FCP-NEXT: vmovdqa64 %zmm8, %zmm16
8499 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm5, %zmm7, %zmm4
8500 ; AVX512-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8501 ; AVX512-FCP-NEXT: vmovdqa 176(%rdi), %xmm4
8502 ; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm0
8503 ; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm31
8504 ; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %xmm15
8505 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm15, %xmm3
8506 ; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
8507 ; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm11
8508 ; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm5
8509 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm11[0],ymm5[1,2],ymm11[3],ymm5[4],ymm11[5],ymm5[6,7],ymm11[8],ymm5[9,10],ymm11[11],ymm5[12],ymm11[13],ymm5[14,15]
8510 ; AVX512-FCP-NEXT: vmovdqa %ymm5, %ymm7
8511 ; AVX512-FCP-NEXT: vpermd %ymm3, %ymm6, %ymm3
8512 ; AVX512-FCP-NEXT: vpshufb %ymm13, %ymm3, %ymm3
8513 ; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm5
8514 ; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm6
8515 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm5[0],ymm6[1],ymm5[2,3],ymm6[4],ymm5[5],ymm6[6],ymm5[7,8],ymm6[9],ymm5[10,11],ymm6[12],ymm5[13],ymm6[14],ymm5[15]
8516 ; AVX512-FCP-NEXT: vmovdqa %ymm5, %ymm12
8517 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm4, %xmm5
8518 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1,2,3],xmm4[4,5],xmm5[6,7]
8519 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm1
8520 ; AVX512-FCP-NEXT: vpor %ymm3, %ymm1, %ymm1
8521 ; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm8
8522 ; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm13
8523 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm13[0],ymm8[1],ymm13[2,3],ymm8[4],ymm13[5],ymm8[6],ymm13[7,8],ymm8[9],ymm13[10,11],ymm8[12],ymm13[13],ymm8[14],ymm13[15]
8524 ; AVX512-FCP-NEXT: vpermd %ymm3, %ymm10, %ymm3
8525 ; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm3
8526 ; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %ymm5
8527 ; AVX512-FCP-NEXT: vmovdqa 288(%rdi), %ymm9
8528 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm9[0,1],ymm5[2],ymm9[3],ymm5[4],ymm9[5,6],ymm5[7],ymm9[8,9],ymm5[10],ymm9[11],ymm5[12],ymm9[13,14],ymm5[15]
8529 ; AVX512-FCP-NEXT: vmovdqa64 %ymm5, %ymm25
8530 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm4, %xmm10
8531 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm10[3,4],xmm4[5,6,7]
8532 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm2
8533 ; AVX512-FCP-NEXT: vpermt2d %ymm0, %ymm17, %ymm3
8534 ; AVX512-FCP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
8535 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5,6,7]
8536 ; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm23
8537 ; AVX512-FCP-NEXT: vpermd %ymm23, %ymm18, %ymm2
8538 ; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm2, %ymm2
8539 ; AVX512-FCP-NEXT: vpternlogq $184, %zmm1, %zmm16, %zmm2
8540 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
8541 ; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8542 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm6[0],ymm12[1],ymm6[2],ymm12[3],ymm6[4,5],ymm12[6],ymm6[7,8],ymm12[9],ymm6[10],ymm12[11],ymm6[12,13],ymm12[14],ymm6[15]
8543 ; AVX512-FCP-NEXT: vmovdqa64 %ymm12, %ymm21
8544 ; AVX512-FCP-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8545 ; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm29
8546 ; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8547 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1
8548 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6],xmm1[7]
8549 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm17 = [2,0,0,0,4,7,1,6]
8550 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm7[0],ymm11[1],ymm7[2,3],ymm11[4],ymm7[5],ymm11[6],ymm7[7,8],ymm11[9],ymm7[10,11],ymm11[12],ymm7[13],ymm11[14],ymm7[15]
8551 ; AVX512-FCP-NEXT: vmovdqa64 %ymm11, %ymm19
8552 ; AVX512-FCP-NEXT: vmovdqa64 %ymm7, %ymm22
8553 ; AVX512-FCP-NEXT: vpermd %ymm1, %ymm17, %ymm1
8554 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,0,1,18,19,20,21,26,27,16,17,30,31,128,128,128,128,128,128]
8555 ; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm1
8556 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [2,3,12,13,6,7,0,1,10,11,4,5,14,15,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
8557 ; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm0
8558 ; AVX512-FCP-NEXT: vpor %ymm1, %ymm0, %ymm10
8559 ; AVX512-FCP-NEXT: vpsrlq $48, %xmm31, %xmm0
8560 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm15, %xmm1
8561 ; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
8562 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm20 = [0,2,5,7,4,7,0,0]
8563 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm8[0],ymm13[1],ymm8[2,3],ymm13[4],ymm8[5],ymm13[6],ymm8[7,8],ymm13[9],ymm8[10,11],ymm13[12],ymm8[13],ymm13[14],ymm8[15]
8564 ; AVX512-FCP-NEXT: vmovdqa %ymm8, %ymm5
8565 ; AVX512-FCP-NEXT: vpermd %ymm0, %ymm20, %ymm4
8566 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [2,3,4,5,4,5,0,1,6,7,8,9,14,15,4,5,18,19,20,21,20,21,16,17,22,23,24,25,30,31,20,21]
8567 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm4
8568 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3,4,5,6,7]
8569 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0,1,2,3],ymm4[4,5,6,7]
8570 ; AVX512-FCP-NEXT: vbroadcasti32x4 {{.*#+}} ymm24 = [1,4,6,3,1,4,6,3]
8571 ; AVX512-FCP-NEXT: # ymm24 = mem[0,1,2,3,0,1,2,3]
8572 ; AVX512-FCP-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8573 ; AVX512-FCP-NEXT: vmovdqa64 %ymm25, %ymm12
8574 ; AVX512-FCP-NEXT: vmovdqu64 %ymm25, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8575 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm12[0],ymm9[1,2],ymm12[3],ymm9[4],ymm12[5],ymm9[6,7],ymm12[8],ymm9[9,10],ymm12[11],ymm9[12],ymm12[13],ymm9[14,15]
8576 ; AVX512-FCP-NEXT: vpermd %ymm1, %ymm24, %ymm0
8577 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,2,3,2,3,4,5,10,11,0,1,14,15,8,9,16,17,18,19,18,19,20,21,26,27,16,17,30,31,24,25]
8578 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm0
8579 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4],ymm0[5,6,7]
8580 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm25 = [1,3,2,3,1,3,6,7]
8581 ; AVX512-FCP-NEXT: vpermd %ymm23, %ymm25, %ymm4
8582 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm14 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
8583 ; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm4, %ymm4
8584 ; AVX512-FCP-NEXT: vpternlogq $184, %zmm10, %zmm16, %zmm4
8585 ; AVX512-FCP-NEXT: vmovdqa64 %zmm16, %zmm10
8586 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm4, %zmm0
8587 ; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8588 ; AVX512-FCP-NEXT: vmovdqa64 %ymm28, %ymm18
8589 ; AVX512-FCP-NEXT: vmovdqa64 %ymm30, %ymm28
8590 ; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm0
8591 ; AVX512-FCP-NEXT: vmovdqa64 %ymm30, %ymm4
8592 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1],ymm4[2],ymm0[3],ymm4[4,5],ymm0[6],ymm4[7,8],ymm0[9],ymm4[10],ymm0[11],ymm4[12,13],ymm0[14],ymm4[15]
8593 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm4
8594 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3],xmm0[4,5,6],xmm4[7]
8595 ; AVX512-FCP-NEXT: vmovdqa64 %ymm27, %ymm7
8596 ; AVX512-FCP-NEXT: vmovdqa64 %ymm26, %ymm4
8597 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm7[1],ymm4[2,3],ymm7[4],ymm4[5],ymm7[6],ymm4[7,8],ymm7[9],ymm4[10,11],ymm7[12],ymm4[13],ymm7[14],ymm4[15]
8598 ; AVX512-FCP-NEXT: vpermd %ymm4, %ymm17, %ymm4
8599 ; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm4, %ymm4
8600 ; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm0
8601 ; AVX512-FCP-NEXT: vpor %ymm4, %ymm0, %ymm0
8602 ; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
8603 ; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Reload
8604 ; AVX512-FCP-NEXT: vmovdqa64 %ymm16, %ymm4
8605 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm11[0],ymm4[1],ymm11[2,3],ymm4[4],ymm11[5],ymm4[6],ymm11[7,8],ymm4[9],ymm11[10,11],ymm4[12],ymm11[13],ymm4[14],ymm11[15]
8606 ; AVX512-FCP-NEXT: vpermd %ymm4, %ymm20, %ymm4
8607 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm3
8608 ; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
8609 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm2
8610 ; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm27 # 16-byte Reload
8611 ; AVX512-FCP-NEXT: vpsrlq $48, %xmm27, %xmm4
8612 ; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
8613 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[3,4,5,6,7]
8614 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
8615 ; AVX512-FCP-NEXT: vmovdqu (%rsp), %ymm8 # 32-byte Reload
8616 ; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
8617 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm8[1,2],ymm4[3],ymm8[4],ymm4[5],ymm8[6,7],ymm4[8],ymm8[9,10],ymm4[11],ymm8[12],ymm4[13],ymm8[14,15]
8618 ; AVX512-FCP-NEXT: vpermd %ymm3, %ymm24, %ymm3
8619 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm3, %ymm1
8620 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5,6,7]
8621 ; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Reload
8622 ; AVX512-FCP-NEXT: vpermd %ymm17, %ymm25, %ymm2
8623 ; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm2, %ymm2
8624 ; AVX512-FCP-NEXT: vpternlogq $184, %zmm0, %zmm10, %zmm2
8625 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm0
8626 ; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8627 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0],ymm12[1],ymm9[2,3],ymm12[4],ymm9[5],ymm12[6],ymm9[7,8],ymm12[9],ymm9[10,11],ymm12[12],ymm9[13],ymm12[14],ymm9[15]
8628 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1
8629 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[3,4],xmm1[5,6,7]
8630 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm15[u,u,u,u,u,u,u,u,4,5,14,15,u,u,u,u]
8631 ; AVX512-FCP-NEXT: vmovdqa64 %xmm15, %xmm20
8632 ; AVX512-FCP-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm31[2],xmm0[3],xmm31[3]
8633 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm24 = [0,3,5,2,5,7,0,0]
8634 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm13[0],ymm5[1],ymm13[2],ymm5[3],ymm13[4,5],ymm5[6],ymm13[7,8],ymm5[9],ymm13[10],ymm5[11],ymm13[12,13],ymm5[14],ymm13[15]
8635 ; AVX512-FCP-NEXT: vmovdqa64 %ymm5, %ymm30
8636 ; AVX512-FCP-NEXT: vmovdqa64 %ymm13, %ymm26
8637 ; AVX512-FCP-NEXT: vpermd %ymm2, %ymm24, %ymm3
8638 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm12 = [0,1,6,7,2,3,2,3,4,5,10,11,0,1,14,15,16,17,22,23,18,19,18,19,20,21,26,27,16,17,30,31]
8639 ; AVX512-FCP-NEXT: vpshufb %ymm12, %ymm3, %ymm3
8640 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3,4,5,6,7]
8641 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3],ymm3[4,5,6,7]
8642 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,8,9,2,3,12,13,6,7,0,1,10,11]
8643 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm1, %xmm1
8644 ; AVX512-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
8645 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm3[0,1,2,3,4],ymm1[5,6,7]
8646 ; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm0
8647 ; AVX512-FCP-NEXT: vmovdqa64 %ymm29, %ymm1
8648 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
8649 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm1, %xmm3
8650 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm1[0,1,2],xmm3[3,4],xmm1[5,6,7]
8651 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm25 = [0,2,0,0,5,7,2,4]
8652 ; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm0
8653 ; AVX512-FCP-NEXT: vmovdqa64 %ymm22, %ymm1
8654 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
8655 ; AVX512-FCP-NEXT: vmovdqa64 %ymm22, %ymm29
8656 ; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm21
8657 ; AVX512-FCP-NEXT: vpermd %ymm1, %ymm25, %ymm3
8658 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,6,7,8,9,14,15,4,5,6,7,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
8659 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm3, %ymm0
8660 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [4,5,14,15,8,9,2,3,12,13,6,7,12,13,14,15]
8661 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm5, %xmm5
8662 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2],ymm0[3,4,5,6,7]
8663 ; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [1,4,6,0,1,4,6,0]
8664 ; AVX512-FCP-NEXT: # ymm5 = mem[0,1,0,1]
8665 ; AVX512-FCP-NEXT: vpermd %ymm23, %ymm5, %ymm14
8666 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
8667 ; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm14, %ymm14
8668 ; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
8669 ; AVX512-FCP-NEXT: vpternlogq $184, %zmm0, %zmm15, %zmm14
8670 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm10, %zmm14, %zmm0
8671 ; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8672 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm8[0],ymm4[1],ymm8[2,3],ymm4[4],ymm8[5],ymm4[6],ymm8[7,8],ymm4[9],ymm8[10,11],ymm4[12],ymm8[13],ymm4[14],ymm8[15]
8673 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm10
8674 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3,4],xmm10[5,6,7]
8675 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm0
8676 ; AVX512-FCP-NEXT: vmovdqa64 %ymm16, %ymm8
8677 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm8[0],ymm11[1],ymm8[2],ymm11[3],ymm8[4,5],ymm11[6],ymm8[7,8],ymm11[9],ymm8[10],ymm11[11],ymm8[12,13],ymm11[14],ymm8[15]
8678 ; AVX512-FCP-NEXT: vmovdqa %ymm11, %ymm13
8679 ; AVX512-FCP-NEXT: vpermd %ymm2, %ymm24, %ymm2
8680 ; AVX512-FCP-NEXT: vpshufb %ymm12, %ymm2, %ymm2
8681 ; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm6[u,u,u,u,u,u,u,u,4,5,14,15,u,u,u,u]
8682 ; AVX512-FCP-NEXT: vmovdqa %xmm6, %xmm14
8683 ; AVX512-FCP-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm4[2],xmm27[2],xmm4[3],xmm27[3]
8684 ; AVX512-FCP-NEXT: vmovdqa64 %xmm27, %xmm6
8685 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm2[3,4,5,6,7]
8686 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
8687 ; AVX512-FCP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8688 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
8689 ; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm11
8690 ; AVX512-FCP-NEXT: vmovdqa64 %ymm28, %ymm12
8691 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm12[0,1],ymm11[2],ymm12[3],ymm11[4],ymm12[5,6],ymm11[7],ymm12[8,9],ymm11[10],ymm12[11],ymm11[12],ymm12[13,14],ymm11[15]
8692 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm2, %xmm4
8693 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[3,4],xmm2[5,6,7]
8694 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
8695 ; AVX512-FCP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm3 # 32-byte Folded Reload
8696 ; AVX512-FCP-NEXT: # ymm3 = ymm7[0],mem[1],ymm7[2,3],mem[4],ymm7[5],mem[6],ymm7[7,8],mem[9],ymm7[10,11],mem[12],ymm7[13],mem[14],ymm7[15]
8697 ; AVX512-FCP-NEXT: vmovdqa64 %ymm7, %ymm16
8698 ; AVX512-FCP-NEXT: vpermd %ymm3, %ymm25, %ymm3
8699 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm3, %ymm1
8700 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
8701 ; AVX512-FCP-NEXT: vpermd %ymm17, %ymm5, %ymm2
8702 ; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm2
8703 ; AVX512-FCP-NEXT: vpternlogq $184, %zmm1, %zmm15, %zmm2
8704 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm28
8705 ; AVX512-FCP-NEXT: vmovdqa64 %xmm31, %xmm15
8706 ; AVX512-FCP-NEXT: vmovdqa64 %xmm20, %xmm7
8707 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm15[0],xmm7[1],xmm15[2,3]
8708 ; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm0 = [6,7,0,1,10,11,0,0,0,0,0,0,0,0,0,0]
8709 ; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm2
8710 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm18 = [1,3,6,0,5,0,0,0]
8711 ; AVX512-FCP-NEXT: vmovdqa64 %ymm30, %ymm9
8712 ; AVX512-FCP-NEXT: vmovdqa64 %ymm26, %ymm10
8713 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm10[0,1],ymm9[2],ymm10[3],ymm9[4],ymm10[5,6],ymm9[7],ymm10[8,9],ymm9[10],ymm10[11],ymm9[12],ymm10[13,14],ymm9[15]
8714 ; AVX512-FCP-NEXT: vpermd %ymm1, %ymm18, %ymm4
8715 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [2,3,2,3,4,5,0,1,6,7,8,9,14,15,4,5,18,19,18,19,20,21,16,17,22,23,24,25,30,31,20,21]
8716 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm4
8717 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[3,4,5,6,7]
8718 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
8719 ; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [2,4,7,0,2,4,7,0]
8720 ; AVX512-FCP-NEXT: # ymm4 = mem[0,1,0,1]
8721 ; AVX512-FCP-NEXT: vpermd %ymm23, %ymm4, %ymm5
8722 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
8723 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm5, %ymm5
8724 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm5, %zmm25
8725 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm6[0],xmm14[1],xmm6[2,3]
8726 ; AVX512-FCP-NEXT: vmovdqa64 %xmm27, %xmm22
8727 ; AVX512-FCP-NEXT: vmovdqa64 %xmm14, %xmm31
8728 ; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm2, %xmm0
8729 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm8[0,1],ymm13[2],ymm8[3],ymm13[4],ymm8[5,6],ymm13[7],ymm8[8,9],ymm13[10],ymm8[11],ymm13[12],ymm8[13,14],ymm13[15]
8730 ; AVX512-FCP-NEXT: vmovdqa64 %ymm13, %ymm20
8731 ; AVX512-FCP-NEXT: vmovdqa64 %ymm8, %ymm30
8732 ; AVX512-FCP-NEXT: vpermd %ymm2, %ymm18, %ymm2
8733 ; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm1
8734 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3,4,5,6,7]
8735 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
8736 ; AVX512-FCP-NEXT: vpermd %ymm17, %ymm4, %ymm1
8737 ; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm1
8738 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm24
8739 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm15[0,1],xmm7[2],xmm15[3]
8740 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm9[0],ymm10[1,2],ymm9[3],ymm10[4],ymm9[5],ymm10[6,7],ymm9[8],ymm10[9,10],ymm9[11],ymm10[12],ymm9[13],ymm10[14,15]
8741 ; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm2 = [8,9,2,3,12,13,0,0,0,0,0,0,0,0,0,0]
8742 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm3
8743 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm27 = [1,4,6,3,6,0,0,0]
8744 ; AVX512-FCP-NEXT: vpermd %ymm1, %ymm27, %ymm1
8745 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [0,1,2,3,4,5,2,3,4,5,10,11,0,1,14,15,16,17,18,19,20,21,18,19,20,21,26,27,16,17,30,31]
8746 ; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm1
8747 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm1[3,4,5,6,7]
8748 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
8749 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm26 = [0,2,1,3,0,2,5,7]
8750 ; AVX512-FCP-NEXT: vpermd %ymm23, %ymm26, %ymm3
8751 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,6,7,4,5,6,7,0,1,6,7,8,9,14,15,16,17,22,23,20,21,22,23,16,17,22,23,24,25,30,31]
8752 ; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm3
8753 ; AVX512-FCP-NEXT: vmovdqa64 %ymm4, %ymm19
8754 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1
8755 ; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
8756 ; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
8757 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm14[1,2],ymm4[3],ymm14[4],ymm4[5],ymm14[6,7],ymm4[8],ymm14[9,10],ymm4[11],ymm14[12],ymm4[13],ymm14[14,15]
8758 ; AVX512-FCP-NEXT: vmovdqa64 %ymm4, %ymm18
8759 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm3, %xmm4
8760 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2],xmm3[3]
8761 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,3,0,0,5,0,2,7]
8762 ; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm15
8763 ; AVX512-FCP-NEXT: vmovdqa64 %ymm29, %ymm10
8764 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm10[0],ymm15[1],ymm10[2],ymm15[3],ymm10[4,5],ymm15[6],ymm10[7,8],ymm15[9],ymm10[10],ymm15[11],ymm10[12,13],ymm15[14],ymm10[15]
8765 ; AVX512-FCP-NEXT: vpermd %ymm5, %ymm4, %ymm5
8766 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [2,3,4,5,10,11,0,1,14,15,14,15,2,3,4,5,18,19,20,21,26,27,16,17,30,31,30,31,18,19,20,21]
8767 ; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm5
8768 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm7 = [6,7,0,1,10,11,4,5,14,15,8,9,12,13,14,15]
8769 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm3, %xmm3
8770 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3,4,5,6,7]
8771 ; AVX512-FCP-NEXT: vpmovsxdq {{.*#+}} zmm5 = [18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0,18446744073709486080,18446744073709551615]
8772 ; AVX512-FCP-NEXT: vpternlogq $184, %zmm3, %zmm5, %zmm25
8773 ; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
8774 ; AVX512-FCP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
8775 ; AVX512-FCP-NEXT: # ymm3 = ymm3[0],mem[1],ymm3[2,3],mem[4],ymm3[5],mem[6],ymm3[7,8],mem[9],ymm3[10,11],mem[12],ymm3[13],mem[14],ymm3[15]
8776 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm3, %xmm8
8777 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm8 = xmm3[0],xmm8[1,2,3],xmm3[4,5],xmm8[6,7]
8778 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
8779 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm8, %xmm8
8780 ; AVX512-FCP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
8781 ; AVX512-FCP-NEXT: vextracti64x4 $1, %zmm25, %ymm9
8782 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm9[0],ymm8[1,2,3,4,5,6,7],ymm9[8],ymm8[9,10,11,12,13,14,15]
8783 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
8784 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm8, %zmm25, %zmm21
8785 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm11[0],ymm12[1,2],ymm11[3],ymm12[4],ymm11[5],ymm12[6,7],ymm11[8],ymm12[9,10],ymm11[11],ymm12[12],ymm11[13],ymm12[14,15]
8786 ; AVX512-FCP-NEXT: vmovdqa64 %ymm12, %ymm23
8787 ; AVX512-FCP-NEXT: vmovdqa64 %ymm11, %ymm25
8788 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm8, %xmm9
8789 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0],xmm8[1],xmm9[2],xmm8[3]
8790 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm8, %xmm7
8791 ; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
8792 ; AVX512-FCP-NEXT: vmovdqa64 %ymm16, %ymm12
8793 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm13[0],ymm12[1],ymm13[2],ymm12[3],ymm13[4,5],ymm12[6],ymm13[7,8],ymm12[9],ymm13[10],ymm12[11],ymm13[12,13],ymm12[14],ymm13[15]
8794 ; AVX512-FCP-NEXT: vpermd %ymm8, %ymm4, %ymm4
8795 ; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm4, %ymm4
8796 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm7[0,1,2],ymm4[3,4,5,6,7]
8797 ; AVX512-FCP-NEXT: vpternlogq $184, %zmm4, %zmm5, %zmm24
8798 ; AVX512-FCP-NEXT: vmovdqa64 %xmm22, %xmm4
8799 ; AVX512-FCP-NEXT: vmovdqa64 %xmm31, %xmm5
8800 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3]
8801 ; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm2
8802 ; AVX512-FCP-NEXT: vmovdqa64 %ymm20, %ymm4
8803 ; AVX512-FCP-NEXT: vmovdqa64 %ymm30, %ymm5
8804 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm5[1,2],ymm4[3],ymm5[4],ymm4[5],ymm5[6,7],ymm4[8],ymm5[9,10],ymm4[11],ymm5[12],ymm4[13],ymm5[14,15]
8805 ; AVX512-FCP-NEXT: vpermd %ymm4, %ymm27, %ymm4
8806 ; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm0
8807 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm0[3,4,5,6,7]
8808 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
8809 ; AVX512-FCP-NEXT: vpermd %ymm17, %ymm26, %ymm2
8810 ; AVX512-FCP-NEXT: vmovdqa64 %ymm19, %ymm4
8811 ; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm2
8812 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
8813 ; AVX512-FCP-NEXT: vmovdqu (%rsp), %ymm9 # 32-byte Reload
8814 ; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
8815 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm11[0],ymm9[1],ymm11[2,3],ymm9[4],ymm11[5],ymm9[6],ymm11[7,8],ymm9[9],ymm11[10,11],ymm9[12],ymm11[13],ymm9[14],ymm11[15]
8816 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm2, %xmm4
8817 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1,2,3],xmm2[4,5],xmm4[6,7]
8818 ; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
8819 ; AVX512-FCP-NEXT: vextracti64x4 $1, %zmm24, %ymm3
8820 ; AVX512-FCP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
8821 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1,2,3,4,5,6,7],ymm3[8],ymm2[9,10,11,12,13,14,15]
8822 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
8823 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm24, %zmm2
8824 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm10[0,1],ymm15[2],ymm10[3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8,9],ymm15[10],ymm10[11],ymm15[12],ymm10[13,14],ymm15[15]
8825 ; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [1,3,0,0,6,0,3,5]
8826 ; AVX512-FCP-NEXT: vpermd %ymm3, %ymm4, %ymm3
8827 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,6,7,8,9,14,15,0,1,6,7,0,1,6,7,16,17,22,23,24,25,30,31,16,17,22,23,16,17,22,23]
8828 ; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm3, %ymm3
8829 ; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm6
8830 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm14[0],ymm6[1],ymm14[2,3],ymm6[4],ymm14[5],ymm6[6],ymm14[7,8],ymm6[9],ymm14[10,11],ymm6[12],ymm14[13],ymm6[14],ymm14[15]
8831 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm6, %xmm7
8832 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3,4],xmm7[5,6,7]
8833 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm7 = [8,9,2,3,12,13,6,7,0,1,10,11,0,1,6,7]
8834 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm6, %xmm6
8835 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2],ymm3[3,4,5,6,7]
8836 ; AVX512-FCP-NEXT: movb $7, %al
8837 ; AVX512-FCP-NEXT: kmovw %eax, %k1
8838 ; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm0, %zmm1 {%k1}
8839 ; AVX512-FCP-NEXT: vextracti64x4 $1, %zmm1, %ymm3
8840 ; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
8841 ; AVX512-FCP-NEXT: vpblendw $181, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
8842 ; AVX512-FCP-NEXT: # ymm6 = mem[0],ymm6[1],mem[2],ymm6[3],mem[4,5],ymm6[6],mem[7,8],ymm6[9],mem[10],ymm6[11],mem[12,13],ymm6[14],mem[15]
8843 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm6, %xmm8
8844 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2,3],xmm6[4,5,6],xmm8[7]
8845 ; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm8 = [6,7,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
8846 ; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm6, %xmm6
8847 ; AVX512-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
8848 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm3[0],ymm6[1,2,3,4,5,6,7],ymm3[8],ymm6[9,10,11,12,13,14,15]
8849 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5,6,7]
8850 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
8851 ; AVX512-FCP-NEXT: vmovdqa64 %ymm23, %ymm3
8852 ; AVX512-FCP-NEXT: vmovdqa64 %ymm25, %ymm6
8853 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm6[1],ymm3[2,3],ymm6[4],ymm3[5],ymm6[6],ymm3[7,8],ymm6[9],ymm3[10,11],ymm6[12],ymm3[13],ymm6[14],ymm3[15]
8854 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm3, %xmm6
8855 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm6[0,1,2],xmm3[3,4],xmm6[5,6,7]
8856 ; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm3, %xmm3
8857 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm13[0,1],ymm12[2],ymm13[3],ymm12[4],ymm13[5,6],ymm12[7],ymm13[8,9],ymm12[10],ymm13[11],ymm12[12],ymm13[13,14],ymm12[15]
8858 ; AVX512-FCP-NEXT: vpermd %ymm6, %ymm4, %ymm4
8859 ; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm4
8860 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3,4,5,6,7]
8861 ; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm0, %zmm0 {%k1}
8862 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm9[0],ymm11[1],ymm9[2],ymm11[3],ymm9[4,5],ymm11[6],ymm9[7,8],ymm11[9],ymm9[10],ymm11[11],ymm9[12,13],ymm11[14],ymm9[15]
8863 ; AVX512-FCP-NEXT: vextracti128 $1, %ymm3, %xmm4
8864 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5,6],xmm4[7]
8865 ; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm3, %xmm3
8866 ; AVX512-FCP-NEXT: vextracti64x4 $1, %zmm0, %ymm4
8867 ; AVX512-FCP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
8868 ; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1,2,3,4,5,6,7],ymm4[8],ymm3[9,10,11,12,13,14,15]
8869 ; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
8870 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
8871 ; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
8872 ; AVX512-FCP-NEXT: vmovaps %zmm3, (%rsi)
8873 ; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
8874 ; AVX512-FCP-NEXT: vmovaps %zmm3, 64(%rsi)
8875 ; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
8876 ; AVX512-FCP-NEXT: vmovaps %zmm3, 64(%rdx)
8877 ; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
8878 ; AVX512-FCP-NEXT: vmovaps %zmm3, (%rdx)
8879 ; AVX512-FCP-NEXT: vmovdqa64 %zmm28, 64(%rcx)
8880 ; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
8881 ; AVX512-FCP-NEXT: vmovaps %zmm3, (%rcx)
8882 ; AVX512-FCP-NEXT: vmovdqa64 %zmm2, 64(%r8)
8883 ; AVX512-FCP-NEXT: vmovdqa64 %zmm21, (%r8)
8884 ; AVX512-FCP-NEXT: vmovdqa64 %zmm0, 64(%r9)
8885 ; AVX512-FCP-NEXT: vmovdqa64 %zmm1, (%r9)
8886 ; AVX512-FCP-NEXT: addq $552, %rsp # imm = 0x228
8887 ; AVX512-FCP-NEXT: vzeroupper
8888 ; AVX512-FCP-NEXT: retq
8889 ;
8890 ; AVX512DQ-LABEL: load_i16_stride5_vf64:
8891 ; AVX512DQ: # %bb.0:
8892 ; AVX512DQ-NEXT: subq $552, %rsp # imm = 0x228
8893 ; AVX512DQ-NEXT: vmovdqa 384(%rdi), %ymm1
8894 ; AVX512DQ-NEXT: vmovdqa 416(%rdi), %ymm2
8895 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm2[1,2],ymm1[3],ymm2[4],ymm1[5],ymm2[6,7],ymm1[8],ymm2[9,10],ymm1[11],ymm2[12],ymm1[13],ymm2[14,15]
8896 ; AVX512DQ-NEXT: vmovdqa %ymm2, %ymm11
8897 ; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8898 ; AVX512DQ-NEXT: vmovdqa %ymm1, %ymm6
8899 ; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8900 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
8901 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6],ymm1[7]
8902 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,6,7,16,17,26,27,20,21,30,31,24,25,128,128,128,128,128,128]
8903 ; AVX512DQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2
8904 ; AVX512DQ-NEXT: vmovdqa 352(%rdi), %ymm3
8905 ; AVX512DQ-NEXT: vmovdqa 320(%rdi), %ymm4
8906 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5],ymm3[6],ymm4[7,8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13],ymm3[14],ymm4[15]
8907 ; AVX512DQ-NEXT: vmovdqa %ymm4, %ymm7
8908 ; AVX512DQ-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8909 ; AVX512DQ-NEXT: vmovdqa %ymm3, %ymm8
8910 ; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8911 ; AVX512DQ-NEXT: vextracti128 $1, %ymm1, %xmm3
8912 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm3[1,2,3],xmm1[4,5],xmm3[6,7]
8913 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
8914 ; AVX512DQ-NEXT: vpshufb %ymm1, %ymm3, %ymm3
8915 ; AVX512DQ-NEXT: vporq %ymm2, %ymm3, %ymm19
8916 ; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm15
8917 ; AVX512DQ-NEXT: vmovdqa 224(%rdi), %ymm13
8918 ; AVX512DQ-NEXT: vmovdqa 176(%rdi), %xmm12
8919 ; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm14
8920 ; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm4
8921 ; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm5
8922 ; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm3
8923 ; AVX512DQ-NEXT: vmovdqa 96(%rdi), %ymm9
8924 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4],ymm3[5],ymm9[6,7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12],ymm3[13],ymm9[14,15]
8925 ; AVX512DQ-NEXT: vmovdqa %ymm3, %ymm10
8926 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
8927 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6],ymm3[7]
8928 ; AVX512DQ-NEXT: vpshufb %ymm0, %ymm2, %ymm0
8929 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10,11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]
8930 ; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
8931 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
8932 ; AVX512DQ-NEXT: vpshufb %ymm1, %ymm2, %ymm1
8933 ; AVX512DQ-NEXT: vpor %ymm0, %ymm1, %ymm0
8934 ; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8935 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm11[0],ymm6[1],ymm11[2,3],ymm6[4],ymm11[5],ymm6[6],ymm11[7,8],ymm6[9],ymm11[10,11],ymm6[12],ymm11[13],ymm6[14],ymm11[15]
8936 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
8937 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
8938 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,8,9,18,19,28,29,22,23,16,17,26,27,128,128,128,128,128,128]
8939 ; AVX512DQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2
8940 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4,5],ymm7[6],ymm8[7,8],ymm7[9],ymm8[10],ymm7[11],ymm8[12,13],ymm7[14],ymm8[15]
8941 ; AVX512DQ-NEXT: vextracti128 $1, %ymm1, %xmm3
8942 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6],xmm3[7]
8943 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm1 = [2,3,12,13,6,7,0,1,10,11,4,5,14,15,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
8944 ; AVX512DQ-NEXT: vpshufb %ymm1, %ymm3, %ymm3
8945 ; AVX512DQ-NEXT: vpor %ymm2, %ymm3, %ymm2
8946 ; AVX512DQ-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
8947 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm9[0],ymm10[1],ymm9[2,3],ymm10[4],ymm9[5],ymm10[6],ymm9[7,8],ymm10[9],ymm9[10,11],ymm10[12],ymm9[13],ymm10[14],ymm9[15]
8948 ; AVX512DQ-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8949 ; AVX512DQ-NEXT: vmovdqu %ymm9, (%rsp) # 32-byte Spill
8950 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
8951 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
8952 ; AVX512DQ-NEXT: vpshufb %ymm0, %ymm2, %ymm0
8953 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
8954 ; AVX512DQ-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8955 ; AVX512DQ-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
8956 ; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
8957 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5,6],xmm3[7]
8958 ; AVX512DQ-NEXT: vpshufb %ymm1, %ymm2, %ymm1
8959 ; AVX512DQ-NEXT: vporq %ymm0, %ymm1, %ymm28
8960 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm6 = [6,7,0,1,10,11,4,5,14,15,14,15,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
8961 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm13[0,1],ymm15[2],ymm13[3],ymm15[4],ymm13[5,6],ymm15[7],ymm13[8,9],ymm15[10],ymm13[11],ymm15[12],ymm13[13,14],ymm15[15]
8962 ; AVX512DQ-NEXT: vmovdqa64 %ymm15, %ymm18
8963 ; AVX512DQ-NEXT: vmovdqa64 %ymm13, %ymm24
8964 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
8965 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4,5,6,7]
8966 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[6,7,6,7,8,9,4,5,14,15,8,9,2,3,12,13,22,23,22,23,24,25,20,21,30,31,24,25,18,19,28,29]
8967 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm12[0],xmm14[1],xmm12[2,3]
8968 ; AVX512DQ-NEXT: vmovdqa64 %xmm12, %xmm16
8969 ; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm30
8970 ; AVX512DQ-NEXT: vpshufb %xmm6, %xmm1, %xmm1
8971 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[3,4,5,6,7]
8972 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
8973 ; AVX512DQ-NEXT: vmovdqa 144(%rdi), %xmm2
8974 ; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm3
8975 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm3[2],xmm2[3]
8976 ; AVX512DQ-NEXT: vmovdqa %xmm3, %xmm7
8977 ; AVX512DQ-NEXT: vmovdqa %xmm2, %xmm11
8978 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
8979 ; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
8980 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
8981 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm9[0],ymm10[1],ymm9[2],ymm10[3],ymm9[4,5],ymm10[6],ymm9[7,8],ymm10[9],ymm9[10],ymm10[11],ymm9[12,13],ymm10[14],ymm9[15]
8982 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
8983 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]
8984 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm5[1,2],ymm4[3],ymm5[4],ymm4[5],ymm5[6,7],ymm4[8],ymm5[9,10],ymm4[11],ymm5[12],ymm4[13],ymm5[14,15]
8985 ; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
8986 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
8987 ; AVX512DQ-NEXT: vpshufb %ymm6, %ymm1, %ymm1
8988 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[6,7,0,1,10,11,4,5,14,15,8,9,12,13,14,15]
8989 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
8990 ; AVX512DQ-NEXT: vmovdqa 256(%rdi), %ymm12
8991 ; AVX512DQ-NEXT: vmovdqa 288(%rdi), %ymm15
8992 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm12[0],ymm15[1],ymm12[2,3],ymm15[4],ymm12[5],ymm15[6],ymm12[7,8],ymm15[9],ymm12[10,11],ymm15[12],ymm12[13],ymm15[14],ymm12[15]
8993 ; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
8994 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
8995 ; AVX512DQ-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
8996 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm2[0,1,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
8997 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
8998 ; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm2
8999 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15]
9000 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
9001 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
9002 ; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9003 ; AVX512DQ-NEXT: vmovdqa 464(%rdi), %xmm8
9004 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm8[3,1,2,3]
9005 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
9006 ; AVX512DQ-NEXT: vmovdqa 448(%rdi), %xmm3
9007 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
9008 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
9009 ; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
9010 ; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9011 ; AVX512DQ-NEXT: vmovdqa %xmm11, %xmm6
9012 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm11[3,1,2,3]
9013 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
9014 ; AVX512DQ-NEXT: vmovdqa %xmm7, %xmm9
9015 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm7[0,2,2,3]
9016 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
9017 ; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
9018 ; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9019 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0],xmm8[1],xmm3[2,3]
9020 ; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm22
9021 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm1 = [6,7,2,3,4,5,6,7,6,7,2,3,12,13,6,7]
9022 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm0, %xmm0
9023 ; AVX512DQ-NEXT: vmovdqa64 %xmm1, %xmm23
9024 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm10
9025 ; AVX512DQ-NEXT: vmovdqa 576(%rdi), %ymm1
9026 ; AVX512DQ-NEXT: vmovdqa 608(%rdi), %ymm2
9027 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1],ymm1[2],ymm2[3],ymm1[4],ymm2[5,6],ymm1[7],ymm2[8,9],ymm1[10],ymm2[11],ymm1[12],ymm2[13,14],ymm1[15]
9028 ; AVX512DQ-NEXT: vmovdqa64 %ymm2, %ymm20
9029 ; AVX512DQ-NEXT: vmovdqa64 %ymm1, %ymm17
9030 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
9031 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm11 = xmm0[0,1,2],xmm1[3,4],xmm0[5,6,7]
9032 ; AVX512DQ-NEXT: vmovdqa 512(%rdi), %ymm5
9033 ; AVX512DQ-NEXT: vmovdqa 544(%rdi), %ymm13
9034 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm13[0],ymm5[1],ymm13[2,3],ymm5[4],ymm13[5],ymm5[6],ymm13[7,8],ymm5[9],ymm13[10,11],ymm5[12],ymm13[13],ymm5[14],ymm13[15]
9035 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
9036 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5,6,7]
9037 ; AVX512DQ-NEXT: vmovdqa64 496(%rdi), %xmm21
9038 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm21[3,1,2,3]
9039 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm1[2,1,2,3,4,5,6,7]
9040 ; AVX512DQ-NEXT: vmovdqa 480(%rdi), %xmm1
9041 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm14 = xmm1[0,2,2,3]
9042 ; AVX512DQ-NEXT: vmovdqa %xmm1, %xmm7
9043 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm14 = xmm14[0,3,2,3,4,5,6,7]
9044 ; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
9045 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm14 = [u,u,u,u,u,u,u,u,8,9,2,3,12,13,6,7,16,17,26,27,26,27,30,31,24,25,18,19,28,29,22,23]
9046 ; AVX512DQ-NEXT: vpshufb %ymm14, %ymm2, %ymm2
9047 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3,4,5,6,7]
9048 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm2 = [12,13,14,15,4,5,14,15,8,9,2,3,12,13,6,7]
9049 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm11
9050 ; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
9051 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm11[5,6,7]
9052 ; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm27 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
9053 ; AVX512DQ-NEXT: vpternlogq $184, %zmm19, %zmm27, %zmm10
9054 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm10, %zmm0
9055 ; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9056 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm15[0,1],ymm12[2],ymm15[3],ymm12[4],ymm15[5,6],ymm12[7],ymm15[8,9],ymm12[10],ymm15[11],ymm12[12],ymm15[13,14],ymm12[15]
9057 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm10
9058 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm10[3,4],xmm0[5,6,7]
9059 ; AVX512DQ-NEXT: vpshufb %xmm2, %xmm0, %xmm0
9060 ; AVX512DQ-NEXT: vmovdqa64 %ymm18, %ymm3
9061 ; AVX512DQ-NEXT: vmovdqa64 %ymm24, %ymm1
9062 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5],ymm3[6],ymm1[7,8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13],ymm3[14],ymm1[15]
9063 ; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm10
9064 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0],ymm2[1,2],ymm10[3],ymm2[4,5,6,7]
9065 ; AVX512DQ-NEXT: vpshufb %ymm14, %ymm2, %ymm2
9066 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm10 = xmm16[3,1,2,3]
9067 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[2,1,2,3,4,5,6,7]
9068 ; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm24
9069 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm11 = xmm30[0,2,2,3]
9070 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,3,2,3,4,5,6,7]
9071 ; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
9072 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1],ymm2[2,3,4,5,6,7]
9073 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9074 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
9075 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm2 = xmm9[0],xmm6[1],xmm9[2,3]
9076 ; AVX512DQ-NEXT: vmovdqa64 %xmm9, %xmm25
9077 ; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm4
9078 ; AVX512DQ-NEXT: vpshufb %xmm4, %xmm2, %xmm2
9079 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
9080 ; AVX512DQ-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm2 # 64-byte Folded Reload
9081 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
9082 ; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9083 ; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm4
9084 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm2 = xmm4[0,1],xmm8[2],xmm4[3]
9085 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,8,9,4,5,14,15,8,9]
9086 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm2, %xmm2
9087 ; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
9088 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm10 = ymm5[0],ymm13[1],ymm5[2,3],ymm13[4],ymm5[5],ymm13[6],ymm5[7,8],ymm13[9],ymm5[10,11],ymm13[12],ymm5[13],ymm13[14],ymm5[15]
9089 ; AVX512DQ-NEXT: vmovdqa64 %ymm13, %ymm26
9090 ; AVX512DQ-NEXT: vmovdqa64 %ymm5, %ymm22
9091 ; AVX512DQ-NEXT: vextracti128 $1, %ymm10, %xmm11
9092 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4,5,6,7]
9093 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm11 = xmm7[0,3,2,3]
9094 ; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm30
9095 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,2,2,3,4,5,6,7]
9096 ; AVX512DQ-NEXT: vpsrlq $48, %xmm21, %xmm13
9097 ; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm13[0],xmm11[1],xmm13[1]
9098 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm13 = [2,3,12,13,0,1,0,1,10,11,4,5,14,15,8,9,18,19,28,29,16,17,16,17,26,27,20,21,30,31,24,25]
9099 ; AVX512DQ-NEXT: vpshufb %ymm13, %ymm10, %ymm10
9100 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0,1,2],xmm10[3,4,5,6,7]
9101 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
9102 ; AVX512DQ-NEXT: vmovdqa64 %ymm20, %ymm9
9103 ; AVX512DQ-NEXT: vmovdqa64 %ymm17, %ymm7
9104 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm11 = ymm7[0],ymm9[1,2],ymm7[3],ymm9[4],ymm7[5],ymm9[6,7],ymm7[8],ymm9[9,10],ymm7[11],ymm9[12],ymm7[13],ymm9[14,15]
9105 ; AVX512DQ-NEXT: vextracti128 $1, %ymm11, %xmm14
9106 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm11 = xmm14[0],xmm11[1],xmm14[2],xmm11[3]
9107 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm14 = [0,1,2,3,6,7,0,1,10,11,4,5,14,15,8,9]
9108 ; AVX512DQ-NEXT: vpshufb %xmm14, %xmm11, %xmm11
9109 ; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
9110 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm11[5,6,7]
9111 ; AVX512DQ-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm2 # 64-byte Folded Reload
9112 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm10, %zmm2, %zmm2
9113 ; AVX512DQ-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9114 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5],ymm1[6],ymm3[7,8],ymm1[9],ymm3[10,11],ymm1[12],ymm3[13],ymm1[14],ymm3[15]
9115 ; AVX512DQ-NEXT: vmovdqa64 %ymm1, %ymm17
9116 ; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm10
9117 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm10[1],ymm2[2],ymm10[3],ymm2[4,5,6,7]
9118 ; AVX512DQ-NEXT: vpshufb %ymm13, %ymm2, %ymm2
9119 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm10 = xmm24[0,3,2,3]
9120 ; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm20
9121 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[1,2,2,3,4,5,6,7]
9122 ; AVX512DQ-NEXT: vmovdqa64 %xmm16, %xmm1
9123 ; AVX512DQ-NEXT: vpsrlq $48, %xmm16, %xmm11
9124 ; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]
9125 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm2[3,4,5,6,7]
9126 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3],ymm2[4,5,6,7]
9127 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm10 = ymm12[0],ymm15[1,2],ymm12[3],ymm15[4],ymm12[5],ymm15[6,7],ymm12[8],ymm15[9,10],ymm12[11],ymm15[12],ymm12[13],ymm15[14,15]
9128 ; AVX512DQ-NEXT: vmovdqa64 %ymm12, %ymm19
9129 ; AVX512DQ-NEXT: vextracti128 $1, %ymm10, %xmm11
9130 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2],xmm10[3]
9131 ; AVX512DQ-NEXT: vpshufb %xmm14, %xmm10, %xmm10
9132 ; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
9133 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm10[5,6,7]
9134 ; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm5
9135 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm10 = xmm5[0,1],xmm6[2],xmm5[3]
9136 ; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm23
9137 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm10, %xmm0
9138 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9139 ; AVX512DQ-NEXT: vpternlogq $184, %zmm28, %zmm27, %zmm0
9140 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
9141 ; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9142 ; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
9143 ; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm18 # 32-byte Reload
9144 ; AVX512DQ-NEXT: vmovdqa64 %ymm18, %ymm0
9145 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm12[1],ymm0[2,3],ymm12[4],ymm0[5],ymm12[6],ymm0[7,8],ymm12[9],ymm0[10,11],ymm12[12],ymm0[13],ymm12[14],ymm0[15]
9146 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
9147 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm2[6],ymm0[7]
9148 ; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Reload
9149 ; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Reload
9150 ; AVX512DQ-NEXT: vmovdqa64 %ymm24, %ymm0
9151 ; AVX512DQ-NEXT: vmovdqa64 %ymm16, %ymm5
9152 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm5[2],ymm0[3],ymm5[4],ymm0[5,6],ymm5[7],ymm0[8,9],ymm5[10],ymm0[11],ymm5[12],ymm0[13,14],ymm5[15]
9153 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm10
9154 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm10 = xmm0[0,1,2],xmm10[3,4],xmm0[5,6,7]
9155 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm0 = [4,5,14,15,8,9,2,3,12,13,10,11,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
9156 ; AVX512DQ-NEXT: vpshufb %ymm0, %ymm2, %ymm2
9157 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[4,5,14,15,8,9,2,3,12,13,6,7,12,13,14,15]
9158 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2],ymm2[3,4,5,6,7]
9159 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm13 = xmm8[0],xmm4[1],xmm8[2,3]
9160 ; AVX512DQ-NEXT: vmovdqa64 %xmm4, %xmm29
9161 ; AVX512DQ-NEXT: vmovdqa64 %xmm8, %xmm28
9162 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
9163 ; AVX512DQ-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
9164 ; AVX512DQ-NEXT: vpternlogq $184, %zmm2, %zmm27, %zmm13
9165 ; AVX512DQ-NEXT: vmovdqa64 %ymm26, %ymm8
9166 ; AVX512DQ-NEXT: vmovdqa64 %ymm22, %ymm4
9167 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm8[0],ymm4[1],ymm8[2],ymm4[3],ymm8[4,5],ymm4[6],ymm8[7,8],ymm4[9],ymm8[10],ymm4[11],ymm8[12,13],ymm4[14],ymm8[15]
9168 ; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm14
9169 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm14[1],ymm2[2,3,4,5,6,7]
9170 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm14 = [4,5,14,15,12,13,2,3,12,13,6,7,0,1,10,11,20,21,30,31,28,29,18,19,28,29,22,23,16,17,26,27]
9171 ; AVX512DQ-NEXT: vpshufb %ymm14, %ymm2, %ymm2
9172 ; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm6
9173 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm11 = xmm30[0,1,1,3]
9174 ; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,7,6,7]
9175 ; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm5
9176 ; AVX512DQ-NEXT: vpunpckhdq {{.*#+}} xmm11 = xmm11[2],xmm21[2],xmm11[3],xmm21[3]
9177 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0,1,2],xmm2[3,4,5,6,7]
9178 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm11[0,1,2,3],ymm2[4,5,6,7]
9179 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm11 = ymm9[0],ymm7[1],ymm9[2,3],ymm7[4],ymm9[5],ymm7[6],ymm9[7,8],ymm7[9],ymm9[10,11],ymm7[12],ymm9[13],ymm7[14],ymm9[15]
9180 ; AVX512DQ-NEXT: vmovdqa64 %ymm7, %ymm21
9181 ; AVX512DQ-NEXT: vextracti128 $1, %ymm11, %xmm10
9182 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm11[3,4],xmm10[5,6,7]
9183 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm11 = [0,1,2,3,8,9,2,3,12,13,6,7,0,1,10,11]
9184 ; AVX512DQ-NEXT: vpshufb %xmm11, %xmm10, %xmm10
9185 ; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
9186 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm10[5,6,7]
9187 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm13, %zmm2
9188 ; AVX512DQ-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9189 ; AVX512DQ-NEXT: vmovdqa64 %ymm17, %ymm2
9190 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10],ymm3[11],ymm2[12,13],ymm3[14],ymm2[15]
9191 ; AVX512DQ-NEXT: vmovdqa64 %ymm3, %ymm22
9192 ; AVX512DQ-NEXT: vmovdqa64 %ymm17, %ymm26
9193 ; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm10
9194 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm10[1],ymm2[2,3,4,5,6,7]
9195 ; AVX512DQ-NEXT: vpshufb %ymm14, %ymm2, %ymm2
9196 ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm10 = xmm20[0,1,1,3]
9197 ; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,7,6,7]
9198 ; AVX512DQ-NEXT: vpunpckhdq {{.*#+}} xmm10 = xmm10[2],xmm1[2],xmm10[3],xmm1[3]
9199 ; AVX512DQ-NEXT: vmovdqa64 %xmm1, %xmm17
9200 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm2[3,4,5,6,7]
9201 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3],ymm2[4,5,6,7]
9202 ; AVX512DQ-NEXT: vmovdqa64 %ymm19, %ymm1
9203 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm10 = ymm15[0],ymm1[1],ymm15[2,3],ymm1[4],ymm15[5],ymm1[6],ymm15[7,8],ymm1[9],ymm15[10,11],ymm1[12],ymm15[13],ymm1[14],ymm15[15]
9204 ; AVX512DQ-NEXT: vmovdqa64 %ymm15, %ymm31
9205 ; AVX512DQ-NEXT: vextracti128 $1, %ymm10, %xmm13
9206 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm10 = xmm13[0,1,2],xmm10[3,4],xmm13[5,6,7]
9207 ; AVX512DQ-NEXT: vpshufb %xmm11, %xmm10, %xmm10
9208 ; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
9209 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm10[5,6,7]
9210 ; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9211 ; AVX512DQ-NEXT: vpblendw $82, (%rsp), %ymm1, %ymm10 # 32-byte Folded Reload
9212 ; AVX512DQ-NEXT: # ymm10 = ymm1[0],mem[1],ymm1[2,3],mem[4],ymm1[5],mem[6],ymm1[7,8],mem[9],ymm1[10,11],mem[12],ymm1[13],mem[14],ymm1[15]
9213 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
9214 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm11[6],ymm10[7]
9215 ; AVX512DQ-NEXT: vpshufb %ymm0, %ymm10, %ymm0
9216 ; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
9217 ; AVX512DQ-NEXT: vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload
9218 ; AVX512DQ-NEXT: # ymm10 = ymm1[0,1],mem[2],ymm1[3],mem[4],ymm1[5,6],mem[7],ymm1[8,9],mem[10],ymm1[11],mem[12],ymm1[13,14],mem[15]
9219 ; AVX512DQ-NEXT: vextracti128 $1, %ymm10, %xmm11
9220 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm11[3,4],xmm10[5,6,7]
9221 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[4,5,14,15,8,9,2,3,12,13,6,7,12,13,14,15]
9222 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2],ymm0[3,4,5,6,7]
9223 ; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm1
9224 ; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm3
9225 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm10 = xmm3[0],xmm1[1],xmm3[2,3]
9226 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
9227 ; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
9228 ; AVX512DQ-NEXT: vpternlogq $184, %zmm0, %zmm27, %zmm10
9229 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm10, %zmm23
9230 ; AVX512DQ-NEXT: vmovdqa64 %ymm18, %ymm13
9231 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm12[0],ymm13[1],ymm12[2],ymm13[3],ymm12[4,5],ymm13[6],ymm12[7,8],ymm13[9],ymm12[10],ymm13[11],ymm12[12,13],ymm13[14],ymm12[15]
9232 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm10 = ymm0[2,3,0,1]
9233 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm10[4],ymm0[5],ymm10[6],ymm0[7]
9234 ; AVX512DQ-NEXT: vmovdqa64 %ymm16, %ymm14
9235 ; AVX512DQ-NEXT: vmovdqa64 %ymm24, %ymm15
9236 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm10 = ymm14[0],ymm15[1,2],ymm14[3],ymm15[4],ymm14[5],ymm15[6,7],ymm14[8],ymm15[9,10],ymm14[11],ymm15[12],ymm14[13],ymm15[14,15]
9237 ; AVX512DQ-NEXT: vextracti128 $1, %ymm10, %xmm11
9238 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2],xmm10[3]
9239 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[6,7,0,1,10,11,4,5,14,15,8,9,12,13,14,15]
9240 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm1 = [6,7,0,1,10,11,4,5,14,15,14,15,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
9241 ; AVX512DQ-NEXT: vpshufb %ymm1, %ymm0, %ymm0
9242 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2],ymm0[3,4,5,6,7]
9243 ; AVX512DQ-NEXT: vmovdqa %ymm4, %ymm3
9244 ; AVX512DQ-NEXT: vmovdqa %ymm8, %ymm4
9245 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm10 = ymm8[0,1],ymm3[2],ymm8[3],ymm3[4],ymm8[5,6],ymm3[7],ymm8[8,9],ymm3[10],ymm8[11],ymm3[12],ymm8[13,14],ymm3[15]
9246 ; AVX512DQ-NEXT: vextracti128 $1, %ymm10, %xmm11
9247 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4,5,6,7]
9248 ; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[6,7,6,7,8,9,4,5,14,15,8,9,2,3,12,13,22,23,22,23,24,25,20,21,30,31,24,25,18,19,28,29]
9249 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm11 = xmm5[0],xmm6[1],xmm5[2,3]
9250 ; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm7
9251 ; AVX512DQ-NEXT: vmovdqa %xmm5, %xmm6
9252 ; AVX512DQ-NEXT: vpshufb %xmm1, %xmm11, %xmm11
9253 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0,1,2],xmm10[3,4,5,6,7]
9254 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
9255 ; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm1
9256 ; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm2
9257 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm2[0,1],xmm1[2],xmm2[3]
9258 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
9259 ; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
9260 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm10, %zmm8, %zmm8
9261 ; AVX512DQ-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm8
9262 ; AVX512DQ-NEXT: vmovdqa %ymm9, %ymm2
9263 ; AVX512DQ-NEXT: vmovdqa64 %ymm21, %ymm10
9264 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm10[0],ymm9[1],ymm10[2,3],ymm9[4],ymm10[5],ymm9[6],ymm10[7,8],ymm9[9],ymm10[10,11],ymm9[12],ymm10[13],ymm9[14],ymm10[15]
9265 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm9
9266 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm9[1,2,3],xmm0[4,5],xmm9[6,7]
9267 ; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
9268 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9269 ; AVX512DQ-NEXT: vextracti64x4 $1, %zmm8, %ymm9
9270 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0],ymm0[1,2,3,4,5,6,7],ymm9[8],ymm0[9,10,11,12,13,14,15]
9271 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
9272 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm8, %zmm25
9273 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm12[0,1],ymm13[2],ymm12[3],ymm13[4],ymm12[5,6],ymm13[7],ymm12[8,9],ymm13[10],ymm12[11],ymm13[12],ymm12[13,14],ymm13[15]
9274 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm0[2,3,0,1]
9275 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4],ymm0[5,6],ymm8[7]
9276 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm8 = ymm15[0],ymm14[1],ymm15[2,3],ymm14[4],ymm15[5],ymm14[6],ymm15[7,8],ymm14[9],ymm15[10,11],ymm14[12],ymm15[13],ymm14[14],ymm15[15]
9277 ; AVX512DQ-NEXT: vextracti128 $1, %ymm8, %xmm9
9278 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm11 = xmm9[0,1,2],xmm8[3,4],xmm9[5,6,7]
9279 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm9 = [8,9,2,3,12,13,6,7,4,5,6,7,4,5,14,15,24,25,18,19,28,29,22,23,20,21,22,23,20,21,30,31]
9280 ; AVX512DQ-NEXT: vpshufb %ymm9, %ymm0, %ymm0
9281 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm8 = [8,9,2,3,12,13,6,7,0,1,10,11,0,1,6,7]
9282 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm11, %xmm11
9283 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2],ymm0[3,4,5,6,7]
9284 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm4 = ymm3[0],ymm4[1,2],ymm3[3],ymm4[4],ymm3[5],ymm4[6,7],ymm3[8],ymm4[9,10],ymm3[11],ymm4[12],ymm3[13],ymm4[14,15]
9285 ; AVX512DQ-NEXT: vextracti128 $1, %ymm4, %xmm5
9286 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3,4,5,6,7]
9287 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm6[0,1],xmm7[2],xmm6[3]
9288 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm3 = [8,9,14,15,4,5,6,7,0,1,10,11,4,5,14,15,24,25,30,31,20,21,22,23,16,17,26,27,20,21,30,31]
9289 ; AVX512DQ-NEXT: vpshufb %ymm3, %ymm4, %ymm4
9290 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm1, %xmm1
9291 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3,4,5,6,7]
9292 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm4[4,5,6,7]
9293 ; AVX512DQ-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
9294 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm4, %zmm1
9295 ; AVX512DQ-NEXT: movb $7, %al
9296 ; AVX512DQ-NEXT: kmovw %eax, %k1
9297 ; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm1 {%k1}
9298 ; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm4
9299 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm10[1],ymm2[2],ymm10[3],ymm2[4,5],ymm10[6],ymm2[7,8],ymm10[9],ymm2[10],ymm10[11],ymm2[12,13],ymm10[14],ymm2[15]
9300 ; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm5
9301 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm5 = xmm0[0,1],xmm5[2,3],xmm0[4,5,6],xmm5[7]
9302 ; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm0 = [6,7,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
9303 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm5
9304 ; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
9305 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm5 = ymm4[0],ymm5[1,2,3,4,5,6,7],ymm4[8],ymm5[9,10,11,12,13,14,15]
9306 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
9307 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm1
9308 ; AVX512DQ-NEXT: vmovdqa64 %ymm22, %ymm2
9309 ; AVX512DQ-NEXT: vmovdqa64 %ymm26, %ymm4
9310 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm4 = ymm2[0],ymm4[1,2],ymm2[3],ymm4[4],ymm2[5],ymm4[6,7],ymm2[8],ymm4[9,10],ymm2[11],ymm4[12],ymm2[13],ymm4[14,15]
9311 ; AVX512DQ-NEXT: vextracti128 $1, %ymm4, %xmm5
9312 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3,4,5,6,7]
9313 ; AVX512DQ-NEXT: vpshufb %ymm3, %ymm4, %ymm3
9314 ; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm2
9315 ; AVX512DQ-NEXT: vmovdqa64 %xmm17, %xmm4
9316 ; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm2[2],xmm4[3]
9317 ; AVX512DQ-NEXT: vpshufb %xmm9, %xmm4, %xmm4
9318 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm3[3,4,5,6,7]
9319 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
9320 ; AVX512DQ-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
9321 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3
9322 ; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
9323 ; AVX512DQ-NEXT: vpblendw $107, (%rsp), %ymm2, %ymm4 # 32-byte Folded Reload
9324 ; AVX512DQ-NEXT: # ymm4 = mem[0,1],ymm2[2],mem[3],ymm2[4],mem[5,6],ymm2[7],mem[8,9],ymm2[10],mem[11],ymm2[12],mem[13,14],ymm2[15]
9325 ; AVX512DQ-NEXT: vpermq {{.*#+}} ymm5 = ymm4[2,3,0,1]
9326 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4],ymm4[5,6],ymm5[7]
9327 ; AVX512DQ-NEXT: vpshufb %ymm9, %ymm4, %ymm4
9328 ; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
9329 ; AVX512DQ-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm5 # 32-byte Folded Reload
9330 ; AVX512DQ-NEXT: # ymm5 = ymm2[0],mem[1],ymm2[2,3],mem[4],ymm2[5],mem[6],ymm2[7,8],mem[9],ymm2[10,11],mem[12],ymm2[13],mem[14],ymm2[15]
9331 ; AVX512DQ-NEXT: vextracti128 $1, %ymm5, %xmm6
9332 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3,4],xmm6[5,6,7]
9333 ; AVX512DQ-NEXT: vpshufb %xmm8, %xmm5, %xmm5
9334 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3,4,5,6,7]
9335 ; AVX512DQ-NEXT: vinserti64x4 $0, %ymm4, %zmm0, %zmm3 {%k1}
9336 ; AVX512DQ-NEXT: vmovdqa64 %ymm19, %ymm2
9337 ; AVX512DQ-NEXT: vmovdqa64 %ymm31, %ymm4
9338 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm2[1],ymm4[2],ymm2[3],ymm4[4,5],ymm2[6],ymm4[7,8],ymm2[9],ymm4[10],ymm2[11],ymm4[12,13],ymm2[14],ymm4[15]
9339 ; AVX512DQ-NEXT: vextracti128 $1, %ymm4, %xmm5
9340 ; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5,6],xmm5[7]
9341 ; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm0
9342 ; AVX512DQ-NEXT: vextracti64x4 $1, %zmm3, %ymm4
9343 ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9344 ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2,3,4,5,6,7],ymm4[8],ymm0[9,10,11,12,13,14,15]
9345 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
9346 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0
9347 ; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
9348 ; AVX512DQ-NEXT: vmovaps %zmm2, (%rsi)
9349 ; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
9350 ; AVX512DQ-NEXT: vmovaps %zmm2, 64(%rsi)
9351 ; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
9352 ; AVX512DQ-NEXT: vmovaps %zmm2, 64(%rdx)
9353 ; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
9354 ; AVX512DQ-NEXT: vmovaps %zmm2, (%rdx)
9355 ; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
9356 ; AVX512DQ-NEXT: vmovaps %zmm2, 64(%rcx)
9357 ; AVX512DQ-NEXT: vmovdqa64 %zmm23, (%rcx)
9358 ; AVX512DQ-NEXT: vmovdqa64 %zmm25, 64(%r8)
9359 ; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
9360 ; AVX512DQ-NEXT: vmovaps %zmm2, (%r8)
9361 ; AVX512DQ-NEXT: vmovdqa64 %zmm1, 64(%r9)
9362 ; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%r9)
9363 ; AVX512DQ-NEXT: addq $552, %rsp # imm = 0x228
9364 ; AVX512DQ-NEXT: vzeroupper
9365 ; AVX512DQ-NEXT: retq
9366 ;
9367 ; AVX512DQ-FCP-LABEL: load_i16_stride5_vf64:
9368 ; AVX512DQ-FCP: # %bb.0:
9369 ; AVX512DQ-FCP-NEXT: subq $552, %rsp # imm = 0x228
9370 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,14,15,4,5,6,7,8,9,10,11,12,13,14,15]
9371 ; AVX512DQ-FCP-NEXT: vmovdqa 496(%rdi), %xmm1
9372 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9373 ; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm1
9374 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
9375 ; AVX512DQ-FCP-NEXT: vmovdqa 480(%rdi), %xmm2
9376 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
9377 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
9378 ; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
9379 ; AVX512DQ-FCP-NEXT: vmovdqa 512(%rdi), %ymm10
9380 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9381 ; AVX512DQ-FCP-NEXT: vmovdqa 544(%rdi), %ymm11
9382 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9383 ; AVX512DQ-FCP-NEXT: vmovdqa 576(%rdi), %ymm7
9384 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9385 ; AVX512DQ-FCP-NEXT: vmovdqa 608(%rdi), %ymm8
9386 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm8, (%rsp) # 32-byte Spill
9387 ; AVX512DQ-FCP-NEXT: vmovdqa 352(%rdi), %ymm4
9388 ; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdi), %ymm5
9389 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
9390 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm5, %ymm28
9391 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm4, %ymm30
9392 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm1, %xmm4
9393 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm1[0],xmm4[1,2,3],xmm1[4,5],xmm4[6,7]
9394 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
9395 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm4
9396 ; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdi), %ymm6
9397 ; AVX512DQ-FCP-NEXT: vmovdqa 416(%rdi), %ymm9
9398 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm6[0],ymm9[1,2],ymm6[3],ymm9[4],ymm6[5],ymm9[6,7],ymm6[8],ymm9[9,10],ymm6[11],ymm9[12],ymm6[13],ymm9[14,15]
9399 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm9, %ymm26
9400 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9401 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm27
9402 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [1,3,0,2,4,6,1,3]
9403 ; AVX512DQ-FCP-NEXT: vpermd %ymm5, %ymm6, %ymm5
9404 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm13 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,2,3,16,17,22,23,24,25,30,31,20,21,128,128,128,128,128,128]
9405 ; AVX512DQ-FCP-NEXT: vpshufb %ymm13, %ymm5, %ymm5
9406 ; AVX512DQ-FCP-NEXT: vpor %ymm5, %ymm4, %ymm4
9407 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm8[0,1],ymm7[2],ymm8[3],ymm7[4],ymm8[5,6],ymm7[7],ymm8[8,9],ymm7[10],ymm8[11],ymm7[12],ymm8[13,14],ymm7[15]
9408 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm5, %xmm7
9409 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm7[3,4],xmm5[5,6,7]
9410 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm11[0],ymm10[1],ymm11[2,3],ymm10[4],ymm11[5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10,11],ymm10[12],ymm11[13],ymm10[14],ymm11[15]
9411 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm10 = [2,4,7,1,4,6,0,0]
9412 ; AVX512DQ-FCP-NEXT: vpermd %ymm7, %ymm10, %ymm7
9413 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [u,u,u,u,u,u,u,u,8,9,14,15,0,1,6,7,16,17,22,23,20,21,22,23,24,25,30,31,16,17,22,23]
9414 ; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm7, %ymm7
9415 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm17 = [8,9,3,2,4,5,7,6]
9416 ; AVX512DQ-FCP-NEXT: vpermt2d %ymm2, %ymm17, %ymm7
9417 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [12,13,14,15,4,5,14,15,8,9,2,3,12,13,6,7]
9418 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm5, %xmm5
9419 ; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
9420 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3,4],ymm5[5,6,7]
9421 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm18 = [0,3,1,3,0,3,5,7]
9422 ; AVX512DQ-FCP-NEXT: vmovdqa 448(%rdi), %ymm7
9423 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9424 ; AVX512DQ-FCP-NEXT: vpermd %ymm7, %ymm18, %ymm7
9425 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm11 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
9426 ; AVX512DQ-FCP-NEXT: vpshufb %ymm11, %ymm7, %ymm7
9427 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm11, %ymm14
9428 ; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
9429 ; AVX512DQ-FCP-NEXT: vpternlogq $184, %zmm4, %zmm8, %zmm7
9430 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm8, %zmm16
9431 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm5, %zmm7, %zmm4
9432 ; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9433 ; AVX512DQ-FCP-NEXT: vmovdqa 176(%rdi), %xmm4
9434 ; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm0
9435 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm31
9436 ; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %xmm15
9437 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm15, %xmm3
9438 ; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
9439 ; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm11
9440 ; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm5
9441 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm11[0],ymm5[1,2],ymm11[3],ymm5[4],ymm11[5],ymm5[6,7],ymm11[8],ymm5[9,10],ymm11[11],ymm5[12],ymm11[13],ymm5[14,15]
9442 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm5, %ymm7
9443 ; AVX512DQ-FCP-NEXT: vpermd %ymm3, %ymm6, %ymm3
9444 ; AVX512DQ-FCP-NEXT: vpshufb %ymm13, %ymm3, %ymm3
9445 ; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm5
9446 ; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm6
9447 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm5[0],ymm6[1],ymm5[2,3],ymm6[4],ymm5[5],ymm6[6],ymm5[7,8],ymm6[9],ymm5[10,11],ymm6[12],ymm5[13],ymm6[14],ymm5[15]
9448 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm5, %ymm12
9449 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm4, %xmm5
9450 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1,2,3],xmm4[4,5],xmm5[6,7]
9451 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm1
9452 ; AVX512DQ-FCP-NEXT: vpor %ymm3, %ymm1, %ymm1
9453 ; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm8
9454 ; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm13
9455 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm13[0],ymm8[1],ymm13[2,3],ymm8[4],ymm13[5],ymm8[6],ymm13[7,8],ymm8[9],ymm13[10,11],ymm8[12],ymm13[13],ymm8[14],ymm13[15]
9456 ; AVX512DQ-FCP-NEXT: vpermd %ymm3, %ymm10, %ymm3
9457 ; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm3
9458 ; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %ymm5
9459 ; AVX512DQ-FCP-NEXT: vmovdqa 288(%rdi), %ymm9
9460 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm9[0,1],ymm5[2],ymm9[3],ymm5[4],ymm9[5,6],ymm5[7],ymm9[8,9],ymm5[10],ymm9[11],ymm5[12],ymm9[13,14],ymm5[15]
9461 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm5, %ymm25
9462 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm4, %xmm10
9463 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm10[3,4],xmm4[5,6,7]
9464 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm2
9465 ; AVX512DQ-FCP-NEXT: vpermt2d %ymm0, %ymm17, %ymm3
9466 ; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
9467 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5,6,7]
9468 ; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm23
9469 ; AVX512DQ-FCP-NEXT: vpermd %ymm23, %ymm18, %ymm2
9470 ; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm2, %ymm2
9471 ; AVX512DQ-FCP-NEXT: vpternlogq $184, %zmm1, %zmm16, %zmm2
9472 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
9473 ; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9474 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm6[0],ymm12[1],ymm6[2],ymm12[3],ymm6[4,5],ymm12[6],ymm6[7,8],ymm12[9],ymm6[10],ymm12[11],ymm6[12,13],ymm12[14],ymm6[15]
9475 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm12, %ymm21
9476 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9477 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm29
9478 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9479 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1
9480 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6],xmm1[7]
9481 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm17 = [2,0,0,0,4,7,1,6]
9482 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm7[0],ymm11[1],ymm7[2,3],ymm11[4],ymm7[5],ymm11[6],ymm7[7,8],ymm11[9],ymm7[10,11],ymm11[12],ymm7[13],ymm11[14],ymm7[15]
9483 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm11, %ymm19
9484 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm7, %ymm22
9485 ; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm17, %ymm1
9486 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,0,1,18,19,20,21,26,27,16,17,30,31,128,128,128,128,128,128]
9487 ; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm1
9488 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [2,3,12,13,6,7,0,1,10,11,4,5,14,15,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
9489 ; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm0
9490 ; AVX512DQ-FCP-NEXT: vpor %ymm1, %ymm0, %ymm10
9491 ; AVX512DQ-FCP-NEXT: vpsrlq $48, %xmm31, %xmm0
9492 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm15, %xmm1
9493 ; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
9494 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm20 = [0,2,5,7,4,7,0,0]
9495 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm8[0],ymm13[1],ymm8[2,3],ymm13[4],ymm8[5],ymm13[6],ymm8[7,8],ymm13[9],ymm8[10,11],ymm13[12],ymm8[13],ymm13[14],ymm8[15]
9496 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm8, %ymm5
9497 ; AVX512DQ-FCP-NEXT: vpermd %ymm0, %ymm20, %ymm4
9498 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [2,3,4,5,4,5,0,1,6,7,8,9,14,15,4,5,18,19,20,21,20,21,16,17,22,23,24,25,30,31,20,21]
9499 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm4
9500 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3,4,5,6,7]
9501 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0,1,2,3],ymm4[4,5,6,7]
9502 ; AVX512DQ-FCP-NEXT: vbroadcasti32x4 {{.*#+}} ymm24 = [1,4,6,3,1,4,6,3]
9503 ; AVX512DQ-FCP-NEXT: # ymm24 = mem[0,1,2,3,0,1,2,3]
9504 ; AVX512DQ-FCP-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9505 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm25, %ymm12
9506 ; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm25, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
9507 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm12[0],ymm9[1,2],ymm12[3],ymm9[4],ymm12[5],ymm9[6,7],ymm12[8],ymm9[9,10],ymm12[11],ymm9[12],ymm12[13],ymm9[14,15]
9508 ; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm24, %ymm0
9509 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,2,3,2,3,4,5,10,11,0,1,14,15,8,9,16,17,18,19,18,19,20,21,26,27,16,17,30,31,24,25]
9510 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm0
9511 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4],ymm0[5,6,7]
9512 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm25 = [1,3,2,3,1,3,6,7]
9513 ; AVX512DQ-FCP-NEXT: vpermd %ymm23, %ymm25, %ymm4
9514 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm14 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
9515 ; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm4, %ymm4
9516 ; AVX512DQ-FCP-NEXT: vpternlogq $184, %zmm10, %zmm16, %zmm4
9517 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm16, %zmm10
9518 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm4, %zmm0
9519 ; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9520 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm28, %ymm18
9521 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm30, %ymm28
9522 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm0
9523 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm30, %ymm4
9524 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1],ymm4[2],ymm0[3],ymm4[4,5],ymm0[6],ymm4[7,8],ymm0[9],ymm4[10],ymm0[11],ymm4[12,13],ymm0[14],ymm4[15]
9525 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm4
9526 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3],xmm0[4,5,6],xmm4[7]
9527 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm27, %ymm7
9528 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm26, %ymm4
9529 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm7[1],ymm4[2,3],ymm7[4],ymm4[5],ymm7[6],ymm4[7,8],ymm7[9],ymm4[10,11],ymm7[12],ymm4[13],ymm7[14],ymm4[15]
9530 ; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm17, %ymm4
9531 ; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm4, %ymm4
9532 ; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm0
9533 ; AVX512DQ-FCP-NEXT: vpor %ymm4, %ymm0, %ymm0
9534 ; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
9535 ; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Reload
9536 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm16, %ymm4
9537 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm11[0],ymm4[1],ymm11[2,3],ymm4[4],ymm11[5],ymm4[6],ymm11[7,8],ymm4[9],ymm11[10,11],ymm4[12],ymm11[13],ymm4[14],ymm11[15]
9538 ; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm20, %ymm4
9539 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm3
9540 ; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
9541 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm2
9542 ; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm27 # 16-byte Reload
9543 ; AVX512DQ-FCP-NEXT: vpsrlq $48, %xmm27, %xmm4
9544 ; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
9545 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[3,4,5,6,7]
9546 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
9547 ; AVX512DQ-FCP-NEXT: vmovdqu (%rsp), %ymm8 # 32-byte Reload
9548 ; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
9549 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm8[1,2],ymm4[3],ymm8[4],ymm4[5],ymm8[6,7],ymm4[8],ymm8[9,10],ymm4[11],ymm8[12],ymm4[13],ymm8[14,15]
9550 ; AVX512DQ-FCP-NEXT: vpermd %ymm3, %ymm24, %ymm3
9551 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm3, %ymm1
9552 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5,6,7]
9553 ; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Reload
9554 ; AVX512DQ-FCP-NEXT: vpermd %ymm17, %ymm25, %ymm2
9555 ; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm2, %ymm2
9556 ; AVX512DQ-FCP-NEXT: vpternlogq $184, %zmm0, %zmm10, %zmm2
9557 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm0
9558 ; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9559 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0],ymm12[1],ymm9[2,3],ymm12[4],ymm9[5],ymm12[6],ymm9[7,8],ymm12[9],ymm9[10,11],ymm12[12],ymm9[13],ymm12[14],ymm9[15]
9560 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1
9561 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[3,4],xmm1[5,6,7]
9562 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm15[u,u,u,u,u,u,u,u,4,5,14,15,u,u,u,u]
9563 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm15, %xmm20
9564 ; AVX512DQ-FCP-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm31[2],xmm0[3],xmm31[3]
9565 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm24 = [0,3,5,2,5,7,0,0]
9566 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm13[0],ymm5[1],ymm13[2],ymm5[3],ymm13[4,5],ymm5[6],ymm13[7,8],ymm5[9],ymm13[10],ymm5[11],ymm13[12,13],ymm5[14],ymm13[15]
9567 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm5, %ymm30
9568 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm13, %ymm26
9569 ; AVX512DQ-FCP-NEXT: vpermd %ymm2, %ymm24, %ymm3
9570 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm12 = [0,1,6,7,2,3,2,3,4,5,10,11,0,1,14,15,16,17,22,23,18,19,18,19,20,21,26,27,16,17,30,31]
9571 ; AVX512DQ-FCP-NEXT: vpshufb %ymm12, %ymm3, %ymm3
9572 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3,4,5,6,7]
9573 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3],ymm3[4,5,6,7]
9574 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,8,9,2,3,12,13,6,7,0,1,10,11]
9575 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm1, %xmm1
9576 ; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
9577 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm3[0,1,2,3,4],ymm1[5,6,7]
9578 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm0
9579 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm29, %ymm1
9580 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
9581 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm1, %xmm3
9582 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm1[0,1,2],xmm3[3,4],xmm1[5,6,7]
9583 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm25 = [0,2,0,0,5,7,2,4]
9584 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm0
9585 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm22, %ymm1
9586 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
9587 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm22, %ymm29
9588 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm21
9589 ; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm25, %ymm3
9590 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,6,7,8,9,14,15,4,5,6,7,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
9591 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm3, %ymm0
9592 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [4,5,14,15,8,9,2,3,12,13,6,7,12,13,14,15]
9593 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm5, %xmm5
9594 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2],ymm0[3,4,5,6,7]
9595 ; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [1,4,6,0,1,4,6,0]
9596 ; AVX512DQ-FCP-NEXT: # ymm5 = mem[0,1,0,1]
9597 ; AVX512DQ-FCP-NEXT: vpermd %ymm23, %ymm5, %ymm14
9598 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
9599 ; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm14, %ymm14
9600 ; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
9601 ; AVX512DQ-FCP-NEXT: vpternlogq $184, %zmm0, %zmm15, %zmm14
9602 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm10, %zmm14, %zmm0
9603 ; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
9604 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm8[0],ymm4[1],ymm8[2,3],ymm4[4],ymm8[5],ymm4[6],ymm8[7,8],ymm4[9],ymm8[10,11],ymm4[12],ymm8[13],ymm4[14],ymm8[15]
9605 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm10
9606 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3,4],xmm10[5,6,7]
9607 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm0
9608 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm16, %ymm8
9609 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm8[0],ymm11[1],ymm8[2],ymm11[3],ymm8[4,5],ymm11[6],ymm8[7,8],ymm11[9],ymm8[10],ymm11[11],ymm8[12,13],ymm11[14],ymm8[15]
9610 ; AVX512DQ-FCP-NEXT: vmovdqa %ymm11, %ymm13
9611 ; AVX512DQ-FCP-NEXT: vpermd %ymm2, %ymm24, %ymm2
9612 ; AVX512DQ-FCP-NEXT: vpshufb %ymm12, %ymm2, %ymm2
9613 ; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm6[u,u,u,u,u,u,u,u,4,5,14,15,u,u,u,u]
9614 ; AVX512DQ-FCP-NEXT: vmovdqa %xmm6, %xmm14
9615 ; AVX512DQ-FCP-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm4[2],xmm27[2],xmm4[3],xmm27[3]
9616 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm27, %xmm6
9617 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm2[3,4,5,6,7]
9618 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
9619 ; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
9620 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
9621 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm11
9622 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm28, %ymm12
9623 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm12[0,1],ymm11[2],ymm12[3],ymm11[4],ymm12[5,6],ymm11[7],ymm12[8,9],ymm11[10],ymm12[11],ymm11[12],ymm12[13,14],ymm11[15]
9624 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm2, %xmm4
9625 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[3,4],xmm2[5,6,7]
9626 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
9627 ; AVX512DQ-FCP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm3 # 32-byte Folded Reload
9628 ; AVX512DQ-FCP-NEXT: # ymm3 = ymm7[0],mem[1],ymm7[2,3],mem[4],ymm7[5],mem[6],ymm7[7,8],mem[9],ymm7[10,11],mem[12],ymm7[13],mem[14],ymm7[15]
9629 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm7, %ymm16
9630 ; AVX512DQ-FCP-NEXT: vpermd %ymm3, %ymm25, %ymm3
9631 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm3, %ymm1
9632 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
9633 ; AVX512DQ-FCP-NEXT: vpermd %ymm17, %ymm5, %ymm2
9634 ; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm2
9635 ; AVX512DQ-FCP-NEXT: vpternlogq $184, %zmm1, %zmm15, %zmm2
9636 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm28
9637 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm31, %xmm15
9638 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm20, %xmm7
9639 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm15[0],xmm7[1],xmm15[2,3]
9640 ; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm0 = [6,7,0,1,10,11,0,0,0,0,0,0,0,0,0,0]
9641 ; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm2
9642 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm18 = [1,3,6,0,5,0,0,0]
9643 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm30, %ymm9
9644 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm26, %ymm10
9645 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm10[0,1],ymm9[2],ymm10[3],ymm9[4],ymm10[5,6],ymm9[7],ymm10[8,9],ymm9[10],ymm10[11],ymm9[12],ymm10[13,14],ymm9[15]
9646 ; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm18, %ymm4
9647 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [2,3,2,3,4,5,0,1,6,7,8,9,14,15,4,5,18,19,18,19,20,21,16,17,22,23,24,25,30,31,20,21]
9648 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm4
9649 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[3,4,5,6,7]
9650 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
9651 ; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [2,4,7,0,2,4,7,0]
9652 ; AVX512DQ-FCP-NEXT: # ymm4 = mem[0,1,0,1]
9653 ; AVX512DQ-FCP-NEXT: vpermd %ymm23, %ymm4, %ymm5
9654 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
9655 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm5, %ymm5
9656 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm5, %zmm25
9657 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm6[0],xmm14[1],xmm6[2,3]
9658 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm27, %xmm22
9659 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm14, %xmm31
9660 ; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm2, %xmm0
9661 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm8[0,1],ymm13[2],ymm8[3],ymm13[4],ymm8[5,6],ymm13[7],ymm8[8,9],ymm13[10],ymm8[11],ymm13[12],ymm8[13,14],ymm13[15]
9662 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm13, %ymm20
9663 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm8, %ymm30
9664 ; AVX512DQ-FCP-NEXT: vpermd %ymm2, %ymm18, %ymm2
9665 ; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm2, %ymm1
9666 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3,4,5,6,7]
9667 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
9668 ; AVX512DQ-FCP-NEXT: vpermd %ymm17, %ymm4, %ymm1
9669 ; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm1
9670 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm24
9671 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm15[0,1],xmm7[2],xmm15[3]
9672 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm9[0],ymm10[1,2],ymm9[3],ymm10[4],ymm9[5],ymm10[6,7],ymm9[8],ymm10[9,10],ymm9[11],ymm10[12],ymm9[13],ymm10[14,15]
9673 ; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm2 = [8,9,2,3,12,13,0,0,0,0,0,0,0,0,0,0]
9674 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm3
9675 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm27 = [1,4,6,3,6,0,0,0]
9676 ; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm27, %ymm1
9677 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [0,1,2,3,4,5,2,3,4,5,10,11,0,1,14,15,16,17,18,19,20,21,18,19,20,21,26,27,16,17,30,31]
9678 ; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm1
9679 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm1[3,4,5,6,7]
9680 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
9681 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm26 = [0,2,1,3,0,2,5,7]
9682 ; AVX512DQ-FCP-NEXT: vpermd %ymm23, %ymm26, %ymm3
9683 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,6,7,4,5,6,7,0,1,6,7,8,9,14,15,16,17,22,23,20,21,22,23,16,17,22,23,24,25,30,31]
9684 ; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm3
9685 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm4, %ymm19
9686 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1
9687 ; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
9688 ; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
9689 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm14[1,2],ymm4[3],ymm14[4],ymm4[5],ymm14[6,7],ymm4[8],ymm14[9,10],ymm4[11],ymm14[12],ymm4[13],ymm14[14,15]
9690 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm4, %ymm18
9691 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm3, %xmm4
9692 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2],xmm3[3]
9693 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,3,0,0,5,0,2,7]
9694 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm15
9695 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm29, %ymm10
9696 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm10[0],ymm15[1],ymm10[2],ymm15[3],ymm10[4,5],ymm15[6],ymm10[7,8],ymm15[9],ymm10[10],ymm15[11],ymm10[12,13],ymm15[14],ymm10[15]
9697 ; AVX512DQ-FCP-NEXT: vpermd %ymm5, %ymm4, %ymm5
9698 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [2,3,4,5,10,11,0,1,14,15,14,15,2,3,4,5,18,19,20,21,26,27,16,17,30,31,30,31,18,19,20,21]
9699 ; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm5
9700 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm7 = [6,7,0,1,10,11,4,5,14,15,8,9,12,13,14,15]
9701 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm3, %xmm3
9702 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3,4,5,6,7]
9703 ; AVX512DQ-FCP-NEXT: vpmovsxdq {{.*#+}} zmm5 = [18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0,18446744073709486080,18446744073709551615]
9704 ; AVX512DQ-FCP-NEXT: vpternlogq $184, %zmm3, %zmm5, %zmm25
9705 ; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
9706 ; AVX512DQ-FCP-NEXT: vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
9707 ; AVX512DQ-FCP-NEXT: # ymm3 = ymm3[0],mem[1],ymm3[2,3],mem[4],ymm3[5],mem[6],ymm3[7,8],mem[9],ymm3[10,11],mem[12],ymm3[13],mem[14],ymm3[15]
9708 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm3, %xmm8
9709 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm8 = xmm3[0],xmm8[1,2,3],xmm3[4,5],xmm8[6,7]
9710 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
9711 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm8, %xmm8
9712 ; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
9713 ; AVX512DQ-FCP-NEXT: vextracti64x4 $1, %zmm25, %ymm9
9714 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm9[0],ymm8[1,2,3,4,5,6,7],ymm9[8],ymm8[9,10,11,12,13,14,15]
9715 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
9716 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm8, %zmm25, %zmm21
9717 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm11[0],ymm12[1,2],ymm11[3],ymm12[4],ymm11[5],ymm12[6,7],ymm11[8],ymm12[9,10],ymm11[11],ymm12[12],ymm11[13],ymm12[14,15]
9718 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm12, %ymm23
9719 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm11, %ymm25
9720 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm8, %xmm9
9721 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0],xmm8[1],xmm9[2],xmm8[3]
9722 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm8, %xmm7
9723 ; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
9724 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm16, %ymm12
9725 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm13[0],ymm12[1],ymm13[2],ymm12[3],ymm13[4,5],ymm12[6],ymm13[7,8],ymm12[9],ymm13[10],ymm12[11],ymm13[12,13],ymm12[14],ymm13[15]
9726 ; AVX512DQ-FCP-NEXT: vpermd %ymm8, %ymm4, %ymm4
9727 ; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm4, %ymm4
9728 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm7[0,1,2],ymm4[3,4,5,6,7]
9729 ; AVX512DQ-FCP-NEXT: vpternlogq $184, %zmm4, %zmm5, %zmm24
9730 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm22, %xmm4
9731 ; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm31, %xmm5
9732 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3]
9733 ; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm2
9734 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm20, %ymm4
9735 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm30, %ymm5
9736 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm5[1,2],ymm4[3],ymm5[4],ymm4[5],ymm5[6,7],ymm4[8],ymm5[9,10],ymm4[11],ymm5[12],ymm4[13],ymm5[14,15]
9737 ; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm27, %ymm4
9738 ; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm0
9739 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm0[3,4,5,6,7]
9740 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
9741 ; AVX512DQ-FCP-NEXT: vpermd %ymm17, %ymm26, %ymm2
9742 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm19, %ymm4
9743 ; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm2
9744 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
9745 ; AVX512DQ-FCP-NEXT: vmovdqu (%rsp), %ymm9 # 32-byte Reload
9746 ; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
9747 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm11[0],ymm9[1],ymm11[2,3],ymm9[4],ymm11[5],ymm9[6],ymm11[7,8],ymm9[9],ymm11[10,11],ymm9[12],ymm11[13],ymm9[14],ymm11[15]
9748 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm2, %xmm4
9749 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1,2,3],xmm2[4,5],xmm4[6,7]
9750 ; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
9751 ; AVX512DQ-FCP-NEXT: vextracti64x4 $1, %zmm24, %ymm3
9752 ; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
9753 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1,2,3,4,5,6,7],ymm3[8],ymm2[9,10,11,12,13,14,15]
9754 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
9755 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm24, %zmm2
9756 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm10[0,1],ymm15[2],ymm10[3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8,9],ymm15[10],ymm10[11],ymm15[12],ymm10[13,14],ymm15[15]
9757 ; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [1,3,0,0,6,0,3,5]
9758 ; AVX512DQ-FCP-NEXT: vpermd %ymm3, %ymm4, %ymm3
9759 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,6,7,8,9,14,15,0,1,6,7,0,1,6,7,16,17,22,23,24,25,30,31,16,17,22,23,16,17,22,23]
9760 ; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm3, %ymm3
9761 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm6
9762 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm14[0],ymm6[1],ymm14[2,3],ymm6[4],ymm14[5],ymm6[6],ymm14[7,8],ymm6[9],ymm14[10,11],ymm6[12],ymm14[13],ymm6[14],ymm14[15]
9763 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm6, %xmm7
9764 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3,4],xmm7[5,6,7]
9765 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm7 = [8,9,2,3,12,13,6,7,0,1,10,11,0,1,6,7]
9766 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm6, %xmm6
9767 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2],ymm3[3,4,5,6,7]
9768 ; AVX512DQ-FCP-NEXT: movb $7, %al
9769 ; AVX512DQ-FCP-NEXT: kmovw %eax, %k1
9770 ; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm0, %zmm1 {%k1}
9771 ; AVX512DQ-FCP-NEXT: vextracti64x4 $1, %zmm1, %ymm3
9772 ; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
9773 ; AVX512DQ-FCP-NEXT: vpblendw $181, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
9774 ; AVX512DQ-FCP-NEXT: # ymm6 = mem[0],ymm6[1],mem[2],ymm6[3],mem[4,5],ymm6[6],mem[7,8],ymm6[9],mem[10],ymm6[11],mem[12,13],ymm6[14],mem[15]
9775 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm6, %xmm8
9776 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2,3],xmm6[4,5,6],xmm8[7]
9777 ; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm8 = [6,7,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
9778 ; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm6, %xmm6
9779 ; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
9780 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm3[0],ymm6[1,2,3,4,5,6,7],ymm3[8],ymm6[9,10,11,12,13,14,15]
9781 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5,6,7]
9782 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
9783 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm23, %ymm3
9784 ; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm25, %ymm6
9785 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm6[1],ymm3[2,3],ymm6[4],ymm3[5],ymm6[6],ymm3[7,8],ymm6[9],ymm3[10,11],ymm6[12],ymm3[13],ymm6[14],ymm3[15]
9786 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm3, %xmm6
9787 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm6[0,1,2],xmm3[3,4],xmm6[5,6,7]
9788 ; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm3, %xmm3
9789 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm13[0,1],ymm12[2],ymm13[3],ymm12[4],ymm13[5,6],ymm12[7],ymm13[8,9],ymm12[10],ymm13[11],ymm12[12],ymm13[13,14],ymm12[15]
9790 ; AVX512DQ-FCP-NEXT: vpermd %ymm6, %ymm4, %ymm4
9791 ; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm4
9792 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3,4,5,6,7]
9793 ; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm0, %zmm0 {%k1}
9794 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm9[0],ymm11[1],ymm9[2],ymm11[3],ymm9[4,5],ymm11[6],ymm9[7,8],ymm11[9],ymm9[10],ymm11[11],ymm9[12,13],ymm11[14],ymm9[15]
9795 ; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm3, %xmm4
9796 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5,6],xmm4[7]
9797 ; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm3, %xmm3
9798 ; AVX512DQ-FCP-NEXT: vextracti64x4 $1, %zmm0, %ymm4
9799 ; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
9800 ; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1,2,3,4,5,6,7],ymm4[8],ymm3[9,10,11,12,13,14,15]
9801 ; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
9802 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
9803 ; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
9804 ; AVX512DQ-FCP-NEXT: vmovaps %zmm3, (%rsi)
9805 ; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
9806 ; AVX512DQ-FCP-NEXT: vmovaps %zmm3, 64(%rsi)
9807 ; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
9808 ; AVX512DQ-FCP-NEXT: vmovaps %zmm3, 64(%rdx)
9809 ; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
9810 ; AVX512DQ-FCP-NEXT: vmovaps %zmm3, (%rdx)
9811 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm28, 64(%rcx)
9812 ; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
9813 ; AVX512DQ-FCP-NEXT: vmovaps %zmm3, (%rcx)
9814 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, 64(%r8)
9815 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm21, (%r8)
9816 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, 64(%r9)
9817 ; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, (%r9)
9818 ; AVX512DQ-FCP-NEXT: addq $552, %rsp # imm = 0x228
9819 ; AVX512DQ-FCP-NEXT: vzeroupper
9820 ; AVX512DQ-FCP-NEXT: retq
9821 ;
9822 ; AVX512BW-LABEL: load_i16_stride5_vf64:
9823 ; AVX512BW: # %bb.0:
9824 ; AVX512BW-NEXT: vmovdqa64 576(%rdi), %zmm2
9825 ; AVX512BW-NEXT: vmovdqa64 384(%rdi), %zmm4
9826 ; AVX512BW-NEXT: vmovdqa64 320(%rdi), %zmm3
9827 ; AVX512BW-NEXT: vmovdqa64 512(%rdi), %zmm5
9828 ; AVX512BW-NEXT: vmovdqa64 448(%rdi), %zmm0
9829 ; AVX512BW-NEXT: vmovdqa64 256(%rdi), %zmm7
9830 ; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm9
9831 ; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm11
9832 ; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm1
9833 ; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm10
9834 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm12 = [16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11]
9835 ; AVX512BW-NEXT: # zmm12 = mem[0,1,2,3,0,1,2,3]
9836 ; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm13
9837 ; AVX512BW-NEXT: vpermt2w %zmm10, %zmm12, %zmm13
9838 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
9839 ; AVX512BW-NEXT: vmovdqa64 %zmm9, %zmm8
9840 ; AVX512BW-NEXT: vpermt2w %zmm11, %zmm6, %zmm8
9841 ; AVX512BW-NEXT: movl $67100672, %eax # imm = 0x3FFE000
9842 ; AVX512BW-NEXT: kmovd %eax, %k1
9843 ; AVX512BW-NEXT: vmovdqu16 %zmm13, %zmm8 {%k1}
9844 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm13 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,34,39,44,49,54,59]
9845 ; AVX512BW-NEXT: vpermt2w %zmm7, %zmm13, %zmm8
9846 ; AVX512BW-NEXT: vpermi2w %zmm5, %zmm0, %zmm12
9847 ; AVX512BW-NEXT: vpermi2w %zmm4, %zmm3, %zmm6
9848 ; AVX512BW-NEXT: vmovdqu16 %zmm12, %zmm6 {%k1}
9849 ; AVX512BW-NEXT: vpermt2w %zmm2, %zmm13, %zmm6
9850 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm14 = [49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44]
9851 ; AVX512BW-NEXT: # zmm14 = mem[0,1,2,3,0,1,2,3]
9852 ; AVX512BW-NEXT: vmovdqa64 %zmm10, %zmm15
9853 ; AVX512BW-NEXT: vpermt2w %zmm1, %zmm14, %zmm15
9854 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm12 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
9855 ; AVX512BW-NEXT: vmovdqa64 %zmm9, %zmm13
9856 ; AVX512BW-NEXT: vpermt2w %zmm11, %zmm12, %zmm13
9857 ; AVX512BW-NEXT: vmovdqu16 %zmm15, %zmm13 {%k1}
9858 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,35,40,45,50,55,60]
9859 ; AVX512BW-NEXT: vpermt2w %zmm7, %zmm15, %zmm13
9860 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm5, %zmm14
9861 ; AVX512BW-NEXT: vpermi2w %zmm4, %zmm3, %zmm12
9862 ; AVX512BW-NEXT: vmovdqu16 %zmm14, %zmm12 {%k1}
9863 ; AVX512BW-NEXT: vpermt2w %zmm2, %zmm15, %zmm12
9864 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm16 = [50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45]
9865 ; AVX512BW-NEXT: # zmm16 = mem[0,1,2,3,0,1,2,3]
9866 ; AVX512BW-NEXT: vmovdqa64 %zmm10, %zmm17
9867 ; AVX512BW-NEXT: vpermt2w %zmm1, %zmm16, %zmm17
9868 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm14 = [34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0]
9869 ; AVX512BW-NEXT: vmovdqa64 %zmm11, %zmm15
9870 ; AVX512BW-NEXT: vpermt2w %zmm9, %zmm14, %zmm15
9871 ; AVX512BW-NEXT: vmovdqu16 %zmm17, %zmm15 {%k1}
9872 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm17 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,36,41,46,51,56,61]
9873 ; AVX512BW-NEXT: vpermt2w %zmm7, %zmm17, %zmm15
9874 ; AVX512BW-NEXT: vpermi2w %zmm0, %zmm5, %zmm16
9875 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm4, %zmm14
9876 ; AVX512BW-NEXT: vmovdqu16 %zmm16, %zmm14 {%k1}
9877 ; AVX512BW-NEXT: vpermt2w %zmm2, %zmm17, %zmm14
9878 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm16 = [19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14]
9879 ; AVX512BW-NEXT: # zmm16 = mem[0,1,2,3,0,1,2,3]
9880 ; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm17
9881 ; AVX512BW-NEXT: vpermt2w %zmm10, %zmm16, %zmm17
9882 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm18 = [35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0]
9883 ; AVX512BW-NEXT: vmovdqa64 %zmm11, %zmm19
9884 ; AVX512BW-NEXT: vpermt2w %zmm9, %zmm18, %zmm19
9885 ; AVX512BW-NEXT: movl $33546240, %eax # imm = 0x1FFE000
9886 ; AVX512BW-NEXT: kmovd %eax, %k1
9887 ; AVX512BW-NEXT: vmovdqu16 %zmm17, %zmm19 {%k1}
9888 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm17 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,32,37,42,47,52,57,62]
9889 ; AVX512BW-NEXT: vpermt2w %zmm7, %zmm17, %zmm19
9890 ; AVX512BW-NEXT: vpermi2w %zmm5, %zmm0, %zmm16
9891 ; AVX512BW-NEXT: vpermi2w %zmm3, %zmm4, %zmm18
9892 ; AVX512BW-NEXT: vmovdqu16 %zmm16, %zmm18 {%k1}
9893 ; AVX512BW-NEXT: vpermt2w %zmm2, %zmm17, %zmm18
9894 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm16 = [4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0]
9895 ; AVX512BW-NEXT: vpermt2w %zmm11, %zmm16, %zmm9
9896 ; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15]
9897 ; AVX512BW-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3]
9898 ; AVX512BW-NEXT: vpermt2w %zmm10, %zmm11, %zmm1
9899 ; AVX512BW-NEXT: movb $7, %al
9900 ; AVX512BW-NEXT: kmovd %eax, %k1
9901 ; AVX512BW-NEXT: vmovdqa64 %zmm9, %zmm1 {%k1}
9902 ; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm9 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,33,38,43,48,53,58,63]
9903 ; AVX512BW-NEXT: vpermt2w %zmm7, %zmm9, %zmm1
9904 ; AVX512BW-NEXT: vpermt2w %zmm5, %zmm11, %zmm0
9905 ; AVX512BW-NEXT: vpermt2w %zmm4, %zmm16, %zmm3
9906 ; AVX512BW-NEXT: vmovdqa64 %zmm3, %zmm0 {%k1}
9907 ; AVX512BW-NEXT: vpermt2w %zmm2, %zmm9, %zmm0
9908 ; AVX512BW-NEXT: vmovdqa64 %zmm6, 64(%rsi)
9909 ; AVX512BW-NEXT: vmovdqa64 %zmm8, (%rsi)
9910 ; AVX512BW-NEXT: vmovdqa64 %zmm12, 64(%rdx)
9911 ; AVX512BW-NEXT: vmovdqa64 %zmm13, (%rdx)
9912 ; AVX512BW-NEXT: vmovdqa64 %zmm14, 64(%rcx)
9913 ; AVX512BW-NEXT: vmovdqa64 %zmm15, (%rcx)
9914 ; AVX512BW-NEXT: vmovdqa64 %zmm18, 64(%r8)
9915 ; AVX512BW-NEXT: vmovdqa64 %zmm19, (%r8)
9916 ; AVX512BW-NEXT: vmovdqa64 %zmm0, 64(%r9)
9917 ; AVX512BW-NEXT: vmovdqa64 %zmm1, (%r9)
9918 ; AVX512BW-NEXT: vzeroupper
9919 ; AVX512BW-NEXT: retq
9920 ;
9921 ; AVX512BW-FCP-LABEL: load_i16_stride5_vf64:
9922 ; AVX512BW-FCP: # %bb.0:
9923 ; AVX512BW-FCP-NEXT: vmovdqa64 576(%rdi), %zmm2
9924 ; AVX512BW-FCP-NEXT: vmovdqa64 384(%rdi), %zmm4
9925 ; AVX512BW-FCP-NEXT: vmovdqa64 320(%rdi), %zmm3
9926 ; AVX512BW-FCP-NEXT: vmovdqa64 512(%rdi), %zmm5
9927 ; AVX512BW-FCP-NEXT: vmovdqa64 448(%rdi), %zmm0
9928 ; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdi), %zmm7
9929 ; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm9
9930 ; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm11
9931 ; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm1
9932 ; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm10
9933 ; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm12 = [16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11]
9934 ; AVX512BW-FCP-NEXT: # zmm12 = mem[0,1,2,3,0,1,2,3]
9935 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm13
9936 ; AVX512BW-FCP-NEXT: vpermt2w %zmm10, %zmm12, %zmm13
9937 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
9938 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm9, %zmm8
9939 ; AVX512BW-FCP-NEXT: vpermt2w %zmm11, %zmm6, %zmm8
9940 ; AVX512BW-FCP-NEXT: movl $67100672, %eax # imm = 0x3FFE000
9941 ; AVX512BW-FCP-NEXT: kmovd %eax, %k1
9942 ; AVX512BW-FCP-NEXT: vmovdqu16 %zmm13, %zmm8 {%k1}
9943 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm13 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,34,39,44,49,54,59]
9944 ; AVX512BW-FCP-NEXT: vpermt2w %zmm7, %zmm13, %zmm8
9945 ; AVX512BW-FCP-NEXT: vpermi2w %zmm5, %zmm0, %zmm12
9946 ; AVX512BW-FCP-NEXT: vpermi2w %zmm4, %zmm3, %zmm6
9947 ; AVX512BW-FCP-NEXT: vmovdqu16 %zmm12, %zmm6 {%k1}
9948 ; AVX512BW-FCP-NEXT: vpermt2w %zmm2, %zmm13, %zmm6
9949 ; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm14 = [49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44]
9950 ; AVX512BW-FCP-NEXT: # zmm14 = mem[0,1,2,3,0,1,2,3]
9951 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm10, %zmm15
9952 ; AVX512BW-FCP-NEXT: vpermt2w %zmm1, %zmm14, %zmm15
9953 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm12 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
9954 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm9, %zmm13
9955 ; AVX512BW-FCP-NEXT: vpermt2w %zmm11, %zmm12, %zmm13
9956 ; AVX512BW-FCP-NEXT: vmovdqu16 %zmm15, %zmm13 {%k1}
9957 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,35,40,45,50,55,60]
9958 ; AVX512BW-FCP-NEXT: vpermt2w %zmm7, %zmm15, %zmm13
9959 ; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm5, %zmm14
9960 ; AVX512BW-FCP-NEXT: vpermi2w %zmm4, %zmm3, %zmm12
9961 ; AVX512BW-FCP-NEXT: vmovdqu16 %zmm14, %zmm12 {%k1}
9962 ; AVX512BW-FCP-NEXT: vpermt2w %zmm2, %zmm15, %zmm12
9963 ; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm16 = [50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45]
9964 ; AVX512BW-FCP-NEXT: # zmm16 = mem[0,1,2,3,0,1,2,3]
9965 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm10, %zmm17
9966 ; AVX512BW-FCP-NEXT: vpermt2w %zmm1, %zmm16, %zmm17
9967 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm14 = [34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0]
9968 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm11, %zmm15
9969 ; AVX512BW-FCP-NEXT: vpermt2w %zmm9, %zmm14, %zmm15
9970 ; AVX512BW-FCP-NEXT: vmovdqu16 %zmm17, %zmm15 {%k1}
9971 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm17 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,36,41,46,51,56,61]
9972 ; AVX512BW-FCP-NEXT: vpermt2w %zmm7, %zmm17, %zmm15
9973 ; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm5, %zmm16
9974 ; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm4, %zmm14
9975 ; AVX512BW-FCP-NEXT: vmovdqu16 %zmm16, %zmm14 {%k1}
9976 ; AVX512BW-FCP-NEXT: vpermt2w %zmm2, %zmm17, %zmm14
9977 ; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm16 = [19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14]
9978 ; AVX512BW-FCP-NEXT: # zmm16 = mem[0,1,2,3,0,1,2,3]
9979 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm17
9980 ; AVX512BW-FCP-NEXT: vpermt2w %zmm10, %zmm16, %zmm17
9981 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm18 = [35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0]
9982 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm11, %zmm19
9983 ; AVX512BW-FCP-NEXT: vpermt2w %zmm9, %zmm18, %zmm19
9984 ; AVX512BW-FCP-NEXT: movl $33546240, %eax # imm = 0x1FFE000
9985 ; AVX512BW-FCP-NEXT: kmovd %eax, %k1
9986 ; AVX512BW-FCP-NEXT: vmovdqu16 %zmm17, %zmm19 {%k1}
9987 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm17 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,32,37,42,47,52,57,62]
9988 ; AVX512BW-FCP-NEXT: vpermt2w %zmm7, %zmm17, %zmm19
9989 ; AVX512BW-FCP-NEXT: vpermi2w %zmm5, %zmm0, %zmm16
9990 ; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm4, %zmm18
9991 ; AVX512BW-FCP-NEXT: vmovdqu16 %zmm16, %zmm18 {%k1}
9992 ; AVX512BW-FCP-NEXT: vpermt2w %zmm2, %zmm17, %zmm18
9993 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm16 = [4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0]
9994 ; AVX512BW-FCP-NEXT: vpermt2w %zmm11, %zmm16, %zmm9
9995 ; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15]
9996 ; AVX512BW-FCP-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3]
9997 ; AVX512BW-FCP-NEXT: vpermt2w %zmm10, %zmm11, %zmm1
9998 ; AVX512BW-FCP-NEXT: movb $7, %al
9999 ; AVX512BW-FCP-NEXT: kmovd %eax, %k1
10000 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm9, %zmm1 {%k1}
10001 ; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm9 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,33,38,43,48,53,58,63]
10002 ; AVX512BW-FCP-NEXT: vpermt2w %zmm7, %zmm9, %zmm1
10003 ; AVX512BW-FCP-NEXT: vpermt2w %zmm5, %zmm11, %zmm0
10004 ; AVX512BW-FCP-NEXT: vpermt2w %zmm4, %zmm16, %zmm3
10005 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, %zmm0 {%k1}
10006 ; AVX512BW-FCP-NEXT: vpermt2w %zmm2, %zmm9, %zmm0
10007 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm6, 64(%rsi)
10008 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm8, (%rsi)
10009 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm12, 64(%rdx)
10010 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm13, (%rdx)
10011 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm14, 64(%rcx)
10012 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm15, (%rcx)
10013 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm18, 64(%r8)
10014 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm19, (%r8)
10015 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, 64(%r9)
10016 ; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, (%r9)
10017 ; AVX512BW-FCP-NEXT: vzeroupper
10018 ; AVX512BW-FCP-NEXT: retq
10019 ;
10020 ; AVX512DQ-BW-LABEL: load_i16_stride5_vf64:
10021 ; AVX512DQ-BW: # %bb.0:
10022 ; AVX512DQ-BW-NEXT: vmovdqa64 576(%rdi), %zmm2
10023 ; AVX512DQ-BW-NEXT: vmovdqa64 384(%rdi), %zmm4
10024 ; AVX512DQ-BW-NEXT: vmovdqa64 320(%rdi), %zmm3
10025 ; AVX512DQ-BW-NEXT: vmovdqa64 512(%rdi), %zmm5
10026 ; AVX512DQ-BW-NEXT: vmovdqa64 448(%rdi), %zmm0
10027 ; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdi), %zmm7
10028 ; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm9
10029 ; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm11
10030 ; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm1
10031 ; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %zmm10
10032 ; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm12 = [16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11]
10033 ; AVX512DQ-BW-NEXT: # zmm12 = mem[0,1,2,3,0,1,2,3]
10034 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm13
10035 ; AVX512DQ-BW-NEXT: vpermt2w %zmm10, %zmm12, %zmm13
10036 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
10037 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm9, %zmm8
10038 ; AVX512DQ-BW-NEXT: vpermt2w %zmm11, %zmm6, %zmm8
10039 ; AVX512DQ-BW-NEXT: movl $67100672, %eax # imm = 0x3FFE000
10040 ; AVX512DQ-BW-NEXT: kmovd %eax, %k1
10041 ; AVX512DQ-BW-NEXT: vmovdqu16 %zmm13, %zmm8 {%k1}
10042 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm13 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,34,39,44,49,54,59]
10043 ; AVX512DQ-BW-NEXT: vpermt2w %zmm7, %zmm13, %zmm8
10044 ; AVX512DQ-BW-NEXT: vpermi2w %zmm5, %zmm0, %zmm12
10045 ; AVX512DQ-BW-NEXT: vpermi2w %zmm4, %zmm3, %zmm6
10046 ; AVX512DQ-BW-NEXT: vmovdqu16 %zmm12, %zmm6 {%k1}
10047 ; AVX512DQ-BW-NEXT: vpermt2w %zmm2, %zmm13, %zmm6
10048 ; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm14 = [49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44]
10049 ; AVX512DQ-BW-NEXT: # zmm14 = mem[0,1,2,3,0,1,2,3]
10050 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm10, %zmm15
10051 ; AVX512DQ-BW-NEXT: vpermt2w %zmm1, %zmm14, %zmm15
10052 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm12 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
10053 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm9, %zmm13
10054 ; AVX512DQ-BW-NEXT: vpermt2w %zmm11, %zmm12, %zmm13
10055 ; AVX512DQ-BW-NEXT: vmovdqu16 %zmm15, %zmm13 {%k1}
10056 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,35,40,45,50,55,60]
10057 ; AVX512DQ-BW-NEXT: vpermt2w %zmm7, %zmm15, %zmm13
10058 ; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm5, %zmm14
10059 ; AVX512DQ-BW-NEXT: vpermi2w %zmm4, %zmm3, %zmm12
10060 ; AVX512DQ-BW-NEXT: vmovdqu16 %zmm14, %zmm12 {%k1}
10061 ; AVX512DQ-BW-NEXT: vpermt2w %zmm2, %zmm15, %zmm12
10062 ; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm16 = [50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45]
10063 ; AVX512DQ-BW-NEXT: # zmm16 = mem[0,1,2,3,0,1,2,3]
10064 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm10, %zmm17
10065 ; AVX512DQ-BW-NEXT: vpermt2w %zmm1, %zmm16, %zmm17
10066 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm14 = [34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0]
10067 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm11, %zmm15
10068 ; AVX512DQ-BW-NEXT: vpermt2w %zmm9, %zmm14, %zmm15
10069 ; AVX512DQ-BW-NEXT: vmovdqu16 %zmm17, %zmm15 {%k1}
10070 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm17 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,36,41,46,51,56,61]
10071 ; AVX512DQ-BW-NEXT: vpermt2w %zmm7, %zmm17, %zmm15
10072 ; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm5, %zmm16
10073 ; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm4, %zmm14
10074 ; AVX512DQ-BW-NEXT: vmovdqu16 %zmm16, %zmm14 {%k1}
10075 ; AVX512DQ-BW-NEXT: vpermt2w %zmm2, %zmm17, %zmm14
10076 ; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm16 = [19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14]
10077 ; AVX512DQ-BW-NEXT: # zmm16 = mem[0,1,2,3,0,1,2,3]
10078 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm17
10079 ; AVX512DQ-BW-NEXT: vpermt2w %zmm10, %zmm16, %zmm17
10080 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm18 = [35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0]
10081 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm11, %zmm19
10082 ; AVX512DQ-BW-NEXT: vpermt2w %zmm9, %zmm18, %zmm19
10083 ; AVX512DQ-BW-NEXT: movl $33546240, %eax # imm = 0x1FFE000
10084 ; AVX512DQ-BW-NEXT: kmovd %eax, %k1
10085 ; AVX512DQ-BW-NEXT: vmovdqu16 %zmm17, %zmm19 {%k1}
10086 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm17 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,32,37,42,47,52,57,62]
10087 ; AVX512DQ-BW-NEXT: vpermt2w %zmm7, %zmm17, %zmm19
10088 ; AVX512DQ-BW-NEXT: vpermi2w %zmm5, %zmm0, %zmm16
10089 ; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm4, %zmm18
10090 ; AVX512DQ-BW-NEXT: vmovdqu16 %zmm16, %zmm18 {%k1}
10091 ; AVX512DQ-BW-NEXT: vpermt2w %zmm2, %zmm17, %zmm18
10092 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm16 = [4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0]
10093 ; AVX512DQ-BW-NEXT: vpermt2w %zmm11, %zmm16, %zmm9
10094 ; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15]
10095 ; AVX512DQ-BW-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3]
10096 ; AVX512DQ-BW-NEXT: vpermt2w %zmm10, %zmm11, %zmm1
10097 ; AVX512DQ-BW-NEXT: movb $7, %al
10098 ; AVX512DQ-BW-NEXT: kmovd %eax, %k1
10099 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm9, %zmm1 {%k1}
10100 ; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm9 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,33,38,43,48,53,58,63]
10101 ; AVX512DQ-BW-NEXT: vpermt2w %zmm7, %zmm9, %zmm1
10102 ; AVX512DQ-BW-NEXT: vpermt2w %zmm5, %zmm11, %zmm0
10103 ; AVX512DQ-BW-NEXT: vpermt2w %zmm4, %zmm16, %zmm3
10104 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, %zmm0 {%k1}
10105 ; AVX512DQ-BW-NEXT: vpermt2w %zmm2, %zmm9, %zmm0
10106 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm6, 64(%rsi)
10107 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm8, (%rsi)
10108 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm12, 64(%rdx)
10109 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm13, (%rdx)
10110 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm14, 64(%rcx)
10111 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm15, (%rcx)
10112 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm18, 64(%r8)
10113 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm19, (%r8)
10114 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, 64(%r9)
10115 ; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, (%r9)
10116 ; AVX512DQ-BW-NEXT: vzeroupper
10117 ; AVX512DQ-BW-NEXT: retq
10118 ;
10119 ; AVX512DQ-BW-FCP-LABEL: load_i16_stride5_vf64:
10120 ; AVX512DQ-BW-FCP: # %bb.0:
10121 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 576(%rdi), %zmm2
10122 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rdi), %zmm4
10123 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rdi), %zmm3
10124 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 512(%rdi), %zmm5
10125 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 448(%rdi), %zmm0
10126 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdi), %zmm7
10127 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm9
10128 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm11
10129 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm1
10130 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm10
10131 ; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm12 = [16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0,1,6,11]
10132 ; AVX512DQ-BW-FCP-NEXT: # zmm12 = mem[0,1,2,3,0,1,2,3]
10133 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm13
10134 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm10, %zmm12, %zmm13
10135 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
10136 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm9, %zmm8
10137 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm11, %zmm6, %zmm8
10138 ; AVX512DQ-BW-FCP-NEXT: movl $67100672, %eax # imm = 0x3FFE000
10139 ; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k1
10140 ; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm13, %zmm8 {%k1}
10141 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm13 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,34,39,44,49,54,59]
10142 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm7, %zmm13, %zmm8
10143 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm5, %zmm0, %zmm12
10144 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm4, %zmm3, %zmm6
10145 ; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm12, %zmm6 {%k1}
10146 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm2, %zmm13, %zmm6
10147 ; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm14 = [49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0,34,39,44]
10148 ; AVX512DQ-BW-FCP-NEXT: # zmm14 = mem[0,1,2,3,0,1,2,3]
10149 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm10, %zmm15
10150 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm1, %zmm14, %zmm15
10151 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm12 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
10152 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm9, %zmm13
10153 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm11, %zmm12, %zmm13
10154 ; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm15, %zmm13 {%k1}
10155 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,35,40,45,50,55,60]
10156 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm7, %zmm15, %zmm13
10157 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm5, %zmm14
10158 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm4, %zmm3, %zmm12
10159 ; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm14, %zmm12 {%k1}
10160 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm2, %zmm15, %zmm12
10161 ; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm16 = [50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0,35,40,45]
10162 ; AVX512DQ-BW-FCP-NEXT: # zmm16 = mem[0,1,2,3,0,1,2,3]
10163 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm10, %zmm17
10164 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm1, %zmm16, %zmm17
10165 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm14 = [34,39,44,49,54,59,0,5,10,15,20,25,30,0,0,0]
10166 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm11, %zmm15
10167 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm9, %zmm14, %zmm15
10168 ; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm17, %zmm15 {%k1}
10169 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm17 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,36,41,46,51,56,61]
10170 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm7, %zmm17, %zmm15
10171 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm5, %zmm16
10172 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm4, %zmm14
10173 ; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm16, %zmm14 {%k1}
10174 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm2, %zmm17, %zmm14
10175 ; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm16 = [19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0,4,9,14]
10176 ; AVX512DQ-BW-FCP-NEXT: # zmm16 = mem[0,1,2,3,0,1,2,3]
10177 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm17
10178 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm10, %zmm16, %zmm17
10179 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm18 = [35,40,45,50,55,60,1,6,11,16,21,26,31,0,0,0]
10180 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm11, %zmm19
10181 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm9, %zmm18, %zmm19
10182 ; AVX512DQ-BW-FCP-NEXT: movl $33546240, %eax # imm = 0x1FFE000
10183 ; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k1
10184 ; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm17, %zmm19 {%k1}
10185 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm17 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,32,37,42,47,52,57,62]
10186 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm7, %zmm17, %zmm19
10187 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm5, %zmm0, %zmm16
10188 ; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm4, %zmm18
10189 ; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm16, %zmm18 {%k1}
10190 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm2, %zmm17, %zmm18
10191 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm16 = [4,9,14,19,24,29,34,39,44,49,54,59,0,0,0,0]
10192 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm11, %zmm16, %zmm9
10193 ; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0,0,5,10,15]
10194 ; AVX512DQ-BW-FCP-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3]
10195 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm10, %zmm11, %zmm1
10196 ; AVX512DQ-BW-FCP-NEXT: movb $7, %al
10197 ; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k1
10198 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm9, %zmm1 {%k1}
10199 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm9 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,33,38,43,48,53,58,63]
10200 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm7, %zmm9, %zmm1
10201 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm5, %zmm11, %zmm0
10202 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm4, %zmm16, %zmm3
10203 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, %zmm0 {%k1}
10204 ; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm2, %zmm9, %zmm0
10205 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm6, 64(%rsi)
10206 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, (%rsi)
10207 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm12, 64(%rdx)
10208 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm13, (%rdx)
10209 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm14, 64(%rcx)
10210 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm15, (%rcx)
10211 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm18, 64(%r8)
10212 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm19, (%r8)
10213 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, 64(%r9)
10214 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, (%r9)
10215 ; AVX512DQ-BW-FCP-NEXT: vzeroupper
10216 ; AVX512DQ-BW-FCP-NEXT: retq
10217 %wide.vec = load <320 x i16>, ptr %in.vec, align 64
10218 %strided.vec0 = shufflevector <320 x i16> %wide.vec, <320 x i16> poison, <64 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35, i32 40, i32 45, i32 50, i32 55, i32 60, i32 65, i32 70, i32 75, i32 80, i32 85, i32 90, i32 95, i32 100, i32 105, i32 110, i32 115, i32 120, i32 125, i32 130, i32 135, i32 140, i32 145, i32 150, i32 155, i32 160, i32 165, i32 170, i32 175, i32 180, i32 185, i32 190, i32 195, i32 200, i32 205, i32 210, i32 215, i32 220, i32 225, i32 230, i32 235, i32 240, i32 245, i32 250, i32 255, i32 260, i32 265, i32 270, i32 275, i32 280, i32 285, i32 290, i32 295, i32 300, i32 305, i32 310, i32 315>
10219 %strided.vec1 = shufflevector <320 x i16> %wide.vec, <320 x i16> poison, <64 x i32> <i32 1, i32 6, i32 11, i32 16, i32 21, i32 26, i32 31, i32 36, i32 41, i32 46, i32 51, i32 56, i32 61, i32 66, i32 71, i32 76, i32 81, i32 86, i32 91, i32 96, i32 101, i32 106, i32 111, i32 116, i32 121, i32 126, i32 131, i32 136, i32 141, i32 146, i32 151, i32 156, i32 161, i32 166, i32 171, i32 176, i32 181, i32 186, i32 191, i32 196, i32 201, i32 206, i32 211, i32 216, i32 221, i32 226, i32 231, i32 236, i32 241, i32 246, i32 251, i32 256, i32 261, i32 266, i32 271, i32 276, i32 281, i32 286, i32 291, i32 296, i32 301, i32 306, i32 311, i32 316>
10220 %strided.vec2 = shufflevector <320 x i16> %wide.vec, <320 x i16> poison, <64 x i32> <i32 2, i32 7, i32 12, i32 17, i32 22, i32 27, i32 32, i32 37, i32 42, i32 47, i32 52, i32 57, i32 62, i32 67, i32 72, i32 77, i32 82, i32 87, i32 92, i32 97, i32 102, i32 107, i32 112, i32 117, i32 122, i32 127, i32 132, i32 137, i32 142, i32 147, i32 152, i32 157, i32 162, i32 167, i32 172, i32 177, i32 182, i32 187, i32 192, i32 197, i32 202, i32 207, i32 212, i32 217, i32 222, i32 227, i32 232, i32 237, i32 242, i32 247, i32 252, i32 257, i32 262, i32 267, i32 272, i32 277, i32 282, i32 287, i32 292, i32 297, i32 302, i32 307, i32 312, i32 317>
10221 %strided.vec3 = shufflevector <320 x i16> %wide.vec, <320 x i16> poison, <64 x i32> <i32 3, i32 8, i32 13, i32 18, i32 23, i32 28, i32 33, i32 38, i32 43, i32 48, i32 53, i32 58, i32 63, i32 68, i32 73, i32 78, i32 83, i32 88, i32 93, i32 98, i32 103, i32 108, i32 113, i32 118, i32 123, i32 128, i32 133, i32 138, i32 143, i32 148, i32 153, i32 158, i32 163, i32 168, i32 173, i32 178, i32 183, i32 188, i32 193, i32 198, i32 203, i32 208, i32 213, i32 218, i32 223, i32 228, i32 233, i32 238, i32 243, i32 248, i32 253, i32 258, i32 263, i32 268, i32 273, i32 278, i32 283, i32 288, i32 293, i32 298, i32 303, i32 308, i32 313, i32 318>
10222 %strided.vec4 = shufflevector <320 x i16> %wide.vec, <320 x i16> poison, <64 x i32> <i32 4, i32 9, i32 14, i32 19, i32 24, i32 29, i32 34, i32 39, i32 44, i32 49, i32 54, i32 59, i32 64, i32 69, i32 74, i32 79, i32 84, i32 89, i32 94, i32 99, i32 104, i32 109, i32 114, i32 119, i32 124, i32 129, i32 134, i32 139, i32 144, i32 149, i32 154, i32 159, i32 164, i32 169, i32 174, i32 179, i32 184, i32 189, i32 194, i32 199, i32 204, i32 209, i32 214, i32 219, i32 224, i32 229, i32 234, i32 239, i32 244, i32 249, i32 254, i32 259, i32 264, i32 269, i32 274, i32 279, i32 284, i32 289, i32 294, i32 299, i32 304, i32 309, i32 314, i32 319>
10223 store <64 x i16> %strided.vec0, ptr %out.vec0, align 64
10224 store <64 x i16> %strided.vec1, ptr %out.vec1, align 64
10225 store <64 x i16> %strided.vec2, ptr %out.vec2, align 64
10226 store <64 x i16> %strided.vec3, ptr %out.vec3, align 64
10227 store <64 x i16> %strided.vec4, ptr %out.vec4, align 64
10228 ret void
10229 }